def compile_script(self, need_mqs=False):
"""compiles the script, if not already compiled.
If need_mqs=True, starts a meqserver and builds the tree as well"""
if not self._module:
self._compile_failed = True
# will reset to False if all goes well\
self.log_progress("compile")
# start meqserver if required
if (need_mqs or int(self.get_option("start_meqserver",
0))) and not self._mqs:
from Timba.Apps import meqserver
# get multithreading option
mt = int(self.get_option("multithreaded", 0))
if mt > 1:
extra = ["-mt", str(mt)]
else:
extra = []
_dprint(1, "starting meqserver", extra)
self._mqs = meqserver.default_mqs(wait_init=10, extra=extra)
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# load config file
tdlconf = self.get_option("tdlconf", ".tdl.conf")
TDLOptions.config.read(tdlconf)
TDLOptions.init_options(self.name, save=False)
# compile TDL module
_dprint(1, "compiling TDL script", self.name)
try:
(self._module, ns,
msg) = Compile.compile_file(self._mqs, self.name)
except:
excinfo = sys.exc_info()
self.log_exc(level=2, *excinfo)
self.fail("compile failed")
excinfo = None
return
# success
self.log(msg, level=2)
self.success("compile")
self._compile_failed = False
pass
def compile_script (self,need_mqs=False):
"""compiles the script, if not already compiled.
If need_mqs=True, starts a meqserver and builds the tree as well""";
if not self._module:
self._compile_failed = True; # will reset to False if all goes well\
self.log_progress("compile");
# start meqserver if required
if ( need_mqs or int(self.get_option("start_meqserver",0)) ) and not self._mqs:
from Timba.Apps import meqserver
# get multithreading option
mt = int(self.get_option("multithreaded",0));
if mt>1:
extra = [ "-mt",str(mt) ];
else:
extra = []
_dprint(1,"starting meqserver",extra);
self._mqs = meqserver.default_mqs(wait_init=10,extra=extra);
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# load config file
tdlconf = self.get_option("tdlconf",".tdl.conf");
TDLOptions.config.read(tdlconf);
TDLOptions.init_options(self.name,save=False);
# compile TDL module
_dprint(1,"compiling TDL script",self.name);
try:
(self._module,ns,msg) = Compile.compile_file(self._mqs,self.name);
except:
excinfo = sys.exc_info();
self.log_exc(level=2,*excinfo);
self.fail("compile failed");
excinfo = None;
return;
# success
self.log(msg,level=2);
self.success("compile");
self._compile_failed = False;
pass;
loaded_options = False;
module = None;
# now parse commands
for cmd in rem_args:
load_match = re_load_config.match(cmd) or re_load_config1.match(cmd);
set_match = re_set_config.match(cmd);
compile_match = re_compile_script.match(cmd) or re_compile_script1.match(cmd);
job_match = re_run_job.match(cmd);
if load_match:
section = load_match.group(1);
print "### Loading config section",section;
TDLOptions.init_options(section,save=False);
loaded_options = True;
elif set_match:
if not loaded_options:
raise RuntimeError,"Config section not yet specified";
name,value = set_match.groups();
print "### Setting option %s=%s"%(name,value);
TDLOptions.set_option(name,value,save=False,from_str=True);
elif compile_match:
script,dum,section = compile_match.groups(None);
print "### Compiling",script;
if not loaded_options and not section:
# this mode reloads default config section
print "### (using options from default section)";
def testMeqtreesBatchJob():
trace_sync = True
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR
print("========== Changing working directory to", newdir)
os.chdir(newdir)
print("========== Making required symlinks")
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))))
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(
PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt")))))
if not os.access(".", os.R_OK | os.W_OK):
print("Directory", os.getcwd(),
"not writable, can't run tests in here.")
print(
"You may choose to run the tests in a different directory by giving it as an argument to this script."
)
sys.exit(1)
## check if we have owlcat or owlcat.sh
owlcat = ""
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat", "owlcat.sh":
tmp = os.path.join(dirname, binary)
if os.path.exists(tmp):
owlcat = tmp
break
if owlcat:
break
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh")
## make simulated MS
print("========== Removing files")
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(
PACKAGE_TEST_DIR))
print("========== Running makems")
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")))
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR))
#TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(
owlcat, PACKAGE_TEST_DIR))
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10"
.format(PACKAGE_TEST_DIR))
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f"""
.format(PACKAGE_TEST_DIR))
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".
format(PACKAGE_TEST_DIR))
run("""diff {0:s}/test-lsm1.txt.tmp {1:s} || diff {0:s}/test-lsm1.txt.tmp {2:s}"""
.format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')),
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm2.txt.reference'))))
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(
owlcat, PACKAGE_TEST_DIR))
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver")
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"))
print("========== Compiling", script)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")))
TDLOptions.config.set(
"calibrate", "ms_sel.msname",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set(
"calibrate", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "lsm.filename",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set(
"calibrate", "cal_g_diag.g_diag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_diag.g_diag.table_name")))
TDLOptions.config.set(
"calibrate", "cal_g_offdiag.g_offdiag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set(
"simulate-model", "lsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set(
"simulate-model", "ms_sel.msname",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set(
"simulate-model", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set(
"simulate-model", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"),
"w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf")))
# needs to re-read because of a Timba perculiarity
mod, ns, msg = Compile.compile_file(mqs,
script,
config="simulate-model")
print("========== Simulating MODEL_DATA ")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## compare against reference image
print("========== Verifying test image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR,
"test-refimage.fits")),
maxdelta=1e-3)
print("========== Compiling script with modified config")
TDLOptions.init_options("simulate-model", save=False)
TDLOptions.set_option("me.g_enable", True)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print("========== Simulating DATA ")
TDLOptions.set_option("ms_sel.output_column", "DATA")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging DATA ")
TDLOptions.set_option("img_sel.imaging_column", "DATA")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"))
print("========== Compiling", script)
mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate")
print("========== Calibrating ")
TDLOptions.get_job_func('cal_G_diag')(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
finally:
print("Stopping meqserver")
# this halts the meqserver
meqserver.stop_default_mqs()
print("========== Making plots of solutions ")
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(
owlcat, PACKAGE_TEST_DIR))
## compare against reference image
print("========== Verifying residual image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3)
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!")
# now we can exit
print("Bye!")
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf");
script = "example-sim.py";
print "========== Compiling batch job 1";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 1");
print "========== Running batch job 1";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Compiling batch job 2";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 2");
print "========== Running batch job 2";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Compiling batch job 2 with modified config";
TDLOptions.init_options("batch job 2",save=False);
TDLOptions.set_option("me.enable_G",False);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print "========== Running batch job 2";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
### Cleanup time
finally:
print "Stopping meqserver";
# this halts the meqserver
meqserver.stop_default_mqs();
# now we can exit
print "Bye!";
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf")
script = "example-sim.py"
print("========== Compiling batch job 1")
mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 1")
print("========== Running batch job 1")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Compiling batch job 2")
mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 2")
print("========== Running batch job 2")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Compiling batch job 2 with modified config")
TDLOptions.init_options("batch job 2", save=False)
TDLOptions.set_option("me.enable_G", False)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print("========== Running batch job 2")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
### Cleanup time
finally:
print("Stopping meqserver")
# this halts the meqserver
meqserver.stop_default_mqs()
# now we can exit
print("Bye!")
module = None
# now parse commands
for cmd in rem_args:
load_match = re_load_config.match(cmd) or re_load_config1.match(
cmd)
set_match = re_set_config.match(cmd)
compile_match = re_compile_script.match(
cmd) or re_compile_script1.match(cmd)
job_match = re_run_job.match(cmd)
if load_match:
section = load_match.group(1)
print "### Loading config section", section
TDLOptions.init_options(section, save=False)
loaded_options = True
elif set_match:
if not loaded_options:
raise RuntimeError, "Config section not yet specified"
name, value = set_match.groups()
print "### Setting option %s=%s" % (name, value)
TDLOptions.set_option(name, value, save=False, from_str=True)
elif compile_match:
script, dum, section = compile_match.groups(None)
print "### Compiling", script
if not loaded_options and not section:
# this mode reloads default config section
print "### (using options from default section)"
## make simulation with perfect MODEL_DATA
script = path("testing-sim.py");
print "========== Compiling",script;
TDLOptions.config.read(path("testing.tdl.conf"));
mod,ns,msg = Compile.compile_file(mqs,script,config="simulate-model");
print "========== Simulating MODEL_DATA ";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Imaging MODEL_DATA ";
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## compare against reference image
print "========== Verifying test image ";
verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',path('test-refimage.fits'),maxdelta=1e-3);
print "========== Compiling script with modified config";
TDLOptions.init_options("simulate-model",save=False);
TDLOptions.set_option("me.g_enable",True);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print "========== Simulating DATA ";
TDLOptions.set_option("ms_sel.output_column","DATA");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Imaging DATA ";
TDLOptions.set_option("img_sel.imaging_column","DATA");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## calibrate
script = path("testing-cal.py");
print "========== Compiling",script;
mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate");
print "========== Calibrating ";
TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True);
def main ():
for optstr in (options.debug or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
debuglevels[context] = int(level);
Timba.utils.verbosity.disable_argv(); # tell verbosity class to not parse its argv
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
Timba.utils.verbosity.set_verbosity_level(context,level);
if not args:
parser.print_help();
sys.exit(1);
if debuglevels:
octopussy.set_debug(debuglevels);
script = args[0];
tdljob = (len(args)>1 and args[1]) or None;
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# this starts a kernel.
if options.compile_only:
mqs = None;
else:
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",options.mt]);
TDLOptions.config.read(options.config);
TDLOptions.init_options(script);
print "************************ Compiling TDL script",script;
# this compiles a script as a TDL module. Any errors will be thrown as
# and exception, so this always returns successfully
(mod,ns,msg) = Compile.compile_file(mqs,script);
if options.compile_only:
print msg;
sys.exit(0);
# if a solve job is not specified, try to find one
if tdljob:
jobfunc = getattr(mod,tdljob,None);
if not jobfunc:
print "Cannot find TDL job named",tdljob;
sys.exit(1);
else:
# does the script define an explicit job list?
joblist = getattr(mod,'_tdl_job_list',[]);
if not joblist:
joblist = [];
# try to build it from implicit function names
for (name,func) in mod.__dict__.iteritems():
if name.startswith("_tdl_job_") and callable(func):
joblist.append(func);
# does the script define a testing function?
testfunc = getattr(mod,'_test_forest',None);
if testfunc:
joblist.insert(0,testfunc);
if not joblist:
print "No TDL jobs found in script",script;
sys.exit(1);
jobfunc = joblist[0];
tdljob = jobfunc.__name__;
# this runs the appropriate job. wait=True is needed to wait
print "************************ Running TDL job",tdljob;
# check if job takes a "wait" argument
(fargs,fvarargs,fvarkw,fdefaults) = inspect.getargspec(jobfunc);
if 'wait' in fargs or fvarkw:
jobopts = dict(wait=True);
else:
jobopts = {};
jobfunc(mqs,None,**jobopts);
print "========== Simulating MODEL_DATA "
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print "========== Imaging MODEL_DATA "
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## compare against reference image
print "========== Verifying test image "
verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',
path('test-refimage.fits'),
maxdelta=1e-3)
print "========== Compiling script with modified config"
TDLOptions.init_options("simulate-model", save=False)
TDLOptions.set_option("me.g_enable", True)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print "========== Simulating DATA "
TDLOptions.set_option("ms_sel.output_column", "DATA")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print "========== Imaging DATA "
TDLOptions.set_option("img_sel.imaging_column", "DATA")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## calibrate
script = path("testing-cal.py")
print "========== Compiling", script
# finally, this is expected to fail
print "Trying to call a non-existing TDL job 'job3'"
try:
TDLOptions.get_job_func('job3')(mqs, None, wait=True)
except NameError:
print "Indeed, 'job3' does not exist."
# This shows how to change configuration on-the-fly.
# TDLOptions.set_option() changes the value of a configuration variable "in memory".
# Note that compile_file() normally rereads the config file, which would defeat the point of any
# on-the-fly changes. To get around this, we can manage the config manually as follows:
print "Recompiling and running for a=5, t=[-1,1]"
# this loads initial config from section [test_b]. The save=False argument tells the system to not
# write the changes we make below to the .tdl.conf file
TDLOptions.init_options("test_b", save=False)
# this makes some changes to the loaded config
TDLOptions.set_option("a", 5)
TDLOptions.set_option("t0", -1)
TDLOptions.set_option("t1", 1)
# this compiles the script. config=None tells it to NOT reload the config file, but to use whatever we have set up above.
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
mod._test_forest(mqs, None, wait=True)
# Make another change to the current config and recompile. Again config=None is necessary,
TDLOptions.set_option("b", -5)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
mod._test_forest(mqs, None, wait=True)
# Since "a" and "b" are compile-time options, we always need to recompile for them to take effect.
# By contrast, runtime options ('t0' and 't1') can be changed without recompiling:
def testMeqtreesBatchJob():
trace_sync = True;
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR;
print("========== Changing working directory to",newdir);
os.chdir(newdir);
print("========== Making required symlinks");
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))));
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(path("test-lsm.txt")))));
if not os.access(".",os.R_OK|os.W_OK):
print("Directory",os.getcwd(),"not writable, can't run tests in here.")
print("You may choose to run the tests in a different directory by giving it as an argument to this script.")
sys.exit(1);
## check if we have owlcat or owlcat.sh
owlcat = "";
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat","owlcat.sh":
tmp = os.path.join(dirname,binary);
if os.path.exists(tmp):
owlcat = tmp;
break;
if owlcat:
break;
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh");
## make simulated MS
print("========== Removing files");
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(PACKAGE_TEST_DIR));
print("========== Running makems");
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")));
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)); #TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(owlcat, PACKAGE_TEST_DIR));
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10".format(PACKAGE_TEST_DIR));
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""".format(PACKAGE_TEST_DIR));
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".format(PACKAGE_TEST_DIR));
run("""diff {0:s}/test-lsm1.txt.tmp {1:s}""".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference'))));
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(owlcat, PACKAGE_TEST_DIR));
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver");
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"));
print("========== Compiling",script);
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")));
TDLOptions.config.set("calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set("calibrate", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set("calibrate", "cal_g_diag.g_diag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name")))
TDLOptions.config.set("calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set("simulate-model", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set("simulate-model", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set("simulate-model", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set("simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))); # needs to re-read because of a Timba perculiarity
mod,ns,msg = Compile.compile_file(mqs, script, config="simulate-model");
print("========== Simulating MODEL_DATA ");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## compare against reference image
print("========== Verifying test image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")),
maxdelta=1e-3);
print("========== Compiling script with modified config");
TDLOptions.init_options("simulate-model",save=False);
TDLOptions.set_option("me.g_enable",True);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print("========== Simulating DATA ");
TDLOptions.set_option("ms_sel.output_column","DATA");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging DATA ");
TDLOptions.set_option("img_sel.imaging_column","DATA");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"));
print("========== Compiling",script);
mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate");
print("========== Calibrating ");
TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
finally:
print("Stopping meqserver");
# this halts the meqserver
meqserver.stop_default_mqs();
print("========== Making plots of solutions ");
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(owlcat, PACKAGE_TEST_DIR));
## compare against reference image
print("========== Verifying residual image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3);
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!");
# now we can exit
print("Bye!");
def import_tdl_module (filename,text=None,config=0):
"""Imports a TDL module.
Parameters:
filename: script location
text: text of module. If none, file will be re-read.
Return value:
a tuple of (module,text), where module is the newly-imported
TDL module, and text is the module text.
Exceptions thrown:
Any import error results in an exception. This is always
a TDL.CumulativeError exception containing an error list.
Other exceptions may indicate an internal failure.
""";
_dprint(1,"importing",filename);
global _current_filename;
_current_filename = filename;
# initialize global nodescope (and repository)
meqds.clear_forest();
try:
reload(Timba.TDL.Settings);
# reset TDL script options, unless config=None ('0' is used as default, causing the filename to be substituted)
TDLOptions.init_script(filename);
if config is not None:
TDLOptions.init_options(config or filename);
# remove .pyo file as that can have cached paths and directories may have changed
# (see bug 677)
try:
os.unlink(os.path.splitext(filename)[0]+'.pyo');
except:
pass;
# open file
infile = file(filename,'r');
if text is None:
text = infile.read();
infile.seek(0);
# infile is now an open input file object, and text is the script
# flush all modules imported via previous TDL run
global _tdlmodlist;
_dprint(1,'clearing out TDL-imported modules',_tdlmodlist);
for m in _tdlmodlist:
try: del sys.modules[m];
except KeyError: pass;
# remember which modules are imported
global _prior_compile_modules;
_prior_compile_modules = set(sys.modules.iterkeys());
modname = '__tdlruntime';
try:
TDLOptions.enable_save_config(False);
imp.acquire_lock();
_tdlmod = imp.load_source(modname,filename,infile);
finally:
TDLOptions.enable_save_config(True);
TDLOptions.save_config();
imp.release_lock();
infile.close();
_update_modlist();
return (_tdlmod,text);
# CumulativeError exceptions returned as is
except TDL.CumulativeError:
_dprint(0,'cumulative error importing TDL file:',filename);
traceback.print_exc();
args = sys.exc_info()[1].args;
_dprint(0,'number of errors in list:',len(args));
_dprint(1,'errors are: {',args,'}');
raise;
# Other exceptions wrapped in a CumulativeError, and
# location information is added in
except:
(etype,exc,tb) = sys.exc_info();
_dprint(0,'exception importing TDL file:',filename);
traceback.print_exception(etype,exc,tb);
# use TDL add_error() to process the error, since this automatically
# adds location information
ns = TDL.NodeScope();
ns.AddError(exc,traceback.extract_tb(tb),error_limit=None);
# re-raise as a CumulativeError
raise TDL.CumulativeError(*ns.GetErrors());
def import_tdl_module(filename, text=None, config=0):
"""Imports a TDL module.
Parameters:
filename: script location
text: text of module. If none, file will be re-read.
Return value:
a tuple of (module,text), where module is the newly-imported
TDL module, and text is the module text.
Exceptions thrown:
Any import error results in an exception. This is always
a TDL.CumulativeError exception containing an error list.
Other exceptions may indicate an internal failure.
"""
_dprint(1, "importing", filename)
global _current_filename
_current_filename = filename
# initialize global nodescope (and repository)
meqds.clear_forest()
try:
reload(Timba.TDL.Settings)
# reset TDL script options, unless config=None ('0' is used as default, causing the filename to be substituted)
TDLOptions.init_script(filename)
if config is not None:
TDLOptions.init_options(config or filename)
# remove .pyo file as that can have cached paths and directories may have changed
# (see bug 677)
try:
os.unlink(os.path.splitext(filename)[0] + '.pyo')
except:
pass
# open file
infile = file(filename, 'r')
if text is None:
text = infile.read()
infile.seek(0)
# infile is now an open input file object, and text is the script
# flush all modules imported via previous TDL run
global _tdlmodlist
_dprint(1, 'clearing out TDL-imported modules', _tdlmodlist)
for m in _tdlmodlist:
try:
del sys.modules[m]
except KeyError:
pass
# remember which modules are imported
global _prior_compile_modules
_prior_compile_modules = set(sys.modules.iterkeys())
modname = '__tdlruntime'
try:
TDLOptions.enable_save_config(False)
imp.acquire_lock()
_tdlmod = imp.load_source(modname, filename, infile)
finally:
TDLOptions.enable_save_config(True)
TDLOptions.save_config()
imp.release_lock()
infile.close()
_update_modlist()
return (_tdlmod, text)
# CumulativeError exceptions returned as is
except TDL.CumulativeError:
_dprint(0, 'cumulative error importing TDL file:', filename)
traceback.print_exc()
args = sys.exc_info()[1].args
_dprint(0, 'number of errors in list:', len(args))
_dprint(1, 'errors are: {', args, '}')
raise
# Other exceptions wrapped in a CumulativeError, and
# location information is added in
except:
(etype, exc, tb) = sys.exc_info()
_dprint(0, 'exception importing TDL file:', filename)
traceback.print_exception(etype, exc, tb)
# use TDL add_error() to process the error, since this automatically
# adds location information
ns = TDL.NodeScope()
ns.AddError(exc, traceback.extract_tb(tb), error_limit=None)
# re-raise as a CumulativeError
raise TDL.CumulativeError(*ns.GetErrors())
# finally, this is expected to fail
print "Trying to call a non-existing TDL job 'job3'";
try:
TDLOptions.get_job_func('job3')(mqs,None,wait=True);
except NameError:
print "Indeed, 'job3' does not exist."
# This shows how to change configuration on-the-fly.
# TDLOptions.set_option() changes the value of a configuration variable "in memory".
# Note that compile_file() normally rereads the config file, which would defeat the point of any
# on-the-fly changes. To get around this, we can manage the config manually as follows:
print "Recompiling and running for a=5, t=[-1,1]";
# this loads initial config from section [test_b]. The save=False argument tells the system to not
# write the changes we make below to the .tdl.conf file
TDLOptions.init_options("test_b",save=False);
# this makes some changes to the loaded config
TDLOptions.set_option("a",5);
TDLOptions.set_option("t0",-1);
TDLOptions.set_option("t1",1);
# this compiles the script. config=None tells it to NOT reload the config file, but to use whatever we have set up above.
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
mod._test_forest(mqs,None,wait=True);
# Make another change to the current config and recompile. Again config=None is necessary,
TDLOptions.set_option("b",-5);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
mod._test_forest(mqs,None,wait=True);
# Since "a" and "b" are compile-time options, we always need to recompile for them to take effect.
# By contrast, runtime options ('t0' and 't1') can be changed without recompiling:
def main():
for optstr in (options.debug or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
debuglevels[context] = int(level)
Timba.utils.verbosity.disable_argv()
# tell verbosity class to not parse its argv
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
Timba.utils.verbosity.set_verbosity_level(context, level)
if not args:
parser.print_help()
sys.exit(1)
if debuglevels:
octopussy.set_debug(debuglevels)
script = args[0]
tdljob = (len(args) > 1 and args[1]) or None
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# this starts a kernel.
if options.compile_only:
mqs = None
else:
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", options.mt])
TDLOptions.config.read(options.config)
TDLOptions.init_options(script)
print(("************************ Compiling TDL script", script))
# this compiles a script as a TDL module. Any errors will be thrown as
# and exception, so this always returns successfully
(mod, ns, msg) = Compile.compile_file(mqs, script)
if options.compile_only:
print(msg)
sys.exit(0)
# if a solve job is not specified, try to find one
if tdljob:
jobfunc = getattr(mod, tdljob, None)
if not jobfunc:
print(("Cannot find TDL job named", tdljob))
sys.exit(1)
else:
# does the script define an explicit job list?
joblist = getattr(mod, '_tdl_job_list', [])
if not joblist:
joblist = []
# try to build it from implicit function names
for (name, func) in list(mod.__dict__.items()):
if name.startswith("_tdl_job_") and callable(func):
joblist.append(func)
# does the script define a testing function?
testfunc = getattr(mod, '_test_forest', None)
if testfunc:
joblist.insert(0, testfunc)
if not joblist:
print(("No TDL jobs found in script", script))
sys.exit(1)
jobfunc = joblist[0]
tdljob = jobfunc.__name__
# this runs the appropriate job. wait=True is needed to wait
print(("************************ Running TDL job", tdljob))
# check if job takes a "wait" argument
(fargs, fvarargs, fvarkw, fdefaults) = inspect.getargspec(jobfunc)
if 'wait' in fargs or fvarkw:
jobopts = dict(wait=True)
else:
jobopts = {}
jobfunc(mqs, None, **jobopts)