def compile (self,script):
try:
self.script = script;
_dprint(0,"compiling",script);
(self.tdlmod,ns,msg) = Compile.compile_file(self.mqs,script);
_dprintf(0,"compiled %s: %s",script,msg);
lines = ["compiled TDL script %s: "%script]+msg.rstrip().split('\n');
self.logs(*[ s+'\n' for s in lines ]);
except:
self.logexc();
raise;
def compile(self, script):
try:
self.script = script
_dprint(0, "compiling", script)
(self.tdlmod, ns, msg) = Compile.compile_file(self.mqs, script)
_dprintf(0, "compiled %s: %s", script, msg)
lines = ["compiled TDL script %s: " % script
] + msg.rstrip().split('\n')
self.logs(*[s + '\n' for s in lines])
except:
self.logexc()
raise
def compile_script(self, need_mqs=False):
"""compiles the script, if not already compiled.
If need_mqs=True, starts a meqserver and builds the tree as well"""
if not self._module:
self._compile_failed = True
# will reset to False if all goes well\
self.log_progress("compile")
# start meqserver if required
if (need_mqs or int(self.get_option("start_meqserver",
0))) and not self._mqs:
from Timba.Apps import meqserver
# get multithreading option
mt = int(self.get_option("multithreaded", 0))
if mt > 1:
extra = ["-mt", str(mt)]
else:
extra = []
_dprint(1, "starting meqserver", extra)
self._mqs = meqserver.default_mqs(wait_init=10, extra=extra)
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# load config file
tdlconf = self.get_option("tdlconf", ".tdl.conf")
TDLOptions.config.read(tdlconf)
TDLOptions.init_options(self.name, save=False)
# compile TDL module
_dprint(1, "compiling TDL script", self.name)
try:
(self._module, ns,
msg) = Compile.compile_file(self._mqs, self.name)
except:
excinfo = sys.exc_info()
self.log_exc(level=2, *excinfo)
self.fail("compile failed")
excinfo = None
return
# success
self.log(msg, level=2)
self.success("compile")
self._compile_failed = False
pass
def compile_script (self,need_mqs=False):
"""compiles the script, if not already compiled.
If need_mqs=True, starts a meqserver and builds the tree as well""";
if not self._module:
self._compile_failed = True; # will reset to False if all goes well\
self.log_progress("compile");
# start meqserver if required
if ( need_mqs or int(self.get_option("start_meqserver",0)) ) and not self._mqs:
from Timba.Apps import meqserver
# get multithreading option
mt = int(self.get_option("multithreaded",0));
if mt>1:
extra = [ "-mt",str(mt) ];
else:
extra = []
_dprint(1,"starting meqserver",extra);
self._mqs = meqserver.default_mqs(wait_init=10,extra=extra);
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# load config file
tdlconf = self.get_option("tdlconf",".tdl.conf");
TDLOptions.config.read(tdlconf);
TDLOptions.init_options(self.name,save=False);
# compile TDL module
_dprint(1,"compiling TDL script",self.name);
try:
(self._module,ns,msg) = Compile.compile_file(self._mqs,self.name);
except:
excinfo = sys.exc_info();
self.log_exc(level=2,*excinfo);
self.fail("compile failed");
excinfo = None;
return;
# success
self.log(msg,level=2);
self.success("compile");
self._compile_failed = False;
pass;
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print "Starting meqserver";
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);
# Once we're connected to a server, some cleanup is required before we can exit the script.
# Since we want to perform this cleanup regardless of whether the script ran to completion
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf");
script = "example-sim.py";
print "========== Compiling batch job 1";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 1");
print "========== Running batch job 1";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Compiling batch job 2";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 2");
print "========== Running batch job 2";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Compiling batch job 2 with modified config";
TDLOptions.init_options("batch job 2",save=False);
TDLOptions.set_option("me.enable_G",False);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print "========== Running batch job 2";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver")
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])
# Once we're connected to a server, some cleanup is required before we can exit the script.
# Since we want to perform this cleanup regardless of whether the script ran to completion
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf")
script = "example-sim.py"
print("========== Compiling batch job 1")
mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 1")
print("========== Running batch job 1")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Compiling batch job 2")
mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 2")
print("========== Running batch job 2")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Compiling batch job 2 with modified config")
TDLOptions.init_options("batch job 2", save=False)
TDLOptions.set_option("me.enable_G", False)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print("========== Running batch job 2")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
loaded_options = True
elif set_match:
if not loaded_options:
raise RuntimeError, "Config section not yet specified"
name, value = set_match.groups()
print "### Setting option %s=%s" % (name, value)
TDLOptions.set_option(name, value, save=False, from_str=True)
elif compile_match:
script, dum, section = compile_match.groups(None)
print "### Compiling", script
if not loaded_options and not section:
# this mode reloads default config section
print "### (using options from default section)"
module, ns, msg = Compile.compile_file(mqs, script)
else:
# this mode uses explicit section, or None if section is not specified
if section:
print """### (using options from config section "%s")""" % section
else:
section = None
print "### (using previously set options)"
module, ns, msg = Compile.compile_file(mqs,
script,
config=section)
print "### ", msg
elif job_match:
if not module:
print "### Error: please specify a script before any TDL jobs"
# you may need the following line for more complicated scripts
# that use TDL options
# from Timba.TDL import TDLOptions
# this starts a kernel.
mqs = meqserver.default_mqs(wait_init=10);
# more complicated scripts might want to invoke TDLOptions here ...
# e.g. the next line of (commented out) python code loads a tdl.conf file.
# Note that it may be better to use a separate config file, rather
# than the default .tdl.conf that the browser creates.
TDLOptions.config.read(".tdl.conf");
# Now compile a script as a TDL module. Any errors will be thrown as
# an exception, so this always returns successfully. We pass in
# __file__ so as to compile ourselves.
(mod,ns,msg) = Compile.compile_file(mqs,__file__);
# This next call runs the _test_forest job.
# Note that wait should be set to True for batch processing
# so that in _test_forest the request is executed with wait=True.
# This makes sure that commands are executed in order.
mod.job_simulate(mqs,None,wait=True);
else:
Timba.TDL._dbg.set_verbose(5);
ns = NodeScope();
_define_forest(ns);
# resolves nodes
ns.Resolve();
print "importing Compile"
from Timba.TDL import Compile
print "importing TDLOptions"
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print "Starting meqserver";
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);
try:
## make simulation with perfect MODEL_DATA
script = path("testing-sim.py");
print "========== Compiling",script;
TDLOptions.config.read(path("testing.tdl.conf"));
mod,ns,msg = Compile.compile_file(mqs,script,config="simulate-model");
print "========== Simulating MODEL_DATA ";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Imaging MODEL_DATA ";
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## compare against reference image
print "========== Verifying test image ";
verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',path('test-refimage.fits'),maxdelta=1e-3);
print "========== Compiling script with modified config";
TDLOptions.init_options("simulate-model",save=False);
TDLOptions.set_option("me.g_enable",True);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print "========== Simulating DATA ";
TDLOptions.set_option("ms_sel.output_column","DATA");
def main():
for optstr in (options.debug or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
debuglevels[context] = int(level)
Timba.utils.verbosity.disable_argv()
# tell verbosity class to not parse its argv
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
Timba.utils.verbosity.set_verbosity_level(context, level)
if not args:
parser.print_help()
sys.exit(1)
if debuglevels:
octopussy.set_debug(debuglevels)
script = args[0]
tdljob = (len(args) > 1 and args[1]) or None
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# this starts a kernel.
if options.compile_only:
mqs = None
else:
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", options.mt])
TDLOptions.config.read(options.config)
TDLOptions.init_options(script)
print(("************************ Compiling TDL script", script))
# this compiles a script as a TDL module. Any errors will be thrown as
# and exception, so this always returns successfully
(mod, ns, msg) = Compile.compile_file(mqs, script)
if options.compile_only:
print(msg)
sys.exit(0)
# if a solve job is not specified, try to find one
if tdljob:
jobfunc = getattr(mod, tdljob, None)
if not jobfunc:
print(("Cannot find TDL job named", tdljob))
sys.exit(1)
else:
# does the script define an explicit job list?
joblist = getattr(mod, '_tdl_job_list', [])
if not joblist:
joblist = []
# try to build it from implicit function names
for (name, func) in list(mod.__dict__.items()):
if name.startswith("_tdl_job_") and callable(func):
joblist.append(func)
# does the script define a testing function?
testfunc = getattr(mod, '_test_forest', None)
if testfunc:
joblist.insert(0, testfunc)
if not joblist:
print(("No TDL jobs found in script", script))
sys.exit(1)
jobfunc = joblist[0]
tdljob = jobfunc.__name__
# this runs the appropriate job. wait=True is needed to wait
print(("************************ Running TDL job", tdljob))
# check if job takes a "wait" argument
(fargs, fvarargs, fvarkw, fdefaults) = inspect.getargspec(jobfunc)
if 'wait' in fargs or fvarkw:
jobopts = dict(wait=True)
else:
jobopts = {}
jobfunc(mqs, None, **jobopts)
# and start editing.
# Note that a config file may contain multiple sections -- see below.
print "Loading config"
TDLOptions.config.read("pipeline.tdl.conf")
# This compiles a script as a TDL module. Any errors will be thrown as an exception (and cause your script to stop).
#
# It is at this point that the config file loaded above takes effect. The default config section for
# a script called /foo/bar.py is simply "[bar]", so in our case the default section is "[pipeline_test]".
# TDL will automatically strip off the directory name and the .py suffix from the script filename to
# obtain a section name. See below for examples of using non-default sections.
#
# compile_file() returns a tuple of three values, the first of which will be needed below.
print "Compiling TDL script"
script = "pipeline_test.py"
mod, ns, msg = Compile.compile_file(mqs, script)
# 'mod' now refers to a Python object that is the compiled module. To execute the _test_forest job within that
# module, we do as follows:
print "Running TDL job"
mod._test_forest(mqs, None, wait=True)
# The wait=True argument causes the thing to not return until the job has been completed.
# None for the second argument tells it that we're running headless (without a GUI parent.)
### Now for some variations
# This shows how to use a different section in the config file.
print "Recompiling and running for test_a"
mod, ns, msg = Compile.compile_file(mqs, script, config="test_a")
# and this shows how to locate and call a "TDL Job" by its job ID
TDLOptions.get_job_func('job1')(mqs, None, wait=True)
from Timba.Apps import app_nogui
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.Meq import meqds
import sys
import traceback
# testing branch
if __name__ == '__main__':
mqs = meqserver.default_mqs(wait_init=10)
# mqs = meqserver.default_mqs(wait_init=5,spawn=None); # if already running
print('meqserver state:', mqs.current_server.state)
(mod, ns, msg) = Compile.compile_file(mqs, 'tdl_test.tdl')
mod._test_forest(mqs, None)
# sync=True makes sure all commands above have completed on the kernel side
# before state is fetched
state = mqs.getnodestate('solver', sync=True)
req = state.request
res = mqs.execute('x', req, wait=True)
print(res)
meq.halt()
def testMeqtreesBatchJob():
trace_sync = True;
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR;
print("========== Changing working directory to",newdir);
os.chdir(newdir);
print("========== Making required symlinks");
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))));
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(path("test-lsm.txt")))));
if not os.access(".",os.R_OK|os.W_OK):
print("Directory",os.getcwd(),"not writable, can't run tests in here.")
print("You may choose to run the tests in a different directory by giving it as an argument to this script.")
sys.exit(1);
## check if we have owlcat or owlcat.sh
owlcat = "";
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat","owlcat.sh":
tmp = os.path.join(dirname,binary);
if os.path.exists(tmp):
owlcat = tmp;
break;
if owlcat:
break;
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh");
## make simulated MS
print("========== Removing files");
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(PACKAGE_TEST_DIR));
print("========== Running makems");
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")));
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)); #TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(owlcat, PACKAGE_TEST_DIR));
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10".format(PACKAGE_TEST_DIR));
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""".format(PACKAGE_TEST_DIR));
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".format(PACKAGE_TEST_DIR));
run("""diff {0:s}/test-lsm1.txt.tmp {1:s}""".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference'))));
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(owlcat, PACKAGE_TEST_DIR));
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver");
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"));
print("========== Compiling",script);
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")));
TDLOptions.config.set("calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set("calibrate", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set("calibrate", "cal_g_diag.g_diag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name")))
TDLOptions.config.set("calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set("simulate-model", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set("simulate-model", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set("simulate-model", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set("simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))); # needs to re-read because of a Timba perculiarity
mod,ns,msg = Compile.compile_file(mqs, script, config="simulate-model");
print("========== Simulating MODEL_DATA ");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## compare against reference image
print("========== Verifying test image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")),
maxdelta=1e-3);
print("========== Compiling script with modified config");
TDLOptions.init_options("simulate-model",save=False);
TDLOptions.set_option("me.g_enable",True);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print("========== Simulating DATA ");
TDLOptions.set_option("ms_sel.output_column","DATA");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging DATA ");
TDLOptions.set_option("img_sel.imaging_column","DATA");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"));
print("========== Compiling",script);
mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate");
print("========== Calibrating ");
TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
finally:
print("Stopping meqserver");
# this halts the meqserver
meqserver.stop_default_mqs();
print("========== Making plots of solutions ");
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(owlcat, PACKAGE_TEST_DIR));
## compare against reference image
print("========== Verifying residual image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3);
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!");
# now we can exit
print("Bye!");
from Timba.Apps import app_nogui
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.Meq import meqds
import sys
import traceback
# testing branch
if __name__ == '__main__':
mqs = meqserver.default_mqs(wait_init=10);
# mqs = meqserver.default_mqs(wait_init=5,spawn=None); # if already running
print 'meqserver state:',mqs.current_server.state;
(mod,ns,msg) = Compile.compile_file(mqs,'tdl_test.tdl');
mod._test_forest(mqs,None);
# sync=True makes sure all commands above have completed on the kernel side
# before state is fetched
state = mqs.getnodestate('solver',sync=True);
req = state.request;
res = mqs.execute('x',req,wait=True);
print res;
meq.halt();
# and start editing.
# Note that a config file may contain multiple sections -- see below.
print "Loading config";
TDLOptions.config.read("pipeline.tdl.conf");
# This compiles a script as a TDL module. Any errors will be thrown as an exception (and cause your script to stop).
#
# It is at this point that the config file loaded above takes effect. The default config section for
# a script called /foo/bar.py is simply "[bar]", so in our case the default section is "[pipeline_test]".
# TDL will automatically strip off the directory name and the .py suffix from the script filename to
# obtain a section name. See below for examples of using non-default sections.
#
# compile_file() returns a tuple of three values, the first of which will be needed below.
print "Compiling TDL script";
script = "pipeline_test.py";
mod,ns,msg = Compile.compile_file(mqs,script);
# 'mod' now refers to a Python object that is the compiled module. To execute the _test_forest job within that
# module, we do as follows:
print "Running TDL job";
mod._test_forest(mqs,None,wait=True);
# The wait=True argument causes the thing to not return until the job has been completed.
# None for the second argument tells it that we're running headless (without a GUI parent.)
### Now for some variations
# This shows how to use a different section in the config file.
print "Recompiling and running for test_a";
mod,ns,msg = Compile.compile_file(mqs,script,config="test_a");
# and this shows how to locate and call a "TDL Job" by its job ID
TDLOptions.get_job_func('job1')(mqs,None,wait=True);
# you may need the following line for more complicated scripts
# that use TDL options
# from Timba.TDL import TDLOptions
# this starts a kernel.
mqs = meqserver.default_mqs(wait_init=10)
# more complicated scripts might want to invoke TDLOptions here ...
# e.g. the next line of (commented out) python code loads a tdl.conf file.
# Note that it may be better to use a separate config file, rather
# than the default .tdl.conf that the browser creates.
TDLOptions.config.read(".tdl.conf")
# Now compile a script as a TDL module. Any errors will be thrown as
# an exception, so this always returns successfully. We pass in
# __file__ so as to compile ourselves.
(mod, ns, msg) = Compile.compile_file(mqs, __file__)
# This next call runs the _test_forest job.
# Note that wait should be set to True for batch processing
# so that in _test_forest the request is executed with wait=True.
# This makes sure that commands are executed in order.
mod.job_simulate(mqs, None, wait=True)
else:
Timba.TDL._dbg.set_verbose(5)
ns = NodeScope()
_define_forest(ns)
# resolves nodes
ns.Resolve()
def testMeqtreesBatchJob():
trace_sync = True
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR
print("========== Changing working directory to", newdir)
os.chdir(newdir)
print("========== Making required symlinks")
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))))
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(
PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt")))))
if not os.access(".", os.R_OK | os.W_OK):
print("Directory", os.getcwd(),
"not writable, can't run tests in here.")
print(
"You may choose to run the tests in a different directory by giving it as an argument to this script."
)
sys.exit(1)
## check if we have owlcat or owlcat.sh
owlcat = ""
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat", "owlcat.sh":
tmp = os.path.join(dirname, binary)
if os.path.exists(tmp):
owlcat = tmp
break
if owlcat:
break
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh")
## make simulated MS
print("========== Removing files")
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(
PACKAGE_TEST_DIR))
print("========== Running makems")
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")))
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR))
#TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(
owlcat, PACKAGE_TEST_DIR))
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10"
.format(PACKAGE_TEST_DIR))
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f"""
.format(PACKAGE_TEST_DIR))
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".
format(PACKAGE_TEST_DIR))
run("""diff {0:s}/test-lsm1.txt.tmp {1:s} || diff {0:s}/test-lsm1.txt.tmp {2:s}"""
.format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')),
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm2.txt.reference'))))
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(
owlcat, PACKAGE_TEST_DIR))
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver")
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"))
print("========== Compiling", script)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")))
TDLOptions.config.set(
"calibrate", "ms_sel.msname",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set(
"calibrate", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "lsm.filename",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set(
"calibrate", "cal_g_diag.g_diag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_diag.g_diag.table_name")))
TDLOptions.config.set(
"calibrate", "cal_g_offdiag.g_offdiag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set(
"simulate-model", "lsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set(
"simulate-model", "ms_sel.msname",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set(
"simulate-model", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set(
"simulate-model", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"),
"w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf")))
# needs to re-read because of a Timba perculiarity
mod, ns, msg = Compile.compile_file(mqs,
script,
config="simulate-model")
print("========== Simulating MODEL_DATA ")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## compare against reference image
print("========== Verifying test image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR,
"test-refimage.fits")),
maxdelta=1e-3)
print("========== Compiling script with modified config")
TDLOptions.init_options("simulate-model", save=False)
TDLOptions.set_option("me.g_enable", True)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print("========== Simulating DATA ")
TDLOptions.set_option("ms_sel.output_column", "DATA")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging DATA ")
TDLOptions.set_option("img_sel.imaging_column", "DATA")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"))
print("========== Compiling", script)
mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate")
print("========== Calibrating ")
TDLOptions.get_job_func('cal_G_diag')(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
finally:
print("Stopping meqserver")
# this halts the meqserver
meqserver.stop_default_mqs()
print("========== Making plots of solutions ")
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(
owlcat, PACKAGE_TEST_DIR))
## compare against reference image
print("========== Verifying residual image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3)
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!")
# now we can exit
print("Bye!")
from Timba.TDL import Compile
print "importing TDLOptions"
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print "Starting meqserver"
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])
try:
## make simulation with perfect MODEL_DATA
script = path("testing-sim.py")
print "========== Compiling", script
TDLOptions.config.read(path("testing.tdl.conf"))
mod, ns, msg = Compile.compile_file(mqs,
script,
config="simulate-model")
print "========== Simulating MODEL_DATA "
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print "========== Imaging MODEL_DATA "
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## compare against reference image
print "========== Verifying test image "
verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',
path('test-refimage.fits'),
maxdelta=1e-3)
loaded_options = True;
elif set_match:
if not loaded_options:
raise RuntimeError,"Config section not yet specified";
name,value = set_match.groups();
print "### Setting option %s=%s"%(name,value);
TDLOptions.set_option(name,value,save=False,from_str=True);
elif compile_match:
script,dum,section = compile_match.groups(None);
print "### Compiling",script;
if not loaded_options and not section:
# this mode reloads default config section
print "### (using options from default section)";
module,ns,msg = Compile.compile_file(mqs,script);
else:
# this mode uses explicit section, or None if section is not specified
if section:
print """### (using options from config section "%s")"""%section;
else:
section = None;
print "### (using previously set options)";
module,ns,msg = Compile.compile_file(mqs,script,config=section);
print "### ",msg;
elif job_match:
if not module:
print "### Error: please specify a script before any TDL jobs";
raise RuntimeError,"TDL job specified before script";
job = job_match.group(1);
def main ():
for optstr in (options.debug or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
debuglevels[context] = int(level);
Timba.utils.verbosity.disable_argv(); # tell verbosity class to not parse its argv
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
Timba.utils.verbosity.set_verbosity_level(context,level);
if not args:
parser.print_help();
sys.exit(1);
if debuglevels:
octopussy.set_debug(debuglevels);
script = args[0];
tdljob = (len(args)>1 and args[1]) or None;
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# this starts a kernel.
if options.compile_only:
mqs = None;
else:
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",options.mt]);
TDLOptions.config.read(options.config);
TDLOptions.init_options(script);
print "************************ Compiling TDL script",script;
# this compiles a script as a TDL module. Any errors will be thrown as
# and exception, so this always returns successfully
(mod,ns,msg) = Compile.compile_file(mqs,script);
if options.compile_only:
print msg;
sys.exit(0);
# if a solve job is not specified, try to find one
if tdljob:
jobfunc = getattr(mod,tdljob,None);
if not jobfunc:
print "Cannot find TDL job named",tdljob;
sys.exit(1);
else:
# does the script define an explicit job list?
joblist = getattr(mod,'_tdl_job_list',[]);
if not joblist:
joblist = [];
# try to build it from implicit function names
for (name,func) in mod.__dict__.iteritems():
if name.startswith("_tdl_job_") and callable(func):
joblist.append(func);
# does the script define a testing function?
testfunc = getattr(mod,'_test_forest',None);
if testfunc:
joblist.insert(0,testfunc);
if not joblist:
print "No TDL jobs found in script",script;
sys.exit(1);
jobfunc = joblist[0];
tdljob = jobfunc.__name__;
# this runs the appropriate job. wait=True is needed to wait
print "************************ Running TDL job",tdljob;
# check if job takes a "wait" argument
(fargs,fvarargs,fvarkw,fdefaults) = inspect.getargspec(jobfunc);
if 'wait' in fargs or fvarkw:
jobopts = dict(wait=True);
else:
jobopts = {};
jobfunc(mqs,None,**jobopts);
