def compile (self,script): try: self.script = script; _dprint(0,"compiling",script); (self.tdlmod,ns,msg) = Compile.compile_file(self.mqs,script); _dprintf(0,"compiled %s: %s",script,msg); lines = ["compiled TDL script %s: "%script]+msg.rstrip().split('\n'); self.logs(*[ s+'\n' for s in lines ]); except: self.logexc(); raise;
def compile(self, script): try: self.script = script _dprint(0, "compiling", script) (self.tdlmod, ns, msg) = Compile.compile_file(self.mqs, script) _dprintf(0, "compiled %s: %s", script, msg) lines = ["compiled TDL script %s: " % script ] + msg.rstrip().split('\n') self.logs(*[s + '\n' for s in lines]) except: self.logexc() raise
def compile_script(self, need_mqs=False): """compiles the script, if not already compiled. If need_mqs=True, starts a meqserver and builds the tree as well""" if not self._module: self._compile_failed = True # will reset to False if all goes well\ self.log_progress("compile") # start meqserver if required if (need_mqs or int(self.get_option("start_meqserver", 0))) and not self._mqs: from Timba.Apps import meqserver # get multithreading option mt = int(self.get_option("multithreaded", 0)) if mt > 1: extra = ["-mt", str(mt)] else: extra = [] _dprint(1, "starting meqserver", extra) self._mqs = meqserver.default_mqs(wait_init=10, extra=extra) from Timba.TDL import Compile from Timba.TDL import TDLOptions # load config file tdlconf = self.get_option("tdlconf", ".tdl.conf") TDLOptions.config.read(tdlconf) TDLOptions.init_options(self.name, save=False) # compile TDL module _dprint(1, "compiling TDL script", self.name) try: (self._module, ns, msg) = Compile.compile_file(self._mqs, self.name) except: excinfo = sys.exc_info() self.log_exc(level=2, *excinfo) self.fail("compile failed") excinfo = None return # success self.log(msg, level=2) self.success("compile") self._compile_failed = False pass
def compile_script (self,need_mqs=False): """compiles the script, if not already compiled. If need_mqs=True, starts a meqserver and builds the tree as well"""; if not self._module: self._compile_failed = True; # will reset to False if all goes well\ self.log_progress("compile"); # start meqserver if required if ( need_mqs or int(self.get_option("start_meqserver",0)) ) and not self._mqs: from Timba.Apps import meqserver # get multithreading option mt = int(self.get_option("multithreaded",0)); if mt>1: extra = [ "-mt",str(mt) ]; else: extra = [] _dprint(1,"starting meqserver",extra); self._mqs = meqserver.default_mqs(wait_init=10,extra=extra); from Timba.TDL import Compile from Timba.TDL import TDLOptions # load config file tdlconf = self.get_option("tdlconf",".tdl.conf"); TDLOptions.config.read(tdlconf); TDLOptions.init_options(self.name,save=False); # compile TDL module _dprint(1,"compiling TDL script",self.name); try: (self._module,ns,msg) = Compile.compile_file(self._mqs,self.name); except: excinfo = sys.exc_info(); self.log_exc(level=2,*excinfo); self.fail("compile failed"); excinfo = None; return; # success self.log(msg,level=2); self.success("compile"); self._compile_failed = False; pass;
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print "Starting meqserver"; mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]); # Once we're connected to a server, some cleanup is required before we can exit the script. # Since we want to perform this cleanup regardless of whether the script ran to completion # or was interrupted by an exception midway through, we use a try...finally block. try: TDLOptions.config.read("batch_sim_example.tdl.conf"); script = "example-sim.py"; print "========== Compiling batch job 1"; mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 1"); print "========== Running batch job 1"; mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print "========== Compiling batch job 2"; mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 2"); print "========== Running batch job 2"; mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print "========== Compiling batch job 2 with modified config"; TDLOptions.init_options("batch job 2",save=False); TDLOptions.set_option("me.enable_G",False); mod,ns,msg = Compile.compile_file(mqs,script,config=None); print "========== Running batch job 2"; mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print("Starting meqserver") mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"]) # Once we're connected to a server, some cleanup is required before we can exit the script. # Since we want to perform this cleanup regardless of whether the script ran to completion # or was interrupted by an exception midway through, we use a try...finally block. try: TDLOptions.config.read("batch_sim_example.tdl.conf") script = "example-sim.py" print("========== Compiling batch job 1") mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 1") print("========== Running batch job 1") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Compiling batch job 2") mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 2") print("========== Running batch job 2") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Compiling batch job 2 with modified config") TDLOptions.init_options("batch job 2", save=False) TDLOptions.set_option("me.enable_G", False) mod, ns, msg = Compile.compile_file(mqs, script, config=None) print("========== Running batch job 2") mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
loaded_options = True elif set_match: if not loaded_options: raise RuntimeError, "Config section not yet specified" name, value = set_match.groups() print "### Setting option %s=%s" % (name, value) TDLOptions.set_option(name, value, save=False, from_str=True) elif compile_match: script, dum, section = compile_match.groups(None) print "### Compiling", script if not loaded_options and not section: # this mode reloads default config section print "### (using options from default section)" module, ns, msg = Compile.compile_file(mqs, script) else: # this mode uses explicit section, or None if section is not specified if section: print """### (using options from config section "%s")""" % section else: section = None print "### (using previously set options)" module, ns, msg = Compile.compile_file(mqs, script, config=section) print "### ", msg elif job_match: if not module: print "### Error: please specify a script before any TDL jobs"
# you may need the following line for more complicated scripts # that use TDL options # from Timba.TDL import TDLOptions # this starts a kernel. mqs = meqserver.default_mqs(wait_init=10); # more complicated scripts might want to invoke TDLOptions here ... # e.g. the next line of (commented out) python code loads a tdl.conf file. # Note that it may be better to use a separate config file, rather # than the default .tdl.conf that the browser creates. TDLOptions.config.read(".tdl.conf"); # Now compile a script as a TDL module. Any errors will be thrown as # an exception, so this always returns successfully. We pass in # __file__ so as to compile ourselves. (mod,ns,msg) = Compile.compile_file(mqs,__file__); # This next call runs the _test_forest job. # Note that wait should be set to True for batch processing # so that in _test_forest the request is executed with wait=True. # This makes sure that commands are executed in order. mod.job_simulate(mqs,None,wait=True); else: Timba.TDL._dbg.set_verbose(5); ns = NodeScope(); _define_forest(ns); # resolves nodes ns.Resolve();
print "importing Compile" from Timba.TDL import Compile print "importing TDLOptions" from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print "Starting meqserver"; mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]); try: ## make simulation with perfect MODEL_DATA script = path("testing-sim.py"); print "========== Compiling",script; TDLOptions.config.read(path("testing.tdl.conf")); mod,ns,msg = Compile.compile_file(mqs,script,config="simulate-model"); print "========== Simulating MODEL_DATA "; mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print "========== Imaging MODEL_DATA "; TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); ## compare against reference image print "========== Verifying test image "; verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',path('test-refimage.fits'),maxdelta=1e-3); print "========== Compiling script with modified config"; TDLOptions.init_options("simulate-model",save=False); TDLOptions.set_option("me.g_enable",True); mod,ns,msg = Compile.compile_file(mqs,script,config=None); print "========== Simulating DATA "; TDLOptions.set_option("ms_sel.output_column","DATA");
def main(): for optstr in (options.debug or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] debuglevels[context] = int(level) Timba.utils.verbosity.disable_argv() # tell verbosity class to not parse its argv for optstr in (options.verbose or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] Timba.utils.verbosity.set_verbosity_level(context, level) if not args: parser.print_help() sys.exit(1) if debuglevels: octopussy.set_debug(debuglevels) script = args[0] tdljob = (len(args) > 1 and args[1]) or None from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions # this starts a kernel. if options.compile_only: mqs = None else: mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", options.mt]) TDLOptions.config.read(options.config) TDLOptions.init_options(script) print(("************************ Compiling TDL script", script)) # this compiles a script as a TDL module. Any errors will be thrown as # and exception, so this always returns successfully (mod, ns, msg) = Compile.compile_file(mqs, script) if options.compile_only: print(msg) sys.exit(0) # if a solve job is not specified, try to find one if tdljob: jobfunc = getattr(mod, tdljob, None) if not jobfunc: print(("Cannot find TDL job named", tdljob)) sys.exit(1) else: # does the script define an explicit job list? joblist = getattr(mod, '_tdl_job_list', []) if not joblist: joblist = [] # try to build it from implicit function names for (name, func) in list(mod.__dict__.items()): if name.startswith("_tdl_job_") and callable(func): joblist.append(func) # does the script define a testing function? testfunc = getattr(mod, '_test_forest', None) if testfunc: joblist.insert(0, testfunc) if not joblist: print(("No TDL jobs found in script", script)) sys.exit(1) jobfunc = joblist[0] tdljob = jobfunc.__name__ # this runs the appropriate job. wait=True is needed to wait print(("************************ Running TDL job", tdljob)) # check if job takes a "wait" argument (fargs, fvarargs, fvarkw, fdefaults) = inspect.getargspec(jobfunc) if 'wait' in fargs or fvarkw: jobopts = dict(wait=True) else: jobopts = {} jobfunc(mqs, None, **jobopts)
# and start editing. # Note that a config file may contain multiple sections -- see below. print "Loading config" TDLOptions.config.read("pipeline.tdl.conf") # This compiles a script as a TDL module. Any errors will be thrown as an exception (and cause your script to stop). # # It is at this point that the config file loaded above takes effect. The default config section for # a script called /foo/bar.py is simply "[bar]", so in our case the default section is "[pipeline_test]". # TDL will automatically strip off the directory name and the .py suffix from the script filename to # obtain a section name. See below for examples of using non-default sections. # # compile_file() returns a tuple of three values, the first of which will be needed below. print "Compiling TDL script" script = "pipeline_test.py" mod, ns, msg = Compile.compile_file(mqs, script) # 'mod' now refers to a Python object that is the compiled module. To execute the _test_forest job within that # module, we do as follows: print "Running TDL job" mod._test_forest(mqs, None, wait=True) # The wait=True argument causes the thing to not return until the job has been completed. # None for the second argument tells it that we're running headless (without a GUI parent.) ### Now for some variations # This shows how to use a different section in the config file. print "Recompiling and running for test_a" mod, ns, msg = Compile.compile_file(mqs, script, config="test_a") # and this shows how to locate and call a "TDL Job" by its job ID TDLOptions.get_job_func('job1')(mqs, None, wait=True)
from Timba.Apps import app_nogui from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.Meq import meqds import sys import traceback # testing branch if __name__ == '__main__': mqs = meqserver.default_mqs(wait_init=10) # mqs = meqserver.default_mqs(wait_init=5,spawn=None); # if already running print('meqserver state:', mqs.current_server.state) (mod, ns, msg) = Compile.compile_file(mqs, 'tdl_test.tdl') mod._test_forest(mqs, None) # sync=True makes sure all commands above have completed on the kernel side # before state is fetched state = mqs.getnodestate('solver', sync=True) req = state.request res = mqs.execute('x', req, wait=True) print(res) meq.halt()
def testMeqtreesBatchJob(): trace_sync = True; # sys.settrace(trace_lines); if len(sys.argv) > 1: newdir = PACKAGE_TEST_DIR; print("========== Changing working directory to",newdir); os.chdir(newdir); print("========== Making required symlinks"); run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA")))); run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt"))))); if not os.access(".",os.R_OK|os.W_OK): print("Directory",os.getcwd(),"not writable, can't run tests in here.") print("You may choose to run the tests in a different directory by giving it as an argument to this script.") sys.exit(1); ## check if we have owlcat or owlcat.sh owlcat = ""; for dirname in os.environ['PATH'].split(':'): for binary in "owlcat","owlcat.sh": tmp = os.path.join(dirname,binary); if os.path.exists(tmp): owlcat = tmp; break; if owlcat: break; if not owlcat: raise RuntimeError("Can't locate owlcat or owlcat.sh"); ## make simulated MS print("========== Removing files"); run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(PACKAGE_TEST_DIR)); print("========== Running makems"); run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg"))); run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)); os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0] run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)); #TODO: this is hacky, bug in CASAcore run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)); run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(owlcat, PACKAGE_TEST_DIR)); run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10".format(PACKAGE_TEST_DIR)); # make test LSMs run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""".format(PACKAGE_TEST_DIR)); run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR)); run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".format(PACKAGE_TEST_DIR)); run("""diff {0:s}/test-lsm1.txt.tmp {1:s}""".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')))); run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR)); run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(owlcat, PACKAGE_TEST_DIR)); print("importing meqserver") from Timba.Apps import meqserver print("importing Compile") from Timba.TDL import Compile print("importing TDLOptions") from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print("Starting meqserver"); mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]); try: ## make simulation with perfect MODEL_DATA script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py")); print("========== Compiling",script); TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf"))); TDLOptions.config.set("calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname"))) TDLOptions.config.set("calibrate", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename"))) TDLOptions.config.set("calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename"))) TDLOptions.config.set("calibrate", "cal_g_diag.g_diag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name"))) TDLOptions.config.set("calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name"))) TDLOptions.config.set("simulate-model", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename"))) TDLOptions.config.set("simulate-model", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname"))) TDLOptions.config.set("simulate-model", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename"))) TDLOptions.config.set("calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")) TDLOptions.config.set("simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")) with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f: TDLOptions.config.write(f) TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))); # needs to re-read because of a Timba perculiarity mod,ns,msg = Compile.compile_file(mqs, script, config="simulate-model"); print("========== Simulating MODEL_DATA "); mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print("========== Imaging MODEL_DATA "); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); ## compare against reference image print("========== Verifying test image "); if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"), path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")), maxdelta=1e-3); print("========== Compiling script with modified config"); TDLOptions.init_options("simulate-model",save=False); TDLOptions.set_option("me.g_enable",True); mod,ns,msg = Compile.compile_file(mqs,script,config=None); print("========== Simulating DATA "); TDLOptions.set_option("ms_sel.output_column","DATA"); mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print("========== Imaging DATA "); TDLOptions.set_option("img_sel.imaging_column","DATA"); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); ## calibrate script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py")); print("========== Compiling",script); mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate"); print("========== Calibrating "); TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True); print("========== Imaging MODEL_DATA "); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); finally: print("Stopping meqserver"); # this halts the meqserver meqserver.stop_default_mqs(); print("========== Making plots of solutions "); run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(owlcat, PACKAGE_TEST_DIR)); ## compare against reference image print("========== Verifying residual image "); if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"), os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"), maxdelta=1e-3); ## all tests succeeded print("========== Break out the bubbly, this hog is airborne!"); # now we can exit print("Bye!");
from Timba.Apps import app_nogui from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.Meq import meqds import sys import traceback # testing branch if __name__ == '__main__': mqs = meqserver.default_mqs(wait_init=10); # mqs = meqserver.default_mqs(wait_init=5,spawn=None); # if already running print 'meqserver state:',mqs.current_server.state; (mod,ns,msg) = Compile.compile_file(mqs,'tdl_test.tdl'); mod._test_forest(mqs,None); # sync=True makes sure all commands above have completed on the kernel side # before state is fetched state = mqs.getnodestate('solver',sync=True); req = state.request; res = mqs.execute('x',req,wait=True); print res; meq.halt();
# and start editing. # Note that a config file may contain multiple sections -- see below. print "Loading config"; TDLOptions.config.read("pipeline.tdl.conf"); # This compiles a script as a TDL module. Any errors will be thrown as an exception (and cause your script to stop). # # It is at this point that the config file loaded above takes effect. The default config section for # a script called /foo/bar.py is simply "[bar]", so in our case the default section is "[pipeline_test]". # TDL will automatically strip off the directory name and the .py suffix from the script filename to # obtain a section name. See below for examples of using non-default sections. # # compile_file() returns a tuple of three values, the first of which will be needed below. print "Compiling TDL script"; script = "pipeline_test.py"; mod,ns,msg = Compile.compile_file(mqs,script); # 'mod' now refers to a Python object that is the compiled module. To execute the _test_forest job within that # module, we do as follows: print "Running TDL job"; mod._test_forest(mqs,None,wait=True); # The wait=True argument causes the thing to not return until the job has been completed. # None for the second argument tells it that we're running headless (without a GUI parent.) ### Now for some variations # This shows how to use a different section in the config file. print "Recompiling and running for test_a"; mod,ns,msg = Compile.compile_file(mqs,script,config="test_a"); # and this shows how to locate and call a "TDL Job" by its job ID TDLOptions.get_job_func('job1')(mqs,None,wait=True);
# you may need the following line for more complicated scripts # that use TDL options # from Timba.TDL import TDLOptions # this starts a kernel. mqs = meqserver.default_mqs(wait_init=10) # more complicated scripts might want to invoke TDLOptions here ... # e.g. the next line of (commented out) python code loads a tdl.conf file. # Note that it may be better to use a separate config file, rather # than the default .tdl.conf that the browser creates. TDLOptions.config.read(".tdl.conf") # Now compile a script as a TDL module. Any errors will be thrown as # an exception, so this always returns successfully. We pass in # __file__ so as to compile ourselves. (mod, ns, msg) = Compile.compile_file(mqs, __file__) # This next call runs the _test_forest job. # Note that wait should be set to True for batch processing # so that in _test_forest the request is executed with wait=True. # This makes sure that commands are executed in order. mod.job_simulate(mqs, None, wait=True) else: Timba.TDL._dbg.set_verbose(5) ns = NodeScope() _define_forest(ns) # resolves nodes ns.Resolve()
def testMeqtreesBatchJob(): trace_sync = True # sys.settrace(trace_lines); if len(sys.argv) > 1: newdir = PACKAGE_TEST_DIR print("========== Changing working directory to", newdir) os.chdir(newdir) print("========== Making required symlinks") run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format( PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA")))) run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format( PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt"))))) if not os.access(".", os.R_OK | os.W_OK): print("Directory", os.getcwd(), "not writable, can't run tests in here.") print( "You may choose to run the tests in a different directory by giving it as an argument to this script." ) sys.exit(1) ## check if we have owlcat or owlcat.sh owlcat = "" for dirname in os.environ['PATH'].split(':'): for binary in "owlcat", "owlcat.sh": tmp = os.path.join(dirname, binary) if os.path.exists(tmp): owlcat = tmp break if owlcat: break if not owlcat: raise RuntimeError("Can't locate owlcat or owlcat.sh") ## make simulated MS print("========== Removing files") run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format( PACKAGE_TEST_DIR)) print("========== Running makems") run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg"))) run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)) os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0] run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)) #TODO: this is hacky, bug in CASAcore run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)) run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format( owlcat, PACKAGE_TEST_DIR)) run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10" .format(PACKAGE_TEST_DIR)) # make test LSMs run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""" .format(PACKAGE_TEST_DIR)) run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""" .format(PACKAGE_TEST_DIR)) run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""". format(PACKAGE_TEST_DIR)) run("""diff {0:s}/test-lsm1.txt.tmp {1:s} || diff {0:s}/test-lsm1.txt.tmp {2:s}""" .format( PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')), path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm2.txt.reference')))) run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""" .format(PACKAGE_TEST_DIR)) run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format( owlcat, PACKAGE_TEST_DIR)) run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format( owlcat, PACKAGE_TEST_DIR)) print("importing meqserver") from Timba.Apps import meqserver print("importing Compile") from Timba.TDL import Compile print("importing TDLOptions") from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print("Starting meqserver") mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"]) try: ## make simulation with perfect MODEL_DATA script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py")) print("========== Compiling", script) TDLOptions.config.read( path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf"))) TDLOptions.config.set( "calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname"))) TDLOptions.config.set( "calibrate", "tiggerlsm.filename", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename"))) TDLOptions.config.set( "calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename"))) TDLOptions.config.set( "calibrate", "cal_g_diag.g_diag.table_name", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name"))) TDLOptions.config.set( "calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name"))) TDLOptions.config.set( "simulate-model", "lsm.filename", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename"))) TDLOptions.config.set( "simulate-model", "ms_sel.msname", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname"))) TDLOptions.config.set( "simulate-model", "tiggerlsm.filename", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename"))) TDLOptions.config.set( "calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")) TDLOptions.config.set( "simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")) with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f: TDLOptions.config.write(f) TDLOptions.config.read( path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))) # needs to re-read because of a Timba perculiarity mod, ns, msg = Compile.compile_file(mqs, script, config="simulate-model") print("========== Simulating MODEL_DATA ") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Imaging MODEL_DATA ") TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) ## compare against reference image print("========== Verifying test image ") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"), path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")), maxdelta=1e-3) print("========== Compiling script with modified config") TDLOptions.init_options("simulate-model", save=False) TDLOptions.set_option("me.g_enable", True) mod, ns, msg = Compile.compile_file(mqs, script, config=None) print("========== Simulating DATA ") TDLOptions.set_option("ms_sel.output_column", "DATA") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Imaging DATA ") TDLOptions.set_option("img_sel.imaging_column", "DATA") TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) ## calibrate script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py")) print("========== Compiling", script) mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate") print("========== Calibrating ") TDLOptions.get_job_func('cal_G_diag')(mqs, None, wait=True) print("========== Imaging MODEL_DATA ") TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) finally: print("Stopping meqserver") # this halts the meqserver meqserver.stop_default_mqs() print("========== Making plots of solutions ") run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""" .format(owlcat, PACKAGE_TEST_DIR)) run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format( owlcat, PACKAGE_TEST_DIR)) run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""" .format(owlcat, PACKAGE_TEST_DIR)) run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format( owlcat, PACKAGE_TEST_DIR)) ## compare against reference image print("========== Verifying residual image ") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"), os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"), maxdelta=1e-3) ## all tests succeeded print("========== Break out the bubbly, this hog is airborne!") # now we can exit print("Bye!")
from Timba.TDL import Compile print "importing TDLOptions" from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print "Starting meqserver" mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"]) try: ## make simulation with perfect MODEL_DATA script = path("testing-sim.py") print "========== Compiling", script TDLOptions.config.read(path("testing.tdl.conf")) mod, ns, msg = Compile.compile_file(mqs, script, config="simulate-model") print "========== Simulating MODEL_DATA " mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print "========== Imaging MODEL_DATA " TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) ## compare against reference image print "========== Verifying test image " verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits', path('test-refimage.fits'), maxdelta=1e-3)
loaded_options = True; elif set_match: if not loaded_options: raise RuntimeError,"Config section not yet specified"; name,value = set_match.groups(); print "### Setting option %s=%s"%(name,value); TDLOptions.set_option(name,value,save=False,from_str=True); elif compile_match: script,dum,section = compile_match.groups(None); print "### Compiling",script; if not loaded_options and not section: # this mode reloads default config section print "### (using options from default section)"; module,ns,msg = Compile.compile_file(mqs,script); else: # this mode uses explicit section, or None if section is not specified if section: print """### (using options from config section "%s")"""%section; else: section = None; print "### (using previously set options)"; module,ns,msg = Compile.compile_file(mqs,script,config=section); print "### ",msg; elif job_match: if not module: print "### Error: please specify a script before any TDL jobs"; raise RuntimeError,"TDL job specified before script"; job = job_match.group(1);
def main (): for optstr in (options.debug or []): opt = optstr.split("=") + ['1']; context,level = opt[:2]; debuglevels[context] = int(level); Timba.utils.verbosity.disable_argv(); # tell verbosity class to not parse its argv for optstr in (options.verbose or []): opt = optstr.split("=") + ['1']; context,level = opt[:2]; Timba.utils.verbosity.set_verbosity_level(context,level); if not args: parser.print_help(); sys.exit(1); if debuglevels: octopussy.set_debug(debuglevels); script = args[0]; tdljob = (len(args)>1 and args[1]) or None; from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions # this starts a kernel. if options.compile_only: mqs = None; else: mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",options.mt]); TDLOptions.config.read(options.config); TDLOptions.init_options(script); print "************************ Compiling TDL script",script; # this compiles a script as a TDL module. Any errors will be thrown as # and exception, so this always returns successfully (mod,ns,msg) = Compile.compile_file(mqs,script); if options.compile_only: print msg; sys.exit(0); # if a solve job is not specified, try to find one if tdljob: jobfunc = getattr(mod,tdljob,None); if not jobfunc: print "Cannot find TDL job named",tdljob; sys.exit(1); else: # does the script define an explicit job list? joblist = getattr(mod,'_tdl_job_list',[]); if not joblist: joblist = []; # try to build it from implicit function names for (name,func) in mod.__dict__.iteritems(): if name.startswith("_tdl_job_") and callable(func): joblist.append(func); # does the script define a testing function? testfunc = getattr(mod,'_test_forest',None); if testfunc: joblist.insert(0,testfunc); if not joblist: print "No TDL jobs found in script",script; sys.exit(1); jobfunc = joblist[0]; tdljob = jobfunc.__name__; # this runs the appropriate job. wait=True is needed to wait print "************************ Running TDL job",tdljob; # check if job takes a "wait" argument (fargs,fvarargs,fvarkw,fdefaults) = inspect.getargspec(jobfunc); if 'wait' in fargs or fvarkw: jobopts = dict(wait=True); else: jobopts = {}; jobfunc(mqs,None,**jobopts);