def __init__ (self,name,log=True,nokill=None): # parse command line self.recording = "-assayrecord" in sys.argv; self._assay_runtime = "-noruntime" not in sys.argv; self._subset_tests = None; # all tests by default # "-assay" limits number of tests for i in range(len(sys.argv)): if sys.argv[i] == "-assay": if i >= len(sys.argv)-1: raise AssaySetupError("-assay argument must be followed by a comma-separated list of test names"); self._subset_tests = sys.argv[i+1].split(","); # if nokill is not explicitly specified, leave kernel running when recording if nokill is None: nokill = self.recording; self.name = name; if app_defaults.args.spawn: self.mqs = meqserver.default_mqs(wait_init=10,debug=False); else: self.mqs = meqserver.default_mqs(wait_init=True,debug=False); self.mqs.whenever("*",self._kernel_event_handler); self.mqs.whenever("node.result",self._node_snapshot_handler); self.tdlmod = self.logger = self.testname = None; self.watching = []; self.testname = None; self.default_tol = 1e-5; # default comparison tolerance self.time_tol = .5; # default runtime tolerance self.hostname = os.popen('hostname').read().rstrip(); self._assay_stat = 0; # open log and dump initial messages if log: self.logger = logger(self.name+".assaylog"); self.logf("start of log for assayer %s",self.name); if self._subset_tests: self.log("will only run the following tests:",*self._subset_tests);
def __init__(self, name, log=True, nokill=None): # parse command line self.recording = "-assayrecord" in sys.argv self._assay_runtime = "-noruntime" not in sys.argv self._subset_tests = None # all tests by default # "-assay" limits number of tests for i in range(len(sys.argv)): if sys.argv[i] == "-assay": if i >= len(sys.argv) - 1: raise AssaySetupError( "-assay argument must be followed by a comma-separated list of test names" ) self._subset_tests = sys.argv[i + 1].split(",") # if nokill is not explicitly specified, leave kernel running when recording if nokill is None: nokill = self.recording self.name = name if app_defaults.args.spawn: self.mqs = meqserver.default_mqs(wait_init=10, debug=False) else: self.mqs = meqserver.default_mqs(wait_init=True, debug=False) self.mqs.whenever("*", self._kernel_event_handler) self.mqs.whenever("node.result", self._node_snapshot_handler) self.tdlmod = self.logger = self.testname = None self.watching = [] self.testname = None self.default_tol = 1e-5 # default comparison tolerance self.time_tol = .5 # default runtime tolerance self.hostname = os.popen('hostname').read().rstrip() self._assay_stat = 0 # open log and dump initial messages if log: self.logger = logger(self.name + ".assaylog") self.logf("start of log for assayer %s", self.name) if self._subset_tests: self.log("will only run the following tests:", *self._subset_tests)
def compile_script(self, need_mqs=False): """compiles the script, if not already compiled. If need_mqs=True, starts a meqserver and builds the tree as well""" if not self._module: self._compile_failed = True # will reset to False if all goes well\ self.log_progress("compile") # start meqserver if required if (need_mqs or int(self.get_option("start_meqserver", 0))) and not self._mqs: from Timba.Apps import meqserver # get multithreading option mt = int(self.get_option("multithreaded", 0)) if mt > 1: extra = ["-mt", str(mt)] else: extra = [] _dprint(1, "starting meqserver", extra) self._mqs = meqserver.default_mqs(wait_init=10, extra=extra) from Timba.TDL import Compile from Timba.TDL import TDLOptions # load config file tdlconf = self.get_option("tdlconf", ".tdl.conf") TDLOptions.config.read(tdlconf) TDLOptions.init_options(self.name, save=False) # compile TDL module _dprint(1, "compiling TDL script", self.name) try: (self._module, ns, msg) = Compile.compile_file(self._mqs, self.name) except: excinfo = sys.exc_info() self.log_exc(level=2, *excinfo) self.fail("compile failed") excinfo = None return # success self.log(msg, level=2) self.success("compile") self._compile_failed = False pass
def compile_script (self,need_mqs=False): """compiles the script, if not already compiled. If need_mqs=True, starts a meqserver and builds the tree as well"""; if not self._module: self._compile_failed = True; # will reset to False if all goes well\ self.log_progress("compile"); # start meqserver if required if ( need_mqs or int(self.get_option("start_meqserver",0)) ) and not self._mqs: from Timba.Apps import meqserver # get multithreading option mt = int(self.get_option("multithreaded",0)); if mt>1: extra = [ "-mt",str(mt) ]; else: extra = [] _dprint(1,"starting meqserver",extra); self._mqs = meqserver.default_mqs(wait_init=10,extra=extra); from Timba.TDL import Compile from Timba.TDL import TDLOptions # load config file tdlconf = self.get_option("tdlconf",".tdl.conf"); TDLOptions.config.read(tdlconf); TDLOptions.init_options(self.name,save=False); # compile TDL module _dprint(1,"compiling TDL script",self.name); try: (self._module,ns,msg) = Compile.compile_file(self._mqs,self.name); except: excinfo = sys.exc_info(); self.log_exc(level=2,*excinfo); self.fail("compile failed"); excinfo = None; return; # success self.log(msg,level=2); self.success("compile"); self._compile_failed = False; pass;
# tell verbosity class to not parse argv -- we do it ourselves here Timba.utils.verbosity.disable_argv(); for optstr in (options.verbose or []): opt = optstr.split("=") + ['1']; context,level = opt[:2]; Timba.utils.verbosity.set_verbosity_level(context,int(level)); # start meqserver from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions TDLOptions.enable_save_config(False); print "### Starting meqserver"; mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",str(options.mt)]); retcode = 0; # use a try...finally block to exit meqserver cleanly at the end try: if not os.path.exists(options.config): print "Config file %s doesn't exist"%options.config; sys.exit(1); print "### Attaching to configuration file",options.config; TDLOptions.config.read(options.config); # disable the writing-out of configuration TDLOptions.config.set_save_filename(None); import re re_load_config = re.compile("^\[(.+)\]$");
def testMeqtreesBatchJob(): trace_sync = True # sys.settrace(trace_lines); if len(sys.argv) > 1: newdir = PACKAGE_TEST_DIR print("========== Changing working directory to", newdir) os.chdir(newdir) print("========== Making required symlinks") run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format( PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA")))) run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format( PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt"))))) if not os.access(".", os.R_OK | os.W_OK): print("Directory", os.getcwd(), "not writable, can't run tests in here.") print( "You may choose to run the tests in a different directory by giving it as an argument to this script." ) sys.exit(1) ## check if we have owlcat or owlcat.sh owlcat = "" for dirname in os.environ['PATH'].split(':'): for binary in "owlcat", "owlcat.sh": tmp = os.path.join(dirname, binary) if os.path.exists(tmp): owlcat = tmp break if owlcat: break if not owlcat: raise RuntimeError("Can't locate owlcat or owlcat.sh") ## make simulated MS print("========== Removing files") run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format( PACKAGE_TEST_DIR)) print("========== Running makems") run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg"))) run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)) os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0] run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)) #TODO: this is hacky, bug in CASAcore run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)) run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format( owlcat, PACKAGE_TEST_DIR)) run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10" .format(PACKAGE_TEST_DIR)) # make test LSMs run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""" .format(PACKAGE_TEST_DIR)) run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""" .format(PACKAGE_TEST_DIR)) run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""". format(PACKAGE_TEST_DIR)) run("""diff {0:s}/test-lsm1.txt.tmp {1:s} || diff {0:s}/test-lsm1.txt.tmp {2:s}""" .format( PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')), path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm2.txt.reference')))) run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""" .format(PACKAGE_TEST_DIR)) run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format( owlcat, PACKAGE_TEST_DIR)) run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format( owlcat, PACKAGE_TEST_DIR)) print("importing meqserver") from Timba.Apps import meqserver print("importing Compile") from Timba.TDL import Compile print("importing TDLOptions") from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print("Starting meqserver") mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"]) try: ## make simulation with perfect MODEL_DATA script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py")) print("========== Compiling", script) TDLOptions.config.read( path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf"))) TDLOptions.config.set( "calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname"))) TDLOptions.config.set( "calibrate", "tiggerlsm.filename", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename"))) TDLOptions.config.set( "calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename"))) TDLOptions.config.set( "calibrate", "cal_g_diag.g_diag.table_name", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name"))) TDLOptions.config.set( "calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name"))) TDLOptions.config.set( "simulate-model", "lsm.filename", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename"))) TDLOptions.config.set( "simulate-model", "ms_sel.msname", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname"))) TDLOptions.config.set( "simulate-model", "tiggerlsm.filename", os.path.join( PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename"))) TDLOptions.config.set( "calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")) TDLOptions.config.set( "simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")) with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f: TDLOptions.config.write(f) TDLOptions.config.read( path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))) # needs to re-read because of a Timba perculiarity mod, ns, msg = Compile.compile_file(mqs, script, config="simulate-model") print("========== Simulating MODEL_DATA ") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Imaging MODEL_DATA ") TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) ## compare against reference image print("========== Verifying test image ") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"), path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")), maxdelta=1e-3) print("========== Compiling script with modified config") TDLOptions.init_options("simulate-model", save=False) TDLOptions.set_option("me.g_enable", True) mod, ns, msg = Compile.compile_file(mqs, script, config=None) print("========== Simulating DATA ") TDLOptions.set_option("ms_sel.output_column", "DATA") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Imaging DATA ") TDLOptions.set_option("img_sel.imaging_column", "DATA") TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) ## calibrate script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py")) print("========== Compiling", script) mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate") print("========== Calibrating ") TDLOptions.get_job_func('cal_G_diag')(mqs, None, wait=True) print("========== Imaging MODEL_DATA ") TDLOptions.get_job_func('make_dirty_image')(mqs, None, wait=True, run_viewer=False) finally: print("Stopping meqserver") # this halts the meqserver meqserver.stop_default_mqs() print("========== Making plots of solutions ") run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""" .format(owlcat, PACKAGE_TEST_DIR)) run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format( owlcat, PACKAGE_TEST_DIR)) run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""" .format(owlcat, PACKAGE_TEST_DIR)) run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format( owlcat, PACKAGE_TEST_DIR)) ## compare against reference image print("========== Verifying residual image ") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists( os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"), os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"), maxdelta=1e-3) ## all tests succeeded print("========== Break out the bubbly, this hog is airborne!") # now we can exit print("Bye!")
# python demo_script.py -run -dmeqserver=3 # # This dumps various messages to stdout, which let you keep # track of how the script is progressing. # # Here's the code required to handle the '-run' flag if '-run' in sys.argv: from Timba.Apps import meqserver from Timba.TDL import Compile # you may need the following line for more complicated scripts # that use TDL options # from Timba.TDL import TDLOptions # this starts a kernel. mqs = meqserver.default_mqs(wait_init=10) # more complicated scripts might want to invoke TDLOptions here ... # e.g. the next line of (commented out) python code loads a tdl.conf file. # Note that it may be better to use a separate config file, rather # than the default .tdl.conf that the browser creates. TDLOptions.config.read(".tdl.conf") # Now compile a script as a TDL module. Any errors will be thrown as # an exception, so this always returns successfully. We pass in # __file__ so as to compile ourselves. (mod, ns, msg) = Compile.compile_file(mqs, __file__) # This next call runs the _test_forest job. # Note that wait should be set to True for batch processing # so that in _test_forest the request is executed with wait=True.
#!/usr/bin/python # -*- coding: utf-8 -*- # # This shows how to run a TDL script in a pipeline (aka batch-mode, aka headless mode) # if __name__ == '__main__': from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print "Starting meqserver"; mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]); # Once we're connected to a server, some cleanup is required before we can exit the script. # Since we want to perform this cleanup regardless of whether the script ran to completion # or was interrupted by an exception midway through, we use a try...finally block. try: TDLOptions.config.read("batch_sim_example.tdl.conf"); script = "example-sim.py"; print "========== Compiling batch job 1"; mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 1"); print "========== Running batch job 1"; mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print "========== Compiling batch job 2"; mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 2"); print "========== Running batch job 2";
# # This shows how to run a TDL script in a pipeline (aka batch-mode, aka headless mode) # from __future__ import absolute_import from __future__ import print_function from __future__ import division if __name__ == '__main__': from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print("Starting meqserver") mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"]) # Once we're connected to a server, some cleanup is required before we can exit the script. # Since we want to perform this cleanup regardless of whether the script ran to completion # or was interrupted by an exception midway through, we use a try...finally block. try: TDLOptions.config.read("batch_sim_example.tdl.conf") script = "example-sim.py" print("========== Compiling batch job 1") mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 1") print("========== Running batch job 1") mod._tdl_job_1_simulate_MS(mqs, None, wait=True) print("========== Compiling batch job 2")
debuglevels[context] = int(level) # tell verbosity class to not parse argv -- we do it ourselves here Timba.utils.verbosity.disable_argv() for optstr in (options.verbose or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] Timba.utils.verbosity.set_verbosity_level(context, int(level)) # start meqserver from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions TDLOptions.enable_save_config(False) print "### Starting meqserver" mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", str(options.mt)]) retcode = 0 # use a try...finally block to exit meqserver cleanly at the end try: if not os.path.exists(options.config): print "Config file %s doesn't exist" % options.config sys.exit(1) print "### Attaching to configuration file", options.config TDLOptions.config.read(options.config) # disable the writing-out of configuration TDLOptions.config.set_save_filename(None) import re re_load_config = re.compile("^\[(.+)\]$")
# python demo_script.py -run -dmeqserver=3 # # This dumps various messages to stdout, which let you keep # track of how the script is progressing. # # Here's the code required to handle the '-run' flag if '-run' in sys.argv: from Timba.Apps import meqserver from Timba.TDL import Compile # you may need the following line for more complicated scripts # that use TDL options # from Timba.TDL import TDLOptions # this starts a kernel. mqs = meqserver.default_mqs(wait_init=10); # more complicated scripts might want to invoke TDLOptions here ... # e.g. the next line of (commented out) python code loads a tdl.conf file. # Note that it may be better to use a separate config file, rather # than the default .tdl.conf that the browser creates. TDLOptions.config.read(".tdl.conf"); # Now compile a script as a TDL module. Any errors will be thrown as # an exception, so this always returns successfully. We pass in # __file__ so as to compile ourselves. (mod,ns,msg) = Compile.compile_file(mqs,__file__); # This next call runs the _test_forest job. # Note that wait should be set to True for batch processing # so that in _test_forest the request is executed with wait=True.
# tell verbosity class to not parse argv -- we do it ourselves here Timba.utils.verbosity.disable_argv() for optstr in (options.verbose or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] Timba.utils.verbosity.set_verbosity_level(context, int(level)) # start meqserver from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions TDLOptions.enable_save_config(False) print("### Starting meqserver") mqs = meqserver.default_mqs( wait_init=10, extra=["-mt", str(options.mt)] + (["-python_memprof"] if options.memprof else [])) retcode = 0 # use a try...finally block to exit meqserver cleanly at the end try: if not os.path.exists(options.config): print(("Config file %s doesn't exist" % options.config)) sys.exit(1) print(("### Attaching to configuration file", options.config)) TDLOptions.config.read(options.config) # disable the writing-out of configuration TDLOptions.config.set_save_filename(None) # but save it manually if options.save_config:
# tell verbosity class to not parse argv -- we do it ourselves here Timba.utils.verbosity.disable_argv(); for optstr in (options.verbose or []): opt = optstr.split("=") + ['1']; context,level = opt[:2]; Timba.utils.verbosity.set_verbosity_level(context,int(level)); # start meqserver from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions TDLOptions.enable_save_config(False); print "### Starting meqserver"; mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",str(options.mt)]+(["-python_memprof"] if options.memprof else [])); retcode = 0; # use a try...finally block to exit meqserver cleanly at the end try: if not os.path.exists(options.config): print "Config file %s doesn't exist"%options.config; sys.exit(1); print "### Attaching to configuration file",options.config; TDLOptions.config.read(options.config); # disable the writing-out of configuration TDLOptions.config.set_save_filename(None); # but save it manually if options.save_config: if ':' in options.save_config:
def main (): for optstr in (options.debug or []): opt = optstr.split("=") + ['1']; context,level = opt[:2]; debuglevels[context] = int(level); Timba.utils.verbosity.disable_argv(); # tell verbosity class to not parse its argv for optstr in (options.verbose or []): opt = optstr.split("=") + ['1']; context,level = opt[:2]; Timba.utils.verbosity.set_verbosity_level(context,level); if not args: parser.print_help(); sys.exit(1); if debuglevels: octopussy.set_debug(debuglevels); script = args[0]; tdljob = (len(args)>1 and args[1]) or None; from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions # this starts a kernel. if options.compile_only: mqs = None; else: mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",options.mt]); TDLOptions.config.read(options.config); TDLOptions.init_options(script); print "************************ Compiling TDL script",script; # this compiles a script as a TDL module. Any errors will be thrown as # and exception, so this always returns successfully (mod,ns,msg) = Compile.compile_file(mqs,script); if options.compile_only: print msg; sys.exit(0); # if a solve job is not specified, try to find one if tdljob: jobfunc = getattr(mod,tdljob,None); if not jobfunc: print "Cannot find TDL job named",tdljob; sys.exit(1); else: # does the script define an explicit job list? joblist = getattr(mod,'_tdl_job_list',[]); if not joblist: joblist = []; # try to build it from implicit function names for (name,func) in mod.__dict__.iteritems(): if name.startswith("_tdl_job_") and callable(func): joblist.append(func); # does the script define a testing function? testfunc = getattr(mod,'_test_forest',None); if testfunc: joblist.insert(0,testfunc); if not joblist: print "No TDL jobs found in script",script; sys.exit(1); jobfunc = joblist[0]; tdljob = jobfunc.__name__; # this runs the appropriate job. wait=True is needed to wait print "************************ Running TDL job",tdljob; # check if job takes a "wait" argument (fargs,fvarargs,fvarkw,fdefaults) = inspect.getargspec(jobfunc); if 'wait' in fargs or fvarkw: jobopts = dict(wait=True); else: jobopts = {}; jobfunc(mqs,None,**jobopts);
def testMeqtreesBatchJob(): trace_sync = True; # sys.settrace(trace_lines); if len(sys.argv) > 1: newdir = PACKAGE_TEST_DIR; print("========== Changing working directory to",newdir); os.chdir(newdir); print("========== Making required symlinks"); run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA")))); run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt"))))); if not os.access(".",os.R_OK|os.W_OK): print("Directory",os.getcwd(),"not writable, can't run tests in here.") print("You may choose to run the tests in a different directory by giving it as an argument to this script.") sys.exit(1); ## check if we have owlcat or owlcat.sh owlcat = ""; for dirname in os.environ['PATH'].split(':'): for binary in "owlcat","owlcat.sh": tmp = os.path.join(dirname,binary); if os.path.exists(tmp): owlcat = tmp; break; if owlcat: break; if not owlcat: raise RuntimeError("Can't locate owlcat or owlcat.sh"); ## make simulated MS print("========== Removing files"); run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(PACKAGE_TEST_DIR)); print("========== Running makems"); run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg"))); run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)); os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0] run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)); #TODO: this is hacky, bug in CASAcore run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR)); run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(owlcat, PACKAGE_TEST_DIR)); run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10".format(PACKAGE_TEST_DIR)); # make test LSMs run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""".format(PACKAGE_TEST_DIR)); run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR)); run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".format(PACKAGE_TEST_DIR)); run("""diff {0:s}/test-lsm1.txt.tmp {1:s}""".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')))); run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR)); run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(owlcat, PACKAGE_TEST_DIR)); print("importing meqserver") from Timba.Apps import meqserver print("importing Compile") from Timba.TDL import Compile print("importing TDLOptions") from Timba.TDL import TDLOptions # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads. # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv). print("Starting meqserver"); mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]); try: ## make simulation with perfect MODEL_DATA script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py")); print("========== Compiling",script); TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf"))); TDLOptions.config.set("calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname"))) TDLOptions.config.set("calibrate", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename"))) TDLOptions.config.set("calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename"))) TDLOptions.config.set("calibrate", "cal_g_diag.g_diag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name"))) TDLOptions.config.set("calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name"))) TDLOptions.config.set("simulate-model", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename"))) TDLOptions.config.set("simulate-model", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname"))) TDLOptions.config.set("simulate-model", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename"))) TDLOptions.config.set("calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")) TDLOptions.config.set("simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")) with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f: TDLOptions.config.write(f) TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))); # needs to re-read because of a Timba perculiarity mod,ns,msg = Compile.compile_file(mqs, script, config="simulate-model"); print("========== Simulating MODEL_DATA "); mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print("========== Imaging MODEL_DATA "); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); ## compare against reference image print("========== Verifying test image "); if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"), path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")), maxdelta=1e-3); print("========== Compiling script with modified config"); TDLOptions.init_options("simulate-model",save=False); TDLOptions.set_option("me.g_enable",True); mod,ns,msg = Compile.compile_file(mqs,script,config=None); print("========== Simulating DATA "); TDLOptions.set_option("ms_sel.output_column","DATA"); mod._tdl_job_1_simulate_MS(mqs,None,wait=True); print("========== Imaging DATA "); TDLOptions.set_option("img_sel.imaging_column","DATA"); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); ## calibrate script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py")); print("========== Compiling",script); mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate"); print("========== Calibrating "); TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True); print("========== Imaging MODEL_DATA "); TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False); finally: print("Stopping meqserver"); # this halts the meqserver meqserver.stop_default_mqs(); print("========== Making plots of solutions "); run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""".format(owlcat, PACKAGE_TEST_DIR)); run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(owlcat, PACKAGE_TEST_DIR)); ## compare against reference image print("========== Verifying residual image "); if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist") if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist") verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"), os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"), maxdelta=1e-3); ## all tests succeeded print("========== Break out the bubbly, this hog is airborne!"); # now we can exit print("Bye!");
def main(): for optstr in (options.debug or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] debuglevels[context] = int(level) Timba.utils.verbosity.disable_argv() # tell verbosity class to not parse its argv for optstr in (options.verbose or []): opt = optstr.split("=") + ['1'] context, level = opt[:2] Timba.utils.verbosity.set_verbosity_level(context, level) if not args: parser.print_help() sys.exit(1) if debuglevels: octopussy.set_debug(debuglevels) script = args[0] tdljob = (len(args) > 1 and args[1]) or None from Timba.Apps import meqserver from Timba.TDL import Compile from Timba.TDL import TDLOptions # this starts a kernel. if options.compile_only: mqs = None else: mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", options.mt]) TDLOptions.config.read(options.config) TDLOptions.init_options(script) print(("************************ Compiling TDL script", script)) # this compiles a script as a TDL module. Any errors will be thrown as # and exception, so this always returns successfully (mod, ns, msg) = Compile.compile_file(mqs, script) if options.compile_only: print(msg) sys.exit(0) # if a solve job is not specified, try to find one if tdljob: jobfunc = getattr(mod, tdljob, None) if not jobfunc: print(("Cannot find TDL job named", tdljob)) sys.exit(1) else: # does the script define an explicit job list? joblist = getattr(mod, '_tdl_job_list', []) if not joblist: joblist = [] # try to build it from implicit function names for (name, func) in list(mod.__dict__.items()): if name.startswith("_tdl_job_") and callable(func): joblist.append(func) # does the script define a testing function? testfunc = getattr(mod, '_test_forest', None) if testfunc: joblist.insert(0, testfunc) if not joblist: print(("No TDL jobs found in script", script)) sys.exit(1) jobfunc = joblist[0] tdljob = jobfunc.__name__ # this runs the appropriate job. wait=True is needed to wait print(("************************ Running TDL job", tdljob)) # check if job takes a "wait" argument (fargs, fvarargs, fvarkw, fdefaults) = inspect.getargspec(jobfunc) if 'wait' in fargs or fvarkw: jobopts = dict(wait=True) else: jobopts = {} jobfunc(mqs, None, **jobopts)