def __init__ (self,name,log=True,nokill=None):
# parse command line
self.recording = "-assayrecord" in sys.argv;
self._assay_runtime = "-noruntime" not in sys.argv;
self._subset_tests = None; # all tests by default
# "-assay" limits number of tests
for i in range(len(sys.argv)):
if sys.argv[i] == "-assay":
if i >= len(sys.argv)-1:
raise AssaySetupError("-assay argument must be followed by a comma-separated list of test names");
self._subset_tests = sys.argv[i+1].split(",");
# if nokill is not explicitly specified, leave kernel running when recording
if nokill is None:
nokill = self.recording;
self.name = name;
if app_defaults.args.spawn:
self.mqs = meqserver.default_mqs(wait_init=10,debug=False);
else:
self.mqs = meqserver.default_mqs(wait_init=True,debug=False);
self.mqs.whenever("*",self._kernel_event_handler);
self.mqs.whenever("node.result",self._node_snapshot_handler);
self.tdlmod = self.logger = self.testname = None;
self.watching = [];
self.testname = None;
self.default_tol = 1e-5; # default comparison tolerance
self.time_tol = .5; # default runtime tolerance
self.hostname = os.popen('hostname').read().rstrip();
self._assay_stat = 0;
# open log and dump initial messages
if log:
self.logger = logger(self.name+".assaylog");
self.logf("start of log for assayer %s",self.name);
if self._subset_tests:
self.log("will only run the following tests:",*self._subset_tests);
def __init__(self, name, log=True, nokill=None):
# parse command line
self.recording = "-assayrecord" in sys.argv
self._assay_runtime = "-noruntime" not in sys.argv
self._subset_tests = None
# all tests by default
# "-assay" limits number of tests
for i in range(len(sys.argv)):
if sys.argv[i] == "-assay":
if i >= len(sys.argv) - 1:
raise AssaySetupError(
"-assay argument must be followed by a comma-separated list of test names"
)
self._subset_tests = sys.argv[i + 1].split(",")
# if nokill is not explicitly specified, leave kernel running when recording
if nokill is None:
nokill = self.recording
self.name = name
if app_defaults.args.spawn:
self.mqs = meqserver.default_mqs(wait_init=10, debug=False)
else:
self.mqs = meqserver.default_mqs(wait_init=True, debug=False)
self.mqs.whenever("*", self._kernel_event_handler)
self.mqs.whenever("node.result", self._node_snapshot_handler)
self.tdlmod = self.logger = self.testname = None
self.watching = []
self.testname = None
self.default_tol = 1e-5
# default comparison tolerance
self.time_tol = .5
# default runtime tolerance
self.hostname = os.popen('hostname').read().rstrip()
self._assay_stat = 0
# open log and dump initial messages
if log:
self.logger = logger(self.name + ".assaylog")
self.logf("start of log for assayer %s", self.name)
if self._subset_tests:
self.log("will only run the following tests:", *self._subset_tests)
def compile_script(self, need_mqs=False):
"""compiles the script, if not already compiled.
If need_mqs=True, starts a meqserver and builds the tree as well"""
if not self._module:
self._compile_failed = True
# will reset to False if all goes well\
self.log_progress("compile")
# start meqserver if required
if (need_mqs or int(self.get_option("start_meqserver",
0))) and not self._mqs:
from Timba.Apps import meqserver
# get multithreading option
mt = int(self.get_option("multithreaded", 0))
if mt > 1:
extra = ["-mt", str(mt)]
else:
extra = []
_dprint(1, "starting meqserver", extra)
self._mqs = meqserver.default_mqs(wait_init=10, extra=extra)
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# load config file
tdlconf = self.get_option("tdlconf", ".tdl.conf")
TDLOptions.config.read(tdlconf)
TDLOptions.init_options(self.name, save=False)
# compile TDL module
_dprint(1, "compiling TDL script", self.name)
try:
(self._module, ns,
msg) = Compile.compile_file(self._mqs, self.name)
except:
excinfo = sys.exc_info()
self.log_exc(level=2, *excinfo)
self.fail("compile failed")
excinfo = None
return
# success
self.log(msg, level=2)
self.success("compile")
self._compile_failed = False
pass
def compile_script (self,need_mqs=False):
"""compiles the script, if not already compiled.
If need_mqs=True, starts a meqserver and builds the tree as well""";
if not self._module:
self._compile_failed = True; # will reset to False if all goes well\
self.log_progress("compile");
# start meqserver if required
if ( need_mqs or int(self.get_option("start_meqserver",0)) ) and not self._mqs:
from Timba.Apps import meqserver
# get multithreading option
mt = int(self.get_option("multithreaded",0));
if mt>1:
extra = [ "-mt",str(mt) ];
else:
extra = []
_dprint(1,"starting meqserver",extra);
self._mqs = meqserver.default_mqs(wait_init=10,extra=extra);
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# load config file
tdlconf = self.get_option("tdlconf",".tdl.conf");
TDLOptions.config.read(tdlconf);
TDLOptions.init_options(self.name,save=False);
# compile TDL module
_dprint(1,"compiling TDL script",self.name);
try:
(self._module,ns,msg) = Compile.compile_file(self._mqs,self.name);
except:
excinfo = sys.exc_info();
self.log_exc(level=2,*excinfo);
self.fail("compile failed");
excinfo = None;
return;
# success
self.log(msg,level=2);
self.success("compile");
self._compile_failed = False;
pass;
# tell verbosity class to not parse argv -- we do it ourselves here
Timba.utils.verbosity.disable_argv();
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
Timba.utils.verbosity.set_verbosity_level(context,int(level));
# start meqserver
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
TDLOptions.enable_save_config(False);
print "### Starting meqserver";
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",str(options.mt)]);
retcode = 0;
# use a try...finally block to exit meqserver cleanly at the end
try:
if not os.path.exists(options.config):
print "Config file %s doesn't exist"%options.config;
sys.exit(1);
print "### Attaching to configuration file",options.config;
TDLOptions.config.read(options.config);
# disable the writing-out of configuration
TDLOptions.config.set_save_filename(None);
import re
re_load_config = re.compile("^\[(.+)\]$");
def testMeqtreesBatchJob():
trace_sync = True
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR
print("========== Changing working directory to", newdir)
os.chdir(newdir)
print("========== Making required symlinks")
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))))
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(
PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt")))))
if not os.access(".", os.R_OK | os.W_OK):
print("Directory", os.getcwd(),
"not writable, can't run tests in here.")
print(
"You may choose to run the tests in a different directory by giving it as an argument to this script."
)
sys.exit(1)
## check if we have owlcat or owlcat.sh
owlcat = ""
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat", "owlcat.sh":
tmp = os.path.join(dirname, binary)
if os.path.exists(tmp):
owlcat = tmp
break
if owlcat:
break
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh")
## make simulated MS
print("========== Removing files")
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(
PACKAGE_TEST_DIR))
print("========== Running makems")
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")))
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR))
#TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(
owlcat, PACKAGE_TEST_DIR))
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10"
.format(PACKAGE_TEST_DIR))
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f"""
.format(PACKAGE_TEST_DIR))
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".
format(PACKAGE_TEST_DIR))
run("""diff {0:s}/test-lsm1.txt.tmp {1:s} || diff {0:s}/test-lsm1.txt.tmp {2:s}"""
.format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')),
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm2.txt.reference'))))
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(
owlcat, PACKAGE_TEST_DIR))
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver")
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"))
print("========== Compiling", script)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")))
TDLOptions.config.set(
"calibrate", "ms_sel.msname",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set(
"calibrate", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "lsm.filename",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set(
"calibrate", "cal_g_diag.g_diag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_diag.g_diag.table_name")))
TDLOptions.config.set(
"calibrate", "cal_g_offdiag.g_offdiag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set(
"simulate-model", "lsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set(
"simulate-model", "ms_sel.msname",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set(
"simulate-model", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set(
"simulate-model", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"),
"w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf")))
# needs to re-read because of a Timba perculiarity
mod, ns, msg = Compile.compile_file(mqs,
script,
config="simulate-model")
print("========== Simulating MODEL_DATA ")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## compare against reference image
print("========== Verifying test image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR,
"test-refimage.fits")),
maxdelta=1e-3)
print("========== Compiling script with modified config")
TDLOptions.init_options("simulate-model", save=False)
TDLOptions.set_option("me.g_enable", True)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print("========== Simulating DATA ")
TDLOptions.set_option("ms_sel.output_column", "DATA")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging DATA ")
TDLOptions.set_option("img_sel.imaging_column", "DATA")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"))
print("========== Compiling", script)
mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate")
print("========== Calibrating ")
TDLOptions.get_job_func('cal_G_diag')(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
finally:
print("Stopping meqserver")
# this halts the meqserver
meqserver.stop_default_mqs()
print("========== Making plots of solutions ")
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(
owlcat, PACKAGE_TEST_DIR))
## compare against reference image
print("========== Verifying residual image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3)
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!")
# now we can exit
print("Bye!")
# python demo_script.py -run -dmeqserver=3
#
# This dumps various messages to stdout, which let you keep
# track of how the script is progressing.
#
# Here's the code required to handle the '-run' flag
if '-run' in sys.argv:
from Timba.Apps import meqserver
from Timba.TDL import Compile
# you may need the following line for more complicated scripts
# that use TDL options
# from Timba.TDL import TDLOptions
# this starts a kernel.
mqs = meqserver.default_mqs(wait_init=10)
# more complicated scripts might want to invoke TDLOptions here ...
# e.g. the next line of (commented out) python code loads a tdl.conf file.
# Note that it may be better to use a separate config file, rather
# than the default .tdl.conf that the browser creates.
TDLOptions.config.read(".tdl.conf")
# Now compile a script as a TDL module. Any errors will be thrown as
# an exception, so this always returns successfully. We pass in
# __file__ so as to compile ourselves.
(mod, ns, msg) = Compile.compile_file(mqs, __file__)
# This next call runs the _test_forest job.
# Note that wait should be set to True for batch processing
# so that in _test_forest the request is executed with wait=True.
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# This shows how to run a TDL script in a pipeline (aka batch-mode, aka headless mode)
#
if __name__ == '__main__':
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print "Starting meqserver";
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);
# Once we're connected to a server, some cleanup is required before we can exit the script.
# Since we want to perform this cleanup regardless of whether the script ran to completion
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf");
script = "example-sim.py";
print "========== Compiling batch job 1";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 1");
print "========== Running batch job 1";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Compiling batch job 2";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 2");
print "========== Running batch job 2";
#
# This shows how to run a TDL script in a pipeline (aka batch-mode, aka headless mode)
#
from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
if __name__ == '__main__':
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver")
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])
# Once we're connected to a server, some cleanup is required before we can exit the script.
# Since we want to perform this cleanup regardless of whether the script ran to completion
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf")
script = "example-sim.py"
print("========== Compiling batch job 1")
mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 1")
print("========== Running batch job 1")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Compiling batch job 2")
debuglevels[context] = int(level)
# tell verbosity class to not parse argv -- we do it ourselves here
Timba.utils.verbosity.disable_argv()
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
Timba.utils.verbosity.set_verbosity_level(context, int(level))
# start meqserver
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
TDLOptions.enable_save_config(False)
print "### Starting meqserver"
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt",
str(options.mt)])
retcode = 0
# use a try...finally block to exit meqserver cleanly at the end
try:
if not os.path.exists(options.config):
print "Config file %s doesn't exist" % options.config
sys.exit(1)
print "### Attaching to configuration file", options.config
TDLOptions.config.read(options.config)
# disable the writing-out of configuration
TDLOptions.config.set_save_filename(None)
import re
re_load_config = re.compile("^\[(.+)\]$")
# python demo_script.py -run -dmeqserver=3
#
# This dumps various messages to stdout, which let you keep
# track of how the script is progressing.
#
# Here's the code required to handle the '-run' flag
if '-run' in sys.argv:
from Timba.Apps import meqserver
from Timba.TDL import Compile
# you may need the following line for more complicated scripts
# that use TDL options
# from Timba.TDL import TDLOptions
# this starts a kernel.
mqs = meqserver.default_mqs(wait_init=10);
# more complicated scripts might want to invoke TDLOptions here ...
# e.g. the next line of (commented out) python code loads a tdl.conf file.
# Note that it may be better to use a separate config file, rather
# than the default .tdl.conf that the browser creates.
TDLOptions.config.read(".tdl.conf");
# Now compile a script as a TDL module. Any errors will be thrown as
# an exception, so this always returns successfully. We pass in
# __file__ so as to compile ourselves.
(mod,ns,msg) = Compile.compile_file(mqs,__file__);
# This next call runs the _test_forest job.
# Note that wait should be set to True for batch processing
# so that in _test_forest the request is executed with wait=True.
# tell verbosity class to not parse argv -- we do it ourselves here
Timba.utils.verbosity.disable_argv()
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
Timba.utils.verbosity.set_verbosity_level(context, int(level))
# start meqserver
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
TDLOptions.enable_save_config(False)
print("### Starting meqserver")
mqs = meqserver.default_mqs(
wait_init=10,
extra=["-mt", str(options.mt)] +
(["-python_memprof"] if options.memprof else []))
retcode = 0
# use a try...finally block to exit meqserver cleanly at the end
try:
if not os.path.exists(options.config):
print(("Config file %s doesn't exist" % options.config))
sys.exit(1)
print(("### Attaching to configuration file", options.config))
TDLOptions.config.read(options.config)
# disable the writing-out of configuration
TDLOptions.config.set_save_filename(None)
# but save it manually
if options.save_config:
# tell verbosity class to not parse argv -- we do it ourselves here
Timba.utils.verbosity.disable_argv();
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
Timba.utils.verbosity.set_verbosity_level(context,int(level));
# start meqserver
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
TDLOptions.enable_save_config(False);
print "### Starting meqserver";
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",str(options.mt)]+(["-python_memprof"] if options.memprof else []));
retcode = 0;
# use a try...finally block to exit meqserver cleanly at the end
try:
if not os.path.exists(options.config):
print "Config file %s doesn't exist"%options.config;
sys.exit(1);
print "### Attaching to configuration file",options.config;
TDLOptions.config.read(options.config);
# disable the writing-out of configuration
TDLOptions.config.set_save_filename(None);
# but save it manually
if options.save_config:
if ':' in options.save_config:
def main ():
for optstr in (options.debug or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
debuglevels[context] = int(level);
Timba.utils.verbosity.disable_argv(); # tell verbosity class to not parse its argv
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1'];
context,level = opt[:2];
Timba.utils.verbosity.set_verbosity_level(context,level);
if not args:
parser.print_help();
sys.exit(1);
if debuglevels:
octopussy.set_debug(debuglevels);
script = args[0];
tdljob = (len(args)>1 and args[1]) or None;
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# this starts a kernel.
if options.compile_only:
mqs = None;
else:
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt",options.mt]);
TDLOptions.config.read(options.config);
TDLOptions.init_options(script);
print "************************ Compiling TDL script",script;
# this compiles a script as a TDL module. Any errors will be thrown as
# and exception, so this always returns successfully
(mod,ns,msg) = Compile.compile_file(mqs,script);
if options.compile_only:
print msg;
sys.exit(0);
# if a solve job is not specified, try to find one
if tdljob:
jobfunc = getattr(mod,tdljob,None);
if not jobfunc:
print "Cannot find TDL job named",tdljob;
sys.exit(1);
else:
# does the script define an explicit job list?
joblist = getattr(mod,'_tdl_job_list',[]);
if not joblist:
joblist = [];
# try to build it from implicit function names
for (name,func) in mod.__dict__.iteritems():
if name.startswith("_tdl_job_") and callable(func):
joblist.append(func);
# does the script define a testing function?
testfunc = getattr(mod,'_test_forest',None);
if testfunc:
joblist.insert(0,testfunc);
if not joblist:
print "No TDL jobs found in script",script;
sys.exit(1);
jobfunc = joblist[0];
tdljob = jobfunc.__name__;
# this runs the appropriate job. wait=True is needed to wait
print "************************ Running TDL job",tdljob;
# check if job takes a "wait" argument
(fargs,fvarargs,fvarkw,fdefaults) = inspect.getargspec(jobfunc);
if 'wait' in fargs or fvarkw:
jobopts = dict(wait=True);
else:
jobopts = {};
jobfunc(mqs,None,**jobopts);
def testMeqtreesBatchJob():
trace_sync = True;
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR;
print("========== Changing working directory to",newdir);
os.chdir(newdir);
print("========== Making required symlinks");
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))));
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(path("test-lsm.txt")))));
if not os.access(".",os.R_OK|os.W_OK):
print("Directory",os.getcwd(),"not writable, can't run tests in here.")
print("You may choose to run the tests in a different directory by giving it as an argument to this script.")
sys.exit(1);
## check if we have owlcat or owlcat.sh
owlcat = "";
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat","owlcat.sh":
tmp = os.path.join(dirname,binary);
if os.path.exists(tmp):
owlcat = tmp;
break;
if owlcat:
break;
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh");
## make simulated MS
print("========== Removing files");
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(PACKAGE_TEST_DIR));
print("========== Running makems");
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")));
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)); #TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(owlcat, PACKAGE_TEST_DIR));
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10".format(PACKAGE_TEST_DIR));
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""".format(PACKAGE_TEST_DIR));
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".format(PACKAGE_TEST_DIR));
run("""diff {0:s}/test-lsm1.txt.tmp {1:s}""".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference'))));
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(owlcat, PACKAGE_TEST_DIR));
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver");
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"));
print("========== Compiling",script);
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")));
TDLOptions.config.set("calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set("calibrate", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set("calibrate", "cal_g_diag.g_diag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name")))
TDLOptions.config.set("calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set("simulate-model", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set("simulate-model", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set("simulate-model", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set("simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))); # needs to re-read because of a Timba perculiarity
mod,ns,msg = Compile.compile_file(mqs, script, config="simulate-model");
print("========== Simulating MODEL_DATA ");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## compare against reference image
print("========== Verifying test image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")),
maxdelta=1e-3);
print("========== Compiling script with modified config");
TDLOptions.init_options("simulate-model",save=False);
TDLOptions.set_option("me.g_enable",True);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print("========== Simulating DATA ");
TDLOptions.set_option("ms_sel.output_column","DATA");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging DATA ");
TDLOptions.set_option("img_sel.imaging_column","DATA");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"));
print("========== Compiling",script);
mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate");
print("========== Calibrating ");
TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
finally:
print("Stopping meqserver");
# this halts the meqserver
meqserver.stop_default_mqs();
print("========== Making plots of solutions ");
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(owlcat, PACKAGE_TEST_DIR));
## compare against reference image
print("========== Verifying residual image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3);
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!");
# now we can exit
print("Bye!");
def main():
for optstr in (options.debug or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
debuglevels[context] = int(level)
Timba.utils.verbosity.disable_argv()
# tell verbosity class to not parse its argv
for optstr in (options.verbose or []):
opt = optstr.split("=") + ['1']
context, level = opt[:2]
Timba.utils.verbosity.set_verbosity_level(context, level)
if not args:
parser.print_help()
sys.exit(1)
if debuglevels:
octopussy.set_debug(debuglevels)
script = args[0]
tdljob = (len(args) > 1 and args[1]) or None
from Timba.Apps import meqserver
from Timba.TDL import Compile
from Timba.TDL import TDLOptions
# this starts a kernel.
if options.compile_only:
mqs = None
else:
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", options.mt])
TDLOptions.config.read(options.config)
TDLOptions.init_options(script)
print(("************************ Compiling TDL script", script))
# this compiles a script as a TDL module. Any errors will be thrown as
# and exception, so this always returns successfully
(mod, ns, msg) = Compile.compile_file(mqs, script)
if options.compile_only:
print(msg)
sys.exit(0)
# if a solve job is not specified, try to find one
if tdljob:
jobfunc = getattr(mod, tdljob, None)
if not jobfunc:
print(("Cannot find TDL job named", tdljob))
sys.exit(1)
else:
# does the script define an explicit job list?
joblist = getattr(mod, '_tdl_job_list', [])
if not joblist:
joblist = []
# try to build it from implicit function names
for (name, func) in list(mod.__dict__.items()):
if name.startswith("_tdl_job_") and callable(func):
joblist.append(func)
# does the script define a testing function?
testfunc = getattr(mod, '_test_forest', None)
if testfunc:
joblist.insert(0, testfunc)
if not joblist:
print(("No TDL jobs found in script", script))
sys.exit(1)
jobfunc = joblist[0]
tdljob = jobfunc.__name__
# this runs the appropriate job. wait=True is needed to wait
print(("************************ Running TDL job", tdljob))
# check if job takes a "wait" argument
(fargs, fvarargs, fvarkw, fdefaults) = inspect.getargspec(jobfunc)
if 'wait' in fargs or fvarkw:
jobopts = dict(wait=True)
else:
jobopts = {}
jobfunc(mqs, None, **jobopts)
