def _load_profile (self,name):
if not self.profiles.has_section(name):
QMessageBox.warning(self,"Error loading profile","""<P>Profile '%s' not found,
check your %s file.</P>"""%(name,PROFILE_FILE));
return;
for name,value in self.profiles.items(name):
try:
TDLOptions.set_option(name,value,save=False,from_str=True);
except ValueError:
dprintf(0,"can't set '%s' to %s, ignoring",name,value);
TDLOptions.save_config();
def _load_profile (self,name):
if not self.profiles.has_section(name):
QMessageBox.warning(self,"Error loading profile","""<P>Profile '%s' not found,
check your %s file.</P>"""%(name,PROFILE_FILE));
return;
for name,value in self.profiles.items(name):
try:
TDLOptions.set_option(name,value,save=False,from_str=True);
except ValueError:
dprintf(0,"can't set '%s' to %s, ignoring",name,value);
TDLOptions.save_config();
set_match = re_set_config.match(cmd);
compile_match = re_compile_script.match(cmd) or re_compile_script1.match(cmd);
job_match = re_run_job.match(cmd);
if load_match:
section = load_match.group(1);
print "### Loading config section",section;
TDLOptions.init_options(section,save=False);
loaded_options = True;
elif set_match:
if not loaded_options:
raise RuntimeError,"Config section not yet specified";
name,value = set_match.groups();
print "### Setting option %s=%s"%(name,value);
TDLOptions.set_option(name,value,save=False,from_str=True);
elif compile_match:
script,dum,section = compile_match.groups(None);
print "### Compiling",script;
if not loaded_options and not section:
# this mode reloads default config section
print "### (using options from default section)";
module,ns,msg = Compile.compile_file(mqs,script);
else:
# this mode uses explicit section, or None if section is not specified
if section:
print """### (using options from config section "%s")"""%section;
else:
section = None;
print "### (using previously set options)";
def testMeqtreesBatchJob():
trace_sync = True
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR
print("========== Changing working directory to", newdir)
os.chdir(newdir)
print("========== Making required symlinks")
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))))
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(
PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt")))))
if not os.access(".", os.R_OK | os.W_OK):
print("Directory", os.getcwd(),
"not writable, can't run tests in here.")
print(
"You may choose to run the tests in a different directory by giving it as an argument to this script."
)
sys.exit(1)
## check if we have owlcat or owlcat.sh
owlcat = ""
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat", "owlcat.sh":
tmp = os.path.join(dirname, binary)
if os.path.exists(tmp):
owlcat = tmp
break
if owlcat:
break
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh")
## make simulated MS
print("========== Removing files")
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(
PACKAGE_TEST_DIR))
print("========== Running makems")
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")))
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR))
#TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(
owlcat, PACKAGE_TEST_DIR))
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10"
.format(PACKAGE_TEST_DIR))
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f"""
.format(PACKAGE_TEST_DIR))
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".
format(PACKAGE_TEST_DIR))
run("""diff {0:s}/test-lsm1.txt.tmp {1:s} || diff {0:s}/test-lsm1.txt.tmp {2:s}"""
.format(
PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')),
path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm2.txt.reference'))))
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
.format(PACKAGE_TEST_DIR))
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(
owlcat, PACKAGE_TEST_DIR))
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver")
mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"))
print("========== Compiling", script)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")))
TDLOptions.config.set(
"calibrate", "ms_sel.msname",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set(
"calibrate", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "lsm.filename",
os.path.join(PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set(
"calibrate", "cal_g_diag.g_diag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_diag.g_diag.table_name")))
TDLOptions.config.set(
"calibrate", "cal_g_offdiag.g_offdiag.table_name",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("calibrate",
"cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set(
"simulate-model", "lsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set(
"simulate-model", "ms_sel.msname",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set(
"simulate-model", "tiggerlsm.filename",
os.path.join(
PACKAGE_TEST_DIR,
TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set(
"calibrate", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set(
"simulate-model", "img_sel.output_fitsname",
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"),
"w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(
path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf")))
# needs to re-read because of a Timba perculiarity
mod, ns, msg = Compile.compile_file(mqs,
script,
config="simulate-model")
print("========== Simulating MODEL_DATA ")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## compare against reference image
print("========== Verifying test image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR,
"test-refimage.fits")),
maxdelta=1e-3)
print("========== Compiling script with modified config")
TDLOptions.init_options("simulate-model", save=False)
TDLOptions.set_option("me.g_enable", True)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print("========== Simulating DATA ")
TDLOptions.set_option("ms_sel.output_column", "DATA")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Imaging DATA ")
TDLOptions.set_option("img_sel.imaging_column", "DATA")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"))
print("========== Compiling", script)
mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate")
print("========== Calibrating ")
TDLOptions.get_job_func('cal_G_diag')(mqs, None, wait=True)
print("========== Imaging MODEL_DATA ")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
finally:
print("Stopping meqserver")
# this halts the meqserver
meqserver.stop_default_mqs()
print("========== Making plots of solutions ")
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(
owlcat, PACKAGE_TEST_DIR))
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png"""
.format(owlcat, PACKAGE_TEST_DIR))
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(
owlcat, PACKAGE_TEST_DIR))
## compare against reference image
print("========== Verifying residual image ")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits")):
raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")):
raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR,
"WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3)
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!")
# now we can exit
print("Bye!")
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf");
script = "example-sim.py";
print "========== Compiling batch job 1";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 1");
print "========== Running batch job 1";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Compiling batch job 2";
mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 2");
print "========== Running batch job 2";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Compiling batch job 2 with modified config";
TDLOptions.init_options("batch job 2",save=False);
TDLOptions.set_option("me.enable_G",False);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print "========== Running batch job 2";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
### Cleanup time
finally:
print "Stopping meqserver";
# this halts the meqserver
meqserver.stop_default_mqs();
# now we can exit
print "Bye!";
# or was interrupted by an exception midway through, we use a try...finally block.
try:
TDLOptions.config.read("batch_sim_example.tdl.conf")
script = "example-sim.py"
print("========== Compiling batch job 1")
mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 1")
print("========== Running batch job 1")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Compiling batch job 2")
mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 2")
print("========== Running batch job 2")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print("========== Compiling batch job 2 with modified config")
TDLOptions.init_options("batch job 2", save=False)
TDLOptions.set_option("me.enable_G", False)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print("========== Running batch job 2")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
### Cleanup time
finally:
print("Stopping meqserver")
# this halts the meqserver
meqserver.stop_default_mqs()
# now we can exit
print("Bye!")
compile_match = re_compile_script.match(
cmd) or re_compile_script1.match(cmd)
job_match = re_run_job.match(cmd)
if load_match:
section = load_match.group(1)
print "### Loading config section", section
TDLOptions.init_options(section, save=False)
loaded_options = True
elif set_match:
if not loaded_options:
raise RuntimeError, "Config section not yet specified"
name, value = set_match.groups()
print "### Setting option %s=%s" % (name, value)
TDLOptions.set_option(name, value, save=False, from_str=True)
elif compile_match:
script, dum, section = compile_match.groups(None)
print "### Compiling", script
if not loaded_options and not section:
# this mode reloads default config section
print "### (using options from default section)"
module, ns, msg = Compile.compile_file(mqs, script)
else:
# this mode uses explicit section, or None if section is not specified
if section:
print """### (using options from config section "%s")""" % section
else:
section = None
print "### (using previously set options)"
script = path("testing-sim.py");
print "========== Compiling",script;
TDLOptions.config.read(path("testing.tdl.conf"));
mod,ns,msg = Compile.compile_file(mqs,script,config="simulate-model");
print "========== Simulating MODEL_DATA ";
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Imaging MODEL_DATA ";
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## compare against reference image
print "========== Verifying test image ";
verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',path('test-refimage.fits'),maxdelta=1e-3);
print "========== Compiling script with modified config";
TDLOptions.init_options("simulate-model",save=False);
TDLOptions.set_option("me.g_enable",True);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print "========== Simulating DATA ";
TDLOptions.set_option("ms_sel.output_column","DATA");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print "========== Imaging DATA ";
TDLOptions.set_option("img_sel.imaging_column","DATA");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## calibrate
script = path("testing-cal.py");
print "========== Compiling",script;
mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate");
print "========== Calibrating ";
TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True);
print "========== Imaging MODEL_DATA ";
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print "========== Imaging MODEL_DATA "
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## compare against reference image
print "========== Verifying test image "
verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',
path('test-refimage.fits'),
maxdelta=1e-3)
print "========== Compiling script with modified config"
TDLOptions.init_options("simulate-model", save=False)
TDLOptions.set_option("me.g_enable", True)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
print "========== Simulating DATA "
TDLOptions.set_option("ms_sel.output_column", "DATA")
mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
print "========== Imaging DATA "
TDLOptions.set_option("img_sel.imaging_column", "DATA")
TDLOptions.get_job_func('make_dirty_image')(mqs,
None,
wait=True,
run_viewer=False)
## calibrate
script = path("testing-cal.py")
print "========== Compiling", script
mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate")
print "Trying to call a non-existing TDL job 'job3'"
try:
TDLOptions.get_job_func('job3')(mqs, None, wait=True)
except NameError:
print "Indeed, 'job3' does not exist."
# This shows how to change configuration on-the-fly.
# TDLOptions.set_option() changes the value of a configuration variable "in memory".
# Note that compile_file() normally rereads the config file, which would defeat the point of any
# on-the-fly changes. To get around this, we can manage the config manually as follows:
print "Recompiling and running for a=5, t=[-1,1]"
# this loads initial config from section [test_b]. The save=False argument tells the system to not
# write the changes we make below to the .tdl.conf file
TDLOptions.init_options("test_b", save=False)
# this makes some changes to the loaded config
TDLOptions.set_option("a", 5)
TDLOptions.set_option("t0", -1)
TDLOptions.set_option("t1", 1)
# this compiles the script. config=None tells it to NOT reload the config file, but to use whatever we have set up above.
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
mod._test_forest(mqs, None, wait=True)
# Make another change to the current config and recompile. Again config=None is necessary,
TDLOptions.set_option("b", -5)
mod, ns, msg = Compile.compile_file(mqs, script, config=None)
mod._test_forest(mqs, None, wait=True)
# Since "a" and "b" are compile-time options, we always need to recompile for them to take effect.
# By contrast, runtime options ('t0' and 't1') can be changed without recompiling:
print "Rerunning for t=[-5,5]"
TDLOptions.set_option("t0", -5)
def testMeqtreesBatchJob():
trace_sync = True;
# sys.settrace(trace_lines);
if len(sys.argv) > 1:
newdir = PACKAGE_TEST_DIR;
print("========== Changing working directory to",newdir);
os.chdir(newdir);
print("========== Making required symlinks");
run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))));
run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(PACKAGE_TEST_DIR,
path(os.path.join(path("test-lsm.txt")))));
if not os.access(".",os.R_OK|os.W_OK):
print("Directory",os.getcwd(),"not writable, can't run tests in here.")
print("You may choose to run the tests in a different directory by giving it as an argument to this script.")
sys.exit(1);
## check if we have owlcat or owlcat.sh
owlcat = "";
for dirname in os.environ['PATH'].split(':'):
for binary in "owlcat","owlcat.sh":
tmp = os.path.join(dirname,binary);
if os.path.exists(tmp):
owlcat = tmp;
break;
if owlcat:
break;
if not owlcat:
raise RuntimeError("Can't locate owlcat or owlcat.sh");
## make simulated MS
print("========== Removing files");
run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(PACKAGE_TEST_DIR));
print("========== Running makems");
run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")));
run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)); #TODO: this is hacky, bug in CASAcore
run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(owlcat, PACKAGE_TEST_DIR));
run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10".format(PACKAGE_TEST_DIR));
# make test LSMs
run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""".format(PACKAGE_TEST_DIR));
run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".format(PACKAGE_TEST_DIR));
run("""diff {0:s}/test-lsm1.txt.tmp {1:s}""".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference'))));
run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(owlcat, PACKAGE_TEST_DIR));
print("importing meqserver")
from Timba.Apps import meqserver
print("importing Compile")
from Timba.TDL import Compile
print("importing TDLOptions")
from Timba.TDL import TDLOptions
# This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
# A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
print("Starting meqserver");
mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);
try:
## make simulation with perfect MODEL_DATA
script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"));
print("========== Compiling",script);
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")));
TDLOptions.config.set("calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname")))
TDLOptions.config.set("calibrate", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename")))
TDLOptions.config.set("calibrate", "cal_g_diag.g_diag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name")))
TDLOptions.config.set("calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name")))
TDLOptions.config.set("simulate-model", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename")))
TDLOptions.config.set("simulate-model", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname")))
TDLOptions.config.set("simulate-model", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
TDLOptions.config.set("calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
TDLOptions.config.set("simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"))
with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f:
TDLOptions.config.write(f)
TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))); # needs to re-read because of a Timba perculiarity
mod,ns,msg = Compile.compile_file(mqs, script, config="simulate-model");
print("========== Simulating MODEL_DATA ");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## compare against reference image
print("========== Verifying test image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"),
path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")),
maxdelta=1e-3);
print("========== Compiling script with modified config");
TDLOptions.init_options("simulate-model",save=False);
TDLOptions.set_option("me.g_enable",True);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
print("========== Simulating DATA ");
TDLOptions.set_option("ms_sel.output_column","DATA");
mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
print("========== Imaging DATA ");
TDLOptions.set_option("img_sel.imaging_column","DATA");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
## calibrate
script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"));
print("========== Compiling",script);
mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate");
print("========== Calibrating ");
TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True);
print("========== Imaging MODEL_DATA ");
TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
finally:
print("Stopping meqserver");
# this halts the meqserver
meqserver.stop_default_mqs();
print("========== Making plots of solutions ");
run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""".format(owlcat, PACKAGE_TEST_DIR));
run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(owlcat, PACKAGE_TEST_DIR));
## compare against reference image
print("========== Verifying residual image ");
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist")
verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
maxdelta=1e-3);
## all tests succeeded
print("========== Break out the bubbly, this hog is airborne!");
# now we can exit
print("Bye!");
print "Trying to call a non-existing TDL job 'job3'";
try:
TDLOptions.get_job_func('job3')(mqs,None,wait=True);
except NameError:
print "Indeed, 'job3' does not exist."
# This shows how to change configuration on-the-fly.
# TDLOptions.set_option() changes the value of a configuration variable "in memory".
# Note that compile_file() normally rereads the config file, which would defeat the point of any
# on-the-fly changes. To get around this, we can manage the config manually as follows:
print "Recompiling and running for a=5, t=[-1,1]";
# this loads initial config from section [test_b]. The save=False argument tells the system to not
# write the changes we make below to the .tdl.conf file
TDLOptions.init_options("test_b",save=False);
# this makes some changes to the loaded config
TDLOptions.set_option("a",5);
TDLOptions.set_option("t0",-1);
TDLOptions.set_option("t1",1);
# this compiles the script. config=None tells it to NOT reload the config file, but to use whatever we have set up above.
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
mod._test_forest(mqs,None,wait=True);
# Make another change to the current config and recompile. Again config=None is necessary,
TDLOptions.set_option("b",-5);
mod,ns,msg = Compile.compile_file(mqs,script,config=None);
mod._test_forest(mqs,None,wait=True);
# Since "a" and "b" are compile-time options, we always need to recompile for them to take effect.
# By contrast, runtime options ('t0' and 't1') can be changed without recompiling:
print "Rerunning for t=[-5,5]";
TDLOptions.set_option("t0",-5);
