def _load_profile (self,name):
   if not self.profiles.has_section(name):
     QMessageBox.warning(self,"Error loading profile","""<P>Profile '%s' not found,
             check your %s file.</P>"""%(name,PROFILE_FILE));
     return;
   for name,value in self.profiles.items(name):
     try:
       TDLOptions.set_option(name,value,save=False,from_str=True);
     except ValueError:
       dprintf(0,"can't set '%s' to %s, ignoring",name,value);
   TDLOptions.save_config();
示例#2
0
 def _load_profile (self,name):
   if not self.profiles.has_section(name):
     QMessageBox.warning(self,"Error loading profile","""<P>Profile '%s' not found,
             check your %s file.</P>"""%(name,PROFILE_FILE));
     return;
   for name,value in self.profiles.items(name):
     try:
       TDLOptions.set_option(name,value,save=False,from_str=True);
     except ValueError:
       dprintf(0,"can't set '%s' to %s, ignoring",name,value);
   TDLOptions.save_config();
      set_match  = re_set_config.match(cmd);
      compile_match = re_compile_script.match(cmd) or re_compile_script1.match(cmd);
      job_match = re_run_job.match(cmd);

      if load_match:
        section = load_match.group(1);
        print "### Loading config section",section;
        TDLOptions.init_options(section,save=False);
        loaded_options = True;

      elif set_match:
        if not loaded_options:
          raise RuntimeError,"Config section not yet specified";
        name,value = set_match.groups();
        print "### Setting option %s=%s"%(name,value);
        TDLOptions.set_option(name,value,save=False,from_str=True);

      elif compile_match:
        script,dum,section = compile_match.groups(None);
        print "### Compiling",script;
        if not loaded_options and not section:
          # this mode reloads default config section
          print "### (using options from default section)";
          module,ns,msg = Compile.compile_file(mqs,script);
        else:
          # this mode uses explicit section, or None if section is not specified
          if section:
            print """### (using options from config section "%s")"""%section;
          else:
            section = None;
            print "### (using previously set options)";
示例#4
0
def testMeqtreesBatchJob():
    trace_sync = True
    #  sys.settrace(trace_lines);

    if len(sys.argv) > 1:
        newdir = PACKAGE_TEST_DIR
        print("========== Changing working directory to", newdir)
        os.chdir(newdir)
        print("========== Making required symlinks")
        run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(
            PACKAGE_TEST_DIR,
            path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))))
        run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(
            PACKAGE_TEST_DIR, path(os.path.join(path("test-lsm.txt")))))

    if not os.access(".", os.R_OK | os.W_OK):
        print("Directory", os.getcwd(),
              "not writable, can't run tests in here.")
        print(
            "You may choose to run the tests in a different directory by giving it as an argument to this script."
        )
        sys.exit(1)

    ## check if we have owlcat or owlcat.sh
    owlcat = ""
    for dirname in os.environ['PATH'].split(':'):
        for binary in "owlcat", "owlcat.sh":
            tmp = os.path.join(dirname, binary)
            if os.path.exists(tmp):
                owlcat = tmp
                break
        if owlcat:
            break
    if not owlcat:
        raise RuntimeError("Can't locate owlcat or owlcat.sh")

    ## make simulated MS
    print("========== Removing files")

    run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(
        PACKAGE_TEST_DIR))
    print("========== Running makems")
    run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")))
    run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
    os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
    run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR))
    #TODO: this is hacky, bug in CASAcore
    run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR))
    run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(
        owlcat, PACKAGE_TEST_DIR))
    run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10"
        .format(PACKAGE_TEST_DIR))
    # make test LSMs
    run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f"""
        .format(PACKAGE_TEST_DIR))
    run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
        .format(PACKAGE_TEST_DIR))
    run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".
        format(PACKAGE_TEST_DIR))
    run("""diff {0:s}/test-lsm1.txt.tmp {1:s} || diff {0:s}/test-lsm1.txt.tmp {2:s}"""
        .format(
            PACKAGE_TEST_DIR,
            path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference')),
            path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm2.txt.reference'))))
    run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f"""
        .format(PACKAGE_TEST_DIR))
    run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(
        owlcat, PACKAGE_TEST_DIR))
    run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(
        owlcat, PACKAGE_TEST_DIR))

    print("importing meqserver")
    from Timba.Apps import meqserver
    print("importing Compile")
    from Timba.TDL import Compile
    print("importing TDLOptions")
    from Timba.TDL import TDLOptions

    # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
    # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
    print("Starting meqserver")
    mqs = meqserver.default_mqs(wait_init=10, extra=["-mt", "2"])

    try:
        ## make simulation with perfect MODEL_DATA
        script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"))
        print("========== Compiling", script)
        TDLOptions.config.read(
            path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")))
        TDLOptions.config.set(
            "calibrate", "ms_sel.msname",
            os.path.join(PACKAGE_TEST_DIR,
                         TDLOptions.config.get("calibrate", "ms_sel.msname")))
        TDLOptions.config.set(
            "calibrate", "tiggerlsm.filename",
            os.path.join(
                PACKAGE_TEST_DIR,
                TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
        TDLOptions.config.set(
            "calibrate", "lsm.filename",
            os.path.join(PACKAGE_TEST_DIR,
                         TDLOptions.config.get("calibrate", "lsm.filename")))
        TDLOptions.config.set(
            "calibrate", "cal_g_diag.g_diag.table_name",
            os.path.join(
                PACKAGE_TEST_DIR,
                TDLOptions.config.get("calibrate",
                                      "cal_g_diag.g_diag.table_name")))
        TDLOptions.config.set(
            "calibrate", "cal_g_offdiag.g_offdiag.table_name",
            os.path.join(
                PACKAGE_TEST_DIR,
                TDLOptions.config.get("calibrate",
                                      "cal_g_offdiag.g_offdiag.table_name")))
        TDLOptions.config.set(
            "simulate-model", "lsm.filename",
            os.path.join(
                PACKAGE_TEST_DIR,
                TDLOptions.config.get("simulate-model", "lsm.filename")))
        TDLOptions.config.set(
            "simulate-model", "ms_sel.msname",
            os.path.join(
                PACKAGE_TEST_DIR,
                TDLOptions.config.get("simulate-model", "ms_sel.msname")))
        TDLOptions.config.set(
            "simulate-model", "tiggerlsm.filename",
            os.path.join(
                PACKAGE_TEST_DIR,
                TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
        TDLOptions.config.set(
            "calibrate", "img_sel.output_fitsname",
            os.path.join(PACKAGE_TEST_DIR,
                         "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
        TDLOptions.config.set(
            "simulate-model", "img_sel.output_fitsname",
            os.path.join(PACKAGE_TEST_DIR,
                         "WSRT.MS.MODEL_DATA.channel.1ch.fits"))
        with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"),
                  "w") as f:
            TDLOptions.config.write(f)
        TDLOptions.config.read(
            path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf")))
        # needs to re-read because of a Timba perculiarity
        mod, ns, msg = Compile.compile_file(mqs,
                                            script,
                                            config="simulate-model")
        print("========== Simulating MODEL_DATA ")
        mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
        print("========== Imaging MODEL_DATA ")
        TDLOptions.get_job_func('make_dirty_image')(mqs,
                                                    None,
                                                    wait=True,
                                                    run_viewer=False)

        ## compare against reference image
        print("========== Verifying test image ")
        if not os.path.exists(
                os.path.join(PACKAGE_TEST_DIR,
                             "WSRT.MS.MODEL_DATA.channel.1ch.fits")):
            raise RuntimeError("Output FITS file does not exist")
        if not os.path.exists(
                os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")):
            raise RuntimeError("Reference FITS file does not exist")
        verify_image(os.path.join(PACKAGE_TEST_DIR,
                                  "WSRT.MS.MODEL_DATA.channel.1ch.fits"),
                     path(os.path.join(PACKAGE_TEST_DIR,
                                       "test-refimage.fits")),
                     maxdelta=1e-3)

        print("========== Compiling script with modified config")
        TDLOptions.init_options("simulate-model", save=False)
        TDLOptions.set_option("me.g_enable", True)
        mod, ns, msg = Compile.compile_file(mqs, script, config=None)
        print("========== Simulating DATA ")
        TDLOptions.set_option("ms_sel.output_column", "DATA")
        mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
        print("========== Imaging DATA ")
        TDLOptions.set_option("img_sel.imaging_column", "DATA")
        TDLOptions.get_job_func('make_dirty_image')(mqs,
                                                    None,
                                                    wait=True,
                                                    run_viewer=False)

        ## calibrate
        script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"))
        print("========== Compiling", script)
        mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate")
        print("========== Calibrating ")
        TDLOptions.get_job_func('cal_G_diag')(mqs, None, wait=True)
        print("========== Imaging MODEL_DATA ")
        TDLOptions.get_job_func('make_dirty_image')(mqs,
                                                    None,
                                                    wait=True,
                                                    run_viewer=False)

    finally:
        print("Stopping meqserver")
        # this halts the meqserver
        meqserver.stop_default_mqs()

    print("========== Making plots of solutions ")
    run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png"""
        .format(owlcat, PACKAGE_TEST_DIR))
    run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(
        owlcat, PACKAGE_TEST_DIR))
    run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png"""
        .format(owlcat, PACKAGE_TEST_DIR))
    run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(
        owlcat, PACKAGE_TEST_DIR))

    ## compare against reference image
    print("========== Verifying residual image ")
    if not os.path.exists(
            os.path.join(PACKAGE_TEST_DIR,
                         "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")):
        raise RuntimeError("Output FITS file does not exist")
    if not os.path.exists(
            os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")):
        raise RuntimeError("Reference FITS file does not exist")
    verify_image(os.path.join(PACKAGE_TEST_DIR,
                              "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
                 os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
                 maxdelta=1e-3)

    ## all tests succeeded
    print("========== Break out the bubbly, this hog is airborne!")

    # now we can exit
    print("Bye!")
  # or was interrupted by an exception midway through, we use a try...finally block.
  try:

    TDLOptions.config.read("batch_sim_example.tdl.conf");

    script = "example-sim.py";
    print "========== Compiling batch job 1";
    mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 1");
    print "========== Running batch job 1";
    mod._tdl_job_1_simulate_MS(mqs,None,wait=True);

    print "========== Compiling batch job 2";
    mod,ns,msg = Compile.compile_file(mqs,script,config="batch job 2");
    print "========== Running batch job 2";
    mod._tdl_job_1_simulate_MS(mqs,None,wait=True);

    print "========== Compiling batch job 2 with modified config";
    TDLOptions.init_options("batch job 2",save=False);
    TDLOptions.set_option("me.enable_G",False);
    mod,ns,msg = Compile.compile_file(mqs,script,config=None);
    print "========== Running batch job 2";
    mod._tdl_job_1_simulate_MS(mqs,None,wait=True);

  ### Cleanup time
  finally:
    print "Stopping meqserver";
    # this halts the meqserver
    meqserver.stop_default_mqs();
    # now we can exit
    print "Bye!";
    # or was interrupted by an exception midway through, we use a try...finally block.
    try:

        TDLOptions.config.read("batch_sim_example.tdl.conf")

        script = "example-sim.py"
        print("========== Compiling batch job 1")
        mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 1")
        print("========== Running batch job 1")
        mod._tdl_job_1_simulate_MS(mqs, None, wait=True)

        print("========== Compiling batch job 2")
        mod, ns, msg = Compile.compile_file(mqs, script, config="batch job 2")
        print("========== Running batch job 2")
        mod._tdl_job_1_simulate_MS(mqs, None, wait=True)

        print("========== Compiling batch job 2 with modified config")
        TDLOptions.init_options("batch job 2", save=False)
        TDLOptions.set_option("me.enable_G", False)
        mod, ns, msg = Compile.compile_file(mqs, script, config=None)
        print("========== Running batch job 2")
        mod._tdl_job_1_simulate_MS(mqs, None, wait=True)

    ### Cleanup time
    finally:
        print("Stopping meqserver")
        # this halts the meqserver
        meqserver.stop_default_mqs()
        # now we can exit
        print("Bye!")
示例#7
0
            compile_match = re_compile_script.match(
                cmd) or re_compile_script1.match(cmd)
            job_match = re_run_job.match(cmd)

            if load_match:
                section = load_match.group(1)
                print "### Loading config section", section
                TDLOptions.init_options(section, save=False)
                loaded_options = True

            elif set_match:
                if not loaded_options:
                    raise RuntimeError, "Config section not yet specified"
                name, value = set_match.groups()
                print "### Setting option %s=%s" % (name, value)
                TDLOptions.set_option(name, value, save=False, from_str=True)

            elif compile_match:
                script, dum, section = compile_match.groups(None)
                print "### Compiling", script
                if not loaded_options and not section:
                    # this mode reloads default config section
                    print "### (using options from default section)"
                    module, ns, msg = Compile.compile_file(mqs, script)
                else:
                    # this mode uses explicit section, or None if section is not specified
                    if section:
                        print """### (using options from config section "%s")""" % section
                    else:
                        section = None
                        print "### (using previously set options)"
示例#8
0
    script = path("testing-sim.py");
    print "========== Compiling",script;
    TDLOptions.config.read(path("testing.tdl.conf"));
    mod,ns,msg = Compile.compile_file(mqs,script,config="simulate-model");
    print "========== Simulating MODEL_DATA ";
    mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
    print "========== Imaging MODEL_DATA ";
    TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);

    ## compare against reference image
    print "========== Verifying test image ";
    verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',path('test-refimage.fits'),maxdelta=1e-3);

    print "========== Compiling script with modified config";
    TDLOptions.init_options("simulate-model",save=False);
    TDLOptions.set_option("me.g_enable",True);
    mod,ns,msg = Compile.compile_file(mqs,script,config=None);
    print "========== Simulating DATA ";
    TDLOptions.set_option("ms_sel.output_column","DATA");
    mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
    print "========== Imaging DATA ";
    TDLOptions.set_option("img_sel.imaging_column","DATA");
    TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);

    ## calibrate
    script = path("testing-cal.py");
    print "========== Compiling",script;
    mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate");
    print "========== Calibrating ";
    TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True);
    print "========== Imaging MODEL_DATA ";
        mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
        print "========== Imaging MODEL_DATA "
        TDLOptions.get_job_func('make_dirty_image')(mqs,
                                                    None,
                                                    wait=True,
                                                    run_viewer=False)

        ## compare against reference image
        print "========== Verifying test image "
        verify_image('WSRT.MS.MODEL_DATA.channel.1ch.fits',
                     path('test-refimage.fits'),
                     maxdelta=1e-3)

        print "========== Compiling script with modified config"
        TDLOptions.init_options("simulate-model", save=False)
        TDLOptions.set_option("me.g_enable", True)
        mod, ns, msg = Compile.compile_file(mqs, script, config=None)
        print "========== Simulating DATA "
        TDLOptions.set_option("ms_sel.output_column", "DATA")
        mod._tdl_job_1_simulate_MS(mqs, None, wait=True)
        print "========== Imaging DATA "
        TDLOptions.set_option("img_sel.imaging_column", "DATA")
        TDLOptions.get_job_func('make_dirty_image')(mqs,
                                                    None,
                                                    wait=True,
                                                    run_viewer=False)

        ## calibrate
        script = path("testing-cal.py")
        print "========== Compiling", script
        mod, ns, msg = Compile.compile_file(mqs, script, config="calibrate")
示例#10
0
        print "Trying to call a non-existing TDL job 'job3'"
        try:
            TDLOptions.get_job_func('job3')(mqs, None, wait=True)
        except NameError:
            print "Indeed, 'job3' does not exist."

        # This shows how to change configuration on-the-fly.
        # TDLOptions.set_option() changes the value of a configuration variable "in memory".
        # Note that compile_file() normally rereads the config file, which would defeat the point of any
        # on-the-fly changes. To get around this, we can manage the config manually as follows:
        print "Recompiling and running for a=5, t=[-1,1]"
        # this loads initial config from section [test_b]. The save=False argument tells the system to not
        # write the changes we make below to the .tdl.conf file
        TDLOptions.init_options("test_b", save=False)
        # this makes some changes to the loaded config
        TDLOptions.set_option("a", 5)
        TDLOptions.set_option("t0", -1)
        TDLOptions.set_option("t1", 1)
        # this compiles the script. config=None tells it to NOT reload the config file, but to use whatever we have set up above.
        mod, ns, msg = Compile.compile_file(mqs, script, config=None)
        mod._test_forest(mqs, None, wait=True)

        # Make another change to the current config and recompile. Again config=None is necessary,
        TDLOptions.set_option("b", -5)
        mod, ns, msg = Compile.compile_file(mqs, script, config=None)
        mod._test_forest(mqs, None, wait=True)

        # Since "a" and "b" are compile-time options, we always need to recompile for them to take effect.
        # By contrast, runtime options ('t0' and 't1') can be changed without recompiling:
        print "Rerunning for t=[-5,5]"
        TDLOptions.set_option("t0", -5)
示例#11
0
def testMeqtreesBatchJob():
  trace_sync = True;
#  sys.settrace(trace_lines);
  
  if len(sys.argv) > 1:
    newdir = PACKAGE_TEST_DIR;
    print("========== Changing working directory to",newdir);
    os.chdir(newdir);
    print("========== Making required symlinks");
    run("rm {0:s}/WSRT_ANTENNA ; ln -s {1:s}".format(PACKAGE_TEST_DIR, 
                                                     path(os.path.join(PACKAGE_TEST_DIR, "WSRT_ANTENNA"))));
    run("rm {0:s}/test-lsm.txt; ln -s {1:s}".format(PACKAGE_TEST_DIR,
                                                    path(os.path.join(path("test-lsm.txt")))));

  if not os.access(".",os.R_OK|os.W_OK):
    print("Directory",os.getcwd(),"not writable, can't run tests in here.")
    print("You may choose to run the tests in a different directory by giving it as an argument to this script.")
    sys.exit(1);

  ## check if we have owlcat or owlcat.sh
  owlcat = "";
  for dirname in os.environ['PATH'].split(':'):
    for binary in "owlcat","owlcat.sh":
      tmp = os.path.join(dirname,binary);
      if os.path.exists(tmp):
        owlcat = tmp;
        break;
    if owlcat:
      break;
  if not owlcat:
    raise RuntimeError("Can't locate owlcat or owlcat.sh");
    
    
  ## make simulated MS
  print("========== Removing files");
  
  run("rm -fr {0:s}/WSRT.MS* {0:s}/WSRT*img {0:s}/WSRT*fits".format(PACKAGE_TEST_DIR));
  print("========== Running makems");
  run("makems %s" % path(os.path.join(PACKAGE_TEST_DIR, "WSRT_makems.cfg")));
  run("mv {0:s}/WSRT.MS_p0 {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
  os.environ["MEQTREES_CATTERY_PATH"] = Cattery.__path__[0]
  run("pyxis {0:s}/WSRT.MS ms.prep".format(PACKAGE_TEST_DIR)); #TODO: this is hacky, bug in CASAcore
  run("ls -ld {0:s}/WSRT.MS".format(PACKAGE_TEST_DIR));
  run("{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS".format(owlcat, PACKAGE_TEST_DIR));
  run("lwimager ms={0:s}/WSRT.MS data=CORRECTED_DATA mode=channel weight=natural npix=10".format(PACKAGE_TEST_DIR));
  # make test LSMs
  run("""tigger-convert {0:s}/test-lsm.txt --rename --format "ra_d dec_d i q u v" --center 0.1deg,60.5deg -f""".format(PACKAGE_TEST_DIR));
  run("""tigger-convert {0:s}/test-lsm.lsm.html {0:s}/test-lsm1.txt --output-format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
  run("""cut -d " " -f 1-10 {0:s}/test-lsm1.txt >{0:s}/test-lsm1.txt.tmp""".format(PACKAGE_TEST_DIR));
  run("""diff {0:s}/test-lsm1.txt.tmp {1:s}""".format(PACKAGE_TEST_DIR, path(os.path.join(PACKAGE_TEST_DIR, 'test-lsm1.txt.reference'))));
  run("""tigger-convert {0:s}/test-lsm1.txt --format "name ra_h dec_d i q u v freq0 spi rm tags..." -f""".format(PACKAGE_TEST_DIR));
  run("""{0:s} plot-ms {1:s}/WSRT.MS DATA:I -o data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
  run("""{0:s} run-imager ms={1:s}/WSRT.MS name_dirty=tmp""".format(owlcat, PACKAGE_TEST_DIR));

  print("importing meqserver")
  from Timba.Apps import meqserver
  print("importing Compile")
  from Timba.TDL import Compile
  print("importing TDLOptions")
  from Timba.TDL import TDLOptions

  # This starts a meqserver. Note how we pass the "-mt 2" option to run two threads.
  # A proper pipeline script may want to get the value of "-mt" from its own arguments (sys.argv).
  print("Starting meqserver");
  mqs = meqserver.default_mqs(wait_init=10,extra=["-mt","2"]);

  try:
    ## make simulation with perfect MODEL_DATA
    script = path(os.path.join(PACKAGE_TEST_DIR, "sim.py"));
    print("========== Compiling",script);
    TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing.tdl.conf")));
    TDLOptions.config.set("calibrate", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "ms_sel.msname")))
    TDLOptions.config.set("calibrate", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "tiggerlsm.filename")))
    TDLOptions.config.set("calibrate", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "lsm.filename")))
    TDLOptions.config.set("calibrate", "cal_g_diag.g_diag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_diag.g_diag.table_name")))
    TDLOptions.config.set("calibrate", "cal_g_offdiag.g_offdiag.table_name", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("calibrate", "cal_g_offdiag.g_offdiag.table_name")))
    TDLOptions.config.set("simulate-model", "lsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "lsm.filename")))
    TDLOptions.config.set("simulate-model", "ms_sel.msname", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "ms_sel.msname")))
    TDLOptions.config.set("simulate-model", "tiggerlsm.filename", os.path.join(PACKAGE_TEST_DIR, TDLOptions.config.get("simulate-model", "tiggerlsm.filename")))
    TDLOptions.config.set("calibrate", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"))
    TDLOptions.config.set("simulate-model", "img_sel.output_fitsname", os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"))
    with open(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"), "w") as f:
      TDLOptions.config.write(f)
    TDLOptions.config.read(path(os.path.join(PACKAGE_TEST_DIR, "testing_tmp.tdl.conf"))); # needs to re-read because of a Timba perculiarity
    mod,ns,msg = Compile.compile_file(mqs, script, config="simulate-model");
    print("========== Simulating MODEL_DATA ");
    mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
    print("========== Imaging MODEL_DATA ");
    TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);

    ## compare against reference image
    print("========== Verifying test image ");
    if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
    if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")): raise RuntimeError("Reference FITS file does not exist")
    verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.MODEL_DATA.channel.1ch.fits"), 
                 path(os.path.join(PACKAGE_TEST_DIR, "test-refimage.fits")), 
                 maxdelta=1e-3);

    print("========== Compiling script with modified config");
    TDLOptions.init_options("simulate-model",save=False);
    TDLOptions.set_option("me.g_enable",True);
    mod,ns,msg = Compile.compile_file(mqs,script,config=None);
    print("========== Simulating DATA ");
    TDLOptions.set_option("ms_sel.output_column","DATA");
    mod._tdl_job_1_simulate_MS(mqs,None,wait=True);
    print("========== Imaging DATA ");
    TDLOptions.set_option("img_sel.imaging_column","DATA");
    TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);

    ## calibrate
    script = path(os.path.join(PACKAGE_TEST_DIR, "cal.py"));
    print("========== Compiling",script);
    mod,ns,msg = Compile.compile_file(mqs,script,config="calibrate");
    print("========== Calibrating ");
    TDLOptions.get_job_func('cal_G_diag')(mqs,None,wait=True);
    print("========== Imaging MODEL_DATA ");
    TDLOptions.get_job_func('make_dirty_image')(mqs,None,wait=True,run_viewer=False);
    
  finally:
    print("Stopping meqserver");
    # this halts the meqserver
    meqserver.stop_default_mqs();
    
  print("========== Making plots of solutions ");
  run("""{0:s} plot-ms {1:s}/WSRT.MS CORRECTED_DATA:I -I ">0" -o {1:s}/corrected_data_i.png""".format(owlcat, PACKAGE_TEST_DIR));
  run("""{0:s} plot-parms -l {1:s}/WSRT.MS/G_diag.fmep""".format(owlcat, PACKAGE_TEST_DIR));
  run("""{0:s} plot-parms {1:s}/WSRT.MS/G_diag.fmep G:*/norm -o {1:s}/parmplot.png""".format(owlcat, PACKAGE_TEST_DIR));
  run("""{0:s} downweigh-redundant-baselines {1:s}/WSRT.MS""".format(owlcat, PACKAGE_TEST_DIR));

  ## compare against reference image
  print("========== Verifying residual image ");
  if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits")): raise RuntimeError("Output FITS file does not exist")
  if not os.path.exists(os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits")): raise RuntimeError("Reference FITS file does not exist")
  verify_image(os.path.join(PACKAGE_TEST_DIR, "WSRT.MS.CORRECTED_DATA.channel.1ch.fits"),
               os.path.join(PACKAGE_TEST_DIR, "test-refresidual.fits"),
               maxdelta=1e-3);

  ## all tests succeeded
  print("========== Break out the bubbly, this hog is airborne!");

  # now we can exit
  print("Bye!");
    print "Trying to call a non-existing TDL job 'job3'";
    try:
      TDLOptions.get_job_func('job3')(mqs,None,wait=True);
    except NameError:
      print "Indeed, 'job3' does not exist."

    # This shows how to change configuration on-the-fly.
    # TDLOptions.set_option() changes the value of a configuration variable "in memory".
    # Note that compile_file() normally rereads the config file, which would defeat the point of any
    # on-the-fly changes. To get around this, we can manage the config manually as follows:
    print "Recompiling and running for a=5, t=[-1,1]";
    # this loads initial config from section [test_b]. The save=False argument tells the system to not
    # write the changes we make below to the .tdl.conf file
    TDLOptions.init_options("test_b",save=False);
    # this makes some changes to the loaded config
    TDLOptions.set_option("a",5);
    TDLOptions.set_option("t0",-1);
    TDLOptions.set_option("t1",1);
    # this compiles the script. config=None tells it to NOT reload the config file, but to use whatever we have set up above.
    mod,ns,msg = Compile.compile_file(mqs,script,config=None);
    mod._test_forest(mqs,None,wait=True);

    # Make another change to the current config and recompile. Again config=None is necessary,
    TDLOptions.set_option("b",-5);
    mod,ns,msg = Compile.compile_file(mqs,script,config=None);
    mod._test_forest(mqs,None,wait=True);

    # Since "a" and "b" are compile-time options, we always need to recompile for them to take effect.
    # By contrast, runtime options ('t0' and 't1') can be changed without recompiling:
    print "Rerunning for t=[-5,5]";
    TDLOptions.set_option("t0",-5);