def test_iopathlist(self): global MSDIR global INPUT global OUTPUT global CABPATH stimela.register_globals() rrr = stimela.Recipe("pathlist", ms_dir=MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": ["testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")], }, cabpath=CABPATH, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"], "testinput2.txt") assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"], "testinput3.txt") assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format(rrr.jobs[0].job.IODEST["input"], rrr.jobs[0].job.IODEST["msfile"], rrr.jobs[0].job.IODEST["output"] )
def test_udocker(self): import sys global MSDIR global INPUT global OUTPUT global UDOCKER if UDOCKER is False: return stimela.register_globals() rrr = stimela.Recipe("singularitypaths", ms_dir=MSDIR, JOB_TYPE="udocker", cabpath="cab/", log_dir="logs") assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": ["testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"], "testinput2.txt") assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"], "testinput3.txt") assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format( rrr.jobs[0].job.IODEST["input"], rrr.jobs[0].job.IODEST["msfile"], rrr.jobs[0].job.IODEST["output"] )
def test_singularity(self): global MSDIR global INPUT global OUTPUT global SINGULARITY if SINGULARITY is False: return stimela.register_globals() rrr = stimela.Recipe("singularitypaths", ms_dir=MSDIR, JOB_TYPE="singularity", cabpath="cab/", singularity_image_dir=os.environ["STIMELA_PULLFOLDER"], log_dir="logs") rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": ["testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"], "testinput2.txt") assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"], "testinput3.txt") assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format( rrr.jobs[0].job.IODEST["input"], rrr.jobs[0].job.IODEST["msfile"], rrr.jobs[0].job.IODEST["output"] )
def test_iopathlist(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("pathlist", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add( "cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": [ "testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output") ], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[ 0] == "/input/testinput2.txt" assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join( "/", "home", os.environ["USER"], "msdir", "testinput3.txt") assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}hello{{reim}}.fits,{}to.fits,{}world.fits".format( "/input/", os.path.join("/", "home", os.environ["USER"], "msdir/"), os.path.join("/", "home", os.environ["USER"], "output/"), )
def test_invalid_choice(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("invchoice", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "d" # only accepts a, b or c }, input=INPUT, output=OUTPUT) with self.assertRaises(PipelineException): rrr.run() #validate and run
def test_required(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testrequired", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla3": 4.0, }, input=INPUT, output=OUTPUT) with self.assertRaises(PipelineException): rrr.run() #validate and run
def test_iopathval(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("ioval", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla2": "testinput2.txt:input", }, input=INPUT, output=OUTPUT) with self.assertRaises(PipelineException): # not exist during validation rrr.run() #validate and run
def test_floattypefail(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testfloattypefail", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla3": "1.0a", }, input=INPUT, output=OUTPUT) with self.assertRaises(PipelineException): rrr.run() #validate and run
def test_floattypesuccess(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testfloattypesuccess", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla3": 4.0, }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[2].value == [4.0]
def test_define_cab(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("customcab", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a" }, input=INPUT, output=OUTPUT) assert len(rrr.jobs) == 1 rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[0].value == "a" assert len(rrr.completed) == 1 assert len(rrr.remaining) == 0
def test_iooverride(self): global MSDIR global INPUT with open(os.path.join(INPUT, "testinput.txt"), "w+") as f: pass global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testiooverrides", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla2": "testinput.txt:input", }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[0].value == "a" assert rrr.jobs[0].job._cab.parameters[1].value == os.path.join(rrr.jobs[0].job.IODEST["input"], "testinput.txt")
def setUpClass(cls): unittest.TestCase.setUpClass() global INPUT, MSDIR, OUTPUT, MS, PREFIX, LSM INPUT = os.path.join(os.path.dirname(__file__), "input") MSDIR = "msdir" OUTPUT = "output" # MS name MS = "meerkat_simulation_example.ms" # Use the NVSS skymodel. This is natively available LSM = "nvss1deg.lsm.html" PREFIX = "stimela-example" # Prefix for output images stimela.register_globals() if not "SINGULARITY_PULLFOLDER" in os.environ: raise ValueError( "ENV SINGULARITY_PULLFOLDER not set! This test requires singularity images to be pulled" )
def test_dismissable(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testdismissable", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla4": sdm("abc"), "bla3": sdm(None) }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[0].value == "a" assert rrr.jobs[0].job._cab.parameters[1].value is None assert rrr.jobs[0].job._cab.parameters[2].value is None assert rrr.jobs[0].job._cab.parameters[3].value == ["abc"]
def setUpClass(cls): unittest.TestCase.setUpClass() # I/O global INPUT INPUT = 'input' global MSDIR MSDIR = 'msdir' global MS MS = '1491291289.1GC.ms' global PREFIX PREFIX = 'deep2' global LABEL LABEL = "test_mkreduction" global OUTPUT OUTPUT = "output_%s" % LABEL stimela.register_globals()
def test_udocker(self): import sys if sys.version_info[0] > 2: return global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("singularitypaths", ms_dir=MSDIR, JOB_TYPE="udocker", cabpath="cab/") assert os.path.exists(MSDIR) rrr.add( "cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": [ "testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output") ], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[ 0] == "/scratch/input/testinput2.txt" assert rrr.jobs[0].job._cab.parameters[4].value[ 1] == "/scratch/msdir/testinput3.txt" assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}hello{{reim}}.fits,{}to.fits,{}world.fits".format( "/scratch/input/", "/scratch/msdir/", "/scratch/output/" )
def test_singularity(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe( "singularitypaths", ms_dir=MSDIR, JOB_TYPE="singularity", cabpath="cab/", singularity_image_dir=os.environ["SINGULARITY_PULLFOLDER"]) assert os.path.exists(MSDIR) rrr.add( "cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": [ "testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output") ], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[ 0] == "/scratch/input/testinput2.txt" assert rrr.jobs[0].job._cab.parameters[4].value[ 1] == "/scratch/msdir/testinput3.txt" assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}hello{{reim}}.fits,{}to.fits,{}world.fits".format( "/scratch/input/", "/scratch/msdir/", "/scratch/output/" )
MANUAL_FLAG_LIST = [] FIRSTGEN_DATA = ["{}.{}.1gc.ms".format(t, PREFIX) for t in TARGET] vermeerkat.log.info("The following fields are available:") for f in FDB: vermeerkat.log.info("\t '{0:s}' index {1:s}{2:s}".format( f, FDB[f], " selected as 'BP'" if f == BPCALIBRATOR else " selected as 'GC'" if f in GCALIBRATOR else " selected as 'ALTCAL'" if f in ALTCAL else " selected as 'TARGET'" if f in TARGET else " not selected")) if not vermeerkat.prompt(dont_prompt=args.dont_prompt): vermeerkat.log.info("Aborted per user request") sys.exit(1) stimela.register_globals() recipe = stimela.Recipe('MEERKAT: basic transfer calibration', ms_dir=MSDIR, singularity_image_dir=os.environ.get( "SINGULARITY_PULLFOLDER", ""), JOB_TYPE=args.containerization) def addmanualflags(recipe, reason, antenna="", spw="", scan="", uvrange="", field=""): """ Read CASA flagdata docs before using """
def setUpClass(cls): unittest.TestCase.setUpClass() # I/O global INPUT INPUT = 'input' global MSDIR MSDIR = 'msdir' global MS MS = 'kat-7-small.ms' global PREFIX PREFIX = 'kat7_small_LBand' # Fields global GCAL GCAL = 'PKS2326-477' global TARGET TARGET = '1' global BPCAL BPCAL = 'PKS1934-638' # Reference antenna global REFANT REFANT = '0' # Calibration tables global ANTPOS_TABLE ANTPOS_TABLE = PREFIX + '.antpos:output' global BPCAL_TABLE BPCAL_TABLE = PREFIX + '.B0:output' global DELAYCAL_TABLE DELAYCAL_TABLE = PREFIX + '.K0:output' global GAINCAL_TABLE GAINCAL_TABLE = PREFIX + '.G0:output' global FLUXSCALE_TABLE FLUXSCALE_TABLE = PREFIX + '.fluxscale:output' global LABEL LABEL = "test_reduction" global OUTPUT OUTPUT = "/tmp/output_%s" % LABEL global MSCONTSUB MSCONTSUB = MS + '.contsub' global SPW SPW = '0:100~355' # Calibration tables global LSM0 LSM0 = PREFIX + '.lsm.html' global SELFCAL_TABLE1 SELFCAL_TABLE1 = PREFIX + '.SF1:output' global IMAGE1 IMAGE1 = PREFIX + 'image1:output' global MASK1 MASK1 = PREFIX + 'mask1.fits' global IMAGE2 IMAGE2 = PREFIX + 'image2:output' global nchans nchans = 256 global chans chans = [100, 355] # Clean-Mask-Clean global imname0 imname0 = PREFIX + 'image0' global maskname0 maskname0 = PREFIX + 'mask0.fits' global maskname01 maskname01 = PREFIX + 'mask01.fits' global imname1 imname1 = PREFIX + 'image1' global corr_ms corr_ms = MS + '-corr.ms' global lsm0 lsm0 = PREFIX + '-LSM0' stimela.register_globals()