def test_iopathlist(self):
global MSDIR
global INPUT
global OUTPUT
global CABPATH
stimela.register_globals()
rrr = stimela.Recipe("pathlist", ms_dir=MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla5": ["testinput2.txt:input",
"testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")],
},
cabpath=CABPATH,
input=INPUT, output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"],
"testinput2.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"],
"testinput3.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format(rrr.jobs[0].job.IODEST["input"],
rrr.jobs[0].job.IODEST["msfile"],
rrr.jobs[0].job.IODEST["output"]
)
def test_udocker(self):
import sys
global MSDIR
global INPUT
global OUTPUT
global UDOCKER
if UDOCKER is False:
return
stimela.register_globals()
rrr = stimela.Recipe("singularitypaths",
ms_dir=MSDIR,
JOB_TYPE="udocker",
cabpath="cab/",
log_dir="logs")
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla5": ["testinput2.txt:input",
"testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")],
}, input=INPUT, output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"],
"testinput2.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"],
"testinput3.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format(
rrr.jobs[0].job.IODEST["input"],
rrr.jobs[0].job.IODEST["msfile"],
rrr.jobs[0].job.IODEST["output"]
)
def test_singularity(self):
global MSDIR
global INPUT
global OUTPUT
global SINGULARITY
if SINGULARITY is False:
return
stimela.register_globals()
rrr = stimela.Recipe("singularitypaths",
ms_dir=MSDIR,
JOB_TYPE="singularity",
cabpath="cab/",
singularity_image_dir=os.environ["STIMELA_PULLFOLDER"],
log_dir="logs")
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla5": ["testinput2.txt:input",
"testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")],
}, input=INPUT, output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"],
"testinput2.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"],
"testinput3.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format(
rrr.jobs[0].job.IODEST["input"],
rrr.jobs[0].job.IODEST["msfile"],
rrr.jobs[0].job.IODEST["output"]
)
def test_iopathlist(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("pathlist", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add(
"cab/custom",
"test1",
{
"bla1":
"a", # only accepts a, b or c
"bla5": [
"testinput2.txt:input", "testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input",
"msfile", "output")
],
},
input=INPUT,
output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[
0] == "/input/testinput2.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(
"/", "home", os.environ["USER"], "msdir", "testinput3.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}hello{{reim}}.fits,{}to.fits,{}world.fits".format(
"/input/",
os.path.join("/", "home", os.environ["USER"], "msdir/"),
os.path.join("/", "home", os.environ["USER"], "output/"),
)
def test_invalid_choice(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("invchoice", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "d" # only accepts a, b or c
}, input=INPUT, output=OUTPUT)
with self.assertRaises(PipelineException):
rrr.run() #validate and run
def test_required(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("testrequired", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla3": 4.0,
}, input=INPUT, output=OUTPUT)
with self.assertRaises(PipelineException):
rrr.run() #validate and run
def test_iopathval(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("ioval", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla2": "testinput2.txt:input",
}, input=INPUT, output=OUTPUT)
with self.assertRaises(PipelineException): # not exist during validation
rrr.run() #validate and run
def test_floattypefail(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("testfloattypefail", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla3": "1.0a",
}, input=INPUT, output=OUTPUT)
with self.assertRaises(PipelineException):
rrr.run() #validate and run
def test_floattypesuccess(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("testfloattypesuccess", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla3": 4.0,
}, input=INPUT, output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[2].value == [4.0]
def test_define_cab(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("customcab", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a"
}, input=INPUT, output=OUTPUT)
assert len(rrr.jobs) == 1
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[0].value == "a"
assert len(rrr.completed) == 1
assert len(rrr.remaining) == 0
def test_iooverride(self):
global MSDIR
global INPUT
with open(os.path.join(INPUT, "testinput.txt"), "w+") as f:
pass
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("testiooverrides", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla2": "testinput.txt:input",
}, input=INPUT, output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[0].value == "a"
assert rrr.jobs[0].job._cab.parameters[1].value == os.path.join(rrr.jobs[0].job.IODEST["input"],
"testinput.txt")
def setUpClass(cls):
unittest.TestCase.setUpClass()
global INPUT, MSDIR, OUTPUT, MS, PREFIX, LSM
INPUT = os.path.join(os.path.dirname(__file__), "input")
MSDIR = "msdir"
OUTPUT = "output"
# MS name
MS = "meerkat_simulation_example.ms"
# Use the NVSS skymodel. This is natively available
LSM = "nvss1deg.lsm.html"
PREFIX = "stimela-example" # Prefix for output images
stimela.register_globals()
if not "SINGULARITY_PULLFOLDER" in os.environ:
raise ValueError(
"ENV SINGULARITY_PULLFOLDER not set! This test requires singularity images to be pulled"
)
def test_dismissable(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("testdismissable", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla4": sdm("abc"),
"bla3": sdm(None)
}, input=INPUT, output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[0].value == "a"
assert rrr.jobs[0].job._cab.parameters[1].value is None
assert rrr.jobs[0].job._cab.parameters[2].value is None
assert rrr.jobs[0].job._cab.parameters[3].value == ["abc"]
def setUpClass(cls):
unittest.TestCase.setUpClass()
# I/O
global INPUT
INPUT = 'input'
global MSDIR
MSDIR = 'msdir'
global MS
MS = '1491291289.1GC.ms'
global PREFIX
PREFIX = 'deep2'
global LABEL
LABEL = "test_mkreduction"
global OUTPUT
OUTPUT = "output_%s" % LABEL
stimela.register_globals()
def test_udocker(self):
import sys
if sys.version_info[0] > 2:
return
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("singularitypaths",
ms_dir=MSDIR,
JOB_TYPE="udocker",
cabpath="cab/")
assert os.path.exists(MSDIR)
rrr.add(
"cab/custom",
"test1",
{
"bla1":
"a", # only accepts a, b or c
"bla5": [
"testinput2.txt:input", "testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input",
"msfile", "output")
],
},
input=INPUT,
output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[
0] == "/scratch/input/testinput2.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[
1] == "/scratch/msdir/testinput3.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}hello{{reim}}.fits,{}to.fits,{}world.fits".format(
"/scratch/input/",
"/scratch/msdir/",
"/scratch/output/"
)
def test_singularity(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe(
"singularitypaths",
ms_dir=MSDIR,
JOB_TYPE="singularity",
cabpath="cab/",
singularity_image_dir=os.environ["SINGULARITY_PULLFOLDER"])
assert os.path.exists(MSDIR)
rrr.add(
"cab/custom",
"test1",
{
"bla1":
"a", # only accepts a, b or c
"bla5": [
"testinput2.txt:input", "testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input",
"msfile", "output")
],
},
input=INPUT,
output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[
0] == "/scratch/input/testinput2.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[
1] == "/scratch/msdir/testinput3.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}hello{{reim}}.fits,{}to.fits,{}world.fits".format(
"/scratch/input/",
"/scratch/msdir/",
"/scratch/output/"
)
MANUAL_FLAG_LIST = []
FIRSTGEN_DATA = ["{}.{}.1gc.ms".format(t, PREFIX) for t in TARGET]
vermeerkat.log.info("The following fields are available:")
for f in FDB:
vermeerkat.log.info("\t '{0:s}' index {1:s}{2:s}".format(
f, FDB[f],
" selected as 'BP'" if f == BPCALIBRATOR else " selected as 'GC'"
if f in GCALIBRATOR else " selected as 'ALTCAL'" if f in ALTCAL else
" selected as 'TARGET'" if f in TARGET else " not selected"))
if not vermeerkat.prompt(dont_prompt=args.dont_prompt):
vermeerkat.log.info("Aborted per user request")
sys.exit(1)
stimela.register_globals()
recipe = stimela.Recipe('MEERKAT: basic transfer calibration',
ms_dir=MSDIR,
singularity_image_dir=os.environ.get(
"SINGULARITY_PULLFOLDER", ""),
JOB_TYPE=args.containerization)
def addmanualflags(recipe,
reason,
antenna="",
spw="",
scan="",
uvrange="",
field=""):
""" Read CASA flagdata docs before using """
def setUpClass(cls):
unittest.TestCase.setUpClass()
# I/O
global INPUT
INPUT = 'input'
global MSDIR
MSDIR = 'msdir'
global MS
MS = 'kat-7-small.ms'
global PREFIX
PREFIX = 'kat7_small_LBand'
# Fields
global GCAL
GCAL = 'PKS2326-477'
global TARGET
TARGET = '1'
global BPCAL
BPCAL = 'PKS1934-638'
# Reference antenna
global REFANT
REFANT = '0'
# Calibration tables
global ANTPOS_TABLE
ANTPOS_TABLE = PREFIX + '.antpos:output'
global BPCAL_TABLE
BPCAL_TABLE = PREFIX + '.B0:output'
global DELAYCAL_TABLE
DELAYCAL_TABLE = PREFIX + '.K0:output'
global GAINCAL_TABLE
GAINCAL_TABLE = PREFIX + '.G0:output'
global FLUXSCALE_TABLE
FLUXSCALE_TABLE = PREFIX + '.fluxscale:output'
global LABEL
LABEL = "test_reduction"
global OUTPUT
OUTPUT = "/tmp/output_%s" % LABEL
global MSCONTSUB
MSCONTSUB = MS + '.contsub'
global SPW
SPW = '0:100~355'
# Calibration tables
global LSM0
LSM0 = PREFIX + '.lsm.html'
global SELFCAL_TABLE1
SELFCAL_TABLE1 = PREFIX + '.SF1:output'
global IMAGE1
IMAGE1 = PREFIX + 'image1:output'
global MASK1
MASK1 = PREFIX + 'mask1.fits'
global IMAGE2
IMAGE2 = PREFIX + 'image2:output'
global nchans
nchans = 256
global chans
chans = [100, 355]
# Clean-Mask-Clean
global imname0
imname0 = PREFIX + 'image0'
global maskname0
maskname0 = PREFIX + 'mask0.fits'
global maskname01
maskname01 = PREFIX + 'mask01.fits'
global imname1
imname1 = PREFIX + 'image1'
global corr_ms
corr_ms = MS + '-corr.ms'
global lsm0
lsm0 = PREFIX + '-LSM0'
stimela.register_globals()
