def test_iopathlist(self):
global MSDIR
global INPUT
global OUTPUT
global CABPATH
stimela.register_globals()
rrr = stimela.Recipe("pathlist", ms_dir=MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla5": ["testinput2.txt:input",
"testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")],
},
cabpath=CABPATH,
input=INPUT, output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"],
"testinput2.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"],
"testinput3.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format(rrr.jobs[0].job.IODEST["input"],
rrr.jobs[0].job.IODEST["msfile"],
rrr.jobs[0].job.IODEST["output"]
)
def test_udocker(self):
import sys
global MSDIR
global INPUT
global OUTPUT
global UDOCKER
if UDOCKER is False:
return
stimela.register_globals()
rrr = stimela.Recipe("singularitypaths",
ms_dir=MSDIR,
JOB_TYPE="udocker",
cabpath="cab/",
log_dir="logs")
assert os.path.exists(MSDIR)
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla5": ["testinput2.txt:input",
"testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")],
}, input=INPUT, output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"],
"testinput2.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"],
"testinput3.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format(
rrr.jobs[0].job.IODEST["input"],
rrr.jobs[0].job.IODEST["msfile"],
rrr.jobs[0].job.IODEST["output"]
)
def test_iopathlist(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("pathlist", ms_dir=MSDIR)
assert os.path.exists(MSDIR)
rrr.add(
"cab/custom",
"test1",
{
"bla1":
"a", # only accepts a, b or c
"bla5": [
"testinput2.txt:input", "testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input",
"msfile", "output")
],
},
input=INPUT,
output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[
0] == "/input/testinput2.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(
"/", "home", os.environ["USER"], "msdir", "testinput3.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}hello{{reim}}.fits,{}to.fits,{}world.fits".format(
"/input/",
os.path.join("/", "home", os.environ["USER"], "msdir/"),
os.path.join("/", "home", os.environ["USER"], "output/"),
)
def test_singularity(self):
global MSDIR
global INPUT
global OUTPUT
global SINGULARITY
if SINGULARITY is False:
return
stimela.register_globals()
rrr = stimela.Recipe("singularitypaths",
ms_dir=MSDIR,
JOB_TYPE="singularity",
cabpath="cab/",
singularity_image_dir=os.environ["STIMELA_PULLFOLDER"],
log_dir="logs")
rrr.add("cab/custom", "test1", {
"bla1": "a", # only accepts a, b or c
"bla5": ["testinput2.txt:input",
"testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")],
}, input=INPUT, output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"],
"testinput2.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"],
"testinput3.txt")
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format(
rrr.jobs[0].job.IODEST["input"],
rrr.jobs[0].job.IODEST["msfile"],
rrr.jobs[0].job.IODEST["output"]
)
def test_udocker(self):
import sys
if sys.version_info[0] > 2:
return
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe("singularitypaths",
ms_dir=MSDIR,
JOB_TYPE="udocker",
cabpath="cab/")
assert os.path.exists(MSDIR)
rrr.add(
"cab/custom",
"test1",
{
"bla1":
"a", # only accepts a, b or c
"bla5": [
"testinput2.txt:input", "testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input",
"msfile", "output")
],
},
input=INPUT,
output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[
0] == "/scratch/input/testinput2.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[
1] == "/scratch/msdir/testinput3.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}hello{{reim}}.fits,{}to.fits,{}world.fits".format(
"/scratch/input/",
"/scratch/msdir/",
"/scratch/output/"
)
def test_singularity(self):
global MSDIR
global INPUT
global OUTPUT
stimela.register_globals()
rrr = stimela.Recipe(
"singularitypaths",
ms_dir=MSDIR,
JOB_TYPE="singularity",
cabpath="cab/",
singularity_image_dir=os.environ["SINGULARITY_PULLFOLDER"])
assert os.path.exists(MSDIR)
rrr.add(
"cab/custom",
"test1",
{
"bla1":
"a", # only accepts a, b or c
"bla5": [
"testinput2.txt:input", "testinput3.txt:msfile",
spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input",
"msfile", "output")
],
},
input=INPUT,
output=OUTPUT)
rrr.run() #validate and run
assert rrr.jobs[0].job._cab.parameters[4].value[
0] == "/scratch/input/testinput2.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[
1] == "/scratch/msdir/testinput3.txt"
assert rrr.jobs[0].job._cab.parameters[4].value[2] == \
"{}hello{{reim}}.fits,{}to.fits,{}world.fits".format(
"/scratch/input/",
"/scratch/msdir/",
"/scratch/output/"
)
def dd_calibrate(field,mslist):
key = 'calibrate_dd'
print("pipeline.output", pipeline.output)
outdir = field+"_ddcal"
dicomod = prefix+"_"+field+"-DD-precal.DicoModel"
dereg = "de-{0:s}.reg".format(field)
output_cubical = OUTPUT+"/"+outdir
test_path = spf("MODEL_DATA")
print("test_path",test_path)
print("output_cubical",output_cubical)
for ms in mslist:
mspref =os.path.splitext(ms)[0].replace('-','_')
step = 'dd_calibrate-{0:s}-{1:s}'.format(mspref,field)
recipe.add('cab/cubical_ddf', step, {
"data-ms" : ms,
"data-column" : config[key]['dd_data_col'],
"out-column" : config[key]['dd_out_data_col'],
"weight-column" : config[key]['dd_weight_col'],
"sol-jones" : "G,DD", # Jones terms to solve
"sol-min-bl" : config[key]['sol_min_bl'], # only solve for |uv| > 300 m
"sol-stall-quorum" : config[key]['dd_sol_stall_quorum'],
"g-type" : config[key]['dd_g_type'],
"g-clip-high" : config[key]['dd_g_clip_high'],
"g-clip-low" : config[key]['dd_g_clip_low'],
"g-solvable" : True,
"g-update-type" : config[key]['dd_g_update_type'],
"g-max-prior-error" : config[key]['dd_g_max_prior_error'],
"dd-max-prior-error" : config[key]['dd_dd_max_prior_error'],
"g-max-post-error" : config[key]['dd_g_max_post_error'],
"dd-max-post-error" : config[key]['dd_dd_max_post_error'],
"g-time-int" : config[key]['dd_g_timeslots_int'],
"g-freq-int" : config[key]['dd_g_chan_int'],
"dist-ncpu" : config[key]['dist_ncpu'],
"dist-nworker" : config[key]['dist_nworker'],
"dist-max-chunks" : config[key]['dist_nworker'],
"dist-max-chunks" : config[key]['dist_nworker'],
# "model-beam-pattern": prefix+"'_$(corr)_$(reim).fits':output",
# "montblanc-feed-type": "linear",
# "model-beam-l-axis" : "px",
# "model-beam-m-axis" : "py",
"g-save-to" : "g_final-cal_{0:s}_{1:s}.parmdb".format(mspref, field),
"dd-save-to" : "dd_cal_final_{0:s}_{1:s}.parmdb".format(mspref, field),
"dd-type" : "complex-2x2",
"dd-clip-high" : 0.0,
"dd-clip-low" : 0.0,
"dd-solvable" : True,
"dd-time-int" : ddsols_t,
"dd-freq-int" : ddsols_f,
"dd-dd-term" : True,
"dd-prop-flags" : 'always',
"dd-fix-dirs" : "0",
"out-subtract-dirs" : "1:",
"model-list" : spf("MODEL_DATA+-{{}}{}@{{}}{}:{{}}{}@{{}}{}".format(dicomod, dereg, dicomod, dereg), "output", "output", "output", "output"),
"out-name" : prefix + "dE_sub",
"out-mode" : 'sr',
"out-model-column" : "MODEL_OUT",
#"data-freq-chunk" : 1*ddsols_f,
#"data-time-chunk" : 1*ddsols_t,
"data-time-chunk" : ddsols_t*int(min(1,config[key]['dist_nworker'])) if (ddsols_f==0 or config[key]['dd_g_chan_int']== 0) else ddsols_t*int(min(1,np.sqrt(config[key]['dist_nworker']))),
"data-freq-chunk" : 0 if (ddsols_f==0 or config[key]['dd_g_chan_int']== 0) else ddsols_f*int(min(1, np.sqrt(config[key]['dist_nworker']))),
"sol-term-iters" : "[50,90,50,90]",
"madmax-plot" : False,
"out-plots" : True,
"madmax-enable" : config[key]['madmax_enable'],
"madmax-threshold" : config[key]['madmax_thr'],
"madmax-global-threshold": config[key]['madmax_global_thr'],
"madmax-estimate" : "corr",
#"out-casa-gaintables" : True,
"degridding-NDegridBand": config['image_dd']['freq_ndegridband'],
'degridding-MaxFacetSize': 0.15,
},
input=INPUT,
#output=OUTPUT+"/"+outdir,
output=output_cubical,
shared_memory=shared_mem,
label='dd_calibrate-{0:s}-{1:s}:: Carry out DD calibration'.format(mspref,field))
def testEndToEndReduction(self):
global INPUT, OUTPUT, MSDIR, MS, LABEL
recipe = stimela.Recipe('Test reduction script',
ms_dir=MSDIR, JOB_TYPE="docker", log_dir="logs")
imname1 = "deep2.1gc"
imname2 = "deep2.2gc"
recipe.add("cab/ddfacet", "ddfacet_test1",
{
"Data-MS": [MS],
"Output-Name": imname1,
"Image-NPix": 2048,
"Image-Cell": 2,
"Cache-Reset": True,
"Freq-NBand": 2,
"Freq-NDegridBand": 4,
"Weight-ColName": "WEIGHT",
"Data-ChunkHours": 0.1,
"Data-Sort": True,
"Log-Boring": True,
"Deconv-MaxMajorIter": 2,
"Deconv-MaxMinorIter": 1500,
"Predict-ColName": "MODEL_DATA"
},
input=INPUT, output=OUTPUT, shared_memory="8gb",
label="image1",
time_out=1800)
recipe.add('cab/tricolour', "flag_data",
{
"ms": MS,
"data-column": "DATA",
"window-backend": 'numpy',
"flagging-strategy": "total_power",
"subtract-model-column": "MODEL_DATA",
},
input=INPUT, output=OUTPUT, label="flag_data",
time_out=1800)
maskname0 = "MASK.fits"
recipe.add('cab/cleanmask', 'mask0', {
"image": '%s.app.restored.fits:output' % (imname1),
"output": '%s:output' % (maskname0),
"dilate": False,
"sigma": 25,
},
input=INPUT,
output=OUTPUT,
label='mask0:: Make mask',
time_out=1800)
recipe.add("cab/ddfacet", "ddfacet_test2",
{
"Data-MS": [MS],
"Output-Name": imname1,
"Image-NPix": 2048,
"Image-Cell": 2.0,
"Cache-Reset": True,
"Freq-NBand": 2,
"Freq-NDegridBand": 4,
"Mask-External": '%s:output' % (maskname0),
"Weight-ColName": "WEIGHT",
"Data-ChunkHours": 0.1,
"Data-Sort": True,
"Log-Boring": True,
"Deconv-MaxMajorIter": 2,
"Deconv-MaxMinorIter": 1500,
},
input=INPUT, output=OUTPUT, shared_memory="24gb",
label="image2",
time_out=1800)
recipe.add("cab/shadems", "shadems_test",
{
'ms': MS,
'xaxis': 'DATA:imag',
'yaxis': 'real',
'col': 'DATA',
'png': '%s_shadems_test_real_imag' % (PREFIX)
},
input=INPUT, output=OUTPUT,
label="shadems_test",
time_out=1800)
# # First selfcal round
recipe.add("cab/cubical_ddf", "cubical_cal",
{
'data-ms': MS,
'data-column': "DATA",
'dist-nworker': 4,
'dist-nthread': 1,
'dist-max-chunks': 20,
'data-freq-chunk': 0,
'data-time-chunk': 1,
'model-list': spf("MODEL_DATA"),
'weight-column': "WEIGHT",
'flags-apply': "FLAG",
'flags-auto-init': "legacy",
'madmax-enable': False,
'madmax-threshold': [0, 0, 10],
'madmax-global-threshold': [0, 0],
'sol-jones': 'g',
'sol-stall-quorum': 0.95,
'out-name': "cubicaltest",
'out-column': "CORRECTED_DATA",
'log-verbose': "solver=2",
'g-type': "complex-2x2",
'g-freq-int': 0,
'g-time-int': 20,
'g-max-iter': 10,
'sol-term-iters': 10,
'g-update-type': "complex-2x2",
}, input=INPUT, output=OUTPUT,
label="cubical",
shared_memory="24gb",
time_out=1800)
recipe.add("cab/cubical", "cubical_cal2",
{
'data-ms': MS,
'data-column': "DATA",
'dist-nworker': 4,
'dist-nthread': 1,
'dist-max-chunks': 20,
'data-freq-chunk': 0,
'data-time-chunk': 1,
'model-list': spf("MODEL_DATA"),
'weight-column': "WEIGHT",
'flags-apply': "FLAG",
'flags-auto-init': "legacy",
'madmax-enable': False,
'madmax-threshold': [0, 0, 10],
'madmax-global-threshold': [0, 0],
'sol-jones': 'g',
'sol-stall-quorum': 0.95,
'out-name': "cubicaltest",
'out-column': "CORRECTED_DATA",
'log-verbose': "solver=2",
'g-type': "complex-2x2",
'g-freq-int': 0,
'g-time-int': 20,
'g-max-iter': 10,
'sol-term-iters': 10,
'out-overwrite' : True,
'g-update-type': "complex-2x2",
}, input=INPUT, output=OUTPUT,
label="cubical",
shared_memory="24gb",
time_out=1800)
recipe.add("cab/ragavi_vis", "ragavi_vis_test",
{
'ms': MS,
'xaxis': 'imaginary',
'yaxis': 'real',
'data-column': 'CORRECTED_DATA',
'htmlname': "%s_ragavi_vis_real_imag" % (PREFIX)
},
input=INPUT, output=OUTPUT,
label="ragavi_vis_test",
time_out=1800)
recipe.run()
def decalibrate(incol="corrected_data",
calincol="corrected_data",
outcol="subtracted_data",
model="model_data",
label='decal',
freq_int=8,
masksig=45,
tagsigma=3.0,
nfacets=4,
solvemode='complex-2x2',
corrtype='sc',
interval=40,
restore=None):
steps = []
steps += image(incol=incol,
label=label,
masksig=masksig,
restore=restore,
nfacets=nfacets)
for ti, t in enumerate(TARGET):
recipe.add(
"cab/catdagger",
"auto_tagger_{}_{}".format(label, ti), {
'ds9-reg-file': "{}.{}.dE.reg".format(label, ti),
'ds9-tag-reg-file': "{}.{}.dE.clusterleads.reg".format(
label, ti),
'sigma': tagsigma,
'min-distance-from-tracking-centre': 350,
'noise-map':
"{}.app.residual.fits:output".format(t + "_" + label),
},
input=INPUT,
output=OUTPUT,
label="auto_tagger_{}_{}".format(label, ti),
shared_memory="250g")
diconame = "{}.DicoModel".format(t + "_" + label)
tagregs = "{}.{}.dE.reg".format(label, ti)
recipe.add("cab/cubical_ddf", "de_calibrate_{}_{}".format(label, ti), {
'data-ms': DATASET,
'data-column': calincol,
'dist-nworker': args.ncubical_workers,
'dist-nthread': args.ncubical_workers,
'dist-max-chunks': args.ncubical_workers,
'data-time-chunk': interval*int(min(1, args.ncubical_workers)) if not args.cubical_split_DI_bandwidth else \
interval*int(min(1, np.sqrt(args.ncubical_workers))),
'data-freq-chunk': 0 if not args.cubical_split_DI_bandwidth else \
freq_int*int(min(1, np.sqrt(args.ncubical_workers))),
'model-list': spf("MODEL_DATA+-{{}}{}@{{}}{}:{{}}{}@{{}}{}".format(diconame, tagregs, diconame, tagregs),
"output", "output", "output", "output"),
#'model-beam-pattern': "'MeerKAT_VBeam_10MHz_53Chans_$(corr)_$(reim).fits':output",
#'model-beam-l-axis' : "X",
#'model-beam-m-axis' : "Y",
'weight-column': "WEIGHT",
'flags-apply': "FLAG",
'flags-auto-init': "legacy",
'madmax-enable': False,
'madmax-threshold': [0,0,10],
'madmax-global-threshold': [0,0],
'sol-jones': 'g,dd',
'sol-min-bl': 110.0,
'sol-max-bl': 0,
'sol-stall-quorum': 0.95,
'sol-term-iters': [50,90,50,90],
'out-name': t + str(time.time()),
'out-mode': corrtype,
'out-column': outcol,
'out-model-column': "MODEL_OUT",
'log-verbose': "solver=2",
'dd-time-int': interval,
'dd-freq-int': freq_int,
'dd-type': solvemode,
'g-type': solvemode,
'g-freq-int': 0,
'g-time-int': interval,
'g-max-iter': 100,
'g-update-type': "phase-diag",
'dd-max-iter': 200,
'dd-clip-high': 0,
'dd-clip-low': 0,
'dd-fix-dirs': "0",
'out-subtract-dirs': '1:',
'dd-dd-term': True,
'dd-max-prior-error': 0.35,
'dd-max-post-error': 0.35,
'g-clip-high': 0,
'g-clip-low': 0,
'g-max-prior-error': 0.35,
'g-max-post-error': 0.35,
'degridding-NDegridBand': args.mfs_predictbands,
'degridding-MaxFacetSize': 0.15
}, input=INPUT, output=OUTPUT, label="de_calibrate_{}_{}".format(label, ti), shared_memory="250g")
recent_corrected_column = outcol
steps += [
"auto_tagger_{}_{}".format(label, ti),
"de_calibrate_{}_{}".format(label, ti)
]
return steps
