def testSingularity(self): global INPUT, MSDIR, OUTPUT, MS, MS_SIM # Start stimela Recipe instance pipeline = stimela.Recipe( "Singularity Test", # Recipe name ms_dir=MSDIR, singularity_image_dir=os.environ["SINGULARITY_PULLFOLDER"], ) pipeline.add( "cab/simms", # Executor image to start container from "simms_example", # Container name { # Parameters to parse to executor container "msname": MS_SIM, "telescope": "kat-7", # Telescope name "direction": "J2000,0deg,-30deg", # Phase tracking centre of observation "synthesis": 0.128, # Synthesis time of observation "dtime": 10, # Integration time in seconds "freq0": "750MHz", # Start frequency of observation "dfreq": "1MHz", # Channel width "nchan": 1 # Number of channels }, input=INPUT, # Input folder output=OUTPUT, # Output folder label="Creating MS", # Process label cpus=2.5, memory_limit="2gb", time_out=300) # Run recipe. The 'steps' added above will be executed in the sequence that they were adde. The 'steps' added above will be # executed in the sequence that they were added pipeline.run()
def test_udocker(self): import sys global MSDIR global INPUT global OUTPUT global UDOCKER if UDOCKER is False: return stimela.register_globals() rrr = stimela.Recipe("singularitypaths", ms_dir=MSDIR, JOB_TYPE="udocker", cabpath="cab/", log_dir="logs") assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": ["testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"], "testinput2.txt") assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"], "testinput3.txt") assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format( rrr.jobs[0].job.IODEST["input"], rrr.jobs[0].job.IODEST["msfile"], rrr.jobs[0].job.IODEST["output"] )
def test_iopathlist(self): global MSDIR global INPUT global OUTPUT global CABPATH stimela.register_globals() rrr = stimela.Recipe("pathlist", ms_dir=MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": ["testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")], }, cabpath=CABPATH, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"], "testinput2.txt") assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"], "testinput3.txt") assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format(rrr.jobs[0].job.IODEST["input"], rrr.jobs[0].job.IODEST["msfile"], rrr.jobs[0].job.IODEST["output"] )
def test_singularity(self): global MSDIR global INPUT global OUTPUT global SINGULARITY if SINGULARITY is False: return stimela.register_globals() rrr = stimela.Recipe("singularitypaths", ms_dir=MSDIR, JOB_TYPE="singularity", cabpath="cab/", singularity_image_dir=os.environ["STIMELA_PULLFOLDER"], log_dir="logs") rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": ["testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output")], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[0] == os.path.join(rrr.jobs[0].job.IODEST["input"], "testinput2.txt") assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join(rrr.jobs[0].job.IODEST["msfile"], "testinput3.txt") assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}/hello{{reim}}.fits,{}/to.fits,{}/world.fits".format( rrr.jobs[0].job.IODEST["input"], rrr.jobs[0].job.IODEST["msfile"], rrr.jobs[0].job.IODEST["output"] )
def test_iopathlist(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("pathlist", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add( "cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": [ "testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output") ], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[ 0] == "/input/testinput2.txt" assert rrr.jobs[0].job._cab.parameters[4].value[1] == os.path.join( "/", "home", os.environ["USER"], "msdir", "testinput3.txt") assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}hello{{reim}}.fits,{}to.fits,{}world.fits".format( "/input/", os.path.join("/", "home", os.environ["USER"], "msdir/"), os.path.join("/", "home", os.environ["USER"], "output/"), )
def get_fields(pipeline, recipe, indir, caltable, cab_name="get_field_info", label=""): _recipe = stimela.Recipe( "get field info", ms_dir=recipe.ms_dir, JOB_TYPE=recipe.JOB_TYPE, singularity_image_dir=recipe.singularity_image_dir, log_dir=pipeline.logs) tfile = tempfile.NamedTemporaryFile(suffix=".json", dir=pipeline.output) tfile.flush _recipe.add("cab/pycasacore", cab_name, { "msname": caltable + ":input", "script": """ from casacore.tables import table import json import os import numpy import codecs INDIR = os.environ["INPUT"] OUTPUT = os.environ["OUTPUT"] tabname = os.path.join(INDIR, "{caltab:s}") tab = table(tabname) uf = numpy.unique(tab.getcol("FIELD_ID")) fields = dict(field_id=list(map(int, uf))) with codecs.open(OUTPUT+'/{fname:s}', 'w', 'utf8') as stdw: a = json.dumps(fields, ensure_ascii=False) stdw.write(a) tab.close() """.format(caltab=caltable, fname=os.path.basename(tfile.name)), }, input=indir, output=pipeline.output) _recipe.run() with codecs.open(tfile.name, "r", "utf8") as stdr: fields = json.load(stdr) tfile.close() return fields
def test_required(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testrequired", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla3": 4.0, }, input=INPUT, output=OUTPUT) with self.assertRaises(PipelineException): rrr.run() #validate and run
def test_invalid_choice(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("invchoice", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "d" # only accepts a, b or c }, input=INPUT, output=OUTPUT) with self.assertRaises(PipelineException): rrr.run() #validate and run
def test_iopathval(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("ioval", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla2": "testinput2.txt:input", }, input=INPUT, output=OUTPUT) with self.assertRaises(PipelineException): # not exist during validation rrr.run() #validate and run
def test_floattypesuccess(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testfloattypesuccess", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla3": 4.0, }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[2].value == [4.0]
def test_floattypefail(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testfloattypefail", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla3": "1.0a", }, input=INPUT, output=OUTPUT) with self.assertRaises(PipelineException): rrr.run() #validate and run
def test_define_cab(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("customcab", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a" }, input=INPUT, output=OUTPUT) assert len(rrr.jobs) == 1 rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[0].value == "a" assert len(rrr.completed) == 1 assert len(rrr.remaining) == 0
def test_dismissable(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testdismissable", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla4": sdm("abc"), "bla3": sdm(None) }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[0].value == "a" assert rrr.jobs[0].job._cab.parameters[1].value is None assert rrr.jobs[0].job._cab.parameters[2].value is None assert rrr.jobs[0].job._cab.parameters[3].value == ["abc"]
def test_iooverride(self): global MSDIR global INPUT with open(os.path.join(INPUT, "testinput.txt"), "w+") as f: pass global OUTPUT stimela.register_globals() rrr = stimela.Recipe("testiooverrides", ms_dir=MSDIR) assert os.path.exists(MSDIR) rrr.add("cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla2": "testinput.txt:input", }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[0].value == "a" assert rrr.jobs[0].job._cab.parameters[1].value == os.path.join(rrr.jobs[0].job.IODEST["input"], "testinput.txt")
def test_udocker(self): import sys if sys.version_info[0] > 2: return global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe("singularitypaths", ms_dir=MSDIR, JOB_TYPE="udocker", cabpath="cab/") assert os.path.exists(MSDIR) rrr.add( "cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": [ "testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output") ], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[ 0] == "/scratch/input/testinput2.txt" assert rrr.jobs[0].job._cab.parameters[4].value[ 1] == "/scratch/msdir/testinput3.txt" assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}hello{{reim}}.fits,{}to.fits,{}world.fits".format( "/scratch/input/", "/scratch/msdir/", "/scratch/output/" )
def test_singularity(self): global MSDIR global INPUT global OUTPUT stimela.register_globals() rrr = stimela.Recipe( "singularitypaths", ms_dir=MSDIR, JOB_TYPE="singularity", cabpath="cab/", singularity_image_dir=os.environ["SINGULARITY_PULLFOLDER"]) assert os.path.exists(MSDIR) rrr.add( "cab/custom", "test1", { "bla1": "a", # only accepts a, b or c "bla5": [ "testinput2.txt:input", "testinput3.txt:msfile", spf("{}hello\{reim\}.fits,{}to.fits,{}world.fits", "input", "msfile", "output") ], }, input=INPUT, output=OUTPUT) rrr.run() #validate and run assert rrr.jobs[0].job._cab.parameters[4].value[ 0] == "/scratch/input/testinput2.txt" assert rrr.jobs[0].job._cab.parameters[4].value[ 1] == "/scratch/msdir/testinput3.txt" assert rrr.jobs[0].job._cab.parameters[4].value[2] == \ "{}hello{{reim}}.fits,{}to.fits,{}world.fits".format( "/scratch/input/", "/scratch/msdir/", "/scratch/output/" )
import stimela INPUT = "input" OUTPUT = "output" MSDIR = "msdir" recipe = stimela.Recipe("wsclean_psf_only", ms_dir=MSDIR) recipe.add("cab/wsclean", "wsclean_psf_only", { "msname": ['fa1_conc.ms'], "name": 'tmp', "column": 'DATA', "weight": 'natural', "npix": 10000, "trim": 8192, "scale": 1.3, "make-psf-only": True, "niter": 1, "pol": 'xx', "channelsout": 1, "joinchannels": False, }, input=INPUT, output=OUTPUT, shared_memory="250gb", label="PSF ONLY") recipe.run()
print('Warning: the number of sources is not divisible by the bins') print(' images will have %d/%d sources' %(total_sources, NUM_SOURCES)) # # create stimela recipe # import stimela INPUT = 'input' OUTPUT = 'output' MSDIR = 'msdir' MS = 'meerkat.ms' recipe = stimela.Recipe(name='Make noise image populated with sources', ms_dir=MSDIR, JOB_TYPE='udocker') for img in range(0, NUM_IMAGES): # determine field centre centre_ra = np.random.uniform(RA_MIN, RA_MAX) centre_dec = np.random.uniform(DEC_MIN, DEC_MAX) # create sky model with open('input/%d-skymodel.txt' %(img), 'w') as f: f.write('#format: ra_d dec_d i\n') for flux in bins: for i in range(0, sources_per_bin): ra = np.random.uniform(centre_ra-OFFSET_RA, centre_ra+OFFSET_RA)
import stimela import os INPUT = "/home/maccagni/FornaxA/data/MeerKAT/fa1/input" OUTPUT = "/home/maccagni/FornaxA/data/MeerKAT/fa1/output" MSDIR = "/home/maccagni/FornaxA/data/MeerKAT/fa1/msdir" calmodel = 'fa1_fast_modelcorr.fits' msname = 'fa1_conc-corr.ms' recipe = stimela.Recipe("Calibrate with meqtrees", ms_dir=MSDIR) calmodel = '{0:s}_{1:d}-nullmodel.txt' with open(os.path.join(INPUT, calmodel), 'w') as stdw: stdw.write('#format: ra_d dec_d i\n') stdw.write('0.0 -30.0 1e-99') # step = 'add_bitflag_column' # recipe.add('cab/msutils', step, # { # "msname" : msname, # "command" : 'copycol' , # "fromcol" : 'FLAG', # "tocol" : 'BITFLAG', # }, # input=INPUT, # output=OUTPUT, # label='Add BITFLAG column ') step = 'calibrate_cubical' recipe.add('cab/cubical', step, {
return 0 # def chdr(filename): # with fits.open(filename, 'update') as mask: # mask[0].header['NAXIS'] = '2' # mask[0].header['NAXIS3'] # mask[0].header['NAXIS4'] # mask.flush() # step= '0' # recipe = stimela.Recipe("Test clean mask making", ms_dir=MSDIR) # recipe.add("cab/cleanmask", step, # { # "image" : image+":output", # "output" : fa1_cleanmask+":output", # "dilate" : False, # "sigma" : 15, # "no-negative" : True, # }, # input=INPUT, # output=OUTPUT, # label="Make a mask out of clean image clean image") step = '1' recipe.add('cab/fitstool', step, {
import stimela PREFIX = "selfcal" recipe = stimela.Recipe("selfcal_simulation", indir="input", outdir="output", cachedir="cachedir") recipe.add( "simms", "makems", { "msname": "meerkat_SourceRecovery.ms", "telescope": "meerkat", "direction": "J2000,0deg,-30deg", "synthesis": 0.5, # in hours "dtime": 5, # in seconds "freq0": 1.42e9, # in hertz "dfreq": 1e6, # in hertz "nchan": 4, }, doc="Create Empty MS") recipe.add( "simulator", "simsky", { "msname": recipe.makems.outputs["msname_out"], "config": "tdlconf.profiles", "use_smearing": False,
vermeerkat.log.info("The following fields are available:") for f in FDB: vermeerkat.log.info("\t '{0:s}' index {1:s}{2:s}".format( f, FDB[f], " selected as 'BP'" if f == BPCALIBRATOR else " selected as 'GC'" if f in GCALIBRATOR else " selected as 'ALTCAL'" if f in ALTCAL else " selected as 'TARGET'" if f in TARGET else " not selected")) if not vermeerkat.prompt(dont_prompt=args.dont_prompt): vermeerkat.log.info("Aborted per user request") sys.exit(1) stimela.register_globals() recipe = stimela.Recipe('MEERKAT: basic transfer calibration', ms_dir=MSDIR, singularity_image_dir=os.environ.get( "SINGULARITY_PULLFOLDER", ""), JOB_TYPE=args.containerization) def addmanualflags(recipe, reason, antenna="", spw="", scan="", uvrange="", field=""): """ Read CASA flagdata docs before using """ recipe.add("cab/casa_flagdata", "handflags", { "vis": ZEROGEN_DATA,
import stimela INPUT = "/Users/maccagni/Projects/FornaX/FornaxA/data/MeerKAT/fa1/input" OUTPUT = "/Users/maccagni/Projects/FornaX/FornaxA/data/MeerKAT/fa1/output" MSDIR = "/Users/maccagni/Projects/FornaX/FornaxA/data/MeerKAT/fa1/msdir" ms = MSDIR + '/fa1_tosub.ms' toutname = MSDIR + '/fa1_conc_tmp-corr.ms' recipe = stimela.Recipe("Subtract model of continuum", ms_dir=MSDIR) recipe.add( "cab/msutils", "sub_model", { "command": 'sumcols', "msname": 'fa1_conc_csub-corr.ms', #"subtract" : True, "col1": 'CORRECTED_DATA', "col2": 'MODEL_DATA', "tocol": 'CORRECTED_DATA' }, input=INPUT, output=OUTPUT, label="Subtract model_data column") recipe.run() #import pyrap.tables as tables #print '' #print '--- Working on file {0:s} ---'.format(ms)
timegain_solint) log.info("Fields being used for solving:") for fi, f in enumerate([field_list[fk] for fk in field_list.keys()]): label = " (BP)" if f in args.bandpass_field else " (POL)" if f in args.polcal_field else "" log.info("\t %d: %s%s" % (fi, f, label)) if not vermeerkat.prompt(dont_prompt=args.dont_prompt): vermeerkat.log.info("Aborted per user request") sys.exit(1) stimela.register_globals() recipe = stimela.Recipe( 'MEERKAT FleetingPol: Interferometric boresight polarization calibration', ms_dir=MSDIR, singularity_image_dir=os.environ.get("SINGULARITY_PULLFOLDER", ""), JOB_TYPE=args.containerization) recipe.add("cab/casa_split", "split_avg_data", { "vis": COMB_MS, "outputvis": BP_CAL_MS, "datacolumn": "corrected", "field": ",".join(args.bandpass_field + args.polcal_field), "timebin": pol_mstimeavg, "width": pol_solchanavg, }, input=INPUT, output=OUTPUT, label="split_avg_data")
def testEndToEndReduction(self): global INPUT, OUTPUT, MSDIR, MS, LABEL recipe = stimela.Recipe('Test reduction script', ms_dir=MSDIR, JOB_TYPE="docker", log_dir="logs") imname1 = "deep2.1gc" imname2 = "deep2.2gc" recipe.add("cab/ddfacet", "ddfacet_test1", { "Data-MS": [MS], "Output-Name": imname1, "Image-NPix": 2048, "Image-Cell": 2, "Cache-Reset": True, "Freq-NBand": 2, "Freq-NDegridBand": 4, "Weight-ColName": "WEIGHT", "Data-ChunkHours": 0.1, "Data-Sort": True, "Log-Boring": True, "Deconv-MaxMajorIter": 2, "Deconv-MaxMinorIter": 1500, "Predict-ColName": "MODEL_DATA" }, input=INPUT, output=OUTPUT, shared_memory="8gb", label="image1", time_out=1800) recipe.add('cab/tricolour', "flag_data", { "ms": MS, "data-column": "DATA", "window-backend": 'numpy', "flagging-strategy": "total_power", "subtract-model-column": "MODEL_DATA", }, input=INPUT, output=OUTPUT, label="flag_data", time_out=1800) maskname0 = "MASK.fits" recipe.add('cab/cleanmask', 'mask0', { "image": '%s.app.restored.fits:output' % (imname1), "output": '%s:output' % (maskname0), "dilate": False, "sigma": 25, }, input=INPUT, output=OUTPUT, label='mask0:: Make mask', time_out=1800) recipe.add("cab/ddfacet", "ddfacet_test2", { "Data-MS": [MS], "Output-Name": imname1, "Image-NPix": 2048, "Image-Cell": 2.0, "Cache-Reset": True, "Freq-NBand": 2, "Freq-NDegridBand": 4, "Mask-External": '%s:output' % (maskname0), "Weight-ColName": "WEIGHT", "Data-ChunkHours": 0.1, "Data-Sort": True, "Log-Boring": True, "Deconv-MaxMajorIter": 2, "Deconv-MaxMinorIter": 1500, }, input=INPUT, output=OUTPUT, shared_memory="24gb", label="image2", time_out=1800) recipe.add("cab/shadems", "shadems_test", { 'ms': MS, 'xaxis': 'DATA:imag', 'yaxis': 'real', 'col': 'DATA', 'png': '%s_shadems_test_real_imag' % (PREFIX) }, input=INPUT, output=OUTPUT, label="shadems_test", time_out=1800) # # First selfcal round recipe.add("cab/cubical_ddf", "cubical_cal", { 'data-ms': MS, 'data-column': "DATA", 'dist-nworker': 4, 'dist-nthread': 1, 'dist-max-chunks': 20, 'data-freq-chunk': 0, 'data-time-chunk': 1, 'model-list': spf("MODEL_DATA"), 'weight-column': "WEIGHT", 'flags-apply': "FLAG", 'flags-auto-init': "legacy", 'madmax-enable': False, 'madmax-threshold': [0, 0, 10], 'madmax-global-threshold': [0, 0], 'sol-jones': 'g', 'sol-stall-quorum': 0.95, 'out-name': "cubicaltest", 'out-column': "CORRECTED_DATA", 'log-verbose': "solver=2", 'g-type': "complex-2x2", 'g-freq-int': 0, 'g-time-int': 20, 'g-max-iter': 10, 'sol-term-iters': 10, 'g-update-type': "complex-2x2", }, input=INPUT, output=OUTPUT, label="cubical", shared_memory="24gb", time_out=1800) recipe.add("cab/cubical", "cubical_cal2", { 'data-ms': MS, 'data-column': "DATA", 'dist-nworker': 4, 'dist-nthread': 1, 'dist-max-chunks': 20, 'data-freq-chunk': 0, 'data-time-chunk': 1, 'model-list': spf("MODEL_DATA"), 'weight-column': "WEIGHT", 'flags-apply': "FLAG", 'flags-auto-init': "legacy", 'madmax-enable': False, 'madmax-threshold': [0, 0, 10], 'madmax-global-threshold': [0, 0], 'sol-jones': 'g', 'sol-stall-quorum': 0.95, 'out-name': "cubicaltest", 'out-column': "CORRECTED_DATA", 'log-verbose': "solver=2", 'g-type': "complex-2x2", 'g-freq-int': 0, 'g-time-int': 20, 'g-max-iter': 10, 'sol-term-iters': 10, 'out-overwrite' : True, 'g-update-type': "complex-2x2", }, input=INPUT, output=OUTPUT, label="cubical", shared_memory="24gb", time_out=1800) recipe.add("cab/ragavi_vis", "ragavi_vis_test", { 'ms': MS, 'xaxis': 'imaginary', 'yaxis': 'real', 'data-column': 'CORRECTED_DATA', 'htmlname': "%s_ragavi_vis_real_imag" % (PREFIX) }, input=INPUT, output=OUTPUT, label="ragavi_vis_test", time_out=1800) recipe.run()
def testEndToEndReduction(self): global INPUT, OUTPUT, MSDIR, MS, LABEL global GAINCAL_TABLE2, FLUXSCALE_TABLE, GAINCAL_TABLE, DELAYCAL_TABLE, BPCAL_TABLE, ANTPOS_TABLE global REFANT, BPCAL, TARGET, GCAL, PREFIX global MSCONTSUB, SPW, LSM0, SELFCAL_TABLE1, corr_ms, lsm0 global IMAGE1, IMAGE2, MASK1, nchans, chans, imname0, maskname0, maskname01, imname1 recipe = stimela.Recipe('Test reduction script', ms_dir=MSDIR, JOB_TYPE="docker") recipe.add('cab/casa_listobs', 'listobs', {"vis": MS}, input=INPUT, output=OUTPUT, label='listobs:: some stats', time_out=300) # It is common for the array to require a small amount of time to settle down at the start of a scan. Consequently, it has # become standard practice to flag the initial samples from the start of each scan. This is known as 'quack' flagging recipe.add('cab/casa_flagdata', 'quack_flagging', { "vis": MS, "mode": 'quack', "quackinterval": 10.0, "quackmode": 'beg', }, input=INPUT, output=OUTPUT, label='quack_flagging:: Quack flagging', time_out=300) # Flag the autocorrelations recipe.add('cab/casa_flagdata', 'autocorr_flagging', { "vis": MS, "mode": 'manual', "autocorr": True, }, input=INPUT, output=OUTPUT, label='autocorr_flagging:: Autocorrelations flagging', time_out=300) # Flag bad channels recipe.add( 'cab/casa_flagdata', 'badchan_flagging', { "vis": MS, "mode": 'manual', "spw": "0:113~113,0:313~313,0:369~369,0:601~607,0:204~204,0:212~212,0:594~600", }, input=INPUT, output=OUTPUT, label='badchan_flagging:: Bad Channel flagging', time_out=300) recipe.add('cab/casa_clearcal', 'clearcal', { "vis": MS, "addmodel": True }, input=INPUT, output=OUTPUT, label='clearcal:: casa clearcal', time_out=300) recipe.add( 'cab/casa_setjy', 'set_flux_scaling', { "vis": MS, "field": BPCAL, "standard": 'Perley-Butler 2010', "usescratch": True, "scalebychan": True, }, input=INPUT, output=OUTPUT, label= 'set_flux_scaling:: Set flux density value for the amplitude calibrator', time_out=300) recipe.add( 'cab/casa_bandpass', 'bandpass_cal', { "vis": MS, "caltable": BPCAL_TABLE, "field": BPCAL, "refant": REFANT, "spw": SPW, "solint": 'inf', "bandtype": 'B', # "opacity" : 0.0, # "gaincurve" : False, }, input=INPUT, output=OUTPUT, label='bandpass_cal:: Bandpass calibration', time_out=300) # display the bandpass solutions. Note that in the plotcal inputs below, the amplitudes are being displayed as a function of # frequency channel. The parameter subplot=221 is used to display multiple plots per page (2 plots per page in the y # direction and 2 in the x direction). The first two commands below show the amplitude solutions (one per each polarization) # and the last two show the phase solutions (one per each polarization). Parameter iteration='antenna' is used to step # through separate plots for each antenna. recipe.add('cab/casa_plotcal', 'plot_bandpass_amp_R', { "caltable": BPCAL_TABLE, "poln": 'R', "xaxis": 'chan', "yaxis": 'amp', "field": BPCAL, "spw": SPW, "subplot": 221, "figfile": PREFIX + '-B0-R-amp.png', }, input=INPUT, output=OUTPUT, label='plot_bandpass_amp_R:: Plot bandpass table. AMP, R', time_out=1200) # Gain calibration - amplitude and phase - first for BPCAL. recipe.add('cab/casa_gaincal', 'gaincal_bp', { "vis": MS, "caltable": GAINCAL_TABLE, "field": "{0:s},{1:s}".format(BPCAL, GCAL), "solint": 'inf', "refant": '', "gaintype": 'G', "calmode": 'ap', "spw": SPW, "solnorm": False, "gaintable": [BPCAL_TABLE], "interp": ['nearest'], }, input=INPUT, output=OUTPUT, label="gaincal:: Gain calibration", time_out=300) # Set fluxscale recipe.add('cab/casa_fluxscale', 'fluxscale', { "vis": MS, "caltable": GAINCAL_TABLE, "fluxtable": FLUXSCALE_TABLE, "reference": [BPCAL], "transfer": [GCAL], "incremental": False, }, input=INPUT, output=OUTPUT, label='fluxscale:: Set fluxscale', time_out=300) # Apply calibration to BPCAL recipe.add( 'cab/casa_applycal', 'applycal_bp', { "vis": MS, "field": BPCAL, "gaintable": [BPCAL_TABLE, FLUXSCALE_TABLE], "gainfield": ['', '', BPCAL], "interp": ['', '', 'nearest'], "calwt": [False], "parang": False, "applymode": "calflag", }, input=INPUT, output=OUTPUT, label='applycal_bp:: Apply calibration to Bandpass Calibrator', time_out=1800) recipe.run() recipe = stimela.Recipe('KAT reduction script 2', ms_dir=MSDIR, JOB_TYPE="docker") # Copy CORRECTED_DATA to DATA, so we can start uv_contsub recipe.add("cab/msutils", "move_corrdata_to_data", { "command": "copycol", "msname": MS, "fromcol": "CORRECTED_DATA", "tocol": "DATA", }, input=INPUT, output=OUTPUT, label="move_corrdata_to_data::msutils", time_out=1800) os.system("rm -rf {}/{}-corr.ms".format(MSDIR, MS[:-3])) recipe.add('cab/casa_split', 'split_corr_data', { "vis": MS, "outputvis": MS[:-3] + '-corr.ms', "field": str(BPCAL), "spw": SPW, "datacolumn": 'data', }, input=INPUT, output=OUTPUT, label='split_corr_data:: Split corrected data from MS', time_out=1800) MS = MS[:-3] + '-corr.ms' recipe.add( 'cab/casa_clearcal', 'prep_split_data', { "vis": MS, "addmodel": True }, input=INPUT, output=OUTPUT, label='prep_split_data:: Prep split data with casa clearcal', time_out=1800) # Clean-Mask-Clean imname0 = PREFIX + 'image0' maskname0 = PREFIX + 'mask0.fits' maskname01 = PREFIX + 'mask01.fits' imname1 = PREFIX + 'image1' recipe.add( 'cab/casa_tclean', 'image_target_field_r1', { "vis": MS, "datacolumn": "corrected", "field": "0", "start": 21, # Other channels don't have any data "nchan": 235 - 21, "width": 1, # Use Briggs weighting to weigh visibilities for imaging "weighting": "briggs", "robust": 0, "imsize": 256, # Image size in pixels "cellsize": "30arcsec", # Size of each square pixel "niter": 100, "stokes": "I", "prefix": '%s:output' % (imname1), }, input=INPUT, output=OUTPUT, label="image_target_field_r1:: Image target field second round", time_out=90) recipe.add('cab/cleanmask', 'mask0', { "image": '%s.image.fits:output' % (imname1), "output": '%s:output' % (maskname0), "dilate": False, "sigma": 20, }, input=INPUT, output=OUTPUT, label='mask0:: Make mask', time_out=1800) lsm0 = PREFIX + '-LSM0' # Source finding for initial model recipe.add( "cab/pybdsm", "extract_init_model", { "image": '%s.image.fits:output' % (imname1), "outfile": '%s:output' % (lsm0), "thresh_pix": 25, "thresh_isl": 15, "port2tigger": True, }, input=INPUT, output=OUTPUT, label= "extract_init_model:: Make initial model from preselfcal image", time_out=1800) # Add bitflag column. To keep track of flagsets. recipe.add("cab/msutils", "msutils", { 'command': 'prep', 'msname': MS, }, input=INPUT, output=OUTPUT, label="prepms::Adds flagsets", time_out=1800) # Not used currently. recipe.add("cab/flagms", "backup_initial_flags", { "msname": MS, "create": True, "nan": True, "flagged-any": "+L", "flag": "legacy", }, input=INPUT, output=OUTPUT, label="backup_initial_flags:: Backup selfcal flags", time_out=1800) # First selfcal round recipe.add( "cab/calibrator", "calibrator_Gjones_subtract_lsm0", { "skymodel": "%s.lsm.html:output" % (lsm0), "msname": MS, "threads": 16, "column": "DATA", "output-data": "CORR_RES", "Gjones": True, # Ad-hoc right now, subject to change "Gjones-solution-intervals": [20, 0], "Gjones-matrix-type": "GainDiagPhase", "tile-size": 512, "field-id": 0, }, input=INPUT, output=OUTPUT, label= "calibrator_Gjones_subtract_lsm0:: Calibrate and subtract LSM0", time_out=1800) # Diversity is a good thing... lets add some DDFacet to this soup bowl imname = PREFIX + 'ddfacet' recipe.add( "cab/ddfacet", "ddfacet_test", { "Data-MS": [MS], "Output-Name": imname, "Image-NPix": 256, "Image-Cell": 30, "Cache-Reset": True, "Freq-NBand": 2, "Weight-ColName": "WEIGHT", "Data-ChunkHours": 10, "Beam-FITSFeed": "rl", "Data-Sort": True, "Log-Boring": True, "Deconv-MaxMajorIter": 1, "Deconv-MaxMinorIter": 20, }, input=INPUT, output=OUTPUT, shared_memory="200gb", label= "image_target_field_r0ddfacet:: Make a test image using ddfacet", time_out=120) lsm1 = PREFIX + '-LSM0' # Source finding for initial model recipe.add( "cab/pybdsm", "extract_init_model", { "image": '%s.app.restored.fits:output' % (imname), "outfile": '%s:output' % (lsm1), "thresh_pix": 25, "thresh_isl": 15, "port2tigger": True, }, input=INPUT, output=OUTPUT, label= "extract_init_model:: Make initial model from preselfcal image", time_out=1800) # Stitch LSMs together lsm2 = PREFIX + '-LSM2' recipe.add("cab/tigger_convert", "stitch_lsms1", { "input-skymodel": "%s.lsm.html:output" % lsm0, "output-skymodel": "%s.lsm.html:output" % lsm2, "rename": True, "force": True, "append": "%s.lsm.html:output" % lsm1, }, input=INPUT, output=OUTPUT, label="stitch_lsms1::Create master lsm file", time_out=300) recipe.add('cab/casa_uvcontsub', 'uvcontsub', { "msname": MS, "field": "0", "fitorder": 1, }, input=INPUT, output=OUTPUT, label='uvcontsub:: Subtract continuum in the UV plane', time_out=1800) # Image HI recipe.add('cab/casa_clean', 'casa_dirty_cube', { "msname": MS + ".contsub", "prefix": PREFIX, "mode": 'channel', "nchan": nchans, "niter": 0, "npix": 256, "cellsize": 30, "weight": 'natural', "port2fits": True, }, input=INPUT, output=OUTPUT, label='casa_dirty_cube:: Make a dirty cube with CASA CLEAN', time_out=1800) recipe.add( 'cab/sofia', 'sofia', { # USE THIS FOR THE WSCLEAN DIRTY CUBE # "import.inFile" : '{:s}-cube.dirty.fits:output'.format(combprefix), # USE THIS FOR THE CASA CLEAN CUBE # CASA CLEAN cube "import.inFile": '{:s}.image.fits:output'.format(PREFIX), "steps.doFlag": False, "steps.doScaleNoise": True, "steps.doSCfind": True, "steps.doMerge": True, "steps.doReliability": False, "steps.doParameterise": False, "steps.doWriteMask": True, "steps.doMom0": False, "steps.doMom1": False, "steps.doWriteCat": True, "flag.regions": [], "scaleNoise.statistic": 'mad', "SCfind.threshold": 4, "SCfind.rmsMode": 'mad', "merge.radiusX": 2, "merge.radiusY": 2, "merge.radiusZ": 2, "merge.minSizeX": 2, "merge.minSizeY": 2, "port2tigger": False, "merge.minSizeZ": 2, }, input=INPUT, output=OUTPUT, label='sofia:: Make SoFiA mask and images', time_out=1800) recipe.run()
import stimela import time import json import re INPUT = "input" OUTPUT = "output" MSDIR = "msdir" MANUAL = False CONFIG = '' if MANUAL: recipe = stimela.Recipe('Make Pipeline', ms_dir=MSDIR) stimela.register_globals() def save_execution_time(data_dict, filename='recipes.json', root='output'): """Save execution time""" filename = '{}/{}_time-it.json'.format(root, filename.split('.')[0]) try: # Extract data from the json data file with open(filename) as data_file: data_existing = json.load(data_file) data_existing.update(data_dict) data = data_existing except IOError: data = data_dict if data: with open(filename, 'w') as f: json.dump(data, f)
PREFIX = "stimela-example" # Prefix for output images try: SINGULARTITY_IMAGE_DIR = os.environ["STIMELA_SINGULARTITY_IMAGE_DIR"] except KeyError: SINGULARTITY_IMAGE_DIR = None # MS name MS = "meerkat_simulation_example.ms" # Use the NVSS skymodel. This is natively available LSM = "nvss1deg.lsm.html" # Start stimela Recipe instance pipeline = stimela.Recipe( "Simulation Example", # Recipe name ms_dir=MSDIR, singularity_image_dir=SINGULARTITY_IMAGE_DIR, log_dir=os.path.join(OUTPUT, "logs"), ) pipeline.JOB_TYPE = "podman" # 1: Make empty MS pipeline.add( "cab/simms", # Executor image to start container from "simms_example", # Container name { # Parameters to parse to executor container "msname": MS, "telescope": "meerkat", # Telescope name "direction": "J2000,0deg,-30deg", # Phase tracking centre of observation "synthesis": 0.128, # Synthesis time of observation
def testBasicSim(self): global INPUT, MSDIR, OUTPUT, MS, PREFIX, LSM # Start stimela Recipe instance pipeline = stimela.Recipe( "Singularity Test", # Recipe name ms_dir=MSDIR, singularity_image_dir=os.environ["SINGULARITY_PULLFOLDER"], ) pipeline.add( "cab/simms", # Executor image to start container from "simms_example", # Container name { # Parameters to parse to executor container "msname": MS, "telescope": "kat-7", # Telescope name "direction": "J2000,0deg,-30deg", # Phase tracking centre of observation "synthesis": 0.128, # Synthesis time of observation "dtime": 10, # Integration time in seconds "freq0": "750MHz", # Start frequency of observation "dfreq": "1MHz", # Channel width "nchan": 1 # Number of channels }, input=INPUT, # Input folder output=OUTPUT, # Output folder label="Creating MS", # Process label cpus=2.5, memory_limit="2gb", time_out=300) pipeline.add("cab/casa_listobs", "listobs_example", {"vis": MS}, input=INPUT, output=OUTPUT, label="Some obs details", time_out=100) pipeline.add( "cab/simulator", "simulator_example", { "msname": MS, "skymodel": LSM, # Sky model to simulate into MS "addnoise": True, # Add thermal noise to visibilities "column": "CORRECTED_DATA", # Simulated data will be saved in this column "sefd": 831, # Compute noise from this SEFD "recenter": True, # Recentre sky model to phase tracking centre of MS "tile-size": 64, "threads": 4, }, input=INPUT, output=OUTPUT, label="Simulating visibilities", time_out=600) # pipeline.add('cab/casa_plotms', # 'plot_vis', # { # "vis" : MS, # "xaxis" : 'uvdist', # "yaxis" : 'amp', # "xdatacolumn" : 'corrected', # "ydatacolumn" : 'corrected', # "plotfile" : PREFIX+'-amp_uvdist.png', # "overwrite" : True, # }, # input=INPUT, # output=OUTPUT, # label='plot_amp_uvdist:: Plot amplitude vs uv-distance', # time_out=600) ## Image # Make things a bit interesting by imaging with different weights # Briggs robust values to use for each image briggs_robust = [2] for i, robust in enumerate(briggs_robust): pipeline.add( "cab/wsclean", "imager_example_robust_{:d}".format(i), { "msname": MS, "weight": "briggs {:d}".format(i), "prefix": "{:s}_robust-{:d}".format(PREFIX, robust), "npix": 2048, # Image size in pixels "cellsize": 2, # Size of each square pixel "clean_iterations": 1000, # Perform 1000 iterarions of clean (Deconvolution) }, input=INPUT, output=OUTPUT, label="Imaging MS, robust={:d}".format(robust), cpus=2, memory_limit="2gb", time_out=600) pipeline.add("cab/casa_rmtables", "delete_ms", { "tablenames": MS + ":msfile", }, input=INPUT, output=OUTPUT, label="Remove MS", time_out=300) # Run recipe. The 'steps' added above will be executed in the sequence that they were adde. The 'steps' added above will be # executed in the sequence that they were added pipeline.run()
def run_workers(self): """ Runs the workers """ report_updated = False for _name, _worker, i in self.workers: try: worker = __import__(_worker) except ImportError: traceback.print_exc() raise ImportError('Worker "{0:s}" could not be found at {1:s}'.format( _worker, self.workers_directory)) if self.config["general"]["cabs"]: log.info("Configuring cab specification overrides") cabspecs_general = self.parse_cabspec_dict(self.config["general"]["cabs"]) else: cabspecs_general = {} active_workers = [] # first, check that workers import, and check their configs for _name, _worker, i in self.workers: config = self.config[_name] if 'enable' in config and not config['enable']: self.skip.append(_worker) continue log.info("Configuring worker {}".format(_name)) try: worker = __import__(_worker) except ImportError: log.error('Error importing worker "{0:s}" from {1:s}'.format(_worker, self.workers_directory)) raise if hasattr(worker, 'check_config'): worker.check_config(config) # check for cab specs cabspecs = cabspecs_general if config["cabs"]: cabspecs = cabspecs.copy() cabspecs.update(self.parse_cabspec_dict(config["cabs"])) active_workers.append((_name, worker, config, cabspecs)) # now run the actual pipeline #for _name, _worker, i in self.workers: for _name, worker, config, cabspecs in active_workers: # Define stimela recipe instance for worker # Also change logger name to avoid duplication of logging info label = getattr(worker, 'LABEL', None) if label is None: # if label is not set, take filename, and split off _worker.py label = os.path.basename(worker.__file__).rsplit("_", 1)[0] recipe = stimela.Recipe(label, ms_dir=self.msdir, singularity_image_dir=self.singularity_image_dir, log_dir=self.logs, cabspecs=cabspecs, logfile=False, # no logfiles for recipes logfile_task=f'{self.logs}/log-{label}-{{task}}-{self.timeNow}.txt') recipe.JOB_TYPE = self.container_tech self.CURRENT_WORKER = _name # Don't allow pipeline-wide resume # functionality os.system('rm -f {}'.format(recipe.resume_file)) # Get recipe steps # 1st get correct section of config file log.info("{0:s}: initializing".format(label), extra=dict(color="GREEN")) worker.worker(self, recipe, config) log.info("{0:s}: running".format(label)) recipe.run() log.info("{0:s}: finished".format(label)) # this should be in the cab cleanup code, no? casa_last = glob.glob(self.output + '/*.last') for file_ in casa_last: os.remove(file_) # update report at end of worker if so configured if self.generate_reports and config["report"]: self.regenerate_reports() report_updated = True else: report_updated = False # generate final report if self.config["general"]["final_report"] and self.generate_reports and not report_updated: self.regenerate_reports() log.info("pipeline run complete")