def flagcaltar_ms(src_dir, msname, ATCA_band, pri, sec, tar):
applycal(
vis=msname,
gaintable=[
f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.B1",
f"{src_dir}/cal_tables/cal_{pri}_{ATCA_band}.F0",
],
gainfield=[pri, pri, sec],
field=tar,
parang=True,
flagbackup=False,
)
flagdata(
vis=msname,
mode="rflag",
field=tar,
datacolumn="corrected",
action="apply",
display="report",
correlation="ABS_ALL",
timedevscale=3.0,
freqdevscale=3.0,
winsize=3,
combinescans=True,
ntime="9999999min",
extendflags=False,
flagbackup=False,
)
return
def quack(project: project.Project, quack_interval_s: float = 5):
"""Flags the first seconds of each scan, defined by quack_interval (in seconds).
"""
casatasks.flagdata(vis=str(project.msfile),
mode='quack',
quackinterval=quack_interval_s,
flagbackup=False)
def flag_ms(visname): # , rawname1, rawname2, rawname3):
print("Flagging antennas affected by shadowing...")
# importatca(
# vis=visname, files=[rawname1, rawname2, rawname3], options="birdie,noac", edge=4
# )
flagmanager(vis=visname, mode="save", versionname="before_online_flagging")
print("Flagging antennas affected by shadowing...")
flagdata(vis=visname, mode="shadow", tolerance=0.0, flagbackup=False)
print("Flagging visibilities with zero amplitudes...")
flagdata(vis=visname, mode="clip", clipzeros=True, flagbackup=False)
print("Quacking visibilities ...")
flagdata(
vis=visname, mode="quack", quackinterval=5.0, quackmode="beg", flagbackup=False
)
flagmanager(vis=visname, mode="save", versionname="after_online_flagging")
flagdata(
vis=visname,
mode="tfcrop",
datacolumn="data",
action="apply",
display="report",
flagbackup=True,
extendpols=True,
correlation="",
flagdimension="freqtime",
growtime=95.0,
growfreq=95.0,
timecutoff=4.0,
timefit="line",
freqfit="poly",
maxnpieces=5,
combinescans=False,
ntime="scan",
extendflags=False,
)
print("Extending flags to all correlations")
flagdata(
vis=visname,
mode="extend",
action="apply",
display="report",
flagbackup=False,
extendpols=True,
correlation="",
growtime=95.0,
growfreq=95.0,
growaround=True,
flagneartime=False,
flagnearfreq=False,
combinescans=False,
ntime="scan",
)
return
def edge_channels(project: project.Project, edge_fraction: float = 0.1):
"""Flangs the edge channels of each subband according to the specified edge_fraction.
For example, 0.1 (default) will imply to flag the 10% of the channels next to the edge of each subband.
"""
edge_channel = int(project.nchannels / (100 * edge_fraction))
start = str(edge_channel - 1)
end = str(project.nchannels - edge_channel)
spw2flag = f"*:0~{start};{end}~{str(project.nchannels - 1)}"
casatasks.flagdata(vis=str(project.msfile),
mode='manual',
spw=spw2flag,
flagbackup=False)
def extend(msfile, params, field=None, grow=90, instance='initial'):
if instance == 'initial':
datacol = 'data'
sources = ''
elif instance == 'postcal':
datacol = 'corrected'
sources = field
else:
print('Please provide a relevant instance.[initial/postcal] ')
cts.flagdata(msfile, mode="extend", field=sources, datacolumn=datacol, clipzeros=True, ntime="scan", extendflags=False, extendpols=True, growtime=grow, growfreq=grow, growaround=False, flagneartime=False, flagnearfreq=False, action="apply", flagbackup=True, overwrite=True, writeflags=True)
return print('Flags extended with a grow of', grow)
def apply_apriori_flagging(project: project.Project):
"""Applies the flags stored in the file associated with the project.
"""
return
casatasks.flagdata(vis=str(project.msfile),
inpfile=str(project.caldir /
f"{project.project_name}.flag"),
reason='any',
action='apply',
flagbackup=False,
savepars=False)
casatasks.flagmanager(vis=str(project.msfile),
mode='save',
versionname='apriori_flags',
comment='A-priori flags prior to any calibration.')
def clipper(msfile, params, field=None, cliplevel=100, instance='initial'):
if instance == 'initial':
datacol = 'data'
sources = ''
elif instance == 'postcal':
datacol = 'corrected'
sources = field
else:
print('Please provide a relevant instance.[initial/postcal] ')
spw = params['general']['spw']
cts.flagdata(msfile , mode='clip', spw=spw, field=sources, clipminmax=cliplevel, datacolumn=datacol, clipoutside=True, clipzeros=True, extendpols=False, action='apply',flagbackup=True, savepars=False, overwrite=True, writeflags=True)
return print('Data flagged with clipper at ', cliplevel)
def flag_quack_integrations(myvis, num_ints=2.5):
tb = table()
tb.open(myvis)
int_time = tb.getcol('INTERVAL')[0]
tb.close()
this_quackinterval = num_ints * int_time
flagdata(vis=myvis,
flagbackup=False,
mode='quack',
quackmode='beg',
quackincrement=False,
quackinterval=this_quackinterval)
def tfcropper(msfile, params, field=None, tcut=10.0, fcut=10.0, instance='initial'):
spw = params['general']['spw']
if instance == 'initial':
cts.flagdata(msfile, mode='tfcrop', datacolumn='data', field='', ntime='scan', spw=spw,
timecutoff=tcut, freqcutoff=fcut, timefit='poly',freqfit='poly',flagdimension='timefreq',
extendflags=False, timedevscale=5.0,freqdevscale=5.0, extendpols=False,growaround=False,
action='apply', flagbackup=True,overwrite=True, writeflags=True)
elif instance == 'postcal':
cts.flagdata(msfile, mode='tfcrop', datacolumn='corrected', field=field, ntime='scan', spw=spw,
timecutoff=tcut, freqcutoff=fcut, timefit='poly',freqfit='line',flagdimension='timefreq',
extendflags=False, timedevscale=5.0,freqdevscale=5.0, extendpols=False,growaround=False,
action='apply', flagbackup=True,overwrite=True, writeflags=True)
return print('Data flagged with tfcrop.')
def gcflagdata(msin, bad_ants, cut_edges=True, bad_chans=None):
"""Flag bad antennas for visibility dataset
:param msin: Visibility dataset in measurement set format path
:type msin: str
:param bad_ants: Bad antennas to flag - can be specified by a list of
antennas, or can be given by a string specifying the data release
:type bad_ants: list or str
:param cut_edges: Specify if the band edges should be flagged
:type cut_edges: bool
:param bad_chans: Specify channels to flag
:type bad_chans: list of strings (e.g. ['207', '377~378'])
:param msin: Flagged visibility dataset
:type msin: Measurement set
"""
if bad_ants == 'IDR2':
JD = int(msin.split('.')[1]) # Get JD from filename
if JD in idr2_bad_ants_casa[0, :]:
bad_ants = get_bad_ants(msin, idr2_bad_ants_casa, verbose=True)
else:
bad_ants = None
print('Visibility dataset {} not in IDR2 - no antennas flagged'.
format(msin))
# Flagging bad antennas known for that JD
if bad_ants is not None:
flagdata(msin, flagbackup=True, mode='manual', \
antenna=str(bad_ants).replace("[", "").replace("]", ""))
# Cutting visibilities at extremes of bandwidth
if cut_edges:
flagdata(msin, flagbackup=True, mode='manual', spw='0:0~65')
flagdata(msin, flagbackup=True, mode='manual', spw='0:930~1024')
# Flagging known bad channels
if bad_chans is not None:
for bad_chan in bad_chans:
flagdata(msin,
flagbackup=True,
mode='manual',
spw='0:' + str(bad_chan))
# Flag autocorrelations
flagdata(msin, autocorr=True)
return msin
def flag_tfcrop(project: project.Project):
"""Runs flagdata with the tfcrop option.
In principle it should work fine for EVN data as long as the data are already calibrated.
"""
casatasks.flagdata(vis=str(project.Project),
mode='tfcrop',
datacolumn='corrected',
field=','.join(project.calibrators),
ntime='scan',
timecutoff=6.0,
freqcutoff=5.0,
timefit='line',
freqfit='line',
flagdimension='freqtime',
extendflags=True,
timedevscale=5.0,
freqdevscale=5.0,
extendpols=False,
growaround=False,
action='apply',
flagbackup=True,
overwrite=True,
writeflags=True)
def make_flagsummary_freq_plot(myvis, flag_dict=None, save_name=None):
'''
This mimics the summary plots made by flagdata, but removes the interactive
part so we can save it.
'''
from casatools import ms
from casatasks import flagdata
myms = ms()
myms.open(myvis)
mymsmd = myms.metadata()
if flag_dict is None or 'spw:channel' not in flag_dict:
flag_dict = flagdata(vis=myvis, mode='summary', spwchan=True, action='calculate')
fig = plt.figure()
ax = fig.add_subplot(111)
spw_nums = mymsmd.spwsforscan(1)
for spw in spw_nums:
spw_freqs = mymsmd.chanfreqs(spw) / 1e9 # GHz
spw_flagfracs = []
for chan in range(len(spw_freqs)):
spw_flagfracs.append(flag_dict['spw:channel'][f"{spw}:{chan}"]['flagged'] / flag_dict['spw:channel'][f'{spw}:{chan}']['total'])
# Plot it.
plt.plot(spw_freqs, spw_flagfracs, drawstyle='steps-mid', label=f"SPW {spw}")
mymsmd.close()
myms.close()
ax.set_ylim([0.0, 1.5])
plt.legend(loc='upper center', frameon=True, ncol=4)
if save_name is None:
save_name = f"{myvis}_spw_flagfraction"
fig.savefig(f"{save_name}.png")
fig.savefig(f"{save_name}.pdf")
plt.close()
def flagcal_ms(img_dir, msname, ATCA_band, pri, sec):
flagmanager(vis=msname, mode="save", versionname="before_rflag")
flagdata(
vis=msname,
mode="rflag",
field=pri,
datacolumn="corrected",
action="apply",
display="report",
correlation="ABS_ALL",
timedevscale=3.0,
freqdevscale=3.0,
winsize=3,
combinescans=True,
ntime="9999999min",
extendflags=False,
flagbackup=False,
)
flagdata(
vis=msname,
mode="rflag",
field=sec,
datacolumn="corrected",
action="apply",
display="report",
correlation="ABS_ALL",
timedevscale=3.0,
freqdevscale=3.0,
winsize=3,
combinescans=True,
ntime="9999999min",
extendflags=False,
flagbackup=False,
)
flagdata(
vis=msname,
mode="extend",
field=pri + "," + sec,
action="apply",
display="report",
flagbackup=False,
extendpols=True,
correlation="",
growtime=95.0,
growfreq=95.0,
growaround=True,
flagneartime=False,
flagnearfreq=False,
combinescans=True,
ntime="9999999min",
)
return
def makeMSFrame(msname='sim_data.ms'):
"""
Construct an empty Measurement Set that has the desired observation setup.
"""
os.system('rm -rf ' + msname)
## Open the simulator
sm.open(ms=msname)
## Read/create an antenna configuration.
## Canned antenna config text files are located here : /home/casa/data/trunk/alma/simmos/*cfg
antennalist = os.path.join(ctsys.resolve("alma/simmos"), "vla.d.cfg")
## Fictitious telescopes can be simulated by specifying x, y, z, d, an, telname, antpos.
## x,y,z are locations in meters in ITRF (Earth centered) coordinates.
## d, an are lists of antenna diameter and name.
## telname and obspos are the name and coordinates of the observatory.
(x, y, z, d, an, an2, telname, obspos) = mysu.readantenna(antennalist)
## Set the antenna configuration
sm.setconfig(telescopename=telname,
x=x,
y=y,
z=z,
dishdiameter=d,
mount=['alt-az'],
antname=an,
coordsystem='local',
referencelocation=me.observatory(telname))
## Set the polarization mode (this goes to the FEED subtable)
sm.setfeed(mode='perfect R L', pol=[''])
## Set the spectral window and polarization (one data-description-id).
## Call multiple times with different names for multiple SPWs or pol setups.
sm.setspwindow(spwname="LBand",
freq='1.0GHz',
deltafreq='0.1GHz',
freqresolution='0.2GHz',
nchannels=10,
stokes='RR LL')
## Setup source/field information (i.e. where the observation phase center is)
## Call multiple times for different pointings or source locations.
sm.setfield(sourcename="fake",
sourcedirection=me.direction(rf='J2000',
v0='19h59m28.5s',
v1='+40d44m01.5s'))
## Set shadow/elevation limits (if you care). These set flags.
sm.setlimits(shadowlimit=0.01, elevationlimit='1deg')
## Leave autocorrelations out of the MS.
sm.setauto(autocorrwt=0.0)
## Set the integration time, and the convention to use for timerange specification
## Note : It is convenient to pick the hourangle mode as all times specified in sm.observe()
## will be relative to when the source transits.
sm.settimes(integrationtime='2000s',
usehourangle=True,
referencetime=me.epoch('UTC', '2019/10/4/00:00:00'))
## Construct MS metadata and UVW values for one scan and ddid
## Call multiple times for multiple scans.
## Call this with different sourcenames (fields) and spw/pol settings as defined above.
## Timesteps will be defined in intervals of 'integrationtime', between starttime and stoptime.
sm.observe(sourcename="fake",
spwname='LBand',
starttime='-5.0h',
stoptime='+5.0h')
## Close the simulator
sm.close()
## Unflag everything (unless you care about elevation/shadow flags)
flagdata(vis=msname, mode='unflag')
def make_flagsummary_uvdist_data(myvis, nbin=25, output_folder="perfield_flagfraction",
intent='*', overwrite=False):
'''
Make a binned flagging fraction vs. uv-distance.
'''
from casatools import ms
from casatasks import flagdata
myms = ms()
myms.open(myvis)
mymsmd = myms.metadata()
# Get VLA antenna ID
antenna_names = mymsmd.antennanames() #returns a list that corresponds to antenna ID
# Get fields matching intent
fieldsnums = mymsmd.fieldsforintent(intent)
if len(fieldsnums) == 0:
raise ValueError("No calibrator intents are in this MS.")
fields = np.array(mymsmd.fieldnames())[fieldsnums]
# Get SPWs
spw_list = mymsmd.spwsforfield(fieldsnums[0])
casalog.post(f"Selecting on fields: {fields}")
print(f"Selecting on fields: {fields}")
for field in fields:
casalog.post(f"Creating uvdist flagging fraction for {field}")
print(f"Creating uvdist flagging fraction for {field}")
baseline_flagging_table = []
save_name = f"{output_folder}/field_{field}_flagfrac_uvdist.txt"
if os.path.exists(save_name) and overwrite:
os.system(f"rm {save_name}")
if not os.path.exists(save_name):
for spw in spw_list:
flag_dict = flagdata(vis=myvis, mode='summary', basecnt=True, action='calculate',
field=field, spw=str(spw))
# Make plot of flagging statistics
# Get information for flagging percentage vs. uvdistance
myms.selectinit()
myms.selectchannel(1, 0, 1, 1) # look at data just for first channel - easily translates
gantdata = myms.getdata(['antenna1','antenna2','uvdist']) # get the points I need
# create adictionary with flagging info
base_dict = create_baseline_dict(antenna_names, gantdata)
# match flagging data to dictionary entry
datamatch = flag_match_baseline(flag_dict['baseline'], base_dict)
# 25 is the number of uvdist bins such that there is minimal error in uvdist.
binned_stats, barwidth = bin_statistics(datamatch, nbin)
spw_vals = [spw] * len(binned_stats[0])
field_vals = [field] * len(binned_stats[0])
baseline_flagging_table.append([field_vals, spw_vals, binned_stats[0], binned_stats[1]])
baseline_flagging_table_hstack = np.hstack(baseline_flagging_table).T
out_table = np.zeros(baseline_flagging_table_hstack.shape[0],
dtype=[("field", 'U32'),
('spw', int),
('uvdist', float),
('frac', float)])
out_table['field'] = baseline_flagging_table_hstack[:, 0].astype('U32')
out_table['spw'] = baseline_flagging_table_hstack[:, 1].astype(int)
out_table['uvdist'] = baseline_flagging_table_hstack[:, 2].astype(float)
out_table['frac'] = baseline_flagging_table_hstack[:, 3].astype(float)
np.savetxt(save_name, out_table, fmt='%s %d %f %f', header="field,spw,uvdist,frac")
mymsmd.close()
myms.close()
def avg_cont(
msfile,
outputvis,
flagchannels="",
maxchanwidth=125,
datacolumn="data",
spws=None,
width_array=None,
):
"""
Produce spectrally averaged continuum measurement sets
Args:
msfile (string): input file name
outputvis (string): output file name
flagchannels: Argument to be passed for flagchannels parameter in
flagdata task
maxchanwidth: Maximum width of channel (MHz). This is the value
recommended by ALMA for Band 6 to avoid bandwidth smearing
datacolumn: Column to pull from for continuum averaging (usually will
be 'data', but may sometimes be 'corrected' if there was flux
rescaling applied)
width_array (array): Argument to be passed to CASA for the width
parameter in split. If not set, all SPWs will be selected by default.
Returns:
None
"""
tb = table()
tb.open(msfile + "/SPECTRAL_WINDOW")
total_bw = tb.getcol("TOTAL_BANDWIDTH")
num_chan = tb.getcol("NUM_CHAN")
tb.close()
timebin = "0s" # default in CASA
if os.path.isdir(msfile + ".flagversions/flags.before_cont_flags"):
# clear out old versions of the flags
flagmanager(vis=msfile, mode="delete", versionname="before_cont_flags")
# save flag state before flagging spectral lines
flagmanager(
vis=msfile,
mode="save",
versionname="before_cont_flags",
comment="Flag states before spectral lines are flagged",
)
# flag spectral lines
flagdata(
vis=msfile,
mode="manual",
spw=flagchannels,
flagbackup=False,
)
os.system("rm -rf " + outputvis)
split(
vis=msfile,
outputvis=outputvis,
spw=spws,
width=width_array,
timebin=timebin,
datacolumn=datacolumn,
intent="OBSERVE_TARGET#ON_SOURCE",
keepflags=False,
)
# restore flagged spectral line channels
flagmanager(vis=msfile, mode="restore", versionname="before_cont_flags")
print("#Averaged continuum dataset saved to %s" % outputvis)
def make_flagsummary_freq_data(myvis, output_folder='perfield_flagfraction',
intent="*", overwrite=False):
'''
This mimics the summary plots made by flagdata, but removes the interactive
part so we can save it.
'''
from casatools import ms
from casatasks import flagdata
myms = ms()
myms.open(myvis)
mymsmd = myms.metadata()
fieldsnums = mymsmd.fieldsforintent(intent)
if len(fieldsnums) == 0:
raise ValueError("No calibrator intents are in this MS.")
fields = np.array(mymsmd.fieldnames())[fieldsnums]
spw_nums = mymsmd.spwsforscan(1)
casalog.post(f"Selecting on fields: {fields}")
print(f"Selecting on fields: {fields}")
for field in fields:
casalog.post(f"Creating freq. flagging fraction for {field}")
print(f"Creating freq. flagging fraction for {field}")
save_name = f"{output_folder}/field_{field}_flagfrac_freq.txt"
if os.path.exists(save_name) and overwrite:
os.system(f"rm {save_name}")
if not os.path.exists(save_name):
flag_dict = flagdata(vis=myvis, mode='summary', spwchan=True, action='calculate',
field=field)
flag_data = []
for spw in spw_nums:
spw_freqs = mymsmd.chanfreqs(spw) / 1e9 # GHz
spw_flagfracs = []
for chan in range(len(spw_freqs)):
spw_flagfracs.append(flag_dict['spw:channel'][f"{spw}:{chan}"]['flagged'] / flag_dict['spw:channel'][f'{spw}:{chan}']['total'])
# Make an equal length SPW column
spw_labels = [spw] * len(spw_freqs)
flag_data.append([spw_labels, np.arange(len(spw_freqs)), spw_freqs, spw_flagfracs])
output_data = np.hstack(flag_data).T
np.savetxt(save_name, output_data, header="spw,channel,freq,frac")
else:
casalog.post(message="File {} already exists. Skipping".format(save_name),
origin='make_qa_tables')
mymsmd.close()
myms.close()
def flag_hi_foreground(myvis,
calibrator_line_range_kms,
hi_spw_num,
cal_intents=["CALIBRATE*"],
test_print=False,
test_run=False):
'''
Define velocity regions to flag for all (or chosen) calibration
fields based on intent.
Parameters
----------
myvis : str
MS name.
calibrator_line_range_kms : dict
Dictionary with velocity range (in LSRK; radio) to flag.
hi_spw_num : int
The SPW of HI in the MS. If None is given, the context is used to
identify the SPW overlapping the HI line (where we can ignore wideband
continuum SPWs).
cal_intents : list, optional
List of the calibrator field intents to apply flagging to.
test_print : bool, optional
Print out additional information for testing purposes.
'''
# Check context for the calibration sources given the list of intents
# to flag.
# Loop through calibrator sources, calling target_foreground_hi_ranges
# Flag the requested range.
# Make a new flagging version marking these calls at the end.
# from taskinit import msmdtool, mstool
from casatools import ms
from casatasks import flagdata, flagmanager
# msmd = msmdtool()
# ms = mstool()
# mymsmd = msmd()
myms = ms()
# if no fields are provided use observe_target intent
# I saw once a calibrator also has this intent so check carefully
# mymsmd.open(myvis)
myms.open(myvis)
mymsmd = myms.metadata()
# Loop through field intents. Default is all calibrators.
field_nums = []
for cal_intent in cal_intents:
field_num = mymsmd.fieldsforintent(cal_intent)
field_nums.extend(list(field_num))
# Unique mapping
field_nums = np.array(list(set(field_nums)))
field_names = np.asarray(mymsmd.fieldnames())[field_nums]
mymsmd.close()
# Loop through the field names, identify in calibrator_line_range_kms,
# and convert mapping from velocity -> freq (LSRK) -> channel.
# myms.open(myvis)
freqs_lsrk = myms.cvelfreqs(spwids=[hi_spw_num], outframe='LSRK')
myms.close()
# in Hz
hi_restfreq = 1.420405752e9
vels_lsrk = lines_freq2vels(freqs_lsrk, hi_restfreq)
for field in field_names:
if field not in calibrator_line_range_kms:
casalog.post('Unable to locate calibrator {} in calibrator list.'.format(field))
casalog.post('Check `calibrator_setup.py` to see if this source is missing')
continue
vel_start = calibrator_line_range_kms[field]['HI'][0]
vel_stop = calibrator_line_range_kms[field]['HI'][1]
# Keep red to blue shifted order.
if vel_start < vel_stop:
vel_stop, vel_start = vel_start, vel_stop
chan_start = np.abs(vels_lsrk - vel_start).argmin()
chan_stop = np.abs(vels_lsrk - vel_stop).argmin()
# Do the flagging and save a new version
if test_print:
print('Field {0} flagging region {1}:{2}~{3}'.format(field, hi_spw_num,
chan_start, chan_stop))
print('Velocity: {0}, {1}'.format(vel_start, vel_stop))
if test_run:
continue
flagdata(myvis, mode='manual', field=field,
spw='{0}:{1}~{2}'.format(hi_spw_num, chan_start, chan_stop),
flagbackup=False)
if not test_run:
flagmanager(myvis, mode='save', versionname='MW_HI_abs_flagging',
comment='Flag Milky Way HI absorption for calibrators.')
def pipeline(msfile, params, doinitial_flagging=True, doflagcal=True, doimagecal=True, douvsub=True, docubeimage=False):
'''
This function takes combines several recipes to construct a pipeline. In particular here it follows a simple procedure.
initial flagging --> setjy+delay calibration --> amplitude calibration + flagging loops on calibrators --> bandpass calibration + flagging loops --> imaging+selfcal loops --> final image --> UVSUB+CVEL --> Cube image with source finder.
'''
with open('yarp_art.txt', 'r') as f:
for line in f:
print(line.rstrip())
print('Running the yarp pipeline on {}'.format(msfile))
if doinitial_flagging:
flag_spw = params['flagging']['spw']
bad_ants = params['flagging']['badants']
flagcmd = params['flagging']['flagcmd']
if len(bad_ants) != 0:
flg.badantflag(msfile, params)
else:
print('No bad antennas found.')
if flagcmd != False:
print('Flagging based on user defined commands ...')
cts.flagdata(msfile, mode='list', inpfile=flagcmd)
else:
None
cts.flagdata(msfile, mode='quack', field='', spw='0', antenna='', correlation='', timerange='',
quackinterval=10, quackmode='beg', action='apply', savepars=True, cmdreason='quackbeg')
cts.flagdata(msfile, mode='quack', field='', spw='0', antenna='', correlation='', timerange='',
quackinterval=10, quackmode='endb', action='apply', savepars=True, cmdreason='quackendb')
if flag_spw != False:
cts.flagdata(msfile, mode='manual', field='', spw=flag_spw, antenna='', correlation='',
timerange='', action='apply', savepars=True, cmdreason='badchannels')
else:
print('Not flagging edge channels')
flg.tfcropper(msfile, params, field='', tcut=6,
fcut=6, instance='initial')
# Flag very long baselines as that can affect the calibration
cts.flagdata(msfile, mode='manual', uvrange='>100klambda')
flg.extend(msfile, params, instance='initial')
#flg.aoflagger(msfile, params)
else:
print('No initial flagging this time.')
# Flagcal begins
if doflagcal:
print('Running the flagcal script...')
nloops = params['calibration']['nloops']
flagcal2(msfile, params, niters=nloops,
flagger='default', interactive=False)
else:
print('No flagcal this time.')
if doimagecal:
print('Doing image and self-calibration...')
target = params['general']['target']
targetcalfile = params['general']['targetcalfile']
avspcfile = targetcalfile[:-3]+'_avspc.ms'
print('Flagging all data above uvran 100klambda ...')
cts.flagdata(targetcalfile, mode='manual', uvrange='>100klambda')
print('Averaging the channels to 1 MHz...')
chanbin = params['imagecal']['chanbin']
cts.mstransform(targetcalfile, field=target, spw='0', chanaverage=True,
chanbin=chanbin, datacolumn='data', outputvis=avspcfile)
print('Rflagging the channel averaged file...')
flg.rflagger(avspcfile, params, field=target,
tcut=10, fcut=10, instance='initial')
print('Flagging the line spws...')
flag_spw = params['imagecal']['spec_line_spw']
cts.flagdata(avspcfile, mode='manual', spw=flag_spw)
print('Doing imaging and self-calibration...')
nloops = params['imagecal']['nloops']
ploops = params['imagecal']['ploops']
aploops = params['imagecal']['aploops']
print('Running {} cycles of self-cal with {} ploops and {} aploops in each cycle...'.format(
str(nloops), str(ploops), str(aploops)))
final_image, selfcaltable = imagecal_dev(
avspcfile, params, nloops=nloops, ploops=ploops, aploops=aploops, flagger='default', interactive=False)
print('Final self-cal table is', selfcaltable)
else:
print('No imaging and self-calibration this time.')
if douvsub:
print('Doing uvsub...')
target = params['general']['target']
targetcalfile = params['general']['targetcalfile']
avspcfile = targetcalfile[:-3]+'_avspc.ms'
nloops = params['imagecal']['nloops']
ploops = params['imagecal']['ploops']
aploops = params['imagecal']['aploops']
outdir = params['general']['outdir']
# apply self-cal table to the full chan resolution file
# selfcaltable = [outdir+'sc_p.gcal.' +
# str(nloops)+str(ploops), outdir+'sc_ap.gcal.'+str(nloops)+str(aploops)]
'''
selfcaltable = [outdir+'sc_p.gcal.' +
str(1)+str(5)]
print(selfcaltable)
cts.applycal(targetcalfile, gaintable=selfcaltable, field='', gainfield='',
applymode='calonly', interp=['linearperobs'], calwt=False, parang=False)
print('Rflagging the self-calibrated data')
'''
linefreespw = params['uvsub']['linefreespw']
linespw = params['uvsub']['linespw']
'''
flg.rflagger(targetcalfile, params, spw=linefreespw, field=target, tcut=6,
fcut=6, instance='postcal') # deep flag the line free channels
flg.rflagger(targetcalfile, params, spw=linespw, field=target, tcut=10,
fcut=10, instance='postcal') # Be more conservative on the line channels
flg.extend(targetcalfile, params, field=target,
grow=80, instance='postcal')
'''
# UVLIN the full chan resolution file
print('Doing uvcontsub...')
fitorder = params['uvsub']['fitorder']
# print(linefreespw)
cts.uvcontsub(targetcalfile, fitspw=linefreespw, fitorder=fitorder, solint='int', combine='scan')
#tempdir = params['general']['temp']
#uvsubber(targetcalfile, params, fitspw=linefreespw, fitorder=fitorder, nterms=1,
# model_image=[tempdir+'AGC203001_cont_sc_p.15.model'])
targetcalfile = params['general']['targetcalfile']
cont_sub_file = targetcalfile+'.uvsub.contsub'
# subprocess.run('mv {} {}'.format(cont_sub_file, outdir),
# shell=True, check=True)
else:
print('No uvsub this time.')
if docubeimage:
print('Doing cube image...')
# image the cube
outdir = params['general']['outdir']
target = params['general']['target']
targetcalfile = params['general']['targetcalfile']
restfreq = '1.420405752GHz'
#cont_sub_file = targetcalfile+'.contsub'
#cont_sub_file = cont_sub_file
cont_sub_file = targetcalfile+'.contsub'
weighting = params['cube']['weighting']
robust = params['cube']['robust']
deconvolver = params['cube']['deconvolver']
# uvran_list = ['0.5~5klambda', '0.5~10klambda', '0.5~20klambda','0.5~40klambda'] # list of uvranges to image the cube
#uvtaper_list = ['4.5klambda', '6klambda', '12klambda', '30klambda']
#imsize_list = [256, 512, 540, 1024]
#cellsize_list = ['8arcsec', '4arcsec', '3arcsec', '1arcsec']
#threshold_list = ['0.44mJy', '1.0mJy', '1.0mJy', '1.0mJy']
uvran_list = ['0.5~6klambda']
uvtaper_list = ['5.5klambda']
imsize_list = [128]
cellsize_list = ['8arcsec']
threshold_list = ['1mJy']
vel_res1 = params['cube']['vel_res']
vel_res = str(vel_res1)+'km/s'
#vel_res = '-14km/s'
file_name = str(vel_res1)+'kmps'
for i in range(len(uvran_list)):
cts.tclean(cont_sub_file,
imagename=outdir+target+'_cube_' +
str(file_name)+'/'+'uvran_'+str(uvran_list[i]),
field='0',
spw='0',
specmode='cube',
nchan=-1,
width=vel_res,
outframe='bary',
veltype='optical',
restfreq=restfreq,
deconvolver='hogbom',
gridder='standard',
uvrange=uvran_list[i],
uvtaper=uvtaper_list[i],
imsize=imsize_list[i],
cell=cellsize_list[i],
threshold=threshold_list[i],
weighting=weighting,
robust=robust,
restoringbeam='common',
interactive=False,
usemask='pb',
pbmask=0.2,
# usemask='auto-multithresh',
# minbeamfrac=0.1,
#sidelobethreshold = 1.5,
# smallscalebias=0.6,
niter=100000
)
else:
print('No cube image this time.')
outdir = general_params['outdir']
subprocess.run('cp {} {}'.format(
'parameters.yaml', outdir), shell=True, check=True)
return print('yarp pipeline ended.')
def badantflag(msfile, params):
general_params = params['general'] # loading general params
badants = params['flagging']['badants']
ants = ','.join(badants)
cts.flagdata(msfile, mode='manual', antenna=ants, action='apply')
return print('Flagged non-working antennas:', ants)
def makeMSFrame(msn = 'sim_data',tel='ALMA'):
"""
Construct an empty Measurement Set that has the desired observation setup
with uvw/scan/field/ddid setup
Construct an empty Measurement Set that has the desired observation setup.
This includes antenna configuration, phase center direction, spectral windows,
date and timerange of the observation, structure of scans/spws/obsidd/fieldids
(and all other MS metadata). Evaluate UVW coordinates for the entire observation
and initialize the DATA column to zero.
"""
msname = msn + '_' + tel + '.ms'
print("Making an MS named : "+msname)
os.system('rm -rf '+msname)
## Open the simulator
sm.open(ms=msname);
## Read/create an antenna configuration.
## Canned antenna config text files are located here : /home/casa/data/trunk/alma/simmos/*cfg
## Fictitious telescopes can be simulated by specifying x, y, z, d, an, telname, antpos.
## x,y,z are locations in meters in ITRF (Earth centered) coordinates.
## d, an are lists of antenna diameter and name.
## telname and obspos are the name and coordinates of the observatory.
if tel=='ALMA':
antennalist = os.path.join( ctsys.resolve("alma/simmos") ,"alma.all.cfg")
(x,y,z,d,an,an2, telname, obspos) = mysu.readantenna(antennalist)
## Pick a subset : Example : four 7m dishes and eight 12m dishes is to
antlist = ['N601','N606','J505','J510', 'A001', 'A012','A025', 'A033','A045', 'A051','A065', 'A078']
obsposname='ALMA'
dir_pointing = me.direction(rf='J2000', v0='19h59m28.5s',v1='-40d44m01.5s')
if tel=='VLA':
antennalist = os.path.join( ctsys.resolve("alma/simmos") ,"vla.d.cfg")
(x,y,z,d,an,an2, telname, obspos) = mysu.readantenna(antennalist)
## Pick a subset : Example : four 7m dishes and eight 12m dishes is to
antlist = ['W01','W02','W03','W05','W07','W09','E01','E02','E03','E05','E07','E09','N01','N02','N03','N05','N07','N09']
#antlist = ['W02','W04','W06','W08','W010','W12','E02','E04','E06','E16','E18','N02','N03','N12','N14','N16','N18']
obsposname='VLA'
dir_pointing = me.direction(rf='J2000', v0='19h59m28.5s',v1='+40d40m00.0s')
if (tel=='NGVLA'):
antennalist = "ngvla-demo-revC.cfg" ## Local copy
##### Or concat two of them programmatically and then read.....
#alist1 = os.path.join( ctsys.resolve("alma/simmos") ,"ngvla-core-revC.cfg")
#alist2 = os.path.join( ctsys.resolve("alma/simmos") ,"ngvla-sba-revC.cfg")
#os.system('cat '+alist1 + ' > ngvla-demo-revC.cfg')
#os.system('cat '+alist2 + ' >> ngvla-demo-revC.cfg')
(x,y,z,d,an,an2,telname, obspos) = mysu.readantenna(antennalist)
## Pick a subset : Example : seven 18m dishes and five 6m dishes is to
antlist = ['m153','m155','m140','m142', 'm130', 'm122','m151', 's012','s006', 's013','s008', 's009']
obsposname='VLA'
dir_pointing = me.direction(rf='J2000', v0='19h59m28.5s',v1='+40d44m01.5s')
telname='NGVLA1'
antmask = np.isin(np.array(an),antlist)
xsel=np.array(x)[antmask]
ysel=np.array(y)[antmask]
zsel=np.array(z)[antmask]
dsel=np.array(d)[antmask]
antsel=np.array(an)[antmask]
newnamelist = np.array(an)[antmask]
# To edit antenna names, edit this list.
# for ii in range(0,len(newnamelist)):
# newnamelist[ii] = newnamelist[ii]+'new'
## Set the antenna configuration
sm.setconfig(telescopename=telname,
x=xsel,
y=ysel,
z=zsel,
dishdiameter=dsel,
mount=['alt-az'],
antname=list(antsel),
#antname=list(newnamelist),
coordsystem='global',
referencelocation=me.observatory(obsposname));
## Set the polarization mode (this goes to the FEED subtable)
sm.setfeed(mode='perfect X Y', pol=['']);
## Set the spectral window and polarization (one data-description-id).
## Call multiple times with different names for multiple SPWs or pol setups.
if tel=='ALMA':
sm.setspwindow(spwname="Band3",
freq='90GHz',
deltafreq='2GHz',
freqresolution='1.0MHz',
nchannels=5,
stokes='XX YY');
useband='Band3'
if tel=='VLA':
sm.setspwindow(spwname="LBand",
freq='1.0GHz',
deltafreq='0.2GHz',
freqresolution='1.0MHz',
nchannels=5,
stokes='RR LL');
useband='LBand'
if (tel=='NGVLA'):
sm.setspwindow(spwname="Band5",
freq='40GHz',
deltafreq='2GHz',
freqresolution='1.0MHz',
nchannels=5,
stokes='XX YY');
useband='Band5'
## Setup source/field information (i.e. where the observation phase center is)
## Call multiple times for different pointings or source locations.
sm.setfield( sourcename="fake1", sourcedirection=dir_pointing);
## Set shadow/elevation limits (if you care). These set flags.
sm.setlimits(shadowlimit=0.01, elevationlimit='1deg');
## Leave autocorrelations out of the MS.
sm.setauto(autocorrwt=0.0);
## Set the integration time, and the convention to use for timerange specification
## Note : It is convenient to pick the hourangle mode as all times specified in sm.observe()
## will be relative to when the source transits.
sm.settimes(integrationtime='2000s',
usehourangle=True,
referencetime=me.epoch('UTC','2020/10/4/00:00:00'));
## Construct MS metadata and UVW values for one scan and ddid
## Call multiple times for multiple scans.
## Call this with different sourcenames (fields) and spw/pol settings as defined above.
## Timesteps will be defined in intervals of 'integrationtime', between starttime and stoptime.
sm.observe(sourcename="fake1",
spwname=useband,
starttime='-5.0h',
stoptime='+5.0h');
## Close the simulator
sm.close()
## Unflag everything (unless you care about elevation/shadow flags)
flagdata(vis=msname,mode='unflag')
