def unfcheck(self, dataset):
self.assertTrue(listobs(vis=dataset, listunfl=True))
listobs(vis=dataset, listfile='listobs1.txt', listunfl=True)
self.assertTrue('nUnflRows' in open('listobs1.txt').read())
listobs(vis=dataset, listfile='listobs2.txt', listunfl=False)
self.assertFalse('nUnflRows' in open('listobs2.txt').read())
def verbosecheck(self, dataset):
listobs(vis=dataset, listfile='listobsNonVerbose.txt', verbose=False)
listobs(vis=dataset, listfile='listobsVerbose', verbose=True)
nonVerbose = os.path.getsize('listobsNonVerbose.txt')
Verbose = os.path.getsize('listobsVerbose')
self.assertGreater(Verbose,
nonVerbose,
msg='Verbose is smaller than non-Verbose')
def test_input(self):
'''Listobs test: Check all if listobs can take ms, mms, time averaged ms, and time averaged mms'''
# See if listobs can take all these forms of MS
self.assertTrue(listobs(vis=mesSet), msg='Fails to take MS file')
self.assertTrue(listobs(vis=multiMesSet), msg='Fails to take MMS file')
self.assertTrue(listobs(vis=timeavg_ms),
msg='Fails to take time-averaged MS file')
self.assertTrue(listobs(vis=timeavg_mms),
msg='Fails to take time-averaged MMS file')
def overwritecheck(self, dataset):
listfile = "listobs.txt"
self.assertTrue(listobs(vis=dataset, listfile=listfile))
# test default value is overwrite=False
self.assertFalse(listobs(vis=dataset, listfile=listfile))
self.assertFalse(
listobs(vis=dataset, listfile=listfile, overwrite=False))
expec = _sha1it(listfile)
self.assertTrue(listobs(vis=dataset, listfile=listfile,
overwrite=True))
got = _sha1it(listfile)
self.assertTrue(got == expec)
def import_rawdata(rawdir='../raw', outdir='./', overwrite=False, **kwargs):
"""This function is used to importasdm and generate the file descriptions
Paramters
---------
rawdir : str
the directory of raw files, ended with .asdm
outdir : str
the output directory
overwrite : bool
overwriting exisiting folder with the same names
kwargs:
the kwargs passed to importasdm
Examples
--------
In the project directory
os.system('mkdir ./data')
import_rawdata(rawdir='../raw', outdir='./data', overwrite=False)
"""
# import the asdm data into outdir
for asdm in glob.glob(os.path.join(rawdir, '*.asdm.sdm')):
basename = os.path.basename(asdm)[:-9]
msfile = basename + '.ms'
msfile_fullpath = os.path.join(outdir, msfile)
if os.path.isdir(msfile_fullpath):
if not overwrite:
print("Reused existing data, set overwrite=True to overwrite.")
continue
else:
print("Overwriting existing files...")
importasdm(asdm=asdm,
vis=os.path.join(outdir, basename + '.ms'),
overwrite=overwrite,
**kwargs)
# generate the data disscription
for obs in glob.glob(os.path.join(outdir, '*.ms')):
obs_listobs = obs + '.listobst.txt'
if os.path.isfile(obs_listobs):
print("Reused existing listobs")
continue
else:
listobs(vis=obs,
listfile=obs_listobs,
verbose=True,
overwrite=overwrite)
def get_listobs(self):
"""Retrieves and stores the information about the scans in the project (runs listobs in CASA).
If the file already exists, it will not be overwriten.
"""
if not self.msfile.exists():
print(f"The MS file {self.msfile.name} does not exist!")
return False
if (self.logdir / f"{self.project_name.lower()}.listobs").exists():
print(
f"{self.project_name.lower()}.listobs already exists and will not be overwriten."
)
else:
casatasks.listobs(
vis=str(self.msfile),
listfile=str(self.logdir /
f"{self.project_name.lower()}.listobs"))
def listobs_jupyter(vis='sim_data_ALMA.ms'):
"""
Print out the contents of listobs.
TODO : Convert the contents to a df (if possible) for a pretty display
"""
listobs(vis=vis, listfile='obslist.txt', verbose=False, overwrite=True)
## print(os.popen('obslist.txt').read()) # ?permission denied?
fp = open('obslist.txt')
for aline in fp.readlines():
print(aline.replace('\n', ''))
fp.close()
tb.open(vis + '/ANTENNA')
print("Dish diameter : " + str(tb.getcol('DISH_DIAMETER')))
print("Antenna name : " + str(tb.getcol('NAME')))
tb.close()
def test_CAS_6733(self):
"""Verify listobs runs to completion on data set in CAS-6733. This was an infinite loop bugfix"""
if CASA6:
vis = casatools.ctsys.resolve(
'regression/unittest/mstransform/CAS-6733.ms')
else:
vis = os.environ.get('CASAPATH').split(
)[0] + '/data/regression/unittest/mstransform/' + "CAS-6733.ms"
self.assertTrue(listobs(vis=vis))
def logreadfunc(self, dataset):
casalog.setlogfile('testlog.log')
listobs(vis=dataset)
casalog.setlogfile(logpath)
if sys.version_info[0] == 3:
print('VERSION', ' ', sys.version_info)
# Check that the file can be read in python session default encoding
with open('testlog.log', 'r') as fout:
list(map(bytes2str, fout.readlines()))
print(list(map(bytes2str, fout.readlines())))
else:
# Check if the file can be decoded as ascii for python 2.7
with open('testlog.log', 'r') as log:
for data in log:
try:
data.decode('ASCII')
except:
self.fail()
def split_ms(src_dir, img_dir, visname, msname, ATCA_band, pri, sec, tar, n_spw):
os.system(f"rm -r {msname}")
os.system(f"rm -r {msname}.flagversions")
os.system("rm -r *.last")
# have removed n_spw for mstransform and included it in the split just before imaging
mstransform(
vis=visname,
outputvis=msname,
datacolumn="data",
field=f"{pri},{sec},{tar}",
# nspw=n_spw,
regridms=True,
# field=f"{sec},{tar}",
# scan="3,>90"
)
listobs(
vis=msname,
listfile=f"{src_dir}/listobs_{ATCA_band}_{tar}.dat",
overwrite=True,
)
flagmanager(vis=msname, mode="save", versionname="after_transform")
return
def selectcheck(self, dataset):
# selectdata should fail for all invalid inputs
self.assertTrue(listobs(vis=dataset, selectdata=False),
msg='Passing False to select data fails on a MS')
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, selectdata=1)
with self.assertRaises(AssertionError):
listobs(vis=dataset, selectdata='str')
else:
self.assertFalse(
listobs(vis=dataset, selectdata=1),
msg='An int is accepted for the select data parameter')
self.assertFalse(
listobs(vis=dataset, selectdata='str'),
msg='A string is accepted for the select data parameter')
self.assertFalse(
listobs(vis=dataset, selectdata=[False]),
msg='Array data type is accepted when it contains values')
# I'm really not sure why these happen. especially the empty list
if not CASA6:
self.assertTrue(listobs(vis=dataset, selectdata=[]))
self.assertTrue(listobs(vis=dataset, selectdata=None))
def split_imgms(data_dir, tar, epoch, ATCA_band, n_spw):
if epoch in ["01", "03", "04", "05"]:
timerange = f"2020/{epoch}/01/00:00:00~2020/{epoch}/30/23:59:59"
else:
timerange = ""
visname = f"{data_dir}data/2020_{tar}_{ATCA_band}.ms"
outputvis = f"{data_dir}data/2020-{epoch}_{tar}_{ATCA_band}.ms"
if os.path.exists(outputvis):
os.system(f"rm -r {outputvis}*")
mstransform(
vis=visname,
outputvis=outputvis,
datacolumn="corrected",
# regridms=True,
field=tar,
timerange=timerange,
)
listobs(
vis=outputvis,
listfile=f"{data_dir}{tar}/listobs_2020-{epoch}_{tar}_{ATCA_band}_preimage.dat",
overwrite=True,
)
return
def arraycheck(self, dataset):
self.assertTrue(listobs(vis=dataset, array='0'),
msg='Listobs fails to recognize valid array in a MS')
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, array=0)
else:
self.assertFalse(
listobs(vis=dataset, array=0),
msg='Listobs fails to recognize invalid data type in a MS')
self.res = listobs(vis=dataset, listfile='listobs.txt')
# These should raise warnings
casalog.setlogfile('testlog.log')
listobs(vis=dataset, array='abc')
self.assertTrue(
'Parse error' in open('testlog.log').read(),
msg='Listobs fails to recognize invalid string in a MS')
listobs(vis=dataset, array='10')
self.assertTrue('The selected table has zero rows'
in open('testlog.log').read(),
msg='Listobs fails to recognize empty table from a MS')
self.assertTrue('ArrayID' in open('listobs.txt').read(),
msg='There is no Array information for a MS')
def obscheck(self, dataset):
self.assertTrue(listobs(vis=dataset, observation=0),
msg='Observation fails to accept Int for a MS')
self.assertTrue(
listobs(vis=dataset, observation='0'),
msg='Observation fails to accept proper string for a MS')
# These should raise warnings
casalog.setlogfile('testlog.log')
listobs(vis=dataset, observation='abc')
self.assertTrue('Parse error' in open('testlog.log').read(),
msg='Listobs fails to identify improper string')
listobs(vis=dataset, observation='10')
self.assertTrue(('The selected table has zero rows')
in open('testlog.log').read(),
msg='Listobs fails to identify an empty table')
self.res = listobs(vis=dataset, listfile='listobs.txt')
self.assertTrue('ObservationID' in open('listobs.txt').read(),
msg='There is no Observation information')
def scancheck(self, dataset):
# Check that listobs runs and that the scan column exists
self.res = listobs(vis=dataset, listfile='listobs.txt')
self.assertTrue('Scan' in open('listobs.txt').read(),
msg='scan does not exist in output')
# Check to see if a valid input and invalid input is returns the proper value (pass/fail)
self.assertTrue(listobs(vis=dataset, scan='1'),
msg='Scan fails to select')
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, scan=1)
else:
self.assertFalse(listobs(vis=dataset, scan=1),
msg='Scan incorrectly accepts an int')
# Make temp log
casalog.setlogfile('testlog.log')
# test for certain warnings appearing in the log
listobs(vis=dataset, scan='1,2')
self.assertFalse('WARN' in open('testlog.log').read(),
msg='A warning is raised for multiple scans')
listobs(vis=dataset, scan=['1', '2'])
self.assertTrue('incorrect data type' in open('testlog.log').read(),
msg='fails to label incorrect data type')
listobs(vis=dataset, scan='abc')
self.assertTrue('Parse error' in open('testlog.log').read(),
msg='fails to recognize improper string')
def fieldcheck(self, dataset):
self.res = listobs(vis=dataset, listfile='listobs.txt')
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, field=1)
else:
self.assertFalse(
listobs(vis=dataset, field=1),
msg='An int was given to the field (requires a string)')
self.assertTrue('FldId' in open('listobs.txt').read(),
msg='Field Id does not exist in a MS')
# section that should raise no warnings
casalog.setlogfile('testlog.log')
listobs(vis=dataset, field='1')
listobs(vis=dataset, field='0~2')
listobs(vis=dataset, field='0,2')
self.assertFalse('WARN' in open('testlog.log').read(),
msg='not accepting proper input for field in a MS')
# section that should be raising warnings
listobs(vis=dataset, field='0-2')
self.assertTrue('No match found for name "0-2"'
in open('testlog.log').read(),
msg='Failed to identify improper delimiter')
listobs(vis=dataset, field='abc')
self.assertTrue('No match found for name "abc"'
in open('testlog.log').read(),
msg='Failed to identify improper string')
def spwcheck(self, dataset):
self.res = listobs(vis=dataset, listfile='listobs.txt')
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, spw=1)
else:
self.assertFalse(listobs(vis=dataset, spw=1),
msg='An int was given to spw (requires string)')
self.assertTrue('SpwID' in open('listobs.txt').read(),
msg='Spw does not exist in a MS')
# section that should raise no warnings
casalog.setlogfile('testlog.log')
listobs(vis=dataset, spw='0')
listobs(vis=dataset, spw='0,1')
listobs(vis=dataset, spw='0~1')
self.assertFalse('WARN' in open('testlog.log').read(),
msg='not accepting proper input for spw in a MS')
# section that should raise warnings
listobs(vis=dataset, spw='3')
self.assertTrue('No match found for 3' in open('testlog.log').read(),
msg='fails to recognize out of range values')
listobs(vis=dataset, spw='0')
self.assertTrue('-1' in open('testlog.log').read(),
msg='Fails to recognize improper delimiter')
listobs(vis=dataset, spw='abc')
self.assertTrue('No match found for "abc"'
in open('testlog.log').read(),
msg='Fails to recognize improper string')
# __Examples__
#
# Make a Measurement Set and inspect it
# In[458]:
makeMSFrame()
# In[459]:
plotData(myplot='uv')
# In[460]:
listobs(vis='sim_data.ms',
listfile='obslist.txt',
verbose=False,
overwrite=True)
## print(os.popen('obslist.txt').read()) # ?permission denied?
fp = open('obslist.txt')
for aline in fp.readlines():
print(aline.replace('\n', ''))
fp.close()
# # Make a True Sky Model (component list and/or image)
#
# Construct a true sky model for which visibilities will be simulated and stored in the DATA column. This could be a component list (with real-world positions and point or gaussian component types), or a CASA image with a real-world coordinate system and pixels containing model sky values. It is possible to also evaluate component lists onto CASA images.
# __Methods__
#
# __Make a source list__
#
def corrcheck(self, dataset):
self.res = listobs(vis=dataset, listfile='listobs.txt')
self.assertTrue('Corrs' in open('listobs.txt').read(),
msg='Corrs does not exist in a MS')
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, correlation=1)
else:
self.assertFalse(
listobs(vis=dataset, correlation=1),
msg='An int was accepted when a string is required')
# section that should not raise warnings
casalog.setlogfile('testlog.log')
listobs(vis=dataset, correlation='XX')
listobs(vis=dataset, correlation='XX,YY')
self.assertFalse(
'WARN' in open('testlog.log').read(),
msg='not accepting proper input for correlation in a MS')
# section that should raise warnings
listobs(vis=dataset, correlation=['XX', 'YY'])
self.assertTrue('incorrect data type' in open('testlog.log').read(),
msg='No warning for using a list was given')
listobs(vis=dataset, correlation='RR')
self.assertTrue('named RR' in open('testlog.log').read(),
msg='No warning for using a absent correlation')
def intentcheck(self, dataset):
# Returns true with a valid input and false with an int
self.assertTrue(listobs(
vis=dataset,
intent=
'CALIBRATE_PHASE.ON_SOURCE,CALIBRATE_POINTING.ON_SOURCE,CALIBRATE_WVR.ON_SOURCE'
),
msg='Fails with valid input')
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, intent=1)
else:
self.assertFalse(listobs(vis=dataset, intent=1),
msg='Accepts int when it should only accept str')
# Test for the existence of the column scan intent
self.res = listobs(vis=dataset, listfile='listobs.txt')
self.assertTrue('ScanIntent' in open('listobs.txt').read(),
msg='There is no ScanIntent information for a MS')
# These shouldn't raise any warnings
casalog.setlogfile('testlog.log')
listobs(vis=dataset, intent='CALIBRATE_PHASE.ON_SOURCE')
listobs(
vis=dataset,
intent=
'CALIBRATE_PHASE.ON_SOURCE,CALIBRATE_POINTING.ON_SOURCE,CALIBRATE_WVR.ON_SOURCE'
)
self.assertFalse(
'WARN' in open('testlog.log').read(),
msg='There are warnings for inputs that should raise none')
# These should raise a warning
listobs(vis=dataset, intent=[])
self.assertTrue('incorrect data type used for field scanintent'
in open('testlog.log').read(),
msg='Incorrect data type list accepted')
listobs(vis=dataset, intent='abc')
self.assertTrue('No match found for "abc"'
in open('testlog.log').read(),
msg='Invalid string accepted without warning')
def measureflux_ms(
src_dir, imagems, fitms, catname, ATCA_band, sourcepar, n_spw, timerange="", field="",
):
try:
split(
vis=imagems, datacolumn="data", outputvis=fitms)
listobs(vis=fitms, filename=f"listobs_{fitms}.dat", overwrite=True)
except:
print("Not splitting")
int_flux_c = []
uvrange = ""
for i in range(n_spw):
spw = str(i)
# If things look like theyre not working, then check the source position! Chances are it can't find the source too far away from the phase centre
outfile = f"{src_dir}/casa_files/{catname}_{spw}.cl"
os.system(f"rm -r {outfile}")
uvmodelfit(
vis=fitms,
niter=15,
comptype="P",
spw=spw,
sourcepar=sourcepar,
outfile=outfile,
uvrange=uvrange,
field=field,
selectdata=True,
timerange=timerange,
)
tbl = table(outfile)
flux = tbl.getcell("Flux", 0)[0].astype("float64")
int_flux_c.append(flux)
print(flux)
if ATCA_band == "C":
np.savetxt(
f"{src_dir}/{catname}.csv",
int_flux_c,
delimiter=",",
header="S_Cband",
)
print(int_flux_c)
elif ATCA_band == "X":
np.savetxt(
f"{src_dir}/{catname}.csv",
int_flux_c,
delimiter=",",
header="S_Xband",
)
print(int_flux_c)
elif ATCA_band == "L":
# int_flux_l = np.array(int_flux_c[::-1])
int_flux_l = int_flux_c
np.savetxt(
f"{src_dir}/{catname}.csv",
int_flux_l,
header="S_Lband",
delimiter=",",
)
print(int_flux_l)
return
def novis(self, dataset):
self.res = listobs(vis=dataset, listfile='listobs.txt')
self.assertFalse('Amp' in open('listobs.txt').read())
myvis = sys.argv[-1]
spw_dict = create_spw_dict(myvis, save_spwdict=False)
# --------------------------------
# Split the calibrated column out into target and calibrator parts.
# --------------------------------
# Keep flags if continuum (to maintain the SPW structure for awproject imaging)
# For lines, don't keep flags.
if "continuum" in myvis:
keep_flags = True
elif "speclines" in myvis:
keep_flags = False
else:
raise ValueError(f"Unsure of the type of data in MS: {myvis}")
split_ms_final_all(myvis,
spw_dict,
data_column='CORRECTED',
target_name_prefix="",
time_bin='0s',
keep_flags=keep_flags,
overwrite=False)
listobs(f"{myvis}.split")
listobs(f"{myvis}.split_calibrators")
def uvrangecheck(self, dataset):
self.assertTrue(listobs(vis=dataset, uvrange='0~100klambda'),
msg='fails to read valid input for uvrange in a MS')
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, uvrange=0)
with self.assertRaises(AssertionError):
listobs(vis=dataset, uvrange=[1, 2])
else:
self.assertFalse(
listobs(vis=dataset, uvrange=0),
msg='Accepts an int when only str should be accepted')
self.assertFalse(
listobs(vis=dataset, uvrange=[1, 2]),
msg='accepts an array of ints when only str should be accepted'
)
# Use temp log
casalog.setlogfile('testlog.log')
# shouldn't raise Warning
listobs(vis=dataset, uvrange='0~100')
listobs(vis=dataset, uvrange='0~100klambda')
listobs(vis=dataset, uvrange='0~50,60~100')
self.assertFalse('WARN' in open('testlog.log').read(),
msg='Warnings are raised for valid inputs')
# should raise warnings
listobs(vis=dataset, uvrange=['0~50', '60~100'])
self.assertTrue('incorrect data type used for field uvdist'
in open('testlog.log').read(),
msg='Fails to raise warning for wrong data type')
listobs(vis=dataset, uvrange='0-100')
self.assertTrue('near char. 2 in string "0-100"'
in open('testlog.log').read(),
msg='Fails to raise warning for wrong delimiter')
listobs(vis=dataset, uvrange='abc')
self.assertTrue('near char. 1 in string "abc"'
in open('testlog.log').read(),
msg='Fails to raise warning for improper string')
def test_wrongInp(self):
if not CASA6:
self.assertFalse(listobs(vis='foo.ms'))
def antcheck(self, dataset):
self.res = listobs(vis=dataset, listfile='listobs.txt')
self.assertTrue('Antennas' in open('listobs.txt').read(),
msg='Antennas section does not exist in MS')
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, antenna=0)
else:
self.assertFalse(
listobs(vis=dataset, antenna=0),
msg='Accepts an int as an argument when a string is required')
# section that should not raise warnings
casalog.setlogfile('testlog.log')
listobs(vis=dataset, antenna='0')
listobs(vis=dataset, antenna='0,DV01')
self.assertFalse('WARN' in open('testlog.log').read(),
msg='Proper inputs raised warnings')
# section that should raise warnings
listobs(vis=dataset, antenna='abc')
self.assertTrue('Antenna Expression: No match found for token(s)'
in open('testlog.log').read(),
msg='No warning raise for invalid string')
listobs(vis=dataset, antenna='3')
self.assertTrue(
'No match found for the antenna specificion [ID(s): [3]]'
in open('testlog.log').read(),
msg='No warning for ID out of range')
# This one is marked as correct by the documentation, but CASA disagrees
listobs(vis=dataset, antenna=['0,DV01'])
self.assertTrue('incorrect data type used for field baseline'
in open('testlog.log').read(),
msg='Failed to recognize list as incorrect data type')
import casatasks vis = "HD143006_continuum.ms" casatasks.listobs(vis)
def timerangecheck(self, dataset):
#check valid entry
self.assertTrue(listobs(vis=dataset, timerange='03:00:00~04:00:00'))
# create temp log and check inputs that raise no warnings
casalog.setlogfile('testlog.log')
listobs(vis=dataset, timerange='3:0:0~4:0:0,4:0:0~5:0:0')
self.assertFalse('WARN' in open('testlog.log').read())
# check that specific warnings are raised
listobs(vis=dataset, timerange='abc')
self.assertTrue(
'Parse error at or near ' in open('testlog.log').read())
listobs(vis=dataset, timerange=[])
self.assertTrue('incorrect data type used for field time' in open(
'testlog.log').read())
listobs(vis=dataset, timerange='03:00:00-04:00:00')
self.assertTrue('near char. 9 in string "03:00:00-04:00:00"' in open(
'testlog.log').read())
listobs(vis=dataset, timerange='3~4')
self.assertTrue(
'MSSelectionNullSelection' in open('testlog.log').read())
# Check that passing an int fails
if CASA6:
with self.assertRaises(AssertionError):
listobs(vis=dataset, timerange=4)
else:
self.assertFalse(listobs(vis=dataset, timerange=4))
params = prms.read('parameters.yaml')
general_params = params['general'] # loading general params
msfile = general_params['msfile']
outdir = general_params['outdir']
fluxcal = general_params['fluxcal']
try:
os.system('mkdir ' + outdir)
os.system('mkdir ' + outdir + 'listscan/')
except:
None
cts.listobs(vis=msfile,
listfile=outdir + 'listscan/listobs.txt',
overwrite=True)
print('listobs saved in ' + outdir + 'listscan/listobs.txt')
#Manually inspect bad antennas
try:
os.system('mkdir ' + outdir + 'ants')
except:
None
for i in fluxcal:
print(i)
subprocess.run(
'casaplotms vis={} field={} spw=0:256 iteraxis=antenna exprange=all plotfile={}+ants/{}.png showgui=False overwrite=True'
.format(msfile, i, outdir, i),
shell=True,
def avgIntervalcheck(self, dataset):
listobs(vis=dataset, listfile='listobs.txt')
self.assertTrue('Average Interval' in open('listobs.txt').read(),
msg='There is no average interval column in a MS')
