def main(options):
ms = options.ms
if ms == '':
logging.error('You have to specify an input MS, use -h for help')
return
cols = options.cols
incol = options.incol
t = pt.table(ms, readonly=False, ack=False)
for col in cols.split(','):
if col not in t.colnames():
logging.info('Adding the output column '+col+' to '+ms+'.')
if incol == '':
# prepare col metadata
cd = t.getcoldesc('DATA')
coldmi = t.getdminfo('DATA')
if options.dysco:
cd['dataManagerType'] = 'DyscoStMan'
cd['dataManagerGroup'] = 'DyscoData'
coldmi = {'NAME': col,'SEQNR': 3,'SPEC': {'dataBitCount': 10,'distribution': 'TruncatedGaussian','distributionTruncation': 2.5,'normalization': 'AF','studentTNu': 0.0,'weightBitCount': 12},'TYPE': 'DyscoStMan'}
# not as performing as standard DATA
else:
coldmi["NAME"] = col
# cd['dataManagerType'] = 'StandardStMan'
# cd['dataManagerGroup'] = 'SSMVar'
# coldmi = {'NAME': col,'SEQNR': 0,'SPEC': {'ActualCacheSize': 2,'BUCKETSIZE': 32768,'IndexLength': 799832,'PERSCACHESIZE': 2},'TYPE': 'StandardStMan'}
cd['comment'] = 'Added by addcol2ms'
t.addcols(pt.makecoldesc(col, cd), coldmi)
# if non dysco is done by default
if options.dysco:
logging.warning('Setting '+col+' = 0')
pt.taql("update $t set "+col+"=0")
else:
# prepare col metadata
coldmi = t.getdminfo(incol)
coldmi['NAME'] = col
cd = t.getcoldesc(incol)
cd['comment'] = 'Added by addcol2ms'
t.addcols(pt.makecoldesc(col, cd), coldmi)
logging.warning('Setting '+col+' = '+incol)
pt.taql("update $t set "+col+"="+incol)
else:
logging.warning('Column '+col+' already exists.')
t.close()
def setupiofiles(inms, outms, incolumn, outcolumn):
"""
if inms!=outms copy it and then work only on outms
"""
if outms == None:
outms = inms
if inms != outms:
t = pt.table(inms)
t.copy(outms, True, True)
t.close()
print("Finished copy.")
# create output column if doesn't exist
to = pt.table(outms, readonly=False)
if not outcolumn in to.colnames():
print("Add column %s" % outcolumn)
ti = pt.table(inms)
coldmi = ti.getdminfo(incolumn)
coldmi['NAME'] = outcolumn
to.addcols(pt.makecoldesc(outcolumn, ti.getcoldesc(incolumn)), coldmi)
pt.taql("update $to set " + outcolumn + "=" + incolumn)
ti.close()
to.close()
return outms
def addcol(ms, incol, outcol):
if outcol not in ms.colnames():
logging.info('Adding column: ' + outcol)
coldmi = ms.getdminfo(incol)
coldmi['NAME'] = outcol
ms.addcols(pt.makecoldesc(outcol, ms.getcoldesc(incol)), coldmi)
if outcol != incol:
# copy columns val
logging.info('Set ' + outcol + '=' + incol)
pt.taql("update $ms set " + outcol + "=" + incol)
def columnAddSimilar(pathMS, columnNameNew, columnNameSimilar, dataManagerInfoNameNew, overwrite = False, fillWithOnes = True, comment = "", verbose = False):
# more to lib_ms
"""
Add a column to a MS that is similar to a pre-existing column (in shape, but not in values).
pathMS: path of the MS
columnNameNew: name of the column to be added
columnNameSimilar: name of the column from which properties are copied (e.g. "DATA")
dataManagerInfoNameNew: string value for the data manager info (DMI) keyword "NAME" (should be unique in the MS)
overwrite: whether or not to overwrite column 'columnNameNew' if it already exists
fillWithOnes: whether or not to fill the newly-made column with ones
verbose: whether or not to produce abundant output
"""
t = tables.table(pathMS, readonly = False)
if (columnExists(t, columnNameNew) and not overwrite):
logger.warning("Attempt to add column '" + columnNameNew + "' aborted, as it already exists and 'overwrite = False' in columnAddSimilar(...).")
else: # Either the column does not exist yet, or it does but overwriting is allowed.
# Remove column if necessary.
if (columnExists(t, columnNameNew)):
logger.info("Removing column '" + columnNameNew + "'...")
t.removecols(columnNameNew)
# Add column.
columnDescription = t.getcoldesc(columnNameSimilar)
dataManagerInfo = t.getdminfo(columnNameSimilar)
if (verbose):
logger.debug("columnDescription:")
logger.debug(columnDescription)
logger.debug("dataManagerInfo:")
logger.debug(dataManagerInfo)
columnDescription["comment"] = ""
# What about adding something here like:
#columnDescription["dataManagerGroup"] = ...?
dataManagerInfo["NAME"] = dataManagerInfoNameNew
if (verbose):
logger.debug("columnDescription (updated):")
logger.debug(columnDescription)
logger.debug("dataManagerInfo (updated):")
logger.debug(dataManagerInfo)
logger.info("Adding column '" + columnNameNew + "'...")
t.addcols(tables.makecoldesc(columnNameNew, columnDescription), dataManagerInfo)
# Fill with ones if desired.
if (fillWithOnes):
logger.info("Filling column '" + columnNameNew + "' with ones...")
columnDataSimilar = t.getcol(columnNameSimilar)
t.putcol(columnNameNew, np.ones_like(columnDataSimilar))
# Close the table to avoid that it is locked for further use.
t.close()
def addcol(ms, incol, outcol):
""" Add a new column to a MS. """
if outcol not in ms.colnames():
logging.info('Adding column: ' + outcol)
coldmi = ms.getdminfo(incol)
coldmi['NAME'] = outcol
ms.addcols(pt.makecoldesc(outcol, ms.getcoldesc(incol)), coldmi)
if (outcol != incol):
# copy columns val
logging.info('Set ' + outcol + '=' + incol)
pt.taql("UPDATE $ms SET " + outcol + "=" + incol)
def test_addcolumns(self):
"""Add columns."""
c1 = makescacoldesc("coli", 0)
c2 = makescacoldesc("cold", 0.)
c3 = makescacoldesc("cols", "")
c4 = makescacoldesc("colb", True)
c5 = makescacoldesc("colc", 0. + 0j)
c6 = makearrcoldesc("colarr", 0.)
t = table("ttable.py_tmp.tab1", maketabdesc((c1, c2, c3, c4, c5,
c6)), ack=False)
t.addrows(2)
cd1 = makecoldesc("col2", t.getcoldesc('coli'))
t.addcols(cd1)
self.assertEqual(t.ncols(), 7)
self.assertIn('col2', t.colnames())
t.renamecol("col2", "ncol2")
self.assertNotIn('col2', t.colnames())
self.assertIn('ncol2', t.colnames())
t.close()
tabledelete("ttable.py_tmp.tab1")
def test_addcolumns(self):
"""Add columns."""
c1 = makescacoldesc("coli", 0)
c2 = makescacoldesc("cold", 0.)
c3 = makescacoldesc("cols", "")
c4 = makescacoldesc("colb", True)
c5 = makescacoldesc("colc", 0. + 0j)
c6 = makearrcoldesc("colarr", 0.)
t = table("ttable.py_tmp.tab1",
maketabdesc((c1, c2, c3, c4, c5, c6)),
ack=False)
t.addrows(2)
cd1 = makecoldesc("col2", t.getcoldesc('coli'))
t.addcols(cd1)
self.assertEqual(t.ncols(), 7)
self.assertIn('col2', t.colnames())
t.renamecol("col2", "ncol2")
self.assertNotIn('col2', t.colnames())
self.assertIn('ncol2', t.colnames())
t.close()
tabledelete("ttable.py_tmp.tab1")
def main(options):
ms = options.ms
if ms == '':
logging.error('You have to specify an input MS, use -h for help')
return
cols = options.cols
incol = options.incol
t = pt.table(ms, readonly=False, ack=False)
for col in cols.split(','):
if col not in t.colnames():
logging.info('Adding the output column ' + col + ' to ' + ms + '.')
if incol == '':
# prepare col metadata
cd = t.getcoldesc('DATA')
coldmi = t.getdminfo('DATA')
if options.dysco:
cd['dataManagerType'] = 'DyscoStMan'
cd['dataManagerGroup'] = 'DyscoData'
coldmi = {
'NAME': col,
'SEQNR': 3,
'SPEC': {
'dataBitCount': 10,
'distribution': 'TruncatedGaussian',
'distributionTruncation': 2.5,
'normalization': 'AF',
'studentTNu': 0.0,
'weightBitCount': 12
},
'TYPE': 'DyscoStMan'
}
# not as performing as standard DATA
else:
coldmi["NAME"] = col
# cd['dataManagerType'] = 'StandardStMan'
# cd['dataManagerGroup'] = 'SSMVar'
# coldmi = {'NAME': col,'SEQNR': 0,'SPEC': {'ActualCacheSize': 2,'BUCKETSIZE': 32768,'IndexLength': 799832,'PERSCACHESIZE': 2},'TYPE': 'StandardStMan'}
cd['comment'] = 'Added by addcol2ms'
t.addcols(pt.makecoldesc(col, cd), coldmi)
# if non dysco is done by default
if options.dysco:
logging.warning('Setting ' + col + ' = 0')
pt.taql("update $t set " + col + "=0")
else:
# prepare col metadata
coldmi = t.getdminfo(incol)
coldmi['NAME'] = col
cd = t.getcoldesc(incol)
cd['comment'] = 'Added by addcol2ms'
t.addcols(pt.makecoldesc(col, cd), coldmi)
logging.warning('Setting ' + col + ' = ' + incol)
pt.taql("update $t set " + col + "=" + incol)
else:
logging.warning('Column ' + col + ' already exists.')
t.close()
)
# clear up ddfcache files to save disk space
os.system('rm -rf *.ddfcache')
# Subtract the columns
if dosubtract:
for ms in msfiles:
t = pt.table(ms, readonly=False, ack=True)
colnames = t.colnames()
if ('PREDICT_SUB' in colnames) and (args['column'] in colnames):
if colname not in colnames:
# Append new column containing all sources
desc = t.getcoldesc(args['column'])
newdesc = pt.makecoldesc(colname, desc)
newdmi = t.getdminfo(args['column'])
newdmi['NAME'] = 'Dysco' + colname
t.addcols(newdesc, newdmi)
for row in range(0, t.nrows(), 3000000):
print('Reading', 'PREDICT_SUB')
f = t.getcol('PREDICT_SUB',
startrow=row,
nrow=3000000,
rowincr=1)
print('Reading', args['column'])
d = t.getcol(args['column'],
startrow=row,
nrow=3000000,
rowincr=1)
def main(data_dir, working_dir, ncpu, keeplongbaselines, chunkhours, predict_column, sub_column, npix_out):
data_dir = os.path.abspath(data_dir)
working_dir = os.path.abspath(working_dir)
try:
os.makedirs(working_dir)
except:
pass
try:
os.makedirs(os.path.join(working_dir, 'SOLSDIR'))
except:
pass
os.chdir(working_dir)
solsdir = os.path.join(data_dir, 'SOLSDIR')
mslist_file, mslist, fullmask, indico, clustercat = get_filenames(data_dir)
predict_dico = os.path.join(data_dir,
'image_full_ampphase_di_m.NS.not_{sub_column}.DicoModel'.format(sub_column=sub_column))
filtered_dico = os.path.join(data_dir,
'image_full_ampphase_di_m.NS.{sub_column}.DicoModel'.format(sub_column=sub_column))
predict_mask = os.path.join(data_dir, 'predict_mask_{sub_column}.fits'.format(
sub_column=sub_column)) # just a name, can be anything
region_file = os.path.join(working_dir, 'box_subtract_region.reg')
if keeplongbaselines:
uvsel = "[0.100000,5000.000000]"
else:
uvsel = "[0.100000,1000.000000]"
robust = -0.5
imagecell = 1.5
data_colname = 'DATA'
columnchecker(mslist, data_colname)
imagenpix = getimsize(fullmask)
npix_out, _ = EstimateNpix(float(npix_out), Padding=1)
# predict
if os.path.isfile(predict_dico):
os.unlink(predict_dico)
if os.path.isfile(predict_mask):
os.unlink(predict_mask)
if os.path.isfile(filtered_dico):
os.unlink(filtered_dico)
make_predict_mask(fullmask, region_file, predict_mask, npix_out=npix_out)
make_predict_dico(indico, predict_dico, predict_mask, npix_out=npix_out)
make_filtered_dico(fullmask, predict_mask, indico, filtered_dico, npix_out=npix_out)
args = dict(chunkhours=chunkhours, mslist_file=mslist_file, data_colname=data_colname, ncpu=ncpu,
clustercat=clustercat,
robust=robust, imagenpix=imagenpix, imagecell=imagecell, predict_mask=predict_mask,
predict_dico=predict_dico, uvsel=uvsel,
solsdir=solsdir, predict_column=predict_column)
logger.info("Predicting...")
cmd_call("DDF.py --Output-Name=image_full_ampphase_di_m.NS_SUB --Data-ChunkHours={chunkhours} --Data-MS={mslist_file} \
--Deconv-PeakFactor=0.001000 --Data-ColName={data_colname} --Parallel-NCPU={ncpu} --Facets-CatNodes={clustercat} \
--Beam-CenterNorm=1 --Deconv-Mode=SSD --Beam-Model=LOFAR --Beam-LOFARBeamMode=A --Weight-Robust={robust} \
--Image-NPix={imagenpix} --CF-wmax=50000 --CF-Nw=100 --Output-Also=onNeds --Image-Cell={imagecell} \
--Facets-NFacets=11 --SSDClean-NEnlargeData=0 --Freq-NDegridBand=1 --Beam-NBand=1 --Facets-DiamMax=1.5 \
--Facets-DiamMin=0.1 --Deconv-RMSFactor=3.000000 --SSDClean-ConvFFTSwitch 10000 --Data-Sort=1 --Cache-Dir=. \
--Log-Memory=1 --Cache-Weight=reset --Output-Mode=Predict --Output-RestoringBeam=6.000000 --Freq-NBand=2 \
--RIME-DecorrMode=FT --SSDClean-SSDSolvePars=[S,Alpha] --SSDClean-BICFactor=0 --Mask-Auto=1 --Mask-SigTh=5.00 \
--Mask-External={predict_mask} --DDESolutions-GlobalNorm=None --DDESolutions-DDModeGrid=AP \
--DDESolutions-DDModeDeGrid=AP --DDESolutions-DDSols=[DDS3_full_smoothed,DDS3_full_slow] \
--Predict-InitDicoModel={predict_dico} --Selection-UVRangeKm={uvsel} --GAClean-MinSizeInit=10 --Cache-Reset=1 \
--Beam-Smooth=1 --Predict-ColName={predict_column} --DDESolutions-SolsDir={solsdir}".format(**args))
# subtract
logger.info("Subtracting...")
for ms in mslist:
with pt.table(ms, readonly=False, ack=True) as t:
colnames = t.colnames()
if sub_column not in colnames:
# Append new column containing all sources
desc = t.getcoldesc(data_colname)
newdesc = pt.makecoldesc(sub_column, desc)
newdmi = t.getdminfo(data_colname)
newdmi['NAME'] = 'Dysco' + sub_column
t.addcols(newdesc, newdmi)
for row in range(0, t.nrows(), 3000000):
logger.info('Reading {}'.format(predict_column))
f = t.getcol(predict_column, startrow=row, nrow=3000000, rowincr=1)
logger.info('Reading', data_colname)
d = t.getcol(data_colname, startrow=row, nrow=3000000, rowincr=1)
logger.info('Writing', sub_column)
t.putcol(sub_column, d - f, startrow=row, nrow=3000000, rowincr=1)
addextraweights(mslist)
cleanup_working_dir(working_dir)
def pipeline_uGMRT_init(pathDirectoryMS, nameMS, verbose=False):
# 1. Download data from NCRA server
# 2. Convert into MS
# 3. Set-up directory structure
pathMS = pathDirectoryMS + '/' + nameMS
if (not os.path.isdir(pathMS)):
print("'" + pathMS + "' is not a valid path.")
sys.exit()
pathDirectoryMain = pathDirectoryMS + '/' + nameMS[:-3] + "-PiLF"
pathDirectoryFieldsTarget = pathDirectoryMain + "/fieldsTarget"
pathDirectoryFieldsCalibrator = pathDirectoryMain + "/fieldsCalibrator"
pathDirectoryLogs = pathDirectoryMain + "/logs"
if (os.path.isdir(pathDirectoryMain)):
shutil.rmtree(pathDirectoryMain)
os.mkdir(pathDirectoryMain)
os.mkdir(pathDirectoryFieldsTarget)
os.mkdir(pathDirectoryFieldsCalibrator)
os.mkdir(pathDirectoryLogs)
# Initialise logging settings.
nameFileLog = "pipeline_uGMRT_init.log"
pathFileLog = pathDirectoryLogs + '/' + nameFileLog
# Initialise logging.
lib_util.printLineBold("Starting log at '" + pathFileLog + "'...")
logging.basicConfig(filename=pathFileLog, level=logging.DEBUG)
logging.info("Started 'pipeline_uGMRT_init.py'!")
# 4. Split up the MS.
columnNameVisibilitiesNew = "DATA"
columnNameFlagsNew = "FLAG"
columnNameWeightsNew = "WEIGHT_SPECTRUM"
scanIDs = getScanIDs(pathMS)
numberOfScans = len(scanIDs)
logging.debug("pathMS:")
logging.debug(pathMS)
# Create temporary paths for the sub-MSs.
pathsMSNew = np.empty(len(scanIDs), dtype=object)
i = 0
for scanID in scanIDs:
pathMSNew = pathDirectoryMain + "/scanID" + str(scanID) + ".MS"
pathsMSNew[i] = pathMSNew
i += 1
logging.debug("pathsMSNew:")
logging.debug(pathsMSNew)
logging.info("- Split-up of original MS '" + pathMS + "' -")
for scanID, pathMSNew, i in zip(scanIDs, pathsMSNew,
np.arange(numberOfScans) + 1):
tables.taql(
"SELECT from $pathMS where SCAN_NUMBER = $scanID giving $pathMSNew as plain"
)
logging.info(
str(i) + " / " + str(numberOfScans) + ". Created MS '" +
pathMSNew + "'.")
# 5. Convert MSs from uGMRT to LOFAR format.
for pathMSNew in pathsMSNew:
logging.info("Starting work on MS at '" + pathMSNew + "'...")
# 5.1
# Removal of columns can give errors when executing LOFAR command 'msoverview'.
logging.info("- Removal of unnecessary data columns -")
t = tables.table(pathMSNew, readonly=False)
# Old array: ["EXPOSURE", "SIGMA_SPECTRUM"]. The column "EXPOSURE" seems necessary for DPPP (and LOFAR's msoverview), though.
for columnName in ["SIGMA_SPECTRUM"
]: # This list could possibly be expanded.
if (lib_util.columnExists(t, columnName)):
t.removecols(columnName)
t.close()
# 5.2
# CORR_TYPE column description comment: 'The polarization type for each correlation product, as a Stokes enum.'
# CORR_PRODUCT column description comment: 'Indices describing receptors of feed going into correlation'
logging.info("- Adaptation of polarisation metadata -")
t = tables.table(pathMSNew + "/POLARIZATION", readonly=False)
correlationTypesNew = np.array([[5, 6, 7, 8]])
correlationProductsNew = np.array([[[0, 0], [0, 1], [1, 0], [1, 1]]])
numberOfCorrelationsNew = 4
t.putcol("CORR_TYPE", correlationTypesNew)
t.putcol("CORR_PRODUCT", correlationProductsNew)
t.putcol("NUM_CORR", numberOfCorrelationsNew)
t.close()
# 5.3
logging.info("- Adaptation of frequency metadata -")
t = tables.table(pathMSNew + "/SPECTRAL_WINDOW", readonly=False)
frequencies = t.getcol("CHAN_FREQ")
frequenciesNew = np.fliplr(frequencies)
t.putcol("CHAN_FREQ", frequenciesNew)
t.close()
if (verbose):
logging.debug("frequencies:")
logging.debug(frequencies)
logging.debug("frequencies (updated):")
logging.debug(frequenciesNew)
# 5.4
logging.info("- Adaptation of field information -")
pathMSNewField = pathMSNew + "/FIELD"
# Remove metadata of other fields in the FIELD subtable
tables.taql(
"delete from $pathMSNewField where rownr() not in (select distinct FIELD_ID from $pathMSNew)"
)
# Set 'SOURCE_ID' to 0 in the FIELD subtable
tables.taql("update $pathMSNewField set SOURCE_ID=0")
# Set 'FIELD_ID' to 0 in the main table
tables.taql("update $pathMSNew set FIELD_ID=0")
# 5.5
logging.info("- Adaptation of intervals -")
times = getTimes(pathMSNew)
intervalPrecise = times[1] - times[0]
# Open the MS as a table in a way that changes can be made.
t = tables.table(pathMSNew, readonly=False)
intervals = t.getcol("INTERVAL")
intervalsNew = np.ones_like(intervals) * intervalPrecise
t.putcol("INTERVAL", intervalsNew)
t.close()
if (verbose):
logging.debug("Time intervals (should be equal):")
logging.debug(times[1:] - times[:-1])
# 5.6
logging.info(
"- Change existing (or create alternative) columns for data, flags and weights -"
)
# Open the MS as a table in a way that changes can be made.
t = tables.table(pathMSNew, readonly=False)
# First, change the 'direction' of the frequency axis.
# If the visibilities were sorted in descending frequency order (id est given for high frequencies first),
# they are converted to ascending frequency order. Note: applying this operation twice returns the MS to its old state!
if (verbose):
logging.info("Loading visibilities...")
visibilities = t.getcol("DATA")
if (verbose):
logging.info("Visibilities loaded!")
visibilities = np.fliplr(visibilities)
visibilitiesNew = np.zeros(
(visibilities.shape[0], visibilities.shape[1], 4),
dtype=np.complex128)
visibilitiesNew[:, :, 0] = visibilities[:, :, 0]
visibilitiesNew[:, :, 3] = visibilities[:, :, 1]
keywordNames = t.colkeywordnames("DATA")
columnDescription = t.getcoldesc("DATA")
dataManagerInfo = t.getdminfo("DATA")
if (verbose):
logging.debug("keywordNames:")
logging.debug(keywordNames)
logging.debug("columnDescription:")
logging.debug(columnDescription)
logging.debug("dataManagerInfo:")
logging.debug(dataManagerInfo)
dataManagerInfo["NAME"] = "TiledDATAMartijn"
dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819],
dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array(
[4, 40, 819], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array(
[4, visibilities.shape[1], visibilities.shape[0]], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array(
[4, visibilities.shape[1]], dtype=np.int32)
if (verbose):
logging.debug("dataManagerInfo (updated):")
logging.debug(dataManagerInfo)
if (verbose):
logging.info("Removing column '" + columnNameVisibilitiesNew +
"', if it exists...")
if (lib_util.columnExists(t, columnNameVisibilitiesNew)):
t.removecols(columnNameVisibilitiesNew)
if (verbose):
logging.info("Adding column '" + columnNameVisibilitiesNew +
"'...")
t.addcols(
tables.makecoldesc(columnNameVisibilitiesNew, columnDescription),
dataManagerInfo)
if (verbose):
logging.info("Filling column '" + columnNameVisibilitiesNew +
"'...")
t.putcol(columnNameVisibilitiesNew, visibilitiesNew)
if (verbose):
logging.info(
"Visibilities flipped along frequency axis and placeholder polarisations added!"
)
if (verbose):
logging.info("Loading flags...")
flags = t.getcol("FLAG")
if (verbose):
logging.info("Flags loaded!")
flags = np.fliplr(flags)
flagsNew = np.zeros((flags.shape[0], flags.shape[1], 4),
dtype=np.bool_)
flagsNew[:, :, 0] = flags[:, :, 0]
flagsNew[:, :, 1] = flags[:, :,
0] # Take over flags from LL correlation
flagsNew[:, :, 2] = flags[:, :,
0] # Take over flags from LL correlation
flagsNew[:, :, 3] = flags[:, :, 1]
keywordNames = t.colkeywordnames("FLAG")
columnDescription = t.getcoldesc("FLAG")
dataManagerInfo = t.getdminfo("FLAG")
if (verbose):
logging.debug("keywordNames:")
logging.debug(keywordNames)
logging.debug("columnDescription:")
logging.debug(columnDescription)
logging.debug("dataManagerInfo:")
logging.debug(dataManagerInfo)
dataManagerInfo["NAME"] = "TiledFlagMartijn"
dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819],
dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array(
[4, 40, 819], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array(
[4, flags.shape[1], flags.shape[0]], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array(
[4, flags.shape[1]], dtype=np.int32)
if (verbose):
logging.debug("dataManagerInfo (updated):")
logging.debug(dataManagerInfo)
if (verbose):
logging.info("Removing column '" + columnNameFlagsNew +
"', if it exists...")
if (lib_util.columnExists(t, columnNameFlagsNew)):
t.removecols(columnNameFlagsNew)
if (verbose):
logging.info("Adding column '" + columnNameFlagsNew + "'...")
t.addcols(tables.makecoldesc(columnNameFlagsNew, columnDescription),
dataManagerInfo)
if (verbose):
logging.info("Filling column '" + columnNameFlagsNew + "'...")
t.putcol(columnNameFlagsNew, flagsNew)
if (verbose):
logging.info(
"Flags flipped along frequency axis and placeholder polarisations added!"
)
if (verbose):
logging.info("Loading weights...")
weights = t.getcol("WEIGHT_SPECTRUM")
if (verbose):
logging.info("Weights loaded!")
weights = np.fliplr(weights)
weightsNew = np.zeros((weights.shape[0], weights.shape[1], 4),
dtype=np.float64)
weightsNew[:, :, 0] = weights[:, :, 0]
weightsNew[:, :, 3] = weights[:, :, 1]
keywordNames = t.colkeywordnames("WEIGHT_SPECTRUM")
columnDescription = t.getcoldesc("WEIGHT_SPECTRUM")
dataManagerInfo = t.getdminfo("WEIGHT_SPECTRUM")
if (verbose):
logging.debug("keywordNames:")
logging.debug(keywordNames)
logging.debug("columnDescription:")
logging.debug(columnDescription)
logging.debug("dataManagerInfo:")
logging.debug(dataManagerInfo)
dataManagerInfo["NAME"] = "TiledWgtSpectrumMartijn"
dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819],
dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array(
[4, 40, 819], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array(
[4, weights.shape[1], weights.shape[0]], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array(
[4, weights.shape[1]], dtype=np.int32)
if (verbose):
logging.debug("dataManagerInfo (updated):")
logging.debug(dataManagerInfo)
if (verbose):
logging.info("Removing column '" + columnNameWeightsNew +
"', if it exists...")
if (lib_util.columnExists(t, columnNameWeightsNew)):
t.removecols(columnNameWeightsNew)
if (verbose):
logging.info("Adding column '" + columnNameWeightsNew + "'...")
t.addcols(tables.makecoldesc(columnNameWeightsNew, columnDescription),
dataManagerInfo)
if (verbose):
logging.info("Filling column '" + columnNameWeightsNew + "'...")
t.putcol(columnNameWeightsNew, weightsNew)
if (verbose):
logging.info(
"Weights flipped along frequency axis and placeholder polarisations added!"
)
t.close()
logging.info("Finished work on MS at '" + pathMSNew + "'!")
# 6. Move the sub-MSs from a temporary place to their right locus in the file tree.
for pathMSNew in pathsMSNew:
MSObject = lib_ms.MS(pathMSNew)
if (MSObject.isCalibrator()):
pathDirectoryCalibrator = pathDirectoryFieldsCalibrator + '/' + MSObject.nameMS
pathMSFinal = pathDirectoryCalibrator + '/' + MSObject.nameMS + ".MS"
if (not os.path.isdir(pathDirectoryCalibrator)):
os.mkdir(pathDirectoryCalibrator)
os.mkdir(pathDirectoryCalibrator + "/plots")
os.mkdir(pathDirectoryCalibrator + "/solutions")
else:
pathDirectoryTarget = pathDirectoryFieldsTarget + '/' + MSObject.getNameField(
)
pathMSFinal = pathDirectoryTarget + "/MSs/" + MSObject.nameMS + ".MS"
if (not os.path.isdir(pathDirectoryTarget)):
os.mkdir(pathDirectoryTarget)
os.mkdir(pathDirectoryTarget + "/plots")
os.mkdir(pathDirectoryTarget + "/MSs")
os.mkdir(pathDirectoryTarget + "/images")
logging.info("Moving MS at '" + pathMSNew + "' to '" + pathMSFinal +
"'...")
MSObject.move(pathMSFinal)
def antialias_ms(msname, tol, outputcolname="DATA_AAF"):
"""
Apply an anti aliasing filter in a parallel way to a measurement set
Params:
msname (str): Name of measurement set
tol (float): Filter response below this limit will be ignored
outputcolname (str): Name of column to write corrected visibilities to (will be added to MS if necessary)
Returns:
None
"""
logger = logging.getLogger("aaf")
logger.info("Anti-aliasing measurement set " + msname)
# 1. Open MS and read subtable 'DATA',
# if necessary create a new column (default "DATA_AAF") to store AAF corrected data.
t1 = datetime.datetime.now()
ms = tables.table(msname, readonly=False, ack=False)
nrows = ms.nrows()
ini_data = tables.tablecolumn(ms, 'DATA')
# If there is no column "DATA_AAF", then create one.
if outputcolname not in ms.colnames():
coldes = tables.makecoldesc(outputcolname, ms.getcoldesc('DATA'))
dmname = ms.getdminfo('DATA')
dmname["NAME"] = 'TiledAAFData'
ms.addcols(coldes, dminfo=dmname)
# 2. Calculate function corr()'s two arguments: subband_response and nconv
# Fixed parameters: Number of channels per subband; Total number of subbands
num_channels = 64
num_subbands = 1024
# Load filter coefficients, pad with zero
coeff = np.loadtxt(file_prefix() + '/share/aaf/Coeffs16384Kaiser-quant.dat')
coeff = np.append(coeff, np.zeros(num_channels * num_subbands - coeff.size))
# Get filter frequency response by doing FFT on filter coefficients
frequency_response = np.abs(np.fft.fft(coeff)) ** 2
# Scaling
frequency_response = frequency_response / np.sum(frequency_response) * num_channels
# We only consider aliasing influence from the neighbouring two bands
subband_response = np.roll(frequency_response, int(1.5 * num_channels))
subband_response = np.reshape(subband_response[0:3 * num_channels], (3, num_channels))
# Tolerance, filter response below that is ignored
# maximum de-convolution length
nconv = int(math.ceil(math.log(tol, subband_response[2, 1] / subband_response[1, 1])))
# 3. Do AAF calibration concurrently (parallel)
num_cpus = multiprocessing.cpu_count()
pool = multiprocessing.Pool(processes=num_cpus - 1)
# Here itertools and the function antialias_list() are just bridges between pool.map and function corr(), because pool.map
# is not suitable for function that has multi arguments.
aafdata = pool.map(antialias_list, itertools.izip(ini_data[0:nrows], itertools.repeat(subband_response), itertools.repeat(nconv)))
# 4. Write AAF corrected data to MS, usually the size of data is very large (for example 109746*16384*4 in my
# current MS), to avoid Memory Error, we wrote the data by four steps.
chunksize = nrows / 4
start_row = np.array([0, chunksize, chunksize * 2, chunksize * 3])
end_row = np.array([chunksize, chunksize * 2, chunksize * 3, nrows])
for parti in range(0, 4):
ms.putcol(outputcolname, np.array(aafdata[start_row[parti]:end_row[parti]]), startrow=start_row[parti],
nrow=end_row[parti] - start_row[parti])
t2 = datetime.datetime.now()
print("Total execution time:", (t2 - t1).total_seconds(), "seconds")
log_msg = "Performed anti-aliasing on MS, wrote result to column " + outputcolname
pt.taql('INSERT INTO {}::HISTORY SET MESSAGE="{}", APPLICATION="apercal"'.format(msname, log_msg))
logger.info(log_msg)
def updateColumns(self, updateFreq):
'''
Update DATA, FLAG and WEIGHT_SPECTRUM columns.
'''
logging.info(
"- Change existing (or create alternative) columns for data, flags and weights -"
)
logging.info("Loading visibilities...")
visibilities = self.t.getcol("DATA")
if (
updateFreq
): # If the frequency channel order was flipped in 'updateFreqMetadata', also flip the data.
visibilities = visibilities[:, ::-1, :]
if (visibilities.shape[2] == 2):
visibilitiesNew = np.zeros(
(visibilities.shape[0], visibilities.shape[1], 4),
dtype=np.complex128)
visibilitiesNew[:, :, 0] = visibilities[:, :, 0]
visibilitiesNew[:, :, 3] = visibilities[:, :, 1]
visibilities = visibilitiesNew
keywordNames = self.t.colkeywordnames("DATA")
columnDescription = self.t.getcoldesc("DATA")
dataManagerInfo = self.t.getdminfo("DATA")
dataManagerInfo["NAME"] = "TiledDATAMartijn"
dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819],
dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array(
[4, 40, 819], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array(
[4, visibilities.shape[1], visibilities.shape[0]], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array(
[4, visibilities.shape[1]], dtype=np.int32)
#logging.debug("keywordNames:")
#logging.debug(keywordNames)
#logging.debug("columnDescription:")
#logging.debug(columnDescription)
#logging.debug("dataManagerInfo:")
#logging.debug(dataManagerInfo)
#logging.debug("dataManagerInfo (updated):")
#logging.debug(dataManagerInfo)
logging.info("Removing column 'DATA', if it exists...")
if (self.columnExists('DATA')):
self.t.removecols('DATA')
logging.info("Adding column 'DATA'...")
self.t.addcols(tables.makecoldesc('DATA', columnDescription),
dataManagerInfo)
logging.info("Filling column 'DATA'...")
self.t.putcol('DATA', visibilities)
logging.info("Loading flags...")
flags = self.t.getcol("FLAG")
if (
updateFreq
): # If the frequency channel order was flipped in 'updateFreqMetadata', also flip the data.
flags = flags[:, ::-1, :]
if (flags.shape[2] == 2):
flagsNew = np.zeros((flags.shape[0], flags.shape[1], 4),
dtype=np.bool_)
flagsNew[:, :, 0] = flags[:, :, 0]
flagsNew[:, :, 1] = flags[:, :,
0] # Take over flags from LL correlation
flagsNew[:, :, 2] = flags[:, :,
0] # Take over flags from LL correlation
flagsNew[:, :, 3] = flags[:, :, 1]
flags = flagsNew
keywordNames = self.t.colkeywordnames("FLAG")
columnDescription = self.t.getcoldesc("FLAG")
dataManagerInfo = self.t.getdminfo("FLAG")
dataManagerInfo["NAME"] = "TiledFlagMartijn"
dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819],
dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array(
[4, 40, 819], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array(
[4, flags.shape[1], flags.shape[0]], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array(
[4, flags.shape[1]], dtype=np.int32)
#logging.debug("keywordNames:")
#logging.debug(keywordNames)
#logging.debug("columnDescription:")
#logging.debug(columnDescription)
#logging.debug("dataManagerInfo:")
#logging.debug(dataManagerInfo)
#logging.debug("dataManagerInfo (updated):")
#logging.debug(dataManagerInfo)
logging.info("Removing column 'FLAG', if it exists...")
if (self.columnExists('FLAG')):
self.t.removecols('FLAG')
logging.info("Adding column 'FLAG'...")
self.t.addcols(tables.makecoldesc('FLAG', columnDescription),
dataManagerInfo)
logging.info("Filling column 'FLAG'...")
self.t.putcol('FLAG', flags)
logging.info("Loading weights...")
weights = self.t.getcol("WEIGHT_SPECTRUM")
if (
updateFreq
): # If the frequency channel order was flipped in 'updateFreqMetadata', also flip the data.
weights = weights[:, ::-1, :]
if (weights.shape[2] == 2):
weightsNew = np.zeros((weights.shape[0], weights.shape[1], 4),
dtype=np.float64)
weightsNew[:, :, 0] = weights[:, :, 0]
weightsNew[:, :, 3] = weights[:, :, 1]
weights = weightsNew
keywordNames = self.t.colkeywordnames("WEIGHT_SPECTRUM")
columnDescription = self.t.getcoldesc("WEIGHT_SPECTRUM")
dataManagerInfo = self.t.getdminfo("WEIGHT_SPECTRUM")
dataManagerInfo["NAME"] = "TiledWgtSpectrumMartijn"
dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819],
dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array(
[4, 40, 819], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array(
[4, weights.shape[1], weights.shape[0]], dtype=np.int32)
dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array(
[4, weights.shape[1]], dtype=np.int32)
#logging.debug("keywordNames:")
#logging.debug(keywordNames)
#logging.debug("columnDescription:")
#logging.debug(columnDescription)
#logging.debug("dataManagerInfo:")
#logging.debug(dataManagerInfo)
#logging.debug("dataManagerInfo (updated):")
#logging.debug(dataManagerInfo)
logging.info("Removing column 'WEIGHT_SPECTRUM', if it exists...")
if (self.columnExists('WEIGHT_SPECTRUM')):
self.t.removecols('WEIGHT_SPECTRUM')
logging.info("Adding column 'WEIGHT_SPECTRUM'...")
self.t.addcols(
tables.makecoldesc('WEIGHT_SPECTRUM', columnDescription),
dataManagerInfo)
logging.info("Filling column 'WEIGHT_SPECTRUM'...")
self.t.putcol('WEIGHT_SPECTRUM', weights)