def main(options): ms = options.ms if ms == '': logging.error('You have to specify an input MS, use -h for help') return cols = options.cols incol = options.incol t = pt.table(ms, readonly=False, ack=False) for col in cols.split(','): if col not in t.colnames(): logging.info('Adding the output column '+col+' to '+ms+'.') if incol == '': # prepare col metadata cd = t.getcoldesc('DATA') coldmi = t.getdminfo('DATA') if options.dysco: cd['dataManagerType'] = 'DyscoStMan' cd['dataManagerGroup'] = 'DyscoData' coldmi = {'NAME': col,'SEQNR': 3,'SPEC': {'dataBitCount': 10,'distribution': 'TruncatedGaussian','distributionTruncation': 2.5,'normalization': 'AF','studentTNu': 0.0,'weightBitCount': 12},'TYPE': 'DyscoStMan'} # not as performing as standard DATA else: coldmi["NAME"] = col # cd['dataManagerType'] = 'StandardStMan' # cd['dataManagerGroup'] = 'SSMVar' # coldmi = {'NAME': col,'SEQNR': 0,'SPEC': {'ActualCacheSize': 2,'BUCKETSIZE': 32768,'IndexLength': 799832,'PERSCACHESIZE': 2},'TYPE': 'StandardStMan'} cd['comment'] = 'Added by addcol2ms' t.addcols(pt.makecoldesc(col, cd), coldmi) # if non dysco is done by default if options.dysco: logging.warning('Setting '+col+' = 0') pt.taql("update $t set "+col+"=0") else: # prepare col metadata coldmi = t.getdminfo(incol) coldmi['NAME'] = col cd = t.getcoldesc(incol) cd['comment'] = 'Added by addcol2ms' t.addcols(pt.makecoldesc(col, cd), coldmi) logging.warning('Setting '+col+' = '+incol) pt.taql("update $t set "+col+"="+incol) else: logging.warning('Column '+col+' already exists.') t.close()
def setupiofiles(inms, outms, incolumn, outcolumn): """ if inms!=outms copy it and then work only on outms """ if outms == None: outms = inms if inms != outms: t = pt.table(inms) t.copy(outms, True, True) t.close() print("Finished copy.") # create output column if doesn't exist to = pt.table(outms, readonly=False) if not outcolumn in to.colnames(): print("Add column %s" % outcolumn) ti = pt.table(inms) coldmi = ti.getdminfo(incolumn) coldmi['NAME'] = outcolumn to.addcols(pt.makecoldesc(outcolumn, ti.getcoldesc(incolumn)), coldmi) pt.taql("update $to set " + outcolumn + "=" + incolumn) ti.close() to.close() return outms
def addcol(ms, incol, outcol): if outcol not in ms.colnames(): logging.info('Adding column: ' + outcol) coldmi = ms.getdminfo(incol) coldmi['NAME'] = outcol ms.addcols(pt.makecoldesc(outcol, ms.getcoldesc(incol)), coldmi) if outcol != incol: # copy columns val logging.info('Set ' + outcol + '=' + incol) pt.taql("update $ms set " + outcol + "=" + incol)
def columnAddSimilar(pathMS, columnNameNew, columnNameSimilar, dataManagerInfoNameNew, overwrite = False, fillWithOnes = True, comment = "", verbose = False): # more to lib_ms """ Add a column to a MS that is similar to a pre-existing column (in shape, but not in values). pathMS: path of the MS columnNameNew: name of the column to be added columnNameSimilar: name of the column from which properties are copied (e.g. "DATA") dataManagerInfoNameNew: string value for the data manager info (DMI) keyword "NAME" (should be unique in the MS) overwrite: whether or not to overwrite column 'columnNameNew' if it already exists fillWithOnes: whether or not to fill the newly-made column with ones verbose: whether or not to produce abundant output """ t = tables.table(pathMS, readonly = False) if (columnExists(t, columnNameNew) and not overwrite): logger.warning("Attempt to add column '" + columnNameNew + "' aborted, as it already exists and 'overwrite = False' in columnAddSimilar(...).") else: # Either the column does not exist yet, or it does but overwriting is allowed. # Remove column if necessary. if (columnExists(t, columnNameNew)): logger.info("Removing column '" + columnNameNew + "'...") t.removecols(columnNameNew) # Add column. columnDescription = t.getcoldesc(columnNameSimilar) dataManagerInfo = t.getdminfo(columnNameSimilar) if (verbose): logger.debug("columnDescription:") logger.debug(columnDescription) logger.debug("dataManagerInfo:") logger.debug(dataManagerInfo) columnDescription["comment"] = "" # What about adding something here like: #columnDescription["dataManagerGroup"] = ...? dataManagerInfo["NAME"] = dataManagerInfoNameNew if (verbose): logger.debug("columnDescription (updated):") logger.debug(columnDescription) logger.debug("dataManagerInfo (updated):") logger.debug(dataManagerInfo) logger.info("Adding column '" + columnNameNew + "'...") t.addcols(tables.makecoldesc(columnNameNew, columnDescription), dataManagerInfo) # Fill with ones if desired. if (fillWithOnes): logger.info("Filling column '" + columnNameNew + "' with ones...") columnDataSimilar = t.getcol(columnNameSimilar) t.putcol(columnNameNew, np.ones_like(columnDataSimilar)) # Close the table to avoid that it is locked for further use. t.close()
def addcol(ms, incol, outcol): """ Add a new column to a MS. """ if outcol not in ms.colnames(): logging.info('Adding column: ' + outcol) coldmi = ms.getdminfo(incol) coldmi['NAME'] = outcol ms.addcols(pt.makecoldesc(outcol, ms.getcoldesc(incol)), coldmi) if (outcol != incol): # copy columns val logging.info('Set ' + outcol + '=' + incol) pt.taql("UPDATE $ms SET " + outcol + "=" + incol)
def test_addcolumns(self): """Add columns.""" c1 = makescacoldesc("coli", 0) c2 = makescacoldesc("cold", 0.) c3 = makescacoldesc("cols", "") c4 = makescacoldesc("colb", True) c5 = makescacoldesc("colc", 0. + 0j) c6 = makearrcoldesc("colarr", 0.) t = table("ttable.py_tmp.tab1", maketabdesc((c1, c2, c3, c4, c5, c6)), ack=False) t.addrows(2) cd1 = makecoldesc("col2", t.getcoldesc('coli')) t.addcols(cd1) self.assertEqual(t.ncols(), 7) self.assertIn('col2', t.colnames()) t.renamecol("col2", "ncol2") self.assertNotIn('col2', t.colnames()) self.assertIn('ncol2', t.colnames()) t.close() tabledelete("ttable.py_tmp.tab1")
def main(options): ms = options.ms if ms == '': logging.error('You have to specify an input MS, use -h for help') return cols = options.cols incol = options.incol t = pt.table(ms, readonly=False, ack=False) for col in cols.split(','): if col not in t.colnames(): logging.info('Adding the output column ' + col + ' to ' + ms + '.') if incol == '': # prepare col metadata cd = t.getcoldesc('DATA') coldmi = t.getdminfo('DATA') if options.dysco: cd['dataManagerType'] = 'DyscoStMan' cd['dataManagerGroup'] = 'DyscoData' coldmi = { 'NAME': col, 'SEQNR': 3, 'SPEC': { 'dataBitCount': 10, 'distribution': 'TruncatedGaussian', 'distributionTruncation': 2.5, 'normalization': 'AF', 'studentTNu': 0.0, 'weightBitCount': 12 }, 'TYPE': 'DyscoStMan' } # not as performing as standard DATA else: coldmi["NAME"] = col # cd['dataManagerType'] = 'StandardStMan' # cd['dataManagerGroup'] = 'SSMVar' # coldmi = {'NAME': col,'SEQNR': 0,'SPEC': {'ActualCacheSize': 2,'BUCKETSIZE': 32768,'IndexLength': 799832,'PERSCACHESIZE': 2},'TYPE': 'StandardStMan'} cd['comment'] = 'Added by addcol2ms' t.addcols(pt.makecoldesc(col, cd), coldmi) # if non dysco is done by default if options.dysco: logging.warning('Setting ' + col + ' = 0') pt.taql("update $t set " + col + "=0") else: # prepare col metadata coldmi = t.getdminfo(incol) coldmi['NAME'] = col cd = t.getcoldesc(incol) cd['comment'] = 'Added by addcol2ms' t.addcols(pt.makecoldesc(col, cd), coldmi) logging.warning('Setting ' + col + ' = ' + incol) pt.taql("update $t set " + col + "=" + incol) else: logging.warning('Column ' + col + ' already exists.') t.close()
) # clear up ddfcache files to save disk space os.system('rm -rf *.ddfcache') # Subtract the columns if dosubtract: for ms in msfiles: t = pt.table(ms, readonly=False, ack=True) colnames = t.colnames() if ('PREDICT_SUB' in colnames) and (args['column'] in colnames): if colname not in colnames: # Append new column containing all sources desc = t.getcoldesc(args['column']) newdesc = pt.makecoldesc(colname, desc) newdmi = t.getdminfo(args['column']) newdmi['NAME'] = 'Dysco' + colname t.addcols(newdesc, newdmi) for row in range(0, t.nrows(), 3000000): print('Reading', 'PREDICT_SUB') f = t.getcol('PREDICT_SUB', startrow=row, nrow=3000000, rowincr=1) print('Reading', args['column']) d = t.getcol(args['column'], startrow=row, nrow=3000000, rowincr=1)
def main(data_dir, working_dir, ncpu, keeplongbaselines, chunkhours, predict_column, sub_column, npix_out): data_dir = os.path.abspath(data_dir) working_dir = os.path.abspath(working_dir) try: os.makedirs(working_dir) except: pass try: os.makedirs(os.path.join(working_dir, 'SOLSDIR')) except: pass os.chdir(working_dir) solsdir = os.path.join(data_dir, 'SOLSDIR') mslist_file, mslist, fullmask, indico, clustercat = get_filenames(data_dir) predict_dico = os.path.join(data_dir, 'image_full_ampphase_di_m.NS.not_{sub_column}.DicoModel'.format(sub_column=sub_column)) filtered_dico = os.path.join(data_dir, 'image_full_ampphase_di_m.NS.{sub_column}.DicoModel'.format(sub_column=sub_column)) predict_mask = os.path.join(data_dir, 'predict_mask_{sub_column}.fits'.format( sub_column=sub_column)) # just a name, can be anything region_file = os.path.join(working_dir, 'box_subtract_region.reg') if keeplongbaselines: uvsel = "[0.100000,5000.000000]" else: uvsel = "[0.100000,1000.000000]" robust = -0.5 imagecell = 1.5 data_colname = 'DATA' columnchecker(mslist, data_colname) imagenpix = getimsize(fullmask) npix_out, _ = EstimateNpix(float(npix_out), Padding=1) # predict if os.path.isfile(predict_dico): os.unlink(predict_dico) if os.path.isfile(predict_mask): os.unlink(predict_mask) if os.path.isfile(filtered_dico): os.unlink(filtered_dico) make_predict_mask(fullmask, region_file, predict_mask, npix_out=npix_out) make_predict_dico(indico, predict_dico, predict_mask, npix_out=npix_out) make_filtered_dico(fullmask, predict_mask, indico, filtered_dico, npix_out=npix_out) args = dict(chunkhours=chunkhours, mslist_file=mslist_file, data_colname=data_colname, ncpu=ncpu, clustercat=clustercat, robust=robust, imagenpix=imagenpix, imagecell=imagecell, predict_mask=predict_mask, predict_dico=predict_dico, uvsel=uvsel, solsdir=solsdir, predict_column=predict_column) logger.info("Predicting...") cmd_call("DDF.py --Output-Name=image_full_ampphase_di_m.NS_SUB --Data-ChunkHours={chunkhours} --Data-MS={mslist_file} \ --Deconv-PeakFactor=0.001000 --Data-ColName={data_colname} --Parallel-NCPU={ncpu} --Facets-CatNodes={clustercat} \ --Beam-CenterNorm=1 --Deconv-Mode=SSD --Beam-Model=LOFAR --Beam-LOFARBeamMode=A --Weight-Robust={robust} \ --Image-NPix={imagenpix} --CF-wmax=50000 --CF-Nw=100 --Output-Also=onNeds --Image-Cell={imagecell} \ --Facets-NFacets=11 --SSDClean-NEnlargeData=0 --Freq-NDegridBand=1 --Beam-NBand=1 --Facets-DiamMax=1.5 \ --Facets-DiamMin=0.1 --Deconv-RMSFactor=3.000000 --SSDClean-ConvFFTSwitch 10000 --Data-Sort=1 --Cache-Dir=. \ --Log-Memory=1 --Cache-Weight=reset --Output-Mode=Predict --Output-RestoringBeam=6.000000 --Freq-NBand=2 \ --RIME-DecorrMode=FT --SSDClean-SSDSolvePars=[S,Alpha] --SSDClean-BICFactor=0 --Mask-Auto=1 --Mask-SigTh=5.00 \ --Mask-External={predict_mask} --DDESolutions-GlobalNorm=None --DDESolutions-DDModeGrid=AP \ --DDESolutions-DDModeDeGrid=AP --DDESolutions-DDSols=[DDS3_full_smoothed,DDS3_full_slow] \ --Predict-InitDicoModel={predict_dico} --Selection-UVRangeKm={uvsel} --GAClean-MinSizeInit=10 --Cache-Reset=1 \ --Beam-Smooth=1 --Predict-ColName={predict_column} --DDESolutions-SolsDir={solsdir}".format(**args)) # subtract logger.info("Subtracting...") for ms in mslist: with pt.table(ms, readonly=False, ack=True) as t: colnames = t.colnames() if sub_column not in colnames: # Append new column containing all sources desc = t.getcoldesc(data_colname) newdesc = pt.makecoldesc(sub_column, desc) newdmi = t.getdminfo(data_colname) newdmi['NAME'] = 'Dysco' + sub_column t.addcols(newdesc, newdmi) for row in range(0, t.nrows(), 3000000): logger.info('Reading {}'.format(predict_column)) f = t.getcol(predict_column, startrow=row, nrow=3000000, rowincr=1) logger.info('Reading', data_colname) d = t.getcol(data_colname, startrow=row, nrow=3000000, rowincr=1) logger.info('Writing', sub_column) t.putcol(sub_column, d - f, startrow=row, nrow=3000000, rowincr=1) addextraweights(mslist) cleanup_working_dir(working_dir)
def pipeline_uGMRT_init(pathDirectoryMS, nameMS, verbose=False): # 1. Download data from NCRA server # 2. Convert into MS # 3. Set-up directory structure pathMS = pathDirectoryMS + '/' + nameMS if (not os.path.isdir(pathMS)): print("'" + pathMS + "' is not a valid path.") sys.exit() pathDirectoryMain = pathDirectoryMS + '/' + nameMS[:-3] + "-PiLF" pathDirectoryFieldsTarget = pathDirectoryMain + "/fieldsTarget" pathDirectoryFieldsCalibrator = pathDirectoryMain + "/fieldsCalibrator" pathDirectoryLogs = pathDirectoryMain + "/logs" if (os.path.isdir(pathDirectoryMain)): shutil.rmtree(pathDirectoryMain) os.mkdir(pathDirectoryMain) os.mkdir(pathDirectoryFieldsTarget) os.mkdir(pathDirectoryFieldsCalibrator) os.mkdir(pathDirectoryLogs) # Initialise logging settings. nameFileLog = "pipeline_uGMRT_init.log" pathFileLog = pathDirectoryLogs + '/' + nameFileLog # Initialise logging. lib_util.printLineBold("Starting log at '" + pathFileLog + "'...") logging.basicConfig(filename=pathFileLog, level=logging.DEBUG) logging.info("Started 'pipeline_uGMRT_init.py'!") # 4. Split up the MS. columnNameVisibilitiesNew = "DATA" columnNameFlagsNew = "FLAG" columnNameWeightsNew = "WEIGHT_SPECTRUM" scanIDs = getScanIDs(pathMS) numberOfScans = len(scanIDs) logging.debug("pathMS:") logging.debug(pathMS) # Create temporary paths for the sub-MSs. pathsMSNew = np.empty(len(scanIDs), dtype=object) i = 0 for scanID in scanIDs: pathMSNew = pathDirectoryMain + "/scanID" + str(scanID) + ".MS" pathsMSNew[i] = pathMSNew i += 1 logging.debug("pathsMSNew:") logging.debug(pathsMSNew) logging.info("- Split-up of original MS '" + pathMS + "' -") for scanID, pathMSNew, i in zip(scanIDs, pathsMSNew, np.arange(numberOfScans) + 1): tables.taql( "SELECT from $pathMS where SCAN_NUMBER = $scanID giving $pathMSNew as plain" ) logging.info( str(i) + " / " + str(numberOfScans) + ". Created MS '" + pathMSNew + "'.") # 5. Convert MSs from uGMRT to LOFAR format. for pathMSNew in pathsMSNew: logging.info("Starting work on MS at '" + pathMSNew + "'...") # 5.1 # Removal of columns can give errors when executing LOFAR command 'msoverview'. logging.info("- Removal of unnecessary data columns -") t = tables.table(pathMSNew, readonly=False) # Old array: ["EXPOSURE", "SIGMA_SPECTRUM"]. The column "EXPOSURE" seems necessary for DPPP (and LOFAR's msoverview), though. for columnName in ["SIGMA_SPECTRUM" ]: # This list could possibly be expanded. if (lib_util.columnExists(t, columnName)): t.removecols(columnName) t.close() # 5.2 # CORR_TYPE column description comment: 'The polarization type for each correlation product, as a Stokes enum.' # CORR_PRODUCT column description comment: 'Indices describing receptors of feed going into correlation' logging.info("- Adaptation of polarisation metadata -") t = tables.table(pathMSNew + "/POLARIZATION", readonly=False) correlationTypesNew = np.array([[5, 6, 7, 8]]) correlationProductsNew = np.array([[[0, 0], [0, 1], [1, 0], [1, 1]]]) numberOfCorrelationsNew = 4 t.putcol("CORR_TYPE", correlationTypesNew) t.putcol("CORR_PRODUCT", correlationProductsNew) t.putcol("NUM_CORR", numberOfCorrelationsNew) t.close() # 5.3 logging.info("- Adaptation of frequency metadata -") t = tables.table(pathMSNew + "/SPECTRAL_WINDOW", readonly=False) frequencies = t.getcol("CHAN_FREQ") frequenciesNew = np.fliplr(frequencies) t.putcol("CHAN_FREQ", frequenciesNew) t.close() if (verbose): logging.debug("frequencies:") logging.debug(frequencies) logging.debug("frequencies (updated):") logging.debug(frequenciesNew) # 5.4 logging.info("- Adaptation of field information -") pathMSNewField = pathMSNew + "/FIELD" # Remove metadata of other fields in the FIELD subtable tables.taql( "delete from $pathMSNewField where rownr() not in (select distinct FIELD_ID from $pathMSNew)" ) # Set 'SOURCE_ID' to 0 in the FIELD subtable tables.taql("update $pathMSNewField set SOURCE_ID=0") # Set 'FIELD_ID' to 0 in the main table tables.taql("update $pathMSNew set FIELD_ID=0") # 5.5 logging.info("- Adaptation of intervals -") times = getTimes(pathMSNew) intervalPrecise = times[1] - times[0] # Open the MS as a table in a way that changes can be made. t = tables.table(pathMSNew, readonly=False) intervals = t.getcol("INTERVAL") intervalsNew = np.ones_like(intervals) * intervalPrecise t.putcol("INTERVAL", intervalsNew) t.close() if (verbose): logging.debug("Time intervals (should be equal):") logging.debug(times[1:] - times[:-1]) # 5.6 logging.info( "- Change existing (or create alternative) columns for data, flags and weights -" ) # Open the MS as a table in a way that changes can be made. t = tables.table(pathMSNew, readonly=False) # First, change the 'direction' of the frequency axis. # If the visibilities were sorted in descending frequency order (id est given for high frequencies first), # they are converted to ascending frequency order. Note: applying this operation twice returns the MS to its old state! if (verbose): logging.info("Loading visibilities...") visibilities = t.getcol("DATA") if (verbose): logging.info("Visibilities loaded!") visibilities = np.fliplr(visibilities) visibilitiesNew = np.zeros( (visibilities.shape[0], visibilities.shape[1], 4), dtype=np.complex128) visibilitiesNew[:, :, 0] = visibilities[:, :, 0] visibilitiesNew[:, :, 3] = visibilities[:, :, 1] keywordNames = t.colkeywordnames("DATA") columnDescription = t.getcoldesc("DATA") dataManagerInfo = t.getdminfo("DATA") if (verbose): logging.debug("keywordNames:") logging.debug(keywordNames) logging.debug("columnDescription:") logging.debug(columnDescription) logging.debug("dataManagerInfo:") logging.debug(dataManagerInfo) dataManagerInfo["NAME"] = "TiledDATAMartijn" dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array( [4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array( [4, visibilities.shape[1], visibilities.shape[0]], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array( [4, visibilities.shape[1]], dtype=np.int32) if (verbose): logging.debug("dataManagerInfo (updated):") logging.debug(dataManagerInfo) if (verbose): logging.info("Removing column '" + columnNameVisibilitiesNew + "', if it exists...") if (lib_util.columnExists(t, columnNameVisibilitiesNew)): t.removecols(columnNameVisibilitiesNew) if (verbose): logging.info("Adding column '" + columnNameVisibilitiesNew + "'...") t.addcols( tables.makecoldesc(columnNameVisibilitiesNew, columnDescription), dataManagerInfo) if (verbose): logging.info("Filling column '" + columnNameVisibilitiesNew + "'...") t.putcol(columnNameVisibilitiesNew, visibilitiesNew) if (verbose): logging.info( "Visibilities flipped along frequency axis and placeholder polarisations added!" ) if (verbose): logging.info("Loading flags...") flags = t.getcol("FLAG") if (verbose): logging.info("Flags loaded!") flags = np.fliplr(flags) flagsNew = np.zeros((flags.shape[0], flags.shape[1], 4), dtype=np.bool_) flagsNew[:, :, 0] = flags[:, :, 0] flagsNew[:, :, 1] = flags[:, :, 0] # Take over flags from LL correlation flagsNew[:, :, 2] = flags[:, :, 0] # Take over flags from LL correlation flagsNew[:, :, 3] = flags[:, :, 1] keywordNames = t.colkeywordnames("FLAG") columnDescription = t.getcoldesc("FLAG") dataManagerInfo = t.getdminfo("FLAG") if (verbose): logging.debug("keywordNames:") logging.debug(keywordNames) logging.debug("columnDescription:") logging.debug(columnDescription) logging.debug("dataManagerInfo:") logging.debug(dataManagerInfo) dataManagerInfo["NAME"] = "TiledFlagMartijn" dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array( [4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array( [4, flags.shape[1], flags.shape[0]], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array( [4, flags.shape[1]], dtype=np.int32) if (verbose): logging.debug("dataManagerInfo (updated):") logging.debug(dataManagerInfo) if (verbose): logging.info("Removing column '" + columnNameFlagsNew + "', if it exists...") if (lib_util.columnExists(t, columnNameFlagsNew)): t.removecols(columnNameFlagsNew) if (verbose): logging.info("Adding column '" + columnNameFlagsNew + "'...") t.addcols(tables.makecoldesc(columnNameFlagsNew, columnDescription), dataManagerInfo) if (verbose): logging.info("Filling column '" + columnNameFlagsNew + "'...") t.putcol(columnNameFlagsNew, flagsNew) if (verbose): logging.info( "Flags flipped along frequency axis and placeholder polarisations added!" ) if (verbose): logging.info("Loading weights...") weights = t.getcol("WEIGHT_SPECTRUM") if (verbose): logging.info("Weights loaded!") weights = np.fliplr(weights) weightsNew = np.zeros((weights.shape[0], weights.shape[1], 4), dtype=np.float64) weightsNew[:, :, 0] = weights[:, :, 0] weightsNew[:, :, 3] = weights[:, :, 1] keywordNames = t.colkeywordnames("WEIGHT_SPECTRUM") columnDescription = t.getcoldesc("WEIGHT_SPECTRUM") dataManagerInfo = t.getdminfo("WEIGHT_SPECTRUM") if (verbose): logging.debug("keywordNames:") logging.debug(keywordNames) logging.debug("columnDescription:") logging.debug(columnDescription) logging.debug("dataManagerInfo:") logging.debug(dataManagerInfo) dataManagerInfo["NAME"] = "TiledWgtSpectrumMartijn" dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array( [4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array( [4, weights.shape[1], weights.shape[0]], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array( [4, weights.shape[1]], dtype=np.int32) if (verbose): logging.debug("dataManagerInfo (updated):") logging.debug(dataManagerInfo) if (verbose): logging.info("Removing column '" + columnNameWeightsNew + "', if it exists...") if (lib_util.columnExists(t, columnNameWeightsNew)): t.removecols(columnNameWeightsNew) if (verbose): logging.info("Adding column '" + columnNameWeightsNew + "'...") t.addcols(tables.makecoldesc(columnNameWeightsNew, columnDescription), dataManagerInfo) if (verbose): logging.info("Filling column '" + columnNameWeightsNew + "'...") t.putcol(columnNameWeightsNew, weightsNew) if (verbose): logging.info( "Weights flipped along frequency axis and placeholder polarisations added!" ) t.close() logging.info("Finished work on MS at '" + pathMSNew + "'!") # 6. Move the sub-MSs from a temporary place to their right locus in the file tree. for pathMSNew in pathsMSNew: MSObject = lib_ms.MS(pathMSNew) if (MSObject.isCalibrator()): pathDirectoryCalibrator = pathDirectoryFieldsCalibrator + '/' + MSObject.nameMS pathMSFinal = pathDirectoryCalibrator + '/' + MSObject.nameMS + ".MS" if (not os.path.isdir(pathDirectoryCalibrator)): os.mkdir(pathDirectoryCalibrator) os.mkdir(pathDirectoryCalibrator + "/plots") os.mkdir(pathDirectoryCalibrator + "/solutions") else: pathDirectoryTarget = pathDirectoryFieldsTarget + '/' + MSObject.getNameField( ) pathMSFinal = pathDirectoryTarget + "/MSs/" + MSObject.nameMS + ".MS" if (not os.path.isdir(pathDirectoryTarget)): os.mkdir(pathDirectoryTarget) os.mkdir(pathDirectoryTarget + "/plots") os.mkdir(pathDirectoryTarget + "/MSs") os.mkdir(pathDirectoryTarget + "/images") logging.info("Moving MS at '" + pathMSNew + "' to '" + pathMSFinal + "'...") MSObject.move(pathMSFinal)
def antialias_ms(msname, tol, outputcolname="DATA_AAF"): """ Apply an anti aliasing filter in a parallel way to a measurement set Params: msname (str): Name of measurement set tol (float): Filter response below this limit will be ignored outputcolname (str): Name of column to write corrected visibilities to (will be added to MS if necessary) Returns: None """ logger = logging.getLogger("aaf") logger.info("Anti-aliasing measurement set " + msname) # 1. Open MS and read subtable 'DATA', # if necessary create a new column (default "DATA_AAF") to store AAF corrected data. t1 = datetime.datetime.now() ms = tables.table(msname, readonly=False, ack=False) nrows = ms.nrows() ini_data = tables.tablecolumn(ms, 'DATA') # If there is no column "DATA_AAF", then create one. if outputcolname not in ms.colnames(): coldes = tables.makecoldesc(outputcolname, ms.getcoldesc('DATA')) dmname = ms.getdminfo('DATA') dmname["NAME"] = 'TiledAAFData' ms.addcols(coldes, dminfo=dmname) # 2. Calculate function corr()'s two arguments: subband_response and nconv # Fixed parameters: Number of channels per subband; Total number of subbands num_channels = 64 num_subbands = 1024 # Load filter coefficients, pad with zero coeff = np.loadtxt(file_prefix() + '/share/aaf/Coeffs16384Kaiser-quant.dat') coeff = np.append(coeff, np.zeros(num_channels * num_subbands - coeff.size)) # Get filter frequency response by doing FFT on filter coefficients frequency_response = np.abs(np.fft.fft(coeff)) ** 2 # Scaling frequency_response = frequency_response / np.sum(frequency_response) * num_channels # We only consider aliasing influence from the neighbouring two bands subband_response = np.roll(frequency_response, int(1.5 * num_channels)) subband_response = np.reshape(subband_response[0:3 * num_channels], (3, num_channels)) # Tolerance, filter response below that is ignored # maximum de-convolution length nconv = int(math.ceil(math.log(tol, subband_response[2, 1] / subband_response[1, 1]))) # 3. Do AAF calibration concurrently (parallel) num_cpus = multiprocessing.cpu_count() pool = multiprocessing.Pool(processes=num_cpus - 1) # Here itertools and the function antialias_list() are just bridges between pool.map and function corr(), because pool.map # is not suitable for function that has multi arguments. aafdata = pool.map(antialias_list, itertools.izip(ini_data[0:nrows], itertools.repeat(subband_response), itertools.repeat(nconv))) # 4. Write AAF corrected data to MS, usually the size of data is very large (for example 109746*16384*4 in my # current MS), to avoid Memory Error, we wrote the data by four steps. chunksize = nrows / 4 start_row = np.array([0, chunksize, chunksize * 2, chunksize * 3]) end_row = np.array([chunksize, chunksize * 2, chunksize * 3, nrows]) for parti in range(0, 4): ms.putcol(outputcolname, np.array(aafdata[start_row[parti]:end_row[parti]]), startrow=start_row[parti], nrow=end_row[parti] - start_row[parti]) t2 = datetime.datetime.now() print("Total execution time:", (t2 - t1).total_seconds(), "seconds") log_msg = "Performed anti-aliasing on MS, wrote result to column " + outputcolname pt.taql('INSERT INTO {}::HISTORY SET MESSAGE="{}", APPLICATION="apercal"'.format(msname, log_msg)) logger.info(log_msg)
def updateColumns(self, updateFreq): ''' Update DATA, FLAG and WEIGHT_SPECTRUM columns. ''' logging.info( "- Change existing (or create alternative) columns for data, flags and weights -" ) logging.info("Loading visibilities...") visibilities = self.t.getcol("DATA") if ( updateFreq ): # If the frequency channel order was flipped in 'updateFreqMetadata', also flip the data. visibilities = visibilities[:, ::-1, :] if (visibilities.shape[2] == 2): visibilitiesNew = np.zeros( (visibilities.shape[0], visibilities.shape[1], 4), dtype=np.complex128) visibilitiesNew[:, :, 0] = visibilities[:, :, 0] visibilitiesNew[:, :, 3] = visibilities[:, :, 1] visibilities = visibilitiesNew keywordNames = self.t.colkeywordnames("DATA") columnDescription = self.t.getcoldesc("DATA") dataManagerInfo = self.t.getdminfo("DATA") dataManagerInfo["NAME"] = "TiledDATAMartijn" dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array( [4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array( [4, visibilities.shape[1], visibilities.shape[0]], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array( [4, visibilities.shape[1]], dtype=np.int32) #logging.debug("keywordNames:") #logging.debug(keywordNames) #logging.debug("columnDescription:") #logging.debug(columnDescription) #logging.debug("dataManagerInfo:") #logging.debug(dataManagerInfo) #logging.debug("dataManagerInfo (updated):") #logging.debug(dataManagerInfo) logging.info("Removing column 'DATA', if it exists...") if (self.columnExists('DATA')): self.t.removecols('DATA') logging.info("Adding column 'DATA'...") self.t.addcols(tables.makecoldesc('DATA', columnDescription), dataManagerInfo) logging.info("Filling column 'DATA'...") self.t.putcol('DATA', visibilities) logging.info("Loading flags...") flags = self.t.getcol("FLAG") if ( updateFreq ): # If the frequency channel order was flipped in 'updateFreqMetadata', also flip the data. flags = flags[:, ::-1, :] if (flags.shape[2] == 2): flagsNew = np.zeros((flags.shape[0], flags.shape[1], 4), dtype=np.bool_) flagsNew[:, :, 0] = flags[:, :, 0] flagsNew[:, :, 1] = flags[:, :, 0] # Take over flags from LL correlation flagsNew[:, :, 2] = flags[:, :, 0] # Take over flags from LL correlation flagsNew[:, :, 3] = flags[:, :, 1] flags = flagsNew keywordNames = self.t.colkeywordnames("FLAG") columnDescription = self.t.getcoldesc("FLAG") dataManagerInfo = self.t.getdminfo("FLAG") dataManagerInfo["NAME"] = "TiledFlagMartijn" dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array( [4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array( [4, flags.shape[1], flags.shape[0]], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array( [4, flags.shape[1]], dtype=np.int32) #logging.debug("keywordNames:") #logging.debug(keywordNames) #logging.debug("columnDescription:") #logging.debug(columnDescription) #logging.debug("dataManagerInfo:") #logging.debug(dataManagerInfo) #logging.debug("dataManagerInfo (updated):") #logging.debug(dataManagerInfo) logging.info("Removing column 'FLAG', if it exists...") if (self.columnExists('FLAG')): self.t.removecols('FLAG') logging.info("Adding column 'FLAG'...") self.t.addcols(tables.makecoldesc('FLAG', columnDescription), dataManagerInfo) logging.info("Filling column 'FLAG'...") self.t.putcol('FLAG', flags) logging.info("Loading weights...") weights = self.t.getcol("WEIGHT_SPECTRUM") if ( updateFreq ): # If the frequency channel order was flipped in 'updateFreqMetadata', also flip the data. weights = weights[:, ::-1, :] if (weights.shape[2] == 2): weightsNew = np.zeros((weights.shape[0], weights.shape[1], 4), dtype=np.float64) weightsNew[:, :, 0] = weights[:, :, 0] weightsNew[:, :, 3] = weights[:, :, 1] weights = weightsNew keywordNames = self.t.colkeywordnames("WEIGHT_SPECTRUM") columnDescription = self.t.getcoldesc("WEIGHT_SPECTRUM") dataManagerInfo = self.t.getdminfo("WEIGHT_SPECTRUM") dataManagerInfo["NAME"] = "TiledWgtSpectrumMartijn" dataManagerInfo["SPEC"]["DEFAULTTILESHAPE"] = np.array([4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["TileShape"] = np.array( [4, 40, 819], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CubeShape"] = np.array( [4, weights.shape[1], weights.shape[0]], dtype=np.int32) dataManagerInfo["SPEC"]["HYPERCUBES"]["*1"]["CellShape"] = np.array( [4, weights.shape[1]], dtype=np.int32) #logging.debug("keywordNames:") #logging.debug(keywordNames) #logging.debug("columnDescription:") #logging.debug(columnDescription) #logging.debug("dataManagerInfo:") #logging.debug(dataManagerInfo) #logging.debug("dataManagerInfo (updated):") #logging.debug(dataManagerInfo) logging.info("Removing column 'WEIGHT_SPECTRUM', if it exists...") if (self.columnExists('WEIGHT_SPECTRUM')): self.t.removecols('WEIGHT_SPECTRUM') logging.info("Adding column 'WEIGHT_SPECTRUM'...") self.t.addcols( tables.makecoldesc('WEIGHT_SPECTRUM', columnDescription), dataManagerInfo) logging.info("Filling column 'WEIGHT_SPECTRUM'...") self.t.putcol('WEIGHT_SPECTRUM', weights)