def addcol(msname='$MS',colname=None,shape=None, data_desc_type='array',valuetype=None,init_with=0,**kw): """ add column to MS msanme : MS to add colmn to colname : column name shape : shape valuetype : data type data_desc_type : 'scalar' for scalar elements and array for 'array' elements init_with : value to initialise the column with """ msname, colname = interpolate_locals('msname colname') tab = pyrap.tables.table(msname,readonly=False) try: tab.getcol(colname) info('Column already exists') except RuntimeError: info('Attempting to add %s column to %s'%(colname,msname)) from pyrap.tables import maketabdesc valuetype = valuetype or 'complex' if shape is None: dshape = list(tab.getcol('DATA').shape) shape = dshape[1:] if data_desc_type=='array': from pyrap.tables import makearrcoldesc coldmi = tab.getdminfo('DATA') # God forbid this (or the TIME) column doesn't exist coldmi['NAME'] = colname.lower() tab.addcols(maketabdesc(makearrcoldesc(colname,init_with,shape=shape,valuetype=valuetype)),coldmi) elif data_desc_type=='scalar': from pyrap.tables import makescacoldesc coldmi = tab.getdminfo('TIME') coldmi['NAME'] = colname.lower() tab.addcols(maketabdesc(makescacoldesc(colname,init_with,valuetype=valuetype)),coldmi) info('Column added successfuly.') if init_with: nrows = dshape[0] rowchunk = nrows//10 if nrows > 1000 else nrows for row0 in range(0,nrows,rowchunk): nr = min(rowchunk,nrows-row0) dshape[0] = nr tab.putcol(colname,numpy.ones(dshape,dtype=valuetype)*init_with,row0,nr) tab.close()
def main(options): inms = options.inms if inms == '': print 'Error: you have to specify an input MS, use -h for help' return outcol = options.outcol t = pt.table(inms, readonly=False, ack=True) if outcol not in t.colnames(): print 'Adding the output column', outcol, 'to', inms coldmi = t.getdminfo('DATA') coldmi['NAME'] = outcol t.addcols( pt.maketabdesc( pt.makearrcoldesc(outcol, 0., valuetype='complex', shape=numpy.array(t.getcell('DATA', 0)).shape)), coldmi) data = t.getcol('DATA') t.putcol(outcol, data) else: print outcol, 'column already exist'
def test_fixed_column_shapes(tmp_path, column, dtype): """ Fixed column, shape and ndim must be supplied """ casa_type = infer_casa_type(dtype) # Column descriptor desc = {'desc': {'_c_order': True, 'comment': '%s column' % column, 'dataManagerGroup': '', 'dataManagerType': '', 'keywords': {}, 'ndim': 2, 'shape': (20, 30), 'maxlen': 0, 'option': 0, 'valueType': casa_type}, 'name': column} table_desc = pt.maketabdesc([desc]) fn = os.path.join(str(tmp_path), "test.table") with pt.table(fn, table_desc, nrow=10, ack=False) as T: # Put 10 rows into the table T.putcol(column, np.zeros((10, 20, 30), dtype=dtype)) assert T.getcol(column).shape == (10, 20, 30) # Must be ndim == 2 err_str = "Table array conformance error" with pytest.raises(RuntimeError, match=err_str): T.putcol(column, np.zeros((5, 40), dtype=dtype)) # shape != (20, 30) with pytest.raises(RuntimeError, match=err_str): T.putcol(column, np.zeros((5, 40, 30), dtype=dtype), startrow=0, nrow=5)
def main(options): cI = numpy.complex(0.,1.) inms = options.inms if inms == '': print 'Error: you have to specify an input MS, use -h for help' return column = options.column outcol = options.outcol t = pt.table(inms, readonly=False, ack=True) if outcol not in t.colnames(): print 'Adding output column',outcol,'to',inms coldmi = t.getdminfo(column) coldmi['NAME'] = outcol t.addcols(pt.maketabdesc(pt.makearrcoldesc(outcol, 0., valuetype='complex', shape=numpy.array(t.getcell(column,0)).shape)), coldmi) print 'Reading input column' data = t.getcol(column) print 'Computing output column' outdata = numpy.transpose(numpy.array([ data[:,:,0]-cI*data[:,:,1]+cI*data[:,:,2]+data[:,:,3], data[:,:,0]+cI*data[:,:,1]+cI*data[:,:,2]-data[:,:,3], data[:,:,0]-cI*data[:,:,1]-cI*data[:,:,2]-data[:,:,3], data[:,:,0]+cI*data[:,:,1]-cI*data[:,:,2]+data[:,:,3]]), (1,2,0)) print 'Finishing up' t.putcol(outcol, outdata)
def test_only_row_shape(tmp_path, column, dtype): """ Missing ndim implies row only! """ casa_type = infer_casa_type(dtype) # Column descriptor desc = {'desc': {'_c_order': True, 'comment': '%s column' % column, 'dataManagerGroup': '', 'dataManagerType': '', 'keywords': {}, 'maxlen': 0, 'option': 0, 'valueType': casa_type}, 'name': column} table_desc = pt.maketabdesc([desc]) fn = os.path.join(str(tmp_path), "test.table") with pt.table(fn, table_desc, nrow=10, ack=False) as T: # Put 10 rows into the table T.putcol(column, np.zeros(10, dtype=dtype)) assert T.getcol(column).shape == (10,) # Must be ndim == 2 err_str = 'Vector<T>: ndim of other array > 1 ndim 1 differs from 2' with pytest.raises(RuntimeError, match=err_str): T.putcol(column, np.zeros((5, 40), dtype=dtype)) # shape != (20, 30) err_str = 'Vector<T>: ndim of other array > 1 ndim 1 differs from 3' with pytest.raises(RuntimeError, match=err_str): T.putcol(column, np.zeros((5, 40, 30), dtype=dtype), startrow=0, nrow=5)
def test_scalar_ndim(tmp_path, column, dtype): """ ndim set to zero implies scalars """ casa_type = infer_casa_type(dtype) # Column descriptor desc = { 'desc': { '_c_order': True, 'comment': f'{column} column', 'dataManagerGroup': '', 'dataManagerType': '', 'keywords': {}, 'maxlen': 0, 'ndim': 0, 'option': 0, 'valueType': casa_type }, 'name': column } table_desc = pt.maketabdesc([desc]) fn = os.path.join(str(tmp_path), "test.table") with pt.table(fn, table_desc, nrow=10, ack=False) as T: for r in range(10): T.putcell(column, r, r) for r in range(10): assert T.getcell(column, r) == r
def addRefColumnToTesttab(self, columnname): if self.verbose: print(bcolors.OKBLUE + "Forwarding reference column " + bcolors.WARNING + columnname + bcolors.OKBLUE + " to " + self.test_MS + bcolors.ENDC) # DEBUG testtab = pt.table(self.test_MS, readonly=False) # Open test_MS in readonly mode reftab = pt.table(self.MS) # Open reference MS in readonly mode # Get column description from testtab testcolumnname = "test_" + columnname testtab.renamecol( columnname, testcolumnname) # rename the existing column in the test table refcol_desc = reftab.getcoldesc(columnname) # Use ForwardColumnEngine to refer column in testtab to reftab columnname testtab.addcols(pt.maketabdesc( [pt.makecoldesc(columnname, refcol_desc)]), dminfo={ '1': { 'TYPE': 'ForwardColumnEngine', 'NAME': 'ForwardData', 'COLUMNS': [columnname], 'SPEC': { 'FORWARDTABLE': reftab.name() } } }) testtab.flush()
def main(options): ms = options.ms if ms == '': print 'Error: you have to specify an input MS, use -h for help' return cols = options.cols incol = options.incol t = pt.table(ms, readonly=False, ack=False) for col in cols.split(','): if col not in t.colnames(): logging.info('Adding the output column '+col+' to '+ms+'.') if incol == '': incol = 'DATA' update = False else: update = True coldmi = t.getdminfo(incol) coldmi['NAME'] = col t.addcols(pt.maketabdesc(pt.makearrcoldesc(col, 0., valuetype=t.col(incol).datatype(), shape=numpy.array(t.getcell(incol,0)).shape)), coldmi) if update: logging.warning('Setting '+col+' = '+incol+'.') t.putcol(col, t.getcol(incol)) else: logging.warning('Column '+col+' already exists.') t.close()
def test_variable_column_dimensions(tmp_path, column, dtype): """ ndim set to -1, we can put anything in the column """ casa_type = infer_casa_type(dtype) # Column descriptor desc = {'desc': {'_c_order': True, 'comment': '%s column' % column, 'dataManagerGroup': '', 'dataManagerType': '', 'keywords': {}, # This allows any shape to go in 'ndim': -1, 'maxlen': 0, 'option': 0, 'valueType': casa_type}, 'name': column} table_desc = pt.maketabdesc([desc]) fn = os.path.join(str(tmp_path), "test.table") with pt.table(fn, table_desc, nrow=10, ack=False) as T: # Put 10 rows into the table T.putcol(column, np.zeros((10, 20, 30), dtype=dtype)) assert T.getcol(column).shape == (10, 20, 30) # Put something differently shaped in the first 5 rows T.putcol(column, np.zeros((5, 40), dtype=dtype), startrow=0, nrow=5) assert T.getcol(column, startrow=0, nrow=5).shape == (5, 40) # The last 5 rows have the original shape assert T.getcol(column, startrow=5, nrow=5).shape == (5, 20, 30) # We can even put a scalar in T.putcell(column, 8, 3) assert T.getcell(column, 8) == 3
def updateObsTable(image, msName, minbl, maxbl, aswvl, usedCounts, visCounts, minTime, maxTime, totTime): obstab = pt.table(image.name() + "/LOFAR_OBSERVATION", readonly=False, ack=False) oritab = pt.table(image.name() + "/LOFAR_ORIGIN", ack=False) minfreq = pt.taql("calc min([select FREQUENCY_MIN from '" + oritab.name() + "'])") maxfreq = pt.taql("calc max([select FREQUENCY_MAX from '" + oritab.name() + "'])") obstab.putcell("OBSERVATION_FREQUENCY_MIN", 0, minfreq[0]) obstab.putcell("OBSERVATION_FREQUENCY_MAX", 0, maxfreq[0]) obstab.putcell("OBSERVATION_FREQUENCY_CENTER", 0, (minfreq[0] + maxfreq[0]) / 2) obstab.putcell("OBSERVATION_INTEGRATION_TIME", 0, totTime) obstab.putcell("OBSERVATION_START", 0, minTime) obstab.putcell("OBSERVATION_END", 0, maxTime) obstab.putcell("TIME_RANGE", 0, (minTime, maxTime)) obstab.putcell("FILENAME", 0, os.path.basename(image.name())) obstab.putcell("FILETYPE", 0, "sky") pt.taql("update '" + obstab.name() + "' set FILEDATE = mjd(date()), " + "RELEASE_DATE = mjd(date()+365)") # Determine minimum and maximum baseline length # If needed, convert from wavelengths to meters. mstab = pt.table(msName, ack=False) if aswvl: minbl *= 2.99792e8 / maxfreq[0] maxbl *= 2.99792e8 / minfreq[0] if minbl <= 0: mbl = pt.taql("calc sqrt(min([select sumsqr(UVW[:2]) from " + msName + "]))") minbl = max(mbl[0], abs(minbl)) if maxbl <= 0: mbl = pt.taql("calc sqrt(max([select sumsqr(UVW[:2]) from " + msName + "]))") if maxbl == 0: maxbl = mbl[0] else: maxbl = min(mbl[0], abs(maxbl)) mstab.close() # Add and fill a few extra columns. col1 = pt.makescacoldesc("MIN_BASELINE_LENGTH", 0, valuetype='double') col2 = pt.makescacoldesc("MAX_BASELINE_LENGTH", 0, valuetype='double') col3 = pt.makearrcoldesc("NVIS_USED", 0, valuetype='int') col4 = pt.makearrcoldesc("NVIS_TOTAL", 0, valuetype='int') obstab.addcols(pt.maketabdesc([col1, col2, col3, col4])) obstab.putcolkeyword("MIN_BASELINE_LENGTH", "QuantumUnits", ["m"]) obstab.putcolkeyword("MAX_BASELINE_LENGTH", "QuantumUnits", ["m"]) obstab.putcell("MIN_BASELINE_LENGTH", 0, minbl) obstab.putcell("MAX_BASELINE_LENGTH", 0, maxbl) # Get sum for all MSs. tusedCounts = usedCounts.sum(axis=0) tvisCounts = visCounts.sum(axis=0) obstab.putcell("NVIS_USED", 0, tusedCounts) obstab.putcell("NVIS_TOTAL", 0, tvisCounts) obstab.close() oritab.close() print "Updated subtable LOFAR_OBSERVATION"
def addcol(ms, incol, outcol): if outcol not in ms.colnames(): logging.info('Adding column: '+outcol) coldmi = ms.getdminfo(incol) coldmi['NAME'] = outcol datatype = ms.col(incol).datatype() ms.addcols(pt.maketabdesc(pt.makearrcoldesc(outcol, 0., valuetype=datatype, shape=np.array(ms.getcell(incol,0)).shape)), coldmi) if outcol != incol: # copy columns val logging.info('Set '+outcol+'='+incol) ms.putcol(outcol, ms.getcol(incol))
def updateObsTable (image, msName, minbl, maxbl, aswvl, usedCounts, visCounts, minTime, maxTime, totTime): obstab = pt.table (image.name() + "/LOFAR_OBSERVATION", readonly=False, ack=False) oritab = pt.table (image.name() + "/LOFAR_ORIGIN", ack=False) minfreq = pt.taql ("calc min([select FREQUENCY_MIN from '" + oritab.name() + "'])") maxfreq = pt.taql ("calc max([select FREQUENCY_MAX from '" + oritab.name() + "'])") obstab.putcell ("OBSERVATION_FREQUENCY_MIN", 0, minfreq[0]); obstab.putcell ("OBSERVATION_FREQUENCY_MAX", 0, maxfreq[0]); obstab.putcell ("OBSERVATION_FREQUENCY_CENTER", 0, (minfreq[0]+maxfreq[0])/2); obstab.putcell ("OBSERVATION_INTEGRATION_TIME", 0, totTime); obstab.putcell ("OBSERVATION_START", 0, minTime); obstab.putcell ("OBSERVATION_END", 0, maxTime); obstab.putcell ("TIME_RANGE", 0, (minTime, maxTime)); obstab.putcell ("FILENAME", 0, os.path.basename(image.name())) obstab.putcell ("FILETYPE", 0, "sky") pt.taql ("update '" + obstab.name() + "' set FILEDATE = mjd(date()), " + "RELEASE_DATE = mjd(date()+365)") # Determine minimum and maximum baseline length # If needed, convert from wavelengths to meters. mstab = pt.table(msName, ack=False) if aswvl: minbl *= 2.99792e8 / maxfreq[0] maxbl *= 2.99792e8 / minfreq[0] if minbl <= 0: mbl = pt.taql ("calc sqrt(min([select sumsqr(UVW[:2]) from " + msName + "]))") minbl = max(mbl[0], abs(minbl)) if maxbl <= 0: mbl = pt.taql ("calc sqrt(max([select sumsqr(UVW[:2]) from " + msName + "]))") if maxbl == 0: maxbl = mbl[0] else: maxbl = min(mbl[0], abs(maxbl)) mstab.close() # Add and fill a few extra columns. col1 = pt.makescacoldesc ("MIN_BASELINE_LENGTH", 0, valuetype='double') col2 = pt.makescacoldesc ("MAX_BASELINE_LENGTH", 0, valuetype='double') col3 = pt.makearrcoldesc ("NVIS_USED", 0, valuetype='int') col4 = pt.makearrcoldesc ("NVIS_TOTAL", 0, valuetype='int') obstab.addcols (pt.maketabdesc ([col1, col2, col3, col4])) obstab.putcolkeyword ("MIN_BASELINE_LENGTH", "QuantumUnits", ["m"]) obstab.putcolkeyword ("MAX_BASELINE_LENGTH", "QuantumUnits", ["m"]) obstab.putcell ("MIN_BASELINE_LENGTH", 0, minbl) obstab.putcell ("MAX_BASELINE_LENGTH", 0, maxbl) # Get sum for all MSs. tusedCounts = usedCounts.sum (axis=0) tvisCounts = visCounts.sum (axis=0) obstab.putcell ("NVIS_USED", 0, tusedCounts) obstab.putcell ("NVIS_TOTAL", 0, tvisCounts) obstab.close() oritab.close() print "Updated subtable LOFAR_OBSERVATION"
def addRefColumnToTesttab(self, columnname): if self.verbose: print bcolors.OKBLUE + "Forwarding reference column " + bcolors.WARNING + columnname + bcolors.OKBLUE + " to " + self.test_MS + bcolors.ENDC # DEBUG testtab=pt.table(self.test_MS, readonly=False) # Open test_MS in readonly mode reftab=pt.table(self.MS) # Open reference MS in readonly mode # Get column description from testtab testcolumnname = "test_" + columnname testtab.renamecol(columnname, testcolumnname) # rename the existing column in the test table refcol_desc=reftab.getcoldesc(columnname) # Use ForwardColumnEngine to refer column in testtab to reftab columnname testtab.addcols(pt.maketabdesc([pt.makecoldesc(columnname, refcol_desc)]), dminfo={'1':{'TYPE':'ForwardColumnEngine', 'NAME':'ForwardData', 'COLUMNS':[columnname], 'SPEC':{'FORWARDTABLE':reftab.name()}}}) testtab.flush()
def main(options): inms = options.inms if inms == '': print 'Error: you have to specify an input MS, use -h for help' return outcol = options.outcol t = pt.table(inms, readonly=False, ack=True) if outcol not in t.colnames(): print 'Adding the output column',outcol,'to',inms coldmi = t.getdminfo('WEIGHT_SPECTRUM') coldmi['NAME'] = outcol t.addcols(pt.maketabdesc(pt.makearrcoldesc(outcol, 0., valuetype='float', shape=numpy.array(t.getcell('WEIGHT_SPECTRUM',0)).shape)), coldmi) data = t.getcol('WEIGHT_SPECTRUM') t.putcol(outcol, data) else: print outcol, 'column already exist'
def test_tiledstman(tmp_path, column, row, shape, dtype): casa_type = infer_casa_type(dtype) # Column descriptor desc = { 'desc': { '_c_order': True, 'comment': f'{column} column', 'dataManagerGroup': 'BAZ-GROUP', 'dataManagerType': 'TiledColumnStMan', 'keywords': {}, 'maxlen': 0, 'ndim': len(shape), 'option': 0, 'shape': shape, 'valueType': casa_type }, 'name': column } table_desc = pt.maketabdesc([desc]) tile_shape = tuple(reversed(shape)) + (row, ) dminfo = { '*1': { 'NAME': 'BAZ-GROUP', 'TYPE': 'TiledColumnStMan', 'SPEC': { 'DEFAULTTILESHAPE': tile_shape }, 'COLUMNS': ['BAZ'], } } fn = os.path.join(str(tmp_path), "test.table") with pt.table(fn, table_desc, dminfo=dminfo, nrow=10, ack=False) as T: dmg = T.getdminfo()['*1'] assert dmg['NAME'] == 'BAZ-GROUP' assert_array_equal(dmg['SPEC']['DEFAULTTILESHAPE'], tile_shape) assert dmg['TYPE'] == 'TiledColumnStMan' assert dmg['COLUMNS'] == ['BAZ']
def test_variable_column_shapes(tmp_path, column, dtype): """ ndim set to 2, but shapes are variable """ casa_type = infer_casa_type(dtype) # Column descriptor desc = { 'desc': { '_c_order': True, 'comment': f'{column} column', 'dataManagerGroup': '', 'dataManagerType': '', 'keywords': {}, 'ndim': 2, 'maxlen': 0, 'option': 0, 'valueType': casa_type }, 'name': column } table_desc = pt.maketabdesc([desc]) fn = os.path.join(str(tmp_path), "test.table") with pt.table(fn, table_desc, nrow=10, ack=False) as T: # Put 10 rows into the table T.putcol(column, np.zeros((10, 20, 30), dtype=dtype)) assert T.getcol(column).shape == (10, 20, 30) # Must be ndim == 2 err_str = "Table array conformance error" with pytest.raises(RuntimeError, match=err_str): T.putcol(column, np.zeros((5, 40), dtype=dtype)) # Put something differently shaped in the first 5 rows T.putcol(column, np.zeros((5, 40, 30), dtype=dtype), startrow=0, nrow=5) assert T.getcol(column, startrow=0, nrow=5).shape == (5, 40, 30) # The last 5 rows have the original shape assert T.getcol(column, startrow=5, nrow=5).shape == (5, 20, 30)
def test_variable_column_descriptor(chunks, dtype, tmp_path): column_meta = [] shapes = {k: sum(c) for k, c in chunks.items()} # Make some visibilities dims = ("row", "chan", "corr") shape = tuple(shapes[d] for d in dims) data_chunks = tuple(chunks[d] for d in dims) data = da.random.random(shape, chunks=data_chunks).astype(dtype) data_var = Variable(dims, data, {}) meta = variable_column_descriptor("DATA", data_var) column_meta.append({"name": "DATA", "desc": meta}) # Make some string names dims = ("row", ) shape = tuple(shapes[d] for d in dims) str_chunks = tuple(chunks[d] for d in dims) np_str_array = np.asarray(["BOB"] * shape[0], dtype=np.object) da_str_array = da.from_array(np_str_array, chunks=str_chunks) str_array_var = Variable(dims, da_str_array, {}) meta = variable_column_descriptor("NAMES", str_array_var) column_meta.append({"name": "NAMES", "desc": meta}) # Create a new table with the column metadata fn = os.path.join(str(tmp_path), "test.ms") tabdesc = pt.maketabdesc(column_meta) with pt.table(fn, tabdesc, readonly=False, ack=False) as T: # Add rows T.addrows(shapes['row']) str_list = np_str_array.tolist() # Put data T.putcol("DATA", data.compute()) T.putcol("NAMES", str_list) # We get out what we put in assert_array_equal(T.getcol("NAMES"), str_list) assert_array_equal(T.getcol("DATA"), data)
def export_ms(hdf_file, ms_file, verbosity=1): """ Convert an HDF file to MS :param hdf_file: Input HDF-MS filename :param ms_file: Output MS filename TODO: Get this working properly. """ pp = PrintLog(verbosity=verbosity) hdul = IdiHdulist(verbosity=1) hdul.read_hdf("testms.h5") main_hdu = hdul["MAIN"] vdict = { 'float32': 'float', 'float64': 'double', 'complex64': 'complex', 'complex128': 'dcomplex', 'int32': 'int', 'uint32': 'uint', 'str': 'string', 'bool': 'bool' } col_descs = [] for col, cdata in main_hdu.data.items(): col = str(col) pp.pp("%16s %s %s" % (col, cdata.shape, cdata.dtype)) if cdata.ndim == 1: vt = vdict[str(cdata.dtype)] cdesc = pt.makescacoldesc(col, cdata[0], valuetype=vt) else: cdesc = pt.makearrcoldesc(col, cdata[0], valuetype=vt) col_descs.append(cdesc) tdesc = pt.maketabdesc(col_descs) t = pt.table("table.ms", tdesc, nrow=main_hdu.n_rows)
def main(options): cI = numpy.complex(0., 1.) inms = options.inms if inms == '': print 'Error: you have to specify an input MS, use -h for help' return column = options.column outcol = options.outcol t = pt.table(inms, readonly=False, ack=True) if outcol not in t.colnames(): print 'Adding output column', outcol, 'to', inms coldmi = t.getdminfo(column) coldmi['NAME'] = outcol t.addcols( pt.maketabdesc( pt.makearrcoldesc(outcol, 0., valuetype='complex', shape=numpy.array(t.getcell(column, 0)).shape)), coldmi) print 'Reading input column' data = t.getcol(column) print 'Computing output column' outdata = numpy.transpose( numpy.array([ data[:, :, 0] - cI * data[:, :, 1] + cI * data[:, :, 2] + data[:, :, 3], data[:, :, 0] + cI * data[:, :, 1] + cI * data[:, :, 2] - data[:, :, 3], data[:, :, 0] - cI * data[:, :, 1] - cI * data[:, :, 2] - data[:, :, 3], data[:, :, 0] + cI * data[:, :, 1] - cI * data[:, :, 2] + data[:, :, 3] ]), (1, 2, 0)) print 'Finishing up' t.putcol(outcol, outdata)
def export_ms(hdf_file, ms_file, verbosity=1): """ Convert an HDF file to MS :param hdf_file: Input HDF-MS filename :param ms_file: Output MS filename TODO: Get this working properly. """ pp = PrintLog(verbosity=verbosity) hdul = IdiHdulist(verbosity=1) hdul.read_hdf("testms.h5") main_hdu = hdul["MAIN"] vdict = {'float32' : 'float', 'float64' : 'double', 'complex64' : 'complex', 'complex128' : 'dcomplex', 'int32' : 'int', 'uint32' : 'uint', 'str' : 'string', 'bool' : 'bool' } col_descs = [] for col, cdata in main_hdu.data.items(): col = str(col) pp.pp("%16s %s %s" % (col, cdata.shape, cdata.dtype)) if cdata.ndim == 1: vt = vdict[str(cdata.dtype)] cdesc = pt.makescacoldesc(col, cdata[0], valuetype=vt) else: cdesc = pt.makearrcoldesc(col, cdata[0], valuetype=vt) col_descs.append(cdesc) tdesc = pt.maketabdesc(col_descs) t = pt.table("table.ms", tdesc, nrow=main_hdu.n_rows)
def addcol(msname, colname=None, shape=None, data_desc_type='array', valuetype=None, init_with=None, coldesc=None, coldmi=None, clone='DATA', rowchunk=None, **kw): """ Add column to MS msanme : MS to add colmn to colname : column name shape : shape valuetype : data type data_desc_type : 'scalar' for scalar elements and array for 'array' elements init_with : value to initialise the column with """ tab = table(msname, readonly=False) if colname in tab.colnames(): print('Column already exists') return 'exists' print('Attempting to add %s column to %s' % (colname, msname)) valuetype = valuetype or 'complex' if coldesc: data_desc = coldesc shape = coldesc['shape'] elif shape: data_desc = maketabdesc( makearrcoldesc(colname, init_with, shape=shape, valuetype=valuetype)) elif valuetype == 'scalar': data_desc = maketabdesc( makearrcoldesc(colname, init_with, valuetype=valuetype)) elif clone: element = tab.getcell(clone, 0) try: shape = element.shape data_desc = maketabdesc( makearrcoldesc(colname, element.flatten()[0], shape=shape, valuetype=valuetype)) except AttributeError: shape = [] data_desc = maketabdesc( makearrcoldesc(colname, element, valuetype=valuetype)) colinfo = [data_desc, coldmi] if coldmi else [data_desc] tab.addcols(*colinfo) print('Column added successfuly.') if init_with is None: tab.close() return 'added' else: spwids = set(tab.getcol('DATA_DESC_ID')) for spw in spwids: print('Initialising {0:s} column with {1}. DDID is {2:d}'.format( colname, init_with, spw)) tab_spw = tab.query('DATA_DESC_ID=={0:d}'.format(spw)) nrows = tab_spw.nrows() rowchunk = rowchunk or nrows / 10 dshape = [0] + [a for a in shape] for row0 in range(0, nrows, rowchunk): nr = min(rowchunk, nrows - row0) dshape[0] = nr print("Wrtiting to column %s (rows %d to %d)" % (colname, row0, row0 + nr - 1)) tab_spw.putcol( colname, numpy.ones(dshape, dtype=type(init_with)) * init_with, row0, nr) tab_spw.close() tab.close() return 'added'
def main(options): cI = numpy.complex(0., 1.) inms = options.inms if inms == '': print 'Error: you have to specify an input MS, use -h for help' return column = options.column outcol = options.outcol t = pt.table(inms, readonly=False, ack=True) if options.back: lincol = options.lincol if lincol not in t.colnames(): print 'Adding the output linear polarization column', lincol, 'to', inms coldmi = t.getdminfo(column) coldmi['NAME'] = lincol t.addcols( pt.maketabdesc( pt.makearrcoldesc(lincol, 0., valuetype='complex', shape=numpy.array(t.getcell(column, 0)).shape)), coldmi) ### RVW EDIT 2012 print 'Reading the input column (circular)', column if column not in t.colnames(): print 'Error: Input column does not exist' return ### RVW EDIT 2012 Input column with the -c switch cirdata = t.getcol(column) #cirdata = t.getcol(outcol) #print 'SHAPE ARRAY', numpy.shape(cirdata) print 'Computing the linear polarization terms...' lindata = numpy.transpose( numpy.array([ 0.5 * (cirdata[:, :, 0] + cirdata[:, :, 1] + cirdata[:, :, 2] + cirdata[:, :, 3]), 0.5 * (cI * cirdata[:, :, 0] - cI * cirdata[:, :, 1] + cI * cirdata[:, :, 2] - cI * cirdata[:, :, 3]), 0.5 * (-cI * cirdata[:, :, 0] - cI * cirdata[:, :, 1] + cI * cirdata[:, :, 2] + cI * cirdata[:, :, 3]), 0.5 * (cirdata[:, :, 0] - cirdata[:, :, 1] - cirdata[:, :, 2] + cirdata[:, :, 3]) ]), (1, 2, 0)) print 'Finishing up...' t.putcol(lincol, lindata) else: if outcol not in t.colnames(): print 'Adding the output column', outcol, 'to', inms coldmi = t.getdminfo(column) coldmi['NAME'] = outcol t.addcols( pt.maketabdesc( pt.makearrcoldesc(outcol, 0., valuetype='complex', shape=numpy.array(t.getcell(column, 0)).shape)), coldmi) print 'Reading the input column (linear)', column data = t.getcol(column) print 'Computing the output column' outdata = numpy.transpose( numpy.array([ 0.5 * (data[:, :, 0] - cI * data[:, :, 1] + cI * data[:, :, 2] + data[:, :, 3]), 0.5 * (data[:, :, 0] + cI * data[:, :, 1] + cI * data[:, :, 2] - data[:, :, 3]), 0.5 * (data[:, :, 0] - cI * data[:, :, 1] - cI * data[:, :, 2] - data[:, :, 3]), 0.5 * (data[:, :, 0] + cI * data[:, :, 1] - cI * data[:, :, 2] + data[:, :, 3]) ]), (1, 2, 0)) print 'Finishing up...' t.putcol(outcol, outdata) if options.poltable: print 'Updating the POLARIZATION table...' tp = pt.table(inms + '/POLARIZATION', readonly=False, ack=True) ### RVW EDIT 2012 if options.back: tp.putcol('CORR_TYPE', numpy.array([[9, 10, 11, 12]], dtype=numpy.int32)) # FROM CIRC-->LIN else: tp.putcol('CORR_TYPE', numpy.array([[5, 6, 7, 8]], dtype=numpy.int32)) # FROM LIN-->CIRC
def test_tiledstman_addcols(tmp_path, column, row, shape, dtype): """ ndim set to zero implies scalars """ casa_type = infer_casa_type(dtype) # Column descriptor desc = { 'desc': { '_c_order': True, 'comment': f'{column} column', 'dataManagerGroup': 'BAZ_GROUP', 'dataManagerType': 'TiledColumnStMan', 'keywords': {}, 'maxlen': 0, 'ndim': len(shape), 'option': 0, 'shape': shape, 'valueType': casa_type }, 'name': column } table_desc = pt.maketabdesc([desc]) tile_shape = tuple(reversed(shape)) + (row, ) dminfo = { '*1': { 'NAME': 'BAZ_GROUP', 'TYPE': 'TiledColumnStMan', 'SPEC': { 'DEFAULTTILESHAPE': tile_shape }, 'COLUMNS': ['BAZ'], } } fn = os.path.join(str(tmp_path), "test.table") with pt.table(fn, table_desc, dminfo=dminfo, nrow=10, ack=False) as T: # Add a new FRED column desc = { 'FRED': { 'dataManagerGroup': 'FRED_GROUP', 'dataManagerType': 'TiledColumnStMan', 'ndim': len(shape), 'shape': shape, 'valueType': casa_type } } dminfo = { '*1': { 'NAME': 'FRED_GROUP', 'TYPE': 'TiledColumnStMan', 'SPEC': { 'DEFAULTTILESHAPE': tile_shape }, 'COLUMNS': ['FRED'], } } T.addcols(desc, dminfo=dminfo) # Trying to add a new QUX column by redefining FRED_GROUP fails desc = { 'QUX': { 'dataManagerGroup': 'FRED_GROUP', 'dataManagerType': 'TiledColumnStMan', 'ndim': len(shape), 'shape': shape, 'valueType': casa_type } } dminfo = { '*1': { 'NAME': 'FRED_GROUP', 'TYPE': 'TiledColumnStMan', 'SPEC': { 'DEFAULTTILESHAPE': tile_shape }, 'COLUMNS': ['FRED', 'QUX'], } } with pytest.raises(RuntimeError, match="Data manager name FRED_GROUP"): T.addcols(desc, dminfo=dminfo) groups = {g['NAME']: g for g in T.getdminfo().values()} assert set(["BAZ_GROUP", "FRED_GROUP"]) == set(groups.keys()) # Adding new QUX column succeeds, but can't # add columns to an existing TiledColumnStMan? # casacore creates a new group, FRED_GROUP_1 T.addcols(desc) groups = {g['NAME']: g for g in T.getdminfo().values()} assert set(["BAZ_GROUP", "FRED_GROUP", "FRED_GROUP_1"]) == set(groups.keys()) # Add ACK and BAR to the ACKBAR_GROUP at the same time succeeds desc = { "ACK": { 'dataManagerGroup': 'ACKBAR_GROUP', 'dataManagerType': 'TiledColumnStMan', 'ndim': len(shape), 'shape': shape, 'valueType': casa_type }, "BAR": { 'dataManagerGroup': 'ACKBAR_GROUP', 'dataManagerType': 'TiledColumnStMan', 'ndim': len(shape), 'shape': shape, 'valueType': casa_type }, } dminfo = { '*1': { 'NAME': 'ACKBAR_GROUP', 'TYPE': 'TiledColumnStMan', 'SPEC': { 'DEFAULTTILESHAPE': tile_shape }, 'COLUMNS': ['ACK', 'BAR'], } } T.addcols(desc, dminfo=dminfo) groups = {g['NAME']: g for g in T.getdminfo().values()} assert set(["BAZ_GROUP", "FRED_GROUP", "FRED_GROUP_1", "ACKBAR_GROUP"]) == set(groups.keys()) assert set(groups["ACKBAR_GROUP"]['COLUMNS']) == set(["ACK", "BAR"])
def main(options): cI = numpy.complex(0.,1.) inms = options.inms if inms == '': print 'Error: you have to specify an input MS, use -h for help' return column = options.column outcol = options.outcol t = pt.table(inms, readonly=False, ack=True) if options.back: lincol = options.lincol if lincol not in t.colnames(): print 'Adding the output linear polarization column',lincol,'to',inms coldmi = t.getdminfo(column) coldmi['NAME'] = lincol t.addcols(pt.maketabdesc(pt.makearrcoldesc(lincol, 0., valuetype='complex', shape=numpy.array(t.getcell(column,0)).shape)), coldmi) ### RVW EDIT 2012 print 'Reading the input column (circular)', column if column not in t.colnames(): print 'Error: Input column does not exist' return ### RVW EDIT 2012 Input column with the -c switch cirdata = t.getcol(column) #cirdata = t.getcol(outcol) #print 'SHAPE ARRAY', numpy.shape(cirdata) print 'Computing the linear polarization terms...' lindata = numpy.transpose(numpy.array([ 0.5*(cirdata[:,:,0]+cirdata[:,:,1]+cirdata[:,:,2]+cirdata[:,:,3]), 0.5*(cI*cirdata[:,:,0]-cI*cirdata[:,:,1]+cI*cirdata[:,:,2]-cI*cirdata[:,:,3]), 0.5*(-cI*cirdata[:,:,0]-cI*cirdata[:,:,1]+cI*cirdata[:,:,2]+cI*cirdata[:,:,3]), 0.5*(cirdata[:,:,0]-cirdata[:,:,1]-cirdata[:,:,2]+cirdata[:,:,3])]), (1,2,0)) print 'Finishing up...' t.putcol(lincol, lindata) else: if outcol not in t.colnames(): print 'Adding the output column',outcol,'to',inms coldmi = t.getdminfo(column) coldmi['NAME'] = outcol t.addcols(pt.maketabdesc(pt.makearrcoldesc(outcol, 0., valuetype='complex', shape=numpy.array(t.getcell(column,0)).shape)), coldmi) print 'Reading the input column (linear)', column data = t.getcol(column) print 'Computing the output column' outdata = numpy.transpose(numpy.array([ 0.5*(data[:,:,0]-cI*data[:,:,1]+cI*data[:,:,2]+data[:,:,3]), 0.5*(data[:,:,0]+cI*data[:,:,1]+cI*data[:,:,2]-data[:,:,3]), 0.5*(data[:,:,0]-cI*data[:,:,1]-cI*data[:,:,2]-data[:,:,3]), 0.5*(data[:,:,0]+cI*data[:,:,1]-cI*data[:,:,2]+data[:,:,3])]), (1,2,0)) print 'Finishing up...' t.putcol(outcol, outdata) if options.poltable: print 'Updating the POLARIZATION table...' tp = pt.table(inms+'/POLARIZATION',readonly=False,ack=True) ### RVW EDIT 2012 if options.back: tp.putcol('CORR_TYPE',numpy.array([[9,10,11,12]],dtype=numpy.int32)) # FROM CIRC-->LIN else: tp.putcol('CORR_TYPE',numpy.array([[5,6,7,8]],dtype=numpy.int32)) # FROM LIN-->CIRC
import pyrap.tables as pt # Get nr of channels t = pt.table(msname + '/SPECTRAL_WINDOW') nf = t.getcell('NUM_CHAN', 0) t.close() # Get nr of receptors and value (for the datatype) t = pt.table(msname + '/SYSCAL', readonly=False) val = t.getcell('TCAL', 0) shp = [nf, len(val)] shpstr = str(shp) # Add a freq-dep column for each freq-indep column t.addcols( pt.maketabdesc(pt.makecoldesc('TCAL_SPECTRUM', val[0], 2, shp), pt.makecoldesc('TSYS_SPECTRUM', val[0], 2, shp))) #etc for other columns t.close() # Copy the freq-indep values to the freq-dep (TaQL's array function extends the values) pt.taql( 'update %s/SYSCAL set TCAL_SPECTRUM=array(TCAL,%s), TSYS_SPECTRUM=array(TSYS,%s)' % (msname, shpstr, shpstr))
def kat_ms_desc_and_dminfo(nbl, nchan, ncorr, model_data=False): """ Creates Table Description and Data Manager Information objecs that describe a MeasurementSet suitable for holding MeerKAT data. Creates additional DATA, IMAGING_WEIGHT and possibly MODEL_DATA and CORRECTED_DATA columns. Columns are given fixed shapes defined by the arguments to this function. :param nbl: Number of baselines. :param nchan: Number of channels. :param ncorr: Number of correlations. :param model_data: Boolean indicated whether MODEL_DATA and CORRECTED_DATA should be added to the Measurement Set. :return: Returns a tuple containing a table description describing the extra columns and hypercolumns, as well as a Data Manager description. """ if not casacore_binding == 'pyrap': raise ValueError("kat_ms_desc_and_dminfo requires the " "casacore binding to operate") # Columns that will be modified. # We want to keep things like their # keywords, dims and shapes modify_columns = { "WEIGHT", "SIGMA", "FLAG", "FLAG_CATEGORY", "UVW", "ANTENNA1", "ANTENNA2" } # Get the required table descriptor for an MS table_desc = tables.required_ms_desc("MAIN") # Take columns we wish to modify extra_table_desc = { c: d for c, d in table_desc.iteritems() if c in modify_columns } # Used to set the SPEC for each Data Manager Group dmgroup_spec = {} def dmspec(coldesc, tile_mem_limit=None): """ Create data manager spec for a given column description, mostly by adding a DEFAULTTILESHAPE that fits into the supplied memory limit. """ # Choose 4MB if none given if tile_mem_limit is None: tile_mem_limit = 4 * 1024 * 1024 # Get the reversed column shape. DEFAULTTILESHAPE is deep in # casacore and its necessary to specify their ordering here # ntilerows is the dim that will change least quickly rev_shape = list(reversed(coldesc["shape"])) ntilerows = 1 np_dtype = MS_TO_NP_TYPE_MAP[coldesc["valueType"].upper()] nbytes = np.dtype(np_dtype).itemsize # Try bump up the number of rows in our tiles while they're # below the memory limit for the tile while np.product(rev_shape + [2 * ntilerows]) * nbytes < tile_mem_limit: ntilerows *= 2 return {"DEFAULTTILESHAPE": np.int32(rev_shape + [ntilerows])} # Update existing columns with shape and data manager information dm_group = 'UVW' shape = [3] extra_table_desc["UVW"].update(options=0, shape=shape, ndim=len(shape), dataManagerGroup=dm_group, dataManagerType='TiledColumnStMan') dmgroup_spec[dm_group] = dmspec(extra_table_desc["UVW"]) dm_group = 'Weight' shape = [ncorr] extra_table_desc["WEIGHT"].update(options=4, shape=shape, ndim=len(shape), dataManagerGroup=dm_group, dataManagerType='TiledColumnStMan') dmgroup_spec[dm_group] = dmspec(extra_table_desc["WEIGHT"]) dm_group = 'Sigma' shape = [ncorr] extra_table_desc["SIGMA"].update(options=4, shape=shape, ndim=len(shape), dataManagerGroup=dm_group, dataManagerType='TiledColumnStMan') dmgroup_spec[dm_group] = dmspec(extra_table_desc["SIGMA"]) dm_group = 'Flag' shape = [nchan, ncorr] extra_table_desc["FLAG"].update(options=4, shape=shape, ndim=len(shape), dataManagerGroup=dm_group, dataManagerType='TiledColumnStMan') dmgroup_spec[dm_group] = dmspec(extra_table_desc["FLAG"]) dm_group = 'FlagCategory' shape = [1, nchan, ncorr] extra_table_desc["FLAG_CATEGORY"].update( options=4, keywords={}, shape=shape, ndim=len(shape), dataManagerGroup=dm_group, dataManagerType='TiledColumnStMan') dmgroup_spec[dm_group] = dmspec(extra_table_desc["FLAG_CATEGORY"]) # Create new columns for integration into the MS additional_columns = [] dm_group = 'Data' shape = [nchan, ncorr] desc = tables.tablecreatearraycoldesc("DATA", 0 + 0j, comment="The Visibility DATA Column", options=4, valuetype='complex', keywords={"UNIT": "Jy"}, shape=shape, ndim=len(shape), datamanagergroup=dm_group, datamanagertype='TiledColumnStMan') dmgroup_spec[dm_group] = dmspec(desc["desc"]) additional_columns.append(desc) dm_group = 'ImagingWeight' shape = [nchan] desc = tables.tablecreatearraycoldesc( "IMAGING_WEIGHT", 0, comment="Weight set by imaging task (e.g. uniform weighting)", options=4, valuetype='float', shape=shape, ndim=len(shape), datamanagergroup=dm_group, datamanagertype='TiledColumnStMan') dmgroup_spec[dm_group] = dmspec(desc["desc"]) additional_columns.append(desc) # Add MODEL_DATA and CORRECTED_DATA if requested if model_data == True: dm_group = 'ModelData' shape = [nchan, ncorr] desc = tables.tablecreatearraycoldesc( "MODEL_DATA", 0 + 0j, comment="The Visibility MODEL_DATA Column", options=4, valuetype='complex', keywords={"UNIT": "Jy"}, shape=shape, ndim=len(shape), datamanagergroup=dm_group, datamanagertype='TiledColumnStMan') dmgroup_spec[dm_group] = dmspec(desc["desc"]) additional_columns.append(desc) dm_group = 'CorrectedData' shape = [nchan, ncorr] desc = tables.tablecreatearraycoldesc( "CORRECTED_DATA", 0 + 0j, comment="The Visibility CORRECTED_DATA Column", options=4, valuetype='complex', keywords={"UNIT": "Jy"}, shape=shape, ndim=len(shape), datamanagergroup=dm_group, datamanagertype='TiledColumnStMan') dmgroup_spec[dm_group] = dmspec(desc["desc"]) additional_columns.append(desc) # Update extra table description with additional columns extra_table_desc.update(tables.maketabdesc(additional_columns)) # Update the original table descriptor with modifications/additions # Need this to construct a complete Data Manager specification # that includes the original columns table_desc.update(extra_table_desc) # Construct DataManager Specification dminfo = tables.makedminfo(table_desc, dmgroup_spec) return extra_table_desc, dminfo
import pyrap.tables as pt # Get nr of channels t = pt.table (msname + '/SPECTRAL_WINDOW') nf = t.getcell('NUM_CHAN', 0) t.close() # Get nr of receptors and value (for the datatype) t = pt.table (msname + '/SYSCAL', readonly=False) val = t.getcell('TCAL', 0) shp = [nf, len(val)] shpstr = str(shp) # Add a freq-dep column for each freq-indep column t.addcols (pt.maketabdesc(pt.makecoldesc('TCAL_SPECTRUM', val[0], 2, shp), pt.makecoldesc('TSYS_SPECTRUM', val[0], 2, shp))) #etc for other columns t.close() # Copy the freq-indep values to the freq-dep (TaQL's array function extends the values) pt.taql ('update %s/SYSCAL set TCAL_SPECTRUM=array(TCAL,%s), TSYS_SPECTRUM=array(TSYS,%s)' % (msname,shpstr,shpstr))