def concateovsa(msname, msfiles, visprefix='./', doclearcal=True, keep_orig_ms=False, cols2rm=["MODEL_DATA", "CORRECTED_DATA"]): concatvis = visprefix + msname msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] if doclearcal: print 'Warning: Corrected column in the input ms file will be cleared!!!' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) else: try: tmpdir = visprefix + '/tmp_ms/' if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = tmpdir + os.path.basename(str(ll)) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) except: print 'Warning: Corrected column not found in the input ms file.' msfiles_ = msfiles if msfiles_: concat(vis=msfiles_, concatvis=concatvis, timesort=True) else: concat(vis=msfiles, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) colnames = tb.colnames() for l in range(len(cols2rm)): if cols2rm[l] in colnames: try: tb.removecols(cols2rm[l]) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
def getColDesc(table, colname): '''Get the description of a column in a table table --> name of table or MS colname --> column name Return a dictionary with the column description''' coldesc = {} try: try: tb.open(table) tcols = tb.colnames() if tcols.__contains__(colname): coldesc = tb.getcoldesc(colname) except: pass finally: tb.close() return coldesc
def concat_slftb(tb_in=[], tb_out=None): if not tb_in: print('tb_in not provided. Abort...') if os.path.exists(tb_out): os.system('rm -r {}'.format(tb_out)) os.system('cp -r {} {}'.format(tb_in[0], tb_out)) tbdata = {} tb.open(tb_out) cols = tb.colnames() tb.close() cols.remove('WEIGHT') for col in cols: tbdata[col] = [] for tbidx, ctb in enumerate(tb_in): tb.open(ctb, nomodify=True) tim0 = tb.getcol(cols[0]) if len(tim0) == 0: continue else: for col in cols: if tbidx == 1 and col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']: tbdata[col].append(tb.getcol(col)[::-1,...]) else: tbdata[col].append(tb.getcol(col)) tb.close() if len(tbdata[cols[0]]) == 0: print('tables have no data. Return') return -1 else: for col in cols: if col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']: tbdata[col] = np.concatenate(tbdata[col], axis=2) else: tbdata[col] = np.concatenate(tbdata[col]) tb.open(tb_out, nomodify=False) nrows = tb.nrows() nrows_new = len(tbdata[cols[0]]) tb.addrows(nrows_new - nrows) for col in cols: tb.putcol(col, tbdata[col]) tb.close() return tb_out
def compTables(referencetab, testtab, excludecols, tolerance=0.001, mode="percentage", startrow=0, nrow=-1, rowincr=1): """ compTables - compare two CASA tables referencetab - the table which is assumed to be correct testtab - the table which is to be compared to referencetab excludecols - list of column names which are to be ignored tolerance - permitted fractional difference (default 0.001 = 0.1 percent) mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees) """ rval = True tb2 = casac.table() tb.open(referencetab) cnames = tb.colnames() tb2.open(testtab) try: for c in cnames: if c in excludecols: continue print "\nTesting column " + c a = 0 try: a = tb.getcol(c, startrow=startrow, nrow=nrow, rowincr=rowincr) except: rval = False print 'Error accessing column ', c, ' in table ', referencetab print sys.exc_info()[0] break b = 0 try: b = tb2.getcol(c, startrow=startrow, nrow=nrow, rowincr=rowincr) except: rval = False print 'Error accessing column ', c, ' in table ', testtab print sys.exc_info()[0] break if not (len(a) == len(b)): print 'Column ', c, ' has different length in tables ', referencetab, ' and ', testtab print a print b rval = False break else: differs = False if not (a == b).all(): for i in range(0, len(a)): if (isinstance(a[i], float)): if ((mode == "percentage") and (abs(a[i] - b[i]) > tolerance * abs(a[i])) ) or ((mode == "absolute") and (abs(a[i] - b[i]) > tolerance)): print "Column " + c + " differs" print "Row=" + str(i) print "Reference file value: " + str(a[i]) print "Input file value: " + str(b[i]) if (mode == "percentage"): print "Tolerance is {0}%; observed difference was {1} %".format( tolerance * 100, 100 * abs(a[i] - b[i]) / abs(a[i])) else: print "Absolute tolerance is {0}; observed difference: {1}".format( tolerance, (abs(a[i] - b[i]))) differs = True rval = False break elif (isinstance(a[i], int) or isinstance(a[i], np.int32)): if (abs(a[i] - b[i]) > 0): print "Column " + c + " differs" print "Row=" + str(i) print "Reference file value: " + str(a[i]) print "Input file value: " + str(b[i]) if (mode == "percentage"): print "tolerance in % should be " + str( 100 * abs(a[i] - b[i]) / abs(a[i])) else: print "absolute tolerance should be " + str( abs(a[i] - b[i])) differs = True rval = False break elif (isinstance(a[i], str) or isinstance(a[i], np.bool_)): if not (a[i] == b[i]): print "Column " + c + " differs" print "Row=" + str(i) print "Reference file value: " + str(a[i]) print "Input file value: " + str(b[i]) if (mode == "percentage"): print "tolerance in % should be " + str( 100 * abs(a[i] - b[i]) / abs(a[i])) else: print "absolute tolerance should be " + str( abs(a[i] - b[i])) differs = True rval = False break elif (isinstance(a[i], list)) or (isinstance( a[i], np.ndarray)): for j in range(0, len(a[i])): if differs: break if ((isinstance(a[i][j], float)) or (isinstance(a[i][j], int))): if ((mode == "percentage") and (abs(a[i][j] - b[i][j]) > tolerance * abs(a[i][j]))) or ( (mode == "absolute") and (abs(a[i][j] - b[i][j]) > tolerance)): print "Column " + c + " differs" print "(Row,Element)=(" + str( j) + "," + str(i) + ")" print "Reference file value: " + str( a[i][j]) print "Input file value: " + str( b[i][j]) if (mode == "percentage"): print "Tolerance in % should be " + str( 100 * abs(a[i][j] - b[i][j]) / abs(a[i][j])) else: print "Absolute tolerance should be " + str( abs(a[i][j] - b[i][j])) differs = True rval = False break elif (isinstance(a[i][j], list)) or (isinstance( a[i][j], np.ndarray)): it = xrange(0, len(a[i][j])) if mode == "percentage": diff = np.abs( np.subtract(a[i][j], b[i][j]) ) > tolerance * np.abs(a[i][j]) it = np.where(diff)[0] elif (mode == "absolute"): diff = np.abs( np.subtract(a[i][j], b[i][j])) > tolerance it = np.where(diff)[0] for k in it: if differs: break if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \ or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \ or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \ ): print "Column " + c + " differs" print "(Row,Channel,Corr)=(" + str( k) + "," + str(j) + "," + str( i) + ")" print "Reference file value: " + str( a[i][j][k]) print "Input file value: " + str( b[i][j][k]) if (mode == "percentage"): print "Tolerance in % should be " + str( 100 * abs(a[i][j][k] - b[i][j][k]) / abs(a[i][j][k])) elif (mode == "absolute"): print "Absolute tolerance should be " + str( abs(a[i][j][k] - b[i][j][k])) elif (mode == "phaseabsdeg"): print "Phase tolerance in degrees should be " + str( phasediffabsdeg( a[i][j][k], b[i][j][k])) else: print "Unknown comparison mode: ", mode differs = True rval = False break else: print "Unknown data type: ", type(a[i]) differs = True rval = False break if not differs: print "Column " + c + " PASSED" finally: tb.close() tb2.close() return rval
def compVarColTables(referencetab, testtab, varcol, tolerance=0.): '''Compare a variable column of two tables. referencetab --> a reference table testtab --> a table to verify varcol --> the name of a variable column (str) Returns True or False. ''' retval = True tb2 = casac.table() tb.open(referencetab) cnames = tb.colnames() tb2.open(testtab) col = varcol if tb.isvarcol(col) and tb2.isvarcol(col): try: # First check if tb.nrows() != tb2.nrows(): print 'Length of %s differ from %s, %s!=%s' % ( referencetab, testtab, len(rk), len(tk)) retval = False else: for therow in xrange(tb.nrows()): rdata = tb.getcell(col, therow) tdata = tb2.getcell(col, therow) # if not (rdata==tdata).all(): if not rdata.all() == tdata.all(): if (tolerance > 0.): differs = False for j in range(0, len(rdata)): ### if (type(rdata[j])==float or type(rdata[j])==int): if ((isinstance(rdata[j], float)) or (isinstance(rdata[j], int))): if (abs(rdata[j] - tdata[j]) > tolerance * abs(rdata[j] + tdata[j])): # print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab # print therow, j # print rdata[j] # print tdata[j] differs = True ### elif (type(rdata[j])==list or type(rdata[j])==np.ndarray): elif (isinstance(rdata[j], list)) or (isinstance( rdata[j], np.ndarray)): for k in range(0, len(rdata[j])): if (abs(rdata[j][k] - tdata[j][k]) > tolerance * abs(rdata[j][k] + tdata[j][k])): # print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab # print therow, j, k # print rdata[j][k] # print tdata[j][k] differs = True if differs: print 'ERROR: Column %s of %s and %s do not agree within tolerance %s' % ( col, referencetab, testtab, tolerance) retval = False break else: print 'ERROR: Column %s of %s and %s do not agree.' % ( col, referencetab, testtab) print 'ERROR: First row to differ is row=%s' % therow retval = False break finally: tb.close() tb2.close() else: print 'Columns are not varcolumns.' retval = False if retval: print 'Column %s of %s and %s agree' % (col, referencetab, testtab) return retval
def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms, doscaling, keep_nsclms, fileidx): from taskinit import tb, casalog filename = filelist[fileidx] uv = aipy.miriad.UV(filename) # try: msname0 = list(filename.split('/')[-1]) msname = visprefix + ''.join(msname0) + '.ms' uv.select('antennae', 0, 1, include=True) uv.select('polarization', -5, -5, include=True) times = [] uv.rewind() for preamble, data in uv.all(): uvw, t, (i, j) = preamble times.append(t) uv.select('clear', -1, -1, include=True) times = ipe.jd2mjds(np.asarray(times)) inttime = np.median((times - np.roll(times, 1))[1:]) / 60 time_steps = len(times) durtim = int((times[-1] - times[0]) / 60 + inttime) time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'].replace('\x00', '') antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.where(uv['sfreq'] > 0)[0] nf = len(good_idx) npol = uv['npol'] nants = uv['nants'] source_id = uv['source'].replace('\x00', '') sfreq = uv['sfreq'][good_idx] sdf = uv['sdf'][good_idx] ra, dec = uv['ra'], uv['dec'] nbl = nants * (nants - 1) / 2 bl2ord = ipe.bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf) nband = len(chan_band) uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 data = ma.masked_array(ma.masked_invalid(data), fill_value=0.0) out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask # if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs if doscaling: out2 = out.copy() for i0 in antlist: for j0 in antlist: if i0 < j0: i, j = i0 - 1, j0 - 1 out2[:, :, :, bl2ord[i, j]] = out[:, :, :, bl2ord[i, j]] / np.sqrt( np.abs(out[:, :, :, bl2ord[i, i]]) * np.abs(out[:, :, :, bl2ord[j, j]])) out2 = out2.reshape(npol, nf, nrows) out2[np.isnan(out2)] = 0 out2[np.isinf(out2)] = 0 # out2 = ma.masked_array(ma.masked_invalid(out2), fill_value=0.0) out = out.reshape(npol, nf, nrows) * 1e4 flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows) uvwarray = np.tile(uvwarray, (1, nband)) sigma = np.ones((4, nrows), dtype=np.float) + 1 sigma = np.tile(sigma, (1, nband)) casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format( filename, (time.time() - time0))) if not nocreatms: modelms = ipe.creatms(filename, visprefix) os.system('mv {} {}'.format(modelms, msname)) else: casalog.post('----------------------------------------') casalog.post('copying standard MS to {0}'.format( msname, (time.time() - time0))) casalog.post('----------------------------------------') os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % modelms + " %s" % msname) casalog.post( 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format( msname, (time.time() - time0))) tb.open(msname, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() nchannels = len(cband['cidx']) for row in range(nrows): if not doscaling or keep_nsclms: tb.putcell('DATA', (row + l * nrows), out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) tb.putcell('FLAG', (row + l * nrows), flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format( (l + 1), time.time() - time1)) tb.putcol('UVW', uvwarray) tb.putcol('SIGMA', sigma) tb.putcol('WEIGHT', 1.0 / sigma**2) timearr = times timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape(nband * npairs * time_steps) tb.putcol('TIME', timearr) tb.putcol('TIME_CENTROID', timearr) scan_id = tb.getcol('SCAN_NUMBER') scan_id *= 0 tb.putcol('SCAN_NUMBER', scan_id) colnames = tb.colnames() cols2rm = ["MODEL_DATA", "CORRECTED_DATA"] for l in range(len(cols2rm)): if cols2rm[l] in colnames: tb.removecols(cols2rm[l]) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the OBSERVATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/OBSERVATION', nomodify=False) tb.putcol( 'TIME_RANGE', np.asarray([times[0] - 0.5 * inttime, times[-1] + 0.5 * inttime]).reshape(2, 1)) tb.putcol('OBSERVER', ['EOVSA team']) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POINTING table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POINTING', nomodify=False) timearr = times.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape(nband * time_steps * nants) tb.putcol('TIME', timearr) tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time) direction = tb.getcol('DIRECTION') direction[0, 0, :] = ra direction[1, 0, :] = dec tb.putcol('DIRECTION', direction) target = tb.getcol('TARGET') target[0, 0, :] = ra target[1, 0, :] = dec tb.putcol('TARGET', target) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the SOURCE table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/SOURCE', nomodify=False) radec = tb.getcol('DIRECTION') radec[0], radec[1] = ra, dec tb.putcol('DIRECTION', radec) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False) pol_id = tb.getcol('POLARIZATION_ID') pol_id *= 0 tb.putcol('POLARIZATION_ID', pol_id) # spw_id = tb.getcol('SPECTRAL_WINDOW_ID') # spw_id *= 0 # tb.putcol('SPECTRAL_WINDOW_ID', spw_id) tb.close() # casalog.post('----------------------------------------') # casalog.post("Updating the POLARIZATION table of" '%s' % msname) # casalog.post('----------------------------------------') # tb.open(msname + '/POLARIZATION/', nomodify=False) # tb.removerows(rownrs=np.arange(1, nband, dtype=int)) # tb.close() casalog.post('----------------------------------------') casalog.post("Updating the FIELD table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/FIELD/', nomodify=False) delay_dir = tb.getcol('DELAY_DIR') delay_dir[0], delay_dir[1] = ra, dec tb.putcol('DELAY_DIR', delay_dir) phase_dir = tb.getcol('PHASE_DIR') phase_dir[0], phase_dir[1] = ra, dec tb.putcol('PHASE_DIR', phase_dir) reference_dir = tb.getcol('REFERENCE_DIR') reference_dir[0], reference_dir[1] = ra, dec tb.putcol('REFERENCE_DIR', reference_dir) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME # del out, flag, uvwarray, uv, timearr, sigma # gc.collect() # if doscaling: if keep_nsclms: msname_scl = visprefix + ''.join(msname0) + '_scl.ms' os.system('cp -r {} {}'.format(msname, msname_scl)) else: msname_scl = msname tb.open(msname_scl, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname_scl) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() for row in range(nrows): tb.putcell( 'DATA', (row + l * nrows), out2[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'. format((l + 1), time.time() - time1)) tb.close() if not (timebin == '0s' and width == 1): msfile = msname + '.split' if doscaling: split(vis=msname_scl, outputvis=msname_scl + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname_scl)) msfile_scl = msname_scl + '.split' if not (doscaling and not keep_nsclms): split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname)) else: msfile = msname if doscaling: msfile_scl = msname_scl casalog.post("finished in --- %s seconds ---" % (time.time() - time0)) if doscaling: return [True, msfile, msfile_scl, durtim] else: return [True, msfile, durtim]
def concat(tb_in=[], tb_out=None): if not tb_in: print('tb_in not provided. Abort...') if os.path.exists(tb_out): os.system('rm -r ' + tb_out) # os.system('cp -r '+tb_in[0]+' '+tb_out) os.system('cp -r ' + tb_in[0] + ' ' + tb_out) tb.open(tb_out + '/SPECTRAL_WINDOW', nomodify=True) nspw = tb.nrows() tb.close() tim = [] fld = [] spw = [] ant1 = [] ant2 = [] intv = [] scan = [] obid = [] cpar = [] para = [] flag = [] snr = [] # wght=[] for ctb in tb_in: tb.open(ctb, nomodify=True) cols = tb.colnames() tim0 = tb.getcol(cols[0]) if len(tim0) == 0: continue else: tim.append(tb.getcol(cols[0])) fld.append(tb.getcol(cols[1])) spw.append(tb.getcol(cols[2])) ant1.append(tb.getcol(cols[3])) ant2.append(tb.getcol(cols[4])) intv.append(tb.getcol(cols[5])) scan.append(tb.getcol(cols[6])) obid.append(tb.getcol(cols[7])) cpar.append(tb.getcol(cols[8])) para.append(tb.getcol(cols[9])) flag.append(tb.getcol(cols[10])) snr.append(tb.getcol(cols[11])) # wght.append(tb.getcol(cols[12])) tb.close() if len(tim) == 0: print('tables have no data. Return') return -1 else: tim = np.concatenate(tim) fld = np.concatenate(fld) spw = np.concatenate(spw) ant1 = np.concatenate(ant1) ant2 = np.concatenate(ant2) intv = np.concatenate(intv) scan = np.concatenate(scan) obid = np.concatenate(obid) cpar = np.concatenate(cpar, axis=2) para = np.concatenate(para, axis=2) flag = np.concatenate(flag, axis=2) snr = np.concatenate(snr, axis=2) # wght=np.concatenate(wght) tb.open(tb_out, nomodify=False) nrows = tb.nrows() nrows_new = len(tim) tb.addrows(nrows_new - nrows) tb.putcol(cols[0], tim) tb.putcol(cols[1], fld) tb.putcol(cols[2], spw) tb.putcol(cols[3], ant1) tb.putcol(cols[4], ant2) tb.putcol(cols[5], intv) tb.putcol(cols[6], scan) tb.putcol(cols[7], obid) tb.putcol(cols[8], cpar) tb.putcol(cols[9], para) tb.putcol(cols[10], flag) tb.putcol(cols[11], snr) tb.close() return tb_out
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False, timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""): if concatvis[-1] == os.path.sep: concatvis = concatvis[:-1] if os.path.sep not in concatvis: visprefix = './' else: visprefix = os.path.dirname(concatvis) + os.path.sep msfiles = vis msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] datacolumn = datacolumn.lower() if datacolumn == 'data': print 'DATA columns will be concatenated.' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) elif datacolumn == 'corrected': # try: print 'CORRECTED columns will be concatenated.' tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = os.path.join(tmpdir, os.path.basename(str(ll))) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) else: raise ValueError('Please set datacolumn to be "data" or "corrected"!') if msfiles_: concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) os.system('rm -rf {}'.format(tmpdir)) else: concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False) nrows = tb.nrows() pol_id = tb.getcol('POLARIZATION_ID') tb.removerows(np.where(pol_id != 0)[0]) tb.close() tb.open(concatvis, nomodify=False) dd_id = tb.getcol('DATA_DESC_ID') idx_dd_id, = np.where(dd_id >= nrows / 2) dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2 tb.putcol('DATA_DESC_ID', dd_id) tb.close() tb.open(concatvis + '/FIELD', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis + '/SOURCE', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) fldid = tb.getcol('FIELD_ID') newfldid = np.zeros(len(fldid), dtype='int') tb.putcol('FIELD_ID', newfldid) colnames = tb.colnames() cols2rm = cols2rm.upper() cols2rm = cols2rm.split(',') for l in range(len(cols2rm)): col = cols2rm[l] + '_DATA' if col in colnames: try: tb.removecols(col) print 'Column {} removed.'.format(col) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
def compTables(referencetab, testtab, excludecols, tolerance=0.001, mode="percentage", startrow = 0, nrow = -1, rowincr = 1): """ compTables - compare two CASA tables referencetab - the table which is assumed to be correct testtab - the table which is to be compared to referencetab excludecols - list of column names which are to be ignored tolerance - permitted fractional difference (default 0.001 = 0.1 percent) mode - comparison is made as "percentage", "absolute", "phaseabsdeg" (for complex numbers = difference of the phases in degrees) """ rval = True tb2 = casac.table() tb.open(referencetab) cnames = tb.colnames() tb2.open(testtab) try: for c in cnames: if c in excludecols: continue print "\nTesting column " + c a = 0 try: a = tb.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr) except: rval = False print 'Error accessing column ', c, ' in table ', referencetab print sys.exc_info()[0] break b = 0 try: b = tb2.getcol(c,startrow=startrow,nrow=nrow,rowincr=rowincr) except: rval = False print 'Error accessing column ', c, ' in table ', testtab print sys.exc_info()[0] break if not (len(a)==len(b)): print 'Column ',c,' has different length in tables ', referencetab, ' and ', testtab print a print b rval = False break else: differs = False if not (a==b).all(): for i in range(0,len(a)): if (isinstance(a[i],float)): if ((mode=="percentage") and (abs(a[i]-b[i]) > tolerance*abs(a[i]))) or ((mode=="absolute") and (abs(a[i]-b[i]) > tolerance)): print "Column " + c + " differs" print "Row=" + str(i) print "Reference file value: " + str(a[i]) print "Input file value: " + str(b[i]) if (mode=="percentage"): print "Tolerance is {0}%; observed difference was {1} %".format (tolerance * 100, 100*abs(a[i]-b[i])/abs(a[i])) else: print "Absolute tolerance is {0}; observed difference: {1}".format (tolerance, (abs(a[i]-b[i]))) differs = True rval = False break elif (isinstance(a[i],int) or isinstance(a[i],np.int32)): if (abs(a[i]-b[i]) > 0): print "Column " + c + " differs" print "Row=" + str(i) print "Reference file value: " + str(a[i]) print "Input file value: " + str(b[i]) if (mode=="percentage"): print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i])) else: print "absolute tolerance should be " + str(abs(a[i]-b[i])) differs = True rval = False break elif (isinstance(a[i],str) or isinstance(a[i],np.bool_)): if not (a[i]==b[i]): print "Column " + c + " differs" print "Row=" + str(i) print "Reference file value: " + str(a[i]) print "Input file value: " + str(b[i]) if (mode=="percentage"): print "tolerance in % should be " + str(100*abs(a[i]-b[i])/abs(a[i])) else: print "absolute tolerance should be " + str(abs(a[i]-b[i])) differs = True rval = False break elif (isinstance(a[i],list)) or (isinstance(a[i],np.ndarray)): for j in range(0,len(a[i])): if differs: break if ((isinstance(a[i][j],float)) or (isinstance(a[i][j],int))): if ((mode=="percentage") and (abs(a[i][j]-b[i][j]) > tolerance*abs(a[i][j]))) or ((mode=="absolute") and (abs(a[i][j]-b[i][j]) > tolerance)): print "Column " + c + " differs" print "(Row,Element)=(" + str(j) + "," + str(i) + ")" print "Reference file value: " + str(a[i][j]) print "Input file value: " + str(b[i][j]) if (mode=="percentage"): print "Tolerance in % should be " + str(100*abs(a[i][j]-b[i][j])/abs(a[i][j])) else: print "Absolute tolerance should be " + str(abs(a[i][j]-b[i][j])) differs = True rval = False break elif (isinstance(a[i][j],list)) or (isinstance(a[i][j],np.ndarray)): for k in range(0,len(a[i][j])): if differs: break if ( ((mode=="percentage") and (abs(a[i][j][k]-b[i][j][k]) > tolerance*abs(a[i][j][k]))) \ or ((mode=="absolute") and (abs(a[i][j][k]-b[i][j][k]) > tolerance)) \ or ((mode=="phaseabsdeg") and (phasediffabsdeg(a[i][j][k],b[i][j][k])>tolerance)) \ ): print "Column " + c + " differs" print "(Row,Channel,Corr)=(" + str(k) + "," + str(j) + "," + str(i) + ")" print "Reference file value: " + str(a[i][j][k]) print "Input file value: " + str(b[i][j][k]) if (mode=="percentage"): print "Tolerance in % should be " + str(100*abs(a[i][j][k]-b[i][j][k])/abs(a[i][j][k])) elif (mode=="absolute"): print "Absolute tolerance should be " + str(abs(a[i][j][k]-b[i][j][k])) elif (mode=="phaseabsdeg"): print "Phase tolerance in degrees should be " + str(phasediffabsdeg(a[i][j][k],b[i][j][k])) else: print "Unknown comparison mode: ",mode differs = True rval = False break else: print "Unknown data type: ",type(a[i]) differs = True rval = False break if not differs: print "Column " + c + " PASSED" finally: tb.close() tb2.close() return rval
def compVarColTables(referencetab, testtab, varcol, tolerance=0.): '''Compare a variable column of two tables. referencetab --> a reference table testtab --> a table to verify varcol --> the name of a variable column (str) Returns True or False. ''' retval = True tb2 = casac.table() tb.open(referencetab) cnames = tb.colnames() tb2.open(testtab) col = varcol if tb.isvarcol(col) and tb2.isvarcol(col): try: # First check if tb.nrows() != tb2.nrows(): print 'Length of %s differ from %s, %s!=%s'%(referencetab,testtab,len(rk),len(tk)) retval = False else: for therow in xrange(tb.nrows()): rdata = tb.getcell(col,therow) tdata = tb2.getcell(col,therow) # if not (rdata==tdata).all(): if not rdata.all()==tdata.all(): if (tolerance>0.): differs=False for j in range(0,len(rdata)): ### if (type(rdata[j])==float or type(rdata[j])==int): if ((isinstance(rdata[j],float)) or (isinstance(rdata[j],int))): if (abs(rdata[j]-tdata[j]) > tolerance*abs(rdata[j]+tdata[j])): # print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab # print therow, j # print rdata[j] # print tdata[j] differs = True ### elif (type(rdata[j])==list or type(rdata[j])==np.ndarray): elif (isinstance(rdata[j],list)) or (isinstance(rdata[j],np.ndarray)): for k in range(0,len(rdata[j])): if (abs(rdata[j][k]-tdata[j][k]) > tolerance*abs(rdata[j][k]+tdata[j][k])): # print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab # print therow, j, k # print rdata[j][k] # print tdata[j][k] differs = True if differs: print 'ERROR: Column %s of %s and %s do not agree within tolerance %s'%(col,referencetab, testtab, tolerance) retval = False break else: print 'ERROR: Column %s of %s and %s do not agree.'%(col,referencetab, testtab) print 'ERROR: First row to differ is row=%s'%therow retval = False break finally: tb.close() tb2.close() else: print 'Columns are not varcolumns.' retval = False if retval: print 'Column %s of %s and %s agree'%(col,referencetab, testtab) return retval
def importeovsa(idbfiles, timebin=None, width=None, visprefix=None, nocreatms=False, doconcat=False, modelms=''): casalog.origin('importeovsa') # # Initialize the helper class # pdh = ParallelDataHelper("importeovsa", locals()) # # # Validate input and output parameters # try: # pdh.setupIO() # except Exception, instance: # casalog.post('%s' % instance, 'ERROR') # return False if type(idbfiles) == Time: filelist = ri.get_trange_files(idbfiles) else: # If input type is not Time, assume that it is the list of files to read filelist = idbfiles if type(filelist) == str: filelist = [filelist] for f in filelist: if not os.path.exists(f): casalog.post("Some files in filelist are invalid. Aborting...") return False if not visprefix: visprefix = './' if not timebin: timebin = '0s' if not width: width = 1 if not modelms: if nocreatms: filename = filelist[0] modelms = ipe.creatms(filename, visprefix) else: if not os.path.exists(modelms): if nocreatms: filename = filelist[0] modelms = ipe.creatms(filename, visprefix) msfile = [] time_concat = [] for filename in filelist: uv = aipy.miriad.UV(filename) # if filename.split('/')[-1][0:3] == 'UDB': # uv_str = uv_hex_rm(uv) # else: # uv_str = uv uv.select('antennae', 0, 1, include=True) uv.select('polarization', -5, -5, include=True) times = [] uv.rewind() for preamble, data in uv.all(): uvw, t, (i, j) = preamble times.append(t) uv.select('clear', -1, -1, include=True) times = ipe.jd2mjds(np.asarray(times)) inttime = np.median((times - np.roll(times, 1))[1:]) / 60 time_steps = len(times) time_concat.append(int((times[-1] - times[0]) / 60 + inttime)) time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'].replace('\x00', '') antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.where(uv['sfreq'] > 0)[0] nf = len(good_idx) npol = uv['npol'] nants = uv['nants'] source_id = uv['source'].replace('\x00', '') sfreq = uv['sfreq'][good_idx] sdf = uv['sdf'][good_idx] ra, dec = uv['ra'], uv['dec'] nbl = nants * (nants - 1) / 2 bl2ord = ipe.bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf) nband = len(chan_band) uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask # if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs out = out.reshape(npol, nf, nrows) flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows) uvwarray = np.tile(uvwarray, (1, nband)) sigma = np.ones((4, nrows), dtype=np.float) + 1 sigma = np.tile(sigma, (1, nband)) casalog.post( 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format( filename, (time.time() - time0))) msname = list(filename.split('/')[-1]) msname = visprefix + ''.join(msname) + '.ms' if not nocreatms: modelms = ipe.creatms(filename, visprefix) os.system('mv {} {}'.format(modelms, msname)) else: casalog.post('----------------------------------------') casalog.post('copying standard MS to {0}'.format( msname, (time.time() - time0))) casalog.post('----------------------------------------') os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % modelms + " %s" % msname) casalog.post( 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'. format(msname, (time.time() - time0))) tb.open(msname, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() nchannels = len(cband['cidx']) for row in range(nrows): tb.putcell('DATA', (row + l * nrows), out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) tb.putcell( 'FLAG', (row + l * nrows), flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'. format((l + 1), time.time() - time1)) tb.putcol('UVW', uvwarray) tb.putcol('SIGMA', sigma) tb.putcol('WEIGHT', 1.0 / sigma**2) timearr = times timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape(nband * npairs * time_steps) tb.putcol('TIME', timearr) tb.putcol('TIME_CENTROID', timearr) scan_id = tb.getcol('SCAN_NUMBER') scan_id *= 0 tb.putcol('SCAN_NUMBER', scan_id) colnames = tb.colnames() cols2rm = ["MODEL_DATA", "CORRECTED_DATA"] for l in range(len(cols2rm)): if cols2rm[l] in colnames: tb.removecols(cols2rm[l]) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the OBSERVATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/OBSERVATION', nomodify=False) tb.putcol( 'TIME_RANGE', np.asarray([times[0] - 0.5 * inttime, times[-1] + 0.5 * inttime]).reshape(2, 1)) tb.putcol('OBSERVER', ['EOVSA team']) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POINTING table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POINTING', nomodify=False) timearr = times.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape(nband * time_steps * nants) tb.putcol('TIME', timearr) tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time) direction = tb.getcol('DIRECTION') direction[0, 0, :] = ra direction[1, 0, :] = dec tb.putcol('DIRECTION', direction) target = tb.getcol('TARGET') target[0, 0, :] = ra target[1, 0, :] = dec tb.putcol('TARGET', target) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the SOURCE table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/SOURCE', nomodify=False) radec = tb.getcol('DIRECTION') radec[0], radec[1] = ra, dec tb.putcol('DIRECTION', radec) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False) pol_id = tb.getcol('POLARIZATION_ID') pol_id *= 0 tb.putcol('POLARIZATION_ID', pol_id) # spw_id = tb.getcol('SPECTRAL_WINDOW_ID') # spw_id *= 0 # tb.putcol('SPECTRAL_WINDOW_ID', spw_id) tb.close() # pdb.set_trace() casalog.post('----------------------------------------') casalog.post("Updating the POLARIZATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POLARIZATION/', nomodify=False) print tb.nrows() tb.removerows(rownrs=np.arange(1, nband, dtype=int)) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the FIELD table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/FIELD/', nomodify=False) delay_dir = tb.getcol('DELAY_DIR') delay_dir[0], delay_dir[1] = ra, dec tb.putcol('DELAY_DIR', delay_dir) phase_dir = tb.getcol('PHASE_DIR') phase_dir[0], phase_dir[1] = ra, dec tb.putcol('PHASE_DIR', phase_dir) reference_dir = tb.getcol('REFERENCE_DIR') reference_dir[0], reference_dir[1] = ra, dec tb.putcol('REFERENCE_DIR', reference_dir) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME del out, flag, uvwarray, uv, timearr, sigma gc.collect() # if not (timebin == '0s' and width == 1): split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname)) msfile.append(msname + '.split') else: msfile.append(msname) casalog.post("finished in --- %s seconds ---" % (time.time() - time0)) if doconcat: msname = list(filelist[0].split('/')[-1]) concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format( int(sum(time_concat))) concat(vis=msfile, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) tb.close() for ll in msfile: os.system('rm -rf {}'.format(ll)) return True