def concateovsa(msname, msfiles, visprefix='./', doclearcal=True, keep_orig_ms=False, cols2rm=["MODEL_DATA", "CORRECTED_DATA"]): concatvis = visprefix + msname msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] if doclearcal: print 'Warning: Corrected column in the input ms file will be cleared!!!' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) else: try: tmpdir = visprefix + '/tmp_ms/' if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = tmpdir + os.path.basename(str(ll)) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) except: print 'Warning: Corrected column not found in the input ms file.' msfiles_ = msfiles if msfiles_: concat(vis=msfiles_, concatvis=concatvis, timesort=True) else: concat(vis=msfiles, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) colnames = tb.colnames() for l in range(len(cols2rm)): if cols2rm[l] in colnames: try: tb.removecols(cols2rm[l]) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
def concat_slftb(tb_in=[], tb_out=None): if not tb_in: print('tb_in not provided. Abort...') if os.path.exists(tb_out): os.system('rm -r {}'.format(tb_out)) os.system('cp -r {} {}'.format(tb_in[0], tb_out)) tbdata = {} tb.open(tb_out) cols = tb.colnames() tb.close() cols.remove('WEIGHT') for col in cols: tbdata[col] = [] for tbidx, ctb in enumerate(tb_in): tb.open(ctb, nomodify=True) tim0 = tb.getcol(cols[0]) if len(tim0) == 0: continue else: for col in cols: if tbidx == 1 and col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']: tbdata[col].append(tb.getcol(col)[::-1,...]) else: tbdata[col].append(tb.getcol(col)) tb.close() if len(tbdata[cols[0]]) == 0: print('tables have no data. Return') return -1 else: for col in cols: if col in ['CPARAM', 'PARAMERR', 'FLAG', 'SNR']: tbdata[col] = np.concatenate(tbdata[col], axis=2) else: tbdata[col] = np.concatenate(tbdata[col]) tb.open(tb_out, nomodify=False) nrows = tb.nrows() nrows_new = len(tbdata[cols[0]]) tb.addrows(nrows_new - nrows) for col in cols: tb.putcol(col, tbdata[col]) tb.close() return tb_out
def flagcaltboutliers(caltable, limit=[]): import numpy as np import numpy.ma as ma # def removeOutliers(x, outlierConstant): # a = np.array(x) # idx, = np.where(np.diff(np.sort(datamag[0, 0, :]))>) # upper_quartile = np.percentile(a, 80) # lower_quartile = np.percentile(a, 20) # IQR = (upper_quartile - lower_quartile) * outlierConstant # quartileSet = (lower_quartile - IQR, upper_quartile + IQR) # return ma.masked_outside(x, quartileSet[1], quartileSet[0]) if not os.path.exists(caltable): return 0 if isinstance(limit, list): if len(limit) == 2: tb.open(caltable, nomodify=False) # subt = tb.query("ANTENNA1==1 && SPECTRAL_WINDOW_ID=10") # data = subt.getcol('CPARAM') # flag = subt.getcol('FLAG') # spw = subt.getcol('SPECTRAL_WINDOW_ID') # datamag = np.abs(data) # mdatamag = ma.masked_outside(datamag, limit[0], limit[1]) # mask = np.logical_or(mdatamag.mask, flag) # dataidx1 = datamag<limit[0] # dataidx2 = datamag>limit[1] # mdatamag = ma.masked_array(mdatamag, mask) # mdatamag[0, 0, :] = removeOutliers(mdatamag[0, 0, :], 5) # mdatamag[1, 0, :] = removeOutliers(mdatamag[1, 0, :], 5) data = tb.getcol('CPARAM') flag = tb.getcol('FLAG') datamag = np.abs(data) dataidx1 = datamag < limit[0] dataidx2 = datamag > limit[1] flag[dataidx1] = True flag[dataidx2] = True tb.putcol('FLAG', flag) return 1 else: print('limit must have two elements. Aborted!') return 0 else: print('limit must be a list. Aborted!')
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False): if miriad: os.system('rm -rf ' + modelvisloc) cmd = 'cp ' + visdataloc + ' ' + modelvisloc os.system(cmd) # get the real and imaginary components real = numpy.real(vis_complex) imag = numpy.imag(vis_complex) # replace data visibilities with model visibilities visfile = fits.open(modelvisloc, mode='update') visibilities = visfile[0].data visheader = visfile[0].header if visheader['NAXIS'] == 7: visibilities['DATA'][:, 0, 0, :, :, :, 0] = real visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag elif visheader['NAXIS'] == 6: visibilities['DATA'][:, 0, 0, :, :, 0] = real visibilities['DATA'][:, 0, 0, :, :, 1] = imag else: print("Visibility dataset has >7 or <6 axes. I can't read this.") #visibilities['DATA'][:, 0, 0, :, :, 2] = wgt # replace the data visibilities with the model visibilities visfile[0].data = visibilities visfile.flush() else: from taskinit import tb print("Writing visibility data to " + modelvisloc) os.system('rm -rf ' + modelvisloc) tb.open(visdataloc) tb.copy(modelvisloc) tb.close() tb.open(modelvisloc, nomodify=False) datashape = tb.getcol('DATA').shape vis_complex = vis_complex.reshape(datashape) tb.putcol('DATA', vis_complex) tb.close()
def clearflagrow(msfile, mode='clear'): ''' :param msfile: :param mode: FLAG_ROW operation default: 'clear': (default) clear the FLAG_ROW 'list': to list existing FLAG_ROW :return: ''' if mode == 'list': tb.open(msfile, nomodify=True) a = tb.getcol('FLAG_ROW') nfrows = np.sum(a) nrows = float(len(a)) print('{:0d} out of {:.0f} ({:.0f}%) rows are flagged in {}'.format(nfrows, nrows, nfrows / nrows * 100, os.path.basename(msfile))) elif mode == 'clear': tb.open(msfile, nomodify=False) a = tb.getcol('FLAG_ROW') a[:] = False tb.putcol('FLAG_ROW', a) print('reset successfully') tb.close()
def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms, doscaling, keep_nsclms, fileidx): from taskinit import tb, casalog filename = filelist[fileidx] uv = aipy.miriad.UV(filename) # try: msname0 = list(filename.split('/')[-1]) msname = visprefix + ''.join(msname0) + '.ms' uv.select('antennae', 0, 1, include=True) uv.select('polarization', -5, -5, include=True) times = [] uv.rewind() for preamble, data in uv.all(): uvw, t, (i, j) = preamble times.append(t) uv.select('clear', -1, -1, include=True) times = ipe.jd2mjds(np.asarray(times)) inttime = np.median((times - np.roll(times, 1))[1:]) / 60 time_steps = len(times) durtim = int((times[-1] - times[0]) / 60 + inttime) time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'].replace('\x00', '') antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.where(uv['sfreq'] > 0)[0] nf = len(good_idx) npol = uv['npol'] nants = uv['nants'] source_id = uv['source'].replace('\x00', '') sfreq = uv['sfreq'][good_idx] sdf = uv['sdf'][good_idx] ra, dec = uv['ra'], uv['dec'] nbl = nants * (nants - 1) / 2 bl2ord = ipe.bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf) nband = len(chan_band) uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 data = ma.masked_array(ma.masked_invalid(data), fill_value=0.0) out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask # if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs if doscaling: out2 = out.copy() for i0 in antlist: for j0 in antlist: if i0 < j0: i, j = i0 - 1, j0 - 1 out2[:, :, :, bl2ord[i, j]] = out[:, :, :, bl2ord[i, j]] / np.sqrt( np.abs(out[:, :, :, bl2ord[i, i]]) * np.abs(out[:, :, :, bl2ord[j, j]])) out2 = out2.reshape(npol, nf, nrows) out2[np.isnan(out2)] = 0 out2[np.isinf(out2)] = 0 # out2 = ma.masked_array(ma.masked_invalid(out2), fill_value=0.0) out = out.reshape(npol, nf, nrows) * 1e4 flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows) uvwarray = np.tile(uvwarray, (1, nband)) sigma = np.ones((4, nrows), dtype=np.float) + 1 sigma = np.tile(sigma, (1, nband)) casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format( filename, (time.time() - time0))) if not nocreatms: modelms = ipe.creatms(filename, visprefix) os.system('mv {} {}'.format(modelms, msname)) else: casalog.post('----------------------------------------') casalog.post('copying standard MS to {0}'.format( msname, (time.time() - time0))) casalog.post('----------------------------------------') os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % modelms + " %s" % msname) casalog.post( 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format( msname, (time.time() - time0))) tb.open(msname, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() nchannels = len(cband['cidx']) for row in range(nrows): if not doscaling or keep_nsclms: tb.putcell('DATA', (row + l * nrows), out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) tb.putcell('FLAG', (row + l * nrows), flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format( (l + 1), time.time() - time1)) tb.putcol('UVW', uvwarray) tb.putcol('SIGMA', sigma) tb.putcol('WEIGHT', 1.0 / sigma**2) timearr = times timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape(nband * npairs * time_steps) tb.putcol('TIME', timearr) tb.putcol('TIME_CENTROID', timearr) scan_id = tb.getcol('SCAN_NUMBER') scan_id *= 0 tb.putcol('SCAN_NUMBER', scan_id) colnames = tb.colnames() cols2rm = ["MODEL_DATA", "CORRECTED_DATA"] for l in range(len(cols2rm)): if cols2rm[l] in colnames: tb.removecols(cols2rm[l]) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the OBSERVATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/OBSERVATION', nomodify=False) tb.putcol( 'TIME_RANGE', np.asarray([times[0] - 0.5 * inttime, times[-1] + 0.5 * inttime]).reshape(2, 1)) tb.putcol('OBSERVER', ['EOVSA team']) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POINTING table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POINTING', nomodify=False) timearr = times.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape(nband * time_steps * nants) tb.putcol('TIME', timearr) tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time) direction = tb.getcol('DIRECTION') direction[0, 0, :] = ra direction[1, 0, :] = dec tb.putcol('DIRECTION', direction) target = tb.getcol('TARGET') target[0, 0, :] = ra target[1, 0, :] = dec tb.putcol('TARGET', target) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the SOURCE table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/SOURCE', nomodify=False) radec = tb.getcol('DIRECTION') radec[0], radec[1] = ra, dec tb.putcol('DIRECTION', radec) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False) pol_id = tb.getcol('POLARIZATION_ID') pol_id *= 0 tb.putcol('POLARIZATION_ID', pol_id) # spw_id = tb.getcol('SPECTRAL_WINDOW_ID') # spw_id *= 0 # tb.putcol('SPECTRAL_WINDOW_ID', spw_id) tb.close() # casalog.post('----------------------------------------') # casalog.post("Updating the POLARIZATION table of" '%s' % msname) # casalog.post('----------------------------------------') # tb.open(msname + '/POLARIZATION/', nomodify=False) # tb.removerows(rownrs=np.arange(1, nband, dtype=int)) # tb.close() casalog.post('----------------------------------------') casalog.post("Updating the FIELD table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/FIELD/', nomodify=False) delay_dir = tb.getcol('DELAY_DIR') delay_dir[0], delay_dir[1] = ra, dec tb.putcol('DELAY_DIR', delay_dir) phase_dir = tb.getcol('PHASE_DIR') phase_dir[0], phase_dir[1] = ra, dec tb.putcol('PHASE_DIR', phase_dir) reference_dir = tb.getcol('REFERENCE_DIR') reference_dir[0], reference_dir[1] = ra, dec tb.putcol('REFERENCE_DIR', reference_dir) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME # del out, flag, uvwarray, uv, timearr, sigma # gc.collect() # if doscaling: if keep_nsclms: msname_scl = visprefix + ''.join(msname0) + '_scl.ms' os.system('cp -r {} {}'.format(msname, msname_scl)) else: msname_scl = msname tb.open(msname_scl, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname_scl) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() for row in range(nrows): tb.putcell( 'DATA', (row + l * nrows), out2[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'. format((l + 1), time.time() - time1)) tb.close() if not (timebin == '0s' and width == 1): msfile = msname + '.split' if doscaling: split(vis=msname_scl, outputvis=msname_scl + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname_scl)) msfile_scl = msname_scl + '.split' if not (doscaling and not keep_nsclms): split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname)) else: msfile = msname if doscaling: msfile_scl = msname_scl casalog.post("finished in --- %s seconds ---" % (time.time() - time0)) if doscaling: return [True, msfile, msfile_scl, durtim] else: return [True, msfile, durtim]
def concat(tb_in=[], tb_out=None): if not tb_in: print('tb_in not provided. Abort...') if os.path.exists(tb_out): os.system('rm -r ' + tb_out) # os.system('cp -r '+tb_in[0]+' '+tb_out) os.system('cp -r ' + tb_in[0] + ' ' + tb_out) tb.open(tb_out + '/SPECTRAL_WINDOW', nomodify=True) nspw = tb.nrows() tb.close() tim = [] fld = [] spw = [] ant1 = [] ant2 = [] intv = [] scan = [] obid = [] cpar = [] para = [] flag = [] snr = [] # wght=[] for ctb in tb_in: tb.open(ctb, nomodify=True) cols = tb.colnames() tim0 = tb.getcol(cols[0]) if len(tim0) == 0: continue else: tim.append(tb.getcol(cols[0])) fld.append(tb.getcol(cols[1])) spw.append(tb.getcol(cols[2])) ant1.append(tb.getcol(cols[3])) ant2.append(tb.getcol(cols[4])) intv.append(tb.getcol(cols[5])) scan.append(tb.getcol(cols[6])) obid.append(tb.getcol(cols[7])) cpar.append(tb.getcol(cols[8])) para.append(tb.getcol(cols[9])) flag.append(tb.getcol(cols[10])) snr.append(tb.getcol(cols[11])) # wght.append(tb.getcol(cols[12])) tb.close() if len(tim) == 0: print('tables have no data. Return') return -1 else: tim = np.concatenate(tim) fld = np.concatenate(fld) spw = np.concatenate(spw) ant1 = np.concatenate(ant1) ant2 = np.concatenate(ant2) intv = np.concatenate(intv) scan = np.concatenate(scan) obid = np.concatenate(obid) cpar = np.concatenate(cpar, axis=2) para = np.concatenate(para, axis=2) flag = np.concatenate(flag, axis=2) snr = np.concatenate(snr, axis=2) # wght=np.concatenate(wght) tb.open(tb_out, nomodify=False) nrows = tb.nrows() nrows_new = len(tim) tb.addrows(nrows_new - nrows) tb.putcol(cols[0], tim) tb.putcol(cols[1], fld) tb.putcol(cols[2], spw) tb.putcol(cols[3], ant1) tb.putcol(cols[4], ant2) tb.putcol(cols[5], intv) tb.putcol(cols[6], scan) tb.putcol(cols[7], obid) tb.putcol(cols[8], cpar) tb.putcol(cols[9], para) tb.putcol(cols[10], flag) tb.putcol(cols[11], snr) tb.close() return tb_out
def calibeovsa(vis=None, caltype=None, interp=None, docalib=True, doflag=True, flagant=None, doimage=False, imagedir=None, antenna=None, timerange=None, spw=None, stokes=None, doconcat=False, msoutdir=None, keep_orig_ms=True): ''' :param vis: EOVSA visibility dataset(s) to be calibrated :param caltype: :param interp: :param docalib: :param qlookimage: :param flagant: :param stokes: :param doconcat: :return: ''' if type(vis) == str: vis = [vis] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] for msfile in vis: casalog.origin('calibeovsa') if not caltype: casalog.post( "Caltype not provided. Perform reference phase calibration and daily phase calibration." ) caltype = [ 'refpha', 'phacal', 'fluxcal' ] ## use this line after the phacal is applied # caltype = ['refcal'] if not os.path.exists(msfile): casalog.post("Input visibility does not exist. Aborting...") continue if msfile.endswith('/'): msfile = msfile[:-1] if not msfile[-3:] in ['.ms', '.MS']: casalog.post( "Invalid visibility. Please provide a proper visibility file ending with .ms" ) # if not caltable: # caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype] # get band information tb.open(msfile + '/SPECTRAL_WINDOW') nspw = tb.nrows() bdname = tb.getcol('NAME') bd_nchan = tb.getcol('NUM_CHAN') bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33 # nchans = tb.getcol('NUM_CHAN') # reffreqs = tb.getcol('REF_FREQUENCY') # cenfreqs = np.zeros((nspw)) tb.close() tb.open(msfile + '/ANTENNA') nant = tb.nrows() antname = tb.getcol('NAME') antlist = [str(ll) for ll in range(len(antname) - 1)] antennas = ','.join(antlist) tb.close() # get time stamp, use the beginning of the file tb.open(msfile + '/OBSERVATION') trs = {'BegTime': [], 'EndTime': []} for ll in range(tb.nrows()): tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600, format='mjd') trs['BegTime'].append(tim0) trs['EndTime'].append(tim1) tb.close() trs['BegTime'] = Time(trs['BegTime']) trs['EndTime'] = Time(trs['EndTime']) btime = np.min(trs['BegTime']) etime = np.max(trs['EndTime']) # ms.open(vis) # summary = ms.summary() # ms.close() # btime = Time(summary['BeginTime'], format='mjd') # etime = Time(summary['EndTime'], format='mjd') ## stop using ms.summary to avoid conflicts with importeovsa t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd') print "This scan observed from {} to {} UTC".format( btime.iso, etime.iso) gaintables = [] if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal' in caltype): refcal = ra.sql2refcalX(btime) pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband) pha[np.where(refcal['flag'] == 1)] = 0. amp = refcal['amp'] amp[np.where(refcal['flag'] == 1)] = 1. t_ref = refcal['timestamp'] # find the start and end time of the local day when refcal is registered try: dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60. except: dhr = -7. bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd') et = Time(bt.mjd + 1., format='mjd') (yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day) dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/' if not os.path.exists(dirname): os.mkdir(dirname) # check if there is any ROACH reboot between the reference calibration found and the current data t_rbts = db.get_reboot(Time([t_ref, btime])) if not t_rbts: casalog.post( "Reference calibration is derived from observation at " + t_ref.iso) print "Reference calibration is derived from observation at " + t_ref.iso else: casalog.post( "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso) casalog.post("Aborting...") print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso print "Aborting..." para_pha = [] para_amp = [] calpha = np.zeros((nspw, 15, 2)) calamp = np.zeros((nspw, 15, 2)) for s in range(nspw): for n in range(15): for p in range(2): calpha[s, n, p] = pha[n, p, bd[s]] calamp[s, n, p] = amp[n, p, bd[s]] para_pha.append(np.degrees(pha[n, p, bd[s]])) para_amp.append(amp[n, p, bd[s]]) if 'fluxcal' in caltype: calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59')) t_bp = Time(calfac['timestamp'], format='lv') if int(t_mid.mjd) == int(t_bp.mjd): accalfac = calfac['accalfac'] # (ant x pol x freq) # tpcalfac = calfac['tpcalfac'] # (ant x pol x freq) caltb_autoamp = dirname + t_bp.isot[:-4].replace( ':', '').replace('-', '') + '.bandpass' if not os.path.exists(caltb_autoamp): bandpass(vis=msfile, caltable=caltb_autoamp, solint='inf', refant='eo01', minblperant=0, minsnr=0, bandtype='B', docallib=False) tb.open(caltb_autoamp, nomodify=False) # (ant x spw) bd_chanidx = np.hstack([[0], bd_nchan.cumsum()]) for ll in range(nspw): antfac = np.sqrt( accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]]) # # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]] antfac = np.moveaxis(antfac, 0, 2) cparam = np.zeros((2, bd_nchan[ll], nant)) cparam[:, :, :-3] = 1.0 / antfac tb.putcol('CPARAM', cparam + 0j, ll * nant, nant) paramerr = tb.getcol('PARAMERR', ll * nant, nant) paramerr = paramerr * 0 tb.putcol('PARAMERR', paramerr, ll * nant, nant) bpflag = tb.getcol('FLAG', ll * nant, nant) bpant1 = tb.getcol('ANTENNA1', ll * nant, nant) bpflagidx, = np.where(bpant1 >= 13) bpflag[:] = False bpflag[:, :, bpflagidx] = True tb.putcol('FLAG', bpflag, ll * nant, nant) bpsnr = tb.getcol('SNR', ll * nant, nant) bpsnr[:] = 100.0 bpsnr[:, :, bpflagidx] = 0.0 tb.putcol('SNR', bpsnr, ll * nant, nant) tb.close() msg_prompt = "Scaling calibration is derived for {}.".format( msfile) casalog.post(msg_prompt) print msg_prompt gaintables.append(caltb_autoamp) else: msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format( t_mid.datetime.strftime('%b %d, %Y'), msfile) casalog.post(msg_prompt) print msg_prompt if ('refpha' in caltype) or ('refcal' in caltype): # caltb_pha = os.path.basename(vis).replace('.ms', '.refpha') # check if the calibration table already exists caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace( '-', '') + '.refpha' if not os.path.exists(caltb_pha): gencal(vis=msfile, caltable=caltb_pha, caltype='ph', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_pha) gaintables.append(caltb_pha) if ('refamp' in caltype) or ('refcal' in caltype): # caltb_amp = os.path.basename(vis).replace('.ms', '.refamp') caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace( '-', '') + '.refamp' if not os.path.exists(caltb_amp): gencal(vis=msfile, caltable=caltb_amp, caltype='amp', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_amp) gaintables.append(caltb_amp) # calibration for the change of delay center between refcal time and beginning of scan -- hopefully none! xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False) if buf: dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv') dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15] xml, buf = ch.read_calX(4, t=t_ref) dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv') dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15] dlycen_ns_diff = dlycen_ns2 - dlycen_ns1 for n in range(2): dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n] print 'Multi-band delay is derived from delay center difference at {} & {}'.format( dly_t1.iso, dly_t2.iso) # print '=====Delays relative to Ant 14=====' # for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]): # ant = antlist[i] # print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl # dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1]) # caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0') caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace( '-', '') + '.dlycen' if not os.path.exists(caltb_dlycen): gencal(vis=msfile, caltable=caltb_dlycen, caltype='mbd', pol='X,Y', antenna=antennas, parameter=dlycen_ns_diff.flatten().tolist()) gaintables.append(caltb_dlycen) if 'phacal' in caltype: phacals = np.array( ra.sql2phacalX([bt, et], neat=True, verbose=False)) if not phacals.any() or len(phacals) == 0: print "Found no phacal records in SQL database, will skip phase calibration" else: # first generate all phacal calibration tables if not already exist t_phas = Time([phacal['t_pha'] for phacal in phacals]) # sort the array in ascending order by t_pha sinds = t_phas.mjd.argsort() t_phas = t_phas[sinds] phacals = phacals[sinds] caltbs_phambd = [] for i, phacal in enumerate(phacals): # filter out phase cals with reference time stamp >30 min away from the provided refcal time if (phacal['t_ref'].jd - refcal['timestamp'].jd) > 30. / 1440.: del phacals[i] del t_phas[i] continue else: t_pha = phacal['t_pha'] phambd_ns = phacal['pslope'] for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n] # set all flagged values to be zero phambd_ns[np.where(phacal['flag'] == 1)] = 0. caltb_phambd = dirname + t_pha.isot[:-4].replace( ':', '').replace('-', '') + '.phambd' caltbs_phambd.append(caltb_phambd) if not os.path.exists(caltb_phambd): gencal(vis=msfile, caltable=caltb_phambd, caltype='mbd', pol='X,Y', antenna=antennas, parameter=phambd_ns.flatten().tolist()) # now decides which table to apply depending on the interpolation method ("neatest" or "linear") if interp == 'nearest': tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd)) dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24. print "Selected nearest phase calibration table at " + t_phas[ tbind].iso gaintables.append(caltbs_phambd[tbind]) if interp == 'linear': # bphacal = ra.sql2phacalX(btime) # ephacal = ra.sql2phacalX(etime,reverse=True) bt_ind, = np.where(t_phas.mjd < btime.mjd) et_ind, = np.where(t_phas.mjd > etime.mjd) if len(bt_ind) == 0 and len(et_ind) == 0: print "No phacal found before or after the ms data within the day of observation" print "Skipping daily phase calibration" elif len(bt_ind) > 0 and len(et_ind) == 0: gaintables.append(caltbs_phambd[bt_ind[-1]]) elif len(bt_ind) == 0 and len(et_ind) > 0: gaintables.append(caltbs_phambd[et_ind[0]]) elif len(bt_ind) > 0 and len(et_ind) > 0: bphacal = phacals[bt_ind[-1]] ephacal = phacals[et_ind[0]] # generate a new table interpolating between two daily phase calibrations t_pha_mean = Time(np.mean( [bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]), format='mjd') phambd_ns = (bphacal['pslope'] + ephacal['pslope']) / 2. for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n] # set all flagged values to be zero phambd_ns[np.where(bphacal['flag'] == 1)] = 0. phambd_ns[np.where(ephacal['flag'] == 1)] = 0. caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace( ':', '').replace('-', '') + '.phambd' if not os.path.exists(caltb_phambd_interp): gencal(vis=msfile, caltable=caltb_phambd_interp, caltype='mbd', pol='X,Y', antenna=antennas, parameter=phambd_ns.flatten().tolist()) print "Using phase calibration table interpolated between records at " + bphacal[ 't_pha'].iso + ' and ' + ephacal['t_pha'].iso gaintables.append(caltb_phambd_interp) if docalib: clearcal(msfile) applycal(vis=msfile, gaintable=gaintables, applymode='calflag', calwt=False) # delete the interpolated phase calibration table try: caltb_phambd_interp except: pass else: if os.path.exists(caltb_phambd_interp): shutil.rmtree(caltb_phambd_interp) if doflag: # flag zeros and NaNs flagdata(vis=msfile, mode='clip', clipzeros=True) if flagant: try: flagdata(vis=msfile, antenna=flagant) except: print "Something wrong with flagant. Abort..." if doimage: from matplotlib import pyplot as plt from suncasa.utils import helioimage2fits as hf from sunpy import map as smap if not antenna: antenna = '0~12' if not stokes: stokes = 'XX' if not timerange: timerange = '' if not spw: spw = '1~3' if not imagedir: imagedir = '.' #(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day) #dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/' #if not os.path.exists(dirname): # os.makedirs(dirname) bds = [spw] nbd = len(bds) imgs = [] for bd in bds: if '~' in bd: bdstr = bd.replace('~', '-') else: bdstr = str(bd).zfill(2) imname = imagedir + '/' + os.path.basename(msfile).replace( '.ms', '.bd' + bdstr) print 'Cleaning image: ' + imname try: clean(vis=msfile, imagename=imname, antenna=antenna, spw=bd, timerange=timerange, imsize=[512], cell=['5.0arcsec'], stokes=stokes, niter=500) except: print 'clean not successfull for band ' + str(bd) else: imgs.append(imname + '.image') junks = ['.flux', '.mask', '.model', '.psf', '.residual'] for junk in junks: if os.path.exists(imname + junk): shutil.rmtree(imname + junk) tranges = [btime.iso + '~' + etime.iso] * nbd fitsfiles = [img.replace('.image', '.fits') for img in imgs] hf.imreg(vis=msfile, timerange=tranges, imagefile=imgs, fitsfile=fitsfiles, usephacenter=False) plt.figure(figsize=(6, 6)) for i, fitsfile in enumerate(fitsfiles): plt.subplot(1, nbd, i + 1) eomap = smap.Map(fitsfile) sz = eomap.data.shape if len(sz) == 4: eomap.data = eomap.data.reshape((sz[2], sz[3])) eomap.plot_settings['cmap'] = plt.get_cmap('jet') eomap.plot() eomap.draw_limb() eomap.draw_grid() plt.show() if doconcat: if len(vis) > 1: # from suncasa.eovsa import concateovsa as ce from suncasa.tasks import concateovsa_cli as ce if msoutdir is None: msoutdir = './' concatvis = os.path.basename(vis[0]) concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms' ce.concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=keep_orig_ms, cols2rm="model,corrected") return [concatvis] else: return vis
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False, timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""): if concatvis[-1] == os.path.sep: concatvis = concatvis[:-1] if os.path.sep not in concatvis: visprefix = './' else: visprefix = os.path.dirname(concatvis) + os.path.sep msfiles = vis msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] datacolumn = datacolumn.lower() if datacolumn == 'data': print 'DATA columns will be concatenated.' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) elif datacolumn == 'corrected': # try: print 'CORRECTED columns will be concatenated.' tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = os.path.join(tmpdir, os.path.basename(str(ll))) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) else: raise ValueError('Please set datacolumn to be "data" or "corrected"!') if msfiles_: concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) os.system('rm -rf {}'.format(tmpdir)) else: concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False) nrows = tb.nrows() pol_id = tb.getcol('POLARIZATION_ID') tb.removerows(np.where(pol_id != 0)[0]) tb.close() tb.open(concatvis, nomodify=False) dd_id = tb.getcol('DATA_DESC_ID') idx_dd_id, = np.where(dd_id >= nrows / 2) dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2 tb.putcol('DATA_DESC_ID', dd_id) tb.close() tb.open(concatvis + '/FIELD', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis + '/SOURCE', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) fldid = tb.getcol('FIELD_ID') newfldid = np.zeros(len(fldid), dtype='int') tb.putcol('FIELD_ID', newfldid) colnames = tb.colnames() cols2rm = cols2rm.upper() cols2rm = cols2rm.split(',') for l in range(len(cols2rm)): col = cols2rm[l] + '_DATA' if col in colnames: try: tb.removecols(col) print 'Column {} removed.'.format(col) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
def pimporteovsa(idbfiles, ncpu=8, timebin=None, width=None, visprefix=None, nocreatms=False, doconcat=False, modelms=''): casalog.origin('pimporteovsa') # # Initialize the helper class # pdh = ParallelDataHelper("importeovsa", locals()) # # # Validate input and output parameters # try: # pdh.setupIO() # except Exception, instance: # casalog.post('%s' % instance, 'ERROR') # return False # import glob # idbfiles = glob.glob('IDB*_unrot')[:4] # ncpu = 4 # timebin = None # width = None # visprefix = None # nocreatms = False # doconcat = False # modelms = '' # if not (type(ncpu) is int): # casalog.post('ncpu should be an integer') # ncpu = 8 if type(idbfiles) == Time: filelist = ri.get_trange_files(idbfiles) else: # If input type is not Time, assume that it is the list of files to read filelist = idbfiles if type(filelist) == str: filelist = [filelist] for f in filelist: if not os.path.exists(f): casalog.post("Some files in filelist are invalid. Aborting...") # return False if not visprefix: visprefix = './' if not timebin: timebin = '0s' if not width: width = 1 if not modelms: if nocreatms: filename = filelist[0] modelms = ipe.creatms(filename, visprefix) else: if not os.path.exists(modelms): if nocreatms: filename = filelist[0] modelms = ipe.creatms(filename, visprefix) iterable = range(len(filelist)) imppart = partial(importeovsa_iter, filelist, timebin, width, visprefix, nocreatms, modelms) t0 = time.time() casalog.post('Perform importeovsa in parallel ...') pool = mp.Pool(ncpu) res = pool.map(imppart, iterable) pool.close() pool.join() t1 = time.time() timelapse = t1 - t0 print 'It took %f secs to complete' % timelapse results = pd.DataFrame({'succeeded': [], 'msfile': [], 'durtim': []}) for r in res: results = results.append( pd.DataFrame({ 'succeeded': [r[0]], 'msfile': [r[1]], 'durtim': [r[2]] })) if doconcat: msname = list(os.path.basename(filelist[0])) concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format( int(results['durtim'].sum())) msfile = results.loc[results['succeeded'] == True, :].msfile.tolist() concat(vis=msfile, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) tb.close() for ll in msfile: os.system('rm -rf {}'.format(ll)) return True
def importeovsa(idbfiles, timebin=None, width=None, visprefix=None, nocreatms=False, doconcat=False, modelms=''): casalog.origin('importeovsa') # # Initialize the helper class # pdh = ParallelDataHelper("importeovsa", locals()) # # # Validate input and output parameters # try: # pdh.setupIO() # except Exception, instance: # casalog.post('%s' % instance, 'ERROR') # return False if type(idbfiles) == Time: filelist = ri.get_trange_files(idbfiles) else: # If input type is not Time, assume that it is the list of files to read filelist = idbfiles if type(filelist) == str: filelist = [filelist] for f in filelist: if not os.path.exists(f): casalog.post("Some files in filelist are invalid. Aborting...") return False if not visprefix: visprefix = './' if not timebin: timebin = '0s' if not width: width = 1 if not modelms: if nocreatms: filename = filelist[0] modelms = ipe.creatms(filename, visprefix) else: if not os.path.exists(modelms): if nocreatms: filename = filelist[0] modelms = ipe.creatms(filename, visprefix) msfile = [] time_concat = [] for filename in filelist: uv = aipy.miriad.UV(filename) # if filename.split('/')[-1][0:3] == 'UDB': # uv_str = uv_hex_rm(uv) # else: # uv_str = uv uv.select('antennae', 0, 1, include=True) uv.select('polarization', -5, -5, include=True) times = [] uv.rewind() for preamble, data in uv.all(): uvw, t, (i, j) = preamble times.append(t) uv.select('clear', -1, -1, include=True) times = ipe.jd2mjds(np.asarray(times)) inttime = np.median((times - np.roll(times, 1))[1:]) / 60 time_steps = len(times) time_concat.append(int((times[-1] - times[0]) / 60 + inttime)) time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'].replace('\x00', '') antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.where(uv['sfreq'] > 0)[0] nf = len(good_idx) npol = uv['npol'] nants = uv['nants'] source_id = uv['source'].replace('\x00', '') sfreq = uv['sfreq'][good_idx] sdf = uv['sdf'][good_idx] ra, dec = uv['ra'], uv['dec'] nbl = nants * (nants - 1) / 2 bl2ord = ipe.bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf) nband = len(chan_band) uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask # if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs out = out.reshape(npol, nf, nrows) flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows) uvwarray = np.tile(uvwarray, (1, nband)) sigma = np.ones((4, nrows), dtype=np.float) + 1 sigma = np.tile(sigma, (1, nband)) casalog.post( 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format( filename, (time.time() - time0))) msname = list(filename.split('/')[-1]) msname = visprefix + ''.join(msname) + '.ms' if not nocreatms: modelms = ipe.creatms(filename, visprefix) os.system('mv {} {}'.format(modelms, msname)) else: casalog.post('----------------------------------------') casalog.post('copying standard MS to {0}'.format( msname, (time.time() - time0))) casalog.post('----------------------------------------') os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % modelms + " %s" % msname) casalog.post( 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'. format(msname, (time.time() - time0))) tb.open(msname, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() nchannels = len(cband['cidx']) for row in range(nrows): tb.putcell('DATA', (row + l * nrows), out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) tb.putcell( 'FLAG', (row + l * nrows), flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'. format((l + 1), time.time() - time1)) tb.putcol('UVW', uvwarray) tb.putcol('SIGMA', sigma) tb.putcol('WEIGHT', 1.0 / sigma**2) timearr = times timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape(nband * npairs * time_steps) tb.putcol('TIME', timearr) tb.putcol('TIME_CENTROID', timearr) scan_id = tb.getcol('SCAN_NUMBER') scan_id *= 0 tb.putcol('SCAN_NUMBER', scan_id) colnames = tb.colnames() cols2rm = ["MODEL_DATA", "CORRECTED_DATA"] for l in range(len(cols2rm)): if cols2rm[l] in colnames: tb.removecols(cols2rm[l]) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the OBSERVATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/OBSERVATION', nomodify=False) tb.putcol( 'TIME_RANGE', np.asarray([times[0] - 0.5 * inttime, times[-1] + 0.5 * inttime]).reshape(2, 1)) tb.putcol('OBSERVER', ['EOVSA team']) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POINTING table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POINTING', nomodify=False) timearr = times.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape(nband * time_steps * nants) tb.putcol('TIME', timearr) tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time) direction = tb.getcol('DIRECTION') direction[0, 0, :] = ra direction[1, 0, :] = dec tb.putcol('DIRECTION', direction) target = tb.getcol('TARGET') target[0, 0, :] = ra target[1, 0, :] = dec tb.putcol('TARGET', target) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the SOURCE table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/SOURCE', nomodify=False) radec = tb.getcol('DIRECTION') radec[0], radec[1] = ra, dec tb.putcol('DIRECTION', radec) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False) pol_id = tb.getcol('POLARIZATION_ID') pol_id *= 0 tb.putcol('POLARIZATION_ID', pol_id) # spw_id = tb.getcol('SPECTRAL_WINDOW_ID') # spw_id *= 0 # tb.putcol('SPECTRAL_WINDOW_ID', spw_id) tb.close() # pdb.set_trace() casalog.post('----------------------------------------') casalog.post("Updating the POLARIZATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POLARIZATION/', nomodify=False) print tb.nrows() tb.removerows(rownrs=np.arange(1, nband, dtype=int)) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the FIELD table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/FIELD/', nomodify=False) delay_dir = tb.getcol('DELAY_DIR') delay_dir[0], delay_dir[1] = ra, dec tb.putcol('DELAY_DIR', delay_dir) phase_dir = tb.getcol('PHASE_DIR') phase_dir[0], phase_dir[1] = ra, dec tb.putcol('PHASE_DIR', phase_dir) reference_dir = tb.getcol('REFERENCE_DIR') reference_dir[0], reference_dir[1] = ra, dec tb.putcol('REFERENCE_DIR', reference_dir) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME del out, flag, uvwarray, uv, timearr, sigma gc.collect() # if not (timebin == '0s' and width == 1): split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname)) msfile.append(msname + '.split') else: msfile.append(msname) casalog.post("finished in --- %s seconds ---" % (time.time() - time0)) if doconcat: msname = list(filelist[0].split('/')[-1]) concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format( int(sum(time_concat))) concat(vis=msfile, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) tb.close() for ll in msfile: os.system('rm -rf {}'.format(ll)) return True
def writeVis(vis_complex, visdataloc, modelvisloc, miriad=False): """ Parameters ---------- vis_complex: numpy.array visdataloc: string uv-data filename miriad: Boolean True: not using CASA, just python data manipulation, hence can run in ipython, False: use CASA taskinit module, must run inside CASA Returns ------- modelvisloc: string uv-model filename """ if miriad: os.system('rm -rf ' + modelvisloc) cmd = 'cp ' + visdataloc + ' ' + modelvisloc os.system(cmd) # get the real and imaginary components real = numpy.real(vis_complex) imag = numpy.imag(vis_complex) # replace data visibilities with model visibilities visfile = fits.open(modelvisloc, mode='update') visibilities = visfile[0].data visheader = visfile[0].header if visheader['NAXIS'] == 7: # expect uu.dim = 4, hence real.ndim also 4 nfreq = visibilities['DATA'][0, 0, 0, 0, :, 0, 0].size # to match uvutil.uvload() instead of fixing array size mismatch with miriad= in the function if nfreq > 1: visibilities['DATA'][:, 0, 0, :, :, :, 0] = real visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag else: try: visibilities['DATA'][:, 0, 0, :, :, :, 0] = real visibilities['DATA'][:, 0, 0, :, :, :, 1] = imag except ValueError: # mistmatach arise when uu.ndim is 3 # which would be the case if nfreq = 0 # vis, 0, 0, spw, chan(freq), pol, (real, imag, weights) visibilities['DATA'][:, 0, 0, :, 0, :, 0] = real visibilities['DATA'][:, 0, 0, :, 0, :, 1] = imag elif visheader['NAXIS'] == 6: visibilities['DATA'][:, 0, 0, :, :, 0] = real visibilities['DATA'][:, 0, 0, :, :, 1] = imag else: print("Visibility dataset has >7 or <6 axes. I can't read this.") #visibilities['DATA'][:, 0, 0, :, :, 2] = wgt # replace the data visibilities with the model visibilities visfile[0].data = visibilities visfile.flush() else: from taskinit import tb print("Writing visibility data to " + modelvisloc) os.system('rm -rf ' + modelvisloc) tb.open(visdataloc) tb.copy(modelvisloc) tb.close() tb.open(modelvisloc, nomodify=False) datashape = tb.getcol('DATA').shape vis_complex = vis_complex.reshape(datashape) tb.putcol('DATA', vis_complex) tb.close()