def concateovsa(msname, msfiles, visprefix='./', doclearcal=True, keep_orig_ms=False, cols2rm=["MODEL_DATA", "CORRECTED_DATA"]): concatvis = visprefix + msname msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] if doclearcal: print 'Warning: Corrected column in the input ms file will be cleared!!!' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) else: try: tmpdir = visprefix + '/tmp_ms/' if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = tmpdir + os.path.basename(str(ll)) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) except: print 'Warning: Corrected column not found in the input ms file.' msfiles_ = msfiles if msfiles_: concat(vis=msfiles_, concatvis=concatvis, timesort=True) else: concat(vis=msfiles, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) colnames = tb.colnames() for l in range(len(cols2rm)): if cols2rm[l] in colnames: try: tb.removecols(cols2rm[l]) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
def splitX(vis, datacolumn2='MODEL_DATA', **kwargs): import os from clearcal_cli import clearcal_cli as clearcal from split_cli import split_cli as split ''' :param msfile: :param datacolumn: :param datacolumn2: :return: ''' kwargs2 = kwargs.copy() datacolumn2 = datacolumn2.upper() outmsfile = kwargs['outputvis'] if outmsfile.endswith('/'): outmsfile = outmsfile[:-1] if os.path.exists(outmsfile): os.system('rm -rf {}'.format(outmsfile)) if os.path.exists('{}.flagversions'.format(outmsfile)): os.system('rm -rf {}.flagversions'.format(outmsfile)) split(vis, **kwargs) for k in ['datacolumn', 'outputvis']: if k in kwargs2: kwargs2.pop(k) kwargs2['outputvis'] = 'tmpms.ms' kwargs2['datacolumn'] = datacolumn2.replace('_DATA', '') if os.path.exists('tmpms.ms'): os.system('rm -rf tmpms.ms') split(vis, **kwargs2) tb.open('tmpms.ms') nrows = tb.nrows() data = [] for row in tqdm(range(nrows), desc='getting {} column'.format(datacolumn2), ascii=True): data.append(tb.getcell('DATA', row)) tb.close() clearcal(outmsfile, addmodel=True) tb.open(outmsfile, nomodify=False) for row in tqdm(range(nrows), desc='writing {} column'.format(datacolumn2), ascii=True): tb.putcell(datacolumn2, row, data[row]) tb.close() os.system('rm -rf tmpms.ms') return outmsfile
casalog.post(str(x), origin='singlefield') mses = list(Qmses.keys()) fullpath_mses = [ '../' + ms[:-3] + "_continuum_split_for_selfcal.ms" for ms in mses if ms in Qmses ] cont_vis = [] for ms in fullpath_mses: splitagain = ms[:-3] + "_wtermmerge.ms" myprint("{0} -> {1}".format(ms, splitagain)) if not os.path.exists(splitagain): assert split(vis=ms, outputvis=splitagain, datacolumn='corrected') cont_vis.append(splitagain) # imagename = '18A-229_Q_mosaic_selfcal_wtermmerge_iter0_dirty' # if not os.path.exists(imagename+".image.tt0.pbcor"): # # do a full-mosaic clean to enable mask creation # tclean( # vis=cont_vis, # spw='', # field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q", # phasecenter='J2000 17h47m19.693 -28d23m11.527', # imsize=[9000,9000], # cell='0.02arcsec', # imagename=imagename, # niter=0, # threshold='1000mJy',
def feature_slfcal(vis, niter=200, spws=['0~1', '2~5', '6~10', '11~20', '21~30', '31~49'], slfcaltbdir='./', bright=None, pols='XX'): ''' Uses images from disk-selfcaled data as model for further self-calibration of outer antennas. This is only a good idea if there are bright active regions that provide strong signal on the londer baselines. ''' trange = ant_trange(vis) if bright is None: bright = [True] * len(spws) # Insert model into ms and do "inf" gaincal, appending to table each subsequent time if os.path.exists('images_init'): os.system('rm -rf images_init') os.system('mv images images_init') clearcal(vis, addmodel=True) fd_images(vis, cleanup=False, niter=niter, spws=spws, bright=bright) # Does shallow clean for selfcal purposes tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d') caltb = os.path.join(slfcaltbdir, tdate + '_d1.pha') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) for s, sp in enumerate(spws): if bright[s]: spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')]) imname = "images/briggs" + spwstr + '.model' if sp == '31~49': # The high-band image is only made to band 43, so adjust the name imname = 'images/briggs31-43.model' ft(vis=vis, spw=sp, model=imname, usescratch=True, incremental=True) if pols == 'XXYY': mstl.modeltransfer(vis, spw=sp) if pols == 'XXYY': mstl.gaincalXY(vis=vis, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda', combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G', minsnr=1.0, calmode='p', append=False) else: gaincal(vis=vis, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda', combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G', minsnr=1.0, calmode='p', append=False) # Apply the corrections to the data and split to a new ms applycal(vis=vis, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False, applymode="calonly") vis1 = 'dslf1_' + vis if os.path.exists(vis1): os.system('rm -rf {}'.format(vis1)) split(vis, outputvis=vis1, datacolumn="corrected") caltb = os.path.join(slfcaltbdir, tdate + '_d2.pha') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) # Move the existing images directory so that a new one will be created if os.path.exists('images_ftcal_rnd1'): os.system('rm -rf images_ftcal_rnd1') # shutil.move('images', 'old_images2') os.system('mv images images_ftcal_rnd1') # Make new model images for another round of selfcal fd_images(vis1, cleanup=False, niter=niter, spws=spws, bright=bright) for s, sp in enumerate(spws): if bright[s]: spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')]) imname = "images/briggs" + spwstr + '.model' if sp == '31~49': # The high-band image is only made to band 43, so adjust the name imname = 'images/briggs31-43.model' ft(vis=vis1, spw=sp, model=imname, usescratch=True) if pols == 'XXYY': mstl.modeltransfer(vis1, spw=sp) if pols == 'XXYY': mstl.gaincalXY(vis=vis1, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda', combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict", gaintype='G', minsnr=1.0, calmode='p', append=False) else: gaincal(vis=vis1, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda', combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict", gaintype='G', minsnr=1.0, calmode='p', append=False) # Apply the corrections to the data and split to a new ms applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False, applymode="calonly") vis2 = 'dslf2_' + vis if os.path.exists(vis2): os.system('rm -rf {}'.format(vis2)) mstl.splitX(vis1, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data") if os.path.exists('images_ftcal_rnd2'): os.system('rm -rf images_ftcal_rnd2') os.system('mv images images_ftcal_rnd2') return vis2
def disk_slfcal(vis, slfcaltbdir='./', active=False, clearcache=False, pols='XX'): ''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name) add a model disk based on the solar disk size for that date and perform multiple selfcal adjustments (two phase and one amplitude), and write out a final selfcaled database with the disk subtracted. Returns the name of the final database. ''' trange = ant_trange(vis) if vis.endswith('/'): vis = vis[:-1] # Use vis name to determine date, and hence number of bands spw2band = np.array([0, 1] + range(4, 52)) defaultfreq = 1.1 + 0.325 * (spw2band + 0.5) # Calculate the center frequency of each spectral window if mstl.get_trange(vis)[0].mjd > 58536: # After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49 nbands = 52 freq = defaultfreq else: # Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30 nbands = 34 freq = 1.419 + np.arange(nbands) / 2. caltbs = [] slashdate = trange[:10] # Verify that the vis is not in the current working directory ''' if os.getcwd() == os.path.dirname(vis): print('Cannot copy vis file onto itself.') print('Please change to a different working directory') return None # Copy original ms to local directory if os.path.exists(os.path.basename(vis)): shutil.rmtree(os.path.basename(vis)) print('Copy {} to working directory {}.'.format(vis, os.getcwd())) shutil.copytree(vis, os.path.basename(vis)) vis = os.path.basename(vis) ''' if not active: clearcal(vis) flagmanager(vis, mode='save', versionname='with-RFI-or-BURSTS') ## automaticaly flag any high amplitudes from flares or RFI flagdata(vis=vis, mode="tfcrop", spw='', action='apply', display='', timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=False) flagmanager(vis, mode='save', versionname='without-RFI-or-BURSTS') dsize, fdens = calc_diskmodel(slashdate, nbands, freq, defaultfreq) diskxmlfile = vis + '.SOLDISK.xml' # Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name) msfile, diskim = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile, active=active) if pols == 'XXYY': caltbs_ = {'XX': [], 'YY': []} pols_ = ['XX', 'YY'] msfileXY = {} for pol in pols_: msfileXY[pol] = '.'.join([msfile, pol]) if os.path.exists(msfileXY[pol]): os.system('rm -rf {}'.format(msfileXY[pol])) mstl.splitX(vis=msfile, outputvis=msfileXY[pol], correlation=pol, datacolumn='data', datacolumn2='MODEL_DATA') tdate = mstl.get_trange(msfile)[0].datetime.strftime('%Y%m%d') caltb = os.path.join(slfcaltbdir, tdate + '_1.pha') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) if pols == 'XXYY': mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, selectdata=True, uvrange="", antenna="0~12&0~12", solint="inf", combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False) for pol in pols_: caltb_ = '.'.join([caltb, pol]) caltbs_[pol].append(caltb_) else: gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="inf", combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False) caltbs.append(caltb) caltb = os.path.join(slfcaltbdir, tdate + '_2.pha') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) # Second round of phase selfcal on the disk using solution interval "1min" if pols == 'XXYY': mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_, selectdata=True, uvrange="", antenna="0~12&0~12", solint="10min", combine="scan", interp="linear", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False) for pol in pols_: caltb_ = '.'.join([caltb, pol]) caltbs_[pol].append(caltb_) else: gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="10min", combine="scan", gaintable=caltbs, interp="linear", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False) caltbs.append(caltb) caltb = os.path.join(slfcaltbdir, tdate + '_3.amp') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) # Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT) if pols == 'XXYY': mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_, selectdata=True, uvrange="", antenna="0~12&0~12", timerange=trange, interp="linear", solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a", append=False) for pol in pols_: caltb_ = '.'.join([caltb, pol]) caltbs_[pol].append(caltb_) else: gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", timerange=trange, gaintable=caltbs, interp="linear", solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a", append=False) mstl.flagcaltboutliers(caltb, limit=[0.125, 8.0]) # mstl.flagcaltboutliers(caltb, limit=[0.5, 2.0]) caltbs.append(caltb) # Split out corrected data and model and do uvsub vis2 = 'slf3_' + msfile if os.path.exists(vis2): os.system('rm -rf {}'.format(vis2)) if os.path.exists(vis2 + '.flagversions'): os.system('rm -rf {}'.format(vis2 + '.flagversions')) # flagmanager(msfile, mode='restore', versionname='with-RFI-or-BURSTS') clearcal(msfile) applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltbs, interp="linear", calwt=False, applymode="calonly") split(msfile, outputvis=vis2, datacolumn="corrected") for sp, dkim in tqdm(enumerate(diskim), desc='Inserting disk model', ascii=True): ft(vis=vis2, spw=str(sp), field='', model=str(dkim), nterms=1, reffreq="", complist="", incremental=False, usescratch=True) # mstl.modeltransfer(msfile, spw='{}'.format(sp)) uvsub(vis=vis2, reverse=False) # Final split to final = 'final_' + msfile if os.path.exists(final): os.system('rm -rf {}'.format(final)) if os.path.exists(final + '.flagversions'): os.system('rm -rf {}'.format(final + '.flagversions')) split(vis2, outputvis=final, datacolumn='corrected') os.system('mv {} {}'.format(msfile + '.flagversions', final + '.flagversions')) # Remove the interim ms files if clearcache: if os.path.exists(msfile): os.system('rm -rf {}'.format(msfile)) if os.path.exists(msfile + '.flagversions'): os.system('rm -rf {}'.format(msfile + '.flagversions')) if os.path.exists(vis2): os.system('rm -rf {}'.format(vis2)) if os.path.exists(vis2 + '.flagversions'): os.system('rm -rf {}'.format(vis2 + '.flagversions')) # Return the name of the selfcaled ms return final, diskxmlfile
def disk_slfcal(vis, slfcaltbdir='./'): ''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name) add a model disk based on the solar disk size for that date and perform multiple selfcal adjustments (two phase and one amplitude), and write out a final selfcaled database with the disk subtracted. Returns the name of the final database. ''' trange = ant_trange(vis) # Use vis name to determine date, and hence number of bands spw2band = np.array([0, 1] + range(4, 52)) defaultfreq = 1.1 + 0.325 * (spw2band + 0.5) # Calculate the center frequency of each spectral window if mstl.get_trange(vis)[0].mjd > 58536: # After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49 nbands = 52 freq = defaultfreq else: # Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30 nbands = 34 freq = np.hstack([[1.419], 2.0 + 0.5 * (np.arange(32)) + (0.5 - 0.081)]) slashdate = trange[:10] # Verify that the vis is not in the current working directory if os.getcwd() == os.path.dirname(vis): print('Cannot copy vis file onto itself.') print('Please change to a different working directory') return None # Copy original ms to local directory if os.path.exists(os.path.basename(vis)): shutil.rmtree(os.path.basename(vis)) print('Copy {} to working directory {}.'.format(vis, os.getcwd())) shutil.copytree(vis, os.path.basename(vis)) vis = os.path.basename(vis) clearcal(vis) ## automaticaly flag impossibly high amplitudes flagdata(vis=vis, mode="tfcrop", spw='', correlation='ABS_XX', action='apply', display='', timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=True) # Default disk size measured for 2019/09/03 # todo add monthly fitting procedure for the disk size and flux density defaultsize = np.array([990.6, 989.4, 988.2, 987.1, 986.0, 984.9, 983.8, 982.7, 981.7, 980.7, 979.7, 978.8, 977.8, 976.9, 976.0, 975.2, 974.3, 973.5, 972.7, 972.0, 971.2, 970.5, 969.8, 969.1, 968.5, 967.8, 967.2, 966.7, 966.1, 965.6, 965.1, 964.6, 964.1, 963.7, 963.3, 962.9, 962.5, 962.1, 961.8, 961.5, 961.3, 961.0, 960.8, 960.6, 960.4, 960.2, 960.1, 960.0, 959.9, 959.8]) # Get current solar distance and modify the default size accordingly fac = eph.get_sunearth_distance('2019/09/03') / eph.get_sunearth_distance(slashdate) newsize = defaultsize * fac.to_value() if nbands == 34: # Interpolate size to 31 spectal windows newsize = np.polyval(np.polyfit(defaultfreq, newsize, 5), freq) dsize = np.array([str(i)[:5] + 'arcsec' for i in newsize], dtype='S12') # These are nominal flux densities * 2, determined on 2019/09/03 defaultfdens = np.array([891282, 954570, 1173229, 1245433, 1373730, 1506802, 1613253, 1702751, 1800721, 1946756, 2096020, 2243951, 2367362, 2525968, 2699795, 2861604, 3054829, 3220450, 3404182, 3602625, 3794312, 3962926, 4164667, 4360683, 4575677, 4767210, 4972824, 5211717, 5444632, 5648266, 5926634, 6144249, 6339863, 6598018, 6802707, 7016012, 7258929, 7454951, 7742816, 7948976, 8203206, 8411834, 8656720, 8908130, 9087766, 9410760, 9571365, 9827078, 10023598, 8896671]) fdens = defaultfdens if nbands == 34: # Interpolate size to 31 spectal windows fdens = np.polyval(np.polyfit(defaultfreq, fdens, 5), freq) diskxmlfile = vis + '.SOLDISK.xml' # Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name) msfile = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile) tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d') caltb = os.path.join(slfcaltbdir, tdate + '_1.pha') if os.path.exists(caltb): os.system('rm -rf {}'.format(caltb)) # Phase selfcal on the disk using solution interval "infinite" gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="inf", combine="scan", refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False) applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False, applymode="calonly") # Split corrected data and model to a new ms for round 2 of phase selfcal vis1 = 'slf_' + msfile if os.path.exists(vis1): os.system('rm -rf {}'.format(vis1)) mstl.splitX(msfile, outputvis=vis1, datacolumn="corrected", datacolumn2="model_data") caltb = os.path.join(slfcaltbdir, tdate + '_2.pha') if os.path.exists(caltb): os.system('rm -rf {}'.format(caltb)) # Second round of phase selfcal on the disk using solution interval "1min" gaincal(vis=vis1, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="1min", combine="scan", refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False) applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False, applymode="calonly") # Split corrected data and model to a new ms vis2 = 'slf2_' + msfile if os.path.exists(vis2): os.system('rm -rf {}'.format(vis2)) mstl.splitX(vis, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data") caltb = os.path.join(slfcaltbdir, tdate + '_3.amp') if os.path.exists(caltb): os.system('rm -rf {}'.format(caltb)) # Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT) gaincal(vis=vis2, caltable=caltb, selectdata=True, uvrange=">0.1Klambda", antenna="0~12&0~12", timerange=trange, solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a", append=False) applycal(vis=vis2, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False, applymode="calonly") # Split out corrected data and model and do uvsub vis3 = 'slf3_' + msfile if os.path.exists(vis3): os.system('rm -rf {}'.format(vis3)) mstl.splitX(vis, outputvis=vis3, datacolumn="corrected", datacolumn2="model_data") uvsub(vis=vis3, reverse=False) # Final split to final = 'final_' + msfile if os.path.exists(final): os.system('rm -rf {}'.format(final)) split(vis3, outputvis=final, datacolumn='corrected') # Remove the interim ms files shutil.rmtree(vis) shutil.rmtree(vis1) shutil.rmtree(vis2) shutil.rmtree(vis3) # Return the name of the selfcaled ms return final, diskxmlfile
def mk_qlook_image(trange, doimport=False, docalib=False, ncpu=10, twidth=12, stokes=None, antenna='0~12', lowcutoff_freq=3.7, imagedir=None, spws=['1~5', '6~10', '11~15', '16~25'], toTb=True, overwrite=True, doslfcal=False, verbose=False): ''' trange: can be 1) a single Time() object: use the entire day 2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00']) 3) a single or a list of UDBms file(s) 4) None -- use current date Time.now() ''' antenna0 = antenna if type(trange) == Time: mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] tsts = [l.to_datetime() for l in mslist['tstlist']] if type(trange) == str: try: date = Time(trange) mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] tsts = [l.to_datetime() for l in mslist['tstlist']] except: vis = [trange] tsts = [] for v in vis: tb.open(v + '/OBSERVATION') tsts.append( Time(tb.getcell('TIME_RANGE')[0] / 24 / 3600, format='mjd').datetime) tb.close() subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] if not stokes: stokes = 'XX' if not imagedir: imagedir = './' imres = { 'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': [], 'Synoptic': { 'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': [] } } for n, msfile in enumerate(vis): msfilebs = os.path.basename(msfile) imdir = imagedir + subdir[n] if not os.path.exists(imdir): os.makedirs(imdir) if doslfcal: slfcalms = './' + msfilebs + '.xx' split(msfile, outputvis=slfcalms, datacolumn='corrected', correlation='XX') cfreqs = getspwfreq(msfile) for spw in spws: antenna = antenna0 if spw == '': continue spwran = [s.zfill(2) for s in spw.split('~')] freqran = [cfreqs[int(s)] for s in spw.split('~')] cfreq = np.mean(freqran) bmsz = max(150. / cfreq, 20.) uvrange = '<10klambda' if doslfcal: slfcal_img = './' + msfilebs + '.slf.spw' + spw.replace( '~', '-') + '.slfimg' slfcal_tb = './' + msfilebs + '.slf.spw' + spw.replace( '~', '-') + '.slftb' try: clean(vis=slfcalms, antenna=antenna, imagename=slfcal_img, spw=spw, mode='mfs', timerange='', imagermode='csclean', psfmode='clark', imsize=[512, 512], cell=['5arcsec'], niter=100, gain=0.05, stokes='I', weighting='natural', restoringbeam=[str(bmsz) + 'arcsec'], pbcor=False, interactive=False, usescratch=True) except: print('error in cleaning spw: ' + spw) break gaincal(vis=slfcalms, refant='0', antenna=antenna, caltable=slfcal_tb, spw=spw, uvrange='', gaintable=[], selectdata=True, timerange='', solint='600s', gaintype='G', calmode='p', combine='', minblperant=3, minsnr=2, append=False) if not os.path.exists(slfcal_tb): print('No solution found in spw: ' + spw) break else: clearcal(slfcalms) delmod(slfcalms) applycal(vis=slfcalms, gaintable=[slfcal_tb], spw=spw, selectdata=True, antenna=antenna, interp='nearest', flagbackup=False, applymode='calonly', calwt=False) msfile = slfcalms imsize = 512 cell = ['5arcsec'] if len(spwran) == 2: spwstr = spwran[0] + '~' + spwran[1] else: spwstr = spwran[0] restoringbeam = ['{0:.1f}arcsec'.format(bmsz)] imagesuffix = '.spw' + spwstr.replace('~', '-') if cfreq > 10.: antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines # else: # antenna = antenna + ';!0&1' # deselect the shortest baselines res = ptclean3(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange, spw=spw, ncpu=ncpu, niter=1000, gain=0.05, antenna=antenna, imsize=imsize, cell=cell, stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam, specmode="mfs", deconvolver="hogbom", datacolumn='data', pbcor=True) if res: imres['Succeeded'] += res['Succeeded'] imres['BeginTime'] += res['BeginTime'] imres['EndTime'] += res['EndTime'] imres['ImageName'] += res['ImageName'] imres['Spw'] += [spwstr] * len(res['ImageName']) imres['Vis'] += [msfile] * len(res['ImageName']) else: continue if len(vis) == 1: # produce the band-by-band whole-day images ms.open(msfile) ms.selectinit() timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True) tim = timfreq['time'] ms.close() cfreqs = getspwfreq(msfile) imdir = imagedir + subdir[0] if not os.path.exists(imdir): os.makedirs(imdir) for spw in spws: antenna = antenna0 if spw == '': spw = '{:d}~{:d}'.format( next(x[0] for x in enumerate(cfreqs) if x[1] > lowcutoff_freq), len(cfreqs) - 1) spwran = [s.zfill(2) for s in spw.split('~')] freqran = [cfreqs[int(s)] for s in spw.split('~')] cfreq = np.mean(freqran) bmsz = max(150. / cfreq, 20.) uvrange = '' imsize = 512 cell = ['5arcsec'] if len(spwran) == 2: spwstr = spwran[0] + '~' + spwran[1] else: spwstr = spwran[0] restoringbeam = ['{0:.1f}arcsec'.format(bmsz)] imagesuffix = '.synoptic.spw' + spwstr.replace('~', '-') antenna = antenna + ';!0&1' # deselect the shortest baselines res = ptclean3(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=len(tim), uvrange=uvrange, spw=spw, ncpu=1, niter=0, gain=0.05, antenna=antenna, imsize=imsize, cell=cell, stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam, specmode="mfs", deconvolver="hogbom", datacolumn='data', pbcor=True) if res: imres['Synoptic']['Succeeded'] += res['Succeeded'] imres['Synoptic']['BeginTime'] += res['BeginTime'] imres['Synoptic']['EndTime'] += res['EndTime'] imres['Synoptic']['ImageName'] += res['ImageName'] imres['Synoptic']['Spw'] += [spwstr] * len(res['ImageName']) imres['Synoptic']['Vis'] += [msfile] * len(res['ImageName']) else: continue # save it for debugging purposes np.savez('imres.npz', imres=imres) return imres
def importeovsa(idbfiles, timebin=None, width=None, visprefix=None, nocreatms=False, doconcat=False, modelms=''): casalog.origin('importeovsa') # # Initialize the helper class # pdh = ParallelDataHelper("importeovsa", locals()) # # # Validate input and output parameters # try: # pdh.setupIO() # except Exception, instance: # casalog.post('%s' % instance, 'ERROR') # return False if type(idbfiles) == Time: filelist = ri.get_trange_files(idbfiles) else: # If input type is not Time, assume that it is the list of files to read filelist = idbfiles if type(filelist) == str: filelist = [filelist] for f in filelist: if not os.path.exists(f): casalog.post("Some files in filelist are invalid. Aborting...") return False if not visprefix: visprefix = './' if not timebin: timebin = '0s' if not width: width = 1 if not modelms: if nocreatms: filename = filelist[0] modelms = creatms(filename, visprefix) else: if not os.path.exists(modelms): if nocreatms: filename = filelist[0] modelms = creatms(filename, visprefix) msfile = [] time_concat = [] for filename in filelist: uv = aipy.miriad.UV(filename) # if filename.split('/')[-1][0:3] == 'UDB': # uv_str = uv_hex_rm(uv) # else: # uv_str = uv uv.select('antennae', 0, 1, include=True) uv.select('polarization', -5, -5, include=True) times = [] uv.rewind() for preamble, data in uv.all(): uvw, t, (i, j) = preamble times.append(t) uv.select('clear', -1, -1, include=True) times = jd2mjds(np.asarray(times)) inttime = np.median((times - np.roll(times, 1))[1:]) / 60 time_steps = len(times) time_concat.append(int((times[-1] - times[0]) / 60 + inttime)) time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'].replace('\x00', '') antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.where(uv['sfreq'] > 0)[0] nf = len(good_idx) npol = uv['npol'] nants = uv['nants'] source_id = uv['source'].replace('\x00', '') sfreq = uv['sfreq'][good_idx] sdf = uv['sdf'][good_idx] ra, dec = uv['ra'], uv['dec'] nbl = nants * (nants - 1) / 2 bl2ord = bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) chan_band = get_band(sfreq=sfreq, sdf=sdf) nband = len(chan_band) uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[ i0, j0]] = -uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs out = out.reshape(npol, nf, nrows) flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows) uvwarray = np.tile(uvwarray, (1, nband)) sigma = np.ones((4, nrows), dtype=np.float) + 1 sigma = np.tile(sigma, (1, nband)) casalog.post( 'IDB File {0} is readed in --- {1:10.2f} seconds ---'.format( filename, (time.time() - time0))) msname = list(filename.split('/')[-1]) msname = visprefix + ''.join(msname) + '-{:d}m.ms'.format( int((times[-1] - times[0]) / 60 + inttime)) if not nocreatms: modelms = creatms(filename, visprefix) os.system('mv {} {}'.format(modelms, msname)) else: casalog.post('----------------------------------------') casalog.post('copying standard MS to {0}'.format( msname, (time.time() - time0))) casalog.post('----------------------------------------') os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % modelms + " %s" % msname) casalog.post( 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'. format(msname, (time.time() - time0))) tb.open(msname, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() nchannels = len(cband['cidx']) for row in range(nrows): tb.putcell('DATA', (row + l * nrows), out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) tb.putcell( 'FLAG', (row + l * nrows), flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'. format((l + 1), time.time() - time1)) tb.putcol('UVW', uvwarray) tb.putcol('SIGMA', sigma) tb.putcol('WEIGHT', 1.0 / sigma**2) timearr = times timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape(nband * npairs * time_steps) tb.putcol('TIME', timearr) tb.putcol('TIME_CENTROID', timearr) scan_id = tb.getcol('SCAN_NUMBER') scan_id *= 0 tb.putcol('SCAN_NUMBER', scan_id) colnames = tb.colnames() cols2rm = ["MODEL_DATA", "CORRECTED_DATA"] for l in range(len(cols2rm)): if cols2rm[l] in colnames: tb.removecols(cols2rm[l]) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the OBSERVATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/OBSERVATION', nomodify=False) tb.putcol( 'TIME_RANGE', np.asarray([times[0] - 0.5 * inttime, times[-1] + 0.5 * inttime]).reshape(2, 1)) tb.putcol('OBSERVER', ['EOVSA team']) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POINTING table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POINTING', nomodify=False) timearr = times.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape(nband * time_steps * nants) tb.putcol('TIME', timearr) tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time) direction = tb.getcol('DIRECTION') direction[0, 0, :] = ra direction[1, 0, :] = dec tb.putcol('DIRECTION', direction) target = tb.getcol('TARGET') target[0, 0, :] = ra target[1, 0, :] = dec tb.putcol('TARGET', target) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the SOURCE table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/SOURCE', nomodify=False) radec = tb.getcol('DIRECTION') radec[0], radec[1] = ra, dec tb.putcol('DIRECTION', radec) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False) pol_id = tb.getcol('POLARIZATION_ID') pol_id *= 0 tb.putcol('POLARIZATION_ID', pol_id) # spw_id = tb.getcol('SPECTRAL_WINDOW_ID') # spw_id *= 0 # tb.putcol('SPECTRAL_WINDOW_ID', spw_id) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POLARIZATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POLARIZATION/', nomodify=False) tb.removerows(rownrs=np.arange(1, nband, dtype=int)) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the FIELD table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/FIELD/', nomodify=False) delay_dir = tb.getcol('DELAY_DIR') delay_dir[0], delay_dir[1] = ra, dec tb.putcol('DELAY_DIR', delay_dir) phase_dir = tb.getcol('PHASE_DIR') phase_dir[0], phase_dir[1] = ra, dec tb.putcol('PHASE_DIR', phase_dir) reference_dir = tb.getcol('REFERENCE_DIR') reference_dir[0], reference_dir[1] = ra, dec tb.putcol('REFERENCE_DIR', reference_dir) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME del out, flag, uvwarray, uv, timearr, sigma gc.collect() # if not (timebin == '0s' and width == 1): split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname)) msfile.append(msname + '.split') else: msfile.append(msname) casalog.post("finished in --- %s seconds ---" % (time.time() - time0)) if doconcat: msname = list(filelist[0].split('/')[-1]) concatvis = visprefix + ''.join(msname) + '-{:d}m.ms'.format( int(sum(time_concat))) concat(vis=msfile, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) tb.close() for ll in msfile: os.system('rm -rf {}'.format(ll)) return True
# flag CH3OH maser #flagdata(vis=ms, mode='manual', # spw='44054800170~44084179830Hz:44054800170~44084179830Hz') tb.open(ms) if 'CORRECTED_DATA' in tb.colnames(): datacolumn = 'corrected' else: datacolumn = 'data' tb.close() split( vis=ms, outputvis=cont_ms, field=",".join(field_names), width=16, datacolumn=datacolumn, spw=Kmses[fullpathms], ) # Unflag CH3OH maser #flagdata(vis=ms, mode='unflag', # spw='44054800170~44084179830Hz:44054800170~44084179830Hz') with open('done_recalibrating_{0}'.format(ms), 'w') as fh: fh.write(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n") fh.write("cont_ms={0}, ms={1}".format(cont_ms, ms) + "\n") os.remove('WORKING') os.chdir(cwd)
field = 'Sgr B2 MN K,Sgr B2 MS K,Sgr B2 SDS K' restfreq = '23.722633335GHz' selfcal_spw = '0~5:23.7178~23.7185GHz' merged_ms = 'NH322_merged.ms' if not os.path.exists(merged_ms): splitvises = [] for ms, spw in spws.items(): basems = os.path.split(ms)[-1] outputvis = 'NH322_{0}'.format(basems) if not os.path.exists(outputvis): split( vis=ms, outputvis=outputvis, field=field, # have to pick these carefully, they're topo. spw='{0}:23.714~23.723GHz'.format(spw), ) msmd.open(outputvis) ref0 = msmd.reffreq(0) msmd.close() if ref0['refer'] == 'TOPO': mst.open(outputvis, nomodify=False) mst.regridspw(outframe='LSRK', mode='vrad', restfreq=23.722633335e9) mst.close()
def get_dspec(vis=None, savespec=True, specfile=None, bl='', uvrange='', field='', scan='', datacolumn='data', domedian=False, timeran=None, spw=None, timebin='0s', verbose=False): from split_cli import split_cli as split msfile = vis if not spw: spw = '' if not timeran: timeran = '' if not bl: bl = '' if domedian: if not uvrange: uvrange = '0.2~0.8km' else: uvrange = '' # Open the ms and plot dynamic spectrum if verbose: print('Splitting selected data...') vis_spl = './tmpms.splitted' if os.path.exists(vis_spl): os.system('rm -rf ' + vis_spl) split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw, uvrange=uvrange, timebin=timebin, datacolumn=datacolumn) ms.open(vis_spl, nomodify=False) if verbose: print('Regridding into a single spectral window...') # print('Reading data spw by spw') ms.cvel(outframe='LSRK', mode='frequency', interp='nearest') ms.selectinit(datadescid=0, reset=True) data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True) ms.close() os.system('rm -rf ' + vis_spl) specamp = data['amplitude'] (npol, nfreq, nbl, ntim) = specamp.shape if verbose: print('npol, nfreq, nbl, ntime:', data['amplitude'].shape) spec = np.swapaxes(specamp, 2, 1) freq = data['axis_info']['freq_axis']['chan_freq'].reshape(nfreq) tim = data['time'] if domedian: if verbose: print('doing median of all the baselines') # mask zero values before median spec_masked = np.ma.masked_where(spec < 1e-9, spec) spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1), fill_value=0.) nbl = 1 ospec = spec_med.reshape((npol, nbl, nfreq, ntim)) else: ospec = spec # Save the dynamic spectral data if savespec: if not specfile: specfile = msfile + '.dspec.npz' if os.path.exists(specfile): os.system('rm -rf ' + specfile) np.savez(specfile, spec=ospec, tim=tim, freq=freq, timeran=timeran, spw=spw, bl=bl, uvrange=uvrange) if verbose: print('Median dynamic spectrum saved as: ' + specfile) return {'spec': ospec, 'tim': tim, 'freq': freq, 'timeran': timeran, 'spw': spw, 'bl': bl, 'uvrange': uvrange}
def get_dspec(vis=None, savespec=True, specfile=None, bl='', uvrange='', field='', scan='', datacolumn='data', domedian=False, timeran=None, spw=None, timebin='0s', regridfreq=False, fillnan=None, verbose=False, usetbtool=False): # from split_cli import split_cli as split if vis.endswith('/'): vis = vis[:-1] msfile = vis if not spw: spw = '' if not timeran: timeran = '' if domedian: if not uvrange: uvrange = '0.2~0.8km' # bl = '' else: uvrange = '' if not bl: bl = '' else: uvrange = '' # Open the ms and plot dynamic spectrum if verbose: print('Splitting selected data...') if usetbtool: try: tb.open(vis + '/POLARIZATION') corrtype = tb.getcell('CORR_TYPE', 0) pols = [stokesenum[p] for p in corrtype] tb.close() except: pols = [] antmask = [] if uvrange is not '' or bl is not '': ms.open(vis) ms.selectinit(datadescid=0) mdata = ms.metadata() antlist = mdata.antennaids() mdata.done() staql = {'uvdist': uvrange, 'baseline': bl, 'spw': spw, 'field': field, 'scan': scan, 'timerange': timeran} ### todo the selection only works for uvrange and bl. To make the selection of other items works, ## I need to make mask for other items. a = ms.msselect(staql) mdata = ms.metadata() baselines = mdata.baselines() for lidx, l in enumerate(antlist): antmask.append(baselines[l][antlist[lidx:]]) antmask = np.hstack(antmask) mdata.done() ms.close() tb.open(vis) spwtb = tbtool() spwtb.open(vis + '/SPECTRAL_WINDOW') ptb = tbtool() ptb.open(vis + '/POLARIZATION') ms.open(vis) spwlist = [] mdata = ms.metadata() nspw = mdata.nspw() nbl = mdata.nbaselines() + mdata.nantennas() nscans = mdata.nscans() spw_nfrq = [] # List of number of frequencies in each spw for i in range(nspw): spw_nfrq.append(mdata.nchan(i)) spw_nfrq = np.array(spw_nfrq) nf = np.sum(spw_nfrq) smry = mdata.summary() scan_ntimes = [] # List of number of times in each scan for iscan in range(nscans): scan_ntimes.append( smry['observationID=0']['arrayID=0']['scan=' + str(iscan)]['fieldID=0']['nrows'] / nspw / nbl) scan_ntimes = np.array(scan_ntimes) scan_ntimes_integer = scan_ntimes.astype(np.int) if len(np.where(scan_ntimes % scan_ntimes_integer != 0)[0]) != 0: # if True: scan_ntimes = [] # List of number of times in each scan for iscan in range(nscans): scan_ntimes.append( len(smry['observationID=0']['arrayID=0']['scan=' + str(iscan)]['fieldID=0'].keys()) - 6) scan_ntimes = np.array(scan_ntimes) else: scan_ntimes = scan_ntimes_integer nt = np.sum(scan_ntimes) times = tb.getcol('TIME') if times[nbl] - times[0] != 0: # This is frequency/scan sort order order = 'f' elif times[nbl * nspw - 1] - times[0] != 0: # This is time sort order order = 't' npol = ptb.getcol('NUM_CORR', 0, 1)[0] ptb.close() freq = np.zeros(nf, float) times = np.zeros(nt, float) if order == 't': specamp = np.zeros((npol, nf, nbl, nt), np.complex) for j in range(nt): fptr = 0 # Loop over spw for i in range(nspw): # Get channel frequencies for this spw (annoyingly comes out as shape (nf, 1) cfrq = spwtb.getcol('CHAN_FREQ', i, 1)[:, 0] if j == 0: # Only need this the first time through spwlist += [i] * len(cfrq) if i == 0: times[j] = tb.getcol('TIME', nbl * (i + nspw * j), 1) # Get the time spec_ = tb.getcol('DATA', nbl * (i + nspw * j), nbl) # Get complex data for this spw flag = tb.getcol('FLAG', nbl * (i + nspw * j), nbl) # Get flags for this spw nfrq = len(cfrq) # Apply flags if type(fillnan) in [int, float]: spec_[flag] = float(fillnan) else: spec_[flag] = 0.0 # Insert data for this spw into larger array specamp[:, fptr:fptr + nfrq, :, j] = spec_ freq[fptr:fptr + nfrq] = cfrq fptr += nfrq else: specf = np.zeros((npol, nf, nt, nbl), np.complex) # Array indexes are swapped iptr = 0 for j in range(nscans): # Loop over scans for i in range(nspw): # Loop over spectral windows s = scan_ntimes[j] f = spw_nfrq[i] s1 = np.sum(scan_ntimes[:j]) # Start time index s2 = np.sum(scan_ntimes[:j + 1]) # End time index f1 = np.sum(spw_nfrq[:i]) # Start freq index f2 = np.sum(spw_nfrq[:i + 1]) # End freq index spec_ = tb.getcol('DATA', iptr, nbl * s) flag = tb.getcol('FLAG', iptr, nbl * s) if j == 0: cfrq = spwtb.getcol('CHAN_FREQ', i, 1)[:, 0] freq[f1:f2] = cfrq spwlist += [i] * len(cfrq) times[s1:s2] = tb.getcol('TIME', iptr, nbl * s).reshape(s, nbl)[:, 0] # Get the times iptr += nbl * s # Apply flags if type(fillnan) in [int, float]: spec_[flag] = float(fillnan) else: spec_[flag] = 0.0 # Insert data for this spw into larger array specf[:, f1:f2, s1:s2] = spec_.reshape(npol, f, s, nbl) # Swap the array indexes back to the desired order specamp = np.swapaxes(specf, 2, 3) tb.close() spwtb.close() ms.close() if len(antmask) > 0: specamp = specamp[:, :, np.where(antmask)[0], :] (npol, nfreq, nbl, ntim) = specamp.shape tim = times else: # Open the ms and plot dynamic spectrum if verbose: print('Splitting selected data...') vis_spl = './tmpms.splitted' if os.path.exists(vis_spl): os.system('rm -rf ' + vis_spl) # split(vis=msfile, outputvis=vis_spl, timerange=timeran, antenna=bl, field=field, scan=scan, spw=spw, # uvrange=uvrange, timebin=timebin, datacolumn=datacolumn) try: from split_cli import split_cli as split split(vis=msfile, outputvis=vis_spl, datacolumn=datacolumn, timerange=timeran, spw=spw, antenna=bl, field=field, scan=scan, uvrange=uvrange, timebin=timebin) except: ms.open(msfile, nomodify=True) ms.split(outputms=vis_spl, whichcol=datacolumn, time=timeran, spw=spw, baseline=bl, field=field, scan=scan, uvrange=uvrange, timebin=timebin) ms.close() if verbose: print('Regridding into a single spectral window...') # print('Reading data spw by spw') try: tb.open(vis_spl + '/POLARIZATION') corrtype = tb.getcell('CORR_TYPE', 0) pols = [stokesenum[p] for p in corrtype] tb.close() except: pols = [] if regridfreq: ms.open(vis_spl, nomodify=False) ms.cvel(outframe='LSRK', mode='frequency', interp='nearest') ms.selectinit(datadescid=0, reset=True) data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True) specamp = data['amplitude'] freq = data['axis_info']['freq_axis']['chan_freq'] else: ms.open(vis_spl) ms.selectinit(datadescid=0, reset=True) spwinfo = ms.getspectralwindowinfo() specamp = [] freq = [] time = [] for descid in range(len(spwinfo.keys())): ms.selectinit(datadescid=0, reset=True) ms.selectinit(datadescid=descid) data = ms.getdata(['amplitude', 'time', 'axis_info'], ifraxis=True) specamp_ = data['amplitude'] freq_ = data['axis_info']['freq_axis']['chan_freq'] time_ = data['time'] if fillnan is not None: flag_ = ms.getdata(['flag', 'time', 'axis_info'], ifraxis=True)['flag'] if type(fillnan) in [int, float, long]: specamp_[flag_] = float(fillnan) else: specamp_[flag_] = 0.0 specamp.append(specamp_) freq.append(freq_) time.append(time_) specamp = np.concatenate(specamp, axis=1) freq = np.concatenate(freq, axis=0) ms.selectinit(datadescid=0, reset=True) ms.close() os.system('rm -rf ' + vis_spl) (npol, nfreq, nbl, ntim) = specamp.shape freq = freq.reshape(nfreq) tim = data['time'] if verbose: print('npol, nfreq, nbl, ntime:', (npol, nfreq, nbl, ntim)) spec = np.swapaxes(specamp, 2, 1) if domedian: if verbose: print('doing median of all the baselines') # mask zero values before median # spec_masked = np.ma.masked_where(spec < 1e-9, spec) # spec_masked2 = np.ma.masked_invalid(spec) # spec_masked = np.ma.masked_array(spec, mask=np.logical_or(spec_masked.mask, spec_masked2.mask)) # spec_med = np.ma.filled(np.ma.median(spec_masked, axis=1), fill_value=0.) spec = np.abs(spec) spec_med = np.nanmedian(spec, axis=1) nbl = 1 ospec = spec_med.reshape((npol, nbl, nfreq, ntim)) else: ospec = spec # Save the dynamic spectral data if savespec: if not specfile: specfile = msfile + '.dspec.npz' if os.path.exists(specfile): os.system('rm -rf ' + specfile) np.savez(specfile, spec=ospec, tim=tim, freq=freq, timeran=timeran, spw=spw, bl=bl, uvrange=uvrange, pol=pols) if verbose: print('Median dynamic spectrum saved as: ' + specfile) return {'spec': ospec, 'tim': tim, 'freq': freq, 'timeran': timeran, 'spw': spw, 'bl': bl, 'uvrange': uvrange, 'pol': pols}
def mk_qlook_image(trange, doimport=False, docalib=False, ncpu=10, twidth=12, stokes=None, antenna='0~12', #imagedir=None, spws=['1~3','4~6','7~9','10~13','14~18','19~28'],verbose=False): imagedir=None, spws=['1~5','6~10','11~15','16~25'], toTb=True, overwrite=True, doslfcal=False, verbose=False): ''' trange: can be 1) a single Time() object: use the entire day 2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00']) 3) a single or a list of UDBms file(s) 4) None -- use current date Time.now() ''' if type(trange) == Time: mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] tsts = [l.to_datetime() for l in mslist['tstlist']] subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts] if type(trange) == str: try: date = Time(trange) mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] except: vis = [trange] subdir = ['/'] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] if not stokes: stokes = 'XX' if not imagedir: imagedir='./' imres = {'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': []} for n, msfile in enumerate(vis): msfilebs=os.path.basename(msfile) imdir = imagedir + subdir[n] if not os.path.exists(imdir): os.makedirs(imdir) if doslfcal: slfcalms = './'+msfilebs+'.xx' split(msfile,outputvis=slfcalms,datacolumn='corrected',correlation='XX') for spw in spws: spwran = [s.zfill(2) for s in spw.split('~')] freqran = [int(s)*0.5+2.9 for s in spw.split('~')] cfreq=np.mean(freqran) bmsz=max(150./cfreq,20.) uvrange='<10klambda' if doslfcal: slfcal_img = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slfimg' slfcal_tb = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slftb' try: clean(vis=slfcalms, antenna=antenna, imagename=slfcal_img, spw=spw, mode='mfs', timerange='', imagermode='csclean', psfmode='clark', imsize=[512,512], cell=['5arcsec'], niter=100, gain=0.05, stokes='I', weighting='natural', restoringbeam=[str(bmsz)+'arcsec'], pbcor=False, interactive=False, usescratch=True) except: print 'error in cleaning spw: '+spw break gaincal(vis=slfcalms, refant='0',antenna=antenna,caltable=slfcal_tb,spw=spw, uvrange='',\ gaintable=[],selectdata=True,timerange='',solint='600s',gaintype='G',calmode='p',\ combine='',minblperant=3,minsnr=2,append=False) if not os.path.exists(slfcal_tb): print 'No solution found in spw: '+spw break else: clearcal(slfcalms) delmod(slfcalms) applycal(vis=slfcalms,gaintable=[slfcal_tb],spw=spw,selectdata=True,\ antenna=antenna,interp='nearest',flagbackup=False,applymode='calonly',calwt=False) msfile=slfcalms if cfreq < 10.: imsize=512 cell=['5arcsec'] else: imsize=1024 cell=['2.5arcsec'] if len(spwran) == 2: spwstr = spwran[0]+'~'+spwran[1] else: spwstr = spwran[0] restoringbeam=['{0:.1f}arcsec'.format(bmsz)] imagesuffix='.spw'+spwstr.replace('~','-') if cfreq > 10.: antenna=antenna+';!0&1;!0&2' #deselect the shortest baselines res=ptclean(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange, spw=spw, ncpu=ncpu, niter=1000, gain=0.05, antenna=antenna,imsize=imsize, cell=cell, stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam, uvtaper=True,outertaper=['30arcsec']) if res: imres['Succeeded'] += res['Succeeded'] imres['BeginTime'] += res['BeginTime'] imres['EndTime'] += res['EndTime'] imres['ImageName'] += res['ImageName'] imres['Spw'] += [spwstr]*len(res['ImageName']) imres['Vis'] += [msfile]*len(res['ImageName']) else: return None #save it for debugging purposes np.savez('imres.npz',imres=imres) return imres
def calibeovsa(vis=None, caltype=None, interp=None, docalib=True, doflag=True, flagant=None, doimage=False, imagedir=None, antenna=None, timerange=None, spw=None, stokes=None, doconcat=False, msoutdir=None, concatvis=None, keep_orig_ms=True): ''' :param vis: EOVSA visibility dataset(s) to be calibrated :param caltype: :param interp: :param docalib: :param qlookimage: :param flagant: :param stokes: :param doconcat: :return: ''' if type(vis) == str: vis = [vis] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] for msfile in vis: casalog.origin('calibeovsa') if not caltype: casalog.post("Caltype not provided. Perform reference phase calibration and daily phase calibration.") caltype = ['refpha', 'phacal', 'fluxcal'] ## use this line after the phacal is applied # caltype = ['refcal'] if not os.path.exists(msfile): casalog.post("Input visibility does not exist. Aborting...") continue if msfile.endswith('/'): msfile = msfile[:-1] if not msfile[-3:] in ['.ms', '.MS']: casalog.post("Invalid visibility. Please provide a proper visibility file ending with .ms") # if not caltable: # caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype] # get band information tb.open(msfile + '/SPECTRAL_WINDOW') nspw = tb.nrows() bdname = tb.getcol('NAME') bd_nchan = tb.getcol('NUM_CHAN') bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33 # nchans = tb.getcol('NUM_CHAN') # reffreqs = tb.getcol('REF_FREQUENCY') # cenfreqs = np.zeros((nspw)) tb.close() tb.open(msfile + '/ANTENNA') nant = tb.nrows() antname = tb.getcol('NAME') antlist = [str(ll) for ll in range(len(antname) - 1)] antennas = ','.join(antlist) tb.close() # get time stamp, use the beginning of the file tb.open(msfile + '/OBSERVATION') trs = {'BegTime': [], 'EndTime': []} for ll in range(tb.nrows()): tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600, format='mjd') trs['BegTime'].append(tim0) trs['EndTime'].append(tim1) tb.close() trs['BegTime'] = Time(trs['BegTime']) trs['EndTime'] = Time(trs['EndTime']) btime = np.min(trs['BegTime']) etime = np.max(trs['EndTime']) # ms.open(vis) # summary = ms.summary() # ms.close() # btime = Time(summary['BeginTime'], format='mjd') # etime = Time(summary['EndTime'], format='mjd') ## stop using ms.summary to avoid conflicts with importeovsa t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd') print "This scan observed from {} to {} UTC".format(btime.iso, etime.iso) gaintables = [] if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal' in caltype): refcal = ra.sql2refcalX(btime) pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband) pha[np.where(refcal['flag'] == 1)] = 0. amp = refcal['amp'] amp[np.where(refcal['flag'] == 1)] = 1. t_ref = refcal['timestamp'] # find the start and end time of the local day when refcal is registered try: dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60. except: dhr = -7. bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd') et = Time(bt.mjd + 1., format='mjd') (yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day) dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/' if not os.path.exists(dirname): os.mkdir(dirname) # check if there is any ROACH reboot between the reference calibration found and the current data t_rbts = db.get_reboot(Time([t_ref, btime])) if not t_rbts: casalog.post("Reference calibration is derived from observation at " + t_ref.iso) print "Reference calibration is derived from observation at " + t_ref.iso else: casalog.post( "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso) casalog.post("Aborting...") print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso print "Aborting..." para_pha = [] para_amp = [] calpha = np.zeros((nspw, 15, 2)) calamp = np.zeros((nspw, 15, 2)) for s in range(nspw): for n in range(15): for p in range(2): calpha[s, n, p] = pha[n, p, bd[s]] calamp[s, n, p] = amp[n, p, bd[s]] para_pha.append(np.degrees(pha[n, p, bd[s]])) para_amp.append(amp[n, p, bd[s]]) if 'fluxcal' in caltype: calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59')) t_bp = Time(calfac['timestamp'], format='lv') if int(t_mid.mjd) == int(t_bp.mjd): accalfac = calfac['accalfac'] # (ant x pol x freq) # tpcalfac = calfac['tpcalfac'] # (ant x pol x freq) caltb_autoamp = dirname + t_bp.isot[:-4].replace(':', '').replace('-', '') + '.bandpass' if not os.path.exists(caltb_autoamp): bandpass(vis=msfile, caltable=caltb_autoamp, solint='inf', refant='eo01', minblperant=0, minsnr=0, bandtype='B', docallib=False) tb.open(caltb_autoamp, nomodify=False) # (ant x spw) bd_chanidx = np.hstack([[0], bd_nchan.cumsum()]) for ll in range(nspw): antfac = np.sqrt(accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]]) # # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]] antfac = np.moveaxis(antfac, 0, 2) cparam = np.zeros((2, bd_nchan[ll], nant)) cparam[:, :, :-3] = 1.0 / antfac tb.putcol('CPARAM', cparam + 0j, ll * nant, nant) paramerr = tb.getcol('PARAMERR', ll * nant, nant) paramerr = paramerr * 0 tb.putcol('PARAMERR', paramerr, ll * nant, nant) bpflag = tb.getcol('FLAG', ll * nant, nant) bpant1 = tb.getcol('ANTENNA1', ll * nant, nant) bpflagidx, = np.where(bpant1 >= 13) bpflag[:] = False bpflag[:, :, bpflagidx] = True tb.putcol('FLAG', bpflag, ll * nant, nant) bpsnr = tb.getcol('SNR', ll * nant, nant) bpsnr[:] = 100.0 bpsnr[:, :, bpflagidx] = 0.0 tb.putcol('SNR', bpsnr, ll * nant, nant) tb.close() msg_prompt = "Scaling calibration is derived for {}.".format(msfile) casalog.post(msg_prompt) print msg_prompt gaintables.append(caltb_autoamp) else: msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format( t_mid.datetime.strftime('%b %d, %Y'), msfile) casalog.post(msg_prompt) print msg_prompt if ('refpha' in caltype) or ('refcal' in caltype): # caltb_pha = os.path.basename(vis).replace('.ms', '.refpha') # check if the calibration table already exists caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace('-', '') + '.refpha' if not os.path.exists(caltb_pha): gencal(vis=msfile, caltable=caltb_pha, caltype='ph', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_pha) gaintables.append(caltb_pha) if ('refamp' in caltype) or ('refcal' in caltype): # caltb_amp = os.path.basename(vis).replace('.ms', '.refamp') caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace('-', '') + '.refamp' if not os.path.exists(caltb_amp): gencal(vis=msfile, caltable=caltb_amp, caltype='amp', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_amp) gaintables.append(caltb_amp) # calibration for the change of delay center between refcal time and beginning of scan -- hopefully none! xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False) if buf: dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv') dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15] xml, buf = ch.read_calX(4, t=t_ref) dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv') dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15] dlycen_ns_diff = dlycen_ns2 - dlycen_ns1 for n in range(2): dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n] print 'Multi-band delay is derived from delay center difference at {} & {}'.format(dly_t1.iso, dly_t2.iso) # print '=====Delays relative to Ant 14=====' # for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]): # ant = antlist[i] # print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl # dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1]) # caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0') caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace('-', '') + '.dlycen' if not os.path.exists(caltb_dlycen): gencal(vis=msfile, caltable=caltb_dlycen, caltype='mbd', pol='X,Y', antenna=antennas, parameter=dlycen_ns_diff.flatten().tolist()) gaintables.append(caltb_dlycen) if 'phacal' in caltype: phacals = np.array(ra.sql2phacalX([bt, et], neat=True, verbose=False)) if not phacals.any() or len(phacals) == 0: print "Found no phacal records in SQL database, will skip phase calibration" else: # first generate all phacal calibration tables if not already exist t_phas = Time([phacal['t_pha'] for phacal in phacals]) # sort the array in ascending order by t_pha sinds = t_phas.mjd.argsort() t_phas = t_phas[sinds] phacals = phacals[sinds] caltbs_phambd = [] for i, phacal in enumerate(phacals): # filter out phase cals with reference time stamp >30 min away from the provided refcal time if (phacal['t_ref'].jd - refcal['timestamp'].jd) > 30. / 1440.: del phacals[i] del t_phas[i] continue else: t_pha = phacal['t_pha'] phambd_ns = phacal['pslope'] for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n] # set all flagged values to be zero phambd_ns[np.where(phacal['flag'] == 1)] = 0. caltb_phambd = dirname + t_pha.isot[:-4].replace(':', '').replace('-', '') + '.phambd' caltbs_phambd.append(caltb_phambd) if not os.path.exists(caltb_phambd): gencal(vis=msfile, caltable=caltb_phambd, caltype='mbd', pol='X,Y', antenna=antennas, parameter=phambd_ns.flatten().tolist()) # now decides which table to apply depending on the interpolation method ("neatest" or "linear") if interp == 'nearest': tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd)) dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24. print "Selected nearest phase calibration table at " + t_phas[tbind].iso gaintables.append(caltbs_phambd[tbind]) if interp == 'linear': # bphacal = ra.sql2phacalX(btime) # ephacal = ra.sql2phacalX(etime,reverse=True) bt_ind, = np.where(t_phas.mjd < btime.mjd) et_ind, = np.where(t_phas.mjd > etime.mjd) if len(bt_ind) == 0 and len(et_ind) == 0: print "No phacal found before or after the ms data within the day of observation" print "Skipping daily phase calibration" elif len(bt_ind) > 0 and len(et_ind) == 0: gaintables.append(caltbs_phambd[bt_ind[-1]]) elif len(bt_ind) == 0 and len(et_ind) > 0: gaintables.append(caltbs_phambd[et_ind[0]]) elif len(bt_ind) > 0 and len(et_ind) > 0: bphacal = phacals[bt_ind[-1]] ephacal = phacals[et_ind[0]] # generate a new table interpolating between two daily phase calibrations t_pha_mean = Time(np.mean([bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]), format='mjd') phambd_ns = (bphacal['pslope'] + ephacal['pslope']) / 2. for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n] # set all flagged values to be zero phambd_ns[np.where(bphacal['flag'] == 1)] = 0. phambd_ns[np.where(ephacal['flag'] == 1)] = 0. caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace(':', '').replace('-', '') + '.phambd' if not os.path.exists(caltb_phambd_interp): gencal(vis=msfile, caltable=caltb_phambd_interp, caltype='mbd', pol='X,Y', antenna=antennas, parameter=phambd_ns.flatten().tolist()) print "Using phase calibration table interpolated between records at " + bphacal['t_pha'].iso + ' and ' + ephacal['t_pha'].iso gaintables.append(caltb_phambd_interp) if docalib: clearcal(msfile) applycal(vis=msfile, gaintable=gaintables, applymode='calflag', calwt=False) # delete the interpolated phase calibration table try: caltb_phambd_interp except: pass else: if os.path.exists(caltb_phambd_interp): shutil.rmtree(caltb_phambd_interp) if doflag: # flag zeros and NaNs flagdata(vis=msfile, mode='clip', clipzeros=True) if flagant: try: flagdata(vis=msfile, antenna=flagant) except: print "Something wrong with flagant. Abort..." if doimage: from matplotlib import pyplot as plt from suncasa.utils import helioimage2fits as hf from sunpy import map as smap if not antenna: antenna = '0~12' if not stokes: stokes = 'XX' if not timerange: timerange = '' if not spw: spw = '1~3' if not imagedir: imagedir = '.' #(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day) #dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/' #if not os.path.exists(dirname): # os.makedirs(dirname) bds = [spw] nbd = len(bds) imgs = [] for bd in bds: if '~' in bd: bdstr = bd.replace('~', '-') else: bdstr = str(bd).zfill(2) imname = imagedir + '/' + os.path.basename(msfile).replace('.ms', '.bd' + bdstr) print 'Cleaning image: ' + imname try: clean(vis=msfile, imagename=imname, antenna=antenna, spw=bd, timerange=timerange, imsize=[512], cell=['5.0arcsec'], stokes=stokes, niter=500) except: print 'clean not successfull for band ' + str(bd) else: imgs.append(imname + '.image') junks = ['.flux', '.mask', '.model', '.psf', '.residual'] for junk in junks: if os.path.exists(imname + junk): shutil.rmtree(imname + junk) tranges = [btime.iso + '~' + etime.iso] * nbd fitsfiles = [img.replace('.image', '.fits') for img in imgs] hf.imreg(vis=msfile, timerange=tranges, imagefile=imgs, fitsfile=fitsfiles, usephacenter=False) plt.figure(figsize=(6, 6)) for i, fitsfile in enumerate(fitsfiles): plt.subplot(1, nbd, i + 1) eomap = smap.Map(fitsfile) sz = eomap.data.shape if len(sz) == 4: eomap.data = eomap.data.reshape((sz[2], sz[3])) eomap.plot_settings['cmap'] = plt.get_cmap('jet') eomap.plot() eomap.draw_limb() eomap.draw_grid() plt.show() if doconcat: from suncasa.tasks import concateovsa_cli as ce # from suncasa.eovsa import concateovsa as ce if msoutdir is None: msoutdir = './' if not concatvis: concatvis = os.path.basename(vis[0]) concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms' else: concatvis = os.path.join(msoutdir, concatvis) if len(vis) > 1: ce.concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=keep_orig_ms, cols2rm="model,corrected") return [concatvis] else: split(vis=vis[0], outputvis=concatvis, datacolumn='corrected') return [concatvis] else: return vis
mses = list(Qmses.keys()) fullpath_mses = [ '../' + ms[:-3] + "_continuum_split_for_selfcal.ms" for ms in mses if ms in Qmses ] cont_vis = [] for ms in fullpath_mses: splitagain = ms[:-3] + "_SgrB2_NM_Q.ms" myprint("{0} -> {1}".format(ms, splitagain)) if not os.path.exists(splitagain): assert split(vis=ms, outputvis=splitagain, field='Sgr B2 NM Q', datacolumn='corrected') cont_vis.append(splitagain) cleanbox_mask_image = 'cleanbox_mask_SgrB2.image' cleanbox_mask = 'cleanbox_mask.mask' mask = cleanbox_mask_image selfcal_fields = "Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q".split(",") selfcal_fields = ['Sgr B2 NM Q'] extrapars = { 'wproject': { 'wprojplanes': 64, #'rotatepastep': 5.0, #'cfcache':'test_wtermmerge.cfcache',
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False, timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""): if concatvis[-1] == os.path.sep: concatvis = concatvis[:-1] if os.path.sep not in concatvis: visprefix = './' else: visprefix = os.path.dirname(concatvis) + os.path.sep msfiles = vis msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] datacolumn = datacolumn.lower() if datacolumn == 'data': print 'DATA columns will be concatenated.' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) elif datacolumn == 'corrected': # try: print 'CORRECTED columns will be concatenated.' tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = os.path.join(tmpdir, os.path.basename(str(ll))) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) else: raise ValueError('Please set datacolumn to be "data" or "corrected"!') if msfiles_: concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) os.system('rm -rf {}'.format(tmpdir)) else: concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False) nrows = tb.nrows() pol_id = tb.getcol('POLARIZATION_ID') tb.removerows(np.where(pol_id != 0)[0]) tb.close() tb.open(concatvis, nomodify=False) dd_id = tb.getcol('DATA_DESC_ID') idx_dd_id, = np.where(dd_id >= nrows / 2) dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2 tb.putcol('DATA_DESC_ID', dd_id) tb.close() tb.open(concatvis + '/FIELD', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis + '/SOURCE', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) fldid = tb.getcol('FIELD_ID') newfldid = np.zeros(len(fldid), dtype='int') tb.putcol('FIELD_ID', newfldid) colnames = tb.colnames() cols2rm = cols2rm.upper() cols2rm = cols2rm.split(',') for l in range(len(cols2rm)): col = cols2rm[l] + '_DATA' if col in colnames: try: tb.removecols(col) print 'Column {} removed.'.format(col) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
flagdata(vis=fullpathms, mode='quack', quackinterval=10) # flag edge channels flagchans = ",".join( ["{0}:0~5;123~128".format(xx) for xx in Qmses[ms].split(",")]) flagdata(vis=fullpathms, mode='manual', spw=flagchans) # flag CH3OH maser flagdata(vis=fullpathms, mode='manual', spw='44054800170~44084179830Hz:44054800170~44084179830Hz') split( vis=fullpathms, outputvis=cont_ms, field=('Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,' 'Sgr B2 S Q,Sgr B2 DS1 Q,Sgr B2 DS2 Q,Sgr B2 DS3 Q'), width=16, spw=Qmses[ms], ) # Unflag CH3OH maser flagdata(vis=fullpathms, mode='unflag', spw='44054800170~44084179830Hz:44054800170~44084179830Hz') clearcal(cont_ms, addmodel=True) #ft(vis=cont_ms, # field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q', # spw='', # model=['../continuum/18A-229_mosaic_for_selfcal.model.tt0.noneg', # '../continuum/18A-229_mosaic_for_selfcal.model.tt1.noneg'],
tint) + 'IDB20170906T' + trange.split(':')[3] + trange.split( ':')[4].split('.')[0] + '.ms.corrected.xx.slfcal' slfcaledms = workdir + 'slfcal/No' + str( tint) + 'IDB20170906T' + trange.split(':')[3] + trange.split( ':')[4].split('.')[0] + '.ms.corrected.xx.slfcaled' calprefix = workdir + 'slfcal/caltbs/No' + str( tint) + 'slf_19' + trange.split(':')[3] + trange.split(':')[4].split( '.')[0] #imgprefix=workdir+'slfcal/images/No'+str(tint)+'slf_19'+trange.split(':')[3]+trange.split(':')[4].split('.')[0] imgprefix = workdir + 'slfcal/images/sbs/No' + str( tint) + 'slf_19' + trange.split(':')[3] + trange.split(':')[4].split( '.')[0] if not os.path.exists(slfcalms): split(vis=refms, outputvis=slfcalms, datacolumn='data', timerange=trange, correlation='XX') listobs(slfcalms, listfile=slfcalms + '.listobs') clearcal(slfcalms) delmod(slfcalms) if doapplyselfcal: caltable_list = [ '/srg/ywei/data/eovsa/caltbs/slf_192920.G0/', '/srg/ywei/data/eovsa/caltbs/slf_192920.G1/', '/srg/ywei/data/eovsa/caltbs/slf_192920.G2/' ] for nc in range(len(caltable_list)): applycal(vis=slfcalms, gaintable=caltable_list[nc], spw=','.join(spws),
def mk_qlook_image(vis, ncpu=10, twidth=12, stokes='I,V', antenna='', imagedir=None, spws=[], toTb=True, overwrite=True, doslfcal=False, phasecenter='', c_external=True): vis = [vis] subdir = ['/'] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] if not imagedir: imagedir = './' imres = {'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': [], 'Freq': []} msfile = vis[0] ms.open(msfile) metadata = ms.metadata() observatory = metadata.observatorynames()[0] # axisInfo = ms.getdata(["axis_info"], ifraxis=True) spwInfo = ms.getspectralwindowinfo() # freqInfo = axisInfo["axis_info"]["freq_axis"]["chan_freq"].swapaxes(0, 1) / 1e9 # freqInfo_ravel = freqInfo.ravel() ms.close() if not spws: if observatory == 'EVLA': spws = ['0', '1', '2', '3', '4', '5', '6', '7'] if observatory == 'EOVSA': spws = ['1~5', '6~10', '11~15', '16~25'] if observatory == 'EOVSA': print 'Provide stokes: ' + str(stokes) + '. However EOVSA has linear feeds. Force stokes to be IV' stokes = 'I,V' msfilebs = os.path.basename(msfile) imdir = imagedir + subdir[0] if not os.path.exists(imdir): os.makedirs(imdir) if doslfcal: slfcalms = './' + msfilebs + '.rr' split(msfile, outputvis=slfcalms, datacolumn='corrected', correlation='RR') for spw in spws: spwran = [s.zfill(2) for s in spw.split('~')] # freqran = [ # (int(s) * spwInfo['0']['TotalWidth'] + spwInfo['0']['RefFreq'] + spwInfo['0']['TotalWidth'] / 2.0) / 1.0e9 # for s in spw.split('~')] spw_ = spw.split('~') if len(spw_) == 2: freqran = [(spwInfo['{}'.format(s)]['RefFreq'] + spwInfo['{}'.format(s)]['TotalWidth'] / 2.0) / 1.0e9 for s in spw.split('~')] elif len(spw_) == 1: s = spw_[0] freqran = np.array([0, spwInfo['{}'.format(s)]['TotalWidth']]) + spwInfo['{}'.format(s)]['RefFreq'] freqran = freqran / 1.0e9 freqran = list(freqran) else: raise ValueError("Keyword 'spw' in wrong format") cfreq = np.mean(freqran) bmsz = max(30. / cfreq, 30.) uvrange = '<3km' if cfreq < 10.: imsize = 512 cell = ['5arcsec'] else: imsize = 1024 cell = ['2.5arcsec'] if len(spwran) == 2: spwstr = spwran[0] + '~' + spwran[1] else: spwstr = spwran[0] restoringbeam = ['{0:.1f}arcsec'.format(bmsz)] imagesuffix = '.spw' + spwstr.replace('~', '-') # if cfreq > 10.: # antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines sto = stokes.replace(',', '') if c_external: cleanscript = os.path.join(imdir, 'ptclean_external.py') resfile = os.path.join(imdir, os.path.basename(msfile) + '.res.npz') os.system('rm -rf {}'.format(cleanscript)) inpdict = {'vis': msfile, 'imageprefix': imdir, 'imagesuffix': imagesuffix, 'twidth': twidth, 'uvrange': uvrange, 'spw': spw, 'ncpu': ncpu, 'niter': 1000, 'gain': 0.05, 'antenna': antenna, 'imsize': imsize, 'cell': cell, 'stokes': sto, 'doreg': True, 'overwrite': overwrite, 'toTb': toTb, 'restoringbeam': restoringbeam, 'uvtaper': True, 'outertaper': ['30arcsec'], 'phasecenter': phasecenter} for key, val in inpdict.items(): if type(val) is str: inpdict[key] = '"{}"'.format(val) fi = open(cleanscript, 'wb') fi.write('from ptclean_cli import ptclean_cli as ptclean \n') fi.write('import numpy as np \n') fi.write( 'res = ptclean(vis={i[vis]},imageprefix={i[imageprefix]},imagesuffix={i[imagesuffix]},twidth={i[twidth]},uvrange={i[uvrange]},spw={i[spw]},ncpu={i[ncpu]},niter={i[niter]},gain={i[gain]},antenna={i[antenna]},imsize={i[imsize]},cell={i[cell]},stokes={i[stokes]},doreg={i[doreg]},overwrite={i[overwrite]},toTb={i[toTb]},restoringbeam={i[restoringbeam]},uvtaper={i[uvtaper]},outertaper={i[outertaper]},phasecenter={i[phasecenter]}) \n'.format( i=inpdict)) fi.write('np.savez("{}",res=res) \n'.format(resfile)) fi.close() os.system('casa --nologger -c {}'.format(cleanscript)) res = np.load(resfile) res = res['res'].item() else: res = ptclean(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange, spw=spw, ncpu=ncpu, niter=1000, gain=0.05, antenna=antenna, imsize=imsize, cell=cell, stokes=sto, doreg=True, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam, uvtaper=True, outertaper=['30arcsec'], phasecenter=phasecenter) if res: imres['Succeeded'] += res['Succeeded'] imres['BeginTime'] += res['BeginTime'] imres['EndTime'] += res['EndTime'] imres['ImageName'] += res['ImageName'] imres['Spw'] += [spwstr] * len(res['ImageName']) imres['Vis'] += [msfile] * len(res['ImageName']) imres['Freq'] += [freqran] * len(res['ImageName']) else: return None # save it for debugging purposes np.savez(os.path.join(imagedir, '{}.imres.npz'.format(os.path.basename(msfile))), imres=imres) return imres
def importeovsa_iter(filelist, timebin, width, visprefix, nocreatms, modelms, doscaling, keep_nsclms, fileidx): from taskinit import tb, casalog filename = filelist[fileidx] uv = aipy.miriad.UV(filename) # try: msname0 = list(filename.split('/')[-1]) msname = visprefix + ''.join(msname0) + '.ms' uv.select('antennae', 0, 1, include=True) uv.select('polarization', -5, -5, include=True) times = [] uv.rewind() for preamble, data in uv.all(): uvw, t, (i, j) = preamble times.append(t) uv.select('clear', -1, -1, include=True) times = ipe.jd2mjds(np.asarray(times)) inttime = np.median((times - np.roll(times, 1))[1:]) / 60 time_steps = len(times) durtim = int((times[-1] - times[0]) / 60 + inttime) time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'].replace('\x00', '') antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.where(uv['sfreq'] > 0)[0] nf = len(good_idx) npol = uv['npol'] nants = uv['nants'] source_id = uv['source'].replace('\x00', '') sfreq = uv['sfreq'][good_idx] sdf = uv['sdf'][good_idx] ra, dec = uv['ra'], uv['dec'] nbl = nants * (nants - 1) / 2 bl2ord = ipe.bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) chan_band = ipe.get_band(sfreq=sfreq, sdf=sdf) nband = len(chan_band) uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 data = ma.masked_array(ma.masked_invalid(data), fill_value=0.0) out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask # if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs if doscaling: out2 = out.copy() for i0 in antlist: for j0 in antlist: if i0 < j0: i, j = i0 - 1, j0 - 1 out2[:, :, :, bl2ord[i, j]] = out[:, :, :, bl2ord[i, j]] / np.sqrt( np.abs(out[:, :, :, bl2ord[i, i]]) * np.abs(out[:, :, :, bl2ord[j, j]])) out2 = out2.reshape(npol, nf, nrows) out2[np.isnan(out2)] = 0 out2[np.isinf(out2)] = 0 # out2 = ma.masked_array(ma.masked_invalid(out2), fill_value=0.0) out = out.reshape(npol, nf, nrows) * 1e4 flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows) uvwarray = np.tile(uvwarray, (1, nband)) sigma = np.ones((4, nrows), dtype=np.float) + 1 sigma = np.tile(sigma, (1, nband)) casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format( filename, (time.time() - time0))) if not nocreatms: modelms = ipe.creatms(filename, visprefix) os.system('mv {} {}'.format(modelms, msname)) else: casalog.post('----------------------------------------') casalog.post('copying standard MS to {0}'.format( msname, (time.time() - time0))) casalog.post('----------------------------------------') os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % modelms + " %s" % msname) casalog.post( 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format( msname, (time.time() - time0))) tb.open(msname, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() nchannels = len(cband['cidx']) for row in range(nrows): if not doscaling or keep_nsclms: tb.putcell('DATA', (row + l * nrows), out[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) tb.putcell('FLAG', (row + l * nrows), flag[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format( (l + 1), time.time() - time1)) tb.putcol('UVW', uvwarray) tb.putcol('SIGMA', sigma) tb.putcol('WEIGHT', 1.0 / sigma**2) timearr = times timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape(nband * npairs * time_steps) tb.putcol('TIME', timearr) tb.putcol('TIME_CENTROID', timearr) scan_id = tb.getcol('SCAN_NUMBER') scan_id *= 0 tb.putcol('SCAN_NUMBER', scan_id) colnames = tb.colnames() cols2rm = ["MODEL_DATA", "CORRECTED_DATA"] for l in range(len(cols2rm)): if cols2rm[l] in colnames: tb.removecols(cols2rm[l]) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the OBSERVATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/OBSERVATION', nomodify=False) tb.putcol( 'TIME_RANGE', np.asarray([times[0] - 0.5 * inttime, times[-1] + 0.5 * inttime]).reshape(2, 1)) tb.putcol('OBSERVER', ['EOVSA team']) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POINTING table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POINTING', nomodify=False) timearr = times.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape(nband * time_steps * nants) tb.putcol('TIME', timearr) tb.putcol('TIME_ORIGIN', timearr) # - 0.5 * delta_time) direction = tb.getcol('DIRECTION') direction[0, 0, :] = ra direction[1, 0, :] = dec tb.putcol('DIRECTION', direction) target = tb.getcol('TARGET') target[0, 0, :] = ra target[1, 0, :] = dec tb.putcol('TARGET', target) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the SOURCE table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/SOURCE', nomodify=False) radec = tb.getcol('DIRECTION') radec[0], radec[1] = ra, dec tb.putcol('DIRECTION', radec) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False) pol_id = tb.getcol('POLARIZATION_ID') pol_id *= 0 tb.putcol('POLARIZATION_ID', pol_id) # spw_id = tb.getcol('SPECTRAL_WINDOW_ID') # spw_id *= 0 # tb.putcol('SPECTRAL_WINDOW_ID', spw_id) tb.close() # casalog.post('----------------------------------------') # casalog.post("Updating the POLARIZATION table of" '%s' % msname) # casalog.post('----------------------------------------') # tb.open(msname + '/POLARIZATION/', nomodify=False) # tb.removerows(rownrs=np.arange(1, nband, dtype=int)) # tb.close() casalog.post('----------------------------------------') casalog.post("Updating the FIELD table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/FIELD/', nomodify=False) delay_dir = tb.getcol('DELAY_DIR') delay_dir[0], delay_dir[1] = ra, dec tb.putcol('DELAY_DIR', delay_dir) phase_dir = tb.getcol('PHASE_DIR') phase_dir[0], phase_dir[1] = ra, dec tb.putcol('PHASE_DIR', phase_dir) reference_dir = tb.getcol('REFERENCE_DIR') reference_dir[0], reference_dir[1] = ra, dec tb.putcol('REFERENCE_DIR', reference_dir) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME # del out, flag, uvwarray, uv, timearr, sigma # gc.collect() # if doscaling: if keep_nsclms: msname_scl = visprefix + ''.join(msname0) + '_scl.ms' os.system('cp -r {} {}'.format(msname, msname_scl)) else: msname_scl = msname tb.open(msname_scl, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname_scl) casalog.post('----------------------------------------') for l, cband in enumerate(chan_band): time1 = time.time() for row in range(nrows): tb.putcell( 'DATA', (row + l * nrows), out2[:, cband['cidx'][0]:cband['cidx'][-1] + 1, row]) casalog.post( '---spw {0:02d} is updated in --- {1:10.2f} seconds ---'. format((l + 1), time.time() - time1)) tb.close() if not (timebin == '0s' and width == 1): msfile = msname + '.split' if doscaling: split(vis=msname_scl, outputvis=msname_scl + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname_scl)) msfile_scl = msname_scl + '.split' if not (doscaling and not keep_nsclms): split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width, keepflags=False) os.system('rm -rf {}'.format(msname)) else: msfile = msname if doscaling: msfile_scl = msname_scl casalog.post("finished in --- %s seconds ---" % (time.time() - time0)) if doscaling: return [True, msfile, msfile_scl, durtim] else: return [True, msfile, durtim]
def importeovsa(idbfiles, timebin=None, width=None, visprefix=None, nocreatms=True, doconcat=False): casalog.origin('importeovsa') # # Initialize the helper class # pdh = ParallelDataHelper("importeovsa", locals()) # # # Validate input and output parameters # try: # pdh.setupIO() # except Exception, instance: # casalog.post('%s' % instance, 'ERROR') # return False if type(idbfiles) == Time: filelist = ri.get_trange_files(idbfiles) else: # If input type is not Time, assume that it is the list of files to read filelist = idbfiles if type(filelist) == str: filelist = [filelist] for f in filelist: if not os.path.exists(f): casalog.post("Some files in filelist are invalid. Aborting...") return False if not visprefix: visprefix = './' if not timebin: timebin = '0s' if not width: width = 1 if nocreatms: filename = filelist[0] modelms = creatms(filename, visprefix) msfile = [] for filename in filelist: uv = aipy.miriad.UV(filename) # if uv['source'].lower() == 'sun': # visprefix = visprefix + 'sun/' # if not os.path.exists(visprefix): # os.mkdir(visprefix) # else: # visprefix = visprefix + 'calibrator/' # if not os.path.exists(visprefix): # os.mkdir(visprefix) # uv.rewind() start_time = 0 # The start and stop times are referenced to ref_time_jd in second end_time = 600 delta_time = 1 time_steps = (end_time - start_time) / delta_time time0 = time.time() if 'antlist' in uv.vartable: ants = uv['antlist'] antlist = map(int, ants.split()) else: antlist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] good_idx = np.arange(len(uv['sfreq'])) ref_time_jd = uv['time'] ref_time_mjd = (ref_time_jd - 2400000.5) * 24. * 3600. + 0.5 * delta_time nf = len(good_idx) npol = uv['npol'] nants = uv['nants'] source_id = uv['source'] ra, dec = uv['ra'], uv['dec'] nbl = nants * (nants - 1) / 2 bl2ord = bl_list2(nants) npairs = nbl + nants flag = np.ones((npol, nf, time_steps, npairs), dtype=bool) out = np.zeros((npol, nf, time_steps, npairs), dtype=np.complex64) # Cross-correlations uvwarray = np.zeros((3, time_steps, npairs), dtype=np.float) bandedge = get_band_edge(nband=34) nband = len(bandedge) - 1 uv.rewind() l = -1 for preamble, data in uv.all(): uvw, t, (i0, j0) = preamble i = antlist.index(i0 + 1) j = antlist.index(j0 + 1) if i > j: # Reverse order of indices j = antlist.index(i0 + 1) i = antlist.index(j0 + 1) # Assumes uv['pol'] is one of -5, -6, -7, -8 k = -5 - uv['pol'] l += 1 out[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.data flag[k, :, l / (npairs * npol), bl2ord[i0, j0]] = data.mask if i != j: if k == 3: uvwarray[:, l / (npairs * npol), bl2ord[i0, j0]] = -uvw * constants.speed_of_light / 1e9 nrows = time_steps * npairs out = out.reshape(npol, nf, nrows) flag = flag.reshape(npol, nf, nrows) uvwarray = uvwarray.reshape(3, nrows) uvwarray = np.tile(uvwarray, (1, nband)) sigma = np.ones((4, nrows), dtype=np.float) + 1 sigma = np.tile(sigma, (1, nband)) casalog.post('IDB File {0} is readed in --- {1:10.2f} seconds ---'.format(filename, (time.time() - time0))) msname = list(filename.split('/')[-1]) msname.insert(11, 'T') msname = visprefix + source_id.upper() + '_' + ''.join(msname[3:]) + '-10m.ms' if not nocreatms: modelms = creatms(filename, visprefix) os.system('mv {} {}'.format(modelms, msname)) else: casalog.post('----------------------------------------') casalog.post('copying standard MS to {0}'.format(msname, (time.time() - time0))) casalog.post('----------------------------------------') os.system("rm -fr %s" % msname) os.system("cp -r " + " %s" % modelms + " %s" % msname) casalog.post( 'Standard MS is copied to {0} in --- {1:10.2f} seconds ---'.format(msname, (time.time() - time0))) tb.open(msname, nomodify=False) casalog.post('----------------------------------------') casalog.post("Updating the main table of" '%s' % msname) casalog.post('----------------------------------------') for l, bdedge in enumerate(bandedge[:-1]): time1 = time.time() nchannels = (bandedge[l + 1] - bandedge[l]) for row in range(nrows): tb.putcell('DATA', (row + l * nrows), out[:, bandedge[l]:bandedge[l + 1], row]) tb.putcell('FLAG', (row + l * nrows), flag[:, bandedge[l]:bandedge[l + 1], row]) casalog.post('---spw {0:02d} is updated in --- {1:10.2f} seconds ---'.format((l + 1), time.time() - time1)) tb.putcol('UVW', uvwarray) tb.putcol('SIGMA', sigma) tb.putcol('WEIGHT', 1.0 / sigma ** 2) timearr = np.arange((time_steps), dtype=np.float) timearr = timearr.reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, npairs)) timearr = timearr.reshape(nband * npairs * time_steps) + ref_time_mjd tb.putcol('TIME', timearr) tb.putcol('TIME_CENTROID', timearr) scan_id = tb.getcol('SCAN_NUMBER') scan_id *= 0 tb.putcol('SCAN_NUMBER', scan_id) colnames = tb.colnames() cols2rm = ["MODEL_DATA", "CORRECTED_DATA"] for l in range(len(cols2rm)): if cols2rm[l] in colnames: tb.removecols(cols2rm[l]) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the OBSERVATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/OBSERVATION', nomodify=False) tb.putcol('TIME_RANGE', np.asarray([ref_time_mjd - 0.5 * delta_time, ref_time_mjd + end_time - 0.5 * delta_time]).reshape( 2, 1)) tb.putcol('OBSERVER', ['EOVSA team']) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POINTING table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POINTING', nomodify=False) timearr = np.arange((time_steps), dtype=np.float).reshape(1, time_steps, 1) timearr = np.tile(timearr, (nband, 1, nants)) timearr = timearr.reshape(nband * time_steps * nants) + ref_time_mjd tb.putcol('TIME', timearr) tb.putcol('TIME_ORIGIN', timearr - 0.5 * delta_time) direction = tb.getcol('DIRECTION') direction[0, 0, :] = ra direction[1, 0, :] = dec tb.putcol('DIRECTION', direction) target = tb.getcol('TARGET') target[0, 0, :] = ra target[1, 0, :] = dec tb.putcol('TARGET', target) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the SOURCE table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/SOURCE', nomodify=False) radec = tb.getcol('DIRECTION') radec[0], radec[1] = ra, dec tb.putcol('DIRECTION', radec) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the DATA_DESCRIPTION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/DATA_DESCRIPTION/', nomodify=False) pol_id = tb.getcol('POLARIZATION_ID') pol_id *= 0 tb.putcol('POLARIZATION_ID', pol_id) # spw_id = tb.getcol('SPECTRAL_WINDOW_ID') # spw_id *= 0 # tb.putcol('SPECTRAL_WINDOW_ID', spw_id) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the POLARIZATION table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/POLARIZATION/', nomodify=False) tb.removerows(rownrs=np.arange(1, nband, dtype=int)) tb.close() casalog.post('----------------------------------------') casalog.post("Updating the FIELD table of" '%s' % msname) casalog.post('----------------------------------------') tb.open(msname + '/FIELD/', nomodify=False) delay_dir = tb.getcol('DELAY_DIR') delay_dir[0], delay_dir[1] = ra, dec tb.putcol('DELAY_DIR', delay_dir) phase_dir = tb.getcol('PHASE_DIR') phase_dir[0], phase_dir[1] = ra, dec tb.putcol('PHASE_DIR', phase_dir) reference_dir = tb.getcol('REFERENCE_DIR') reference_dir[0], reference_dir[1] = ra, dec tb.putcol('REFERENCE_DIR', reference_dir) name = np.array([source_id], dtype='|S{0}'.format(len(source_id) + 1)) tb.putcol('NAME', name) tb.close() # FIELD: DELAY_DIR, PHASE_DIR, REFERENCE_DIR, NAME del out, flag, uvwarray, uv, timearr, sigma gc.collect() # if not (timebin == '0s' and width == 1): split(vis=msname, outputvis=msname + '.split', datacolumn='data', timebin=timebin, width=width) os.system('rm -rf {}'.format(msname)) msfile.append(msname + '.split') else: msfile.append(msname) casalog.post("finished in --- %s seconds ---" % (time.time() - time0)) # msname = visprefix + source_id.upper() + '_' + ''.join(msname[3:]) + '-10m.ms' if doconcat: msname = list(filelist[0].split('/')[-1]) msname.insert(11, 'T') concatvis = visprefix + ''.join(msname[3:]) + '-{}m.ms'.format(10.0 * len(msfile)) concat(vis=msfile, concatvis=concatvis) for ll in msfile: os.system('rm -rf {}'.format(ll)) return True
from astropy.io import fits from astropy import wcs mses = list(Qmses.keys()) fullpath_mses = ['../' + ms[:-3] + "_continuum.ms" for ms in mses if ms in Qmses] raw_and_corr_vis = 'continuum_concatenated_raw_and_corr.ms' if not os.path.exists(raw_and_corr_vis): assert concat(vis=fullpath_mses, concatvis=raw_and_corr_vis, # should be used but isn't freqtol='5MHz', ) cont_vis = 'continuum_concatenated_selfcal_wterms.ms' if not os.path.exists(cont_vis): assert split(vis=raw_and_corr_vis, outputvis=cont_vis, datacolumn='corrected') # this is OK because there should be no corrected column clearcal(vis=cont_vis, addmodel=True) imagename = '18A-229_Q_mosaic_selfcal_iter0_dirty' if not os.path.exists(imagename+".image.tt0.pbcor"): # do a full-mosaic clean to enable mask creation tclean( vis=cont_vis, spw='', field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q", phasecenter='J2000 17h47m19.693 -28d23m11.527', imsize=[9000,9000], cell='0.02arcsec',
# # this is OK because there should be no corrected column #clearcal(vis=cont_vis, addmodel=True) def myprint(x): print(x) casalog.post(str(x), origin='singlefield') selfcal_mses = {} for ms in fullpath_mses: outms = ms[:-3]+"_split_for_selfcal_mergedates.ms" if not os.path.exists(outms): # TODO: come back later & keep in DS... split(vis=ms, outputvis=outms, datacolumn='corrected', field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q") msname = ms[16:21] if msname == '03_06': if 'T12' in ms: msname = '03_06_T12' else: msname = '03_06_T01' myprint(msname) selfcal_mses[msname] = outms myprint(selfcal_mses) cont_vis = list(selfcal_mses.values())