def concateovsa(msname, msfiles, visprefix='./', doclearcal=True, keep_orig_ms=False, cols2rm=["MODEL_DATA", "CORRECTED_DATA"]): concatvis = visprefix + msname msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] if doclearcal: print 'Warning: Corrected column in the input ms file will be cleared!!!' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) else: try: tmpdir = visprefix + '/tmp_ms/' if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = tmpdir + os.path.basename(str(ll)) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) except: print 'Warning: Corrected column not found in the input ms file.' msfiles_ = msfiles if msfiles_: concat(vis=msfiles_, concatvis=concatvis, timesort=True) else: concat(vis=msfiles, concatvis=concatvis, timesort=True) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) colnames = tb.colnames() for l in range(len(cols2rm)): if cols2rm[l] in colnames: try: tb.removecols(cols2rm[l]) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
def splitX(vis, datacolumn2='MODEL_DATA', **kwargs): import os from clearcal_cli import clearcal_cli as clearcal from split_cli import split_cli as split ''' :param msfile: :param datacolumn: :param datacolumn2: :return: ''' kwargs2 = kwargs.copy() datacolumn2 = datacolumn2.upper() outmsfile = kwargs['outputvis'] if outmsfile.endswith('/'): outmsfile = outmsfile[:-1] if os.path.exists(outmsfile): os.system('rm -rf {}'.format(outmsfile)) if os.path.exists('{}.flagversions'.format(outmsfile)): os.system('rm -rf {}.flagversions'.format(outmsfile)) split(vis, **kwargs) for k in ['datacolumn', 'outputvis']: if k in kwargs2: kwargs2.pop(k) kwargs2['outputvis'] = 'tmpms.ms' kwargs2['datacolumn'] = datacolumn2.replace('_DATA', '') if os.path.exists('tmpms.ms'): os.system('rm -rf tmpms.ms') split(vis, **kwargs2) tb.open('tmpms.ms') nrows = tb.nrows() data = [] for row in tqdm(range(nrows), desc='getting {} column'.format(datacolumn2), ascii=True): data.append(tb.getcell('DATA', row)) tb.close() clearcal(outmsfile, addmodel=True) tb.open(outmsfile, nomodify=False) for row in tqdm(range(nrows), desc='writing {} column'.format(datacolumn2), ascii=True): tb.putcell(datacolumn2, row, data[row]) tb.close() os.system('rm -rf tmpms.ms') return outmsfile
fullpath_mses = ['../' + ms[:-3] + "_continuum.ms" for ms in mses if ms in Qmses] raw_and_corr_vis = 'continuum_concatenated_raw_and_corr.ms' if not os.path.exists(raw_and_corr_vis): assert concat(vis=fullpath_mses, concatvis=raw_and_corr_vis, # should be used but isn't freqtol='5MHz', ) cont_vis = 'continuum_concatenated_selfcal_wterms.ms' if not os.path.exists(cont_vis): assert split(vis=raw_and_corr_vis, outputvis=cont_vis, datacolumn='corrected') # this is OK because there should be no corrected column clearcal(vis=cont_vis, addmodel=True) imagename = '18A-229_Q_mosaic_selfcal_iter0_dirty' if not os.path.exists(imagename+".image.tt0.pbcor"): # do a full-mosaic clean to enable mask creation tclean( vis=cont_vis, spw='', field="Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,Sgr B2 S Q", phasecenter='J2000 17h47m19.693 -28d23m11.527', imsize=[9000,9000], cell='0.02arcsec', imagename=imagename, niter=0, threshold='1000mJy',
def mk_qlook_image(trange, doimport=False, docalib=False, ncpu=10, twidth=12, stokes=None, antenna='0~12', lowcutoff_freq=3.7, imagedir=None, spws=['1~5', '6~10', '11~15', '16~25'], toTb=True, overwrite=True, doslfcal=False, verbose=False): ''' trange: can be 1) a single Time() object: use the entire day 2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00']) 3) a single or a list of UDBms file(s) 4) None -- use current date Time.now() ''' antenna0 = antenna if type(trange) == Time: mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] tsts = [l.to_datetime() for l in mslist['tstlist']] if type(trange) == str: try: date = Time(trange) mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] tsts = [l.to_datetime() for l in mslist['tstlist']] except: vis = [trange] tsts = [] for v in vis: tb.open(v + '/OBSERVATION') tsts.append( Time(tb.getcell('TIME_RANGE')[0] / 24 / 3600, format='mjd').datetime) tb.close() subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] if not stokes: stokes = 'XX' if not imagedir: imagedir = './' imres = { 'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': [], 'Synoptic': { 'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': [] } } for n, msfile in enumerate(vis): msfilebs = os.path.basename(msfile) imdir = imagedir + subdir[n] if not os.path.exists(imdir): os.makedirs(imdir) if doslfcal: slfcalms = './' + msfilebs + '.xx' split(msfile, outputvis=slfcalms, datacolumn='corrected', correlation='XX') cfreqs = getspwfreq(msfile) for spw in spws: antenna = antenna0 if spw == '': continue spwran = [s.zfill(2) for s in spw.split('~')] freqran = [cfreqs[int(s)] for s in spw.split('~')] cfreq = np.mean(freqran) bmsz = max(150. / cfreq, 20.) uvrange = '<10klambda' if doslfcal: slfcal_img = './' + msfilebs + '.slf.spw' + spw.replace( '~', '-') + '.slfimg' slfcal_tb = './' + msfilebs + '.slf.spw' + spw.replace( '~', '-') + '.slftb' try: clean(vis=slfcalms, antenna=antenna, imagename=slfcal_img, spw=spw, mode='mfs', timerange='', imagermode='csclean', psfmode='clark', imsize=[512, 512], cell=['5arcsec'], niter=100, gain=0.05, stokes='I', weighting='natural', restoringbeam=[str(bmsz) + 'arcsec'], pbcor=False, interactive=False, usescratch=True) except: print('error in cleaning spw: ' + spw) break gaincal(vis=slfcalms, refant='0', antenna=antenna, caltable=slfcal_tb, spw=spw, uvrange='', gaintable=[], selectdata=True, timerange='', solint='600s', gaintype='G', calmode='p', combine='', minblperant=3, minsnr=2, append=False) if not os.path.exists(slfcal_tb): print('No solution found in spw: ' + spw) break else: clearcal(slfcalms) delmod(slfcalms) applycal(vis=slfcalms, gaintable=[slfcal_tb], spw=spw, selectdata=True, antenna=antenna, interp='nearest', flagbackup=False, applymode='calonly', calwt=False) msfile = slfcalms imsize = 512 cell = ['5arcsec'] if len(spwran) == 2: spwstr = spwran[0] + '~' + spwran[1] else: spwstr = spwran[0] restoringbeam = ['{0:.1f}arcsec'.format(bmsz)] imagesuffix = '.spw' + spwstr.replace('~', '-') if cfreq > 10.: antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines # else: # antenna = antenna + ';!0&1' # deselect the shortest baselines res = ptclean3(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange, spw=spw, ncpu=ncpu, niter=1000, gain=0.05, antenna=antenna, imsize=imsize, cell=cell, stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam, specmode="mfs", deconvolver="hogbom", datacolumn='data', pbcor=True) if res: imres['Succeeded'] += res['Succeeded'] imres['BeginTime'] += res['BeginTime'] imres['EndTime'] += res['EndTime'] imres['ImageName'] += res['ImageName'] imres['Spw'] += [spwstr] * len(res['ImageName']) imres['Vis'] += [msfile] * len(res['ImageName']) else: continue if len(vis) == 1: # produce the band-by-band whole-day images ms.open(msfile) ms.selectinit() timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True) tim = timfreq['time'] ms.close() cfreqs = getspwfreq(msfile) imdir = imagedir + subdir[0] if not os.path.exists(imdir): os.makedirs(imdir) for spw in spws: antenna = antenna0 if spw == '': spw = '{:d}~{:d}'.format( next(x[0] for x in enumerate(cfreqs) if x[1] > lowcutoff_freq), len(cfreqs) - 1) spwran = [s.zfill(2) for s in spw.split('~')] freqran = [cfreqs[int(s)] for s in spw.split('~')] cfreq = np.mean(freqran) bmsz = max(150. / cfreq, 20.) uvrange = '' imsize = 512 cell = ['5arcsec'] if len(spwran) == 2: spwstr = spwran[0] + '~' + spwran[1] else: spwstr = spwran[0] restoringbeam = ['{0:.1f}arcsec'.format(bmsz)] imagesuffix = '.synoptic.spw' + spwstr.replace('~', '-') antenna = antenna + ';!0&1' # deselect the shortest baselines res = ptclean3(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=len(tim), uvrange=uvrange, spw=spw, ncpu=1, niter=0, gain=0.05, antenna=antenna, imsize=imsize, cell=cell, stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam, specmode="mfs", deconvolver="hogbom", datacolumn='data', pbcor=True) if res: imres['Synoptic']['Succeeded'] += res['Succeeded'] imres['Synoptic']['BeginTime'] += res['BeginTime'] imres['Synoptic']['EndTime'] += res['EndTime'] imres['Synoptic']['ImageName'] += res['ImageName'] imres['Synoptic']['Spw'] += [spwstr] * len(res['ImageName']) imres['Synoptic']['Vis'] += [msfile] * len(res['ImageName']) else: continue # save it for debugging purposes np.savez('imres.npz', imres=imres) return imres
imname = contimagename + "_robust{0}_selfcal{1}".format( robust, selfcaliter) if not os.path.exists(imname + ".image.tt0"): okfields, notokfields = goodenough_field_solutions(caltable, minsnr=5) if len(okfields) == 0: logprint("All fields flagged out of gaincal solns!", origin='contim_selfcal') raise ValueError("All fields flagged out of gaincal solns!") okfields_str = ",".join(["{0}".format(x) for x in okfields]) logprint("Fields {0} had min snr 5, fields {1} did not".format( okfields, notokfields), origin='contim_selfcal') clearcal(vis=selfcal_ms, addmodel=True) # use gainfield so we interpolate the good solutions to the other # fields applycal(vis=selfcal_ms, gainfield=okfields_str, gaintable=cals, interp="linear", applymode='calonly', calwt=False) # do not run the clean if no mask exists assert os.path.exists(maskname), "Mask {0} was not found.".format( maskname) # do this even if the output file exists: we need to populate the # modelcolumn
def calibeovsa(vis=None, caltype=None, interp=None, docalib=True, doflag=True, flagant=None, doimage=False, imagedir=None, antenna=None, timerange=None, spw=None, stokes=None, doconcat=False, msoutdir=None, keep_orig_ms=True): ''' :param vis: EOVSA visibility dataset(s) to be calibrated :param caltype: :param interp: :param docalib: :param qlookimage: :param flagant: :param stokes: :param doconcat: :return: ''' if type(vis) == str: vis = [vis] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] for msfile in vis: casalog.origin('calibeovsa') if not caltype: casalog.post( "Caltype not provided. Perform reference phase calibration and daily phase calibration." ) caltype = [ 'refpha', 'phacal', 'fluxcal' ] ## use this line after the phacal is applied # caltype = ['refcal'] if not os.path.exists(msfile): casalog.post("Input visibility does not exist. Aborting...") continue if msfile.endswith('/'): msfile = msfile[:-1] if not msfile[-3:] in ['.ms', '.MS']: casalog.post( "Invalid visibility. Please provide a proper visibility file ending with .ms" ) # if not caltable: # caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype] # get band information tb.open(msfile + '/SPECTRAL_WINDOW') nspw = tb.nrows() bdname = tb.getcol('NAME') bd_nchan = tb.getcol('NUM_CHAN') bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33 # nchans = tb.getcol('NUM_CHAN') # reffreqs = tb.getcol('REF_FREQUENCY') # cenfreqs = np.zeros((nspw)) tb.close() tb.open(msfile + '/ANTENNA') nant = tb.nrows() antname = tb.getcol('NAME') antlist = [str(ll) for ll in range(len(antname) - 1)] antennas = ','.join(antlist) tb.close() # get time stamp, use the beginning of the file tb.open(msfile + '/OBSERVATION') trs = {'BegTime': [], 'EndTime': []} for ll in range(tb.nrows()): tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600, format='mjd') trs['BegTime'].append(tim0) trs['EndTime'].append(tim1) tb.close() trs['BegTime'] = Time(trs['BegTime']) trs['EndTime'] = Time(trs['EndTime']) btime = np.min(trs['BegTime']) etime = np.max(trs['EndTime']) # ms.open(vis) # summary = ms.summary() # ms.close() # btime = Time(summary['BeginTime'], format='mjd') # etime = Time(summary['EndTime'], format='mjd') ## stop using ms.summary to avoid conflicts with importeovsa t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd') print "This scan observed from {} to {} UTC".format( btime.iso, etime.iso) gaintables = [] if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal' in caltype): refcal = ra.sql2refcalX(btime) pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband) pha[np.where(refcal['flag'] == 1)] = 0. amp = refcal['amp'] amp[np.where(refcal['flag'] == 1)] = 1. t_ref = refcal['timestamp'] # find the start and end time of the local day when refcal is registered try: dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60. except: dhr = -7. bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd') et = Time(bt.mjd + 1., format='mjd') (yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day) dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/' if not os.path.exists(dirname): os.mkdir(dirname) # check if there is any ROACH reboot between the reference calibration found and the current data t_rbts = db.get_reboot(Time([t_ref, btime])) if not t_rbts: casalog.post( "Reference calibration is derived from observation at " + t_ref.iso) print "Reference calibration is derived from observation at " + t_ref.iso else: casalog.post( "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso) casalog.post("Aborting...") print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso print "Aborting..." para_pha = [] para_amp = [] calpha = np.zeros((nspw, 15, 2)) calamp = np.zeros((nspw, 15, 2)) for s in range(nspw): for n in range(15): for p in range(2): calpha[s, n, p] = pha[n, p, bd[s]] calamp[s, n, p] = amp[n, p, bd[s]] para_pha.append(np.degrees(pha[n, p, bd[s]])) para_amp.append(amp[n, p, bd[s]]) if 'fluxcal' in caltype: calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59')) t_bp = Time(calfac['timestamp'], format='lv') if int(t_mid.mjd) == int(t_bp.mjd): accalfac = calfac['accalfac'] # (ant x pol x freq) # tpcalfac = calfac['tpcalfac'] # (ant x pol x freq) caltb_autoamp = dirname + t_bp.isot[:-4].replace( ':', '').replace('-', '') + '.bandpass' if not os.path.exists(caltb_autoamp): bandpass(vis=msfile, caltable=caltb_autoamp, solint='inf', refant='eo01', minblperant=0, minsnr=0, bandtype='B', docallib=False) tb.open(caltb_autoamp, nomodify=False) # (ant x spw) bd_chanidx = np.hstack([[0], bd_nchan.cumsum()]) for ll in range(nspw): antfac = np.sqrt( accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]]) # # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]] antfac = np.moveaxis(antfac, 0, 2) cparam = np.zeros((2, bd_nchan[ll], nant)) cparam[:, :, :-3] = 1.0 / antfac tb.putcol('CPARAM', cparam + 0j, ll * nant, nant) paramerr = tb.getcol('PARAMERR', ll * nant, nant) paramerr = paramerr * 0 tb.putcol('PARAMERR', paramerr, ll * nant, nant) bpflag = tb.getcol('FLAG', ll * nant, nant) bpant1 = tb.getcol('ANTENNA1', ll * nant, nant) bpflagidx, = np.where(bpant1 >= 13) bpflag[:] = False bpflag[:, :, bpflagidx] = True tb.putcol('FLAG', bpflag, ll * nant, nant) bpsnr = tb.getcol('SNR', ll * nant, nant) bpsnr[:] = 100.0 bpsnr[:, :, bpflagidx] = 0.0 tb.putcol('SNR', bpsnr, ll * nant, nant) tb.close() msg_prompt = "Scaling calibration is derived for {}.".format( msfile) casalog.post(msg_prompt) print msg_prompt gaintables.append(caltb_autoamp) else: msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format( t_mid.datetime.strftime('%b %d, %Y'), msfile) casalog.post(msg_prompt) print msg_prompt if ('refpha' in caltype) or ('refcal' in caltype): # caltb_pha = os.path.basename(vis).replace('.ms', '.refpha') # check if the calibration table already exists caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace( '-', '') + '.refpha' if not os.path.exists(caltb_pha): gencal(vis=msfile, caltable=caltb_pha, caltype='ph', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_pha) gaintables.append(caltb_pha) if ('refamp' in caltype) or ('refcal' in caltype): # caltb_amp = os.path.basename(vis).replace('.ms', '.refamp') caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace( '-', '') + '.refamp' if not os.path.exists(caltb_amp): gencal(vis=msfile, caltable=caltb_amp, caltype='amp', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_amp) gaintables.append(caltb_amp) # calibration for the change of delay center between refcal time and beginning of scan -- hopefully none! xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False) if buf: dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv') dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15] xml, buf = ch.read_calX(4, t=t_ref) dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv') dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15] dlycen_ns_diff = dlycen_ns2 - dlycen_ns1 for n in range(2): dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n] print 'Multi-band delay is derived from delay center difference at {} & {}'.format( dly_t1.iso, dly_t2.iso) # print '=====Delays relative to Ant 14=====' # for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]): # ant = antlist[i] # print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl # dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1]) # caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0') caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace( '-', '') + '.dlycen' if not os.path.exists(caltb_dlycen): gencal(vis=msfile, caltable=caltb_dlycen, caltype='mbd', pol='X,Y', antenna=antennas, parameter=dlycen_ns_diff.flatten().tolist()) gaintables.append(caltb_dlycen) if 'phacal' in caltype: phacals = np.array( ra.sql2phacalX([bt, et], neat=True, verbose=False)) if not phacals.any() or len(phacals) == 0: print "Found no phacal records in SQL database, will skip phase calibration" else: # first generate all phacal calibration tables if not already exist t_phas = Time([phacal['t_pha'] for phacal in phacals]) # sort the array in ascending order by t_pha sinds = t_phas.mjd.argsort() t_phas = t_phas[sinds] phacals = phacals[sinds] caltbs_phambd = [] for i, phacal in enumerate(phacals): # filter out phase cals with reference time stamp >30 min away from the provided refcal time if (phacal['t_ref'].jd - refcal['timestamp'].jd) > 30. / 1440.: del phacals[i] del t_phas[i] continue else: t_pha = phacal['t_pha'] phambd_ns = phacal['pslope'] for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n] # set all flagged values to be zero phambd_ns[np.where(phacal['flag'] == 1)] = 0. caltb_phambd = dirname + t_pha.isot[:-4].replace( ':', '').replace('-', '') + '.phambd' caltbs_phambd.append(caltb_phambd) if not os.path.exists(caltb_phambd): gencal(vis=msfile, caltable=caltb_phambd, caltype='mbd', pol='X,Y', antenna=antennas, parameter=phambd_ns.flatten().tolist()) # now decides which table to apply depending on the interpolation method ("neatest" or "linear") if interp == 'nearest': tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd)) dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24. print "Selected nearest phase calibration table at " + t_phas[ tbind].iso gaintables.append(caltbs_phambd[tbind]) if interp == 'linear': # bphacal = ra.sql2phacalX(btime) # ephacal = ra.sql2phacalX(etime,reverse=True) bt_ind, = np.where(t_phas.mjd < btime.mjd) et_ind, = np.where(t_phas.mjd > etime.mjd) if len(bt_ind) == 0 and len(et_ind) == 0: print "No phacal found before or after the ms data within the day of observation" print "Skipping daily phase calibration" elif len(bt_ind) > 0 and len(et_ind) == 0: gaintables.append(caltbs_phambd[bt_ind[-1]]) elif len(bt_ind) == 0 and len(et_ind) > 0: gaintables.append(caltbs_phambd[et_ind[0]]) elif len(bt_ind) > 0 and len(et_ind) > 0: bphacal = phacals[bt_ind[-1]] ephacal = phacals[et_ind[0]] # generate a new table interpolating between two daily phase calibrations t_pha_mean = Time(np.mean( [bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]), format='mjd') phambd_ns = (bphacal['pslope'] + ephacal['pslope']) / 2. for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n] # set all flagged values to be zero phambd_ns[np.where(bphacal['flag'] == 1)] = 0. phambd_ns[np.where(ephacal['flag'] == 1)] = 0. caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace( ':', '').replace('-', '') + '.phambd' if not os.path.exists(caltb_phambd_interp): gencal(vis=msfile, caltable=caltb_phambd_interp, caltype='mbd', pol='X,Y', antenna=antennas, parameter=phambd_ns.flatten().tolist()) print "Using phase calibration table interpolated between records at " + bphacal[ 't_pha'].iso + ' and ' + ephacal['t_pha'].iso gaintables.append(caltb_phambd_interp) if docalib: clearcal(msfile) applycal(vis=msfile, gaintable=gaintables, applymode='calflag', calwt=False) # delete the interpolated phase calibration table try: caltb_phambd_interp except: pass else: if os.path.exists(caltb_phambd_interp): shutil.rmtree(caltb_phambd_interp) if doflag: # flag zeros and NaNs flagdata(vis=msfile, mode='clip', clipzeros=True) if flagant: try: flagdata(vis=msfile, antenna=flagant) except: print "Something wrong with flagant. Abort..." if doimage: from matplotlib import pyplot as plt from suncasa.utils import helioimage2fits as hf from sunpy import map as smap if not antenna: antenna = '0~12' if not stokes: stokes = 'XX' if not timerange: timerange = '' if not spw: spw = '1~3' if not imagedir: imagedir = '.' #(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day) #dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/' #if not os.path.exists(dirname): # os.makedirs(dirname) bds = [spw] nbd = len(bds) imgs = [] for bd in bds: if '~' in bd: bdstr = bd.replace('~', '-') else: bdstr = str(bd).zfill(2) imname = imagedir + '/' + os.path.basename(msfile).replace( '.ms', '.bd' + bdstr) print 'Cleaning image: ' + imname try: clean(vis=msfile, imagename=imname, antenna=antenna, spw=bd, timerange=timerange, imsize=[512], cell=['5.0arcsec'], stokes=stokes, niter=500) except: print 'clean not successfull for band ' + str(bd) else: imgs.append(imname + '.image') junks = ['.flux', '.mask', '.model', '.psf', '.residual'] for junk in junks: if os.path.exists(imname + junk): shutil.rmtree(imname + junk) tranges = [btime.iso + '~' + etime.iso] * nbd fitsfiles = [img.replace('.image', '.fits') for img in imgs] hf.imreg(vis=msfile, timerange=tranges, imagefile=imgs, fitsfile=fitsfiles, usephacenter=False) plt.figure(figsize=(6, 6)) for i, fitsfile in enumerate(fitsfiles): plt.subplot(1, nbd, i + 1) eomap = smap.Map(fitsfile) sz = eomap.data.shape if len(sz) == 4: eomap.data = eomap.data.reshape((sz[2], sz[3])) eomap.plot_settings['cmap'] = plt.get_cmap('jet') eomap.plot() eomap.draw_limb() eomap.draw_grid() plt.show() if doconcat: if len(vis) > 1: # from suncasa.eovsa import concateovsa as ce from suncasa.tasks import concateovsa_cli as ce if msoutdir is None: msoutdir = './' concatvis = os.path.basename(vis[0]) concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms' ce.concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=keep_orig_ms, cols2rm="model,corrected") return [concatvis] else: return vis
def concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=True, cols2rm="model,corrected", freqtol="", dirtol="", respectname=False, timesort=True, copypointing=True, visweightscale=[], forcesingleephemfield=""): if concatvis[-1] == os.path.sep: concatvis = concatvis[:-1] if os.path.sep not in concatvis: visprefix = './' else: visprefix = os.path.dirname(concatvis) + os.path.sep msfiles = vis msfiles_ = [] for idx, ll in enumerate(msfiles): if str(ll).endswith('/'): msfiles[idx] = str(ll)[:-1] datacolumn = datacolumn.lower() if datacolumn == 'data': print 'DATA columns will be concatenated.' for ll in msfiles: clearcal(vis=str(ll), addmodel=True) elif datacolumn == 'corrected': # try: print 'CORRECTED columns will be concatenated.' tmpdir = os.path.join(visprefix, 'tmp_ms') + os.path.sep if not os.path.exists(tmpdir): os.makedirs(tmpdir) for ll in msfiles: msfile_ = os.path.join(tmpdir, os.path.basename(str(ll))) msfiles_.append(msfile_) split(vis=str(ll), outputvis=msfile_, datacolumn='corrected') clearcal(vis=msfile_, addmodel=True) else: raise ValueError('Please set datacolumn to be "data" or "corrected"!') if msfiles_: concat(vis=msfiles_, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) os.system('rm -rf {}'.format(tmpdir)) else: concat(vis=msfiles, concatvis=concatvis, freqtol=freqtol, dirtol=dirtol, respectname=respectname, timesort=timesort, copypointing=copypointing, visweightscale=visweightscale, forcesingleephemfield=forcesingleephemfield) # Change all observation ids to be the same (zero) tb.open(concatvis + '/OBSERVATION', nomodify=False) nobs = tb.nrows() tim0 = tb.getcell('TIME_RANGE', 0)[0] tim1 = tb.getcell('TIME_RANGE', nobs - 1)[1] tb.removerows([i + 1 for i in range(nobs - 1)]) tb.putcell('TIME_RANGE', 0, [tim0, tim1]) tb.close() tb.open(concatvis + '/DATA_DESCRIPTION', nomodify=False) nrows = tb.nrows() pol_id = tb.getcol('POLARIZATION_ID') tb.removerows(np.where(pol_id != 0)[0]) tb.close() tb.open(concatvis, nomodify=False) dd_id = tb.getcol('DATA_DESC_ID') idx_dd_id, = np.where(dd_id >= nrows / 2) dd_id[idx_dd_id] = dd_id[idx_dd_id] - nrows / 2 tb.putcol('DATA_DESC_ID', dd_id) tb.close() tb.open(concatvis + '/FIELD', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis + '/SOURCE', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() tb.open(concatvis, nomodify=False) obsid = tb.getcol('OBSERVATION_ID') newobsid = np.zeros(len(obsid), dtype='int') tb.putcol('OBSERVATION_ID', newobsid) fldid = tb.getcol('FIELD_ID') newfldid = np.zeros(len(fldid), dtype='int') tb.putcol('FIELD_ID', newfldid) colnames = tb.colnames() cols2rm = cols2rm.upper() cols2rm = cols2rm.split(',') for l in range(len(cols2rm)): col = cols2rm[l] + '_DATA' if col in colnames: try: tb.removecols(col) print 'Column {} removed.'.format(col) except: pass tb.close() if msfiles_ != [] and msfiles_ != msfiles: for ll in msfiles_: os.system('rm -rf {}'.format(ll)) if not keep_orig_ms: for ll in msfiles: os.system('rm -rf {}'.format(ll))
def feature_slfcal(vis, niter=200, spws=['0~1', '2~5', '6~10', '11~20', '21~30', '31~49'], slfcaltbdir='./', bright=None, pols='XX'): ''' Uses images from disk-selfcaled data as model for further self-calibration of outer antennas. This is only a good idea if there are bright active regions that provide strong signal on the londer baselines. ''' trange = ant_trange(vis) if bright is None: bright = [True] * len(spws) # Insert model into ms and do "inf" gaincal, appending to table each subsequent time if os.path.exists('images_init'): os.system('rm -rf images_init') os.system('mv images images_init') clearcal(vis, addmodel=True) fd_images(vis, cleanup=False, niter=niter, spws=spws, bright=bright) # Does shallow clean for selfcal purposes tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d') caltb = os.path.join(slfcaltbdir, tdate + '_d1.pha') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) for s, sp in enumerate(spws): if bright[s]: spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')]) imname = "images/briggs" + spwstr + '.model' if sp == '31~49': # The high-band image is only made to band 43, so adjust the name imname = 'images/briggs31-43.model' ft(vis=vis, spw=sp, model=imname, usescratch=True, incremental=True) if pols == 'XXYY': mstl.modeltransfer(vis, spw=sp) if pols == 'XXYY': mstl.gaincalXY(vis=vis, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda', combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G', minsnr=1.0, calmode='p', append=False) else: gaincal(vis=vis, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda', combine="scan", antenna='0~12&0~12', refant='0', refantmode="strict", solint='inf', gaintype='G', minsnr=1.0, calmode='p', append=False) # Apply the corrections to the data and split to a new ms applycal(vis=vis, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False, applymode="calonly") vis1 = 'dslf1_' + vis if os.path.exists(vis1): os.system('rm -rf {}'.format(vis1)) split(vis, outputvis=vis1, datacolumn="corrected") caltb = os.path.join(slfcaltbdir, tdate + '_d2.pha') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) # Move the existing images directory so that a new one will be created if os.path.exists('images_ftcal_rnd1'): os.system('rm -rf images_ftcal_rnd1') # shutil.move('images', 'old_images2') os.system('mv images images_ftcal_rnd1') # Make new model images for another round of selfcal fd_images(vis1, cleanup=False, niter=niter, spws=spws, bright=bright) for s, sp in enumerate(spws): if bright[s]: spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')]) imname = "images/briggs" + spwstr + '.model' if sp == '31~49': # The high-band image is only made to band 43, so adjust the name imname = 'images/briggs31-43.model' ft(vis=vis1, spw=sp, model=imname, usescratch=True) if pols == 'XXYY': mstl.modeltransfer(vis1, spw=sp) if pols == 'XXYY': mstl.gaincalXY(vis=vis1, caltable=caltb, pols=pols, selectdata=True, timerange=trange, uvrange='>1.5Klambda', combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict", gaintype='G', minsnr=1.0, calmode='p', append=False) else: gaincal(vis=vis1, caltable=caltb, selectdata=True, timerange=trange, uvrange='>1.5Klambda', combine="scan", antenna='0~12&0~12', refant='0', solint='10min', refantmode="strict", gaintype='G', minsnr=1.0, calmode='p', append=False) # Apply the corrections to the data and split to a new ms applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="linear", calwt=False, applymode="calonly") vis2 = 'dslf2_' + vis if os.path.exists(vis2): os.system('rm -rf {}'.format(vis2)) mstl.splitX(vis1, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data") if os.path.exists('images_ftcal_rnd2'): os.system('rm -rf images_ftcal_rnd2') os.system('mv images images_ftcal_rnd2') return vis2
def disk_slfcal(vis, slfcaltbdir='./', active=False, clearcache=False, pols='XX'): ''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name) add a model disk based on the solar disk size for that date and perform multiple selfcal adjustments (two phase and one amplitude), and write out a final selfcaled database with the disk subtracted. Returns the name of the final database. ''' trange = ant_trange(vis) if vis.endswith('/'): vis = vis[:-1] # Use vis name to determine date, and hence number of bands spw2band = np.array([0, 1] + range(4, 52)) defaultfreq = 1.1 + 0.325 * (spw2band + 0.5) # Calculate the center frequency of each spectral window if mstl.get_trange(vis)[0].mjd > 58536: # After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49 nbands = 52 freq = defaultfreq else: # Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30 nbands = 34 freq = 1.419 + np.arange(nbands) / 2. caltbs = [] slashdate = trange[:10] # Verify that the vis is not in the current working directory ''' if os.getcwd() == os.path.dirname(vis): print('Cannot copy vis file onto itself.') print('Please change to a different working directory') return None # Copy original ms to local directory if os.path.exists(os.path.basename(vis)): shutil.rmtree(os.path.basename(vis)) print('Copy {} to working directory {}.'.format(vis, os.getcwd())) shutil.copytree(vis, os.path.basename(vis)) vis = os.path.basename(vis) ''' if not active: clearcal(vis) flagmanager(vis, mode='save', versionname='with-RFI-or-BURSTS') ## automaticaly flag any high amplitudes from flares or RFI flagdata(vis=vis, mode="tfcrop", spw='', action='apply', display='', timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=False) flagmanager(vis, mode='save', versionname='without-RFI-or-BURSTS') dsize, fdens = calc_diskmodel(slashdate, nbands, freq, defaultfreq) diskxmlfile = vis + '.SOLDISK.xml' # Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name) msfile, diskim = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile, active=active) if pols == 'XXYY': caltbs_ = {'XX': [], 'YY': []} pols_ = ['XX', 'YY'] msfileXY = {} for pol in pols_: msfileXY[pol] = '.'.join([msfile, pol]) if os.path.exists(msfileXY[pol]): os.system('rm -rf {}'.format(msfileXY[pol])) mstl.splitX(vis=msfile, outputvis=msfileXY[pol], correlation=pol, datacolumn='data', datacolumn2='MODEL_DATA') tdate = mstl.get_trange(msfile)[0].datetime.strftime('%Y%m%d') caltb = os.path.join(slfcaltbdir, tdate + '_1.pha') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) if pols == 'XXYY': mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, selectdata=True, uvrange="", antenna="0~12&0~12", solint="inf", combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False) for pol in pols_: caltb_ = '.'.join([caltb, pol]) caltbs_[pol].append(caltb_) else: gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="inf", combine="scan", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False) caltbs.append(caltb) caltb = os.path.join(slfcaltbdir, tdate + '_2.pha') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) # Second round of phase selfcal on the disk using solution interval "1min" if pols == 'XXYY': mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_, selectdata=True, uvrange="", antenna="0~12&0~12", solint="10min", combine="scan", interp="linear", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False) for pol in pols_: caltb_ = '.'.join([caltb, pol]) caltbs_[pol].append(caltb_) else: gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", solint="10min", combine="scan", gaintable=caltbs, interp="linear", refant="0", refantmode="strict", minsnr=1.0, gaintype="G", calmode="p", append=False) caltbs.append(caltb) caltb = os.path.join(slfcaltbdir, tdate + '_3.amp') if os.path.exists(caltb): os.system('rm -rf {}*'.format(caltb)) # Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT) if pols == 'XXYY': mstl.gaincalXY(vis=msfile, caltable=caltb, pols=pols, msfileXY=msfileXY, gaintableXY=caltbs_, selectdata=True, uvrange="", antenna="0~12&0~12", timerange=trange, interp="linear", solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a", append=False) for pol in pols_: caltb_ = '.'.join([caltb, pol]) caltbs_[pol].append(caltb_) else: gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="", antenna="0~12&0~12", timerange=trange, gaintable=caltbs, interp="linear", solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a", append=False) mstl.flagcaltboutliers(caltb, limit=[0.125, 8.0]) # mstl.flagcaltboutliers(caltb, limit=[0.5, 2.0]) caltbs.append(caltb) # Split out corrected data and model and do uvsub vis2 = 'slf3_' + msfile if os.path.exists(vis2): os.system('rm -rf {}'.format(vis2)) if os.path.exists(vis2 + '.flagversions'): os.system('rm -rf {}'.format(vis2 + '.flagversions')) # flagmanager(msfile, mode='restore', versionname='with-RFI-or-BURSTS') clearcal(msfile) applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltbs, interp="linear", calwt=False, applymode="calonly") split(msfile, outputvis=vis2, datacolumn="corrected") for sp, dkim in tqdm(enumerate(diskim), desc='Inserting disk model', ascii=True): ft(vis=vis2, spw=str(sp), field='', model=str(dkim), nterms=1, reffreq="", complist="", incremental=False, usescratch=True) # mstl.modeltransfer(msfile, spw='{}'.format(sp)) uvsub(vis=vis2, reverse=False) # Final split to final = 'final_' + msfile if os.path.exists(final): os.system('rm -rf {}'.format(final)) if os.path.exists(final + '.flagversions'): os.system('rm -rf {}'.format(final + '.flagversions')) split(vis2, outputvis=final, datacolumn='corrected') os.system('mv {} {}'.format(msfile + '.flagversions', final + '.flagversions')) # Remove the interim ms files if clearcache: if os.path.exists(msfile): os.system('rm -rf {}'.format(msfile)) if os.path.exists(msfile + '.flagversions'): os.system('rm -rf {}'.format(msfile + '.flagversions')) if os.path.exists(vis2): os.system('rm -rf {}'.format(vis2)) if os.path.exists(vis2 + '.flagversions'): os.system('rm -rf {}'.format(vis2 + '.flagversions')) # Return the name of the selfcaled ms return final, diskxmlfile
split( vis=fullpathms, outputvis=cont_ms, field=('Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q,' 'Sgr B2 S Q,Sgr B2 DS1 Q,Sgr B2 DS2 Q,Sgr B2 DS3 Q'), width=16, spw=Qmses[ms], ) # Unflag CH3OH maser flagdata(vis=fullpathms, mode='unflag', spw='44054800170~44084179830Hz:44054800170~44084179830Hz') clearcal(cont_ms, addmodel=True) #ft(vis=cont_ms, # field='Sgr B2 N Q,Sgr B2 NM Q,Sgr B2 MS Q', # spw='', # model=['../continuum/18A-229_mosaic_for_selfcal.model.tt0.noneg', # '../continuum/18A-229_mosaic_for_selfcal.model.tt1.noneg'], # nterms=2) model = '../continuum/18A-229_mosaic_for_selfcal.model.tt0.noneg' # this image is reliable, even if the phase is no longer directly measured. # All of the other approaches have done really weird things. # Definitely DO NOT use this for amplitude self-cal!! model = '../reduction_scripts/DePree_NM_regridNM_jypix.image' if os.path.exists(model):
def mk_qlook_image(trange, doimport=False, docalib=False, ncpu=10, twidth=12, stokes=None, antenna='0~12', #imagedir=None, spws=['1~3','4~6','7~9','10~13','14~18','19~28'],verbose=False): imagedir=None, spws=['1~5','6~10','11~15','16~25'], toTb=True, overwrite=True, doslfcal=False, verbose=False): ''' trange: can be 1) a single Time() object: use the entire day 2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00']) 3) a single or a list of UDBms file(s) 4) None -- use current date Time.now() ''' if type(trange) == Time: mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] tsts = [l.to_datetime() for l in mslist['tstlist']] subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts] if type(trange) == str: try: date = Time(trange) mslist = trange2ms(trange=trange, doimport=doimport) vis = mslist['ms'] except: vis = [trange] subdir = ['/'] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] if not stokes: stokes = 'XX' if not imagedir: imagedir='./' imres = {'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': []} for n, msfile in enumerate(vis): msfilebs=os.path.basename(msfile) imdir = imagedir + subdir[n] if not os.path.exists(imdir): os.makedirs(imdir) if doslfcal: slfcalms = './'+msfilebs+'.xx' split(msfile,outputvis=slfcalms,datacolumn='corrected',correlation='XX') for spw in spws: spwran = [s.zfill(2) for s in spw.split('~')] freqran = [int(s)*0.5+2.9 for s in spw.split('~')] cfreq=np.mean(freqran) bmsz=max(150./cfreq,20.) uvrange='<10klambda' if doslfcal: slfcal_img = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slfimg' slfcal_tb = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slftb' try: clean(vis=slfcalms, antenna=antenna, imagename=slfcal_img, spw=spw, mode='mfs', timerange='', imagermode='csclean', psfmode='clark', imsize=[512,512], cell=['5arcsec'], niter=100, gain=0.05, stokes='I', weighting='natural', restoringbeam=[str(bmsz)+'arcsec'], pbcor=False, interactive=False, usescratch=True) except: print 'error in cleaning spw: '+spw break gaincal(vis=slfcalms, refant='0',antenna=antenna,caltable=slfcal_tb,spw=spw, uvrange='',\ gaintable=[],selectdata=True,timerange='',solint='600s',gaintype='G',calmode='p',\ combine='',minblperant=3,minsnr=2,append=False) if not os.path.exists(slfcal_tb): print 'No solution found in spw: '+spw break else: clearcal(slfcalms) delmod(slfcalms) applycal(vis=slfcalms,gaintable=[slfcal_tb],spw=spw,selectdata=True,\ antenna=antenna,interp='nearest',flagbackup=False,applymode='calonly',calwt=False) msfile=slfcalms if cfreq < 10.: imsize=512 cell=['5arcsec'] else: imsize=1024 cell=['2.5arcsec'] if len(spwran) == 2: spwstr = spwran[0]+'~'+spwran[1] else: spwstr = spwran[0] restoringbeam=['{0:.1f}arcsec'.format(bmsz)] imagesuffix='.spw'+spwstr.replace('~','-') if cfreq > 10.: antenna=antenna+';!0&1;!0&2' #deselect the shortest baselines res=ptclean(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange, spw=spw, ncpu=ncpu, niter=1000, gain=0.05, antenna=antenna,imsize=imsize, cell=cell, stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam, uvtaper=True,outertaper=['30arcsec']) if res: imres['Succeeded'] += res['Succeeded'] imres['BeginTime'] += res['BeginTime'] imres['EndTime'] += res['EndTime'] imres['ImageName'] += res['ImageName'] imres['Spw'] += [spwstr]*len(res['ImageName']) imres['Vis'] += [msfile]*len(res['ImageName']) else: return None #save it for debugging purposes np.savez('imres.npz',imres=imres) return imres
def disk_slfcal(vis, slfcaltbdir='./'): ''' Starting with the name of a calibrated ms (vis, which must have 'UDByyyymmdd' in the name) add a model disk based on the solar disk size for that date and perform multiple selfcal adjustments (two phase and one amplitude), and write out a final selfcaled database with the disk subtracted. Returns the name of the final database. ''' trange = ant_trange(vis) # Use vis name to determine date, and hence number of bands spw2band = np.array([0, 1] + range(4, 52)) defaultfreq = 1.1 + 0.325 * (spw2band + 0.5) # Calculate the center frequency of each spectral window if mstl.get_trange(vis)[0].mjd > 58536: # After 2019 Feb 22, the band numbers changed to 1-52, and spw from 0-49 nbands = 52 freq = defaultfreq else: # Before 2019 Feb 22, the band numbers were 1-34, and spw from 0-30 nbands = 34 freq = np.hstack([[1.419], 2.0 + 0.5 * (np.arange(32)) + (0.5 - 0.081)]) slashdate = trange[:10] # Verify that the vis is not in the current working directory if os.getcwd() == os.path.dirname(vis): print('Cannot copy vis file onto itself.') print('Please change to a different working directory') return None # Copy original ms to local directory if os.path.exists(os.path.basename(vis)): shutil.rmtree(os.path.basename(vis)) print('Copy {} to working directory {}.'.format(vis, os.getcwd())) shutil.copytree(vis, os.path.basename(vis)) vis = os.path.basename(vis) clearcal(vis) ## automaticaly flag impossibly high amplitudes flagdata(vis=vis, mode="tfcrop", spw='', correlation='ABS_XX', action='apply', display='', timecutoff=3.0, freqcutoff=2.0, maxnpieces=2, flagbackup=True) # Default disk size measured for 2019/09/03 # todo add monthly fitting procedure for the disk size and flux density defaultsize = np.array([990.6, 989.4, 988.2, 987.1, 986.0, 984.9, 983.8, 982.7, 981.7, 980.7, 979.7, 978.8, 977.8, 976.9, 976.0, 975.2, 974.3, 973.5, 972.7, 972.0, 971.2, 970.5, 969.8, 969.1, 968.5, 967.8, 967.2, 966.7, 966.1, 965.6, 965.1, 964.6, 964.1, 963.7, 963.3, 962.9, 962.5, 962.1, 961.8, 961.5, 961.3, 961.0, 960.8, 960.6, 960.4, 960.2, 960.1, 960.0, 959.9, 959.8]) # Get current solar distance and modify the default size accordingly fac = eph.get_sunearth_distance('2019/09/03') / eph.get_sunearth_distance(slashdate) newsize = defaultsize * fac.to_value() if nbands == 34: # Interpolate size to 31 spectal windows newsize = np.polyval(np.polyfit(defaultfreq, newsize, 5), freq) dsize = np.array([str(i)[:5] + 'arcsec' for i in newsize], dtype='S12') # These are nominal flux densities * 2, determined on 2019/09/03 defaultfdens = np.array([891282, 954570, 1173229, 1245433, 1373730, 1506802, 1613253, 1702751, 1800721, 1946756, 2096020, 2243951, 2367362, 2525968, 2699795, 2861604, 3054829, 3220450, 3404182, 3602625, 3794312, 3962926, 4164667, 4360683, 4575677, 4767210, 4972824, 5211717, 5444632, 5648266, 5926634, 6144249, 6339863, 6598018, 6802707, 7016012, 7258929, 7454951, 7742816, 7948976, 8203206, 8411834, 8656720, 8908130, 9087766, 9410760, 9571365, 9827078, 10023598, 8896671]) fdens = defaultfdens if nbands == 34: # Interpolate size to 31 spectal windows fdens = np.polyval(np.polyfit(defaultfreq, fdens, 5), freq) diskxmlfile = vis + '.SOLDISK.xml' # Insert the disk model (msfile is the same as vis, and will be used as the "original" vis file name) msfile = insertdiskmodel(vis, dsize=dsize, fdens=fdens, xmlfile=diskxmlfile) tdate = mstl.get_trange(vis)[0].datetime.strftime('%Y%m%d') caltb = os.path.join(slfcaltbdir, tdate + '_1.pha') if os.path.exists(caltb): os.system('rm -rf {}'.format(caltb)) # Phase selfcal on the disk using solution interval "infinite" gaincal(vis=msfile, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="inf", combine="scan", refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False) applycal(vis=msfile, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False, applymode="calonly") # Split corrected data and model to a new ms for round 2 of phase selfcal vis1 = 'slf_' + msfile if os.path.exists(vis1): os.system('rm -rf {}'.format(vis1)) mstl.splitX(msfile, outputvis=vis1, datacolumn="corrected", datacolumn2="model_data") caltb = os.path.join(slfcaltbdir, tdate + '_2.pha') if os.path.exists(caltb): os.system('rm -rf {}'.format(caltb)) # Second round of phase selfcal on the disk using solution interval "1min" gaincal(vis=vis1, caltable=caltb, selectdata=True, uvrange="<3.0Klambda", antenna="0~12&0~12", solint="1min", combine="scan", refant="0", refantmode="flex", minsnr=1.0, gaintype="G", calmode="p", append=False) applycal(vis=vis1, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False, applymode="calonly") # Split corrected data and model to a new ms vis2 = 'slf2_' + msfile if os.path.exists(vis2): os.system('rm -rf {}'.format(vis2)) mstl.splitX(vis, outputvis=vis2, datacolumn="corrected", datacolumn2="model_data") caltb = os.path.join(slfcaltbdir, tdate + '_3.amp') if os.path.exists(caltb): os.system('rm -rf {}'.format(caltb)) # Final round of amplitude selfcal with 1-h solution interval (restrict to 16-24 UT) gaincal(vis=vis2, caltable=caltb, selectdata=True, uvrange=">0.1Klambda", antenna="0~12&0~12", timerange=trange, solint="60min", combine="scan", refant="10", refantmode="flex", minsnr=1.0, gaintype="G", calmode="a", append=False) applycal(vis=vis2, selectdata=True, antenna="0~12", gaintable=caltb, interp="nearest", calwt=False, applymode="calonly") # Split out corrected data and model and do uvsub vis3 = 'slf3_' + msfile if os.path.exists(vis3): os.system('rm -rf {}'.format(vis3)) mstl.splitX(vis, outputvis=vis3, datacolumn="corrected", datacolumn2="model_data") uvsub(vis=vis3, reverse=False) # Final split to final = 'final_' + msfile if os.path.exists(final): os.system('rm -rf {}'.format(final)) split(vis3, outputvis=final, datacolumn='corrected') # Remove the interim ms files shutil.rmtree(vis) shutil.rmtree(vis1) shutil.rmtree(vis2) shutil.rmtree(vis3) # Return the name of the selfcaled ms return final, diskxmlfile