def verifyMS(msname, expnumspws, expnumchan, inspw, expchanfreqs=[], ignoreflags=False): '''Function to verify spw and channels information in an MS msname --> name of MS to verify expnumspws --> expected number of SPWs in the MS expnumchan --> expected number of channels in spw inspw --> SPW ID expchanfreqs --> numpy array with expected channel frequencies ignoreflags --> do not check the FLAG column Returns a list with True or False and a state message''' msg = '' tb.open(msname+'/SPECTRAL_WINDOW') nc = tb.getcell("NUM_CHAN", inspw) nr = tb.nrows() cf = tb.getcell("CHAN_FREQ", inspw) tb.close() # After channel selection/average, need to know the exact row number to check, # ignore this check in these cases. if not ignoreflags: tb.open(msname) dimdata = tb.getcell("FLAG", 0)[0].size tb.close() if not (nr==expnumspws): msg = "Found "+str(nr)+", expected "+str(expnumspws)+" spectral windows in "+msname return [False,msg] if not (nc == expnumchan): msg = "Found "+ str(nc) +", expected "+str(expnumchan)+" channels in spw "+str(inspw)+" in "+msname return [False,msg] if not ignoreflags and (dimdata != expnumchan): msg = "Found "+ str(dimdata) +", expected "+str(expnumchan)+" channels in FLAG column in "+msname return [False,msg] if not (expchanfreqs==[]): print "Testing channel frequencies ..." # print cf # print expchanfreqs if not (expchanfreqs.size == expnumchan): msg = "Internal error: array of expected channel freqs should have dimension ", expnumchan return [False,msg] df = (cf - expchanfreqs)/expchanfreqs if not (abs(df) < 1E-8).all: msg = "channel frequencies in spw "+str(inspw)+" differ from expected values by (relative error) "+str(df) return [False,msg] return [True,msg]
def compVarColTables(referencetab, testtab, varcol, tolerance=0.): '''Compare a variable column of two tables. referencetab --> a reference table testtab --> a table to verify varcol --> the name of a variable column (str) Returns True or False. ''' retval = True tb2 = casac.table() tb.open(referencetab) cnames = tb.colnames() tb2.open(testtab) col = varcol if tb.isvarcol(col) and tb2.isvarcol(col): try: # First check if tb.nrows() != tb2.nrows(): print 'Length of %s differ from %s, %s!=%s' % ( referencetab, testtab, len(rk), len(tk)) retval = False else: for therow in xrange(tb.nrows()): rdata = tb.getcell(col, therow) tdata = tb2.getcell(col, therow) # if not (rdata==tdata).all(): if not rdata.all() == tdata.all(): if (tolerance > 0.): differs = False for j in range(0, len(rdata)): ### if (type(rdata[j])==float or type(rdata[j])==int): if ((isinstance(rdata[j], float)) or (isinstance(rdata[j], int))): if (abs(rdata[j] - tdata[j]) > tolerance * abs(rdata[j] + tdata[j])): # print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab # print therow, j # print rdata[j] # print tdata[j] differs = True ### elif (type(rdata[j])==list or type(rdata[j])==np.ndarray): elif (isinstance(rdata[j], list)) or (isinstance( rdata[j], np.ndarray)): for k in range(0, len(rdata[j])): if (abs(rdata[j][k] - tdata[j][k]) > tolerance * abs(rdata[j][k] + tdata[j][k])): # print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab # print therow, j, k # print rdata[j][k] # print tdata[j][k] differs = True if differs: print 'ERROR: Column %s of %s and %s do not agree within tolerance %s' % ( col, referencetab, testtab, tolerance) retval = False break else: print 'ERROR: Column %s of %s and %s do not agree.' % ( col, referencetab, testtab) print 'ERROR: First row to differ is row=%s' % therow retval = False break finally: tb.close() tb2.close() else: print 'Columns are not varcolumns.' retval = False if retval: print 'Column %s of %s and %s agree' % (col, referencetab, testtab) return retval
def compVarColTables(referencetab, testtab, varcol, tolerance=0.): '''Compare a variable column of two tables. referencetab --> a reference table testtab --> a table to verify varcol --> the name of a variable column (str) Returns True or False. ''' retval = True tb2 = casac.table() tb.open(referencetab) cnames = tb.colnames() tb2.open(testtab) col = varcol if tb.isvarcol(col) and tb2.isvarcol(col): try: # First check if tb.nrows() != tb2.nrows(): print 'Length of %s differ from %s, %s!=%s'%(referencetab,testtab,len(rk),len(tk)) retval = False else: for therow in xrange(tb.nrows()): rdata = tb.getcell(col,therow) tdata = tb2.getcell(col,therow) # if not (rdata==tdata).all(): if not rdata.all()==tdata.all(): if (tolerance>0.): differs=False for j in range(0,len(rdata)): ### if (type(rdata[j])==float or type(rdata[j])==int): if ((isinstance(rdata[j],float)) or (isinstance(rdata[j],int))): if (abs(rdata[j]-tdata[j]) > tolerance*abs(rdata[j]+tdata[j])): # print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab # print therow, j # print rdata[j] # print tdata[j] differs = True ### elif (type(rdata[j])==list or type(rdata[j])==np.ndarray): elif (isinstance(rdata[j],list)) or (isinstance(rdata[j],np.ndarray)): for k in range(0,len(rdata[j])): if (abs(rdata[j][k]-tdata[j][k]) > tolerance*abs(rdata[j][k]+tdata[j][k])): # print 'Column ', col,' differs in tables ', referencetab, ' and ', testtab # print therow, j, k # print rdata[j][k] # print tdata[j][k] differs = True if differs: print 'ERROR: Column %s of %s and %s do not agree within tolerance %s'%(col,referencetab, testtab, tolerance) retval = False break else: print 'ERROR: Column %s of %s and %s do not agree.'%(col,referencetab, testtab) print 'ERROR: First row to differ is row=%s'%therow retval = False break finally: tb.close() tb2.close() else: print 'Columns are not varcolumns.' retval = False if retval: print 'Column %s of %s and %s agree'%(col,referencetab, testtab) return retval
def calibeovsa(vis=None, caltype=None, interp=None, docalib=True, doflag=True, flagant=None, doimage=False, imagedir=None, antenna=None, timerange=None, spw=None, stokes=None, doconcat=False, msoutdir=None, concatvis=None, keep_orig_ms=True): ''' :param vis: EOVSA visibility dataset(s) to be calibrated :param caltype: :param interp: :param docalib: :param qlookimage: :param flagant: :param stokes: :param doconcat: :return: ''' if type(vis) == str: vis = [vis] for idx, f in enumerate(vis): if f[-1] == '/': vis[idx] = f[:-1] for msfile in vis: casalog.origin('calibeovsa') if not caltype: casalog.post("Caltype not provided. Perform reference phase calibration and daily phase calibration.") caltype = ['refpha', 'phacal', 'fluxcal'] ## use this line after the phacal is applied # caltype = ['refcal'] if not os.path.exists(msfile): casalog.post("Input visibility does not exist. Aborting...") continue if msfile.endswith('/'): msfile = msfile[:-1] if not msfile[-3:] in ['.ms', '.MS']: casalog.post("Invalid visibility. Please provide a proper visibility file ending with .ms") # if not caltable: # caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype] # get band information tb.open(msfile + '/SPECTRAL_WINDOW') nspw = tb.nrows() bdname = tb.getcol('NAME') bd_nchan = tb.getcol('NUM_CHAN') bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33 # nchans = tb.getcol('NUM_CHAN') # reffreqs = tb.getcol('REF_FREQUENCY') # cenfreqs = np.zeros((nspw)) tb.close() tb.open(msfile + '/ANTENNA') nant = tb.nrows() antname = tb.getcol('NAME') antlist = [str(ll) for ll in range(len(antname) - 1)] antennas = ','.join(antlist) tb.close() # get time stamp, use the beginning of the file tb.open(msfile + '/OBSERVATION') trs = {'BegTime': [], 'EndTime': []} for ll in range(tb.nrows()): tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600, format='mjd') trs['BegTime'].append(tim0) trs['EndTime'].append(tim1) tb.close() trs['BegTime'] = Time(trs['BegTime']) trs['EndTime'] = Time(trs['EndTime']) btime = np.min(trs['BegTime']) etime = np.max(trs['EndTime']) # ms.open(vis) # summary = ms.summary() # ms.close() # btime = Time(summary['BeginTime'], format='mjd') # etime = Time(summary['EndTime'], format='mjd') ## stop using ms.summary to avoid conflicts with importeovsa t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd') print "This scan observed from {} to {} UTC".format(btime.iso, etime.iso) gaintables = [] if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal' in caltype): refcal = ra.sql2refcalX(btime) pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband) pha[np.where(refcal['flag'] == 1)] = 0. amp = refcal['amp'] amp[np.where(refcal['flag'] == 1)] = 1. t_ref = refcal['timestamp'] # find the start and end time of the local day when refcal is registered try: dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60. except: dhr = -7. bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd') et = Time(bt.mjd + 1., format='mjd') (yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day) dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/' if not os.path.exists(dirname): os.mkdir(dirname) # check if there is any ROACH reboot between the reference calibration found and the current data t_rbts = db.get_reboot(Time([t_ref, btime])) if not t_rbts: casalog.post("Reference calibration is derived from observation at " + t_ref.iso) print "Reference calibration is derived from observation at " + t_ref.iso else: casalog.post( "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso) casalog.post("Aborting...") print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso print "Aborting..." para_pha = [] para_amp = [] calpha = np.zeros((nspw, 15, 2)) calamp = np.zeros((nspw, 15, 2)) for s in range(nspw): for n in range(15): for p in range(2): calpha[s, n, p] = pha[n, p, bd[s]] calamp[s, n, p] = amp[n, p, bd[s]] para_pha.append(np.degrees(pha[n, p, bd[s]])) para_amp.append(amp[n, p, bd[s]]) if 'fluxcal' in caltype: calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59')) t_bp = Time(calfac['timestamp'], format='lv') if int(t_mid.mjd) == int(t_bp.mjd): accalfac = calfac['accalfac'] # (ant x pol x freq) # tpcalfac = calfac['tpcalfac'] # (ant x pol x freq) caltb_autoamp = dirname + t_bp.isot[:-4].replace(':', '').replace('-', '') + '.bandpass' if not os.path.exists(caltb_autoamp): bandpass(vis=msfile, caltable=caltb_autoamp, solint='inf', refant='eo01', minblperant=0, minsnr=0, bandtype='B', docallib=False) tb.open(caltb_autoamp, nomodify=False) # (ant x spw) bd_chanidx = np.hstack([[0], bd_nchan.cumsum()]) for ll in range(nspw): antfac = np.sqrt(accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]]) # # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]] antfac = np.moveaxis(antfac, 0, 2) cparam = np.zeros((2, bd_nchan[ll], nant)) cparam[:, :, :-3] = 1.0 / antfac tb.putcol('CPARAM', cparam + 0j, ll * nant, nant) paramerr = tb.getcol('PARAMERR', ll * nant, nant) paramerr = paramerr * 0 tb.putcol('PARAMERR', paramerr, ll * nant, nant) bpflag = tb.getcol('FLAG', ll * nant, nant) bpant1 = tb.getcol('ANTENNA1', ll * nant, nant) bpflagidx, = np.where(bpant1 >= 13) bpflag[:] = False bpflag[:, :, bpflagidx] = True tb.putcol('FLAG', bpflag, ll * nant, nant) bpsnr = tb.getcol('SNR', ll * nant, nant) bpsnr[:] = 100.0 bpsnr[:, :, bpflagidx] = 0.0 tb.putcol('SNR', bpsnr, ll * nant, nant) tb.close() msg_prompt = "Scaling calibration is derived for {}.".format(msfile) casalog.post(msg_prompt) print msg_prompt gaintables.append(caltb_autoamp) else: msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format( t_mid.datetime.strftime('%b %d, %Y'), msfile) casalog.post(msg_prompt) print msg_prompt if ('refpha' in caltype) or ('refcal' in caltype): # caltb_pha = os.path.basename(vis).replace('.ms', '.refpha') # check if the calibration table already exists caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace('-', '') + '.refpha' if not os.path.exists(caltb_pha): gencal(vis=msfile, caltable=caltb_pha, caltype='ph', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_pha) gaintables.append(caltb_pha) if ('refamp' in caltype) or ('refcal' in caltype): # caltb_amp = os.path.basename(vis).replace('.ms', '.refamp') caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace('-', '') + '.refamp' if not os.path.exists(caltb_amp): gencal(vis=msfile, caltable=caltb_amp, caltype='amp', antenna=antennas, pol='X,Y', spw='0~' + str(nspw - 1), parameter=para_amp) gaintables.append(caltb_amp) # calibration for the change of delay center between refcal time and beginning of scan -- hopefully none! xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False) if buf: dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv') dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15] xml, buf = ch.read_calX(4, t=t_ref) dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv') dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15] dlycen_ns_diff = dlycen_ns2 - dlycen_ns1 for n in range(2): dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n] print 'Multi-band delay is derived from delay center difference at {} & {}'.format(dly_t1.iso, dly_t2.iso) # print '=====Delays relative to Ant 14=====' # for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]): # ant = antlist[i] # print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl # dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1]) # caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0') caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace('-', '') + '.dlycen' if not os.path.exists(caltb_dlycen): gencal(vis=msfile, caltable=caltb_dlycen, caltype='mbd', pol='X,Y', antenna=antennas, parameter=dlycen_ns_diff.flatten().tolist()) gaintables.append(caltb_dlycen) if 'phacal' in caltype: phacals = np.array(ra.sql2phacalX([bt, et], neat=True, verbose=False)) if not phacals.any() or len(phacals) == 0: print "Found no phacal records in SQL database, will skip phase calibration" else: # first generate all phacal calibration tables if not already exist t_phas = Time([phacal['t_pha'] for phacal in phacals]) # sort the array in ascending order by t_pha sinds = t_phas.mjd.argsort() t_phas = t_phas[sinds] phacals = phacals[sinds] caltbs_phambd = [] for i, phacal in enumerate(phacals): # filter out phase cals with reference time stamp >30 min away from the provided refcal time if (phacal['t_ref'].jd - refcal['timestamp'].jd) > 30. / 1440.: del phacals[i] del t_phas[i] continue else: t_pha = phacal['t_pha'] phambd_ns = phacal['pslope'] for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n] # set all flagged values to be zero phambd_ns[np.where(phacal['flag'] == 1)] = 0. caltb_phambd = dirname + t_pha.isot[:-4].replace(':', '').replace('-', '') + '.phambd' caltbs_phambd.append(caltb_phambd) if not os.path.exists(caltb_phambd): gencal(vis=msfile, caltable=caltb_phambd, caltype='mbd', pol='X,Y', antenna=antennas, parameter=phambd_ns.flatten().tolist()) # now decides which table to apply depending on the interpolation method ("neatest" or "linear") if interp == 'nearest': tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd)) dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24. print "Selected nearest phase calibration table at " + t_phas[tbind].iso gaintables.append(caltbs_phambd[tbind]) if interp == 'linear': # bphacal = ra.sql2phacalX(btime) # ephacal = ra.sql2phacalX(etime,reverse=True) bt_ind, = np.where(t_phas.mjd < btime.mjd) et_ind, = np.where(t_phas.mjd > etime.mjd) if len(bt_ind) == 0 and len(et_ind) == 0: print "No phacal found before or after the ms data within the day of observation" print "Skipping daily phase calibration" elif len(bt_ind) > 0 and len(et_ind) == 0: gaintables.append(caltbs_phambd[bt_ind[-1]]) elif len(bt_ind) == 0 and len(et_ind) > 0: gaintables.append(caltbs_phambd[et_ind[0]]) elif len(bt_ind) > 0 and len(et_ind) > 0: bphacal = phacals[bt_ind[-1]] ephacal = phacals[et_ind[0]] # generate a new table interpolating between two daily phase calibrations t_pha_mean = Time(np.mean([bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]), format='mjd') phambd_ns = (bphacal['pslope'] + ephacal['pslope']) / 2. for n in range(2): phambd_ns[:, n] -= phambd_ns[0, n] # set all flagged values to be zero phambd_ns[np.where(bphacal['flag'] == 1)] = 0. phambd_ns[np.where(ephacal['flag'] == 1)] = 0. caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace(':', '').replace('-', '') + '.phambd' if not os.path.exists(caltb_phambd_interp): gencal(vis=msfile, caltable=caltb_phambd_interp, caltype='mbd', pol='X,Y', antenna=antennas, parameter=phambd_ns.flatten().tolist()) print "Using phase calibration table interpolated between records at " + bphacal['t_pha'].iso + ' and ' + ephacal['t_pha'].iso gaintables.append(caltb_phambd_interp) if docalib: clearcal(msfile) applycal(vis=msfile, gaintable=gaintables, applymode='calflag', calwt=False) # delete the interpolated phase calibration table try: caltb_phambd_interp except: pass else: if os.path.exists(caltb_phambd_interp): shutil.rmtree(caltb_phambd_interp) if doflag: # flag zeros and NaNs flagdata(vis=msfile, mode='clip', clipzeros=True) if flagant: try: flagdata(vis=msfile, antenna=flagant) except: print "Something wrong with flagant. Abort..." if doimage: from matplotlib import pyplot as plt from suncasa.utils import helioimage2fits as hf from sunpy import map as smap if not antenna: antenna = '0~12' if not stokes: stokes = 'XX' if not timerange: timerange = '' if not spw: spw = '1~3' if not imagedir: imagedir = '.' #(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day) #dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/' #if not os.path.exists(dirname): # os.makedirs(dirname) bds = [spw] nbd = len(bds) imgs = [] for bd in bds: if '~' in bd: bdstr = bd.replace('~', '-') else: bdstr = str(bd).zfill(2) imname = imagedir + '/' + os.path.basename(msfile).replace('.ms', '.bd' + bdstr) print 'Cleaning image: ' + imname try: clean(vis=msfile, imagename=imname, antenna=antenna, spw=bd, timerange=timerange, imsize=[512], cell=['5.0arcsec'], stokes=stokes, niter=500) except: print 'clean not successfull for band ' + str(bd) else: imgs.append(imname + '.image') junks = ['.flux', '.mask', '.model', '.psf', '.residual'] for junk in junks: if os.path.exists(imname + junk): shutil.rmtree(imname + junk) tranges = [btime.iso + '~' + etime.iso] * nbd fitsfiles = [img.replace('.image', '.fits') for img in imgs] hf.imreg(vis=msfile, timerange=tranges, imagefile=imgs, fitsfile=fitsfiles, usephacenter=False) plt.figure(figsize=(6, 6)) for i, fitsfile in enumerate(fitsfiles): plt.subplot(1, nbd, i + 1) eomap = smap.Map(fitsfile) sz = eomap.data.shape if len(sz) == 4: eomap.data = eomap.data.reshape((sz[2], sz[3])) eomap.plot_settings['cmap'] = plt.get_cmap('jet') eomap.plot() eomap.draw_limb() eomap.draw_grid() plt.show() if doconcat: from suncasa.tasks import concateovsa_cli as ce # from suncasa.eovsa import concateovsa as ce if msoutdir is None: msoutdir = './' if not concatvis: concatvis = os.path.basename(vis[0]) concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms' else: concatvis = os.path.join(msoutdir, concatvis) if len(vis) > 1: ce.concateovsa(vis, concatvis, datacolumn='corrected', keep_orig_ms=keep_orig_ms, cols2rm="model,corrected") return [concatvis] else: split(vis=vis[0], outputvis=concatvis, datacolumn='corrected') return [concatvis] else: return vis
def calc_phasecenter_from_solxy(vis, timerange='', xycen=None, usemsphacenter=True): ''' return the phase center in RA and DEC of a given solar coordinates :param vis: input measurement sets file :param timerange: can be a string or astropy.time.core.Time object, or a 2-element list of string or Time object :param xycen: solar x-pos and y-pos in arcsec :param usemsphacenter: :return: phasecenter midtim: mid time of the given timerange ''' tb.open(vis + '/POINTING') tst = Time(tb.getcell('TIME_ORIGIN', 0) / 24. / 3600., format='mjd') ted = Time(tb.getcell('TIME_ORIGIN', tb.nrows() - 1) / 24. / 3600., format='mjd') tb.close() datstr = tst.iso[:10] if isinstance(timerange, Time): try: (sttim, edtim) = timerange except: sttim = timerange edtim = sttim else: if timerange == '': sttim = tst edtim = ted else: try: (tstart, tend) = timerange.split('~') if tstart[2] == ':': sttim = Time(datstr + 'T' + tstart) edtim = Time(datstr + 'T' + tend) # timerange = '{0}/{1}~{0}/{2}'.format(datstr.replace('-', '/'), tstart, tend) else: sttim = Time(qa.quantity(tstart, 'd')['value'], format='mjd') edtim = Time(qa.quantity(tend, 'd')['value'], format='mjd') except: try: if timerange[2] == ':': sttim = Time(datstr + 'T' + timerange) edtim = sttim else: sttim = Time(qa.quantity(timerange, 'd')['value'], format='mjd') edtim = sttim except ValueError: print("keyword 'timerange' in wrong format") ms.open(vis) metadata = ms.metadata() observatory = metadata.observatorynames()[0] ms.close() midtim_mjd = (sttim.mjd + edtim.mjd) / 2. midtim = Time(midtim_mjd, format='mjd') eph = read_horizons(t0=midtim) if observatory == 'EOVSA' or (not usemsphacenter): print('This is EOVSA data') # use RA and DEC from FIELD ID 0 tb.open(vis + '/FIELD') phadir = tb.getcol('PHASE_DIR').flatten() tb.close() ra0 = phadir[0] dec0 = phadir[1] else: ra0 = eph['ra'][0] dec0 = eph['dec'][0] if not xycen: # use solar disk center as default phasecenter = 'J2000 ' + str(ra0) + 'rad ' + str(dec0) + 'rad' else: x0 = np.radians(xycen[0] / 3600.) y0 = np.radians(xycen[1] / 3600.) p0 = np.radians(eph['p0'][0]) # p angle in radians raoff = -((x0) * np.cos(p0) - y0 * np.sin(p0)) / np.cos(eph['dec'][0]) decoff = (x0) * np.sin(p0) + y0 * np.cos(p0) newra = ra0 + raoff newdec = dec0 + decoff phasecenter = 'J2000 ' + str(newra) + 'rad ' + str(newdec) + 'rad' return phasecenter, midtim
def imreg(vis=None, ephem=None, msinfo=None, imagefile=None, timerange=None, reftime=None, fitsfile=None, beamfile=None, offsetfile=None, toTb=None, sclfactor=1.0, verbose=False, p_ang=False, overwrite=True, usephacenter=True, deletehistory=False, subregion=[], docompress=False): ''' main routine to register CASA images Required Inputs: vis: STRING. CASA measurement set from which the image is derived imagefile: STRING or LIST. name of the input CASA image timerange: STRING or LIST. timerange used to generate the CASA image, must have the same length as the input images. Each element should be in CASA standard time format, e.g., '2012/03/03/12:00:00~2012/03/03/13:00:00' Optional Inputs: msinfo: DICTIONARY. CASA MS information, output from read_msinfo. If not provided, generate one from the supplied vis ephem: DICTIONARY. solar ephem, output from read_horizons. If not provided, query JPL Horizons based on time info of the vis (internet connection required) fitsfile: STRING or LIST. name of the output registered fits files reftime: STRING or LIST. Each element should be in CASA standard time format, e.g., '2012/03/03/12:00:00' offsetfile: optionally provide an offset with a series of solar x and y offsets with timestamps toTb: Bool. Convert the default Jy/beam to brightness temperature? sclfactor: scale the image values up by its value (to compensate VLA 20 dB attenuator) verbose: Bool. Show more diagnostic info if True. usephacenter: Bool -- if True, correct for the RA and DEC in the ms file based on solar empheris. Otherwise assume the phasecenter is correctly pointed to the solar disk center (EOVSA case) subregion: Region selection. See 'help par.region' for details. Usage: >>> from suncasa.utils import helioimage2fits as hf >>> hf.imreg(vis='mydata.ms', imagefile='myimage.image', fitsfile='myimage.fits', timerange='2017/08/21/20:21:10~2017/08/21/20:21:18') The output fits file is 'myimage.fits' History: BC (sometime in 2014): function was first wrote, followed by a number of edits by BC and SY BC (2019-07-16): Added checks for stokes parameter. Verified that for converting from Jy/beam to brightness temperature, the convention of 2*k_b*T should always be used. I.e., for unpolarized source, stokes I, RR, LL, XX, YY, etc. in the output CASA images from (t)clean should all have same values of radio intensity (in Jy/beam) and brightness temperature (in K). ''' ia = iatool() if deletehistory: ms_clearhistory(vis) if not imagefile: raise ValueError('Please specify input image') if not timerange: raise ValueError('Please specify timerange of the input image') if type(imagefile) == str: imagefile = [imagefile] if type(timerange) == str: timerange = [timerange] if not fitsfile: fitsfile = [img + '.fits' for img in imagefile] if type(fitsfile) == str: fitsfile = [fitsfile] nimg = len(imagefile) if len(timerange) != nimg: raise ValueError( 'Number of input images does not equal to number of timeranges!') if len(fitsfile) != nimg: raise ValueError( 'Number of input images does not equal to number of output fits files!' ) nimg = len(imagefile) if verbose: print(str(nimg) + ' images to process...') if reftime: # use as reference time to find solar disk RA and DEC to register the image, but not the actual timerange associated with the image if type(reftime) == str: reftime = [reftime] * nimg if len(reftime) != nimg: raise ValueError( 'Number of reference times does not match that of input images!' ) helio = ephem_to_helio(vis, ephem=ephem, msinfo=msinfo, reftime=reftime, usephacenter=usephacenter) else: # use the supplied timerange to register the image helio = ephem_to_helio(vis, ephem=ephem, msinfo=msinfo, reftime=timerange, usephacenter=usephacenter) if toTb: (bmajs, bmins, bpas, beamunits, bpaunits) = getbeam(imagefile=imagefile, beamfile=beamfile) for n, img in enumerate(imagefile): if verbose: print('processing image #' + str(n) + ' ' + img) fitsf = fitsfile[n] timeran = timerange[n] # obtain duration of the image as FITS header exptime try: [tbg0, tend0] = timeran.split('~') tbg_d = qa.getvalue(qa.convert(qa.totime(tbg0), 'd'))[0] tend_d = qa.getvalue(qa.convert(qa.totime(tend0), 'd'))[0] tdur_s = (tend_d - tbg_d) * 3600. * 24. dateobs = qa.time(qa.quantity(tbg_d, 'd'), form='fits', prec=10)[0] except: print('Error in converting the input timerange: ' + str(timeran) + '. Proceeding to the next image...') continue hel = helio[n] if not os.path.exists(img): warnings.warn('{} does not existed!'.format(img)) else: if os.path.exists(fitsf) and not overwrite: raise ValueError( 'Specified fits file already exists and overwrite is set to False. Aborting...' ) else: p0 = hel['p0'] tb.open(img + '/logtable', nomodify=False) nobs = tb.nrows() tb.removerows([i + 1 for i in range(nobs - 1)]) tb.close() ia.open(img) imr = ia.rotate(pa=str(-p0) + 'deg') if subregion is not []: imr = imr.subimage(region=subregion) imr.tofits(fitsf, history=False, overwrite=overwrite) imr.close() imsum = ia.summary() ia.close() ia.done() # construct the standard fits header # RA and DEC of the reference pixel crpix1 and crpix2 (imra, imdec) = (imsum['refval'][0], imsum['refval'][1]) # find out the difference of the image center to the CASA phase center # RA and DEC difference in arcseconds ddec = degrees((imdec - hel['dec_fld'])) * 3600. dra = degrees((imra - hel['ra_fld']) * cos(hel['dec_fld'])) * 3600. # Convert into image heliocentric offsets prad = -radians(hel['p0']) dx = (-dra) * cos(prad) - ddec * sin(prad) dy = (-dra) * sin(prad) + ddec * cos(prad) if offsetfile: try: offset = np.load(offsetfile) except: raise ValueError( 'The specified offsetfile does not exist!') reftimes_d = offset['reftimes_d'] xoffs = offset['xoffs'] yoffs = offset['yoffs'] timg_d = hel['reftime'] ind = bisect.bisect_left(reftimes_d, timg_d) xoff = xoffs[ind - 1] yoff = yoffs[ind - 1] else: xoff = hel['refx'] yoff = hel['refy'] if verbose: print( 'offset of image phase center to visibility phase center (arcsec): dx={0:.2f}, dy={1:.2f}' .format(dx, dy)) print( 'offset of visibility phase center to solar disk center (arcsec): dx={0:.2f}, dy={1:.2f}' .format(xoff, yoff)) (crval1, crval2) = (xoff + dx, yoff + dy) # update the fits header to heliocentric coordinates hdu = pyfits.open(fitsf, mode='update') hdu[0].verify('fix') header = hdu[0].header dshape = hdu[0].data.shape ndim = hdu[0].data.ndim (cdelt1, cdelt2) = (-header['cdelt1'] * 3600., header['cdelt2'] * 3600. ) # Original CDELT1, 2 are for RA and DEC in degrees header['cdelt1'] = cdelt1 header['cdelt2'] = cdelt2 header['cunit1'] = 'arcsec' header['cunit2'] = 'arcsec' header['crval1'] = crval1 header['crval2'] = crval2 header['ctype1'] = 'HPLN-TAN' header['ctype2'] = 'HPLT-TAN' header['date-obs'] = dateobs # begin time of the image if not p_ang: hel['p0'] = 0 try: # this works for pyfits version of CASA 4.7.0 but not CASA 4.6.0 if tdur_s: header.set('exptime', tdur_s) else: header.set('exptime', 1.) header.set('p_angle', hel['p0']) header.set( 'dsun_obs', sun.sunearth_distance(Time(dateobs)).to(u.meter).value) header.set( 'rsun_obs', sun.solar_semidiameter_angular_size(Time(dateobs)).value) header.set('rsun_ref', sun.constants.radius.value) header.set('hgln_obs', 0.) header.set( 'hglt_obs', sun.heliographic_solar_center(Time(dateobs))[1].value) except: # this works for astropy.io.fits if tdur_s: header.append(('exptime', tdur_s)) else: header.append(('exptime', 1.)) header.append(('p_angle', hel['p0'])) header.append( ('dsun_obs', sun.sunearth_distance(Time(dateobs)).to(u.meter).value)) header.append( ('rsun_obs', sun.solar_semidiameter_angular_size(Time(dateobs)).value)) header.append(('rsun_ref', sun.constants.radius.value)) header.append(('hgln_obs', 0.)) header.append( ('hglt_obs', sun.heliographic_solar_center(Time(dateobs))[1].value)) # check if stokes parameter exist exist_stokes = False stokes_mapper = { 'I': 1, 'Q': 2, 'U': 3, 'V': 4, 'RR': -1, 'LL': -2, 'RL': -3, 'LR': -4, 'XX': -5, 'YY': -6, 'XY': -7, 'YX': -8 } if 'CRVAL3' in header.keys(): if header['CTYPE3'] == 'STOKES': stokenum = header['CRVAL3'] exist_stokes = True if 'CRVAL4' in header.keys(): if header['CTYPE4'] == 'STOKES': stokenum = header['CRVAL4'] exist_stokes = True if exist_stokes: stokesstr = stokes_mapper.keys()[stokes_mapper.values().index( stokenum)] if verbose: print('This image is in Stokes ' + stokesstr) else: print( 'STOKES Information does not seem to exist! Assuming Stokes I' ) stokenum = 1 # intensity units to brightness temperature if toTb: # get restoring beam info bmaj = bmajs[n] bmin = bmins[n] beamunit = beamunits[n] data = hdu[ 0].data # remember the data order is reversed due to the FITS convension keys = header.keys() values = header.values() # which axis is frequency? faxis = keys[values.index('FREQ')][-1] faxis_ind = ndim - int(faxis) # find out the polarization of this image k_b = qa.constants('k')['value'] c_l = qa.constants('c')['value'] # Always use 2*kb for all polarizations const = 2. * k_b / c_l**2 if header['BUNIT'].lower() == 'jy/beam': header['BUNIT'] = 'K' header['BTYPE'] = 'Brightness Temperature' for i in range(dshape[faxis_ind]): nu = header['CRVAL' + faxis] + header['CDELT' + faxis] * ( i + 1 - header['CRPIX' + faxis]) if header['CUNIT' + faxis] == 'KHz': nu *= 1e3 if header['CUNIT' + faxis] == 'MHz': nu *= 1e6 if header['CUNIT' + faxis] == 'GHz': nu *= 1e9 if len(bmaj) > 1: # multiple (per-plane) beams bmajtmp = bmaj[i] bmintmp = bmin[i] else: # one single beam bmajtmp = bmaj[0] bmintmp = bmin[0] if beamunit == 'arcsec': bmaj0 = np.radians(bmajtmp / 3600.) bmin0 = np.radians(bmintmp / 3600.) if beamunit == 'arcmin': bmaj0 = np.radians(bmajtmp / 60.) bmin0 = np.radians(bmintmp / 60.) if beamunit == 'deg': bmaj0 = np.radians(bmajtmp) bmin0 = np.radians(bmintmp) if beamunit == 'rad': bmaj0 = bmajtmp bmin0 = bmintmp beam_area = bmaj0 * bmin0 * np.pi / (4. * log(2.)) factor = const * nu**2 # SI unit jy_to_si = 1e-26 # print(nu/1e9, beam_area, factor) factor2 = sclfactor # if sclfactor: # factor2 = 100. if faxis == '3': data[:, i, :, :] *= jy_to_si / beam_area / factor * factor2 if faxis == '4': data[ i, :, :, :] *= jy_to_si / beam_area / factor * factor2 header = fu.headerfix(header) hdu.flush() hdu.close() if ndim - np.count_nonzero(np.array(dshape) == 1) > 3: docompress = False ''' Caveat: only 1D, 2D, or 3D images are currently supported by the astropy fits compression. If a n-dimensional image data array does not have at least n-3 single-dimensional entries, force docompress to be False ''' print( 'warning: The fits data contains more than 3 non squeezable dimensions. Skipping fits compression..' ) if docompress: fitsftmp = fitsf + ".tmp.fits" os.system("mv {} {}".format(fitsf, fitsftmp)) hdu = pyfits.open(fitsftmp) hdu[0].verify('fix') header = hdu[0].header data = hdu[0].data fu.write_compressed_image_fits(fitsf, data, header, compression_type='RICE_1', quantize_level=4.0) os.system("rm -rf {}".format(fitsftmp)) if deletehistory: ms_restorehistory(vis) return fitsfile
def read_horizons(t0=None, dur=None, vis=None, observatory=None, verbose=False): ''' This function visits JPL Horizons to retrieve J2000 topocentric RA and DEC of the solar disk center as a function of time. Keyword arguments: t0: Referece time in astropy.Time format dur: duration of the returned coordinates. Default to 2 minutes vis: CASA visibility dataset (in measurement set format). If provided, use entire duration from the visibility data observatory: observatory code (from JPL Horizons). If not provided, use information from visibility. if no visibility found, use earth center (code=500) verbose: True to provide extra information Usage: >>> from astropy.time import Time >>> out = read_horizons(t0=Time('2017-09-10 16:00:00'), observatory='-81') >>> out = read_horizons(vis = 'mydata.ms') History: BC (sometime in 2014): function was first wrote, followed by a number of edits by BC and SY BC (2019-07-16): Added docstring documentation ''' import urllib2 import ssl if not t0 and not vis: t0 = Time.now() if not dur: dur = 1. / 60. / 24. # default to 2 minutes if t0: try: btime = Time(t0) except: print('input time ' + str(t0) + ' not recognized') return -1 if vis: if not os.path.exists(vis): print('Input ms data ' + vis + ' does not exist! ') return -1 try: # ms.open(vis) # summary = ms.summary() # ms.close() # btime = Time(summary['BeginTime'], format='mjd') # etime = Time(summary['EndTime'], format='mjd') ## alternative way to avoid conflicts with importeovsa, if needed -- more time consuming if observatory == 'geocentric': observatory = '500' else: ms.open(vis) metadata = ms.metadata() if metadata.observatorynames()[0] == 'EVLA': observatory = '-5' elif metadata.observatorynames()[0] == 'EOVSA': observatory = '-81' elif metadata.observatorynames()[0] == 'ALMA': observatory = '-7' ms.close() tb.open(vis) btime_vis = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd') etime_vis = Time(tb.getcell('TIME', tb.nrows() - 1) / 24. / 3600., format='mjd') tb.close() if verbose: print("Beginning time of this scan " + btime_vis.iso) print("End time of this scan " + etime_vis.iso) # extend the start and end time for jpl horizons by 0.5 hr on each end btime = Time(btime_vis.mjd - 0.5 / 24., format='mjd') dur = etime_vis.mjd - btime_vis.mjd + 1.0 / 24. except: print('error in reading ms file: ' + vis + ' to obtain the ephemeris!') return -1 # default the observatory to geocentric, if none provided if not observatory: observatory = '500' etime = Time(btime.mjd + dur, format='mjd') try: cmdstr = "https://ssd.jpl.nasa.gov/horizons_batch.cgi?batch=1&TABLE_TYPE='OBSERVER'&QUANTITIES='1,17,20'&CSV_FORMAT='YES'&ANG_FORMAT='DEG'&CAL_FORMAT='BOTH'&SOLAR_ELONG='0,180'&CENTER='{}@399'&COMMAND='10'&START_TIME='".format( observatory ) + btime.iso.replace( ' ', ',' ) + "'&STOP_TIME='" + etime.iso[:-4].replace( ' ', ',' ) + "'&STEP_SIZE='1m'&SKIP_DAYLT='NO'&EXTRA_PREC='YES'&APPARENT='REFRACTED'" cmdstr = cmdstr.replace("'", "%27") try: context = ssl._create_unverified_context() f = urllib2.urlopen(cmdstr, context=context) except: f = urllib2.urlopen(cmdstr) lines = f.readlines() f.close() except: # todo use geocentric coordinate for the new VLA data import requests, collections params = collections.OrderedDict() params['batch'] = '1' params['TABLE_TYPE'] = "'OBSERVER'" params['QUANTITIES'] = "'1,17,20'" params['CSV_FORMAT'] = "'YES'" params['ANG_FORMAT'] = "'DEG'" params['CAL_FORMAT'] = "'BOTH'" params['SOLAR_ELONG'] = "'0,180'" if observatory == '500': params['CENTER'] = "'500'" else: params['CENTER'] = "'{}@399'".format(observatory) params['COMMAND'] = "'10'" params['START_TIME'] = "'{}'".format(btime.iso[:-4].replace(' ', ',')) params['STOP_TIME'] = "'{}'".format(etime.iso[:-4].replace(' ', ',')) params['STEP_SIZE'] = "'1m'" params['SKIP_DAYLT'] = "'NO'" params['EXTRA_PREC'] = "'YES'" params['APPAENT'] = "'REFRACTED'" results = requests.get("https://ssd.jpl.nasa.gov/horizons_batch.cgi", params=params) lines = [ll for ll in results.iter_lines()] nline = len(lines) istart = 0 for i in range(nline): line = lines[i] if line[0:5] == '$$SOE': # start recording istart = i + 1 if line[0:5] == '$$EOE': # end recording iend = i newlines = lines[istart:iend] nrec = len(newlines) ephem_ = [] t = [] ra = [] dec = [] p0 = [] delta = [] for line in newlines: items = line.split(',') t.append(Time(float(items[1]), format='jd').mjd) ra.append(np.radians(float(items[4]))) dec.append(np.radians(float(items[5]))) p0.append(float(items[6])) delta.append(float(items[8])) # convert list of dictionary to a dictionary of arrays ephem = {'time': t, 'ra': ra, 'dec': dec, 'p0': p0, 'delta': delta} return ephem
def read_horizons(vis): import urllib2 import ssl if not os.path.exists(vis): print 'Input ms data ' + vis + ' does not exist! ' return -1 try: # ms.open(vis) # summary = ms.summary() # ms.close() # btime = Time(summary['BeginTime'], format='mjd') # etime = Time(summary['EndTime'], format='mjd') ## alternative way to avoid conflicts with importeovsa, if needed -- more time consuming ms.open(vis) metadata = ms.metadata() if metadata.observatorynames()[0] == 'EVLA': observatory_code = '-5' elif metadata.observatorynames()[0] == 'EOVSA': observatory_code = '-81' elif metadata.observatorynames()[0] == 'ALMA': observatory_code = '-7' ms.close() tb.open(vis) btime = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd') etime = Time(tb.getcell('TIME', tb.nrows() - 1) / 24. / 3600., format='mjd') tb.close() print "Beginning time of this scan " + btime.iso print "End time of this scan " + etime.iso cmdstr = "http://ssd.jpl.nasa.gov/horizons_batch.cgi?batch=l&TABLE_TYPE='OBSERVER'&QUANTITIES='1,17,20'&CSV_FORMAT='YES'&ANG_FORMAT='DEG'&CAL_FORMAT='BOTH'&SOLAR_ELONG='0,180'&CENTER='{}@399'&COMMAND='10'&START_TIME='".format( observatory_code ) + btime.iso.replace( ' ', ',' ) + "'&STOP_TIME='" + etime.iso[:-4].replace( ' ', ',' ) + "'&STEP_SIZE='1 m'&SKIP_DAYLT='NO'&EXTRA_PREC='YES'&APPARENT='REFRACTED'" try: context = ssl._create_unverified_context() f = urllib2.urlopen(cmdstr, context=context) except: f = urllib2.urlopen(cmdstr) except: print 'error in reading ms file: ' + vis + ' to obtain the ephemeris!' return -1 # inputs: # ephemfile: # OBSERVER output from JPL Horizons for topocentric coordinates with for example # target=Sun, observer=VLA=-5 # extra precision, quantities 1,17,20, REFRACTION # routine goes through file to find $$SOE which is start of ephemeris and ends with $$EOE # outputs: a Python dictionary containing the following: # timestr: date and time as a string # time: modified Julian date # ra: right ascention, in rad # dec: declination, in rad # rastr: ra in string # decstr: dec in string # p0: solar p angle, CCW with respect to the celestial north pole # delta: distance from the disk center to the observer, in AU # delta_dot: time derivative of delta, in the light of sight direction. Negative means it is moving toward the observer # # initialize the return dictionary ephem0 = dict.fromkeys(['time', 'ra', 'dec', 'delta', 'p0']) lines = f.readlines() f.close() nline = len(lines) istart = 0 for i in range(nline): line = lines[i] if line[0:5] == '$$SOE': # start recording istart = i + 1 if line[0:5] == '$$EOE': # end recording iend = i newlines = lines[istart:iend] nrec = len(newlines) ephem_ = [] t = [] ra = [] dec = [] p0 = [] delta = [] for line in newlines: items = line.split(',') # t.append({'unit':'mjd','value':Time(float(items[1]),format='jd').mjd}) # ra.append({'unit': 'rad', 'value': np.radians(float(items[4]))}) # dec.append({'unit': 'rad', 'value': np.radians(float(items[5]))}) # p0.append({'unit': 'deg', 'value': float(items[6])}) # delta.append({'unit': 'au', 'value': float(items[8])}) t.append(Time(float(items[1]), format='jd').mjd) ra.append(np.radians(float(items[4]))) dec.append(np.radians(float(items[5]))) p0.append(float(items[6])) delta.append(float(items[8])) # convert list of dictionary to a dictionary of arrays ephem = {'time': t, 'ra': ra, 'dec': dec, 'p0': p0, 'delta': delta} return ephem