def fd_images(vis, cleanup=False, niter=None, spws=['0~1', '2~5', '6~10', '11~20', '21~30', '31~43'], imgoutdir='./',
bright=None, stokes="XX"):
''' Create standard full-disk images in "images" subdirectory of the current directory.
If cleanup is True, delete those images after completion, leaving only the fits images.
'''
# Check if "images" directory exists (if not, create it and mark it for later deletion)
try:
if os.stat('images'):
rm_images = False # Mark as not removeable
except:
os.mkdir('images')
if cleanup:
rm_images = True # Mark as removeable
else:
rm_images = False # Mark as not removeable
trange = ant_trange(vis)
tdate = trange.replace('/', '')[:8]
if niter is None:
niter = 5000
if bright is None:
bright = [True] * len(spws)
imagefile = []
fitsfile = []
for s, sp in enumerate(spws):
if bright[s]:
spwstr = '-'.join(['{:02d}'.format(int(sp_)) for sp_ in sp.split('~')])
imname = "images/briggs" + spwstr
# tclean(vis=vis, selectdata=True, spw=sp, timerange=trange,
# antenna="0~12", datacolumn="corrected", imagename=imname, imsize=[1024], cell=['2.5arcsec'],
# stokes="XX", projection="SIN", specmode="mfs", interpolation="linear", deconvolver="multiscale",
# scales=[0, 5, 15, 30], nterms=2, smallscalebias=0.6, restoration=True, weighting="briggs", robust=0,
# niter=niter, gain=0.05, savemodel="none")
os.system('rm -rf {}.*'.format(imname))
tclean(vis=vis, selectdata=True, spw=sp, timerange=trange,
antenna="0~12", datacolumn="data", imagename=imname, imsize=[1024], cell=['2.5arcsec'],
stokes=stokes, projection="SIN", specmode="mfs", interpolation="linear", deconvolver="multiscale",
scales=[0, 5, 15, 30], nterms=2, smallscalebias=0.6, restoration=True, weighting="briggs", robust=0,
niter=niter, gain=0.05, savemodel="none", usemask='auto-multithresh', pbmask=0.0,
sidelobethreshold=1.0, noisethreshold=2.5, lownoisethreshold=1.5, negativethreshold=5.0,
smoothfactor=1.0, minbeamfrac=0.3, cutthreshold=0.01, growiterations=75, dogrowprune=True,
minpercentchange=-1.0)
outfits = os.path.join(imgoutdir, 'eovsa_' + tdate + '.spw' + spwstr + '.tb.fits')
if os.path.exists(outfits):
os.system('rm -rf {}'.format(outfits))
imagefile.append(imname + '.image')
fitsfile.append(outfits)
hf.imreg(vis=vis, imagefile=imagefile, fitsfile=fitsfile, timerange=[trange] * len(fitsfile), toTb=True,
usephacenter=False, overwrite=True)
if rm_images:
shutil.rmtree('images') # Remove all images and the folder named images
# To add disk model image to the images, I can try scipy.ndimage routines gaussian_filter() and zoom()
return fitsfile
phasecenter=phasecenter,
mask='',
pbcor=pbcor,
interactive=False)
except:
print('cleaning spw ' + sp + ' unsuccessful. Proceed to next spw')
continue
if pbcor:
imn = imname + '.image.pbcor'
else:
imn = imname + '.image'
if os.path.exists(imn):
hf.imreg(vis=vis,
imagefile=imn,
fitsfile=fitsfile,
timerange=trange,
usephacenter=False,
toTb=True,
verbose=False)
fitsfiles.append(fitsfile)
else:
print('cleaning spw ' + sp + ' unsuccessful. Proceed to next spw')
junks = [
'.flux', '.model', '.psf', '.residual', '.mask', '.image', '.pb',
'.image.pbcor', '.sumwt'
]
for junk in junks:
if os.path.exists(imname + junk):
os.system('rm -rf ' + imname + junk)
else:
print('fits file ' + fitsfile + ' already exists, skip clean...')
def clean_iter(
tim, vis, imageprefix, imagesuffix, twidth, doreg, usephacenter,
reftime, ephem, msinfo, toTb, overwrite, selectdata, field, spw,
uvrange, antenna, scan, observation, intent, datacolumn, imsize, cell,
phasecenter, stokes, projection, startmodel, specmode, reffreq, nchan,
start, width, outframe, veltype, restfreq, interpolation, gridder,
facets, chanchunks, wprojplanes, vptable, usepointing, mosweight,
aterm, psterm, wbawp, conjbeams, cfcache, computepastep, rotatepastep,
pblimit, normtype, deconvolver, scales, nterms, smallscalebias,
restoration, restoringbeam, pbcor, outlierfile, weighting, robust,
npixels, uvtaper, niter, gain, threshold, nsigma, cycleniter,
cyclefactor, minpsffraction, maxpsffraction, interactive, usemask,
mask, pbmask, sidelobethreshold, noisethreshold, lownoisethreshold,
negativethreshold, smoothfactor, minbeamfrac, cutthreshold,
growiterations, dogrowprune, minpercentchange, verbose, restart,
savemodel, calcres, calcpsf, parallel, subregion, tmpdir, btidx):
from tclean_cli import tclean_cli as tclean
from split_cli import split_cli as split
bt = btidx # 0
if bt + twidth < len(tim) - 1:
et = btidx + twidth - 1
else:
et = len(tim) - 1
if bt == 0:
bt_d = tim[bt] - ((tim[bt + 1] - tim[bt]) / 2)
else:
bt_d = tim[bt] - ((tim[bt] - tim[bt - 1]) / 2)
if et == (len(tim) - 1) or et == -1:
et_d = tim[et] + ((tim[et] - tim[et - 1]) / 2)
else:
et_d = tim[et] + ((tim[et + 1] - tim[et]) / 2)
timerange = qa.time(qa.quantity(bt_d, 's'), prec=9, form='ymd')[0] + '~' + \
qa.time(qa.quantity(et_d, 's'), prec=9, form='ymd')[0]
btstr = qa.time(qa.quantity(bt_d, 's'), prec=9, form='fits')[0]
etstr = qa.time(qa.quantity(et_d, 's'), prec=9, form='fits')[0]
print('cleaning timerange: ' + timerange)
image0 = btstr.replace(':', '').replace('-', '')
imname = imageprefix + image0 + imagesuffix
# ms_tmp = tmpdir + image0 + '.ms'
# print('checkpoint 1')
# # split(vis=vis, outputvis=ms_tmp, field=field, scan=scan, antenna=antenna, timerange=timerange,
# # datacolumn=datacolumn)
# ms.open(vis)
# print('checkpoint 1-1')
# ms.split(ms_tmp,field=field, scan=scan, baseline=antenna, time=timerange,whichcol=datacolumn)
# print('checkpoint 1-2')
# ms.close()
# print('checkpoint 2')
if overwrite or (len(glob.glob(imname + '*')) == 0):
os.system('rm -rf {}*'.format(imname))
try:
tclean(vis=vis,
selectdata=selectdata,
field=field,
spw=spw,
timerange=timerange,
uvrange=uvrange,
antenna=antenna,
scan=scan,
observation=observation,
intent=intent,
datacolumn=datacolumn,
imagename=imname,
imsize=imsize,
cell=cell,
phasecenter=phasecenter,
stokes=stokes,
projection=projection,
startmodel=startmodel,
specmode=specmode,
reffreq=reffreq,
nchan=nchan,
start=start,
width=width,
outframe=outframe,
veltype=veltype,
restfreq=restfreq,
interpolation=interpolation,
gridder=gridder,
facets=facets,
chanchunks=chanchunks,
wprojplanes=wprojplanes,
vptable=vptable,
usepointing=usepointing,
mosweight=mosweight,
aterm=aterm,
psterm=psterm,
wbawp=wbawp,
conjbeams=conjbeams,
cfcache=cfcache,
computepastep=computepastep,
rotatepastep=rotatepastep,
pblimit=pblimit,
normtype=normtype,
deconvolver=deconvolver,
scales=scales,
nterms=nterms,
smallscalebias=smallscalebias,
restoration=restoration,
restoringbeam=restoringbeam,
pbcor=pbcor,
outlierfile=outlierfile,
weighting=weighting,
robust=robust,
npixels=npixels,
uvtaper=uvtaper,
niter=niter,
gain=gain,
threshold=threshold,
nsigma=nsigma,
cycleniter=cycleniter,
cyclefactor=cyclefactor,
minpsffraction=minpsffraction,
maxpsffraction=maxpsffraction,
interactive=interactive,
usemask=usemask,
mask=mask,
pbmask=pbmask,
sidelobethreshold=sidelobethreshold,
noisethreshold=noisethreshold,
lownoisethreshold=lownoisethreshold,
negativethreshold=negativethreshold,
smoothfactor=smoothfactor,
minbeamfrac=minbeamfrac,
cutthreshold=cutthreshold,
growiterations=growiterations,
dogrowprune=dogrowprune,
minpercentchange=minpercentchange,
verbose=verbose,
restart=restart,
savemodel=savemodel,
calcres=calcres,
calcpsf=calcpsf,
parallel=parallel)
# print('checkpoint 3')
if pbcor:
clnjunks = [
'.flux', '.mask', '.model', '.psf', '.residual', '.pb',
'.sumwt', '.image'
]
else:
clnjunks = [
'.flux', '.mask', '.model', '.psf', '.residual', '.pb',
'.sumwt', '.image.pbcor'
]
for clnjunk in clnjunks:
if os.path.exists(imname + clnjunk):
shutil.rmtree(imname + clnjunk)
if pbcor:
os.system('mv {} {}'.format(imname + '.image.pbcor',
imname + '.image'))
except:
print('error in cleaning image: ' + btstr)
return [False, btstr, etstr, '']
else:
print(imname + ' exists. Clean task aborted.')
if doreg and not os.path.isfile(imname + '.fits'):
# ephem.keys()
# msinfo.keys()
try:
# check if ephemfile and msinfofile exist
if not ephem:
print(
"ephemeris info does not exist, querying from JPL Horizons on the fly"
)
ephem = hf.read_horizons(vis=vis)
if not msinfo:
print("ms info not provided, generating one on the fly")
msinfo = hf.read_msinfo(vis)
hf.imreg(vis=vis,
ephem=ephem,
msinfo=msinfo,
timerange=timerange,
reftime=reftime,
imagefile=imname + '.image',
fitsfile=imname + '.fits',
toTb=toTb,
scl100=False,
usephacenter=usephacenter,
subregion=subregion)
if os.path.exists(imname + '.fits'):
shutil.rmtree(imname + '.image')
return [True, btstr, etstr, imname + '.fits']
else:
return [False, btstr, etstr, '']
except:
print('error in registering image: ' + btstr)
return [False, btstr, etstr, imname + '.image']
else:
if os.path.exists(imname + '.image'):
return [True, btstr, etstr, imname + '.image']
else:
return [False, btstr, etstr, '']
def WEI_plot(dofile=True,
vis=None,
timerange=None,
spw='',
aiafits='',
imagehead='',
workdir='',
spwCol=3,
phasecenter='J2000 11h00m48 06d14m60'):
if vis[-1] == '/':
vis = vis[:-1]
ms.open(vis)
spwInfo = ms.getspectralwindowinfo()
ms.close()
tb.open(vis)
starttim = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
endtim = Time(tb.getcell('TIME',
tb.nrows() - 1) / 24. / 3600.,
format='mjd')
tb.close()
tb.open(vis + '/SPECTRAL_WINDOW')
reffreqs = tb.getcol('REF_FREQUENCY')
bdwds = tb.getcol('TOTAL_BANDWIDTH')
cfreqs = reffreqs + bdwds / 2.
tb.close()
sbeam = 35.
#get timerange from vis file
if not timerange:
timerange = '{0}~{1}'.format(
starttim.iso.replace('-', '/').replace(' ', '/'),
endtim.iso.replace('-', '/').replace(' ', '/'))
nspw = len(spwInfo)
#draw plot aia
fig = plt.figure(figsize=(12, 7), dpi=100)
gs1 = gridspec.GridSpec(4, 3)
gs1.update(left=0.08, right=0.32, wspace=0.05)
ax1 = plt.subplot(gs1[11])
aiamap = smap.Map(aiafits)
aiamap.plot(axes=ax1)
#do clean spwCol by spwCol
for cur_spwCol in range(0, np.floor_divide(nspw, spwCol)):
if ((cur_spwCol + 1) * spwCol) < nspw:
cur_spwRange = str(cur_spwCol * spwCol + 1) + '~' + str(
(cur_spwCol + 1) * spwCol)
else:
cur_spwRange = str(cur_spwCol * spwCol + 1) + '~' + '31'
imagename = imagehead + cur_spwRange + 'SPWs'
cur_eovsaFits = imagename + '.fits'
if cur_spwCol < 6:
cur_mask = '/srg/ywei/data/eovsa/mask/sep_6mask_' + str(
cur_spwCol + 1) + '.rgn'
else:
cur_mask = '/srg/ywei/data/eovsa/mask/sep_6/mask_6.rgn'
if dofile:
#clean(vis=vis, spw=cur_spwRange, timerange=timerange, imagename=imagename, imsize=[256,256], niter=100, cell=['2arcsec'] )
#clean(vis=vis, spw=cur_spwRange, timerange=timerange, imagename=imagename, imsize=[512,512], niter=1000, cell=['1arcsec'],stokes='XX', gain=0.05,weighting='briggs', mode='mfs',imagermode='csclean',psfmode='clark',robust=0.0,restoringbeam = ['10.0arcsec'], mask=cur_mask,pbcor=True)
clean(vis=vis,
spw=cur_spwRange,
timerange=timerange,
imagename=imagename,
imsize=[512, 512],
niter=1000,
cell=['1arcsec'],
stokes='XX',
gain=0.05,
weighting='briggs',
mode='mfs',
imagermode='csclean',
psfmode='clark',
robust=0.0,
restoringbeam=['10.0arcsec'],
mask='',
pbcor=True)
print 'fits name =' + str(cur_eovsaFits)
hf.imreg(vis=vis,
imagefile=imagename + '.image',
fitsfile=imagename + '.fits',
timerange=timerange)
#plot eovsa
cur_emap = smap.Map(cur_eovsaFits)
cur_axname = plt.subplot(gs1[cur_spwCol + 1])
(npol, nf, nx, ny) = cur_emap.data.shape
print 'shape = ' + str(cur_emap.data.shape)
if npol != 1:
print 'To be determined'
else:
cur_emap.data = cur_emap.data.reshape((512, 512))
cur_emap.plot_settings['cmap'] = plt.get_cmap('jet')
cur_emap.plot(axes=cur_axname)
stokes=pol,
weighting='briggs',
robust=0.0,
restoringbeam=[str(bm) + 'arcsec'],
#phasecenter='J2000 11h14m09 04d52m53',
phasecenter='J2000 11h00m48 06d14m60',
#mask='clean_mask.rgn',
#mask=cur_mask,
mask='',
pbcor=True,
interactive=False,
usescratch=False)
#hf.imreg(vis=slfcaledms,imagefile=imname+'.image',fitsfile=fitsfile,timerange=trange,usephacenter=True,toTb=True,verbose=False)
hf.imreg(vis=slfcaledms,
ephem=eph,
imagefile=imname + '.image',
fitsfile=fitsfile,
timerange=trange,
verbose=False)
#imname_ori=imgprefix+'_ori_'+spwran.replace('~','-')
#fitsfile_ori=imname_ori+'.fits'
#clean(vis=refms,
# antenna=antennas,
# imagename=imname_ori,
# spw=sp,
# #mode='channel',
# mode='mfs',
# timerange=trange,
# imagermode='csclean',
# psfmode='clark',
# imsize=[256,256],
# cell=['2arcsec'],
def calibeovsa(vis=None,
caltype=None,
interp=None,
docalib=True,
doflag=True,
flagant=None,
doimage=False,
imagedir=None,
antenna=None,
timerange=None,
spw=None,
stokes=None,
doconcat=False,
msoutdir=None,
keep_orig_ms=True):
'''
:param vis: EOVSA visibility dataset(s) to be calibrated
:param caltype:
:param interp:
:param docalib:
:param qlookimage:
:param flagant:
:param stokes:
:param doconcat:
:return:
'''
if type(vis) == str:
vis = [vis]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
for msfile in vis:
casalog.origin('calibeovsa')
if not caltype:
casalog.post(
"Caltype not provided. Perform reference phase calibration and daily phase calibration."
)
caltype = [
'refpha', 'phacal', 'fluxcal'
] ## use this line after the phacal is applied # caltype = ['refcal']
if not os.path.exists(msfile):
casalog.post("Input visibility does not exist. Aborting...")
continue
if msfile.endswith('/'):
msfile = msfile[:-1]
if not msfile[-3:] in ['.ms', '.MS']:
casalog.post(
"Invalid visibility. Please provide a proper visibility file ending with .ms"
)
# if not caltable:
# caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype]
# get band information
tb.open(msfile + '/SPECTRAL_WINDOW')
nspw = tb.nrows()
bdname = tb.getcol('NAME')
bd_nchan = tb.getcol('NUM_CHAN')
bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33
# nchans = tb.getcol('NUM_CHAN')
# reffreqs = tb.getcol('REF_FREQUENCY')
# cenfreqs = np.zeros((nspw))
tb.close()
tb.open(msfile + '/ANTENNA')
nant = tb.nrows()
antname = tb.getcol('NAME')
antlist = [str(ll) for ll in range(len(antname) - 1)]
antennas = ','.join(antlist)
tb.close()
# get time stamp, use the beginning of the file
tb.open(msfile + '/OBSERVATION')
trs = {'BegTime': [], 'EndTime': []}
for ll in range(tb.nrows()):
tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600,
format='mjd')
trs['BegTime'].append(tim0)
trs['EndTime'].append(tim1)
tb.close()
trs['BegTime'] = Time(trs['BegTime'])
trs['EndTime'] = Time(trs['EndTime'])
btime = np.min(trs['BegTime'])
etime = np.max(trs['EndTime'])
# ms.open(vis)
# summary = ms.summary()
# ms.close()
# btime = Time(summary['BeginTime'], format='mjd')
# etime = Time(summary['EndTime'], format='mjd')
## stop using ms.summary to avoid conflicts with importeovsa
t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd')
print "This scan observed from {} to {} UTC".format(
btime.iso, etime.iso)
gaintables = []
if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal'
in caltype):
refcal = ra.sql2refcalX(btime)
pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband)
pha[np.where(refcal['flag'] == 1)] = 0.
amp = refcal['amp']
amp[np.where(refcal['flag'] == 1)] = 1.
t_ref = refcal['timestamp']
# find the start and end time of the local day when refcal is registered
try:
dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60.
except:
dhr = -7.
bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd')
et = Time(bt.mjd + 1., format='mjd')
(yr, mon, day) = (bt.datetime.year, bt.datetime.month,
bt.datetime.day)
dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/'
if not os.path.exists(dirname):
os.mkdir(dirname)
# check if there is any ROACH reboot between the reference calibration found and the current data
t_rbts = db.get_reboot(Time([t_ref, btime]))
if not t_rbts:
casalog.post(
"Reference calibration is derived from observation at " +
t_ref.iso)
print "Reference calibration is derived from observation at " + t_ref.iso
else:
casalog.post(
"Oh crap! Roach reboot detected between the reference calibration time "
+ t_ref.iso + ' and the current observation at ' +
btime.iso)
casalog.post("Aborting...")
print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso
print "Aborting..."
para_pha = []
para_amp = []
calpha = np.zeros((nspw, 15, 2))
calamp = np.zeros((nspw, 15, 2))
for s in range(nspw):
for n in range(15):
for p in range(2):
calpha[s, n, p] = pha[n, p, bd[s]]
calamp[s, n, p] = amp[n, p, bd[s]]
para_pha.append(np.degrees(pha[n, p, bd[s]]))
para_amp.append(amp[n, p, bd[s]])
if 'fluxcal' in caltype:
calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59'))
t_bp = Time(calfac['timestamp'], format='lv')
if int(t_mid.mjd) == int(t_bp.mjd):
accalfac = calfac['accalfac'] # (ant x pol x freq)
# tpcalfac = calfac['tpcalfac'] # (ant x pol x freq)
caltb_autoamp = dirname + t_bp.isot[:-4].replace(
':', '').replace('-', '') + '.bandpass'
if not os.path.exists(caltb_autoamp):
bandpass(vis=msfile,
caltable=caltb_autoamp,
solint='inf',
refant='eo01',
minblperant=0,
minsnr=0,
bandtype='B',
docallib=False)
tb.open(caltb_autoamp, nomodify=False) # (ant x spw)
bd_chanidx = np.hstack([[0], bd_nchan.cumsum()])
for ll in range(nspw):
antfac = np.sqrt(
accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]])
# # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]]
antfac = np.moveaxis(antfac, 0, 2)
cparam = np.zeros((2, bd_nchan[ll], nant))
cparam[:, :, :-3] = 1.0 / antfac
tb.putcol('CPARAM', cparam + 0j, ll * nant, nant)
paramerr = tb.getcol('PARAMERR', ll * nant, nant)
paramerr = paramerr * 0
tb.putcol('PARAMERR', paramerr, ll * nant, nant)
bpflag = tb.getcol('FLAG', ll * nant, nant)
bpant1 = tb.getcol('ANTENNA1', ll * nant, nant)
bpflagidx, = np.where(bpant1 >= 13)
bpflag[:] = False
bpflag[:, :, bpflagidx] = True
tb.putcol('FLAG', bpflag, ll * nant, nant)
bpsnr = tb.getcol('SNR', ll * nant, nant)
bpsnr[:] = 100.0
bpsnr[:, :, bpflagidx] = 0.0
tb.putcol('SNR', bpsnr, ll * nant, nant)
tb.close()
msg_prompt = "Scaling calibration is derived for {}.".format(
msfile)
casalog.post(msg_prompt)
print msg_prompt
gaintables.append(caltb_autoamp)
else:
msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format(
t_mid.datetime.strftime('%b %d, %Y'), msfile)
casalog.post(msg_prompt)
print msg_prompt
if ('refpha' in caltype) or ('refcal' in caltype):
# caltb_pha = os.path.basename(vis).replace('.ms', '.refpha')
# check if the calibration table already exists
caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refpha'
if not os.path.exists(caltb_pha):
gencal(vis=msfile,
caltable=caltb_pha,
caltype='ph',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_pha)
gaintables.append(caltb_pha)
if ('refamp' in caltype) or ('refcal' in caltype):
# caltb_amp = os.path.basename(vis).replace('.ms', '.refamp')
caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refamp'
if not os.path.exists(caltb_amp):
gencal(vis=msfile,
caltable=caltb_amp,
caltype='amp',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_amp)
gaintables.append(caltb_amp)
# calibration for the change of delay center between refcal time and beginning of scan -- hopefully none!
xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False)
if buf:
dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv')
dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15]
xml, buf = ch.read_calX(4, t=t_ref)
dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv')
dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15]
dlycen_ns_diff = dlycen_ns2 - dlycen_ns1
for n in range(2):
dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n]
print 'Multi-band delay is derived from delay center difference at {} & {}'.format(
dly_t1.iso, dly_t2.iso)
# print '=====Delays relative to Ant 14====='
# for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]):
# ant = antlist[i]
# print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl
# dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1])
# caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0')
caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace(
'-', '') + '.dlycen'
if not os.path.exists(caltb_dlycen):
gencal(vis=msfile,
caltable=caltb_dlycen,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=dlycen_ns_diff.flatten().tolist())
gaintables.append(caltb_dlycen)
if 'phacal' in caltype:
phacals = np.array(
ra.sql2phacalX([bt, et], neat=True, verbose=False))
if not phacals.any() or len(phacals) == 0:
print "Found no phacal records in SQL database, will skip phase calibration"
else:
# first generate all phacal calibration tables if not already exist
t_phas = Time([phacal['t_pha'] for phacal in phacals])
# sort the array in ascending order by t_pha
sinds = t_phas.mjd.argsort()
t_phas = t_phas[sinds]
phacals = phacals[sinds]
caltbs_phambd = []
for i, phacal in enumerate(phacals):
# filter out phase cals with reference time stamp >30 min away from the provided refcal time
if (phacal['t_ref'].jd -
refcal['timestamp'].jd) > 30. / 1440.:
del phacals[i]
del t_phas[i]
continue
else:
t_pha = phacal['t_pha']
phambd_ns = phacal['pslope']
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(phacal['flag'] == 1)] = 0.
caltb_phambd = dirname + t_pha.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
caltbs_phambd.append(caltb_phambd)
if not os.path.exists(caltb_phambd):
gencal(vis=msfile,
caltable=caltb_phambd,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
# now decides which table to apply depending on the interpolation method ("neatest" or "linear")
if interp == 'nearest':
tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd))
dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24.
print "Selected nearest phase calibration table at " + t_phas[
tbind].iso
gaintables.append(caltbs_phambd[tbind])
if interp == 'linear':
# bphacal = ra.sql2phacalX(btime)
# ephacal = ra.sql2phacalX(etime,reverse=True)
bt_ind, = np.where(t_phas.mjd < btime.mjd)
et_ind, = np.where(t_phas.mjd > etime.mjd)
if len(bt_ind) == 0 and len(et_ind) == 0:
print "No phacal found before or after the ms data within the day of observation"
print "Skipping daily phase calibration"
elif len(bt_ind) > 0 and len(et_ind) == 0:
gaintables.append(caltbs_phambd[bt_ind[-1]])
elif len(bt_ind) == 0 and len(et_ind) > 0:
gaintables.append(caltbs_phambd[et_ind[0]])
elif len(bt_ind) > 0 and len(et_ind) > 0:
bphacal = phacals[bt_ind[-1]]
ephacal = phacals[et_ind[0]]
# generate a new table interpolating between two daily phase calibrations
t_pha_mean = Time(np.mean(
[bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]),
format='mjd')
phambd_ns = (bphacal['pslope'] +
ephacal['pslope']) / 2.
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(bphacal['flag'] == 1)] = 0.
phambd_ns[np.where(ephacal['flag'] == 1)] = 0.
caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
if not os.path.exists(caltb_phambd_interp):
gencal(vis=msfile,
caltable=caltb_phambd_interp,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
print "Using phase calibration table interpolated between records at " + bphacal[
't_pha'].iso + ' and ' + ephacal['t_pha'].iso
gaintables.append(caltb_phambd_interp)
if docalib:
clearcal(msfile)
applycal(vis=msfile,
gaintable=gaintables,
applymode='calflag',
calwt=False)
# delete the interpolated phase calibration table
try:
caltb_phambd_interp
except:
pass
else:
if os.path.exists(caltb_phambd_interp):
shutil.rmtree(caltb_phambd_interp)
if doflag:
# flag zeros and NaNs
flagdata(vis=msfile, mode='clip', clipzeros=True)
if flagant:
try:
flagdata(vis=msfile, antenna=flagant)
except:
print "Something wrong with flagant. Abort..."
if doimage:
from matplotlib import pyplot as plt
from suncasa.utils import helioimage2fits as hf
from sunpy import map as smap
if not antenna:
antenna = '0~12'
if not stokes:
stokes = 'XX'
if not timerange:
timerange = ''
if not spw:
spw = '1~3'
if not imagedir:
imagedir = '.'
#(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day)
#dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/'
#if not os.path.exists(dirname):
# os.makedirs(dirname)
bds = [spw]
nbd = len(bds)
imgs = []
for bd in bds:
if '~' in bd:
bdstr = bd.replace('~', '-')
else:
bdstr = str(bd).zfill(2)
imname = imagedir + '/' + os.path.basename(msfile).replace(
'.ms', '.bd' + bdstr)
print 'Cleaning image: ' + imname
try:
clean(vis=msfile,
imagename=imname,
antenna=antenna,
spw=bd,
timerange=timerange,
imsize=[512],
cell=['5.0arcsec'],
stokes=stokes,
niter=500)
except:
print 'clean not successfull for band ' + str(bd)
else:
imgs.append(imname + '.image')
junks = ['.flux', '.mask', '.model', '.psf', '.residual']
for junk in junks:
if os.path.exists(imname + junk):
shutil.rmtree(imname + junk)
tranges = [btime.iso + '~' + etime.iso] * nbd
fitsfiles = [img.replace('.image', '.fits') for img in imgs]
hf.imreg(vis=msfile,
timerange=tranges,
imagefile=imgs,
fitsfile=fitsfiles,
usephacenter=False)
plt.figure(figsize=(6, 6))
for i, fitsfile in enumerate(fitsfiles):
plt.subplot(1, nbd, i + 1)
eomap = smap.Map(fitsfile)
sz = eomap.data.shape
if len(sz) == 4:
eomap.data = eomap.data.reshape((sz[2], sz[3]))
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot()
eomap.draw_limb()
eomap.draw_grid()
plt.show()
if doconcat:
if len(vis) > 1:
# from suncasa.eovsa import concateovsa as ce
from suncasa.tasks import concateovsa_cli as ce
if msoutdir is None:
msoutdir = './'
concatvis = os.path.basename(vis[0])
concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms'
ce.concateovsa(vis,
concatvis,
datacolumn='corrected',
keep_orig_ms=keep_orig_ms,
cols2rm="model,corrected")
return [concatvis]
else:
return vis
timerange=trange,
imagermode='csclean',
psfmode='clark',
imsize=[512, 512],
cell=['5arcsec'],
niter=1000,
gain=0.05,
stokes=pol,
restoringbeam=['15arcsec'],
interactive=False,
pbcor=True,
usescratch=True)
hf.imreg(vis=slfcalms,
imagefile=img_init + '.image',
fitsfile=img_init + '.fits',
timerange=trange,
usephacenter=False,
verbose=True)
from sunpy import map as smap
from matplotlib import pyplot as plt
eomap = smap.Map(img_init + '.fits')
eomap.data = eomap.data.reshape((512, 512))
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot()
eomap.draw_limb()
eomap.draw_grid()
plt.show()
viewer(img_init + '.image')
if dopartsun:
def clean_iter(
tim, freq, vis, imageprefix, imagesuffix, ncpu, twidth, doreg,
usephacenter, reftime, ephem, msinfo, toTb, overwrite, outlierfile,
field, spw, selectdata, uvrange, antenna, scan, observation, intent,
mode, resmooth, gridmode, wprojplanes, facets, cfcache, rotpainc,
painc, aterm, psterm, mterm, wbawp, conjbeams, epjtable, interpolation,
niter, gain, threshold, psfmode, imagermode, ftmachine, mosweight,
scaletype, multiscale, negcomponent, smallscalebias, interactive, mask,
nchan, start, width, outframe, veltype, imsize, cell, phasecenter,
restfreq, stokes, weighting, robust, uvtaper, outertaper, innertaper,
modelimage, restoringbeam, pbcor, minpb, usescratch, noise, npixels,
npercycle, cyclefactor, cyclespeedup, nterms, reffreq, chaniter,
flatnoise, allowchunk, btidx):
from taskinit import ms
from taskinit import qa
# from __casac__.quanta import quanta as qa
from __main__ import default, inp
#from clean import clean
from clean_cli import clean_cli as clean
bt = btidx # 0
if bt + twidth < len(tim) - 1:
et = btidx + twidth - 1
else:
et = len(tim) - 1
# tim_d = tim/3600./24.-np.fix(tim/3600./24.)
if bt == 0:
bt_d = tim[bt] - ((tim[bt + 1] - tim[bt]) / 2)
else:
bt_d = tim[bt] - ((tim[bt] - tim[bt - 1]) / 2)
if et == (len(tim) - 1) or et == -1:
et_d = tim[et] + ((tim[et] - tim[et - 1]) / 2)
else:
et_d = tim[et] + ((tim[et + 1] - tim[et]) / 2)
#
# bt_d=tim[bt]
# et_d=tim[et]+0.005
timerange = qa.time(qa.quantity(bt_d, 's'), prec=9, form='ymd')[0] + '~' + \
qa.time(qa.quantity(et_d, 's'), prec=9, form='ymd')[0]
tmid = (bt_d + et_d) / 2.
btstr = qa.time(qa.quantity(bt_d, 's'), prec=9, form='fits')[0]
etstr = qa.time(qa.quantity(et_d, 's'), prec=9, form='fits')[0]
print 'cleaning timerange: ' + timerange
image0 = btstr.replace(':', '').replace('-', '')
imname = imageprefix + image0 + imagesuffix
if overwrite or (len(glob.glob(imname + '*')) == 0):
# inp(taskname = 'clean')
os.system('rm -rf {}*'.format(imname))
try:
clean(vis=vis,
imagename=imname,
outlierfile=outlierfile,
field=field,
spw=spw,
selectdata=selectdata,
timerange=timerange,
uvrange=uvrange,
antenna=antenna,
scan=scan,
observation=str(observation),
intent=intent,
mode=mode,
resmooth=resmooth,
gridmode=gridmode,
wprojplanes=wprojplanes,
facets=facets,
cfcache=cfcache,
rotpainc=rotpainc,
painc=painc,
psterm=psterm,
aterm=aterm,
mterm=mterm,
wbawp=wbawp,
conjbeams=conjbeams,
epjtable=epjtable,
interpolation=interpolation,
niter=niter,
gain=gain,
threshold=threshold,
psfmode=psfmode,
imagermode=imagermode,
ftmachine=ftmachine,
mosweight=mosweight,
scaletype=scaletype,
multiscale=multiscale,
negcomponent=negcomponent,
smallscalebias=smallscalebias,
interactive=interactive,
mask=mask,
nchan=nchan,
start=start,
width=width,
outframe=outframe,
veltype=veltype,
imsize=imsize,
cell=cell,
phasecenter=phasecenter,
restfreq=restfreq,
stokes=stokes,
weighting=weighting,
robust=robust,
uvtaper=uvtaper,
outertaper=outertaper,
innertaper=innertaper,
modelimage=modelimage,
restoringbeam=restoringbeam,
pbcor=pbcor,
minpb=minpb,
usescratch=usescratch,
noise=noise,
npixels=npixels,
npercycle=npercycle,
cyclefactor=cyclefactor,
cyclespeedup=cyclespeedup,
nterms=nterms,
reffreq=reffreq,
chaniter=chaniter,
flatnoise=flatnoise,
allowchunk=False)
clnjunks = ['.flux', '.mask', '.model', '.psf', '.residual']
for clnjunk in clnjunks:
if os.path.exists(imname + clnjunk):
shutil.rmtree(imname + clnjunk)
except:
print('error in cleaning image: ' + btstr)
return [False, btstr, etstr, '']
else:
print imname + ' exists. Clean task aborted.'
if doreg and not os.path.isfile(imname + '.fits'):
#ephem.keys()
#msinfo.keys()
try:
# check if ephemfile and msinfofile exist
if not ephem:
print(
"ephemeris info does not exist, querying from JPL Horizons on the fly"
)
ephem = hf.read_horizons(vis)
if not msinfo:
print("ms info not provided, generating one on the fly")
msinfo = hf.read_msinfo(vis)
hf.imreg(vis=vis,
ephem=ephem,
msinfo=msinfo,
timerange=timerange,
reftime=reftime,
imagefile=imname + '.image',
fitsfile=imname + '.fits',
toTb=toTb,
scl100=False,
usephacenter=usephacenter)
if os.path.exists(imname + '.fits'):
shutil.rmtree(imname + '.image')
return [True, btstr, etstr, imname + '.fits']
else:
return [False, btstr, etstr, '']
except:
print('error in registering image: ' + btstr)
return [False, btstr, etstr, imname + '.image']
else:
if os.path.exists(imname + '.image'):
return [True, btstr, etstr, imname + '.image']
else:
return [False, btstr, etstr, '']
imagename=im_init,
spw='1~15',
specmode='mfs',
timerange=trange,
imsize=[npix],
cell=['5arcsec'],
niter=1000,
gain=0.05,
stokes='I',
restoringbeam=['30arcsec'],
interactive=False,
pbcor=True)
hf.imreg(vis=slfcalms,
imagefile=im_init + '.image.pbcor',
fitsfile=im_init + '.fits',
timerange=trange,
usephacenter=False,
verbose=True)
clnjunks = [
'.flux', '.mask', '.model', '.psf', '.residual', '.sumwt', '.pb',
'.image'
]
for clnjunk in clnjunks:
if os.path.exists(im_init + clnjunk):
os.system('rm -rf ' + im_init + clnjunk)
from sunpy import map as smap
from matplotlib import pyplot as plt
fig = plt.figure(figsize=(6, 6))
ax = fig.add_subplot(111)
eomap = smap.Map(im_init + '.fits')
def svplot(vis, timerange=None, spw='', workdir='./', specfile=None, bl=None, uvrange=None,
stokes='RR,LL', dmin=None, dmax=None,
goestime=None, reftime=None,
xycen=None, fov=[500.,500.], xyrange=None, restoringbeam=[''], robust=0.0,
niter=500, imsize=[512], cell=['5.0arcsec'],interactive=False,
usemsphacenter=True, imagefile=None, fitsfile=None, plotaia=True,
aiawave=171, aiafits=None, savefig=False, mkmovie=False, overwrite=True, ncpu=10, twidth=1, verbose=True):
'''
Required inputs:
vis: calibrated CASA measurement set
Important optional inputs:
timerange: timerange for clean. Standard CASA time selection format.
If not provided, use the entire range (*BE CAREFUL, COULD BE VERY SLOW*)
spw: spectral window selection following the CASA syntax.
Examples: spw='1:2~60' (spw id 1, channel range 2-60); spw='*:1.2~1.3GHz' (selects all channels within 1.2-1.3 GHz; note the *)
specfile: supply dynamic spectrum save file (from suncasa.utils.dspec2.get_dspec()). Otherwise
generate a median dynamic spectrum on the fly
Optional inputs:
bl: baseline to generate dynamic spectrum
uvrange: uvrange to select baselines for generating dynamic spectrum
stokes: polarization of the clean image, can be 'RR,LL' or 'I,V'
dmin,dmax: color bar parameter
goestime: goes plot time, example ['2016/02/18 18:00:00','2016/02/18 23:00:00']
rhessisav: rhessi savefile
reftime: reftime for the image
xycen: center of the image in helioprojective coordinates (HPLN/HPLT), in arcseconds. Example: [900, -150.]
fov: field of view in arcsecs. Example: [500., 500.]
xyrange: field of view in solar XY coordinates. Format: [[x1,x2],[y1,y2]]. Example: [[900., 1200.],[0,300]]
***NOTE: THIS PARAMETER OVERWRITES XYCEN AND FOV***
aiawave: wave length of aia file in a
imagefile: if imagefile provided, use it. Otherwise do clean and generate a new one.
fitsfile: if fitsfile provided, use it. Otherwise generate a new one
savefig: whether to save the figure
Example:
'''
if xycen:
xc, yc = xycen
xlen, ylen = fov
if parse_version(sunpy.__version__)>parse_version('0.8.0'):
xyrange = [[xc - xlen / 2.0, yc - ylen / 2.0], [xc + xlen / 2.0, yc + ylen / 2.0]]
else:
xyrange = [[xc - xlen / 2.0, xc + xlen / 2.0], [yc - ylen / 2.0, yc + ylen / 2.0]]
stokes_allowed = ['RR,LL', 'I,V', 'RRLL', 'IV']
if not stokes in stokes_allowed:
print 'wrong stokes parameter ' + str(stokes) + '. Allowed values are ' + ', '.join(stokes_allowed)
return -1
if stokes == 'RRLL':
stokes = 'RR,LL'
if stokes == 'IV':
stokes = 'I,V'
if vis[-1] == '/':
vis = vis[:-1]
if not os.path.exists(vis):
print 'input measurement not exist'
return -1
if aiafits is None:
aiafits = ''
# split the data
# generating dynamic spectrum
if not os.path.exists(workdir):
os.makedirs(workdir)
if specfile:
try:
specdata = np.load(specfile)
except:
print('Provided dynamic spectrum file not numpy npz. Generating one from the visibility data')
specfile = os.path.join(workdir, os.path.basename(vis) + '.dspec.npz')
dspec_external(vis, workdir=workdir, specfile=specfile)
specdata = np.load(specfile) # specdata = ds.get_dspec(vis, domedian=True, verbose=True)
else:
print('Dynamic spectrum file not provided; Generating one from the visibility data')
# specdata = ds.get_dspec(vis, domedian=True, verbose=True)
specfile = os.path.join(workdir, os.path.basename(vis) + '.dspec.npz')
dspec_external(vis, workdir=workdir, specfile=specfile)
specdata = np.load(specfile)
tb.open(vis)
starttim = Time(tb.getcell('TIME', 0) / 24. / 3600., format='mjd')
endtim = Time(tb.getcell('TIME', tb.nrows() - 1) / 24. / 3600., format='mjd')
tb.close()
datstr = starttim.iso[:10]
if timerange is None or timerange == '':
starttim1 = starttim
endtim1 = endtim
timerange = '{0}~{1}'.format(starttim.iso.replace('-', '/').replace(' ', '/'), endtim.iso.replace('-', '/').replace(' ', '/'))
else:
try:
(tstart, tend) = timerange.split('~')
if tstart[2] == ':':
starttim1 = Time(datstr + 'T' + tstart)
endtim1 = Time(datstr + 'T' + tend)
timerange = '{0}/{1}~{0}/{2}'.format(datstr.replace('-', '/'), tstart, tend)
else:
starttim1 = Time(qa.quantity(tstart, 'd')['value'], format='mjd')
endtim1 = Time(qa.quantity(tend, 'd')['value'], format='mjd')
except ValueError:
print "keyword 'timerange' in wrong format"
midtime_mjd = (starttim1.mjd + endtim1.mjd) / 2.
if vis.endswith('/'):
vis = vis[:-1]
visname = os.path.basename(vis)
bt = starttim1.plot_date
et = endtim1.plot_date
# find out min and max frequency for plotting in dynamic spectrum
ms.open(vis)
metadata = ms.metadata()
observatory = metadata.observatorynames()[0]
spwInfo = ms.getspectralwindowinfo()
nspw = len(spwInfo)
if not spw:
spw = '0~' + str(nspw - 1)
staql = {'timerange': timerange, 'spw': spw}
if ms.msselect(staql, onlyparse=True):
ndx = ms.msselectedindices()
chan_sel = ndx['channel']
nspw = chan_sel.shape[0]
bspw = chan_sel[0, 0]
bchan = chan_sel[0, 1]
espw = chan_sel[-1, 0]
echan = chan_sel[-1, 2]
bfreq = spwInfo[str(bspw)]['Chan1Freq'] + spwInfo[str(bspw)]['ChanWidth'] * bchan
efreq = spwInfo[str(espw)]['Chan1Freq'] + spwInfo[str(espw)]['ChanWidth'] * echan
bfreqghz = bfreq / 1e9
efreqghz = efreq / 1e9
if verbose:
print 'selected timerange {}'.format(timerange)
print 'selected frequency range {0:6.3f} to {1:6.3f} GHz'.format(bfreqghz, efreqghz)
else:
print "spw or timerange selection failed. Aborting..."
ms.close()
return -1
ms.close()
if observatory == 'EOVSA':
print 'Provide stokes: ' + str(stokes) + '. However EOVSA has linear feeds. Force stokes to be IV'
stokes = 'I,V'
if mkmovie:
plt.ioff()
# fig = plt.figure(figsize=(12, 7.5), dpi=100)
if fitsfile:
pass
else:
if not imagefile:
# from ptclean_cli import ptclean_cli as ptclean
eph = hf.read_horizons(t0=Time(midtime_mjd, format='mjd'))
if observatory == 'EOVSA' or (not usemsphacenter):
phasecenter = ''
else:
phasecenter = 'J2000 ' + str(eph['ra'][0])[:15] + 'rad ' + str(eph['dec'][0])[:15] + 'rad'
print 'use phasecenter: ' + phasecenter
qlookfitsdir = os.path.join(workdir, 'qlookfits/')
qlookfigdir = os.path.join(workdir, 'qlookimgs/')
imresfile = os.path.join(qlookfitsdir, '{}.imres.npz'.format(os.path.basename(vis)))
if overwrite:
imres = mk_qlook_image(vis, twidth=twidth, ncpu=ncpu, imagedir=qlookfitsdir, phasecenter=phasecenter, stokes=stokes,
c_external=True)
else:
if os.path.exists(imresfile):
imres = np.load(imresfile)
imres = imres['imres'].item()
else:
print('Image results file not found; Creating new images.')
imres = mk_qlook_image(vis, twidth=twidth, ncpu=ncpu, imagedir=qlookfitsdir, phasecenter=phasecenter, stokes=stokes,
c_external=True)
if not os.path.exists(qlookfigdir):
os.makedirs(qlookfigdir)
plt_qlook_image(imres, figdir=qlookfigdir, specdata=specdata, verbose=True, stokes=stokes, fov=xyrange)
else:
spec = specdata['spec']
(npol, nbl, nfreq, ntim) = spec.shape
tidx = range(ntim)
fidx = range(nfreq)
tim = specdata['tim']
freq = specdata['freq']
freqghz = freq / 1e9
spec_tim = Time(specdata['tim'] / 3600. / 24., format='mjd')
timstrr = spec_tim.plot_date
plt.ion()
fig = plt.figure(figsize=(12, 7), dpi=100)
gs1 = gridspec.GridSpec(3, 1)
gs1.update(left=0.08, right=0.32, wspace=0.05)
gs2 = gridspec.GridSpec(2, 2)
gs2.update(left=0.38, right=0.98, hspace=0.02, wspace=0.02)
spec_1 = np.absolute(spec[0, 0, :, :])
spec_2 = np.absolute(spec[1, 0, :, :])
if observatory == 'EVLA':
# circular feeds
polstr = ['RR', 'LL']
if observatory == 'EOVSA' or observatory == 'ALMA':
# linear feeds
polstr = ['XX', 'YY']
print 'plot the dynamic spectrum in pol ' + ' & '.join(polstr)
ax1 = plt.subplot(gs1[0])
ax1.pcolormesh(timstrr, freqghz, spec_1, cmap='jet', vmin=dmin, vmax=dmax)
ax1.set_xlim(timstrr[tidx[0]], timstrr[tidx[-1]])
ax1.xaxis_date()
ax1.xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
# ax1.set_xticklabels(['']*10)
ax1.set_ylim(freqghz[fidx[0]], freqghz[fidx[-1]])
ax1.set_ylabel('Frequency (GHz)', fontsize=10)
ax1.set_title(observatory + ' ' + datstr + ' ' + polstr[0] + ' & ' + polstr[1], fontsize=12)
ax1.set_autoscale_on(False)
ax1.add_patch(patches.Rectangle((bt, bfreqghz), et - bt, efreqghz - bfreqghz, ec='w', fill=False))
ax1.plot([(bt + et) / 2.], [(bfreqghz + efreqghz) / 2.], '*w', ms=12)
for tick in ax1.get_xticklabels():
tick.set_fontsize(8)
for tick in ax1.get_yticklabels():
tick.set_fontsize(8)
ax2 = plt.subplot(gs1[1])
ax2.pcolormesh(timstrr, freqghz, spec_2, cmap='jet', vmin=dmin, vmax=dmax)
ax2.set_xlim(timstrr[tidx[0]], timstrr[tidx[-1]])
ax2.xaxis_date()
ax2.xaxis.set_major_formatter(DateFormatter("%H:%M:%S"))
ax2.set_ylim(freqghz[fidx[0]], freqghz[fidx[-1]])
ax2.set_ylabel('Frequency (GHz)', fontsize=10)
for tick in ax2.get_xticklabels():
tick.set_fontsize(8)
for tick in ax2.get_yticklabels():
tick.set_fontsize(8)
ax2.set_autoscale_on(False)
ax2.add_patch(patches.Rectangle((bt, bfreqghz), et - bt, efreqghz - bfreqghz, ec='w', fill=False))
ax2.plot([(bt + et) / 2.], [(bfreqghz + efreqghz) / 2.], '*w', ms=12)
# Second part: GOES plot
if goestime:
btgoes = goestime[0]
etgoes = goestime[1]
else:
datstrg = datstr.replace('-', '/')
btgoes = datstrg + ' ' + qa.time(qa.quantity(tim[0] - 1800, 's'), form='clean', prec=9)[0]
etgoes = datstrg + ' ' + qa.time(qa.quantity(tim[tidx[-1] - 1] + 1800, 's'), form='clean', prec=9)[0]
if verbose:
print 'Acquire GOES soft X-ray data in from ' + btgoes + ' to ' + etgoes
ax3 = plt.subplot(gs1[2])
try:
from sunpy import lightcurve as lc
from sunpy.time import TimeRange
goest = lc.GOESLightCurve.create(TimeRange(btgoes, etgoes))
except:
goesscript = os.path.join(workdir, 'goes.py')
goesdatafile = os.path.join(workdir, 'goes.dat')
os.system('rm -rf {}'.format(goesscript))
fi = open(goesscript, 'wb')
fi.write('import os \n')
fi.write('from sunpy.time import TimeRange \n')
fi.write('from sunpy import lightcurve as lc \n')
fi.write('import pickle \n')
fi.write('goesplottim = TimeRange("{0}", "{1}") \n'.format(btgoes, etgoes))
fi.write('goes = lc.GOESLightCurve.create(goesplottim) \n')
fi.write('fi2 = open("{}", "wb") \n'.format(goesdatafile))
fi.write('pickle.dump(goes, fi2) \n')
fi.write('fi2.close()')
fi.close()
try:
os.system('python {}'.format(goesscript))
os.system('rm -rf {}'.format(goesscript))
except NameError:
print "Bad input names"
except ValueError:
print "Bad input values"
except:
print "Unexpected error:", sys.exc_info()[0]
print "Error in generating GOES light curves. Proceed without GOES..."
if os.path.exists(goesdatafile):
fi1 = file(goesdatafile, 'rb')
goest = pickle.load(fi1)
fi1.close()
try:
dates = mpl.dates.date2num(parse_time(goest.data.index))
goesdif = np.diff(goest.data['xrsb'])
gmax = np.nanmax(goesdif)
gmin = np.nanmin(goesdif)
ran = gmax - gmin
db = 2.8 / ran
goesdifp = goesdif * db + gmin + (-6)
ax3.plot_date(dates, np.log10(goest.data['xrsb']), '-', label='1.0--8.0 $\AA$', color='red', lw=2)
ax3.plot_date(dates[0:-1], goesdifp, '-', label='derivate', color='blue', lw=0.4)
ax3.set_ylim([-7, -3])
ax3.set_yticks([-7, -6, -5, -4, -3])
ax3.set_yticklabels([r'$10^{-7}$', r'$10^{-6}$', r'$10^{-5}$', r'$10^{-4}$', r'$10^{-3}$'])
ax3.set_title('Goes Soft X-ray', fontsize=12)
ax3.set_ylabel('Watts m$^{-2}$')
ax3.set_xlabel(datetime.datetime.isoformat(goest.data.index[0])[0:10])
ax3.axvspan(dates[899], dates[dates.size - 899], alpha=0.2)
ax2 = ax3.twinx()
# ax2.set_yscale("log")
ax2.set_ylim([-7, -3])
ax2.set_yticks([-7, -6, -5, -4, -3])
ax2.set_yticklabels(['B', 'C', 'M', 'X', ''])
ax3.yaxis.grid(True, 'major')
ax3.xaxis.grid(False, 'major')
ax3.legend(prop={'size': 6})
formatter = mpl.dates.DateFormatter('%H:%M')
ax3.xaxis.set_major_formatter(formatter)
ax3.fmt_xdata = mpl.dates.DateFormatter('%H:%M')
except:
print 'Error in downloading GOES soft X-ray data. Proceeding with out soft X-ray plot.'
# third part
# start to download the fits files
if plotaia:
if not aiafits:
newlist = []
items = glob.glob('*.fits')
for names in items:
str1 = starttim1.iso[:4] + '_' + starttim1.iso[5:7] + '_' + starttim1.iso[8:10] + 't' + starttim1.iso[
11:13] + '_' + starttim1.iso[14:16]
str2 = str(aiawave)
if names.endswith(".fits"):
if names.find(str1) != -1 and names.find(str2) != -1:
newlist.append(names)
newlist.append('0')
if newlist and os.path.exists(newlist[0]):
aiafits = newlist[0]
else:
print 'downloading the aiafits file'
wave1 = aiawave - 3
wave2 = aiawave + 3
t1 = Time(starttim1.mjd - 0.02 / 24., format='mjd')
t2 = Time(endtim1.mjd + 0.02 / 24., format='mjd')
try:
from sunpy.net import vso
client = vso.VSOClient()
qr = client.query(vso.attrs.Time(t1.iso, t2.iso), vso.attrs.Instrument('aia'), vso.attrs.Wave(wave1 * u.AA, wave2 * u.AA))
res = client.get(qr, path='{file}')
except:
SdoDownloadscript = os.path.join(workdir, 'SdoDownload.py')
os.system('rm -rf {}'.format(SdoDownloadscript))
fi = open(SdoDownloadscript, 'wb')
fi.write('from sunpy.net import vso \n')
fi.write('from astropy import units as u \n')
fi.write('client = vso.VSOClient() \n')
fi.write(
"qr = client.query(vso.attrs.Time('{0}', '{1}'), vso.attrs.Instrument('aia'), vso.attrs.Wave({2} * u.AA, {3} * u.AA)) \n".format(
t1.iso, t2.iso, wave1, wave2))
fi.write("res = client.get(qr, path='{file}') \n")
fi.close()
try:
os.system('python {}'.format(SdoDownloadscript))
except NameError:
print "Bad input names"
except ValueError:
print "Bad input values"
except:
print "Unexpected error:", sys.exc_info()[0]
print "Error in Downloading AIA fits files. Proceed without AIA..."
# Here something is needed to check whether it has finished downloading the fits files or not
if not aiafits:
newlist = []
items = glob.glob('*.fits')
for nm in items:
str1 = starttim1.iso[:4] + '_' + starttim1.iso[5:7] + '_' + starttim1.iso[8:10] + 't' + starttim1.iso[
11:13] + '_' + starttim1.iso[14:16]
str2 = str(aiawave)
if nm.find(str1) != -1 and nm.find(str2) != -1:
newlist.append(nm)
if newlist:
aiafits = newlist[0]
print 'AIA fits ' + aiafits + ' selected'
else:
print 'no AIA fits files found. Proceed without AIA'
try:
aiamap = smap.Map(aiafits)
except:
print 'error in reading aiafits. Proceed without AIA'
# RCP or I
ax4 = plt.subplot(gs2[0, 0])
ax5 = plt.subplot(gs2[1, 0])
# LCP or V
ax6 = plt.subplot(gs2[0, 1])
ax7 = plt.subplot(gs2[1, 1])
if fitsfile:
pass
else:
if not imagefile:
eph = hf.read_horizons(t0=Time(midtime_mjd, format='mjd'))
if observatory == 'EOVSA' or (not usemsphacenter):
print 'This is EOVSA data'
# use RA and DEC from FIELD ID 0
tb.open(vis+'/FIELD')
phadir = tb.getcol('PHASE_DIR').flatten()
tb.close()
ra0 = phadir[0]
dec0 = phadir[1]
if stokes == 'RRLL' or stokes == 'RR,LL':
print 'Provide stokes: ' + str(stokes) + '. However EOVSA has linear feeds. Force stokes to be IV'
stokes = 'I,V'
else:
ra0 = eph['ra'][0]
dec0 = eph['dec'][0]
if not xycen:
# use solar disk center as default
phasecenter = 'J2000 ' + str(ra0) + 'rad ' + str(dec0) + 'rad'
else:
x0 = np.radians(xycen[0]/3600.)
y0 = np.radians(xycen[1]/3600.)
p0 = np.radians(eph['p0'][0]) # p angle in radians
raoff = -((x0) * np.cos(p0) - y0 * np.sin(p0))/np.cos(eph['dec'][0])
decoff = (x0) * np.sin(p0) + y0 * np.cos(p0)
newra = ra0 + raoff
newdec = dec0 + decoff
phasecenter = 'J2000 ' + str(newra) + 'rad ' + str(newdec) + 'rad'
imagename = os.path.join(workdir, visname + '.outim')
if os.path.exists(imagename + '.image') or os.path.exists(imagename + '.flux'):
os.system('rm -rf ' + imagename + '.*')
sto = stokes.replace(',', '')
print 'do clean for ' + timerange + ' in spw ' + spw + ' stokes ' + sto
print 'Original phasecenter: '+ str(ra0) + str(dec0)
print 'use phasecenter: ' + phasecenter
clean(vis=vis, imagename=imagename, selectdata=True, spw=spw, timerange=timerange, stokes=sto,
niter=niter, interactive=interactive, npercycle=50, imsize=imsize, cell=cell, restoringbeam=restoringbeam,
weighting='briggs', robust=robust, phasecenter=phasecenter)
os.system('rm -rf ' + imagename + '.psf')
os.system('rm -rf ' + imagename + '.flux')
os.system('rm -rf ' + imagename + '.model')
os.system('rm -rf ' + imagename + '.mask')
os.system('rm -rf ' + imagename + '.residual')
imagefile = imagename + '.image'
fitsfile = imagefile + '.fits'
hf.imreg(vis=vis, ephem=eph, imagefile=imagefile, timerange=timerange, reftime=reftime, fitsfile=fitsfile, verbose=True, overwrite=True)
print 'fits file ' + fitsfile + ' selected'
ax4.cla()
ax5.cla()
ax6.cla()
ax7.cla()
rfits = fitsfile
try:
hdulist = fits.open(rfits)
hdu = hdulist[0]
(npol, nf, nx, ny) = hdu.data.shape
rmap = smap.Map(hdu.data[0, 0, :, :], hdu.header)
except:
print 'radio fits file not recognized by sunpy.map. Aborting...'
return -1
if npol > 1:
rmap1 = smap.Map(hdu.data[0, 0, :, :], hdu.header)
rmap2 = smap.Map(hdu.data[1, 0, :, :], hdu.header)
XX, YY = np.meshgrid(np.arange(rmap.data.shape[1]), np.arange(rmap.data.shape[0]))
try:
rmapx, rmapy = rmap.pixel_to_data(XX * u.pix, YY * u.pix)
except:
rmapxy = rmap.pixel_to_data(XX * u.pix, YY * u.pix)
rmapx = rmapxy.Tx
rmapy = rmapxy.Ty
if not xyrange:
if xycen:
x0 = xycen[0] * u.arcsec
y0 = xycen[1] * u.arcsec
if not xycen:
row, col = rmap1.data.shape
positon = np.nanargmax(rmap1.data)
m, n = divmod(positon, col)
x0 = rmap1.xrange[0] + rmap1.scale[1] * (n + 0.5) * u.pix
y0 = rmap1.yrange[0] + rmap1.scale[0] * (m + 0.5) * u.pix
if len(fov) == 1:
fov=[fov]*2
sz_x = fov[0] * u.arcsec
sz_y = fov[1] * u.arcsec
x1 = x0 - sz_x/2.
x2 = x0 + sz_x/2.
y1 = y0 - sz_y/2.
y2 = y0 + sz_y/2.
xyrange = [[x1.value, x2.value], [y1.value, y2.value]]
else:
sz_x = xyrange[0][1] - xyrange[0][0]
sz_y = xyrange[1][1] - xyrange[1][0]
clevels1 = np.linspace(0.2, 0.9, 5)
if stokes.split(',')[1] == 'V':
clevels2 = np.array([0.8, -0.6, -0.4, -0.2, 0.2, 0.4, 0.6, 0.8])
else:
clevels2 = np.linspace(0.2, 0.9, 5)
if 'aiamap' in vars():
aiamap.plot_settings['cmap'] = plt.get_cmap('binary')
if rmap:
title = 'AIA {0:.0f} + {1} {2:6.3f} GHz'.format(aiamap.wavelength.value, observatory, (bfreqghz + efreqghz) / 2.0)
else:
title = 'AIA {0:.0f}'.format(aiamap.wavelength.value)
aiamap.plot(axes=ax4)
ax4.set_title(title + ' ' + stokes.split(',')[0], fontsize=12)
aiamap.draw_limb()
aiamap.draw_grid()
aiamap.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
aiamap.plot(axes=ax6)
ax6.set_title(title + ' ' + stokes.split(',')[1], fontsize=12)
aiamap.draw_limb()
aiamap.draw_grid()
aiamap.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
if rmap:
ax4.contour(rmapx.value, rmapy.value, rmap1.data, levels=clevels1 * np.nanmax(rmap1.data), cmap=cm.jet)
ax6.contour(rmapx.value, rmapy.value, rmap2.data, levels=clevels2 * np.nanmax(rmap2.data), cmap=cm.RdBu)
ax4.text(0.02, 0.02, 'AIA {0:.0f} '.format(aiamap.wavelength.value) + aiamap.date.strftime('%H:%M:%S'),
verticalalignment='bottom', horizontalalignment='left', transform=ax4.transAxes, color='k',
fontsize=10)
ax6.text(0.02, 0.02, 'AIA {0:.0f} '.format(aiamap.wavelength.value) + aiamap.date.strftime('%H:%M:%S'),
verticalalignment='bottom', horizontalalignment='left', transform=ax6.transAxes, color='k',
fontsize=10)
else:
title = '{0} {1:6.3f} GHz'.format(observatory, (bfreqghz + efreqghz) / 2.0)
rmap1.plot(axes=ax4, cmap=cm.jet)
ax4.set_title(title + ' ' + stokes.split(',')[0], fontsize=12)
rmap1.draw_limb()
rmap1.draw_grid()
rmap1.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
rmap2.plot(axes=ax6, cmap=cm.RdBu)
ax6.set_title(title + ' ' + stokes.split(',')[1], fontsize=12)
rmap2.draw_limb()
rmap2.draw_grid()
# ax4.contour(rmapx.value, rmapy.value, rmap1.data, levels=np.linspace(0.2, 0.9, 5) * np.nanmax(rmap1.data),
# cmap=cm.gray)
# ax6.contour(rmapx.value, rmapy.value, rmap2.data, levels=np.linspace(0.2, 0.9, 5) * np.nanmax(rmap2.data),
# cmap=cm.gray)
rmap2.draw_rectangle((xyrange[0][0], xyrange[1][0]) * u.arcsec, sz_x, sz_y)
ax4.set_xlim(-1200, 1200)
ax4.set_ylim(-1200, 1200)
ax6.set_xlim(-1200, 1200)
ax6.set_ylim(-1200, 1200)
try:
subrmap1 = rmap1.submap(xyrange[0] * u.arcsec, xyrange[1] * u.arcsec)
subrmap2 = rmap2.submap(xyrange[0] * u.arcsec, xyrange[1] * u.arcsec)
except:
bl = SkyCoord(xyrange[0][0] * u.arcsec, xyrange[1][0] * u.arcsec, frame=rmap1.coordinate_frame)
tr = SkyCoord(xyrange[0][1] * u.arcsec, xyrange[1][1] * u.arcsec, frame=rmap1.coordinate_frame)
subrmap1 = rmap1.submap(bl, tr)
subrmap2 = rmap2.submap(bl, tr)
XX, YY = np.meshgrid(np.arange(subrmap1.data.shape[1]), np.arange(subrmap1.data.shape[0]))
try:
subrmapx, subrmapy = subrmap1.pixel_to_data(XX * u.pix, YY * u.pix)
except:
subrmapxy = subrmap1.pixel_to_data(XX * u.pix, YY * u.pix)
subrmapx = subrmapxy.Tx
subrmapy = subrmapxy.Ty
if 'aiamap' in vars():
try:
subaiamap = aiamap.submap(xyrange[0] * u.arcsec, xyrange[1] * u.arcsec)
except:
bl = SkyCoord(xyrange[0][0] * u.arcsec, xyrange[1][0] * u.arcsec, frame=aiamap.coordinate_frame)
tr = SkyCoord(xyrange[0][1] * u.arcsec, xyrange[1][1] * u.arcsec, frame=aiamap.coordinate_frame)
subaiamap = aiamap.submap(bl, tr)
subaiamap.plot(axes=ax5, title='')
subaiamap.draw_limb()
subaiamap.draw_grid()
subaiamap.plot(axes=ax7, title='')
subaiamap.draw_limb()
subaiamap.draw_grid()
ax5.contour(subrmapx.value, subrmapy.value, subrmap1.data, levels=clevels1 * np.nanmax(subrmap1.data), cmap=cm.jet)
ax7.contour(subrmapx.value, subrmapy.value, subrmap2.data, levels=clevels2 * np.nanmax(subrmap2.data),
cmap=cm.RdBu) # subaiamap.draw_rectangle((fov[0][0], fov[1][0]) * u.arcsec, 400 * u.arcsec, 400 * u.arcsec)
else:
subrmap1.plot(axes=ax5, cmap=cm.jet, title='')
subrmap1.draw_limb()
subrmap1.draw_grid()
subrmap2.plot(axes=ax7, cmap=cm.RdBu, title='')
subrmap2.draw_limb()
subrmap2.draw_grid() # ax5.contour(subrmapx.value, subrmapy.value, subrmap1.data, # levels=clevels1 * np.nanmax(subrmap1.data), cmap=cm.gray) # ax7.contour(subrmapx.value, subrmapy.value, subrmap2.data, # levels=clevels2 * np.nanmax(subrmap2.data), cmap=cm.gray) # subrmap1.draw_rectangle((fov[0][0], fov[1][0]) * u.arcsec, 400 * u.arcsec, 400 * u.arcsec) # subrmap2.draw_rectangle((fov[0][0], fov[1][0]) * u.arcsec, 400 * u.arcsec, 400 * u.arcsec)
ax5.set_xlim(xyrange[0])
ax5.set_ylim(xyrange[1])
ax5.text(0.02, 0.02, observatory + ' ' + rmap.date.strftime('%H:%M:%S.%f')[:-3], verticalalignment='bottom',
horizontalalignment='left', transform=ax5.transAxes, color='k', fontsize=10)
ax7.set_xlim(xyrange[0])
ax7.set_ylim(xyrange[1])
ax7.text(0.02, 0.02, observatory + ' ' + rmap.date.strftime('%H:%M:%S.%f')[:-3], verticalalignment='bottom',
horizontalalignment='left', transform=ax7.transAxes, color='k', fontsize=10)
fig.show()
tb.close()
timerange = '{0}~{1}'.format(
starttim.iso.replace('-', '/').replace(' ', '/'),
endtim.iso.replace('-', '/').replace(' ', '/'))
cur_imagefile = []
cur_fitsfile = []
vis = '/srg/ywei/data/eovsa/sep6_dofinal/IDB20170906T190319-195320.ms.corrected'
for sp in range(4):
index = 0
cur_imagefile.append('slf_192920.spw' + str(sp + index * 4 + 1).zfill(2) +
'.slfcal0.image/')
cur_fitsfile.append(tdir + 'slf_192920.spw' +
str(sp + index * 4 + 1).zfill(2) + '.slfcal0.fits')
print cur_imagefile[0]
hf.imreg(vis=vis,
imagefile=cur_imagefile[0],
fitsfile=cur_fitsfile[0],
timerange=timerange)
cur_imagefile.append('slf_192920.spw' + str(sp + index * 4 + 1).zfill(2) +
'.slfcal1.image/')
cur_fitsfile.append(tdir + 'slf_192920.spw' +
str(sp + index * 4 + 1).zfill(2) + '.slfcal1.fits')
hf.imreg(vis=vis,
imagefile=cur_imagefile[1],
fitsfile=cur_fitsfile[1],
timerange=timerange)
cur_imagefile.append('slf_192920.spw' + str(sp + index * 4 + 1).zfill(2) +
'.slfcal2.image/')
cur_fitsfile.append(tdir + 'slf_192920.spw' +
str(sp + index * 4 + 1).zfill(2) + '.slfcal2.fits')
hf.imreg(vis=vis,
imagefile=cur_imagefile[2],
