def pipeline(year=None,
month=None,
day=None,
ndays=1,
clearcache=True,
overwrite=True,
doimport=True,
pols='XX'):
workdir = '/data1/workdir/'
os.chdir(workdir)
# Set to run 5 days earlier than the current date
if year is None:
mjdnow = Time.now().mjd
t = Time(mjdnow - 2, format='mjd')
else:
# Uncomment below and set date to run for a given date
t = Time('{}-{:02d}-{:02d} 20:00'.format(year, month, day))
for d in range(ndays):
t1 = Time(t.mjd - d, format='mjd')
datestr = t1.iso[:10]
subdir = os.path.join(workdir, t1.datetime.strftime('%Y%m%d/'))
if not os.path.exists(subdir):
os.makedirs(subdir)
else:
os.system('rm -rf {}/*'.format(subdir))
vis_corrected = calib_pipeline(datestr,
overwrite=overwrite,
doimport=doimport,
workdir=subdir,
clearcache=False,
pols=pols)
if clearcache:
os.chdir(workdir)
os.system('rm -rf {}'.format(subdir))
def qlook_image_pipeline(date, twidth=10, ncpu=15, doimport=False, docalib=False):
''' date: date string or Time object. e.g., '2017-07-15' or Time('2017-07-15')
'''
import pytz
from datetime import datetime
if date is None:
date = Time.now()
try:
date = Time(date)
except:
print('date format not recognised. Abort....')
return None
qlookfitsdir = os.getenv('EOVSAQLOOKFITS')
qlookfigdir = os.getenv('EOVSAQLOOKFIG')
if not qlookfitsdir:
qlookfitsdir='/data1/eovsa/qlookfits/'
if not qlookfigdir:
qlookfigdir='/common/webplots/qlookimg_10m/'
imagedir=qlookfitsdir
if docalib:
vis=calib_pipeline(date,doimport=doimport)
imres=mk_qlook_image(date, twidth=twidth, ncpu=ncpu, doimport=doimport, docalib=docalib, imagedir=imagedir,verbose=True)
figdir=qlookfigdir
plt_qlook_image(imres,figdir=figdir,verbose=True)
def pipeline1(year=None,
month=None,
day=None,
clearcache=True,
overwrite=True,
doimport=True):
from suncasa.eovsa import eovsa_pipeline as ep
import os
from eovsapy.util import Time
workdir = '/data1/workdir/'
os.chdir(workdir)
# Set to run 5 days earlier than the current date
if year is None:
mjdnow = Time.now().mjd
t = Time(mjdnow - 2, format='mjd')
else:
# Uncomment below and set date to run for a given date
t = Time('{}-{:02d}-{:02d} 20:00'.format(year, month, day))
print(t.iso)
datestr = t.iso[:10]
subdir = t.datetime.strftime('%Y%m%d/')
if not os.path.exists(subdir):
os.makedirs(subdir)
vis_corrected = ep.calib_pipeline(datestr,
overwrite=overwrite,
doimport=doimport,
workdir=os.path.join(workdir, subdir))
if clearcache:
os.chdir(workdir)
os.system('rm -rf ' + subdir)
def qlook_image_pipeline(date,
twidth=10,
ncpu=15,
doimport=False,
docalib=False,
synoptic=False):
''' date: date string or Time object. e.g., '2017-07-15' or Time('2017-07-15')
'''
import pytz
from datetime import datetime
if date is None:
date = Time.now()
try:
date = Time(date)
except:
print('date format not recognised. Abort....')
return None
qlookfitsdir = os.getenv('EOVSAQLOOKFITS')
qlookfigdir = os.getenv('EOVSAQLOOKFIG')
synopticfigdir = os.getenv('EOVSASYNOPTICFIG')
if not qlookfitsdir:
qlookfitsdir = '/data1/eovsa/fits/qlook_10m/'
if not qlookfigdir:
qlookfigdir = '/common/webplots/qlookimg_10m/'
if not synopticfigdir:
synopticfigdir = '/common/webplots/SynopticImg/'
imagedir = qlookfitsdir
if synoptic:
vis_synoptic = os.path.join(
udbmsdir, date.datetime.strftime("%Y%m"),
'UDB' + date.datetime.strftime("%Y%m%d") + '.ms')
if os.path.exists(vis_synoptic):
date = vis_synoptic
else:
print(
'Whole-day ms file {} not existed. About..... Use pipeline1.py to make one.'
.format(vis_synoptic))
return None
if docalib:
vis = calib_pipeline(date, doimport=doimport, synoptic=synoptic)
imres = mk_qlook_image(date,
twidth=twidth,
ncpu=ncpu,
doimport=doimport,
docalib=docalib,
imagedir=imagedir,
verbose=True)
figdir = qlookfigdir
plt_qlook_image(imres, figdir=figdir, verbose=True)
if imres['Synoptic']['Succeeded']:
figdir = synopticfigdir
plt_qlook_image(imres['Synoptic'],
figdir=figdir,
verbose=True,
synoptic=True)
def qlook_image_pipeline(date,
twidth=10,
ncpu=15,
doimport=False,
docalib=False,
synoptic=False,
overwrite=True):
''' date: date string or Time object. e.g., '2017-07-15' or Time('2017-07-15')
'''
import pytz
from datetime import datetime
if date is None:
date = Time.now()
try:
date = Time(date)
except:
print('date format not recognised. Abort....')
return None
if date.mjd > Time('2019-02-02 12:00:00').mjd:
## the last '' window is for fullBD synthesis image. Now obsolete.
# spws = ['6~10', '11~20', '21~30', '31~43', '']
# spws = ['6~10', '11~20', '21~30', '31~43']
spws = ['0~1', '2~5', '6~10', '11~20', '21~30', '31~49']
else:
## the last '' window is for fullBD synthesis image. Now obsolete.
# spws = ['1~5', '6~10', '11~15', '16~25', '']
spws = ['1~3', '4~6', '7~10', '10~14', '15~20', '21~30']
if docalib:
vis = calib_pipeline(date, doimport=doimport, synoptic=synoptic)
imagedir = qlookfitsdir
if synoptic:
vis_synoptic = os.path.join(
udbmsdir, date.datetime.strftime("%Y%m"),
'UDB' + date.datetime.strftime("%Y%m%d") + '.ms')
if os.path.exists(vis_synoptic):
date = vis_synoptic
else:
print(
'Whole-day ms file {} not existed. About..... Use pipeline to make one.'
.format(vis_synoptic))
return None
imres = mk_qlook_image(date,
twidth=twidth,
ncpu=ncpu,
doimport=doimport,
docalib=docalib,
imagedir=imagedir,
spws=spws,
verbose=True,
overwrite=overwrite)
figdir = qlookfigdir
plt_qlook_image(imres, figdir=figdir, verbose=True)
def calib_pipeline(trange,doimport=False):
'''
trange: can be 1) a single Time() object: use the entire day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) a single or a list of UDBms file(s)
4) None -- use current date Time.now()
'''
if type(trange) == Time:
mslist = trange2ms(trange=trange, doimport=doimport)
invis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts]
if type(trange) == str:
try:
date = Time(trange)
mslist = trange2ms(trange=trange, doimport=doimport)
invis = mslist['ms']
except:
invis = [trange]
subdir = ['/']
for idx, f in enumerate(invis):
if f[-1] == '/':
invis[idx] = f[:-1]
vis=calibeovsa.calibeovsa(invis, caltype=['refpha','phacal'], interp='nearest',
doflag=True, flagant='13~15', doimage=False, doconcat=False)
return vis
def trange2ms(trange=None,
doimport=False,
verbose=False,
doscaling=False,
overwrite=True):
'''This finds all solar UDBms files within a timerange; If the UDBms file does not exist
in EOVSAUDBMSSCL, create one by calling importeovsa
Required inputs:
trange - can be 1) a single string or Time() object in UTC: use the entire day, e.g., '2017-08-01' or Time('2017-08-01')
if just a date, find all scans withing the same date in local time.
if a complete time stamp, find the local date first (which may be different from that provided,
and return all scans within that day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) None -- use current date Time.now()
doimport - Boolean. If true, call importeovsa to import UDB files that are missing from
those found in the directory specified in EOVSAUDBMSSCL. Otherwise, return
a list of ms files it has found.
doscaling - Boolean. If true, scale cross-correlation amplitudes by using auto-correlations
verbose - Boolean. If true, return more information
'''
import glob
if trange is None:
trange = Time.now()
if type(trange) == list or type(trange) == str:
try:
trange = Time(trange)
except:
print('trange format not recognised. Abort....')
return None
# if type(trange) == Time:
try:
# if single Time object, the following line would report an error
nt = len(trange)
if len(trange) > 1:
# more than one value
trange = Time([trange[0], trange[-1]])
tdatetime = trange[0].to_datetime()
else:
# single value in a list
if trange[0].mjd == np.fix(trange[0].mjd):
# if only date is given, move the time from 00 to 12 UT
trange[0] = Time(trange[0].mjd + 0.5, format='mjd')
tdatetime = trange[0].to_datetime()
dhr = trange[0].LocalTime.utcoffset().total_seconds(
) / 60 / 60 / 24
btime = Time(np.fix(trange[0].mjd + dhr) - dhr, format='mjd')
etime = Time(btime.mjd + 1, format='mjd')
trange = Time([btime, etime])
except:
# the case of a single Time object
if trange.mjd == np.fix(trange.mjd):
# if only date is given, move the time from 00 to 12 UT
trange = Time(trange.mjd + 0.5, format='mjd')
tdatetime = trange.to_datetime()
dhr = trange.LocalTime.utcoffset().total_seconds() / 60 / 60 / 24
btime = Time(np.fix(trange.mjd + dhr) - dhr, format='mjd')
etime = Time(btime.mjd + 1, format='mjd')
trange = Time([btime, etime])
print('Selected timerange in UTC: ', trange.iso)
if doscaling:
udbmspath = udbmsscldir
else:
udbmspath = udbmsdir
inpath = '{}{}/'.format(udbdir, tdatetime.strftime("%Y"))
outpath = '{}{}/'.format(udbmspath, tdatetime.strftime("%Y%m"))
if not os.path.exists(outpath):
if verbose:
print(outpath + ' does not exist. Making a new directory.')
os.makedirs(outpath)
msfiles = []
else:
msfiles = [
os.path.basename(ll).split('.')[0]
for ll in glob.glob('{}UDB*.ms'.format(outpath))
]
msfile_synoptic = os.path.join(
outpath, 'UDB' + tdatetime.strftime("%Y%m%d") + '.ms')
if overwrite and doimport:
if os.path.exists(msfile_synoptic):
os.system('rm -rf {}'.format(msfile_synoptic))
sclist = ra.findfiles(trange, projid='NormalObserving', srcid='Sun')
udbfilelist = sclist['scanlist']
udbfilelist = [os.path.basename(ll) for ll in udbfilelist]
if os.path.exists(msfile_synoptic):
return {
'mspath': outpath,
'udbpath': inpath,
'udbfile': sorted(udbfilelist),
'udb2ms': [],
'ms': [msfile_synoptic],
'tstlist': sclist['tstlist'],
'tedlist': sclist['tedlist']
}
else:
udbfilelist_set = set(udbfilelist)
msfiles = udbfilelist_set.intersection(msfiles)
filelist = udbfilelist_set - msfiles
filelist = sorted(list(filelist))
if filelist and doimport:
import multiprocessing as mprocs
# ncpu = mprocs.cpu_count()
# if ncpu > 10:
# ncpu = 10
# if ncpu > len(filelist):
# ncpu = len(filelist)
ncpu = 1
timporteovsa.importeovsa(idbfiles=[inpath + ll for ll in filelist],
ncpu=ncpu,
timebin="0s",
width=1,
visprefix=outpath,
nocreatms=False,
doconcat=False,
modelms="",
doscaling=doscaling,
keep_nsclms=False,
udb_corr=True)
msfiles = [
os.path.basename(ll).split('.')[0]
for ll in glob.glob('{}UDB*.ms'.format(outpath))
]
udbfilelist_set = set(udbfilelist)
msfiles = udbfilelist_set.intersection(msfiles)
filelist = udbfilelist_set - msfiles
filelist = sorted(list(filelist))
return {
'mspath': outpath,
'udbpath': inpath,
'udbfile': sorted(udbfilelist),
'udb2ms': filelist,
'ms': [outpath + ll + '.ms' for ll in sorted(list(msfiles))],
'tstlist': sclist['tstlist'],
'tedlist': sclist['tedlist']
}
def plt_qlook_image(imres, figdir=None, verbose=True, synoptic=False):
from matplotlib import pyplot as plt
from sunpy import map as smap
from sunpy import sun
from matplotlib import colors
import astropy.units as u
from suncasa.utils import plot_mapX as pmX
# from matplotlib import gridspec as gridspec
if not figdir:
figdir = './'
nspw = len(set(imres['Spw']))
plttimes = list(set(imres['BeginTime']))
ntime = len(plttimes)
# sort the imres according to time
images = np.array(imres['ImageName'])
btimes = Time(imres['BeginTime'])
etimes = Time(imres['EndTime'])
spws = np.array(imres['Spw'])
suc = np.array(imres['Succeeded'])
inds = btimes.argsort()
images_sort = images[inds].reshape(ntime, nspw)
btimes_sort = btimes[inds].reshape(ntime, nspw)
suc_sort = suc[inds].reshape(ntime, nspw)
if verbose:
print('{0:d} figures to plot'.format(ntime))
plt.ioff()
fig = plt.figure(figsize=(8, 8))
plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
axs = []
ims = []
pltst = 0
for i in range(ntime):
plt.ioff()
plttime = btimes_sort[i, 0]
tofd = plttime.mjd - np.fix(plttime.mjd)
suci = suc_sort[i]
if not synoptic:
if tofd < 16. / 24. or sum(
suci
) < nspw - 2: # if time of the day is before 16 UT (and 24 UT), skip plotting (because the old antennas are not tracking)
continue
else:
if pltst == 0:
i0 = i
pltst = 1
else:
if pltst == 0:
i0 = i
pltst = 1
if i == i0:
if synoptic:
timetext = fig.text(0.01,
0.98,
plttime.iso[:10],
color='w',
fontweight='bold',
fontsize=12,
ha='left')
else:
timetext = fig.text(0.01,
0.98,
plttime.iso[:19],
color='w',
fontweight='bold',
fontsize=12,
ha='left')
else:
if synoptic:
timetext.set_text(plttime.iso[:10])
else:
timetext.set_text(plttime.iso[:19])
if verbose:
print('Plotting image at: ', plttime.iso)
for n in range(nspw):
plt.ioff()
if i == i0:
if nspw == 1:
ax = fig.add_subplot(111)
else:
ax = fig.add_subplot(nspw / 2, 2, n + 1)
axs.append(ax)
else:
ax = axs[n]
image = images_sort[i, n]
if suci[n] or os.path.exists(image):
try:
eomap = smap.Map(image)
except:
continue
data = eomap.data
sz = data.shape
if len(sz) == 4:
data = data.reshape((sz[2], sz[3]))
data[np.isnan(data)] = 0.0
# add a basin flux to the image to avoid negative values
data = data + 0.8e5
data[data < 0] = 0.0
data = np.sqrt(data)
eomap = smap.Map(data, eomap.meta)
# resample the image for plotting
dim = u.Quantity([256, 256], u.pixel)
eomap = eomap.resample(dim)
else:
# make an empty map
data = np.zeros((256, 256))
header = {
"DATE-OBS": plttime.isot,
"EXPTIME": 0.,
"CDELT1": 10.,
"NAXIS1": 256,
"CRVAL1": 0.,
"CRPIX1": 128.5,
"CUNIT1": "arcsec",
"CTYPE1": "HPLN-TAN",
"CDELT2": 10.,
"NAXIS2": 256,
"CRVAL2": 0.,
"CRPIX2": 128.5,
"CUNIT2": "arcsec",
"CTYPE2": "HPLT-TAN",
"HGLT_OBS":
sun.heliographic_solar_center(plttime)[1].value,
"HGLN_OBS": 0.,
"RSUN_OBS":
sun.solar_semidiameter_angular_size(plttime).value,
"RSUN_REF": sun.constants.radius.value,
"DSUN_OBS":
sun.sunearth_distance(plttime).to(u.meter).value,
}
eomap = smap.Map(data, header)
if i == i0:
eomap_ = pmX.Sunmap(eomap)
# im = eomap_.imshow(axes=ax, cmap='jet', norm=colors.LogNorm(vmin=0.1, vmax=1e8))
im = eomap_.imshow(axes=ax,
cmap='jet',
norm=colors.Normalize(vmin=150, vmax=700))
ims.append(im)
if not synoptic:
eomap_.draw_limb(axes=ax)
eomap_.draw_grid(axes=ax)
ax.set_xlim([-1080, 1080])
ax.set_ylim([-1080, 1080])
try:
cfreq = eomap.meta['crval3'] / 1.0e9
bdwid = eomap.meta['cdelt3'] / 1.0e9
ax.text(0.98,
0.01,
'{0:.1f} - {1:.1f} GHz'.format(
cfreq - bdwid / 2.0, cfreq + bdwid / 2.0),
color='w',
transform=ax.transAxes,
fontweight='bold',
ha='right')
except:
pass
ax.set_title(' ')
ax.set_xlabel('')
ax.set_ylabel('')
ax.set_xticklabels([''])
ax.set_yticklabels([''])
else:
ims[n].set_data(eomap.data)
fig_tdt = plttime.to_datetime()
if synoptic:
fig_subdir = fig_tdt.strftime("%Y/")
figname = 'eovsa_qlimg_' + plttime.iso[:10].replace('-',
'') + '.png'
else:
fig_subdir = fig_tdt.strftime("%Y/%m/%d/")
figname = 'eovsa_qlimg_' + plttime.isot.replace(':', '').replace(
'-', '')[:15] + '.png'
figdir_ = figdir + fig_subdir
if not os.path.exists(figdir_):
os.makedirs(figdir_)
if verbose:
print('Saving plot to :' + figdir_ + figname)
plt.savefig(figdir_ + figname)
plt.close(fig)
def mk_qlook_image(trange,
doimport=False,
docalib=False,
ncpu=10,
twidth=12,
stokes=None,
antenna='0~12',
lowcutoff_freq=3.7,
imagedir=None,
spws=['1~5', '6~10', '11~15', '16~25'],
toTb=True,
overwrite=True,
doslfcal=False,
verbose=False):
'''
trange: can be 1) a single Time() object: use the entire day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) a single or a list of UDBms file(s)
4) None -- use current date Time.now()
'''
antenna0 = antenna
if type(trange) == Time:
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
if type(trange) == str:
try:
date = Time(trange)
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
except:
vis = [trange]
tsts = []
for v in vis:
tb.open(v + '/OBSERVATION')
tsts.append(
Time(tb.getcell('TIME_RANGE')[0] / 24 / 3600,
format='mjd').datetime)
tb.close()
subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not stokes:
stokes = 'XX'
if not imagedir:
imagedir = './'
imres = {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': [],
'Synoptic': {
'Succeeded': [],
'BeginTime': [],
'EndTime': [],
'ImageName': [],
'Spw': [],
'Vis': []
}
}
for n, msfile in enumerate(vis):
msfilebs = os.path.basename(msfile)
imdir = imagedir + subdir[n]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './' + msfilebs + '.xx'
split(msfile,
outputvis=slfcalms,
datacolumn='corrected',
correlation='XX')
cfreqs = getspwfreq(msfile)
for spw in spws:
antenna = antenna0
if spw == '':
continue
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = '<10klambda'
if doslfcal:
slfcal_img = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slfimg'
slfcal_tb = './' + msfilebs + '.slf.spw' + spw.replace(
'~', '-') + '.slftb'
try:
clean(vis=slfcalms,
antenna=antenna,
imagename=slfcal_img,
spw=spw,
mode='mfs',
timerange='',
imagermode='csclean',
psfmode='clark',
imsize=[512, 512],
cell=['5arcsec'],
niter=100,
gain=0.05,
stokes='I',
weighting='natural',
restoringbeam=[str(bmsz) + 'arcsec'],
pbcor=False,
interactive=False,
usescratch=True)
except:
print('error in cleaning spw: ' + spw)
break
gaincal(vis=slfcalms,
refant='0',
antenna=antenna,
caltable=slfcal_tb,
spw=spw,
uvrange='',
gaintable=[],
selectdata=True,
timerange='',
solint='600s',
gaintype='G',
calmode='p',
combine='',
minblperant=3,
minsnr=2,
append=False)
if not os.path.exists(slfcal_tb):
print('No solution found in spw: ' + spw)
break
else:
clearcal(slfcalms)
delmod(slfcalms)
applycal(vis=slfcalms,
gaintable=[slfcal_tb],
spw=spw,
selectdata=True,
antenna=antenna,
interp='nearest',
flagbackup=False,
applymode='calonly',
calwt=False)
msfile = slfcalms
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.spw' + spwstr.replace('~', '-')
if cfreq > 10.:
antenna = antenna + ';!0&1;!0&2' # deselect the shortest baselines
# else:
# antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=twidth,
uvrange=uvrange,
spw=spw,
ncpu=ncpu,
niter=1000,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr] * len(res['ImageName'])
imres['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
if len(vis) == 1:
# produce the band-by-band whole-day images
ms.open(msfile)
ms.selectinit()
timfreq = ms.getdata(['time', 'axis_info'], ifraxis=True)
tim = timfreq['time']
ms.close()
cfreqs = getspwfreq(msfile)
imdir = imagedir + subdir[0]
if not os.path.exists(imdir):
os.makedirs(imdir)
for spw in spws:
antenna = antenna0
if spw == '':
spw = '{:d}~{:d}'.format(
next(x[0] for x in enumerate(cfreqs)
if x[1] > lowcutoff_freq),
len(cfreqs) - 1)
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [cfreqs[int(s)] for s in spw.split('~')]
cfreq = np.mean(freqran)
bmsz = max(150. / cfreq, 20.)
uvrange = ''
imsize = 512
cell = ['5arcsec']
if len(spwran) == 2:
spwstr = spwran[0] + '~' + spwran[1]
else:
spwstr = spwran[0]
restoringbeam = ['{0:.1f}arcsec'.format(bmsz)]
imagesuffix = '.synoptic.spw' + spwstr.replace('~', '-')
antenna = antenna + ';!0&1' # deselect the shortest baselines
res = ptclean3(vis=msfile,
imageprefix=imdir,
imagesuffix=imagesuffix,
twidth=len(tim),
uvrange=uvrange,
spw=spw,
ncpu=1,
niter=0,
gain=0.05,
antenna=antenna,
imsize=imsize,
cell=cell,
stokes=stokes,
doreg=True,
usephacenter=False,
overwrite=overwrite,
toTb=toTb,
restoringbeam=restoringbeam,
specmode="mfs",
deconvolver="hogbom",
datacolumn='data',
pbcor=True)
if res:
imres['Synoptic']['Succeeded'] += res['Succeeded']
imres['Synoptic']['BeginTime'] += res['BeginTime']
imres['Synoptic']['EndTime'] += res['EndTime']
imres['Synoptic']['ImageName'] += res['ImageName']
imres['Synoptic']['Spw'] += [spwstr] * len(res['ImageName'])
imres['Synoptic']['Vis'] += [msfile] * len(res['ImageName'])
else:
continue
# save it for debugging purposes
np.savez('imres.npz', imres=imres)
return imres
def calibeovsa(vis=None,
caltype=None,
interp=None,
docalib=True,
doflag=True,
flagant=None,
doimage=False,
imagedir=None,
antenna=None,
timerange=None,
spw=None,
stokes=None,
doconcat=False,
msoutdir=None,
keep_orig_ms=True):
'''
:param vis: EOVSA visibility dataset(s) to be calibrated
:param caltype:
:param interp:
:param docalib:
:param qlookimage:
:param flagant:
:param stokes:
:param doconcat:
:return:
'''
if type(vis) == str:
vis = [vis]
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
for msfile in vis:
casalog.origin('calibeovsa')
if not caltype:
casalog.post(
"Caltype not provided. Perform reference phase calibration and daily phase calibration."
)
caltype = [
'refpha', 'phacal', 'fluxcal'
] ## use this line after the phacal is applied # caltype = ['refcal']
if not os.path.exists(msfile):
casalog.post("Input visibility does not exist. Aborting...")
continue
if msfile.endswith('/'):
msfile = msfile[:-1]
if not msfile[-3:] in ['.ms', '.MS']:
casalog.post(
"Invalid visibility. Please provide a proper visibility file ending with .ms"
)
# if not caltable:
# caltable=[os.path.basename(vis).replace('.ms','.'+c) for c in caltype]
# get band information
tb.open(msfile + '/SPECTRAL_WINDOW')
nspw = tb.nrows()
bdname = tb.getcol('NAME')
bd_nchan = tb.getcol('NUM_CHAN')
bd = [int(b[4:]) - 1 for b in bdname] # band index from 0 to 33
# nchans = tb.getcol('NUM_CHAN')
# reffreqs = tb.getcol('REF_FREQUENCY')
# cenfreqs = np.zeros((nspw))
tb.close()
tb.open(msfile + '/ANTENNA')
nant = tb.nrows()
antname = tb.getcol('NAME')
antlist = [str(ll) for ll in range(len(antname) - 1)]
antennas = ','.join(antlist)
tb.close()
# get time stamp, use the beginning of the file
tb.open(msfile + '/OBSERVATION')
trs = {'BegTime': [], 'EndTime': []}
for ll in range(tb.nrows()):
tim0, tim1 = Time(tb.getcell('TIME_RANGE', ll) / 24 / 3600,
format='mjd')
trs['BegTime'].append(tim0)
trs['EndTime'].append(tim1)
tb.close()
trs['BegTime'] = Time(trs['BegTime'])
trs['EndTime'] = Time(trs['EndTime'])
btime = np.min(trs['BegTime'])
etime = np.max(trs['EndTime'])
# ms.open(vis)
# summary = ms.summary()
# ms.close()
# btime = Time(summary['BeginTime'], format='mjd')
# etime = Time(summary['EndTime'], format='mjd')
## stop using ms.summary to avoid conflicts with importeovsa
t_mid = Time((btime.mjd + etime.mjd) / 2., format='mjd')
print "This scan observed from {} to {} UTC".format(
btime.iso, etime.iso)
gaintables = []
if ('refpha' in caltype) or ('refamp' in caltype) or ('refcal'
in caltype):
refcal = ra.sql2refcalX(btime)
pha = refcal['pha'] # shape is 15 (nant) x 2 (npol) x 34 (nband)
pha[np.where(refcal['flag'] == 1)] = 0.
amp = refcal['amp']
amp[np.where(refcal['flag'] == 1)] = 1.
t_ref = refcal['timestamp']
# find the start and end time of the local day when refcal is registered
try:
dhr = t_ref.LocalTime.utcoffset().total_seconds() / 60. / 60.
except:
dhr = -7.
bt = Time(np.fix(t_ref.mjd + dhr / 24.) - dhr / 24., format='mjd')
et = Time(bt.mjd + 1., format='mjd')
(yr, mon, day) = (bt.datetime.year, bt.datetime.month,
bt.datetime.day)
dirname = caltbdir + str(yr) + str(mon).zfill(2) + '/'
if not os.path.exists(dirname):
os.mkdir(dirname)
# check if there is any ROACH reboot between the reference calibration found and the current data
t_rbts = db.get_reboot(Time([t_ref, btime]))
if not t_rbts:
casalog.post(
"Reference calibration is derived from observation at " +
t_ref.iso)
print "Reference calibration is derived from observation at " + t_ref.iso
else:
casalog.post(
"Oh crap! Roach reboot detected between the reference calibration time "
+ t_ref.iso + ' and the current observation at ' +
btime.iso)
casalog.post("Aborting...")
print "Oh crap! Roach reboot detected between the reference calibration time " + t_ref.iso + ' and the current observation at ' + btime.iso
print "Aborting..."
para_pha = []
para_amp = []
calpha = np.zeros((nspw, 15, 2))
calamp = np.zeros((nspw, 15, 2))
for s in range(nspw):
for n in range(15):
for p in range(2):
calpha[s, n, p] = pha[n, p, bd[s]]
calamp[s, n, p] = amp[n, p, bd[s]]
para_pha.append(np.degrees(pha[n, p, bd[s]]))
para_amp.append(amp[n, p, bd[s]])
if 'fluxcal' in caltype:
calfac = pc.get_calfac(Time(t_mid.iso.split(' ')[0] + 'T23:59:59'))
t_bp = Time(calfac['timestamp'], format='lv')
if int(t_mid.mjd) == int(t_bp.mjd):
accalfac = calfac['accalfac'] # (ant x pol x freq)
# tpcalfac = calfac['tpcalfac'] # (ant x pol x freq)
caltb_autoamp = dirname + t_bp.isot[:-4].replace(
':', '').replace('-', '') + '.bandpass'
if not os.path.exists(caltb_autoamp):
bandpass(vis=msfile,
caltable=caltb_autoamp,
solint='inf',
refant='eo01',
minblperant=0,
minsnr=0,
bandtype='B',
docallib=False)
tb.open(caltb_autoamp, nomodify=False) # (ant x spw)
bd_chanidx = np.hstack([[0], bd_nchan.cumsum()])
for ll in range(nspw):
antfac = np.sqrt(
accalfac[:, :, bd_chanidx[ll]:bd_chanidx[ll + 1]])
# # antfac *= tpcalfac[:, :,bd_chanidx[ll]:bd_chanidx[ll + 1]]
antfac = np.moveaxis(antfac, 0, 2)
cparam = np.zeros((2, bd_nchan[ll], nant))
cparam[:, :, :-3] = 1.0 / antfac
tb.putcol('CPARAM', cparam + 0j, ll * nant, nant)
paramerr = tb.getcol('PARAMERR', ll * nant, nant)
paramerr = paramerr * 0
tb.putcol('PARAMERR', paramerr, ll * nant, nant)
bpflag = tb.getcol('FLAG', ll * nant, nant)
bpant1 = tb.getcol('ANTENNA1', ll * nant, nant)
bpflagidx, = np.where(bpant1 >= 13)
bpflag[:] = False
bpflag[:, :, bpflagidx] = True
tb.putcol('FLAG', bpflag, ll * nant, nant)
bpsnr = tb.getcol('SNR', ll * nant, nant)
bpsnr[:] = 100.0
bpsnr[:, :, bpflagidx] = 0.0
tb.putcol('SNR', bpsnr, ll * nant, nant)
tb.close()
msg_prompt = "Scaling calibration is derived for {}.".format(
msfile)
casalog.post(msg_prompt)
print msg_prompt
gaintables.append(caltb_autoamp)
else:
msg_prompt = "Caution: No TPCAL is available on {}. No scaling calibration is derived for {}.".format(
t_mid.datetime.strftime('%b %d, %Y'), msfile)
casalog.post(msg_prompt)
print msg_prompt
if ('refpha' in caltype) or ('refcal' in caltype):
# caltb_pha = os.path.basename(vis).replace('.ms', '.refpha')
# check if the calibration table already exists
caltb_pha = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refpha'
if not os.path.exists(caltb_pha):
gencal(vis=msfile,
caltable=caltb_pha,
caltype='ph',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_pha)
gaintables.append(caltb_pha)
if ('refamp' in caltype) or ('refcal' in caltype):
# caltb_amp = os.path.basename(vis).replace('.ms', '.refamp')
caltb_amp = dirname + t_ref.isot[:-4].replace(':', '').replace(
'-', '') + '.refamp'
if not os.path.exists(caltb_amp):
gencal(vis=msfile,
caltable=caltb_amp,
caltype='amp',
antenna=antennas,
pol='X,Y',
spw='0~' + str(nspw - 1),
parameter=para_amp)
gaintables.append(caltb_amp)
# calibration for the change of delay center between refcal time and beginning of scan -- hopefully none!
xml, buf = ch.read_calX(4, t=[t_ref, btime], verbose=False)
if buf:
dly_t2 = Time(stf.extract(buf[0], xml['Timestamp']), format='lv')
dlycen_ns2 = stf.extract(buf[0], xml['Delaycen_ns'])[:15]
xml, buf = ch.read_calX(4, t=t_ref)
dly_t1 = Time(stf.extract(buf, xml['Timestamp']), format='lv')
dlycen_ns1 = stf.extract(buf, xml['Delaycen_ns'])[:15]
dlycen_ns_diff = dlycen_ns2 - dlycen_ns1
for n in range(2):
dlycen_ns_diff[:, n] -= dlycen_ns_diff[0, n]
print 'Multi-band delay is derived from delay center difference at {} & {}'.format(
dly_t1.iso, dly_t2.iso)
# print '=====Delays relative to Ant 14====='
# for i, dl in enumerate(dlacen_ns_diff[:, 0] - dlacen_ns_diff[13, 0]):
# ant = antlist[i]
# print 'Ant eo{0:02d}: x {1:.2f} ns & y {2:.2f} ns'.format(int(ant) + 1, dl
# dlacen_ns_diff[i, 1] - dlacen_ns_diff[13, 1])
# caltb_mbd0 = os.path.basename(vis).replace('.ms', '.mbd0')
caltb_dlycen = dirname + dly_t2.isot[:-4].replace(':', '').replace(
'-', '') + '.dlycen'
if not os.path.exists(caltb_dlycen):
gencal(vis=msfile,
caltable=caltb_dlycen,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=dlycen_ns_diff.flatten().tolist())
gaintables.append(caltb_dlycen)
if 'phacal' in caltype:
phacals = np.array(
ra.sql2phacalX([bt, et], neat=True, verbose=False))
if not phacals.any() or len(phacals) == 0:
print "Found no phacal records in SQL database, will skip phase calibration"
else:
# first generate all phacal calibration tables if not already exist
t_phas = Time([phacal['t_pha'] for phacal in phacals])
# sort the array in ascending order by t_pha
sinds = t_phas.mjd.argsort()
t_phas = t_phas[sinds]
phacals = phacals[sinds]
caltbs_phambd = []
for i, phacal in enumerate(phacals):
# filter out phase cals with reference time stamp >30 min away from the provided refcal time
if (phacal['t_ref'].jd -
refcal['timestamp'].jd) > 30. / 1440.:
del phacals[i]
del t_phas[i]
continue
else:
t_pha = phacal['t_pha']
phambd_ns = phacal['pslope']
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(phacal['flag'] == 1)] = 0.
caltb_phambd = dirname + t_pha.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
caltbs_phambd.append(caltb_phambd)
if not os.path.exists(caltb_phambd):
gencal(vis=msfile,
caltable=caltb_phambd,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
# now decides which table to apply depending on the interpolation method ("neatest" or "linear")
if interp == 'nearest':
tbind = np.argmin(np.abs(t_phas.mjd - t_mid.mjd))
dt = np.min(np.abs(t_phas.mjd - t_mid.mjd)) * 24.
print "Selected nearest phase calibration table at " + t_phas[
tbind].iso
gaintables.append(caltbs_phambd[tbind])
if interp == 'linear':
# bphacal = ra.sql2phacalX(btime)
# ephacal = ra.sql2phacalX(etime,reverse=True)
bt_ind, = np.where(t_phas.mjd < btime.mjd)
et_ind, = np.where(t_phas.mjd > etime.mjd)
if len(bt_ind) == 0 and len(et_ind) == 0:
print "No phacal found before or after the ms data within the day of observation"
print "Skipping daily phase calibration"
elif len(bt_ind) > 0 and len(et_ind) == 0:
gaintables.append(caltbs_phambd[bt_ind[-1]])
elif len(bt_ind) == 0 and len(et_ind) > 0:
gaintables.append(caltbs_phambd[et_ind[0]])
elif len(bt_ind) > 0 and len(et_ind) > 0:
bphacal = phacals[bt_ind[-1]]
ephacal = phacals[et_ind[0]]
# generate a new table interpolating between two daily phase calibrations
t_pha_mean = Time(np.mean(
[bphacal['t_pha'].mjd, ephacal['t_pha'].mjd]),
format='mjd')
phambd_ns = (bphacal['pslope'] +
ephacal['pslope']) / 2.
for n in range(2):
phambd_ns[:, n] -= phambd_ns[0, n]
# set all flagged values to be zero
phambd_ns[np.where(bphacal['flag'] == 1)] = 0.
phambd_ns[np.where(ephacal['flag'] == 1)] = 0.
caltb_phambd_interp = dirname + t_pha_mean.isot[:-4].replace(
':', '').replace('-', '') + '.phambd'
if not os.path.exists(caltb_phambd_interp):
gencal(vis=msfile,
caltable=caltb_phambd_interp,
caltype='mbd',
pol='X,Y',
antenna=antennas,
parameter=phambd_ns.flatten().tolist())
print "Using phase calibration table interpolated between records at " + bphacal[
't_pha'].iso + ' and ' + ephacal['t_pha'].iso
gaintables.append(caltb_phambd_interp)
if docalib:
clearcal(msfile)
applycal(vis=msfile,
gaintable=gaintables,
applymode='calflag',
calwt=False)
# delete the interpolated phase calibration table
try:
caltb_phambd_interp
except:
pass
else:
if os.path.exists(caltb_phambd_interp):
shutil.rmtree(caltb_phambd_interp)
if doflag:
# flag zeros and NaNs
flagdata(vis=msfile, mode='clip', clipzeros=True)
if flagant:
try:
flagdata(vis=msfile, antenna=flagant)
except:
print "Something wrong with flagant. Abort..."
if doimage:
from matplotlib import pyplot as plt
from suncasa.utils import helioimage2fits as hf
from sunpy import map as smap
if not antenna:
antenna = '0~12'
if not stokes:
stokes = 'XX'
if not timerange:
timerange = ''
if not spw:
spw = '1~3'
if not imagedir:
imagedir = '.'
#(yr, mon, day) = (bt.datetime.year, bt.datetime.month, bt.datetime.day)
#dirname = imagedir + str(yr) + '/' + str(mon).zfill(2) + '/' + str(day).zfill(2) + '/'
#if not os.path.exists(dirname):
# os.makedirs(dirname)
bds = [spw]
nbd = len(bds)
imgs = []
for bd in bds:
if '~' in bd:
bdstr = bd.replace('~', '-')
else:
bdstr = str(bd).zfill(2)
imname = imagedir + '/' + os.path.basename(msfile).replace(
'.ms', '.bd' + bdstr)
print 'Cleaning image: ' + imname
try:
clean(vis=msfile,
imagename=imname,
antenna=antenna,
spw=bd,
timerange=timerange,
imsize=[512],
cell=['5.0arcsec'],
stokes=stokes,
niter=500)
except:
print 'clean not successfull for band ' + str(bd)
else:
imgs.append(imname + '.image')
junks = ['.flux', '.mask', '.model', '.psf', '.residual']
for junk in junks:
if os.path.exists(imname + junk):
shutil.rmtree(imname + junk)
tranges = [btime.iso + '~' + etime.iso] * nbd
fitsfiles = [img.replace('.image', '.fits') for img in imgs]
hf.imreg(vis=msfile,
timerange=tranges,
imagefile=imgs,
fitsfile=fitsfiles,
usephacenter=False)
plt.figure(figsize=(6, 6))
for i, fitsfile in enumerate(fitsfiles):
plt.subplot(1, nbd, i + 1)
eomap = smap.Map(fitsfile)
sz = eomap.data.shape
if len(sz) == 4:
eomap.data = eomap.data.reshape((sz[2], sz[3]))
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot()
eomap.draw_limb()
eomap.draw_grid()
plt.show()
if doconcat:
if len(vis) > 1:
# from suncasa.eovsa import concateovsa as ce
from suncasa.tasks import concateovsa_cli as ce
if msoutdir is None:
msoutdir = './'
concatvis = os.path.basename(vis[0])
concatvis = msoutdir + '/' + concatvis.split('.')[0] + '_concat.ms'
ce.concateovsa(vis,
concatvis,
datacolumn='corrected',
keep_orig_ms=keep_orig_ms,
cols2rm="model,corrected")
return [concatvis]
else:
return vis
def plt_qlook_image(imres, figdir=None, verbose=True, synoptic=False):
from matplotlib import pyplot as plt
from sunpy import map as smap
from sunpy import sun
from matplotlib import colors
import astropy.units as u
if not figdir:
figdir = './'
nspw = len(set(imres['Spw']))
plttimes = list(set(imres['BeginTime']))
ntime = len(plttimes)
# sort the imres according to time
images = np.array(imres['ImageName'])
btimes = Time(imres['BeginTime'])
etimes = Time(imres['EndTime'])
spws = np.array(imres['Spw'])
suc = np.array(imres['Succeeded'])
inds = btimes.argsort()
images_sort = images[inds].reshape(ntime, nspw)
btimes_sort = btimes[inds].reshape(ntime, nspw)
suc_sort = suc[inds].reshape(ntime, nspw)
spws_sort = spws[inds].reshape(ntime, nspw)
if verbose:
print '{0:d} figures to plot'.format(ntime)
plt.ioff()
fig = plt.figure(figsize=(8, 8))
plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
for i in range(ntime):
plt.ioff()
plt.clf()
plttime = btimes_sort[i, 0]
tofd = plttime.mjd - np.fix(plttime.mjd)
suci = suc_sort[i]
if not synoptic:
if tofd < 16. / 24. or sum(
suci
) < nspw - 2: # if time of the day is before 16 UT (and 24 UT), skip plotting (because the old antennas are not tracking)
continue
#fig=plt.figure(figsize=(9,6))
#fig.suptitle('EOVSA @ '+plttime.iso[:19])
if synoptic:
fig.text(0.01,
0.98,
plttime.iso[:10],
color='w',
fontweight='bold',
fontsize=12,
ha='left')
else:
fig.text(0.01,
0.98,
plttime.iso[:19],
color='w',
fontweight='bold',
fontsize=12,
ha='left')
if verbose:
print 'Plotting image at: ', plttime.iso
for n in range(nspw):
plt.ioff()
image = images_sort[i, n]
#fig.add_subplot(nspw/3, 3, n+1)
fig.add_subplot(nspw / 2, 2, n + 1)
if suci[n]:
try:
eomap = smap.Map(image)
except:
continue
sz = eomap.data.shape
if len(sz) == 4:
eomap.data = eomap.data.reshape((sz[2], sz[3]))
eomap.data[np.isnan(eomap.data)] = 0.0
#resample the image for plotting
dim = u.Quantity([256, 256], u.pixel)
eomap = eomap.resample(dim)
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot_settings['norm'] = colors.Normalize(vmin=-1e5,
vmax=1e6)
eomap.plot()
if not synoptic:
eomap.draw_limb()
eomap.draw_grid()
ax = plt.gca()
ax.set_xlim([-1080, 1080])
ax.set_ylim([-1080, 1080])
spwran = spws_sort[i, n]
freqran = [int(s) * 0.5 + 2.9 for s in spwran.split('~')]
ax.text(0.98,
0.01,
'{0:.1f} - {1:.1f} GHz'.format(freqran[0], freqran[1]),
color='w',
transform=ax.transAxes,
fontweight='bold',
ha='right')
ax.set_title(' ')
#ax.set_title('spw '+spws_sort[i,n])
#ax.text(0.01,0.02, plttime.isot,transform=ax.transAxes,color='white')
ax.set_xlabel('')
ax.set_ylabel('')
ax.set_xticklabels([''])
ax.set_yticklabels([''])
else:
#make an empty map
data = np.zeros((512, 512))
header = {
"DATE-OBS": plttime.isot,
"EXPTIME": 0.,
"CDELT1": 5.,
"NAXIS1": 512,
"CRVAL1": 0.,
"CRPIX1": 257,
"CUNIT1": "arcsec",
"CTYPE1": "HPLN-TAN",
"CDELT2": 5.,
"NAXIS2": 512,
"CRVAL2": 0.,
"CRPIX2": 257,
"CUNIT2": "arcsec",
"CTYPE2": "HPLT-TAN",
"HGLT_OBS":
sun.heliographic_solar_center(plttime)[1].value,
"HGLN_OBS": 0.,
"RSUN_OBS":
sun.solar_semidiameter_angular_size(plttime).value,
"RSUN_REF": sun.constants.radius.value,
"DSUN_OBS":
sun.sunearth_distance(plttime).to(u.meter).value,
}
eomap = smap.Map(data, header)
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot_settings['norm'] = colors.Normalize(vmin=-1e5,
vmax=1e6)
eomap.plot()
if not synoptic:
eomap.draw_limb()
eomap.draw_grid()
ax = plt.gca()
ax.set_xlim([-1080, 1080])
ax.set_ylim([-1080, 1080])
#ax.set_title('spw '+spwran+'( )'))
spwran = spws_sort[i, n]
freqran = [int(s) * 0.5 + 2.9 for s in spwran.split('~')]
spwran = spws_sort[i, n]
#ax.set_title('{0:.1f} - {1:.1f} GHz'.format(freqran[0],freqran[1]))
ax.text(0.98,
0.01,
'{0:.1f} - {1:.1f} GHz'.format(freqran[0], freqran[1]),
color='w',
transform=ax.transAxes,
fontweight='bold',
ha='right')
ax.set_title(' ')
#ax.text(0.01,0.02, plttime.isot,transform=ax.transAxes,color='white')
ax.set_xlabel('')
ax.set_ylabel('')
ax.set_xticklabels([''])
ax.set_yticklabels([''])
fig_tdt = plttime.to_datetime()
if synoptic:
fig_subdir = fig_tdt.strftime("%Y/")
figname = 'eovsa_qlimg_' + plttime.iso[:10].replace('-',
'') + '.png'
else:
fig_subdir = fig_tdt.strftime("%Y/%m/%d/")
figname = 'eovsa_qlimg_' + plttime.isot.replace(':', '').replace(
'-', '')[:15] + '.png'
figdir_ = figdir + fig_subdir
if not os.path.exists(figdir_):
os.makedirs(figdir_)
if verbose:
print 'Saving plot to :' + figdir_ + figname
plt.savefig(figdir_ + figname)
plt.close(fig)
def mk_qlook_image(trange, doimport=False, docalib=False, ncpu=10, twidth=12, stokes=None, antenna='0~12',
#imagedir=None, spws=['1~3','4~6','7~9','10~13','14~18','19~28'],verbose=False):
imagedir=None, spws=['1~5','6~10','11~15','16~25'], toTb=True, overwrite=True,
doslfcal=False, verbose=False):
'''
trange: can be 1) a single Time() object: use the entire day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) a single or a list of UDBms file(s)
4) None -- use current date Time.now()
'''
if type(trange) == Time:
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
tsts = [l.to_datetime() for l in mslist['tstlist']]
subdir = [tst.strftime("%Y/%m/%d/") for tst in tsts]
if type(trange) == str:
try:
date = Time(trange)
mslist = trange2ms(trange=trange, doimport=doimport)
vis = mslist['ms']
except:
vis = [trange]
subdir = ['/']
for idx, f in enumerate(vis):
if f[-1] == '/':
vis[idx] = f[:-1]
if not stokes:
stokes = 'XX'
if not imagedir:
imagedir='./'
imres = {'Succeeded': [], 'BeginTime': [], 'EndTime': [], 'ImageName': [], 'Spw': [], 'Vis': []}
for n, msfile in enumerate(vis):
msfilebs=os.path.basename(msfile)
imdir = imagedir + subdir[n]
if not os.path.exists(imdir):
os.makedirs(imdir)
if doslfcal:
slfcalms = './'+msfilebs+'.xx'
split(msfile,outputvis=slfcalms,datacolumn='corrected',correlation='XX')
for spw in spws:
spwran = [s.zfill(2) for s in spw.split('~')]
freqran = [int(s)*0.5+2.9 for s in spw.split('~')]
cfreq=np.mean(freqran)
bmsz=max(150./cfreq,20.)
uvrange='<10klambda'
if doslfcal:
slfcal_img = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slfimg'
slfcal_tb = './'+msfilebs+'.slf.spw'+spw.replace('~','-')+'.slftb'
try:
clean(vis=slfcalms,
antenna=antenna,
imagename=slfcal_img,
spw=spw,
mode='mfs',
timerange='',
imagermode='csclean',
psfmode='clark',
imsize=[512,512],
cell=['5arcsec'],
niter=100,
gain=0.05,
stokes='I',
weighting='natural',
restoringbeam=[str(bmsz)+'arcsec'],
pbcor=False,
interactive=False,
usescratch=True)
except:
print 'error in cleaning spw: '+spw
break
gaincal(vis=slfcalms, refant='0',antenna=antenna,caltable=slfcal_tb,spw=spw, uvrange='',\
gaintable=[],selectdata=True,timerange='',solint='600s',gaintype='G',calmode='p',\
combine='',minblperant=3,minsnr=2,append=False)
if not os.path.exists(slfcal_tb):
print 'No solution found in spw: '+spw
break
else:
clearcal(slfcalms)
delmod(slfcalms)
applycal(vis=slfcalms,gaintable=[slfcal_tb],spw=spw,selectdata=True,\
antenna=antenna,interp='nearest',flagbackup=False,applymode='calonly',calwt=False)
msfile=slfcalms
if cfreq < 10.:
imsize=512
cell=['5arcsec']
else:
imsize=1024
cell=['2.5arcsec']
if len(spwran) == 2:
spwstr = spwran[0]+'~'+spwran[1]
else:
spwstr = spwran[0]
restoringbeam=['{0:.1f}arcsec'.format(bmsz)]
imagesuffix='.spw'+spwstr.replace('~','-')
if cfreq > 10.:
antenna=antenna+';!0&1;!0&2' #deselect the shortest baselines
res=ptclean(vis=msfile, imageprefix=imdir, imagesuffix=imagesuffix, twidth=twidth, uvrange=uvrange,
spw=spw, ncpu=ncpu, niter=1000, gain=0.05, antenna=antenna,imsize=imsize, cell=cell,
stokes=stokes, doreg=True, usephacenter=False, overwrite=overwrite, toTb=toTb, restoringbeam=restoringbeam,
uvtaper=True,outertaper=['30arcsec'])
if res:
imres['Succeeded'] += res['Succeeded']
imres['BeginTime'] += res['BeginTime']
imres['EndTime'] += res['EndTime']
imres['ImageName'] += res['ImageName']
imres['Spw'] += [spwstr]*len(res['ImageName'])
imres['Vis'] += [msfile]*len(res['ImageName'])
else:
return None
#save it for debugging purposes
np.savez('imres.npz',imres=imres)
return imres
def plt_qlook_image(imres, figdir=None, verbose=True):
from matplotlib import pyplot as plt
from sunpy import map as smap
from sunpy import sun
import astropy.units as u
if not figdir:
figdir = './'
nspw = len(set(imres['Spw']))
plttimes = list(set(imres['BeginTime']))
ntime = len(plttimes)
# sort the imres according to time
images = np.array(imres['ImageName'])
btimes = Time(imres['BeginTime'])
etimes = Time(imres['EndTime'])
spws = np.array(imres['Spw'])
suc = np.array(imres['Succeeded'])
inds = btimes.argsort()
images_sort = images[inds].reshape(ntime, nspw)
btimes_sort = btimes[inds].reshape(ntime, nspw)
suc_sort = suc[inds].reshape(ntime, nspw)
spws_sort = spws[inds].reshape(ntime, nspw)
if verbose:
print '{0:d} figures to plot'.format(ntime)
for i in range(ntime):
#for i in range(1):
plt.ioff()
fig = plt.figure(figsize=(11, 6))
plttime = btimes_sort[i, 0]
fig.suptitle('EOVSA @ ' + plttime.iso[:19])
if verbose:
print 'Plotting image at: ', plttime.iso
suci = suc_sort[i]
for n in range(nspw):
plt.ioff()
image = images_sort[i, n]
fig.add_subplot(nspw / 3, 3, n + 1)
if suci[n]:
try:
eomap = smap.Map(image)
except:
continue
sz = eomap.data.shape
if len(sz) == 4:
eomap.data = eomap.data.reshape((sz[2], sz[3]))
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot()
eomap.draw_limb()
eomap.draw_grid()
ax = plt.gca()
ax.set_xlim([-1050, 1050])
ax.set_ylim([-1050, 1050])
ax.set_title('spw ' + spws_sort[i, n])
#ax.text(0.01,0.02, plttime.isot,transform=ax.transAxes,color='white')
if n != nspw - 3:
ax.set_xlabel('')
ax.set_ylabel('')
ax.set_xticklabels([''])
ax.set_yticklabels([''])
else:
#make an empty map
data = np.zeros((512, 512))
header = {
"DATE-OBS": plttime.isot,
"EXPTIME": 0.,
"CDELT1": 5.,
"NAXIS1": 512,
"CRVAL1": 0.,
"CRPIX1": 257,
"CUNIT1": "arcsec",
"CTYPE1": "HPLN-TAN",
"CDELT2": 5.,
"NAXIS2": 512,
"CRVAL2": 0.,
"CRPIX2": 257,
"CUNIT2": "arcsec",
"CTYPE2": "HPLT-TAN",
"HGLT_OBS":
sun.heliographic_solar_center(plttime)[1].value,
"HGLN_OBS": 0.,
"RSUN_OBS":
sun.solar_semidiameter_angular_size(plttime).value,
"RSUN_REF": sun.constants.radius.value,
"DSUN_OBS":
sun.sunearth_distance(plttime).to(u.meter).value,
}
eomap = smap.Map(data, header)
eomap.plot_settings['cmap'] = plt.get_cmap('jet')
eomap.plot()
eomap.draw_limb()
eomap.draw_grid()
ax = plt.gca()
ax.set_xlim([-1050, 1050])
ax.set_ylim([-1050, 1050])
ax.set_title('spw ' + spws_sort[i, n])
#ax.text(0.01,0.02, plttime.isot,transform=ax.transAxes,color='white')
if n != 6:
ax.set_xlabel('')
ax.set_ylabel('')
ax.set_xticklabels([''])
ax.set_yticklabels([''])
figname = 'eovsa_qlimg_' + plttime.isot.replace(':', '').replace(
'-', '')[:15] + '.png'
fig_tdt = plttime.to_datetime()
fig_subdir = fig_tdt.strftime("%Y/%m/%d/")
figdir_ = figdir + fig_subdir
if not os.path.exists(figdir_):
os.makedirs(figdir_)
if verbose:
print 'Saving plot to :' + figdir_ + figname
plt.savefig(figdir_ + figname)
plt.close(fig)