def pipeline(year=None,
month=None,
day=None,
ndays=1,
clearcache=True,
overwrite=True,
doimport=True,
pols='XX'):
workdir = '/data1/workdir/'
os.chdir(workdir)
# Set to run 5 days earlier than the current date
if year is None:
mjdnow = Time.now().mjd
t = Time(mjdnow - 2, format='mjd')
else:
# Uncomment below and set date to run for a given date
t = Time('{}-{:02d}-{:02d} 20:00'.format(year, month, day))
for d in range(ndays):
t1 = Time(t.mjd - d, format='mjd')
datestr = t1.iso[:10]
subdir = os.path.join(workdir, t1.datetime.strftime('%Y%m%d/'))
if not os.path.exists(subdir):
os.makedirs(subdir)
else:
os.system('rm -rf {}/*'.format(subdir))
vis_corrected = calib_pipeline(datestr,
overwrite=overwrite,
doimport=doimport,
workdir=subdir,
clearcache=False,
pols=pols)
if clearcache:
os.chdir(workdir)
os.system('rm -rf {}'.format(subdir))
def qlook_image_pipeline(date, twidth=10, ncpu=15, doimport=False, docalib=False):
''' date: date string or Time object. e.g., '2017-07-15' or Time('2017-07-15')
'''
import pytz
from datetime import datetime
if date is None:
date = Time.now()
try:
date = Time(date)
except:
print('date format not recognised. Abort....')
return None
qlookfitsdir = os.getenv('EOVSAQLOOKFITS')
qlookfigdir = os.getenv('EOVSAQLOOKFIG')
if not qlookfitsdir:
qlookfitsdir='/data1/eovsa/qlookfits/'
if not qlookfigdir:
qlookfigdir='/common/webplots/qlookimg_10m/'
imagedir=qlookfitsdir
if docalib:
vis=calib_pipeline(date,doimport=doimport)
imres=mk_qlook_image(date, twidth=twidth, ncpu=ncpu, doimport=doimport, docalib=docalib, imagedir=imagedir,verbose=True)
figdir=qlookfigdir
plt_qlook_image(imres,figdir=figdir,verbose=True)
def pipeline1(year=None,
month=None,
day=None,
clearcache=True,
overwrite=True,
doimport=True):
from suncasa.eovsa import eovsa_pipeline as ep
import os
from eovsapy.util import Time
workdir = '/data1/workdir/'
os.chdir(workdir)
# Set to run 5 days earlier than the current date
if year is None:
mjdnow = Time.now().mjd
t = Time(mjdnow - 2, format='mjd')
else:
# Uncomment below and set date to run for a given date
t = Time('{}-{:02d}-{:02d} 20:00'.format(year, month, day))
print(t.iso)
datestr = t.iso[:10]
subdir = t.datetime.strftime('%Y%m%d/')
if not os.path.exists(subdir):
os.makedirs(subdir)
vis_corrected = ep.calib_pipeline(datestr,
overwrite=overwrite,
doimport=doimport,
workdir=os.path.join(workdir, subdir))
if clearcache:
os.chdir(workdir)
os.system('rm -rf ' + subdir)
def qlook_image_pipeline(date,
twidth=10,
ncpu=15,
doimport=False,
docalib=False,
synoptic=False):
''' date: date string or Time object. e.g., '2017-07-15' or Time('2017-07-15')
'''
import pytz
from datetime import datetime
if date is None:
date = Time.now()
try:
date = Time(date)
except:
print('date format not recognised. Abort....')
return None
qlookfitsdir = os.getenv('EOVSAQLOOKFITS')
qlookfigdir = os.getenv('EOVSAQLOOKFIG')
synopticfigdir = os.getenv('EOVSASYNOPTICFIG')
if not qlookfitsdir:
qlookfitsdir = '/data1/eovsa/fits/qlook_10m/'
if not qlookfigdir:
qlookfigdir = '/common/webplots/qlookimg_10m/'
if not synopticfigdir:
synopticfigdir = '/common/webplots/SynopticImg/'
imagedir = qlookfitsdir
if synoptic:
vis_synoptic = os.path.join(
udbmsdir, date.datetime.strftime("%Y%m"),
'UDB' + date.datetime.strftime("%Y%m%d") + '.ms')
if os.path.exists(vis_synoptic):
date = vis_synoptic
else:
print(
'Whole-day ms file {} not existed. About..... Use pipeline1.py to make one.'
.format(vis_synoptic))
return None
if docalib:
vis = calib_pipeline(date, doimport=doimport, synoptic=synoptic)
imres = mk_qlook_image(date,
twidth=twidth,
ncpu=ncpu,
doimport=doimport,
docalib=docalib,
imagedir=imagedir,
verbose=True)
figdir = qlookfigdir
plt_qlook_image(imres, figdir=figdir, verbose=True)
if imres['Synoptic']['Succeeded']:
figdir = synopticfigdir
plt_qlook_image(imres['Synoptic'],
figdir=figdir,
verbose=True,
synoptic=True)
def qlook_image_pipeline(date,
twidth=10,
ncpu=15,
doimport=False,
docalib=False,
synoptic=False,
overwrite=True):
''' date: date string or Time object. e.g., '2017-07-15' or Time('2017-07-15')
'''
import pytz
from datetime import datetime
if date is None:
date = Time.now()
try:
date = Time(date)
except:
print('date format not recognised. Abort....')
return None
if date.mjd > Time('2019-02-02 12:00:00').mjd:
## the last '' window is for fullBD synthesis image. Now obsolete.
# spws = ['6~10', '11~20', '21~30', '31~43', '']
# spws = ['6~10', '11~20', '21~30', '31~43']
spws = ['0~1', '2~5', '6~10', '11~20', '21~30', '31~49']
else:
## the last '' window is for fullBD synthesis image. Now obsolete.
# spws = ['1~5', '6~10', '11~15', '16~25', '']
spws = ['1~3', '4~6', '7~10', '10~14', '15~20', '21~30']
if docalib:
vis = calib_pipeline(date, doimport=doimport, synoptic=synoptic)
imagedir = qlookfitsdir
if synoptic:
vis_synoptic = os.path.join(
udbmsdir, date.datetime.strftime("%Y%m"),
'UDB' + date.datetime.strftime("%Y%m%d") + '.ms')
if os.path.exists(vis_synoptic):
date = vis_synoptic
else:
print(
'Whole-day ms file {} not existed. About..... Use pipeline to make one.'
.format(vis_synoptic))
return None
imres = mk_qlook_image(date,
twidth=twidth,
ncpu=ncpu,
doimport=doimport,
docalib=docalib,
imagedir=imagedir,
spws=spws,
verbose=True,
overwrite=overwrite)
figdir = qlookfigdir
plt_qlook_image(imres, figdir=figdir, verbose=True)
def trange2ms(trange=None,
doimport=False,
verbose=False,
doscaling=False,
overwrite=True):
'''This finds all solar UDBms files within a timerange; If the UDBms file does not exist
in EOVSAUDBMSSCL, create one by calling importeovsa
Required inputs:
trange - can be 1) a single string or Time() object in UTC: use the entire day, e.g., '2017-08-01' or Time('2017-08-01')
if just a date, find all scans withing the same date in local time.
if a complete time stamp, find the local date first (which may be different from that provided,
and return all scans within that day
2) a range of Time(), e.g., Time(['2017-08-01 00:00','2017-08-01 23:00'])
3) None -- use current date Time.now()
doimport - Boolean. If true, call importeovsa to import UDB files that are missing from
those found in the directory specified in EOVSAUDBMSSCL. Otherwise, return
a list of ms files it has found.
doscaling - Boolean. If true, scale cross-correlation amplitudes by using auto-correlations
verbose - Boolean. If true, return more information
'''
import glob
if trange is None:
trange = Time.now()
if type(trange) == list or type(trange) == str:
try:
trange = Time(trange)
except:
print('trange format not recognised. Abort....')
return None
# if type(trange) == Time:
try:
# if single Time object, the following line would report an error
nt = len(trange)
if len(trange) > 1:
# more than one value
trange = Time([trange[0], trange[-1]])
tdatetime = trange[0].to_datetime()
else:
# single value in a list
if trange[0].mjd == np.fix(trange[0].mjd):
# if only date is given, move the time from 00 to 12 UT
trange[0] = Time(trange[0].mjd + 0.5, format='mjd')
tdatetime = trange[0].to_datetime()
dhr = trange[0].LocalTime.utcoffset().total_seconds(
) / 60 / 60 / 24
btime = Time(np.fix(trange[0].mjd + dhr) - dhr, format='mjd')
etime = Time(btime.mjd + 1, format='mjd')
trange = Time([btime, etime])
except:
# the case of a single Time object
if trange.mjd == np.fix(trange.mjd):
# if only date is given, move the time from 00 to 12 UT
trange = Time(trange.mjd + 0.5, format='mjd')
tdatetime = trange.to_datetime()
dhr = trange.LocalTime.utcoffset().total_seconds() / 60 / 60 / 24
btime = Time(np.fix(trange.mjd + dhr) - dhr, format='mjd')
etime = Time(btime.mjd + 1, format='mjd')
trange = Time([btime, etime])
print('Selected timerange in UTC: ', trange.iso)
if doscaling:
udbmspath = udbmsscldir
else:
udbmspath = udbmsdir
inpath = '{}{}/'.format(udbdir, tdatetime.strftime("%Y"))
outpath = '{}{}/'.format(udbmspath, tdatetime.strftime("%Y%m"))
if not os.path.exists(outpath):
if verbose:
print(outpath + ' does not exist. Making a new directory.')
os.makedirs(outpath)
msfiles = []
else:
msfiles = [
os.path.basename(ll).split('.')[0]
for ll in glob.glob('{}UDB*.ms'.format(outpath))
]
msfile_synoptic = os.path.join(
outpath, 'UDB' + tdatetime.strftime("%Y%m%d") + '.ms')
if overwrite and doimport:
if os.path.exists(msfile_synoptic):
os.system('rm -rf {}'.format(msfile_synoptic))
sclist = ra.findfiles(trange, projid='NormalObserving', srcid='Sun')
udbfilelist = sclist['scanlist']
udbfilelist = [os.path.basename(ll) for ll in udbfilelist]
if os.path.exists(msfile_synoptic):
return {
'mspath': outpath,
'udbpath': inpath,
'udbfile': sorted(udbfilelist),
'udb2ms': [],
'ms': [msfile_synoptic],
'tstlist': sclist['tstlist'],
'tedlist': sclist['tedlist']
}
else:
udbfilelist_set = set(udbfilelist)
msfiles = udbfilelist_set.intersection(msfiles)
filelist = udbfilelist_set - msfiles
filelist = sorted(list(filelist))
if filelist and doimport:
import multiprocessing as mprocs
# ncpu = mprocs.cpu_count()
# if ncpu > 10:
# ncpu = 10
# if ncpu > len(filelist):
# ncpu = len(filelist)
ncpu = 1
timporteovsa.importeovsa(idbfiles=[inpath + ll for ll in filelist],
ncpu=ncpu,
timebin="0s",
width=1,
visprefix=outpath,
nocreatms=False,
doconcat=False,
modelms="",
doscaling=doscaling,
keep_nsclms=False,
udb_corr=True)
msfiles = [
os.path.basename(ll).split('.')[0]
for ll in glob.glob('{}UDB*.ms'.format(outpath))
]
udbfilelist_set = set(udbfilelist)
msfiles = udbfilelist_set.intersection(msfiles)
filelist = udbfilelist_set - msfiles
filelist = sorted(list(filelist))
return {
'mspath': outpath,
'udbpath': inpath,
'udbfile': sorted(udbfilelist),
'udb2ms': filelist,
'ms': [outpath + ll + '.ms' for ll in sorted(list(msfiles))],
'tstlist': sclist['tstlist'],
'tedlist': sclist['tedlist']
}