def extract_archive_data(tstart, tstop):
"""
extract fits data files from archive
input: tstart --- starting time
tstop --- stopping time
output: extracted epheio fits files
"""
#
#--- extract data of the given data period
#
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=ephin\n'
line = line + 'level=0\n'
line = line + 'filetype=epheio\n'
line = line + 'tstart=' + str(tstart) + '\n'
line = line + 'tstop=' + str(tstop) + '\n'
line = line + 'go\n'
flist = mcf.run_arc5gl_process(line)
if len(flist) > 0:
combine_fits_files(flist)
return 'ztemp.fits'
else:
return False
def run_arc5gl(operation, dataset, detector, level, filetype, obsid):
"""
run arc5gl to extract fits file
input: operation --- operation, retrive/browse
dataset --- data set, usually flight
level --- level
filetype --- file type, eg evt1
obsid --- obsid
output: fits --- extracted fits file name
"""
line = 'operation=' + operation + '\n'
line = line + 'dataset=' + dataset + '\n'
line = line + 'detector=' + detector + '\n'
line = line + 'level=' + level + '\n'
line = line + 'filetype=' + filetype + '\n'
line = line + 'obsid=' + str(obsid)+ '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
try:
fits = out[0]
except:
fits =''
return fits
def call_arc5gl(op, detector, level, tstart='', tstop='', sub='', ifile=''):
"""
using arc5gl to extract a fits file list or a file itself
input: op ---- operation: retreive/browse
detector --- detector
level --- level
tstart --- starting time if file name is provided, it is ignored
tstop --- stopping time if file name is provided, it is ignored
sub --- sub detector name; default ""
ifile --- file name; default ""
output: flist --- a list of fits file, either results of browse or extracted file name
"""
line = 'operation=' + op + '\n'
line = line + 'dataset=flight\n'
line = line + 'detector=' + detector + '\n'
if sub != '':
line = line + 'subdetector=' + sub + '\n'
line = line + 'level=' + str(level) + '\n'
if ifile == '':
line = line + 'tstart=' + str(tstart) + '\n'
line = line + 'tstop=' + str(tstop) + '\n'
else:
line = line + 'filename=' + ifile + '\n'
line = line + 'go\n'
flist = mcf.run_arc5gl_process(line)
return flist
def run_arc5gl_browse(start, stop, lev='Lev1'):
"""
run arc5gl to get a list of fits files in the given time period
input: start --- starting time in the format of <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
stop --- stoping time in the format of <yyyy>-<mm>-<dd>T<hh>:<mm>:<ss>
lev --- data level
outfile --- output file name, default: zspace
output: a list of fits file names in <outfile>
"""
line = 'operation=browse\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
if lev == 'Lev1':
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
else:
line = line + 'level=2\n'
line = line + 'filetype=evt2\n'
line = line + 'tstart=' + start + '\n'
line = line + 'tstop=' + stop + '\n'
line = line + 'go\n'
f_list = mcf.run_arc5gl_process(line)
return f_list
def find_pitch_angle(start, stop):
"""
create a table of time and sun pitch angle
input: start --- starting time in seconds from 1998.1.1
stop --- stopping time in seconds from 1998.1.1
output: time --- a list of time in seconds from 1998.1.1
pitch --- a list of ptich in degree
"""
#
#--- extract pcad aspsol fits files for the given time period
#
line = 'operation=retrieve\n'
line = line + 'dataset = flight\n'
line = line + 'detector = pcad\n'
line = line + 'subdetector = aca\n'
line = line + 'level = 1\n'
line = line + 'filetype =aspsol\n'
line = line + 'tstart = ' + str(start) + '\n'
line = line + 'tstop = ' + str(stop) + '\n'
line = line + 'go\n'
data_list = mcf.run_arc5gl_process(line)
[cols, tbdata] = ecf.read_fits_file(data_list[0])
for ent in data_list:
cmd = 'rm -rf ' + ent
os.system(cmd)
time = tbdata['time']
ra = tbdata['ra']
dec = tbdata['dec']
time_list = []
pitch_list = []
prev = 0.0
m = 0
for k in range(0, len(time)):
#
#--- select one data every 5 mins
#
itime = int(time[k])
if itime == prev:
continue
if m % 300 == 0:
#
#--- get the sun angle
#
pitch = find_chandra_pitch(time[k], ra[k], dec[k])
time_list.append(itime)
pitch_list.append(pitch)
prev = itime
m += 1
time = numpy.array(time_list)
pitch = numpy.array(pitch_list)
return [time, pitch]
def get_ephin_data(start, stop, out_dir):
"""
extract ephin data and create ephin_data file
input: start --- start time in seconds from 1998.1.1
stop --- stop time in seconds from 1998.1.1
out_dir --- output directory
output: out_dir/ehin_data
"""
#
#--- first create a list of ephin fits file for the month
#
line = 'operation=browse\n'
line = line + 'dataset=flight\n'
line = line + 'detector=ephin\n'
line = line + 'level=1\n'
line = line + 'filetype=ephrates\n'
line = line + 'tstart=' + start + '\n'
line = line + 'tstop=' + stop + '\n'
line = line + 'go\n'
data = mcf.run_arc5gl_process(line)
#
#--- extract ephin fits file one by one and analyze
#
for ent in data:
mc = re.search('fits', ent)
if mc is not None:
atemp = re.split('\s+', ent)
fits = atemp[0]
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=ephin\n'
line = line + 'level=1\n'
line = line + 'filetype=ephrates\n'
line = line + 'filename=' + fits + '\n'
line = line + 'go\n'
chk = mcf.run_arc5gl_process(line)
cmd = 'gzip -d *fits.gz'
os.system(cmd)
extract_ephin_data(fits, out_dir)
def get_data(start, stop, year, msid_list, out_dir):
"""
update eph l1 related data for the given data peirod
input: start --- start time in seconds from 1998.1.1
stop --- stop time in seconds from 1998.1.1
year --- data extracted year
msid_list --- list of msids
out_dir --- output_directory
output: <out_dir>/<msid>_full_data_<year>.fits
"""
print(str(start) + '<-->' + str(stop))
line = 'operation=retrieve\n'
line = line + 'dataset = flight\n'
line = line + 'detector = ephin\n'
line = line + 'level = 0\n'
line = line + 'filetype =ephhk \n'
line = line + 'tstart = ' + str(start) + '\n'
line = line + 'tstop = ' + str(stop) + '\n'
line = line + 'go\n'
data_list = mcf.run_arc5gl_process(line)
#
#--- uppend the data to the local fits data files
#
for fits in data_list:
[cols, tbdata] = ecf.read_fits_file(fits)
time = tbdata['time']
for col in msid_list:
#
#--- ignore columns with "ST_" (standard dev) and time
#
mdata = tbdata[col]
cdata = [time, mdata]
ocols = ['time', col.lower()]
if not os.path.isdir(out_dir):
cmd = 'mkdir ' + out_dir
os.system(cmd)
ofits = out_dir + col.lower() + '_full_data_' + str(year) + '.fits'
if os.path.isfile(ofits):
ecf.update_fits_file(ofits, ocols, cdata)
else:
ecf.create_fits_file(ofits, ocols, cdata)
mcf.rm_files(fits)
def run_script():
for year in range(1999,2015):
if mcf.is_leapyear(year):
dend = 367
else:
dend = 366
syear = str(year)
lyear = syear[2] + syear[3]
for yday in range(1,dend):
if year == 1999 and yday < 239:
continue
if year == 2014 and yday > 316:
break
lyday = str(yday)
if yday < 10:
lyday = '00' + lyday
elif yday < 100:
lyday = '0' + lyday
dtime = str(year) + ':' + lyday
start = dtime + ':00:00:00'
stop = dtime + ':23:59:59'
line = 'operation = retrieve\n'
line = line + 'dataset = flight\n'
line = line + 'detector = telem\n'
line = line + 'level = raw\n'
line = line + 'tstart = ' + start + '\n'
line = line + 'tstop = ' + stop + '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
cmd = 'ls * > ' + zspace
os.system(cmd)
test = open(zspace, 'r').read()
mc = re.search('sto', test)
if mc is not None:
os.system('rm *log*')
os.system('gzip -fd *gz')
os.system('ls *.sto > xtmpnew')
os.system('nice ./filters_ccdm')
esd.extract_sim_data()
os.system('rm -rf *.sto *.tl')
def get_data(start, stop, year, msid_list):
print(str(start) + '<-->' + str(stop))
line = 'operation=retrieve\n'
line = line + 'dataset = flight\n'
line = line + 'detector = ephin\n'
line = line + 'level = 0\n'
line = line + 'filetype =ephhk \n'
line = line + 'tstart = ' + start + '\n'
line = line + 'tstop = ' + stop + '\n'
line = line + 'go\n'
data_list = mcf.run_arc5gl_process(line)
#
#--- uppend the data to the local fits data files
#
for fits in data_list:
[cols, tbdata] = ecf.read_fits_file(fits)
time = tbdata['time']
for col in msid_list:
#
#--- ignore columns with "ST_" (standard dev) and time
#
mdata = tbdata[col]
cdata = [time, mdata]
ocols = ['time', col.lower()]
if not os.path.isdir(out_dir):
cmd = 'mkdir ' + out_dir
os.system(cmd)
ofits = out_dir + col.lower() + '_full_data_' + str(year) + '.fits'
if os.path.isfile(ofits):
update_fits_file(ofits, ocols, cdata)
else:
create_fits_file(ofits, ocols, cdata)
mcf.rm_files(fits)
def extract_dither_info_from_pacd(start, stop):
"""
extract dither information from archieved pacad data
input: start --- start time
stop --- stop time
output: t_list --- a list of time
d_list --- a list of data (DISA/ENAB)
"""
#
#--- set arc5gl command
#
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=pcad\n'
line = line + 'subdetector=eng\n'
line = line + 'level=0\n'
line = line + 'filetype=pcad8eng\n'
line = line + 'tstart=' + str(start) + '\n'
line = line + 'tstop=' + str(stop) + '\n'
line = line + 'go\n'
#
#--- run arc5gl
#
f_list = mcf.run_arc5gl_process(line)
if len(f_list) < 1:
return []
#
#--- extract the data
#
t_list = []
d_list = []
for ent in f_list:
hout = pyfits.open(ent)
data = hout[1].data
t_list = t_list + list(data['time'])
d_list = d_list + list(data['AODITHEN'])
hout.close()
mcf.rm_files(ent)
t_list = [float(ent) for ent in t_list]
return [t_list, d_list]
def extract_evt1_data(start, stop):
"""
extract all acis evt1 file during the period
input: start --- start time
stop --- stop time
output: ./acisf_evt1.fits
out --- a list of acis evt1 file extracted
"""
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'tstart=' + str(start) + '\n'
line = line + 'tstop=' + str(stop) + '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
return out
def extract_data(start, stop):
"""
using arc5gl, extract hk data
input: start --- start time (any format accepted by arc5gl)
stop --- stop time (any format accepted by arc5gl)
output: extracted hrc hk0 fits data files
flist --- a list of hk0 fits files
"""
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=hrc\n'
line = line + 'level=0\n'
line = line + 'filetype=hrchk\n'
line = line + 'tstart=' + str(start) + '\n'
line = line + 'tstop=' + str(stop) + '\n'
line = line + 'go\n'
flist = mcf.run_arc5gl_process(line)
return flist
def extract_acis_count_rate(start, stop, dir_name):
"""
extract acis count rate data
input: year --- year
month --- month
dir_name --- output dir name
output: <dir_name>/ccd<#ccd>
"""
#
#--- make a list of data fits file
#
data_list = get_data_list_from_archive(start, stop)
if len(data_list) == 0:
print("No data")
exit(1)
for ifile in data_list:
#
#--- extract the fits file with arc5gl
#
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'filename=' + ifile + '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
cmd = 'gzip -d ' + ifile + '.gz'
os.system(cmd)
#
#--- extract data and update/create the count rate data
#
print("Extracting: " + ifile)
extract_data(ifile, dir_name)
mcf.rm_files(ifile)
def get_data_list_from_archive(start, stop):
"""
compare the current input list to the old one and select out the data which are not used
input: start --- start time in seconds from 1998.1.1
stop --- stop time in seconds from 1998.1.1
output: file_list --- a list of acis evt1 file list
"""
#
#--- create data list with arc5gl
#
line = 'operation=browse\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'tstart=' + str(start) + '\n'
line = line + 'tstop=' + str(stop) + '\n'
line = line + 'go\n'
data = mcf.run_arc5gl_process(line)
#
#--- choose files with only non-calibration data
#
file_list = []
for ent in data:
mc = re.search('acisf', ent)
if mc is None:
continue
ftemp = re.split('\s+', ent)
atemp = re.split('acisf', ftemp[0])
btemp = re.split('_', atemp[1])
ctemp = re.split('N', btemp[0])
mark = int(ctemp[0])
if mark < 50000:
file_list.append(ftemp[0])
return file_list
def run_arc5gl(start, stop):
"""
extract acis and hrc evt1a.fits files using arc4gl
input: start --- start time in the format of 2018-01-01:00:00:00
stop --- stop time
output: fits files (e.g., acisf17108_001N002_evt1a.fits.gz)
"""
#
#--- read a template and create the current command file
#
ifile = house_keeping +'arc_template'
with open(ifile, 'r') as f:
line = f.read()
line = line.replace('#START#', start)
line = line.replace('#STOP#', stop)
flist = mcf.run_arc5gl_process(line)
#
#--- remove unwanted fits files
#
os.system('rm *src1a*')
def use_arc5gl(operation,
dataset,
detector,
level,
filetype,
start,
stop,
deposit='./'):
"""
extract data using arc5gl.
input: start --- stop (year and ydate)
operation --- (e.g., retrive)
dataset ---(e.g. flight)
detector --- (e.g. hrc)
level --- (eg 0, 1, 2)
filetype ---(e.g, evt1)
start --- starting time
stop --- stopping time
deposit --- where to deposit output fits file
output: data --- a list of fits file extracted
"""
#
#--- set arc5gl commend
#
line = 'operation=' + operation + '\n'
line = line + 'dataset=' + dataset + '\n'
line = line + 'detector=' + detector + '\n'
line = line + 'level=' + str(level) + '\n'
line = line + 'filetype=' + filetype + '\n'
line = line + 'tstart=' + str(start) + '\n'
line = line + 'tstop=' + str(stop) + '\n'
line = line + 'go\n'
data = mcf.run_arc5gl_process(line)
return data
def extract_hrchk(start, stop):
"""
extract hrchk fits data files from archive
input: start --- starting time in yyyy-mm-ddThh:mm:ss
stop --- stopping time in yyyy-mm-ddThh:mm:ss
output: data --- a list of fits files extracted
"""
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=hrc\n'
line = line + 'level=0\n'
line = line + 'filetype=hrchk\n'
line = line + 'tstart=' + str(start) + '\n'
line = line + 'tstop=' + str(stop) + '\n'
line = line + 'go\n'
cdata = mcf.run_arc5gl_process(line)
if len(cdata) > 0:
cmd = 'chmod 777 *fits.gz'
os.system(cmd)
return cdata
def run_arc5gl(start, stop):
"""
extract data from archive using arc5gl
input: start --- starting time in the format of mm/dd/yy,hh/mm/ss. hh/mm/ss is optional
stop --- stoping time
output: extracted data set
"""
#
#--- write arc5gl command
#
line = 'operation=retrieve\n'
line = line + 'dataset =flight\n'
line = line + 'detector=telem\n'
line = line + 'level =raw\n'
line = line + 'tstart=' + str(start) + '\n'
line = line + 'tstop=' + str(stop) + '\n'
line = line + 'go\n'
#
#--- extract data
#
out = mcf.run_arc5gl_process(line)
return out
def make_obsid_list(start, stop):
"""
create a list of acis observation for a given period
input: start --- starting time
stop --- stopping time
output: a_lis --- a list of obsids
"""
line = 'operation=browse\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'tstart=' + str(start) + '\n'
line = line + 'tstop=' + str(stop) + '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
#
#--- save obsids
#
a_list = []
if len(out) > 0:
for ent in out:
atemp = re.split('acisf', ent)
btemp = re.split('_', atemp[1])
obsid = btemp[0]
#
#--- make sure that obsid is numeric
#
try:
chk = float(obsid)
except:
continue
a_list.append(obsid)
return a_list
def extract_data_arc5gl(detector, level, filetype, tstart, tstop, sub=''):
"""
extract data using arc5gl
input: detector --- detector name
level --- level
filetype --- file name
tstart --- starting time
tstop --- stopping time
sub --- subdetector name; defalut "" --- no sub detector
output: cols --- a list of col name
tdata --- a list of arrays of data
"""
#
#--- extract ephin hk lev 0 fits data
#
line = 'operation=retrieve\n'
line = line + 'dataset = flight\n'
line = line + 'detector = ' + detector + '\n'
if sub != '':
line = line + 'subdetector = ' + sub + '\n'
line = line + 'level = ' + level + '\n'
line = line + 'filetype = ' + filetype + '\n'
line = line + 'tstart = ' + str(tstart) + '\n'
line = line + 'tstop = ' + str(tstop) + '\n'
line = line + 'go\n'
flist = mcf.run_arc5gl_process(line)
if len(flist) < 1:
print("\t\tNo data")
return [[], []]
#
#--- combined them
#
flen = len(flist)
if flen == 0:
return [[], []]
elif flen == 1:
cmd = 'cp ' + flist[0] + ' ./ztemp.fits'
os.system(cmd)
else:
mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits')
if flen > 2:
for k in range(2, flen):
mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits')
cmd = 'mv out.fits ztemp.fits'
os.system(cmd)
#
#--- remove indivisual fits files
#
for ent in flist:
cmd = 'rm -rf ' + ent
os.system(cmd)
#
#--- return data
#
[cols, tbdata] = ecf.read_fits_file('ztemp.fits')
cmd = 'rm -f ztemp.fits out.fits'
os.system(cmd)
return [cols, tbdata]
def update_simdiag_data(date=''):
"""
collect sim diag msids
input: date ---- the date in yyyymmdd format. if not given, yesterday's date is used
output: fits file data related to grad and comp
"""
#
#--- read group names which need special treatment
#
#sfile = house_keeping + 'msid_list_simdiag'
sfile = './msid_list_simsupple'
data = mcf.read_data_file(sfile)
cols = []
g_dir = {}
for ent in data:
atemp = re.split('\s+', ent)
cols.append(atemp[0])
g_dir[atemp[0]] = atemp[1]
#
#--- create msid <---> unit dictionary
#
[udict, ddict] = ecf.read_unit_list()
#
#--- read mta database
#
mta_db = ecf.read_mta_database()
#
#--- read mta msid <---> sql msid conversion list
#
mta_cross = ecf.read_cross_check_table()
day_list = []
for year in range(1999, 2021):
cyear = str(year)
for mon in range(1, 13):
if year == 1999:
if mon < 8:
continue
if year == 2020:
if mon > 1:
break
cmon = str(mon)
if mon < 10:
cmon = '0' + cmon
if mcf.is_leapyear(year):
lday = mday_list2[mon - 1]
else:
lday = mday_list[mon - 1]
for day in range(1, lday + 1):
cday = str(day)
if day < 10:
cday = '0' + cday
sday = cyear + '-' + cmon + '-' + cday
day_list.append(sday)
for sday in day_list:
if sday == '2020-01-17':
break
print("Date: " + sday)
start = sday + 'T00:00:00'
stop = sday + 'T23:59:59'
line = 'operation=retrieve\n'
line = line + 'dataset = flight\n'
line = line + 'detector = sim\n'
line = line + 'level = 0\n'
line = line + 'filetype = sim\n'
line = line + 'tstart = ' + start + '\n'
line = line + 'tstop = ' + stop + '\n'
line = line + 'go\n'
flist = mcf.run_arc5gl_process(line)
if len(flist) < 1:
print("\t\tNo data")
continue
#
#--- combined them
#
flen = len(flist)
if flen == 0:
continue
elif flen == 1:
cmd = 'cp ' + flist[0] + ' ./ztemp.fits'
os.system(cmd)
else:
mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits')
if flen > 2:
for k in range(2, flen):
mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits')
cmd = 'mv out.fits ztemp.fits'
os.system(cmd)
#
#--- remove indivisual fits files
#
for ent in flist:
cmd = 'rm -rf ' + ent
os.system(cmd)
#
#--- read out the data for the full day
#
[cols_xxx, tbdata] = ecf.read_fits_file('ztemp.fits')
cmd = 'rm -f ztemp.fits out.fits'
os.system(cmd)
#
#--- get time data in the list form
#
dtime = list(tbdata.field('time'))
for k in range(0, len(cols)):
#
#---- extract data in a list form
#
col = cols[k]
data = list(tbdata.field(col))
#
#--- change col name to msid
#
msid = col.lower()
#
#--- get limit data table for the msid
#
try:
tchk = convert_unit_indicator(udict[msid])
except:
tchk = 0
glim = ecf.get_limit(msid, tchk, mta_db, mta_cross)
#
#--- update database
#
tstart = convert_time_format(start)
tstop = convert_time_format(stop)
update_database(msid,
g_dir[msid],
dtime,
data,
glim,
pstart=tstart,
pstop=tstop)
def acis_dose_get_data(startYear='', startMonth='', stopYear='', stopMonth=''):
"""
extract ACIS evt1 data from a month and create combined image file.
input: startYear --- year of starting time
startMonth --- month of starting time
stopYear --- year of stopping time
stopMonth --- month of stopping time
"""
if startYear == '' or startMonth == '' or stopYear == '' or stopMonth == '':
startYear = raw_input('Start Year: ')
startYear = int(float(startYear))
startMonth = raw_input('Start Month: ')
startMonth = int(float(startMonth))
stopYear = raw_input('Stop Year: ')
stopYear = int(float(stopYear))
stopMonth = raw_input('Stop Month: ')
stopMonth = int(float(stopMonth))
#
#--- start extracting the data for the year/month period
#
for year in range(startYear, stopYear + 1):
#
#--- create a list of month appropriate for the year
#
month_list = expf.make_month_list(year, startYear, stopYear,
startMonth, stopMonth)
for month in month_list:
smon = mcf.add_leading_zero(month)
start = str(year) + '-' + smon + '-01T00:00:00'
nextMonth = month + 1
nyear = year
if nextMonth > 12:
nextMonth = 1
nyear += 1
smon = mcf.add_leading_zero(nextMonth)
stop = str(nyear) + '-' + smon + '-01T00:00:00'
#
#--- using ar5gl, get a list of file names
#
line = 'operation=browse\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'tstart=' + start + '\n'
line = line + 'tstop=' + stop + '\n'
line = line + 'go\n'
fitsList = mcf.run_arc5gl_process(line)
#
#--- extract each evt1 file, extract the central part, and combine them into a one file
#
for fits in fitsList:
print("FITS File: " + fits)
atemp = re.split('\s+', line)
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'filename=' + fits + '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
#
#--- check whether the fits file actually extracted and if so, ungip the file
#
if len(out) < 1:
continue
cmd = 'gzip -d ' + out[0]
os.system(cmd)
line = fits + '[EVENTS][bin tdetx=2800:5200:1, tdety=1650:4150:1][option type=i4]'
#
#--- create an image file
#
ichk = expf.create_image(line, 'ztemp.fits')
#
#--- combined images
#
if ichk > 0:
expf.combine_image('ztemp.fits', 'total.fits')
mcf.rm_files(fits)
mcf.rm_files('ztemp.fits')
#
#--- rename the file
#
lyear = str(startYear)
lmon = mcf.add_leading_zero(startMonth)
outfile = './ACIS_' + lmon + '_' + lyear + '_full.fits'
cmd = 'mv total.fits ' + outfile
os.system(cmd)
#
#--- trim the extreme values
#
upper = find_10th(outfile)
if mcf.is_neumeric(upper):
outfile2 = './ACIS_' + lmon + '_' + lyear + '.fits'
cmd = ' dmimgthresh infile=' + outfile + ' outfile='
cmd = cmd + outfile2 + ' cut="0:' + str(
upper) + '" value=0 clobber=yes'
expf.run_ascds(cmd)
else:
cmd = 'cp -f ' + outfile + ' ' + outfile2
os.system(cmd)
cmd = 'gzip ' + outfile
os.system(cmd)
#
#--- move full one to the data dir; keep other in <exc_dir> to be used to create cumlative files
#
cmd = 'mv ' + outfile + '* ' + mon_acis_dir + '/.'
os.system(cmd)
def update_eph_data(date=''):
"""
collect grad and comp data for trending
input: date ---- the date in yyyymmdd format. if not given, yesterday's date is used
output: fits file data related to grad and comp
"""
#
#--- read group names which need special treatment
#
#sfile = 'eph_list'
#glist = mcf.read_data_file(sfile)
#
#--- create msid <---> unit dictionary
#
[udict, ddict] = ecf.read_unit_list()
#
#--- read mta database
#
mta_db = ecf.read_mta_database()
#
#--- read mta msid <---> sql msid conversion list
#
mta_cross = ecf.read_cross_check_table()
day_list = []
for year in range(2000, 2019): #---- CHANGE CHANGE CHAGE!!!!!
lyear = year
for mon in range(1, 13):
#if year == 2018 and mon > 1:
# break
#if year == 2017 and mon < 11:
# continue
cmon = str(mon)
if mon < 10:
cmon = '0' + cmon
nmon = mon + 1
if nmon > 12:
nmon = 1
lyear += 1
clmon = str(nmon)
if nmon < 10:
clmon = '0' + clmon
start = str(year) + '-' + cmon + '-01T00:00:00'
stop = str(lyear) + '-' + clmon + '-01T00:00:00'
print "Period: " + str(start) + "<--->" + str(stop)
for group in glist:
print "Group: " + group
#
#---CHANGE THE DETECTOR/FILETYPE BEFORE RUNNING IF IT IS DIFFERENT FROM EPHHK
#
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=ephin\n'
line = line + 'level=0\n'
line = line + 'filetype=ephhk\n'
line = line + 'tstart=' + start + '\n'
line = line + 'tstop=' + stop + '\n'
line = line + 'go\n'
flist = mcf.run_arc5gl_process(line)
if len(flist) < 1:
print "\t\tNo data"
continue
#
#--- combined them
#
flen = len(flist)
if flen == 0:
continue
elif flen == 1:
cmd = 'cp ' + flist[0] + ' ./ztemp.fits'
os.system(cmd)
else:
mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits')
if flen > 2:
for k in range(2, flen):
mfo.appendFitsTable('ztemp.fits', flist[k],
'out.fits')
cmd = 'mv out.fits ztemp.fits'
os.system(cmd)
#
#--- remove indivisual fits files
#
for ent in flist:
cmd = 'rm -rf ' + ent
os.system(cmd)
#
#--- read out the data
#
[cols, tbdata] = ecf.read_fits_file('ztemp.fits')
cmd = 'rm -f ztemp.fits out.fits'
os.system(cmd)
#
#--- get time data in the list form
#
dtime = list(tbdata.field('time'))
for k in range(1, len(cols)):
#
#--- select col name without ST_ (which is standard dev)
#
col = cols[k]
mc = re.search('ST_', col)
if mc is not None:
continue
mc = re.search('quality', col, re.IGNORECASE)
if mc is not None:
continue
mc = re.search('mjf', col, re.IGNORECASE)
if mc is not None:
continue
mc = re.search('gap', col, re.IGNORECASE)
if mc is not None:
continue
mc = re.search('dataqual', col, re.IGNORECASE)
if mc is not None:
continue
mc = re.search('tlm_fmt', col, re.IGNORECASE)
if mc is not None:
continue
#
#---- extract data in a list form
#
data = list(tbdata.field(col))
#
#--- change col name to msid
#
msid = col.lower()
#
#--- get limit data table for the msid
#
try:
tchk = convert_unit_indicator(udict[msid])
except:
tchk = 0
glim = ecf.get_limit(msid, tchk, mta_db, mta_cross)
#
#--- update database
#
update_database(msid, group, dtime, data, glim)
def process_evt(lev='Lev1'):
"""
process lev1 or 2 ACIS SIB data
input: lev -- which level data to process; defalut: 'Lev1'
it also reads acis_obs file to find fits file names
output: processed fits files in <out_dir>/lres/
"""
ldir = cor_dir + lev + '/'
indir = ldir + 'Input/'
outdir = ldir + 'Outdir/'
repdir = ldir + 'Reportdir/'
data = mcf.read_data_file('./acis_obs')
for obs in data:
atemp = re.split('\s+', obs)
obsid = atemp[0].strip()
print("OBSID: " + str(obsid))
#
#--- extract evt 1/2 file
#
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=acis\n'
if lev == 'Lev1':
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
else:
line = line + 'level=2\n'
line = line + 'filetype=evt2\n'
line = line + 'obsid=' + str(obsid) + '\n'
line = line + 'go\n'
flist = mcf.run_arc5gl_process(line)
for fits in flist:
#
#--- exclude bright sources from the file
#
es.exclude_sources(fits)
cmd = 'mv *cleaned*fits ' + indir + '/. 2>/dev/null'
os.system(cmd)
cmd = 'rm -rf *fits*'
os.system(cmd)
#
#--- extract acis evt1 files from archeive, and compute SIB
#--- at some occasion, the process dies with unknown reason; so repeat twice
#--- to cover the failed case
#
try:
scf.sib_corr_comp_sib(lev)
except:
try:
scf.sib_corr_comp_sib(lev)
except:
pass
#
#--- clean up the files
#
cmd = 'rm -rf ' + indir + '/*fits ' + outdir + '/*fits ' + outdir + '/*ped* '
os.system(cmd)
def recover_hrcveto_data():
"""
recover hrc veto data
input: none
output: fits file data related to grad and comp
"""
#
#--- read group names which need special treatment
#
#sfile = 'eph_list'
#glist = mcf.read_data_file(sfile)
glist = ['Hrcveto']
#
#--- create msid <---> unit dictionary
#
[udict, ddict] = ecf.read_unit_list()
#
#--- read mta database
#
mta_db = ecf.read_mta_database()
#
#--- read mta msid <---> sql msid conversion list
#
mta_cross = ecf.read_cross_check_table()
day_list = []
for year in range(1999, 2018):
lyear = year
cyear = str(year)
for mon in range(1, 13):
if year == 1999:
if mon < 8:
continue
if year == 2017:
if mon > 10:
break
cmon = str(mon)
if mon < 10:
cmon = '0' + cmon
nmon = mon + 1
if nmon > 12:
nmon = 1
lyear += 1
cnmon = str(nmon)
if nmon < 10:
cnmon = '0' + cnmon
start = str(year) + '-' + cmon + '-01T00:00:00'
stop = str(lyear) + '-' + cnmon + '-01T00:00:00'
for group in glist:
print "Group: " + group + ' : ' + str(start) + '<-->' + str(stop)
line = 'operation=retrieve\n'
line = line + 'dataset = flight\n'
line = line + 'detector = hrc\n'
line = line + 'level = 0\n'
line = line + 'filetype = hrcss\n'
line = line + 'tstart = ' + start + '\n'
line = line + 'tstop = ' + stop + '\n'
line = line + 'go\n'
flinst = mcf.run_arc5gl_process(line)
if len(flist) < 1:
print "\t\tNo data"
continue
#
#--- combined them
#
flen = len(flist)
if flen == 0:
continue
elif flen == 1:
cmd = 'cp ' + flist[0] + ' ./ztemp.fits'
os.system(cmd)
else:
mfo. appendFitsTable(flist[0], flist[1], 'ztemp.fits')
if flen > 2:
for k in range(2, flen):
mfo. appendFitsTable('ztemp.fits', flist[k], 'out.fits')
cmd = 'mv out.fits ztemp.fits'
os.system(cmd)
#
#--- remove indivisual fits files
#
for ent in flist:
cmd = 'rm -rf ' + ent
os.system(cmd)
#
#--- read out the data for the full day
#
[cols, tbdata] = ecf.read_fits_file('ztemp.fits')
cols = ['TLEVART', 'VLEVART', 'SHEVART']
cmd = 'rm -f ztemp.fits out.fits'
os.system(cmd)
#
#--- get time data in the list form
#
dtime = list(tbdata.field('time'))
for col in cols:
#
#---- extract data in a list form
#
data = list(tbdata.field(col))
#
#--- change col name to msid
#
msid = col.lower()
#
#--- get limit data table for the msid
#
try:
tchk = convert_unit_indicator(udict[msid])
except:
tchk = 0
glim = ecf.get_limit(msid, tchk, mta_db, mta_cross)
#
#--- update database
#
update_database(msid, group, dtime, data, glim)
def hrc_dose_get_data(startYear='NA',
startMonth='NA',
stopYear='NA',
stopMonth='NA'):
"""
extract HRC evt1 data from a month and create cumulative data fits file.
input: startYear --- start year
startMonth --- start month
stopYear --- stop year
stopMonth --- stop month
output: image fits files for the month and cumulative cases
"""
#
#--- if the dates are not given, set them to the last month
#
if startYear == 'NA':
[stopYear, stopMonth, day] = mcf.today_date()
startYear = stopYear
stopMonth -= 1
startMonth = stopMonth
if startMonth < 1:
startMonth = 12
startYear -= 1
#
#--- start extracting the data for the year/month period
#
for year in range(startYear, stopYear + 1):
lyear = str(year)
syear = lyear[2] + lyear[3]
#
#--- create a list of month appropriate for the year
#
month_list = expf.make_month_list(year, startYear, stopYear,
startMonth, stopMonth)
for month in month_list:
smonth = mcf.add_leading_zero(month)
#
#--- output file name settings
#
outfile_i = './HRCI_' + str(smonth) + '_' + str(lyear) + '.fits'
outfile_s = './HRCS_' + str(smonth) + '_' + str(lyear) + '.fits'
#
#--- using ar5gl, get file names
#
smonth = mcf.add_leading_zero(month)
syear = str(year)
start = syear + '-' + smonth + '-01T00:00:00'
nextMonth = month + 1
if nextMonth > 12:
lyear = year + 1
nextMonth = 1
else:
lyear = year
smonth = mcf.add_leading_zero(nextMonth)
syear = str(lyear)
stop = str(lyear) + '-' + smonth + '-01T00:00:00'
line = 'operation=browse\n'
line = line + 'dataset=flight\n'
line = line + 'detector=hrc\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'tstart=' + start + '\n'
line = line + 'tstop=' + stop + '\n'
line = line + 'go\n'
fitsList = mcf.run_arc5gl_process(line)
#
#--- extract each evt1 file, extract the central part, and combine them into a one file
#
#--- set counters for how many hrc-i and hrc-s are extracted
#
hrci_cnt = 0
hrcs_cnt = 0
for fitsName in fitsList:
print("Fits file: " + fitsName)
m = re.search('fits', fitsName)
if m is None:
continue
try:
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=hrc\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'filename=' + fitsName + '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
if len(out) < 1:
continue
ofits = out[0]
cmd = 'gzip -d ' + ofits
os.system(cmd)
ofits = ofits.replace('.gz', '')
except:
continue
#
#--- checking which HRC (S or I)
#
hout = pyfits.open(ofits)
data = hout[1].header
detector = data['DETNAM']
hout.close()
#
#--- creating the center part image
#
line = set_cmd_line(ofits, detector)
ichk = expf.create_image(line, 'ztemp.fits')
#
#--- for HRC S
#
if detector == 'HRC-S' and ichk > 0:
expf.combine_image('ztemp.fits', 'total_s.fits')
hrcs_cnt += 1
#
#--- for HRC I
#
elif detector == 'HRC-I' and ichk > 0:
expf.combine_image('ztemp.fits', 'total_i.fits')
hrci_cnt += 1
mcf.rm_files('out.fits')
mcf.rm_files(ofits)
#
#--- move the file to a depository
#
if hrcs_cnt > 0:
cmd = 'mv total_s.fits ' + web_dir + 'Month_hrc/' + outfile_s
os.system(cmd)
cmd = 'gzip ' + web_dir + '/Month_hrc/*.fits'
os.system(cmd)
if hrci_cnt > 0:
cmd = 'mv total_i.fits ' + web_dir + 'Month_hrc/' + outfile_i
os.system(cmd)
cmd = 'gzip ' + web_dir + '/Month_hrc/*.fits'
os.system(cmd)
createCumulative(year, month, 'HRC-S', web_dir)
createCumulative(year, month, 'HRC-I', web_dir)
def get_data(tstart, tstop, year, grad_list, out_dir):
"""
update msid data in msid_list for the given data period
input: start --- start time in seconds from 1998.1.1
stop --- stop time in seconds from 1998.1.1
year --- the year in which data is extracted
grad_list --- a list of group name in grads
out_dir --- output_directory
"""
print("Period: " + str(tstart) + '<-->' + str(tstop) + ' in Year: ' +
str(year))
#
#--- extract ecach group data
#
for group in grad_list:
print(group)
line = 'operation=retrieve\n'
line = line + 'dataset = mta\n'
line = line + 'detector = grad\n'
line = line + 'level = 0.5\n'
line = line + 'filetype = ' + group + '\n'
line = line + 'tstart = ' + str(tstart) + '\n'
line = line + 'tstop = ' + str(tstop) + '\n'
line = line + 'go\n'
data_list = mcf.run_arc5gl_process(line)
#
#--- read the first fits file and prep for the data list
#
[cols, tbdata] = ecf.read_fits_file(data_list[0])
col_list = []
for ent in cols:
if ent.lower() == 'time':
continue
mc = re.search('st_', ent.lower())
if mc is not None:
continue
col_list.append(ent)
mcf.rm_files(data_list[0])
tdata = tbdata['time']
mdata = []
for col in col_list:
mdata.append(tbdata[col])
#
#--- read the rest of the data
#
clen = len(col_list)
for k in range(1, len(data_list)):
fits = data_list[k]
[cols, tbdata] = ecf.read_fits_file(fits)
tdata = numpy.append(tdata, tbdata['time'])
for m in range(0, clen):
cdata = tbdata[col_list[m]]
mdata[m] = numpy.append(mdata[m], cdata)
mcf.rm_files(fits)
dout = out_dir + group.capitalize() + '/'
if not os.path.isdir(dout):
cmd = 'mkdir ' + dout
os.system(cmd)
#
#--- write out the data to fits file
#
for k in range(0, clen):
col = col_list[k]
ocols = ['time', col.lower()]
cdata = [tdata, mdata[k]]
ofits = dout + col.lower() + '_full_data_' + str(year) + '.fits'
if os.path.isfile(ofits):
ecf.update_fits_file(ofits, ocols, cdata)
else:
ecf.create_fits_file(ofits, ocols, cdata)
#
#--- zip the fits file from the last year at the beginning of the year
#
ecf.check_zip_possible(dout)
