def extract_hrcveto_data():
"""
extract hrc veto data
input: none
output: fits file data related to grad and comp
"""
#
#--- set basic information
#
group = 'Hrcveto'
cols = ['TLEVART', 'VLEVART', 'SHEVART']
[udict, ddict, mta_db, mta_cross] = ecf.get_basic_info_dict()
#
#--- find the date to be filled
#
ctime = ecf.find_the_last_entry_time(testfits)
start = Chandra.Time.DateTime(ctime).date
today = time.strftime("%Y:%j:00:00:00", time.gmtime())
ctime = Chandra.Time.DateTime(today).secs - 43200.0
stop = Chandra.Time.DateTime(ctime).date
print("Group: " + group + ': ' + str(start) + '<-->' + str(stop))
[xxx, tbdata] = uds.extract_data_arc5gl('hrc', '0', 'hrcss', start, stop)
#
#--- get time data in the list form
#
dtime = list(tbdata.field('time'))
for col in cols:
#
#---- extract data in a list form
#
data = list(tbdata.field(col))
#
#--- change col name to msid
#
msid = col.lower()
#
#--- get limit data table for the msid
#
try:
tchk = ecf.convert_unit_indicator(udict[msid])
except:
tchk = 0
glim = ecf.get_limit(msid, tchk, mta_db, mta_cross)
#
#--- update database
#
uds.update_database(msid, group, dtime, data, glim)
def update_simdiag_data(date=''):
"""
collect sim diag msids
input: date ---- the date in yyyymmdd format. if not given, yesterday's date is used
output: fits file data related to simdiag
"""
#
#--- read group names which need special treatment
#
sfile = house_keeping + 'msid_list_simdiag'
data = mcf.read_data_file(sfile)
cols = []
g_dir = {}
for ent in data:
atemp = re.split('\s+', ent)
cols.append(atemp[0])
g_dir[atemp[0]] = atemp[1]
#
#--- get the basic information
#
[udict, ddict, mta_db, mta_cross] = ecf.get_basic_info_dict()
#
#--- find date to read the data
#
if date == '':
date_list = ecf.create_date_list_to_yestaday(testfits)
else:
date_list = [date]
for sday in date_list:
sday = sday[:4] + '-' + sday[4:6] + '-' + sday[6:]
print("Date: " + sday)
start = sday + 'T00:00:00'
stop = sday + 'T23:59:59'
[xxxx, tbdata] = uds.extract_data_arc5gl('sim', '0', 'simdiag', start,
stop)
#
#--- get time data in the list form
#
try: #---03/07/19
dtime = list(tbdata.field('time'))
except:
continue
for k in range(0, len(cols)):
col = cols[k]
#
#---- extract data in a list form
#
data = list(tbdata.field(col))
#
#--- change col name to msid
#
msid = col.lower()
#
#--- get limit data table for the msid
#
try:
tchk = ecf.convert_unit_indicator(udict[msid])
except:
tchk = 0
glim = ecf.get_limit(msid, tchk, mta_db, mta_cross)
#
#--- update database
#
uds.update_database(msid, g_dir[msid], dtime, data, glim)
def update_grad_and_comp_data(date=''):
"""
collect grad and comp data for trending
input: date ---- the data colletion end date in yyyymmdd format. if not given, yesterday's date is used
output: fits file data related to grad and comp
"""
#
#--- read group names which need special treatment
#
sfile = house_keeping + 'mp_process_list'
glist = mcf.read_data_file(sfile)
#
#--- create msid <---> unit dictionary
#
[udict, ddict] = ecf.read_unit_list()
#
#--- read mta database
#
mta_db = ecf.read_mta_database()
#
#--- read mta msid <---> sql msid conversion list
#
mta_cross = ecf.read_cross_check_table()
#
#--- find date to read the data
#
if date == '':
date_list = ecf.create_date_list_to_yestaday(testfits)
else:
date_list = [date]
for day in date_list:
#
#--- find the names of the fits files of the day of the group
#
print("Date: " + str(day))
for group in glist:
print("Group: " + str(group))
cmd = 'ls /data/mta_www/mp_reports/' + day + '/' + group + '/data/mta*fits* > ' + zspace
os.system(cmd)
flist = mcf.read_data_file(zspace, remove=1)
#
#--- combined them
#
flen = len(flist)
if flen == 0:
continue
elif flen == 1:
cmd = 'cp ' + flist[0] + ' ./ztemp.fits'
os.system(cmd)
else:
mfo.appendFitsTable(flist[0], flist[1], 'ztemp.fits')
if flen > 2:
for k in range(2, flen):
mfo.appendFitsTable('ztemp.fits', flist[k], 'out.fits')
cmd = 'mv out.fits ztemp.fits'
os.system(cmd)
#
#--- read out the data for the full day
#
[cols, tbdata] = ecf.read_fits_file('ztemp.fits')
cmd = 'rm -f ztemp.fits out.fits'
os.system(cmd)
#
#--- get time data in the list form
#
dtime = list(tbdata.field('time'))
for k in range(1, len(cols)):
#
#--- select col name without ST_ (which is standard dev)
#
col = cols[k]
mc = re.search('ST_', col)
if mc is not None:
continue
#
#---- extract data in a list form
#
data = list(tbdata.field(col))
#
#--- change col name to msid
#
msid = col.lower()
#
#--- get limit data table for the msid
#
try:
tchk = ecf.convert_unit_indicator(udict[msid])
except:
tchk = 0
glim = ecf.get_limit(msid, tchk, mta_db, mta_cross)
#
#--- update database
#
uds.update_database(msid, group, dtime, data, glim)
def update_eph_data_from_comm(date = ''):
"""
collect eph data for trending
input: date ---- the data collection end date in yyyymmdd format.
if not given, yesterday's date is used
output: fits file data related to grad and comp
"""
#
#--- read group names which need special treatment
#
#sfile = house_keeping + 'eph_list'
#glist = mcf.read_data_file(sfile)
glist = ['ephhk',]
#
#--- create msid <---> unit dictionary
#
[udict, ddict] = ecf.read_unit_list()
#
#--- read mta database
#
mta_db = ecf.read_mta_database()
#
#--- read mta msid <---> sql msid conversion list
#
mta_cross = ecf.read_cross_check_table()
#
#--- find date to read the data
#
if date == '':
yesterday = datetime.date.today() - datetime.timedelta(1)
yesterday = str(yesterday).replace('-', '')
date_list = create_date_list(yesterday)
else:
date_list = [date]
error_message = ''
for day in date_list:
#
#--- find the names of the fits files of the day of the group
#
dline = "Date: " + str(day)
print(dline)
for group in glist:
print("Group: " + str(group))
cmd = 'ls /data/mta_www/mp_reports/' + day + '/' + group + '/data/* > ' + zspace
os.system(cmd)
tlist = mcf.read_data_file(zspace, remove=1)
flist = []
for ent in tlist:
mc = re.search('_STephhk_static_eio0.fits', ent)
if mc is not None:
flist.append(ent)
#
#--- combined them
#
flen = len(flist)
if flen == 0:
continue
elif flen == 1:
cmd = 'cp ' + flist[0] + ' ./ztemp.fits'
os.system(cmd)
else:
mcf.rm_files('ztemp.fits')
mfo. appendFitsTable(flist[0], flist[1], 'ztemp.fits')
if flen > 2:
for k in range(2, flen):
mfo. appendFitsTable('ztemp.fits', flist[k], 'out.fits')
cmd = 'mv out.fits ztemp.fits'
os.system(cmd)
#
#--- read out the data for the full day
#
[cols, tbdata] = ecf.read_fits_file('ztemp.fits')
cmd = 'rm -f ztemp.fits out.fits'
os.system(cmd)
#
#--- get time data in the list form
#
dtime = list(tbdata.field('time'))
for k in range(1, len(cols)):
#
#--- select col name without ST_ (which is standard dev)
#
col = cols[k]
mc = re.search('ST_', col)
if mc is not None:
continue
mc = re.search('quality', col, re.IGNORECASE)
if mc is not None:
continue
#
#---- extract data in a list form
#
data = list(tbdata.field(col))
#
#--- change col name to msid
#
msid = col.lower()
#
#--- get limit data table for the msid
#
try:
tchk = ecf.convert_unit_indicator(udict[msid])
except:
tchk = 0
glim = ecf.get_limit(msid, tchk, mta_db, mta_cross)
#
#--- update database
#
wline = uds.update_database(msid, group.capitalize(), dtime, data, glim)
if wline != "":
error_message = error_message + dline + '\n' + wline
#
#--- if there are errors, sending error message
#
if error_message != "":
error_message = 'MTA limit trend EPH got problems: \n' + error_message
fo = open(zspace, 'w')
fo.write(error_message)
fo.close()
cmd = 'cat ' + zspace + ' | mailx -s "Subject: EPH data update problem "'
cmd = cmd + '*****@*****.**'
os.system(cmd)
mcf.rm_files(zspace)
def update_simsuppl_data(date = ''):
"""
collect sim msids data
input: date ---- the date in yyyymmdd format. if not given, yesterday's date is used
output: fits file data related to sim
"""
#
#--- read group names which need special treatment
#
sfile = house_keeping + 'msid_list_simactu_supple'
data = ecf.read_file_data(sfile)
cols = []
g_dir = {}
for ent in data:
atemp = re.split('\s+', ent)
cols.append(atemp[0])
g_dir[atemp[0]] = atemp[1]
#
#--- get the basic information
#
[udict, ddict, mta_db, mta_cross] = ecf.get_basic_info_dict()
#
#--- find date to read the data
#
if date == '':
yesterday = datetime.date.today() - datetime.timedelta(1)
yesterday = str(yesterday).replace('-', '')
date_list = find_the_last_entry_time(yesterday)
else:
date_list = [date]
for sday in date_list:
print "Date: " + sday
start = sday + 'T00:00:00'
stop = sday + 'T23:59:59'
[xxx, tbdata] = uds.extract_data_arc5gl('sim', '0', 'sim', start, stop)
#
#--- get time data in the list form
#
dtime = list(tbdata.field('time'))
for k in range(0, len(cols)):
#
#--- select col name without ST_ (which is standard dev)
#
col = cols[k]
#
#---- extract data in a list form
#
data = list(tbdata.field(col))
#
#--- change col name to msid
#
msid = col.lower()
#
#--- get limit data table for the msid
#
try:
tchk = ecf.convert_unit_indicator(udict[msid])
except:
tchk = 0
glim = ecf.get_limit(msid, tchk, mta_db, mta_cross)
#
#--- update database
#
uds.update_database(msid, g_dir[msid], dtime, data, glim)