def update_data_files():
"""
main function to run all function to create/update ACIS Dose Plots
input: none, but data are extracted according to current date
output: all data, plots, and html
"""
(uyear, umon, mon_name) = check_date() #--- mon_name is this month's data/plot directory
dir_save = [mon_name]
#
#--- ACIS count rate data
#
data_list = get_data_list()
for dfile in data_list:
#
#--- check when the data is created
#
(tyear, tmonth, tdate) = find_date_obs(dfile)
if tyear == 1900:
continue
#
#--- choose an appropriate output directory
#
cmonth = mcf.change_month_format(tmonth) #--- convert digit to letter
ucmon = cmonth.upper()
tmon_name = web_dir + '/' + ucmon + str(tyear)
#
#--- if the directory is not the current output directory (mon_name), add to dir_list
#
chk = 0
for test in dir_save:
if tmon_name == test:
chk = 1
break
if chk == 0:
dir_save.append(tmon_name)
#
#--- now actually extract data and update/create the count rate data
#
extract_data(dfile, tmon_name)
#
#--- if date is before Nov 2018, extract ephin data
#
if (tyear == 2018) and (tmonth < 11):
ephin_data(dir_save)
elif tyear < 2018:
ephin_data(dir_save)
#
#-- clean the data files
#
for ent in dir_save:
cleanUp(ent)
return dir_save
def extract_monitor_data(year, month, fname, cols, ofile):
"""
extract monitor data from fits files
input: year --- year
month --- month
fname --- none-variable part of fits file name
cols --- a list of column names
ofile --- output file name
"""
#
#--- find the last entry time (seconds from 1998.1.1)
#
if os.path.isfile(ofile):
odata = mcf.read_data_file(ofile)
if len(odata) > 0:
atemp = re.split('\s+', odata[-1])
cut = float(atemp[0])
else:
cut = 0.0
else:
cut = 0.0
lyear = str(year)
ddir = data_dir + 'Fits_data/' + mcf.change_month_format(month).upper()
ddir = ddir + lyear[2] + lyear[3] + '/'
#
#--- data is a list of lists of column data
#
data = read_fits_data(ddir, fname, cols)
if data:
line = ''
#
#--- make sure that the data extracted will be after "cut" date
#
dlen = len(data[0])
kstart = dlen
for k in range(0, dlen):
if data[0][k] > cut:
kstart = k
break
if kstart == dlen:
dlen = 0
if dlen > 0:
for k in range(kstart, dlen):
for m in range(0, len(cols)):
if cols[m] == 'time':
line = line + '%d\t' % data[m][k]
else:
line = line + '%2.3e\t' % data[m][k]
line = line + '\n'
with open(ofile, 'a') as fo:
fo.write(line)
def sdate_to_ldate_with_space(sdate):
"""
change date in second from 1998.1.1 to MMM dd
input: stime --- time in seconds from 1998.1.1
output: ldate --- date in form of MMM dd (e.g. Aug 19)
"""
atemp = re.split('\/', sdate)
mon = int(float(atemp[0]))
lmon = mcf.change_month_format(mon)
ldate = lmon + ' ' + atemp[1]
return ldate
def get_data_for_month(year, month):
"""
extract one month amount of data and update data files
input: year --- year of the data
month --- month of the data
output: updated data files:
<MMM><YYYY>/ccd#
<MMM><YYYY>/ephin_data
Note: there is no ephin data after 2018 Nov
"""
#
#--- if the year/month are not given, extract data of the last month
#
if year == '':
out = time.strftime('%Y:%m' % time.gmtime)
atemp = re.split(':', out)
year = int(float(atemp[0]))
month = int(float(atemp[1]))
month -= 1
if month < 1:
month = 12
year -= 1
cmonth = mcf.change_month_format(month) #--- convert digit to letter month
ucmon = cmonth.upper()
dir_name = data_dir + '/' + ucmon + str(year) + '/' #--- output directory
if os.path.isdir(dir_name):
cmd = 'rm -rf ' + dir_name + '*'
else:
cmd = 'mkdir -p ' + dir_name
os.system(cmd)
#
#--- get acis count rate data
#
extract_acis_count_rate(year, month, dir_name)
#
#--- get ephin rate data; no data after Nov 2018
#
if year < 2018:
get_ephin_data(year, month, dir_name)
elif (year == 2018) and (month < 11):
get_ephin_data(year, month, dir_name)
#
#-- clean the data files
#
cleanUp(dir_name)
def get_obsdate():
"""
read sot database and make a list of obsids and its observation dates
Input: none, but read data from <sot_direcotry>
Output: obs_dict --- a dictionary of obsid <--> starting date
"""
#
#--- read sot data
#
data = mcf.read_data_file(sot_directory)
obsid_list = []
start_date = []
index_date = []
obs_dict = {}
for ent in data:
temp = re.split('\^', ent)
obsid = temp[1]
#
#--- check the data are valid
#
try:
atemp = re.split('\s+', temp[13])
mon = mcf.change_month_format(atemp[0])
date = atemp[1]
year = atemp[2][2] + atemp[2][3]
except:
continue
#
#--- starting date in form of 05/23/14
#
lmon = mcf.add_leading_zero(mon)
ldate = mcf.add_leading_zero(date)
dline = lmon + '/' + ldate + '/' + year
try:
obs_dict[int(obsid)] = dline
except:
pass
return obs_dict
def cleanup_sib_dir(lev, mon, year):
"""
clean up the working directories
input: lev --- data level
mon --- month of the data processed
year --- year of the data processd
output: none
"""
lmon = mcf.change_month_format(mon)
lmon = lmon.lower()
cmd = 'mv ' + cor_dir + lev + '/Outdir/lres '
cmd = cmd + cor_dir + lev + '/Outdir/lres_' + lmon +str(year) + '_modified'
os.system(cmd)
cmd = 'rm -rf ' + cor_dir + lev + '/Outdir/ctirm_dir'
os.system(cmd)
cmd = 'rm -rf ' + cor_dir + lev + '/Outdir/filtered'
os.system(cmd)
cmd = 'rm -rf ' + cor_dir + lev + '/Outdir/hres'
os.system(cmd)
def create_display_data_table():
"""
create a readable data table for html page
Input: none, but read from <data_dir>/ccd<ccd>_<node>
Output: <web_dir>/ccd<ccd>_<node>
"""
for ccd in range(0, 10):
for node in range(0, 4):
ifile = 'ccd' + str(ccd) + '_' + str(node)
#
#--- read the original data file
#
infile = data_dir + ifile
data = mcf.read_data_file(infile)
#
#--- adding heading
#
line = "#\n#Date Mn K alpha Al K alpha "
line = line + "Ti K alpha Slope Sigma Int Sigma\n#\n"
for ent in data:
atemp = re.split('\s+', ent)
#
#--- converting the date format from chandra time into <mon> <year> for html data display
#
stime = int(atemp[0])
out = mcf.convert_date_format(stime,
ifmt='chandra',
ofmt='%Y:%m:%d')
atemp = re.split(':', out)
lmon = mcf.change_month_format(atemp[1])
ldate = lmon + ' ' + atemp[0]
lout = ent.replace(atemp[0], ldate)
line = line + lout + '\n'
outfile = web_dir + 'Data/' + ifile
with open(outfile, 'w') as fo:
fo.write(line)
def find_prev_date(cmon, lev):
"""
find the last extreacted obsid date
input: cmon --- current month
lev --- data level
output: date --- the date which to be used to start extracting data
"""
afile = cor_dir + lev + '/acis_obs'
#
#--- check whether file exist
#
if os.path.isfile(afile):
data = mcf.read_data_file(afile)
if len(data) > 0:
atemp = re.split('\s+', data[-1])
try:
btemp = re.split('T', atemp[-4])
ctemp = re.split('-', btemp[0])
dmon = int(float(ctemp[1]))
day = str(int(float(ctemp[2])))
except:
mon = atemp[-7]
dmon = mcf.change_month_format(mon)
day = atemp[-6]
#
#--- just in a case acis_obs is from the last month..
#
if dmon == cmon:
date = day
else:
date = '1'
else:
date = '1'
else:
date = '1'
return date
def create_weekly_report(date, year, debug=0):
"""
main script to set up the weekly report template for the week
input: date --- date in the format of mmdd (e.g. 0910)
year --- year in the format of yyyy (e.g. 2015)
debug --- if it is other than 0, print out some output
output: weekly report /data/mta4/www/REPORT/<yyyy>/<mm><dd>.html
/data/mta4/www/REPORT/<yyyy>/<mm><dd>_fptemp.png
it also creates local copies in <data_dir>
"""
#
#--- if the test is requested, create Test directory
#
if debug != 0:
os.system('mkdir -p /data/mta/Script/Weekly/TEST/')
hodir = '/data/mta/Script/Weekly/TEST/'
else:
hodir = '/data/mta4/www/REPORTS/'
#
#--- one day in seconds
#
oned = 86400
#
#--- set various time formats
#
syear = str(year) #--- 4 digit year
yrd2 = year[2] + year[3] #--- 2 digit year
year = int(float(year)) #--- integer year
date = str(date)
smon = date[0] + date[1] #--- two digit month
mon = int(float(smon)) #--- integer month
lmon = mcf.change_month_format(mon) #--- month in letter (e.g.Mar)
sday = date[2] + date[3] #--- two digit mday
day = int(float(sday)) #--- integer mday
ltime = str(year) + ':' + str(mon) + ':' + str(day) + 'T00:00:00'
ltime = time.strftime('%Y:%j:%H:%M:%S',
time.strptime(ltime, '%Y:%m:%dT%H:%M:%S'))
stop = int(Chandra.Time.DateTime(ltime).secs) + oned
atemp = re.split(':', ltime)
ydate = int(float(atemp[1]))
day_n = stop - 7 * oned
day01 = stop - 5 * oned
day0 = stop - 6 * oned
lday0 = stime_to_ddate(day0)
sday0 = sdate_to_ldate(lday0)
start = day0
lday1 = stime_to_ddate(day01)
tout = Chandra.Time.DateTime(day0).date
atemp = re.split('\.', tout)
tout = atemp[0]
ttemp = re.split(':', tout)
iru_start = str(ttemp[0]) + '_' + str(ttemp[1])
#
#--- year of the beginning of the period; could be different from that of the end
#
byear = ttemp[0]
lday0 = stime_to_ddate(day0)
day1 = stop - 5 * oned
lday1 = stime_to_ddate(day1)
day2 = stop - 4 * oned
lday2 = stime_to_ddate(day2)
sday2 = sdate_to_ldate(lday2)
day3 = stop - 3 * oned
lday3 = stime_to_ddate(day3)
day4 = stop - 2 * oned
lday4 = stime_to_ddate(day4)
sday4 = sdate_to_ldate(lday4)
day5 = stop - 1 * oned
lday5 = stime_to_ddate(day5)
day6 = stop
lday6 = stime_to_ddate(day6)
sday6 = sdate_to_ldate(lday6)
tout = Chandra.Time.DateTime(day6).date
atemp = re.split('\.', tout)
tout = atemp[0]
ttemp = re.split(':', tout)
iru_stop = '_' + str(ttemp[1])
day7 = stop + 1 * oned
lday7 = stime_to_ddate(day7)
#
#---- setting file name
#
atemp = re.split('\/', lday6)
file_date = atemp[0] + atemp[1]
file_date2 = atemp[0] + '/' + atemp[1]
file_name = file_date + '.html'
#
#--- title
#
titledate = lday0 + ' - ' + lday6
ldate = sdate_to_ldate(lday6)
ldate_sp = sdate_to_ldate_with_space(lday6)
#
#--- focal temp file name
#
paft.plot_acis_focal_temp(year, ydate)
fftp.find_focal_temp_peaks(year, mon, day, 0.3)
fptemp = file_date + '_fptemp.png'
fpext_range = str(start) + ' ' + str(stop)
fpstart = str(start)
fplsub = '"' + sday0 + '", "' + sday2 + '", "' + sday4 + '", "' + sday6 + '"'
fpdsub = str(day0) + ', ' + str(day2) + ', ' + str(day4) + ', ' + str(day6)
#
#--- IRU span
#
irudate = iru_start + iru_stop
irudate = str(syear) + '/' + iru_start + iru_stop
#
#--- telemetry data table
#
tel_start = start - oned
tel_stop = stop - oned
telem_table = ctt.get_telem_data(tel_start, tel_stop)
#
#--- bad pixcel data table
#
bad_pix_table = cbpt.create_bad_pixel_table()
#
#--- recent observation table
#
ro_stop = stop + oned
r_obs_table = frobs.find_recent_observations(ro_stop)
#
#--- find trending dates/title
#
it_date = syear + ':' + smon + ':' + sday
[title, c_t_date, last_trend_date] = find_inst_trend_name(it_date)
#
#--- index.html input
#
s1 = sday0[0:3] + ' ' + sday0[3:5]
s2 = sday6[0:3] + ' ' + sday6[3:5]
index = '<td> <a href="./' + str(
year) + '/' + file_date + '.html">' + s1 + ' - ' + s2 + '</a>'
#
#--- debugging output
#
if debug != 0:
print("file_name; " + file_name)
print("title date: " + titledate)
print("ldate: " + ldate)
print("fptemp: " + fptemp)
print("fpext_range: " + fpext_range)
print("fpstart: " + fpstart)
print(" fplsub: " + fplsub)
print(" fpdsub: " + fpdsub)
print("irudate: " + irudate)
print("title: " + title)
print("last_trend_date: " + last_trend_date)
print("index: " + index)
#
#--- create a work directory
#
outdir = wdir + 'Data/' + ldate + '/'
cmd = 'mkdir -p ' + outdir
os.system(cmd)
#
#--- read the template for the weekly, and start replacing dates etc
#
tfile = tdir + 'this_week'
linput = read_template(tfile)
linput = linput.replace('#DDATE#', file_date)
linput = linput.replace('#IRUSPAN1#', irudate)
linput = linput.replace('#IRUSPAN2#', irudate)
linput = linput.replace('#TITLE#', title)
linput = linput.replace('#TITLEDATE#', titledate)
atemp = last_trend_date
atemp = atemp.replace('/', '')
linput = linput.replace('#PREVREPORT#', atemp)
atemp = re.split('/', last_trend_date)
pmon = int(float(atemp[0]))
lmon = mcf.change_month_format(pmon)
line = lmon + ' ' + atemp[1]
#
#--- the previous report could be from the last year
#
ryear = syear
if mon < pmon:
ryear = year - 1
ryear = str(ryear)
linput = linput.replace('#RYEAR#', ryear)
linput = linput.replace('#PREVDATE#', line)
atitle = str(title)
atitle = atitle.replace(' ', '_')
[temp1, temp2, temp3, temp4, temp5, temp6, temp7,
temp8] = read_cti_values()
linput = linput.replace('#ATEMP#', temp3)
linput = linput.replace('#ATEMP2#', temp4)
linput = linput.replace('#DTEMP#', temp7)
linput = linput.replace('#DTEMP2#', temp8)
[val, step, tval] = read_sim()
linput = linput.replace('#WSTEP#', step)
linput = linput.replace('#WMOVE#', val)
linput = linput.replace('#TMOVE#', tval)
#
#--- read the focal temp peak list
#
tstop = stop + 86400
[fcnt, fdata] = read_focal_temp_data(fptemp, outdir)
linput = linput.replace('#TEMPPEAK#', str(fcnt))
linput = linput.replace('#TEMPLIST#', fdata)
#
#--- bad pixel
#
linput = linput.replace('BAD_PIXEL_TABLE', bad_pix_table)
#
#--- photon
#
linput = linput.replace('PHOTON_TABLE', r_obs_table)
#
#--- telem data
#
linput = linput.replace('TELEM_TABLE', telem_table)
#
#--- trend data
#
trend = set_trend_data_input(title)
linput = linput.replace('#TREND#', trend)
#
#--- write out the weekly report
#
ofile = outdir + file_name
with open(ofile, 'w') as fo:
fo.write(linput)
#
#--- move files
#
move_files(date, year, outdir, file_name, fptemp, hodir)
#
#--- send out email to admin; notify the job complete
#
send_email_to_admin(date, year)
def get_data_for_day(year, month, day=''):
"""
extract one month amount of data and update data files
input: year --- year of the data
month --- month of the data
output: updated data files:
<MMM><YYYY>/ccd#
<MMM><YYYY>/ephin_data
Note: there is no ephin data after 2018 Nov
"""
#
#--- if the date is given...
#
chk = 0
if year != '':
lmon = mcf.add_leading_zero(month)
lday = mcf.add_leading_zero(day)
if day != '':
start = str(year) + ':' + lmon + ':' + lday + ':00:00:00'
stop = str(year) + ':' + lmon + ':' + lday + ':23:59:59'
#
#-- if the day part is not given, compute for the entire month
#
else:
nyear = year
nmon = mon + 1
if nmon > 12:
nyear += 1
nmon = 1
nmon = mcf.add_leading_zero(nmon)
start = str(year) + ':' + lmon + ':' + ':01:00:00:00'
stop = str(nyear) + ':' + nmom + ':' + ':01:00:00:00'
chk = 1
start = mcf.convert_date_format(start,
ifmt='%Y:%m:%d:%H:%M:%S',
ofmt='chandra')
stop = mcf.convert_date_format(stop,
ifmt='%Y:%m:%d:%H:%M:%S',
ofmt='chandra')
#
#--- if the year/month are not given, extract data of the day before
#
if year == '':
out = time.strftime('%Y:%j:00:00:00', time.gmtime())
stop = Chandra.Time.DateTime(out).secs
start = stop - 86400.0
out = mcf.convert_date_format(start, ifmt='chandra', ofmt='%Y:%m')
out = re.split(':', out)
year = int(float(out[0]))
month = int(float(out[1]))
#
#--- set output directory name
#
cmonth = mcf.change_month_format(month) #--- convert digit to letter month
ucmon = cmonth.upper()
dir_name = data_dir + '/' + ucmon + str(year) + '/' #--- output directory
if not os.path.isdir(dir_name):
cmd = 'mkdir -p ' + dir_name
os.system(cmd)
#
#--- if the entire month data needs to be extracted, remove all previous data
#
if chk == 1:
cmd = 'rm -rf ' + dir_name + '/ccd*'
os.system(cmd)
#
#--- get acis count rate data
#
extract_acis_count_rate(start, stop, dir_name)
#
#--- get ephin rate data; no data after Nov 2018
#
if year < 2018:
get_ephin_data(start, stop, dir_name)
elif (year == 2018) and (month < 11):
get_ephin_data(start, stop, dir_name)
#
#-- clean the data files
#
cleanUp(dir_name)
def print_html_page(indir, outdir, hrc, data):
"""
create HTML page to display HRC historical data.
input: indir --- input dir
outdir --- output dir
hrc --- hrc
data --- a list of lists of data
output: <outdir>/<hrc>.html
"""
#
#--- open the data ; except the first three, all others are lists of data
#
(date, year, month, mean_acc, std_acc, min_acc, min_apos, \
max_acc, max_apos, asig1, asig2, asig3, mean_dff, std_dff,\
min_dff, min_dpos, max_dff, max_dpos, dsig1, dsig2, dsig3) = data
outdir = outdir + '/' + hrc + '.html'
#
#--- start writing the html page
#
line = '<!DOCTYPE html>\n'
line = line + '<html>\n'
line = line + '<head>\n'
line = line + '<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\n'
line = line + '<style type="text/css">\n'
line = line + 'body{background-color:#FFEBCD;}\n'
line = line + 'table{text-align:center;margin-left:auto;margin-right:auto;'
line = line + 'border-style:solid;border-spacing:8px;border-width:2px;'
line = line + 'border-collapse:separate}\n'
line = line + 'a:link {color:green;}\n'
line = line + 'a:visited {color:red;}\n'
line = line + 'td{text-align:center;padding:8px}\n'
line = line + '</style>\n'
if hrc == 'hrci':
hname = 'HRC I'
wname = 'HRCI'
else:
hname = 'HRC S'
wname = 'HRCS'
line = line + '<title>' + hname + ' History Data</title>\n'
line = line + '</head>\n'
line = line + '<body>\n'
line = line + '<h2 style="text-align:center">Data: ' + hname + '</h2>\n'
line = line + '<div style="padding-bottom:30px">\n'
line = line + '<table border=1>\n'
line = line + '<tr><th> </th><th> </th><th colspan=11>'
line = line + 'Monthly</th><th colspan=11>Cumulative</th></tr>\n'
line = line + '<tr style="color:blue"><th>Year</th><th>Month</th>\n'
line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>\n'
line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>'
line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n'
line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>'
line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>'
line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n'
for i in range(0, len(date)):
smonth = mcf.add_leading_zero(month[i])
cmonth = mcf.change_month_format(month[i])
syear = str(int(year[i]))
#
#--- monthly HRC dose data
#
if mean_dff[i] == 0 and std_dff[i] == 0:
#
#--- for the case there is no data for this month
#
line = line + '<tr><td>%d</td><td>%d</td>' % (year[i], month[i])
line = line + '<td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td>\n'
line = line + '<td>NA</td><td>NA</td><td>NA</td><td>No Data</td><td>No Image</td>\n'
#line = line + '</tr>\n'
else:
line = line + '<tr>'
line = line + '<td>%d</td>' % year[i]
line = line + '<td>%d</td>' % month[i]
line = line + '<td>%4.4f</td>' % mean_dff[i]
line = line + '<td>%4.4f</td>' % std_dff[i]
line = line + '<td>%4.1f</td>' % min_dff[i]
line = line + '<td>%s</td>' % min_dpos[i]
line = line + '<td>%4.1f</td>' % max_dff[i]
line = line + '<td>%s</td>' % max_dpos[i]
line = line + '<td>%4.1f</td>' % dsig1[i]
line = line + '<td>%s</td>' % dsig2[i]
line = line + '<td>%4.1f</td>\n' % dsig3[i]
line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month_hrc/'
line = line + wname + '_' + smonth + '_' + syear + '.fits.gz'
line = line + '">fits</a></td>\n'
line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/'
line = line + wname + '_' + smonth + '_' + syear + '.png'
line = line + '">map</a></td>\n'
#
#---- cumulative HRC dose data
#
line = line + '<td>%4.4f</td>' % mean_acc[i]
line = line + '<td>%4.4f</td>' % std_acc[i]
line = line + '<td>%4.1f</td>' % min_acc[i]
line = line + '<td>%s</td>' % min_apos[i]
line = line + '<td>%4.1f</td>' % max_acc[i]
line = line + '<td>%s</td>' % max_apos[i]
line = line + '<td>%4.1f</td>' % asig1[i]
line = line + '<td>%s</td>' % asig2[i]
line = line + '<td>%4.1f</td>\n' % asig3[i]
line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative_hrc/'
line = line + wname + '_08_1999_' + smonth + '_' + syear + '.fits.gz'
line = line + '">fits</a></td>\n'
line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/'
line = line + wname + '_08_1999_' + smonth + '_' + syear + '.png'
line = line + '">map</a></td>\n'
line = line + '</tr>\n\n'
#
#--- put header every new year so that we can read data easier
#
if month[i] % 12 == 0 and i != (len(date) - 1):
line = line + '\n<tr style="color:blue"><th>Year</th><th>Month</th>'
line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>'
line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>'
line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n'
line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>'
line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>'
line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n\n'
line = line + '</table>\n\n'
line = line + "</div>\n"
line = line + '<hr />\n'
#
#--- set current date
#
[tyear, tmon, tday] = mcf.today_date()
lmon = mcf.add_leading_zero(tmon)
lday = mcf.add_leading_zero(tday)
line = line + '<p style="padding-top:10px;padding-bottom:10px">'
line = line + '<strong style="font-size:105%;float:right">Last Update: '
line = line + lmon + '/' + lday + '/' + str(tyear) + '</strong></p>\n'
line = line + '<p>If you have any questions about this page, contact '
line = line + ' <a href="mailto:[email protected]">[email protected].</a></p>\n'
line = line + '</body>\n'
line = line + '</html>\n'
with open(outdir, 'w') as fo:
fo.write(line)
def create_html_page(data, hrc, sec, ctop):
"""
create HTML page to display HRC historical data.
input: data --- a list of lists of data:
date,year,month,mean_acc,std_acc,min_acc,min_apos,
max_acc,max_apos,asig1, asig2, asig3, mean_dff,std_dff,
min_dff, min_dpos,max_dff,max_dpos, dsig1, dsig2, dsig3
hrc --- hrci or hrcs
sec --- section
ctop --- max number of section
output: <web_dir>/Sub_html/<inst>_<sec>.html
"""
#
#--- today's date
#
today = time.strftime('%Y:%m:%d', time.gmtime())
atemp = re.split(':', today)
year = int(float(atemp[0]))
mon = int(float(atemp[1]))
day = int(float(atemp[2]))
smon = str(mon)
if mon < 10:
smon = '0' + smon
sday = str(day)
if day < 10:
sday = '0' + sday
#
#--- start composing a html page
#
aline = read_template('sub_top1')
if hrc == 'hrci':
hname = 'HRC I'
wname = 'HRCI'
inst = 'i'
else:
hname = 'HRC S'
wname = 'HRCS'
inst = 's'
aline = aline + '<title>' + hname + ' Section: ' + str(
sec) + ' History Data</title>\n'
aline = aline + "</head>\n"
aline = aline + '<body style="color:white;background-color:black">\n'
aline = aline + '<div><a href="../hrc_exposure_map.html">Back to Top</a>\n'
aline = aline + '<h2 style="text-align:center">Data: ' + hname
aline = aline + ' Section: ' + str(sec) + '\n'
if sec == 0:
aline = aline + ' (<a href="./hrc' + inst + '_' + str(
sec + 1) + '.html">Next</a>)'
elif sec == ctop - 1:
aline = aline + ' (<a href="./hrc' + inst + '_' + str(
sec - 1) + '.html">Prev</a>)'
else:
aline = aline + ' (<a href="./hrc' + inst + '_' + str(
sec - 1) + '.html">Prev</a> : '
aline = aline + '<a href="./hrc' + inst + '_' + str(
sec + 1) + '.html">Next</a>)'
aline = aline + '</h2>\n'
aline = aline + "<div style='padding-bottom:30px'>\n"
aline = aline + '<table border=1>\n'
aline = aline + read_template('sub_col_header')
#
#--- open the data to indivisual lists
#
[year,month,mean_acc,std_acc,min_acc,min_apos, \
max_acc,max_apos,asig1, asig2, asig3, mean_dff,std_dff,\
min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3] = data
dlen = len(year)
for i in range(0, dlen):
smonth = str(int(month[i]))
if month[i] < 10:
smonth = '0' + smonth
syear = int(year[i])
#
#--- converting digit to letters, i.e. 1 to Jan
#
cmonth = mcf.change_month_format(month[i])
#
#--- monthly HRC dose data
#
if mean_dff[i] == 0 and std_dff[i] == 0:
aline = aline + '<tr><td>%d</td><td>%d</td>' % (year[i], month[i])
aline = aline + read_template('sub_nodata')
else:
aline = aline + '<tr><td>%d</td>' % year[i]
aline = aline + '<td>%d</td>' % month[i]
try:
aline = aline + '<td>%4.4f</td>' % mean_dff[i]
aline = aline + '<td>%4.4f</td>' % std_dff[i]
#aline = aline + '<td>%4.1f</td>' % min_dff[i]
#aline = aline + '<td>%s</td>' % min_dpos[i]
aline = aline + '<td>%4.1f</td>' % max_dff[i]
aline = aline + '<td>%s</td>' % max_dpos[i]
aline = aline + '<td>%4.1f</td>' % dsig1[i]
aline = aline + '<td>%s</td>' % dsig2[i]
aline = aline + '<td>%4.1f</td>\n' % dsig3[i]
# if hrc == 'hrci':
# aline = aline + '<td><a href="' + data_i_dir + '/Month/'
# else:
# aline = aline + '<td><a href="' + data_s_dir + '/Month/'
# aline = aline + wname + '_' + smonth + '_' + str(syear) + '.fits.gz">fits</a></td>\n'
aline = aline + '<td><a href="' + web_address + 'Image/' + wname + '/Month/'
aline = aline + wname + '_' + smonth + '_' + str(syear)
aline = aline + '_' + str(sec) + '.html">map</a></td>\n'
except:
#
#--- for the case there is no data, print 'na'
#
aline = aline + '<td>na</td>'
aline = aline + '<td>na</td>'
aline = aline + '<td>na</td>'
aline = aline + '<td>na</td>'
aline = aline + '<td>na</td>'
aline = aline + '<td>na</td>'
aline = aline + '<td>na</td>\n'
aline = aline + '<td>na</td>\n'
#
#---- cumulative HRC dose data
#
aline = aline + '<td>%4.4f</td>' % mean_acc[i]
aline = aline + '<td>%4.4f</td>' % std_acc[i]
#aline = aline + '<td>%4.1f</td>' % min_acc[i]
#aline = aline + '<td>%s</td>' % min_apos[i]
aline = aline + '<td>%4.1f</td>' % max_acc[i]
aline = aline + '<td>%s</td>' % max_apos[i]
aline = aline + '<td>%4.1f</td>' % asig1[i]
aline = aline + '<td>%s</td>' % asig2[i]
aline = aline + '<td>%4.1f</td>\n' % asig3[i]
# if hrc == 'hrci':
# aline = aline + '<td><a href="' + data_i_dir + '/Cumulative/'
# else:
# aline = aline + '<td><a href="' + data_s_dir + '/Cumulative/'
# aline = aline + wname + '_' + smonth + '_' + str(syear) + '.fits.gz">fits</a></td>\n'
aline = aline + '<td><a href="' + web_address + 'Image/' + wname + '/Cumulative/'
aline = aline + wname + '_08_1999_' + smonth + '_' + str(
syear) + '_' + str(sec) + '.html">map</a></td>\n'
#
#--- put header every new year so that we can read data easier
#
if month[i] % 12 == 0 and i != (dlen - 1):
aline = aline + read_template('sub_col_header')
aline = aline + '</table>\n\n'
aline = aline + '<div style="padding_top:10px; padding_bottom:10px;">'
aline = aline + '<a href="../hrc_exposure_map.html">Back to Top</a>\n'
#
#--- add today's date as update date
#
today = time.strftime('%Y:%m:%d', time.gmtime())
atemp = re.split(':', today)
cmon = mcf.change_month_format(int(float(atemp[1])))
today = cmon + ' ' + atemp[2] + ', ' + atemp[0]
aline = aline + read_template('sub_footer').replace('#UPDATE#', today)
outdir = web_dir + 'Sub_html/' + hrc + '_' + str(sec) + '.html'
with open(outdir, 'w') as fo:
fo.write(aline)
def ephin_data(dir_save):
"""
extract ephin data
input: dir_save --- a list of directory
output: <out_dir>/ephin_data --- ephin data (300 sec accumulation)
Note: EPHIN data is not available after Nov 2018
"""
data_list = get_ephin_list()
for ifile in data_list:
try:
(tyear, tmonth, tdate) = find_date_obs(ifile)
if tyear == 1900:
continue
cmonth = mcf.change_month_format(tmonth)
ucmon = cmonth.upper()
tmon_name = web_dir + '/' + ucmon + str(tyear)
chk = 0
for test in dir_save:
if tmon_name == test:
chk = 1
break
if chk == 0:
dir_save.append(tmon_name)
extract_ephin_data(ifile, tmon_name)
except:
pass
#----------------------------------------------------------------------------------------
#--- check_date: check wether there is an output directory and if it is not, create one
#---------------------------------------------------------------------------------------
def check_date():
"""
check wether there is an output directory and if it is not, create one
input: none
output: year --- the current year
mon --- the current month
mdir --- the current output direcotry
"""
#
#--- find today's date
#
out = time.strftime('%Y:%j:00:00:00', time.gmtime())
stime = Chandra.Time.DateTime(out).secs)
#
#--- find 10 days ago and create the output directory based on that date
#
s10 = stime - 10 * 86400.0
out = mcf.convert_date_format(s10, ifmt='chandra', ofmt='%Y:%m:%d')
atemp = re.split(':', out)
year = int(float(atemp[0]))
mon = int(float(atemp[1]))
lmon = mcf.change_month_format(mon)
mdir = web_dir + lmon.upper() + str(year)
if not os.path.isdir(mdir)
cmd = 'mkdir -p ' + mfile
os.system(cmd)
return (year, mon, mdir)
#---------------------------------------------------------------------------------------
#-- get_data_list: compare the current input list to the old one and select data -
#---------------------------------------------------------------------------------------
def get_data_list():
"""
compare the current input list to the old one and select out the data which are not used
input: house_keeping/old_file_list --- previous data list
house_keeping/bad_fits_file --- bad fits data list
the data are read from /dsops/ap/sdp/cache/*/acis/*evt1.fits (if it is an actual run)
output: input_data --- the data list
"""
#
#--- create a current data file list
#
cmd = 'ls -d /dsops/ap/sdp/cache/*/acis/*evt1.fits > ' + zspace
os.system(cmd)
#
#--- use only non-calibration data files
#
data = mcf.read_data_file(zspace, remove=1)
file_list = find_obs_data_files(data)
#
#--- read the old file list
#
ifile = house_keeping + 'old_file_list'
old_list = mcf.read_data_file(ifile)
#
#--- read bad fits file list
#
try:
file2 = house_keeping + 'bad_fits_file'
bad_list = mcf.read_data_file(file2)
except:
bad_list = []
#
#--- update "old_file_list"
#
with open(ifile, 'w') as fo:
for line in file_list:
fo.write(line + '\n')
#
#--- compara two files and select out new file names
#
n_list = list(numpy.setdiff1d(file_list, old_list))
if len(bad_list) > 0:
n_list = list(numpy.setdiff1d(n_list, bad_list))
return n_list
#---------------------------------------------------------------------------------------
#-- find_obs_data_files: choose files with only non-calibration data files --
#---------------------------------------------------------------------------------------
def find_obs_data_files(data):
"""
choose files with only non-calibration data files
input: data --- a list of data file names
output: file_list --- a list of non-calibration data file
"""
file_list = []
for ent in data:
atemp = re.split('acisf', ent)
btemp = re.split('_', atemp[1])
#
#--- for the older format : acisf16218N001_evt1.fits
#--- for the new format : acisf16218_000N001_evt1.fits
#
try:
val = float(btemp[0])
mark= int(float(val))
except:
ctemp = re.split('N', btemp[1])
mark = int(float(ctemp[0]))
if mark < 50000:
file_list.append(ent)
return file_list
#---------------------------------------------------------------------------------------
#--- find_date_obs: find observation time ---
#---------------------------------------------------------------------------------------
def find_date_obs(ifile):
"""
find observation time
input: file --- input fits file name
output: year, month, and date
"""
try:
dout = pyfits.open(ifile)
date = dout[0].header['DATE-OBS']
atemp = re.split('T', date)
btemp = re.split('-', atemp[0])
year = int(btemp[0])
month = int(btemp[1])
date = int(btemp[2])
return (year, month, date)
except:
return (1900, 1, 1)
#---------------------------------------------------------------------------------------
#--- extract_data: extract time and ccd_id from the fits file and create count rate data
#---------------------------------------------------------------------------------------
def extract_data(file, out_dir ):
"""
extract time and ccd_id from the fits file and create count rate data
input: file --- fits file data
out_dir --- the directory in which data will be saved
output: ccd<ccd>--- 5 min accumulated count rate data file
"""
#
#--- extract time and ccd id information from the given file
#
data = pyfits.getdata(file, 0)
time_col = data.field('TIME')
ccdid_col = data.field('CCD_ID')
#
#--- initialize
#
diff = 0
chk = 0
ccd_c = [0 for x in range(0, 10)]
ccd_h = [[] for x in range(0, 10)]
#
#--- check each line and count the numbers of ccd in the each 300 sec intervals
#
for k in range(0, len(time_col)):
try:
ftime = float(time_col[k])
ccd_id = int(ccdid_col[k])
except:
continue
if ftime > 0:
if chk == 0:
ccd_c[ccd_id] += 1
stime = ftime
diff = 0
chk = 1
elif diff >= 300.0:
#
#--- print out counts per 300 sec
#
for i in range(0, 10):
line = str(stime) + '\t' + str(ccd_c[i]) + '\n'
ccd_h[i].append(line)
#
#--- re initialize for the next round
#
ccd_c[i] = 0
ccd_c[ccd_id] += 1
stime = ftime
diff = 0
chk = 0
#
#--- accumurate the count until the 300 sec interval is reached
#
else:
diff = ftime - stime
ccd_c[ccd_id] += 1
#
#--- for the case the last interval is less than 300 sec,
#--- estimate the the numbers of hit and adjust
#
if diff > 0 and diff < 300:
ratio = 300.0 / diff
for i in range(0, 10):
ccd_c[i] *= ratio
line = str(stime) + '\t' + str(ccd_c[i]) + '\n'
ccd_h[i].append(line)
for i in range(0, 10):
ifile = out_dir + '/ccd' + str(i)
with open(ifile, 'a') as fo:
for ent in ccd_h[i]:
fo.write(ent)
#---------------------------------------------------------------------------------------
#--- get_ephin_list: create an ephin data list ---
#---------------------------------------------------------------------------------------
def get_ephin_list():
"""
create an ephin data list
input: ephin data --- /dsops/ap/sdp/cache/*/ephin/*lc1.fits
output: input_data_list: a list of ephin data file which is not extract
"""
cmd = 'ls -d /dsops/ap/sdp/cache/*/ephin/*lc1.fits > '
cmd = cmd + house_keeping + '/ephin_dir_list'
os.system(cmd)
#
#--- get a list of the current entries
#
ifile = house_keeping + '/ephin_dir_list'
data = mcf.read_data_file(ifile)
#
#--- read the list which we read the last time
#
ifile = house_keeping + '/ephin_old_dir_list'
old = mcf.read_data_file(ifile)
#
#--- find the last entry of the list
#
last_entry = old[len(old)-1]
#
#--- select data which is new
#
input_data_list = []
chk = 0
for ent in data:
if chk == 0:
if ent == last_entry:
chk = 1
else:
input_data_list.append(ent)
#
#--- replace the old list with the new one
#
ifile = house_keeping + '/ephin_dir_list'
if os.path.isfile(ifile):
cmd = 'mv ' + house_keeping + '/ephin_dir_list ' + house_keeping + '/ephin_old_dir_list'
os.system(cmd)
return input_data_list
#---------------------------------------------------------------------------------------
#-- extract_ephin_data: extract ephine data from a given data file name and save it in out_dir
#---------------------------------------------------------------------------------------
def extract_ephin_data(ifile, out_dir):
"""
extract ephine data from a given data file name and save it in out_dir
input: ifile --- ephin data file name
out_dir --- directory which the data is saved
output: <out_dir>/ephin_data --- ephin data (300 sec accumulation)
"""
#
#--- extract time and ccd id information from the given file
#
data = pyfits.getdata(ifile, 1)
time_r = data.field("TIME")
scp4_r = data.field("SCP4")
sce150_r = data.field("SCE150")
sce300_r = data.field("SCE300")
sce1500_r = data.field("SCE1300")
#
#--- initialize
#
diff = 0
chk = 0
ephin_data = []
#
#--- sdata[0]: scp4, sdata[1]: sce150, sdata[2]: sce300, and sdata[3]: sce1300
#
sdata = [0 for x in range(0, 4)]
#
#--- check each line and count the numbers of ccd in the each 300 sec intervals
#
for k in range(0, len(time_r)):
try:
ftime = float(time_r[k])
val1 = float(scp4_r[k])
val2 = float(sce150_r[k])
val3 = float(sce300_r[k])
val4 = float(sce1500_r[k])
except:
continue
if ftime > 0:
if chk == 0:
sdata[0] += val1
sdata[1] += val2
sdata[2] += val3
sdata[3] += val4
stime = ftime
diff = 0
chk = 1
elif diff >= 300.0:
#
#--- print out counts per 300 sec
#
line = str(stime) + '\t'
for j in range(0, 4):
line = line + str(sdata[j]) + '\t'
sdata[j] = 0
line = line + '\n'
ephin_data.append(line)
chk = 0
#
#--- re initialize for the next round
#
try:
val1 = float(scp4_r[k])
val2 = float(sce150_r[k])
val3 = float(sce300_r[k])
val4 = float(sce1500_r[k])
except:
continue
sdata[0] += val1
sdata[1] += val2
sdata[2] += val3
sdata[3] += val4
stime = ftime
diff = 0
#
#--- accumurate the count until the 300 sec interval is reached
#
else:
diff = ftime - stime
try:
val1 = float(scp4_r[k])
val2 = float(sce150_r[k])
val3 = float(sce300_r[k])
val4 = float(sce1500_r[k])
except:
continue
sdata[0] += val1
sdata[1] += val2
sdata[2] += val3
sdata[3] += val4
#
#--- for the case the last interval is less than 300 sec,
#--- estimate the the numbers of hit and adjust
#
if diff > 0 and diff < 300:
line = str(stime) + '\t'
ratio = 300.0 / diff
for j in range(0, 4):
var = sdata[j] * ratio
line = line + str(var) + '\t'
line = line + '\n'
ephin_data.append(line)
ifile = out_dir + '/ephin_rate'
with open(ifile, 'a') as fo:
for ent in ephin_data:
fo.write(ent)
#---------------------------------------------------------------------------------------
#-- cleanUp: sort and remove duplicated lines in all files in given data directory ---
#---------------------------------------------------------------------------------------
def cleanUp(cdir):
"""
sort and remove duplicated lines in all files in given data directory
input cdir --- directory name
output cdir/files --- cleaned up files
"""
if os.listdir(cdir) != []:
cmd = 'ls ' + cdir + '/* > ' + zspace
os.system(cmd)
data = mcf.read_data_file(zspace, remove=1)
for ifile in data:
#
#--- avoid html and png files
#
try:
m = re.search('\.', ifile)
if m is None:
mcf.remove_duplicated_lines(ifile, chk = 1, srt=1)
except:
pass
#-----------------------------------------------------------------------------------------
#-- TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST ---
#-----------------------------------------------------------------------------------------
class TestFunctions(unittest.TestCase):
"""
testing functions
"""
def test_check_date(self):
out = check_date()
print("YEAR: MON: Dir: " + str(out))
#------------------------------------------------------------
if __name__ == "__main__":
unittest.main()
def create_sub_html_page(ifile=''):
"""
read the old html page and create new sub aca subpage
input: ifile --- old html page, if not given, read from /Temp_comp_area/acatrd.html
output: <web_dir>/<MMM><yy>/acatrd.html. plots are also copied to that directory
"""
ichk = 0
if ifile == '':
ifile = exc_dir + '/Temp_comp_area/acatrd.html'
ichk = 1
#
#--- create the data lists from the old formatted html page
#
data, tsave = extract_date_from_old_html(ifile)
#
#--- create a new table
#
htable = create_html_table(data)
#
#--- find the year and month of the data created
#
out = Chandra.Time.DateTime(0.5 * (tsave[0] + tsave[1])).date
atemp = re.split('\.', out)
out = time.strftime('%Y:%m:%d', time.strptime(atemp[0], '%Y:%j:%H:%M:%S'))
atemp = re.split(':', out)
year = atemp[0]
mon = atemp[1]
dyear = int(float(year))
dmon = int(float(mon))
lmon = mcf.change_month_format(mon)
#
#--- read the template
#
tfile = house_keeping + 'Template/sub_page'
with open(tfile, 'r') as f:
template = f.read()
#
#--- substitute the values
#
template = template.replace('#YEAR#', year)
template = template.replace('#MON#', lmon)
template = template.replace('#TABLE#', htable)
#
#--- substitute magnitude slope and std values
#
ifile = data_dir + 'monthly_mag_stats'
template = substitue_mag_stat(template, ifile, dyear, dmon)
#
#--- substitute slot slope and std values
#
ifile = data_dir + 'diff_mtatr_month_slope'
template = substitue_slot_stat(template, ifile, dyear, dmon, 'POS')
ifile = data_dir + 'diff_mtatr_month_slope'
template = substitue_slot_stat(template, ifile, dyear, dmon, 'DIFF')
ifile = data_dir + 'acacent_mtatr_month_slope'
template = substitue_slot_stat(template, ifile, dyear, dmon, 'ANGY')
template = substitue_slot_stat(template,
ifile,
dyear,
dmon,
'ANGZ',
col_start=10)
#
#--- substitute slot tables
#
gtable = make_slot_star_table(dyear, dmon, 'guide_gsst')
template = template.replace('#GUIDE_LOOK#', gtable)
ftable = make_slot_star_table(dyear, dmon, 'fid_gsst')
template = template.replace('#FID_LOOK#', ftable)
mtable = make_slot_star_table(dyear, dmon, 'monitor_gsst')
template = template.replace('#MON_LOOK#', mtable)
#
#--- check the output directory and if it does not exist, create
#
odir = web_dir + lmon.upper() + year[2] + year[3] + '/'
if not os.path.isdir(odir):
cmd = 'mkdir -p ' + odir
os.system(cmd)
#
#--- write the new html page
#
ofile = odir + 'acatrd.html'
with open(ofile, 'w') as fo:
fo.write(template)
def make_slot_star_table(dyear, dmon, tail):
"""
create a slot data table for the monthly html page
input: dyear --- year
dmon --- month
tail --- a part of fits file name used to find the file
output: line --- a html table string
"""
#
#--- find data fits file
#
lyear = str(dyear)
ddir = data_dir + 'Fits_data/' + mcf.change_month_format(
dmon).upper() + lyear[2] + lyear[3] + '/'
cmd = 'ls ' + ddir + '*_' + tail + '.fits.gz > ' + zspace
os.system(cmd)
#
#--- if there is no data fits file, say so
#
out = mcf.read_data_file(zspace, remove=1)
if len(out) == 0:
return '<h3>No Slot Data</h3>'
#
#--- read the data fits file
#
fits = out[0].strip()
fout = pyfits.open(fits)
fdata = fout[1].data
fout.close()
#
#--- extract each column data
#
good = fdata['good']
marginal = fdata['marginal']
bad = fdata['bad']
ra_rms = fdata['rms_ra_err']
dec_rms = fdata['rms_dec_err']
mag_rms = fdata['rms_delta_mag']
angy = fdata['avg_angynea']
angz = fdata['avg_angznea']
nums = fdata['number']
num_nea = fdata['number_nea']
#
#--- create slot data html data table
#
line = '<table border=1 cellpadding=3>\n'
line = line + '<tr>\n'
line = line + '<th>slot</th><th>rms_ra_err (deg)</th><th>rms_dec_err (deg)</th>\n'
line = line + '<th>rms_delta_mag (marcsec)</th><th>number</th>\n'
line = line + '<th>avg_angynea (arcsec)</th><th>avg_angznea (arcsec)</th>\n'
line = line + '<th>number_nea</th>\n'
line = line + '<th>good</th><th>marginal</th><th>bad</th>\n'
line = line + '</tr>\n'
for k in range(0, 8):
line = line + '<tr>\n'
line = line + '<th>' + str(k) + '</th>'
line = line + '<td>%2.3e</td>' % ra_rms[k]
line = line + '<td>%2.3e</td>' % dec_rms[k]
line = line + '<td>%2.3e</td>' % mag_rms[k]
line = line + '<td>%4d</td>' % nums[k]
line = line + '<td>%2.3e</td>' % angy[k]
line = line + '<td>%2.3e</td>' % angz[k]
line = line + '<td>%4d</td>' % num_nea[k]
line = line + '<td>%2.2f</td>' % good[k]
line = line + '<td>%2.2f</td>' % marginal[k]
line = line + '<td>%2.2f</td>' % bad[k]
line = line + '</tr>\n'
line = line + '<tr>\n'
line = line + '<th>AVG</th>'
line = line + '<td>%2.3e</td>' % numpy.mean(ra_rms)
line = line + '<td>%2.3e</td>' % numpy.mean(dec_rms)
line = line + '<td>%2.3e</td>' % numpy.mean(mag_rms)
line = line + '<td>%4d</td>' % numpy.mean(nums)
line = line + '<td>%2.3e</td>' % numpy.mean(angy)
line = line + '<td>%2.3e</td>' % numpy.mean(angz)
line = line + '<td>%4d</td>' % numpy.mean(num_nea)
line = line + '<td>%2.2f</td>' % numpy.mean(good)
line = line + '<td>%2.2f</td>' % numpy.mean(marginal)
line = line + '<td>%2.2f</td>' % numpy.mean(bad)
line = line + '</tr>\n'
line = line + '</table>'
return line
def acis_dose_monthly_report(year='NA', month='NA'):
"""
create monthly report tables
input: year --- year
month --- month
output: monthly_diff_<mm>_<yyyy>
monthly_acc_<mm>_<yyyy>
"""
#
#--- if year and/or month is not given, find the latest year month entry
#
if year == 'NA' or month == 'NA':
ifile = data_out + 'i_2_n_0_acc_out'
data = mcf.read_data_file(ifile)
line = data[-1]
atemp = re.split('\s+|\t+', line)
year = int(atemp[0])
month = int(atemp[1])
syear = str(year)
smon = mcf.add_leading_zero(month)
dline = ''
aline = ''
#
#--- convert month in digit to month in letter
#
lmon = mcf.change_month_format(month)
lmon = lmon.lower()
#
#--- find monthly stat
#
diff = mon_acis_dir + 'ACIS_' + smon + '_' + syear + '.fits.gz'
(mean, std, dmin, dmax) = getstat(diff)
dline = dline + 'ACIS_' + lmon + syear[2] + syear[3] + ' 6004901 '
dline = dline + '%3.3f %3.3f %3.1f %4d\n\n' % (
mean, std, dmin, dmax)
#
#--- find cumulative stat
#
acc = cum_acis_dir + 'ACIS_07_1999_' + smon + '_' + syear + '.fits.gz'
(mean, std, amin, amax) = getstat(acc)
aline = aline + 'ACIS_total 6004901 '
aline = aline + '%3.3f %3.3f %3.1f %6d\n\n' % (
mean, std, amin, amax)
#
#--- now print stat for each section
#
for inst in ('i', 's'): #--- acis i or s
for ccd in (2, 3): #--- ccd 2 or 3 (i2/i3/s2/s3)
dline = dline + '\n'
aline = aline + '\n'
for node in (0, 1, 2, 3): #--- node
for dtype in ('dff', 'acc'): #--- monthly or cumulative
ifile = data_out + inst + '_' + str(ccd) + '_n_'
ifile = ifile + str(node) + '_' + dtype + '_out'
data = mcf.read_data_file(ifile)
if year == 'NA' or month == 'NA':
out = data[-1]
atemp = re.split('\s+', out)
else:
for ent in data:
atemp = re.split('\s+', ent)
if int(atemp[0]) == year and int(
atemp[1]) == month:
break
mline = inst.upper() + str(ccd) + ' node ' + str(
node) + ' 262654\t'
mline = mline + '%3.6f\t' % (float(atemp[2]))
mline = mline + '%3.6f\t' % (float(atemp[3]))
mline = mline + '%3.1f\t' % (float(atemp[4]))
mline = mline + '%5.1f\n' % (float(atemp[6]))
if dtype == 'dff':
dline = dline + mline
else:
aline = aline + mline
ofile = './monthly_diff_' + smon + '_' + syear
with open(ofile, 'w') as fo:
fo.write(dline)
ofile = './monthly_acc_' + smon + '_' + syear
with open(ofile, 'w') as fo:
fo.write(aline)
def find_scheduled_obs():
"""
find MP scheduled observations
input: none but read from /data/mpcrit1/mplogs/* directory
ouptput <obs_ss>/scheduled_obs_list
"""
#
#--- find today's date
#
out = time.strftime('%Y:%m:%d', time.gmtime())
atemp = re.split(':', out)
year = int(float(atemp[0]))
mon = int(float(atemp[1]))
mday = int(float(atemp[2]))
umon = mcf.change_month_format(mon).upper()
lmon = mcf.change_month_format(mon).lower()
#
#--- next month
#
nyear = year
nmon = mon + 1
if mon > 12:
nyear += 1
nmon = 1
unmon = mcf.change_month_format(nmon).upper()
lnmon = mcf.change_month_format(nmon).lower()
#
#--- find all MP OR lists scheduled to start after today
#
cmd = 'ls -rtd /data/mpcrit1/mplogs/' + str(
year) + '/' + umon + '* > ' + zspace
os.system(cmd)
d_list = mcf.read_data_file(zspace, remove=1)
#
#--- find direcotries hold data after today's date
#--- the directory name: <MMM><dd><yy>, e.g. JUL2020
#
n_list = []
for ent in d_list:
atemp = re.split(umon, ent)
date = atemp[1][0] + atemp[1][1]
date = int(float(date))
if date > mday:
n_list.append(ent)
#
#--- find all *.or files and mp persons responsible for that schedule
#
[f_list, u_list] = find_file_and_mp(n_list)
#
#--- check the next month, if directory exists
#
cmd = 'ls /data/mpcrit1/mplogs/' + str(nyear) + '> ' + zspace
os.system(cmd)
with open(zspace, 'r') as f:
test = f.read()
mcf.rm_files(zspace)
mc = re.search(unmon, test)
if mc is not None:
cmd = 'ls -rltd /data/mpcrit1/mplogs/' + str(
nyear) + '/' + unmon + '* >> ' + zspace
os.system(cmd)
d_list = mcf.read_data_file(zspace, remove=1)
[f_list2, u_list2] = find_file_and_mp(d_list, nchk=1)
f_list = f_list + f_list2
u_list = u_list + u_list2
#
#--- get all observations under schedules
#
m_list = [] #--- a list of obsids
o_dict = {} #--- a dictionary to keep info related obsids
for k in range(0, len(f_list)):
ifile = f_list[k]
mp_person = u_list[k]
[o_dict, m_list] = find_obsids(ifile, o_dict, m_list, mp_person)
#
#--- remove duplicate ans sort it
#
m_list = sorted(list(set(m_list)))
#
#--- print out the scheduled list
#
line = ''
for obsid in m_list:
line = line + str(obsid) + '\t' + o_dict[obsid][-1] + '\n'
ofile = obs_ss + 'scheduled_obs_list'
with open(ofile, 'w') as fo:
fo.write(line)
def update_one_year_pages(iyear=''):
"""
upate yearly html page
input: year --- year. if it is not given, update all years
output: <web_dir>/aca_trend_year<yyyy>.html
"""
#
#--- find the current year and month
#
out = time.strftime('%Y:%m:%d', time.gmtime())
atemp = re.split(':', out)
this_year = int(float(atemp[0]))
this_month = int(float(atemp[1]))
this_day = int(float(atemp[2]))
if this_day < 5:
this_month -= 1
if this_month < 1:
this_month = 12
this_year -= 1
#
#--- set the year interval to create the pages
#
if iyear == '':
syear = 1999
eyear = this_year + 1
else:
syear = iyear
eyear = iyear + 1
for year in range(syear, eyear):
lyear = str(year)
syear = lyear[2] + lyear[3]
tfile = house_keeping + 'Template/one_year_template'
with open(tfile, 'r') as f:
template = f.read()
template = template.replace('#YEAR#', lyear)
template = template.replace('#SYEAR#', syear)
#
#--- substitue slope and std of magnitude data
#
dfile = 'yearly_mag_stats'
template = substitue_year_page(template, dfile, year, 14, 'MAG')
#
#--- substitute slope and std of pos_err
#
dfile = 'pos_err_mtatr_year_slope'
template = substitue_year_page(template, dfile, year, 8, 'POS')
#
#--- substitue slope and std of magnitue difference
#
dfile = 'diff_mtatr_year_slope'
template = substitue_year_page(template, dfile, year, 8, 'DIFF')
#
#--- substitute slope and std of angy
#
dfile = 'acacent_mtatr_year_slope'
template = substitue_year_page(template, dfile, year, 8, 'ANGY')
#
#--- subsititute slope and std of angz
#
dfile = 'acacent2_mtatr_year_slope'
template = substitue_year_page(template, dfile, year, 8, 'ANGZ')
#
#--- create a table to link to the each month of this year
#
if year == 1999:
smon = 8
emon = 12
elif year == this_year:
smon = 1
emon = this_month
else:
smon = 1
emon = 12
line = '<table border=1 cellspacing=2 cellpadding=2>\n'
line = line + '<tr>\n'
for mon in range(1, 13):
lmon = mcf.change_month_format(mon).upper()
if mon < smon:
line = line + '<td>' + lmon + '</td>\n'
elif mon > emon:
line = line + '<td>' + lmon + '</td>\n'
else:
line = line + '<td><a href="./' + lmon + syear
line = line + '/acatrd.html">' + lmon + '</a>\n'
line = line + '</table>\n'
template = template.replace('#LTABLE#', line)
#
#--- update the html page
#
outname = web_dir + 'aca_trend_year' + str(year) + '.html'
with open(outname, 'w') as fo:
fo.write(template)
def run_hrc_stowed_background_monthly_update(lyear='', lmonth=''):
"""
run monthly update for the hrc stowed background data
input: lyear, lmonth --- the year and month of the data to be processed
if they are not given, the last month of year and year will be used
output: newly proccessed data: <data_dir>/<yyyy><mmm>/*
if it is jan or jul, cummulative maps for the year (and the entire period) are created
"""
#
#--- clean up exc dir
#
cmd = 'rm -rf ' + exc_dir + 'Temp_dir/* 2> /dev/null'
os.system(cmd)
cmd = 'rm -rf ' + exc_dir + 'param 2> /dev/null'
os.system(cmd)
cmd = 'mkdir ' + exc_dir + 'param 2> /dev/null'
os.system(cmd)
#
#--- if date (year/month) to be processed is not given, find the last month's year, month
#
if lyear == '':
ctime = time.gmtime() #--- today's date information
lyear = ctime.tm_year
lmonth = ctime.tm_mon -1
if lmonth < 1:
lyear -= 1
lmonth = 12
chk = 0
else:
chk = 1
#
#--- check whether the data was already processed; if not, run the process
#--- except re-analysis is requested by given year and month
#
smonth = mcf.change_month_format(lmonth)
cdir = data_dir + str(lyear) + smonth.upper()
if chk == 0 and (not os.path.isdir(cdir)):
chk = 1
smail = 0
if chk == 1:
try:
hsb.hrc_stowed_background(lyear, lmonth, lyear, lmonth)
#
#--- if the processed month is june, septempber or december, update maps and html pages
#
if (lmonth == 12) or (lmonth == 6) or (lmonth == 9):
#
#--- update total event number tables
#
chif.yearly_cummulative_image(lyear)
get_yearly_evt_count() #---- create yearly event counts
hpm.plot_hrc_map(lyear)
hpm.plot_hrc_map('total') #---- creating cumulative maps
hpm.plot_hrc_map('total', 1) #---- creating cumulative histogram for the front page
hpr.plot_hrc_trend_data() #---- creating trending plots
uhhp.create_html_pages(lyear) #---- updating all html pages
uhhp.update_stat_data_table() #---- updating stat html pages
create_cumulative_image()
if test_data_creation(lyear, lmonth):
smail = 1
else:
smail = 3
except:
smail = 2
else:
smail = 0
#
#--- remove the temp file if it is still around
#
cmd = 'rm -f /tmp/zspace*hrc* 2> /dev/null'
os.system(cmd)
cmd = 'rm -rf ' + exc_dir + 'Temp_dir/* 2> /dev/null'
os.system(cmd)
#
#--- sending mail
#
if smail == 1:
message = 'hrc stowed proccess ran normally for the period: '
message = message + str(lyear) + ' : ' + str(lmonth) + '\n'
header = '"Subject: HRC Stowed Data Process Complated"'
elif smail == 2:
message = 'hrc stowed proccess ran into problems with the data for the period: '
message = message + str(lyear) + ' : ' + str(lmonth) + '\n'
header = '"Subject: HRC Stowed Data Process Failed"'
elif smail == 3:
message = 'hrc stowed proccess did not find any data for the period: '
message = message + str(lyear) + ' : ' + str(lmonth) + '\n'
header = '"Subject: HRC Stowed Data Process: No Data Found"'
if smail > 0:
send_email(message, header)