def update_main_html():
"""
update main html page
input: none but read a template from <house_keeping>
output: <web_dir>/expousre.html
"""
[tyear, mon, day] = mcf.today_date()
today = mcf.change_month_format(mon)
today = today + ' ' + mcf.add_leading_zero(day)
today = today + ', ' + str(tyear)
syear = tyear
smon = mon - 1
if smon < 1:
smon = 12
syear = tyear - 1
lyear = str(syear)
lmon = mcf.add_leading_zero(smon)
line = lmon + '_' + lyear
aline = mcf.change_month_format(smon) + ' ' + lyear
ifile = house_keeping + 'template'
with open(ifile, 'r') as f:
data = f.read()
data = data.replace('#ODATE#', line)
data = data.replace('#DATE#', aline)
data = data.replace('#TODAY#', today)
ifile = web_dir + 'exposure.html'
with open(ifile, 'w') as fo:
fo.write(data)
def update_plt_html_date():
"""
update html pages for plots; just replacing date
no input, but get the list from plot_dir
"""
[lyear, lmon, lday] = mcf.today_date()
date = mcf.add_leading_zero(lmon) + '/'
date = date + mcf.add_leading_zero(lday) + '/' + str(lyear)
cmd = 'ls ' + plot_dir + '/*html>./ztemp'
os.system(cmd)
data = mcf.read_data_file('./ztemp', remove=1)
for ent in data:
hdat = mcf.read_data_file(ent)
sline = ''
for oline in hdat:
m = re.search('Last Update', oline)
if m is not None:
sline = sline + 'Last Update: ' + date + '\n'
else:
sline = sline + oline + '\n'
with open(ent, 'w') as fo:
fo.write(sline)
def create_hrc_maps(year= 'NA', month= 'NA'):
"""
create HRC image maps for given year and month
input: year --- year
month --- month
output: <outdir>/<out.png
"""
#
#--- if year and month are not given, set the date of the last month
#
if year == 'NA' or month == 'NA':
[year, month, day] = mcf.today_date()
month -= 1
if month < 1:
month = 12
year -= 1
#
#--- create images
#
hrc_dose_conv_to_png(mon_hrc_dir, img_dir, year, month)
hrc_dose_conv_to_png(cum_hrc_dir, img_dir, year, month)
def write_html(ccd, sec, data, outfile):
"""
write a html page:
input: ccd --- ccd; e.g., i2, s3
sec --- section 0 - 3
data --- a list of lists of data
date, year,month,mean_acc,std_acc,min_acc,min_apos,
max_acc,max_apos,m10_acc, m10_apos,mean_dff,std_dff,
min_dff, min_dpos,max_dff,max_dpos,m10_dff,m10_dpos
outfile --- output file name
output: outfile
"""
#
#--- open the list of lists
#
[date, year,month,mean_acc,std_acc,min_acc,min_apos,\
max_acc,max_apos, asig1, asig2, asig3, mean_dff,std_dff,\
min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3] = data
#
#--- today's date
#
[lyear, lmon, lday] = mcf.today_date()
#
#--- start writing a html page
#
sline = '<!DOCTYPE html>\n'
sline = sline + '<html>\n'
sline = sline + '<head>\n'
sline = sline + '<title>ACIS ' + ccd.upper() + ' Section '
sline = sline + str(sec) + ' History Data</title>\n'
sline = sline + "<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />"
#
#--- css style sheet
#
sline = sline + '<style type="text/css">\n'
sline = sline + 'body{background-color:#FFEBCD;}\n'
sline = sline + 'table{text-align:center;margin-left:auto;margin-right:'
sline = sline + 'auto;border-style:solid;border-spacing:8px;border-width:'
sline = sline + '2px;border-collapse:separate}\n'
sline = sline + 'td {text-align:center}\n'
sline = sline + 'a:link {color:green;}\n'
sline = sline + 'a:visited {color:red;}\n'
sline = sline + '</style>\n'
sline = sline + '</head>\n'
sline = sline + '<body> \n'
# sline = sline + '<br /><h3> Last Update: '
# sline = sline + mcf.add_leading_zero(lmon) + '/'
# sline = sline + mcf.add_leading_zero(lday) + '/'
# sline = sline + str(lyear) + '</h3>\n'
sline = sline + '<table border=1>\n'
sline = sline + header_write() + '\n'
for i in range(0, len(year)):
sline = sline + '<tr>\n'
sline = sline + '<td>' + str(year[i]) + '</td>\t'
sline = sline + '<td>' + str(month[i]) + '</td>\t'
sline = sline + '<td>' + str(mean_dff[i]) + '</td>\t'
sline = sline + '<td>' + str(std_dff[i]) + '</td>\t'
sline = sline + '<td>' + str(min_dff[i]) + '</td>\t'
sline = sline + '<td>' + str(min_dpos[i]) + '</td>\t'
sline = sline + '<td>' + str(max_dff[i]) + '</td>\t'
sline = sline + '<td>' + str(max_dpos[i]) + '</td>\t'
sline = sline + '<td>' + str(dsig1[i]) + '</td>\t'
sline = sline + '<td>' + str(dsig2[i]) + '</td>\t'
sline = sline + '<td>' + str(dsig3[i]) + '</td>\t'
syear = str(int(year[i]))
smon = mcf.add_leading_zero(month[i])
lccd = ccd.replace('_', '')
ifile = 'ACIS_' + smon + '_' + syear + '_' + lccd
sline = sline + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month/'
sline = sline + ifile + '.fits.gz">fits</a></td>\n'
sline = sline + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/'
sline = sline + ifile + '.png">map</a></td>\n\n'
sline = sline + '<td>' + str(mean_acc[i]) + '</td>\t'
sline = sline + '<td>' + str(std_acc[i]) + '</td>\t'
sline = sline + '<td>' + str(min_acc[i]) + '</td>\t'
sline = sline + '<td>' + str(min_apos[i]) + '</td>\t'
sline = sline + '<td>' + str(max_acc[i]) + '</td>\t'
sline = sline + '<td>' + str(max_apos[i]) + '</td>\t'
sline = sline + '<td>' + str(asig1[i]) + '</td>\t'
sline = sline + '<td>' + str(asig2[i]) + '</td>\t'
sline = sline + '<td>' + str(asig3[i]) + '</td>\n'
ifile = 'ACIS_07_1999_' + smon + '_' + syear + '_' + lccd
sline = sline + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative/'
sline = sline + ifile + '.fits.gz">fits</a></td>\n'
sline = sline + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/'
sline = sline + ifile + '.png">map</a></td>'
sline = sline + '</tr>\n\n'
#
#--- put header every new year so that we can read data easier
#
if month[i] % 12 == 0 and i != (len(year) - 1):
sline = sline + header_write() + '\n'
sline = sline + '</table>\n\n'
sline = sline + '<br /><br /><hr /><br />\n'
sline = sline + '<br /><strong style="font-size:105%;float:right">Last Update: '
sline = sline + mcf.add_leading_zero(lmon) + '/'
sline = sline + mcf.add_leading_zero(lday) + '/'
sline = sline + str(lyear) + '</strong>\n'
sline = sline + '<p>If you have any questions about this page, contact '
sline = sline + '<a href="mailto:[email protected]">'
sline = sline + '[email protected].</a></p>\n'
sline = sline + '</body>\n'
sline = sline + '</html>\n'
with open(outfile, 'w') as fo:
fo.write(sline)
def hrc_dose_run(year='NA', month='NA'):
"""
run all needed HRC scripts to extract data and create images
input: year --- year
month --- month
"""
#
#--- if year and month are given, get that year and month for data extraction
#
if year != 'NA' and str(year).isdigit() and str(month).isdigit():
lyear = int(year)
lmonth = int(month)
#
#--- if year and month are not given, use a month before the current year/month
#
else:
[year, month, day] = mcf.today_date()
lyear = year
lmonth = month - 1
if lmonth < 1:
lmonth = 12
lyear -= 1
#
#--- extracting data
#
try:
hgdata.hrc_dose_get_data(lyear, lmonth, lyear, lmonth)
except:
print("Data Extraction Failed!!")
subject = 'HRC data extraction problem'
content = 'Extraction of HRC data for exposure map trend failed.'
expf.send_warning_email(subject, content)
# exit(1)
#
#--- computing statistics
#
try:
hstat.hrc_dose_extract_stat_data_month(lyear, lmonth)
except:
print("Stat computation failed")
pass
#
#--- creating html pages
#
try:
hhtml.hrc_dose_make_data_html()
except:
print("HTML page construction failed")
pass
#
#--- plotting histories
#
try:
hplot.hrc_dose_plot_exposure_stat()
except:
print("Trend plotting failed")
pass
#
#--- creating map images
#
try:
himg.create_hrc_maps(lyear, lmonth)
except:
print("HRC map construction failed")
pass
#
#--- plotting monthly report trend
#
try:
monthly.hrc_dose_plot_monthly_report()
except:
print("Plot for monthly report failed")
pass
def print_html_page(indir, outdir, hrc, data):
"""
create HTML page to display HRC historical data.
input: indir --- input dir
outdir --- output dir
hrc --- hrc
data --- a list of lists of data
output: <outdir>/<hrc>.html
"""
#
#--- open the data ; except the first three, all others are lists of data
#
(date, year, month, mean_acc, std_acc, min_acc, min_apos, \
max_acc, max_apos, asig1, asig2, asig3, mean_dff, std_dff,\
min_dff, min_dpos, max_dff, max_dpos, dsig1, dsig2, dsig3) = data
outdir = outdir + '/' + hrc + '.html'
#
#--- start writing the html page
#
line = '<!DOCTYPE html>\n'
line = line + '<html>\n'
line = line + '<head>\n'
line = line + '<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\n'
line = line + '<style type="text/css">\n'
line = line + 'body{background-color:#FFEBCD;}\n'
line = line + 'table{text-align:center;margin-left:auto;margin-right:auto;'
line = line + 'border-style:solid;border-spacing:8px;border-width:2px;'
line = line + 'border-collapse:separate}\n'
line = line + 'a:link {color:green;}\n'
line = line + 'a:visited {color:red;}\n'
line = line + 'td{text-align:center;padding:8px}\n'
line = line + '</style>\n'
if hrc == 'hrci':
hname = 'HRC I'
wname = 'HRCI'
else:
hname = 'HRC S'
wname = 'HRCS'
line = line + '<title>' + hname + ' History Data</title>\n'
line = line + '</head>\n'
line = line + '<body>\n'
line = line + '<h2 style="text-align:center">Data: ' + hname + '</h2>\n'
line = line + '<div style="padding-bottom:30px">\n'
line = line + '<table border=1>\n'
line = line + '<tr><th> </th><th> </th><th colspan=11>'
line = line + 'Monthly</th><th colspan=11>Cumulative</th></tr>\n'
line = line + '<tr style="color:blue"><th>Year</th><th>Month</th>\n'
line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>\n'
line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>'
line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n'
line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>'
line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>'
line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n'
for i in range(0, len(date)):
smonth = mcf.add_leading_zero(month[i])
cmonth = mcf.change_month_format(month[i])
syear = str(int(year[i]))
#
#--- monthly HRC dose data
#
if mean_dff[i] == 0 and std_dff[i] == 0:
#
#--- for the case there is no data for this month
#
line = line + '<tr><td>%d</td><td>%d</td>' % (year[i], month[i])
line = line + '<td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td>\n'
line = line + '<td>NA</td><td>NA</td><td>NA</td><td>No Data</td><td>No Image</td>\n'
#line = line + '</tr>\n'
else:
line = line + '<tr>'
line = line + '<td>%d</td>' % year[i]
line = line + '<td>%d</td>' % month[i]
line = line + '<td>%4.4f</td>' % mean_dff[i]
line = line + '<td>%4.4f</td>' % std_dff[i]
line = line + '<td>%4.1f</td>' % min_dff[i]
line = line + '<td>%s</td>' % min_dpos[i]
line = line + '<td>%4.1f</td>' % max_dff[i]
line = line + '<td>%s</td>' % max_dpos[i]
line = line + '<td>%4.1f</td>' % dsig1[i]
line = line + '<td>%s</td>' % dsig2[i]
line = line + '<td>%4.1f</td>\n' % dsig3[i]
line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month_hrc/'
line = line + wname + '_' + smonth + '_' + syear + '.fits.gz'
line = line + '">fits</a></td>\n'
line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/'
line = line + wname + '_' + smonth + '_' + syear + '.png'
line = line + '">map</a></td>\n'
#
#---- cumulative HRC dose data
#
line = line + '<td>%4.4f</td>' % mean_acc[i]
line = line + '<td>%4.4f</td>' % std_acc[i]
line = line + '<td>%4.1f</td>' % min_acc[i]
line = line + '<td>%s</td>' % min_apos[i]
line = line + '<td>%4.1f</td>' % max_acc[i]
line = line + '<td>%s</td>' % max_apos[i]
line = line + '<td>%4.1f</td>' % asig1[i]
line = line + '<td>%s</td>' % asig2[i]
line = line + '<td>%4.1f</td>\n' % asig3[i]
line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative_hrc/'
line = line + wname + '_08_1999_' + smonth + '_' + syear + '.fits.gz'
line = line + '">fits</a></td>\n'
line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/'
line = line + wname + '_08_1999_' + smonth + '_' + syear + '.png'
line = line + '">map</a></td>\n'
line = line + '</tr>\n\n'
#
#--- put header every new year so that we can read data easier
#
if month[i] % 12 == 0 and i != (len(date) - 1):
line = line + '\n<tr style="color:blue"><th>Year</th><th>Month</th>'
line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>'
line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>'
line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n'
line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>'
line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>'
line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n\n'
line = line + '</table>\n\n'
line = line + "</div>\n"
line = line + '<hr />\n'
#
#--- set current date
#
[tyear, tmon, tday] = mcf.today_date()
lmon = mcf.add_leading_zero(tmon)
lday = mcf.add_leading_zero(tday)
line = line + '<p style="padding-top:10px;padding-bottom:10px">'
line = line + '<strong style="font-size:105%;float:right">Last Update: '
line = line + lmon + '/' + lday + '/' + str(tyear) + '</strong></p>\n'
line = line + '<p>If you have any questions about this page, contact '
line = line + ' <a href="mailto:[email protected]">[email protected].</a></p>\n'
line = line + '</body>\n'
line = line + '</html>\n'
with open(outdir, 'w') as fo:
fo.write(line)
def hrc_dose_get_data(startYear='NA',
startMonth='NA',
stopYear='NA',
stopMonth='NA'):
"""
extract HRC evt1 data from a month and create cumulative data fits file.
input: startYear --- start year
startMonth --- start month
stopYear --- stop year
stopMonth --- stop month
output: image fits files for the month and cumulative cases
"""
#
#--- if the dates are not given, set them to the last month
#
if startYear == 'NA':
[stopYear, stopMonth, day] = mcf.today_date()
startYear = stopYear
stopMonth -= 1
startMonth = stopMonth
if startMonth < 1:
startMonth = 12
startYear -= 1
#
#--- start extracting the data for the year/month period
#
for year in range(startYear, stopYear + 1):
lyear = str(year)
syear = lyear[2] + lyear[3]
#
#--- create a list of month appropriate for the year
#
month_list = expf.make_month_list(year, startYear, stopYear,
startMonth, stopMonth)
for month in month_list:
smonth = mcf.add_leading_zero(month)
#
#--- output file name settings
#
outfile_i = './HRCI_' + str(smonth) + '_' + str(lyear) + '.fits'
outfile_s = './HRCS_' + str(smonth) + '_' + str(lyear) + '.fits'
#
#--- using ar5gl, get file names
#
smonth = mcf.add_leading_zero(month)
syear = str(year)
start = syear + '-' + smonth + '-01T00:00:00'
nextMonth = month + 1
if nextMonth > 12:
lyear = year + 1
nextMonth = 1
else:
lyear = year
smonth = mcf.add_leading_zero(nextMonth)
syear = str(lyear)
stop = str(lyear) + '-' + smonth + '-01T00:00:00'
line = 'operation=browse\n'
line = line + 'dataset=flight\n'
line = line + 'detector=hrc\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'tstart=' + start + '\n'
line = line + 'tstop=' + stop + '\n'
line = line + 'go\n'
fitsList = mcf.run_arc5gl_process(line)
#
#--- extract each evt1 file, extract the central part, and combine them into a one file
#
#--- set counters for how many hrc-i and hrc-s are extracted
#
hrci_cnt = 0
hrcs_cnt = 0
for fitsName in fitsList:
print("Fits file: " + fitsName)
m = re.search('fits', fitsName)
if m is None:
continue
try:
line = 'operation=retrieve\n'
line = line + 'dataset=flight\n'
line = line + 'detector=hrc\n'
line = line + 'level=1\n'
line = line + 'filetype=evt1\n'
line = line + 'filename=' + fitsName + '\n'
line = line + 'go\n'
out = mcf.run_arc5gl_process(line)
if len(out) < 1:
continue
ofits = out[0]
cmd = 'gzip -d ' + ofits
os.system(cmd)
ofits = ofits.replace('.gz', '')
except:
continue
#
#--- checking which HRC (S or I)
#
hout = pyfits.open(ofits)
data = hout[1].header
detector = data['DETNAM']
hout.close()
#
#--- creating the center part image
#
line = set_cmd_line(ofits, detector)
ichk = expf.create_image(line, 'ztemp.fits')
#
#--- for HRC S
#
if detector == 'HRC-S' and ichk > 0:
expf.combine_image('ztemp.fits', 'total_s.fits')
hrcs_cnt += 1
#
#--- for HRC I
#
elif detector == 'HRC-I' and ichk > 0:
expf.combine_image('ztemp.fits', 'total_i.fits')
hrci_cnt += 1
mcf.rm_files('out.fits')
mcf.rm_files(ofits)
#
#--- move the file to a depository
#
if hrcs_cnt > 0:
cmd = 'mv total_s.fits ' + web_dir + 'Month_hrc/' + outfile_s
os.system(cmd)
cmd = 'gzip ' + web_dir + '/Month_hrc/*.fits'
os.system(cmd)
if hrci_cnt > 0:
cmd = 'mv total_i.fits ' + web_dir + 'Month_hrc/' + outfile_i
os.system(cmd)
cmd = 'gzip ' + web_dir + '/Month_hrc/*.fits'
os.system(cmd)
createCumulative(year, month, 'HRC-S', web_dir)
createCumulative(year, month, 'HRC-I', web_dir)
def acis_dose_control(year = '', month = ''):
"""
monthly acis dose update control script
input: optional year and month
"""
if year == '' or month == '':
[year, month, day] = mcf.today_date()
month = month -1
if month < 1:
month = 12
year -= 1
else:
year = int(year)
month = int(month)
syear = str(year)
smon = mcf.add_leading_zero(month)
#
#--- extract data
#
try:
getd.acis_dose_get_data(year, month, year, month)
except:
print("Data Extraction Failed!!")
subject = 'ACIS data extraction problem'
content = 'Extraction of ACIS data for exposure map trend failed.'
expf.send_warning_email(subject, content)
exit(1)
#
#--- create cumulative data
#
file1 = 'ACIS_' + smon + '_' + syear + '.fits'
file2 = 'ACIS_' + smon + '_' + syear + '.fits.gz'
chk = 0
for test in os.listdir('./'):
if fnmatch.fnmatch(test, file2):
chk = 1
break
if chk == 0:
afile = file1
else:
afile = file2
try:
cuml.acis_create_cumulative(afile)
except:
print("Cumurative Data Creating Failed!!")
subject = 'ACIS crumulativedata problem'
content = 'Creation of Cumulative ACIS data for exposure map trend failed.'
expf.send_warning_email(subject, content)
exit(1)
#
#--- compute statistics
#
astat.acis_dose_extract_stat_data_month(year, month)
#
#--- plot data
#
aplot.acis_dose_plot_exposure_stat(clean='Yes')
#
#--- create images (you need to use ds9 to create a better image)
#
aimg.create_acis_maps(year, month)
#
#--- update html pages
#
ahtml.acis_dose_make_data_html()
#
#--- print monthly output
#
arport.acis_dose_monthly_report()