def update_main_html(): """ update main html page input: none but read a template from <house_keeping> output: <web_dir>/expousre.html """ [tyear, mon, day] = mcf.today_date() today = mcf.change_month_format(mon) today = today + ' ' + mcf.add_leading_zero(day) today = today + ', ' + str(tyear) syear = tyear smon = mon - 1 if smon < 1: smon = 12 syear = tyear - 1 lyear = str(syear) lmon = mcf.add_leading_zero(smon) line = lmon + '_' + lyear aline = mcf.change_month_format(smon) + ' ' + lyear ifile = house_keeping + 'template' with open(ifile, 'r') as f: data = f.read() data = data.replace('#ODATE#', line) data = data.replace('#DATE#', aline) data = data.replace('#TODAY#', today) ifile = web_dir + 'exposure.html' with open(ifile, 'w') as fo: fo.write(data)
def update_plt_html_date(): """ update html pages for plots; just replacing date no input, but get the list from plot_dir """ [lyear, lmon, lday] = mcf.today_date() date = mcf.add_leading_zero(lmon) + '/' date = date + mcf.add_leading_zero(lday) + '/' + str(lyear) cmd = 'ls ' + plot_dir + '/*html>./ztemp' os.system(cmd) data = mcf.read_data_file('./ztemp', remove=1) for ent in data: hdat = mcf.read_data_file(ent) sline = '' for oline in hdat: m = re.search('Last Update', oline) if m is not None: sline = sline + 'Last Update: ' + date + '\n' else: sline = sline + oline + '\n' with open(ent, 'w') as fo: fo.write(sline)
def create_hrc_maps(year= 'NA', month= 'NA'): """ create HRC image maps for given year and month input: year --- year month --- month output: <outdir>/<out.png """ # #--- if year and month are not given, set the date of the last month # if year == 'NA' or month == 'NA': [year, month, day] = mcf.today_date() month -= 1 if month < 1: month = 12 year -= 1 # #--- create images # hrc_dose_conv_to_png(mon_hrc_dir, img_dir, year, month) hrc_dose_conv_to_png(cum_hrc_dir, img_dir, year, month)
def write_html(ccd, sec, data, outfile): """ write a html page: input: ccd --- ccd; e.g., i2, s3 sec --- section 0 - 3 data --- a list of lists of data date, year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,m10_acc, m10_apos,mean_dff,std_dff, min_dff, min_dpos,max_dff,max_dpos,m10_dff,m10_dpos outfile --- output file name output: outfile """ # #--- open the list of lists # [date, year,month,mean_acc,std_acc,min_acc,min_apos,\ max_acc,max_apos, asig1, asig2, asig3, mean_dff,std_dff,\ min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3] = data # #--- today's date # [lyear, lmon, lday] = mcf.today_date() # #--- start writing a html page # sline = '<!DOCTYPE html>\n' sline = sline + '<html>\n' sline = sline + '<head>\n' sline = sline + '<title>ACIS ' + ccd.upper() + ' Section ' sline = sline + str(sec) + ' History Data</title>\n' sline = sline + "<meta http-equiv='Content-Type' content='text/html; charset=utf-8' />" # #--- css style sheet # sline = sline + '<style type="text/css">\n' sline = sline + 'body{background-color:#FFEBCD;}\n' sline = sline + 'table{text-align:center;margin-left:auto;margin-right:' sline = sline + 'auto;border-style:solid;border-spacing:8px;border-width:' sline = sline + '2px;border-collapse:separate}\n' sline = sline + 'td {text-align:center}\n' sline = sline + 'a:link {color:green;}\n' sline = sline + 'a:visited {color:red;}\n' sline = sline + '</style>\n' sline = sline + '</head>\n' sline = sline + '<body> \n' # sline = sline + '<br /><h3> Last Update: ' # sline = sline + mcf.add_leading_zero(lmon) + '/' # sline = sline + mcf.add_leading_zero(lday) + '/' # sline = sline + str(lyear) + '</h3>\n' sline = sline + '<table border=1>\n' sline = sline + header_write() + '\n' for i in range(0, len(year)): sline = sline + '<tr>\n' sline = sline + '<td>' + str(year[i]) + '</td>\t' sline = sline + '<td>' + str(month[i]) + '</td>\t' sline = sline + '<td>' + str(mean_dff[i]) + '</td>\t' sline = sline + '<td>' + str(std_dff[i]) + '</td>\t' sline = sline + '<td>' + str(min_dff[i]) + '</td>\t' sline = sline + '<td>' + str(min_dpos[i]) + '</td>\t' sline = sline + '<td>' + str(max_dff[i]) + '</td>\t' sline = sline + '<td>' + str(max_dpos[i]) + '</td>\t' sline = sline + '<td>' + str(dsig1[i]) + '</td>\t' sline = sline + '<td>' + str(dsig2[i]) + '</td>\t' sline = sline + '<td>' + str(dsig3[i]) + '</td>\t' syear = str(int(year[i])) smon = mcf.add_leading_zero(month[i]) lccd = ccd.replace('_', '') ifile = 'ACIS_' + smon + '_' + syear + '_' + lccd sline = sline + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month/' sline = sline + ifile + '.fits.gz">fits</a></td>\n' sline = sline + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' sline = sline + ifile + '.png">map</a></td>\n\n' sline = sline + '<td>' + str(mean_acc[i]) + '</td>\t' sline = sline + '<td>' + str(std_acc[i]) + '</td>\t' sline = sline + '<td>' + str(min_acc[i]) + '</td>\t' sline = sline + '<td>' + str(min_apos[i]) + '</td>\t' sline = sline + '<td>' + str(max_acc[i]) + '</td>\t' sline = sline + '<td>' + str(max_apos[i]) + '</td>\t' sline = sline + '<td>' + str(asig1[i]) + '</td>\t' sline = sline + '<td>' + str(asig2[i]) + '</td>\t' sline = sline + '<td>' + str(asig3[i]) + '</td>\n' ifile = 'ACIS_07_1999_' + smon + '_' + syear + '_' + lccd sline = sline + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative/' sline = sline + ifile + '.fits.gz">fits</a></td>\n' sline = sline + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' sline = sline + ifile + '.png">map</a></td>' sline = sline + '</tr>\n\n' # #--- put header every new year so that we can read data easier # if month[i] % 12 == 0 and i != (len(year) - 1): sline = sline + header_write() + '\n' sline = sline + '</table>\n\n' sline = sline + '<br /><br /><hr /><br />\n' sline = sline + '<br /><strong style="font-size:105%;float:right">Last Update: ' sline = sline + mcf.add_leading_zero(lmon) + '/' sline = sline + mcf.add_leading_zero(lday) + '/' sline = sline + str(lyear) + '</strong>\n' sline = sline + '<p>If you have any questions about this page, contact ' sline = sline + '<a href="mailto:[email protected]">' sline = sline + '[email protected].</a></p>\n' sline = sline + '</body>\n' sline = sline + '</html>\n' with open(outfile, 'w') as fo: fo.write(sline)
def hrc_dose_run(year='NA', month='NA'): """ run all needed HRC scripts to extract data and create images input: year --- year month --- month """ # #--- if year and month are given, get that year and month for data extraction # if year != 'NA' and str(year).isdigit() and str(month).isdigit(): lyear = int(year) lmonth = int(month) # #--- if year and month are not given, use a month before the current year/month # else: [year, month, day] = mcf.today_date() lyear = year lmonth = month - 1 if lmonth < 1: lmonth = 12 lyear -= 1 # #--- extracting data # try: hgdata.hrc_dose_get_data(lyear, lmonth, lyear, lmonth) except: print("Data Extraction Failed!!") subject = 'HRC data extraction problem' content = 'Extraction of HRC data for exposure map trend failed.' expf.send_warning_email(subject, content) # exit(1) # #--- computing statistics # try: hstat.hrc_dose_extract_stat_data_month(lyear, lmonth) except: print("Stat computation failed") pass # #--- creating html pages # try: hhtml.hrc_dose_make_data_html() except: print("HTML page construction failed") pass # #--- plotting histories # try: hplot.hrc_dose_plot_exposure_stat() except: print("Trend plotting failed") pass # #--- creating map images # try: himg.create_hrc_maps(lyear, lmonth) except: print("HRC map construction failed") pass # #--- plotting monthly report trend # try: monthly.hrc_dose_plot_monthly_report() except: print("Plot for monthly report failed") pass
def print_html_page(indir, outdir, hrc, data): """ create HTML page to display HRC historical data. input: indir --- input dir outdir --- output dir hrc --- hrc data --- a list of lists of data output: <outdir>/<hrc>.html """ # #--- open the data ; except the first three, all others are lists of data # (date, year, month, mean_acc, std_acc, min_acc, min_apos, \ max_acc, max_apos, asig1, asig2, asig3, mean_dff, std_dff,\ min_dff, min_dpos, max_dff, max_dpos, dsig1, dsig2, dsig3) = data outdir = outdir + '/' + hrc + '.html' # #--- start writing the html page # line = '<!DOCTYPE html>\n' line = line + '<html>\n' line = line + '<head>\n' line = line + '<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\n' line = line + '<style type="text/css">\n' line = line + 'body{background-color:#FFEBCD;}\n' line = line + 'table{text-align:center;margin-left:auto;margin-right:auto;' line = line + 'border-style:solid;border-spacing:8px;border-width:2px;' line = line + 'border-collapse:separate}\n' line = line + 'a:link {color:green;}\n' line = line + 'a:visited {color:red;}\n' line = line + 'td{text-align:center;padding:8px}\n' line = line + '</style>\n' if hrc == 'hrci': hname = 'HRC I' wname = 'HRCI' else: hname = 'HRC S' wname = 'HRCS' line = line + '<title>' + hname + ' History Data</title>\n' line = line + '</head>\n' line = line + '<body>\n' line = line + '<h2 style="text-align:center">Data: ' + hname + '</h2>\n' line = line + '<div style="padding-bottom:30px">\n' line = line + '<table border=1>\n' line = line + '<tr><th> </th><th> </th><th colspan=11>' line = line + 'Monthly</th><th colspan=11>Cumulative</th></tr>\n' line = line + '<tr style="color:blue"><th>Year</th><th>Month</th>\n' line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>\n' line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>' line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n' line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>' line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>' line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n' for i in range(0, len(date)): smonth = mcf.add_leading_zero(month[i]) cmonth = mcf.change_month_format(month[i]) syear = str(int(year[i])) # #--- monthly HRC dose data # if mean_dff[i] == 0 and std_dff[i] == 0: # #--- for the case there is no data for this month # line = line + '<tr><td>%d</td><td>%d</td>' % (year[i], month[i]) line = line + '<td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td>\n' line = line + '<td>NA</td><td>NA</td><td>NA</td><td>No Data</td><td>No Image</td>\n' #line = line + '</tr>\n' else: line = line + '<tr>' line = line + '<td>%d</td>' % year[i] line = line + '<td>%d</td>' % month[i] line = line + '<td>%4.4f</td>' % mean_dff[i] line = line + '<td>%4.4f</td>' % std_dff[i] line = line + '<td>%4.1f</td>' % min_dff[i] line = line + '<td>%s</td>' % min_dpos[i] line = line + '<td>%4.1f</td>' % max_dff[i] line = line + '<td>%s</td>' % max_dpos[i] line = line + '<td>%4.1f</td>' % dsig1[i] line = line + '<td>%s</td>' % dsig2[i] line = line + '<td>%4.1f</td>\n' % dsig3[i] line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month_hrc/' line = line + wname + '_' + smonth + '_' + syear + '.fits.gz' line = line + '">fits</a></td>\n' line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' line = line + wname + '_' + smonth + '_' + syear + '.png' line = line + '">map</a></td>\n' # #---- cumulative HRC dose data # line = line + '<td>%4.4f</td>' % mean_acc[i] line = line + '<td>%4.4f</td>' % std_acc[i] line = line + '<td>%4.1f</td>' % min_acc[i] line = line + '<td>%s</td>' % min_apos[i] line = line + '<td>%4.1f</td>' % max_acc[i] line = line + '<td>%s</td>' % max_apos[i] line = line + '<td>%4.1f</td>' % asig1[i] line = line + '<td>%s</td>' % asig2[i] line = line + '<td>%4.1f</td>\n' % asig3[i] line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative_hrc/' line = line + wname + '_08_1999_' + smonth + '_' + syear + '.fits.gz' line = line + '">fits</a></td>\n' line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' line = line + wname + '_08_1999_' + smonth + '_' + syear + '.png' line = line + '">map</a></td>\n' line = line + '</tr>\n\n' # #--- put header every new year so that we can read data easier # if month[i] % 12 == 0 and i != (len(date) - 1): line = line + '\n<tr style="color:blue"><th>Year</th><th>Month</th>' line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>' line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>' line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n' line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>' line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>' line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n\n' line = line + '</table>\n\n' line = line + "</div>\n" line = line + '<hr />\n' # #--- set current date # [tyear, tmon, tday] = mcf.today_date() lmon = mcf.add_leading_zero(tmon) lday = mcf.add_leading_zero(tday) line = line + '<p style="padding-top:10px;padding-bottom:10px">' line = line + '<strong style="font-size:105%;float:right">Last Update: ' line = line + lmon + '/' + lday + '/' + str(tyear) + '</strong></p>\n' line = line + '<p>If you have any questions about this page, contact ' line = line + ' <a href="mailto:[email protected]">[email protected].</a></p>\n' line = line + '</body>\n' line = line + '</html>\n' with open(outdir, 'w') as fo: fo.write(line)
def hrc_dose_get_data(startYear='NA', startMonth='NA', stopYear='NA', stopMonth='NA'): """ extract HRC evt1 data from a month and create cumulative data fits file. input: startYear --- start year startMonth --- start month stopYear --- stop year stopMonth --- stop month output: image fits files for the month and cumulative cases """ # #--- if the dates are not given, set them to the last month # if startYear == 'NA': [stopYear, stopMonth, day] = mcf.today_date() startYear = stopYear stopMonth -= 1 startMonth = stopMonth if startMonth < 1: startMonth = 12 startYear -= 1 # #--- start extracting the data for the year/month period # for year in range(startYear, stopYear + 1): lyear = str(year) syear = lyear[2] + lyear[3] # #--- create a list of month appropriate for the year # month_list = expf.make_month_list(year, startYear, stopYear, startMonth, stopMonth) for month in month_list: smonth = mcf.add_leading_zero(month) # #--- output file name settings # outfile_i = './HRCI_' + str(smonth) + '_' + str(lyear) + '.fits' outfile_s = './HRCS_' + str(smonth) + '_' + str(lyear) + '.fits' # #--- using ar5gl, get file names # smonth = mcf.add_leading_zero(month) syear = str(year) start = syear + '-' + smonth + '-01T00:00:00' nextMonth = month + 1 if nextMonth > 12: lyear = year + 1 nextMonth = 1 else: lyear = year smonth = mcf.add_leading_zero(nextMonth) syear = str(lyear) stop = str(lyear) + '-' + smonth + '-01T00:00:00' line = 'operation=browse\n' line = line + 'dataset=flight\n' line = line + 'detector=hrc\n' line = line + 'level=1\n' line = line + 'filetype=evt1\n' line = line + 'tstart=' + start + '\n' line = line + 'tstop=' + stop + '\n' line = line + 'go\n' fitsList = mcf.run_arc5gl_process(line) # #--- extract each evt1 file, extract the central part, and combine them into a one file # #--- set counters for how many hrc-i and hrc-s are extracted # hrci_cnt = 0 hrcs_cnt = 0 for fitsName in fitsList: print("Fits file: " + fitsName) m = re.search('fits', fitsName) if m is None: continue try: line = 'operation=retrieve\n' line = line + 'dataset=flight\n' line = line + 'detector=hrc\n' line = line + 'level=1\n' line = line + 'filetype=evt1\n' line = line + 'filename=' + fitsName + '\n' line = line + 'go\n' out = mcf.run_arc5gl_process(line) if len(out) < 1: continue ofits = out[0] cmd = 'gzip -d ' + ofits os.system(cmd) ofits = ofits.replace('.gz', '') except: continue # #--- checking which HRC (S or I) # hout = pyfits.open(ofits) data = hout[1].header detector = data['DETNAM'] hout.close() # #--- creating the center part image # line = set_cmd_line(ofits, detector) ichk = expf.create_image(line, 'ztemp.fits') # #--- for HRC S # if detector == 'HRC-S' and ichk > 0: expf.combine_image('ztemp.fits', 'total_s.fits') hrcs_cnt += 1 # #--- for HRC I # elif detector == 'HRC-I' and ichk > 0: expf.combine_image('ztemp.fits', 'total_i.fits') hrci_cnt += 1 mcf.rm_files('out.fits') mcf.rm_files(ofits) # #--- move the file to a depository # if hrcs_cnt > 0: cmd = 'mv total_s.fits ' + web_dir + 'Month_hrc/' + outfile_s os.system(cmd) cmd = 'gzip ' + web_dir + '/Month_hrc/*.fits' os.system(cmd) if hrci_cnt > 0: cmd = 'mv total_i.fits ' + web_dir + 'Month_hrc/' + outfile_i os.system(cmd) cmd = 'gzip ' + web_dir + '/Month_hrc/*.fits' os.system(cmd) createCumulative(year, month, 'HRC-S', web_dir) createCumulative(year, month, 'HRC-I', web_dir)
def acis_dose_control(year = '', month = ''): """ monthly acis dose update control script input: optional year and month """ if year == '' or month == '': [year, month, day] = mcf.today_date() month = month -1 if month < 1: month = 12 year -= 1 else: year = int(year) month = int(month) syear = str(year) smon = mcf.add_leading_zero(month) # #--- extract data # try: getd.acis_dose_get_data(year, month, year, month) except: print("Data Extraction Failed!!") subject = 'ACIS data extraction problem' content = 'Extraction of ACIS data for exposure map trend failed.' expf.send_warning_email(subject, content) exit(1) # #--- create cumulative data # file1 = 'ACIS_' + smon + '_' + syear + '.fits' file2 = 'ACIS_' + smon + '_' + syear + '.fits.gz' chk = 0 for test in os.listdir('./'): if fnmatch.fnmatch(test, file2): chk = 1 break if chk == 0: afile = file1 else: afile = file2 try: cuml.acis_create_cumulative(afile) except: print("Cumurative Data Creating Failed!!") subject = 'ACIS crumulativedata problem' content = 'Creation of Cumulative ACIS data for exposure map trend failed.' expf.send_warning_email(subject, content) exit(1) # #--- compute statistics # astat.acis_dose_extract_stat_data_month(year, month) # #--- plot data # aplot.acis_dose_plot_exposure_stat(clean='Yes') # #--- create images (you need to use ds9 to create a better image) # aimg.create_acis_maps(year, month) # #--- update html pages # ahtml.acis_dose_make_data_html() # #--- print monthly output # arport.acis_dose_monthly_report()