def update_data_files(): """ main function to run all function to create/update ACIS Dose Plots input: none, but data are extracted according to current date output: all data, plots, and html """ (uyear, umon, mon_name) = check_date() #--- mon_name is this month's data/plot directory dir_save = [mon_name] # #--- ACIS count rate data # data_list = get_data_list() for dfile in data_list: # #--- check when the data is created # (tyear, tmonth, tdate) = find_date_obs(dfile) if tyear == 1900: continue # #--- choose an appropriate output directory # cmonth = mcf.change_month_format(tmonth) #--- convert digit to letter ucmon = cmonth.upper() tmon_name = web_dir + '/' + ucmon + str(tyear) # #--- if the directory is not the current output directory (mon_name), add to dir_list # chk = 0 for test in dir_save: if tmon_name == test: chk = 1 break if chk == 0: dir_save.append(tmon_name) # #--- now actually extract data and update/create the count rate data # extract_data(dfile, tmon_name) # #--- if date is before Nov 2018, extract ephin data # if (tyear == 2018) and (tmonth < 11): ephin_data(dir_save) elif tyear < 2018: ephin_data(dir_save) # #-- clean the data files # for ent in dir_save: cleanUp(ent) return dir_save
def extract_monitor_data(year, month, fname, cols, ofile): """ extract monitor data from fits files input: year --- year month --- month fname --- none-variable part of fits file name cols --- a list of column names ofile --- output file name """ # #--- find the last entry time (seconds from 1998.1.1) # if os.path.isfile(ofile): odata = mcf.read_data_file(ofile) if len(odata) > 0: atemp = re.split('\s+', odata[-1]) cut = float(atemp[0]) else: cut = 0.0 else: cut = 0.0 lyear = str(year) ddir = data_dir + 'Fits_data/' + mcf.change_month_format(month).upper() ddir = ddir + lyear[2] + lyear[3] + '/' # #--- data is a list of lists of column data # data = read_fits_data(ddir, fname, cols) if data: line = '' # #--- make sure that the data extracted will be after "cut" date # dlen = len(data[0]) kstart = dlen for k in range(0, dlen): if data[0][k] > cut: kstart = k break if kstart == dlen: dlen = 0 if dlen > 0: for k in range(kstart, dlen): for m in range(0, len(cols)): if cols[m] == 'time': line = line + '%d\t' % data[m][k] else: line = line + '%2.3e\t' % data[m][k] line = line + '\n' with open(ofile, 'a') as fo: fo.write(line)
def sdate_to_ldate_with_space(sdate): """ change date in second from 1998.1.1 to MMM dd input: stime --- time in seconds from 1998.1.1 output: ldate --- date in form of MMM dd (e.g. Aug 19) """ atemp = re.split('\/', sdate) mon = int(float(atemp[0])) lmon = mcf.change_month_format(mon) ldate = lmon + ' ' + atemp[1] return ldate
def get_data_for_month(year, month): """ extract one month amount of data and update data files input: year --- year of the data month --- month of the data output: updated data files: <MMM><YYYY>/ccd# <MMM><YYYY>/ephin_data Note: there is no ephin data after 2018 Nov """ # #--- if the year/month are not given, extract data of the last month # if year == '': out = time.strftime('%Y:%m' % time.gmtime) atemp = re.split(':', out) year = int(float(atemp[0])) month = int(float(atemp[1])) month -= 1 if month < 1: month = 12 year -= 1 cmonth = mcf.change_month_format(month) #--- convert digit to letter month ucmon = cmonth.upper() dir_name = data_dir + '/' + ucmon + str(year) + '/' #--- output directory if os.path.isdir(dir_name): cmd = 'rm -rf ' + dir_name + '*' else: cmd = 'mkdir -p ' + dir_name os.system(cmd) # #--- get acis count rate data # extract_acis_count_rate(year, month, dir_name) # #--- get ephin rate data; no data after Nov 2018 # if year < 2018: get_ephin_data(year, month, dir_name) elif (year == 2018) and (month < 11): get_ephin_data(year, month, dir_name) # #-- clean the data files # cleanUp(dir_name)
def get_obsdate(): """ read sot database and make a list of obsids and its observation dates Input: none, but read data from <sot_direcotry> Output: obs_dict --- a dictionary of obsid <--> starting date """ # #--- read sot data # data = mcf.read_data_file(sot_directory) obsid_list = [] start_date = [] index_date = [] obs_dict = {} for ent in data: temp = re.split('\^', ent) obsid = temp[1] # #--- check the data are valid # try: atemp = re.split('\s+', temp[13]) mon = mcf.change_month_format(atemp[0]) date = atemp[1] year = atemp[2][2] + atemp[2][3] except: continue # #--- starting date in form of 05/23/14 # lmon = mcf.add_leading_zero(mon) ldate = mcf.add_leading_zero(date) dline = lmon + '/' + ldate + '/' + year try: obs_dict[int(obsid)] = dline except: pass return obs_dict
def cleanup_sib_dir(lev, mon, year): """ clean up the working directories input: lev --- data level mon --- month of the data processed year --- year of the data processd output: none """ lmon = mcf.change_month_format(mon) lmon = lmon.lower() cmd = 'mv ' + cor_dir + lev + '/Outdir/lres ' cmd = cmd + cor_dir + lev + '/Outdir/lres_' + lmon +str(year) + '_modified' os.system(cmd) cmd = 'rm -rf ' + cor_dir + lev + '/Outdir/ctirm_dir' os.system(cmd) cmd = 'rm -rf ' + cor_dir + lev + '/Outdir/filtered' os.system(cmd) cmd = 'rm -rf ' + cor_dir + lev + '/Outdir/hres' os.system(cmd)
def create_display_data_table(): """ create a readable data table for html page Input: none, but read from <data_dir>/ccd<ccd>_<node> Output: <web_dir>/ccd<ccd>_<node> """ for ccd in range(0, 10): for node in range(0, 4): ifile = 'ccd' + str(ccd) + '_' + str(node) # #--- read the original data file # infile = data_dir + ifile data = mcf.read_data_file(infile) # #--- adding heading # line = "#\n#Date Mn K alpha Al K alpha " line = line + "Ti K alpha Slope Sigma Int Sigma\n#\n" for ent in data: atemp = re.split('\s+', ent) # #--- converting the date format from chandra time into <mon> <year> for html data display # stime = int(atemp[0]) out = mcf.convert_date_format(stime, ifmt='chandra', ofmt='%Y:%m:%d') atemp = re.split(':', out) lmon = mcf.change_month_format(atemp[1]) ldate = lmon + ' ' + atemp[0] lout = ent.replace(atemp[0], ldate) line = line + lout + '\n' outfile = web_dir + 'Data/' + ifile with open(outfile, 'w') as fo: fo.write(line)
def find_prev_date(cmon, lev): """ find the last extreacted obsid date input: cmon --- current month lev --- data level output: date --- the date which to be used to start extracting data """ afile = cor_dir + lev + '/acis_obs' # #--- check whether file exist # if os.path.isfile(afile): data = mcf.read_data_file(afile) if len(data) > 0: atemp = re.split('\s+', data[-1]) try: btemp = re.split('T', atemp[-4]) ctemp = re.split('-', btemp[0]) dmon = int(float(ctemp[1])) day = str(int(float(ctemp[2]))) except: mon = atemp[-7] dmon = mcf.change_month_format(mon) day = atemp[-6] # #--- just in a case acis_obs is from the last month.. # if dmon == cmon: date = day else: date = '1' else: date = '1' else: date = '1' return date
def create_weekly_report(date, year, debug=0): """ main script to set up the weekly report template for the week input: date --- date in the format of mmdd (e.g. 0910) year --- year in the format of yyyy (e.g. 2015) debug --- if it is other than 0, print out some output output: weekly report /data/mta4/www/REPORT/<yyyy>/<mm><dd>.html /data/mta4/www/REPORT/<yyyy>/<mm><dd>_fptemp.png it also creates local copies in <data_dir> """ # #--- if the test is requested, create Test directory # if debug != 0: os.system('mkdir -p /data/mta/Script/Weekly/TEST/') hodir = '/data/mta/Script/Weekly/TEST/' else: hodir = '/data/mta4/www/REPORTS/' # #--- one day in seconds # oned = 86400 # #--- set various time formats # syear = str(year) #--- 4 digit year yrd2 = year[2] + year[3] #--- 2 digit year year = int(float(year)) #--- integer year date = str(date) smon = date[0] + date[1] #--- two digit month mon = int(float(smon)) #--- integer month lmon = mcf.change_month_format(mon) #--- month in letter (e.g.Mar) sday = date[2] + date[3] #--- two digit mday day = int(float(sday)) #--- integer mday ltime = str(year) + ':' + str(mon) + ':' + str(day) + 'T00:00:00' ltime = time.strftime('%Y:%j:%H:%M:%S', time.strptime(ltime, '%Y:%m:%dT%H:%M:%S')) stop = int(Chandra.Time.DateTime(ltime).secs) + oned atemp = re.split(':', ltime) ydate = int(float(atemp[1])) day_n = stop - 7 * oned day01 = stop - 5 * oned day0 = stop - 6 * oned lday0 = stime_to_ddate(day0) sday0 = sdate_to_ldate(lday0) start = day0 lday1 = stime_to_ddate(day01) tout = Chandra.Time.DateTime(day0).date atemp = re.split('\.', tout) tout = atemp[0] ttemp = re.split(':', tout) iru_start = str(ttemp[0]) + '_' + str(ttemp[1]) # #--- year of the beginning of the period; could be different from that of the end # byear = ttemp[0] lday0 = stime_to_ddate(day0) day1 = stop - 5 * oned lday1 = stime_to_ddate(day1) day2 = stop - 4 * oned lday2 = stime_to_ddate(day2) sday2 = sdate_to_ldate(lday2) day3 = stop - 3 * oned lday3 = stime_to_ddate(day3) day4 = stop - 2 * oned lday4 = stime_to_ddate(day4) sday4 = sdate_to_ldate(lday4) day5 = stop - 1 * oned lday5 = stime_to_ddate(day5) day6 = stop lday6 = stime_to_ddate(day6) sday6 = sdate_to_ldate(lday6) tout = Chandra.Time.DateTime(day6).date atemp = re.split('\.', tout) tout = atemp[0] ttemp = re.split(':', tout) iru_stop = '_' + str(ttemp[1]) day7 = stop + 1 * oned lday7 = stime_to_ddate(day7) # #---- setting file name # atemp = re.split('\/', lday6) file_date = atemp[0] + atemp[1] file_date2 = atemp[0] + '/' + atemp[1] file_name = file_date + '.html' # #--- title # titledate = lday0 + ' - ' + lday6 ldate = sdate_to_ldate(lday6) ldate_sp = sdate_to_ldate_with_space(lday6) # #--- focal temp file name # paft.plot_acis_focal_temp(year, ydate) fftp.find_focal_temp_peaks(year, mon, day, 0.3) fptemp = file_date + '_fptemp.png' fpext_range = str(start) + ' ' + str(stop) fpstart = str(start) fplsub = '"' + sday0 + '", "' + sday2 + '", "' + sday4 + '", "' + sday6 + '"' fpdsub = str(day0) + ', ' + str(day2) + ', ' + str(day4) + ', ' + str(day6) # #--- IRU span # irudate = iru_start + iru_stop irudate = str(syear) + '/' + iru_start + iru_stop # #--- telemetry data table # tel_start = start - oned tel_stop = stop - oned telem_table = ctt.get_telem_data(tel_start, tel_stop) # #--- bad pixcel data table # bad_pix_table = cbpt.create_bad_pixel_table() # #--- recent observation table # ro_stop = stop + oned r_obs_table = frobs.find_recent_observations(ro_stop) # #--- find trending dates/title # it_date = syear + ':' + smon + ':' + sday [title, c_t_date, last_trend_date] = find_inst_trend_name(it_date) # #--- index.html input # s1 = sday0[0:3] + ' ' + sday0[3:5] s2 = sday6[0:3] + ' ' + sday6[3:5] index = '<td> <a href="./' + str( year) + '/' + file_date + '.html">' + s1 + ' - ' + s2 + '</a>' # #--- debugging output # if debug != 0: print("file_name; " + file_name) print("title date: " + titledate) print("ldate: " + ldate) print("fptemp: " + fptemp) print("fpext_range: " + fpext_range) print("fpstart: " + fpstart) print(" fplsub: " + fplsub) print(" fpdsub: " + fpdsub) print("irudate: " + irudate) print("title: " + title) print("last_trend_date: " + last_trend_date) print("index: " + index) # #--- create a work directory # outdir = wdir + 'Data/' + ldate + '/' cmd = 'mkdir -p ' + outdir os.system(cmd) # #--- read the template for the weekly, and start replacing dates etc # tfile = tdir + 'this_week' linput = read_template(tfile) linput = linput.replace('#DDATE#', file_date) linput = linput.replace('#IRUSPAN1#', irudate) linput = linput.replace('#IRUSPAN2#', irudate) linput = linput.replace('#TITLE#', title) linput = linput.replace('#TITLEDATE#', titledate) atemp = last_trend_date atemp = atemp.replace('/', '') linput = linput.replace('#PREVREPORT#', atemp) atemp = re.split('/', last_trend_date) pmon = int(float(atemp[0])) lmon = mcf.change_month_format(pmon) line = lmon + ' ' + atemp[1] # #--- the previous report could be from the last year # ryear = syear if mon < pmon: ryear = year - 1 ryear = str(ryear) linput = linput.replace('#RYEAR#', ryear) linput = linput.replace('#PREVDATE#', line) atitle = str(title) atitle = atitle.replace(' ', '_') [temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8] = read_cti_values() linput = linput.replace('#ATEMP#', temp3) linput = linput.replace('#ATEMP2#', temp4) linput = linput.replace('#DTEMP#', temp7) linput = linput.replace('#DTEMP2#', temp8) [val, step, tval] = read_sim() linput = linput.replace('#WSTEP#', step) linput = linput.replace('#WMOVE#', val) linput = linput.replace('#TMOVE#', tval) # #--- read the focal temp peak list # tstop = stop + 86400 [fcnt, fdata] = read_focal_temp_data(fptemp, outdir) linput = linput.replace('#TEMPPEAK#', str(fcnt)) linput = linput.replace('#TEMPLIST#', fdata) # #--- bad pixel # linput = linput.replace('BAD_PIXEL_TABLE', bad_pix_table) # #--- photon # linput = linput.replace('PHOTON_TABLE', r_obs_table) # #--- telem data # linput = linput.replace('TELEM_TABLE', telem_table) # #--- trend data # trend = set_trend_data_input(title) linput = linput.replace('#TREND#', trend) # #--- write out the weekly report # ofile = outdir + file_name with open(ofile, 'w') as fo: fo.write(linput) # #--- move files # move_files(date, year, outdir, file_name, fptemp, hodir) # #--- send out email to admin; notify the job complete # send_email_to_admin(date, year)
def get_data_for_day(year, month, day=''): """ extract one month amount of data and update data files input: year --- year of the data month --- month of the data output: updated data files: <MMM><YYYY>/ccd# <MMM><YYYY>/ephin_data Note: there is no ephin data after 2018 Nov """ # #--- if the date is given... # chk = 0 if year != '': lmon = mcf.add_leading_zero(month) lday = mcf.add_leading_zero(day) if day != '': start = str(year) + ':' + lmon + ':' + lday + ':00:00:00' stop = str(year) + ':' + lmon + ':' + lday + ':23:59:59' # #-- if the day part is not given, compute for the entire month # else: nyear = year nmon = mon + 1 if nmon > 12: nyear += 1 nmon = 1 nmon = mcf.add_leading_zero(nmon) start = str(year) + ':' + lmon + ':' + ':01:00:00:00' stop = str(nyear) + ':' + nmom + ':' + ':01:00:00:00' chk = 1 start = mcf.convert_date_format(start, ifmt='%Y:%m:%d:%H:%M:%S', ofmt='chandra') stop = mcf.convert_date_format(stop, ifmt='%Y:%m:%d:%H:%M:%S', ofmt='chandra') # #--- if the year/month are not given, extract data of the day before # if year == '': out = time.strftime('%Y:%j:00:00:00', time.gmtime()) stop = Chandra.Time.DateTime(out).secs start = stop - 86400.0 out = mcf.convert_date_format(start, ifmt='chandra', ofmt='%Y:%m') out = re.split(':', out) year = int(float(out[0])) month = int(float(out[1])) # #--- set output directory name # cmonth = mcf.change_month_format(month) #--- convert digit to letter month ucmon = cmonth.upper() dir_name = data_dir + '/' + ucmon + str(year) + '/' #--- output directory if not os.path.isdir(dir_name): cmd = 'mkdir -p ' + dir_name os.system(cmd) # #--- if the entire month data needs to be extracted, remove all previous data # if chk == 1: cmd = 'rm -rf ' + dir_name + '/ccd*' os.system(cmd) # #--- get acis count rate data # extract_acis_count_rate(start, stop, dir_name) # #--- get ephin rate data; no data after Nov 2018 # if year < 2018: get_ephin_data(start, stop, dir_name) elif (year == 2018) and (month < 11): get_ephin_data(start, stop, dir_name) # #-- clean the data files # cleanUp(dir_name)
def print_html_page(indir, outdir, hrc, data): """ create HTML page to display HRC historical data. input: indir --- input dir outdir --- output dir hrc --- hrc data --- a list of lists of data output: <outdir>/<hrc>.html """ # #--- open the data ; except the first three, all others are lists of data # (date, year, month, mean_acc, std_acc, min_acc, min_apos, \ max_acc, max_apos, asig1, asig2, asig3, mean_dff, std_dff,\ min_dff, min_dpos, max_dff, max_dpos, dsig1, dsig2, dsig3) = data outdir = outdir + '/' + hrc + '.html' # #--- start writing the html page # line = '<!DOCTYPE html>\n' line = line + '<html>\n' line = line + '<head>\n' line = line + '<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\n' line = line + '<style type="text/css">\n' line = line + 'body{background-color:#FFEBCD;}\n' line = line + 'table{text-align:center;margin-left:auto;margin-right:auto;' line = line + 'border-style:solid;border-spacing:8px;border-width:2px;' line = line + 'border-collapse:separate}\n' line = line + 'a:link {color:green;}\n' line = line + 'a:visited {color:red;}\n' line = line + 'td{text-align:center;padding:8px}\n' line = line + '</style>\n' if hrc == 'hrci': hname = 'HRC I' wname = 'HRCI' else: hname = 'HRC S' wname = 'HRCS' line = line + '<title>' + hname + ' History Data</title>\n' line = line + '</head>\n' line = line + '<body>\n' line = line + '<h2 style="text-align:center">Data: ' + hname + '</h2>\n' line = line + '<div style="padding-bottom:30px">\n' line = line + '<table border=1>\n' line = line + '<tr><th> </th><th> </th><th colspan=11>' line = line + 'Monthly</th><th colspan=11>Cumulative</th></tr>\n' line = line + '<tr style="color:blue"><th>Year</th><th>Month</th>\n' line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>\n' line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>' line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n' line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>' line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>' line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n' for i in range(0, len(date)): smonth = mcf.add_leading_zero(month[i]) cmonth = mcf.change_month_format(month[i]) syear = str(int(year[i])) # #--- monthly HRC dose data # if mean_dff[i] == 0 and std_dff[i] == 0: # #--- for the case there is no data for this month # line = line + '<tr><td>%d</td><td>%d</td>' % (year[i], month[i]) line = line + '<td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td><td>NA</td>\n' line = line + '<td>NA</td><td>NA</td><td>NA</td><td>No Data</td><td>No Image</td>\n' #line = line + '</tr>\n' else: line = line + '<tr>' line = line + '<td>%d</td>' % year[i] line = line + '<td>%d</td>' % month[i] line = line + '<td>%4.4f</td>' % mean_dff[i] line = line + '<td>%4.4f</td>' % std_dff[i] line = line + '<td>%4.1f</td>' % min_dff[i] line = line + '<td>%s</td>' % min_dpos[i] line = line + '<td>%4.1f</td>' % max_dff[i] line = line + '<td>%s</td>' % max_dpos[i] line = line + '<td>%4.1f</td>' % dsig1[i] line = line + '<td>%s</td>' % dsig2[i] line = line + '<td>%4.1f</td>\n' % dsig3[i] line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Month_hrc/' line = line + wname + '_' + smonth + '_' + syear + '.fits.gz' line = line + '">fits</a></td>\n' line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' line = line + wname + '_' + smonth + '_' + syear + '.png' line = line + '">map</a></td>\n' # #---- cumulative HRC dose data # line = line + '<td>%4.4f</td>' % mean_acc[i] line = line + '<td>%4.4f</td>' % std_acc[i] line = line + '<td>%4.1f</td>' % min_acc[i] line = line + '<td>%s</td>' % min_apos[i] line = line + '<td>%4.1f</td>' % max_acc[i] line = line + '<td>%s</td>' % max_apos[i] line = line + '<td>%4.1f</td>' % asig1[i] line = line + '<td>%s</td>' % asig2[i] line = line + '<td>%4.1f</td>\n' % asig3[i] line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Cumulative_hrc/' line = line + wname + '_08_1999_' + smonth + '_' + syear + '.fits.gz' line = line + '">fits</a></td>\n' line = line + '<td><a href="https://cxc.cfa.harvard.edu/mta_days/mta_max_exp/Images/' line = line + wname + '_08_1999_' + smonth + '_' + syear + '.png' line = line + '">map</a></td>\n' line = line + '</tr>\n\n' # #--- put header every new year so that we can read data easier # if month[i] % 12 == 0 and i != (len(date) - 1): line = line + '\n<tr style="color:blue"><th>Year</th><th>Month</th>' line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>' line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>' line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th>\n' line = line + '<th>Mean</th><th>SD</th><th>Min</th><th>Min Position</th>' line = line + '<th>Max</th><th>Max Position</th><th>68% Level</th>' line = line + '<th>95% Level</th><th>99.7% Level</th><th>Data</th><th>Map</th></tr>\n\n' line = line + '</table>\n\n' line = line + "</div>\n" line = line + '<hr />\n' # #--- set current date # [tyear, tmon, tday] = mcf.today_date() lmon = mcf.add_leading_zero(tmon) lday = mcf.add_leading_zero(tday) line = line + '<p style="padding-top:10px;padding-bottom:10px">' line = line + '<strong style="font-size:105%;float:right">Last Update: ' line = line + lmon + '/' + lday + '/' + str(tyear) + '</strong></p>\n' line = line + '<p>If you have any questions about this page, contact ' line = line + ' <a href="mailto:[email protected]">[email protected].</a></p>\n' line = line + '</body>\n' line = line + '</html>\n' with open(outdir, 'w') as fo: fo.write(line)
def create_html_page(data, hrc, sec, ctop): """ create HTML page to display HRC historical data. input: data --- a list of lists of data: date,year,month,mean_acc,std_acc,min_acc,min_apos, max_acc,max_apos,asig1, asig2, asig3, mean_dff,std_dff, min_dff, min_dpos,max_dff,max_dpos, dsig1, dsig2, dsig3 hrc --- hrci or hrcs sec --- section ctop --- max number of section output: <web_dir>/Sub_html/<inst>_<sec>.html """ # #--- today's date # today = time.strftime('%Y:%m:%d', time.gmtime()) atemp = re.split(':', today) year = int(float(atemp[0])) mon = int(float(atemp[1])) day = int(float(atemp[2])) smon = str(mon) if mon < 10: smon = '0' + smon sday = str(day) if day < 10: sday = '0' + sday # #--- start composing a html page # aline = read_template('sub_top1') if hrc == 'hrci': hname = 'HRC I' wname = 'HRCI' inst = 'i' else: hname = 'HRC S' wname = 'HRCS' inst = 's' aline = aline + '<title>' + hname + ' Section: ' + str( sec) + ' History Data</title>\n' aline = aline + "</head>\n" aline = aline + '<body style="color:white;background-color:black">\n' aline = aline + '<div><a href="../hrc_exposure_map.html">Back to Top</a>\n' aline = aline + '<h2 style="text-align:center">Data: ' + hname aline = aline + ' Section: ' + str(sec) + '\n' if sec == 0: aline = aline + ' (<a href="./hrc' + inst + '_' + str( sec + 1) + '.html">Next</a>)' elif sec == ctop - 1: aline = aline + ' (<a href="./hrc' + inst + '_' + str( sec - 1) + '.html">Prev</a>)' else: aline = aline + ' (<a href="./hrc' + inst + '_' + str( sec - 1) + '.html">Prev</a> : ' aline = aline + '<a href="./hrc' + inst + '_' + str( sec + 1) + '.html">Next</a>)' aline = aline + '</h2>\n' aline = aline + "<div style='padding-bottom:30px'>\n" aline = aline + '<table border=1>\n' aline = aline + read_template('sub_col_header') # #--- open the data to indivisual lists # [year,month,mean_acc,std_acc,min_acc,min_apos, \ max_acc,max_apos,asig1, asig2, asig3, mean_dff,std_dff,\ min_dff, min_dpos,max_dff,max_dpos,dsig1, dsig2, dsig3] = data dlen = len(year) for i in range(0, dlen): smonth = str(int(month[i])) if month[i] < 10: smonth = '0' + smonth syear = int(year[i]) # #--- converting digit to letters, i.e. 1 to Jan # cmonth = mcf.change_month_format(month[i]) # #--- monthly HRC dose data # if mean_dff[i] == 0 and std_dff[i] == 0: aline = aline + '<tr><td>%d</td><td>%d</td>' % (year[i], month[i]) aline = aline + read_template('sub_nodata') else: aline = aline + '<tr><td>%d</td>' % year[i] aline = aline + '<td>%d</td>' % month[i] try: aline = aline + '<td>%4.4f</td>' % mean_dff[i] aline = aline + '<td>%4.4f</td>' % std_dff[i] #aline = aline + '<td>%4.1f</td>' % min_dff[i] #aline = aline + '<td>%s</td>' % min_dpos[i] aline = aline + '<td>%4.1f</td>' % max_dff[i] aline = aline + '<td>%s</td>' % max_dpos[i] aline = aline + '<td>%4.1f</td>' % dsig1[i] aline = aline + '<td>%s</td>' % dsig2[i] aline = aline + '<td>%4.1f</td>\n' % dsig3[i] # if hrc == 'hrci': # aline = aline + '<td><a href="' + data_i_dir + '/Month/' # else: # aline = aline + '<td><a href="' + data_s_dir + '/Month/' # aline = aline + wname + '_' + smonth + '_' + str(syear) + '.fits.gz">fits</a></td>\n' aline = aline + '<td><a href="' + web_address + 'Image/' + wname + '/Month/' aline = aline + wname + '_' + smonth + '_' + str(syear) aline = aline + '_' + str(sec) + '.html">map</a></td>\n' except: # #--- for the case there is no data, print 'na' # aline = aline + '<td>na</td>' aline = aline + '<td>na</td>' aline = aline + '<td>na</td>' aline = aline + '<td>na</td>' aline = aline + '<td>na</td>' aline = aline + '<td>na</td>' aline = aline + '<td>na</td>\n' aline = aline + '<td>na</td>\n' # #---- cumulative HRC dose data # aline = aline + '<td>%4.4f</td>' % mean_acc[i] aline = aline + '<td>%4.4f</td>' % std_acc[i] #aline = aline + '<td>%4.1f</td>' % min_acc[i] #aline = aline + '<td>%s</td>' % min_apos[i] aline = aline + '<td>%4.1f</td>' % max_acc[i] aline = aline + '<td>%s</td>' % max_apos[i] aline = aline + '<td>%4.1f</td>' % asig1[i] aline = aline + '<td>%s</td>' % asig2[i] aline = aline + '<td>%4.1f</td>\n' % asig3[i] # if hrc == 'hrci': # aline = aline + '<td><a href="' + data_i_dir + '/Cumulative/' # else: # aline = aline + '<td><a href="' + data_s_dir + '/Cumulative/' # aline = aline + wname + '_' + smonth + '_' + str(syear) + '.fits.gz">fits</a></td>\n' aline = aline + '<td><a href="' + web_address + 'Image/' + wname + '/Cumulative/' aline = aline + wname + '_08_1999_' + smonth + '_' + str( syear) + '_' + str(sec) + '.html">map</a></td>\n' # #--- put header every new year so that we can read data easier # if month[i] % 12 == 0 and i != (dlen - 1): aline = aline + read_template('sub_col_header') aline = aline + '</table>\n\n' aline = aline + '<div style="padding_top:10px; padding_bottom:10px;">' aline = aline + '<a href="../hrc_exposure_map.html">Back to Top</a>\n' # #--- add today's date as update date # today = time.strftime('%Y:%m:%d', time.gmtime()) atemp = re.split(':', today) cmon = mcf.change_month_format(int(float(atemp[1]))) today = cmon + ' ' + atemp[2] + ', ' + atemp[0] aline = aline + read_template('sub_footer').replace('#UPDATE#', today) outdir = web_dir + 'Sub_html/' + hrc + '_' + str(sec) + '.html' with open(outdir, 'w') as fo: fo.write(aline)
def ephin_data(dir_save): """ extract ephin data input: dir_save --- a list of directory output: <out_dir>/ephin_data --- ephin data (300 sec accumulation) Note: EPHIN data is not available after Nov 2018 """ data_list = get_ephin_list() for ifile in data_list: try: (tyear, tmonth, tdate) = find_date_obs(ifile) if tyear == 1900: continue cmonth = mcf.change_month_format(tmonth) ucmon = cmonth.upper() tmon_name = web_dir + '/' + ucmon + str(tyear) chk = 0 for test in dir_save: if tmon_name == test: chk = 1 break if chk == 0: dir_save.append(tmon_name) extract_ephin_data(ifile, tmon_name) except: pass #---------------------------------------------------------------------------------------- #--- check_date: check wether there is an output directory and if it is not, create one #--------------------------------------------------------------------------------------- def check_date(): """ check wether there is an output directory and if it is not, create one input: none output: year --- the current year mon --- the current month mdir --- the current output direcotry """ # #--- find today's date # out = time.strftime('%Y:%j:00:00:00', time.gmtime()) stime = Chandra.Time.DateTime(out).secs) # #--- find 10 days ago and create the output directory based on that date # s10 = stime - 10 * 86400.0 out = mcf.convert_date_format(s10, ifmt='chandra', ofmt='%Y:%m:%d') atemp = re.split(':', out) year = int(float(atemp[0])) mon = int(float(atemp[1])) lmon = mcf.change_month_format(mon) mdir = web_dir + lmon.upper() + str(year) if not os.path.isdir(mdir) cmd = 'mkdir -p ' + mfile os.system(cmd) return (year, mon, mdir) #--------------------------------------------------------------------------------------- #-- get_data_list: compare the current input list to the old one and select data - #--------------------------------------------------------------------------------------- def get_data_list(): """ compare the current input list to the old one and select out the data which are not used input: house_keeping/old_file_list --- previous data list house_keeping/bad_fits_file --- bad fits data list the data are read from /dsops/ap/sdp/cache/*/acis/*evt1.fits (if it is an actual run) output: input_data --- the data list """ # #--- create a current data file list # cmd = 'ls -d /dsops/ap/sdp/cache/*/acis/*evt1.fits > ' + zspace os.system(cmd) # #--- use only non-calibration data files # data = mcf.read_data_file(zspace, remove=1) file_list = find_obs_data_files(data) # #--- read the old file list # ifile = house_keeping + 'old_file_list' old_list = mcf.read_data_file(ifile) # #--- read bad fits file list # try: file2 = house_keeping + 'bad_fits_file' bad_list = mcf.read_data_file(file2) except: bad_list = [] # #--- update "old_file_list" # with open(ifile, 'w') as fo: for line in file_list: fo.write(line + '\n') # #--- compara two files and select out new file names # n_list = list(numpy.setdiff1d(file_list, old_list)) if len(bad_list) > 0: n_list = list(numpy.setdiff1d(n_list, bad_list)) return n_list #--------------------------------------------------------------------------------------- #-- find_obs_data_files: choose files with only non-calibration data files -- #--------------------------------------------------------------------------------------- def find_obs_data_files(data): """ choose files with only non-calibration data files input: data --- a list of data file names output: file_list --- a list of non-calibration data file """ file_list = [] for ent in data: atemp = re.split('acisf', ent) btemp = re.split('_', atemp[1]) # #--- for the older format : acisf16218N001_evt1.fits #--- for the new format : acisf16218_000N001_evt1.fits # try: val = float(btemp[0]) mark= int(float(val)) except: ctemp = re.split('N', btemp[1]) mark = int(float(ctemp[0])) if mark < 50000: file_list.append(ent) return file_list #--------------------------------------------------------------------------------------- #--- find_date_obs: find observation time --- #--------------------------------------------------------------------------------------- def find_date_obs(ifile): """ find observation time input: file --- input fits file name output: year, month, and date """ try: dout = pyfits.open(ifile) date = dout[0].header['DATE-OBS'] atemp = re.split('T', date) btemp = re.split('-', atemp[0]) year = int(btemp[0]) month = int(btemp[1]) date = int(btemp[2]) return (year, month, date) except: return (1900, 1, 1) #--------------------------------------------------------------------------------------- #--- extract_data: extract time and ccd_id from the fits file and create count rate data #--------------------------------------------------------------------------------------- def extract_data(file, out_dir ): """ extract time and ccd_id from the fits file and create count rate data input: file --- fits file data out_dir --- the directory in which data will be saved output: ccd<ccd>--- 5 min accumulated count rate data file """ # #--- extract time and ccd id information from the given file # data = pyfits.getdata(file, 0) time_col = data.field('TIME') ccdid_col = data.field('CCD_ID') # #--- initialize # diff = 0 chk = 0 ccd_c = [0 for x in range(0, 10)] ccd_h = [[] for x in range(0, 10)] # #--- check each line and count the numbers of ccd in the each 300 sec intervals # for k in range(0, len(time_col)): try: ftime = float(time_col[k]) ccd_id = int(ccdid_col[k]) except: continue if ftime > 0: if chk == 0: ccd_c[ccd_id] += 1 stime = ftime diff = 0 chk = 1 elif diff >= 300.0: # #--- print out counts per 300 sec # for i in range(0, 10): line = str(stime) + '\t' + str(ccd_c[i]) + '\n' ccd_h[i].append(line) # #--- re initialize for the next round # ccd_c[i] = 0 ccd_c[ccd_id] += 1 stime = ftime diff = 0 chk = 0 # #--- accumurate the count until the 300 sec interval is reached # else: diff = ftime - stime ccd_c[ccd_id] += 1 # #--- for the case the last interval is less than 300 sec, #--- estimate the the numbers of hit and adjust # if diff > 0 and diff < 300: ratio = 300.0 / diff for i in range(0, 10): ccd_c[i] *= ratio line = str(stime) + '\t' + str(ccd_c[i]) + '\n' ccd_h[i].append(line) for i in range(0, 10): ifile = out_dir + '/ccd' + str(i) with open(ifile, 'a') as fo: for ent in ccd_h[i]: fo.write(ent) #--------------------------------------------------------------------------------------- #--- get_ephin_list: create an ephin data list --- #--------------------------------------------------------------------------------------- def get_ephin_list(): """ create an ephin data list input: ephin data --- /dsops/ap/sdp/cache/*/ephin/*lc1.fits output: input_data_list: a list of ephin data file which is not extract """ cmd = 'ls -d /dsops/ap/sdp/cache/*/ephin/*lc1.fits > ' cmd = cmd + house_keeping + '/ephin_dir_list' os.system(cmd) # #--- get a list of the current entries # ifile = house_keeping + '/ephin_dir_list' data = mcf.read_data_file(ifile) # #--- read the list which we read the last time # ifile = house_keeping + '/ephin_old_dir_list' old = mcf.read_data_file(ifile) # #--- find the last entry of the list # last_entry = old[len(old)-1] # #--- select data which is new # input_data_list = [] chk = 0 for ent in data: if chk == 0: if ent == last_entry: chk = 1 else: input_data_list.append(ent) # #--- replace the old list with the new one # ifile = house_keeping + '/ephin_dir_list' if os.path.isfile(ifile): cmd = 'mv ' + house_keeping + '/ephin_dir_list ' + house_keeping + '/ephin_old_dir_list' os.system(cmd) return input_data_list #--------------------------------------------------------------------------------------- #-- extract_ephin_data: extract ephine data from a given data file name and save it in out_dir #--------------------------------------------------------------------------------------- def extract_ephin_data(ifile, out_dir): """ extract ephine data from a given data file name and save it in out_dir input: ifile --- ephin data file name out_dir --- directory which the data is saved output: <out_dir>/ephin_data --- ephin data (300 sec accumulation) """ # #--- extract time and ccd id information from the given file # data = pyfits.getdata(ifile, 1) time_r = data.field("TIME") scp4_r = data.field("SCP4") sce150_r = data.field("SCE150") sce300_r = data.field("SCE300") sce1500_r = data.field("SCE1300") # #--- initialize # diff = 0 chk = 0 ephin_data = [] # #--- sdata[0]: scp4, sdata[1]: sce150, sdata[2]: sce300, and sdata[3]: sce1300 # sdata = [0 for x in range(0, 4)] # #--- check each line and count the numbers of ccd in the each 300 sec intervals # for k in range(0, len(time_r)): try: ftime = float(time_r[k]) val1 = float(scp4_r[k]) val2 = float(sce150_r[k]) val3 = float(sce300_r[k]) val4 = float(sce1500_r[k]) except: continue if ftime > 0: if chk == 0: sdata[0] += val1 sdata[1] += val2 sdata[2] += val3 sdata[3] += val4 stime = ftime diff = 0 chk = 1 elif diff >= 300.0: # #--- print out counts per 300 sec # line = str(stime) + '\t' for j in range(0, 4): line = line + str(sdata[j]) + '\t' sdata[j] = 0 line = line + '\n' ephin_data.append(line) chk = 0 # #--- re initialize for the next round # try: val1 = float(scp4_r[k]) val2 = float(sce150_r[k]) val3 = float(sce300_r[k]) val4 = float(sce1500_r[k]) except: continue sdata[0] += val1 sdata[1] += val2 sdata[2] += val3 sdata[3] += val4 stime = ftime diff = 0 # #--- accumurate the count until the 300 sec interval is reached # else: diff = ftime - stime try: val1 = float(scp4_r[k]) val2 = float(sce150_r[k]) val3 = float(sce300_r[k]) val4 = float(sce1500_r[k]) except: continue sdata[0] += val1 sdata[1] += val2 sdata[2] += val3 sdata[3] += val4 # #--- for the case the last interval is less than 300 sec, #--- estimate the the numbers of hit and adjust # if diff > 0 and diff < 300: line = str(stime) + '\t' ratio = 300.0 / diff for j in range(0, 4): var = sdata[j] * ratio line = line + str(var) + '\t' line = line + '\n' ephin_data.append(line) ifile = out_dir + '/ephin_rate' with open(ifile, 'a') as fo: for ent in ephin_data: fo.write(ent) #--------------------------------------------------------------------------------------- #-- cleanUp: sort and remove duplicated lines in all files in given data directory --- #--------------------------------------------------------------------------------------- def cleanUp(cdir): """ sort and remove duplicated lines in all files in given data directory input cdir --- directory name output cdir/files --- cleaned up files """ if os.listdir(cdir) != []: cmd = 'ls ' + cdir + '/* > ' + zspace os.system(cmd) data = mcf.read_data_file(zspace, remove=1) for ifile in data: # #--- avoid html and png files # try: m = re.search('\.', ifile) if m is None: mcf.remove_duplicated_lines(ifile, chk = 1, srt=1) except: pass #----------------------------------------------------------------------------------------- #-- TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST TEST --- #----------------------------------------------------------------------------------------- class TestFunctions(unittest.TestCase): """ testing functions """ def test_check_date(self): out = check_date() print("YEAR: MON: Dir: " + str(out)) #------------------------------------------------------------ if __name__ == "__main__": unittest.main()
def create_sub_html_page(ifile=''): """ read the old html page and create new sub aca subpage input: ifile --- old html page, if not given, read from /Temp_comp_area/acatrd.html output: <web_dir>/<MMM><yy>/acatrd.html. plots are also copied to that directory """ ichk = 0 if ifile == '': ifile = exc_dir + '/Temp_comp_area/acatrd.html' ichk = 1 # #--- create the data lists from the old formatted html page # data, tsave = extract_date_from_old_html(ifile) # #--- create a new table # htable = create_html_table(data) # #--- find the year and month of the data created # out = Chandra.Time.DateTime(0.5 * (tsave[0] + tsave[1])).date atemp = re.split('\.', out) out = time.strftime('%Y:%m:%d', time.strptime(atemp[0], '%Y:%j:%H:%M:%S')) atemp = re.split(':', out) year = atemp[0] mon = atemp[1] dyear = int(float(year)) dmon = int(float(mon)) lmon = mcf.change_month_format(mon) # #--- read the template # tfile = house_keeping + 'Template/sub_page' with open(tfile, 'r') as f: template = f.read() # #--- substitute the values # template = template.replace('#YEAR#', year) template = template.replace('#MON#', lmon) template = template.replace('#TABLE#', htable) # #--- substitute magnitude slope and std values # ifile = data_dir + 'monthly_mag_stats' template = substitue_mag_stat(template, ifile, dyear, dmon) # #--- substitute slot slope and std values # ifile = data_dir + 'diff_mtatr_month_slope' template = substitue_slot_stat(template, ifile, dyear, dmon, 'POS') ifile = data_dir + 'diff_mtatr_month_slope' template = substitue_slot_stat(template, ifile, dyear, dmon, 'DIFF') ifile = data_dir + 'acacent_mtatr_month_slope' template = substitue_slot_stat(template, ifile, dyear, dmon, 'ANGY') template = substitue_slot_stat(template, ifile, dyear, dmon, 'ANGZ', col_start=10) # #--- substitute slot tables # gtable = make_slot_star_table(dyear, dmon, 'guide_gsst') template = template.replace('#GUIDE_LOOK#', gtable) ftable = make_slot_star_table(dyear, dmon, 'fid_gsst') template = template.replace('#FID_LOOK#', ftable) mtable = make_slot_star_table(dyear, dmon, 'monitor_gsst') template = template.replace('#MON_LOOK#', mtable) # #--- check the output directory and if it does not exist, create # odir = web_dir + lmon.upper() + year[2] + year[3] + '/' if not os.path.isdir(odir): cmd = 'mkdir -p ' + odir os.system(cmd) # #--- write the new html page # ofile = odir + 'acatrd.html' with open(ofile, 'w') as fo: fo.write(template)
def make_slot_star_table(dyear, dmon, tail): """ create a slot data table for the monthly html page input: dyear --- year dmon --- month tail --- a part of fits file name used to find the file output: line --- a html table string """ # #--- find data fits file # lyear = str(dyear) ddir = data_dir + 'Fits_data/' + mcf.change_month_format( dmon).upper() + lyear[2] + lyear[3] + '/' cmd = 'ls ' + ddir + '*_' + tail + '.fits.gz > ' + zspace os.system(cmd) # #--- if there is no data fits file, say so # out = mcf.read_data_file(zspace, remove=1) if len(out) == 0: return '<h3>No Slot Data</h3>' # #--- read the data fits file # fits = out[0].strip() fout = pyfits.open(fits) fdata = fout[1].data fout.close() # #--- extract each column data # good = fdata['good'] marginal = fdata['marginal'] bad = fdata['bad'] ra_rms = fdata['rms_ra_err'] dec_rms = fdata['rms_dec_err'] mag_rms = fdata['rms_delta_mag'] angy = fdata['avg_angynea'] angz = fdata['avg_angznea'] nums = fdata['number'] num_nea = fdata['number_nea'] # #--- create slot data html data table # line = '<table border=1 cellpadding=3>\n' line = line + '<tr>\n' line = line + '<th>slot</th><th>rms_ra_err (deg)</th><th>rms_dec_err (deg)</th>\n' line = line + '<th>rms_delta_mag (marcsec)</th><th>number</th>\n' line = line + '<th>avg_angynea (arcsec)</th><th>avg_angznea (arcsec)</th>\n' line = line + '<th>number_nea</th>\n' line = line + '<th>good</th><th>marginal</th><th>bad</th>\n' line = line + '</tr>\n' for k in range(0, 8): line = line + '<tr>\n' line = line + '<th>' + str(k) + '</th>' line = line + '<td>%2.3e</td>' % ra_rms[k] line = line + '<td>%2.3e</td>' % dec_rms[k] line = line + '<td>%2.3e</td>' % mag_rms[k] line = line + '<td>%4d</td>' % nums[k] line = line + '<td>%2.3e</td>' % angy[k] line = line + '<td>%2.3e</td>' % angz[k] line = line + '<td>%4d</td>' % num_nea[k] line = line + '<td>%2.2f</td>' % good[k] line = line + '<td>%2.2f</td>' % marginal[k] line = line + '<td>%2.2f</td>' % bad[k] line = line + '</tr>\n' line = line + '<tr>\n' line = line + '<th>AVG</th>' line = line + '<td>%2.3e</td>' % numpy.mean(ra_rms) line = line + '<td>%2.3e</td>' % numpy.mean(dec_rms) line = line + '<td>%2.3e</td>' % numpy.mean(mag_rms) line = line + '<td>%4d</td>' % numpy.mean(nums) line = line + '<td>%2.3e</td>' % numpy.mean(angy) line = line + '<td>%2.3e</td>' % numpy.mean(angz) line = line + '<td>%4d</td>' % numpy.mean(num_nea) line = line + '<td>%2.2f</td>' % numpy.mean(good) line = line + '<td>%2.2f</td>' % numpy.mean(marginal) line = line + '<td>%2.2f</td>' % numpy.mean(bad) line = line + '</tr>\n' line = line + '</table>' return line
def acis_dose_monthly_report(year='NA', month='NA'): """ create monthly report tables input: year --- year month --- month output: monthly_diff_<mm>_<yyyy> monthly_acc_<mm>_<yyyy> """ # #--- if year and/or month is not given, find the latest year month entry # if year == 'NA' or month == 'NA': ifile = data_out + 'i_2_n_0_acc_out' data = mcf.read_data_file(ifile) line = data[-1] atemp = re.split('\s+|\t+', line) year = int(atemp[0]) month = int(atemp[1]) syear = str(year) smon = mcf.add_leading_zero(month) dline = '' aline = '' # #--- convert month in digit to month in letter # lmon = mcf.change_month_format(month) lmon = lmon.lower() # #--- find monthly stat # diff = mon_acis_dir + 'ACIS_' + smon + '_' + syear + '.fits.gz' (mean, std, dmin, dmax) = getstat(diff) dline = dline + 'ACIS_' + lmon + syear[2] + syear[3] + ' 6004901 ' dline = dline + '%3.3f %3.3f %3.1f %4d\n\n' % ( mean, std, dmin, dmax) # #--- find cumulative stat # acc = cum_acis_dir + 'ACIS_07_1999_' + smon + '_' + syear + '.fits.gz' (mean, std, amin, amax) = getstat(acc) aline = aline + 'ACIS_total 6004901 ' aline = aline + '%3.3f %3.3f %3.1f %6d\n\n' % ( mean, std, amin, amax) # #--- now print stat for each section # for inst in ('i', 's'): #--- acis i or s for ccd in (2, 3): #--- ccd 2 or 3 (i2/i3/s2/s3) dline = dline + '\n' aline = aline + '\n' for node in (0, 1, 2, 3): #--- node for dtype in ('dff', 'acc'): #--- monthly or cumulative ifile = data_out + inst + '_' + str(ccd) + '_n_' ifile = ifile + str(node) + '_' + dtype + '_out' data = mcf.read_data_file(ifile) if year == 'NA' or month == 'NA': out = data[-1] atemp = re.split('\s+', out) else: for ent in data: atemp = re.split('\s+', ent) if int(atemp[0]) == year and int( atemp[1]) == month: break mline = inst.upper() + str(ccd) + ' node ' + str( node) + ' 262654\t' mline = mline + '%3.6f\t' % (float(atemp[2])) mline = mline + '%3.6f\t' % (float(atemp[3])) mline = mline + '%3.1f\t' % (float(atemp[4])) mline = mline + '%5.1f\n' % (float(atemp[6])) if dtype == 'dff': dline = dline + mline else: aline = aline + mline ofile = './monthly_diff_' + smon + '_' + syear with open(ofile, 'w') as fo: fo.write(dline) ofile = './monthly_acc_' + smon + '_' + syear with open(ofile, 'w') as fo: fo.write(aline)
def find_scheduled_obs(): """ find MP scheduled observations input: none but read from /data/mpcrit1/mplogs/* directory ouptput <obs_ss>/scheduled_obs_list """ # #--- find today's date # out = time.strftime('%Y:%m:%d', time.gmtime()) atemp = re.split(':', out) year = int(float(atemp[0])) mon = int(float(atemp[1])) mday = int(float(atemp[2])) umon = mcf.change_month_format(mon).upper() lmon = mcf.change_month_format(mon).lower() # #--- next month # nyear = year nmon = mon + 1 if mon > 12: nyear += 1 nmon = 1 unmon = mcf.change_month_format(nmon).upper() lnmon = mcf.change_month_format(nmon).lower() # #--- find all MP OR lists scheduled to start after today # cmd = 'ls -rtd /data/mpcrit1/mplogs/' + str( year) + '/' + umon + '* > ' + zspace os.system(cmd) d_list = mcf.read_data_file(zspace, remove=1) # #--- find direcotries hold data after today's date #--- the directory name: <MMM><dd><yy>, e.g. JUL2020 # n_list = [] for ent in d_list: atemp = re.split(umon, ent) date = atemp[1][0] + atemp[1][1] date = int(float(date)) if date > mday: n_list.append(ent) # #--- find all *.or files and mp persons responsible for that schedule # [f_list, u_list] = find_file_and_mp(n_list) # #--- check the next month, if directory exists # cmd = 'ls /data/mpcrit1/mplogs/' + str(nyear) + '> ' + zspace os.system(cmd) with open(zspace, 'r') as f: test = f.read() mcf.rm_files(zspace) mc = re.search(unmon, test) if mc is not None: cmd = 'ls -rltd /data/mpcrit1/mplogs/' + str( nyear) + '/' + unmon + '* >> ' + zspace os.system(cmd) d_list = mcf.read_data_file(zspace, remove=1) [f_list2, u_list2] = find_file_and_mp(d_list, nchk=1) f_list = f_list + f_list2 u_list = u_list + u_list2 # #--- get all observations under schedules # m_list = [] #--- a list of obsids o_dict = {} #--- a dictionary to keep info related obsids for k in range(0, len(f_list)): ifile = f_list[k] mp_person = u_list[k] [o_dict, m_list] = find_obsids(ifile, o_dict, m_list, mp_person) # #--- remove duplicate ans sort it # m_list = sorted(list(set(m_list))) # #--- print out the scheduled list # line = '' for obsid in m_list: line = line + str(obsid) + '\t' + o_dict[obsid][-1] + '\n' ofile = obs_ss + 'scheduled_obs_list' with open(ofile, 'w') as fo: fo.write(line)
def update_one_year_pages(iyear=''): """ upate yearly html page input: year --- year. if it is not given, update all years output: <web_dir>/aca_trend_year<yyyy>.html """ # #--- find the current year and month # out = time.strftime('%Y:%m:%d', time.gmtime()) atemp = re.split(':', out) this_year = int(float(atemp[0])) this_month = int(float(atemp[1])) this_day = int(float(atemp[2])) if this_day < 5: this_month -= 1 if this_month < 1: this_month = 12 this_year -= 1 # #--- set the year interval to create the pages # if iyear == '': syear = 1999 eyear = this_year + 1 else: syear = iyear eyear = iyear + 1 for year in range(syear, eyear): lyear = str(year) syear = lyear[2] + lyear[3] tfile = house_keeping + 'Template/one_year_template' with open(tfile, 'r') as f: template = f.read() template = template.replace('#YEAR#', lyear) template = template.replace('#SYEAR#', syear) # #--- substitue slope and std of magnitude data # dfile = 'yearly_mag_stats' template = substitue_year_page(template, dfile, year, 14, 'MAG') # #--- substitute slope and std of pos_err # dfile = 'pos_err_mtatr_year_slope' template = substitue_year_page(template, dfile, year, 8, 'POS') # #--- substitue slope and std of magnitue difference # dfile = 'diff_mtatr_year_slope' template = substitue_year_page(template, dfile, year, 8, 'DIFF') # #--- substitute slope and std of angy # dfile = 'acacent_mtatr_year_slope' template = substitue_year_page(template, dfile, year, 8, 'ANGY') # #--- subsititute slope and std of angz # dfile = 'acacent2_mtatr_year_slope' template = substitue_year_page(template, dfile, year, 8, 'ANGZ') # #--- create a table to link to the each month of this year # if year == 1999: smon = 8 emon = 12 elif year == this_year: smon = 1 emon = this_month else: smon = 1 emon = 12 line = '<table border=1 cellspacing=2 cellpadding=2>\n' line = line + '<tr>\n' for mon in range(1, 13): lmon = mcf.change_month_format(mon).upper() if mon < smon: line = line + '<td>' + lmon + '</td>\n' elif mon > emon: line = line + '<td>' + lmon + '</td>\n' else: line = line + '<td><a href="./' + lmon + syear line = line + '/acatrd.html">' + lmon + '</a>\n' line = line + '</table>\n' template = template.replace('#LTABLE#', line) # #--- update the html page # outname = web_dir + 'aca_trend_year' + str(year) + '.html' with open(outname, 'w') as fo: fo.write(template)
def run_hrc_stowed_background_monthly_update(lyear='', lmonth=''): """ run monthly update for the hrc stowed background data input: lyear, lmonth --- the year and month of the data to be processed if they are not given, the last month of year and year will be used output: newly proccessed data: <data_dir>/<yyyy><mmm>/* if it is jan or jul, cummulative maps for the year (and the entire period) are created """ # #--- clean up exc dir # cmd = 'rm -rf ' + exc_dir + 'Temp_dir/* 2> /dev/null' os.system(cmd) cmd = 'rm -rf ' + exc_dir + 'param 2> /dev/null' os.system(cmd) cmd = 'mkdir ' + exc_dir + 'param 2> /dev/null' os.system(cmd) # #--- if date (year/month) to be processed is not given, find the last month's year, month # if lyear == '': ctime = time.gmtime() #--- today's date information lyear = ctime.tm_year lmonth = ctime.tm_mon -1 if lmonth < 1: lyear -= 1 lmonth = 12 chk = 0 else: chk = 1 # #--- check whether the data was already processed; if not, run the process #--- except re-analysis is requested by given year and month # smonth = mcf.change_month_format(lmonth) cdir = data_dir + str(lyear) + smonth.upper() if chk == 0 and (not os.path.isdir(cdir)): chk = 1 smail = 0 if chk == 1: try: hsb.hrc_stowed_background(lyear, lmonth, lyear, lmonth) # #--- if the processed month is june, septempber or december, update maps and html pages # if (lmonth == 12) or (lmonth == 6) or (lmonth == 9): # #--- update total event number tables # chif.yearly_cummulative_image(lyear) get_yearly_evt_count() #---- create yearly event counts hpm.plot_hrc_map(lyear) hpm.plot_hrc_map('total') #---- creating cumulative maps hpm.plot_hrc_map('total', 1) #---- creating cumulative histogram for the front page hpr.plot_hrc_trend_data() #---- creating trending plots uhhp.create_html_pages(lyear) #---- updating all html pages uhhp.update_stat_data_table() #---- updating stat html pages create_cumulative_image() if test_data_creation(lyear, lmonth): smail = 1 else: smail = 3 except: smail = 2 else: smail = 0 # #--- remove the temp file if it is still around # cmd = 'rm -f /tmp/zspace*hrc* 2> /dev/null' os.system(cmd) cmd = 'rm -rf ' + exc_dir + 'Temp_dir/* 2> /dev/null' os.system(cmd) # #--- sending mail # if smail == 1: message = 'hrc stowed proccess ran normally for the period: ' message = message + str(lyear) + ' : ' + str(lmonth) + '\n' header = '"Subject: HRC Stowed Data Process Complated"' elif smail == 2: message = 'hrc stowed proccess ran into problems with the data for the period: ' message = message + str(lyear) + ' : ' + str(lmonth) + '\n' header = '"Subject: HRC Stowed Data Process Failed"' elif smail == 3: message = 'hrc stowed proccess did not find any data for the period: ' message = message + str(lyear) + ' : ' + str(lmonth) + '\n' header = '"Subject: HRC Stowed Data Process: No Data Found"' if smail > 0: send_email(message, header)