def mv_old_file(dom): dom -= 30 if dom > 0: [year, ydate] = tcnv.DOMtoYdate(dom) sydate = str(ydate) if ydate < 10: sydate = '00' + sydate elif ydate < 100: sydate = '0' + sydate atime = str(year) + ':' + sydate + ':00:00:00' stime = tcnv.axTimeMTA(atime) cmd = 'ls ' + house_keeping + '/Defect/CCD*/* > ' + zspace os.system(cmd) fs = open(zspace, 'r') ldata = [line.strip() for line in fs.readlines()] fs.close() mcf.rm_file(zspace) for ent in ldata: atemp = re.split('\/acis', ent) btemp = re.split('_', atemp[1]) if int(btemp[0]) < stime: out = ent out = out.replace('Defect', 'Defect/Save') cmd = 'mv ' + ent + ' ' + out os.system(cmd)
def stime_to_ddate(stime): """ change data in second from 1998.1.1 to mm/dd/yy format input: stime --- time in seconds from 1998.1.1 output: dtime --- date in the form of mm/dd/yy (e.g. 08/19/15) """ tlist = tcnv.axTimeMTA(stime) atemp = re.split(':', tlist) year = int(float(atemp[0])) ydate = int(float(atemp[1])) [mon, date] = tcnv.changeYdateToMonDate(year, ydate) lyear = str(atemp[0]) syr = lyear[2] + lyear[3] smon = str(mon) if mon < 10: smon = '0' + smon sday = str(date) if date < 10: sday = '0' + sday dtime = smon + '/' + sday + '/' + syr return dtime
def stime_to_ddate2(stime): """ change data in second from 1998.1.1 to yyyymmdd format input: stime --- time in seconds from 1998.1.1 output: dtime --- date in the form of yyyymmdd (e.g. 20150819) """ tlist = tcnv.axTimeMTA(stime) atemp = re.split(':', tlist) year = int(float(atemp[0])) lyear = str(atemp[0]) ydate = int(float(atemp[1])) [mon, date] = tcnv.changeYdateToMonDate(year, ydate) smon = str(mon) if mon < 10: smon = '0' + smon sday = str(date) if date < 10: sday = '0' + sday dtime = lyear + smon + sday return dtime
def get_data_period(): """ set start and stop time of the data period input: none output: [start, stop] --- in format of mm/dd/yy """ # #--- find the last date of the data update # maxt = 0 for dname in ('I2cp.dat', 'I3cp.dat', 'S2cp.dat', 'S3cp.dat'): file = web_dir + dname f = open(file, 'r') data = [line.strip() for line in f.readlines()] f.close() atemp = re.split('\s+', data[-1]) val = float(atemp[0]) if val > maxt: maxt = int(val) atemp = tcnv.axTimeMTA(str(maxt)) ttemp = re.split(':', atemp) year = int(float(ttemp[0])) ydate = int(float(ttemp[1])) (month, mdate) = tcnv.changeYdateToMonDate(year, ydate) smon = str(month) if month < 10: smon = '0' + smon sday = str(mdate) if mdate < 10: sday = '0' + sday start = smon + '/' + sday + '/' + ttemp[0][2] + ttemp[0][3] # #--- find today's date; it will be the last date of the period # ctime = datetime.datetime.now() syear = str(ctime.year) month = ctime.month cmon = str(month) if month < 10: cmon = '0' + cmon date = ctime.day cday = str(date) if date < 10: cday = '0' + cday stop = cmon + '/' + cday + '/' + syear[2] + syear[3] return [start, stop]
def read_rad_zone_data(start, stop): """ extract radiation zone information for a given time span input: start --- starting time stop --- stopping time data are read from /data/mta/Script/Interrupt/house_keeping/rad_zone_info output: rad_start --- a list of radiation zone starting time rad_stop --- a list of radiation zone stopping time """ f = open('/data/mta/Script/Interrupt/house_keeping/rad_zone_info', 'r') data = [line.strip() for line in f.readlines()] f.close() rad_start = [] rad_stop = [] rad_in = 0 for ent in data: atemp = re.split('\s+', ent) try: val = float(atemp[1]) #---- check time is in digit (dom) except: continue [year, ydate] = tcnv.DOMtoYdate(float(atemp[1])) # #--- convert time to sec from 1998.1.1 # day_part = int(ydate) rest = ydate - day_part hour = int(24 * rest) rest = 24 * rest - hour minutes = int(60 * rest) ltime = str(year) + ':' + str(day_part) + ':' + str(hour) + ':' + str(minutes) + ':00' stime = tcnv.axTimeMTA(ltime) if stime >= start and stime < stop: if atemp[0] == 'ENTRY' and rad_in == 0: rad_start.append(stime) rad_in = 1 elif atemp[0] == 'EXIT' and rad_in == 1: rad_stop.append(stime) rad_in = 0 elif stime >=stop: break else: continue if len(rad_stop) < len(rad_start): rad_stop.append(stop) return [rad_start, rad_stop]
def stime_to_ydate(stime): """ convert time in sec from 1998.1.1 to ydate. no year info input: stime ---- time in sec from 1998.1.1 output: ydate ---- ydate """ time = tcnv.axTimeMTA(int(stime)) atemp = re.split(':', time) ydate = float(atemp[1]) + float(atemp[2])/24.0 + float(atemp[3])/1440.0 + float(atemp[4]) / 86400.0 return ydate
def sec1998tofracday(stime): """ convert time from seconds from 1998.1.1 to fractional yday input: stime --- time in seconds from 1998.1.1 output: lday --- fractional year date. igore year """ ptime = tcnv.axTimeMTA(stime) atemp = re.split(':', ptime) day = float(atemp[1]) hh = float(atemp[2]) mm = float(atemp[3]) dtime = day + hh/24.0 + mm/ 1440.0 lday = "%.2f" % round(dtime, 2) return lday
def current_time(): """ THIS FUNCTION IS NOT USED IN THIS SCRIPT ANYMORE find current time in seconds from 1998.1.1 input: none output: dtime --- the current time in seconds from 1998.1.1 """ stime = time.time() out = time.gmtime() stime -= tcorrect ct = str(out[0]) + ':' + str(out[7]) + ':' + str(out[3]) + ':' + str(out[4]) + ':' + str(out[5]) dtime = tcnv.axTimeMTA(ct) return dtime
def make_data_list(year, mon, day, tail): """ make a data list from the last entry date to the most current data input: year --- the year of the last entry mon --- the month of the last entry day --- the day of month of the last entry tail --- the suffix of the data file output dlist --- a list of data names """ # #--- convert the date into seconds from 1998.1.1 # dst = tcnv.convertDateToTime2(year, mon, day) # #--- find today's time # today = tcnv.currentTime() cyear = today[0] cmon = today[1] cday = today[2] cdst = tcnv.convertDateToTime2(cyear, cmon, cday) dlist = [] # #--- check the current date is larger than the date indicated #--- if so find how many days between and retrieve data for each day # if cdst > dst: step = int(( cdst - dst)/86400) if step >= 1: head = make_header(year, mon, day) name = head + tail dlist.append(name) for i in range(2, step): sdate = int(cdst - 86400.0 * i) out = tcnv.axTimeMTA(sdate) atemp = re.split(':', out) year = int(atemp[0]) ydate = int(atemp[1]) [mon, day] = tcnv.changeYdateToMonDate(year, ydate) head = make_header(year, mon, day) name = head + tail dlist.append(name) return dlist
def create_display_data_table(): """ create a readable data table for html page Input: none, but read from <data_dir>/ccd<ccd>_<node> Output: <web_dir>/ccd<ccd>_<node> """ for ccd in range(0, 10): for node in range(0, 4): file = 'ccd' + str(ccd) + '_' + str(node) infile = data_dir + file outfile = web_dir + 'Data/' + file f = open(infile, 'r') data = [line.strip() for line in f.readlines()] f.close() fo = open(outfile, 'w') # #--- adding heading # line = "#\n#Date Mn K alpha Al K alpha Ti K alpha Slope Sigma Int Sigma\n#\n" fo.write(line) for ent in data: atemp = re.split('\s+', ent) stime = int(atemp[0]) # #--- converting the date into <mon> <year> form (e.g. May 2013) # ltime = tcnv.axTimeMTA(stime) btemp = re.split(':', ltime) year = btemp[0] [mon, mdate] = tcnv.changeYdateToMonDate(int(year), int(btemp[1])) lmon = tcnv.changeMonthFormat(mon) line = lmon + ' ' + year for j in range(1, len(atemp)): line = line + '\t' + atemp[j] line = line + '\n' fo.write(line) fo.close()
def extract_sim_data(): """ extract sim data from PRIMARYCCDM_*.*.tl input: none but read from <dumpdir>/PRIMARYCCDM_*.*.tl output: <outdir>sim_data.out """ # #--- find the time of the last entry from the sim_data.out # sfile = outdir + 'sim_data.out' f = open(sfile, 'r') data = [line.strip() for line in f.readlines()] f.close() # #--- cleaning up the data; drop the data which the date starts from ":" e.g. :2014 # pdata = [] for ent in data: if re.search('^:', ent): continue else: pdata.append(ent) # #--- the last entiry values # if len(pdata) > 0: atemp = re.split('\s+', pdata[len(pdata)-1]) ltime = tcnv.axTimeMTA(atemp[0]) #--- converting time to sec from 1998.1.1 time_2 = atemp[0] col1_2 = atemp[1] col2_2 = atemp[2] col3_2 = atemp[3] else: ltime = 0 time_2 = 0 col1_2 = '' col2_2 = '' col3_2 = '' # #--- check whether input files exists # cmd = 'ls -rt ' + dumpdir + 'PRIMARYCCDM_*.*.tl >' + zspace os.system(cmd) f = open(zspace, 'r') data = [line.strip() for line in f.readlines()] f.close() cmd = 'rm ' + zspace os.system(cmd) dlen = len(data) if dlen < 1: exit(1) # #--- files exist. read the data from the last 10 files # tlist = data[dlen-40:] for ent in tlist: cmd = 'cat ' +ent + ' >> ' + zspace os.system(cmd) f = open(zspace, 'r') data = [line.strip() for line in f.readlines()] f.close() cmd = 'rm ' + zspace os.system(cmd) prev = '' fo = open('./temp_save', 'w') # #--- go though each data line # for ent in data: try: # #--- expect the first letter of the data line is numeric (e.g. 2014). # val = float(ent[0]) except: continue # #--- only data with "FMT" format will be used # mc = re.search('FMT', ent) if mc is None: continue atemp = re.split('\t+', ent) # #--- if there are less than 20 entries, something wrong; skip it # if len(atemp) < 20: continue # #--- convert time format # time = atemp[0] time = time.strip(); time = time.replace(' ', ':') time = time.replace(':::', ':00') time = time.replace('::', ':0') # #--- if the time is exactly same as one before, skip it # if time == time_2: continue # #--- if the time is already in the database, keip it # stime = tcnv.axTimeMTA(time) if stime <= ltime: continue # #--- use only data which tscpos and fapos have numeric values # tscpos = atemp[4].strip() fapos = atemp[5].strip() if tscpos == "" or fapos == "": continue else: tscpos = int(float(tscpos)) fapos = int(float(fapos)) # aopc = atemp[11].strip() # if aopc == '': # aopc = '0' mpwm = atemp[12].strip() if mcf.chkNumeric(mpwm): mpwm = int(float(mpwm)) mpwm = str(mpwm) else: mpwm = '0' # #--- we want to print only beginning and ending of the same data entries. #--- skip the line if all three entiries are same as one before, except the last one # if col1_2 == tscpos and col2_2 == fapos and col3_2 == mpwm: time_2 = time continue line = time + '\t' + str(tscpos) + '\t' + str(fapos) + '\t' + mpwm + '\n' if line == prev: continue else: pline = time_2 + '\t' + str(col1_2) + '\t' + str(col2_2) + '\t' + str(col3_2) + '\n' fo.write(pline) fo.write(line) prev = line time_2 = time col1_2 = tscpos col2_2 = fapos col3_2 = mpwm fo.close() sfile2 = sfile + '~' cmd = 'cp ' + sfile + ' ' + sfile2 os.system(cmd) cmd = 'cat ./temp_save >> ' + sfile os.system(cmd)
def acis_gain_plot_trend(): """ plotting trends of gain and offset Input: none, but read from <data_dir> Output: <web_dir>/Plots/gain_plot_ccd<ccd>.png <web_dir>/Plots/offset_plot_ccd<ccd>.png """ for ccd in range(0, 10): # #--- plotting 4 nodes on one panel, but gain and offset separately # Xset_gain = [] Yset_gain = [] Eset_gain = [] yMin_gain = [] yMax_gain = [] Label_gain = [] Xset_offset = [] Yset_offset = [] Eset_offset = [] yMin_offset = [] yMax_offset = [] Label_offset = [] for node in range(0, 4): # #--- read data for given CCD and Node # # file = data_dir + 'ccd' + str(ccd) + '_' + str(node) f = open(file, 'r') data = [line.strip() for line in f.readlines()] f.close() time = [] gain = [] gerr = [] #--- error for gain offset = [] oerr = [] #--- error for offset # #--- setting lower and upper limits to remove outlyers # sum = 0 sum2 = 0 for ent in data: atemp = re.split('\s+', ent) gval = float(atemp[4]) sum += gval sum2 += gval * gval avg = sum / len(data) sig = math.sqrt(sum2 / len(data) - avg * avg) blim = avg - 3.0 * sig tlim = avg + 3.0 * sig for ent in data: atemp = re.split('\s+', ent) # #--- convert time into year date (e.g.2012.14) # gval = float(atemp[4]) if (gval <= blim) or (gval >= tlim): continue stime = tcnv.axTimeMTA(int(atemp[0])) btemp = re.split(':', stime) year = int(btemp[0]) ydate = int(btemp[1]) chk = 4.0 * int(0.25 * year) if chk == year: base = 366.0 else: base = 365.0 ytime = float(year) + float(ydate) / base time.append(ytime) gain.append(float(atemp[4])) gerr.append(float(atemp[5])) offset.append(float(atemp[6])) oerr.append(float(atemp[7])) xmax = max(time) Xset_gain.append(time) Yset_gain.append(gain) Eset_gain.append(gerr) # #--- set plotting range # avg = mean(gain) ymin = avg - 0.002 ymin = round(ymin, 3) - 0.001 ymax = avg + 0.002 ymax = round(ymax, 3) + 0.001 yMin_gain.append(ymin) yMax_gain.append(ymax) name = 'Gain (ADU/eV) Node' + str(node) Label_gain.append(name) Xset_offset.append(time) Yset_offset.append(offset) Eset_offset.append(oerr) avg = mean(offset) ymin = avg - 8.0 ymin = int(ymin) ymax = avg + 8.0 ymax = int(ymax) yMin_offset.append(ymin) yMax_offset.append(ymax) name = 'Offset (ADU) Node' + str(node) Label_offset.append(name) xmin = int(2000) xmax = int(xmax) + 1 xname = 'Time (year)' # #--- actual plotting starts here # yname = 'Gain' plotPanel(xmin, xmax, yMin_gain, yMax_gain, Xset_gain, Yset_gain, Eset_gain, xname, yname, Label_gain) outname = web_dir + "/Plots/gain_plot_ccd" + str(ccd) + '.png' cmd = 'mv out.png ' + outname os.system(cmd) yname = 'Offset' plotPanel(xmin, xmax, yMin_offset, yMax_offset, Xset_offset, Yset_offset, Eset_offset, xname, yname, Label_offset) outname = web_dir + "/Plots/offset_plot_ccd" + str(ccd) + '.png' cmd = 'mv out.png ' + outname os.system(cmd)
def plot_data(start, stop, mag_plot=1): """ create a configulation display panel for a give time period input: start --- starting time in sec from 1998.1.1 stop --- stopping time in sec from 1998.1.1 """ # #---- set a few parameters # if mag_plot == 0: pnum = 5 else: pnum = 6 mpl.rcParams['font.size'] = 11 mpl.rcParams['font.weight'] = 'strong' props = font_manager.FontProperties(size=6) plt.subplots_adjust(hspace=0.05) plt.subplots_adjust(wspace=0.12) # #--- set a few others # xpos = stime_to_ydate(stop) + 0.1 ystep = 0.2 # #--- read sim information # [acis_i_start, acis_i_stop, acis_s_start, \ acis_s_stop, hrc_i_start, hrc_i_stop, \ hrc_s_start, hrc_s_stop, hetg_start, \ hetg_stop, letg_start, letg_stop, \ radmon_start, radmon_stop, fmt, time] = erd.find_sim_position(start, stop) # #--- hetg /letg information plot # start_set = [hetg_start, letg_start] stop_set = [hetg_stop, letg_stop] ax1 = plt.subplot(pnum, 1, 1) plot_strip_box(ax1,start, stop, start_set, stop_set, color1) plt.text(xpos, 0.8, "HETG", color=color1[0]) plt.text(xpos, 0.6, "LETG", color=color1[1]) # #--- acis /hrc information plot # start_set = [acis_i_start, acis_s_start, hrc_i_start, hrc_s_start] stop_set = [acis_i_stop, acis_s_stop, hrc_i_stop, hrc_s_stop] ax2 = plt.subplot(pnum, 1, 2) plot_strip_box(ax2,start, stop, start_set, stop_set, color1) plt.text(xpos, 0.9, "ACIS I ", color=color1[0]) plt.text(xpos, 0.7, "ACIS S ", color=color1[1]) plt.text(xpos, 0.5, "HRC I ", color=color1[2]) plt.text(xpos, 0.3, "HRC S ", color=color1[3]) # #--- cti information plot # [cti_start, cti_stop] = erd.read_ccd_data(start, stop) start_set = [cti_start] stop_set = [cti_stop] ax3 = plt.subplot(pnum, 1, 3) plot_strip_box(ax3, start, stop, start_set, stop_set, color1) # #--- altitude information plot # [atime, alt, magx, magy, magz, crm] = erd.read_orbit_data(start, stop) plot_line(ax3, start, stop, atime, alt) plt.text(xpos, 0.8, "CTI ", color=color1[0]) plt.text(xpos, 0.7, "Check ", color=color1[0]) plt.text(xpos, 0.4, "Altitude ", color="green") # #--- radmon information plot # start_set = [radmon_start] stop_set = [radmon_stop] ax4 = plt.subplot(pnum, 1, 4) plot_strip_box(ax4, start, stop, start_set, stop_set, color1) # #--- hrc sheild rate plot # [htime, rate] = erd.read_hrc_data(start, stop) plot_line(ax4, start, stop, htime, rate) # #--- goes p3 rate plot # # [gtime, p1, p2, p3] = erd.read_goes_data(start, stop) # plot_points(ax4, start, stop, gtime, p3, color='lime', pts=0.5,lw=0) plt.text(xpos, 0.8, "Radmon", color=color1[0]) plt.text(xpos, 0.6, "HRC", color="green") plt.text(xpos, 0.5, "Shield", color="green") plt.text(xpos, 0.4, "Rate", color="green") # plt.text(xpos, 0.2, "GOES P3", color="green") # #--- magnetsphere plot # if mag_plot != 0: [start_set, stop_set] = find_mag_region(start, stop) axm = plt.subplot(6, 1, 5) plot_strip_box(axm, start, stop, start_set, stop_set, color1) # plt.text(xpos, 0.9, "Solar", color=color1[0]) # plt.text(xpos, 0.8, "Wind", color=color1[0]) plt.text(xpos, 0.6, "Magneto-", color=color1[1]) plt.text(xpos, 0.5, "sheath", color=color1[1]) plt.text(xpos, 0.3, "Magneto-", color=color1[2]) plt.text(xpos, 0.2, "sphere", color=color1[2]) # #--- often the data are not available; so make a note on the plot # # diff1 = stop - start # tlast = time[len(time)-1] # diff2 = tlast - start # ratio = diff2 / diff1 # if ratio < 0.7: # xnote = 0.5 * (stop - tlast) # plt.text(xnote, 0.5, "No Data", color='maroon') # #--- FMT format information plot # [start_set, stop_set] = find_fmt_region(start, stop, fmt, time) ax5 = plt.subplot(pnum, 1, pnum) plot_strip_box(ax5, start, stop, start_set, stop_set,color4) plt.text(xpos, 0.9, "FMT1", color=color4[0]) plt.text(xpos, 0.7, "FMT2", color=color4[1]) plt.text(xpos, 0.5, "FMT3", color=color4[2]) plt.text(xpos, 0.3, "FMT4", color=color4[3]) plt.text(xpos, 0.1, "FMT5", color=color4[4]) # #--- plot x axis tick label only at the bottom ones # if mag_plot == 0: ax_list = [ax1, ax2, ax3, ax4, ax5] else: ax_list = [ax1, ax2, ax3, ax4, ax5, axm] for ax in ax_list: for label in ax.get_yticklabels(): label.set_visible(False) if ax != ax5: for label in ax.get_xticklabels(): label.set_visible(False) else: pass # #--- x axis label # mid = int(0.5 * (start + stop)) ltime = tcnv.axTimeMTA(mid) atemp = re.split(':', ltime) year = int(atemp[0]) ydate = int(atemp[1]) [mon, day] = tcnv.changeYdateToMonDate(year,ydate) xlabel = "Time (DOY) " + str(atemp[0]) ax5.set_xlabel(xlabel) # #--- set the size of the plotting area in inch (width: 10.0in, height 5.0in) # fig = matplotlib.pyplot.gcf() fig.set_size_inches(12.0, 10.0) # #--- save the plot in png format # syear = str(year) smon = tcnv.changeMonthFormat(mon) smon = smon.lower() outname = 'rad_use_' + smon + syear[2] + syear[3] + '.png' plt.savefig(outname, format='png', dpi=100) plt.close('all')
def report_error(): """ read errors from <cup_usr_list>_error_list, sort it out, clean, and send out email Input: none but read from <cup_usr_list>_error_list Output: email sent out """ # #--- find the current time # [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime("Local") # #--- create surfix for files which will be saved in Past_errors directory # smon = str(mon) if mon < 10: smon = '0' + smon sday = str(day) if day < 10: sday = '0' + sday tail = str(year) + smon + sday for tag in cpu_usr_list: efile = house_keeping + 'Records/' + tag + '_error_list' pfile = house_keeping + 'Records/Past_errors/' + tag + '_error_list_' + tail prev_line = '' chk = mcf.chkFile(efile) if chk > 0: # #--- read error messages from the file # f = open(efile, 'r') data = [line.strip() for line in f.readlines()] f.close() # #--- sort the data so that we can correct messages to each cron job together # data.sort() task_list = [] time_list = [] mssg_list = [] for ent in data: atemp = re.split(' : ' , ent) task_list.append(atemp[0]) stime = int(atemp[1]) dtime = tcnv.axTimeMTA(stime) time_list.append(dtime) mssg_list.append(atemp[2]) # #--- write out cron job name # fo = open(zspace, 'w') cname = task_list[0] line = '\n\n' + cname + '\n____________________\n\n' fo.write(line) for i in range(0, len(mssg_list)): if task_list[i] != cname: cname = task_list[i] line = '\n\n' + cname + '\n____________________\n\n' fo.write(line) # #--- create each line. if it is exactly same as one line before, skip it # line = time_list[i] + ' : ' + mssg_list[i] + '\n' if line != prev_line: fo.write(line) prev_line = line fo.close() # #--- send email out # # cmd = 'cp ' + zspace + ' ' + '/data/mta/Script/Cron_check/Scripts/' + tag # os.system(cmd) send_mail(tag, email_list) # #--- move the error list to Past_errors directory # cmd = 'mv ' + efile + ' ' + pfile os.system(cmd)
# #--- pylab plotting routine related modules # from pylab import * import matplotlib.pyplot as plt import matplotlib.font_manager as font_manager import matplotlib.lines as lines from matplotlib.transforms import Bbox mag_plot = 1 if len(sys.argv) == 3: start = sys.argv[1] stop = sys.argv[2] else: print "Input: start stop in the format of 2014:204:00:00:00 or in seconds from 1998.1.1" exit(1) if __name__ == "__main__": try: start = int(float(start)) stop = int(float(stop)) except: start = tcnv.axTimeMTA(start) stop = tcnv.axTimeMTA(stop) plot_data(start, stop, mag_plot)
def sci_run_add_to_rad_zone_list(file='NA'): """ adding radiation zone list to rad_zone_list. input: file --- the name of inputfile containing: e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto output <house_keeping>/rad_zone_list """ # #--- check whether the list alread exists; if it does, read which ones are already in the list # cmd = 'ls ' + house_keeping + '* > ./ztemp' os.system(cmd) test = open('./ztemp').read() os.system('rm -f ./ztemp') m1 = re.search('rad_zone_list', test) m2 = re.search('rad_zone_list~', test) eventList = [] echk = 0 if m1 is not None: line = house_keeping + 'rad_zone_list' f = open(line, 'r') data = [line.strip() for line in f.readlines()] f.close() for ent in data: atemp = re.split('\s+|\t+', ent) eventList.append(atemp[0]) echk = 1 # #--- if file is not given (if it is NA), ask the file input # if file == 'NA': file = raw_input('Please put the intrrupt timing list: ') f = open(file, 'r') data = [line.strip() for line in f.readlines()] f.close() # #--- put the list in the reverse order # data.reverse() for ent in data: if not ent: break # #--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30) #--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list #--- to each date # etemp = re.split('\s+', ent) echk = 0 for comp in eventList: if comp == etemp[0]: echk = 1 break if echk == 0: atemp = re.split(':', etemp[1]) year = atemp[0] month = atemp[1] date = atemp[2] hour = atemp[3] mins = atemp[4] # #--- convert to dom/sec1998 # ydate = tcnv.findYearDate(int(year), int(month), int(date)) dom = tcnv.findDOM(int(year), int(ydate)) line = year + ':' + str(int(ydate)) + ':' + hour + ':' + mins + ':00' csec = tcnv.axTimeMTA(line) # #--- end date # atemp = re.split(':', etemp[2]) eyear = atemp[0] emonth = atemp[1] edate = atemp[2] ehour = atemp[3] emins = atemp[4] ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate)) line = eyear + ':' + str(int(ydate)) + ':' + ehour + ':' + emins + ':00' csec2 = tcnv.axTimeMTA(line) # #--- date stamp for the list # list_date = str(year) + str(month) + str(date) # #--- check radiation zones for 3 days before to 5 days after from the interruptiondate #--- if the interruption lasted longer than 5 days, extend the range 7 more days # begin = dom - 3 end = dom + 5 diff = csec2 - csec if diff > 432000: end += 7 # #--- read radiation zone infornation # infile = house_keeping + '/rad_zone_info' f = open(infile, 'r') rdata = [line.strip() for line in f.readlines()] f.close() status = [] rdate = [] chk = 0 last_st= '' cnt = 0 for line in rdata: atemp = re.split('\s+', line) dtime = float(atemp[1]) #--- dom of the entry or exit if chk == 0 and atemp[0] == 'ENTRY' and dtime >= begin: status.append(atemp[0]) rdate.append(dtime) chk += 1 last_st = atemp[0] cnt += 1 elif chk > 0 and dtime >= begin and dtime <= end: status.append(atemp[0]) rdate.append(dtime) last_st = atemp[0] cnt += 1 elif atemp[1] > end and last_st == 'EXIT': break elif atemp[1] > end and last_st == 'ENTRY': status.append(atemp[0]) rdate.append(dtime) cnt += 1 break f = open('./temp_zone', 'w') # #--- a format of the output is, e.g.: '20120313 (4614.2141112963,4614.67081268519):...' # line = list_date + '\t' f.write(line) upper = cnt -1 i = 0; while i < cnt: line = '(' + str(rdate[i]) + ',' f.write(line) i += 1 if i < upper: line = str(rdate[i]) + '):' f.write(line) else: line = str(rdate[i]) + ')\n' f.write(line) i += 1 f.close() # #--- append the past rad zone list # oldFile = house_keeping + '/rad_zone_list~' crtFile = house_keeping + '/rad_zone_list' if m1 is not None: cmd = 'cat '+ './temp_zone ' + crtFile + ' > ./temp_comb' os.system(cmd) else: os.system('mv .temp_zone ./temp_comb') os.system('rm ./temp_zone') # #--- save the old file and move the update file to rad_zone_list # if m2 is not None: cmd = 'chmod 775 ' + crtFile + ' ' + oldFile os.system(cmd) if m1 is not None: cmd = 'mv ' + crtFile + ' ' + oldFile os.system(cmd) cmd = 'chmod 644 ' + oldFile os.system(cmd) cmd = 'mv ' + './temp_comb ' + crtFile os.system(cmd)
def sci_run_compute_gap(file = 'NA'): 'for a given file name which contains a list like: "20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto", recompute the lost science time (excluding radiation zone) ' # #--- if file is not given (if it is NA), ask the file input # if file == 'NA': file = raw_input('Please put the intrrupt timing list: ') f = open(file, 'r') data = [line.strip() for line in f.readlines()] f.close() # #--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30) #--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list #--- to each date # update = [] for ent in data: if not ent: #--- if it is a blank line end the operation break etemp = re.split('\s+', ent) atemp = re.split(':', etemp[1]) year = atemp[0] month = atemp[1] date = atemp[2] hour = atemp[3] mins = atemp[4] # #--- convert to dom/sec1998 # ydate = tcnv.findYearDate(int(year), int(month), int(date)) #--- a function from convertTimeFormat dom = tcnv.findDOM(int(year), int(ydate), int(hour), int(mins), 0) #--- a function from convertTimeFormat line = year + ':' + str(ydate) + ':' + hour + ':' + mins + ':00' csec = tcnv.axTimeMTA(line) #--- a function from convertTimeFormat # #--- end date # atemp = re.split(':', etemp[2]) eyear = atemp[0] emonth = atemp[1] edate = atemp[2] ehour = atemp[3] emins = atemp[4] ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate)) dom2 = tcnv.findDOM(int(eyear), int(ydate), int(ehour), int(emins), 0) line = eyear + ':' + str(ydate) + ':' + ehour + ':' + emins + ':00' csec2 = tcnv.axTimeMTA(line) # #--- date stamp for the list # list_date = str(year) + str(month) + str(date) # #--- read radiation zone information from "rad_zone_list" and add up the time overlap with #--- radiatio zones with the interruption time period # line = house_keeping + '/rad_zone_list' f = open(line, 'r') rlist = [line.strip() for line in f.readlines()] f.close() sum = 0 for record in rlist: atemp = re.split('\s+', record) if list_date == atemp[0]: btemp = re.split(':', atemp[1]) for period in btemp: t1 = re.split('\(', period) t2 = re.split('\)', t1[1]) t3 = re.split('\,', t2[0]) pstart = float(t3[0]) pend = float(t3[1]) if pstart >= dom and pstart < dom2: if pend <= dom2: diff = pend - pstart sum += diff else: diff = dom2 - pstart sum += diff elif pstart < dom2 and pend > dom: if pend <= dom2: diff = pend - dom sum += diff else: diff = dom2 - dom sum += diff break sum *= 86400 #--- change unit from day to sec sciLost = (csec2 - csec - sum) / 1000 #--- total science time lost excluding radiation zone passes line = etemp[0] + '\t' + etemp[1] + '\t' + etemp[2] + '\t' + "%.1f" % sciLost + '\t' + etemp[4] update.append(line) # #--- update the file # oldfile = file + '~' cmd = 'mv ' + file + ' ' + oldfile os.system(cmd) f = open(file, 'w') for ent in update: f.write(ent) f.write('\n') f.close()
def acis_gain_plot_trend(): """ plotting trends of gain and offset Input: none, but read from <data_dir> Output: <web_dir>/Plots/gain_plot_ccd<ccd>.png <web_dir>/Plots/offset_plot_ccd<ccd>.png """ for ccd in range(0, 10): # #--- plotting 4 nodes on one panel, but gain and offset separately # Xset_gain = [] Yset_gain = [] Eset_gain = [] yMin_gain = [] yMax_gain = [] Label_gain = [] Xset_offset = [] Yset_offset = [] Eset_offset = [] yMin_offset = [] yMax_offset = [] Label_offset= [] for node in range(0, 4): # #--- read data for given CCD and Node # # file = data_dir + 'ccd' + str(ccd) + '_' + str(node) f = open(file, 'r') data = [line.strip() for line in f.readlines()] f.close() time = [] gain = [] gerr = [] #--- error for gain offset = [] oerr = [] #--- error for offset # #--- setting lower and upper limits to remove outlyers # sum = 0 sum2 = 0 for ent in data: atemp = re.split('\s+', ent) gval = float(atemp[4]) sum += gval sum2 += gval * gval avg = sum / len(data) sig = math.sqrt(sum2/len(data) - avg * avg) blim = avg - 3.0 * sig; tlim = avg + 3.0 * sig; for ent in data: atemp = re.split('\s+', ent) # #--- convert time into year date (e.g.2012.14) # gval = float(atemp[4]) if (gval <= blim) or (gval >= tlim): continue stime = tcnv.axTimeMTA(int(atemp[0])) btemp = re.split(':', stime) year = int(btemp[0]) ydate = int(btemp[1]) chk = 4.0 * int (0.25 * year) if chk == year: base = 366.0 else: base = 365.0 ytime = float(year) + float(ydate) / base time.append(ytime) gain.append(float(atemp[4])) gerr.append(float(atemp[5])) offset.append(float(atemp[6])) oerr.append(float(atemp[7])) xmax = max(time) Xset_gain.append(time) Yset_gain.append(gain) Eset_gain.append(gerr) # #--- set plotting range # avg = mean(gain) ymin = avg - 0.002 ymin = round(ymin, 3) -0.001 ymax = avg + 0.002 ymax = round(ymax, 3) +0.001 yMin_gain.append(ymin) yMax_gain.append(ymax) name = 'Gain (ADU/eV) Node' + str(node) Label_gain.append(name) Xset_offset.append(time) Yset_offset.append(offset) Eset_offset.append(oerr) avg = mean(offset) ymin = avg - 8.0 ymin = int(ymin) ymax = avg + 8.0 ymax = int(ymax) yMin_offset.append(ymin) yMax_offset.append(ymax) name = 'Offset (ADU) Node' + str(node) Label_offset.append(name) xmin = int(2000) xmax = int(xmax) + 1 xname = 'Time (year)' # #--- actual plotting starts here # yname = 'Gain' plotPanel(xmin, xmax, yMin_gain, yMax_gain, Xset_gain, Yset_gain, Eset_gain, xname, yname, Label_gain) outname = web_dir + "/Plots/gain_plot_ccd" + str(ccd) + '.png' cmd = 'mv out.png ' + outname os.system(cmd) yname = 'Offset' plotPanel(xmin, xmax, yMin_offset, yMax_offset, Xset_offset, Yset_offset, Eset_offset, xname, yname, Label_offset) outname = web_dir + "/Plots/offset_plot_ccd" + str(ccd) + '.png' cmd = 'mv out.png ' + outname os.system(cmd)
def create_weekly_report(date, year, debug = 0): """ main script to set up the weekly report template for the week input: date --- date in the format of mmdd (e.g. 0910) year --- year in the format of yyyy (e.g. 2015) debug --- if it is other than 0, print out some output output: a direcotry containing templete (e.g. Sep10) """ # #--- if the test is requested, create Test directory # if debug != 0: os.system('mkdir /data/mta/Script/Weekly/Test/') odir = '/data/mta/Script/Weekly/Test/' else: odir = '/data/mta4/www/REPORTS/' oned = 86400 syear = str(year) #--- 4 digit year yrd2 = year[2] + year[3] #--- 2 digit year year = int(float(year)) #--- integer year date = str(date) smon = date[0] + date[1] #--- two digit month mon = int(float(smon)) #--- integer month lmon = tcnv.changeMonthFormat(mon) #--- month in letter (e.g.Mar) sday = date[2] + date[3] #--- two digit mday day = int(float(sday)) #--- integer mday stop = tcnv.convertDateToCTime(year, mon, day, 0, 0, 0) day_n = stop - 7 * oned # tout = tcnv.axTimeMTA(day_n) # ttemp = re.split(':', tout) # iru_start = str(ttemp[0]) + '_' + str(ttemp[1]) #day01 = stop - 6 * oned #day0 = stop - 7 * oned day01 = stop - 5 * oned day0 = stop - 6 * oned lday0 = stime_to_ddate(day0) sday0 = sdate_to_ldate(lday0) start = day0 lday1 = stime_to_ddate(day01) tout = tcnv.axTimeMTA(day0) ttemp = re.split(':', tout) iru_start = str(ttemp[0]) + '_' + str(ttemp[1]) # #--- year of the beginning of the period; could be different from that of the end # byear = ttemp[0] lday0 = stime_to_ddate(day0) day1 = stop - 5 * oned lday1 = stime_to_ddate(day1) day2 = stop - 4 * oned lday2 = stime_to_ddate(day2) sday2 = sdate_to_ldate(lday2) day3 = stop - 3 * oned lday3 = stime_to_ddate(day3) day4 = stop - 2 * oned lday4 = stime_to_ddate(day4) sday4 = sdate_to_ldate(lday4) day5 = stop - 1 * oned lday5 = stime_to_ddate(day5) day6 = stop lday6 = stime_to_ddate(day6) sday6 = sdate_to_ldate(lday6) #tout = tcnv.axTimeMTA(day5) tout = tcnv.axTimeMTA(day6) ttemp = re.split(':', tout) iru_stop = '_' + str(ttemp[1]) day7 = stop + 1 * oned lday7 = stime_to_ddate(day7) # #---- setting file name # atemp = re.split('\/', lday6) file_date = atemp[0] + atemp[1] file_date2 = atemp[0] + '/' + atemp[1] file_name = file_date + '.html' # #--- title # titledate = lday0 + ' - ' + lday6 ldate = sdate_to_ldate(lday6) ldate_sp = sdate_to_ldate_with_space(lday6) # #--- focal temp file name # fptemp = file_date + '_fptemp.gif' fpext_range = str(start)+' '+ str(stop) fpstart = str(start) fplsub = '"'+ sday0 + '", "' + sday2 + '", "' + sday4 + '", "' + sday6 + '"' fpdsub = str(day0) + ', ' + str(day2) + ', ' + str(day4) + ', ' + str(day6) # #--- IRU span # irudate = iru_start + iru_stop # #--- telemetry idl command # telmstart = stime_to_ddate2(start) telmstop = stime_to_ddate2(stop) telem_command = 'weekly_telem,' + telmstart + ',' + telmstop # #--- telemetry header line # daylist = '|' + lday0 +'|' + lday1 + '|' + lday2 +'|' + lday3 +'|' + lday4 +'|' + lday5 +'|' + lday6 # #--- find trending dates/title # tfile = tdir + 'trending_order' f = open(tfile, 'r') data = [line.strip() for line in f.readlines()] f.close() data.reverse() chk = 0 for ent in data: if ent == '\s+' or ent == '': continue atemp = re.split(' : ', ent) adate = atemp[0].strip() aname = atemp[1].strip() if chk == 0: if adate == file_date2: title = aname chk = 1 else: if aname == title: last_trend_date = adate break # #--- index.html input # s1 = sday0[0:3] + ' ' + sday0[3:5] s2 = sday6[0:3] + ' ' + sday6[3:5] index = '<td> <a href="./' + str(year) + '/' + file_date + '.html">' + s1 + ' - ' + s2 + '</a>' # #--- debugging output # if debug != 0: print "file_name; " + file_name print "title date: " + titledate print "ldate: " + ldate print "fptemp: " + fptemp print "fpext_range: " + fpext_range print "fpstart: " + fpstart print " fplsub: " + fplsub print " fpdsub: " + fpdsub print "irudate: " + irudate print "telmstart: " + telmstart print "telmstop: " + telmstop print "telmcommand: " + telem_command print "daylist: " + daylist print "title: " + title print "last_trend_date: " + last_trend_date print "index: " + index # #--- create a work directory # cmd = 'mkdir ' + wdir + ldate os.system(cmd) outdir = wdir + ldate + '/' cmd = 'cp ' + tdir + 'get_ftemp_data.perl' + ' ' + outdir os.system(cmd) cmd = 'cp ' + tdir + 'subst3.perl' + ' ' + outdir os.system(cmd) cmd = 'cp ' + tdir + 'test' + ' ' + outdir os.system(cmd) cmd = 'mkdir ' + outdir + '/param' os.system(cmd) # #--- create instruction page # tfile = tdir + 'instruction' f = open(tfile, 'r') input = f.read() f.close() input = input.replace('#LDATE_S#', ldate_sp) input = input.replace('#LDATE#', ldate) input = input.replace('#DDATE#', file_date) input = input.replace('#YEAR#', syear) input = input.replace('#DAYLIST#', daylist) input = input.replace('#TELM_CMD#',telem_command) input = input.replace('#INDEX#', index) ofile = outdir + ldate.lower() fo = open(ofile, 'w') fo.write(input) fo.close() # #--- focal temp related files # tfile = tdir + 'get_ftemp_wrap_script' f = open(tfile, 'r') input = f.read() f.close() input = input.replace('#MANDD#', ldate) ofile = outdir + 'get_ftemp_wrap_script' fo = open(ofile, 'w') fo.write(input) fo.close() cmd = 'chmod 755 ' + outdir + 'get_ftemp_wrap_script' os.system(cmd) tfile = tdir + 'get_ftemp_main_script' f = open(tfile, 'r') input = f.read() f.close() #input = input.replace('#START#', str(start)) #input = input.replace('#STOP#', str(stop)) input = input.replace('#START#', str(start + 86400)) input = input.replace('#STOP#', str(stop + 86400)) ofile = outdir + 'get_ftemp_main_script' fo = open(ofile, 'w') fo.write(input) fo.close() cmd = 'chmod 755 ' + outdir + 'get_ftemp_main_script' os.system(cmd) tfile = tdir + 'plot_erad_time.pro' f = open(tfile, 'r') input = f.read() f.close() input = input.replace('#START#', str(start)) input = input.replace('#SDATELIST#', fpdsub) input = input.replace('#LDATELIST#', fplsub) ofile = outdir + 'plot_erad_time.pro' fo = open(ofile, 'w') fo.write(input) fo.close() cmd = 'cp -f ' + outdir + 'plot_erad_time.pro ./Focal/' os.system(cmd) # #--- this_week file # tfile = tdir + 'this_week' f = open(tfile, 'r') input = f.read() f.close() input = input.replace('#DDATE#', file_date) input = input.replace('#IRUSPAN1#', irudate) input = input.replace('#IRUSPAN2#', irudate) input = input.replace('#TITLE#', title) input = input.replace('#TITLEDATE#',titledate) atemp = last_trend_date atemp = atemp.replace('/', '') input = input.replace('#PREVREPORT#', atemp) atemp = re.split('/', last_trend_date) pmon = int(float(atemp[0])) lmon = tcnv.changeMonthFormat(pmon) line = lmon + ' ' + atemp[1] # #--- the previous report could be from the last year # ryear = syear if mon < pmon: ryear = year -1 ryear = str(ryear) input = input.replace('#RYEAR#', ryear) input = input.replace('#PREVDATE#', line) atitle = str(title) atitle = atitle.replace(' ', '_') #file = tdir + 'Headers/' + atitle #fs = open(file, 'r') #trend = fs.read() #fs.close() #input = input.replace("#TREND#", trend) [temp1, temp2, temp3, temp4] = read_cti_values() input = input.replace('#ATEMP#', temp1) input = input.replace('#ATEMP2#', temp2) input = input.replace('#DTEMP#', temp3) input = input.replace('#DTEMP2#', temp4) [val, step] = read_sim() input = input.replace('#WMOVE#', val) input = input.replace('#WSTEP#', step) # #--- make photon and bad pixel output # run_bad_pix_and_photon(outdir) # #--- run to get focal temp fits files # tstop = stop + 86400 [fcnt, fdata] = run_focal_temp_data(outdir, start, stop, fptemp) [fcnt, fdata] = run_focal_temp_data_new() input = input.replace('#TEMPPEAK#', str(fcnt)) input = input.replace('#TEMPLIST#', fdata) # #--- bad pixel # file = outdir + 'bad_pix_list' fs = open(file, 'r') bdata = fs.read() fs.close() input = input.replace('BAD_PIXEL_TABLE', bdata) # #--- photon # file = outdir + 'photons' fs = open(file, 'r') pdata = fs.read() fs.close() input = input.replace('PHOTON_TABLE', pdata) # #--- telem data # update_weekly_telem(year, byear, mon) tdata = run_telem_data(telem_command, daylist, outdir) input = input.replace('TELEM_TABLE', tdata) # #--- trend data # trend = set_trend_data_input(str(date)) input = input.replace('#TREND#', trend) # #--- write out the weekly report # ofile = outdir + file_name fo = open(ofile, 'w') fo.write(input) fo.close() # #--- clean up # cmd = 'rm ./out ./out2' os.system(cmd) # #--- move files # move_files(date, year, outdir, file_name, fptemp, odir) # #--- send out email to admin; notify the job complete # send_email_to_admin(date, year)
def extract_sim_data(): """ extract sim data from PRIMARYCCDM_*.*.tl input: none but read from <dumpdir>/PRIMARYCCDM_*.*.tl output: <outdir>sim_data.out """ # #--- find the time of the last entry from the sim_data.out # sfile = outdir + 'sim_data.out' f = open(sfile, 'r') data = [line.strip() for line in f.readlines()] f.close() # #--- cleaning up the data; drop the data which the date starts from ":" e.g. :2014 # pdata = [] for ent in data: if re.search('^:', ent): continue else: pdata.append(ent) # #--- the last entiry values # if len(pdata) > 0: atemp = re.split('\s+', pdata[len(pdata) - 1]) ltime = tcnv.axTimeMTA( atemp[0]) #--- converting time to sec from 1998.1.1 time_2 = atemp[0] col1_2 = atemp[1] col2_2 = atemp[2] col3_2 = atemp[3] else: ltime = 0 time_2 = 0 col1_2 = '' col2_2 = '' col3_2 = '' # #--- check whether input files exists # cmd = 'ls -rt ' + dumpdir + 'PRIMARYCCDM_*.*.tl >' + zspace os.system(cmd) f = open(zspace, 'r') data = [line.strip() for line in f.readlines()] f.close() cmd = 'rm ' + zspace os.system(cmd) dlen = len(data) if dlen < 1: exit(1) # #--- files exist. read the data from the last 10 files # tlist = data[dlen - 40:] for ent in tlist: cmd = 'cat ' + ent + ' >> ' + zspace os.system(cmd) f = open(zspace, 'r') data = [line.strip() for line in f.readlines()] f.close() cmd = 'rm ' + zspace os.system(cmd) prev = '' fo = open('./temp_save', 'w') # #--- go though each data line # for ent in data: try: # #--- expect the first letter of the data line is numeric (e.g. 2014). # val = float(ent[0]) except: continue # #--- only data with "FMT" format will be used # mc = re.search('FMT', ent) if mc is None: continue atemp = re.split('\t+', ent) # #--- if there are less than 20 entries, something wrong; skip it # if len(atemp) < 20: continue # #--- convert time format # time = atemp[0] time = time.strip() time = time.replace(' ', ':') time = time.replace(':::', ':00') time = time.replace('::', ':0') # #--- if the time is exactly same as one before, skip it # if time == time_2: continue # #--- if the time is already in the database, keip it # stime = tcnv.axTimeMTA(time) if stime <= ltime: continue # #--- use only data which tscpos and fapos have numeric values # tscpos = atemp[4].strip() fapos = atemp[5].strip() if tscpos == "" or fapos == "": continue else: tscpos = int(float(tscpos)) fapos = int(float(fapos)) # aopc = atemp[11].strip() # if aopc == '': # aopc = '0' mpwm = atemp[12].strip() if mcf.chkNumeric(mpwm): mpwm = int(float(mpwm)) mpwm = str(mpwm) else: mpwm = '0' # #--- we want to print only beginning and ending of the same data entries. #--- skip the line if all three entiries are same as one before, except the last one # if col1_2 == tscpos and col2_2 == fapos and col3_2 == mpwm: time_2 = time continue line = time + '\t' + str(tscpos) + '\t' + str( fapos) + '\t' + mpwm + '\n' if line == prev: continue else: pline = time_2 + '\t' + str(col1_2) + '\t' + str( col2_2) + '\t' + str(col3_2) + '\n' fo.write(pline) fo.write(line) prev = line time_2 = time col1_2 = tscpos col2_2 = fapos col3_2 = mpwm fo.close() sfile2 = sfile + '~' cmd = 'cp ' + sfile + ' ' + sfile2 os.system(cmd) cmd = 'cat ./temp_save >> ' + sfile os.system(cmd)
def sci_run_add_to_rad_zone_list(file='NA'): 'adding radiation zone list to rad_zone_list. input: file name containing: e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto' # #--- if file is not given (if it is NA), ask the file input # if file == 'NA': file = raw_input('Please put the intrrupt timing list: ') f = open(file, 'r') data = [line.strip() for line in f.readlines()] data.reverse() f.close() for ent in data: # #--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30) #--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list #--- to each date # etemp = re.split('\s+', ent) atemp = re.split(':', etemp[1]) year = atemp[0] month = atemp[1] date = atemp[2] hour = atemp[3] mins = atemp[4] # #--- convert to dom/sec1998 # ydate = tcnv.findYearDate(int(year), int(month), int(date)) dom = tcnv.findDOM(int(year), int(ydate)) line = year + ':' + str(int(ydate)) + ':' + hour + ':' + mins + ':00' csec = tcnv.axTimeMTA(line) # #--- end date # atemp = re.split(':', etemp[2]) eyear = atemp[0] emonth = atemp[1] edate = atemp[2] ehour = atemp[3] emins = atemp[4] ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate)) dom2 = tcnv.findDOM(int(eyear), int(ydate)) line = eyear + ':' + str(int(ydate)) + ':' + ehour + ':' + emins + ':00' csec2 = tcnv.axTimeMTA(line) # #--- date stamp for the list # list_date = str(year) + str(month) + str(date) # #--- check radiation zones for 3 days before to 5 days after from the interruptiondate # begin = dom - 3 end = dom2 + 5 # #--- read radiation zone infornation # infile = house_keeping + '/rad_zone_info' f = open(infile, 'r') rdata = [line.strip() for line in f.readlines()] f.close() status = [] rdate = [] chk = 0 last_st= '' cnt = 0 for line in rdata: atemp = re.split('\s+', line) dtime = float(atemp[1]) #--- dom of the entry or exit if chk == 0 and atemp[0] == 'ENTRY' and dtime >= begin: status.append(atemp[0]) rdate.append(dtime) chk += 1 last_st = atemp[0] cnt += 1 elif chk > 0 and dtime >= begin and dtime <= end: status.append(atemp[0]) rdate.append(dtime) last_st = atemp[0] cnt += 1 elif atemp[1] > end and last_st == 'EXIT': break elif atemp[1] > end and last_st == 'ENTRY': status.append(atemp[0]) rdate.append(dtime) cnt += 1 break f = open('./temp_zone', 'w') # #--- a format of the output is, e.g.: '20120313 (4614.2141112963,4614.67081268519):...' # line = list_date + '\t' f.write(line) i = 0 while i < cnt: line = '(' + str(rdate[i]) + ',' f.write(line) i += 1 if i < cnt-1: line = str(rdate[i]) + '):' f.write(line) else: line = str(rdate[i]) + ')\n' f.write(line) i += 1 f.close() # #--- append the past rad zone list # oldFile = house_keeping + '/rad_zone_list~' crtFile = house_keeping + '/rad_zone_list' #> cmd = 'cat ' './temp_zone ' + crtFile + ' > /tmp/mta/temp_comb' cmd = 'cat '+ './temp_zone ' + crtFile + ' > ./temp_comb' os.system(cmd) #> os.systm('rm /tmp/mta/temp_zone') os.system('rm ./temp_zone') # #--- save the old file and move the update file to rad_zone_list # cmd = 'chmod 775 ' + crtFile + ' ' + oldFile os.system(cmd) cmd = 'mv ' + crtFile + ' ' + oldFile os.system(cmd) cmd = 'chmod 644 ' + oldFile os.system(cmd) #> cmd = 'mv ' + '/tmp/mta/temp_comb ' + crtFile cmd = 'mv ' + './temp_comb ' + crtFile os.system(cmd)
def report_error(): """ read errors from <cup_usr_list>_error_list, sort it out, clean, and send out email Input: none but read from <cup_usr_list>_error_list Output: email sent out """ # #--- find the current time # [year, mon, day, hours, min, sec, weekday, yday, dst] = tcnv.currentTime("Local") # #--- create surfix for files which will be saved in Past_errors directory # smon = str(mon) if mon < 10: smon = '0' + smon sday = str(day) if day < 10: sday = '0' + sday tail = str(year) + smon + sday for tag in cpu_usr_list: efile = house_keeping + 'Records/' + tag + '_error_list' pfile = house_keeping + 'Records/Past_errors/' + tag + '_error_list_' + tail prev_line = '' chk = mcf.chkFile(efile) if chk > 0: # #--- read error messages from the file # f = open(efile, 'r') data = [line.strip() for line in f.readlines()] f.close() # #--- sort the data so that we can correct messages to each cron job together # data.sort() task_list = [] time_list = [] mssg_list = [] for ent in data: atemp = re.split(' : ' , ent) task_list.append(atemp[0]) stime = int(atemp[1]) dtime = tcnv.axTimeMTA(stime) time_list.append(dtime) mssg_list.append(atemp[2]) # #--- write out cron job name # fo = open(zspace, 'w') cname = task_list[0] line = '\n\n' + cname + '\n____________________\n\n' fo.write(line) for i in range(0, len(mssg_list)): if task_list[i] != cname: cname = task_list[i] line = '\n\n' + cname + '\n____________________\n\n' fo.write(line) # #--- create each line. if it is exactly same as one line before, skip it # line = time_list[i] + ' : ' + mssg_list[i] + '\n' if line != prev_line: fo.write(line) prev_line = line fo.close() # #--- send email out # # cmd = 'cp ' + zspace + ' ' + '/data/mta/Script/Cron_check/Scripts/' + tag # os.system(cmd) send_mail(tag, email_list) # #--- move the error list to Past_errors directory # cmd = 'mv ' + efile + ' ' + pfile os.system(cmd) cmd = 'chmod 755 ' + pfile os.system(cmd)
def sci_run_compute_gap(file = 'NA'): 'for a given file name which contains a list like: "20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto", recompute the lost science time (excluding radiation zone) ' # #--- if file is not given (if it is NA), ask the file input # if file == 'NA': file = raw_input('Please put the intrrupt timing list: ') f = open(file, 'r') data = [line.strip() for line in f.readlines()] f.close() # #--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30) #--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list #--- to each date # update = [] for ent in data: if not ent: #--- if it is a blank line end the operation break etemp = re.split('\s+', ent) atemp = re.split(':', etemp[1]) year = atemp[0] month = atemp[1] date = atemp[2] hour = atemp[3] mins = atemp[4] # #--- convert to dom/sec1998 # ydate = tcnv.findYearDate(int(year), int(month), int(date)) #--- a function from convertTimeFormat dom = tcnv.findDOM(int(year), int(ydate), int(hour), int(mins), 0) #--- a function from convertTimeFormat line = year + ':' + str(ydate) + ':' + hour + ':' + mins + ':00' csec = tcnv.axTimeMTA(line) #--- a function from convertTimeFormat # #--- end date # atemp = re.split(':', etemp[2]) eyear = atemp[0] emonth = atemp[1] edate = atemp[2] ehour = atemp[3] emins = atemp[4] ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate)) dom2 = tcnv.findDOM(int(eyear), int(ydate), int(ehour), int(emins), 0) line = eyear + ':' + str(ydate) + ':' + ehour + ':' + emins + ':00' csec2 = tcnv.axTimeMTA(line) # #--- date stamp for the list # list_date = str(year) + str(month) + str(date) # #--- read radiation zone information from "rad_zone_list" and add up the time overlap with #--- radiatio zones with the interruption time period # line = house_keeping + '/rad_zone_list' f = open(line, 'r') rlist = [line.strip() for line in f.readlines()] f.close() sum = 0 for record in rlist: atemp = re.split('\s+', record) if list_date == atemp[0]: btemp = re.split(':', atemp[1]) for period in btemp: t1 = re.split('\(', period) t2 = re.split('\)', t1[1]) t3 = re.split('\,', t2[0]) pstart = float(t3[0]) pend = float(t3[1]) if pstart >= dom and pstart < dom2: if pend <= dom2: diff = pend - pstart sum += diff else: diff = dom2 - pstart sum += diff elif pstart < dom2 and pend > dom: if pend <= dom2: diff = pend - dom sum += diff else: diff = dom2 - dom sum += diff break sum *= 86400 #--- change unit from day to sec sciLost = (csec2 - csec - sum) / 1000 #--- total science time lost excluding radiation zone passes line = etemp[0] + '\t' + etemp[1] + '\t' + etemp[2] + '\t' + "%.1f" % sciLost + '\t' + etemp[4] update.append(line) return update
def extract_new_data_sub(dir, col_list, start, stop, year, month): # #--- extract data # line = 'operation=retrieve\n' line = line + 'dataset = mta\n' line = line + 'detector = grad\n' line = line + 'level = 0.5\n' line = line + 'filetype =' + dir + '\n' line = line + 'tstart=' + start + '\n' line = line + 'tstop=' + stop + '\n' line = line + 'go\n' fo = open(zspace, 'w') fo.write(line) fo.close() cmd1 = "/usr/bin/env PERL5LIB=" #cmd2 = ' echo ' + hakama + ' |arc4gl -U' + dare + ' -Sarcocc -i' + zspace cmd2 = ' /proj/axaf/simul/bin/arc5gl -user isobe -script ' + zspace cmd = cmd1 + cmd2 bash(cmd, env=ascdsenv) mcf.rm_file(zspace) cmd = 'mv *gz ./Working_dir/.' os.system(cmd) cmd = 'gzip -d ./Working_dir/*.gz' os.system(cmd) cmd = 'ls ./Working_dir/*.fits > ' + zspace os.system(cmd) f = open(zspace, 'r') data = [line.strip() for line in f.readlines()] f.close() mcf.rm_file(zspace) if len(data) == 0: return 'na' # #--- extract each column data and stored in arrays # elen = len(col_list) dat_save = [[] for x in range(0, elen)] time_dat = [] for file in data: dout = fits.getdata(file, 1) time = dout.field('time') for ent in time: time_dat.append(ent) for k in range(0, elen): col = col_list[k] sta = 'ST_' + col val = dout.field(col) chk = dout.field(sta) for j in range(0, len(chk)): if chk[j] >= 0: dat_save[k].append(val[j]) # #--- set the beginning of the and the end of the first day of the data # ydate = tcnv.findYearDate(year, month, 1) tent = str(year) + ':' + str(ydate) + ':00:00:00' begin = tcnv.axTimeMTA(tent) smonth = month + 1 syear = year if smonth > 12: smonth = 1 syear += 1 ydate = tcnv.findYearDate(syear, smonth, 1) tent = str(syear) + ':' + str(ydate) + ':00:00:00' stop = tcnv.axTimeMTA(tent) end = begin + 86400 if end > stop: end = stop dom = int(tcnv.stimeToDom(begin)) sline = str(dom) dsumming = [[] for x in range(0, elen)] # #--- find mean and std of each columns # for k in range(0, len(time_dat)): if time_dat[k] >= begin and time_dat[k] < end: for i in range(0, elen): dsumming[i].append(dat_save[i][k]) elif time_dat[k] < begin: continue elif time_dat[k] >= end: for i in range(0, elen): narray = numpy.array(dsumming[i]) avg = numpy.mean(narray) avg = '%.4f' % round(avg, 4) err = numpy.std(narray) err = '%.4f' % round(err, 4) sline = sline + '\t' + str(avg) + '\t' + str(err) sline = sline + '\n' # #--- reintialize for the next day # begin += 86400 if begin >= stop: break end = begin + 86400 if end > stop: end = stop dom = int(tcnv.stimeToDom(begin)) sline = sline + str(dom) dsumming = [[] for x in range(0, elen)] cmd = 'rm ./Working_dir/*.fits*' os.system(cmd) return sline
def get_new_value(year, month): """ extract aorwspd values from dataseeker input: year/month output: [av1, av2, av3, av4, av5, av6] --- six values of aorwspd(1-6) """ # #--- set month long time interval in sec from 1998.1.1 # year2 = year month2 = month -1 if month2 < 1: mont2 = 12 year2 -= 1 ydate = tcnv.findYearDate(year, month, 15) t_in = str(year) + ':' + str(ydate) + ':00:00:00' time1 = tcnv.axTimeMTA(t_in) ydate = tcnv.findYearDate(year2, month2, 15) t_in = str(year2) + ':' + str(ydate) + ':00:00:00' time2 = tcnv.axTimeMTA(t_in) # #--- set command to call dataseeker # f = open('./test', 'w') #-- we need an empty "test" file to run dataseeker f.close() mcf.mk_empty_dir('param') #-- make empty param directory line = 'columns=_aorwspd1_avg,' line = line + '_aorwspd2_avg,' line = line + '_aorwspd3_avg,' line = line + '_aorwspd4_avg,' line = line + '_aorwspd5_avg,' line = line + '_aorwspd6_avg' line = line + ' timestart=' + str(time2) line = line + ' timestop=' + str(time1) cmd = 'punlearn dataseeker; dataseeker.pl infile=test outfile=ztemp.fits search_crit=\'' cmd = 'dataseeker.pl infile=test outfile=ztemp.fits search_crit="' cmd = cmd + line + '" loginFile="' + loginfile + '"' # #--- run dataseeker # bash("/usr/bin/env PERL5LIB='' " + cmd, env=ascdsenv) mcf.rm_file(zspace) mcf.rm_file('./test') os.system('rm -rf ./param') # #--- read fits file # try: dout = pyfits.getdata('./ztemp.fits') aw1 = dout.field('AORWSPD1_AVG') aw2 = dout.field('AORWSPD2_AVG') aw3 = dout.field('AORWSPD3_AVG') aw4 = dout.field('AORWSPD4_AVG') aw5 = dout.field('AORWSPD5_AVG') aw6 = dout.field('AORWSPD6_AVG') except: aw1 = aw2 = aw3 = aw4 = aw5 = aw6 = [] mcf.rm_file("./ztemp.fits") # #--- create monthly "sum" of the reaction wheel rotations #--- dataseeker gives 5 min avg of the value; one day is 24 hr x 60 min / 5min = 288. # sum1 = sum2 = sum3 = sum4 = sum5 = sum6 = 0.0 for i in range(0, len(aw1)): sum1 += abs(aw1[i]) sum2 += abs(aw2[i]) sum3 += abs(aw3[i]) sum4 += abs(aw4[i]) sum5 += abs(aw5[i]) sum6 += abs(aw6[i]) av1 = sum1 / 288 av2 = sum2 / 288 av3 = sum3 / 288 av4 = sum4 / 288 av5 = sum5 / 288 av6 = sum6 / 288 return [av1, av2, av3, av4, av5, av6]
def find_sim_position(start, stop): """ extract sim position realated data for a given time span input start --- starting time stop --- stopping time data are in: /data/mta_www/mta_temp/mta_states/MJ/<yyyy>//comprehensive_data_summary<yyyy> output: acis_i_start/ acis_i_stop --- acis i starting and stopping time acis_s_start/ acis_s_stop --- acis s starting and stopping time hrc_i_start/ hrc_i_stop --- hrc i starting and stopping time hrc_s_start/ hrc_s_stop --- hrc s starting and stopping time hetg_start/ hetg_stop --- hetg starting and stopping time letg_start/ letg_stop --- letg starting and stopping time radmon_start/ radmon_stop --- radmon starting and stopping time fmt --- fmt format list time --- time list """ # #--- find year of starting and stopping time. this will be used to fine which data set we need to use # ntime = tcnv.axTimeMTA(start) atemp = re.split(':', ntime) syear = int(atemp[0]) ntime = tcnv.axTimeMTA(stop) atemp = re.split(':', ntime) lyear = int(atemp[0]) # #-- read all data sets contain the appropriate data # data = [] for year in range(syear, lyear+1): file = '/data/mta_www/mta_temp/mta_states/MJ/' + str(year) + '/comprehensive_data_summary' + str(year) f = open(file, 'r') dtmp = [line.strip() for line in f.readlines()] f.close() data = data + dtmp acis_i_start = [] acis_i_stop = [] acis_s_start = [] acis_s_stop = [] hrc_i_start = [] hrc_i_stop = [] hrc_s_start = [] hrc_s_stop = [] hetg_start = [] hetg_stop = [] letg_start = [] letg_stop = [] radmon_start = [] radmon_stop = [] fmt = [] time = [] acis_i_in = 0 acis_s_in = 0 hrc_i_in = 0 hrc_s_in = 0 hetg_in = 0 letg_in = 0 radmon_on = 0 for ent in data: # #--- use only data line starting with year (which should be a digits) # try: val = float(ent[0]) except: continue atemp = re.split('\s+', ent) stime = tcnv.axTimeMTA(atemp[0]) #--- converting time to sec from 1998.1.1 if stime < start: continue if stime > stop: break scpos = float(atemp[1]) #--- 3TSCPOS hpos = float(atemp[9]) #--- 4HPOSARO lpos = float(atemp[11]) #--- 4LPOSARO time.append(stime) # #--- ACIS I # if scpos > 89000 and acis_i_in == 0: acis_i_start.append(stime) acis_i_in = 1 elif scpos < 89000 and acis_i_in == 1: acis_i_stop.append(stime) acis_i_in = 0 # #--- ACIS S # elif scpos < 80000 and scpos > 71000 and acis_s_in == 0: acis_s_start.append(stime) acis_s_in = 1 elif (scpos > 80000 or scpos < 71000) and acis_s_in == 1: acis_s_stop.append(stime) acis_s_in = 0 # #--- HRC I # elif scpos < -45000 and scpos > -55000 and hrc_i_in == 0: hrc_i_start.append(stime) hrc_i_in = 1 elif (scpos > -45000 or scpos < -55000) and hrc_i_in == 1: hrc_i_stop.append(stime) hrc_i_in = 0 # #--- HRC S # elif scpos < -90000 and hrc_s_in == 0: hrc_s_start.append(stime) hrc_s_in = 1 elif scpos > -90000 and hrc_s_in == 1: hrc_s_stop.append(stime) hrc_s_in = 0 # #--- HETIG # if hpos < 20 and hetg_in == 0: hetg_start.append(stime) hetg_in = 1 elif hpos > 60 and hetg_in == 1: hetg_stop.append(stime) hetg_in = 0 # #--- LETG # if lpos < 20 and letg_in == 0: letg_start.append(stime) letg_in = 1 elif lpos > 60 and letg_in == 1: letg_stop.append(stime) letg_in = 0 # #--- RADMON # if atemp[5] == 'DISA' and radmon_on == 0: radmon_start.append(stime) radmon_on = 1 elif atemp[5] == 'ENAB' and radmon_on == 1: radmon_stop.append(stime) radmon_on = 0 # #--- FMT Format # fmt.append(atemp[7]) # #--- for the case the period is not closed during the time interval given #--- add "stop" time to close the period # if len(acis_i_stop) < len(acis_i_start): acis_i_stop.append(stop) if len(acis_s_stop) < len(acis_s_start): acis_s_stop.append(stop) if len(hrc_i_stop) < len(hrc_i_start): hrc_i_stop.append(stop) if len(hrc_s_stop) < len(hrc_s_start): hrc_s_stop.append(stop) if len(hetg_stop) < len(hetg_start): hetg_stop.append(stop) if len(letg_stop) < len(letg_start): letg_stop.append(stop) if radmon_on == 1: radmon_stop.append(stop) return[acis_i_start, acis_i_stop, acis_s_start, acis_s_stop, hrc_i_start, hrc_i_stop, \ hrc_s_start, hrc_s_stop, hetg_start, hetg_stop, letg_start, letg_stop, \ radmon_start, radmon_stop, fmt, time]
def create_weekly_report(date, year, debug=0): """ main script to set up the weekly report template for the week input: date --- date in the format of mmdd (e.g. 0910) year --- year in the format of yyyy (e.g. 2015) debug --- if it is other than 0, print out some output output: a direcotry containing templete (e.g. Sep10) """ # #--- if the test is requested, create Test directory # if debug != 0: os.system('mkdir /data/mta/Script/Weekly/Test/') odir = '/data/mta/Script/Weekly/Test/' else: odir = '/data/mta4/www/REPORTS/' oned = 86400 syear = str(year) #--- 4 digit year yrd2 = year[2] + year[3] #--- 2 digit year year = int(float(year)) #--- integer year date = str(date) smon = date[0] + date[1] #--- two digit month mon = int(float(smon)) #--- integer month lmon = tcnv.changeMonthFormat(mon) #--- month in letter (e.g.Mar) sday = date[2] + date[3] #--- two digit mday day = int(float(sday)) #--- integer mday stop = tcnv.convertDateToCTime(year, mon, day, 0, 0, 0) day_n = stop - 7 * oned # tout = tcnv.axTimeMTA(day_n) # ttemp = re.split(':', tout) # iru_start = str(ttemp[0]) + '_' + str(ttemp[1]) #day01 = stop - 6 * oned #day0 = stop - 7 * oned day01 = stop - 5 * oned day0 = stop - 6 * oned lday0 = stime_to_ddate(day0) sday0 = sdate_to_ldate(lday0) start = day0 lday1 = stime_to_ddate(day01) tout = tcnv.axTimeMTA(day0) ttemp = re.split(':', tout) iru_start = str(ttemp[0]) + '_' + str(ttemp[1]) # #--- year of the beginning of the period; could be different from that of the end # byear = ttemp[0] lday0 = stime_to_ddate(day0) day1 = stop - 5 * oned lday1 = stime_to_ddate(day1) day2 = stop - 4 * oned lday2 = stime_to_ddate(day2) sday2 = sdate_to_ldate(lday2) day3 = stop - 3 * oned lday3 = stime_to_ddate(day3) day4 = stop - 2 * oned lday4 = stime_to_ddate(day4) sday4 = sdate_to_ldate(lday4) day5 = stop - 1 * oned lday5 = stime_to_ddate(day5) day6 = stop lday6 = stime_to_ddate(day6) sday6 = sdate_to_ldate(lday6) #tout = tcnv.axTimeMTA(day5) tout = tcnv.axTimeMTA(day6) ttemp = re.split(':', tout) iru_stop = '_' + str(ttemp[1]) day7 = stop + 1 * oned lday7 = stime_to_ddate(day7) # #---- setting file name # atemp = re.split('\/', lday6) file_date = atemp[0] + atemp[1] file_date2 = atemp[0] + '/' + atemp[1] file_name = file_date + '.html' # #--- title # titledate = lday0 + ' - ' + lday6 ldate = sdate_to_ldate(lday6) ldate_sp = sdate_to_ldate_with_space(lday6) # #--- focal temp file name # fptemp = file_date + '_fptemp.gif' fpext_range = str(start) + ' ' + str(stop) fpstart = str(start) fplsub = '"' + sday0 + '", "' + sday2 + '", "' + sday4 + '", "' + sday6 + '"' fpdsub = str(day0) + ', ' + str(day2) + ', ' + str(day4) + ', ' + str(day6) # #--- IRU span # irudate = iru_start + iru_stop # #--- telemetry idl command # telmstart = stime_to_ddate2(start) telmstop = stime_to_ddate2(stop) telem_command = 'weekly_telem,' + telmstart + ',' + telmstop # #--- telemetry header line # daylist = '|' + lday0 + '|' + lday1 + '|' + lday2 + '|' + lday3 + '|' + lday4 + '|' + lday5 + '|' + lday6 # #--- find trending dates/title # tfile = tdir + 'trending_order' f = open(tfile, 'r') data = [line.strip() for line in f.readlines()] f.close() data.reverse() chk = 0 for ent in data: if ent == '\s+' or ent == '': continue atemp = re.split(' : ', ent) adate = atemp[0].strip() aname = atemp[1].strip() file_date2.strip() if chk == 0: if str(adate) == str(file_date2): title = aname chk = 1 else: if aname == title: last_trend_date = adate break # #--- index.html input # s1 = sday0[0:3] + ' ' + sday0[3:5] s2 = sday6[0:3] + ' ' + sday6[3:5] index = '<td> <a href="./' + str( year) + '/' + file_date + '.html">' + s1 + ' - ' + s2 + '</a>' # #--- debugging output # if debug != 0: print "file_name; " + file_name print "title date: " + titledate print "ldate: " + ldate print "fptemp: " + fptemp print "fpext_range: " + fpext_range print "fpstart: " + fpstart print " fplsub: " + fplsub print " fpdsub: " + fpdsub print "irudate: " + irudate print "telmstart: " + telmstart print "telmstop: " + telmstop print "telmcommand: " + telem_command print "daylist: " + daylist print "title: " + title print "last_trend_date: " + last_trend_date print "index: " + index # #--- create a work directory # cmd = 'mkdir ' + wdir + ldate os.system(cmd) outdir = wdir + ldate + '/' cmd = 'cp ' + tdir + 'get_ftemp_data.perl' + ' ' + outdir os.system(cmd) cmd = 'cp ' + tdir + 'subst3.perl' + ' ' + outdir os.system(cmd) cmd = 'cp ' + tdir + 'test' + ' ' + outdir os.system(cmd) cmd = 'mkdir ' + outdir + '/param' os.system(cmd) # #--- create instruction page # tfile = tdir + 'instruction' f = open(tfile, 'r') input = f.read() f.close() input = input.replace('#LDATE_S#', ldate_sp) input = input.replace('#LDATE#', ldate) input = input.replace('#DDATE#', file_date) input = input.replace('#YEAR#', syear) input = input.replace('#DAYLIST#', daylist) input = input.replace('#TELM_CMD#', telem_command) input = input.replace('#INDEX#', index) ofile = outdir + ldate.lower() fo = open(ofile, 'w') fo.write(input) fo.close() # #--- focal temp related files # tfile = tdir + 'get_ftemp_wrap_script' f = open(tfile, 'r') input = f.read() f.close() input = input.replace('#MANDD#', ldate) ofile = outdir + 'get_ftemp_wrap_script' fo = open(ofile, 'w') fo.write(input) fo.close() cmd = 'chmod 755 ' + outdir + 'get_ftemp_wrap_script' os.system(cmd) tfile = tdir + 'get_ftemp_main_script' f = open(tfile, 'r') input = f.read() f.close() #input = input.replace('#START#', str(start)) #input = input.replace('#STOP#', str(stop)) input = input.replace('#START#', str(start + 86400)) input = input.replace('#STOP#', str(stop + 86400)) ofile = outdir + 'get_ftemp_main_script' fo = open(ofile, 'w') fo.write(input) fo.close() cmd = 'chmod 755 ' + outdir + 'get_ftemp_main_script' os.system(cmd) tfile = tdir + 'plot_erad_time.pro' f = open(tfile, 'r') input = f.read() f.close() input = input.replace('#START#', str(start)) input = input.replace('#SDATELIST#', fpdsub) input = input.replace('#LDATELIST#', fplsub) ofile = outdir + 'plot_erad_time.pro' fo = open(ofile, 'w') fo.write(input) fo.close() cmd = 'cp -f ' + outdir + 'plot_erad_time.pro ./Focal/' os.system(cmd) # #--- this_week file # tfile = tdir + 'this_week' f = open(tfile, 'r') input = f.read() f.close() input = input.replace('#DDATE#', file_date) input = input.replace('#IRUSPAN1#', irudate) input = input.replace('#IRUSPAN2#', irudate) input = input.replace('#TITLE#', title) input = input.replace('#TITLEDATE#', titledate) atemp = last_trend_date atemp = atemp.replace('/', '') input = input.replace('#PREVREPORT#', atemp) atemp = re.split('/', last_trend_date) pmon = int(float(atemp[0])) lmon = tcnv.changeMonthFormat(pmon) line = lmon + ' ' + atemp[1] # #--- the previous report could be from the last year # ryear = syear if mon < pmon: ryear = year - 1 ryear = str(ryear) input = input.replace('#RYEAR#', ryear) input = input.replace('#PREVDATE#', line) atitle = str(title) atitle = atitle.replace(' ', '_') #file = tdir + 'Headers/' + atitle #fs = open(file, 'r') #trend = fs.read() #fs.close() #input = input.replace("#TREND#", trend) [temp1, temp2, temp3, temp4] = read_cti_values() input = input.replace('#ATEMP#', temp1) input = input.replace('#ATEMP2#', temp2) input = input.replace('#DTEMP#', temp3) input = input.replace('#DTEMP2#', temp4) [val, step] = read_sim() input = input.replace('#WMOVE#', val) input = input.replace('#WSTEP#', step) # #--- make photon and bad pixel output # run_bad_pix_and_photon(outdir) # #--- run to get focal temp fits files # tstop = stop + 86400 [fcnt, fdata] = run_focal_temp_data(outdir, start, stop, fptemp) [fcnt, fdata] = run_focal_temp_data_new() input = input.replace('#TEMPPEAK#', str(fcnt)) input = input.replace('#TEMPLIST#', fdata) # #--- bad pixel # file = outdir + 'bad_pix_list' fs = open(file, 'r') bdata = fs.read() fs.close() input = input.replace('BAD_PIXEL_TABLE', bdata) # #--- photon # file = outdir + 'photons' fs = open(file, 'r') pdata = fs.read() fs.close() input = input.replace('PHOTON_TABLE', pdata) # #--- telem data # update_weekly_telem(year, byear, mon) tdata = run_telem_data(telem_command, daylist, outdir) input = input.replace('TELEM_TABLE', tdata) # #--- trend data # trend = set_trend_data_input(str(date)) input = input.replace('#TREND#', trend) # #--- write out the weekly report # ofile = outdir + file_name fo = open(ofile, 'w') fo.write(input) fo.close() # #--- clean up # cmd = 'rm ./out ./out2' os.system(cmd) # #--- move files # move_files(date, year, outdir, file_name, fptemp, odir) # #--- send out email to admin; notify the job complete # send_email_to_admin(date, year)
def useArc5gl(operation, dataset, detector, level, filetype, startYear = 0, startYdate = 0, stopYear = 0 , stopYdate = 0, deposit = './Working_dir/'): """ extract data using arc5gl. input: start --- stop (year and ydate) operation --- (e.g., retrive) dataset ---(e.g. flight) detector --- (e.g. hrc) level --- (eg 0, 1, 2) filetype ---(e.g, evt1) startYear startYdate stopYear stopYdate output: data --- a list of fits file extracted """ # #--- use arc5gl to extract ephin data # (year1, month1, day1, hours1, minute1, second1, ydate1) = tcnv.dateFormatCon(startYear, startYdate) (year2, month2, day2, hours2, minute2, second2, ydate2) = tcnv.dateFormatCon(stopYear, stopYdate) stringYear1 = str(year1) stringYear2 = str(year2) arc_start = str(month1) + '/' + str(day1) + '/' + stringYear1[2] + stringYear1[3] arc_start = arc_start + ',' + str(hours1) + ':'+ str(minute1) + ':00' arc_stop = str(month2) + '/' + str(day2) + '/' + stringYear2[2] + stringYear2[3] arc_stop = arc_stop + ',' + str(hours2) + ':'+ str(minute2) + ':00' intime = str(startYear) + ':' + adjust_ydate_format(startYdate) + ':00:00:00' arc_start = tcnv.axTimeMTA(intime) intime = str(stopYear) + ':' + adjust_ydate_format(stopYdate) + ':00:00:00' arc_stop = tcnv.axTimeMTA(intime) #print "I AM HERE: " + str(arc_start) + "<--->" + str(arc_stop) line = 'operation=' + operation + '\n' line = line + 'dataset=' + dataset + '\n' line = line + 'detector=' + detector + '\n' line = line + 'level=' + str(level) + '\n' line = line + 'filetype=' + filetype + '\n' line = line + 'tstart=' + str(arc_start) + '\n' line = line + 'tstop=' + str(arc_stop) + '\n' line = line + 'go\n' f = open(zspace, 'w') f.write(line) f.close() cmd = 'cd ' + deposit + '; arc5gl -user isobe -script ' + zspace + ' >./zlist' os.system(cmd) cmd = 'rm ' + zspace os.system(cmd) infile = deposit + '/zlist' f = open(infile, 'r') data = [line.strip() for line in f.readlines()] f.close() cmd = 'rm ' + infile os.system(cmd) return data
def sib_corr_comb(start, stop, lev): """ combined fits files into one per ccd input: start --- start time of the interval <yyyy>:<ddd>:<hh>:<mm>:<ss> stop --- stop time of the interval <yyyy>:<ddd>:<hh>:<mm>:<ss> lev --- data level "Lev1" or "Lve2" output: combined data: lres_ccd<ccd>_merged.fits in Data directory """ # #--- convert the time to seconds from 1998.1.1 # tstart = tcnv.axTimeMTA(start) tstop = tcnv.axTimeMTA(stop) # #--- make a list of data fits files # lres = s_dir + lev + '/Outdir/lres/' cmd = 'ls ' + lres + '*fits > ' + zspace os.system(cmd) data = scf.read_file(zspace, remove=1) # #--- initialize ccd_list<ccd> # for ccd in range(0, 10): exec 'ccd_list%s = []' % (str(ccd)) for ent in data: # #--- check whether the data are inside of the specified time period # [tmin, tmax] = find_time_interval(ent) if tmin >= tstart and tmax <= tstop: btemp = re.split('_acis', ent) head = btemp[0] # #--- add the fits file to ccd_list # for ccd in range(0, 10): chk = 'acis' + str(ccd) mc = re.search(chk, ent) if mc is not None: line = str(ent) exec "ccd_list%s.append('%s')" % (str(ccd), line) break # #--- combined all fits files of a specific ccd into one fits file # for ccd in range(0, 10): exec "alist = ccd_list%s" % (str(ccd)) if len(alist) > 0: # #--- the first of the list is simply copied to temp.fits # cmd = 'cp ' + alist[0] + ' temp.fits' os.system(cmd) for k in range(1, len(alist)): cmd = 'dmmerge "' + alist[ k] + ',temp.fits" outfile=zmerged.fits outBlock=""' cmd = cmd + 'columnList="" clobber="yes"' try: scf.run_ascds(cmd) except: continue cmd = 'mv ./zmerged.fits ./temp.fits' os.system(cmd) cmd = 'mv ./temp.fits ' + s_dir + lev + '/Data/lres_ccd' + str( ccd) + '_merged.fits' os.system(cmd)
# #--- pylab plotting routine related modules # from pylab import * import matplotlib.pyplot as plt import matplotlib.font_manager as font_manager import matplotlib.lines as lines #from matplotlib.transforms import Bbox mag_plot = 1 if len(sys.argv) == 3: start = sys.argv[1] stop = sys.argv[2] else: print "Input: start stop in the format of 2014:204:00:00:00 or in seconds from 1998.1.1" exit(1) if __name__ == "__main__": try: start = int(float(start)) stop = int(float(stop)) except: start = tcnv.axTimeMTA(start) stop = tcnv.axTimeMTA(stop) plot_data(start, stop, mag_plot)
def sib_corr_comb(start, stop, lev): """ combined fits files into one per ccd input: start --- start time of the interval <yyyy>:<ddd>:<hh>:<mm>:<ss> stop --- stop time of the interval <yyyy>:<ddd>:<hh>:<mm>:<ss> lev --- data level "Lev1" or "Lve2" output: combined data: lres_ccd<ccd>_merged.fits in Data directory """ # #--- convert the time to seconds from 1998.1.1 # tstart = tcnv.axTimeMTA(start) tstop = tcnv.axTimeMTA(stop) # #--- make a list of data fits files # lres = s_dir + lev + '/Outdir/lres/' cmd = 'ls ' + lres + '*fits > ' + zspace os.system(cmd) data = scf.read_file(zspace, remove=1) # #--- initialize ccd_list<ccd> # for ccd in range(0, 10): exec 'ccd_list%s = []' % (str(ccd)) for ent in data: # #--- check whether the data are inside of the specified time period # [tmin, tmax] = find_time_interval(ent) if tmin >= tstart and tmax <= tstop: btemp = re.split('_acis', ent) head = btemp[0] # #--- add the fits file to ccd_list # for ccd in range(0, 10): chk = 'acis' + str(ccd) mc = re.search(chk, ent) if mc is not None: line = str(ent) exec "ccd_list%s.append('%s')" % (str(ccd), line) break # #--- combined all fits files of a specific ccd into one fits file # for ccd in range(0, 10): exec "alist = ccd_list%s" % (str(ccd)) if len(alist) > 0: # #--- the first of the list is simply copied to temp.fits # cmd = 'cp ' + alist[0] + ' temp.fits' os.system(cmd) for k in range(1, len(alist)): cmd = 'dmmerge "' + alist[k] + ',temp.fits" outfile=zmerged.fits outBlock=""' cmd = cmd + 'columnList="" clobber="yes"' scf.run_ascds(cmd) cmd = 'mv ./zmerged.fits ./temp.fits' os.system(cmd) cmd = 'mv ./temp.fits ' + s_dir + lev + '/Data/lres_ccd' + str(ccd) + '_merged.fits' os.system(cmd)
def sci_run_add_to_rad_zone_list(file='NA'): 'adding radiation zone list to rad_zone_list. input: file name containing: e.g. 20120313 2012:03:13:22:41 2012:03:14:13:57 53.3 auto' # #--- check whether the list alread exists; if it does, read which ones are already in the list # cmd = 'ls ' + house_keeping + '* > ./ztemp' os.system(cmd) # f = open('./ztemp', 'r') # test = f.readlines() # f.close() test = open('./ztemp').read() m1 = re.search('rad_zone_list', test) m2 = re.search('rad_zone_list~', test) eventList = [] echk = 0 if m1 is not None: line = house_keeping + 'rad_zone_list' f = open(line, 'r') data = [line.strip() for line in f.readlines()] f.close() for ent in data: atemp = re.split('\s+|\t+', ent) eventList.append(atemp[0]) echk = 1 # #--- if file is not given (if it is NA), ask the file input # if file == 'NA': file = raw_input('Please put the intrrupt timing list: ') f = open(file, 'r') data = [line.strip() for line in f.readlines()] f.close() # #--- put the list in the reverse order # data.reverse() for ent in data: if not ent: break # #--- a starting date of the interruption in yyyy:mm:dd:hh:mm (e.g., 2006:03:20:10:30) #--- there could be multiple lines of date; in that is the case, the scripts add the rad zone list #--- to each date # etemp = re.split('\s+', ent) echk = 0 for comp in eventList: if comp == etemp[0]: echk = 1 break if echk == 0: atemp = re.split(':', etemp[1]) year = atemp[0] month = atemp[1] date = atemp[2] hour = atemp[3] mins = atemp[4] # #--- convert to dom/sec1998 # ydate = tcnv.findYearDate(int(year), int(month), int(date)) dom = tcnv.findDOM(int(year), int(ydate)) line = year + ':' + str(int(ydate)) + ':' + hour + ':' + mins + ':00' csec = tcnv.axTimeMTA(line) # #--- end date # atemp = re.split(':', etemp[2]) eyear = atemp[0] emonth = atemp[1] edate = atemp[2] ehour = atemp[3] emins = atemp[4] ydate = tcnv.findYearDate(int(eyear), int(emonth), int(edate)) line = eyear + ':' + str(int(ydate)) + ':' + ehour + ':' + emins + ':00' csec2 = tcnv.axTimeMTA(line) # #--- date stamp for the list # list_date = str(year) + str(month) + str(date) # #--- check radiation zones for 3 days before to 5 days after from the interruptiondate #--- if the interruption lasted longer than 5 days, extend the range 7 more days # begin = dom - 3 end = dom + 5 diff = csec2 - csec if diff > 432000: end += 7 # #--- read radiation zone infornation # infile = house_keeping + '/rad_zone_info' f = open(infile, 'r') rdata = [line.strip() for line in f.readlines()] f.close() status = [] rdate = [] chk = 0 last_st= '' cnt = 0 for line in rdata: atemp = re.split('\s+', line) dtime = float(atemp[1]) #--- dom of the entry or exit if chk == 0 and atemp[0] == 'ENTRY' and dtime >= begin: status.append(atemp[0]) rdate.append(dtime) chk += 1 last_st = atemp[0] cnt += 1 elif chk > 0 and dtime >= begin and dtime <= end: status.append(atemp[0]) rdate.append(dtime) last_st = atemp[0] cnt += 1 elif atemp[1] > end and last_st == 'EXIT': break elif atemp[1] > end and last_st == 'ENTRY': status.append(atemp[0]) rdate.append(dtime) cnt += 1 break f = open('./temp_zone', 'w') # #--- a format of the output is, e.g.: '20120313 (4614.2141112963,4614.67081268519):...' # line = list_date + '\t' f.write(line) upper = cnt -1 i = 0; while i < cnt: line = '(' + str(rdate[i]) + ',' f.write(line) i += 1 if i < upper: line = str(rdate[i]) + '):' f.write(line) else: line = str(rdate[i]) + ')\n' f.write(line) i += 1 f.close() # #--- append the past rad zone list # oldFile = house_keeping + '/rad_zone_list~' crtFile = house_keeping + '/rad_zone_list' if m1 is not None: cmd = 'cat '+ './temp_zone ' + crtFile + ' > ./temp_comb' os.system(cmd) else: os.system('mv .temp_zone ./temp_comb') os.system('rm ./temp_zone') # #--- save the old file and move the update file to rad_zone_list # if m2 is not None: cmd = 'chmod 775 ' + crtFile + ' ' + oldFile os.system(cmd) if m1 is not None: cmd = 'mv ' + crtFile + ' ' + oldFile os.system(cmd) cmd = 'chmod 644 ' + oldFile os.system(cmd) cmd = 'mv ' + './temp_comb ' + crtFile os.system(cmd)