def test_compute_stats(self): msid = '1crbt' start = 155088000.0 stop = 155865600.0 comp = [2592, 145.86, 145.22, 1.04, 145.22, 147.61, 0, 0, 0, 0] l_list = ecf.set_limit_list(msid) out = set_limits_for_period(l_list, start, stop) [start, stop, y_min, y_max, r_min, r_max] = out[0] data = get_data(msid, start, stop) out = compute_stats(data, y_min, y_max, r_min, r_max) self.assertEquals(out, comp)
def update_glimmon_database(): """ updata database related glimmon msids input: none but read from several static list and get data with dataseeker outpu: updated data file: <data_dir>/<msid>_data """ # #--- read all msids and go through one by one # msid_list = read_msids() for msid in msid_list: # #--- read limit table for the msid # try: l_list = ecf.set_limit_list(msid) except: l_list = [] print "MSID: " + str(msid) update_data(msid, l_list)
def create_limit_table(msid, group, unit, xmin, xmax): """ create a limit table for msid input: msid --- msid unit --- unit xmin --- xmin xmax --- xmax output: <web_dir>/Limit_table/<msid>_limit_table.html """ # #--- read limit data # pmsid = drop_suffix(msid) l_list = ecf.set_limit_list(pmsid) # #--- read header part # title = msid + ' limit table' repl = [["#MSID#", title], ["#JAVASCRIPT#", ''], ["#STYLE#", ""]] line = read_template('html_head', repl) # #--- except a few, all temperatures are in K # if unit == 'DEGF': tline = msid.upper() + ' (K)' elif unit == 'DEGC': tline = msid.upper() + ' (K)' elif unit == '': tline = msid.upper() else: tline = msid.upper() + ' (' + unit + ')' bgline = '<th style="background-color:' line = line + '<h2>' + tline + '</h2>\n' line = line + '<table border=1 cellpadding=2>\n' line = line + '<tr><th>Start Time</th>\n' line = line + '<th>Stop Time</th>\n' line = line + bgline + 'yellow">Yellow Lower</th>\n' line = line + bgline + 'yellow">Yellow Upper</th>\n' line = line + bgline + 'red">Red Lower</th>\n' line = line + bgline + 'red">Red Upper</th>\n' line = line + '</tr>\n' for k in range(0, len(l_list)): alist = l_list[k] [astart, byear, base] = convert_stime_into_year(float(alist[0])) [astop, byear, base] = convert_stime_into_year(float(alist[1])) # #--- there are often the data with <start>=<stop>, drop them # if astart == astop: continue astart = float('%4.2f' % (round(astart, 2))) astop = float('%4.2f' % (round(astop, 2))) if k == 0: if astart > xmin: astart = '---' if k == (len(l_list) - 1): astop = "---" # #--- alist: ymin, ymax, rmin, rmax in position of 2 to 5 # tlist = [astart, astop] + alist[2:6] # #--- create each row # line = line + '<tr>\n' for tval in tlist: line = line + '<td style="text-align:center;">' + str( tval) + '</td>\n' line = line + '</tr>\n' line = line + '</table>\n' line = line + '</body>\n</html>\n' o_dir = web_dir + group + '/' check_dir_exist(o_dir) o_dir = o_dir + 'Limit_table/' check_dir_exist(o_dir) file_name = o_dir + msid + '_limit_table.html' with open(file_name, 'w') as fo: fo.write(line)
def create_limit_table(msid, unit, xmin, xmax): """ create a limit table for msid input: msid --- msid unit --- unit xmin --- xmin xmax --- xmax output: <web_dir>/Limit_table/<msid>_limit_table.html """ # #--- read limit data # pmsid = msid.replace('_i^', '') pmsid = pmsid.replace('_s^', '') pmsid = pmsid.replace('_off^', '') l_list = ecf.set_limit_list(pmsid) line = '<!DOCTYPE html>\n' line = line + '<html>\n' line = line + '<head>\n' line = line + '\t<title>' + msid + ' limit table</title>\n' line = line + '</head>\n' line = line + '<body style="background-color:#FFE4B5; padding-top:20px;background-color:#FAEBD7;' line = line + 'font-family:Georgia, "Times New Roman", Times, serif;">\n' # #--- except a few, all temperatures are in K # if unit == 'DEGF': tline = msid.upper() + ' (K)' elif unit == 'DEGC': tline = msid.upper() + ' (K)' elif unit == '': tline = msid.upper() else: tline = msid.upper() + ' (' + unit + ')' line = line + '<h2>' + tline + '</h2>\n' line = line + '<table border=1 cellpadding=2>\n' line = line + '<tr><th>Start Time</th>\n' line = line + '<th>Stop Time</th>\n' line = line + '<th style="background-color:yellow">Yellow Lower</th>\n' line = line + '<th style="background-color:yellow">Yellow Upper</th>\n' line = line + '<th style="background-color:red">Red Lower</th>\n' line = line + '<th style="background-color:red">Red Upper</th>\n' line = line + '</tr>\n' for k in range(0, len(l_list)): alist = l_list[k] astart = ecf.stime_to_frac_year(float(alist[0])) astop = ecf.stime_to_frac_year(float(alist[1])) # #--- there are often the data with <start>=<stop>, drop them # if astart == astop: continue astart = '%4.2f' % (round(astart,2)) astop = '%4.2f' % (round(astop, 2)) if k == 0: if astart > xmin: astart = '---' if k == (len(l_list) -1): astop = "---" yminl = float(alist[2]) ymaxl = float(alist[3]) rminl = float(alist[4]) rmaxl = float(alist[5]) line = line + '<tr>\n' line = line + '<td style="text-align:center;">' + str(astart) + '</td>\n' line = line + '<td style="text-align:center;">' + str(astop) + '</td>\n' line = line + '<td style="text-align:center;">' + str(yminl) + '</td>\n' line = line + '<td style="text-align:center;">' + str(ymaxl) + '</td>\n' line = line + '<td style="text-align:center;">' + str(rminl) + '</td>\n' line = line + '<td style="text-align:center;">' + str(rmaxl) + '</td>\n' line = line + '</tr>\n' line = line + '</table>\n' line = line + '</body>\n</html>\n' file_name = web_dir + 'Limit_table/' + msid + '_limit_table.html' fo = open(file_name, 'w') fo.write(line) fo.close()
def plot_interactive_trend(msid, unit): """ create interactive trend plot input: msid --- msid unit --- unit of msid output: pout --- plot in html format <web_dir>/Future/<msid>_<loc> --- plots in html format saved in the directory if it may violate the limit in future loc: low or top """ # #--- read data # pdata = read_data(msid) if pdata == na: print msid + ': empty data file' return na if len(pdata[0]) < 10: return False # #--- set the x plotting range # xmin = 1999 xmax = ecf.current_time() + 1 xmax = "%4.1f" % round(xmax, 1) xmax = float(xmax) xdiff = xmax - xmin xpos = xmin + 0.05 * xdiff # #--- compute predict trends # [tlim, tmax, min_a, min_b, max_a, max_b, xcent, y_avg, y_min, y_max] = predict_trend(pdata, xmin, xmax) # #--- set y plotting range # [ymin, ymax] = set_y_range(pdata, y_min, y_max, msid) ydiff = ymax - ymin ypos = ymax - 0.1 * ydiff # #--- create a limit table for this msid # create_limit_table(msid, unit, xmin, xmax) # #--- hrc has 4 different categories (all, hrc i, hrc s, off); use the same limit range # pmsid = msid.replace('_i^', '') pmsid = pmsid.replace('_s^', '') pmsid = pmsid.replace('_off^', '') l_list = ecf.set_limit_list(pmsid) l_len = len(l_list) if l_len > 0: [trb_save, rb1_save, rb2_save, tyb_save, yb1_save, yb2_save, tyt_save, \ yt1_save, yt2_save, trt_save, rt1_save, rt2_save] = set_warning_area(l_list, xmin, xmax, ymin, ymax) # #--- create violation notification # if l_len == 0: vtdata = [0, 0, 0, 0] ved.incert_data(msid, vtdata) pchk_l = 0 pchk_u = 0 gchk = 0 elif xmax - tmax > 3.0: vtdata = [0, 0, 0, 0] ved.incert_data(msid, vtdata) pchk_l = 0 pchk_u = 0 gchk = -1 elif (abs(min_b) == 999) or (abs(max_b) == 999): vtdata = [0, 0, 0, 0] ved.incert_data(msid, vtdata) pchk_l = 0 pchk_u = 0 gchk = 0 else: [wline, wline2, pchk_l, pchk_u] = create_violation_notification(msid, pdata, min_a, min_b, max_a, max_b, tmax) gchk = 1 # #--- readjust y plotting range # if pchk_l > 0: if ymin > yb2_save[-1]: ymin = yb2_save[-1] - abs(ymin - yb2_save[-1]) if pchk_u > 0: if ymax < yt1_save[-1]: ymax = yt1_save[-1] + abs(ymax - yt1_save[-1]) # #--- open and set plotting surface # plt.close('all') fig, ax = plt.subplots(1, figsize=(8,6)) props = font_manager.FontProperties(size=14) mpl.rcParams['font.size'] = 14 mpl.rcParams['font.weight'] = 'bold' # #--- set plotting axes # ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) ax.set_xlabel('Time (year)') if unit != '': if unit in ('DEGF', 'DEGC'): unit = 'K' uline = msid + ' (' + unit + ')' else: uline = msid ax.yaxis.labelpad = 30 ax.set_ylabel(uline) # #--- shade limit areas # if l_len > 0: if len(trb_save) > 0: ax.fill_between(trb_save, rb1_save, rb2_save, facecolor='red', alpha=0.2, interpolate=True) if len(tyb_save) > 0: ax.fill_between(tyb_save, yb1_save, yb2_save, facecolor='yellow', alpha=0.2, interpolate=True) if len(tyt_save) > 0: ax.fill_between(tyt_save, yt1_save, yt2_save, facecolor='yellow', alpha=0.2, interpolate=True) if len(trt_save) > 0: ax.fill_between(trt_save, rt1_save, rt2_save, facecolor='red', alpha=0.2, interpolate=True) # #--- moving average # try: ax.fill_between(xcent, y_min, y_max, facecolor='#00FFFF', alpha=0.3, interpolate=True) except: pass # #--- plot a future possible envelope # if gchk > 0: pbegin = min_a + min_b * tlim pend = min_a + min_b * xmax ax.plot([tlim, xmax], [pbegin, pend], color='green', lw=4, linestyle='dashed') pbegin = max_a + max_b * tlim pend = max_a + max_b * xmax ax.plot([tlim, xmax], [pbegin, pend], color='green', lw=4, linestyle='dashed') if wline != '': plt.text(xpos, ypos, wline, color='red') if wline2 != '': ypos = ymax -0.15 * ydiff plt.text(xpos, ypos, wline2, color='red') elif gchk < 0: plt.text(xpos, ypos, 'More than the last 2 years of data are missing (no violation check)', color='red') # #---- trending plots # points = ax.scatter(pdata[0], pdata[5], color=pdata[17], marker='o', s=80 ,lw=0) # #--- pop up page are created here # labels = create_label_html(pdata[1:17], msid, unit) # #--- link the popup page to the plot # plugins.connect(fig, mpld3.plugins.PointHTMLTooltip(points, labels, css=css, voffset=20, hoffset=-50)) # #--- set the size of plot # fig.set_size_inches(11.0, 8.0) fig.tight_layout() # #--- convert the plot into html format # pout = mpld3.fig_to_html(fig) # #--- save the trend plot separately if the future violation is predicted # if pchk_l > 0: save_plot_html(msid, fig, 'low') if pchk_u > 0: save_plot_html(msid, fig, 'top') plt.close('all') # #--- return plot in html format # return pout
def set_y_range(pdata, y_min, y_max, msid): """ find plotting y range input: pdata --- a list of lists of data xtime = pdata[0] dnum = pdata[1] start = pdata[2] stop = pdata[3] avg = pdata[4] med = pdata[5] std = pdata[6] dmin = pdata[7] dmax = pdata[8] ylow = pdata[9] ytop = pdata[10] rlow = pdata[11] rtop = pdata[12] yl_lim = pdata[13] yu_lim = pdata[14] rl_lim = pdata[15] ru_lim = pdata[16] pcolor = pdata[17] y_min --- a list of lower envelope y_max --- a list of ypper envelope output: [ymin, ymax] """ # #--- remove all dummy values # udata = [] for ent in pdata[5]: if ent in [-999, -998,-99, 99, 998, 999]: continue else: udata.append(ent) # #--- remove all extreme outlayers before computer med and std # udata.sort() lcnt = len(udata) p = int(0.05 * lcnt) test = udata[p:lcnt-p] med = numpy.mean(numpy.array(test)) std = numpy.std(numpy.array(test)) ymin = med - 3.5 * std ymax = med + 3.5 * std if ymin == ymax: tlist = ecf.set_limit_list(msid) ybot = tlist[-1][4] ytop = tlist[-1][5] if (ybot in [-999, -998]) or (ytop in [998, 999]): harea = 1.0 else: harea = 0.5 * abs(ytop - ybot) ymin = med - harea ymax = med + harea # #--- check ymin and ymax are reasoble # if ymin < 0.0: mcut = 1.05 * med else: mcut = 0.95 * med if ymin > mcut: if ymin < 0.0: ymin = 1.1 * ymin else: ymin = 0.9 * ymin if ymax < 0.0: mcut = 0.95 * med else: mcut = 1.05 * med if ymax < mcut: if ymax < 0.0: ymax = 0.9 * ymax else: ymax = 1.1 * ymax # #--- adjust ymin/ymax if the envelop is larger than the original ymin/ymax # try: out = sorted(y_min) test = out[1] except: test = ymin if test < ymin: if test > 0: ymin = 0.95 * test else: ymin = 1.05 * test try: out = sorted(y_max) test = out[-2] except: test = ymax if test > ymax: if test > 0: ymax = 1.05 * test else: ymax = 0.95 * test return [ymin, ymax]
def update_mta_comp_database(): """ updata database of mta computed msids input: none but read from /data/mta4/Deriv/*fits files outpu: updated data file: <data_dir>/<msid>_data """ # #--- get a list of data fits file names # infile = house_keeping + 'mta_comp_fits_files' data = ecf.read_file_data(infile) for fits in data: # #--- hrc has 4 different cases (all data, hrc i, hrc s, and off). tail contain which one this one is #--- if this is not hrc (or hrc all), tail = 2 # mc = re.search('hrc', fits) if mc is not None: atemp = re.split('_', fits) btemp = re.split('.fits', atemp[1]) tail = btemp[0] else: tail = 2 [cols, tbdata] = ecf.read_fits_file(fits) time = [] for ent in tbdata.field('time'): stime = float(ent) # #--- check whether the time is in dom # if stime < 31536000: stime = ecf.dom_to_stime(float(ent)) time.append(stime) for col in cols: col = col.lower() # #--- we need only *_avg columns # mc = re.search('_avg', col) if mc is not None: vals = tbdata.field(col) ctime = [] cvals = [] for m in range(0, len(time)): # #--- skip the data value "nan" and dummy values (-999, -998, -99, 99, 998, 999) # if str(vals[m]) in ['nan', 'NaN', 'NAN']: continue nval = float(vals[m]) if nval in [-999, -998, -99, 99, 998, 999]: continue else: ctime.append(time[m]) cvals.append(nval) atemp = re.split('_', col) msid = atemp[-2] if mcf.chkNumeric(tail): oname = msid else: oname = msid + '_' + tail print "MSID: " + str(oname) cmd = 'rm ' + data_dir + oname + '_data' os.system(cmd) # #--- read limit table for the msid # l_list = ecf.set_limit_list(msid) if len(l_list) == 0: try: l_list = mta_db[msid] except: l_list = [] update_data(msid, l_list, dset=tail, time=ctime, vals=cvals)
def create_limit_table(msid, unit, xmin, xmax): """ create a limit table for msid input: msid --- msid unit --- unit xmin --- xmin xmax --- xmax output: <web_dir>/Limit_table/<msid>_limit_table.html """ # #--- read limit data # pmsid = msid.replace('_i^', '') pmsid = pmsid.replace('_s^', '') pmsid = pmsid.replace('_off^', '') l_list = ecf.set_limit_list(pmsid) line = '<!DOCTYPE html>\n' line = line + '<html>\n' line = line + '<head>\n' line = line + '\t<title>' + msid + ' limit table</title>\n' line = line + '</head>\n' line = line + '<body style="background-color:#FFE4B5; padding-top:20px;background-color:#FAEBD7;' line = line + 'font-family:Georgia, "Times New Roman", Times, serif;">\n' # #--- except a few, all temperatures are in K # if unit == 'DEGF': tline = msid.upper() + ' (K)' elif unit == 'DEGC': tline = msid.upper() + ' (K)' elif unit == '': tline = msid.upper() else: tline = msid.upper() + ' (' + unit + ')' line = line + '<h2>' + tline + '</h2>\n' line = line + '<table border=1 cellpadding=2>\n' line = line + '<tr><th>Start Time</th>\n' line = line + '<th>Stop Time</th>\n' line = line + '<th style="background-color:yellow">Yellow Lower</th>\n' line = line + '<th style="background-color:yellow">Yellow Upper</th>\n' line = line + '<th style="background-color:red">Red Lower</th>\n' line = line + '<th style="background-color:red">Red Upper</th>\n' line = line + '</tr>\n' for k in range(0, len(l_list)): alist = l_list[k] astart = ecf.stime_to_frac_year(float(alist[0])) astop = ecf.stime_to_frac_year(float(alist[1])) # #--- there are often the data with <start>=<stop>, drop them # if astart == astop: continue astart = '%4.2f' % (round(astart, 2)) astop = '%4.2f' % (round(astop, 2)) if k == 0: if astart > xmin: astart = '---' if k == (len(l_list) - 1): astop = "---" yminl = float(alist[2]) ymaxl = float(alist[3]) rminl = float(alist[4]) rmaxl = float(alist[5]) line = line + '<tr>\n' line = line + '<td style="text-align:center;">' + str( astart) + '</td>\n' line = line + '<td style="text-align:center;">' + str( astop) + '</td>\n' line = line + '<td style="text-align:center;">' + str( yminl) + '</td>\n' line = line + '<td style="text-align:center;">' + str( ymaxl) + '</td>\n' line = line + '<td style="text-align:center;">' + str( rminl) + '</td>\n' line = line + '<td style="text-align:center;">' + str( rmaxl) + '</td>\n' line = line + '</tr>\n' line = line + '</table>\n' line = line + '</body>\n</html>\n' file_name = web_dir + 'Limit_table/' + msid + '_limit_table.html' fo = open(file_name, 'w') fo.write(line) fo.close()
def plot_interactive_trend(msid, unit): """ create interactive trend plot input: msid --- msid unit --- unit of msid output: pout --- plot in html format <web_dir>/Future/<msid>_<loc> --- plots in html format saved in the directory if it may violate the limit in future loc: low or top """ # #--- read data # pdata = read_data(msid) if pdata == na: print msid + ': empty data file' return na if len(pdata[0]) < 10: return False # #--- set the x plotting range # xmin = 1999 xmax = ecf.current_time() + 1 xmax = "%4.1f" % round(xmax, 1) xmax = float(xmax) xdiff = xmax - xmin xpos = xmin + 0.05 * xdiff # #--- compute predict trends # [tlim, tmax, min_a, min_b, max_a, max_b, xcent, y_avg, y_min, y_max] = predict_trend(pdata, xmin, xmax) # #--- set y plotting range # [ymin, ymax] = set_y_range(pdata, y_min, y_max, msid) ydiff = ymax - ymin ypos = ymax - 0.1 * ydiff # #--- create a limit table for this msid # create_limit_table(msid, unit, xmin, xmax) # #--- hrc has 4 different categories (all, hrc i, hrc s, off); use the same limit range # pmsid = msid.replace('_i^', '') pmsid = pmsid.replace('_s^', '') pmsid = pmsid.replace('_off^', '') l_list = ecf.set_limit_list(pmsid) l_len = len(l_list) if l_len > 0: [trb_save, rb1_save, rb2_save, tyb_save, yb1_save, yb2_save, tyt_save, \ yt1_save, yt2_save, trt_save, rt1_save, rt2_save] = set_warning_area(l_list, xmin, xmax, ymin, ymax) # #--- create violation notification # if l_len == 0: vtdata = [0, 0, 0, 0] ved.incert_data(msid, vtdata) pchk_l = 0 pchk_u = 0 gchk = 0 elif xmax - tmax > 3.0: vtdata = [0, 0, 0, 0] ved.incert_data(msid, vtdata) pchk_l = 0 pchk_u = 0 gchk = -1 elif (abs(min_b) == 999) or (abs(max_b) == 999): vtdata = [0, 0, 0, 0] ved.incert_data(msid, vtdata) pchk_l = 0 pchk_u = 0 gchk = 0 else: [wline, wline2, pchk_l, pchk_u] = create_violation_notification(msid, pdata, min_a, min_b, max_a, max_b, tmax) gchk = 1 # #--- readjust y plotting range # if pchk_l > 0: if ymin > yb2_save[-1]: ymin = yb2_save[-1] - abs(ymin - yb2_save[-1]) if pchk_u > 0: if ymax < yt1_save[-1]: ymax = yt1_save[-1] + abs(ymax - yt1_save[-1]) # #--- open and set plotting surface # plt.close('all') fig, ax = plt.subplots(1, figsize=(8, 6)) props = font_manager.FontProperties(size=14) mpl.rcParams['font.size'] = 14 mpl.rcParams['font.weight'] = 'bold' # #--- set plotting axes # ax.set_xlim(xmin, xmax) ax.set_ylim(ymin, ymax) ax.set_xlabel('Time (year)') if unit != '': if unit in ('DEGF', 'DEGC'): unit = 'K' uline = msid + ' (' + unit + ')' else: uline = msid ax.yaxis.labelpad = 30 ax.set_ylabel(uline) # #--- shade limit areas # if l_len > 0: if len(trb_save) > 0: ax.fill_between(trb_save, rb1_save, rb2_save, facecolor='red', alpha=0.2, interpolate=True) if len(tyb_save) > 0: ax.fill_between(tyb_save, yb1_save, yb2_save, facecolor='yellow', alpha=0.2, interpolate=True) if len(tyt_save) > 0: ax.fill_between(tyt_save, yt1_save, yt2_save, facecolor='yellow', alpha=0.2, interpolate=True) if len(trt_save) > 0: ax.fill_between(trt_save, rt1_save, rt2_save, facecolor='red', alpha=0.2, interpolate=True) # #--- moving average # try: ax.fill_between(xcent, y_min, y_max, facecolor='#00FFFF', alpha=0.3, interpolate=True) except: pass # #--- plot a future possible envelope # if gchk > 0: pbegin = min_a + min_b * tlim pend = min_a + min_b * xmax ax.plot([tlim, xmax], [pbegin, pend], color='green', lw=4, linestyle='dashed') pbegin = max_a + max_b * tlim pend = max_a + max_b * xmax ax.plot([tlim, xmax], [pbegin, pend], color='green', lw=4, linestyle='dashed') if wline != '': plt.text(xpos, ypos, wline, color='red') if wline2 != '': ypos = ymax - 0.15 * ydiff plt.text(xpos, ypos, wline2, color='red') elif gchk < 0: plt.text( xpos, ypos, 'More than the last 2 years of data are missing (no violation check)', color='red') # #---- trending plots # points = ax.scatter(pdata[0], pdata[5], color=pdata[17], marker='o', s=80, lw=0) # #--- pop up page are created here # labels = create_label_html(pdata[1:17], msid, unit) # #--- link the popup page to the plot # plugins.connect( fig, mpld3.plugins.PointHTMLTooltip(points, labels, css=css, voffset=20, hoffset=-50)) # #--- set the size of plot # fig.set_size_inches(11.0, 8.0) fig.tight_layout() # #--- convert the plot into html format # pout = mpld3.fig_to_html(fig) # #--- save the trend plot separately if the future violation is predicted # if pchk_l > 0: save_plot_html(msid, fig, 'low') if pchk_u > 0: save_plot_html(msid, fig, 'top') plt.close('all') # #--- return plot in html format # return pout
def set_y_range(pdata, y_min, y_max, msid): """ find plotting y range input: pdata --- a list of lists of data xtime = pdata[0] dnum = pdata[1] start = pdata[2] stop = pdata[3] avg = pdata[4] med = pdata[5] std = pdata[6] dmin = pdata[7] dmax = pdata[8] ylow = pdata[9] ytop = pdata[10] rlow = pdata[11] rtop = pdata[12] yl_lim = pdata[13] yu_lim = pdata[14] rl_lim = pdata[15] ru_lim = pdata[16] pcolor = pdata[17] y_min --- a list of lower envelope y_max --- a list of ypper envelope output: [ymin, ymax] """ # #--- remove all dummy values # udata = [] for ent in pdata[5]: if ent in [-999, -998, -99, 99, 998, 999]: continue else: udata.append(ent) # #--- remove all extreme outlayers before computer med and std # udata.sort() lcnt = len(udata) p = int(0.05 * lcnt) test = udata[p:lcnt - p] med = numpy.mean(numpy.array(test)) std = numpy.std(numpy.array(test)) ymin = med - 3.5 * std ymax = med + 3.5 * std if ymin == ymax: tlist = ecf.set_limit_list(msid) ybot = tlist[-1][4] ytop = tlist[-1][5] if (ybot in [-999, -998]) or (ytop in [998, 999]): harea = 1.0 else: harea = 0.5 * abs(ytop - ybot) ymin = med - harea ymax = med + harea # #--- check ymin and ymax are reasoble # if ymin < 0.0: mcut = 1.05 * med else: mcut = 0.95 * med if ymin > mcut: if ymin < 0.0: ymin = 1.1 * ymin else: ymin = 0.9 * ymin if ymax < 0.0: mcut = 0.95 * med else: mcut = 1.05 * med if ymax < mcut: if ymax < 0.0: ymax = 0.9 * ymax else: ymax = 1.1 * ymax # #--- adjust ymin/ymax if the envelop is larger than the original ymin/ymax # try: out = sorted(y_min) test = out[1] except: test = ymin if test < ymin: if test > 0: ymin = 0.95 * test else: ymin = 1.05 * test try: out = sorted(y_max) test = out[-2] except: test = ymax if test > ymax: if test > 0: ymax = 1.05 * test else: ymax = 0.95 * test return [ymin, ymax]
def update_mta_comp_database(): """ updata database of mta computed msids input: none but read from /data/mta4/Deriv/*fits files outpu: updated data file: <data_dir>/<msid>_data """ # #--- get a list of data fits file names # infile = house_keeping + 'mta_comp_fits_files' data = ecf.read_file_data(infile) for fits in data: # #--- hrc has 4 different cases (all data, hrc i, hrc s, and off). tail contain which one this one is #--- if this is not hrc (or hrc all), tail = 2 # mc = re.search('hrc', fits) if mc is not None: atemp = re.split('_', fits) btemp = re.split('.fits', atemp[1]) tail = btemp[0] else: tail = 2 [cols, tbdata] = ecf.read_fits_file(fits) time = [] for ent in tbdata.field('time'): stime = float(ent) # #--- check whether the time is in dom # if stime < 31536000: stime = ecf.dom_to_stime(float(ent)) time.append(stime) for col in cols: col = col.lower() # #--- we need only *_avg columns # mc = re.search('_avg', col) if mc is not None: vals = tbdata.field(col) ctime = [] cvals = [] for m in range(0, len(time)): # #--- skip the data value "nan" and dummy values (-999, -998, -99, 99, 998, 999) # if str(vals[m]) in ['nan', 'NaN', 'NAN']: continue nval = float(vals[m]) if nval in [-999, -998, -99, 99, 998, 999]: continue else: ctime.append(time[m]) cvals.append(nval) atemp = re.split('_', col) msid = atemp[-2] if mcf.chkNumeric(tail): oname = msid else: oname = msid + '_' + tail print "MSID: " + str(oname) cmd = 'rm ' + data_dir + oname + '_data' os.system(cmd) # #--- read limit table for the msid # l_list = ecf.set_limit_list(msid) if len(l_list) == 0: try: l_list = mta_db[msid] except: l_list = [] update_data(msid, l_list, dset = tail, time=ctime, vals=cvals)