def test_compute_stats(self):
msid = '1crbt'
start = 155088000.0
stop = 155865600.0
comp = [2592, 145.86, 145.22, 1.04, 145.22, 147.61, 0, 0, 0, 0]
l_list = ecf.set_limit_list(msid)
out = set_limits_for_period(l_list, start, stop)
[start, stop, y_min, y_max, r_min, r_max] = out[0]
data = get_data(msid, start, stop)
out = compute_stats(data, y_min, y_max, r_min, r_max)
self.assertEquals(out, comp)
def test_compute_stats(self):
msid = '1crbt'
start = 155088000.0
stop = 155865600.0
comp = [2592, 145.86, 145.22, 1.04, 145.22, 147.61, 0, 0, 0, 0]
l_list = ecf.set_limit_list(msid)
out = set_limits_for_period(l_list, start, stop)
[start, stop, y_min, y_max, r_min, r_max] = out[0]
data = get_data(msid, start, stop)
out = compute_stats(data, y_min, y_max, r_min, r_max)
self.assertEquals(out, comp)
def update_glimmon_database():
"""
updata database related glimmon msids
input: none but read from several static list and get data with dataseeker
outpu: updated data file: <data_dir>/<msid>_data
"""
#
#--- read all msids and go through one by one
#
msid_list = read_msids()
for msid in msid_list:
#
#--- read limit table for the msid
#
try:
l_list = ecf.set_limit_list(msid)
except:
l_list = []
print "MSID: " + str(msid)
update_data(msid, l_list)
def update_glimmon_database():
"""
updata database related glimmon msids
input: none but read from several static list and get data with dataseeker
outpu: updated data file: <data_dir>/<msid>_data
"""
#
#--- read all msids and go through one by one
#
msid_list = read_msids()
for msid in msid_list:
#
#--- read limit table for the msid
#
try:
l_list = ecf.set_limit_list(msid)
except:
l_list = []
print "MSID: " + str(msid)
update_data(msid, l_list)
def create_limit_table(msid, group, unit, xmin, xmax):
"""
create a limit table for msid
input: msid --- msid
unit --- unit
xmin --- xmin
xmax --- xmax
output: <web_dir>/Limit_table/<msid>_limit_table.html
"""
#
#--- read limit data
#
pmsid = drop_suffix(msid)
l_list = ecf.set_limit_list(pmsid)
#
#--- read header part
#
title = msid + ' limit table'
repl = [["#MSID#", title], ["#JAVASCRIPT#", ''], ["#STYLE#", ""]]
line = read_template('html_head', repl)
#
#--- except a few, all temperatures are in K
#
if unit == 'DEGF':
tline = msid.upper() + ' (K)'
elif unit == 'DEGC':
tline = msid.upper() + ' (K)'
elif unit == '':
tline = msid.upper()
else:
tline = msid.upper() + ' (' + unit + ')'
bgline = '<th style="background-color:'
line = line + '<h2>' + tline + '</h2>\n'
line = line + '<table border=1 cellpadding=2>\n'
line = line + '<tr><th>Start Time</th>\n'
line = line + '<th>Stop Time</th>\n'
line = line + bgline + 'yellow">Yellow Lower</th>\n'
line = line + bgline + 'yellow">Yellow Upper</th>\n'
line = line + bgline + 'red">Red Lower</th>\n'
line = line + bgline + 'red">Red Upper</th>\n'
line = line + '</tr>\n'
for k in range(0, len(l_list)):
alist = l_list[k]
[astart, byear, base] = convert_stime_into_year(float(alist[0]))
[astop, byear, base] = convert_stime_into_year(float(alist[1]))
#
#--- there are often the data with <start>=<stop>, drop them
#
if astart == astop:
continue
astart = float('%4.2f' % (round(astart, 2)))
astop = float('%4.2f' % (round(astop, 2)))
if k == 0:
if astart > xmin:
astart = '---'
if k == (len(l_list) - 1):
astop = "---"
#
#--- alist: ymin, ymax, rmin, rmax in position of 2 to 5
#
tlist = [astart, astop] + alist[2:6]
#
#--- create each row
#
line = line + '<tr>\n'
for tval in tlist:
line = line + '<td style="text-align:center;">' + str(
tval) + '</td>\n'
line = line + '</tr>\n'
line = line + '</table>\n'
line = line + '</body>\n</html>\n'
o_dir = web_dir + group + '/'
check_dir_exist(o_dir)
o_dir = o_dir + 'Limit_table/'
check_dir_exist(o_dir)
file_name = o_dir + msid + '_limit_table.html'
with open(file_name, 'w') as fo:
fo.write(line)
def create_limit_table(msid, unit, xmin, xmax):
"""
create a limit table for msid
input: msid --- msid
unit --- unit
xmin --- xmin
xmax --- xmax
output: <web_dir>/Limit_table/<msid>_limit_table.html
"""
#
#--- read limit data
#
pmsid = msid.replace('_i^', '')
pmsid = pmsid.replace('_s^', '')
pmsid = pmsid.replace('_off^', '')
l_list = ecf.set_limit_list(pmsid)
line = '<!DOCTYPE html>\n'
line = line + '<html>\n'
line = line + '<head>\n'
line = line + '\t<title>' + msid + ' limit table</title>\n'
line = line + '</head>\n'
line = line + '<body style="background-color:#FFE4B5; padding-top:20px;background-color:#FAEBD7;'
line = line + 'font-family:Georgia, "Times New Roman", Times, serif;">\n'
#
#--- except a few, all temperatures are in K
#
if unit == 'DEGF':
tline = msid.upper() + ' (K)'
elif unit == 'DEGC':
tline = msid.upper() + ' (K)'
elif unit == '':
tline = msid.upper()
else:
tline = msid.upper() + ' (' + unit + ')'
line = line + '<h2>' + tline + '</h2>\n'
line = line + '<table border=1 cellpadding=2>\n'
line = line + '<tr><th>Start Time</th>\n'
line = line + '<th>Stop Time</th>\n'
line = line + '<th style="background-color:yellow">Yellow Lower</th>\n'
line = line + '<th style="background-color:yellow">Yellow Upper</th>\n'
line = line + '<th style="background-color:red">Red Lower</th>\n'
line = line + '<th style="background-color:red">Red Upper</th>\n'
line = line + '</tr>\n'
for k in range(0, len(l_list)):
alist = l_list[k]
astart = ecf.stime_to_frac_year(float(alist[0]))
astop = ecf.stime_to_frac_year(float(alist[1]))
#
#--- there are often the data with <start>=<stop>, drop them
#
if astart == astop:
continue
astart = '%4.2f' % (round(astart,2))
astop = '%4.2f' % (round(astop, 2))
if k == 0:
if astart > xmin:
astart = '---'
if k == (len(l_list) -1):
astop = "---"
yminl = float(alist[2])
ymaxl = float(alist[3])
rminl = float(alist[4])
rmaxl = float(alist[5])
line = line + '<tr>\n'
line = line + '<td style="text-align:center;">' + str(astart) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(astop) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(yminl) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(ymaxl) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(rminl) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(rmaxl) + '</td>\n'
line = line + '</tr>\n'
line = line + '</table>\n'
line = line + '</body>\n</html>\n'
file_name = web_dir + 'Limit_table/' + msid + '_limit_table.html'
fo = open(file_name, 'w')
fo.write(line)
fo.close()
def plot_interactive_trend(msid, unit):
"""
create interactive trend plot
input: msid --- msid
unit --- unit of msid
output: pout --- plot in html format
<web_dir>/Future/<msid>_<loc> --- plots in html format saved in the directory
if it may violate the limit in future loc: low or top
"""
#
#--- read data
#
pdata = read_data(msid)
if pdata == na:
print msid + ': empty data file'
return na
if len(pdata[0]) < 10:
return False
#
#--- set the x plotting range
#
xmin = 1999
xmax = ecf.current_time() + 1
xmax = "%4.1f" % round(xmax, 1)
xmax = float(xmax)
xdiff = xmax - xmin
xpos = xmin + 0.05 * xdiff
#
#--- compute predict trends
#
[tlim, tmax, min_a, min_b, max_a, max_b, xcent, y_avg, y_min, y_max] = predict_trend(pdata, xmin, xmax)
#
#--- set y plotting range
#
[ymin, ymax] = set_y_range(pdata, y_min, y_max, msid)
ydiff = ymax - ymin
ypos = ymax - 0.1 * ydiff
#
#--- create a limit table for this msid
#
create_limit_table(msid, unit, xmin, xmax)
#
#--- hrc has 4 different categories (all, hrc i, hrc s, off); use the same limit range
#
pmsid = msid.replace('_i^', '')
pmsid = pmsid.replace('_s^', '')
pmsid = pmsid.replace('_off^', '')
l_list = ecf.set_limit_list(pmsid)
l_len = len(l_list)
if l_len > 0:
[trb_save, rb1_save, rb2_save, tyb_save, yb1_save, yb2_save, tyt_save, \
yt1_save, yt2_save, trt_save, rt1_save, rt2_save] = set_warning_area(l_list, xmin, xmax, ymin, ymax)
#
#--- create violation notification
#
if l_len == 0:
vtdata = [0, 0, 0, 0]
ved.incert_data(msid, vtdata)
pchk_l = 0
pchk_u = 0
gchk = 0
elif xmax - tmax > 3.0:
vtdata = [0, 0, 0, 0]
ved.incert_data(msid, vtdata)
pchk_l = 0
pchk_u = 0
gchk = -1
elif (abs(min_b) == 999) or (abs(max_b) == 999):
vtdata = [0, 0, 0, 0]
ved.incert_data(msid, vtdata)
pchk_l = 0
pchk_u = 0
gchk = 0
else:
[wline, wline2, pchk_l, pchk_u] = create_violation_notification(msid, pdata, min_a, min_b, max_a, max_b, tmax)
gchk = 1
#
#--- readjust y plotting range
#
if pchk_l > 0:
if ymin > yb2_save[-1]:
ymin = yb2_save[-1] - abs(ymin - yb2_save[-1])
if pchk_u > 0:
if ymax < yt1_save[-1]:
ymax = yt1_save[-1] + abs(ymax - yt1_save[-1])
#
#--- open and set plotting surface
#
plt.close('all')
fig, ax = plt.subplots(1, figsize=(8,6))
props = font_manager.FontProperties(size=14)
mpl.rcParams['font.size'] = 14
mpl.rcParams['font.weight'] = 'bold'
#
#--- set plotting axes
#
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
ax.set_xlabel('Time (year)')
if unit != '':
if unit in ('DEGF', 'DEGC'):
unit = 'K'
uline = msid + ' (' + unit + ')'
else:
uline = msid
ax.yaxis.labelpad = 30
ax.set_ylabel(uline)
#
#--- shade limit areas
#
if l_len > 0:
if len(trb_save) > 0:
ax.fill_between(trb_save, rb1_save, rb2_save, facecolor='red', alpha=0.2, interpolate=True)
if len(tyb_save) > 0:
ax.fill_between(tyb_save, yb1_save, yb2_save, facecolor='yellow', alpha=0.2, interpolate=True)
if len(tyt_save) > 0:
ax.fill_between(tyt_save, yt1_save, yt2_save, facecolor='yellow', alpha=0.2, interpolate=True)
if len(trt_save) > 0:
ax.fill_between(trt_save, rt1_save, rt2_save, facecolor='red', alpha=0.2, interpolate=True)
#
#--- moving average
#
try:
ax.fill_between(xcent, y_min, y_max, facecolor='#00FFFF', alpha=0.3, interpolate=True)
except:
pass
#
#--- plot a future possible envelope
#
if gchk > 0:
pbegin = min_a + min_b * tlim
pend = min_a + min_b * xmax
ax.plot([tlim, xmax], [pbegin, pend], color='green', lw=4, linestyle='dashed')
pbegin = max_a + max_b * tlim
pend = max_a + max_b * xmax
ax.plot([tlim, xmax], [pbegin, pend], color='green', lw=4, linestyle='dashed')
if wline != '':
plt.text(xpos, ypos, wline, color='red')
if wline2 != '':
ypos = ymax -0.15 * ydiff
plt.text(xpos, ypos, wline2, color='red')
elif gchk < 0:
plt.text(xpos, ypos, 'More than the last 2 years of data are missing (no violation check)', color='red')
#
#---- trending plots
#
points = ax.scatter(pdata[0], pdata[5], color=pdata[17], marker='o', s=80 ,lw=0)
#
#--- pop up page are created here
#
labels = create_label_html(pdata[1:17], msid, unit)
#
#--- link the popup page to the plot
#
plugins.connect(fig, mpld3.plugins.PointHTMLTooltip(points, labels, css=css, voffset=20, hoffset=-50))
#
#--- set the size of plot
#
fig.set_size_inches(11.0, 8.0)
fig.tight_layout()
#
#--- convert the plot into html format
#
pout = mpld3.fig_to_html(fig)
#
#--- save the trend plot separately if the future violation is predicted
#
if pchk_l > 0:
save_plot_html(msid, fig, 'low')
if pchk_u > 0:
save_plot_html(msid, fig, 'top')
plt.close('all')
#
#--- return plot in html format
#
return pout
def set_y_range(pdata, y_min, y_max, msid):
"""
find plotting y range
input: pdata --- a list of lists of data
xtime = pdata[0]
dnum = pdata[1]
start = pdata[2]
stop = pdata[3]
avg = pdata[4]
med = pdata[5]
std = pdata[6]
dmin = pdata[7]
dmax = pdata[8]
ylow = pdata[9]
ytop = pdata[10]
rlow = pdata[11]
rtop = pdata[12]
yl_lim = pdata[13]
yu_lim = pdata[14]
rl_lim = pdata[15]
ru_lim = pdata[16]
pcolor = pdata[17]
y_min --- a list of lower envelope
y_max --- a list of ypper envelope
output: [ymin, ymax]
"""
#
#--- remove all dummy values
#
udata = []
for ent in pdata[5]:
if ent in [-999, -998,-99, 99, 998, 999]:
continue
else:
udata.append(ent)
#
#--- remove all extreme outlayers before computer med and std
#
udata.sort()
lcnt = len(udata)
p = int(0.05 * lcnt)
test = udata[p:lcnt-p]
med = numpy.mean(numpy.array(test))
std = numpy.std(numpy.array(test))
ymin = med - 3.5 * std
ymax = med + 3.5 * std
if ymin == ymax:
tlist = ecf.set_limit_list(msid)
ybot = tlist[-1][4]
ytop = tlist[-1][5]
if (ybot in [-999, -998]) or (ytop in [998, 999]):
harea = 1.0
else:
harea = 0.5 * abs(ytop - ybot)
ymin = med - harea
ymax = med + harea
#
#--- check ymin and ymax are reasoble
#
if ymin < 0.0:
mcut = 1.05 * med
else:
mcut = 0.95 * med
if ymin > mcut:
if ymin < 0.0:
ymin = 1.1 * ymin
else:
ymin = 0.9 * ymin
if ymax < 0.0:
mcut = 0.95 * med
else:
mcut = 1.05 * med
if ymax < mcut:
if ymax < 0.0:
ymax = 0.9 * ymax
else:
ymax = 1.1 * ymax
#
#--- adjust ymin/ymax if the envelop is larger than the original ymin/ymax
#
try:
out = sorted(y_min)
test = out[1]
except:
test = ymin
if test < ymin:
if test > 0:
ymin = 0.95 * test
else:
ymin = 1.05 * test
try:
out = sorted(y_max)
test = out[-2]
except:
test = ymax
if test > ymax:
if test > 0:
ymax = 1.05 * test
else:
ymax = 0.95 * test
return [ymin, ymax]
def update_mta_comp_database():
"""
updata database of mta computed msids
input: none but read from /data/mta4/Deriv/*fits files
outpu: updated data file: <data_dir>/<msid>_data
"""
#
#--- get a list of data fits file names
#
infile = house_keeping + 'mta_comp_fits_files'
data = ecf.read_file_data(infile)
for fits in data:
#
#--- hrc has 4 different cases (all data, hrc i, hrc s, and off). tail contain which one this one is
#--- if this is not hrc (or hrc all), tail = 2
#
mc = re.search('hrc', fits)
if mc is not None:
atemp = re.split('_', fits)
btemp = re.split('.fits', atemp[1])
tail = btemp[0]
else:
tail = 2
[cols, tbdata] = ecf.read_fits_file(fits)
time = []
for ent in tbdata.field('time'):
stime = float(ent)
#
#--- check whether the time is in dom
#
if stime < 31536000:
stime = ecf.dom_to_stime(float(ent))
time.append(stime)
for col in cols:
col = col.lower()
#
#--- we need only *_avg columns
#
mc = re.search('_avg', col)
if mc is not None:
vals = tbdata.field(col)
ctime = []
cvals = []
for m in range(0, len(time)):
#
#--- skip the data value "nan" and dummy values (-999, -998, -99, 99, 998, 999)
#
if str(vals[m]) in ['nan', 'NaN', 'NAN']:
continue
nval = float(vals[m])
if nval in [-999, -998, -99, 99, 998, 999]:
continue
else:
ctime.append(time[m])
cvals.append(nval)
atemp = re.split('_', col)
msid = atemp[-2]
if mcf.chkNumeric(tail):
oname = msid
else:
oname = msid + '_' + tail
print "MSID: " + str(oname)
cmd = 'rm ' + data_dir + oname + '_data'
os.system(cmd)
#
#--- read limit table for the msid
#
l_list = ecf.set_limit_list(msid)
if len(l_list) == 0:
try:
l_list = mta_db[msid]
except:
l_list = []
update_data(msid, l_list, dset=tail, time=ctime, vals=cvals)
def create_limit_table(msid, unit, xmin, xmax):
"""
create a limit table for msid
input: msid --- msid
unit --- unit
xmin --- xmin
xmax --- xmax
output: <web_dir>/Limit_table/<msid>_limit_table.html
"""
#
#--- read limit data
#
pmsid = msid.replace('_i^', '')
pmsid = pmsid.replace('_s^', '')
pmsid = pmsid.replace('_off^', '')
l_list = ecf.set_limit_list(pmsid)
line = '<!DOCTYPE html>\n'
line = line + '<html>\n'
line = line + '<head>\n'
line = line + '\t<title>' + msid + ' limit table</title>\n'
line = line + '</head>\n'
line = line + '<body style="background-color:#FFE4B5; padding-top:20px;background-color:#FAEBD7;'
line = line + 'font-family:Georgia, "Times New Roman", Times, serif;">\n'
#
#--- except a few, all temperatures are in K
#
if unit == 'DEGF':
tline = msid.upper() + ' (K)'
elif unit == 'DEGC':
tline = msid.upper() + ' (K)'
elif unit == '':
tline = msid.upper()
else:
tline = msid.upper() + ' (' + unit + ')'
line = line + '<h2>' + tline + '</h2>\n'
line = line + '<table border=1 cellpadding=2>\n'
line = line + '<tr><th>Start Time</th>\n'
line = line + '<th>Stop Time</th>\n'
line = line + '<th style="background-color:yellow">Yellow Lower</th>\n'
line = line + '<th style="background-color:yellow">Yellow Upper</th>\n'
line = line + '<th style="background-color:red">Red Lower</th>\n'
line = line + '<th style="background-color:red">Red Upper</th>\n'
line = line + '</tr>\n'
for k in range(0, len(l_list)):
alist = l_list[k]
astart = ecf.stime_to_frac_year(float(alist[0]))
astop = ecf.stime_to_frac_year(float(alist[1]))
#
#--- there are often the data with <start>=<stop>, drop them
#
if astart == astop:
continue
astart = '%4.2f' % (round(astart, 2))
astop = '%4.2f' % (round(astop, 2))
if k == 0:
if astart > xmin:
astart = '---'
if k == (len(l_list) - 1):
astop = "---"
yminl = float(alist[2])
ymaxl = float(alist[3])
rminl = float(alist[4])
rmaxl = float(alist[5])
line = line + '<tr>\n'
line = line + '<td style="text-align:center;">' + str(
astart) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(
astop) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(
yminl) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(
ymaxl) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(
rminl) + '</td>\n'
line = line + '<td style="text-align:center;">' + str(
rmaxl) + '</td>\n'
line = line + '</tr>\n'
line = line + '</table>\n'
line = line + '</body>\n</html>\n'
file_name = web_dir + 'Limit_table/' + msid + '_limit_table.html'
fo = open(file_name, 'w')
fo.write(line)
fo.close()
def plot_interactive_trend(msid, unit):
"""
create interactive trend plot
input: msid --- msid
unit --- unit of msid
output: pout --- plot in html format
<web_dir>/Future/<msid>_<loc> --- plots in html format saved in the directory
if it may violate the limit in future loc: low or top
"""
#
#--- read data
#
pdata = read_data(msid)
if pdata == na:
print msid + ': empty data file'
return na
if len(pdata[0]) < 10:
return False
#
#--- set the x plotting range
#
xmin = 1999
xmax = ecf.current_time() + 1
xmax = "%4.1f" % round(xmax, 1)
xmax = float(xmax)
xdiff = xmax - xmin
xpos = xmin + 0.05 * xdiff
#
#--- compute predict trends
#
[tlim, tmax, min_a, min_b, max_a, max_b, xcent, y_avg, y_min,
y_max] = predict_trend(pdata, xmin, xmax)
#
#--- set y plotting range
#
[ymin, ymax] = set_y_range(pdata, y_min, y_max, msid)
ydiff = ymax - ymin
ypos = ymax - 0.1 * ydiff
#
#--- create a limit table for this msid
#
create_limit_table(msid, unit, xmin, xmax)
#
#--- hrc has 4 different categories (all, hrc i, hrc s, off); use the same limit range
#
pmsid = msid.replace('_i^', '')
pmsid = pmsid.replace('_s^', '')
pmsid = pmsid.replace('_off^', '')
l_list = ecf.set_limit_list(pmsid)
l_len = len(l_list)
if l_len > 0:
[trb_save, rb1_save, rb2_save, tyb_save, yb1_save, yb2_save, tyt_save, \
yt1_save, yt2_save, trt_save, rt1_save, rt2_save] = set_warning_area(l_list, xmin, xmax, ymin, ymax)
#
#--- create violation notification
#
if l_len == 0:
vtdata = [0, 0, 0, 0]
ved.incert_data(msid, vtdata)
pchk_l = 0
pchk_u = 0
gchk = 0
elif xmax - tmax > 3.0:
vtdata = [0, 0, 0, 0]
ved.incert_data(msid, vtdata)
pchk_l = 0
pchk_u = 0
gchk = -1
elif (abs(min_b) == 999) or (abs(max_b) == 999):
vtdata = [0, 0, 0, 0]
ved.incert_data(msid, vtdata)
pchk_l = 0
pchk_u = 0
gchk = 0
else:
[wline, wline2, pchk_l,
pchk_u] = create_violation_notification(msid, pdata, min_a, min_b,
max_a, max_b, tmax)
gchk = 1
#
#--- readjust y plotting range
#
if pchk_l > 0:
if ymin > yb2_save[-1]:
ymin = yb2_save[-1] - abs(ymin - yb2_save[-1])
if pchk_u > 0:
if ymax < yt1_save[-1]:
ymax = yt1_save[-1] + abs(ymax - yt1_save[-1])
#
#--- open and set plotting surface
#
plt.close('all')
fig, ax = plt.subplots(1, figsize=(8, 6))
props = font_manager.FontProperties(size=14)
mpl.rcParams['font.size'] = 14
mpl.rcParams['font.weight'] = 'bold'
#
#--- set plotting axes
#
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
ax.set_xlabel('Time (year)')
if unit != '':
if unit in ('DEGF', 'DEGC'):
unit = 'K'
uline = msid + ' (' + unit + ')'
else:
uline = msid
ax.yaxis.labelpad = 30
ax.set_ylabel(uline)
#
#--- shade limit areas
#
if l_len > 0:
if len(trb_save) > 0:
ax.fill_between(trb_save,
rb1_save,
rb2_save,
facecolor='red',
alpha=0.2,
interpolate=True)
if len(tyb_save) > 0:
ax.fill_between(tyb_save,
yb1_save,
yb2_save,
facecolor='yellow',
alpha=0.2,
interpolate=True)
if len(tyt_save) > 0:
ax.fill_between(tyt_save,
yt1_save,
yt2_save,
facecolor='yellow',
alpha=0.2,
interpolate=True)
if len(trt_save) > 0:
ax.fill_between(trt_save,
rt1_save,
rt2_save,
facecolor='red',
alpha=0.2,
interpolate=True)
#
#--- moving average
#
try:
ax.fill_between(xcent,
y_min,
y_max,
facecolor='#00FFFF',
alpha=0.3,
interpolate=True)
except:
pass
#
#--- plot a future possible envelope
#
if gchk > 0:
pbegin = min_a + min_b * tlim
pend = min_a + min_b * xmax
ax.plot([tlim, xmax], [pbegin, pend],
color='green',
lw=4,
linestyle='dashed')
pbegin = max_a + max_b * tlim
pend = max_a + max_b * xmax
ax.plot([tlim, xmax], [pbegin, pend],
color='green',
lw=4,
linestyle='dashed')
if wline != '':
plt.text(xpos, ypos, wline, color='red')
if wline2 != '':
ypos = ymax - 0.15 * ydiff
plt.text(xpos, ypos, wline2, color='red')
elif gchk < 0:
plt.text(
xpos,
ypos,
'More than the last 2 years of data are missing (no violation check)',
color='red')
#
#---- trending plots
#
points = ax.scatter(pdata[0],
pdata[5],
color=pdata[17],
marker='o',
s=80,
lw=0)
#
#--- pop up page are created here
#
labels = create_label_html(pdata[1:17], msid, unit)
#
#--- link the popup page to the plot
#
plugins.connect(
fig,
mpld3.plugins.PointHTMLTooltip(points,
labels,
css=css,
voffset=20,
hoffset=-50))
#
#--- set the size of plot
#
fig.set_size_inches(11.0, 8.0)
fig.tight_layout()
#
#--- convert the plot into html format
#
pout = mpld3.fig_to_html(fig)
#
#--- save the trend plot separately if the future violation is predicted
#
if pchk_l > 0:
save_plot_html(msid, fig, 'low')
if pchk_u > 0:
save_plot_html(msid, fig, 'top')
plt.close('all')
#
#--- return plot in html format
#
return pout
def set_y_range(pdata, y_min, y_max, msid):
"""
find plotting y range
input: pdata --- a list of lists of data
xtime = pdata[0]
dnum = pdata[1]
start = pdata[2]
stop = pdata[3]
avg = pdata[4]
med = pdata[5]
std = pdata[6]
dmin = pdata[7]
dmax = pdata[8]
ylow = pdata[9]
ytop = pdata[10]
rlow = pdata[11]
rtop = pdata[12]
yl_lim = pdata[13]
yu_lim = pdata[14]
rl_lim = pdata[15]
ru_lim = pdata[16]
pcolor = pdata[17]
y_min --- a list of lower envelope
y_max --- a list of ypper envelope
output: [ymin, ymax]
"""
#
#--- remove all dummy values
#
udata = []
for ent in pdata[5]:
if ent in [-999, -998, -99, 99, 998, 999]:
continue
else:
udata.append(ent)
#
#--- remove all extreme outlayers before computer med and std
#
udata.sort()
lcnt = len(udata)
p = int(0.05 * lcnt)
test = udata[p:lcnt - p]
med = numpy.mean(numpy.array(test))
std = numpy.std(numpy.array(test))
ymin = med - 3.5 * std
ymax = med + 3.5 * std
if ymin == ymax:
tlist = ecf.set_limit_list(msid)
ybot = tlist[-1][4]
ytop = tlist[-1][5]
if (ybot in [-999, -998]) or (ytop in [998, 999]):
harea = 1.0
else:
harea = 0.5 * abs(ytop - ybot)
ymin = med - harea
ymax = med + harea
#
#--- check ymin and ymax are reasoble
#
if ymin < 0.0:
mcut = 1.05 * med
else:
mcut = 0.95 * med
if ymin > mcut:
if ymin < 0.0:
ymin = 1.1 * ymin
else:
ymin = 0.9 * ymin
if ymax < 0.0:
mcut = 0.95 * med
else:
mcut = 1.05 * med
if ymax < mcut:
if ymax < 0.0:
ymax = 0.9 * ymax
else:
ymax = 1.1 * ymax
#
#--- adjust ymin/ymax if the envelop is larger than the original ymin/ymax
#
try:
out = sorted(y_min)
test = out[1]
except:
test = ymin
if test < ymin:
if test > 0:
ymin = 0.95 * test
else:
ymin = 1.05 * test
try:
out = sorted(y_max)
test = out[-2]
except:
test = ymax
if test > ymax:
if test > 0:
ymax = 1.05 * test
else:
ymax = 0.95 * test
return [ymin, ymax]
def update_mta_comp_database():
"""
updata database of mta computed msids
input: none but read from /data/mta4/Deriv/*fits files
outpu: updated data file: <data_dir>/<msid>_data
"""
#
#--- get a list of data fits file names
#
infile = house_keeping + 'mta_comp_fits_files'
data = ecf.read_file_data(infile)
for fits in data:
#
#--- hrc has 4 different cases (all data, hrc i, hrc s, and off). tail contain which one this one is
#--- if this is not hrc (or hrc all), tail = 2
#
mc = re.search('hrc', fits)
if mc is not None:
atemp = re.split('_', fits)
btemp = re.split('.fits', atemp[1])
tail = btemp[0]
else:
tail = 2
[cols, tbdata] = ecf.read_fits_file(fits)
time = []
for ent in tbdata.field('time'):
stime = float(ent)
#
#--- check whether the time is in dom
#
if stime < 31536000:
stime = ecf.dom_to_stime(float(ent))
time.append(stime)
for col in cols:
col = col.lower()
#
#--- we need only *_avg columns
#
mc = re.search('_avg', col)
if mc is not None:
vals = tbdata.field(col)
ctime = []
cvals = []
for m in range(0, len(time)):
#
#--- skip the data value "nan" and dummy values (-999, -998, -99, 99, 998, 999)
#
if str(vals[m]) in ['nan', 'NaN', 'NAN']:
continue
nval = float(vals[m])
if nval in [-999, -998, -99, 99, 998, 999]:
continue
else:
ctime.append(time[m])
cvals.append(nval)
atemp = re.split('_', col)
msid = atemp[-2]
if mcf.chkNumeric(tail):
oname = msid
else:
oname = msid + '_' + tail
print "MSID: " + str(oname)
cmd = 'rm ' + data_dir + oname + '_data'
os.system(cmd)
#
#--- read limit table for the msid
#
l_list = ecf.set_limit_list(msid)
if len(l_list) == 0:
try:
l_list = mta_db[msid]
except:
l_list = []
update_data(msid, l_list, dset = tail, time=ctime, vals=cvals)