def main(opt):
opt, args = get_options()
if not os.path.exists(opt.outdir):
os.mkdir(opt.outdir)
config_logging(opt.outdir, opt.verbose)
# Store info relevant to processing for use in outputs
proc = dict(
run_user=os.environ['USER'],
run_time=time.ctime(),
errors=[],
)
logger.info(
'#####################################################################'
)
logger.info(
'# %s run at %s by %s' %
(os.path.dirname(__file__), proc['run_time'], proc['run_user']))
logger.info('# version = %s' % VERSION)
logger.info('# characteristics version = %s' % characteristics.VERSION)
logger.info(
'#####################################################################\n'
)
logger.info('Command line options:\n%s\n' % pformat(opt.__dict__))
# Connect to database (NEED TO USE aca_read)
tnow = DateTime(opt.run_start_time).secs
tstart = tnow
# Get temperature telemetry for 3 weeks prior to min(tstart, NOW)
tlm = get_telem_values(tstart, [
'sim_z', 'dp_pitch', 'aoacaseq', 'aodithen', 'cacalsta', 'cobsrqid',
'aofunlst', 'aopcadmd', '4ootgsel', '4ootgmtn', 'aocmdqt1', 'aocmdqt2',
'aocmdqt3', '1de28avo', '1deicacu', '1dp28avo', '1dpicacu', '1dp28bvo',
'1dpicbcu'
],
days=opt.days,
name_map={
'sim_z': 'tscpos',
'cobsrqid': 'obsid'
})
tlm['tscpos'] = tlm['tscpos'] * -397.7225924607
outdir = opt.outdir
states = get_states(tlm[0].date, tlm[-1].date)
write_states(opt, states)
tlm = Ska.Numpy.add_column(tlm, 'power', smoothed_power(tlm))
# Get bad time intervals
bad_time_mask = get_bad_mask(tlm)
# Interpolate states onto the tlm.date grid
state_vals = cmd_states.interpolate_states(states, tlm['date'])
# "Forgive" dither intervals with dark current replicas
# This will also exclude dither disables that are in cmd states for standard dark cals
dark_mask = np.zeros(len(tlm), dtype='bool')
dark_times = []
# Find dither "disable" states from tlm
dith_disa_states = logical_intervals(tlm['date'],
tlm['aodithen'] == 'DISA')
for state in dith_disa_states:
# Index back into telemetry for each of these constant dither disable states
idx0 = np.searchsorted(tlm['date'], state['tstart'], side='left')
idx1 = np.searchsorted(tlm['date'], state['tstop'], side='right')
# If any samples have aca calibration flag, mark interval for exclusion.
if np.any(tlm['cacalsta'][idx0:idx1] != 'OFF '):
dark_mask[idx0:idx1] = True
dark_times.append({
'start': state['datestart'],
'stop': state['datestop']
})
# Calculate the 4th term of the commanded quaternions
cmd_q4 = np.sqrt(
np.abs(1.0 - tlm['aocmdqt1']**2 - tlm['aocmdqt2']**2 -
tlm['aocmdqt3']**2))
raw_tlm_q = np.vstack(
[tlm['aocmdqt1'], tlm['aocmdqt2'], tlm['aocmdqt3'],
cmd_q4]).transpose()
# Calculate angle/roll differences in state cmd vs tlm cmd quaternions
raw_state_q = np.vstack([state_vals[n]
for n in ['q1', 'q2', 'q3', 'q4']]).transpose()
tlm_q = normalize(raw_tlm_q)
# only use values that aren't NaNs
good = np.isnan(np.sum(tlm_q, axis=-1)) == False
# and are in NPNT
npnt = tlm['aopcadmd'] == 'NPNT'
# and are in KALM after the first 2 sample of the transition
not_kalm = tlm['aoacaseq'] != 'KALM'
kalm = (not_kalm | np.hstack([[False, False], not_kalm[:-2]])) == False
# and aren't during momentum unloads or in the first 2 samples after unloads
unload = tlm['aofunlst'] != 'NONE'
no_unload = (unload | np.hstack([[False, False], unload[:-2]])) == False
ok = good & npnt & kalm & no_unload & ~bad_time_mask
state_q = normalize(raw_state_q)
dot_q = np.sum(tlm_q[ok] * state_q[ok], axis=-1)
dot_q[dot_q > 1] = 1
angle_diff = np.degrees(2 * np.arccos(dot_q))
angle_diff = np.min([angle_diff, 360 - angle_diff], axis=0)
roll_diff = Quat(tlm_q[ok]).roll - Quat(state_q[ok]).roll
roll_diff = np.min([roll_diff, 360 - roll_diff], axis=0)
for msid in MODE_SOURCE:
tlm_col = np.zeros(len(tlm))
state_col = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
tlm_col[tlm[MODE_SOURCE[msid]] == mode] = idx
state_col[state_vals[msid] == mode] = idx
tlm = Ska.Numpy.add_column(tlm, msid, tlm_col)
state_vals = Ska.Numpy.add_column(state_vals, "{}_pred".format(msid),
state_col)
for msid in ['letg', 'hetg']:
txt = np.repeat('RETR', len(tlm))
# use a combination of the select telemetry and the insertion telem to
# approximate the state_vals values
txt[(tlm['4ootgsel'] == msid.upper())
& (tlm['4ootgmtn'] == 'INSE')] = 'INSE'
tlm_col = np.zeros(len(tlm))
state_col = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
tlm_col[txt == mode] = idx
state_col[state_vals[msid] == mode] = idx
tlm = Ska.Numpy.add_column(tlm, msid, tlm_col)
state_vals = Ska.Numpy.add_column(state_vals, "{}_pred".format(msid),
state_col)
diff_only = {
'pointing': {
'diff': angle_diff * 3600,
'date': tlm['date'][ok]
},
'roll': {
'diff': roll_diff * 3600,
'date': tlm['date'][ok]
}
}
pred = {
'dp_pitch': state_vals.pitch,
'obsid': state_vals.obsid,
'dither': state_vals['dither_pred'],
'pcad_mode': state_vals['pcad_mode_pred'],
'letg': state_vals['letg_pred'],
'hetg': state_vals['hetg_pred'],
'tscpos': state_vals.simpos,
'power': state_vals.power,
'pointing': 1,
'roll': 1
}
plots_validation = []
valid_viols = []
logger.info('Making validation plots and quantile table')
quantiles = (1, 5, 16, 50, 84, 95, 99)
# store lines of quantile table in a string and write out later
quant_table = ''
quant_head = ",".join(['MSID'] + ["quant%d" % x for x in quantiles])
quant_table += quant_head + "\n"
for fig_id, msid in enumerate(sorted(pred)):
plot = dict(msid=msid.upper())
fig = plt.figure(10 + fig_id, figsize=(7, 3.5))
fig.clf()
scale = SCALES.get(msid, 1.0)
ax = None
if msid not in diff_only:
if msid in MODE_MSIDS:
state_msid = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
state_msid[state_vals[msid] == mode] = idx
ticklocs, fig, ax = plot_cxctime(tlm['date'],
tlm[msid],
fig=fig,
fmt='-r')
ticklocs, fig, ax = plot_cxctime(tlm['date'],
state_msid,
fig=fig,
fmt='-b')
plt.yticks(range(len(MODE_MSIDS[msid])), MODE_MSIDS[msid])
else:
ticklocs, fig, ax = plot_cxctime(tlm['date'],
tlm[msid] / scale,
fig=fig,
fmt='-r')
ticklocs, fig, ax = plot_cxctime(tlm['date'],
pred[msid] / scale,
fig=fig,
fmt='-b')
else:
ticklocs, fig, ax = plot_cxctime(diff_only[msid]['date'],
diff_only[msid]['diff'] / scale,
fig=fig,
fmt='-k')
plot['diff_only'] = msid in diff_only
ax.set_title(TITLE[msid])
ax.set_ylabel(LABELS[msid])
xlims = ax.get_xlim()
ylims = ax.get_ylim()
bad_times = list(characteristics.bad_times)
# Add the time intervals of dark current calibrations that have been excluded from
# the diffs to the "bad_times" for validation so they also can be marked with grey
# rectangles in the plot. This is only really visible with interactive/zoomed plot.
if msid in ['dither', 'pcad_mode']:
bad_times.extend(dark_times)
# Add "background" grey rectangles for excluded time regions to vs-time plot
for bad in bad_times:
bad_start = cxc2pd([DateTime(bad['start']).secs])[0]
bad_stop = cxc2pd([DateTime(bad['stop']).secs])[0]
if not ((bad_stop >= xlims[0]) & (bad_start <= xlims[1])):
continue
rect = matplotlib.patches.Rectangle((bad_start, ylims[0]),
bad_stop - bad_start,
ylims[1] - ylims[0],
alpha=.2,
facecolor='black',
edgecolor='none')
ax.add_patch(rect)
filename = msid + '_valid.png'
outfile = os.path.join(outdir, filename)
logger.info('Writing plot file %s' % outfile)
plt.tight_layout()
plt.margins(0.05)
fig.savefig(outfile)
plot['lines'] = filename
if msid not in diff_only:
ok = ~bad_time_mask
if msid in ['dither', 'pcad_mode']:
# For these two validations also ignore intervals during a dark current calibration
ok &= ~dark_mask
diff = tlm[msid][ok] - pred[msid][ok]
else:
diff = diff_only[msid]['diff']
# Sort the diffs in-place because we're just using them in aggregate
diff = np.sort(diff)
# if there are only a few residuals, don't bother with histograms
if msid.upper() in validation_scale_count:
plot['samples'] = len(diff)
plot['diff_count'] = np.count_nonzero(diff)
plot['n_changes'] = 1 + np.count_nonzero(pred[msid][1:] -
pred[msid][0:-1])
if (plot['diff_count'] <
(plot['n_changes'] * validation_scale_count[msid.upper()])):
plots_validation.append(plot)
continue
# if the msid exceeds the diff count, add a validation violation
else:
viol = {
'msid':
"{}_diff_count".format(msid),
'value':
plot['diff_count'],
'limit':
plot['n_changes'] * validation_scale_count[msid.upper()],
'quant':
None,
}
valid_viols.append(viol)
logger.info(
'WARNING: %s %d discrete diffs exceed limit of %d' %
(msid, plot['diff_count'],
plot['n_changes'] * validation_scale_count[msid.upper()]))
# Make quantiles
if (msid != 'obsid'):
quant_line = "%s" % msid
for quant in quantiles:
quant_val = diff[(len(diff) * quant) // 100]
plot['quant%02d' % quant] = FMTS[msid] % quant_val
quant_line += (',' + FMTS[msid] % quant_val)
quant_table += quant_line + "\n"
for histscale in ('lin', 'log'):
fig = plt.figure(20 + fig_id, figsize=(4, 3))
fig.clf()
ax = fig.gca()
ax.hist(diff / scale, bins=50, log=(histscale == 'log'))
ax.set_title(msid.upper() + ' residuals: telem - cmd states',
fontsize=11)
ax.set_xlabel(LABELS[msid])
fig.subplots_adjust(bottom=0.18)
plt.tight_layout()
filename = '%s_valid_hist_%s.png' % (msid, histscale)
outfile = os.path.join(outdir, filename)
logger.info('Writing plot file %s' % outfile)
fig.savefig(outfile)
plot['hist' + histscale] = filename
plots_validation.append(plot)
filename = os.path.join(outdir, 'validation_quant.csv')
logger.info('Writing quantile table %s' % filename)
f = open(filename, 'w')
f.write(quant_table)
f.close()
# If run_start_time is specified this is likely for regression testing
# or other debugging. In this case write out the full predicted and
# telemetered dataset as a pickle.
if opt.run_start_time:
filename = os.path.join(outdir, 'validation_data.pkl')
logger.info('Writing validation data %s' % filename)
f = open(filename, 'w')
pickle.dump({'pred': pred, 'tlm': tlm}, f, protocol=-1)
f.close()
valid_viols.extend(make_validation_viols(plots_validation))
if len(valid_viols) > 0:
# generate daily plot url if outdir in expected year/day format
daymatch = re.match('.*(\d{4})/(\d{3})', opt.outdir)
if daymatch:
url = os.path.join(URL, daymatch.group(1), daymatch.group(2))
logger.info('validation warning(s) at %s' % url)
else:
logger.info('validation warning(s) in output at %s' % opt.outdir)
write_index_rst(opt, proc, plots_validation, valid_viols)
rst_to_html(opt, proc)
def make_check_plots(outdir, states, times, temps, tstart, tstop, char):
"""
Make output plots.
:param opt: options
:param states: commanded states
:param times: time stamps (sec) for temperature arrays
:param temps: dict of temperatures
:param tstart: load start time
:rtype: dict of review information including plot file names
"""
plots = {}
# Start time of loads being reviewed expressed in units for plotdate()
load_start = cxc2pd([tstart])[0]
load_stop = cxc2pd([tstop])[0]
# Add labels for obsids
id_xs = [cxc2pd([states[0]['tstart']])[0]]
id_labels = [str(states[0]['obsid'])]
for s0, s1 in zip(states[:-1], states[1:]):
if s0['obsid'] != s1['obsid']:
id_xs.append(cxc2pd([s1['tstart']])[0])
id_labels.append(str(s1['obsid']))
logger.info('Making temperature check plots')
for fig_id, msid in enumerate(('aca',)):
temp_ymax = max(char['ccd_temp_red_limit'], np.max(temps))
temp_ymin = min(char['ccd_temp_yellow_limit'], np.min(temps))
plots[msid] = plot_two(fig_id=fig_id + 1,
x=times,
y=temps,
x2=pointpair(states['tstart'], states['tstop']),
y2=pointpair(states['pitch']),
xlabel='Date',
ylabel='Temperature (C)',
ylabel2='Pitch (deg)',
ylim=(temp_ymin - .05 * (temp_ymax - temp_ymin),
temp_ymax + .05 * (temp_ymax - temp_ymin)),
ylim2=(40, 180),
figsize=(9, 5),
)
ax = plots[msid]['ax']
plots[msid]['ax'].axhline(y=char['ccd_temp_yellow_limit'],
linestyle='--', color='g', linewidth=2.0)
plots[msid]['ax'].axhline(y=char['ccd_temp_red_limit'],
linestyle='--', color='r', linewidth=2.0)
plt.subplots_adjust(bottom=0.1)
pad = 1
lineid_plot.plot_line_ids([cxc2pd([times[0]])[0] - pad, cxc2pd([times[-1]])[0] + pad],
[ax.get_ylim()[0], ax.get_ylim()[0]],
id_xs, id_labels, box_axes_space=0.12,
ax=ax,
label1_size=7)
plt.tight_layout()
plt.subplots_adjust(top=.85)
xlims = ax.get_xlim()
ylims = ax.get_ylim()
pre_rect = matplotlib.patches.Rectangle((xlims[0], ylims[0]),
load_start - xlims[0],
ylims[1] - ylims[0],
alpha=.1,
facecolor='black',
edgecolor='none')
ax.add_patch(pre_rect)
post_rect = matplotlib.patches.Rectangle((load_stop, ylims[0]),
xlims[-1] - load_stop,
ylims[1] - ylims[0],
alpha=.1,
facecolor='black',
edgecolor='none')
ax.add_patch(post_rect)
filename = MSID_PLOT_NAME[msid]
outfile = os.path.join(outdir, filename)
logger.info('Writing plot file %s' % outfile)
plots[msid]['fig'].savefig(outfile)
plots[msid]['filename'] = filename
return plots
def main():
"""
Generate the Replan Central timeline plot.
"""
import matplotlib.patches
import matplotlib.pyplot as plt
from Ska.Matplotlib import plot_cxctime
# TODO: refactor this into smaller functions where possible.
# Basic setup. Set times and get input states, radzones and comms.
now = DateTime('2012:249:00:35:00' if args.test else None)
now = DateTime(now.date[:14] + ':00') # truncate to 0 secs
start = now - 1.0
stop = start + args.hours / 24.0
states = fetch_states(start, stop,
server='/proj/sot/ska/data/cmd_states/cmd_states.h5')
radzones = get_radzones()
comms = get_comms()
# Get the ACIS ops fluence estimate and current 2hr avg flux
fluence_date, fluence0 = get_fluence(ACIS_FLUENCE_FILE)
if fluence_date.secs < now.secs:
fluence_date = now
avg_flux = get_avg_flux(ACE_RATES_FILE)
# Get the realtime ACE P3 and HRC proxy values over the time range
goes_x_times, goes_x_vals = get_goes_x(start.secs, now.secs)
p3_times, p3_vals = get_ace_p3(start.secs, now.secs)
hrc_times, hrc_vals = get_hrc(start.secs, now.secs)
# For testing: inject predefined values for different scenarios
if args.test_scenario:
p3_vals, avg_flux, fluence0 = get_test_vals(
args.test_scenario, p3_times, p3_vals, avg_flux, fluence0)
# Compute the predicted fluence based on the current 2hr average flux.
fluence_times = np.arange(fluence_date.secs, stop.secs, args.dt)
rates = np.ones_like(fluence_times) * max(avg_flux, 0.0) * args.dt
fluence = calc_fluence(fluence_times, fluence0, rates, states)
zero_fluence_at_radzone(fluence_times, fluence, radzones)
# Initialize the main plot figure
plt.rc('legend', fontsize=10)
fig = plt.figure(1, figsize=(9, 5))
fig.clf()
fig.patch.set_alpha(0.0)
ax = fig.add_axes(AXES_LOC, axis_bgcolor='w')
ax.yaxis.tick_right()
ax.yaxis.set_label_position('right')
ax.yaxis.set_offset_position('right')
ax.patch.set_alpha(1.0)
# Plot lines at 1.0 and 2.0 (10^9) corresponding to fluence yellow
# and red limits. Also plot the fluence=0 line in black.
x0, x1 = cxc2pd([fluence_times[0], fluence_times[-1]])
plt.plot([x0, x1], [0.0, 0.0], '-k')
plt.plot([x0, x1], [1.0, 1.0], '--b', lw=2.0)
plt.plot([x0, x1], [2.0, 2.0], '--r', lw=2.0)
# Draw dummy lines off the plot for the legend
lx = [fluence_times[0], fluence_times[-1]]
ly = [-1, -1]
plot_cxctime(lx, ly, '-k', lw=3, label='None', fig=fig, ax=ax)
plot_cxctime(lx, ly, '-r', lw=3, label='HETG', fig=fig, ax=ax)
plot_cxctime(lx, ly, '-c', lw=3, label='LETG', fig=fig, ax=ax)
# Make a z-valued curve where the z value corresponds to the grating state.
x = cxc2pd(fluence_times)
y = fluence
z = np.zeros(len(fluence_times), dtype=np.int)
for state in states:
ok = ((state['tstart'] < fluence_times)
& (fluence_times <= state['tstop']))
if state['hetg'] == 'INSR':
z[ok] = 1
elif state['letg'] == 'INSR':
z[ok] = 2
plot_multi_line(x, y, z, [0, 1, 2], ['k', 'r', 'c'], ax)
# Plot 10, 50, 90 percentiles of fluence
p3_slope = get_p3_slope(p3_times, p3_vals)
if p3_slope is not None and avg_flux > 0:
p3_fits, p3_samps, fluences = cfd.get_fluences(
os.path.join(args.data_dir, 'ACE_hourly_avg.npy'))
hrs, fl10, fl50, fl90 = cfd.get_fluence_percentiles(
avg_flux, p3_slope, p3_fits, p3_samps, fluences,
args.min_flux_samples, args.max_slope_samples)
fluence_hours = (fluence_times - fluence_times[0]) / 3600.0
for fl_y, linecolor in zip((fl10, fl50, fl90),
('-g', '-b', '-r')):
fl_y = Ska.Numpy.interpolate(fl_y, hrs, fluence_hours)
rates = np.diff(fl_y)
fl_y_atten = calc_fluence(fluence_times[:-1], fluence0, rates, states)
zero_fluence_at_radzone(fluence_times[:-1], fl_y_atten, radzones)
plt.plot(x0 + fluence_hours[:-1] / 24.0, fl_y_atten, linecolor)
# Set x and y axis limits
x0, x1 = cxc2pd([start.secs, stop.secs])
plt.xlim(x0, x1)
y0 = -0.45
y1 = 2.55
plt.ylim(y0, y1)
id_xs = []
id_labels = []
# Draw comm passes
next_comm = None
for comm in comms:
t0 = DateTime(comm['bot_date']['value']).secs
t1 = DateTime(comm['eot_date']['value']).secs
pd0, pd1 = cxc2pd([t0, t1])
if pd1 >= x0 and pd0 <= x1:
p = matplotlib.patches.Rectangle((pd0, y0),
pd1 - pd0,
y1 - y0,
alpha=0.2,
facecolor='r',
edgecolor='none')
ax.add_patch(p)
id_xs.append((pd0 + pd1) / 2)
id_labels.append('{}:{}'.format(comm['station']['value'][4:6],
comm['track_local']['value'][:9]))
if (next_comm is None and DateTime(comm['bot_date']['value']).secs > now.secs):
next_comm = comm
# Draw radiation zones
for rad0, rad1 in radzones:
t0 = DateTime(rad0).secs
t1 = DateTime(rad1).secs
if t0 < stop.secs and t1 > start.secs:
if t0 < start.secs:
t0 = start.secs
if t1 > stop.secs:
t1 = stop.secs
pd0, pd1 = cxc2pd([t0, t1])
p = matplotlib.patches.Rectangle((pd0, y0),
pd1 - pd0,
y1 - y0,
alpha=0.2,
facecolor='b',
edgecolor='none')
ax.add_patch(p)
# Draw now line
plt.plot(cxc2pd([now.secs, now.secs]), [y0, y1], '-g', lw=4)
id_xs.extend(cxc2pd([now.secs]))
id_labels.append('NOW')
# Add labels for obsids
id_xs.extend(cxc2pd([start.secs]))
id_labels.append(str(states[0]['obsid']))
for s0, s1 in zip(states[:-1], states[1:]):
if s0['obsid'] != s1['obsid']:
id_xs.append(cxc2pd([s1['tstart']])[0])
id_labels.append(str(s1['obsid']))
plt.grid()
plt.ylabel('Attenuated fluence / 1e9')
plt.legend(loc='upper center', labelspacing=0.15)
lineid_plot.plot_line_ids(cxc2pd([start.secs, stop.secs]),
[y1, y1],
id_xs, id_labels, ax=ax,
box_axes_space=0.14,
label1_size=10)
# Plot observed GOES X-ray rates and limits
pd = cxc2pd(goes_x_times)
lgoesx = log_scale(goes_x_vals * 1e8)
plt.plot(pd, lgoesx, '-m', alpha=0.3, lw=1.5)
plt.plot(pd, lgoesx, '.m', mec='m', ms=3)
# Plot observed ACE P3 rates and limits
lp3 = log_scale(p3_vals)
pd = cxc2pd(p3_times)
ox = cxc2pd([start.secs, now.secs])
oy1 = log_scale(12000.)
plt.plot(ox, [oy1, oy1], '--b', lw=2)
oy1 = log_scale(55000.)
plt.plot(ox, [oy1, oy1], '--r', lw=2)
plt.plot(pd, lp3, '-k', alpha=0.3, lw=3)
plt.plot(pd, lp3, '.k', mec='k', ms=3)
# Plot observed HRC shield proxy rates and limits
pd = cxc2pd(hrc_times)
lhrc = log_scale(hrc_vals)
plt.plot(pd, lhrc, '-c', alpha=0.3, lw=3)
plt.plot(pd, lhrc, '.c', mec='c', ms=3)
# Draw SI state
times = np.arange(start.secs, stop.secs, 300)
state_vals = interpolate_states(states, times)
y_si = -0.23
x = cxc2pd(times)
y = np.zeros_like(times) + y_si
z = np.zeros_like(times, dtype=np.float) # 0 => ACIS
z[state_vals['simpos'] < 0] = 1.0 # HRC
plot_multi_line(x, y, z, [0, 1], ['c', 'r'], ax)
dx = (x1 - x0) * 0.01
plt.text(x1 + dx, y_si, 'HRC/ACIS',
ha='left', va='center', size='small')
# Draw log scale y-axis on left
ax2 = fig.add_axes(AXES_LOC, axis_bgcolor='w',
frameon=False)
ax2.set_autoscale_on(False)
ax2.xaxis.set_visible(False)
ax2.set_xlim(0, 1)
ax2.set_yscale('log')
ax2.set_ylim(np.power(10.0, np.array([y0, y1]) * 2 + 1))
ax2.set_ylabel('ACE flux / HRC proxy / GOES X-ray')
ax2.text(-0.015, 2.5e3, 'M', ha='right', color='m', weight='demibold')
ax2.text(-0.015, 2.5e4, 'X', ha='right', color='m', weight='semibold')
# Draw dummy lines off the plot for the legend
lx = [0, 1]
ly = [1, 1]
ax2.plot(lx, ly, '-k', lw=3, label='ACE')
ax2.plot(lx, ly, '-c', lw=3, label='HRC')
ax2.plot(lx, ly, '-m', lw=3, label='GOES-X')
ax2.legend(loc='upper left', labelspacing=0.15)
plt.draw()
plt.savefig(os.path.join(args.data_dir, 'timeline.png'))
write_states_json(os.path.join(args.data_dir, 'timeline_states.js'),
fig, ax, states, start, stop, now,
next_comm,
fluence, fluence_times,
p3_vals, p3_times, avg_flux,
hrc_vals, hrc_times)
def write_states_json(fn, fig, ax, states, start, stop, now,
next_comm,
fluences, fluence_times,
p3s, p3_times, p3_avg,
hrcs, hrc_times):
"""
Generate JSON data file that contains all the annotation values used in the
javascript-driven annotated plot on Replan Central. This creates a data structure
with state values for each 10-minute time step along the X-axis of the plot. All of
the hard work (formatting etc) is done here so the javascript is very simple.
"""
formats = {'ra': '{:10.4f}',
'dec': '{:10.4f}',
'roll': '{:10.4f}',
'pitch': '{:8.2f}',
'obsid': '{:5d}',
}
start = start - 1
tstop = (stop + 1).secs
tstart = DateTime(start.date[:8] + ':00:00:00').secs
times = np.arange(tstart, tstop, 600)
pds = cxc2pd(times) # Convert from CXC time to plotdate times
# Set up matplotlib transforms
data_to_disp = ax.transData.transform
ax_to_disp = ax.transAxes.transform
disp_to_ax = ax.transAxes.inverted().transform
disp_to_fig = fig.transFigure.inverted().transform
disp_xy = ax_to_disp([(0, 0), (1, 1)])
fig_xy = disp_to_fig(disp_xy)
data = {'ax_x': fig_xy[:, 0].tolist(),
'ax_y': fig_xy[:, 1].tolist()}
outs = []
now_idx = 0
now_secs = now.secs
state_names = ('obsid', 'simpos', 'pitch', 'ra', 'dec', 'roll',
'pcad_mode', 'si_mode', 'power_cmd', 'letg', 'hetg')
# Get all the state values that occur within the range of the plot
disp_xy = data_to_disp([(pd, 0.0) for pd in pds])
ax_xy = disp_to_ax(disp_xy)
ok = (ax_xy[:, 0] > 0.0) & (ax_xy[:, 0] < 1.0)
times = times[ok]
pds = pds[ok]
state_vals = interpolate_states(states, times)
# Set the current values
p3_now = p3s[-1]
hrc_now = hrcs[-1]
fluence_now = fluences[0]
fluences = Ska.Numpy.interpolate(fluences, fluence_times, times)
p3s = Ska.Numpy.interpolate(p3s, p3_times, times)
hrcs = Ska.Numpy.interpolate(hrcs, hrc_times, times)
# Iterate through each time step and create corresponding data structure
# with pre-formatted values for display in the output table.
NOT_AVAIL = 'N/A'
for time, pd, state_val, fluence, p3, hrc in izip(times, pds, state_vals,
fluences, p3s, hrcs):
out = {}
out['date'] = date_zulu(time)
for name in state_names:
val = state_val[name].tolist()
fval = formats.get(name, '{}').format(val)
out[name] = re.sub(' ', ' ', fval)
out['ccd_fep'] = '{}, {}'.format(state_val['ccd_count'],
state_val['fep_count'])
out['vid_clock'] = '{}, {}'.format(state_val['vid_board'],
state_val['clocking'])
out['si'] = get_si(state_val['simpos'])
out['now_dt'] = get_fmt_dt(time, now_secs)
if time < now_secs:
now_idx += 1
out['fluence'] = '{:.2f}e9'.format(fluence_now)
out['p3'] = '{:.0f}'.format(p3) if p3 > 0 else NOT_AVAIL
out['hrc'] = '{:.0f}'.format(hrc)
else:
out['fluence'] = '{:.2f}e9'.format(fluence)
out['p3'] = '{:.0f}'.format(p3_now) if p3_now > 0 else NOT_AVAIL
out['hrc'] = '{:.0f}'.format(hrc_now)
outs.append(out)
data['states'] = outs
data['now_idx'] = now_idx
data['now_date'] = date_zulu(now)
data['p3_avg_now'] = '{:.0f}'.format(p3_avg) if p3_avg > 0 else NOT_AVAIL
data['p3_now'] = '{:.0f}'.format(p3_now) if p3_now > 0 else NOT_AVAIL
data['hrc_now'] = '{:.0f}'.format(hrc_now)
track = next_comm['track_local']['value']
data['track_time'] = (' ' + track[15:19] + track[:4]
+ ' ' + track[10:13])
data['track_dt'] = get_fmt_dt(next_comm['bot_date']['value'], now_secs)
data['track_station'] = '{}-{}'.format(next_comm['site']['value'],
next_comm['station']['value'][4:6])
data['track_activity'] = next_comm['activity']['value'][:14]
# Finally write this all out as a simple javascript program that defines a single
# variable ``data``.
with open(fn, 'w') as f:
f.write('var data = {}'.format(json.dumps(data)))
def main(opt):
opt, args = get_options()
if not os.path.exists(opt.outdir):
os.mkdir(opt.outdir)
config_logging(opt.outdir, opt.verbose)
# Store info relevant to processing for use in outputs
proc = dict(run_user=os.environ['USER'],
run_time=time.ctime(),
errors=[],
)
logger.info('#####################################################################')
logger.info('# %s run at %s by %s' % (os.path.dirname(__file__),
proc['run_time'], proc['run_user']))
logger.info('# version = %s' % VERSION)
logger.info('# characteristics version = %s' % characteristics.VERSION)
logger.info('#####################################################################\n')
logger.info('Command line options:\n%s\n' % pformat(opt.__dict__))
# Connect to database (NEED TO USE aca_read)
tnow = DateTime(opt.run_start_time).secs
tstart = tnow
# Get temperature telemetry for 3 weeks prior to min(tstart, NOW)
tlm = get_telem_values(tstart,
['sim_z', 'dp_pitch', 'aoacaseq',
'aodithen', 'cobsrqid', 'aofunlst',
'aopcadmd', '4ootgsel', '4ootgmtn',
'aocmdqt1', 'aocmdqt2', 'aocmdqt3',
'1de28avo', '1deicacu',
'1dp28avo', '1dpicacu',
'1dp28bvo', '1dpicbcu'],
days=opt.days,
name_map={'sim_z': 'tscpos',
'cobsrqid': 'obsid'})
tlm['tscpos'] = tlm['tscpos'] * -397.7225924607
outdir = opt.outdir
states = get_states(tlm[0].date, tlm[-1].date)
write_states(opt, states)
tlm = Ska.Numpy.add_column(tlm, 'power', smoothed_power(tlm))
# Get bad time intervals
bad_time_mask = get_bad_mask(tlm)
# Interpolate states onto the tlm.date grid
state_vals = cmd_states.interpolate_states(states, tlm['date'])
# Calculate the 4th term of the commanded quaternions
cmd_q4 = np.sqrt(np.abs(1.0
- tlm['aocmdqt1']**2
- tlm['aocmdqt2']**2
- tlm['aocmdqt3']**2))
raw_tlm_q = np.vstack([tlm['aocmdqt1'],
tlm['aocmdqt2'],
tlm['aocmdqt3'],
cmd_q4]).transpose()
# Calculate angle/roll differences in state cmd vs tlm cmd quaternions
raw_state_q = np.vstack([state_vals[n] for n
in ['q1', 'q2', 'q3', 'q4']]).transpose()
tlm_q = normalize(raw_tlm_q)
# only use values that aren't NaNs
good = np.isnan(np.sum(tlm_q, axis=-1)) == False
# and are in NPNT
npnt = tlm['aopcadmd'] == 'NPNT'
# and are in KALM after the first 2 sample of the transition
not_kalm = tlm['aoacaseq'] != 'KALM'
kalm = (not_kalm | np.hstack([[False, False], not_kalm[:-2]])) == False
# and aren't during momentum unloads or in the first 2 samples after unloads
unload = tlm['aofunlst'] != 'NONE'
no_unload = (unload | np.hstack([[False, False], unload[:-2]])) == False
ok = good & npnt & kalm & no_unload & ~bad_time_mask
state_q = normalize(raw_state_q)
dot_q = np.sum(tlm_q[ok] * state_q[ok], axis=-1)
dot_q[dot_q > 1] = 1
angle_diff = np.degrees(2 * np.arccos(dot_q))
angle_diff = np.min([angle_diff, 360 - angle_diff], axis=0)
roll_diff = Quat(tlm_q[ok]).roll - Quat(state_q[ok]).roll
roll_diff = np.min([roll_diff, 360 - roll_diff], axis=0)
for msid in MODE_SOURCE:
tlm_col = np.zeros(len(tlm))
state_col = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
tlm_col[tlm[MODE_SOURCE[msid]] == mode] = idx
state_col[state_vals[msid] == mode] = idx
tlm = Ska.Numpy.add_column(tlm, msid, tlm_col)
state_vals = Ska.Numpy.add_column(state_vals,
"{}_pred".format(msid), state_col)
for msid in ['letg', 'hetg']:
txt = np.repeat('RETR', len(tlm))
# use a combination of the select telemetry and the insertion telem to
# approximate the state_vals values
txt[(tlm['4ootgsel'] == msid.upper())
& (tlm['4ootgmtn'] == 'INSE')] = 'INSE'
tlm_col = np.zeros(len(tlm))
state_col = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
tlm_col[txt == mode] = idx
state_col[state_vals[msid] == mode] = idx
tlm = Ska.Numpy.add_column(tlm, msid, tlm_col)
state_vals = Ska.Numpy.add_column(state_vals,
"{}_pred".format(msid), state_col)
diff_only = {'pointing': {'diff': angle_diff * 3600,
'date': tlm['date'][ok]},
'roll': {'diff': roll_diff * 3600,
'date': tlm['date'][ok]}}
pred = {'dp_pitch': state_vals.pitch,
'obsid': state_vals.obsid,
'dither': state_vals['dither_pred'],
'pcad_mode': state_vals['pcad_mode_pred'],
'letg': state_vals['letg_pred'],
'hetg': state_vals['hetg_pred'],
'tscpos': state_vals.simpos,
'power': state_vals.power,
'pointing': 1,
'roll': 1}
plots_validation = []
valid_viols = []
logger.info('Making validation plots and quantile table')
quantiles = (1, 5, 16, 50, 84, 95, 99)
# store lines of quantile table in a string and write out later
quant_table = ''
quant_head = ",".join(['MSID'] + ["quant%d" % x for x in quantiles])
quant_table += quant_head + "\n"
for fig_id, msid in enumerate(sorted(pred)):
plot = dict(msid=msid.upper())
fig = plt.figure(10 + fig_id, figsize=(7, 3.5))
fig.clf()
scale = SCALES.get(msid, 1.0)
ax = None
if msid not in diff_only:
if msid in MODE_MSIDS:
state_msid = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
state_msid[state_vals[msid] == mode] = idx
ticklocs, fig, ax = plot_cxctime(tlm['date'],
tlm[msid], fig=fig, fmt='-r')
ticklocs, fig, ax = plot_cxctime(tlm['date'],
state_msid, fig=fig, fmt='-b')
plt.yticks(range(len(MODE_MSIDS[msid])), MODE_MSIDS[msid])
else:
ticklocs, fig, ax = plot_cxctime(tlm['date'],
tlm[msid] / scale, fig=fig, fmt='-r')
ticklocs, fig, ax = plot_cxctime(tlm['date'],
pred[msid] / scale, fig=fig, fmt='-b')
else:
ticklocs, fig, ax = plot_cxctime(diff_only[msid]['date'],
diff_only[msid]['diff'] / scale, fig=fig, fmt='-k')
plot['diff_only'] = msid in diff_only
ax.set_title(TITLE[msid])
ax.set_ylabel(LABELS[msid])
xlims = ax.get_xlim()
ylims = ax.get_ylim()
for bad in characteristics.bad_times:
bad_start = cxc2pd([DateTime(bad['start']).secs])[0]
bad_stop = cxc2pd([DateTime(bad['stop']).secs])[0]
if not ((bad_stop >= xlims[0]) & (bad_start <= xlims[1])):
continue
rect = matplotlib.patches.Rectangle((bad_start, ylims[0]),
bad_stop - bad_start,
ylims[1] - ylims[0],
alpha=.2,
facecolor='black',
edgecolor='none')
ax.add_patch(rect)
filename = msid + '_valid.png'
outfile = os.path.join(outdir, filename)
logger.info('Writing plot file %s' % outfile)
plt.tight_layout()
fig.savefig(outfile)
plot['lines'] = filename
if msid not in diff_only:
diff = tlm[msid][~bad_time_mask] - pred[msid][~bad_time_mask]
diff = np.sort(diff)
else:
diff = np.sort(diff_only[msid]['diff'])
# if there are only a few residuals, don't bother with histograms
if msid.upper() in validation_scale_count:
plot['samples'] = len(diff)
plot['diff_count'] = np.count_nonzero(diff)
plot['n_changes'] = 1 + np.count_nonzero(pred[msid][1:] - pred[msid][0:-1])
if (plot['diff_count'] <
(plot['n_changes'] * validation_scale_count[msid.upper()])):
plots_validation.append(plot)
continue
# if the msid exceeds the diff count, add a validation violation
else:
viol = {'msid': "{}_diff_count".format(msid),
'value': plot['diff_count'],
'limit': plot['n_changes'] * validation_scale_count[msid.upper()],
'quant': None,
}
valid_viols.append(viol)
logger.info('WARNING: %s %d discrete diffs exceed limit of %d' %
(msid, plot['diff_count'],
plot['n_changes'] * validation_scale_count[msid.upper()]))
# Make quantiles
if (msid != 'obsid'):
quant_line = "%s" % msid
for quant in quantiles:
quant_val = diff[(len(diff) * quant) // 100]
plot['quant%02d' % quant] = FMTS[msid] % quant_val
quant_line += (',' + FMTS[msid] % quant_val)
quant_table += quant_line + "\n"
for histscale in ('lin', 'log'):
fig = plt.figure(20 + fig_id, figsize=(4, 3))
fig.clf()
ax = fig.gca()
ax.hist(diff / scale, bins=50, log=(histscale == 'log'))
ax.set_title(msid.upper() + ' residuals: telem - cmd states', fontsize=11)
ax.set_xlabel(LABELS[msid])
fig.subplots_adjust(bottom=0.18)
plt.tight_layout()
filename = '%s_valid_hist_%s.png' % (msid, histscale)
outfile = os.path.join(outdir, filename)
logger.info('Writing plot file %s' % outfile)
fig.savefig(outfile)
plot['hist' + histscale] = filename
plots_validation.append(plot)
filename = os.path.join(outdir, 'validation_quant.csv')
logger.info('Writing quantile table %s' % filename)
f = open(filename, 'w')
f.write(quant_table)
f.close()
# If run_start_time is specified this is likely for regression testing
# or other debugging. In this case write out the full predicted and
# telemetered dataset as a pickle.
if opt.run_start_time:
filename = os.path.join(outdir, 'validation_data.pkl')
logger.info('Writing validation data %s' % filename)
f = open(filename, 'w')
pickle.dump({'pred': pred, 'tlm': tlm}, f, protocol=-1)
f.close()
valid_viols.extend(make_validation_viols(plots_validation))
if len(valid_viols) > 0:
# generate daily plot url if outdir in expected year/day format
daymatch = re.match('.*(\d{4})/(\d{3})', opt.outdir)
if daymatch:
url = os.path.join(URL, daymatch.group(1), daymatch.group(2))
logger.info('validation warning(s) at %s' % url)
else:
logger.info('validation warning(s) in output at %s' % opt.outdir)
write_index_rst(opt, proc, plots_validation, valid_viols)
rst_to_html(opt, proc)