def get_q_atts_transforms(telems, slot, dt):
"""
Get quaternions and associated transforms, matched to the times of yag/zag data
in slot. Apply a time offset ``dt`` to account for latencies in telemetry
and ACA image readout.
"""
logger.verbose('Interpolating quaternions for slot {}'.format(slot))
yz_times = telems['aoacyan{}'.format(slot)].times
q_times = telems['aoattqt1'].times
qs = np.empty((len(yz_times), 4), dtype=np.float64)
for ii in range(4):
q_vals = telems['aoattqt{}'.format(ii + 1)].vals
qs[:, ii] = Ska.Numpy.interpolate(q_vals, q_times + dt, yz_times, sorted=True)
q_atts = quaternion.Quat(quaternion.normalize(qs))
transforms = q_atts.transform # N x 3 x 3
return q_atts, transforms
def consecutive(data, stepsize=1):
return np.split(data, np.where(np.diff(data) != stepsize)[0] + 1)
ds = events.dwells.filter(start='2008:007')
#ds = events.dwells.filter(obsid=17321)
low_obsids = []
for d in ds:
obsid = d.get_obsid()
print "obsid {} start {}".format(obsid, d.manvr.start)
pcad_data = get_pcad(d)
q_atts = Quat(
normalize(
np.column_stack([
pcad_data['AOATTQT1'].vals, pcad_data['AOATTQT2'].vals,
pcad_data['AOATTQT3'].vals, pcad_data['AOATTQT4'].vals
])))
try:
cat = get_catalog(obsid, d.manvr.start)
except (ValueError, IndexError) as e:
print "Skipping {} {}".format(obsid, e)
continue
gcat = cat['cat'][(cat['cat']['type'] == 'BOT') |
(cat['cat']['type'] == 'GUI')]
yag_offs = np.zeros((len(pcad_data['AOKALSTR'].vals), len(gcat)))
zag_offs = np.zeros((len(pcad_data['AOKALSTR'].vals), len(gcat)))
slot_ok_old = {}
slot_ok_new = {}
for idx, entry in enumerate(gcat):
slot = entry['slot']
def main(opt):
opt, args = get_options()
if not os.path.exists(opt.outdir):
os.mkdir(opt.outdir)
config_logging(opt.outdir, opt.verbose)
# Store info relevant to processing for use in outputs
proc = dict(
run_user=os.environ['USER'],
run_time=time.ctime(),
errors=[],
)
logger.info(
'#####################################################################'
)
logger.info(
'# %s run at %s by %s' %
(os.path.dirname(__file__), proc['run_time'], proc['run_user']))
logger.info('# version = %s' % VERSION)
logger.info('# characteristics version = %s' % characteristics.VERSION)
logger.info(
'#####################################################################\n'
)
logger.info('Command line options:\n%s\n' % pformat(opt.__dict__))
# Connect to database (NEED TO USE aca_read)
tnow = DateTime(opt.run_start_time).secs
tstart = tnow
# Get temperature telemetry for 3 weeks prior to min(tstart, NOW)
tlm = get_telem_values(tstart, [
'sim_z', 'dp_pitch', 'aoacaseq', 'aodithen', 'cacalsta', 'cobsrqid',
'aofunlst', 'aopcadmd', '4ootgsel', '4ootgmtn', 'aocmdqt1', 'aocmdqt2',
'aocmdqt3', '1de28avo', '1deicacu', '1dp28avo', '1dpicacu', '1dp28bvo',
'1dpicbcu'
],
days=opt.days,
name_map={
'sim_z': 'tscpos',
'cobsrqid': 'obsid'
})
tlm['tscpos'] = tlm['tscpos'] * -397.7225924607
outdir = opt.outdir
states = get_states(tlm[0].date, tlm[-1].date)
write_states(opt, states)
tlm = Ska.Numpy.add_column(tlm, 'power', smoothed_power(tlm))
# Get bad time intervals
bad_time_mask = get_bad_mask(tlm)
# Interpolate states onto the tlm.date grid
state_vals = cmd_states.interpolate_states(states, tlm['date'])
# "Forgive" dither intervals with dark current replicas
# This will also exclude dither disables that are in cmd states for standard dark cals
dark_mask = np.zeros(len(tlm), dtype='bool')
dark_times = []
# Find dither "disable" states from tlm
dith_disa_states = logical_intervals(tlm['date'],
tlm['aodithen'] == 'DISA')
for state in dith_disa_states:
# Index back into telemetry for each of these constant dither disable states
idx0 = np.searchsorted(tlm['date'], state['tstart'], side='left')
idx1 = np.searchsorted(tlm['date'], state['tstop'], side='right')
# If any samples have aca calibration flag, mark interval for exclusion.
if np.any(tlm['cacalsta'][idx0:idx1] != 'OFF '):
dark_mask[idx0:idx1] = True
dark_times.append({
'start': state['datestart'],
'stop': state['datestop']
})
# Calculate the 4th term of the commanded quaternions
cmd_q4 = np.sqrt(
np.abs(1.0 - tlm['aocmdqt1']**2 - tlm['aocmdqt2']**2 -
tlm['aocmdqt3']**2))
raw_tlm_q = np.vstack(
[tlm['aocmdqt1'], tlm['aocmdqt2'], tlm['aocmdqt3'],
cmd_q4]).transpose()
# Calculate angle/roll differences in state cmd vs tlm cmd quaternions
raw_state_q = np.vstack([state_vals[n]
for n in ['q1', 'q2', 'q3', 'q4']]).transpose()
tlm_q = normalize(raw_tlm_q)
# only use values that aren't NaNs
good = np.isnan(np.sum(tlm_q, axis=-1)) == False
# and are in NPNT
npnt = tlm['aopcadmd'] == 'NPNT'
# and are in KALM after the first 2 sample of the transition
not_kalm = tlm['aoacaseq'] != 'KALM'
kalm = (not_kalm | np.hstack([[False, False], not_kalm[:-2]])) == False
# and aren't during momentum unloads or in the first 2 samples after unloads
unload = tlm['aofunlst'] != 'NONE'
no_unload = (unload | np.hstack([[False, False], unload[:-2]])) == False
ok = good & npnt & kalm & no_unload & ~bad_time_mask
state_q = normalize(raw_state_q)
dot_q = np.sum(tlm_q[ok] * state_q[ok], axis=-1)
dot_q[dot_q > 1] = 1
angle_diff = np.degrees(2 * np.arccos(dot_q))
angle_diff = np.min([angle_diff, 360 - angle_diff], axis=0)
roll_diff = Quat(tlm_q[ok]).roll - Quat(state_q[ok]).roll
roll_diff = np.min([roll_diff, 360 - roll_diff], axis=0)
for msid in MODE_SOURCE:
tlm_col = np.zeros(len(tlm))
state_col = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
tlm_col[tlm[MODE_SOURCE[msid]] == mode] = idx
state_col[state_vals[msid] == mode] = idx
tlm = Ska.Numpy.add_column(tlm, msid, tlm_col)
state_vals = Ska.Numpy.add_column(state_vals, "{}_pred".format(msid),
state_col)
for msid in ['letg', 'hetg']:
txt = np.repeat('RETR', len(tlm))
# use a combination of the select telemetry and the insertion telem to
# approximate the state_vals values
txt[(tlm['4ootgsel'] == msid.upper())
& (tlm['4ootgmtn'] == 'INSE')] = 'INSE'
tlm_col = np.zeros(len(tlm))
state_col = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
tlm_col[txt == mode] = idx
state_col[state_vals[msid] == mode] = idx
tlm = Ska.Numpy.add_column(tlm, msid, tlm_col)
state_vals = Ska.Numpy.add_column(state_vals, "{}_pred".format(msid),
state_col)
diff_only = {
'pointing': {
'diff': angle_diff * 3600,
'date': tlm['date'][ok]
},
'roll': {
'diff': roll_diff * 3600,
'date': tlm['date'][ok]
}
}
pred = {
'dp_pitch': state_vals.pitch,
'obsid': state_vals.obsid,
'dither': state_vals['dither_pred'],
'pcad_mode': state_vals['pcad_mode_pred'],
'letg': state_vals['letg_pred'],
'hetg': state_vals['hetg_pred'],
'tscpos': state_vals.simpos,
'power': state_vals.power,
'pointing': 1,
'roll': 1
}
plots_validation = []
valid_viols = []
logger.info('Making validation plots and quantile table')
quantiles = (1, 5, 16, 50, 84, 95, 99)
# store lines of quantile table in a string and write out later
quant_table = ''
quant_head = ",".join(['MSID'] + ["quant%d" % x for x in quantiles])
quant_table += quant_head + "\n"
for fig_id, msid in enumerate(sorted(pred)):
plot = dict(msid=msid.upper())
fig = plt.figure(10 + fig_id, figsize=(7, 3.5))
fig.clf()
scale = SCALES.get(msid, 1.0)
ax = None
if msid not in diff_only:
if msid in MODE_MSIDS:
state_msid = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
state_msid[state_vals[msid] == mode] = idx
ticklocs, fig, ax = plot_cxctime(tlm['date'],
tlm[msid],
fig=fig,
fmt='-r')
ticklocs, fig, ax = plot_cxctime(tlm['date'],
state_msid,
fig=fig,
fmt='-b')
plt.yticks(range(len(MODE_MSIDS[msid])), MODE_MSIDS[msid])
else:
ticklocs, fig, ax = plot_cxctime(tlm['date'],
tlm[msid] / scale,
fig=fig,
fmt='-r')
ticklocs, fig, ax = plot_cxctime(tlm['date'],
pred[msid] / scale,
fig=fig,
fmt='-b')
else:
ticklocs, fig, ax = plot_cxctime(diff_only[msid]['date'],
diff_only[msid]['diff'] / scale,
fig=fig,
fmt='-k')
plot['diff_only'] = msid in diff_only
ax.set_title(TITLE[msid])
ax.set_ylabel(LABELS[msid])
xlims = ax.get_xlim()
ylims = ax.get_ylim()
bad_times = list(characteristics.bad_times)
# Add the time intervals of dark current calibrations that have been excluded from
# the diffs to the "bad_times" for validation so they also can be marked with grey
# rectangles in the plot. This is only really visible with interactive/zoomed plot.
if msid in ['dither', 'pcad_mode']:
bad_times.extend(dark_times)
# Add "background" grey rectangles for excluded time regions to vs-time plot
for bad in bad_times:
bad_start = cxc2pd([DateTime(bad['start']).secs])[0]
bad_stop = cxc2pd([DateTime(bad['stop']).secs])[0]
if not ((bad_stop >= xlims[0]) & (bad_start <= xlims[1])):
continue
rect = matplotlib.patches.Rectangle((bad_start, ylims[0]),
bad_stop - bad_start,
ylims[1] - ylims[0],
alpha=.2,
facecolor='black',
edgecolor='none')
ax.add_patch(rect)
filename = msid + '_valid.png'
outfile = os.path.join(outdir, filename)
logger.info('Writing plot file %s' % outfile)
plt.tight_layout()
plt.margins(0.05)
fig.savefig(outfile)
plot['lines'] = filename
if msid not in diff_only:
ok = ~bad_time_mask
if msid in ['dither', 'pcad_mode']:
# For these two validations also ignore intervals during a dark current calibration
ok &= ~dark_mask
diff = tlm[msid][ok] - pred[msid][ok]
else:
diff = diff_only[msid]['diff']
# Sort the diffs in-place because we're just using them in aggregate
diff = np.sort(diff)
# if there are only a few residuals, don't bother with histograms
if msid.upper() in validation_scale_count:
plot['samples'] = len(diff)
plot['diff_count'] = np.count_nonzero(diff)
plot['n_changes'] = 1 + np.count_nonzero(pred[msid][1:] -
pred[msid][0:-1])
if (plot['diff_count'] <
(plot['n_changes'] * validation_scale_count[msid.upper()])):
plots_validation.append(plot)
continue
# if the msid exceeds the diff count, add a validation violation
else:
viol = {
'msid':
"{}_diff_count".format(msid),
'value':
plot['diff_count'],
'limit':
plot['n_changes'] * validation_scale_count[msid.upper()],
'quant':
None,
}
valid_viols.append(viol)
logger.info(
'WARNING: %s %d discrete diffs exceed limit of %d' %
(msid, plot['diff_count'],
plot['n_changes'] * validation_scale_count[msid.upper()]))
# Make quantiles
if (msid != 'obsid'):
quant_line = "%s" % msid
for quant in quantiles:
quant_val = diff[(len(diff) * quant) // 100]
plot['quant%02d' % quant] = FMTS[msid] % quant_val
quant_line += (',' + FMTS[msid] % quant_val)
quant_table += quant_line + "\n"
for histscale in ('lin', 'log'):
fig = plt.figure(20 + fig_id, figsize=(4, 3))
fig.clf()
ax = fig.gca()
ax.hist(diff / scale, bins=50, log=(histscale == 'log'))
ax.set_title(msid.upper() + ' residuals: telem - cmd states',
fontsize=11)
ax.set_xlabel(LABELS[msid])
fig.subplots_adjust(bottom=0.18)
plt.tight_layout()
filename = '%s_valid_hist_%s.png' % (msid, histscale)
outfile = os.path.join(outdir, filename)
logger.info('Writing plot file %s' % outfile)
fig.savefig(outfile)
plot['hist' + histscale] = filename
plots_validation.append(plot)
filename = os.path.join(outdir, 'validation_quant.csv')
logger.info('Writing quantile table %s' % filename)
f = open(filename, 'w')
f.write(quant_table)
f.close()
# If run_start_time is specified this is likely for regression testing
# or other debugging. In this case write out the full predicted and
# telemetered dataset as a pickle.
if opt.run_start_time:
filename = os.path.join(outdir, 'validation_data.pkl')
logger.info('Writing validation data %s' % filename)
f = open(filename, 'w')
pickle.dump({'pred': pred, 'tlm': tlm}, f, protocol=-1)
f.close()
valid_viols.extend(make_validation_viols(plots_validation))
if len(valid_viols) > 0:
# generate daily plot url if outdir in expected year/day format
daymatch = re.match('.*(\d{4})/(\d{3})', opt.outdir)
if daymatch:
url = os.path.join(URL, daymatch.group(1), daymatch.group(2))
logger.info('validation warning(s) at %s' % url)
else:
logger.info('validation warning(s) in output at %s' % opt.outdir)
write_index_rst(opt, proc, plots_validation, valid_viols)
rst_to_html(opt, proc)
def kal(dwell, telem, limit=20, catalog=None, nowflags=False):
cat = catalog
# Track status
fids = np.column_stack([(telem['AOACFID{}'.format(slot)].vals == 'FID ')
for slot in range(0, 8)])
trak = np.column_stack([(telem['AOACFCT{}'.format(slot)].vals == 'TRAK')
for slot in range(0, 8)])
# Flags
ir = np.column_stack([(telem['AOACIIR{}'.format(slot)].vals == 'OK ')
for slot in range(0, 8)])
sp = np.column_stack([(telem['AOACISP{}'.format(slot)].vals == 'OK ')
for slot in range(0, 8)])
dp_date = DateTime('2013:297:11:25:52.000').secs
dp = np.column_stack([((telem['AOACIDP{}'.format(slot)].vals == 'OK ')
| (telem['AOKALSTR'].times > dp_date))
for slot in range(0, 8)])
if dwell.start > '2015:251':
# I'm not sure about the fetch grid if we use fetch interpolate, so just use
# a sorted search to see if the MSS flag should apply
mss = telem['AOACIMSS'].vals == 'ENAB'
mss_times = telem['AOACIMSS'].times
mss_at_times = mss[
np.searchsorted(mss_times[1:-1], telem['AOACIMS0'].times) - 1]
ms = np.column_stack([((telem['AOACIMS{}'.format(slot)].vals == 'OK ')
| ~mss_at_times) for slot in range(0, 8)])
else:
ms = np.column_stack([(telem['AOACIMS{}'.format(slot)].vals == 'OK ')
for slot in range(0, 8)])
# use rolled-by-4 for ~last 4.1 sample
last_trak = np.roll(trak, 4, axis=0)
last_trak[0] = True
# Calc centroid residuals using CYAN/CZAN
q_atts = Quat(
normalize(
np.column_stack([
telem['AOATTQT1'].vals, telem['AOATTQT2'].vals,
telem['AOATTQT3'].vals, telem['AOATTQT4'].vals
])))
# guide and bot slots
gcat = cat[(cat['type'] == 'BOT') | (cat['type'] == 'GUI')]
# make a couple of structures for the offsets
yag_offs = np.zeros((len(telem['AOKALSTR'].vals), 8))
zag_offs = np.zeros((len(telem['AOKALSTR'].vals), 8))
for idx, entry in enumerate(gcat):
slot = entry['slot']
#ok = telem['AOACFCT{}'.format(slot)].vals == 'TRAK'
star = agasc.get_star(entry['id'], date=dwell.manvr.start)
eci = radec2eci(star['RA_PMCORR'], star['DEC_PMCORR'])
d_aca = np.dot(q_atts.transform.transpose(0, 2, 1), eci)
yag = np.degrees(np.arctan2(d_aca[:, 1], d_aca[:, 0])) * 3600
zag = np.degrees(np.arctan2(d_aca[:, 2], d_aca[:, 0])) * 3600
yag_offs[:, slot] = yag - telem['AOACYAN{}'.format(entry['slot'])].vals
zag_offs[:, slot] = zag - telem['AOACZAN{}'.format(entry['slot'])].vals
if nowflags:
kal = (~fids & trak & last_trak & ir & sp
& (np.abs(yag_offs) < limit) & (np.abs(zag_offs) < limit))
else:
kal = (~fids & trak & last_trak & ir & sp & dp & ms
& (np.abs(yag_offs) < limit) & (np.abs(zag_offs) < limit))
return telem['AOKALSTR'].times, kal
def main(opt):
opt, args = get_options()
if not os.path.exists(opt.outdir):
os.mkdir(opt.outdir)
config_logging(opt.outdir, opt.verbose)
# Store info relevant to processing for use in outputs
proc = dict(run_user=os.environ['USER'],
run_time=time.ctime(),
errors=[],
)
logger.info('#####################################################################')
logger.info('# %s run at %s by %s' % (os.path.dirname(__file__),
proc['run_time'], proc['run_user']))
logger.info('# version = %s' % VERSION)
logger.info('# characteristics version = %s' % characteristics.VERSION)
logger.info('#####################################################################\n')
logger.info('Command line options:\n%s\n' % pformat(opt.__dict__))
# Connect to database (NEED TO USE aca_read)
tnow = DateTime(opt.run_start_time).secs
tstart = tnow
# Get temperature telemetry for 3 weeks prior to min(tstart, NOW)
tlm = get_telem_values(tstart,
['sim_z', 'dp_pitch', 'aoacaseq',
'aodithen', 'cobsrqid', 'aofunlst',
'aopcadmd', '4ootgsel', '4ootgmtn',
'aocmdqt1', 'aocmdqt2', 'aocmdqt3',
'1de28avo', '1deicacu',
'1dp28avo', '1dpicacu',
'1dp28bvo', '1dpicbcu'],
days=opt.days,
name_map={'sim_z': 'tscpos',
'cobsrqid': 'obsid'})
tlm['tscpos'] = tlm['tscpos'] * -397.7225924607
outdir = opt.outdir
states = get_states(tlm[0].date, tlm[-1].date)
write_states(opt, states)
tlm = Ska.Numpy.add_column(tlm, 'power', smoothed_power(tlm))
# Get bad time intervals
bad_time_mask = get_bad_mask(tlm)
# Interpolate states onto the tlm.date grid
state_vals = cmd_states.interpolate_states(states, tlm['date'])
# Calculate the 4th term of the commanded quaternions
cmd_q4 = np.sqrt(np.abs(1.0
- tlm['aocmdqt1']**2
- tlm['aocmdqt2']**2
- tlm['aocmdqt3']**2))
raw_tlm_q = np.vstack([tlm['aocmdqt1'],
tlm['aocmdqt2'],
tlm['aocmdqt3'],
cmd_q4]).transpose()
# Calculate angle/roll differences in state cmd vs tlm cmd quaternions
raw_state_q = np.vstack([state_vals[n] for n
in ['q1', 'q2', 'q3', 'q4']]).transpose()
tlm_q = normalize(raw_tlm_q)
# only use values that aren't NaNs
good = np.isnan(np.sum(tlm_q, axis=-1)) == False
# and are in NPNT
npnt = tlm['aopcadmd'] == 'NPNT'
# and are in KALM after the first 2 sample of the transition
not_kalm = tlm['aoacaseq'] != 'KALM'
kalm = (not_kalm | np.hstack([[False, False], not_kalm[:-2]])) == False
# and aren't during momentum unloads or in the first 2 samples after unloads
unload = tlm['aofunlst'] != 'NONE'
no_unload = (unload | np.hstack([[False, False], unload[:-2]])) == False
ok = good & npnt & kalm & no_unload & ~bad_time_mask
state_q = normalize(raw_state_q)
dot_q = np.sum(tlm_q[ok] * state_q[ok], axis=-1)
dot_q[dot_q > 1] = 1
angle_diff = np.degrees(2 * np.arccos(dot_q))
angle_diff = np.min([angle_diff, 360 - angle_diff], axis=0)
roll_diff = Quat(tlm_q[ok]).roll - Quat(state_q[ok]).roll
roll_diff = np.min([roll_diff, 360 - roll_diff], axis=0)
for msid in MODE_SOURCE:
tlm_col = np.zeros(len(tlm))
state_col = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
tlm_col[tlm[MODE_SOURCE[msid]] == mode] = idx
state_col[state_vals[msid] == mode] = idx
tlm = Ska.Numpy.add_column(tlm, msid, tlm_col)
state_vals = Ska.Numpy.add_column(state_vals,
"{}_pred".format(msid), state_col)
for msid in ['letg', 'hetg']:
txt = np.repeat('RETR', len(tlm))
# use a combination of the select telemetry and the insertion telem to
# approximate the state_vals values
txt[(tlm['4ootgsel'] == msid.upper())
& (tlm['4ootgmtn'] == 'INSE')] = 'INSE'
tlm_col = np.zeros(len(tlm))
state_col = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
tlm_col[txt == mode] = idx
state_col[state_vals[msid] == mode] = idx
tlm = Ska.Numpy.add_column(tlm, msid, tlm_col)
state_vals = Ska.Numpy.add_column(state_vals,
"{}_pred".format(msid), state_col)
diff_only = {'pointing': {'diff': angle_diff * 3600,
'date': tlm['date'][ok]},
'roll': {'diff': roll_diff * 3600,
'date': tlm['date'][ok]}}
pred = {'dp_pitch': state_vals.pitch,
'obsid': state_vals.obsid,
'dither': state_vals['dither_pred'],
'pcad_mode': state_vals['pcad_mode_pred'],
'letg': state_vals['letg_pred'],
'hetg': state_vals['hetg_pred'],
'tscpos': state_vals.simpos,
'power': state_vals.power,
'pointing': 1,
'roll': 1}
plots_validation = []
valid_viols = []
logger.info('Making validation plots and quantile table')
quantiles = (1, 5, 16, 50, 84, 95, 99)
# store lines of quantile table in a string and write out later
quant_table = ''
quant_head = ",".join(['MSID'] + ["quant%d" % x for x in quantiles])
quant_table += quant_head + "\n"
for fig_id, msid in enumerate(sorted(pred)):
plot = dict(msid=msid.upper())
fig = plt.figure(10 + fig_id, figsize=(7, 3.5))
fig.clf()
scale = SCALES.get(msid, 1.0)
ax = None
if msid not in diff_only:
if msid in MODE_MSIDS:
state_msid = np.zeros(len(tlm))
for mode, idx in zip(MODE_MSIDS[msid], count()):
state_msid[state_vals[msid] == mode] = idx
ticklocs, fig, ax = plot_cxctime(tlm['date'],
tlm[msid], fig=fig, fmt='-r')
ticklocs, fig, ax = plot_cxctime(tlm['date'],
state_msid, fig=fig, fmt='-b')
plt.yticks(range(len(MODE_MSIDS[msid])), MODE_MSIDS[msid])
else:
ticklocs, fig, ax = plot_cxctime(tlm['date'],
tlm[msid] / scale, fig=fig, fmt='-r')
ticklocs, fig, ax = plot_cxctime(tlm['date'],
pred[msid] / scale, fig=fig, fmt='-b')
else:
ticklocs, fig, ax = plot_cxctime(diff_only[msid]['date'],
diff_only[msid]['diff'] / scale, fig=fig, fmt='-k')
plot['diff_only'] = msid in diff_only
ax.set_title(TITLE[msid])
ax.set_ylabel(LABELS[msid])
xlims = ax.get_xlim()
ylims = ax.get_ylim()
for bad in characteristics.bad_times:
bad_start = cxc2pd([DateTime(bad['start']).secs])[0]
bad_stop = cxc2pd([DateTime(bad['stop']).secs])[0]
if not ((bad_stop >= xlims[0]) & (bad_start <= xlims[1])):
continue
rect = matplotlib.patches.Rectangle((bad_start, ylims[0]),
bad_stop - bad_start,
ylims[1] - ylims[0],
alpha=.2,
facecolor='black',
edgecolor='none')
ax.add_patch(rect)
filename = msid + '_valid.png'
outfile = os.path.join(outdir, filename)
logger.info('Writing plot file %s' % outfile)
plt.tight_layout()
fig.savefig(outfile)
plot['lines'] = filename
if msid not in diff_only:
diff = tlm[msid][~bad_time_mask] - pred[msid][~bad_time_mask]
diff = np.sort(diff)
else:
diff = np.sort(diff_only[msid]['diff'])
# if there are only a few residuals, don't bother with histograms
if msid.upper() in validation_scale_count:
plot['samples'] = len(diff)
plot['diff_count'] = np.count_nonzero(diff)
plot['n_changes'] = 1 + np.count_nonzero(pred[msid][1:] - pred[msid][0:-1])
if (plot['diff_count'] <
(plot['n_changes'] * validation_scale_count[msid.upper()])):
plots_validation.append(plot)
continue
# if the msid exceeds the diff count, add a validation violation
else:
viol = {'msid': "{}_diff_count".format(msid),
'value': plot['diff_count'],
'limit': plot['n_changes'] * validation_scale_count[msid.upper()],
'quant': None,
}
valid_viols.append(viol)
logger.info('WARNING: %s %d discrete diffs exceed limit of %d' %
(msid, plot['diff_count'],
plot['n_changes'] * validation_scale_count[msid.upper()]))
# Make quantiles
if (msid != 'obsid'):
quant_line = "%s" % msid
for quant in quantiles:
quant_val = diff[(len(diff) * quant) // 100]
plot['quant%02d' % quant] = FMTS[msid] % quant_val
quant_line += (',' + FMTS[msid] % quant_val)
quant_table += quant_line + "\n"
for histscale in ('lin', 'log'):
fig = plt.figure(20 + fig_id, figsize=(4, 3))
fig.clf()
ax = fig.gca()
ax.hist(diff / scale, bins=50, log=(histscale == 'log'))
ax.set_title(msid.upper() + ' residuals: telem - cmd states', fontsize=11)
ax.set_xlabel(LABELS[msid])
fig.subplots_adjust(bottom=0.18)
plt.tight_layout()
filename = '%s_valid_hist_%s.png' % (msid, histscale)
outfile = os.path.join(outdir, filename)
logger.info('Writing plot file %s' % outfile)
fig.savefig(outfile)
plot['hist' + histscale] = filename
plots_validation.append(plot)
filename = os.path.join(outdir, 'validation_quant.csv')
logger.info('Writing quantile table %s' % filename)
f = open(filename, 'w')
f.write(quant_table)
f.close()
# If run_start_time is specified this is likely for regression testing
# or other debugging. In this case write out the full predicted and
# telemetered dataset as a pickle.
if opt.run_start_time:
filename = os.path.join(outdir, 'validation_data.pkl')
logger.info('Writing validation data %s' % filename)
f = open(filename, 'w')
pickle.dump({'pred': pred, 'tlm': tlm}, f, protocol=-1)
f.close()
valid_viols.extend(make_validation_viols(plots_validation))
if len(valid_viols) > 0:
# generate daily plot url if outdir in expected year/day format
daymatch = re.match('.*(\d{4})/(\d{3})', opt.outdir)
if daymatch:
url = os.path.join(URL, daymatch.group(1), daymatch.group(2))
logger.info('validation warning(s) at %s' % url)
else:
logger.info('validation warning(s) in output at %s' % opt.outdir)
write_index_rst(opt, proc, plots_validation, valid_viols)
rst_to_html(opt, proc)