def starcheck_orig_link(obsid):
"""
Return a link to the original starcheck products for an obsid
"""
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
if mp_dir is None:
return None
starcheck_html = "{top}{mp_dir}starcheck.html#obsid{obsid}".format(
top="https://icxc.harvard.edu/mp/mplogs", mp_dir=mp_dir, obsid=str(obsid)
)
return starcheck_html
def starcheck_orig_link(obsid):
"""
Return a link to the original starcheck products for an obsid
"""
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
if mp_dir is None:
return None
starcheck_html = "{top}{mp_dir}starcheck.html#obsid{obsid}".format(
top="https://icxc.harvard.edu/mp/mplogs",
mp_dir=mp_dir,
obsid=str(obsid))
return starcheck_html
def get_starcheck(obsid):
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
return (sc, mp_dir, status)
def main(obsid):
report_root = REPORT_ROOT
strobs = "%05d" % obsid
chunk_dir = strobs[0:2]
topdir = os.path.join(report_root, chunk_dir)
outdir = os.path.join(topdir, strobs)
if not os.path.exists(outdir):
os.makedirs(outdir)
jinja_env = jinja2.Environment(
loader=jinja2.PackageLoader('mica.report'))
jinja_env.line_comment_prefix = '##'
jinja_env.line_statement_prefix = '#'
logger.info("Making report for {}".format(obsid))
logger.debug("Getting target info from axafapstat")
summary = target_summary(obsid)
# Links
logger.debug("Looking up obsid links")
all_progress = {'science': ['ocat',
'long_term', 'short_term',
'starcheck', 'observed',
'engineering', 'aspect_1',
'cxcds_vv', 'released'],
'er': ['starcheck', 'observed',
'engineering', 'cxcds_vv']}
report_status = {}
er = summary is None and obsid > 40000
progress = all_progress['er' if er else 'science']
if er:
links = obs_links(obsid)
else:
if summary is None:
raise ValueError("Obsid not found in target table")
report_status['ocat'] = summary['status']
links = obs_links(obsid, summary['seq_nbr'], summary['lts_lt_plan'])
if not er and (summary['status'] in
['canceled', 'unobserved', 'untriggered']):
logger.debug(
"Obsid {obsid} has status {status}".format(
obsid=obsid, status=summary['status']))
if summary is not None:
if summary['lts_lt_plan'] is not None:
report_status['long_term'] = summary['lts_lt_plan']
if summary['soe_st_sched_date']:
report_status['short_term'] = summary['soe_st_sched_date']
last_sched = ''
if not er:
if summary['lts_lt_plan']:
last_sched = "in LTS for {}".format(
str(summary['lts_lt_plan']))
if summary['soe_st_sched_date']:
last_sched = "in ST sched for {}".format(
str(summary['soe_st_sched_date']))
## Starcheck
logger.debug("Fetching starcheck catalog")
try:
if summary is not None and summary['lts_lt_plan'] is not None:
plan_date = Time(summary['lts_lt_plan'])
if plan_date.cxcsec > (CxoTime.now() + 21).secs:
raise LookupError("No starcheck expected for {} lts date".format(str(plan)))
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
obs_sc, mp_dir, status = get_starcheck(obsid)
logger.debug("Plotting starcheck catalog to {}".format(os.path.join(outdir, 'starcheck.png')))
if obs_sc['obs']['point_ra'] is None:
raise LookupError("Observation has no pointing.")
if len(obs_sc['cat']) == 0:
raise LookupError("Observation has no catalog")
fig, cat, obs = catalog.plot(obsid, mp_dir)
sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
fig.savefig(os.path.join(outdir, 'starcheck.png'))
plt.close('all')
except LookupError as detail:
logger.info("No starcheck catalog. Writing out OCAT info only")
logger.info(detail)
template = jinja_env.get_template('report.html')
page = template.render(obsid=obsid,
target=summary,
links=links,
temps=None,
pred_temp=None,
cat_table=None,
er=er if er else None,
last_sched=last_sched,
obs=None,
version=version)
full_report_file = os.path.join(outdir, 'index.html')
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, 'w')
f.write(page)
f.close()
notes = {'report_version': REPORT_VERSION,
'vv_version': None,
'vv_revision': None,
'aspect_1_id': None,
'last_sched': last_sched,
'ocat_status': report_status.get('ocat'),
'long_term': str(report_status.get('long_term')),
'short_term': str(report_status.get('short_term')),
'starcheck': report_status.get('starcheck'),
'obsid': obsid,
'checked_date': CxoTime.now().date}
f = open(os.path.join(outdir, 'notes.json'), 'w')
f.write(json.dumps(notes,
sort_keys=True,
indent=4))
f.close()
save_state_in_db(obsid, notes)
return
if not er and 'shortterm' in links:
dir_match = re.match(r'/\d{4}/(\w{3}\d{4})/ofls(\w)', mp_dir)
mp_label = "{}{}".format(dir_match.group(1),
dir_match.group(2).upper())
last_sched = 'in <A HREF="{}">{}</A> at {}'.format(
links['shortterm']['link'], mp_label, str(obs_sc['obs']['mp_starcat_time']))
report_status['starcheck'] = mp_dir
# engineering data available
logger.debug("Getting acq and trak stats")
acqs = get_obs_acq_stats(obsid)
trak = get_obs_trak_stats(obsid)
temps = get_obs_temps(obsid, outdir)
pred_temp = sc.get('pred_temp')
if acqs or trak:
last_sched = "eng. data available"
er_status = None
if er:
stat_map = {'ran': 'ran on',
'approved': 'approved for',
'ran_pretimelines': 'ran on',
'planned': 'planned for'}
er_status = "{} {}".format(stat_map[status], obs_sc['obs']['mp_starcat_time'])
run_obspar = None
vv = None
logger.info("Processing ER; no V&V available")
else:
# obspar ingested
try:
run_obspar = obspar.get_obspar(obsid, version='last')
except:
run_obspar = None
# v&v available
try:
vv = get_vv(obsid, version='last')
except LookupError:
vv = None
try:
if vv is None or 'vv_version' not in vv or vv['vv_version'] < WANT_VV_VERSION:
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version='last')
for slot in vv['slots']:
if isinstance(vv['slots'][slot]['dy_med'], list):
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version='last')
break
vv_files = get_vv_files(obsid, version='last')
last_sched = "complete through mica v&v"
except LookupError:
logger.info("No V&V available")
vv = None
if vv is not None:
report_status['mica_vv'] = True
for file in vv_files:
newfile = os.path.join(outdir, os.path.basename(file))
if not os.path.exists(newfile):
logger.debug("linking {} into {}".format(file, outdir))
os.symlink(file, newfile)
asp_dir = asp_l1.get_obs_dirs(obsid)['last']
asp_logs = sorted(glob(os.path.join(asp_dir, "asp_l1_f*log*gz")))
for log, interval in zip(asp_logs, vv['intervals']):
logmatch = re.search(r'(.*log)\.gz', os.path.basename(log))
if logmatch:
newlogname = "{}.txt".format(logmatch.group(1))
newlog = os.path.join(outdir, newlogname)
if not os.path.exists(newlog):
logger.debug("copying/gunzipping asp log {}".format(newlog))
logtext = gzip.open(log, 'rt').readlines()
f = open(newlog, 'w')
f.writelines(logtext)
f.close()
interval['loglink'] = newlogname
aiprops = get_aiprops(obsid)
aiprops_template = jinja_env.get_template('aiprops.html')
aiprops_page = aiprops_template.render(obsid=obsid, aiprops=aiprops)
aiprops_page_file = os.path.join(outdir, 'aiprops.html')
logger.debug("AIPROPS report to {}".format(aiprops_page_file))
f = open(aiprops_page_file, 'w')
f.write(aiprops_page)
f.close()
props_template = jinja_env.get_template('props.html')
props_page = props_template.render(obsid=obsid, vv=vv)
props_page_file = os.path.join(outdir, 'props.html')
logger.debug("GS/FIDPROPS report to {}".format(props_page_file))
f = open(props_page_file, 'w')
f.write(props_page)
f.close()
for slot in vv['slots']:
if 'n_pts' not in vv['slots'][slot]:
continue
slot_template = jinja_env.get_template('vv_slots_single.html')
slot_page = slot_template.render(obsid=obsid,
vv=vv,
slot=slot)
slot_page_file = os.path.join(outdir, "slot_{}.html".format(slot))
logger.debug("VV SLOT report to {}".format(slot_page_file))
f = open(slot_page_file, 'w')
f.write(slot_page)
f.close()
official_notes = official_vv_notes(obsid, summary)
if official_notes:
for rep in official_notes:
if rep['comments'] == 'Hidden':
rep['comments'] = """
<A target="_blank" HREF="{}">{}</A><BR>(<A target="_blank" HREF="https://icxc.cfa.harvard.edu/soft/vv/vv_login.html">LOGIN</A> once first)</BR>""".format(links['vv']['link'], links['vv']['label'])
vv_template = jinja_env.get_template('vv.html')
vv['has_errors'] = (('errors' in vv) and (len(vv['errors']))) or None
vv_page = vv_template.render(obsid=obsid,
vv=vv,
obspar=run_obspar,
official_vv_notes=official_notes,
)
vv_page_file = os.path.join(outdir, 'vv.html')
logger.debug("VV report to {}".format(vv_page_file))
f = open(vv_page_file, 'w')
f.write(vv_page)
f.close()
cat_table = catalog_info(obs_sc['cat'], acqs, trak, vv)
for row, cat_row in zip(obs_sc['cat'], cat_table):
if row['type'] != 'FID':
if row['id'] is not None:
s = star_info(row['id'])
star_template = jinja_env.get_template('star.html')
star_page = star_template.render(
star=s['agasc_info'],
acqs=s['acqs'],
traks=s['traks'],
agg_acq=s['agg_acq'],
agg_trak=s['agg_trak'])
star_page_file = os.path.join(outdir, 'star_%d.html' % int(row['id']))
logger.debug("Writing out star info to {}".format(star_page_file))
f = open(star_page_file, 'w')
f.write(star_page)
f.close()
cat_row['idlink'] = (
'<A HREF="star_{id}.html" STYLE="text-decoration: none;"'
'ONMOUSEOVER="return overlib '
'(\'ACQ total:{n_acq} noid:{n_noid} <BR /> '
'GUI total:{n_gui} bad:{n_bad} fail:{n_fail} obc_bad:{n_obc_bad} '
'<BR /> Avg Mag {avg_mag:4.2f}\', WIDTH, 220);", ONMOUSEOUT="return nd();"> '
'{id}</A>'.format(id=int(row['id']),
n_acq=s['agg_acq']['n_acqs'],
n_noid=s['agg_acq']['n_acq_noid'],
n_gui=s['agg_trak']['n_guis'],
n_bad=s['agg_trak']['n_bad'],
n_fail=s['agg_trak']['n_fail'],
n_obc_bad=s['agg_trak']['n_obc_bad'],
avg_mag=(s['agg_trak']['avg_mag']
or s['agg_acq']['avg_mag']
or 13.94)))
else:
cat_row['idlink'] = " "
else:
if 'id' in row:
cat_row['idlink'] = int(row['id'])
else:
cat_row['idlink'] = ''
template = jinja_env.get_template('report.html')
page = template.render(cat_table=cat_table,
obs=obs,
sc=obs_sc,
vv=vv,
links=links,
target=summary,
temps=temps,
pred_temp=pred_temp,
er=er if er else None,
er_status=er_status,
last_sched=last_sched,
obsid=obsid,
version=version)
full_report_file = os.path.join(outdir, 'index.html')
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, 'w')
f.write(page)
f.close()
cat_file = os.path.join(outdir, 'star_table.json')
f = open(cat_file, 'w')
f.write(json.dumps(cat_table, sort_keys=True, indent=4))
f.close()
notes = {'report_version': REPORT_VERSION,
'vv_version': None,
'vv_revision': None,
'aspect_1_id': None,
'last_sched': last_sched,
'ocat_status': report_status.get('ocat'),
'long_term': str(report_status.get('long_term')),
'short_term': str(report_status.get('short_term')),
'starcheck': report_status.get('starcheck'),
'obsid': obsid,
'checked_date': CxoTime.now().date}
if vv:
notes['vv_version'] = vv.get('vv_version')
notes['vv_revision'] = vv.get('revision')
notes['aspect_1_id'] = vv.get('aspect_1_id')
f = open(os.path.join(outdir, 'notes.json'), 'w')
f.write(json.dumps(notes,
sort_keys=True,
indent=4))
f.close()
save_state_in_db(obsid, notes)
def obs_links(obsid, sequence=None, plan=None):
links = {
'obscat': {
'label': "Target Param : {}".format(obsid),
'link': "https://icxc.cfa.harvard.edu/cgi-bin/mp/target_param.cgi?{}".format(obsid)},
'mp_dir': None,
'shortterm': None,
'fot_dir': None,
'fot_daily': None,
'cen_dash': None,
'starcheck_html': None,
'vv': None}
if sequence is not None:
links['seq_sum'] = {
'label': "Seq. Summary : {}".format(sequence),
'link': "https://icxc.harvard.edu/cgi-bin/mp/target.cgi?{}".format(sequence)}
mp_dir = None
# if this is a science observation, only try to get a star catalog if it has a home
# in the schedule either in the past or the near future
if plan is not None:
plan_date = Time(plan)
if plan_date < Time.now() + 21 * u.day:
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
else:
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
# Check for centroid dashboard
strobs = "%05d" % obsid
chunk_dir = strobs[0:2]
cen_dash_file = "{}/{}/index.html".format(chunk_dir, strobs)
if os.path.exists(os.path.join('/proj/sot/ska/www/ASPECT_ICXC/centroid_reports/',
cen_dash_file)):
cen_url = 'https://icxc.cfa.harvard.edu/aspect/centroid_reports/'
links['cen_dash'] = {'link': "{}/{}".format(cen_url, cen_dash_file),
'label': 'Centroid Dashboard'}
if mp_dir is not None:
dir_match = re.match(r'/\d{4}/(\w{3}\d{4})/ofls(\w)', mp_dir)
mp_label = "{}{}".format(dir_match.group(1),
dir_match.group(2).upper())
mp_date_m = re.match(r'(\d{4}):(\d{3}):\d{2}:\d{2}:\d{2}\.\d{3}', mp_date)
if mp_date_m:
year = int(mp_date_m.group(1))
doy = int(mp_date_m.group(2))
dtm = datetime.datetime(year, 1, 1) + datetime.timedelta(doy - 1)
month = dtm.strftime("%b")
dom = dtm.strftime("%d")
links['fot_daily'] = {
'label': "Daily Plots {}:{:03d}".format(year, int(doy)),
'link': "{root}/{year}/{upper_month}/{lower_month}{day_of_month:02d}_{doy:03d}/".format(
root=DAILY_PLOT_ROOT, year=year, upper_month=month.upper(),
lower_month=month.lower(), day_of_month=int(dom), doy=int(doy))}
vv, vvid = official_vv(obsid)
links['shortterm'] = {'link': guess_shortterm(mp_dir),
'label': "Short Term Sched. {}".format(mp_label)}
links['fot_dir'] = {'link': guess_fot_summary(mp_dir),
'label': "FOT Approved Sched. {}".format(mp_label)}
links['starcheck_html'] = {'link': "{top}{mp_dir}starcheck.html#obsid{obsid}".format(top="https://icxc.harvard.edu/mp/mplogs", obsid=obsid, mp_dir=mp_dir),
'label': "Starcheck obsid {}".format(obsid)}
links['mp_dir'] = {'link': mp_dir,
'label': "Starcheck obsid {}".format(obsid)}
links['vv'] = {'link': vv,
'label': "CXCDS V&V (id={})".format(vvid)}
return links
def get_starcheck(obsid):
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
sc = starcheck.obsid(obsid, mp_dir)
return (sc, mp_dir, status)
def main(obsid, config=None, report_root=None):
if config is None:
config = DEFAULT_CONFIG
if report_root is None:
report_root = config["report_root"]
global ACA_DB
if ACA_DB is None or not ACA_DB.conn._is_connected:
ACA_DB = Ska.DBI.DBI(dbi="sybase", server="sybase", user="******")
strobs = "%05d" % obsid
chunk_dir = strobs[0:2]
topdir = os.path.join(report_root, chunk_dir)
outdir = os.path.join(topdir, strobs)
if not os.path.exists(outdir):
os.makedirs(outdir)
jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.join(os.environ["SKA"], "data", "mica", "templates"))
)
jinja_env.line_comment_prefix = "##"
jinja_env.line_statement_prefix = "#"
logger.info("Making report for {}".format(obsid))
logger.debug("Getting target info from axafapstat")
summary = target_summary(obsid)
# Links
logger.debug("Looking up obsid links")
all_progress = {
"science": [
"ocat",
"long_term",
"short_term",
"starcheck",
"observed",
"engineering",
"aspect_1",
"cxcds_vv",
"released",
],
"er": ["starcheck", "observed", "engineering", "cxcds_vv"],
}
report_status = {}
er = summary is None and obsid > 40000
progress = all_progress["er" if er else "science"]
if er:
links = obs_links(obsid)
else:
if summary is None:
raise ValueError("Obsid not found in target table")
report_status["ocat"] = summary["status"]
links = obs_links(obsid, summary["seq_nbr"], summary["lts_lt_plan"])
if not er and (summary["status"] in ["canceled", "unobserved", "untriggered"]):
logger.debug("Obsid {obsid} has status {status}".format(obsid=obsid, status=summary["status"]))
if summary is not None:
if summary["lts_lt_plan"] is not None:
report_status["long_term"] = summary["lts_lt_plan"]
if summary["soe_st_sched_date"]:
report_status["short_term"] = summary["soe_st_sched_date"]
last_sched = ""
if not er:
if summary["lts_lt_plan"]:
last_sched = "in LTS for {}".format(str(summary["lts_lt_plan"]))
if summary["soe_st_sched_date"]:
last_sched = "in ST sched for {}".format(str(summary["soe_st_sched_date"]))
## Starcheck
logger.debug("Fetching starcheck catalog")
try:
if summary is not None and summary["lts_lt_plan"] is not None:
plan = summary["lts_lt_plan"]
plan_date = DateTime(
"{:4d}:{:03d}:{:02d}:{:02d}:{:02d}.000".format(
plan.year, plan.dayofyear, plan.hour, plan.minute, plan.second
)
)
if plan_date.secs > (DateTime() + 21).secs:
raise LookupError("No starcheck expected for {} lts date".format(str(plan)))
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
obs_sc, mp_dir, status = get_starcheck(obsid)
logger.debug("Plotting starcheck catalog to {}".format(os.path.join(outdir, "starcheck.png")))
if obs_sc["obs"][0]["point_ra"] is None:
raise LookupError("Observation has no pointing.")
if len(obs_sc["catalog"]) == 0:
raise LookupError("Observation has no catalog")
fig, cat, obs = catalog.plot(obsid, mp_dir)
sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
fig.savefig(os.path.join(outdir, "starcheck.png"))
plt.close("all")
except LookupError as detail:
logger.info("No starcheck catalog. Writing out OCAT info only")
logger.info(detail)
template = jinja_env.get_template("report.html")
page = template.render(
obsid=obsid,
target=summary,
links=links,
temps=None,
pred_temp=None,
cat_table=None,
er=er if er else None,
last_sched=last_sched,
obs=None,
version=version,
)
full_report_file = os.path.join(outdir, "index.html")
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, "w")
f.write(page)
f.close()
notes = {
"report_version": REPORT_VERSION,
"vv_version": None,
"vv_revision": None,
"aspect_1_id": None,
"last_sched": last_sched,
"ocat_status": report_status.get("ocat"),
"long_term": str(report_status.get("long_term")),
"short_term": str(report_status.get("short_term")),
"starcheck": report_status.get("starcheck"),
"obsid": obsid,
"checked_date": DateTime().date,
}
f = open(os.path.join(outdir, "notes.json"), "w")
f.write(json.dumps(notes, sort_keys=True, indent=4))
f.close()
save_state_in_db(obsid, notes, config)
return
if not er and "shortterm" in links:
dir_match = re.match("/\d{4}/(\w{3}\d{4})/ofls(\w)", mp_dir)
mp_label = "{}{}".format(dir_match.group(1), dir_match.group(2).upper())
last_sched = 'in <A HREF="{}">{}</A> at {}'.format(
links["shortterm"]["link"], mp_label, str(obs_sc["obs"][0]["mp_starcat_time"])
)
report_status["starcheck"] = mp_dir
# engineering data available
logger.debug("Getting acq and trak stats")
acqs = get_obs_acq_stats(obsid)
trak = get_obs_trak_stats(obsid)
temps = get_obs_temps(obsid, outdir)
pred_temp = sc["pred_temp"]
if acqs or trak:
last_sched = "eng. data available"
er_status = None
if er:
stat_map = {"ran": "ran on", "approved": "approved for", "ran_pretimelines": "ran on", "planned": "planned for"}
er_status = "{} {}".format(stat_map[status], obs_sc["obs"][0]["mp_starcat_time"])
run_obspar = None
vv = None
logger.info("Processing ER; no V&V available")
else:
# obspar ingested
try:
run_obspar = obspar.get_obspar(obsid, version="last")
except:
run_obspar = None
# v&v available
try:
vv = get_vv(obsid, version="last")
except LookupError:
vv = None
try:
if vv is None or "vv_version" not in vv or vv["vv_version"] < WANT_VV_VERSION:
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version="last")
for slot in vv["slots"]:
if isinstance(vv["slots"][slot]["dy_med"], list):
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version="last")
break
vv_files = get_vv_files(obsid, version="last")
last_sched = "complete through mica v&v"
except LookupError:
logger.info("No V&V available")
vv = None
if vv is not None:
report_status["mica_vv"] = True
for file in vv_files:
newfile = os.path.join(outdir, os.path.basename(file))
if not os.path.exists(newfile):
logger.debug("linking {} into {}".format(file, outdir))
bash("ln -s {} {}".format(file, outdir))
asp_dir = asp_l1.get_obs_dirs(obsid)["last"]
asp_logs = sorted(glob(os.path.join(asp_dir, "asp*log*gz")))
for log, interval in zip(asp_logs, vv["intervals"]):
logmatch = re.search("(.*log)\.gz", os.path.basename(log))
if logmatch:
newlogname = "{}.txt".format(logmatch.group(1))
newlog = os.path.join(outdir, newlogname)
if not os.path.exists(newlog):
logger.debug("copying/gunzipping asp log {}".format(newlog))
logtext = gzip.open(log).readlines()
f = open(newlog, "w")
f.writelines(logtext)
f.close()
interval["loglink"] = newlogname
aiprops = get_aiprops(obsid)
aiprops_template = jinja_env.get_template("aiprops.html")
aiprops_page = aiprops_template.render(obsid=obsid, aiprops=aiprops)
aiprops_page_file = os.path.join(outdir, "aiprops.html")
logger.debug("AIPROPS report to {}".format(aiprops_page_file))
f = open(aiprops_page_file, "w")
f.write(aiprops_page)
f.close()
props_template = jinja_env.get_template("props.html")
props_page = props_template.render(obsid=obsid, vv=vv)
props_page_file = os.path.join(outdir, "props.html")
logger.debug("GS/FIDPROPS report to {}".format(props_page_file))
f = open(props_page_file, "w")
f.write(props_page)
f.close()
for slot in vv["slots"]:
if "n_pts" not in vv["slots"][slot]:
continue
slot_template = jinja_env.get_template("vv_slots_single.html")
slot_page = slot_template.render(obsid=obsid, vv=vv, slot=slot)
slot_page_file = os.path.join(outdir, "slot_{}.html".format(slot))
logger.debug("VV SLOT report to {}".format(slot_page_file))
f = open(slot_page_file, "w")
f.write(slot_page)
f.close()
official_notes = official_vv_notes(obsid, summary)
if official_notes:
for rep in official_notes:
if rep["comments"] == "Hidden":
rep[
"comments"
] = """
<A target="_blank" HREF="{}">{}</A><BR>(<A target="_blank" HREF="https://icxc.cfa.harvard.edu/soft/vv/vv_login.html">LOGIN</A> once first)</BR>""".format(
links["vv"]["link"], links["vv"]["label"]
)
vv_template = jinja_env.get_template("vv.html")
vv["has_errors"] = (("errors" in vv) and (len(vv["errors"]))) or None
vv_page = vv_template.render(obsid=obsid, vv=vv, obspar=run_obspar, official_vv_notes=official_notes)
vv_page_file = os.path.join(outdir, "vv.html")
logger.debug("VV report to {}".format(vv_page_file))
f = open(vv_page_file, "w")
f.write(vv_page)
f.close()
cat_table = catalog_info(obs_sc["catalog"], acqs, trak, vv)
for row, cat_row in zip(obs_sc["catalog"], cat_table):
if row["type"] != "FID":
if row["id"] is not None:
s = star_info(row["id"])
star_template = jinja_env.get_template("star.html")
star_page = star_template.render(
star=s["agasc_info"], acqs=s["acqs"], traks=s["traks"], agg_acq=s["agg_acq"], agg_trak=s["agg_trak"]
)
star_page_file = os.path.join(outdir, "star_%d.html" % int(row["id"]))
logger.debug("Writing out star info to {}".format(star_page_file))
f = open(star_page_file, "w")
f.write(star_page)
f.close()
cat_row["idlink"] = (
'<A HREF="star_{id}.html" STYLE="text-decoration: none;"'
'ONMOUSEOVER="return overlib '
"('ACQ total:{n_acq} noid:{n_noid} <BR /> "
"GUI total:{n_gui} bad:{n_bad} fail:{n_fail} obc_bad:{n_obc_bad} "
'<BR /> Avg Mag {avg_mag:4.2f}\', WIDTH, 220);", ONMOUSEOUT="return nd();"> '
"{id}</A>".format(
id=int(row["id"]),
n_acq=s["agg_acq"]["n_acqs"],
n_noid=s["agg_acq"]["n_acq_noid"],
n_gui=s["agg_trak"]["n_guis"],
n_bad=s["agg_trak"]["n_bad"],
n_fail=s["agg_trak"]["n_fail"],
n_obc_bad=s["agg_trak"]["n_obc_bad"],
avg_mag=(s["agg_trak"]["avg_mag"] or s["agg_acq"]["avg_mag"] or 13.94),
)
)
else:
cat_row["idlink"] = " "
else:
if "id" in row:
cat_row["idlink"] = int(row["id"])
else:
cat_row["idlink"] = ""
template = jinja_env.get_template("report.html")
page = template.render(
cat_table=cat_table,
obs=obs,
sc=obs_sc,
vv=vv,
links=links,
target=summary,
temps=temps,
pred_temp=pred_temp,
er=er if er else None,
er_status=er_status,
last_sched=last_sched,
obsid=obsid,
version=version,
)
full_report_file = os.path.join(outdir, "index.html")
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, "w")
f.write(page)
f.close()
cat_file = os.path.join(outdir, "star_table.json")
f = open(cat_file, "w")
f.write(json.dumps(cat_table, sort_keys=True, indent=4))
f.close()
notes = {
"report_version": REPORT_VERSION,
"vv_version": None,
"vv_revision": None,
"aspect_1_id": None,
"last_sched": last_sched,
"ocat_status": report_status.get("ocat"),
"long_term": str(report_status.get("long_term")),
"short_term": str(report_status.get("short_term")),
"starcheck": report_status.get("starcheck"),
"obsid": obsid,
"checked_date": DateTime().date,
}
if vv:
notes["vv_version"] = vv.get("vv_version")
notes["vv_revision"] = vv.get("revision")
notes["aspect_1_id"] = vv.get("aspect_1_id")
f = open(os.path.join(outdir, "notes.json"), "w")
f.write(json.dumps(notes, sort_keys=True, indent=4))
f.close()
save_state_in_db(obsid, notes, config)
def obs_links(obsid, sequence=None, plan=None):
links = {
"obscat": {
"label": "Target Param : {}".format(obsid),
"link": "https://icxc.cfa.harvard.edu/cgi-bin/mp/target_param.cgi?{}".format(obsid),
},
"mp_dir": None,
"shortterm": None,
"fot_dir": None,
"fot_daily": None,
"starcheck_html": None,
"vv": None,
}
if sequence is not None:
links["seq_sum"] = {
"label": "Seq. Summary : {}".format(sequence),
"link": "https://icxc.harvard.edu/cgi-bin/mp/target.cgi?{}".format(sequence),
}
mp_dir = None
# if this is a science observation, only try to get a star catalog if it has a home
# in the schedule either in the past or the near future
if plan is not None:
plan_date = DateTime(
"{:4d}:{:03d}:{:02d}:{:02d}:{:02d}.000".format(
plan.year, plan.dayofyear, plan.hour, plan.minute, plan.second
)
)
if plan_date.secs < (DateTime() + 21).secs:
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
else:
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
if mp_dir is not None:
dir_match = re.match("/\d{4}/(\w{3}\d{4})/ofls(\w)", mp_dir)
mp_label = "{}{}".format(dir_match.group(1), dir_match.group(2).upper())
mp_date_m = re.match("(\d{4}):(\d{3}):\d{2}:\d{2}:\d{2}\.\d{3}", mp_date)
if mp_date_m:
year = int(mp_date_m.group(1))
doy = int(mp_date_m.group(2))
dtm = datetime.datetime(year, 1, 1) + datetime.timedelta(doy - 1)
month = dtm.strftime("%b")
dom = dtm.strftime("%d")
links["fot_daily"] = {
"label": "Daily Plots {}:{:03d}".format(year, int(doy)),
"link": "{root}/{year}/{upper_month}/{lower_month}{day_of_month:02d}_{doy:03d}/".format(
root=DAILY_PLOT_ROOT,
year=year,
upper_month=month.upper(),
lower_month=month.lower(),
day_of_month=int(dom),
doy=int(doy),
),
}
vv, vvid = official_vv(obsid)
links["shortterm"] = {"link": guess_shortterm(mp_dir), "label": "Short Term Sched. {}".format(mp_label)}
links["fot_dir"] = {"link": guess_fot_summary(mp_dir), "label": "FOT Approved Sched. {}".format(mp_label)}
links["starcheck_html"] = {
"link": "{top}{mp_dir}starcheck.html#obsid{obsid}".format(
top="https://icxc.harvard.edu/mp/mplogs", obsid=obsid, mp_dir=mp_dir
),
"label": "Starcheck obsid {}".format(obsid),
}
links["mp_dir"] = {"link": mp_dir, "label": "Starcheck obsid {}".format(obsid)}
links["vv"] = {"link": vv, "label": "CXCDS V&V (id={})".format(vvid)}
return links
def main(obsid):
report_root = REPORT_ROOT
strobs = "%05d" % obsid
chunk_dir = strobs[0:2]
topdir = os.path.join(report_root, chunk_dir)
outdir = os.path.join(topdir, strobs)
if not os.path.exists(outdir):
os.makedirs(outdir)
jinja_env = jinja2.Environment(
loader=jinja2.PackageLoader('mica.report'))
jinja_env.line_comment_prefix = '##'
jinja_env.line_statement_prefix = '#'
logger.info("Making report for {}".format(obsid))
logger.debug("Getting target info from axafapstat")
summary = target_summary(obsid)
# Links
logger.debug("Looking up obsid links")
all_progress = {'science': ['ocat',
'long_term', 'short_term',
'starcheck', 'observed',
'engineering', 'aspect_1',
'cxcds_vv', 'released'],
'er': ['starcheck', 'observed',
'engineering', 'cxcds_vv']}
report_status = {}
er = summary is None and obsid > 40000
progress = all_progress['er' if er else 'science']
if er:
links = obs_links(obsid)
else:
if summary is None:
raise ValueError("Obsid not found in target table")
report_status['ocat'] = summary['status']
links = obs_links(obsid, summary['seq_nbr'], summary['lts_lt_plan'])
if not er and (summary['status'] in
['canceled', 'unobserved', 'untriggered']):
logger.debug(
"Obsid {obsid} has status {status}".format(
obsid=obsid, status=summary['status']))
if summary is not None:
if summary['lts_lt_plan'] is not None:
report_status['long_term'] = summary['lts_lt_plan']
if summary['soe_st_sched_date']:
report_status['short_term'] = summary['soe_st_sched_date']
last_sched = ''
if not er:
if summary['lts_lt_plan']:
last_sched = "in LTS for {}".format(
str(summary['lts_lt_plan']))
if summary['soe_st_sched_date']:
last_sched = "in ST sched for {}".format(
str(summary['soe_st_sched_date']))
## Starcheck
logger.debug("Fetching starcheck catalog")
try:
if summary is not None and summary['lts_lt_plan'] is not None:
plan = summary['lts_lt_plan']
plan_date = DateTime("{:4d}:{:03d}:{:02d}:{:02d}:{:02d}.000".format(
plan.year, plan.dayofyear, plan.hour, plan.minute, plan.second))
if plan_date.secs > (DateTime() + 21).secs:
raise LookupError("No starcheck expected for {} lts date".format(str(plan)))
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
obs_sc, mp_dir, status = get_starcheck(obsid)
logger.debug("Plotting starcheck catalog to {}".format(os.path.join(outdir, 'starcheck.png')))
if obs_sc['obs']['point_ra'] is None:
raise LookupError("Observation has no pointing.")
if len(obs_sc['cat']) == 0:
raise LookupError("Observation has no catalog")
fig, cat, obs = catalog.plot(obsid, mp_dir)
sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
fig.savefig(os.path.join(outdir, 'starcheck.png'))
plt.close('all')
except LookupError as detail:
logger.info("No starcheck catalog. Writing out OCAT info only")
logger.info(detail)
template = jinja_env.get_template('report.html')
page = template.render(obsid=obsid,
target=summary,
links=links,
temps=None,
pred_temp=None,
cat_table=None,
er=er if er else None,
last_sched=last_sched,
obs=None,
version=version)
full_report_file = os.path.join(outdir, 'index.html')
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, 'w')
f.write(page)
f.close()
notes = {'report_version': REPORT_VERSION,
'vv_version': None,
'vv_revision': None,
'aspect_1_id': None,
'last_sched': last_sched,
'ocat_status': report_status.get('ocat'),
'long_term': str(report_status.get('long_term')),
'short_term': str(report_status.get('short_term')),
'starcheck': report_status.get('starcheck'),
'obsid': obsid,
'checked_date': DateTime().date}
f = open(os.path.join(outdir, 'notes.json'), 'w')
f.write(json.dumps(notes,
sort_keys=True,
indent=4))
f.close()
save_state_in_db(obsid, notes)
return
if not er and 'shortterm' in links:
dir_match = re.match('/\d{4}/(\w{3}\d{4})/ofls(\w)', mp_dir)
mp_label = "{}{}".format(dir_match.group(1),
dir_match.group(2).upper())
last_sched = 'in <A HREF="{}">{}</A> at {}'.format(
links['shortterm']['link'], mp_label, str(obs_sc['obs']['mp_starcat_time']))
report_status['starcheck'] = mp_dir
# engineering data available
logger.debug("Getting acq and trak stats")
acqs = get_obs_acq_stats(obsid)
trak = get_obs_trak_stats(obsid)
temps = get_obs_temps(obsid, outdir)
pred_temp = sc.get('pred_temp')
if acqs or trak:
last_sched = "eng. data available"
er_status = None
if er:
stat_map = {'ran': 'ran on',
'approved': 'approved for',
'ran_pretimelines': 'ran on',
'planned': 'planned for'}
er_status = "{} {}".format(stat_map[status], obs_sc['obs']['mp_starcat_time'])
run_obspar = None
vv = None
logger.info("Processing ER; no V&V available")
else:
# obspar ingested
try:
run_obspar = obspar.get_obspar(obsid, version='last')
except:
run_obspar = None
# v&v available
try:
vv = get_vv(obsid, version='last')
except LookupError:
vv = None
try:
if vv is None or 'vv_version' not in vv or vv['vv_version'] < WANT_VV_VERSION:
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version='last')
for slot in vv['slots']:
if isinstance(vv['slots'][slot]['dy_med'], list):
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version='last')
break
vv_files = get_vv_files(obsid, version='last')
last_sched = "complete through mica v&v"
except LookupError:
logger.info("No V&V available")
vv = None
if vv is not None:
report_status['mica_vv'] = True
for file in vv_files:
newfile = os.path.join(outdir, os.path.basename(file))
if not os.path.exists(newfile):
logger.debug("linking {} into {}".format(file, outdir))
bash("ln -s {} {}".format(file, outdir))
asp_dir = asp_l1.get_obs_dirs(obsid)['last']
asp_logs = sorted(glob(os.path.join(asp_dir, "asp*log*gz")))
for log, interval in zip(asp_logs, vv['intervals']):
logmatch = re.search('(.*log)\.gz', os.path.basename(log))
if logmatch:
newlogname = "{}.txt".format(logmatch.group(1))
newlog = os.path.join(outdir, newlogname)
if not os.path.exists(newlog):
logger.debug("copying/gunzipping asp log {}".format(newlog))
logtext = gzip.open(log).readlines()
f = open(newlog, 'w')
f.writelines(logtext)
f.close()
interval['loglink'] = newlogname
aiprops = get_aiprops(obsid)
aiprops_template = jinja_env.get_template('aiprops.html')
aiprops_page = aiprops_template.render(obsid=obsid, aiprops=aiprops)
aiprops_page_file = os.path.join(outdir, 'aiprops.html')
logger.debug("AIPROPS report to {}".format(aiprops_page_file))
f = open(aiprops_page_file, 'w')
f.write(aiprops_page)
f.close()
props_template = jinja_env.get_template('props.html')
props_page = props_template.render(obsid=obsid, vv=vv)
props_page_file = os.path.join(outdir, 'props.html')
logger.debug("GS/FIDPROPS report to {}".format(props_page_file))
f = open(props_page_file, 'w')
f.write(props_page)
f.close()
for slot in vv['slots']:
if 'n_pts' not in vv['slots'][slot]:
continue
slot_template = jinja_env.get_template('vv_slots_single.html')
slot_page = slot_template.render(obsid=obsid,
vv=vv,
slot=slot)
slot_page_file = os.path.join(outdir, "slot_{}.html".format(slot))
logger.debug("VV SLOT report to {}".format(slot_page_file))
f = open(slot_page_file, 'w')
f.write(slot_page)
f.close()
official_notes = official_vv_notes(obsid, summary)
if official_notes:
for rep in official_notes:
if rep['comments'] == 'Hidden':
rep['comments'] = """
<A target="_blank" HREF="{}">{}</A><BR>(<A target="_blank" HREF="https://icxc.cfa.harvard.edu/soft/vv/vv_login.html">LOGIN</A> once first)</BR>""".format(links['vv']['link'], links['vv']['label'])
vv_template = jinja_env.get_template('vv.html')
vv['has_errors'] = (('errors' in vv) and (len(vv['errors']))) or None
vv_page = vv_template.render(obsid=obsid,
vv=vv,
obspar=run_obspar,
official_vv_notes=official_notes,
)
vv_page_file = os.path.join(outdir, 'vv.html')
logger.debug("VV report to {}".format(vv_page_file))
f = open(vv_page_file, 'w')
f.write(vv_page)
f.close()
cat_table = catalog_info(obs_sc['cat'], acqs, trak, vv)
for row, cat_row in zip(obs_sc['cat'], cat_table):
if row['type'] != 'FID':
if row['id'] is not None:
s = star_info(row['id'])
star_template = jinja_env.get_template('star.html')
star_page = star_template.render(
star=s['agasc_info'],
acqs=s['acqs'],
traks=s['traks'],
agg_acq=s['agg_acq'],
agg_trak=s['agg_trak'])
star_page_file = os.path.join(outdir, 'star_%d.html' % int(row['id']))
logger.debug("Writing out star info to {}".format(star_page_file))
f = open(star_page_file, 'w')
f.write(star_page)
f.close()
cat_row['idlink'] = (
'<A HREF="star_{id}.html" STYLE="text-decoration: none;"'
'ONMOUSEOVER="return overlib '
'(\'ACQ total:{n_acq} noid:{n_noid} <BR /> '
'GUI total:{n_gui} bad:{n_bad} fail:{n_fail} obc_bad:{n_obc_bad} '
'<BR /> Avg Mag {avg_mag:4.2f}\', WIDTH, 220);", ONMOUSEOUT="return nd();"> '
'{id}</A>'.format(id=int(row['id']),
n_acq=s['agg_acq']['n_acqs'],
n_noid=s['agg_acq']['n_acq_noid'],
n_gui=s['agg_trak']['n_guis'],
n_bad=s['agg_trak']['n_bad'],
n_fail=s['agg_trak']['n_fail'],
n_obc_bad=s['agg_trak']['n_obc_bad'],
avg_mag=(s['agg_trak']['avg_mag']
or s['agg_acq']['avg_mag']
or 13.94)))
else:
cat_row['idlink'] = " "
else:
if 'id' in row:
cat_row['idlink'] = int(row['id'])
else:
cat_row['idlink'] = ''
template = jinja_env.get_template('report.html')
page = template.render(cat_table=cat_table,
obs=obs,
sc=obs_sc,
vv=vv,
links=links,
target=summary,
temps=temps,
pred_temp=pred_temp,
er=er if er else None,
er_status=er_status,
last_sched=last_sched,
obsid=obsid,
version=version)
full_report_file = os.path.join(outdir, 'index.html')
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, 'w')
f.write(page)
f.close()
cat_file = os.path.join(outdir, 'star_table.json')
f = open(cat_file, 'w')
f.write(json.dumps(cat_table, sort_keys=True, indent=4))
f.close()
notes = {'report_version': REPORT_VERSION,
'vv_version': None,
'vv_revision': None,
'aspect_1_id': None,
'last_sched': last_sched,
'ocat_status': report_status.get('ocat'),
'long_term': str(report_status.get('long_term')),
'short_term': str(report_status.get('short_term')),
'starcheck': report_status.get('starcheck'),
'obsid': obsid,
'checked_date': DateTime().date}
if vv:
notes['vv_version'] = vv.get('vv_version')
notes['vv_revision'] = vv.get('revision')
notes['aspect_1_id'] = vv.get('aspect_1_id')
f = open(os.path.join(outdir, 'notes.json'), 'w')
f.write(json.dumps(notes,
sort_keys=True,
indent=4))
f.close()
save_state_in_db(obsid, notes)
def obs_links(obsid, sequence=None, plan=None):
links = {
'obscat': {
'label': "Target Param : {}".format(obsid),
'link': "https://icxc.cfa.harvard.edu/cgi-bin/mp/target_param.cgi?{}".format(obsid)},
'mp_dir': None,
'shortterm': None,
'fot_dir': None,
'fot_daily': None,
'cen_dash': None,
'starcheck_html': None,
'vv': None}
if sequence is not None:
links['seq_sum'] = {
'label': "Seq. Summary : {}".format(sequence),
'link': "https://icxc.harvard.edu/cgi-bin/mp/target.cgi?{}".format(sequence)}
mp_dir = None
# if this is a science observation, only try to get a star catalog if it has a home
# in the schedule either in the past or the near future
if plan is not None:
plan_date = DateTime("{:4d}:{:03d}:{:02d}:{:02d}:{:02d}.000".format(
plan.year, plan.dayofyear, plan.hour, plan.minute, plan.second))
if plan_date.secs < (DateTime() + 21).secs:
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
else:
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
# Check for centroid dashboard
strobs = "%05d" % obsid
chunk_dir = strobs[0:2]
cen_dash_file = "{}/{}/index.html".format(chunk_dir, strobs)
if os.path.exists(os.path.join('/proj/sot/ska/www/ASPECT_ICXC/centroid_dashboard/',
cen_dash_file)):
cen_url = 'https://icxc.cfa.harvard.edu/aspect/centroid_dashboard/'
links['cen_dash'] = {'link': "{}/{}".format(cen_url, cen_dash_file),
'label': 'Centroid Dashboard'}
if mp_dir is not None:
dir_match = re.match('/\d{4}/(\w{3}\d{4})/ofls(\w)', mp_dir)
mp_label = "{}{}".format(dir_match.group(1),
dir_match.group(2).upper())
mp_date_m = re.match('(\d{4}):(\d{3}):\d{2}:\d{2}:\d{2}\.\d{3}', mp_date)
if mp_date_m:
year = int(mp_date_m.group(1))
doy = int(mp_date_m.group(2))
dtm = datetime.datetime(year, 1, 1) + datetime.timedelta(doy - 1)
month = dtm.strftime("%b")
dom = dtm.strftime("%d")
links['fot_daily'] = {
'label': "Daily Plots {}:{:03d}".format(year, int(doy)),
'link': "{root}/{year}/{upper_month}/{lower_month}{day_of_month:02d}_{doy:03d}/".format(
root=DAILY_PLOT_ROOT, year=year, upper_month=month.upper(),
lower_month=month.lower(), day_of_month=int(dom), doy=int(doy))}
vv, vvid = official_vv(obsid)
links['shortterm'] = {'link': guess_shortterm(mp_dir),
'label': "Short Term Sched. {}".format(mp_label)}
links['fot_dir'] = {'link': guess_fot_summary(mp_dir),
'label': "FOT Approved Sched. {}".format(mp_label)}
links['starcheck_html'] = {'link': "{top}{mp_dir}starcheck.html#obsid{obsid}".format(top="https://icxc.harvard.edu/mp/mplogs", obsid=obsid, mp_dir=mp_dir),
'label': "Starcheck obsid {}".format(obsid)}
links['mp_dir'] = {'link': mp_dir,
'label': "Starcheck obsid {}".format(obsid)}
links['vv'] = {'link': vv,
'label': "CXCDS V&V (id={})".format(vvid)}
return links
