def _file_vv(obi):
"""
Save processed V&V data to per-obsid archive
"""
obsid = int(obi.info()['obsid'])
version = int(obi.info()['revision'])
# set up directory for data
strobs = "%05d_v%02d" % (obsid, version)
chunk_dir = strobs[0:2]
chunk_dir_path = os.path.join(FILES['data_root'], chunk_dir)
obs_dir = os.path.join(chunk_dir_path, strobs)
if not os.path.exists(obs_dir):
logger.info("making directory %s" % obs_dir)
os.makedirs(obs_dir)
else:
logger.info("obsid dir %s already exists" % obs_dir)
for f in glob(os.path.join(obi.tempdir, "*")):
os.chmod(f, 0775)
shutil.copy(f, obs_dir)
os.remove(f)
logger.info("moved VV files to {}".format(obs_dir))
os.removedirs(obi.tempdir)
logger.info("removed directory {}".format(obi.tempdir))
# make any desired link
obs_ln = os.path.join(FILES['data_root'], chunk_dir, "%05d" % obsid)
obs_ln_last = os.path.join(
FILES['data_root'], chunk_dir, "%05d_last" % obsid)
obsdirs = asp_l1_arch.get_obs_dirs(obsid)
isdefault = 0
if 'default' in obsdirs:
if (os.path.realpath(obsdirs[version])
== os.path.realpath(obsdirs['default'])):
if os.path.islink(obs_ln):
os.unlink(obs_ln)
os.symlink(os.path.relpath(obs_dir, chunk_dir_path), obs_ln)
isdefault = 1
if 'last' in obsdirs:
if ('default' in obsdirs
and (os.path.realpath(obsdirs['last'])
!= os.path.realpath(obsdirs['default']))
or 'default' not in obsdirs):
if (os.path.realpath(obsdirs[version])
== os.path.realpath(obsdirs['last'])):
if os.path.islink(obs_ln_last):
os.unlink(obs_ln_last)
os.symlink(os.path.relpath(obs_dir, chunk_dir_path),
obs_ln_last)
if ('default' in obsdirs
and (os.path.realpath(obsdirs['last'])
== os.path.realpath(obsdirs['default']))):
if os.path.exists(obs_ln_last):
os.unlink(obs_ln_last)
obi.isdefault = isdefault
def _file_vv(obi):
"""
Save processed V&V data to per-obsid archive
"""
obsid = int(obi.info()['obsid'])
version = int(obi.info()['revision'])
# set up directory for data
strobs = "%05d_v%02d" % (obsid, version)
chunk_dir = strobs[0:2]
chunk_dir_path = os.path.join(FILES['data_root'], chunk_dir)
obs_dir = os.path.join(chunk_dir_path, strobs)
if not os.path.exists(obs_dir):
logger.info("making directory %s" % obs_dir)
os.makedirs(obs_dir)
else:
logger.info("obsid dir %s already exists" % obs_dir)
for f in glob(os.path.join(obi.tempdir, "*")):
os.chmod(f, 0o775)
shutil.copy(f, obs_dir)
os.remove(f)
logger.info("moved VV files to {}".format(obs_dir))
os.removedirs(obi.tempdir)
logger.info("removed directory {}".format(obi.tempdir))
# make any desired link
obs_ln = os.path.join(FILES['data_root'], chunk_dir, "%05d" % obsid)
obs_ln_last = os.path.join(FILES['data_root'], chunk_dir,
"%05d_last" % obsid)
obsdirs = asp_l1_arch.get_obs_dirs(obsid)
isdefault = 0
if 'default' in obsdirs:
if (os.path.realpath(obsdirs[version]) == os.path.realpath(
obsdirs['default'])):
if os.path.islink(obs_ln):
os.unlink(obs_ln)
os.symlink(os.path.relpath(obs_dir, chunk_dir_path), obs_ln)
isdefault = 1
if 'last' in obsdirs:
if ('default' in obsdirs and (os.path.realpath(obsdirs['last']) !=
os.path.realpath(obsdirs['default']))
or 'default' not in obsdirs):
if (os.path.realpath(obsdirs[version]) == os.path.realpath(
obsdirs['last'])):
if os.path.islink(obs_ln_last):
os.unlink(obs_ln_last)
os.symlink(os.path.relpath(obs_dir, chunk_dir_path),
obs_ln_last)
if ('default' in obsdirs and (os.path.realpath(
obsdirs['last']) == os.path.realpath(obsdirs['default']))):
if os.path.exists(obs_ln_last):
os.unlink(obs_ln_last)
obi.isdefault = isdefault
def get_arch_vv(obsid, version='last'):
"""
Given obsid and version, find archived ASP1 and obspar products and
run V&V. Effort is made to find the obspar that was actually used during
creation of the ASP1 products.
:param obsid: obsid
:param version: 'last', 'default', or revision number of ASP1 products
:returns: mica.vv.Obi V&V object
"""
logger.info("Generating V&V for obsid {}".format(obsid))
asp_l1_dirs = asp_l1_arch.get_obs_dirs(obsid)
if asp_l1_dirs is None or version not in asp_l1_dirs:
raise LookupError(
"Requested version {} not in asp_l1 archive".format(version))
l1_dir = asp_l1_dirs[version]
# find the obspar that matches the requested aspect_1 products
# this is in the aspect processing table
asp_l1_proc = Ska.DBI.DBI(dbi="sqlite", server=FILES['asp1_proc_table'])
asp_obs = asp_l1_proc.fetchall(
"SELECT * FROM aspect_1_proc where obsid = {}".format(obsid))
asp_proc = None
if len(asp_obs) == 0:
return None
if version == 'last':
asp_proc = asp_obs[asp_obs['aspect_1_id'] == np.max(
asp_obs['aspect_1_id'])][0]
if version == 'default':
asp_proc = asp_obs[asp_obs['isdefault'] == 1][0]
if asp_proc is None:
asp_proc = asp_obs[asp_obs['revision'] == version][0]
obspar_dirs = obspar_arch.get_obs_dirs(obsid)
if obspar_dirs is None or asp_proc['obspar_version'] not in obspar_dirs:
# try to update the obspar archive with the missing version
config = obspar_arch.CONFIG.copy()
config.update(dict(obsid=obsid, version=asp_proc['obspar_version']))
oa = obsid_archive.ObsArchive(config)
oa.logger.setLevel(logging.INFO)
oa.logger.addHandler(logging.StreamHandler())
oa.update()
obspar_dirs = obspar_arch.get_obs_dirs(obsid)
try:
obspar_file = glob(
os.path.join(obspar_dirs[asp_proc['obspar_version']],
'axaf*par*'))[0]
except IndexError:
raise LookupError(f"Requested version {version} not in obspar archive")
return Obi(obspar_file, l1_dir, temproot=FILES['temp_root'])
def get_arch_vv(obsid, version='last'):
"""
Given obsid and version, find archived ASP1 and obspar products and
run V&V. Effort is made to find the obspar that was actually used during
creation of the ASP1 products.
:param obsid: obsid
:param version: 'last', 'default', or revision number of ASP1 products
:returns: mica.vv.Obi V&V object
"""
logger.info("Generating V&V for obsid {}".format(obsid))
asp_l1_dirs = asp_l1_arch.get_obs_dirs(obsid)
if asp_l1_dirs is None or version not in asp_l1_dirs:
raise LookupError("Requested version {} not in asp_l1 archive".format(version))
l1_dir = asp_l1_dirs[version]
# find the obspar that matches the requested aspect_1 products
# this is in the aspect processing table
asp_l1_proc = Ska.DBI.DBI(dbi="sqlite", server=FILES['asp1_proc_table'])
asp_obs = asp_l1_proc.fetchall(
"SELECT * FROM aspect_1_proc where obsid = {}".format(
obsid))
asp_proc = None
if len(asp_obs) == 0:
return None
if version == 'last':
asp_proc = asp_obs[asp_obs['aspect_1_id']
== np.max(asp_obs['aspect_1_id'])][0]
if version == 'default':
asp_proc = asp_obs[asp_obs['isdefault'] == 1][0]
if asp_proc is None:
asp_proc = asp_obs[asp_obs['revision'] == version][0]
obspar_dirs = obspar_arch.get_obs_dirs(obsid)
if asp_proc['obspar_version'] not in obspar_dirs:
# try to update the obspar archive with the missing version
config = obspar_arch.CONFIG.copy()
config.update(dict(obsid=obsid, version=asp_proc['obspar_version']))
oa = obsid_archive.ObsArchive(config)
oa.logger.setLevel(logging.INFO)
oa.logger.addHandler(logging.StreamHandler())
oa.update()
obspar_dirs = obspar_arch.get_obs_dirs(obsid)
obspar_file = glob(os.path.join(obspar_dirs[asp_proc['obspar_version']],
'axaf*par*'))[0]
return Obi(obspar_file, l1_dir, temproot=FILES['temp_root'])
def main(obsid):
report_root = REPORT_ROOT
strobs = "%05d" % obsid
chunk_dir = strobs[0:2]
topdir = os.path.join(report_root, chunk_dir)
outdir = os.path.join(topdir, strobs)
if not os.path.exists(outdir):
os.makedirs(outdir)
jinja_env = jinja2.Environment(
loader=jinja2.PackageLoader('mica.report'))
jinja_env.line_comment_prefix = '##'
jinja_env.line_statement_prefix = '#'
logger.info("Making report for {}".format(obsid))
logger.debug("Getting target info from axafapstat")
summary = target_summary(obsid)
# Links
logger.debug("Looking up obsid links")
all_progress = {'science': ['ocat',
'long_term', 'short_term',
'starcheck', 'observed',
'engineering', 'aspect_1',
'cxcds_vv', 'released'],
'er': ['starcheck', 'observed',
'engineering', 'cxcds_vv']}
report_status = {}
er = summary is None and obsid > 40000
progress = all_progress['er' if er else 'science']
if er:
links = obs_links(obsid)
else:
if summary is None:
raise ValueError("Obsid not found in target table")
report_status['ocat'] = summary['status']
links = obs_links(obsid, summary['seq_nbr'], summary['lts_lt_plan'])
if not er and (summary['status'] in
['canceled', 'unobserved', 'untriggered']):
logger.debug(
"Obsid {obsid} has status {status}".format(
obsid=obsid, status=summary['status']))
if summary is not None:
if summary['lts_lt_plan'] is not None:
report_status['long_term'] = summary['lts_lt_plan']
if summary['soe_st_sched_date']:
report_status['short_term'] = summary['soe_st_sched_date']
last_sched = ''
if not er:
if summary['lts_lt_plan']:
last_sched = "in LTS for {}".format(
str(summary['lts_lt_plan']))
if summary['soe_st_sched_date']:
last_sched = "in ST sched for {}".format(
str(summary['soe_st_sched_date']))
## Starcheck
logger.debug("Fetching starcheck catalog")
try:
if summary is not None and summary['lts_lt_plan'] is not None:
plan_date = Time(summary['lts_lt_plan'])
if plan_date.cxcsec > (CxoTime.now() + 21).secs:
raise LookupError("No starcheck expected for {} lts date".format(str(plan)))
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
obs_sc, mp_dir, status = get_starcheck(obsid)
logger.debug("Plotting starcheck catalog to {}".format(os.path.join(outdir, 'starcheck.png')))
if obs_sc['obs']['point_ra'] is None:
raise LookupError("Observation has no pointing.")
if len(obs_sc['cat']) == 0:
raise LookupError("Observation has no catalog")
fig, cat, obs = catalog.plot(obsid, mp_dir)
sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
fig.savefig(os.path.join(outdir, 'starcheck.png'))
plt.close('all')
except LookupError as detail:
logger.info("No starcheck catalog. Writing out OCAT info only")
logger.info(detail)
template = jinja_env.get_template('report.html')
page = template.render(obsid=obsid,
target=summary,
links=links,
temps=None,
pred_temp=None,
cat_table=None,
er=er if er else None,
last_sched=last_sched,
obs=None,
version=version)
full_report_file = os.path.join(outdir, 'index.html')
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, 'w')
f.write(page)
f.close()
notes = {'report_version': REPORT_VERSION,
'vv_version': None,
'vv_revision': None,
'aspect_1_id': None,
'last_sched': last_sched,
'ocat_status': report_status.get('ocat'),
'long_term': str(report_status.get('long_term')),
'short_term': str(report_status.get('short_term')),
'starcheck': report_status.get('starcheck'),
'obsid': obsid,
'checked_date': CxoTime.now().date}
f = open(os.path.join(outdir, 'notes.json'), 'w')
f.write(json.dumps(notes,
sort_keys=True,
indent=4))
f.close()
save_state_in_db(obsid, notes)
return
if not er and 'shortterm' in links:
dir_match = re.match(r'/\d{4}/(\w{3}\d{4})/ofls(\w)', mp_dir)
mp_label = "{}{}".format(dir_match.group(1),
dir_match.group(2).upper())
last_sched = 'in <A HREF="{}">{}</A> at {}'.format(
links['shortterm']['link'], mp_label, str(obs_sc['obs']['mp_starcat_time']))
report_status['starcheck'] = mp_dir
# engineering data available
logger.debug("Getting acq and trak stats")
acqs = get_obs_acq_stats(obsid)
trak = get_obs_trak_stats(obsid)
temps = get_obs_temps(obsid, outdir)
pred_temp = sc.get('pred_temp')
if acqs or trak:
last_sched = "eng. data available"
er_status = None
if er:
stat_map = {'ran': 'ran on',
'approved': 'approved for',
'ran_pretimelines': 'ran on',
'planned': 'planned for'}
er_status = "{} {}".format(stat_map[status], obs_sc['obs']['mp_starcat_time'])
run_obspar = None
vv = None
logger.info("Processing ER; no V&V available")
else:
# obspar ingested
try:
run_obspar = obspar.get_obspar(obsid, version='last')
except:
run_obspar = None
# v&v available
try:
vv = get_vv(obsid, version='last')
except LookupError:
vv = None
try:
if vv is None or 'vv_version' not in vv or vv['vv_version'] < WANT_VV_VERSION:
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version='last')
for slot in vv['slots']:
if isinstance(vv['slots'][slot]['dy_med'], list):
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version='last')
break
vv_files = get_vv_files(obsid, version='last')
last_sched = "complete through mica v&v"
except LookupError:
logger.info("No V&V available")
vv = None
if vv is not None:
report_status['mica_vv'] = True
for file in vv_files:
newfile = os.path.join(outdir, os.path.basename(file))
if not os.path.exists(newfile):
logger.debug("linking {} into {}".format(file, outdir))
os.symlink(file, newfile)
asp_dir = asp_l1.get_obs_dirs(obsid)['last']
asp_logs = sorted(glob(os.path.join(asp_dir, "asp_l1_f*log*gz")))
for log, interval in zip(asp_logs, vv['intervals']):
logmatch = re.search(r'(.*log)\.gz', os.path.basename(log))
if logmatch:
newlogname = "{}.txt".format(logmatch.group(1))
newlog = os.path.join(outdir, newlogname)
if not os.path.exists(newlog):
logger.debug("copying/gunzipping asp log {}".format(newlog))
logtext = gzip.open(log, 'rt').readlines()
f = open(newlog, 'w')
f.writelines(logtext)
f.close()
interval['loglink'] = newlogname
aiprops = get_aiprops(obsid)
aiprops_template = jinja_env.get_template('aiprops.html')
aiprops_page = aiprops_template.render(obsid=obsid, aiprops=aiprops)
aiprops_page_file = os.path.join(outdir, 'aiprops.html')
logger.debug("AIPROPS report to {}".format(aiprops_page_file))
f = open(aiprops_page_file, 'w')
f.write(aiprops_page)
f.close()
props_template = jinja_env.get_template('props.html')
props_page = props_template.render(obsid=obsid, vv=vv)
props_page_file = os.path.join(outdir, 'props.html')
logger.debug("GS/FIDPROPS report to {}".format(props_page_file))
f = open(props_page_file, 'w')
f.write(props_page)
f.close()
for slot in vv['slots']:
if 'n_pts' not in vv['slots'][slot]:
continue
slot_template = jinja_env.get_template('vv_slots_single.html')
slot_page = slot_template.render(obsid=obsid,
vv=vv,
slot=slot)
slot_page_file = os.path.join(outdir, "slot_{}.html".format(slot))
logger.debug("VV SLOT report to {}".format(slot_page_file))
f = open(slot_page_file, 'w')
f.write(slot_page)
f.close()
official_notes = official_vv_notes(obsid, summary)
if official_notes:
for rep in official_notes:
if rep['comments'] == 'Hidden':
rep['comments'] = """
<A target="_blank" HREF="{}">{}</A><BR>(<A target="_blank" HREF="https://icxc.cfa.harvard.edu/soft/vv/vv_login.html">LOGIN</A> once first)</BR>""".format(links['vv']['link'], links['vv']['label'])
vv_template = jinja_env.get_template('vv.html')
vv['has_errors'] = (('errors' in vv) and (len(vv['errors']))) or None
vv_page = vv_template.render(obsid=obsid,
vv=vv,
obspar=run_obspar,
official_vv_notes=official_notes,
)
vv_page_file = os.path.join(outdir, 'vv.html')
logger.debug("VV report to {}".format(vv_page_file))
f = open(vv_page_file, 'w')
f.write(vv_page)
f.close()
cat_table = catalog_info(obs_sc['cat'], acqs, trak, vv)
for row, cat_row in zip(obs_sc['cat'], cat_table):
if row['type'] != 'FID':
if row['id'] is not None:
s = star_info(row['id'])
star_template = jinja_env.get_template('star.html')
star_page = star_template.render(
star=s['agasc_info'],
acqs=s['acqs'],
traks=s['traks'],
agg_acq=s['agg_acq'],
agg_trak=s['agg_trak'])
star_page_file = os.path.join(outdir, 'star_%d.html' % int(row['id']))
logger.debug("Writing out star info to {}".format(star_page_file))
f = open(star_page_file, 'w')
f.write(star_page)
f.close()
cat_row['idlink'] = (
'<A HREF="star_{id}.html" STYLE="text-decoration: none;"'
'ONMOUSEOVER="return overlib '
'(\'ACQ total:{n_acq} noid:{n_noid} <BR /> '
'GUI total:{n_gui} bad:{n_bad} fail:{n_fail} obc_bad:{n_obc_bad} '
'<BR /> Avg Mag {avg_mag:4.2f}\', WIDTH, 220);", ONMOUSEOUT="return nd();"> '
'{id}</A>'.format(id=int(row['id']),
n_acq=s['agg_acq']['n_acqs'],
n_noid=s['agg_acq']['n_acq_noid'],
n_gui=s['agg_trak']['n_guis'],
n_bad=s['agg_trak']['n_bad'],
n_fail=s['agg_trak']['n_fail'],
n_obc_bad=s['agg_trak']['n_obc_bad'],
avg_mag=(s['agg_trak']['avg_mag']
or s['agg_acq']['avg_mag']
or 13.94)))
else:
cat_row['idlink'] = " "
else:
if 'id' in row:
cat_row['idlink'] = int(row['id'])
else:
cat_row['idlink'] = ''
template = jinja_env.get_template('report.html')
page = template.render(cat_table=cat_table,
obs=obs,
sc=obs_sc,
vv=vv,
links=links,
target=summary,
temps=temps,
pred_temp=pred_temp,
er=er if er else None,
er_status=er_status,
last_sched=last_sched,
obsid=obsid,
version=version)
full_report_file = os.path.join(outdir, 'index.html')
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, 'w')
f.write(page)
f.close()
cat_file = os.path.join(outdir, 'star_table.json')
f = open(cat_file, 'w')
f.write(json.dumps(cat_table, sort_keys=True, indent=4))
f.close()
notes = {'report_version': REPORT_VERSION,
'vv_version': None,
'vv_revision': None,
'aspect_1_id': None,
'last_sched': last_sched,
'ocat_status': report_status.get('ocat'),
'long_term': str(report_status.get('long_term')),
'short_term': str(report_status.get('short_term')),
'starcheck': report_status.get('starcheck'),
'obsid': obsid,
'checked_date': CxoTime.now().date}
if vv:
notes['vv_version'] = vv.get('vv_version')
notes['vv_revision'] = vv.get('revision')
notes['aspect_1_id'] = vv.get('aspect_1_id')
f = open(os.path.join(outdir, 'notes.json'), 'w')
f.write(json.dumps(notes,
sort_keys=True,
indent=4))
f.close()
save_state_in_db(obsid, notes)
def main(obsid, config=None, report_root=None):
if config is None:
config = DEFAULT_CONFIG
if report_root is None:
report_root = config["report_root"]
global ACA_DB
if ACA_DB is None or not ACA_DB.conn._is_connected:
ACA_DB = Ska.DBI.DBI(dbi="sybase", server="sybase", user="******")
strobs = "%05d" % obsid
chunk_dir = strobs[0:2]
topdir = os.path.join(report_root, chunk_dir)
outdir = os.path.join(topdir, strobs)
if not os.path.exists(outdir):
os.makedirs(outdir)
jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.join(os.environ["SKA"], "data", "mica", "templates"))
)
jinja_env.line_comment_prefix = "##"
jinja_env.line_statement_prefix = "#"
logger.info("Making report for {}".format(obsid))
logger.debug("Getting target info from axafapstat")
summary = target_summary(obsid)
# Links
logger.debug("Looking up obsid links")
all_progress = {
"science": [
"ocat",
"long_term",
"short_term",
"starcheck",
"observed",
"engineering",
"aspect_1",
"cxcds_vv",
"released",
],
"er": ["starcheck", "observed", "engineering", "cxcds_vv"],
}
report_status = {}
er = summary is None and obsid > 40000
progress = all_progress["er" if er else "science"]
if er:
links = obs_links(obsid)
else:
if summary is None:
raise ValueError("Obsid not found in target table")
report_status["ocat"] = summary["status"]
links = obs_links(obsid, summary["seq_nbr"], summary["lts_lt_plan"])
if not er and (summary["status"] in ["canceled", "unobserved", "untriggered"]):
logger.debug("Obsid {obsid} has status {status}".format(obsid=obsid, status=summary["status"]))
if summary is not None:
if summary["lts_lt_plan"] is not None:
report_status["long_term"] = summary["lts_lt_plan"]
if summary["soe_st_sched_date"]:
report_status["short_term"] = summary["soe_st_sched_date"]
last_sched = ""
if not er:
if summary["lts_lt_plan"]:
last_sched = "in LTS for {}".format(str(summary["lts_lt_plan"]))
if summary["soe_st_sched_date"]:
last_sched = "in ST sched for {}".format(str(summary["soe_st_sched_date"]))
## Starcheck
logger.debug("Fetching starcheck catalog")
try:
if summary is not None and summary["lts_lt_plan"] is not None:
plan = summary["lts_lt_plan"]
plan_date = DateTime(
"{:4d}:{:03d}:{:02d}:{:02d}:{:02d}.000".format(
plan.year, plan.dayofyear, plan.hour, plan.minute, plan.second
)
)
if plan_date.secs > (DateTime() + 21).secs:
raise LookupError("No starcheck expected for {} lts date".format(str(plan)))
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
obs_sc, mp_dir, status = get_starcheck(obsid)
logger.debug("Plotting starcheck catalog to {}".format(os.path.join(outdir, "starcheck.png")))
if obs_sc["obs"][0]["point_ra"] is None:
raise LookupError("Observation has no pointing.")
if len(obs_sc["catalog"]) == 0:
raise LookupError("Observation has no catalog")
fig, cat, obs = catalog.plot(obsid, mp_dir)
sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
fig.savefig(os.path.join(outdir, "starcheck.png"))
plt.close("all")
except LookupError as detail:
logger.info("No starcheck catalog. Writing out OCAT info only")
logger.info(detail)
template = jinja_env.get_template("report.html")
page = template.render(
obsid=obsid,
target=summary,
links=links,
temps=None,
pred_temp=None,
cat_table=None,
er=er if er else None,
last_sched=last_sched,
obs=None,
version=version,
)
full_report_file = os.path.join(outdir, "index.html")
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, "w")
f.write(page)
f.close()
notes = {
"report_version": REPORT_VERSION,
"vv_version": None,
"vv_revision": None,
"aspect_1_id": None,
"last_sched": last_sched,
"ocat_status": report_status.get("ocat"),
"long_term": str(report_status.get("long_term")),
"short_term": str(report_status.get("short_term")),
"starcheck": report_status.get("starcheck"),
"obsid": obsid,
"checked_date": DateTime().date,
}
f = open(os.path.join(outdir, "notes.json"), "w")
f.write(json.dumps(notes, sort_keys=True, indent=4))
f.close()
save_state_in_db(obsid, notes, config)
return
if not er and "shortterm" in links:
dir_match = re.match("/\d{4}/(\w{3}\d{4})/ofls(\w)", mp_dir)
mp_label = "{}{}".format(dir_match.group(1), dir_match.group(2).upper())
last_sched = 'in <A HREF="{}">{}</A> at {}'.format(
links["shortterm"]["link"], mp_label, str(obs_sc["obs"][0]["mp_starcat_time"])
)
report_status["starcheck"] = mp_dir
# engineering data available
logger.debug("Getting acq and trak stats")
acqs = get_obs_acq_stats(obsid)
trak = get_obs_trak_stats(obsid)
temps = get_obs_temps(obsid, outdir)
pred_temp = sc["pred_temp"]
if acqs or trak:
last_sched = "eng. data available"
er_status = None
if er:
stat_map = {"ran": "ran on", "approved": "approved for", "ran_pretimelines": "ran on", "planned": "planned for"}
er_status = "{} {}".format(stat_map[status], obs_sc["obs"][0]["mp_starcat_time"])
run_obspar = None
vv = None
logger.info("Processing ER; no V&V available")
else:
# obspar ingested
try:
run_obspar = obspar.get_obspar(obsid, version="last")
except:
run_obspar = None
# v&v available
try:
vv = get_vv(obsid, version="last")
except LookupError:
vv = None
try:
if vv is None or "vv_version" not in vv or vv["vv_version"] < WANT_VV_VERSION:
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version="last")
for slot in vv["slots"]:
if isinstance(vv["slots"][slot]["dy_med"], list):
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version="last")
break
vv_files = get_vv_files(obsid, version="last")
last_sched = "complete through mica v&v"
except LookupError:
logger.info("No V&V available")
vv = None
if vv is not None:
report_status["mica_vv"] = True
for file in vv_files:
newfile = os.path.join(outdir, os.path.basename(file))
if not os.path.exists(newfile):
logger.debug("linking {} into {}".format(file, outdir))
bash("ln -s {} {}".format(file, outdir))
asp_dir = asp_l1.get_obs_dirs(obsid)["last"]
asp_logs = sorted(glob(os.path.join(asp_dir, "asp*log*gz")))
for log, interval in zip(asp_logs, vv["intervals"]):
logmatch = re.search("(.*log)\.gz", os.path.basename(log))
if logmatch:
newlogname = "{}.txt".format(logmatch.group(1))
newlog = os.path.join(outdir, newlogname)
if not os.path.exists(newlog):
logger.debug("copying/gunzipping asp log {}".format(newlog))
logtext = gzip.open(log).readlines()
f = open(newlog, "w")
f.writelines(logtext)
f.close()
interval["loglink"] = newlogname
aiprops = get_aiprops(obsid)
aiprops_template = jinja_env.get_template("aiprops.html")
aiprops_page = aiprops_template.render(obsid=obsid, aiprops=aiprops)
aiprops_page_file = os.path.join(outdir, "aiprops.html")
logger.debug("AIPROPS report to {}".format(aiprops_page_file))
f = open(aiprops_page_file, "w")
f.write(aiprops_page)
f.close()
props_template = jinja_env.get_template("props.html")
props_page = props_template.render(obsid=obsid, vv=vv)
props_page_file = os.path.join(outdir, "props.html")
logger.debug("GS/FIDPROPS report to {}".format(props_page_file))
f = open(props_page_file, "w")
f.write(props_page)
f.close()
for slot in vv["slots"]:
if "n_pts" not in vv["slots"][slot]:
continue
slot_template = jinja_env.get_template("vv_slots_single.html")
slot_page = slot_template.render(obsid=obsid, vv=vv, slot=slot)
slot_page_file = os.path.join(outdir, "slot_{}.html".format(slot))
logger.debug("VV SLOT report to {}".format(slot_page_file))
f = open(slot_page_file, "w")
f.write(slot_page)
f.close()
official_notes = official_vv_notes(obsid, summary)
if official_notes:
for rep in official_notes:
if rep["comments"] == "Hidden":
rep[
"comments"
] = """
<A target="_blank" HREF="{}">{}</A><BR>(<A target="_blank" HREF="https://icxc.cfa.harvard.edu/soft/vv/vv_login.html">LOGIN</A> once first)</BR>""".format(
links["vv"]["link"], links["vv"]["label"]
)
vv_template = jinja_env.get_template("vv.html")
vv["has_errors"] = (("errors" in vv) and (len(vv["errors"]))) or None
vv_page = vv_template.render(obsid=obsid, vv=vv, obspar=run_obspar, official_vv_notes=official_notes)
vv_page_file = os.path.join(outdir, "vv.html")
logger.debug("VV report to {}".format(vv_page_file))
f = open(vv_page_file, "w")
f.write(vv_page)
f.close()
cat_table = catalog_info(obs_sc["catalog"], acqs, trak, vv)
for row, cat_row in zip(obs_sc["catalog"], cat_table):
if row["type"] != "FID":
if row["id"] is not None:
s = star_info(row["id"])
star_template = jinja_env.get_template("star.html")
star_page = star_template.render(
star=s["agasc_info"], acqs=s["acqs"], traks=s["traks"], agg_acq=s["agg_acq"], agg_trak=s["agg_trak"]
)
star_page_file = os.path.join(outdir, "star_%d.html" % int(row["id"]))
logger.debug("Writing out star info to {}".format(star_page_file))
f = open(star_page_file, "w")
f.write(star_page)
f.close()
cat_row["idlink"] = (
'<A HREF="star_{id}.html" STYLE="text-decoration: none;"'
'ONMOUSEOVER="return overlib '
"('ACQ total:{n_acq} noid:{n_noid} <BR /> "
"GUI total:{n_gui} bad:{n_bad} fail:{n_fail} obc_bad:{n_obc_bad} "
'<BR /> Avg Mag {avg_mag:4.2f}\', WIDTH, 220);", ONMOUSEOUT="return nd();"> '
"{id}</A>".format(
id=int(row["id"]),
n_acq=s["agg_acq"]["n_acqs"],
n_noid=s["agg_acq"]["n_acq_noid"],
n_gui=s["agg_trak"]["n_guis"],
n_bad=s["agg_trak"]["n_bad"],
n_fail=s["agg_trak"]["n_fail"],
n_obc_bad=s["agg_trak"]["n_obc_bad"],
avg_mag=(s["agg_trak"]["avg_mag"] or s["agg_acq"]["avg_mag"] or 13.94),
)
)
else:
cat_row["idlink"] = " "
else:
if "id" in row:
cat_row["idlink"] = int(row["id"])
else:
cat_row["idlink"] = ""
template = jinja_env.get_template("report.html")
page = template.render(
cat_table=cat_table,
obs=obs,
sc=obs_sc,
vv=vv,
links=links,
target=summary,
temps=temps,
pred_temp=pred_temp,
er=er if er else None,
er_status=er_status,
last_sched=last_sched,
obsid=obsid,
version=version,
)
full_report_file = os.path.join(outdir, "index.html")
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, "w")
f.write(page)
f.close()
cat_file = os.path.join(outdir, "star_table.json")
f = open(cat_file, "w")
f.write(json.dumps(cat_table, sort_keys=True, indent=4))
f.close()
notes = {
"report_version": REPORT_VERSION,
"vv_version": None,
"vv_revision": None,
"aspect_1_id": None,
"last_sched": last_sched,
"ocat_status": report_status.get("ocat"),
"long_term": str(report_status.get("long_term")),
"short_term": str(report_status.get("short_term")),
"starcheck": report_status.get("starcheck"),
"obsid": obsid,
"checked_date": DateTime().date,
}
if vv:
notes["vv_version"] = vv.get("vv_version")
notes["vv_revision"] = vv.get("revision")
notes["aspect_1_id"] = vv.get("aspect_1_id")
f = open(os.path.join(outdir, "notes.json"), "w")
f.write(json.dumps(notes, sort_keys=True, indent=4))
f.close()
save_state_in_db(obsid, notes, config)
def main(obsid):
report_root = REPORT_ROOT
strobs = "%05d" % obsid
chunk_dir = strobs[0:2]
topdir = os.path.join(report_root, chunk_dir)
outdir = os.path.join(topdir, strobs)
if not os.path.exists(outdir):
os.makedirs(outdir)
jinja_env = jinja2.Environment(
loader=jinja2.PackageLoader('mica.report'))
jinja_env.line_comment_prefix = '##'
jinja_env.line_statement_prefix = '#'
logger.info("Making report for {}".format(obsid))
logger.debug("Getting target info from axafapstat")
summary = target_summary(obsid)
# Links
logger.debug("Looking up obsid links")
all_progress = {'science': ['ocat',
'long_term', 'short_term',
'starcheck', 'observed',
'engineering', 'aspect_1',
'cxcds_vv', 'released'],
'er': ['starcheck', 'observed',
'engineering', 'cxcds_vv']}
report_status = {}
er = summary is None and obsid > 40000
progress = all_progress['er' if er else 'science']
if er:
links = obs_links(obsid)
else:
if summary is None:
raise ValueError("Obsid not found in target table")
report_status['ocat'] = summary['status']
links = obs_links(obsid, summary['seq_nbr'], summary['lts_lt_plan'])
if not er and (summary['status'] in
['canceled', 'unobserved', 'untriggered']):
logger.debug(
"Obsid {obsid} has status {status}".format(
obsid=obsid, status=summary['status']))
if summary is not None:
if summary['lts_lt_plan'] is not None:
report_status['long_term'] = summary['lts_lt_plan']
if summary['soe_st_sched_date']:
report_status['short_term'] = summary['soe_st_sched_date']
last_sched = ''
if not er:
if summary['lts_lt_plan']:
last_sched = "in LTS for {}".format(
str(summary['lts_lt_plan']))
if summary['soe_st_sched_date']:
last_sched = "in ST sched for {}".format(
str(summary['soe_st_sched_date']))
## Starcheck
logger.debug("Fetching starcheck catalog")
try:
if summary is not None and summary['lts_lt_plan'] is not None:
plan = summary['lts_lt_plan']
plan_date = DateTime("{:4d}:{:03d}:{:02d}:{:02d}:{:02d}.000".format(
plan.year, plan.dayofyear, plan.hour, plan.minute, plan.second))
if plan_date.secs > (DateTime() + 21).secs:
raise LookupError("No starcheck expected for {} lts date".format(str(plan)))
mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
obs_sc, mp_dir, status = get_starcheck(obsid)
logger.debug("Plotting starcheck catalog to {}".format(os.path.join(outdir, 'starcheck.png')))
if obs_sc['obs']['point_ra'] is None:
raise LookupError("Observation has no pointing.")
if len(obs_sc['cat']) == 0:
raise LookupError("Observation has no catalog")
fig, cat, obs = catalog.plot(obsid, mp_dir)
sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
fig.savefig(os.path.join(outdir, 'starcheck.png'))
plt.close('all')
except LookupError as detail:
logger.info("No starcheck catalog. Writing out OCAT info only")
logger.info(detail)
template = jinja_env.get_template('report.html')
page = template.render(obsid=obsid,
target=summary,
links=links,
temps=None,
pred_temp=None,
cat_table=None,
er=er if er else None,
last_sched=last_sched,
obs=None,
version=version)
full_report_file = os.path.join(outdir, 'index.html')
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, 'w')
f.write(page)
f.close()
notes = {'report_version': REPORT_VERSION,
'vv_version': None,
'vv_revision': None,
'aspect_1_id': None,
'last_sched': last_sched,
'ocat_status': report_status.get('ocat'),
'long_term': str(report_status.get('long_term')),
'short_term': str(report_status.get('short_term')),
'starcheck': report_status.get('starcheck'),
'obsid': obsid,
'checked_date': DateTime().date}
f = open(os.path.join(outdir, 'notes.json'), 'w')
f.write(json.dumps(notes,
sort_keys=True,
indent=4))
f.close()
save_state_in_db(obsid, notes)
return
if not er and 'shortterm' in links:
dir_match = re.match('/\d{4}/(\w{3}\d{4})/ofls(\w)', mp_dir)
mp_label = "{}{}".format(dir_match.group(1),
dir_match.group(2).upper())
last_sched = 'in <A HREF="{}">{}</A> at {}'.format(
links['shortterm']['link'], mp_label, str(obs_sc['obs']['mp_starcat_time']))
report_status['starcheck'] = mp_dir
# engineering data available
logger.debug("Getting acq and trak stats")
acqs = get_obs_acq_stats(obsid)
trak = get_obs_trak_stats(obsid)
temps = get_obs_temps(obsid, outdir)
pred_temp = sc.get('pred_temp')
if acqs or trak:
last_sched = "eng. data available"
er_status = None
if er:
stat_map = {'ran': 'ran on',
'approved': 'approved for',
'ran_pretimelines': 'ran on',
'planned': 'planned for'}
er_status = "{} {}".format(stat_map[status], obs_sc['obs']['mp_starcat_time'])
run_obspar = None
vv = None
logger.info("Processing ER; no V&V available")
else:
# obspar ingested
try:
run_obspar = obspar.get_obspar(obsid, version='last')
except:
run_obspar = None
# v&v available
try:
vv = get_vv(obsid, version='last')
except LookupError:
vv = None
try:
if vv is None or 'vv_version' not in vv or vv['vv_version'] < WANT_VV_VERSION:
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version='last')
for slot in vv['slots']:
if isinstance(vv['slots'][slot]['dy_med'], list):
mica.vv.process.process(obsid, version="last")
vv = get_vv(obsid, version='last')
break
vv_files = get_vv_files(obsid, version='last')
last_sched = "complete through mica v&v"
except LookupError:
logger.info("No V&V available")
vv = None
if vv is not None:
report_status['mica_vv'] = True
for file in vv_files:
newfile = os.path.join(outdir, os.path.basename(file))
if not os.path.exists(newfile):
logger.debug("linking {} into {}".format(file, outdir))
bash("ln -s {} {}".format(file, outdir))
asp_dir = asp_l1.get_obs_dirs(obsid)['last']
asp_logs = sorted(glob(os.path.join(asp_dir, "asp*log*gz")))
for log, interval in zip(asp_logs, vv['intervals']):
logmatch = re.search('(.*log)\.gz', os.path.basename(log))
if logmatch:
newlogname = "{}.txt".format(logmatch.group(1))
newlog = os.path.join(outdir, newlogname)
if not os.path.exists(newlog):
logger.debug("copying/gunzipping asp log {}".format(newlog))
logtext = gzip.open(log).readlines()
f = open(newlog, 'w')
f.writelines(logtext)
f.close()
interval['loglink'] = newlogname
aiprops = get_aiprops(obsid)
aiprops_template = jinja_env.get_template('aiprops.html')
aiprops_page = aiprops_template.render(obsid=obsid, aiprops=aiprops)
aiprops_page_file = os.path.join(outdir, 'aiprops.html')
logger.debug("AIPROPS report to {}".format(aiprops_page_file))
f = open(aiprops_page_file, 'w')
f.write(aiprops_page)
f.close()
props_template = jinja_env.get_template('props.html')
props_page = props_template.render(obsid=obsid, vv=vv)
props_page_file = os.path.join(outdir, 'props.html')
logger.debug("GS/FIDPROPS report to {}".format(props_page_file))
f = open(props_page_file, 'w')
f.write(props_page)
f.close()
for slot in vv['slots']:
if 'n_pts' not in vv['slots'][slot]:
continue
slot_template = jinja_env.get_template('vv_slots_single.html')
slot_page = slot_template.render(obsid=obsid,
vv=vv,
slot=slot)
slot_page_file = os.path.join(outdir, "slot_{}.html".format(slot))
logger.debug("VV SLOT report to {}".format(slot_page_file))
f = open(slot_page_file, 'w')
f.write(slot_page)
f.close()
official_notes = official_vv_notes(obsid, summary)
if official_notes:
for rep in official_notes:
if rep['comments'] == 'Hidden':
rep['comments'] = """
<A target="_blank" HREF="{}">{}</A><BR>(<A target="_blank" HREF="https://icxc.cfa.harvard.edu/soft/vv/vv_login.html">LOGIN</A> once first)</BR>""".format(links['vv']['link'], links['vv']['label'])
vv_template = jinja_env.get_template('vv.html')
vv['has_errors'] = (('errors' in vv) and (len(vv['errors']))) or None
vv_page = vv_template.render(obsid=obsid,
vv=vv,
obspar=run_obspar,
official_vv_notes=official_notes,
)
vv_page_file = os.path.join(outdir, 'vv.html')
logger.debug("VV report to {}".format(vv_page_file))
f = open(vv_page_file, 'w')
f.write(vv_page)
f.close()
cat_table = catalog_info(obs_sc['cat'], acqs, trak, vv)
for row, cat_row in zip(obs_sc['cat'], cat_table):
if row['type'] != 'FID':
if row['id'] is not None:
s = star_info(row['id'])
star_template = jinja_env.get_template('star.html')
star_page = star_template.render(
star=s['agasc_info'],
acqs=s['acqs'],
traks=s['traks'],
agg_acq=s['agg_acq'],
agg_trak=s['agg_trak'])
star_page_file = os.path.join(outdir, 'star_%d.html' % int(row['id']))
logger.debug("Writing out star info to {}".format(star_page_file))
f = open(star_page_file, 'w')
f.write(star_page)
f.close()
cat_row['idlink'] = (
'<A HREF="star_{id}.html" STYLE="text-decoration: none;"'
'ONMOUSEOVER="return overlib '
'(\'ACQ total:{n_acq} noid:{n_noid} <BR /> '
'GUI total:{n_gui} bad:{n_bad} fail:{n_fail} obc_bad:{n_obc_bad} '
'<BR /> Avg Mag {avg_mag:4.2f}\', WIDTH, 220);", ONMOUSEOUT="return nd();"> '
'{id}</A>'.format(id=int(row['id']),
n_acq=s['agg_acq']['n_acqs'],
n_noid=s['agg_acq']['n_acq_noid'],
n_gui=s['agg_trak']['n_guis'],
n_bad=s['agg_trak']['n_bad'],
n_fail=s['agg_trak']['n_fail'],
n_obc_bad=s['agg_trak']['n_obc_bad'],
avg_mag=(s['agg_trak']['avg_mag']
or s['agg_acq']['avg_mag']
or 13.94)))
else:
cat_row['idlink'] = " "
else:
if 'id' in row:
cat_row['idlink'] = int(row['id'])
else:
cat_row['idlink'] = ''
template = jinja_env.get_template('report.html')
page = template.render(cat_table=cat_table,
obs=obs,
sc=obs_sc,
vv=vv,
links=links,
target=summary,
temps=temps,
pred_temp=pred_temp,
er=er if er else None,
er_status=er_status,
last_sched=last_sched,
obsid=obsid,
version=version)
full_report_file = os.path.join(outdir, 'index.html')
logger.info("Writing out full report to {}".format(full_report_file))
f = open(full_report_file, 'w')
f.write(page)
f.close()
cat_file = os.path.join(outdir, 'star_table.json')
f = open(cat_file, 'w')
f.write(json.dumps(cat_table, sort_keys=True, indent=4))
f.close()
notes = {'report_version': REPORT_VERSION,
'vv_version': None,
'vv_revision': None,
'aspect_1_id': None,
'last_sched': last_sched,
'ocat_status': report_status.get('ocat'),
'long_term': str(report_status.get('long_term')),
'short_term': str(report_status.get('short_term')),
'starcheck': report_status.get('starcheck'),
'obsid': obsid,
'checked_date': DateTime().date}
if vv:
notes['vv_version'] = vv.get('vv_version')
notes['vv_revision'] = vv.get('revision')
notes['aspect_1_id'] = vv.get('aspect_1_id')
f = open(os.path.join(outdir, 'notes.json'), 'w')
f.write(json.dumps(notes,
sort_keys=True,
indent=4))
f.close()
save_state_in_db(obsid, notes)