예제 #1
0
def _file_vv(obi):
    """
    Save processed V&V data to per-obsid archive
    """
    obsid = int(obi.info()['obsid'])
    version = int(obi.info()['revision'])
    # set up directory for data
    strobs = "%05d_v%02d" % (obsid, version)
    chunk_dir = strobs[0:2]
    chunk_dir_path = os.path.join(FILES['data_root'], chunk_dir)
    obs_dir = os.path.join(chunk_dir_path, strobs)
    if not os.path.exists(obs_dir):
        logger.info("making directory %s" % obs_dir)
        os.makedirs(obs_dir)
    else:
        logger.info("obsid dir %s already exists" % obs_dir)
    for f in glob(os.path.join(obi.tempdir, "*")):
        os.chmod(f, 0775)
        shutil.copy(f, obs_dir)
        os.remove(f)
    logger.info("moved VV files to {}".format(obs_dir))
    os.removedirs(obi.tempdir)
    logger.info("removed directory {}".format(obi.tempdir))
    # make any desired link
    obs_ln = os.path.join(FILES['data_root'], chunk_dir, "%05d" % obsid)
    obs_ln_last = os.path.join(
        FILES['data_root'], chunk_dir, "%05d_last" % obsid)
    obsdirs = asp_l1_arch.get_obs_dirs(obsid)
    isdefault = 0
    if 'default' in obsdirs:
        if (os.path.realpath(obsdirs[version])
                == os.path.realpath(obsdirs['default'])):
            if os.path.islink(obs_ln):
                os.unlink(obs_ln)
            os.symlink(os.path.relpath(obs_dir, chunk_dir_path), obs_ln)
            isdefault = 1
    if 'last' in obsdirs:
        if ('default' in obsdirs
                and (os.path.realpath(obsdirs['last'])
                     != os.path.realpath(obsdirs['default']))
                or 'default' not in obsdirs):
            if (os.path.realpath(obsdirs[version])
                    == os.path.realpath(obsdirs['last'])):
                if os.path.islink(obs_ln_last):
                    os.unlink(obs_ln_last)
                os.symlink(os.path.relpath(obs_dir, chunk_dir_path),
                           obs_ln_last)
        if ('default' in obsdirs
            and (os.path.realpath(obsdirs['last'])
                 == os.path.realpath(obsdirs['default']))):
            if os.path.exists(obs_ln_last):
                os.unlink(obs_ln_last)
    obi.isdefault = isdefault
예제 #2
0
def _file_vv(obi):
    """
    Save processed V&V data to per-obsid archive
    """
    obsid = int(obi.info()['obsid'])
    version = int(obi.info()['revision'])
    # set up directory for data
    strobs = "%05d_v%02d" % (obsid, version)
    chunk_dir = strobs[0:2]
    chunk_dir_path = os.path.join(FILES['data_root'], chunk_dir)
    obs_dir = os.path.join(chunk_dir_path, strobs)
    if not os.path.exists(obs_dir):
        logger.info("making directory %s" % obs_dir)
        os.makedirs(obs_dir)
    else:
        logger.info("obsid dir %s already exists" % obs_dir)
    for f in glob(os.path.join(obi.tempdir, "*")):
        os.chmod(f, 0o775)
        shutil.copy(f, obs_dir)
        os.remove(f)
    logger.info("moved VV files to {}".format(obs_dir))
    os.removedirs(obi.tempdir)
    logger.info("removed directory {}".format(obi.tempdir))
    # make any desired link
    obs_ln = os.path.join(FILES['data_root'], chunk_dir, "%05d" % obsid)
    obs_ln_last = os.path.join(FILES['data_root'], chunk_dir,
                               "%05d_last" % obsid)
    obsdirs = asp_l1_arch.get_obs_dirs(obsid)
    isdefault = 0
    if 'default' in obsdirs:
        if (os.path.realpath(obsdirs[version]) == os.path.realpath(
                obsdirs['default'])):
            if os.path.islink(obs_ln):
                os.unlink(obs_ln)
            os.symlink(os.path.relpath(obs_dir, chunk_dir_path), obs_ln)
            isdefault = 1
    if 'last' in obsdirs:
        if ('default' in obsdirs and (os.path.realpath(obsdirs['last']) !=
                                      os.path.realpath(obsdirs['default']))
                or 'default' not in obsdirs):
            if (os.path.realpath(obsdirs[version]) == os.path.realpath(
                    obsdirs['last'])):
                if os.path.islink(obs_ln_last):
                    os.unlink(obs_ln_last)
                os.symlink(os.path.relpath(obs_dir, chunk_dir_path),
                           obs_ln_last)
        if ('default' in obsdirs and (os.path.realpath(
                obsdirs['last']) == os.path.realpath(obsdirs['default']))):
            if os.path.exists(obs_ln_last):
                os.unlink(obs_ln_last)
    obi.isdefault = isdefault
예제 #3
0
def get_arch_vv(obsid, version='last'):
    """
    Given obsid and version, find archived ASP1 and obspar products and
    run V&V.  Effort is made to find the obspar that was actually used during
    creation of the ASP1 products.

    :param obsid: obsid
    :param version: 'last', 'default', or revision number of ASP1 products
    :returns: mica.vv.Obi V&V object
    """
    logger.info("Generating V&V for obsid {}".format(obsid))
    asp_l1_dirs = asp_l1_arch.get_obs_dirs(obsid)
    if asp_l1_dirs is None or version not in asp_l1_dirs:
        raise LookupError(
            "Requested version {} not in asp_l1 archive".format(version))
    l1_dir = asp_l1_dirs[version]
    # find the obspar that matches the requested aspect_1 products
    # this is in the aspect processing table
    asp_l1_proc = Ska.DBI.DBI(dbi="sqlite", server=FILES['asp1_proc_table'])
    asp_obs = asp_l1_proc.fetchall(
        "SELECT * FROM aspect_1_proc where obsid = {}".format(obsid))
    asp_proc = None
    if len(asp_obs) == 0:
        return None
    if version == 'last':
        asp_proc = asp_obs[asp_obs['aspect_1_id'] == np.max(
            asp_obs['aspect_1_id'])][0]
    if version == 'default':
        asp_proc = asp_obs[asp_obs['isdefault'] == 1][0]
    if asp_proc is None:
        asp_proc = asp_obs[asp_obs['revision'] == version][0]
    obspar_dirs = obspar_arch.get_obs_dirs(obsid)
    if obspar_dirs is None or asp_proc['obspar_version'] not in obspar_dirs:
        # try to update the obspar archive with the missing version
        config = obspar_arch.CONFIG.copy()
        config.update(dict(obsid=obsid, version=asp_proc['obspar_version']))
        oa = obsid_archive.ObsArchive(config)
        oa.logger.setLevel(logging.INFO)
        oa.logger.addHandler(logging.StreamHandler())
        oa.update()
        obspar_dirs = obspar_arch.get_obs_dirs(obsid)
    try:
        obspar_file = glob(
            os.path.join(obspar_dirs[asp_proc['obspar_version']],
                         'axaf*par*'))[0]
    except IndexError:
        raise LookupError(f"Requested version {version} not in obspar archive")
    return Obi(obspar_file, l1_dir, temproot=FILES['temp_root'])
예제 #4
0
def get_arch_vv(obsid, version='last'):
    """
    Given obsid and version, find archived ASP1 and obspar products and
    run V&V.  Effort is made to find the obspar that was actually used during
    creation of the ASP1 products.

    :param obsid: obsid
    :param version: 'last', 'default', or revision number of ASP1 products
    :returns: mica.vv.Obi V&V object
    """
    logger.info("Generating V&V for obsid {}".format(obsid))
    asp_l1_dirs = asp_l1_arch.get_obs_dirs(obsid)
    if asp_l1_dirs is None or version not in asp_l1_dirs:
        raise LookupError("Requested version {} not in asp_l1 archive".format(version))
    l1_dir = asp_l1_dirs[version]
    # find the obspar that matches the requested aspect_1 products
    # this is in the aspect processing table
    asp_l1_proc = Ska.DBI.DBI(dbi="sqlite", server=FILES['asp1_proc_table'])
    asp_obs = asp_l1_proc.fetchall(
        "SELECT * FROM aspect_1_proc where obsid = {}".format(
            obsid))
    asp_proc = None
    if len(asp_obs) == 0:
        return None
    if version == 'last':
        asp_proc = asp_obs[asp_obs['aspect_1_id']
                           == np.max(asp_obs['aspect_1_id'])][0]
    if version == 'default':
        asp_proc = asp_obs[asp_obs['isdefault'] == 1][0]
    if asp_proc is None:
        asp_proc = asp_obs[asp_obs['revision'] == version][0]
    obspar_dirs = obspar_arch.get_obs_dirs(obsid)
    if asp_proc['obspar_version'] not in obspar_dirs:
        # try to update the obspar archive with the missing version
        config = obspar_arch.CONFIG.copy()
        config.update(dict(obsid=obsid, version=asp_proc['obspar_version']))
        oa = obsid_archive.ObsArchive(config)
        oa.logger.setLevel(logging.INFO)
        oa.logger.addHandler(logging.StreamHandler())
        oa.update()
        obspar_dirs = obspar_arch.get_obs_dirs(obsid)
    obspar_file = glob(os.path.join(obspar_dirs[asp_proc['obspar_version']],
                                    'axaf*par*'))[0]
    return Obi(obspar_file, l1_dir, temproot=FILES['temp_root'])
예제 #5
0
def main(obsid):
    report_root = REPORT_ROOT
    strobs = "%05d" % obsid
    chunk_dir = strobs[0:2]
    topdir = os.path.join(report_root, chunk_dir)
    outdir = os.path.join(topdir, strobs)

    if not os.path.exists(outdir):
        os.makedirs(outdir)

    jinja_env = jinja2.Environment(
        loader=jinja2.PackageLoader('mica.report'))
    jinja_env.line_comment_prefix = '##'
    jinja_env.line_statement_prefix = '#'

    logger.info("Making report for {}".format(obsid))
    logger.debug("Getting target info from axafapstat")
    summary = target_summary(obsid)
    # Links
    logger.debug("Looking up obsid links")

    all_progress = {'science': ['ocat',
                                 'long_term', 'short_term',
                                 'starcheck', 'observed',
                                 'engineering', 'aspect_1',
                                 'cxcds_vv', 'released'],
                    'er': ['starcheck', 'observed',
                           'engineering', 'cxcds_vv']}
    report_status = {}

    er = summary is None and obsid > 40000
    progress = all_progress['er' if er else 'science']
    if er:
        links = obs_links(obsid)
    else:
        if summary is None:
            raise ValueError("Obsid not found in target table")
        report_status['ocat'] = summary['status']
        links = obs_links(obsid, summary['seq_nbr'], summary['lts_lt_plan'])

    if not er and (summary['status'] in
                   ['canceled', 'unobserved', 'untriggered']):
        logger.debug(
            "Obsid {obsid} has status {status}".format(
                obsid=obsid, status=summary['status']))

    if summary is not None:
        if summary['lts_lt_plan'] is not None:
            report_status['long_term'] = summary['lts_lt_plan']
        if summary['soe_st_sched_date']:
            report_status['short_term'] =  summary['soe_st_sched_date']

    last_sched = ''
    if not er:
        if summary['lts_lt_plan']:
            last_sched = "in LTS for {}".format(
                str(summary['lts_lt_plan']))
        if summary['soe_st_sched_date']:
            last_sched = "in ST sched for {}".format(
                str(summary['soe_st_sched_date']))

    ## Starcheck
    logger.debug("Fetching starcheck catalog")
    try:
        if summary is not None and summary['lts_lt_plan'] is not None:
            plan_date = Time(summary['lts_lt_plan'])
            if plan_date.cxcsec > (CxoTime.now() + 21).secs:
                raise LookupError("No starcheck expected for {} lts date".format(str(plan)))
        mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
        obs_sc, mp_dir, status = get_starcheck(obsid)
        logger.debug("Plotting starcheck catalog to {}".format(os.path.join(outdir, 'starcheck.png')))
        if obs_sc['obs']['point_ra'] is None:
            raise LookupError("Observation has no pointing.")
        if len(obs_sc['cat']) == 0:
            raise LookupError("Observation has no catalog")
        fig, cat, obs = catalog.plot(obsid, mp_dir)
        sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
        fig.savefig(os.path.join(outdir, 'starcheck.png'))
        plt.close('all')
    except LookupError as detail:
        logger.info("No starcheck catalog.  Writing out OCAT info only")
        logger.info(detail)
        template = jinja_env.get_template('report.html')
        page = template.render(obsid=obsid,
                               target=summary,
                               links=links,
                               temps=None,
                               pred_temp=None,
                               cat_table=None,
                               er=er if er else None,
                               last_sched=last_sched,
                               obs=None,
                               version=version)
        full_report_file = os.path.join(outdir, 'index.html')
        logger.info("Writing out full report to {}".format(full_report_file))
        f = open(full_report_file, 'w')
        f.write(page)
        f.close()
        notes = {'report_version': REPORT_VERSION,
                 'vv_version': None,
                 'vv_revision': None,
                 'aspect_1_id': None,
                 'last_sched': last_sched,
                 'ocat_status': report_status.get('ocat'),
                 'long_term': str(report_status.get('long_term')),
                 'short_term': str(report_status.get('short_term')),
                 'starcheck': report_status.get('starcheck'),
                 'obsid': obsid,
                 'checked_date': CxoTime.now().date}
        f = open(os.path.join(outdir, 'notes.json'), 'w')
        f.write(json.dumps(notes,
                           sort_keys=True,
                           indent=4))
        f.close()
        save_state_in_db(obsid, notes)
        return

    if not er and 'shortterm' in links:
        dir_match = re.match(r'/\d{4}/(\w{3}\d{4})/ofls(\w)', mp_dir)
        mp_label = "{}{}".format(dir_match.group(1),
                                 dir_match.group(2).upper())
        last_sched = 'in <A HREF="{}">{}</A> at {}'.format(
            links['shortterm']['link'], mp_label, str(obs_sc['obs']['mp_starcat_time']))


    report_status['starcheck'] = mp_dir

    # engineering data available
    logger.debug("Getting acq and trak stats")
    acqs = get_obs_acq_stats(obsid)
    trak = get_obs_trak_stats(obsid)
    temps = get_obs_temps(obsid, outdir)
    pred_temp = sc.get('pred_temp')
    if acqs or trak:
        last_sched = "eng. data available"

    er_status = None
    if er:
        stat_map = {'ran': 'ran on',
                    'approved': 'approved for',
                    'ran_pretimelines': 'ran on',
                    'planned': 'planned for'}
        er_status = "{} {}".format(stat_map[status], obs_sc['obs']['mp_starcat_time'])
        run_obspar = None
        vv = None
        logger.info("Processing ER; no V&V available")
    else:
        # obspar ingested
        try:
            run_obspar = obspar.get_obspar(obsid, version='last')
        except:
            run_obspar = None

        # v&v available
        try:
            vv = get_vv(obsid, version='last')
        except LookupError:
            vv = None

        try:
            if vv is None or 'vv_version' not in vv or vv['vv_version'] < WANT_VV_VERSION:
                mica.vv.process.process(obsid, version="last")
                vv = get_vv(obsid, version='last')
            for slot in vv['slots']:
                if isinstance(vv['slots'][slot]['dy_med'], list):
                    mica.vv.process.process(obsid, version="last")
                    vv = get_vv(obsid, version='last')
                    break
            vv_files = get_vv_files(obsid, version='last')
            last_sched = "complete through mica v&v"
        except LookupError:
            logger.info("No V&V available")
            vv = None

    if vv is not None:
        report_status['mica_vv'] = True
        for file in vv_files:
            newfile = os.path.join(outdir, os.path.basename(file))
            if not os.path.exists(newfile):
                logger.debug("linking {} into {}".format(file, outdir))
                os.symlink(file, newfile)
        asp_dir = asp_l1.get_obs_dirs(obsid)['last']
        asp_logs = sorted(glob(os.path.join(asp_dir, "asp_l1_f*log*gz")))
        for log, interval in zip(asp_logs, vv['intervals']):
            logmatch = re.search(r'(.*log)\.gz', os.path.basename(log))
            if logmatch:
                newlogname = "{}.txt".format(logmatch.group(1))
                newlog = os.path.join(outdir, newlogname)
                if not os.path.exists(newlog):
                    logger.debug("copying/gunzipping asp log {}".format(newlog))
                    logtext = gzip.open(log, 'rt').readlines()
                    f = open(newlog, 'w')
                    f.writelines(logtext)
                    f.close()
                interval['loglink'] = newlogname

        aiprops = get_aiprops(obsid)
        aiprops_template = jinja_env.get_template('aiprops.html')
        aiprops_page = aiprops_template.render(obsid=obsid, aiprops=aiprops)
        aiprops_page_file = os.path.join(outdir, 'aiprops.html')
        logger.debug("AIPROPS report to {}".format(aiprops_page_file))
        f = open(aiprops_page_file, 'w')
        f.write(aiprops_page)
        f.close()

        props_template = jinja_env.get_template('props.html')
        props_page = props_template.render(obsid=obsid, vv=vv)
        props_page_file = os.path.join(outdir, 'props.html')
        logger.debug("GS/FIDPROPS report to {}".format(props_page_file))
        f = open(props_page_file, 'w')
        f.write(props_page)
        f.close()

        for slot in vv['slots']:
            if 'n_pts' not in vv['slots'][slot]:
                continue
            slot_template = jinja_env.get_template('vv_slots_single.html')
            slot_page = slot_template.render(obsid=obsid,
                                             vv=vv,
                                             slot=slot)
            slot_page_file = os.path.join(outdir, "slot_{}.html".format(slot))
            logger.debug("VV SLOT report to {}".format(slot_page_file))
            f = open(slot_page_file, 'w')
            f.write(slot_page)
            f.close()

        official_notes = official_vv_notes(obsid, summary)
        if official_notes:
            for rep in official_notes:
                if rep['comments'] == 'Hidden':
                    rep['comments'] = """
<A target="_blank" HREF="{}">{}</A><BR>(<A target="_blank" HREF="https://icxc.cfa.harvard.edu/soft/vv/vv_login.html">LOGIN</A> once first)</BR>""".format(links['vv']['link'], links['vv']['label'])
        vv_template = jinja_env.get_template('vv.html')
        vv['has_errors'] = (('errors' in vv) and (len(vv['errors']))) or None
        vv_page = vv_template.render(obsid=obsid,
                                     vv=vv,
                                     obspar=run_obspar,
                                     official_vv_notes=official_notes,
                                     )
        vv_page_file = os.path.join(outdir, 'vv.html')
        logger.debug("VV report to {}".format(vv_page_file))
        f = open(vv_page_file, 'w')
        f.write(vv_page)
        f.close()

    cat_table = catalog_info(obs_sc['cat'], acqs, trak, vv)

    for row, cat_row in zip(obs_sc['cat'], cat_table):
        if row['type'] != 'FID':
            if row['id'] is not None:
                s = star_info(row['id'])
                star_template = jinja_env.get_template('star.html')
                star_page = star_template.render(
                    star=s['agasc_info'],
                    acqs=s['acqs'],
                    traks=s['traks'],
                    agg_acq=s['agg_acq'],
                    agg_trak=s['agg_trak'])
                star_page_file = os.path.join(outdir, 'star_%d.html' % int(row['id']))
                logger.debug("Writing out star info to {}".format(star_page_file))
                f = open(star_page_file, 'w')
                f.write(star_page)
                f.close()
                cat_row['idlink'] = (
                    '<A HREF="star_{id}.html" STYLE="text-decoration: none;"'
                    'ONMOUSEOVER="return overlib '
                    '(\'ACQ total:{n_acq} noid:{n_noid} <BR /> '
                    'GUI total:{n_gui} bad:{n_bad} fail:{n_fail} obc_bad:{n_obc_bad} '
                    '<BR /> Avg Mag {avg_mag:4.2f}\', WIDTH, 220);", ONMOUSEOUT="return nd();"> '
                    '{id}</A>'.format(id=int(row['id']),
                                      n_acq=s['agg_acq']['n_acqs'],
                                      n_noid=s['agg_acq']['n_acq_noid'],
                                      n_gui=s['agg_trak']['n_guis'],
                                      n_bad=s['agg_trak']['n_bad'],
                                      n_fail=s['agg_trak']['n_fail'],
                                      n_obc_bad=s['agg_trak']['n_obc_bad'],
                                      avg_mag=(s['agg_trak']['avg_mag']
                                               or s['agg_acq']['avg_mag']
                                               or 13.94)))
            else:
                cat_row['idlink'] = "&nbsp;"
        else:
            if 'id' in row:
                cat_row['idlink'] = int(row['id'])
            else:
                cat_row['idlink'] = ''
    template = jinja_env.get_template('report.html')

    page = template.render(cat_table=cat_table,
                           obs=obs,
                           sc=obs_sc,
                           vv=vv,
                           links=links,
                           target=summary,
                           temps=temps,
                           pred_temp=pred_temp,
                           er=er if er else None,
                           er_status=er_status,
                           last_sched=last_sched,
                           obsid=obsid,
                           version=version)
    full_report_file = os.path.join(outdir, 'index.html')
    logger.info("Writing out full report to {}".format(full_report_file))
    f = open(full_report_file, 'w')
    f.write(page)
    f.close()

    cat_file = os.path.join(outdir, 'star_table.json')
    f = open(cat_file, 'w')
    f.write(json.dumps(cat_table, sort_keys=True, indent=4))
    f.close()

    notes = {'report_version': REPORT_VERSION,
             'vv_version': None,
             'vv_revision': None,
             'aspect_1_id': None,
             'last_sched': last_sched,
             'ocat_status': report_status.get('ocat'),
             'long_term': str(report_status.get('long_term')),
             'short_term': str(report_status.get('short_term')),
             'starcheck': report_status.get('starcheck'),
             'obsid': obsid,
             'checked_date': CxoTime.now().date}
    if vv:
        notes['vv_version'] = vv.get('vv_version')
        notes['vv_revision'] = vv.get('revision')
        notes['aspect_1_id'] = vv.get('aspect_1_id')
    f = open(os.path.join(outdir, 'notes.json'), 'w')
    f.write(json.dumps(notes,
                       sort_keys=True,
                       indent=4))
    f.close()
    save_state_in_db(obsid, notes)
예제 #6
0
파일: report.py 프로젝트: sot/mica
def main(obsid, config=None, report_root=None):

    if config is None:
        config = DEFAULT_CONFIG
    if report_root is None:
        report_root = config["report_root"]

    global ACA_DB
    if ACA_DB is None or not ACA_DB.conn._is_connected:
        ACA_DB = Ska.DBI.DBI(dbi="sybase", server="sybase", user="******")

    strobs = "%05d" % obsid
    chunk_dir = strobs[0:2]
    topdir = os.path.join(report_root, chunk_dir)
    outdir = os.path.join(topdir, strobs)

    if not os.path.exists(outdir):
        os.makedirs(outdir)

    jinja_env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(os.path.join(os.environ["SKA"], "data", "mica", "templates"))
    )
    jinja_env.line_comment_prefix = "##"
    jinja_env.line_statement_prefix = "#"

    logger.info("Making report for {}".format(obsid))
    logger.debug("Getting target info from axafapstat")
    summary = target_summary(obsid)
    # Links
    logger.debug("Looking up obsid links")

    all_progress = {
        "science": [
            "ocat",
            "long_term",
            "short_term",
            "starcheck",
            "observed",
            "engineering",
            "aspect_1",
            "cxcds_vv",
            "released",
        ],
        "er": ["starcheck", "observed", "engineering", "cxcds_vv"],
    }
    report_status = {}

    er = summary is None and obsid > 40000
    progress = all_progress["er" if er else "science"]
    if er:
        links = obs_links(obsid)
    else:
        if summary is None:
            raise ValueError("Obsid not found in target table")
        report_status["ocat"] = summary["status"]
        links = obs_links(obsid, summary["seq_nbr"], summary["lts_lt_plan"])

    if not er and (summary["status"] in ["canceled", "unobserved", "untriggered"]):
        logger.debug("Obsid {obsid} has status {status}".format(obsid=obsid, status=summary["status"]))

    if summary is not None:
        if summary["lts_lt_plan"] is not None:
            report_status["long_term"] = summary["lts_lt_plan"]
        if summary["soe_st_sched_date"]:
            report_status["short_term"] = summary["soe_st_sched_date"]

    last_sched = ""
    if not er:
        if summary["lts_lt_plan"]:
            last_sched = "in LTS for {}".format(str(summary["lts_lt_plan"]))
        if summary["soe_st_sched_date"]:
            last_sched = "in ST sched for {}".format(str(summary["soe_st_sched_date"]))

    ## Starcheck
    logger.debug("Fetching starcheck catalog")
    try:
        if summary is not None and summary["lts_lt_plan"] is not None:
            plan = summary["lts_lt_plan"]
            plan_date = DateTime(
                "{:4d}:{:03d}:{:02d}:{:02d}:{:02d}.000".format(
                    plan.year, plan.dayofyear, plan.hour, plan.minute, plan.second
                )
            )
            if plan_date.secs > (DateTime() + 21).secs:
                raise LookupError("No starcheck expected for {} lts date".format(str(plan)))
        mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
        obs_sc, mp_dir, status = get_starcheck(obsid)
        logger.debug("Plotting starcheck catalog to {}".format(os.path.join(outdir, "starcheck.png")))
        if obs_sc["obs"][0]["point_ra"] is None:
            raise LookupError("Observation has no pointing.")
        if len(obs_sc["catalog"]) == 0:
            raise LookupError("Observation has no catalog")
        fig, cat, obs = catalog.plot(obsid, mp_dir)
        sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
        fig.savefig(os.path.join(outdir, "starcheck.png"))
        plt.close("all")
    except LookupError as detail:
        logger.info("No starcheck catalog.  Writing out OCAT info only")
        logger.info(detail)
        template = jinja_env.get_template("report.html")
        page = template.render(
            obsid=obsid,
            target=summary,
            links=links,
            temps=None,
            pred_temp=None,
            cat_table=None,
            er=er if er else None,
            last_sched=last_sched,
            obs=None,
            version=version,
        )
        full_report_file = os.path.join(outdir, "index.html")
        logger.info("Writing out full report to {}".format(full_report_file))
        f = open(full_report_file, "w")
        f.write(page)
        f.close()
        notes = {
            "report_version": REPORT_VERSION,
            "vv_version": None,
            "vv_revision": None,
            "aspect_1_id": None,
            "last_sched": last_sched,
            "ocat_status": report_status.get("ocat"),
            "long_term": str(report_status.get("long_term")),
            "short_term": str(report_status.get("short_term")),
            "starcheck": report_status.get("starcheck"),
            "obsid": obsid,
            "checked_date": DateTime().date,
        }
        f = open(os.path.join(outdir, "notes.json"), "w")
        f.write(json.dumps(notes, sort_keys=True, indent=4))
        f.close()
        save_state_in_db(obsid, notes, config)
        return

    if not er and "shortterm" in links:
        dir_match = re.match("/\d{4}/(\w{3}\d{4})/ofls(\w)", mp_dir)
        mp_label = "{}{}".format(dir_match.group(1), dir_match.group(2).upper())
        last_sched = 'in <A HREF="{}">{}</A> at {}'.format(
            links["shortterm"]["link"], mp_label, str(obs_sc["obs"][0]["mp_starcat_time"])
        )

    report_status["starcheck"] = mp_dir

    # engineering data available
    logger.debug("Getting acq and trak stats")
    acqs = get_obs_acq_stats(obsid)
    trak = get_obs_trak_stats(obsid)
    temps = get_obs_temps(obsid, outdir)
    pred_temp = sc["pred_temp"]
    if acqs or trak:
        last_sched = "eng. data available"

    er_status = None
    if er:
        stat_map = {"ran": "ran on", "approved": "approved for", "ran_pretimelines": "ran on", "planned": "planned for"}
        er_status = "{} {}".format(stat_map[status], obs_sc["obs"][0]["mp_starcat_time"])
        run_obspar = None
        vv = None
        logger.info("Processing ER; no V&V available")
    else:
        # obspar ingested
        try:
            run_obspar = obspar.get_obspar(obsid, version="last")
        except:
            run_obspar = None

        # v&v available
        try:
            vv = get_vv(obsid, version="last")
        except LookupError:
            vv = None

        try:
            if vv is None or "vv_version" not in vv or vv["vv_version"] < WANT_VV_VERSION:
                mica.vv.process.process(obsid, version="last")
                vv = get_vv(obsid, version="last")
            for slot in vv["slots"]:
                if isinstance(vv["slots"][slot]["dy_med"], list):
                    mica.vv.process.process(obsid, version="last")
                    vv = get_vv(obsid, version="last")
                    break
            vv_files = get_vv_files(obsid, version="last")
            last_sched = "complete through mica v&v"
        except LookupError:
            logger.info("No V&V available")
            vv = None

    if vv is not None:
        report_status["mica_vv"] = True
        for file in vv_files:
            newfile = os.path.join(outdir, os.path.basename(file))
            if not os.path.exists(newfile):
                logger.debug("linking {} into {}".format(file, outdir))
                bash("ln -s {} {}".format(file, outdir))
        asp_dir = asp_l1.get_obs_dirs(obsid)["last"]
        asp_logs = sorted(glob(os.path.join(asp_dir, "asp*log*gz")))
        for log, interval in zip(asp_logs, vv["intervals"]):
            logmatch = re.search("(.*log)\.gz", os.path.basename(log))
            if logmatch:
                newlogname = "{}.txt".format(logmatch.group(1))
                newlog = os.path.join(outdir, newlogname)
                if not os.path.exists(newlog):
                    logger.debug("copying/gunzipping asp log {}".format(newlog))
                    logtext = gzip.open(log).readlines()
                    f = open(newlog, "w")
                    f.writelines(logtext)
                    f.close()
                interval["loglink"] = newlogname

        aiprops = get_aiprops(obsid)
        aiprops_template = jinja_env.get_template("aiprops.html")
        aiprops_page = aiprops_template.render(obsid=obsid, aiprops=aiprops)
        aiprops_page_file = os.path.join(outdir, "aiprops.html")
        logger.debug("AIPROPS report to {}".format(aiprops_page_file))
        f = open(aiprops_page_file, "w")
        f.write(aiprops_page)
        f.close()

        props_template = jinja_env.get_template("props.html")
        props_page = props_template.render(obsid=obsid, vv=vv)
        props_page_file = os.path.join(outdir, "props.html")
        logger.debug("GS/FIDPROPS report to {}".format(props_page_file))
        f = open(props_page_file, "w")
        f.write(props_page)
        f.close()

        for slot in vv["slots"]:
            if "n_pts" not in vv["slots"][slot]:
                continue
            slot_template = jinja_env.get_template("vv_slots_single.html")
            slot_page = slot_template.render(obsid=obsid, vv=vv, slot=slot)
            slot_page_file = os.path.join(outdir, "slot_{}.html".format(slot))
            logger.debug("VV SLOT report to {}".format(slot_page_file))
            f = open(slot_page_file, "w")
            f.write(slot_page)
            f.close()

        official_notes = official_vv_notes(obsid, summary)
        if official_notes:
            for rep in official_notes:
                if rep["comments"] == "Hidden":
                    rep[
                        "comments"
                    ] = """
<A target="_blank" HREF="{}">{}</A><BR>(<A target="_blank" HREF="https://icxc.cfa.harvard.edu/soft/vv/vv_login.html">LOGIN</A> once first)</BR>""".format(
                        links["vv"]["link"], links["vv"]["label"]
                    )
        vv_template = jinja_env.get_template("vv.html")
        vv["has_errors"] = (("errors" in vv) and (len(vv["errors"]))) or None
        vv_page = vv_template.render(obsid=obsid, vv=vv, obspar=run_obspar, official_vv_notes=official_notes)
        vv_page_file = os.path.join(outdir, "vv.html")
        logger.debug("VV report to {}".format(vv_page_file))
        f = open(vv_page_file, "w")
        f.write(vv_page)
        f.close()

    cat_table = catalog_info(obs_sc["catalog"], acqs, trak, vv)

    for row, cat_row in zip(obs_sc["catalog"], cat_table):
        if row["type"] != "FID":
            if row["id"] is not None:
                s = star_info(row["id"])
                star_template = jinja_env.get_template("star.html")
                star_page = star_template.render(
                    star=s["agasc_info"], acqs=s["acqs"], traks=s["traks"], agg_acq=s["agg_acq"], agg_trak=s["agg_trak"]
                )
                star_page_file = os.path.join(outdir, "star_%d.html" % int(row["id"]))
                logger.debug("Writing out star info to {}".format(star_page_file))
                f = open(star_page_file, "w")
                f.write(star_page)
                f.close()
                cat_row["idlink"] = (
                    '<A HREF="star_{id}.html" STYLE="text-decoration: none;"'
                    'ONMOUSEOVER="return overlib '
                    "('ACQ total:{n_acq} noid:{n_noid} <BR /> "
                    "GUI total:{n_gui} bad:{n_bad} fail:{n_fail} obc_bad:{n_obc_bad} "
                    '<BR /> Avg Mag {avg_mag:4.2f}\', WIDTH, 220);", ONMOUSEOUT="return nd();"> '
                    "{id}</A>".format(
                        id=int(row["id"]),
                        n_acq=s["agg_acq"]["n_acqs"],
                        n_noid=s["agg_acq"]["n_acq_noid"],
                        n_gui=s["agg_trak"]["n_guis"],
                        n_bad=s["agg_trak"]["n_bad"],
                        n_fail=s["agg_trak"]["n_fail"],
                        n_obc_bad=s["agg_trak"]["n_obc_bad"],
                        avg_mag=(s["agg_trak"]["avg_mag"] or s["agg_acq"]["avg_mag"] or 13.94),
                    )
                )
            else:
                cat_row["idlink"] = "&nbsp;"
        else:
            if "id" in row:
                cat_row["idlink"] = int(row["id"])
            else:
                cat_row["idlink"] = ""
    template = jinja_env.get_template("report.html")

    page = template.render(
        cat_table=cat_table,
        obs=obs,
        sc=obs_sc,
        vv=vv,
        links=links,
        target=summary,
        temps=temps,
        pred_temp=pred_temp,
        er=er if er else None,
        er_status=er_status,
        last_sched=last_sched,
        obsid=obsid,
        version=version,
    )
    full_report_file = os.path.join(outdir, "index.html")
    logger.info("Writing out full report to {}".format(full_report_file))
    f = open(full_report_file, "w")
    f.write(page)
    f.close()

    cat_file = os.path.join(outdir, "star_table.json")
    f = open(cat_file, "w")
    f.write(json.dumps(cat_table, sort_keys=True, indent=4))
    f.close()

    notes = {
        "report_version": REPORT_VERSION,
        "vv_version": None,
        "vv_revision": None,
        "aspect_1_id": None,
        "last_sched": last_sched,
        "ocat_status": report_status.get("ocat"),
        "long_term": str(report_status.get("long_term")),
        "short_term": str(report_status.get("short_term")),
        "starcheck": report_status.get("starcheck"),
        "obsid": obsid,
        "checked_date": DateTime().date,
    }
    if vv:
        notes["vv_version"] = vv.get("vv_version")
        notes["vv_revision"] = vv.get("revision")
        notes["aspect_1_id"] = vv.get("aspect_1_id")
    f = open(os.path.join(outdir, "notes.json"), "w")
    f.write(json.dumps(notes, sort_keys=True, indent=4))
    f.close()
    save_state_in_db(obsid, notes, config)
예제 #7
0
파일: report.py 프로젝트: sot/mica
def main(obsid):
    report_root = REPORT_ROOT
    strobs = "%05d" % obsid
    chunk_dir = strobs[0:2]
    topdir = os.path.join(report_root, chunk_dir)
    outdir = os.path.join(topdir, strobs)

    if not os.path.exists(outdir):
        os.makedirs(outdir)

    jinja_env = jinja2.Environment(
        loader=jinja2.PackageLoader('mica.report'))
    jinja_env.line_comment_prefix = '##'
    jinja_env.line_statement_prefix = '#'

    logger.info("Making report for {}".format(obsid))
    logger.debug("Getting target info from axafapstat")
    summary = target_summary(obsid)
    # Links
    logger.debug("Looking up obsid links")

    all_progress = {'science': ['ocat',
                                 'long_term', 'short_term',
                                 'starcheck', 'observed',
                                 'engineering', 'aspect_1',
                                 'cxcds_vv', 'released'],
                    'er': ['starcheck', 'observed',
                           'engineering', 'cxcds_vv']}
    report_status = {}

    er = summary is None and obsid > 40000
    progress = all_progress['er' if er else 'science']
    if er:
        links = obs_links(obsid)
    else:
        if summary is None:
            raise ValueError("Obsid not found in target table")
        report_status['ocat'] = summary['status']
        links = obs_links(obsid, summary['seq_nbr'], summary['lts_lt_plan'])

    if not er and (summary['status'] in
                   ['canceled', 'unobserved', 'untriggered']):
        logger.debug(
            "Obsid {obsid} has status {status}".format(
                obsid=obsid, status=summary['status']))

    if summary is not None:
        if summary['lts_lt_plan'] is not None:
            report_status['long_term'] = summary['lts_lt_plan']
        if summary['soe_st_sched_date']:
            report_status['short_term'] =  summary['soe_st_sched_date']

    last_sched = ''
    if not er:
        if summary['lts_lt_plan']:
            last_sched = "in LTS for {}".format(
                str(summary['lts_lt_plan']))
        if summary['soe_st_sched_date']:
            last_sched = "in ST sched for {}".format(
                str(summary['soe_st_sched_date']))

    ## Starcheck
    logger.debug("Fetching starcheck catalog")
    try:
        if summary is not None and summary['lts_lt_plan'] is not None:
            plan = summary['lts_lt_plan']
            plan_date = DateTime("{:4d}:{:03d}:{:02d}:{:02d}:{:02d}.000".format(
                    plan.year, plan.dayofyear, plan.hour, plan.minute, plan.second))
            if plan_date.secs > (DateTime() + 21).secs:
                raise LookupError("No starcheck expected for {} lts date".format(str(plan)))
        mp_dir, status, mp_date = starcheck.get_mp_dir(obsid)
        obs_sc, mp_dir, status = get_starcheck(obsid)
        logger.debug("Plotting starcheck catalog to {}".format(os.path.join(outdir, 'starcheck.png')))
        if obs_sc['obs']['point_ra'] is None:
            raise LookupError("Observation has no pointing.")
        if len(obs_sc['cat']) == 0:
            raise LookupError("Observation has no catalog")
        fig, cat, obs = catalog.plot(obsid, mp_dir)
        sc = starcheck.get_starcheck_catalog(obsid, mp_dir)
        fig.savefig(os.path.join(outdir, 'starcheck.png'))
        plt.close('all')
    except LookupError as detail:
        logger.info("No starcheck catalog.  Writing out OCAT info only")
        logger.info(detail)
        template = jinja_env.get_template('report.html')
        page = template.render(obsid=obsid,
                               target=summary,
                               links=links,
                               temps=None,
                               pred_temp=None,
                               cat_table=None,
                               er=er if er else None,
                               last_sched=last_sched,
                               obs=None,
                               version=version)
        full_report_file = os.path.join(outdir, 'index.html')
        logger.info("Writing out full report to {}".format(full_report_file))
        f = open(full_report_file, 'w')
        f.write(page)
        f.close()
        notes = {'report_version': REPORT_VERSION,
                 'vv_version': None,
                 'vv_revision': None,
                 'aspect_1_id': None,
                 'last_sched': last_sched,
                 'ocat_status': report_status.get('ocat'),
                 'long_term': str(report_status.get('long_term')),
                 'short_term': str(report_status.get('short_term')),
                 'starcheck': report_status.get('starcheck'),
                 'obsid': obsid,
                 'checked_date': DateTime().date}
        f = open(os.path.join(outdir, 'notes.json'), 'w')
        f.write(json.dumps(notes,
                           sort_keys=True,
                           indent=4))
        f.close()
        save_state_in_db(obsid, notes)
        return

    if not er and 'shortterm' in links:
        dir_match = re.match('/\d{4}/(\w{3}\d{4})/ofls(\w)', mp_dir)
        mp_label = "{}{}".format(dir_match.group(1),
                                 dir_match.group(2).upper())
        last_sched = 'in <A HREF="{}">{}</A> at {}'.format(
            links['shortterm']['link'], mp_label, str(obs_sc['obs']['mp_starcat_time']))


    report_status['starcheck'] = mp_dir

    # engineering data available
    logger.debug("Getting acq and trak stats")
    acqs = get_obs_acq_stats(obsid)
    trak = get_obs_trak_stats(obsid)
    temps = get_obs_temps(obsid, outdir)
    pred_temp = sc.get('pred_temp')
    if acqs or trak:
        last_sched = "eng. data available"

    er_status = None
    if er:
        stat_map = {'ran': 'ran on',
                    'approved': 'approved for',
                    'ran_pretimelines': 'ran on',
                    'planned': 'planned for'}
        er_status = "{} {}".format(stat_map[status], obs_sc['obs']['mp_starcat_time'])
        run_obspar = None
        vv = None
        logger.info("Processing ER; no V&V available")
    else:
        # obspar ingested
        try:
            run_obspar = obspar.get_obspar(obsid, version='last')
        except:
            run_obspar = None

        # v&v available
        try:
            vv = get_vv(obsid, version='last')
        except LookupError:
            vv = None

        try:
            if vv is None or 'vv_version' not in vv or vv['vv_version'] < WANT_VV_VERSION:
                mica.vv.process.process(obsid, version="last")
                vv = get_vv(obsid, version='last')
            for slot in vv['slots']:
                if isinstance(vv['slots'][slot]['dy_med'], list):
                    mica.vv.process.process(obsid, version="last")
                    vv = get_vv(obsid, version='last')
                    break
            vv_files = get_vv_files(obsid, version='last')
            last_sched = "complete through mica v&v"
        except LookupError:
            logger.info("No V&V available")
            vv = None

    if vv is not None:
        report_status['mica_vv'] = True
        for file in vv_files:
            newfile = os.path.join(outdir, os.path.basename(file))
            if not os.path.exists(newfile):
                logger.debug("linking {} into {}".format(file, outdir))
                bash("ln -s {} {}".format(file, outdir))
        asp_dir = asp_l1.get_obs_dirs(obsid)['last']
        asp_logs = sorted(glob(os.path.join(asp_dir, "asp*log*gz")))
        for log, interval in zip(asp_logs, vv['intervals']):
            logmatch = re.search('(.*log)\.gz', os.path.basename(log))
            if logmatch:
                newlogname = "{}.txt".format(logmatch.group(1))
                newlog = os.path.join(outdir, newlogname)
                if not os.path.exists(newlog):
                    logger.debug("copying/gunzipping asp log {}".format(newlog))
                    logtext = gzip.open(log).readlines()
                    f = open(newlog, 'w')
                    f.writelines(logtext)
                    f.close()
                interval['loglink'] = newlogname

        aiprops = get_aiprops(obsid)
        aiprops_template = jinja_env.get_template('aiprops.html')
        aiprops_page = aiprops_template.render(obsid=obsid, aiprops=aiprops)
        aiprops_page_file = os.path.join(outdir, 'aiprops.html')
        logger.debug("AIPROPS report to {}".format(aiprops_page_file))
        f = open(aiprops_page_file, 'w')
        f.write(aiprops_page)
        f.close()

        props_template = jinja_env.get_template('props.html')
        props_page = props_template.render(obsid=obsid, vv=vv)
        props_page_file = os.path.join(outdir, 'props.html')
        logger.debug("GS/FIDPROPS report to {}".format(props_page_file))
        f = open(props_page_file, 'w')
        f.write(props_page)
        f.close()

        for slot in vv['slots']:
            if 'n_pts' not in vv['slots'][slot]:
                continue
            slot_template = jinja_env.get_template('vv_slots_single.html')
            slot_page = slot_template.render(obsid=obsid,
                                             vv=vv,
                                             slot=slot)
            slot_page_file = os.path.join(outdir, "slot_{}.html".format(slot))
            logger.debug("VV SLOT report to {}".format(slot_page_file))
            f = open(slot_page_file, 'w')
            f.write(slot_page)
            f.close()

        official_notes = official_vv_notes(obsid, summary)
        if official_notes:
            for rep in official_notes:
                if rep['comments'] == 'Hidden':
                    rep['comments'] = """
<A target="_blank" HREF="{}">{}</A><BR>(<A target="_blank" HREF="https://icxc.cfa.harvard.edu/soft/vv/vv_login.html">LOGIN</A> once first)</BR>""".format(links['vv']['link'], links['vv']['label'])
        vv_template = jinja_env.get_template('vv.html')
        vv['has_errors'] = (('errors' in vv) and (len(vv['errors']))) or None
        vv_page = vv_template.render(obsid=obsid,
                                     vv=vv,
                                     obspar=run_obspar,
                                     official_vv_notes=official_notes,
                                     )
        vv_page_file = os.path.join(outdir, 'vv.html')
        logger.debug("VV report to {}".format(vv_page_file))
        f = open(vv_page_file, 'w')
        f.write(vv_page)
        f.close()

    cat_table = catalog_info(obs_sc['cat'], acqs, trak, vv)

    for row, cat_row in zip(obs_sc['cat'], cat_table):
        if row['type'] != 'FID':
            if row['id'] is not None:
                s = star_info(row['id'])
                star_template = jinja_env.get_template('star.html')
                star_page = star_template.render(
                    star=s['agasc_info'],
                    acqs=s['acqs'],
                    traks=s['traks'],
                    agg_acq=s['agg_acq'],
                    agg_trak=s['agg_trak'])
                star_page_file = os.path.join(outdir, 'star_%d.html' % int(row['id']))
                logger.debug("Writing out star info to {}".format(star_page_file))
                f = open(star_page_file, 'w')
                f.write(star_page)
                f.close()
                cat_row['idlink'] = (
                    '<A HREF="star_{id}.html" STYLE="text-decoration: none;"'
                    'ONMOUSEOVER="return overlib '
                    '(\'ACQ total:{n_acq} noid:{n_noid} <BR /> '
                    'GUI total:{n_gui} bad:{n_bad} fail:{n_fail} obc_bad:{n_obc_bad} '
                    '<BR /> Avg Mag {avg_mag:4.2f}\', WIDTH, 220);", ONMOUSEOUT="return nd();"> '
                    '{id}</A>'.format(id=int(row['id']),
                                      n_acq=s['agg_acq']['n_acqs'],
                                      n_noid=s['agg_acq']['n_acq_noid'],
                                      n_gui=s['agg_trak']['n_guis'],
                                      n_bad=s['agg_trak']['n_bad'],
                                      n_fail=s['agg_trak']['n_fail'],
                                      n_obc_bad=s['agg_trak']['n_obc_bad'],
                                      avg_mag=(s['agg_trak']['avg_mag']
                                               or s['agg_acq']['avg_mag']
                                               or 13.94)))
            else:
                cat_row['idlink'] = "&nbsp;"
        else:
            if 'id' in row:
                cat_row['idlink'] = int(row['id'])
            else:
                cat_row['idlink'] = ''
    template = jinja_env.get_template('report.html')

    page = template.render(cat_table=cat_table,
                           obs=obs,
                           sc=obs_sc,
                           vv=vv,
                           links=links,
                           target=summary,
                           temps=temps,
                           pred_temp=pred_temp,
                           er=er if er else None,
                           er_status=er_status,
                           last_sched=last_sched,
                           obsid=obsid,
                           version=version)
    full_report_file = os.path.join(outdir, 'index.html')
    logger.info("Writing out full report to {}".format(full_report_file))
    f = open(full_report_file, 'w')
    f.write(page)
    f.close()

    cat_file = os.path.join(outdir, 'star_table.json')
    f = open(cat_file, 'w')
    f.write(json.dumps(cat_table, sort_keys=True, indent=4))
    f.close()

    notes = {'report_version': REPORT_VERSION,
             'vv_version': None,
             'vv_revision': None,
             'aspect_1_id': None,
             'last_sched': last_sched,
             'ocat_status': report_status.get('ocat'),
             'long_term': str(report_status.get('long_term')),
             'short_term': str(report_status.get('short_term')),
             'starcheck': report_status.get('starcheck'),
             'obsid': obsid,
             'checked_date': DateTime().date}
    if vv:
        notes['vv_version'] = vv.get('vv_version')
        notes['vv_revision'] = vv.get('revision')
        notes['aspect_1_id'] = vv.get('aspect_1_id')
    f = open(os.path.join(outdir, 'notes.json'), 'w')
    f.write(json.dumps(notes,
                       sort_keys=True,
                       indent=4))
    f.close()
    save_state_in_db(obsid, notes)