logger.warning('WARNING: conversion from xml to ASCII segment file failed.')
    
                    if opts.force:
                        logger.info(traceback.print_exc())
                        raise e
                    else:
                        gpsstart += stride 
                        continue 

            ### if we aren't building auxmvc vectors, we re-use pat files from realtime job
            ### this requires us to redefine the 'science-segments' as the intersection of scisegs with realtime segs
            ### we call this intersection "idq_segs"
            else: ### we're re-using pat files!
                try:
                    ### determine segments from realtime filenames
                    realtime_segs = idq.get_idq_segments(realtimedir, gpsstart - lookback, gpsstart + stride, suffix='.pat')

                    ### read in science segments
                    (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name)

                    ### take the intersection of these segments
                    idq_segs = event.andsegments([scisegs, realtime_segs])

                    ### write segment file
                    idqseg_path = idq.idqsegascii(output_dir, '_%s'%dq_name, gpsstart - lookback, lookback+stride)
                    f = open(idqseg_path, 'w')
                    for seg in idq_segs:
                        print >> f, seg[0], seg[1]
                    f.close()
 ### we may want to remove the unsafe channels, but this could be tricky and we don't want to throw away GW channels accidentally
                    ovlsegs = idqseg_path
        seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]

    ### science segments xml filename
    seg_file = idq.segxml(gdbdir, "%s_%s" % (filetag, dq_name), opts.start,
                          opts.end - opts.start)
    if opts.verbose:
        print '  writing science segments to file : ' + seg_file
    ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))

    (scisegs, coveredseg) = idq.extract_dq_segments(
        seg_file, dq_name)  ### read in segments from xml file

if opts.verbose:
    print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir,
                               opts.start,
                               opts.end,
                               suffix='.dat')

if opts.verbose:
    print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments([scisegs, idqsegs])

### write segment file
if opts.ignore_science_segments:
    idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start,
                                  opts.end - opts.start)
else:
    idqseg_path = idq.idqsegascii(gdbdir, '%s_%s' % (filetag, dq_name),
                                  opts.start, opts.end - opts.start)
if opts.verbose:
    print "  writing : " + idqseg_path
Exemple #3
0
        except Exception as e:
            traceback.print_exc()
            logger.info(
                'ERROR: segment generation failed. Skipping this calibration period.'
            )

            if opts.force:  ### we are require successful training or else we want errors
                logger.info(traceback.print_exc())
                raise e
            else:  ### we don't care if any particular training job fails
                gpsstart += stride
                continue

    logger.info('finding idq segments')
    idqsegs = idq.get_idq_segments(realtimedir,
                                   gpsstart - lookback,
                                   gpsstart + stride,
                                   suffix='.dat')

    logger.info(
        'taking intersection between science segments and idq segments')
    idqsegs = event.andsegments([scisegs, idqsegs])

    ### write segment file
    if opts.ignore_science_segments:
        idqseg_path = idq.idqsegascii(output_dir, '', gpsstart - lookback,
                                      lookback + stride)
    else:
        idqseg_path = idq.idqsegascii(output_dir, '_%s' % dq_name,
                                      gpsstart - lookback, lookback + stride)
    f = open(idqseg_path, 'w')
    for seg in idqsegs:
    ### write seg_xml_file to disk
    lsctables.use_in(ligolw.LIGOLWContentHandler)
    xmldoc = ligolw_utils.load_fileobj(seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]

    ### science segments xml filename
    seg_file = idq.segxml(gdbdir, "%s_%s"%(filetag, dq_name), opts.start , opts.end-opts.start )
    if opts.verbose:
        print '  writing science segments to file : '+seg_file
    ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))

    (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file

if opts.verbose:
    print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir, opts.start, opts.end, suffix='.dat')

if opts.verbose:
    print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments( [scisegs, idqsegs] )

### write segment file
if opts.ignore_science_segments:
    idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start, opts.end-opts.start)
else:
    idqseg_path = idq.idqsegascii(gdbdir, '%s_%s'%(filetag, dq_name), opts.start, opts.end-opts.start)
if opts.verbose:
    print "  writing : "+idqseg_path
f = open(idqseg_path, 'w')
for seg in idqsegs:
    print >> f, seg[0], seg[1]
def check_calibartion(
    realtimedir,
    start,
    end,
    classifier,
    FAPthrs,
    verbose=False,
    ):
    """
....checks the pipeline's calibration at each "FAPthr in FAPThrs"
        """

        # =================================================
        # grab idq_segments so we compute meaningful livetimes
        # =================================================

    if verbose:
        print 'getting idq_segements'

    idq_segs = idq.get_idq_segments(realtimedir, start, end,
                                    suffix='.npy.gz')
    idq_livetime = event.livetime(idq_segs)

    # =================================================
    # grab relevant data
    # =================================================

    if verbose:
        print 'looking for *_fap_*.npy.gz files in', opts.realtimedir

    fapfilenames = [filename for filename in
                    idq.get_all_files_in_range(realtimedir, start, end,
                    pad=0, suffix='.npy.gz') if '_fap_' in filename
                    and classifier in filename]

    if opts.verbose:
        print 'discovered %d files' % len(fapfilenames)
        print 'building time-series'

    (times, timeseries) = idq_gdb_utils.combine_ts(fapfilenames)

        # =================================================
        # check calibration
        # =================================================

    segments = []
    deadtimes = []
    statedFAPs = []
    for FAPthr in FAPthrs:
        if verbose:
            print 'computing segements for FAPthr =', FAPthr

        segs = []
        max_statedFAP = None
        for (t, ts) in zip(times, timeseries):
            (_segs, _min_ts) = timeseries_to_segments(t, -ts, -FAPthr)  # we want FAP <= FAPthr <--> -FAP >= FAPthr
            segs += _segs
            if _min_ts != None:
                statedFAP = -_min_ts
                if max_statedFAP < statedFAP:
                    max_statedFAP = statedFAP

        segs = event.andsegments([segs, idq_segs])
        segments.append(segs)
        deadtimes.append(1.0 * event.livetime(segs) / idq_livetime)
        statedFAPs.append(statedFAP)

    return (idq_segs, segments, deadtimes, statedFAPs)
    xmldoc = ligolw_utils.load_fileobj(seg_xml_file, contenthandler=ligolw.LIGOLWContentHandler)[0]

    ### science segments xml filename
    seg_file = idq.segxml(opts.output_dir, "_%s"%dq_name, startgps, stride)
    logger.info('writing science segments to file : '+seg_file)
    ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))

    (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file

### modify scisegs to account for shifts
logger.info('modifying scisegs to account for shifts')
modified_scisegs = [(s-opts.right_padding-opts.t_lag, e+opts.left_padding-opts.t_lag) for s, e in scisegs]

### find idq segments
logger.info('finding idq segments')
idqsegs = idq.get_idq_segments(realtimedir, lookup_startgps, lookup_endgps, suffix='.dat')

logger.info('taking intersection between modified science segments and idq segments')
idqsegs = event.andsegments( [modified_scisegs, idqsegs] )

### write segment file
if opts.ignore_science_segments:
    idqseg_path = idq.idqsegascii(opts.output_dir, '', startgps, stride)
else:
    idqseg_path = idq.idqsegascii(opts.output_dir, '_%s'%dq_name, startgps , stride)
f = open(idqseg_path, 'w')
for seg in idqsegs:
    print >> f, seg[0], seg[1]
f.close()

#========================
    seg_file = idq.segxml(opts.output_dir, "_%s" % dq_name, startgps, stride)
    logger.info('writing science segments to file : ' + seg_file)
    ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))

    (scisegs, coveredseg) = idq.extract_dq_segments(
        seg_file, dq_name)  ### read in segments from xml file

### modify scisegs to account for shifts
logger.info('modifying scisegs to account for shifts')
modified_scisegs = [(s - opts.right_padding - opts.t_lag,
                     e + opts.left_padding - opts.t_lag) for s, e in scisegs]

### find idq segments
logger.info('finding idq segments')
idqsegs = idq.get_idq_segments(realtimedir,
                               lookup_startgps,
                               lookup_endgps,
                               suffix='.dat')

logger.info(
    'taking intersection between modified science segments and idq segments')
idqsegs = event.andsegments([modified_scisegs, idqsegs])

### write segment file
if opts.ignore_science_segments:
    idqseg_path = idq.idqsegascii(opts.output_dir, '', startgps, stride)
else:
    idqseg_path = idq.idqsegascii(opts.output_dir, '_%s' % dq_name, startgps,
                                  stride)
f = open(idqseg_path, 'w')
for seg in idqsegs:
    print >> f, seg[0], seg[1]