def check_calibration(segs, times, timeseries, FAPthrs):
    """
    checks the pipeline's calibration at each "FAPthr in FAPThrs"

    this may be sped up with a call to timeseries_to_livetime() instead of idq.timeseries_to_segments() -> event.livetime
    however, we currently use some segment logic and it is not clear that we can avoid actually generating segments
    """
    idq_livetime = event.livetime(segs)

    segments = []
    deadtimes = []
    statedFAPs = []
    errs = []
    for FAPthr in FAPthrs:
        SEGS = []
        max_statedFAP = 0.0
        for (t, ts) in zip(times, timeseries):
            (_segs, _min_ts) = idq.timeseries_to_segments(
                t, -ts, -FAPthr)  # we want FAP <= FAPthr <--> -FAP >= -FAPthr
            SEGS += (
                _segs
            )  ### ensure this is a list in case the behavior of timeseries_to_segments changes...
            if _min_ts != None:
                statedFAP = -_min_ts
                if max_statedFAP < statedFAP:
                    max_statedFAP = statedFAP

        SEGS = event.andsegments([SEGS, segs])
        segments.append(SEGS)

        SEGS_livetime = event.livetime(SEGS)

        if not idq_livetime:
            if SEGS_livetime:
                raise ValueError(
                    "something is weird with segments... idq_livetime is zero but SEGS_livetime is not"
                )
            else:
                deadtime = 0.0
        else:
            deadtime = 1.0 * SEGS_livetime / idq_livetime
            if deadtime > 1.0:
                raise ValueError(
                    "deadtime > 1.0, something is weird...\n  SEGS_livetime = %f\n  idq_livetime = %f"
                    % (SEGS_livetime, idq_livetime))
        deadtimes.append(deadtime)

        statedFAPs.append(max_statedFAP)

        if max_statedFAP > 0:
            err = deadtime / max_statedFAP - 1
        elif deadtime:
            err = 1
        else:
            err = 0
        errs.append(err)

    return segments, deadtimes, statedFAPs, errs
Exemplo n.º 2
0
def check_calibration( segs, times, timeseries, FAPthrs):
    """
    checks the pipeline's calibration at each "FAPthr in FAPThrs"

    this may be sped up with a call to timeseries_to_livetime() instead of idq.timeseries_to_segments() -> event.livetime
    however, we currently use some segment logic and it is not clear that we can avoid actually generating segments
    """
    idq_livetime = event.livetime(segs)

    segments = []
    deadtimes = []
    statedFAPs = []
    errs = []
    for FAPthr in FAPthrs:
        SEGS = []
        max_statedFAP = 0.0
        for (t, ts) in zip(times, timeseries):
            (_segs, _min_ts) = idq.timeseries_to_segments(t, -ts, -FAPthr)  # we want FAP <= FAPthr <--> -FAP >= -FAPthr
            SEGS += (_segs) ### ensure this is a list in case the behavior of timeseries_to_segments changes...
            if _min_ts != None:
                statedFAP = -_min_ts
                if max_statedFAP < statedFAP:
                    max_statedFAP = statedFAP

        SEGS = event.andsegments([SEGS, segs])
        segments.append(SEGS)

        SEGS_livetime = event.livetime(SEGS)

        if not idq_livetime:
            if SEGS_livetime:
                raise ValueError("something is weird with segments... idq_livetime is zero but SEGS_livetime is not")
            else:
                deadtime = 0.0
        else:
            deadtime = 1.0 * SEGS_livetime / idq_livetime
            if deadtime > 1.0:
                raise ValueError("deadtime > 1.0, something is weird...\n  SEGS_livetime = %f\n  idq_livetime = %f"%(SEGS_livetime, idq_livetime))
        deadtimes.append( deadtime )

        statedFAPs.append(max_statedFAP)

        if max_statedFAP > 0:
            err = deadtime/max_statedFAP - 1
        elif deadtime:
            err = 1
        else:
            err = 0
        errs.append( err )

    return segments, deadtimes, statedFAPs, errs
Exemplo n.º 3
0
            if opts.force:  ### we are require successful training or else we want errors
                logger.info(traceback.print_exc())
                raise e
            else:  ### we don't care if any particular training job fails
                gpsstart += stride
                continue

    logger.info('finding idq segments')
    idqsegs = idq.get_idq_segments(realtimedir,
                                   gpsstart - lookback,
                                   gpsstart + stride,
                                   suffix='.dat')

    logger.info(
        'taking intersection between science segments and idq segments')
    idqsegs = event.andsegments([scisegs, idqsegs])

    ### write segment file
    if opts.ignore_science_segments:
        idqseg_path = idq.idqsegascii(output_dir, '', gpsstart - lookback,
                                      lookback + stride)
    else:
        idqseg_path = idq.idqsegascii(output_dir, '_%s' % dq_name,
                                      gpsstart - lookback, lookback + stride)
    f = open(idqseg_path, 'w')
    for seg in idqsegs:
        print >> f, seg[0], seg[1]
    f.close()

    #===============================================================================================
    # update mappings via uroc files
    ### science segments xml filename
    seg_file = idq.segxml(gdbdir, "%s_%s"%(filetag, dq_name), opts.start , opts.end-opts.start )
    if opts.verbose:
        print '  writing science segments to file : '+seg_file
    ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))

    (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file

if opts.verbose:
    print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir, opts.start, opts.end, suffix='.dat')

if opts.verbose:
    print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments( [scisegs, idqsegs] )

### write segment file
if opts.ignore_science_segments:
    idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start, opts.end-opts.start)
else:
    idqseg_path = idq.idqsegascii(gdbdir, '%s_%s'%(filetag, dq_name), opts.start, opts.end-opts.start)
if opts.verbose:
    print "  writing : "+idqseg_path
f = open(idqseg_path, 'w')
for seg in idqsegs:
    print >> f, seg[0], seg[1]
f.close()

#=================================================
Exemplo n.º 5
0
def extract_ovl_vconfigs( rank_frames, channame, traindir, start, end, metric='eff/dt' ):
    """
    returns a dictionary mapping active vconfigs to segments
    does NOT include "none" channel
    """
    vconfigs = []
    for rnkfr in rank_frames:
        trained, calib = idq.extract_timeseries_ranges( rnkfr )
        classifier = idq.extract_fap_name( rnkfr ) 

        vetolist = glob.glob( "%s/%d_%d/ovl/ovl/*vetolist.eval"%(traindir, trained[0], trained[1]) )        
        if len(vetolist) != 1:
            raise ValueError( "trouble finding a single vetolist file for : %s"%rnkfr )
        vetolist=vetolist[0]
        v = event.loadstringtable( vetolist )

        rankmap = { 0:[(None, None, None, None, 0, 0)] }

        for line in v:
            metric_exp = float(line[ovl.vD['metric_exp']])
            if metric == 'eff/dt':
                rnk = ovl.effbydt_to_rank( metric_exp )
            elif metric == 'vsig':
                rnk = ovl.vsig_to_rank( metric_exp )
            elif metric == 'useP': 
                rnk = ovl.useP_to_rank( metric_exp )
            else:
                raise ValueError("metric=%s not understood"%metric)
            if rankmap.has_key(rnk):
                rankmap[rnk].append( (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk ))
            else:
                rankmap[rnk] = [(line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]), float(line[ovl.vD['vwin']]), metric, metric_exp, rnk )]

        for key, value in rankmap.items():
            rankmap[key] = tuple(value)

        t, ts = idq.combine_gwf( [rnkfr], [channame])
        t = t[0]
        truth = (start <= t)*(t <= end)
        t = t[truth]
        ts = ts[0][truth]
        if not len(ts):
            continue

        configs = rankmap[ts[0]]
        segStart = t[0]
        for T, TS in zip(t, ts):
            if rankmap[TS] != configs:
                vconfigs.append( (configs, [segStart, T] ) )
                segStart = T
                configs = rankmap[TS]
            else:
                pass 
        vconfigs.append( (configs, [segStart, T+t[1]-t[0]] ) )

    configs = {}
    for vconfig, seg in vconfigs:
        if configs.has_key( vconfig ):
            configs[vconfig].append( seg )
        else:
            configs[vconfig] = [ seg ]
    for key, value in configs.items():
        value = event.andsegments( [event.fixsegments( value ), [[start,end]] ] )
        if event.livetime( value ):
            configs[key] = event.fixsegments( value )
        else:
            raise ValueError("somehow picked up a config with zero livetime...")

    return vconfigs, configs, {"vchan":0, "vthr":1, "vwin":2, "metric":3, "metric_exp":4, "rank":5}
            ### this requires us to redefine the 'science-segments' as the intersection of scisegs with realtime segs
            ### we call this intersection "idq_segs"
            else:  ### we're re-using pat files!
                try:
                    ### determine segments from realtime filenames
                    realtime_segs = idq.get_idq_segments(realtimedir,
                                                         gpsstart - lookback,
                                                         gpsstart + stride,
                                                         suffix='.pat')

                    ### read in science segments
                    (scisegs,
                     coveredseg) = idq.extract_dq_segments(seg_file, dq_name)

                    ### take the intersection of these segments
                    idq_segs = event.andsegments([scisegs, realtime_segs])

                    ### write segment file
                    idqseg_path = idq.idqsegascii(output_dir, '_%s' % dq_name,
                                                  gpsstart - lookback,
                                                  lookback + stride)
                    f = open(idqseg_path, 'w')
                    for seg in idq_segs:
                        print >> f, seg[0], seg[1]
                    f.close()
                    ### we may want to remove the unsafe channels, but this could be tricky and we don't want to throw away GW channels accidentally
                    ovlsegs = idqseg_path

                except Exception as e:
                    traceback.print_exc()
                    logger.info(
        except Exception as e:
            traceback.print_exc()
            logger.info('ERROR: segment generation failed. Skipping this calibration period.')

            if opts.force: ### we are require successful training or else we want errors
                logger.info(traceback.print_exc())
                raise e
            else: ### we don't care if any particular training job fails
                gpsstart += stride
                continue

    logger.info('finding idq segments')
    idqsegs = idq.get_idq_segments(realtimedir, gpsstart-lookback, gpsstart+stride, suffix='.dat')

    logger.info('taking intersection between science segments and idq segments')
    idqsegs = event.andsegments( [scisegs, idqsegs] )

    ### write segment file
    if opts.ignore_science_segments:
        idqseg_path = idq.idqsegascii(output_dir, '', gpsstart-lookback, lookback+stride)
    else:
        idqseg_path = idq.idqsegascii(output_dir, '_%s'%dq_name, gpsstart - lookback, lookback+stride)
    f = open(idqseg_path, 'w')
    for seg in idqsegs:
        print >> f, seg[0], seg[1]
    f.close()

    #===============================================================================================
    # update mappings via uroc files
    #===============================================================================================
Exemplo n.º 8
0
#-------------------------------------------------

### query segments
segs = [[start, end]]
for ifo in opts.observatory:

    segfilename = "%s/%s1_%s-%d-%d.xml"%(opts.output_dir, ifo, opts.flag.replace(":","_"), start, end-start)
    cmd = "ligolw_segment_query_dqsegdb -t %s -q -a %s1:%s -s %d -e %d -o %s"%(opts.segdb_url, ifo, opts.flag, start, end, segfilename)
    if opts.verbose:
        print "querying %s segments for %s\n    %s"%(opts.flag, ifo, cmd)

    output = sp.Popen( cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE ).communicate()

    ### iterate over segments, computing PSDs
    xmldoc = ligolw_utils.load_filename(segfilename, contenthandler=lsctables.use_in(ligolw.LIGOLWContentHandler))
    segs = event.andsegments( [segs, [[row.start_time, row.end_time] for row in table.get_table(xmldoc, lsctables.SegmentTable.tableName)]] ) ### take intersection of segments

### write intersection to ascii file
segfilename = "%s/intersection-%d-%d.seg"%(opts.output_dir, start, end-start)
if opts.verbose:
    print "found %d sec of joint livetime"%(event.livetime(segs))
    print "writing : %s"%(segfilename)
file_obj = open(segfilename, 'w')
for s, e in segs:
    print >> file_obj, s, e
file_obj.close()

#-------------------------------------------------

### iterate through segments and estimte PSD for each IFO
if opts.verbose:
Exemplo n.º 9
0
for gps in args:
    print "gps : %.9f" % (gps)

    minwin = opts.window

    ### go find triggers
    if opts.verbose:
        print "\tdiscoverying KW triggers within [%.9f, %.9f]" % (gps - opts.window, gps + opts.window)

    ### figure out which files you want
    filenames = []
    coverage = []
    for gdsdir in kwgdsdirs:
        for filename in idq.get_all_files_in_range(gdsdir, gps - opts.window, gps + opts.window, pad=0, suffix=".trg"):
            seg = idq.extract_start_stop(filename, suffix=".trg")
            if not event.livetime(event.andsegments([coverage, [seg]])):
                coverage = event.fixsegments(coverage + [seg])
                filenames.append(filename)

    ### figure out the extent of the coverage
    if len(event.include([[gps]], coverage, tcent=0)) == 0:
        if opts.force:
            if opts.verbose:
                print "no triggers found for gps : %.3f" % (gps)
            continue
        else:
            raise ValueError("no triggers found for gps : %.3f" % (gps))
    for s, e in coverage:
        if s < gps:
            if gps - s < minwin:
                minwin = gps - s
Exemplo n.º 10
0
for gps in args:
    print "gps : %.9f"%(gps)

    minwin = opts.window

    ### go find triggers
    if opts.verbose:
        print "\tdiscoverying KW triggers within [%.9f, %.9f]"%(gps-opts.window, gps+opts.window)

    ### figure out which files you want
    filenames = []
    coverage = []
    for gdsdir in kwgdsdirs:
        for filename in idq.get_all_files_in_range(gdsdir, gps-opts.window, gps+opts.window, pad=0, suffix=".trg"):
            seg = idq.extract_start_stop(filename, suffix=".trg")
            if not event.livetime(event.andsegments([coverage, [seg]])):
                coverage = event.fixsegments( coverage + [seg] )
                filenames.append( filename )

    ### figure out the extent of the coverage
    if len(event.include([[gps]], coverage, tcent=0)) == 0:
        if opts.force:
            if opts.verbose:
                print "no triggers found for gps : %.3f"%(gps)
            continue
        else:
            raise ValueError("no triggers found for gps : %.3f"%(gps))
    for s, e in coverage:
        if s < gps:
            if gps-s < minwin:
                minwin = gps-s
    ### science segments xml filename
    seg_file = idq.segxml(gdbdir, "%s_%s"%(filetag, dq_name), opts.start , opts.end-opts.start )
    if opts.verbose:
        print '  writing science segments to file : '+seg_file
    ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))

    (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file

if opts.verbose:
    print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir, opts.start, opts.end, suffix='.dat')

if opts.verbose:
    print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments( [scisegs, idqsegs] )

### write segment file
if opts.ignore_science_segments:
    idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start, opts.end-opts.start)
else:
    idqseg_path = idq.idqsegascii(gdbdir, '%s_%s'%(filetag, dq_name), opts.start, opts.end-opts.start)
if opts.verbose:
    print "  writing : "+idqseg_path
f = open(idqseg_path, 'w')
for seg in idqsegs:
    print >> f, seg[0], seg[1]
f.close()

#=================================================
Exemplo n.º 12
0
if opts.dq_segments:
    # load dq segments
    if opts.verbose:
        print "reading segments from %s" % opts.dq_segments

    (dq_segments, covered_segments) = \
        idq.extract_dq_segments(open(opts.dq_segments, 'r'),
                                opts.dq_segments_name)

    # sort and merge segments
    dq_segments = event.fixsegments(dq_segments)

    ### filter patfiles by scisegs, keep only those events with non-zero overlap with science time
    patfiles = [
        pat for pat in patfiles if event.livetime(
            event.andsegments(
                [dq_segments, [idq.extract_start_stop(pat, suffix=".pat")]]))
    ]

    if opts.verbose:
        print "%d patfiles remain after taking overlap with segments" % len(
            patfiles)

if len(patfiles) == 0:
    print 'No *.pat files found in the gps range ' \
        + str(gps_start_time) + ' - ' + str(gps_end_time)
    print 'Exiting with status 2.'
    sys.exit(2)

# load auxmvc vector samples
auxmvc_samples = auxmvc_utils.ReadMVSCTriggers(patfiles, Classified=False)
def check_calibartion(
    realtimedir,
    start,
    end,
    classifier,
    FAPthrs,
    verbose=False,
    ):
    """
....checks the pipeline's calibration at each "FAPthr in FAPThrs"
        """

        # =================================================
        # grab idq_segments so we compute meaningful livetimes
        # =================================================

    if verbose:
        print 'getting idq_segements'

    idq_segs = idq.get_idq_segments(realtimedir, start, end,
                                    suffix='.npy.gz')
    idq_livetime = event.livetime(idq_segs)

    # =================================================
    # grab relevant data
    # =================================================

    if verbose:
        print 'looking for *_fap_*.npy.gz files in', opts.realtimedir

    fapfilenames = [filename for filename in
                    idq.get_all_files_in_range(realtimedir, start, end,
                    pad=0, suffix='.npy.gz') if '_fap_' in filename
                    and classifier in filename]

    if opts.verbose:
        print 'discovered %d files' % len(fapfilenames)
        print 'building time-series'

    (times, timeseries) = idq_gdb_utils.combine_ts(fapfilenames)

        # =================================================
        # check calibration
        # =================================================

    segments = []
    deadtimes = []
    statedFAPs = []
    for FAPthr in FAPthrs:
        if verbose:
            print 'computing segements for FAPthr =', FAPthr

        segs = []
        max_statedFAP = None
        for (t, ts) in zip(times, timeseries):
            (_segs, _min_ts) = timeseries_to_segments(t, -ts, -FAPthr)  # we want FAP <= FAPthr <--> -FAP >= FAPthr
            segs += _segs
            if _min_ts != None:
                statedFAP = -_min_ts
                if max_statedFAP < statedFAP:
                    max_statedFAP = statedFAP

        segs = event.andsegments([segs, idq_segs])
        segments.append(segs)
        deadtimes.append(1.0 * event.livetime(segs) / idq_livetime)
        statedFAPs.append(statedFAP)

    return (idq_segs, segments, deadtimes, statedFAPs)
Exemplo n.º 14
0
        'general', 'kw')).items():  ### this is kinda ugly...
    if opts.verbose:
        print(
            "  searching for KW trgfiles corresponding to %s in %s within [%.3f, %.3f]"
            % (kwconf, dirname, segs[0][0], segs[-1][1]))

    ### iterate over all trg files found in that directory
    for trgfile in idq.get_all_files_in_range(dirname,
                                              segs[0][0],
                                              segs[-1][1],
                                              pad=0,
                                              suffix='.trg'):
        ### check whether there is some overlap
        ### not gauranteed if there are gaps between min and max gps times
        if event.livetime(
                event.andsegments(
                    [[idq.extract_start_stop(trgfile, suffix='.trg')], segs])):
            if opts.verbose:
                print("    kept : " + trgfile)
            kw_trgfiles.append(trgfile)

        elif opts.verbose:
            print("    discarded : " + trgfile)

#---

if opts.verbose:
    print("evaluating %d times using %d KW trgfiles" %
          (Ngps, len(kw_trgfiles)))
### set up output pointers
if opts.output_filename:
    datfile = os.path.basename(opts.output_filename)
    seg_file = idq.segxml(opts.output_dir, "_%s"%dq_name, startgps, stride)
    logger.info('writing science segments to file : '+seg_file)
    ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))

    (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name) ### read in segments from xml file

### modify scisegs to account for shifts
logger.info('modifying scisegs to account for shifts')
modified_scisegs = [(s-opts.right_padding-opts.t_lag, e+opts.left_padding-opts.t_lag) for s, e in scisegs]

### find idq segments
logger.info('finding idq segments')
idqsegs = idq.get_idq_segments(realtimedir, lookup_startgps, lookup_endgps, suffix='.dat')

logger.info('taking intersection between modified science segments and idq segments')
idqsegs = event.andsegments( [modified_scisegs, idqsegs] )

### write segment file
if opts.ignore_science_segments:
    idqseg_path = idq.idqsegascii(opts.output_dir, '', startgps, stride)
else:
    idqseg_path = idq.idqsegascii(opts.output_dir, '_%s'%dq_name, startgps , stride)
f = open(idqseg_path, 'w')
for seg in idqsegs:
    print >> f, seg[0], seg[1]
f.close()

#========================
# go findeth the frame data
#========================
logger.info('  finding all *fap*.gwf files')
Exemplo n.º 16
0
                        gpsstart += stride 
                        continue 

            ### if we aren't building auxmvc vectors, we re-use pat files from realtime job
            ### this requires us to redefine the 'science-segments' as the intersection of scisegs with realtime segs
            ### we call this intersection "idq_segs"
            else: ### we're re-using pat files!
                try:
                    ### determine segments from realtime filenames
                    realtime_segs = idq.get_idq_segments(realtimedir, gpsstart - lookback, gpsstart + stride, suffix='.pat')

                    ### read in science segments
                    (scisegs, coveredseg) = idq.extract_dq_segments(seg_file, dq_name)

                    ### take the intersection of these segments
                    idq_segs = event.andsegments([scisegs, realtime_segs])

                    ### write segment file
                    idqseg_path = idq.idqsegascii(output_dir, '_%s'%dq_name, gpsstart - lookback, lookback+stride)
                    f = open(idqseg_path, 'w')
                    for seg in idq_segs:
                        print >> f, seg[0], seg[1]
                    f.close()
 ### we may want to remove the unsafe channels, but this could be tricky and we don't want to throw away GW channels accidentally
                    ovlsegs = idqseg_path

                except Exception as e:
                    traceback.print_exc()
                    logger.info('WARNING: failed to generate iDQ segments from realtime output.')

                    if opts.force:
        print '  writing science segments to file : ' + seg_file
    ligolw_utils.write_filename(xmldoc, seg_file, gz=seg_file.endswith(".gz"))

    (scisegs, coveredseg) = idq.extract_dq_segments(
        seg_file, dq_name)  ### read in segments from xml file

if opts.verbose:
    print 'finding idq segments'
idqsegs = idq.get_idq_segments(realtimedir,
                               opts.start,
                               opts.end,
                               suffix='.dat')

if opts.verbose:
    print 'taking intersection between science segments and idq segments'
idqsegs = event.andsegments([scisegs, idqsegs])

### write segment file
if opts.ignore_science_segments:
    idqseg_path = idq.idqsegascii(gdbdir, filetag, opts.start,
                                  opts.end - opts.start)
else:
    idqseg_path = idq.idqsegascii(gdbdir, '%s_%s' % (filetag, dq_name),
                                  opts.start, opts.end - opts.start)
if opts.verbose:
    print "  writing : " + idqseg_path
f = open(idqseg_path, 'w')
for seg in idqsegs:
    print >> f, seg[0], seg[1]
f.close()
#---

### look up KW trg files that intersect segs
if opts.verbose:
    print( "finding relevant kw_trgfiles" )
kw_trgfiles = []
### iterate over different configurations used in training
for kwconf, dirname in eval(config.get('general', 'kw')).items(): ### this is kinda ugly...
    if opts.verbose:
        print( "  searching for KW trgfiles corresponding to %s in %s within [%.3f, %.3f]"%(kwconf, dirname, segs[0][0], segs[-1][1]) )

    ### iterate over all trg files found in that directory
    for trgfile in idq.get_all_files_in_range(dirname, segs[0][0], segs[-1][1], pad=0, suffix='.trg'):
        ### check whether there is some overlap 
        ### not gauranteed if there are gaps between min and max gps times
        if event.livetime(event.andsegments([[idq.extract_start_stop(trgfile, suffix='.trg')], segs])): 
            if opts.verbose:
                print( "    kept : "+trgfile )
            kw_trgfiles.append( trgfile )

        elif opts.verbose:
            print( "    discarded : "+trgfile )

#---

if opts.verbose:
    print( "evaluating %d times using %d KW trgfiles"%(Ngps, len(kw_trgfiles) ) )
### set up output pointers
if opts.output_filename:
    datfile = os.path.basename(opts.output_filename)
    output_dir = os.path.dirname(opts.output_filename)
Exemplo n.º 19
0
def extract_ovl_vconfigs(rank_frames,
                         channame,
                         traindir,
                         start,
                         end,
                         metric='eff/dt'):
    """
    returns a dictionary mapping active vconfigs to segments
    does NOT include "none" channel
    """
    vconfigs = []
    for rnkfr in rank_frames:
        trained, calib = idq.extract_timeseries_ranges(rnkfr)
        classifier = idq.extract_fap_name(rnkfr)

        vetolist = glob.glob("%s/%d_%d/ovl/ovl/*vetolist.eval" %
                             (traindir, trained[0], trained[1]))
        if len(vetolist) != 1:
            raise ValueError(
                "trouble finding a single vetolist file for : %s" % rnkfr)
        vetolist = vetolist[0]
        v = event.loadstringtable(vetolist)

        rankmap = {0: [(None, None, None, None, 0, 0)]}

        for line in v:
            metric_exp = float(line[ovl.vD['metric_exp']])
            if metric == 'eff/dt':
                rnk = ovl.effbydt_to_rank(metric_exp)
            elif metric == 'vsig':
                rnk = ovl.vsig_to_rank(metric_exp)
            elif metric == 'useP':
                rnk = ovl.useP_to_rank(metric_exp)
            else:
                raise ValueError("metric=%s not understood" % metric)
            if rankmap.has_key(rnk):
                rankmap[rnk].append(
                    (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]),
                     float(line[ovl.vD['vwin']]), metric, metric_exp, rnk))
            else:
                rankmap[rnk] = [
                    (line[ovl.vD['vchan']], float(line[ovl.vD['vthr']]),
                     float(line[ovl.vD['vwin']]), metric, metric_exp, rnk)
                ]

        for key, value in rankmap.items():
            rankmap[key] = tuple(value)

        t, ts = idq.combine_gwf([rnkfr], [channame])
        t = t[0]
        truth = (start <= t) * (t <= end)
        t = t[truth]
        ts = ts[0][truth]
        if not len(ts):
            continue

        configs = rankmap[ts[0]]
        segStart = t[0]
        for T, TS in zip(t, ts):
            if rankmap[TS] != configs:
                vconfigs.append((configs, [segStart, T]))
                segStart = T
                configs = rankmap[TS]
            else:
                pass
        vconfigs.append((configs, [segStart, T + t[1] - t[0]]))

    configs = {}
    for vconfig, seg in vconfigs:
        if configs.has_key(vconfig):
            configs[vconfig].append(seg)
        else:
            configs[vconfig] = [seg]
    for key, value in configs.items():
        value = event.andsegments([event.fixsegments(value), [[start, end]]])
        if event.livetime(value):
            configs[key] = event.fixsegments(value)
        else:
            raise ValueError(
                "somehow picked up a config with zero livetime...")

    return vconfigs, configs, {
        "vchan": 0,
        "vthr": 1,
        "vwin": 2,
        "metric": 3,
        "metric_exp": 4,
        "rank": 5
    }
### modify scisegs to account for shifts
logger.info('modifying scisegs to account for shifts')
modified_scisegs = [(s - opts.right_padding - opts.t_lag,
                     e + opts.left_padding - opts.t_lag) for s, e in scisegs]

### find idq segments
logger.info('finding idq segments')
idqsegs = idq.get_idq_segments(realtimedir,
                               lookup_startgps,
                               lookup_endgps,
                               suffix='.dat')

logger.info(
    'taking intersection between modified science segments and idq segments')
idqsegs = event.andsegments([modified_scisegs, idqsegs])

### write segment file
if opts.ignore_science_segments:
    idqseg_path = idq.idqsegascii(opts.output_dir, '', startgps, stride)
else:
    idqseg_path = idq.idqsegascii(opts.output_dir, '_%s' % dq_name, startgps,
                                  stride)
f = open(idqseg_path, 'w')
for seg in idqsegs:
    print >> f, seg[0], seg[1]
f.close()

#========================
# go findeth the frame data
#========================