def find_kw(channel, start, end, base=None): """Find KW trigger XML files """ span = Segment(to_gps(start), to_gps(end)) channel = get_channel(channel) ifo = channel.ifo if base is None and channel.name.split(':', 1)[-1] == 'GDS-CALIB_STRAIN': tag = '%s-KW_HOFT' % ifo[0].upper() base = '/gds-%s/dmt/triggers/%s' % (ifo.lower(), tag) elif base is None: tag = '%s-KW_TRIGGERS' % ifo[0].upper() base = '/gds-%s/dmt/triggers/%s' % (ifo.lower(), tag) gps5 = int('%.5s' % start) end5 = int('%.5s' % end) out = Cache() append = out.append while gps5 <= end5: trigglob = os.path.join( base, '%s-%d' % (tag, gps5), '%s-*-*.xml' % tag) found = glob.glob(trigglob) for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) gps5 += 1 out.sort(key=lambda e: e.path) vprint(" Found %d files for %s (KW)\n" % (len(out), channel.ndsname)) return out
def parse_command_line(): parser = OptionParser( version="%prog CVS $Id$", usage="%prog [options]", description= "Constructs the likelihood-ratio based coincidence stage for an excess power analysis. The input consists of one or more LAL caches listing the sqlite database trigger files, and a list of segments giving the time intervals that should be considered to be independent. The LAL caches list all trigger files together, that is injections, time slides, and zero-lag. The individual trigger files are self-describing, so the analysis codes can autodetect their type. Each segment will be analyzed using the files that intersect it: the likelihood ratios will be constructed from the injections and time-lag triggers contained in files that intersect the segment, and that data used to assign likelihoods to the injections, time-lag, and zero-lag coincs in all files that intersect the same segment." ) parser.add_option( "--input-cache", metavar="filename", action="append", default=[], help= "Add the contents of this cache file to the list of files from which to draw statistics." ) parser.add_option( "--round-robin-cache", metavar="filename", action="append", default=[], help= "Add the contents of this cache file to the list of files from which to draw injection statistics in a round-robin way." ) parser.add_option( "--condor-log-dir", metavar="path", default=".", help="Set the directory for Condor log files (default = \".\").") parser.add_option( "--config-file", metavar="filename", default="power.ini", help="Set .ini configuration file name (default = \"power.ini\").") parser.add_option( "--distribution-segments", metavar="filename", help= "Read boundaries for distribution data intervals from this segwizard format segments file (required)." ) parser.add_option("-v", "--verbose", action="store_true", help="Be verbose.") options, filenames = parser.parse_args() if options.distribution_segments is None: raise ValueError, "missing required argument --distribution-segments" options.distribution_segments = segmentsUtils.fromsegwizard( file(options.distribution_segments), coltype=LIGOTimeGPS) options.input_cache = set([ CacheEntry(line) for filename in options.input_cache for line in file(filename) ]) options.round_robin_cache = [ set(map(CacheEntry, file(filename))) for filename in options.round_robin_cache ] return options, (filenames or [])
def find_dmt_omega(channel, start, end, base=None): """Find DMT-Omega trigger XML files """ span = Segment(to_gps(start), to_gps(end)) channel = get_channel(channel) ifo = channel.ifo if base is None and channel.name.split(':', 1)[-1] == 'GDS-CALIB_STRAIN': base = '/gds-%s/dmt/triggers/%s-HOFT_Omega' % ( ifo.lower(), ifo[0].upper()) elif base is None: raise NotImplementedError("This method doesn't know how to locate DMT " "Omega trigger files for %r" % str(channel)) gps5 = int('%.5s' % start) end5 = int('%.5s' % end) out = Cache() append = out.append while gps5 <= end5: trigglob = os.path.join( base, str(gps5), '%s-%s_%s_%s_OmegaC-*-*.xml' % ( ifo, channel.system, channel.subsystem, channel.signal)) found = glob.glob(trigglob) for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) gps5 += 1 out.sort(key=lambda e: e.path) vprint(" Found %d files for %s (DMT-Omega)\n" % (len(out), channel.ndsname)) return out
def read_losc_state(filename, channel, group=None, start=None, end=None, copy=False): """Read a `StateVector` from a LOSC-format HDF file. """ h5file = open_hdf5(filename) if group: channel = '%s/%s' % (group, channel) # find data dataset = _find_dataset(h5file, '%s/DQmask' % channel) maskset = _find_dataset(h5file, '%s/DQDescriptions' % channel) # read data nddata = dataset.value bits = list(maskset.value) # read metadata try: epoch = dataset.attrs['Xstart'] except KeyError: try: ce = CacheEntry.from_T050017(h5file.filename) except ValueError: epoch = None else: epoch = ce.segment[0] try: dt = dataset.attrs['Xspacing'] except KeyError: dt = Quantity(1, 's') else: xunit = Unit(dataset.attrs['Xunit']) dt = Quantity(dt, xunit) return StateVector(nddata, bits=bits, epoch=epoch, name='Data quality', sample_rate=(1/dt).to('Hertz'), copy=copy)
def read_losc_state(filename, channel, group=None, start=None, end=None): """Read a `StateVector` from a LOSC-format HDF file. """ h5file = open_hdf5(filename) if group: channel = '%s/%s' % (group, channel) # find data dataset = _find_dataset(h5file, '%s/DQmask' % channel) maskset = _find_dataset(h5file, '%s/DQDescriptions' % channel) # read data nddata = dataset.value bits = list(maskset.value) # read metadata try: epoch = dataset.attrs['Xstart'] except KeyError: try: from glue.lal import CacheEntry except ImportError: epoch = None else: ce = CacheEntry.from_T050017(h5file.filename) epoch = ce.segment[0] try: dt = dataset.attrs['Xspacing'] except KeyError: dt = Quantity(1, 's') else: xunit = Unit(dataset.attrs['Xunit']) dt = Quantity(dt, xunit) return StateVector(nddata, bits=bits, epoch=epoch, sample_rate=(1 / dt).to('Hertz'), name='Data quality')
def FrameCachetoLALCache(fcache): lcache = LALCache() files = fcache.get_files() for f in files: lcache.append(LALCacheEntry.from_T050017(f)) return lcache
def setUpClass(cls): cls.FRAMES = {} cls._tempdir = tempfile.mkdtemp(prefix='gwsumm-test-data-') # get data for channel in LOSC_DATA: cls.FRAMES[channel] = Cache() for gwf in LOSC_DATA[channel]: target = os.path.join(cls._tempdir, os.path.basename(gwf)) download(gwf, target) cls.FRAMES[channel].append(CacheEntry.from_T050017(target))
def parse_command_line(): parser = OptionParser( version = "Name: %%prog\n%s" % git_version.verbose_msg, usage = "%prog [options] [file ...]", description = "%prog does blah blah blah." ) parser.add_option("-p", "--live-time-program", metavar = "name", default="lalapps_ring", help = "Set the name of the program whose entries in the search_summary table will set the search live time. Required.") parser.add_option("--veto-segments-name", help = "Set the name of the segments to extract from the segment tables and use as the veto list.") parser.add_option("--categories", metavar = "{\"frequency-ifos-oninstruments\",\"oninstruments\"}", default="oninstruments", help = "Select the event categorization algorithm. Default oninstruments") parser.add_option("-b", "--frequency-bins", metavar = "frequency,frequency[,frequency,...]", help = "Set the boundaries of the frequency bins in Hz. The lowest and highest bounds must be explicitly listed. Example \"0,5,inf\". Required if frequency-based categorization algorithm has been selected.") parser.add_option("--rank-by", metavar = "{\"snr\",\"uncombined-ifar\",\"likelihood\"}", default="snr", help = "Select the event ranking method. Default is snr") parser.add_option("-t", "--tmp-space", metavar = "path", help = "Path to a directory suitable for use as a work area while manipulating the database file. The database file will be worked on in this directory, and then moved to the final location when complete. This option is intended to improve performance when running in a networked environment, where there might be a local disk with higher bandwidth than is available to the filesystem on which the final output will reside.") parser.add_option("-n","--extrapolation-num",action="store",type="int",default=0, metavar="num",help="number of time-slide points to use in FAR extrapolation" ) parser.add_option("-g", "--input-cache", help="cache of sqlite files") parser.add_option("-v", "--verbose", action = "store_true", help = "Be verbose.") options, filenames = parser.parse_args() if options.input_cache is not None: filenames.extend([CacheEntry(c).path() for c in open(options.input_cache)]) # # categories and ranking # if options.categories not in ("frequency-ifos-oninstruments", "oninstruments"): raise ValueError, "missing or unrecognized --categories option" if options.rank_by not in ("snr", "uncombined-ifar", "likelihood"): raise ValueError, "missing or unrecognized --rank-by option" options.populate_column = "false_alarm_rate" # # parse frequency bins # if options.categories in ("frequency-ifos-oninstruments",): if options.frequency_bins is None: raise ValueError, "--frequency-bins required with category algorithm \"%s\"" % options.categories options.frequency_bins = sorted(map(float, options.frequency_bins.split(","))) if len(options.frequency_bins) < 2: raise ValueError, "must set at least two frequency bin boundaries (i.e., define at least one frequency bin)" options.frequency_bins = rate.IrregularBins(options.frequency_bins) # # other # if options.live_time_program is None: raise ValueError, "missing required option -p or --live-time-program" # # done # return options, (filenames or [None])
def read_losc_state(filename, channel, group=None, start=None, end=None, copy=False): """Read a `StateVector` from a LOSC-format HDF file. Parameters ---------- filename : `str` path to LOSC-format HDF5 file to read. channel : `str` name of HDF5 dataset to read. group : `str`, optional name of containing HDF5 group for ``channel``. If not given, the first dataset named ``channel`` will be assumed as the right one. start : `Time`, `~gwpy.time.LIGOTimeGPS`, optional start GPS time of desired data end : `Time`, `~gwpy.time.LIGOTimeGPS`, optional end GPS time of desired data copy : `bool`, default: `False` create a fresh-memory copy of the underlying array Returns ------- data : :class`~gwpy.timeseries.TimeSeries` a new `TimeSeries` containing the data read from disk """ h5file = open_hdf5(filename) if group: channel = '%s/%s' % (group, channel) # find data dataset = _find_dataset(h5file, '%s/DQmask' % channel) maskset = _find_dataset(h5file, '%s/DQDescriptions' % channel) # read data nddata = dataset.value bits = list(maskset.value) # read metadata try: epoch = dataset.attrs['Xstart'] except KeyError: try: ce = CacheEntry.from_T050017(h5file.filename) except ValueError: epoch = None else: epoch = ce.segment[0] try: dt = dataset.attrs['Xspacing'] except KeyError: dt = Quantity(1, 's') else: xunit = parse_unit(dataset.attrs['Xunits']) dt = Quantity(dt, xunit) return StateVector(nddata, bits=bits, epoch=epoch, name='Data quality', dx=dt, copy=copy)
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower().startswith('omicron'): etg = '?' + etg[1:] # construct search gpsstart = to_gps(gpsstart).seconds gpsend = to_gps(gpsend).seconds span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5]) + 1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]' * 10)) # test for channel-level directory if not glob.glob(searchbase): raise ValueError("No channel-level directory found at %s. Either the " "channel name or ETG names are wrong, or this " "channel is not configured for this ETG." % searchbase) # perform and cache results out = Cache() append = out.append for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end=' ') found = set( map(os.path.realpath, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) n = 0 for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) n += 1 if verbose: gprint("%d found" % n) out.sort(key=lambda e: e.path) return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False): """Find the paths of trigger files that represent the given observatory, channel, and ETG (event trigger generator) for a given GPS [start, end) segment. """ if etg.lower().startswith('omicron'): etg = '?' + etg[1:] # construct search gpsstart = to_gps(gpsstart).seconds gpsend = to_gps(gpsend).seconds span = Segment(gpsstart, gpsend) ifo, channel = channel.split(':', 1) trigtype = "%s_%s" % (channel, etg.lower()) epoch = '*' searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype) gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1) trigform = ('%s-%s_%s-%s-*.xml*' % (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]'*10)) # test for channel-level directory if not glob.glob(searchbase): raise ValueError("No channel-level directory found at %s. Either the " "channel name or ETG names are wrong, or this " "channel is not configured for this ETG." % searchbase) # perform and cache results out = Cache() append = out.append for gpsdir in gpsdirs: gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform) if verbose: gprint("Searching %s..." % os.path.split(gpssearchpath)[0], end=' ') found = set(map( os.path.realpath, glob.glob(os.path.join(searchbase, str(gpsdir), trigform)))) n = 0 for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) n += 1 if verbose: gprint("%d found" % n) out.sort(key=lambda e: e.path) return out
def get_output_cache(self): """ Returns a LAL cache of the output file name. Calling this method also induces the output name to get set, so it must be at least once. """ if not self.output_cache: self.output_cache = [ CacheEntry( self.get_ifo(), self.__usertag, segments.segment(LIGOTimeGPS(self.get_start()), LIGOTimeGPS(self.get_end())), "file://localhost" + os.path.abspath(self.get_output())) ] return self.output_cache
def get_files(self): """ Return Find all files described by this FrameCacheEntry. """ filenames = glob.glob(os.path.join(self.path,\ '%s-%s*-%s.*' % (self.observatory,\ self.description,\ self.duration))) cache = [e.path for e in\ LALCache([LALCacheEntry.from_T050017(f) for f in filenames])\ if e.observatory==self.observatory and\ e.description==self.description and\ self.segment.intersects(e.segment) and\ abs(e.segment)==self.duration] return cache
def fromlalcache(cachefile, coltype=int): """ Construct a segmentlist representing the times spanned by the files identified in the LAL cache contained in the file object file. The segmentlist will be created with segments whose boundaries are of type coltype, which should raise ValueError if it cannot convert its string argument. Example: >>> from lal import LIGOTimeGPS >>> cache_seglists = fromlalcache(open(filename), coltype = LIGOTimeGPS).coalesce() See also: glue.lal.CacheEntry """ return segments.segmentlist( CacheEntry(l, coltype=coltype).segment for l in cachefile)
def process(self, config=GWSummConfigParser(), **kwargs): # set params self.rounds = None if not os.path.isdir(self.directory): self.rounds = None return # get some basic info ifo = config.get('DEFAULT', 'ifo') # read the configuration d = os.path.realpath(self.directory).rstrip('/') self.conf = dict() confs = glob(os.path.join(d, '%s-HVETO_CONF-*-*.txt' % ifo)) if len(confs) != 1: self.rounds = 'FAIL' return conffile = confs[0] try: with open(conffile) as f: self.conf = dict() lines = f.readlines()[3:] for line in lines: try: key, val = line.split(': ', 1) self.conf[key.strip()] = eval(val) except (ValueError, SyntaxError, NameError): pass except IOError: self.rounds = 'FAIL' return else: etg = self.conf.pop('AUXtype', None) if 'DEfnm' in self.conf: name = re_quote.sub('', self.conf['DEfnm']) self.primary = '%s:%s' % (ifo, name) if 'DEtype' in self.conf: hetg = re_quote.sub('', self.conf['DEtype']) if re.search('_%s\Z' % hetg, self.primary, re.I): self.primary = self.primary[:-len(hetg)-1] else: self.primary = None # find the segments try: ce = CacheEntry.from_T050017(conffile) except (ValueError): start = int(self.span[0]) duration = int(abs(self.span)) span = self.span else: start = int(ce.segment[0]) duration = int(abs(ce.segment)) span = ce.segment try: statefile = self.conf['dqfnm'] except KeyError: statefile = '%s-HVETO_DQ_SEGS-%d-%d.txt' % (ifo, start, duration) if not os.path.isfile(os.path.join(self.directory, statefile)): self.rounds = 'NOSEGMENTS' return # find the results table resultsfile = os.path.join(self.directory, 'summary_stats.txt') if not os.path.isfile(resultsfile): self.rounds = 'FAIL' return # determine the Hveto state cache = Cache([CacheEntry.from_T050017( os.path.join(self.directory, statefile))]) segments = SegmentList.read(cache) globalv.SEGMENTS[self.states[0].definition] = DataQualityFlag( self.states[0].definition, known=[span], active=segments) self.finalize_states(config=config, query=False) # read results file self.rounds = [] with open(resultsfile, 'r') as f: for line in f.readlines(): self.rounds.append(dict(zip(self.summaryrows, line.split(' ')[1:]))) # fix channel name c = '%s:%s' % (ifo, self.rounds[-1]['Winning channel']) if etg and re.search('_%s\Z' % etg, c, re.I): c = c.rsplit('_', 1)[0] self.rounds[-1]['Winning channel'] = c # read starting triggers rawfile = ('%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' % (ifo, start, duration)) cache = Cache([CacheEntry.from_T050017( os.path.join(self.directory, rawfile))]) get_triggers('%s:hveto_start' % ifo, 'hveto', [self.span], config=config, cache=cache, return_=False) get_triggers('%s:hveto_vetoed_all' % ifo, 'hveto', [self.span], config=config, cache=Cache(), return_=False) for r in range(1, len(self.rounds) + 1): # read round veto triggers rawfile = ('%s-HVETO_VETOED_TRIGS_ROUND_%d-%d-%d.txt' % (ifo, r, start, duration)) cache = Cache([CacheEntry.from_T050017( os.path.join(self.directory, rawfile))]) trigs = get_triggers('%s:hveto_vetoed_round %d' % (ifo, r), 'hveto', [self.span], config=config, cache=cache) add_triggers(trigs, '%s:hveto_vetoed_all,hveto' % ifo, segments=SegmentList([self.span])) # read round veto segments segfile = ('%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' % (ifo, r, start, duration)) cache = Cache([CacheEntry.from_T050017( os.path.join(self.directory, segfile))]) get_segments('%s:hveto_veto_segs_round_%d' % (ifo, r), [self.span], config=config, cache=cache, return_=False) for plot in self.plots: if isinstance(plot, HvetoSegmentSummaryPlot): plot.find_flags() kwargs['trigcache'] = Cache() kwargs['segmentcache'] = Cache() super(HvetoTab, self).process(config=config, **kwargs) # find some plots for plot in ['OVERAL_HISTOGRAM', 'OVERAL_EFF_DT'][::-1]: filename = ( '%s-HVETO_%s-%d-%d.png' % (ifo, plot, start, duration)) plotfile = os.path.join(self.directory, filename) if os.path.isfile(plotfile): p = SummaryPlot(os.path.join(self.url, filename), new=False) p.state = self.states[0] self.plots.insert(0, p) # delete data from archive del globalv.SEGMENTS[self.states[0].definition] for row in range(1, len(self.rounds) + 1): del globalv.SEGMENTS['%s:hveto_veto_segs_round_%s' % (ifo, row)]
#!/usr/bin/python import sys from glue.lal import CacheEntry from glue.ligolw import lsctables, utils for filename in (CacheEntry(line).path for line in file(sys.argv[1])): xmldoc = utils.load_filename(filename, gz=(filename or "stdin").endswith(".gz")) try: lsctables.table.get_table(xmldoc, lsctables.SnglInspiralTable.tableName) except ValueError: xmldoc.childNodes[-1].appendChild( lsctables.New( lsctables.SnglInspiralTable, columns=("process_id", "ifo", "search", "channel", "end_time", "end_time_ns", "end_time_gmst", "impulse_time", "impulse_time_ns", "template_duration", "event_duration", "amplitude", "eff_distance", "coa_phase", "mass1", "mass2", "mchirp", "mtotal", "eta", "kappa", "chi", "tau0", "tau2", "tau3", "tau4", "tau5", "ttotal", "psi0", "psi3", "alpha", "alpha1", "alpha2", "alpha3", "alpha4", "alpha5", "alpha6", "beta", "f_final", "snr", "chisq", "chisq_dof", "bank_chisq", "bank_chisq_dof", "cont_chisq", "cont_chisq_dof", "sigmasq", "rsqveto_duration", "Gamma0", "Gamma1", "Gamma2", "Gamma3", "Gamma4", "Gamma5", "Gamma6", "Gamma7", "Gamma8", "Gamma9", "event_id"))) utils.write_filename(filename, xmldoc, gz=(filename or "stdout").endswith(".gz"))
def find_frames(ifo, frametype, gpsstart, gpsend, config=GWSummConfigParser(), urltype='file', gaps='warn', onerror='raise'): """Query the datafind server for GWF files for the given type Parameters ---------- ifo : `str` prefix for the IFO of interest (either one or two characters) frametype : `str` name of the frametype to find gpsstart : `int` GPS start time of the query gpsend : `int` GPS end time of the query config : `~ConfigParser.ConfigParser`, optional configuration with `[datafind]` section containing `server` specification, otherwise taken from the environment urltype : `str`, optional what type of file paths to return, default: `file` gaps : `str`, optional what to do when gaps are detected, one of - `ignore` : do nothing - `warn` : display the existence of gaps but carry on - `raise` : raise an exception onerror : `str`, optional what to do when the `~glue.datafind` query itself fails, same options as for ``gaps`` Returns ------- cache : `~glue.lal.Cache` a list of structured frame file descriptions matching the ifo and frametype requested """ vprint(' Finding %s-%s frames for [%d, %d)...' % (ifo[0], frametype, int(gpsstart), int(gpsend))) # find datafind host:port try: host = config.get('datafind', 'server') except (NoOptionError, NoSectionError): try: host = os.environ['LIGO_DATAFIND_SERVER'] except KeyError: host = None port = None else: try: host, port = host.rsplit(':', 1) except ValueError: port = None else: port = int(port) else: port = config.getint('datafind', 'port') # get credentials if port == 80: cert = None key = None else: cert, key = datafind.find_credential() # XXX HACK: LLO changed frame types on Dec 6 2013: LLOCHANGE = 1070291904 if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE: frametype = frametype[-1] # query frames ifo = ifo[0].upper() gpsstart = int(floor(gpsstart)) gpsend = int(ceil(min(globalv.NOW, gpsend))) if gpsend <= gpsstart: return Cache() # parse match try: frametype, match = frametype.split('|', 1) except ValueError: match = None def _query(): if cert is not None: dfconn = datafind.GWDataFindHTTPSConnection(host=host, port=port, cert_file=cert, key_file=key) else: dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port) return dfconn.find_frame_urls(ifo[0].upper(), frametype, gpsstart, gpsend, urltype=urltype, on_gaps=gaps, match=match) try: cache = _query() except RuntimeError as e: sleep(1) try: cache = _query() except RuntimeError: if 'Invalid GPS times' in str(e): e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), ) if onerror in ['ignore', None]: pass elif onerror in ['warn']: warnings.warn('Caught %s: %s' % (type(e).__name__, str(e))) else: raise cache = Cache() # XXX: if querying for day of LLO frame type change, do both if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T'] and gpsstart < LLOCHANGE < gpsend): start = len(cache) and cache[-1].segment[1] or gpsstart if start < gpsend: cache.extend( dfconn.find_frame_urls(ifo[0].upper(), 'L1_%s' % frametype, start, gpsend, urltype=urltype, on_gaps=gaps)[1:]) # extend cache beyond datafind's knowledge to reduce latency try: latest = cache[-1] ngps = len( re_gwf_gps_epoch.search(os.path.dirname( latest.path)).groupdict()['gpsepoch']) except (IndexError, AttributeError): pass else: while True: s, e = latest.segment if s >= gpsend: break # replace GPS time of file basename new = latest.path.replace('-%d-' % s, '-%d-' % e) # replace GPS epoch in dirname new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps]) if os.path.isfile(new): latest = CacheEntry.from_T050017(new) cache.append(latest) else: break # validate files existing and return cache, _ = cache.checkfilesexist() vprint(' %d found.\n' % len(cache)) return cache
# extract observatory observatory = (options.observatory and options.observatory.strip()) or "+".join( sorted(seglists)) # extract description if options.description: description = options.description else: if process_ids is None: description = set(searchsumm.getColumnByName("comment")) else: description = set(row.comment for row in searchsumm if row.process_id in process_ids) if len(description) < 1: raise ValueError, "%s: no matching rows found in search summary table" % filename if len(description) > 1: raise ValueError, "%s: comments in matching rows of search summary table are not identical" % filename description = description.pop().strip() or None # set URL url = "file://localhost" + os.path.abspath(filename) # write cache entry print >> options.output, str( CacheEntry(observatory, description, seglists.extent_all(), url)) # allow garbage collection xmldoc.unlink()
# # Using time slide information, construct segment lists describing times # requiring trigger construction. # if options.verbose: print >>sys.stderr, "Computing segments for which lalapps_power jobs are required ..." background_time_slides = {} background_seglistdict = segments.segmentlistdict() if options.do_noninjections: for filename in options.background_time_slides: cache_entry = CacheEntry(None, None, None, "file://localhost" + os.path.abspath(filename)) background_time_slides[cache_entry] = ligolw_tisi.load_time_slides(filename, verbose = options.verbose, gz = filename.endswith(".gz")).values() background_seglistdict |= compute_segment_lists(seglistdict, background_time_slides[cache_entry], options.minimum_gap, options.timing_params, full_segments = options.full_segments, verbose = options.verbose) injection_time_slides = {} injection_seglistdict = segments.segmentlistdict() if options.do_injections: for filename in options.injection_time_slides: cache_entry = CacheEntry(None, None, None, "file://localhost" + os.path.abspath(filename)) injection_time_slides[cache_entry] = ligolw_tisi.load_time_slides(filename, verbose = options.verbose, gz = filename.endswith(".gz")).values() injection_seglistdict |= compute_segment_lists(seglistdict, injection_time_slides[cache_entry], options.minimum_gap, options.timing_params, full_segments = options.full_segments, verbose = options.verbose) # apply time shifts to segment lists to shift tiling phases, but take # intersection with original segments to stay within allowed times. Note:
def process(self, config=GWSummConfigParser(), **kwargs): # set params self.rounds = None if not os.path.isdir(self.directory): self.rounds = None return # get some basic info ifo = config.get('DEFAULT', 'ifo') # read the configuration d = os.path.realpath(self.directory).rstrip('/') self.conf = dict() confs = glob(os.path.join(d, '%s-HVETO_CONF-*-*.txt' % ifo)) if len(confs) != 1: self.rounds = 'FAIL' return conffile = confs[0] try: with open(conffile) as f: self.conf = dict() lines = f.readlines()[3:] for line in lines: try: key, val = line.split(': ', 1) self.conf[key.strip()] = eval(val) except (ValueError, SyntaxError, NameError): pass except IOError: self.rounds = 'FAIL' return else: etg = self.conf.pop('AUXtype', None) if 'DEfnm' in self.conf: name = re_quote.sub('', self.conf['DEfnm']) self.primary = '%s:%s' % (ifo, name) if 'DEtype' in self.conf: hetg = re_quote.sub('', self.conf['DEtype']) if re.search('_%s\Z' % hetg, self.primary, re.I): self.primary = self.primary[:-len(hetg) - 1] else: self.primary = None # find the segments try: ce = CacheEntry.from_T050017(conffile) except (ValueError): start = int(self.span[0]) duration = int(abs(self.span)) span = self.span else: start = int(ce.segment[0]) duration = int(abs(ce.segment)) span = ce.segment try: statefile = self.conf['dqfnm'] except KeyError: statefile = '%s-HVETO_DQ_SEGS-%d-%d.txt' % (ifo, start, duration) if not os.path.isfile(os.path.join(self.directory, statefile)): self.rounds = 'NOSEGMENTS' return # find the results table resultsfile = os.path.join(self.directory, 'summary_stats.txt') if not os.path.isfile(resultsfile): self.rounds = 'FAIL' return # determine the Hveto state cache = Cache( [CacheEntry.from_T050017(os.path.join(self.directory, statefile))]) segments = SegmentList.read(cache) globalv.SEGMENTS[self.states[0].definition] = DataQualityFlag( self.states[0].definition, known=[span], active=segments) self.finalize_states(config=config, query=False) # read results file self.rounds = [] with open(resultsfile, 'r') as f: for line in f.readlines(): self.rounds.append( dict(zip(self.summaryrows, line.split(' ')[1:]))) # fix channel name c = '%s:%s' % (ifo, self.rounds[-1]['Winning channel']) if etg and re.search('_%s\Z' % etg, c, re.I): c = c.rsplit('_', 1)[0] self.rounds[-1]['Winning channel'] = c # read starting triggers rawfile = ('%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' % (ifo, start, duration)) cache = Cache( [CacheEntry.from_T050017(os.path.join(self.directory, rawfile))]) get_triggers('%s:hveto_start' % ifo, 'hveto', [self.span], config=config, cache=cache, tablename='sngl_burst', return_=False) get_triggers('%s:hveto_vetoed_all' % ifo, 'hveto', [self.span], config=config, cache=Cache(), tablename='sngl_burst') for r in range(1, len(self.rounds) + 1): # read round veto triggers rawfile = ('%s-HVETO_VETOED_TRIGS_ROUND_%d-%d-%d.txt' % (ifo, r, start, duration)) cache = Cache([ CacheEntry.from_T050017(os.path.join(self.directory, rawfile)) ]) trigs = get_triggers('%s:hveto_vetoed_round %d' % (ifo, r), 'hveto', [self.span], config=config, cache=cache, tablename='sngl_burst') globalv.TRIGGERS['%s:hveto_vetoed_all,hveto' % ifo].extend(trigs) # read round veto segments segfile = ('%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' % (ifo, r, start, duration)) cache = Cache([ CacheEntry.from_T050017(os.path.join(self.directory, segfile)) ]) get_segments('%s:hveto_veto_segs_round_%d' % (ifo, r), [self.span], config=config, cache=cache, return_=False) for plot in self.plots: if isinstance(plot, HvetoSegmentSummaryPlot): plot.find_flags() kwargs['trigcache'] = Cache() kwargs['segmentcache'] = Cache() super(HvetoTab, self).process(config=config, **kwargs) # find some plots for plot in ['OVERAL_HISTOGRAM', 'OVERAL_EFF_DT'][::-1]: filename = ('%s-HVETO_%s-%d-%d.png' % (ifo, plot, start, duration)) plotfile = os.path.join(self.directory, filename) if os.path.isfile(plotfile): p = SummaryPlot(os.path.join(self.url, filename), new=False) p.state = self.states[0] self.plots.insert(0, p) # delete data from archive del globalv.SEGMENTS[self.states[0].definition] for row in range(1, len(self.rounds) + 1): del globalv.SEGMENTS['%s:hveto_veto_segs_round_%s' % (ifo, row)]
#! /usr/bin/python from __future__ import print_function from glue.lal import Cache from glue.lal import CacheEntry import sys for line in sys.stdin: c = CacheEntry.from_T050017(line) print(str(c), end=' ')
def parse_command_line(): parser = OptionParser(usage="%prog [options] ...", description="FIXME") parser.add_option("-f", "--config-file", metavar="filename", help="Use this configuration file (required).") parser.add_option( "-l", "--log-path", metavar="path", help="Make condor put log files in this directory (required).") parser.add_option( "--background-time-slides", metavar="filename", action="append", help= "Set the name of the file from which to obtain the time slide table for use in the background branch of the pipeline (required). This option can be given multiple times to parallelize the background analysis across time slides. You will want to make sure the time slide files have distinct vectors to not repeat the same analysis multiple times, and in particular you'll want to make sure only one of them has a zero-lag vector in it." ) parser.add_option( "--injection-time-slides", metavar="filename", help= "Set the name of the file from which to obtain the time slide table for use in the injection branch of the pipeline (required)." ) parser.add_option( "--segments-file", metavar="filename", help= "Set the name of the LIGO Light-Weight XML file from which to obtain segment lists (required). See ligolw_segments and ligolw_segment_query for more information on constructing an XML-format segments file. See also --segments-name." ) parser.add_option( "--segments-name", metavar="name", default="segments", help= "Set the name of the segment lists to retrieve from the segments file (default = \"segments\"). See also --segments-file." ) parser.add_option( "--vetoes-file", metavar="filename", help= "Set the name of the LIGO Light-Weight XML file from which to obtain veto segment lists (optional). See ligolw_segments and ligolw_segment_query for more information on constructing an XML-format segments file. See also --vetos-name." ) parser.add_option( "--vetoes-name", metavar="name", default="vetoes", help= "Set the name of the segment lists to retrieve from the veto segments file (default = \"vetoes\"). See also --vetoes-file." ) parser.add_option("-v", "--verbose", action="store_true", help="Be verbose.") options, filenames = parser.parse_args() required_options = [ "log_path", "config_file", "background_time_slides", "injection_time_slides", "segments_file" ] missing_options = [ option for option in required_options if getattr(options, option) is None ] if missing_options: raise ValueError, "missing required options %s" % ", ".join( sorted("--%s" % option.replace("_", "-") for option in missing_options)) if options.vetoes_file is not None: options.vetoes_cache = set([ CacheEntry( None, None, None, "file://localhost" + os.path.abspath(options.vetoes_file)) ]) else: options.vetoes_cache = set() options.injection_time_slides = [options.injection_time_slides] return options, filenames