def find_kw(channel, start, end, base=None):
"""Find KW trigger XML files
"""
span = Segment(to_gps(start), to_gps(end))
channel = get_channel(channel)
ifo = channel.ifo
if base is None and channel.name.split(':', 1)[-1] == 'GDS-CALIB_STRAIN':
tag = '%s-KW_HOFT' % ifo[0].upper()
base = '/gds-%s/dmt/triggers/%s' % (ifo.lower(), tag)
elif base is None:
tag = '%s-KW_TRIGGERS' % ifo[0].upper()
base = '/gds-%s/dmt/triggers/%s' % (ifo.lower(), tag)
gps5 = int('%.5s' % start)
end5 = int('%.5s' % end)
out = Cache()
append = out.append
while gps5 <= end5:
trigglob = os.path.join(
base, '%s-%d' % (tag, gps5), '%s-*-*.xml' % tag)
found = glob.glob(trigglob)
for f in found:
ce = CacheEntry.from_T050017(f)
if ce.segment.intersects(span):
append(ce)
gps5 += 1
out.sort(key=lambda e: e.path)
vprint(" Found %d files for %s (KW)\n"
% (len(out), channel.ndsname))
return out
def parse_command_line():
parser = OptionParser(
version="%prog CVS $Id$",
usage="%prog [options]",
description=
"Constructs the likelihood-ratio based coincidence stage for an excess power analysis. The input consists of one or more LAL caches listing the sqlite database trigger files, and a list of segments giving the time intervals that should be considered to be independent. The LAL caches list all trigger files together, that is injections, time slides, and zero-lag. The individual trigger files are self-describing, so the analysis codes can autodetect their type. Each segment will be analyzed using the files that intersect it: the likelihood ratios will be constructed from the injections and time-lag triggers contained in files that intersect the segment, and that data used to assign likelihoods to the injections, time-lag, and zero-lag coincs in all files that intersect the same segment."
)
parser.add_option(
"--input-cache",
metavar="filename",
action="append",
default=[],
help=
"Add the contents of this cache file to the list of files from which to draw statistics."
)
parser.add_option(
"--round-robin-cache",
metavar="filename",
action="append",
default=[],
help=
"Add the contents of this cache file to the list of files from which to draw injection statistics in a round-robin way."
)
parser.add_option(
"--condor-log-dir",
metavar="path",
default=".",
help="Set the directory for Condor log files (default = \".\").")
parser.add_option(
"--config-file",
metavar="filename",
default="power.ini",
help="Set .ini configuration file name (default = \"power.ini\").")
parser.add_option(
"--distribution-segments",
metavar="filename",
help=
"Read boundaries for distribution data intervals from this segwizard format segments file (required)."
)
parser.add_option("-v",
"--verbose",
action="store_true",
help="Be verbose.")
options, filenames = parser.parse_args()
if options.distribution_segments is None:
raise ValueError, "missing required argument --distribution-segments"
options.distribution_segments = segmentsUtils.fromsegwizard(
file(options.distribution_segments), coltype=LIGOTimeGPS)
options.input_cache = set([
CacheEntry(line) for filename in options.input_cache
for line in file(filename)
])
options.round_robin_cache = [
set(map(CacheEntry, file(filename)))
for filename in options.round_robin_cache
]
return options, (filenames or [])
def find_dmt_omega(channel, start, end, base=None):
"""Find DMT-Omega trigger XML files
"""
span = Segment(to_gps(start), to_gps(end))
channel = get_channel(channel)
ifo = channel.ifo
if base is None and channel.name.split(':', 1)[-1] == 'GDS-CALIB_STRAIN':
base = '/gds-%s/dmt/triggers/%s-HOFT_Omega' % (
ifo.lower(), ifo[0].upper())
elif base is None:
raise NotImplementedError("This method doesn't know how to locate DMT "
"Omega trigger files for %r" % str(channel))
gps5 = int('%.5s' % start)
end5 = int('%.5s' % end)
out = Cache()
append = out.append
while gps5 <= end5:
trigglob = os.path.join(
base, str(gps5),
'%s-%s_%s_%s_OmegaC-*-*.xml' % (
ifo, channel.system, channel.subsystem, channel.signal))
found = glob.glob(trigglob)
for f in found:
ce = CacheEntry.from_T050017(f)
if ce.segment.intersects(span):
append(ce)
gps5 += 1
out.sort(key=lambda e: e.path)
vprint(" Found %d files for %s (DMT-Omega)\n"
% (len(out), channel.ndsname))
return out
def read_losc_state(filename, channel, group=None, start=None, end=None,
copy=False):
"""Read a `StateVector` from a LOSC-format HDF file.
"""
h5file = open_hdf5(filename)
if group:
channel = '%s/%s' % (group, channel)
# find data
dataset = _find_dataset(h5file, '%s/DQmask' % channel)
maskset = _find_dataset(h5file, '%s/DQDescriptions' % channel)
# read data
nddata = dataset.value
bits = list(maskset.value)
# read metadata
try:
epoch = dataset.attrs['Xstart']
except KeyError:
try:
ce = CacheEntry.from_T050017(h5file.filename)
except ValueError:
epoch = None
else:
epoch = ce.segment[0]
try:
dt = dataset.attrs['Xspacing']
except KeyError:
dt = Quantity(1, 's')
else:
xunit = Unit(dataset.attrs['Xunit'])
dt = Quantity(dt, xunit)
return StateVector(nddata, bits=bits, epoch=epoch, name='Data quality',
sample_rate=(1/dt).to('Hertz'), copy=copy)
def read_losc_state(filename, channel, group=None, start=None, end=None):
"""Read a `StateVector` from a LOSC-format HDF file.
"""
h5file = open_hdf5(filename)
if group:
channel = '%s/%s' % (group, channel)
# find data
dataset = _find_dataset(h5file, '%s/DQmask' % channel)
maskset = _find_dataset(h5file, '%s/DQDescriptions' % channel)
# read data
nddata = dataset.value
bits = list(maskset.value)
# read metadata
try:
epoch = dataset.attrs['Xstart']
except KeyError:
try:
from glue.lal import CacheEntry
except ImportError:
epoch = None
else:
ce = CacheEntry.from_T050017(h5file.filename)
epoch = ce.segment[0]
try:
dt = dataset.attrs['Xspacing']
except KeyError:
dt = Quantity(1, 's')
else:
xunit = Unit(dataset.attrs['Xunit'])
dt = Quantity(dt, xunit)
return StateVector(nddata,
bits=bits,
epoch=epoch,
sample_rate=(1 / dt).to('Hertz'),
name='Data quality')
def FrameCachetoLALCache(fcache):
lcache = LALCache()
files = fcache.get_files()
for f in files:
lcache.append(LALCacheEntry.from_T050017(f))
return lcache
def setUpClass(cls):
cls.FRAMES = {}
cls._tempdir = tempfile.mkdtemp(prefix='gwsumm-test-data-')
# get data
for channel in LOSC_DATA:
cls.FRAMES[channel] = Cache()
for gwf in LOSC_DATA[channel]:
target = os.path.join(cls._tempdir, os.path.basename(gwf))
download(gwf, target)
cls.FRAMES[channel].append(CacheEntry.from_T050017(target))
def setUpClass(cls):
cls.FRAMES = {}
cls._tempdir = tempfile.mkdtemp(prefix='gwsumm-test-data-')
# get data
for channel in LOSC_DATA:
cls.FRAMES[channel] = Cache()
for gwf in LOSC_DATA[channel]:
target = os.path.join(cls._tempdir, os.path.basename(gwf))
download(gwf, target)
cls.FRAMES[channel].append(CacheEntry.from_T050017(target))
def parse_command_line():
parser = OptionParser(
version = "Name: %%prog\n%s" % git_version.verbose_msg,
usage = "%prog [options] [file ...]",
description = "%prog does blah blah blah."
)
parser.add_option("-p", "--live-time-program", metavar = "name", default="lalapps_ring", help = "Set the name of the program whose entries in the search_summary table will set the search live time. Required.")
parser.add_option("--veto-segments-name", help = "Set the name of the segments to extract from the segment tables and use as the veto list.")
parser.add_option("--categories", metavar = "{\"frequency-ifos-oninstruments\",\"oninstruments\"}", default="oninstruments", help = "Select the event categorization algorithm. Default oninstruments")
parser.add_option("-b", "--frequency-bins", metavar = "frequency,frequency[,frequency,...]", help = "Set the boundaries of the frequency bins in Hz. The lowest and highest bounds must be explicitly listed. Example \"0,5,inf\". Required if frequency-based categorization algorithm has been selected.")
parser.add_option("--rank-by", metavar = "{\"snr\",\"uncombined-ifar\",\"likelihood\"}", default="snr", help = "Select the event ranking method. Default is snr")
parser.add_option("-t", "--tmp-space", metavar = "path", help = "Path to a directory suitable for use as a work area while manipulating the database file. The database file will be worked on in this directory, and then moved to the final location when complete. This option is intended to improve performance when running in a networked environment, where there might be a local disk with higher bandwidth than is available to the filesystem on which the final output will reside.")
parser.add_option("-n","--extrapolation-num",action="store",type="int",default=0, metavar="num",help="number of time-slide points to use in FAR extrapolation" )
parser.add_option("-g", "--input-cache", help="cache of sqlite files")
parser.add_option("-v", "--verbose", action = "store_true", help = "Be verbose.")
options, filenames = parser.parse_args()
if options.input_cache is not None: filenames.extend([CacheEntry(c).path() for c in open(options.input_cache)])
#
# categories and ranking
#
if options.categories not in ("frequency-ifos-oninstruments", "oninstruments"):
raise ValueError, "missing or unrecognized --categories option"
if options.rank_by not in ("snr", "uncombined-ifar", "likelihood"):
raise ValueError, "missing or unrecognized --rank-by option"
options.populate_column = "false_alarm_rate"
#
# parse frequency bins
#
if options.categories in ("frequency-ifos-oninstruments",):
if options.frequency_bins is None:
raise ValueError, "--frequency-bins required with category algorithm \"%s\"" % options.categories
options.frequency_bins = sorted(map(float, options.frequency_bins.split(",")))
if len(options.frequency_bins) < 2:
raise ValueError, "must set at least two frequency bin boundaries (i.e., define at least one frequency bin)"
options.frequency_bins = rate.IrregularBins(options.frequency_bins)
#
# other
#
if options.live_time_program is None:
raise ValueError, "missing required option -p or --live-time-program"
#
# done
#
return options, (filenames or [None])
def read_losc_state(filename, channel, group=None, start=None, end=None,
copy=False):
"""Read a `StateVector` from a LOSC-format HDF file.
Parameters
----------
filename : `str`
path to LOSC-format HDF5 file to read.
channel : `str`
name of HDF5 dataset to read.
group : `str`, optional
name of containing HDF5 group for ``channel``. If not given,
the first dataset named ``channel`` will be assumed as the right
one.
start : `Time`, `~gwpy.time.LIGOTimeGPS`, optional
start GPS time of desired data
end : `Time`, `~gwpy.time.LIGOTimeGPS`, optional
end GPS time of desired data
copy : `bool`, default: `False`
create a fresh-memory copy of the underlying array
Returns
-------
data : :class`~gwpy.timeseries.TimeSeries`
a new `TimeSeries` containing the data read from disk
"""
h5file = open_hdf5(filename)
if group:
channel = '%s/%s' % (group, channel)
# find data
dataset = _find_dataset(h5file, '%s/DQmask' % channel)
maskset = _find_dataset(h5file, '%s/DQDescriptions' % channel)
# read data
nddata = dataset.value
bits = list(maskset.value)
# read metadata
try:
epoch = dataset.attrs['Xstart']
except KeyError:
try:
ce = CacheEntry.from_T050017(h5file.filename)
except ValueError:
epoch = None
else:
epoch = ce.segment[0]
try:
dt = dataset.attrs['Xspacing']
except KeyError:
dt = Quantity(1, 's')
else:
xunit = parse_unit(dataset.attrs['Xunits'])
dt = Quantity(dt, xunit)
return StateVector(nddata, bits=bits, epoch=epoch, name='Data quality',
dx=dt, copy=copy)
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False):
"""Find the paths of trigger files that represent the given
observatory, channel, and ETG (event trigger generator) for a given
GPS [start, end) segment.
"""
if etg.lower().startswith('omicron'):
etg = '?' + etg[1:]
# construct search
gpsstart = to_gps(gpsstart).seconds
gpsend = to_gps(gpsend).seconds
span = Segment(gpsstart, gpsend)
ifo, channel = channel.split(':', 1)
trigtype = "%s_%s" % (channel, etg.lower())
epoch = '*'
searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype)
gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5]) + 1)
trigform = ('%s-%s_%s-%s-*.xml*' %
(ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]' * 10))
# test for channel-level directory
if not glob.glob(searchbase):
raise ValueError("No channel-level directory found at %s. Either the "
"channel name or ETG names are wrong, or this "
"channel is not configured for this ETG." %
searchbase)
# perform and cache results
out = Cache()
append = out.append
for gpsdir in gpsdirs:
gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform)
if verbose:
gprint("Searching %s..." % os.path.split(gpssearchpath)[0],
end=' ')
found = set(
map(os.path.realpath,
glob.glob(os.path.join(searchbase, str(gpsdir), trigform))))
n = 0
for f in found:
ce = CacheEntry.from_T050017(f)
if ce.segment.intersects(span):
append(ce)
n += 1
if verbose:
gprint("%d found" % n)
out.sort(key=lambda e: e.path)
return out
def find_trigger_urls(channel, etg, gpsstart, gpsend, verbose=False):
"""Find the paths of trigger files that represent the given
observatory, channel, and ETG (event trigger generator) for a given
GPS [start, end) segment.
"""
if etg.lower().startswith('omicron'):
etg = '?' + etg[1:]
# construct search
gpsstart = to_gps(gpsstart).seconds
gpsend = to_gps(gpsend).seconds
span = Segment(gpsstart, gpsend)
ifo, channel = channel.split(':', 1)
trigtype = "%s_%s" % (channel, etg.lower())
epoch = '*'
searchbase = os.path.join(TRIGFIND_BASE_PATH, epoch, ifo, trigtype)
gpsdirs = range(int(str(gpsstart)[:5]), int(str(gpsend)[:5])+1)
trigform = ('%s-%s_%s-%s-*.xml*'
% (ifo, re_dash.sub('_', channel), etg.lower(), '[0-9]'*10))
# test for channel-level directory
if not glob.glob(searchbase):
raise ValueError("No channel-level directory found at %s. Either the "
"channel name or ETG names are wrong, or this "
"channel is not configured for this ETG."
% searchbase)
# perform and cache results
out = Cache()
append = out.append
for gpsdir in gpsdirs:
gpssearchpath = os.path.join(searchbase, str(gpsdir), trigform)
if verbose:
gprint("Searching %s..." % os.path.split(gpssearchpath)[0],
end=' ')
found = set(map(
os.path.realpath,
glob.glob(os.path.join(searchbase, str(gpsdir), trigform))))
n = 0
for f in found:
ce = CacheEntry.from_T050017(f)
if ce.segment.intersects(span):
append(ce)
n += 1
if verbose:
gprint("%d found" % n)
out.sort(key=lambda e: e.path)
return out
def get_output_cache(self):
"""
Returns a LAL cache of the output file name. Calling this
method also induces the output name to get set, so it must
be at least once.
"""
if not self.output_cache:
self.output_cache = [
CacheEntry(
self.get_ifo(), self.__usertag,
segments.segment(LIGOTimeGPS(self.get_start()),
LIGOTimeGPS(self.get_end())),
"file://localhost" + os.path.abspath(self.get_output()))
]
return self.output_cache
def get_files(self):
"""
Return Find all files described by this FrameCacheEntry.
"""
filenames = glob.glob(os.path.join(self.path,\
'%s-%s*-%s.*' % (self.observatory,\
self.description,\
self.duration)))
cache = [e.path for e in\
LALCache([LALCacheEntry.from_T050017(f) for f in filenames])\
if e.observatory==self.observatory and\
e.description==self.description and\
self.segment.intersects(e.segment) and\
abs(e.segment)==self.duration]
return cache
def fromlalcache(cachefile, coltype=int):
"""
Construct a segmentlist representing the times spanned by the files
identified in the LAL cache contained in the file object file. The
segmentlist will be created with segments whose boundaries are of
type coltype, which should raise ValueError if it cannot convert
its string argument.
Example:
>>> from lal import LIGOTimeGPS
>>> cache_seglists = fromlalcache(open(filename), coltype = LIGOTimeGPS).coalesce()
See also:
glue.lal.CacheEntry
"""
return segments.segmentlist(
CacheEntry(l, coltype=coltype).segment for l in cachefile)
def process(self, config=GWSummConfigParser(), **kwargs):
# set params
self.rounds = None
if not os.path.isdir(self.directory):
self.rounds = None
return
# get some basic info
ifo = config.get('DEFAULT', 'ifo')
# read the configuration
d = os.path.realpath(self.directory).rstrip('/')
self.conf = dict()
confs = glob(os.path.join(d, '%s-HVETO_CONF-*-*.txt' % ifo))
if len(confs) != 1:
self.rounds = 'FAIL'
return
conffile = confs[0]
try:
with open(conffile) as f:
self.conf = dict()
lines = f.readlines()[3:]
for line in lines:
try:
key, val = line.split(': ', 1)
self.conf[key.strip()] = eval(val)
except (ValueError, SyntaxError, NameError):
pass
except IOError:
self.rounds = 'FAIL'
return
else:
etg = self.conf.pop('AUXtype', None)
if 'DEfnm' in self.conf:
name = re_quote.sub('', self.conf['DEfnm'])
self.primary = '%s:%s' % (ifo, name)
if 'DEtype' in self.conf:
hetg = re_quote.sub('', self.conf['DEtype'])
if re.search('_%s\Z' % hetg, self.primary, re.I):
self.primary = self.primary[:-len(hetg)-1]
else:
self.primary = None
# find the segments
try:
ce = CacheEntry.from_T050017(conffile)
except (ValueError):
start = int(self.span[0])
duration = int(abs(self.span))
span = self.span
else:
start = int(ce.segment[0])
duration = int(abs(ce.segment))
span = ce.segment
try:
statefile = self.conf['dqfnm']
except KeyError:
statefile = '%s-HVETO_DQ_SEGS-%d-%d.txt' % (ifo, start, duration)
if not os.path.isfile(os.path.join(self.directory, statefile)):
self.rounds = 'NOSEGMENTS'
return
# find the results table
resultsfile = os.path.join(self.directory, 'summary_stats.txt')
if not os.path.isfile(resultsfile):
self.rounds = 'FAIL'
return
# determine the Hveto state
cache = Cache([CacheEntry.from_T050017(
os.path.join(self.directory, statefile))])
segments = SegmentList.read(cache)
globalv.SEGMENTS[self.states[0].definition] = DataQualityFlag(
self.states[0].definition, known=[span], active=segments)
self.finalize_states(config=config, query=False)
# read results file
self.rounds = []
with open(resultsfile, 'r') as f:
for line in f.readlines():
self.rounds.append(dict(zip(self.summaryrows,
line.split(' ')[1:])))
# fix channel name
c = '%s:%s' % (ifo, self.rounds[-1]['Winning channel'])
if etg and re.search('_%s\Z' % etg, c, re.I):
c = c.rsplit('_', 1)[0]
self.rounds[-1]['Winning channel'] = c
# read starting triggers
rawfile = ('%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt'
% (ifo, start, duration))
cache = Cache([CacheEntry.from_T050017(
os.path.join(self.directory, rawfile))])
get_triggers('%s:hveto_start' % ifo, 'hveto', [self.span],
config=config, cache=cache, return_=False)
get_triggers('%s:hveto_vetoed_all' % ifo, 'hveto', [self.span],
config=config, cache=Cache(), return_=False)
for r in range(1, len(self.rounds) + 1):
# read round veto triggers
rawfile = ('%s-HVETO_VETOED_TRIGS_ROUND_%d-%d-%d.txt'
% (ifo, r, start, duration))
cache = Cache([CacheEntry.from_T050017(
os.path.join(self.directory, rawfile))])
trigs = get_triggers('%s:hveto_vetoed_round %d' % (ifo, r), 'hveto',
[self.span], config=config, cache=cache)
add_triggers(trigs, '%s:hveto_vetoed_all,hveto' % ifo,
segments=SegmentList([self.span]))
# read round veto segments
segfile = ('%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt'
% (ifo, r, start, duration))
cache = Cache([CacheEntry.from_T050017(
os.path.join(self.directory, segfile))])
get_segments('%s:hveto_veto_segs_round_%d' % (ifo, r), [self.span],
config=config, cache=cache, return_=False)
for plot in self.plots:
if isinstance(plot, HvetoSegmentSummaryPlot):
plot.find_flags()
kwargs['trigcache'] = Cache()
kwargs['segmentcache'] = Cache()
super(HvetoTab, self).process(config=config, **kwargs)
# find some plots
for plot in ['OVERAL_HISTOGRAM', 'OVERAL_EFF_DT'][::-1]:
filename = (
'%s-HVETO_%s-%d-%d.png' % (ifo, plot, start, duration))
plotfile = os.path.join(self.directory, filename)
if os.path.isfile(plotfile):
p = SummaryPlot(os.path.join(self.url, filename), new=False)
p.state = self.states[0]
self.plots.insert(0, p)
# delete data from archive
del globalv.SEGMENTS[self.states[0].definition]
for row in range(1, len(self.rounds) + 1):
del globalv.SEGMENTS['%s:hveto_veto_segs_round_%s' % (ifo, row)]
#!/usr/bin/python
import sys
from glue.lal import CacheEntry
from glue.ligolw import lsctables, utils
for filename in (CacheEntry(line).path for line in file(sys.argv[1])):
xmldoc = utils.load_filename(filename,
gz=(filename or "stdin").endswith(".gz"))
try:
lsctables.table.get_table(xmldoc,
lsctables.SnglInspiralTable.tableName)
except ValueError:
xmldoc.childNodes[-1].appendChild(
lsctables.New(
lsctables.SnglInspiralTable,
columns=("process_id", "ifo", "search", "channel", "end_time",
"end_time_ns", "end_time_gmst", "impulse_time",
"impulse_time_ns", "template_duration",
"event_duration", "amplitude", "eff_distance",
"coa_phase", "mass1", "mass2", "mchirp", "mtotal",
"eta", "kappa", "chi", "tau0", "tau2", "tau3", "tau4",
"tau5", "ttotal", "psi0", "psi3", "alpha", "alpha1",
"alpha2", "alpha3", "alpha4", "alpha5", "alpha6",
"beta", "f_final", "snr", "chisq", "chisq_dof",
"bank_chisq", "bank_chisq_dof", "cont_chisq",
"cont_chisq_dof", "sigmasq", "rsqveto_duration",
"Gamma0", "Gamma1", "Gamma2", "Gamma3", "Gamma4",
"Gamma5", "Gamma6", "Gamma7", "Gamma8", "Gamma9",
"event_id")))
utils.write_filename(filename,
xmldoc,
gz=(filename or "stdout").endswith(".gz"))
def find_frames(ifo,
frametype,
gpsstart,
gpsend,
config=GWSummConfigParser(),
urltype='file',
gaps='warn',
onerror='raise'):
"""Query the datafind server for GWF files for the given type
Parameters
----------
ifo : `str`
prefix for the IFO of interest (either one or two characters)
frametype : `str`
name of the frametype to find
gpsstart : `int`
GPS start time of the query
gpsend : `int`
GPS end time of the query
config : `~ConfigParser.ConfigParser`, optional
configuration with `[datafind]` section containing `server`
specification, otherwise taken from the environment
urltype : `str`, optional
what type of file paths to return, default: `file`
gaps : `str`, optional
what to do when gaps are detected, one of
- `ignore` : do nothing
- `warn` : display the existence of gaps but carry on
- `raise` : raise an exception
onerror : `str`, optional
what to do when the `~glue.datafind` query itself fails, same
options as for ``gaps``
Returns
-------
cache : `~glue.lal.Cache`
a list of structured frame file descriptions matching the ifo and
frametype requested
"""
vprint(' Finding %s-%s frames for [%d, %d)...' %
(ifo[0], frametype, int(gpsstart), int(gpsend)))
# find datafind host:port
try:
host = config.get('datafind', 'server')
except (NoOptionError, NoSectionError):
try:
host = os.environ['LIGO_DATAFIND_SERVER']
except KeyError:
host = None
port = None
else:
try:
host, port = host.rsplit(':', 1)
except ValueError:
port = None
else:
port = int(port)
else:
port = config.getint('datafind', 'port')
# get credentials
if port == 80:
cert = None
key = None
else:
cert, key = datafind.find_credential()
# XXX HACK: LLO changed frame types on Dec 6 2013:
LLOCHANGE = 1070291904
if re.match('L1_{CRMT}', frametype) and gpsstart < LLOCHANGE:
frametype = frametype[-1]
# query frames
ifo = ifo[0].upper()
gpsstart = int(floor(gpsstart))
gpsend = int(ceil(min(globalv.NOW, gpsend)))
if gpsend <= gpsstart:
return Cache()
# parse match
try:
frametype, match = frametype.split('|', 1)
except ValueError:
match = None
def _query():
if cert is not None:
dfconn = datafind.GWDataFindHTTPSConnection(host=host,
port=port,
cert_file=cert,
key_file=key)
else:
dfconn = datafind.GWDataFindHTTPConnection(host=host, port=port)
return dfconn.find_frame_urls(ifo[0].upper(),
frametype,
gpsstart,
gpsend,
urltype=urltype,
on_gaps=gaps,
match=match)
try:
cache = _query()
except RuntimeError as e:
sleep(1)
try:
cache = _query()
except RuntimeError:
if 'Invalid GPS times' in str(e):
e.args = ('%s: %d ... %s' % (str(e), gpsstart, gpsend), )
if onerror in ['ignore', None]:
pass
elif onerror in ['warn']:
warnings.warn('Caught %s: %s' % (type(e).__name__, str(e)))
else:
raise
cache = Cache()
# XXX: if querying for day of LLO frame type change, do both
if (ifo[0].upper() == 'L' and frametype in ['C', 'R', 'M', 'T']
and gpsstart < LLOCHANGE < gpsend):
start = len(cache) and cache[-1].segment[1] or gpsstart
if start < gpsend:
cache.extend(
dfconn.find_frame_urls(ifo[0].upper(),
'L1_%s' % frametype,
start,
gpsend,
urltype=urltype,
on_gaps=gaps)[1:])
# extend cache beyond datafind's knowledge to reduce latency
try:
latest = cache[-1]
ngps = len(
re_gwf_gps_epoch.search(os.path.dirname(
latest.path)).groupdict()['gpsepoch'])
except (IndexError, AttributeError):
pass
else:
while True:
s, e = latest.segment
if s >= gpsend:
break
# replace GPS time of file basename
new = latest.path.replace('-%d-' % s, '-%d-' % e)
# replace GPS epoch in dirname
new = new.replace('%s/' % str(s)[:ngps], '%s/' % str(e)[:ngps])
if os.path.isfile(new):
latest = CacheEntry.from_T050017(new)
cache.append(latest)
else:
break
# validate files existing and return
cache, _ = cache.checkfilesexist()
vprint(' %d found.\n' % len(cache))
return cache
# extract observatory
observatory = (options.observatory
and options.observatory.strip()) or "+".join(
sorted(seglists))
# extract description
if options.description:
description = options.description
else:
if process_ids is None:
description = set(searchsumm.getColumnByName("comment"))
else:
description = set(row.comment for row in searchsumm
if row.process_id in process_ids)
if len(description) < 1:
raise ValueError, "%s: no matching rows found in search summary table" % filename
if len(description) > 1:
raise ValueError, "%s: comments in matching rows of search summary table are not identical" % filename
description = description.pop().strip() or None
# set URL
url = "file://localhost" + os.path.abspath(filename)
# write cache entry
print >> options.output, str(
CacheEntry(observatory, description, seglists.extent_all(), url))
# allow garbage collection
xmldoc.unlink()
#
# Using time slide information, construct segment lists describing times
# requiring trigger construction.
#
if options.verbose:
print >>sys.stderr, "Computing segments for which lalapps_power jobs are required ..."
background_time_slides = {}
background_seglistdict = segments.segmentlistdict()
if options.do_noninjections:
for filename in options.background_time_slides:
cache_entry = CacheEntry(None, None, None, "file://localhost" + os.path.abspath(filename))
background_time_slides[cache_entry] = ligolw_tisi.load_time_slides(filename, verbose = options.verbose, gz = filename.endswith(".gz")).values()
background_seglistdict |= compute_segment_lists(seglistdict, background_time_slides[cache_entry], options.minimum_gap, options.timing_params, full_segments = options.full_segments, verbose = options.verbose)
injection_time_slides = {}
injection_seglistdict = segments.segmentlistdict()
if options.do_injections:
for filename in options.injection_time_slides:
cache_entry = CacheEntry(None, None, None, "file://localhost" + os.path.abspath(filename))
injection_time_slides[cache_entry] = ligolw_tisi.load_time_slides(filename, verbose = options.verbose, gz = filename.endswith(".gz")).values()
injection_seglistdict |= compute_segment_lists(seglistdict, injection_time_slides[cache_entry], options.minimum_gap, options.timing_params, full_segments = options.full_segments, verbose = options.verbose)
# apply time shifts to segment lists to shift tiling phases, but take
# intersection with original segments to stay within allowed times. Note:
def read_losc_state(filename,
channel,
group=None,
start=None,
end=None,
copy=False):
"""Read a `StateVector` from a LOSC-format HDF file.
Parameters
----------
filename : `str`
path to LOSC-format HDF5 file to read.
channel : `str`
name of HDF5 dataset to read.
group : `str`, optional
name of containing HDF5 group for ``channel``. If not given,
the first dataset named ``channel`` will be assumed as the right
one.
start : `Time`, `~gwpy.time.LIGOTimeGPS`, optional
start GPS time of desired data
end : `Time`, `~gwpy.time.LIGOTimeGPS`, optional
end GPS time of desired data
copy : `bool`, default: `False`
create a fresh-memory copy of the underlying array
Returns
-------
data : :class`~gwpy.timeseries.TimeSeries`
a new `TimeSeries` containing the data read from disk
"""
h5file = open_hdf5(filename)
if group:
channel = '%s/%s' % (group, channel)
# find data
dataset = _find_dataset(h5file, '%s/DQmask' % channel)
maskset = _find_dataset(h5file, '%s/DQDescriptions' % channel)
# read data
nddata = dataset.value
bits = list(maskset.value)
# read metadata
try:
epoch = dataset.attrs['Xstart']
except KeyError:
try:
ce = CacheEntry.from_T050017(h5file.filename)
except ValueError:
epoch = None
else:
epoch = ce.segment[0]
try:
dt = dataset.attrs['Xspacing']
except KeyError:
dt = Quantity(1, 's')
else:
xunit = parse_unit(dataset.attrs['Xunits'])
dt = Quantity(dt, xunit)
return StateVector(nddata,
bits=bits,
epoch=epoch,
name='Data quality',
dx=dt,
copy=copy)
def process(self, config=GWSummConfigParser(), **kwargs):
# set params
self.rounds = None
if not os.path.isdir(self.directory):
self.rounds = None
return
# get some basic info
ifo = config.get('DEFAULT', 'ifo')
# read the configuration
d = os.path.realpath(self.directory).rstrip('/')
self.conf = dict()
confs = glob(os.path.join(d, '%s-HVETO_CONF-*-*.txt' % ifo))
if len(confs) != 1:
self.rounds = 'FAIL'
return
conffile = confs[0]
try:
with open(conffile) as f:
self.conf = dict()
lines = f.readlines()[3:]
for line in lines:
try:
key, val = line.split(': ', 1)
self.conf[key.strip()] = eval(val)
except (ValueError, SyntaxError, NameError):
pass
except IOError:
self.rounds = 'FAIL'
return
else:
etg = self.conf.pop('AUXtype', None)
if 'DEfnm' in self.conf:
name = re_quote.sub('', self.conf['DEfnm'])
self.primary = '%s:%s' % (ifo, name)
if 'DEtype' in self.conf:
hetg = re_quote.sub('', self.conf['DEtype'])
if re.search('_%s\Z' % hetg, self.primary, re.I):
self.primary = self.primary[:-len(hetg) - 1]
else:
self.primary = None
# find the segments
try:
ce = CacheEntry.from_T050017(conffile)
except (ValueError):
start = int(self.span[0])
duration = int(abs(self.span))
span = self.span
else:
start = int(ce.segment[0])
duration = int(abs(ce.segment))
span = ce.segment
try:
statefile = self.conf['dqfnm']
except KeyError:
statefile = '%s-HVETO_DQ_SEGS-%d-%d.txt' % (ifo, start, duration)
if not os.path.isfile(os.path.join(self.directory, statefile)):
self.rounds = 'NOSEGMENTS'
return
# find the results table
resultsfile = os.path.join(self.directory, 'summary_stats.txt')
if not os.path.isfile(resultsfile):
self.rounds = 'FAIL'
return
# determine the Hveto state
cache = Cache(
[CacheEntry.from_T050017(os.path.join(self.directory, statefile))])
segments = SegmentList.read(cache)
globalv.SEGMENTS[self.states[0].definition] = DataQualityFlag(
self.states[0].definition, known=[span], active=segments)
self.finalize_states(config=config, query=False)
# read results file
self.rounds = []
with open(resultsfile, 'r') as f:
for line in f.readlines():
self.rounds.append(
dict(zip(self.summaryrows,
line.split(' ')[1:])))
# fix channel name
c = '%s:%s' % (ifo, self.rounds[-1]['Winning channel'])
if etg and re.search('_%s\Z' % etg, c, re.I):
c = c.rsplit('_', 1)[0]
self.rounds[-1]['Winning channel'] = c
# read starting triggers
rawfile = ('%s-HVETO_RAW_TRIGS_ROUND_0-%d-%d.txt' %
(ifo, start, duration))
cache = Cache(
[CacheEntry.from_T050017(os.path.join(self.directory, rawfile))])
get_triggers('%s:hveto_start' % ifo,
'hveto', [self.span],
config=config,
cache=cache,
tablename='sngl_burst',
return_=False)
get_triggers('%s:hveto_vetoed_all' % ifo,
'hveto', [self.span],
config=config,
cache=Cache(),
tablename='sngl_burst')
for r in range(1, len(self.rounds) + 1):
# read round veto triggers
rawfile = ('%s-HVETO_VETOED_TRIGS_ROUND_%d-%d-%d.txt' %
(ifo, r, start, duration))
cache = Cache([
CacheEntry.from_T050017(os.path.join(self.directory, rawfile))
])
trigs = get_triggers('%s:hveto_vetoed_round %d' % (ifo, r),
'hveto', [self.span],
config=config,
cache=cache,
tablename='sngl_burst')
globalv.TRIGGERS['%s:hveto_vetoed_all,hveto' % ifo].extend(trigs)
# read round veto segments
segfile = ('%s-HVETO_VETO_SEGS_ROUND_%d-%d-%d.txt' %
(ifo, r, start, duration))
cache = Cache([
CacheEntry.from_T050017(os.path.join(self.directory, segfile))
])
get_segments('%s:hveto_veto_segs_round_%d' % (ifo, r), [self.span],
config=config,
cache=cache,
return_=False)
for plot in self.plots:
if isinstance(plot, HvetoSegmentSummaryPlot):
plot.find_flags()
kwargs['trigcache'] = Cache()
kwargs['segmentcache'] = Cache()
super(HvetoTab, self).process(config=config, **kwargs)
# find some plots
for plot in ['OVERAL_HISTOGRAM', 'OVERAL_EFF_DT'][::-1]:
filename = ('%s-HVETO_%s-%d-%d.png' % (ifo, plot, start, duration))
plotfile = os.path.join(self.directory, filename)
if os.path.isfile(plotfile):
p = SummaryPlot(os.path.join(self.url, filename), new=False)
p.state = self.states[0]
self.plots.insert(0, p)
# delete data from archive
del globalv.SEGMENTS[self.states[0].definition]
for row in range(1, len(self.rounds) + 1):
del globalv.SEGMENTS['%s:hveto_veto_segs_round_%s' % (ifo, row)]
#! /usr/bin/python
from __future__ import print_function
from glue.lal import Cache
from glue.lal import CacheEntry
import sys
for line in sys.stdin:
c = CacheEntry.from_T050017(line)
print(str(c), end=' ')
def parse_command_line():
parser = OptionParser(usage="%prog [options] ...", description="FIXME")
parser.add_option("-f",
"--config-file",
metavar="filename",
help="Use this configuration file (required).")
parser.add_option(
"-l",
"--log-path",
metavar="path",
help="Make condor put log files in this directory (required).")
parser.add_option(
"--background-time-slides",
metavar="filename",
action="append",
help=
"Set the name of the file from which to obtain the time slide table for use in the background branch of the pipeline (required). This option can be given multiple times to parallelize the background analysis across time slides. You will want to make sure the time slide files have distinct vectors to not repeat the same analysis multiple times, and in particular you'll want to make sure only one of them has a zero-lag vector in it."
)
parser.add_option(
"--injection-time-slides",
metavar="filename",
help=
"Set the name of the file from which to obtain the time slide table for use in the injection branch of the pipeline (required)."
)
parser.add_option(
"--segments-file",
metavar="filename",
help=
"Set the name of the LIGO Light-Weight XML file from which to obtain segment lists (required). See ligolw_segments and ligolw_segment_query for more information on constructing an XML-format segments file. See also --segments-name."
)
parser.add_option(
"--segments-name",
metavar="name",
default="segments",
help=
"Set the name of the segment lists to retrieve from the segments file (default = \"segments\"). See also --segments-file."
)
parser.add_option(
"--vetoes-file",
metavar="filename",
help=
"Set the name of the LIGO Light-Weight XML file from which to obtain veto segment lists (optional). See ligolw_segments and ligolw_segment_query for more information on constructing an XML-format segments file. See also --vetos-name."
)
parser.add_option(
"--vetoes-name",
metavar="name",
default="vetoes",
help=
"Set the name of the segment lists to retrieve from the veto segments file (default = \"vetoes\"). See also --vetoes-file."
)
parser.add_option("-v",
"--verbose",
action="store_true",
help="Be verbose.")
options, filenames = parser.parse_args()
required_options = [
"log_path", "config_file", "background_time_slides",
"injection_time_slides", "segments_file"
]
missing_options = [
option for option in required_options
if getattr(options, option) is None
]
if missing_options:
raise ValueError, "missing required options %s" % ", ".join(
sorted("--%s" % option.replace("_", "-")
for option in missing_options))
if options.vetoes_file is not None:
options.vetoes_cache = set([
CacheEntry(
None, None, None,
"file://localhost" + os.path.abspath(options.vetoes_file))
])
else:
options.vetoes_cache = set()
options.injection_time_slides = [options.injection_time_slides]
return options, filenames
