def get_loud_trigs(fList, veto_file, new_snr_cut):
""" Return a list(s) of single inspiral triggers that are above the
new snr threshold for every combination of file in the file list
and application of veto in the veto file list.
"""
trigs = lsctables.New(lsctables.SnglInspiralTable)
searched_segs = segments.segmentlist()
for fname in fList:
xmldoc = utils.load_filename(fname, gz=True)
tbl = lsctables.table.get_table(xmldoc, lsctables.SnglInspiralTable.tableName)
trigs.extend([tbl[i] for i in (tbl.get_new_snr() > new_snr_cut).nonzero()[0]])
search_summary = lsctables.table.get_table(xmldoc, lsctables.SearchSummaryTable.tableName)
searched_segs += search_summary.get_outlist()
if isinstance(veto_file, list):
# If we have multiple veto files, return results for applying each one
lt = []
tg = []
for vf in veto_file:
veto_segs = segmentsUtils.fromsegwizard(open(vf))
segs_after_veto = searched_segs - veto_segs
print vf, 'livetime', abs(segs_after_veto)
tg.append(trigs.veto(veto_segs))
lt.append(abs(segs_after_veto))
return tg, lt
else:
veto_segs = segmentsUtils.fromsegwizard(open(veto_file))
segs_after_veto = searched_segs - veto_segs
print veto_file, 'livetime', abs(segs_after_veto)
return trigs.veto(veto_segs), abs(segs_after_veto)
def read_segfile(segfile):
"""
Read segments in segwizard form with sanity check and return
glue.segments.segmentlist
"""
try:
seg_list =\
segmentsUtils.fromsegwizard(open(segfile),coltype=float,strict=False)
except(IOError):
print >> sys.stderr, """
Error: file contains no segments or glue.segmentsUtils.fromsegwizard is
not reading segments correctly. Please check the seg file.
(possibly comments in the file is causing this)
%s
"""%segfile
raise
seg_list.coalesce()
if seg_list == [] or abs(seg_list) == 0:
print >> sys.stderr, """
Warning: File contains no segments or glue.segmentsUtils.fromsegwizard is
not reading segments correctly. Please check the seg file.
(Possibly comments in the file is causing this.)
Attempting to ignore.
"""
return seg_list
def parse_config_file(options):
if options.verbose:
print >>sys.stderr, "reading %s ..." % options.config_file
config = ConfigParser.SafeConfigParser()
config.read(options.config_file)
options.tag = config.get("pipeline", "user_tag")
options.enable_clustering = config.getboolean("pipeline", "enable_clustering")
seglistdict = segments.segmentlistdict()
tiling_phase = {}
for ifo in config.get("pipeline", "ifos").split():
seglistdict[ifo] = segmentsUtils.fromsegwizard(file(config.get("pipeline", "seglist_%s" % ifo)), coltype = LIGOTimeGPS).coalesce()
try:
offset = config.getfloat("pipeline", "tiling_phase_%s" % ifo)
except ConfigParser.NoOptionError:
offset = 0.0
if offset:
tiling_phase[ifo] = offset
options.psds_per_power = config.getint("pipeline", "psds_per_power")
options.psds_per_injection = config.getint("pipeline", "psds_per_injection")
options.timing_params = power.TimingParameters(config)
return seglistdict, tiling_phase, config
def read_segfile(segfile):
"""
Read segments in segwizard form with sanity check and return
glue.segments.segmentlist
"""
try:
seg_list =\
segmentsUtils.fromsegwizard(open(segfile),coltype=float,strict=False)
except(IOError):
print >> sys.stderr, """
Error: file contains no segments or glue.segmentsUtils.fromsegwizard is
not reading segments correctly. Please check the seg file.
(possibly comments in the file is causing this)
%s
"""%segfile
raise
seg_list.coalesce()
if seg_list == [] or abs(seg_list) == 0:
print >> sys.stderr, """
Warning: File contains no segments or glue.segmentsUtils.fromsegwizard is
not reading segments correctly. Please check the seg file.
(Possibly comments in the file is causing this.)
Attempting to ignore.
"""
return seg_list
def from_segwizard(f,
coalesce=True,
gpstype=LIGOTimeGPS,
strict=True,
nproc=1):
"""Read segments from a segwizard format file into a `SegmentList`
"""
if nproc != 1:
return SegmentList.read(f,
coalesce=coalesce,
gpstype=gpstype,
strict=strict,
nproc=nproc,
format='cache')
# format list of files and read in serial
files = file_list(f)
segs = SegmentList()
for fp in files:
if isinstance(fp, file):
fp = fp.name
with open(fp, 'r') as fobj:
segs += SegmentList(
map(
Segment,
segmentsUtils.fromsegwizard(fobj,
coltype=gpstype,
strict=strict)))
if coalesce:
segs.coalesce()
return segs
def parse_command_line():
parser = OptionParser(
version="%prog CVS $Id$",
usage="%prog [options]",
description=
"Constructs the likelihood-ratio based coincidence stage for an excess power analysis. The input consists of one or more LAL caches listing the sqlite database trigger files, and a list of segments giving the time intervals that should be considered to be independent. The LAL caches list all trigger files together, that is injections, time slides, and zero-lag. The individual trigger files are self-describing, so the analysis codes can autodetect their type. Each segment will be analyzed using the files that intersect it: the likelihood ratios will be constructed from the injections and time-lag triggers contained in files that intersect the segment, and that data used to assign likelihoods to the injections, time-lag, and zero-lag coincs in all files that intersect the same segment."
)
parser.add_option(
"--input-cache",
metavar="filename",
action="append",
default=[],
help=
"Add the contents of this cache file to the list of files from which to draw statistics."
)
parser.add_option(
"--round-robin-cache",
metavar="filename",
action="append",
default=[],
help=
"Add the contents of this cache file to the list of files from which to draw injection statistics in a round-robin way."
)
parser.add_option(
"--condor-log-dir",
metavar="path",
default=".",
help="Set the directory for Condor log files (default = \".\").")
parser.add_option(
"--config-file",
metavar="filename",
default="power.ini",
help="Set .ini configuration file name (default = \"power.ini\").")
parser.add_option(
"--distribution-segments",
metavar="filename",
help=
"Read boundaries for distribution data intervals from this segwizard format segments file (required)."
)
parser.add_option("-v",
"--verbose",
action="store_true",
help="Be verbose.")
options, filenames = parser.parse_args()
if options.distribution_segments is None:
raise ValueError, "missing required argument --distribution-segments"
options.distribution_segments = segmentsUtils.fromsegwizard(
file(options.distribution_segments), coltype=lal.LIGOTimeGPS)
options.input_cache = set([
CacheEntry(line) for filename in options.input_cache
for line in file(filename)
])
options.round_robin_cache = [
set(map(CacheEntry, file(filename)))
for filename in options.round_robin_cache
]
return options, (filenames or [])
def from_segwizard(source,
coalesce=True,
gpstype=LIGOTimeGPS,
strict=True,
nproc=1):
"""Read segments from a segwizard format file into a `SegmentList`
"""
from glue import segmentsUtils
if nproc != 1:
return SegmentList.read(source,
coalesce=coalesce,
gpstype=gpstype,
strict=strict,
nproc=nproc,
format='cache')
# format list of files and read in serial
files = file_list(source)
segs = SegmentList()
for file_ in files:
with open(file_, 'r') as fobj:
raw = segmentsUtils.fromsegwizard(fobj,
coltype=gpstype,
strict=strict)
segs.extend(SegmentList(map(Segment, raw)))
if coalesce:
segs.coalesce()
return segs
def parse_config_file(options):
if options.verbose:
print >> sys.stderr, "reading %s ..." % options.config_file
config = ConfigParser.SafeConfigParser()
config.read(options.config_file)
options.tag = config.get("pipeline", "user_tag")
options.enable_clustering = config.getboolean("pipeline",
"enable_clustering")
seglistdict = segments.segmentlistdict()
tiling_phase = {}
for ifo in config.get("pipeline", "ifos").split():
seglistdict[ifo] = segmentsUtils.fromsegwizard(
file(config.get("pipeline", "seglist_%s" % ifo)),
coltype=LIGOTimeGPS).coalesce()
try:
offset = config.getfloat("pipeline", "tiling_phase_%s" % ifo)
except ConfigParser.NoOptionError:
offset = 0.0
if offset:
tiling_phase[ifo] = offset
options.psds_per_power = config.getint("pipeline", "psds_per_power")
options.psds_per_injection = config.getint("pipeline",
"psds_per_injection")
options.timing_params = power.TimingParameters(config)
return seglistdict, tiling_phase, config
def insert_from_segwizard(self,
fileobj,
instruments,
name,
version=None,
comment=None):
"""
Parse the contents of the file object fileobj as a
segwizard-format segment list, and insert the result as a
new list of "active" segments into this LigolwSegments
object. A new entry will be created in the segment_definer
table for the segment list, and instruments, name and
comment are used to populate the entry's metadata. Note
that the "valid" segments are left empty, nominally
indicating that there are no periods of validity. Returns
the newly created LigolwSegmentList object.
"""
ligolw_segment_list = LigolwSegmentList(
active=segmentsUtils.fromsegwizard(fileobj,
coltype=lsctables.LIGOTimeGPS),
instruments=instruments,
name=name,
version=version,
comment=comment)
self.add(ligolw_segment_list)
return ligolw_segment_list
def test_tofromseqwizard(self): """ Check that the segwizard writing routine's output is parsed correctly. """ data = StringIO.StringIO() correct = segments.segmentlist([segments.segment(10, 100), segments.segment(110, 120), segments.segment(125, 130), segments.segment(0, 200)]) segmentsUtils.tosegwizard(data, correct) data.seek(0) self.assertEqual(correct, segmentsUtils.fromsegwizard(data, strict=True))
def readSegFiles(segdir):
times = {}
for name,fileName in\
zip(["buffer", "off", "on"],\
["bufferSeg.txt","offSourceSeg.txt","onSourceSeg.txt"]):
segs = segmentsUtils.fromsegwizard(
open(os.path.join(segdir, fileName), 'r'))
if len(segs) > 1:
raise AttributeError, 'More than one segment, an error has occured.'
times[name] = segs[0]
return times
def insert_from_segwizard(self, fileobj, instruments, name, version = None, comment = None): """ Parse the contents of the file object fileobj as a segwizard-format segment list, and insert the result as a new list of "active" segments into this LigolwSegments object. A new entry will be created in the segment_definer table for the segment list, and instruments, name and comment are used to populate the entry's metadata. Note that the "valid" segments are left empty, nominally indicating that there are no periods of validity. """ self.add(LigolwSegmentList(active = segmentsUtils.fromsegwizard(fileobj, coltype = LIGOTimeGPS), instruments = instruments, name = name, version = version, comment = comment))
def readSegFiles(segdir):
times = {}
for name,fileName in\
zip(["buffer", "off", "on"],\
["bufferSeg.txt","offSourceSeg.txt","onSourceSeg.txt"]):
segs = segmentsUtils.fromsegwizard(open(os.path.join(segdir,fileName),
'r'))
if len(segs)>1:
raise AttributeError, 'More than one segment, an error has occured.'
times[name] = segs[0]
return times
def from_segwizard(fobj, coltype=float, strict=True):
"""Read segments from a segwizard format file into a `SegmentList`
"""
if isinstance(fobj, basestring):
fobj = open(fobj, 'r')
close = True
else:
close = False
segs = SegmentList(segmentsUtils.fromsegwizard(fobj, coltype=coltype,
strict=strict))
if close:
fobj.close()
return segs
def test_fromsegwizard(self):
"""
Test segwizard parsing.
"""
data = StringIO.StringIO("""# This is a comment
# This is another comment
# Again a comment
1 10 100 90
2 110 120 10# Here's a comment
3 125 130 5 # Another one
4 0 200 200""")
correct = segments.segmentlist([segments.segment(10, 100), segments.segment(110, 120), segments.segment(125, 130), segments.segment(0, 200)])
self.assertEqual(correct, segmentsUtils.fromsegwizard(data, strict=True))
def test_tofromseqwizard(self):
"""
Check that the segwizard writing routine's output is parsed
correctly.
"""
data = StringIO.StringIO()
correct = segments.segmentlist([
segments.segment(10, 100),
segments.segment(110, 120),
segments.segment(125, 130),
segments.segment(0, 200)
])
segmentsUtils.tosegwizard(data, correct)
data.seek(0)
self.assertEqual(correct, segmentsUtils.fromsegwizard(data,
strict=True))
def parse_command_line():
parser = OptionParser(
version="%prog CVS $Id$",
usage="%prog [options]",
description="Constructs the likelihood-ratio based coincidence stage for an excess power analysis. The input consists of one or more LAL caches listing the sqlite database trigger files, and a list of segments giving the time intervals that should be considered to be independent. The LAL caches list all trigger files together, that is injections, time slides, and zero-lag. The individual trigger files are self-describing, so the analysis codes can autodetect their type. Each segment will be analyzed using the files that intersect it: the likelihood ratios will be constructed from the injections and time-lag triggers contained in files that intersect the segment, and that data used to assign likelihoods to the injections, time-lag, and zero-lag coincs in all files that intersect the same segment.",
)
parser.add_option(
"--input-cache",
metavar="filename",
action="append",
default=[],
help="Add the contents of this cache file to the list of files from which to draw statistics.",
)
parser.add_option(
"--round-robin-cache",
metavar="filename",
action="append",
default=[],
help="Add the contents of this cache file to the list of files from which to draw injection statistics in a round-robin way.",
)
parser.add_option(
"--condor-log-dir", metavar="path", default=".", help='Set the directory for Condor log files (default = ".").'
)
parser.add_option(
"--config-file",
metavar="filename",
default="power.ini",
help='Set .ini configuration file name (default = "power.ini").',
)
parser.add_option(
"--distribution-segments",
metavar="filename",
help="Read boundaries for distribution data intervals from this segwizard format segments file (required).",
)
parser.add_option("-v", "--verbose", action="store_true", help="Be verbose.")
options, filenames = parser.parse_args()
if options.distribution_segments is None:
raise ValueError, "missing required argument --distribution-segments"
options.distribution_segments = segmentsUtils.fromsegwizard(
file(options.distribution_segments), coltype=LIGOTimeGPS
)
options.input_cache = set([CacheEntry(line) for filename in options.input_cache for line in file(filename)])
options.round_robin_cache = [set(map(CacheEntry, file(filename))) for filename in options.round_robin_cache]
return options, (filenames or [])
def readVetoFiles( self ):
"""
Reads the veto segments given by veto-files (if any)
"""
self.vetodict = dict()
# loop over the IFO names
for ifoName in self.colors:
self.vetodict[ifoName]=None
# create the attribute name and check if it exists
attributeName = 'followup_vetofile_'+ifoName.lower()
if hasattr( self.opts, attributeName):
# get the filename
filename = getattr( self.opts, attributeName )
if filename:
#read the segment lists
self.vetodict[ifoName] = segmentsUtils.fromsegwizard(open(filename))
def readVetoFiles( self ):
"""
Reads the veto segments given by veto-files (if any)
"""
self.vetodict = dict()
# loop over the IFO names
for ifoName in self.colors:
self.vetodict[ifoName]=None
# create the attribute name and check if it exists
attributeName = 'followup_vetofile_'+ifoName.lower()
if hasattr( self.opts, attributeName):
# get the filename
filename = getattr( self.opts, attributeName )
if filename:
#read the segment lists
self.vetodict[ifoName] = segmentsUtils.fromsegwizard(open(filename))
def test_fromsegwizard(self):
"""
Test segwizard parsing.
"""
data = StringIO.StringIO("""# This is a comment
# This is another comment
# Again a comment
1 10 100 90
2 110 120 10# Here's a comment
3 125 130 5 # Another one
4 0 200 200""")
correct = segments.segmentlist([
segments.segment(10, 100),
segments.segment(110, 120),
segments.segment(125, 130),
segments.segment(0, 200)
])
self.assertEqual(correct, segmentsUtils.fromsegwizard(data,
strict=True))
def from_segwizard(f, coalesce=True, gpstype=LIGOTimeGPS, strict=True,
nproc=1):
"""Read segments from a segwizard format file into a `SegmentList`
"""
if nproc != 1:
return SegmentList.read(f, coalesce=coalesce, gpstype=gpstype,
strict=strict, nproc=nproc, format='cache')
# format list of files and read in serial
files = file_list(f)
segs = SegmentList()
for fp in files:
if isinstance(fp, file):
fp = fp.name
with open(fp, 'r') as fobj:
segs += SegmentList(map(Segment, segmentsUtils.fromsegwizard(
fobj, coltype=gpstype, strict=strict)))
if coalesce:
segs.coalesce()
return segs
def get_exttrig_trials(on_segs, off_segs, veto_files):
"""
Return a tuple of (off-source time bins, off-source veto mask,
index of trial that is on source).
The off-source veto mask is a one-dimensional boolean array where True
means vetoed.
@param on_segs: On-source segments
@param off_segs: Off-source segments
@param veto_files: List of filenames containing vetoes
"""
# Check that offsource length is a multiple of the onsource segment length
trial_len = int(abs(on_segs))
if abs(off_segs) % trial_len != 0:
raise ValueError, "The provided file's analysis segment is not "\
"divisible by the fold time."
extent = (off_segs | on_segs).extent()
# generate bins for trials
num_trials = int(abs(extent)) // trial_len
trial_bins = rate.LinearBins(extent[0], extent[1], num_trials)
# incorporate veto file; in trial_veto_mask, True means vetoed.
trial_veto_mask = numpy.zeros(num_trials, dtype=numpy.bool8)
for veto_file in veto_files:
new_veto_segs = segmentsUtils.fromsegwizard(open(veto_file),
coltype=int)
if new_veto_segs.intersects(on_segs):
print >>sys.stderr, "warning: %s overlaps on-source segment" \
% veto_file
trial_veto_mask |= rate.bins_spanned(trial_bins,
new_veto_segs,
dtype=numpy.bool8)
# identify onsource trial index
onsource_mask = rate.bins_spanned(trial_bins, on_segs, dtype=numpy.bool8)
if sum(onsource_mask) != 1:
raise ValueError, "on-source segment spans more or less than one trial"
onsource_ind = numpy.arange(len(onsource_mask))[onsource_mask]
return trial_bins, trial_veto_mask, onsource_ind
def get_exttrig_trials(on_segs, off_segs, veto_files):
"""
Return a tuple of (off-source time bins, off-source veto mask,
index of trial that is on source).
The off-source veto mask is a one-dimensional boolean array where True
means vetoed.
@param on_segs: On-source segments
@param off_segs: Off-source segments
@param veto_files: List of filenames containing vetoes
"""
# Check that offsource length is a multiple of the onsource segment length
trial_len = int(abs(on_segs))
if abs(off_segs) % trial_len != 0:
raise ValueError, "The provided file's analysis segment is not "\
"divisible by the fold time."
extent = (off_segs | on_segs).extent()
# generate bins for trials
num_trials = int(abs(extent)) // trial_len
trial_bins = rate.LinearBins(extent[0], extent[1], num_trials)
# incorporate veto file; in trial_veto_mask, True means vetoed.
trial_veto_mask = numpy.zeros(num_trials, dtype=numpy.bool8)
for veto_file in veto_files:
new_veto_segs = segmentsUtils.fromsegwizard(open(veto_file),
coltype=int)
if new_veto_segs.intersects(on_segs):
print >>sys.stderr, "warning: %s overlaps on-source segment" \
% veto_file
trial_veto_mask |= rate.bins_spanned(trial_bins, new_veto_segs,
dtype=numpy.bool8)
# identify onsource trial index
onsource_mask = rate.bins_spanned(trial_bins, on_segs, dtype=numpy.bool8)
if sum(onsource_mask) != 1:
raise ValueError, "on-source segment spans more or less than one trial"
onsource_ind = numpy.arange(len(onsource_mask))[onsource_mask]
return trial_bins, trial_veto_mask, onsource_ind
help="make things verbose" )
parser.add_option("-H","--h1-segments",action="store",type="string",\
default=None, metavar=" H1_SEGMENTS", help="H1 input segment to read" )
parser.add_option("-K","--h2-segments",action="store",type="string",\
default=None, metavar=" H2_SEGMENTS", help="H2 input segment to read" )
parser.add_option("-L","--l1-segments",action="store",type="string",\
default=None, metavar=" L1_SEGMENTS", help="L1 input segment to read" )
parser.add_option("-i","--injection-file",action="store",type="string",\
default=None, metavar=" INJFILE", help="An injection file" )
parser.add_option("-g","--glitch-time",action="store",type="int",\
default=None, metavar=" GLITCH_TIME",\
help="produce plot of triggers around GLITCH_TIME")
(opts, args) = parser.parse_args()
h1seglist = segmentsUtils.fromsegwizard(file(opts.h1_segments)).coalesce()
h1seglist = cleanlist(h1seglist, 2064)
h1seglist = h1seglist.contract(72)
h2seglist = segmentsUtils.fromsegwizard(file(opts.h2_segments)).coalesce()
h2seglist = cleanlist(h2seglist, 2064)
h2seglist = h2seglist.contract(72)
l1seglist = segmentsUtils.fromsegwizard(file(opts.l1_segments))
l1seglist = cleanlist(l1seglist, 2064)
l1seglist = l1seglist.contract(72)
triplelist = h1seglist & h2seglist & l1seglist
triplelist.coalesce()
h1h2doublelist = h1seglist & h2seglist
# # Livetime segments # # Livetime and live segments # Query from a segment database if opt.segment_database: livesegs = hveto.query_segments_db( opt.segment_database, opt.gps_start, opt.gps_end, opt.segment_definer ) # Name of livetime file to process elif opt.livetime_segments: # Get segments from an XML file try: livesegs = hveto.query_segments_xml( opt.livetime_segments, opt.gps_start, opt.gps_end, opt.segment_definer) # Okay, try segwizard except: livesegs = fromsegwizard( open(opt.livetime_segments) ).coalesce() if not opt.gps_start and not opt.gps_end: runseg = segment(livesegs[0][0], livesegs[-1][1]) else: runseg = segment(opt.gps_start, opt.gps_end) runlive = livetime = float(abs(livesegs)) print "Live time before vetoes: %f, covering %2.2f%% of %s" % (livetime, 100*livetime/abs(runseg), str(runseg)) if runlive == 0: exit() lalsegl = gl.lalseg_from_seglist( livesegs ) chanlist = sorted(gl.get_chan_list( trig_seq )) print "Channels present:\n\t" + "\n\t".join(chanlist) # Preprocessing. Remove triggers which are outside the livetime segments or # below our minimum SNR threshold
def exttrig_dataquery(grb_name,
grb_time,
grb_ra,
grb_dec,
offset,
config_file,
extend=False,
useold=False,
make_plots=False,
make_xml=False):
'''
Finds science time of all available IFOs.
'''
##############################################################################
# get segment duration and minimum amount of science time
##############################################################################
# read the configuration file
cp = ConfigParser.ConfigParser()
cp.read(config_file)
# define hardcoded variables
basic_ifolist = ifolist = ['H1', 'H2', 'L1', 'V1']
catlist = [1, 2, 3]
sensitivity_dict = {"H1": 1, "L1": 2, "H2": 3, "V1": 4, "G1": 5}
# get segment length from configuration file
pad_data = int(cp.get('data', 'pad-data'))
if cp.has_option('data', 'segment-duration'):
blockDuration = segmentDuration = psdDuration = int(
cp.get('data', 'segment-duration'))
elif cp.has_option('data', 'segment-length'):
blockDuration = segmentDuration = psdDuration = int(
cp.get('data', 'segment-length')) / int(
cp.get('data', 'sample-rate'))
else:
raise ValueError, "EXIT: Cannot find segment-duration in [data] section of configuration file!"
# get sample rate
if cp.has_option('data', 'sample-rate'):
sampleRate = int(cp.get('data', 'sample-rate'))
print ">> Sample rate has been set to: %d" % sampleRate
print
else:
print ">> ERROR: Need to specify sample-rate in [data] section of configuration file in order to calculate inputs for downstream processes."
sys.exit()
# if not extend option then need to get block duration
if not extend:
if cp.has_option('data', 'block-duration'):
blockDuration = int(cp.get('data', 'block-duration'))
elif cp.has_option('data', 'segment-length'):
s_length = int(cp.get('data', 'segment-length'))
s_num = int(cp.get('data', 'number-of-segments'))
s_rate = int(cp.get('data', 'sample-rate'))
s_overlap = int(cp.get('inspiral', 'segment-overlap'))
# calculate blockDuration
blockDuration = (s_length * s_num -
(s_num - 1) * s_overlap) / s_rate
else:
raise ValueError, "EXIT: Cannot find block-duration in [data] section of configuration file! Either set block-duration or use --extend option."
# calculate the minimum amount of science time need and how the length of quanta to be added on both ends of the analysis time
minscilength = blockDuration + 2 * pad_data
quanta = segmentDuration / 2
# if extend beyond minscilength; add segments of quanta length to each end of segment
print ">> Minimum science segment length is: %ss" % minscilength
print
if extend:
print ">> Will extend minimum science segment by quanta of: %ss" % quanta
print
##############################################################################
# get list of segments for each IFO and put in science txt file
##############################################################################
if not useold:
# external call to ligolw_segment_query
query_start = int(grb_time - offset)
query_end = int(grb_time + offset)
for ifo in ifolist:
if cp.has_option('segments', '%s-segments' % ifo.lower()):
segmentName = cp.get('segments', '%s-segments' % ifo.lower())
check_segment_availability(grb_name, grb_time, query_start,
query_end, offset, ifo, segmentName)
##############################################################################
# get veto files
##############################################################################
if not useold:
# get and read veto definer file
veto_file_url = cp.get('exttrig', 'cvs_veto_definer')
veto_file_path, headers = urllib.urlretrieve(
veto_file_url, os.path.basename(veto_file_url))
# do ligolw_segments_from_cats
deltat = 500
args = {
'start_time': int(grb_time - offset - deltat),
'end_time': int(grb_time + offset + deltat),
'veto_file_path': veto_file_path
}
cmd = "ligolw_segments_from_cats --database --veto-file={veto_file_path} --separate-categories --gps-start-time {start_time} --gps-end-time {end_time} --output-dir=. --individual-results".format(
**args)
print '>>', cmd
print
process = subprocess.Popen([cmd],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output, err = process.communicate()
# Rename the veto files for easier handling
veto_files = glob.glob(
'./*VETOTIME_CAT*{start_time}*xml'.format(**args))
for filename in veto_files:
p = filename.split('-')
newname = "%s-%s_grb%s.xml" % (p[0], p[1], grb_name)
shutil.move(filename, newname)
##############################################################################
# look in txt files and find segment with onsource and minscilength
##############################################################################
# create segment that is +/- offset of GRB time
onsource = [grb_time - int(cp.get('exttrig','onsource_left')),\
grb_time + int(cp.get('exttrig','onsource_right'))]
onSourceSegment = segments.segment(onsource[0], onsource[1])
# get segments in science txt files; see if segments length at least minscilength
# if no then discard them; if yes then put in segdict[ifo] and ifo in ifolist
basic_segdict = segdict = segments.segmentlistdict()
for ifo in ifolist:
# check configuration file
if not cp.has_option('segments', '%s-segments' % ifo.lower()):
continue
# find segment with onsource and check it is at least minscilength
ifo_segfile = '%s-science_grb%s.txt' % (ifo, grb_name)
if os.path.exists(ifo_segfile):
tmplist = segmentsUtils.fromsegwizard(open(ifo_segfile))
try:
s = tmplist.find(onSourceSegment)
except ValueError:
# if onsource not in segments then move on to next IFO
continue
if abs(tmplist[s]) >= minscilength:
segdict[ifo] = segments.segmentlist([tmplist[s]])
basic_segdict[ifo] = segments.segmentlist([s for s in tmplist])
ifolist = segdict.keys()
if len(ifolist) < 2:
print "EXIT: Less than 2 interferometers have available data!"
sys.exit()
##############################################################################
# apply vetoes
##############################################################################
# apply
print ">> Vetoes that overlap with science segments:"
for ifo in ifolist:
# flag; True if IFO not vetoed
cat_flag = True
for cat in catlist:
# create list and check for overlaps
xmlsegfile = "./%s-VETOTIME_CAT%s_grb%s.xml" % (ifo, cat, grb_name)
if os.path.exists(xmlsegfile) and cat_flag:
testseg = segments.segment(
[segdict[ifo][0][0], segdict[ifo][0][1]])
list_overlaps = []
# load the content of the veto-file
xmldoc = utils.load_filename(xmlsegfile,
gz=False,
contenthandler=lsctables.use_in(
ligolw.LIGOLWContentHandler))
segs = table.get_table(xmldoc,
lsctables.SegmentTable.tableName)
segdefs = table.get_table(xmldoc,
lsctables.SegmentDefTable.tableName)
# create a mapping between the segments and their definitions
defdict = {}
for segdef in segdefs:
defdict[segdef.segment_def_id] = segdef.name
# find veto segments that intersect science segment of IFO with onsource
for seg in segs:
s = segments.segment(seg.start_time, seg.end_time)
if testseg.intersects(s):
id = seg.segment_def_id
list_overlaps.append(
[defdict[id], seg.start_time, seg.end_time])
# cut veto CAT1 segments out of science segment; CAT1,2,3 veto IFO if in onsource will veto IFO
for name, segstart, segend in list_overlaps:
print "CAT%s IFO %s, Start: %d End: %d because %s" % (
cat, ifo, segstart, segend, name)
s = segments.segment(segstart, segend)
if onSourceSegment.intersects(s):
segdict.pop(ifo, None)
cat_flag = False
break
if cat == 1:
vetoes = segments.segmentlist(
segments.segment(s[1], s[2]) for s in list_overlaps)
segdict[ifo] -= vetoes
# get list of IFOs
ifolist = segdict.keys()
print
if len(ifolist) < 2:
print "EXIT: After vetoes, less than 2 interferometers have available data!"
sys.exit()
##############################################################################
# determine segment to be analyzed
##############################################################################
# sort from most sensitive to least sensitive
def sensitivity_cmp(ifo1, ifo2):
return cmp(sensitivity_dict[ifo1], sensitivity_dict[ifo2])
ifolist.sort(sensitivity_cmp)
# compares IFOs and finds the segment to analyze
# now try getting off-source segments
# start trying with all IFOs
# work our way through subsets; beginning with most sensitive combinations
test_combos = itertools.chain(*itertools.imap(
lambda n: iterutils.choices(ifolist, n), xrange(len(ifolist), 1, -1)))
off_source_segment = None
the_ifo_combo = []
for ifo_combo in test_combos:
# find conincident science time of IFOs
trial_seglist = segdict.intersection(ifo_combo)
if abs(trial_seglist) < minscilength:
print "EXIT: IFOs do not overlap enough for minscilength", abs(
trial_seglist)
sys.exit()
else:
pass
# find segment with grb_time inside
try:
super_seg = trial_seglist[trial_seglist.find(
onSourceSegment)].contract(pad_data)
except ValueError:
print "EXIT: ValueError with super_seg"
sys.exit()
if onSourceSegment not in super_seg:
print "EXIT: onsource not in super_seg"
sys.exit()
# find int division of onsource time intervals before and after grb
tplus = (super_seg[1] - onSourceSegment[1])
tminus = (onSourceSegment[0] - super_seg[0])
# get minimum number of onsource time intervals in offsource
tmin = (minscilength - 2 * pad_data - abs(onSourceSegment))
# cut to get minscilength
if tplus + tminus > tmin:
half_max = tmin // 2
if tplus < half_max:
print ">> Left sticks out so cut it."
remainder = tmin - tplus
tminus = min(remainder, tminus)
elif tminus < half_max:
print ">> Right sticks out so cut it."
remainder = tmin - tminus
tplus = min(remainder, tplus)
else:
print ">> Both sides stick out so cut as symmetrically as possible."
tminus = half_max
tplus = tmin - half_max # odd trial sticks out on right
if tplus + tminus < tmin:
offsource = None
temp_segment = segments.segment(
(onSourceSegment[0] - tminus - pad_data,
onSourceSegment[1] + tplus + pad_data))
if temp_segment is not None:
offsource = temp_segment
ifolist = list(ifo_combo)
if extend:
# extend as many adjacent 128 second blocks as possible
begin_time = offsource[0] - quanta * (
abs(super_seg[0] - offsource[0]) // quanta)
end_time = offsource[1] + quanta * (
abs(super_seg[1] - offsource[1]) // quanta)
offsource = segments.segment((begin_time, end_time))
break
print
# check length at least minscilength
if abs(offsource) < minscilength:
print abs(offsource), minscilength
print "EXIT: Calculated offsource segment but less than minscilength!"
sys.exit()
# check if no detectors can be used then exit
if len(ifolist) < 2:
print "EXIT: Calculated offsource segment but less than two IFOs!"
sys.exit()
# check edge case
if abs(offsource[0] -
onsource[0]) < pad_data or abs(offsource[1] -
onsource[1]) < pad_data:
print "WARNING: GRB time close to edge of offsource. Its within the padding time."
# concatenate "H1L1V1", etc.
ifolist.sort()
ifotag = "".join(ifolist)
print ">> Offsource segment for %s GRB is:" % ifotag
print "Start:", offsource[0], "End:", offsource[1], "Duration:", offsource[
1] - offsource[0], "Left:", grb_time - offsource[
0], "Right:", offsource[1] - grb_time
print
##############################################################################
# output
##############################################################################
# write analyse txt files
for ifo in basic_ifolist:
if ifo in ifolist:
analysisFP = open('%s-analyse_grb%s.txt' % (ifo, grb_name), 'w')
analysisFP.write('# seg\t start \t stop \t duration\n')
analysisFP.write(
'0\t %d\t %d\t %d\n' %
(offsource[0], offsource[1], offsource[1] - offsource[0]))
else:
analysisFP = open('%s-analyse_grb%s.txt' % (ifo, grb_name), 'w')
analysisFP.write('# seg\t start \t stop \t duration\n')
# calculate blockDuration
blockDuration = int(abs(offsource[0] - offsource[1])) - 2 * pad_data
# calculate psdDuration
# gets largest power of two such that blockDuration/psdDuration = psdRatio
# could have done a binary & operator that is faster but this is more user-friendly I believe
min_psdDuration = int(cp.get('exttrig', 'min-psd-length'))
psdRatio = int(cp.get('exttrig', 'psd-ratio'))
psdDuration = 2**int(numpy.log2(blockDuration / psdRatio))
if psdDuration < min_psdDuration:
print "EXIT: PSD segment duration is too short. It is %ds but needs to be at least %ds in length." % (
psdDuration, min_psdDuration)
sys.exit()
# some downstream processes (e.g. lalapps_tmpltbank) cannot handle these inputs
if cp.has_option('data', 'segment-duration'):
cp.remove_option('data', 'segment-duration')
cp.remove_option('data', 'block-duration')
# some downstream processes (e.g. lalapps_tmpltbank) requires these options to run
print ">> Using sample rate of %d to calculate inputs for downstream processes." % sampleRate
print
segmentLength = segmentDuration * sampleRate
segmentCount = blockDuration / (
segmentDuration /
2) - 1 # subtract 1 because one segment length is overlapped
segmentOverlap = segmentLength / 2
cp.set('data', 'segment-length', segmentLength)
cp.set('data', 'number-of-segments', segmentCount)
cp.set('inspiral', 'segment-overlap', segmentOverlap)
# set values for [coh_PTF_inspral] section in configuration file
cp.set('coh_PTF_inspiral', 'block-duration', blockDuration)
cp.set('coh_PTF_inspiral', 'segment-duration', segmentDuration)
cp.set('coh_PTF_inspiral', 'psd-segment-duration', psdDuration)
cp.set('coh_PTF_inspiral', 'pad-data', pad_data)
f = open('grb%s.ini' % grb_name, 'w')
cp.write(f)
f.close()
print ">> The [data] section of the configuration file has been edited with the following values:"
print "sample-rate=", sampleRate
print "segment-length=", segmentLength
print "number-of-segments=", segmentCount
print "segment-overlap=", segmentOverlap
print
print ">> The [coh_PTF_inspiral] section of the configuration file has been edited with the following values:"
print "block-duration =", blockDuration
print "segment-duration =", segmentDuration
print "psd-segment-duration =", psdDuration
print "pad-data =", pad_data
print
# plot segments
offSourceSegment = segments.segment(offsource[0], offsource[1])
plot_window = segments.segment(grb_time - offset, grb_time + offset)
plot_segments(basic_segdict, onSourceSegment, offSourceSegment, grb_time,
plot_window, "segment_plot_%s.png" % grb_name, grb_name)
# make xml file
if make_xml:
# create a new xml document with an ExtTriggers Table
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
tbl = lsctables.New(lsctables.ExtTriggersTable)
xmldoc.childNodes[-1].appendChild(tbl)
# set the values we need
row = lsctables.ExtTriggersTable()
row.process_id = None
row.det_alts = None
row.det_band = None
row.det_fluence = None
row.det_fluence_int = None
row.det_name = None
row.det_peak = None
row.det_peak_int = None
row.det_snr = ''
row.email_time = 0
row.event_dec = float(grb_dec)
row.event_dec_err = 0.0
row.event_epoch = ''
row.event_err_type = ''
row.event_ra = float(grb_ra)
row.event_ra_err = 0.0
row.start_time = grb_time
row.start_time_ns = 0
row.event_type = ''
row.event_z = 0.0
row.event_z_err = 0.0
row.notice_comments = ''
row.notice_id = ''
row.notice_sequence = ''
row.notice_time = 0
row.notice_type = ''
row.notice_url = ''
row.obs_fov_dec = 0.0
row.obs_fov_dec_width = 0.0
row.obs_fov_ra = 0.0
row.obs_fov_ra_width = 0.0
row.obs_loc_ele = 0.0
row.obs_loc_lat = 0.0
row.obs_loc_long = 0.0
row.ligo_fave_lho = 0.0
row.ligo_fave_llo = 0.0
row.ligo_delay = 0.0
row.event_number_gcn = 9999
row.event_number_grb = grb_name
row.event_status = 0
# insert into the table and write file
tbl.extend([row])
filename = 'grb%s.xml' % grb_name
utils.write_filename(xmldoc, filename)
# plot all vetoes
if make_plots:
vetodict = segments.segmentlistdict()
for cat in catlist:
for ifo in ifolist:
vetofile = "%s-VETOTIME_CAT%s_grb%s.xml" % (ifo, cat, grb_name)
xmldoc = utils.load_filename(vetofile,
gz=False,
contenthandler=lsctables.use_in(
ligolw.LIGOLWContentHandler))
segs = table.get_table(xmldoc,
lsctables.SegmentTable.tableName)
segdefs = table.get_table(xmldoc,
lsctables.SegmentDefTable.tableName)
vetodict[ifo] = segments.segmentlist(
segments.segment(s.start_time, s.end_time) for s in segs)
if vetodict:
plot_segments(vetodict, onSourceSegment, offSourceSegment,
grb_time, plot_window,
"veto_plot_CAT%s_%s.png" % (cat, grb_name),
"%s CAT%s" % (grb_name, cat))
# return
return 'grb%s.ini' % grb_name, ifolist, onSourceSegment, offSourceSegment
if iend > opts.end_time:
iend = opts.end_time
search_epochs.append(segments.segment(istart, iend))
istart += opts.interval
# FIXME: the writing out of the segments should be done at the end so
# that successfully generated dags, etc can be maintained from run to
# run
segmentsUtils.tosegwizard(file("multi_hipe_selectedsegs.txt", 'w'),
search_epochs)
##############################################################################
# Read in all the segment lists
ifolist = []
segdict = {}
if opts.h1_segments:
tmplist = segmentsUtils.fromsegwizard(file(opts.h1_segments)).coalesce()
segdict["H1"] = cleanlist(tmplist, minsciseg)
ifolist.append("H1")
if opts.h2_segments:
tmplist = segmentsUtils.fromsegwizard(file(opts.h2_segments)).coalesce()
segdict["H2"] = cleanlist(tmplist, minsciseg)
ifolist.append("H2")
if opts.l1_segments:
tmplist = segmentsUtils.fromsegwizard(file(opts.l1_segments))
segdict["L1"] = cleanlist(tmplist, minsciseg)
ifolist.append("L1")
if opts.v1_segments:
tmplist = segmentsUtils.fromsegwizard(file(opts.v1_segments))
help="make things verbose" )
parser.add_option("-H","--h1-segments",action="store",type="string",\
default=None, metavar=" H1_SEGMENTS", help="H1 input segment to read" )
parser.add_option("-K","--h2-segments",action="store",type="string",\
default=None, metavar=" H2_SEGMENTS", help="H2 input segment to read" )
parser.add_option("-L","--l1-segments",action="store",type="string",\
default=None, metavar=" L1_SEGMENTS", help="L1 input segment to read" )
parser.add_option("-i","--injection-file",action="store",type="string",\
default=None, metavar=" INJFILE", help="An injection file" )
parser.add_option("-g","--glitch-time",action="store",type="int",\
default=None, metavar=" GLITCH_TIME",\
help="produce plot of triggers around GLITCH_TIME")
( opts , args ) = parser.parse_args()
h1seglist = segmentsUtils.fromsegwizard(file(opts.h1_segments)).coalesce()
h1seglist = cleanlist(h1seglist, 2064)
h1seglist = h1seglist.contract(72)
h2seglist = segmentsUtils.fromsegwizard(file(opts.h2_segments)).coalesce()
h2seglist = cleanlist(h2seglist, 2064)
h2seglist = h2seglist.contract(72)
l1seglist = segmentsUtils.fromsegwizard(file(opts.l1_segments))
l1seglist = cleanlist(l1seglist, 2064)
l1seglist = l1seglist.contract(72)
triplelist = h1seglist & h2seglist & l1seglist
triplelist.coalesce()
h1h2doublelist = h1seglist & h2seglist
# Get segments from segdb
seg_flag = ifo + ':DMT-ANALYSIS_READY:1'
xml_file = ifo + '_seg.xml'
txt_file = ifo + '_seg.txt'
seg_cmd = 'ligolw_segment_query_dqsegdb --query-segments --segment-url https://segments.ligo.org --gps-start-time ' + \
str(seg_start) + ' --gps-end-time ' + str(seg_stop) + ' --include-segments ' + seg_flag + ' --output-file ' + xml_file
os.system(seg_cmd)
xml_cmd = "ligolw_print --table segment --column start_time --column end_time --delimiter ' ' " + xml_file +\
" | awk '{print NR \" \" $1 \" \" $2 \" \" $2-$1}' > " + txt_file
os.system(xml_cmd)
sciseg = segmentsUtils.fromsegwizard(open(txt_file))
sciseg.coalesce()
print 'Number of segments is', len(sciseg)
# get CAT1 veto segs and remove them from the scisegs
# figure out how much time is in cat1, typical duration of flag, whether it breaks up scisegs into tiny pieces...
cat1_file = ifo + '-VETOTIME_CAT1-1127271617-2111400.xml'
txt1_file = ifo + '_cat1.txt'
cat2_file = ifo + '-VETOTIME_CAT2-1127271617-2111400.xml'
txt2_file = ifo + '_cat2.txt'
xml_cmd = "ligolw_print --table segment --column start_time --column end_time --delimiter ' ' " + cat1_file +\
" | awk '{print NR \" \" $1 \" \" $2 \" \" $2-$1}' > " + txt1_file
os.system(xml_cmd)
#xml_cmd = "ligolw_print --table segment --column start_time --column end_time --delimiter ' ' " + cat2_file +\
# " | awk '{print NR \" \" $1 \" \" $2 \" \" $2-$1}' > " + txt2_file
from glue import segmentsUtils
parser=OptionParser()
opts,args = parser.parse_args()
config=ConfigParser.ConfigParser()
config.read(args[0])
if config.has_option('analysis','ifos'):
ifos=ast.literal_eval(config.get('analysis','ifos'))
else:
ifos=['H1','L1']#,'V1']
segments={}
if config.has_option('datafind','veto-categories'):
veto_categories=ast.literal_eval(config.get('datafind','veto-categories'))
else: veto_categories=[]
for ifo in ifos:
(segFileName,dqVetoes)=inspiralutils.findSegmentsToAnalyze(config, ifo,
veto_categories, generate_segments=True,
use_available_data=False, data_quality_vetoes=False)
segfile=open(segFileName)
segments[ifo]=segmentsUtils.fromsegwizard(segfile)
print segments[ifo]
# --- Run segdb query
#
if cp.has_option('datafind','veto-categories'):
veto_categories=ast.literal_eval(cp.get('datafind','veto-categories'))
else: veto_categories=[]
curdir=os.getcwd()
os.chdir(segment_dir)
(segFileName,dqVetoes)=inspiralutils.findSegmentsToAnalyze(cp, ifo,
veto_categories, generate_segments=True,
use_available_data=False, data_quality_vetoes=False)
segfile=open(segFileName)
segmentList[ifo]=segmentsUtils.fromsegwizard(segfile)
segmentList[ifo].coalesce()
segfile.close()
if segmentList[ifo] == []:
print >> sys.stderr, "No matching segments for %s"%ifo
sys.exit()
os.chdir(curdir)
# --------------------------------------------------------------------
# Set up cache files to point to local copies of frames in the working
# directory
if opts.copy_frames:
def exttrig_dataquery(grb_name, grb_time, grb_ra, grb_dec, offset, config_file, extend=False, useold=False, make_plots=False, make_xml=False):
'''
Finds science time of all available IFOs.
'''
##############################################################################
# get segment duration and minimum amount of science time
##############################################################################
# read the configuration file
cp = ConfigParser.ConfigParser()
cp.read(config_file)
# define hardcoded variables
basic_ifolist = ifolist = ['H1','H2','L1','V1']
catlist = [1,2,3]
sensitivity_dict = {"H1": 1, "L1": 2, "H2": 3, "V1": 4, "G1": 5}
# get segment length from configuration file
pad_data = int(cp.get('data','pad-data'))
if cp.has_option('data','segment-duration'):
blockDuration = segmentDuration = psdDuration = int(cp.get('data','segment-duration'))
elif cp.has_option('data','segment-length'):
blockDuration = segmentDuration = psdDuration = int(cp.get('data','segment-length')) /int(cp.get('data','sample-rate'))
else:
raise ValueError, "EXIT: Cannot find segment-duration in [data] section of configuration file!"
# get sample rate
if cp.has_option('data','sample-rate'):
sampleRate = int(cp.get('data', 'sample-rate'))
print ">> Sample rate has been set to: %d"%sampleRate
print
else:
print ">> ERROR: Need to specify sample-rate in [data] section of configuration file in order to calculate inputs for downstream processes."
sys.exit()
# if not extend option then need to get block duration
if not extend:
if cp.has_option('data','block-duration'):
blockDuration = int(cp.get('data','block-duration'))
elif cp.has_option('data','segment-length'):
s_length = int(cp.get('data', 'segment-length'))
s_num = int(cp.get('data', 'number-of-segments'))
s_rate = int(cp.get('data', 'sample-rate'))
s_overlap = int(cp.get('inspiral', 'segment-overlap'))
# calculate blockDuration
blockDuration = ( s_length * s_num - ( s_num - 1 ) * s_overlap ) / s_rate
else:
raise ValueError, "EXIT: Cannot find block-duration in [data] section of configuration file! Either set block-duration or use --extend option."
# calculate the minimum amount of science time need and how the length of quanta to be added on both ends of the analysis time
minscilength = blockDuration + 2 * pad_data
quanta = segmentDuration / 2
# if extend beyond minscilength; add segments of quanta length to each end of segment
print ">> Minimum science segment length is: %ss"%minscilength
print
if extend:
print ">> Will extend minimum science segment by quanta of: %ss"%quanta
print
##############################################################################
# get list of segments for each IFO and put in science txt file
##############################################################################
if not useold:
# external call to ligolw_segment_query
query_start = int(grb_time - offset)
query_end = int(grb_time + offset)
for ifo in ifolist:
if cp.has_option('segments','%s-segments'%ifo.lower()):
segmentName = cp.get('segments','%s-segments'%ifo.lower())
check_segment_availability(grb_name, grb_time, query_start, query_end, offset, ifo, segmentName)
##############################################################################
# get veto files
##############################################################################
if not useold:
# get and read veto definer file
veto_file_url = cp.get('exttrig','cvs_veto_definer')
veto_file_path,headers = urllib.urlretrieve(veto_file_url,os.path.basename(veto_file_url))
# do ligolw_segments_from_cats
deltat = 500
args = {'start_time' : int(grb_time - offset - deltat),
'end_time' : int(grb_time + offset + deltat),
'veto_file_path' : veto_file_path}
cmd = "ligolw_segments_from_cats --database --veto-file={veto_file_path} --separate-categories --gps-start-time {start_time} --gps-end-time {end_time} --output-dir=. --individual-results".format(**args)
print '>>',cmd
print
process = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output,err = process.communicate()
# Rename the veto files for easier handling
veto_files = glob.glob('./*VETOTIME_CAT*{start_time}*xml'.format(**args))
for filename in veto_files:
p = filename.split('-')
newname = "%s-%s_grb%s.xml"%(p[0], p[1], grb_name)
shutil.move(filename, newname)
##############################################################################
# look in txt files and find segment with onsource and minscilength
##############################################################################
# create segment that is +/- offset of GRB time
onsource = [grb_time - int(cp.get('exttrig','onsource_left')),\
grb_time + int(cp.get('exttrig','onsource_right'))]
onSourceSegment = segments.segment(onsource[0], onsource[1])
# get segments in science txt files; see if segments length at least minscilength
# if no then discard them; if yes then put in segdict[ifo] and ifo in ifolist
basic_segdict = segdict = segments.segmentlistdict()
for ifo in ifolist:
# check configuration file
if not cp.has_option('segments','%s-segments' % ifo.lower()):
continue
# find segment with onsource and check it is at least minscilength
ifo_segfile = '%s-science_grb%s.txt' %(ifo, grb_name)
if os.path.exists(ifo_segfile):
tmplist = segmentsUtils.fromsegwizard(open(ifo_segfile))
try:
s = tmplist.find(onSourceSegment)
except ValueError:
# if onsource not in segments then move on to next IFO
continue
if abs(tmplist[s]) >= minscilength:
segdict[ifo] = segments.segmentlist([tmplist[s]])
basic_segdict[ifo] = segments.segmentlist([s for s in tmplist])
ifolist = segdict.keys()
if len(ifolist) < 2:
print "EXIT: Less than 2 interferometers have available data!"
sys.exit()
##############################################################################
# apply vetoes
##############################################################################
# apply
print ">> Vetoes that overlap with science segments:"
for ifo in ifolist:
# flag; True if IFO not vetoed
cat_flag = True
for cat in catlist:
# create list and check for overlaps
xmlsegfile = "./%s-VETOTIME_CAT%s_grb%s.xml" %(ifo, cat, grb_name)
if os.path.exists(xmlsegfile) and cat_flag:
testseg = segments.segment([segdict[ifo][0][0],segdict[ifo][0][1]])
list_overlaps = []
# load the content of the veto-file
xmldoc = utils.load_filename(xmlsegfile, gz = False, contenthandler = lsctables.use_in(ligolw.LIGOLWContentHandler))
segs = lsctables.SegmentTable.get_table(xmldoc)
segdefs = lsctables.SegmentDefTable.get_table(xmldoc)
# create a mapping between the segments and their definitions
defdict = {}
for segdef in segdefs:
defdict[segdef.segment_def_id] = segdef.name
# find veto segments that intersect science segment of IFO with onsource
for seg in segs:
s = segments.segment(seg.start_time, seg.end_time)
if testseg.intersects(s):
id = seg.segment_def_id
list_overlaps.append([defdict[id], seg.start_time, seg.end_time])
# cut veto CAT1 segments out of science segment; CAT1,2,3 veto IFO if in onsource will veto IFO
for name, segstart, segend in list_overlaps:
print "CAT%s IFO %s, Start: %d End: %d because %s"%(cat, ifo, segstart, segend, name)
s = segments.segment(segstart, segend)
if onSourceSegment.intersects(s):
segdict.pop(ifo, None)
cat_flag = False
break
if cat == 1:
vetoes = segments.segmentlist(segments.segment(s[1], s[2]) for s in list_overlaps)
segdict[ifo] -= vetoes
# get list of IFOs
ifolist = segdict.keys()
print
if len(ifolist) < 2:
print "EXIT: After vetoes, less than 2 interferometers have available data!"
sys.exit()
##############################################################################
# determine segment to be analyzed
##############################################################################
# sort from most sensitive to least sensitive
def sensitivity_cmp(ifo1, ifo2):
return cmp(sensitivity_dict[ifo1], sensitivity_dict[ifo2])
ifolist.sort(sensitivity_cmp)
# compares IFOs and finds the segment to analyze
# now try getting off-source segments
# start trying with all IFOs
# work our way through subsets; beginning with most sensitive combinations
test_combos = itertools.chain(*itertools.imap(lambda n: iterutils.choices(ifolist, n), xrange(len(ifolist), 1, -1)))
off_source_segment = None
the_ifo_combo = []
for ifo_combo in test_combos:
# find conincident science time of IFOs
trial_seglist = segdict.intersection(ifo_combo)
if abs(trial_seglist) < minscilength:
print "EXIT: IFOs do not overlap enough for minscilength",abs(trial_seglist)
sys.exit()
else:
pass
# find segment with grb_time inside
try:
super_seg = trial_seglist[trial_seglist.find(onSourceSegment)].contract(pad_data)
except ValueError:
print "EXIT: ValueError with super_seg"
sys.exit()
if onSourceSegment not in super_seg:
print "EXIT: onsource not in super_seg"
sys.exit()
# find int division of onsource time intervals before and after grb
tplus = (super_seg[1] - onSourceSegment[1])
tminus = (onSourceSegment[0] - super_seg[0])
# get minimum number of onsource time intervals in offsource
tmin = ( minscilength - 2*pad_data - abs(onSourceSegment) )
# cut to get minscilength
if tplus + tminus > tmin:
half_max = tmin // 2
if tplus < half_max:
print ">> Left sticks out so cut it."
remainder = tmin - tplus
tminus = min(remainder, tminus)
elif tminus < half_max:
print ">> Right sticks out so cut it."
remainder = tmin - tminus
tplus = min(remainder, tplus)
else:
print ">> Both sides stick out so cut as symmetrically as possible."
tminus = half_max
tplus = tmin - half_max # odd trial sticks out on right
if tplus + tminus < tmin:
offsource = None
temp_segment = segments.segment((onSourceSegment[0] - tminus - pad_data, onSourceSegment[1] + tplus + pad_data))
if temp_segment is not None:
offsource = temp_segment
ifolist = list(ifo_combo)
if extend:
# extend as many adjacent 128 second blocks as possible
begin_time = offsource[0] - quanta * (abs(super_seg[0]-offsource[0])//quanta)
end_time = offsource[1] + quanta * (abs(super_seg[1]-offsource[1])//quanta)
offsource = segments.segment((begin_time,end_time))
break
print
# check length at least minscilength
if abs(offsource) < minscilength:
print abs(offsource),minscilength
print "EXIT: Calculated offsource segment but less than minscilength!"
sys.exit()
# check if no detectors can be used then exit
if len(ifolist) < 2:
print "EXIT: Calculated offsource segment but less than two IFOs!"
sys.exit()
# check edge case
if abs(offsource[0]-onsource[0]) < pad_data or abs(offsource[1]-onsource[1]) < pad_data:
print "WARNING: GRB time close to edge of offsource. Its within the padding time."
# concatenate "H1L1V1", etc.
ifolist.sort()
ifotag = "".join(ifolist)
print ">> Offsource segment for %s GRB is:"%ifotag
print "Start:",offsource[0],"End:",offsource[1],"Duration:",offsource[1]-offsource[0],"Left:",grb_time-offsource[0],"Right:",offsource[1]-grb_time
print
##############################################################################
# output
##############################################################################
# write analyse txt files
for ifo in basic_ifolist:
if ifo in ifolist:
analysisFP = open('%s-analyse_grb%s.txt' %(ifo,grb_name),'w')
analysisFP.write('# seg\t start \t stop \t duration\n')
analysisFP.write('0\t %d\t %d\t %d\n' %(offsource[0],offsource[1],offsource[1]-offsource[0]))
else:
analysisFP = open('%s-analyse_grb%s.txt' %(ifo,grb_name),'w')
analysisFP.write('# seg\t start \t stop \t duration\n')
# calculate blockDuration
blockDuration = int(abs(offsource[0]-offsource[1])) - 2 * pad_data
# calculate psdDuration
# gets largest power of two such that blockDuration/psdDuration = psdRatio
# could have done a binary & operator that is faster but this is more user-friendly I believe
min_psdDuration = int(cp.get('exttrig', 'min-psd-length'))
psdRatio = int(cp.get('exttrig', 'psd-ratio'))
psdDuration = 2**int(numpy.log2(blockDuration/psdRatio))
if psdDuration < min_psdDuration:
print "EXIT: PSD segment duration is too short. It is %ds but needs to be at least %ds in length."%(psdDuration,min_psdDuration)
sys.exit()
# some downstream processes (e.g. lalapps_tmpltbank) cannot handle these inputs
if cp.has_option('data', 'segment-duration'):
cp.remove_option('data', 'segment-duration')
cp.remove_option('data', 'block-duration')
# some downstream processes (e.g. lalapps_tmpltbank) requires these options to run
print ">> Using sample rate of %d to calculate inputs for downstream processes."%sampleRate
print
segmentLength = segmentDuration*sampleRate
segmentCount = blockDuration/(segmentDuration/2) - 1 # subtract 1 because one segment length is overlapped
segmentOverlap = segmentLength/2
cp.set('data', 'segment-length', segmentLength)
cp.set('data', 'number-of-segments', segmentCount)
cp.set('inspiral', 'segment-overlap', segmentOverlap)
# set values for [coh_PTF_inspral] section in configuration file
cp.set('coh_PTF_inspiral', 'block-duration', blockDuration)
cp.set('coh_PTF_inspiral', 'segment-duration', segmentDuration)
cp.set('coh_PTF_inspiral', 'psd-segment-duration', psdDuration)
cp.set('coh_PTF_inspiral', 'pad-data', pad_data)
f = open('grb%s.ini'%grb_name,'w')
cp.write(f)
f.close()
print ">> The [data] section of the configuration file has been edited with the following values:"
print "sample-rate=",sampleRate
print "segment-length=",segmentLength
print "number-of-segments=",segmentCount
print "segment-overlap=",segmentOverlap
print
print ">> The [coh_PTF_inspiral] section of the configuration file has been edited with the following values:"
print "block-duration =",blockDuration
print "segment-duration =",segmentDuration
print "psd-segment-duration =",psdDuration
print "pad-data =",pad_data
print
# plot segments
offSourceSegment = segments.segment(offsource[0], offsource[1])
plot_window = segments.segment(grb_time-offset, grb_time+offset)
plot_segments(basic_segdict, onSourceSegment, offSourceSegment, grb_time, plot_window, "segment_plot_%s.png"%grb_name, grb_name)
# make xml file
if make_xml:
# create a new xml document with an ExtTriggers Table
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
tbl = lsctables.New(lsctables.ExtTriggersTable)
xmldoc.childNodes[-1].appendChild(tbl)
# set the values we need
row = lsctables.ExtTriggersTable()
row.process_id = None
row.det_alts = None
row.det_band = None
row.det_fluence = None
row.det_fluence_int = None
row.det_name = None
row.det_peak = None
row.det_peak_int = None
row.det_snr = ''
row.email_time = 0
row.event_dec = float(grb_dec)
row.event_dec_err = 0.0
row.event_epoch = ''
row.event_err_type = ''
row.event_ra = float(grb_ra)
row.event_ra_err = 0.0
row.start_time = grb_time
row.start_time_ns = 0
row.event_type = ''
row.event_z = 0.0
row.event_z_err = 0.0
row.notice_comments = ''
row.notice_id = ''
row.notice_sequence = ''
row.notice_time = 0
row.notice_type = ''
row.notice_url = ''
row.obs_fov_dec = 0.0
row.obs_fov_dec_width = 0.0
row.obs_fov_ra = 0.0
row.obs_fov_ra_width = 0.0
row.obs_loc_ele = 0.0
row.obs_loc_lat = 0.0
row.obs_loc_long = 0.0
row.ligo_fave_lho = 0.0
row.ligo_fave_llo = 0.0
row.ligo_delay = 0.0
row.event_number_gcn = 9999
row.event_number_grb = grb_name
row.event_status = 0
# insert into the table and write file
tbl.extend([row])
filename = 'grb%s.xml' % grb_name
utils.write_filename(xmldoc, filename)
# plot all vetoes
if make_plots:
vetodict = segments.segmentlistdict()
for cat in catlist:
for ifo in ifolist:
vetofile = "%s-VETOTIME_CAT%s_grb%s.xml" % (ifo, cat, grb_name)
xmldoc = utils.load_filename(vetofile, gz = False, contenthandler = lsctables.use_in(ligolw.LIGOLWContentHandler))
segs = lsctables.SegmentTable.get_table(xmldoc)
segdefs = lsctables.SegmentDefTable.get_table(xmldoc)
vetodict[ifo] = segments.segmentlist(segments.segment(s.start_time, s.end_time) for s in segs)
if vetodict:
plot_segments(vetodict, onSourceSegment, offSourceSegment, grb_time, plot_window, "veto_plot_CAT%s_%s.png"%(cat,grb_name), "%s CAT%s"%(grb_name, cat))
# return
return 'grb%s.ini'%grb_name, ifolist, onSourceSegment, offSourceSegment
iend = istart + opts.interval
if iend > opts.end_time:
iend = opts.end_time
search_epochs.append(segments.segment(istart,iend))
istart += opts.interval
# FIXME: the writing out of the segments should be done at the end so
# that successfully generated dags, etc can be maintained from run to
# run
segmentsUtils.tosegwizard(file("multi_hipe_selectedsegs.txt",'w'),search_epochs)
##############################################################################
# Read in all the segment lists
ifolist = []
segdict = {}
if opts.h1_segments:
tmplist = segmentsUtils.fromsegwizard(file(opts.h1_segments)).coalesce()
segdict["H1"] = cleanlist(tmplist, minsciseg)
ifolist.append("H1")
if opts.h2_segments:
tmplist = segmentsUtils.fromsegwizard(file(opts.h2_segments)).coalesce()
segdict["H2"] = cleanlist(tmplist, minsciseg)
ifolist.append("H2")
if opts.l1_segments:
tmplist = segmentsUtils.fromsegwizard(file(opts.l1_segments))
segdict["L1"] = cleanlist(tmplist, minsciseg)
ifolist.append("L1")
if opts.v1_segments:
tmplist = segmentsUtils.fromsegwizard(file(opts.v1_segments))
