def generated_vdb_ascii(json_str, filepath):
res_dict = json.loads(json_str)
active_list = res_dict['active']
active_segments = segments.segmentlist(
[segments.segment(x[0], x[1]) for x in active_list])
known_list = res_dict['known']
known_segments = segments.segmentlist(
[segments.segment(x[0], x[1]) for x in known_list])
query_start = res_dict['query_information']['start']
query_stop = res_dict['query_information']['end']
if query_start != 0 and query_stop != 0:
requested_span = segments.segmentlist(
[segments.segment(query_start, query_stop)])
else:
requested_span = segments.segmentlist(
[segments.segment(0, 9999999999)])
active_segments_string = ',1 \n'.join(
[str(i[0]) + "," + str(i[1]) for i in active_segments]) + ",1 \n"
unknown_segments = requested_span - known_segments
unknown_segments_string = ',-1 \n'.join(
[str(i[0]) + "," + str(i[1]) for i in unknown_segments]) + ",-1 \n"
known_not_active_segments = known_segments - active_segments
known_not_active_segments_string = ',0 \n'.join(
[str(i[0]) + "," + str(i[1])
for i in known_not_active_segments]) + ",0 \n"
output_fileh = open(filepath, 'w+')
query_info_string = json.dumps(res_dict['query_information'], indent=1)
output_fileh.writelines(query_info_string)
output_fileh.write('\n')
output_fileh.writelines(active_segments_string)
output_fileh.writelines(unknown_segments_string)
output_fileh.writelines(known_not_active_segments_string)
output_fileh.close()
return filepath
def inj_seg(self, exclude_coinc_flags=None):
""" Returns a segmentlist that is the union of all excitation,
segdb and bitmasked channels.
"""
if exclude_coinc_flags is None:
exclude_coinc_flags = []
tmp_list = segments.segmentlist([])
for key in self.exc_dict.keys():
if key[3:] not in exclude_coinc_flags:
tmp_list.extend(self.exc_dict[key])
for key in self.seg_dict.keys():
if key[3:] not in exclude_coinc_flags:
tmp_list.extend(self.seg_dict[key])
for key in self.bitmask_dict.keys():
if key[3:] not in exclude_coinc_flags:
tmp_list.extend(self.bitmask_dict[key])
if self.schedule_time:
seg = segments.segment(self.schedule_time, self.schedule_time + 1)
seg_list = segments.segmentlist([seg])
tmp_list.extend(seg_list)
for time in self.gracedb_time:
seg = segments.segment(time, time + 1)
seg_list = segments.segmentlist([seg])
tmp_list.extend(seg_list)
return tmp_list
def split_segment(seg, min_segment_length, pad, overlap,
short_segment_duration, max_job_length):
# avoid infinite loop
if min_segment_length + 2 * pad <= overlap:
raise ValueError, "infinite loop: min_segment_length + 2 * pad must be > overlap"
# clip max_job_length down to an allowed size
max_job_length = clip_segment_length(max_job_length, pad,
short_segment_duration)
seglist = segments.segmentlist()
while abs(seg) >= min_segment_length + 2 * pad:
# try to use max_job_length each time
if abs(seg) >= max_job_length:
seglist.append(segments.segment(seg[0], seg[0] + max_job_length))
else:
seglist.append(
segments.segment(
seg[0], seg[0] + clip_segment_length(
abs(seg), pad, short_segment_duration)))
assert abs(seglist[-1]) != 0 # safety-check for no-op
# bounds must be integers
if abs((int(seglist[-1][0]) - seglist[-1][0]) /
seglist[-1][0]) > 1e-14 or abs(
(int(seglist[-1][1]) - seglist[-1][1]) /
seglist[-1][1]) > 1e-14:
raise ValueError, "segment %s does not have integer boundaries" % str(
seglist[-1])
# advance segment
seg = segments.segment(seglist[-1][1] - overlap, seg[1])
if not seglist:
raise ValueError, "unable to use segment %s" % str(seg)
return seglist
def subdivide(seg, length, min_len=0):
"""
Subdivide a segment into smaller segments based on a given length. Enforce a given minimum length at the end, if necessary. If the remainder segment is smaller than the minimum length, then the last two segments will span the remainder plus penultimate segment, with the span divided evenly between the two.
Input segment: (0, 10] subdivide 3 min 2
Output segment(s): (0, 3], (3, 6], (6, 8], (8, 10]
"""
assert length >= min_len
if abs(seg) < min_len:
return segmentlist([])
if abs(seg) <= length:
return segmentlist([seg])
subsegl = segmentlist([])
for i in range(int(float(abs(seg)) / length)):
st = seg[0]
subsegl.append(segment(st + length * i, st + length * (i + 1)))
# Make an attempt to subdivide evenly.
if float(abs(seg)) % length <= min_len:
s1 = subsegl.pop()
rem_len = float(abs(s1)) + (float(abs(seg)) % length)
s2 = segment(seg[1] - rem_len / 2, seg[1])
s1 = segment(s1[0], seg[1] - rem_len / 2)
subsegl.append(s1)
subsegl.append(s2)
else:
subsegl.append(segment(subsegl[-1][1], seg[1]))
return subsegl
def fromsegmentxml(file, dict=False, id=None):
"""
Read a glue.segments.segmentlist from the file object file containing an
xml segment table.
Arguments:
file : file object
file object for segment xml file
Keyword Arguments:
dict : [ True | False ]
returns a glue.segments.segmentlistdict containing coalesced
glue.segments.segmentlists keyed by seg_def.name for each entry in the
contained segment_def_table. Default False
id : int
returns a glue.segments.segmentlist object containing only those
segments matching the given segment_def_id integer
"""
# load xmldocument and SegmentDefTable and SegmentTables
xmldoc, digest = utils.load_fileobj(file,
gz=file.name.endswith(".gz"),
contenthandler=lsctables.use_in(
ligolw.LIGOLWContentHandler))
seg_def_table = lsctables.SegmentDefTable.get_table(xmldoc)
seg_table = lsctables.SegmentTable.get_table(xmldoc)
if dict:
segs = segments.segmentlistdict()
else:
segs = segments.segmentlist()
seg_id = {}
for seg_def in seg_def_table:
seg_id[int(seg_def.segment_def_id)] = str(seg_def.name)
if dict:
segs[str(seg_def.name)] = segments.segmentlist()
for seg in seg_table:
if dict:
segs[seg_id[int(seg.segment_def_id)]]\
.append(segments.segment(seg.start_time, seg.end_time))
continue
if id and int(seg.segment_def_id) == id:
segs.append(segments.segment(seg.start_time, seg.end_time))
continue
segs.append(segments.segment(seg.start_time, seg.end_time))
if dict:
for seg_name in seg_id.values():
segs[seg_name] = segs[seg_name].coalesce()
else:
segs = segs.coalesce()
xmldoc.unlink()
return segs
def subdivide(seg, length, min_len=0):
"""
Subdivide a segment into smaller segments based on a given length. Enforce a given minimum length at the end, if necessary. If the remainder segment is smaller than the minimum length, then the last two segments will span the remainder plus penultimate segment, with the span divided evenly between the two.
Input segment: (0, 10] subdivide 3 min 2
Output segment(s): (0, 3], (3, 6], (6, 8], (8, 10]
"""
assert length >= min_len
if abs(seg) < min_len:
return segmentlist([])
if abs(seg) <= length:
return segmentlist([seg])
subsegl = segmentlist([])
for i in range(int(float(abs(seg))/length)):
st = seg[0]
subsegl.append(segment(st+length*i, st+length*(i+1)))
# Make an attempt to subdivide evenly.
if float(abs(seg)) % length <= min_len:
s1 = subsegl.pop()
rem_len = float(abs(s1)) + (float(abs(seg)) % length)
s2 = segment(seg[1]-rem_len/2, seg[1])
s1 = segment(s1[0], seg[1]-rem_len/2)
subsegl.append(s1)
subsegl.append(s2)
else:
subsegl.append(segment(subsegl[-1][1], seg[1]))
return subsegl
def do_summary_table(xmldoc, sim_tree, liv_tree):
try:
search_summary = lsctables.SearchSummaryTable.get_table(xmldoc)
except ValueError:
search_summary = lsctables.New(lsctables.SearchSummaryTable, [
"process_id", "nevents", "ifos", "comment", "in_start_time",
"in_start_time_ns", "out_start_time", "out_start_time_ns",
"in_end_time", "in_end_time_ns", "out_end_time", "out_end_time_ns"
])
xmldoc.childNodes[0].appendChild(search_summary)
process_id_type = lsctables.ProcessID
runids = set()
for i in range(0, sim_tree.GetEntries()):
sim_tree.GetEntry(i)
# Id for the run processed by WaveBurst -> process ID
if sim_tree.run in runids:
continue
row = search_summary.RowType()
row.process_id = process_id_type(sim_tree.run)
runids.add(sim_tree.run)
# Search Summary Table
# events found in the run -> nevents
setattr(row, "nevents", sim_tree.GetEntries())
# Imstruments involved in the search
row.ifos = lsctables.ifos_from_instrument_set(
get_ifos_from_index(
branch_array_to_list(sim_tree.ifo, sim_tree.ndim)))
setattr(row, "comment", "waveburst")
# Begin and end time of the segment
# TODO: This is a typical offset on either side of the job for artifacts
# It can, and probably will change in the future, and should not be hardcoded
# TODO: Make this work properly. We need a gps end from the livetime
waveoffset = 8
livetime = 600
#live_entries = live_tree.GetEntries()
# This is WAAAAAAAAAAAAAY too slow
#for l in range(0, live_entries):
#liv_tree.GetEntry(l)
#livetime = max(livetime, liv_tree.live)
#if livetime < 0:
#sys.exit("Could not find livetime, cannot fill all of summary table.")
# in -- with waveoffset
# out -- without waveoffset
row.set_in(
segments.segment(LIGOTimeGPS(sim_tree.gps - waveoffset),
LIGOTimeGPS(sim_tree.gps + livetime +
waveoffset)))
row.set_out(
segments.segment(LIGOTimeGPS(sim_tree.gps),
LIGOTimeGPS(sim_tree.gps + livetime)))
search_summary.append(row)
def inj_seg(self, exclude_coinc_flags=None):
""" Returns a segmentlist that is the union of all excitation,
segdb and bitmasked channels.
"""
if exclude_coinc_flags is None:
exclude_coinc_flags = []
tmp_list = segments.segmentlist([])
for key in self.exc_dict.keys():
if key[3:] not in exclude_coinc_flags:
tmp_list.extend(self.exc_dict[key])
for key in self.seg_dict.keys():
if key[3:] not in exclude_coinc_flags:
tmp_list.extend(self.seg_dict[key])
for key in self.bitmask_dict.keys():
if key[3:] not in exclude_coinc_flags:
tmp_list.extend(self.bitmask_dict[key])
if self.schedule_time:
seg = segments.segment(self.schedule_time, self.schedule_time + 1)
seg_list = segments.segmentlist([seg])
tmp_list.extend(seg_list)
for time in self.gracedb_time:
seg = segments.segment(time, time + 1)
seg_list = segments.segmentlist([seg])
tmp_list.extend(seg_list)
return tmp_list
def generated_vdb_ascii(json_str, filepath):
res_dict = json.loads(json_str)
active_list = res_dict["active"]
active_segments = segments.segmentlist([segments.segment(x[0], x[1]) for x in active_list])
known_list = res_dict["known"]
known_segments = segments.segmentlist([segments.segment(x[0], x[1]) for x in known_list])
query_start = res_dict["query_information"]["start"]
query_stop = res_dict["query_information"]["end"]
if query_start != 0 and query_stop != 0:
requested_span = segments.segmentlist([segments.segment(query_start, query_stop)])
else:
requested_span = segments.segmentlist([segments.segment(0, 9999999999)])
active_segments_string = ",1 \n".join([str(i[0]) + "," + str(i[1]) for i in active_segments]) + ",1 \n"
unknown_segments = requested_span - known_segments
unknown_segments_string = ",-1 \n".join([str(i[0]) + "," + str(i[1]) for i in unknown_segments]) + ",-1 \n"
known_not_active_segments = known_segments - active_segments
known_not_active_segments_string = (
",0 \n".join([str(i[0]) + "," + str(i[1]) for i in known_not_active_segments]) + ",0 \n"
)
output_fileh = open(filepath, "w+")
query_info_string = json.dumps(res_dict["query_information"], indent=1)
output_fileh.writelines(query_info_string)
output_fileh.write("\n")
output_fileh.writelines(active_segments_string)
output_fileh.writelines(unknown_segments_string)
output_fileh.writelines(known_not_active_segments_string)
output_fileh.close()
return filepath
def generated_vdb_ascii(json_dict,filepath):
#res_dict=json.loads(json_str)
res_dict=json_dict
active_list=res_dict['active']
active_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in active_list])
active_segments.coalesce()
known_list=res_dict['known']
known_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in known_list])
known_segments.coalesce()
query_start=res_dict['query_information']['start']
query_stop=res_dict['query_information']['end']
if query_start!=0 and query_stop!=0:
requested_span=segments.segmentlist([segments.segment(query_start,query_stop)])
else:
requested_span=segments.segmentlist([segments.segment(0,9999999999)])
active_segments_string=',1 \n'.join([str(i[0])+","+str(i[1]) for i in active_segments])+",1 \n"
unknown_segments=requested_span-known_segments
unknown_segments_string=',-1 \n'.join([str(i[0])+","+str(i[1]) for i in unknown_segments])+",-1 \n"
known_not_active_segments=known_segments-active_segments
known_not_active_segments_string=',0 \n'.join([str(i[0])+","+str(i[1]) for i in known_not_active_segments])+",0 \n"
output_fileh=open(filepath,'a')
query_info_string=json.dumps(res_dict['query_information'], indent=1)
output_fileh.writelines(query_info_string)
output_fileh.write('\n')
output_fileh.writelines(active_segments_string)
output_fileh.writelines(unknown_segments_string)
output_fileh.writelines(known_not_active_segments_string)
output_fileh.close()
return filepath
def fromsegwizard(file, coltype=int, strict=True):
"""
Read a segmentlist from the file object file containing a segwizard
compatible segment list. Parsing stops on the first line that
cannot be parsed (which is consumed). The segmentlist will be
created with segment whose boundaries are of type coltype, which
should raise ValueError if it cannot convert its string argument.
Two-column, three-column, and four-column segwizard files are
recognized, but the entire file must be in the same format, which
is decided by the first parsed line. If strict is True and the
file is in three- or four-column format, then each segment's
duration is checked against that column in the input file.
NOTE: the output is a segmentlist as described by the file; if
the segments in the input file are not coalesced or out of order,
then thusly shall be the output of this function. It is
recommended that this function's output be coalesced before use.
"""
commentpat = re.compile(r"\s*([#;].*)?\Z", re.DOTALL)
twocolsegpat = re.compile(r"\A\s*([\d.+-eE]+)\s+([\d.+-eE]+)\s*\Z")
threecolsegpat = re.compile(
r"\A\s*([\d.+-eE]+)\s+([\d.+-eE]+)\s+([\d.+-eE]+)\s*\Z")
fourcolsegpat = re.compile(
r"\A\s*([\d]+)\s+([\d.+-eE]+)\s+([\d.+-eE]+)\s+([\d.+-eE]+)\s*\Z")
format = None
l = segments.segmentlist()
for line in file:
line = commentpat.split(line)[0]
if not line:
continue
try:
[tokens] = fourcolsegpat.findall(line)
num = int(tokens[0])
seg = segments.segment(map(coltype, tokens[1:3]))
duration = coltype(tokens[3])
this_line_format = 4
except ValueError:
try:
[tokens] = threecolsegpat.findall(line)
seg = segments.segment(map(coltype, tokens[0:2]))
duration = coltype(tokens[2])
this_line_format = 3
except ValueError:
try:
[tokens] = twocolsegpat.findall(line)
seg = segments.segment(map(coltype, tokens[0:2]))
duration = abs(seg)
this_line_format = 2
except ValueError:
break
if strict:
if abs(seg) != duration:
raise ValueError("segment '%s' has incorrect duration" % line)
if format is None:
format = this_line_format
elif format != this_line_format:
raise ValueError("segment '%s' format mismatch" % line)
l.append(seg)
return l
def fromsegmentxml(file, dict=False, id=None):
"""
Read a glue.segments.segmentlist from the file object file containing an
xml segment table.
Arguments:
file : file object
file object for segment xml file
Keyword Arguments:
dict : [ True | False ]
returns a glue.segments.segmentlistdict containing coalesced
glue.segments.segmentlists keyed by seg_def.name for each entry in the
contained segment_def_table. Default False
id : int
returns a glue.segments.segmentlist object containing only those
segments matching the given segment_def_id integer
"""
# load xmldocument and SegmentDefTable and SegmentTables
xmldoc, digest = utils.load_fileobj(file, gz=file.name.endswith(".gz"))
seg_def_table = table.get_table(xmldoc, lsctables.SegmentDefTable.tableName)
seg_table = table.get_table(xmldoc, lsctables.SegmentTable.tableName)
if dict:
segs = segments.segmentlistdict()
else:
segs = segments.segmentlist()
seg_id = {}
for seg_def in seg_def_table:
seg_id[int(seg_def.segment_def_id)] = str(seg_def.name)
if dict:
segs[str(seg_def.name)] = segments.segmentlist()
for seg in seg_table:
if dict:
segs[seg_id[int(seg.segment_def_id)]]\
.append(segments.segment(seg.start_time, seg.end_time))
continue
if id and int(seg.segment_def_id)==id:
segs.append(segments.segment(seg.start_time, seg.end_time))
continue
segs.append(segments.segment(seg.start_time, seg.end_time))
if dict:
for seg_name in seg_id.values():
segs[seg_name] = segs[seg_name].coalesce()
else:
segs = segs.coalesce()
xmldoc.unlink()
return segs
def fromsegwizard(file, coltype = int, strict = True):
"""
Read a segmentlist from the file object file containing a segwizard
compatible segment list. Parsing stops on the first line that
cannot be parsed (which is consumed). The segmentlist will be
created with segment whose boundaries are of type coltype, which
should raise ValueError if it cannot convert its string argument.
Two-column, three-column, and four-column segwizard files are
recognized, but the entire file must be in the same format, which
is decided by the first parsed line. If strict is True and the
file is in three- or four-column format, then each segment's
duration is checked against that column in the input file.
NOTE: the output is a segmentlist as described by the file; if
the segments in the input file are not coalesced or out of order,
then thusly shall be the output of this function. It is
recommended that this function's output be coalesced before use.
"""
commentpat = re.compile(r"\s*([#;].*)?\Z", re.DOTALL)
twocolsegpat = re.compile(r"\A\s*([\d.+-eE]+)\s+([\d.+-eE]+)\s*\Z")
threecolsegpat = re.compile(r"\A\s*([\d.+-eE]+)\s+([\d.+-eE]+)\s+([\d.+-eE]+)\s*\Z")
fourcolsegpat = re.compile(r"\A\s*([\d]+)\s+([\d.+-eE]+)\s+([\d.+-eE]+)\s+([\d.+-eE]+)\s*\Z")
format = None
l = segments.segmentlist()
for line in file:
line = commentpat.split(line)[0]
if not line:
continue
try:
[tokens] = fourcolsegpat.findall(line)
num = int(tokens[0])
seg = segments.segment(map(coltype, tokens[1:3]))
duration = coltype(tokens[3])
this_line_format = 4
except ValueError:
try:
[tokens] = threecolsegpat.findall(line)
seg = segments.segment(map(coltype, tokens[0:2]))
duration = coltype(tokens[2])
this_line_format = 3
except ValueError:
try:
[tokens] = twocolsegpat.findall(line)
seg = segments.segment(map(coltype, tokens[0:2]))
duration = abs(seg)
this_line_format = 2
except ValueError:
break
if strict:
if abs(seg) != duration:
raise ValueError("segment '%s' has incorrect duration" % line)
if format is None:
format = this_line_format
elif format != this_line_format:
raise ValueError("segment '%s' format mismatch" % line)
l.append(seg)
return l
def get_manually(gps_start_time, gps_end_time):
db_location = os.environ['S6_SEGMENT_SERVER']
segment_connection = segmentdb_utils.setup_database(db_location)
engine = query_engine.LdbdQueryEngine(segment_connection)
# 1. Get v1 science segments
sql = "SELECT segment.start_time, segment.end_time "
sql += "FROM segment_definer, segment "
sql += "WHERE segment.segment_def_id = segment_definer.segment_def_id "
sql += "AND segment_definer.ifos = 'H1' "
sql += "AND segment.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.name = 'DMT-SCIENCE' "
sql += "AND segment_definer.version = 1 "
sql += "AND NOT (%s > segment.end_time OR segment.start_time > %s)" % (
gps_start_time, gps_end_time)
v1_science_segments = segmentlist(
[segment(row[0], row[1]) for row in engine.query(sql)]).coalesce()
# 2. Get v2 science summaries
sql = "SELECT segment_summary.start_time, segment_summary.end_time "
sql += "FROM segment_definer, segment_summary "
sql += "WHERE segment_summary.segment_def_id = segment_definer.segment_def_id "
sql += "AND segment_definer.ifos = 'H1' "
sql += "AND segment_summary.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.name = 'DMT-SCIENCE' "
sql += "AND segment_definer.version = 2 "
sql += "AND NOT (%s > segment_summary.end_time OR segment_summary.start_time > %s)" % (
gps_start_time, gps_end_time)
v2_science_summaries = segmentlist(
[segment(row[0], row[1]) for row in engine.query(sql)]).coalesce()
# 1. Get v2 science segments
sql = "SELECT segment.start_time, segment.end_time "
sql += "FROM segment_definer, segment "
sql += "WHERE segment.segment_def_id = segment_definer.segment_def_id "
sql += "AND segment_definer.ifos = 'H1' "
sql += "AND segment.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.name = 'DMT-SCIENCE' "
sql += "AND segment_definer.version = 2 "
sql += "AND NOT (%s > segment.end_time OR segment.start_time > %s)" % (
gps_start_time, gps_end_time)
v2_science_segments = segmentlist(
[segment(row[0], row[1]) for row in engine.query(sql)]).coalesce()
result = (v1_science_segments - v2_science_summaries) + v2_science_segments
result.coalesce()
result &= segmentlist([segment(gps_start_time, gps_end_time)])
return result
def _query(self, channel, start, end):
"Do we know where the frame file is?"
if segment(start, end) in self._remotecoverage:
return True
urls = query_LDR(self.host, self.port, channel[0], self.frametype, start, end, urlType="file")
if urls:
new = Cache.from_urls(urls, coltype=int)
new.sort(key=operator.attrgetter("segment"))
self.add_cache(new)
return segment(start, end) in self._remotecoverage
def test_tofromseqwizard(self): """ Check that the segwizard writing routine's output is parsed correctly. """ data = StringIO.StringIO() correct = segments.segmentlist([segments.segment(10, 100), segments.segment(110, 120), segments.segment(125, 130), segments.segment(0, 200)]) segmentsUtils.tosegwizard(data, correct) data.seek(0) self.assertEqual(correct, segmentsUtils.fromsegwizard(data, strict=True))
def randomlist(n): def r(): return random.randint(1,5000) if n < 1: raise ValueError, "randomlist(n): n must be >= 1" x = r() l = segments.segmentlist([segments.segment(x, x + r())]) for i in range(n - 1): x = l[-1][1] + r() l.append(segments.segment(x, x + r())) return l
def test_optimized_query(engine):
res = segmentdb_utils.query_segments( engine, 'segment_summary', [ ('H1','DMT-TESTSEG_2',1,924900000,924900016,0,0),
('H1','DMT-TESTSEG_3',1,924900000,924900016,0,0) ] )
if res[0] != segmentlist([segment(924900000, 924900010)]):
return False
if res[1] != segmentlist([segment(924900008, 924900016)]):
return False
return True
def coalesceResultDictionary(result_dict):
out_result_dict=result_dict
active_seg_python_list=[seg.segment(i[0],i[1]) for i in result_dict[0]['active']]
active_seg_list=seg.segmentlist(active_seg_python_list)
active_seg_list.coalesce()
out_result_dict[0]['active']=active_seg_list
known_seg_python_list=[seg.segment(i[0],i[1]) for i in result_dict[0]['known']]
known_seg_list=seg.segmentlist(known_seg_python_list)
known_seg_list.coalesce()
out_result_dict[0]['known']=known_seg_list
return out_result_dict
def pad_and_truncate(row_start, row_end):
tmp = segmentlist([segment(row_start + start_pad, row_end + end_pad)])
# No coalesce needed as a list with a single segment is already coalesced
tmp &= search_span_list
# The intersection is guaranteed to be non-empty if the row passed match()
# PR 2969: The above comment is incorrect. Negative padding may cause
# an empty intersection.
if len(tmp) == 0:
return segment(0,0)
else:
return tmp[0]
def test_basic_seg_summary(engine):
res = segmentdb_utils.query_segments( engine, 'segment_summary', [ ('H1','DMT-TESTSEG_1',1,924900000,924900016,0,0) ] )
if res != [ segmentlist( [segment(924900000, 924900016)] ) ]:
return False
res = segmentdb_utils.query_segments( engine, 'segment_summary', [ ('H1','DMT-TESTSEG_1',2,924900000,924900016,0,0) ] )
if res != [ segmentlist( [segment(924900008, 924900010)] ) ]:
return False
return True
def associate_psds_to_segments(opt, fd_segments, gwstrain, flen, delta_f, flow,
dyn_range_factor=1., precision=None):
"""Generate a set of overlapping PSDs covering the data in GWstrain.
Then associate these PSDs with the appropriate segment in strain_segments.
Parameters
-----------
opt : object
Result of parsing the CLI with OptionParser, or any object with the
required attributes (psd_model, psd_file, asd_file, psd_estimation,
psd_segment_length, psd_segment_stride, psd_inverse_length, psd_output).
fd_segments : StrainSegments.fourier_segments() object
The fourier transforms of the various analysis segments. The psd
attribute of each segment is updated to point to the appropriate PSD.
gwstrain : Strain object
The timeseries of raw data on which to estimate PSDs.
flen : int
The length in samples of the output PSDs.
delta_f : float
The frequency step of the output PSDs.
flow: float
The low frequncy cutoff to use when calculating the PSD.
dyn_range_factor : {1, float}
For PSDs taken from models or text files, if `dyn_range_factor` is
not None, then the PSD is multiplied by `dyn_range_factor` ** 2.
precision : str, choices (None,'single','double')
If not specified, or specified as None, the precision of the returned
PSD will match the precision of the data, if measuring a PSD, or will
match the default precision of the model if using an analytical PSD.
If 'single' the PSD will be converted to float32, if not already in
that precision. If 'double' the PSD will be converted to float64, if
not already in that precision.
"""
psds_and_times = generate_overlapping_psds(opt, gwstrain, flen, delta_f,
flow, dyn_range_factor=dyn_range_factor,
precision=precision)
for fd_segment in fd_segments:
best_psd = None
psd_overlap = 0
inp_seg = segments.segment(fd_segment.seg_slice.start,
fd_segment.seg_slice.stop)
for start_idx, end_idx, psd in psds_and_times:
psd_seg = segments.segment(start_idx, end_idx)
if psd_seg.intersects(inp_seg):
curr_overlap = abs(inp_seg & psd_seg)
if curr_overlap > psd_overlap:
psd_overlap = curr_overlap
best_psd = psd
if best_psd is None:
err_msg = "No PSDs found intersecting segment!"
raise ValueError(err_msg)
fd_segment.psd = best_psd
def do_summary_table(xmldoc, sim_tree, liv_tree):
try:
search_summary = table.get_table(xmldoc, lsctables.SearchSummaryTable.tableName)
except ValueError:
search_summary = lsctables.New(lsctables.SearchSummaryTable,
["process_id", "nevents", "ifos", "comment", "in_start_time",
"in_start_time_ns", "out_start_time", "out_start_time_ns",
"in_end_time", "in_end_time_ns", "out_end_time", "out_end_time_ns"])
xmldoc.childNodes[0].appendChild(search_summary)
process_id_type = type(table.get_table(xmldoc, lsctables.ProcessTable.tableName).next_id)
runids = set()
for i in range(0, sim_tree.GetEntries()) :
sim_tree.GetEntry(i)
# Id for the run processed by WaveBurst -> process ID
if sim_tree.run in runids :
continue
row = search_summary.RowType()
row.process_id = process_id_type(sim_tree.run)
runids.add(sim_tree.run)
# Search Summary Table
# events found in the run -> nevents
setattr(row, "nevents", sim_tree.GetEntries())
# Imstruments involved in the search
row.ifos = lsctables.ifos_from_instrument_set( get_ifos_from_index( branch_array_to_list ( sim_tree.ifo, sim_tree.ndim ) ) )
setattr(row, "comment", "waveburst")
# Begin and end time of the segment
# TODO: This is a typical offset on either side of the job for artifacts
# It can, and probably will change in the future, and should not be hardcoded
# TODO: Make this work properly. We need a gps end from the livetime
waveoffset = 8
livetime = 600
#live_entries = live_tree.GetEntries()
# This is WAAAAAAAAAAAAAY too slow
#for l in range(0, live_entries):
#liv_tree.GetEntry(l)
#livetime = max(livetime, liv_tree.live)
#if livetime < 0:
#sys.exit("Could not find livetime, cannot fill all of summary table.")
# in -- with waveoffset
# out -- without waveoffset
row.set_in(segments.segment(LIGOTimeGPS(sim_tree.gps - waveoffset), LIGOTimeGPS(sim_tree.gps + livetime + waveoffset)))
row.set_out(segments.segment(LIGOTimeGPS(sim_tree.gps), LIGOTimeGPS(sim_tree.gps + livetime)))
search_summary.append(row)
def test_optimized_query(engine):
res = segmentdb_utils.query_segments(
engine, 'segment_summary',
[('H1', 'DMT-TESTSEG_2', 1, 924900000, 924900016, 0, 0),
('H1', 'DMT-TESTSEG_3', 1, 924900000, 924900016, 0, 0)])
if res[0] != segmentlist([segment(924900000, 924900010)]):
return False
if res[1] != segmentlist([segment(924900008, 924900016)]):
return False
return True
def get_manually(gps_start_time, gps_end_time):
db_location = os.environ["S6_SEGMENT_SERVER"]
segment_connection = segmentdb_utils.setup_database(db_location)
engine = query_engine.LdbdQueryEngine(segment_connection)
# 1. Get v1 science segments
sql = "SELECT segment.start_time, segment.end_time "
sql += "FROM segment_definer, segment "
sql += "WHERE segment.segment_def_id = segment_definer.segment_def_id "
sql += "AND segment_definer.ifos = 'H1' "
sql += "AND segment.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.name = 'DMT-SCIENCE' "
sql += "AND segment_definer.version = 1 "
sql += "AND NOT (%s > segment.end_time OR segment.start_time > %s)" % (gps_start_time, gps_end_time)
v1_science_segments = segmentlist([segment(row[0], row[1]) for row in engine.query(sql)]).coalesce()
# 2. Get v2 science summaries
sql = "SELECT segment_summary.start_time, segment_summary.end_time "
sql += "FROM segment_definer, segment_summary "
sql += "WHERE segment_summary.segment_def_id = segment_definer.segment_def_id "
sql += "AND segment_definer.ifos = 'H1' "
sql += "AND segment_summary.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.name = 'DMT-SCIENCE' "
sql += "AND segment_definer.version = 2 "
sql += "AND NOT (%s > segment_summary.end_time OR segment_summary.start_time > %s)" % (gps_start_time, gps_end_time)
v2_science_summaries = segmentlist([segment(row[0], row[1]) for row in engine.query(sql)]).coalesce()
# 1. Get v2 science segments
sql = "SELECT segment.start_time, segment.end_time "
sql += "FROM segment_definer, segment "
sql += "WHERE segment.segment_def_id = segment_definer.segment_def_id "
sql += "AND segment_definer.ifos = 'H1' "
sql += "AND segment.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.name = 'DMT-SCIENCE' "
sql += "AND segment_definer.version = 2 "
sql += "AND NOT (%s > segment.end_time OR segment.start_time > %s)" % (gps_start_time, gps_end_time)
v2_science_segments = segmentlist([segment(row[0], row[1]) for row in engine.query(sql)]).coalesce()
result = (v1_science_segments - v2_science_summaries) + v2_science_segments
result.coalesce()
result &= segmentlist([segment(gps_start_time, gps_end_time)])
return result
def build_segment_list_one(
engine, gps_start_time, gps_end_time, ifo, segment_name, version=None, start_pad=0, end_pad=0
):
"""Builds a list of segments satisfying the given criteria """
seg_result = segmentlist([])
sum_result = segmentlist([])
# Is there any way to get segment and segement summary in one query?
# Maybe some sort of outer join where we keep track of which segment
# summaries we've already seen.
sql = "SELECT segment_summary.start_time, segment_summary.end_time "
sql += "FROM segment_definer, segment_summary "
sql += "WHERE segment_summary.segment_def_id = segment_definer.segment_def_id "
sql += "AND segment_definer.ifos = '%s' " % ifo
if engine.__class__ == query_engine.LdbdQueryEngine:
sql += "AND segment_summary.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.name = '%s' " % segment_name
sql += "AND segment_definer.version = %s " % version
sql += "AND NOT (%s > segment_summary.end_time OR segment_summary.start_time > %s)" % (gps_start_time, gps_end_time)
rows = engine.query(sql)
for sum_start_time, sum_end_time in rows:
sum_start_time = (sum_start_time < gps_start_time) and gps_start_time or sum_start_time
sum_end_time = (sum_end_time > gps_end_time) and gps_end_time or sum_end_time
sum_result |= segmentlist([segment(sum_start_time, sum_end_time)])
# We can't use queries paramaterized with ? since the ldbd protocol doesn't support it...
sql = "SELECT segment.start_time + %d, segment.end_time + %d " % (start_pad, end_pad)
sql += "FROM segment, segment_definer "
sql += "WHERE segment.segment_def_id = segment_definer.segment_def_id "
if engine.__class__ == query_engine.LdbdQueryEngine:
sql += "AND segment.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.ifos = '%s' " % ifo
sql += "AND segment_definer.name = '%s' " % segment_name
sql += "AND segment_definer.version = %s " % version
sql += "AND NOT (%s > segment.end_time OR segment.start_time > %s)" % (gps_start_time, gps_end_time)
rows = engine.query(sql)
for seg_start_time, seg_end_time in rows:
seg_start_time = (seg_start_time < gps_start_time) and gps_start_time or seg_start_time
seg_end_time = (seg_end_time > gps_end_time) and gps_end_time or seg_end_time
seg_result |= segmentlist([segment(seg_start_time, seg_end_time)])
engine.close()
return sum_result, seg_result
def new_plots(instrument, amplitude_func, amplitude_lbl, plots): l = ( FreqVsTime(instrument), HrssVsFreqScatter(instrument, amplitude_func, amplitude_lbl), SimBurstUtils.Efficiency_hrss_vs_freq((instrument,), amplitude_func, amplitude_lbl, 0.1), TriggerCountHistogram(instrument), RecoveredVsInjectedhrss(instrument, amplitude_func, amplitude_lbl), RecoveredPerInjectedhrssVsFreq(instrument, amplitude_func, amplitude_lbl), RecoveredPerInjectedhrssVsBandwidth(instrument, amplitude_func, amplitude_lbl), RecoveredTimeOffset(instrument, segments.segment(-0.03, +0.03), 0.00015), RecoveredFrequencyOffset(instrument, segments.segment(-1.0, +1.0), .002), RecoveredVsInjectedFreq(instrument, amplitude_func) ) return [l[i] for i in plots]
def new_plots(instrument, amplitude_func, amplitude_lbl, plots): l = ( FreqVsTime(instrument), HrssVsFreqScatter(instrument, amplitude_func, amplitude_lbl), SimBurstUtils.Efficiency_hrss_vs_freq((instrument,), amplitude_func, amplitude_lbl, 0.1), TriggerCountHistogram(instrument), RecoveredVsInjectedhrss(instrument, amplitude_func, amplitude_lbl), RecoveredPerInjectedhrssVsFreq(instrument, amplitude_func, amplitude_lbl), RecoveredPerInjectedhrssVsBandwidth(instrument, amplitude_func, amplitude_lbl), RecoveredTimeOffset(instrument, segments.segment(-0.03, +0.03), 0.00015), RecoveredFrequencyOffset(instrument, segments.segment(-1.0, +1.0), .002), RecoveredVsInjectedFreq(instrument, amplitude_func) ) return [l[i] for i in plots]
def test_fromsegwizard(self):
"""
Test segwizard parsing.
"""
data = StringIO.StringIO("""# This is a comment
# This is another comment
# Again a comment
1 10 100 90
2 110 120 10# Here's a comment
3 125 130 5 # Another one
4 0 200 200""")
correct = segments.segmentlist([segments.segment(10, 100), segments.segment(110, 120), segments.segment(125, 130), segments.segment(0, 200)])
self.assertEqual(correct, segmentsUtils.fromsegwizard(data, strict=True))
def build_segment_list_one(engine, gps_start_time, gps_end_time, ifo, segment_name, version = None, start_pad = 0, end_pad = 0):
"""Builds a list of segments satisfying the given criteria """
seg_result = segmentlist([])
sum_result = segmentlist([])
# Is there any way to get segment and segement summary in one query?
# Maybe some sort of outer join where we keep track of which segment
# summaries we've already seen.
sql = "SELECT segment_summary.start_time, segment_summary.end_time "
sql += "FROM segment_definer, segment_summary "
sql += "WHERE segment_summary.segment_def_id = segment_definer.segment_def_id "
sql += "AND segment_definer.ifos = '%s' " % ifo
if engine.__class__ == query_engine.LdbdQueryEngine:
sql += "AND segment_summary.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.name = '%s' " % segment_name
sql += "AND segment_definer.version = %s " % version
sql += "AND NOT (%s > segment_summary.end_time OR segment_summary.start_time > %s)" % (gps_start_time, gps_end_time)
rows = engine.query(sql)
for sum_start_time, sum_end_time in rows:
sum_start_time = (sum_start_time < gps_start_time) and gps_start_time or sum_start_time
sum_end_time = (sum_end_time > gps_end_time) and gps_end_time or sum_end_time
sum_result |= segmentlist([segment(sum_start_time, sum_end_time)])
# We can't use queries paramaterized with ? since the ldbd protocol doesn't support it...
sql = "SELECT segment.start_time + %d, segment.end_time + %d " % (start_pad, end_pad)
sql += "FROM segment, segment_definer "
sql += "WHERE segment.segment_def_id = segment_definer.segment_def_id "
if engine.__class__ == query_engine.LdbdQueryEngine:
sql += "AND segment.segment_def_cdb = segment_definer.creator_db "
sql += "AND segment_definer.ifos = '%s' " % ifo
sql += "AND segment_definer.name = '%s' " % segment_name
sql += "AND segment_definer.version = %s " % version
sql += "AND NOT (%s > segment.end_time OR segment.start_time > %s)" % (gps_start_time, gps_end_time)
rows = engine.query(sql)
for seg_start_time, seg_end_time in rows:
seg_start_time = (seg_start_time < gps_start_time) and gps_start_time or seg_start_time
seg_end_time = (seg_end_time > gps_end_time) and gps_end_time or seg_end_time
seg_result |= segmentlist([segment(seg_start_time, seg_end_time)])
engine.close()
return sum_result, seg_result
def __init__(self):
# set defaults
now = XLALUTCToGPS(time.gmtime())
self.segment = segments.segment(now, now + (-1 * 3600))
self.ratewidth = 60.0
self.freqwidth = 16.0
self.band = segments.segment(0.0, 2500.0)
self.set_instrument("H1")
self.seglist = segments.segmentlist([self.segment])
self.filename = None
self.format = None # force update
self.set_format("png")
self.cluster = 0
return self
def __init__(self):
# set defaults
now = XLALUTCToGPS(time.gmtime())
self.segment = segments.segment(now, now + (-1 * 3600))
self.ratewidth = 60.0
self.freqwidth = 16.0
self.band = segments.segment(0.0, 2500.0)
self.set_instrument("H1")
self.seglist = segments.segmentlist([self.segment])
self.filename = None
self.format = None # force update
self.set_format("png")
self.cluster = 0
return self
def test_basic_seg_summary(engine):
res = segmentdb_utils.query_segments(
engine, 'segment_summary',
[('H1', 'DMT-TESTSEG_1', 1, 924900000, 924900016, 0, 0)])
if res != [segmentlist([segment(924900000, 924900016)])]:
return False
res = segmentdb_utils.query_segments(
engine, 'segment_summary',
[('H1', 'DMT-TESTSEG_1', 2, 924900000, 924900016, 0, 0)])
if res != [segmentlist([segment(924900008, 924900010)])]:
return False
return True
def convert_json_list_to_segmentlist(jsonlist):
"""
Helper function used to convert json list of lists type object to a
segmentlist object
"""
segment_list=segments.segmentlist([segments.segment(x[0],x[1]) for x in jsonlist])
return segment_list
def get_slide_coincs_from_cache(cachefile, pattern, match, verb, coinc_stat):
full_coinc_table = []
cache = cachefile.sieve(description=pattern, exact_match=match)
found, missed = cache.checkfilesexist()
files = found.pfnlist()
if not len(files):
print >>sys.stderr, "cache contains no files with " + pattern + " description"
return None
# split the time slide files into 105 groups to aid with I/O
num_files=len(files)
#Changed by Tristan Miller as a memory fix
#groups_of_files = split_seq(files,105)
groups_of_files = split_seq(files,50)
for filegroup in groups_of_files:
if filegroup:
# extract the coinc table
coinc_table = SnglInspiralUtils.ReadSnglInspiralFromFiles(filegroup, mangle_event_id=False, verbose=verb, non_lsc_tables_ok=False)
segDict = SearchSummaryUtils.GetSegListFromSearchSummaries(filegroup)
rings = segments.segmentlist(iterutils.flatten(segDict.values()))
rings.sort()
for k,ring in enumerate(rings):
rings[k] = segments.segment(rings[k][0], rings[k][1] + 10**(-9))
shift_vector = {"H1": 0, "H2": 0, "L1": 5, "V1": 5}
if coinc_table:
SnglInspiralUtils.slideTriggersOnRingWithVector(coinc_table, shift_vector, rings)
full_coinc_table.extend(CoincInspiralUtils.coincInspiralTable(coinc_table,coinc_stat))
return full_coinc_table
def _query(self, channel, start, end):
"Do we know where the frame file is?"
if segment(start, end) in self._remotecoverage:
return True
urls = query_LDR(self.host,
self.port,
channel[0],
self.frametype,
start,
end,
urlType="file")
if urls:
new = Cache.from_urls(urls, coltype=int)
new.sort(key=operator.attrgetter("segment"))
self.add_cache(new)
return segment(start, end) in self._remotecoverage
def fromfilenames(filenames, coltype=int):
"""
Return a segmentlist describing the intervals spanned by the files
whose names are given in the list filenames. The segmentlist is
constructed by parsing the file names, and the boundaries of each
segment are coerced to type coltype.
The file names are parsed using a generalization of the format
described in Technical Note LIGO-T010150-00-E, which allows the
start time and duration appearing in the file name to be
non-integers.
NOTE: the output is a segmentlist as described by the file names;
if the file names are not in time order, or describe overlaping
segments, then thusly shall be the output of this function. It is
recommended that this function's output be coalesced before use.
"""
pattern = re.compile(r"-([\d.]+)-([\d.]+)\.[\w_+#]+\Z")
l = segments.segmentlist()
for name in filenames:
[(s, d)] = pattern.findall(name.strip().rstrip(".gz"))
s = coltype(s)
d = coltype(d)
l.append(segments.segment(s, s + d))
return l
def read_segfile_xml(segfile,verbose):
"""
Read segment file in ligolw xml type and return in glue.segments.segmentlist
format.
"""
from glue.ligolw import ligolw
from glue.ligolw import table
from glue.ligolw import utils
def ContentHandler(xmldoc):
return ligolw.PartialLIGOLWContentHandler(xmldoc, lambda name, attrs:\
(name == ligolw.Table.tagName) and\
(table.StripTableName(attrs["Name"]) in ["segment"]))
try:
table.use_in(ligolw.PartialLIGOLWContentHandler)
except AttributeError:
# old glue did not allow .use_in().
# FIXME: remove when we can require the latest version of glue
pass
xmldoc = utils.load_url(segfile, verbose = verbose,gz = segfile.endswith(".gz"), contenthandler = ContentHandler)
seg_list = segmentlist()
for table_elem in xmldoc.getElements(lambda e:\
(e.tagName == ligolw.Table.tagName)):
for row in table_elem:
seg_list.append(segment(row.start_time, row.end_time))
xmldoc.unlink()
return seg_list
def find_segments(doc, key, use_segment_table = True):
key_pieces = key.split(':')
while len(key_pieces) < 3:
key_pieces.append('*')
filter_func = lambda x: str(x.ifos) == key_pieces[0] and (str(x.name) == key_pieces[1] or key_pieces[1] == '*') and (str(x.version) == key_pieces[2] or key_pieces[2] == '*')
# Find all segment definers matching the critieria
seg_def_table = lsctables.SegmentDefTable.get_table(doc)
seg_defs = filter(filter_func, seg_def_table)
seg_def_ids = map(lambda x: str(x.segment_def_id), seg_defs)
# Find all segments belonging to those definers
if use_segment_table:
seg_table = lsctables.SegmentTable.get_table(doc)
seg_entries = filter(lambda x: str(x.segment_def_id) in seg_def_ids, seg_table)
else:
seg_sum_table = lsctables.SegmentSumTable.get_table(doc)
seg_entries = filter(lambda x: str(x.segment_def_id) in seg_def_ids, seg_sum_table)
# Combine into a segmentlist
ret = segmentlist(map(lambda x: segment(x.start_time, x.end_time), seg_entries))
ret.coalesce()
return ret
def fetch(self, channel, start, end):
"""
Retrieve data, caching file locations and the files themselves.
"""
seg = segment(start, end)
if not self._query(channel, start, end):
raise ValueError("%s not found in cache" %
repr(segmentlist([seg]) - self._remotecoverage))
# Need to cache files locally
# Note: seg *will* be in self._cachecoverage if self.scratchdir is None.
if seg not in self._cachecoverage:
for f, s in zip(self._remotefiles, self._remotesegs):
if seg.intersects(s) and s not in self._cachecoverage:
dest = os.path.join(self._scratchdir, os.path.split(f)[-1])
if self._verbose:
print "Copying %s -->\n %s." % (f, dest)
shutil.copy(f, dest)
ind = bisect_right(self._cachedsegs, s)
self._cachedfiles.insert(ind, dest)
self._cachedsegs.insert(ind, s)
self._cachecoverage |= segmentlist([s])
assert seg in self._cachecoverage
# Finally, return the cached data
return self._fetch(channel, start, end)
def from_T050017(cls, url, coltype = LIGOTimeGPS):
"""
Parse a URL in the style of T050017-00 into a CacheEntry. The
T050017-00 file name format is, essentially,
observatory-description-start-duration.extension
Example:
>>> c = CacheEntry.from_T050017("file://localhost/data/node144/frames/S5/strain-L2/LLO/L-L1_RDS_C03_L2-8365/L-L1_RDS_C03_L2-836562330-83.gwf")
>>> c.observatory
'L'
>>> c.host
'localhost'
>>> os.path.basename(c.path)
'L-L1_RDS_C03_L2-836562330-83.gwf'
"""
match = cls._url_regex.search(url)
if not match:
raise ValueError("could not convert %s to CacheEntry" % repr(url))
observatory = match.group("obs")
description = match.group("dsc")
# FIXME: remove typecasts when LIGOTimeGPS can be passed a unicode
start = str(match.group("strt"))
duration = str(match.group("dur"))
if start == "-" and duration == "-":
# no segment information
segment = None
else:
segment = segments.segment(coltype(start), coltype(start) + coltype(duration))
return cls(observatory, description, segment, url)
def get_output(self): if self._AnalysisNode__output is None: if None in (self.get_start(), self.get_end(), self.get_ifo(), self.__usertag): raise ValueError, "start time, end time, ifo, or user tag has not been set" seg = segments.segment(LIGOTimeGPS(self.get_start()), LIGOTimeGPS(self.get_end())) self.set_output(os.path.join(self.output_dir, "%s-POWER_%s-%d-%d.xml.gz" % (self.get_ifo(), self.__usertag, int(self.get_start()), int(self.get_end()) - int(self.get_start())))) return self._AnalysisNode__output
def exportGPSEventToDisk(tevent, dir, cnt, dag, filename=None):
"""
"""
#If directy coincHeadings not there make the directory if
#filename is None
headingDir = dir + "/coinc_info"
ifos = tevent.instruments
instruments = tevent.instruments
time = tevent.time
idKeyStr = "%s_%s" % (str(time), instruments)
if filename == None:
filename = "coincEvent.info"
stfu_pipe.mkdir(headingDir)
filename = os.path.normpath(headingDir + '/' + idKeyStr + '_' +
filename)
fp = open(filename, 'w')
fp.write("#DIR\t\t\tRANK\tFAR\t\tSNR\tIFOS\tINSTRUMENTS\tTIME\t\tMASS\n")
fp.write("%-16s\t%d\t%0.2e\t%.2f\t%s\t%s\t\t%.3f\t%.2f\n" %
(dir, cnt, 0, 0, ifos, instruments, float(time), 0))
fp.write("#DIR\t\t\tIFO\tTIME\t\tSNR\tCHISQ\tMASS1\tMASS2\n")
rowString = "%-16s\t%s\t%.3f\t%.2f\t%.2f\t%.2f\t%.2f\n"
content = list()
for ifo in tevent.ifos_list:
content.append(
rowString %
(dir, ifo, float(time), float(0), float(0), float(0), float(0)))
cache = lal.CacheEntry(instruments, "COINC_INFO_" + dir.upper(),
segments.segment(float(time), float(time)),
"file://localhost/" + os.path.abspath(filename))
dag.output_cache.append(cache)
fp.writelines(content)
fp.close()
return os.path.split(filename)[1]
def segmentlist_range(start, stop, period):
"""
Analogous to Python's range() builtin, this generator yields a
sequence of continuous adjacent segments each of length "period"
with the first starting at "start" and the last ending not after
"stop". Note that the segments generated do not form a coalesced
list (they are not disjoint). start, stop, and period can be any
objects which support basic arithmetic operations.
Example:
>>> from glue.segments import *
>>> segmentlist(segmentlist_range(0, 15, 5))
[segment(0, 5), segment(5, 10), segment(10, 15)]
>>> segmentlist(segmentlist_range('', 'xxx', 'x'))
[segment('', 'x'), segment('x', 'xx'), segment('xx', 'xxx')]
"""
n = 1
b = start
while True:
a, b = b, start + n * period
if b > stop:
break
yield segments.segment(a, b)
n += 1
def get_output(self): if self._AnalysisNode__output is None: if None in (self.get_start(), self.get_end(), self.get_ifo(), self.__usertag): raise ValueError, "start time, end time, ifo, or user tag has not been set" seg = segments.segment(lal.LIGOTimeGPS(self.get_start()), lal.LIGOTimeGPS(self.get_end())) self.set_output(os.path.join(self.output_dir, "%s-POWER_%s-%d-%d.xml.gz" % (self.get_ifo(), self.__usertag, int(self.get_start()), int(self.get_end()) - int(self.get_start())))) return self._AnalysisNode__output
def read_segfile_xml(segfile, verbose):
"""
Read segment file in ligolw xml type and return in glue.segments.segmentlist
format.
"""
from glue.ligolw import ligolw, utils, lsctables, table
lsctables.use_in(ligolw.LIGOLWContentHandler)
def ContentHandler(xmldoc):
return ligolw.LIGOLWContentHandler(xmldoc, lambda name, attrs:\
(name == ligolw.Table.tagName) and\
(table.StripTableName(attrs["Name"]) in ["segment"]))
utils.ContentHandler = ContentHandler
xmldoc = utils.load_url(segfile,
verbose=verbose,
gz=segfile.endswith(".gz"),
contenthandler=ligolw.LIGOLWContentHandler)
seg_list = segmentlist()
rows = table.get_table(xmldoc, lsctables.VetoDefTable.tableName)
for row in rows:
seg_list.append(segment(row.start_time, row.end_time))
xmldoc.unlink()
return seg_list
def test_tofromseqwizard(self):
"""
Check that the segwizard writing routine's output is parsed
correctly.
"""
data = StringIO.StringIO()
correct = segments.segmentlist([
segments.segment(10, 100),
segments.segment(110, 120),
segments.segment(125, 130),
segments.segment(0, 200)
])
segmentsUtils.tosegwizard(data, correct)
data.seek(0)
self.assertEqual(correct, segmentsUtils.fromsegwizard(data,
strict=True))
def tosegmentxml(file, segs):
"""
Write the glue.segments.segmentlist object segs to file object file in xml
format with appropriate tables.
"""
# generate empty document
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
xmldoc.childNodes[-1].appendChild(lsctables.New(lsctables.ProcessTable))
xmldoc.childNodes[-1].appendChild(lsctables.New(lsctables.ProcessParamsTable))
# append process to table
process = ligolw_process.append_process(xmldoc,\
program='pylal.dq.dqSegmentUtils',\
version=__version__,\
cvs_repository='lscsoft',\
cvs_entry_time=__date__)
gpssegs = segments.segmentlist()
for seg in segs:
gpssegs.append(segments.segment(LIGOTimeGPS(seg[0]), LIGOTimeGPS(seg[1])))
# append segs and seg definer
segments_tables = ligolw_segments.LigolwSegments(xmldoc)
segments_tables.add(ligolw_segments.LigolwSegmentList(active=gpssegs))
# finalise
segments_tables.coalesce()
segments_tables.optimize()
segments_tables.finalize(process)
ligolw_process.set_process_end_time(process)
# write file
utils.write_fileobj(xmldoc, file, gz=False)
def fetch(self, channel, start, end):
"""
Retrieve data, caching file locations and the files themselves.
"""
seg = segment(start, end)
if not self._query(channel, start, end):
raise ValueError("%s not found in cache" % repr(segmentlist([seg]) - self._remotecoverage))
# Need to cache files locally
# Note: seg *will* be in self._cachecoverage if self.scratchdir is None.
if seg not in self._cachecoverage:
for f,s in zip(self._remotefiles, self._remotesegs):
if seg.intersects(s) and s not in self._cachecoverage:
dest = os.path.join(self._scratchdir, os.path.split(f)[-1])
if self._verbose:
print "Copying %s -->\n %s." % (f, dest)
shutil.copy(f, dest)
ind = bisect_right(self._cachedsegs, s)
self._cachedfiles.insert(ind, dest)
self._cachedsegs.insert(ind, s)
self._cachecoverage |= segmentlist([s])
assert seg in self._cachecoverage
# Finally, return the cached data
return self._fetch(channel, start, end)
def matches(row):
return (
row[0].strip() == ifo
and row[1] == name
and int(row[2]) == int(version)
and search_span.intersects(segment(row[3] + start_pad, row[4] + start_pad))
)
def get_valid_times(self):
overlap = int(self.get_opt('segment-duration')) / 4
pad_data = int(self.get_opt('pad-data'))
valid_start = self.data_seg[0] + pad_data + overlap
valid_end = self.data_seg[1] - pad_data - overlap
return self.data_seg, segments.segment(valid_start, valid_end)
def from_bitstream(bitstream, start, dt, minlen=1):
"""
Convert consecutive True values in a bit stream (boolean-castable
iterable) to a stream of segments. Require minlen consecutive True
samples to comprise a segment.
Example:
>>> list(from_bitstream((True, True, False, True, False), 0, 1))
[segment(0, 2), segment(3, 4)]
>>> list(from_bitstream([[], [[]], [[]], [], []], 1013968613, 0.125))
[segment(1013968613.125, 1013968613.375)]
"""
bitstream = iter(bitstream)
i = 0
while 1:
if bitstream.next():
# found start of True block; find the end
j = i + 1
try:
while bitstream.next():
j += 1
finally: # make sure StopIteration doesn't kill final segment
if j - i >= minlen:
yield segments.segment(start + i * dt, start + j * dt)
i = j # advance to end of block
i += 1
def expand_version_number(engine, segdef):
ifo, name, version, start_time, end_time, start_pad, end_pad = segdef
if version != '*':
return [segdef]
# Start looking at the full interval
intervals = segmentlist([segment(start_time, end_time)])
# Find the maximum version number
sql = "SELECT max(version) FROM segment_definer "
sql += "WHERE segment_definer.ifos = '%s' " % ifo
sql += "AND segment_definer.name = '%s' " % name
rows = engine.query(sql)
try:
version = len(rows[0]) and rows[0][0] or 1
except:
version = None
results = []
while version > 0:
for interval in intervals:
segs = query_segments(engine, 'segment_summary', [(ifo, name, version, interval[0], interval[1], 0, 0)])
for seg in segs[0]:
results.append( (ifo, name, version, seg[0], seg[1], 0, 0) )
intervals.coalesce()
intervals -= segs[0]
version -= 1
return results
def get_valid_times(self):
overlap = int(self.get_opt('segment-duration')) / 4
pad_data = int(self.get_opt('pad-data'))
valid_start = self.data_seg[0] + pad_data + overlap
valid_end = self.data_seg[1] - pad_data - overlap
return self.data_seg, segments.segment(valid_start, valid_end)
def from_T050017(cls, url, coltype=LIGOTimeGPS):
"""
Parse a URL in the style of T050017-00 into a CacheEntry. The
T050017-00 file name format is, essentially,
observatory-description-start-duration.extension
Example:
>>> c = CacheEntry.from_T050017("file://localhost/data/node144/frames/S5/strain-L2/LLO/L-L1_RDS_C03_L2-8365/L-L1_RDS_C03_L2-836562330-83.gwf")
>>> c.observatory
'L'
>>> c.host
'localhost'
>>> os.path.basename(c.path)
'L-L1_RDS_C03_L2-836562330-83.gwf'
"""
match = cls._url_regex.search(url)
if not match:
raise ValueError("could not convert %s to CacheEntry" % repr(url))
observatory = match.group("obs")
description = match.group("dsc")
# FIXME: remove typecasts when LIGOTimeGPS can be passed a unicode
start = str(match.group("strt"))
duration = str(match.group("dur"))
if start == "-" and duration == "-":
# no segment information
segment = None
else:
segment = segments.segment(coltype(start),
coltype(start) + coltype(duration))
return cls(observatory, description, segment, url)
def new_plots(plots):
deltat_seg = segments.segment(-0.3, +0.3)
deltat_width = 0.03125
l = [
RateContours("H2", "H1"),
ConfidenceContours("H2", "H1", magnitude_a, "Confidence", 1, 10**10),
ConfidenceContours("H2", "L1", magnitude_a, "Confidence", 1, 10**10),
ConfidenceContours("L1", "H1", magnitude_a, "Confidence", 1, 10**10),
ConfidenceContours("H2", "H1", magnitude_b,
r"Power / D.o.F. / ($F_{+}^{2} + F_{\times}^{2}$)",
1, 10**10),
ConfidenceContours("H2", "L1", magnitude_b,
r"Power / D.o.F. / ($F_{+}^{2} + F_{\times}^{2}$)",
1, 10**10),
ConfidenceContours("L1", "H1", magnitude_b,
r"Power / D.o.F. / ($F_{+}^{2} + F_{\times}^{2}$)",
1, 10**10),
ConfidenceContourProjection(
numpy.array((-1 / math.sqrt(2), +1 / math.sqrt(2), 0), "Float64"),
numpy.array(
(-1 / math.sqrt(4), -1 / math.sqrt(4), +1 / math.sqrt(2)),
"Float64"), magnitude_b, 10**5),
RateVsConfidence("H1"),
RateVsConfidence("H2"),
RateVsConfidence("L1")
]
return [l[i] for i in plots]
