def vn_ldas_gap_check(vn_xml_file,vn_flag,ldas_gap_xml_file):
# read in summary intervals of vn_flag
vndoc = ligolw_utils.load_url(vn_xml_file, gz = (vn_xml_file or "stdin").endswith(".gz"))
vn_sums=segmentdb_utils.find_segments(vndoc,"%s" % vn_flag,use_segment_table=False)
# read in segment and summary intervals of LDAS allowed gaps
ldasdoc = ligolw_utils.load_url(ldas_gap_xml_file, gz = (ldas_gap_xml_file or "stdin").endswith(".gz"))
ldas_gaps=segmentdb_utils.find_segments(ldasdoc,'H1:RESULT:1')
ldas_sums=segmentdb_utils.find_segments(ldasdoc,'H1:DCH-MISSING_LDAS_C02_L2:1',use_segment_table=False)
if len(ldas_sums) == 0:
message = "no LDAS info was found within the specified time in the database!"
print(message)
return
# if valid Vn file and ldas files both found, do further testings:
if len(vn_sums) > 1 :
vn_sum_gaps=segmentlist([segment(vn_sums[0][0],vn_sums[-1][1])]) - vn_sums
if vn_sum_gaps == ldas_gaps:
message = ""
else:
message = "test failed. Found unexpected gaps in LDAS frames: %s \n" % vn_sum_gaps - ldas_gaps
print(message)
return
else:
if (len(vn_sums & ldas_gaps) == 0):
message = ""
else:
message = "test failed. Found overlap between %s summary intervals and LDAS allowed gaps: \n" % (vn_flag, vn_sums - ldas_gaps)
print(message)
return
print(message)
def run_segment_operation(outdoc, filenames, segments, use_segment_table, operation, result_name = 'RESULT', preserve = True):
"""
Performs an operation (intersect or union) across a set of segments.
That is, given a set of files each with segment definers DMT-FLAG1,
DMT-FLAG2 etc and a list of segments DMT-FLAG1,DMT-FLAG1 this returns
RESULT = (table 1's DMT-FLAG1 union table 2's DMT-FLAG1 union ...)
operation
(table 1's DMT-FLAG2 union table 2's DMT-FLAG2 union ...)
operation
etc
"""
proc_id = table.get_table(outdoc, lsctables.ProcessTable.tableName)[0].process_id
if preserve:
indoc = ligolw_add.ligolw_add(outdoc, filenames)
else:
indoc = ligolw_add.ligolw_add(ligolw.Document(), filenames)
# Start with a segment covering all of time, then
# intersect with each of the fields of interest
keys = segments.split(',')
if operation == INTERSECT:
sgmntlist = glue.segments.segmentlist([glue.segments.segment(-glue.segments.infinity(), glue.segments.infinity())])
for key in keys:
sgmntlist &= find_segments(indoc, key, use_segment_table)
elif operation == UNION:
sgmntlist = glue.segments.segmentlist([])
for key in keys:
sgmntlist |= find_segments(indoc, key, use_segment_table)
elif operation == DIFF:
sgmntlist = find_segments(indoc, keys[0], use_segment_table)
for key in keys[1:]:
sgmntlist -= find_segments(indoc, key, use_segment_table)
else:
raise NameError("%s is not a known operation (intersect, union or diff)" % operation)
# Add a segment definer and segments
seg_def_id = add_to_segment_definer(outdoc, proc_id, '', result_name, 1)
if use_segment_table:
add_to_segment(outdoc, proc_id, seg_def_id, sgmntlist)
else:
add_to_segment_summary(outdoc, proc_id, seg_def_id, sgmntlist)
return outdoc, abs(sgmntlist)
def run_segment_operation(outdoc, filenames, segments, use_segment_table, operation, result_name = 'RESULT', preserve = True):
"""
Performs an operation (intersect or union) across a set of segments.
That is, given a set of files each with segment definers DMT-FLAG1,
DMT-FLAG2 etc and a list of segments DMT-FLAG1,DMT-FLAG1 this returns
RESULT = (table 1's DMT-FLAG1 union table 2's DMT-FLAG1 union ...)
operation
(table 1's DMT-FLAG2 union table 2's DMT-FLAG2 union ...)
operation
etc
"""
proc_id = lsctables.ProcessTable.get_table(outdoc)[0].process_id
if preserve:
indoc = ligolw_add.ligolw_add(outdoc, filenames)
else:
indoc = ligolw_add.ligolw_add(ligolw.Document(), filenames)
# Start with a segment covering all of time, then
# intersect with each of the fields of interest
keys = segments.split(',')
if operation == INTERSECT:
sgmntlist = glue.segments.segmentlist([glue.segments.segment(-glue.segments.infinity(), glue.segments.infinity())])
for key in keys:
sgmntlist &= find_segments(indoc, key, use_segment_table)
elif operation == UNION:
sgmntlist = glue.segments.segmentlist([])
for key in keys:
sgmntlist |= find_segments(indoc, key, use_segment_table)
elif operation == DIFF:
sgmntlist = find_segments(indoc, keys[0], use_segment_table)
for key in keys[1:]:
sgmntlist -= find_segments(indoc, key, use_segment_table)
else:
raise NameError("%s is not a known operation (intersect, union or diff)" % operation)
# Add a segment definer and segments
seg_def_id = add_to_segment_definer(outdoc, proc_id, '', result_name, 1)
if use_segment_table:
add_to_segment(outdoc, proc_id, seg_def_id, sgmntlist)
else:
add_to_segment_summary(outdoc, proc_id, seg_def_id, sgmntlist)
return outdoc, abs(sgmntlist)
def vn_ldas_gap_check(vn_xml_file, vn_flag, ldas_gap_xml_file):
# read in summary intervals of vn_flag
vndoc = ligolw_utils.load_url(vn_xml_file,
gz=(vn_xml_file or "stdin").endswith(".gz"))
vn_sums = segmentdb_utils.find_segments(vndoc,
"%s" % vn_flag,
use_segment_table=False)
# read in segment and summary intervals of LDAS allowed gaps
ldasdoc = ligolw_utils.load_url(ldas_gap_xml_file,
gz=(ldas_gap_xml_file
or "stdin").endswith(".gz"))
ldas_gaps = segmentdb_utils.find_segments(ldasdoc, 'H1:RESULT:1')
ldas_sums = segmentdb_utils.find_segments(ldasdoc,
'H1:DCH-MISSING_LDAS_C02_L2:1',
use_segment_table=False)
if len(ldas_sums) == 0:
message = "no LDAS info was found within the specified time in the database!"
print(message)
return
# if valid Vn file and ldas files both found, do further testings:
if len(vn_sums) > 1:
vn_sum_gaps = segmentlist([segment(vn_sums[0][0], vn_sums[-1][1])
]) - vn_sums
if vn_sum_gaps == ldas_gaps:
message = ""
else:
message = "test failed. Found unexpected gaps in LDAS frames: %s \n" % vn_sum_gaps - ldas_gaps
print(message)
return
else:
if (len(vn_sums & ldas_gaps) == 0):
message = ""
else:
message = "test failed. Found overlap between %s summary intervals and LDAS allowed gaps: \n" % (
vn_flag, vn_sums - ldas_gaps)
print(message)
return
print(message)
def run_file_operation(outdoc,
filenames,
use_segment_table,
operation,
preserve=True):
"""
Performs an operation (intersect or union) across a set of files.
That is, given a set of files each with segment definers DMT-FLAG1,
DMT-FLAG2 etc the result is a file where
DMT-FLAG1 = (file 1's DMT-FLAG1 operation file 2's DMT-FLAG1 operation ...)
DMT-FLAG2 = (file 1's DMT-FLAG2 operation file 2's DMT-FLAG2 operation ...)
etc
"""
proc_id = table.get_table(outdoc,
lsctables.ProcessTable.tableName)[0].process_id
# load up the files into individual documents
xmldocs = [
ligolw_add.ligolw_add(ligolw.Document(), [fname])
for fname in filenames
]
# Get the list of dinstinct segment_definers across all docs
segment_definers = {}
def register_definer(seg_def):
key = (seg_def.ifos, seg_def.name, seg_def.version)
segment_definers[key] = True
return key
for xmldoc in xmldocs:
seg_def_table = table.get_table(xmldoc,
lsctables.SegmentDefTable.tableName)
map(register_definer, seg_def_table)
# For each unique segment definer, find the intersection
for ifo, name, version in segment_definers:
if operation == INTERSECT:
# If I were feeling especially functional-ist I'd write this
# with reduce()
result = glue.segments.segmentlist([
glue.segments.segment(-glue.segments.infinity(),
glue.segments.infinity())
])
for xmldoc in xmldocs:
result &= find_segments(xmldoc,
'%s:%s:%d' % (ifo, name, version),
use_segment_table)
elif operation == UNION:
result = glue.segments.segmentlist([])
for xmldoc in xmldocs:
result |= find_segments(xmldoc,
'%s:%s:%d' % (ifo, name, version),
use_segment_table)
elif operation == DIFF:
result = find_segments(xmldocs[0],
'%s:%s:%d' % (ifo, name, version),
use_segment_table)
for xmldoc in xmldocs[1:]:
result -= find_segments(xmldoc,
'%s:%s:%d' % (ifo, name, version),
use_segment_table)
else:
raise NameError(
"%s is not a known operation (intersect, union or diff)" %
operation)
# Add a segment definer for the result
seg_def_id = add_to_segment_definer(outdoc, proc_id, ifo, name,
version)
# Add the segments
if use_segment_table:
add_to_segment(outdoc, proc_id, seg_def_id, result)
else:
add_to_segment_summary(outdoc, proc_id, seg_def_id, result)
# If we're preserving, also load up everything into the output document.
if preserve:
# Add them to the output document
map(lambda x: outdoc.appendChild(x.childNodes[0]), xmldocs)
# Merge the ligolw elements and tables
ligolw_add.merge_ligolws(outdoc)
ligolw_add.merge_compatible_tables(outdoc)
return outdoc, abs(result)
def run_file_operation(outdoc, filenames, use_segment_table, operation, preserve = True):
"""
Performs an operation (intersect or union) across a set of files.
That is, given a set of files each with segment definers DMT-FLAG1,
DMT-FLAG2 etc the result is a file where
DMT-FLAG1 = (file 1's DMT-FLAG1 operation file 2's DMT-FLAG1 operation ...)
DMT-FLAG2 = (file 1's DMT-FLAG2 operation file 2's DMT-FLAG2 operation ...)
etc
"""
proc_id = table.get_table(outdoc, lsctables.ProcessTable.tableName)[0].process_id
# load up the files into individual documents
xmldocs = [ligolw_add.ligolw_add(ligolw.Document(), [fname]) for fname in filenames]
# Get the list of dinstinct segment_definers across all docs
segment_definers = {}
def register_definer(seg_def):
key = (seg_def.ifos, seg_def.name, seg_def.version)
segment_definers[key] = True
return key
for xmldoc in xmldocs:
seg_def_table = table.get_table(xmldoc, lsctables.SegmentDefTable.tableName)
map (register_definer, seg_def_table)
# For each unique segment definer, find the intersection
for ifo, name, version in segment_definers:
if operation == INTERSECT:
# If I were feeling especially functional-ist I'd write this
# with reduce()
result = glue.segments.segmentlist([glue.segments.segment(-glue.segments.infinity(), glue.segments.infinity())])
for xmldoc in xmldocs:
result &= find_segments(xmldoc, '%s:%s:%d' % (ifo, name, version), use_segment_table)
elif operation == UNION:
result = glue.segments.segmentlist([])
for xmldoc in xmldocs:
result |= find_segments(xmldoc, '%s:%s:%d' % (ifo, name, version), use_segment_table)
elif operation == DIFF:
result = find_segments(xmldocs[0], '%s:%s:%d' % (ifo, name, version), use_segment_table)
for xmldoc in xmldocs[1:]:
result -= find_segments(xmldoc, '%s:%s:%d' % (ifo, name, version), use_segment_table)
else:
raise NameError ("%s is not a known operation (intersect, union or diff)" % operation)
# Add a segment definer for the result
seg_def_id = add_to_segment_definer(outdoc, proc_id, ifo, name, version)
# Add the segments
if use_segment_table:
add_to_segment(outdoc, proc_id, seg_def_id, result)
else:
add_to_segment_summary(outdoc, proc_id, seg_def_id, result)
# If we're preserving, also load up everything into the output document.
if preserve:
# Add them to the output document
map(lambda x: outdoc.appendChild(x.childNodes[0]), xmldocs)
# Merge the ligolw elements and tables
ligolw_add.merge_ligolws(outdoc)
ligolw_add.merge_compatible_tables(outdoc)
return outdoc, abs(result)
