def round_robin(round_robin_cache_nodes, round_robin_cache): parents = list( iterutils.choices(round_robin_cache_nodes, len(round_robin_cache_nodes) - 1)) parents.reverse() parents = [reduce(lambda a, b: a | b, seq) for seq in parents] return zip( parents, [cache for (cache, ) in iterutils.choices(round_robin_cache, 1)])
def round_robin_and_flatten(injection_coinc_node_groups, injection_likelihood_node_groups): # see the documentation for glue.iterutils.choices() for an # explanation of the procedure used here to round-robin the node # lists A = list(iterutils.choices(injection_coinc_node_groups, 1)) B = list(iterutils.choices(injection_likelihood_node_groups, len(injection_likelihood_node_groups) - 1)) B.reverse() A = [flatten_node_groups(node_groups) for node_groups in A] B = [flatten_node_groups(node_groups) for node_groups in B] return zip(A, B)
def round_robin_and_flatten(injection_coinc_node_groups, injection_likelihood_node_groups): # see the documentation for glue.iterutils.choices() for an # explanation of the procedure used here to round-robin the node # lists A = list(iterutils.choices(injection_coinc_node_groups, 1)) B = list( iterutils.choices(injection_likelihood_node_groups, len(injection_likelihood_node_groups) - 1)) B.reverse() A = [flatten_node_groups(node_groups) for node_groups in A] B = [flatten_node_groups(node_groups) for node_groups in B] return zip(A, B)
def detector_combos( instruments ): out = [] instruments = tuple(instruments) for i in range(1, len(instruments)): X = list(iterutils.choices(instruments, i)) Y = list(iterutils.choices(instruments, len(instruments) - i)) Y.reverse() out.extend(zip(X,Y)) #out.extend(zip(X, Y)) instruments = list(instruments) instruments.sort() instruments = tuple(instruments) out.append((instruments, ())) return out
def triangulators(timing_uncertainties): """ Return a dictionary of snglcoinc.TOATriangulator objects initialized for a variety of instrument combinations. timing_uncertainties is a dictionary of instrument->\Delta t pairs. The return value is a dictionary of (instrument tuple)->TOATrangulator mappings. The instrument names in each tuple are sorted in alphabetical order, and the triangulators are constructed with the instruments in that order (the the documentation for snglcoinc.TOATriangulator for more information). Example: >>> x = triangulators({"H1": 0.005, "L1": 0.005, "V1": 0.005}) constructs a dictionary of triangulators for every combination of two or more instruments that can be constructed from those three. The program lalapps_string_plot_binj can be used to measure the timing uncertainties for the instruments in a search. """ allinstruments = sorted(timing_uncertainties.keys()) triangulators = {} for n in range(2, len(allinstruments) + 1): for instruments in iterutils.choices(allinstruments, n): triangulators[instruments] = snglcoinc.TOATriangulator([ lal.cached_detector_by_prefix[instrument].location for instrument in instruments ], [ timing_uncertainties[instrument] for instrument in instruments ]) return triangulators
def replicate_threshold(threshold, instruments): """ From a single threshold and a list of instruments, return a dictionary whose keys are every instrument pair (both orders), and whose values are all the same single threshold. Example: >>> replicate_threshold(6, ["H1", "H2"]) {("H1", "H2"): 6, ("H2", "H1"): 6} """ instruments = sorted(instruments) thresholds = dict((pair, threshold) for pair in iterutils.choices(instruments, 2)) instruments.reverse() thresholds.update(dict((pair,threshold) for pair in iterutils.choices(instruments, 2))) return thresholds
def replicate_threshold(e_thinca_parameter, instruments): """ From a single threshold and a list of instruments, return a dictionary whose keys are every instrument pair (both orders), and whose values are all the same single threshold. Example: >>> replicate_threshold(6, ["H1", "H2"]) {("H1", "H2"): 6, ("H2", "H1"): 6} """ instruments = sorted(instruments) thresholds = dict((pair, e_thinca_parameter) for pair in iterutils.choices(instruments, 2)) instruments.reverse() thresholds.update(dict((pair, e_thinca_parameter) for pair in iterutils.choices(instruments, 2))) return thresholds
def triangulators(timing_uncertainties): """ Return a dictionary of snglcoinc.TOATriangulator objects initialized for a variety of instrument combinations. timing_uncertainties is a dictionary of instrument->\Delta t pairs. The return value is a dictionary of (instrument tuple)->TOATrangulator mappings. The instrument names in each tuple are sorted in alphabetical order, and the triangulators are constructed with the instruments in that order (the the documentation for snglcoinc.TOATriangulator for more information). Example: >>> x = triangulators({"H1": 0.005, "L1": 0.005, "V1": 0.005}) constructs a dictionary of triangulators for every combination of two or more instruments that can be constructed from those three. The program lalapps_string_plot_binj can be used to measure the timing uncertainties for the instruments in a search. """ allinstruments = sorted(timing_uncertainties.keys()) triangulators = {} for n in range(2, len(allinstruments) + 1): for instruments in iterutils.choices(allinstruments, n): triangulators[instruments] = snglcoinc.TOATriangulator([inject.cached_detector[inject.prefix_to_name[instrument]].location for instrument in instruments], [timing_uncertainties[instrument] for instrument in instruments]) return triangulators
def populate_experiment_table(xmldoc, search_group, trigger_program, lars_id, instruments, comments=None, add_inst_subsets=False, verbose=False): """ Populate the experiment table using the given entries. If add_inst_subsets is set to True, will write additional entries for every possible sub-combination of the given instrument set. Returns a dictionary of experiment_ids keyed by the instrument set. @xmldoc: xmldoc to get/write table to @lars_id: lars_id of the experiment @search_group: lsc group that performed the experiment (e.g., cbc) @trigger_program: name of the program that performed the analysis (e.g., inspiral, ringdown, etc.) @comments: any desired comments @add_inst_subsets: will write an entry for every possible subset of @instruments @verbose: be verbose """ if verbose: print >> sys.stderr, "\tPopulating the Experiment table..." # find the experiment table or create one if needed try: expr_table = lsctables.ExperimentTable.get_table(xmldoc) except ValueError: expr_table = xmldoc.childNodes[0].appendChild( lsctables.New(lsctables.ExperimentTable)) # determine experiment start and end times expr_start_time, expr_end_time = get_experiment_times(xmldoc) # write entry to the experiment table for the given instruments if it doesn't already exist experiment_ids = {} for nn in range(len(instruments), 1, -1): if not add_inst_subsets and (nn != len(instruments)): break # add every possible sub-combination of the instrument set if # they're not already in the table for sub_combo in iterutils.choices(list(instruments), nn): if frozenset(sub_combo) not in experiment_ids: experiment_ids[frozenset( sub_combo)] = expr_table.write_new_expr_id( search_group, trigger_program, lars_id, sub_combo, expr_start_time, expr_end_time, comments=comments) return experiment_ids
def optimize(self): """ Identifies segment lists that differ only in their instruments --- they have the same valid and active segments, the same name, version and the same comment --- and then deletes all but one of them, leaving just a single list having the union of the instruments. """ self.sort() segment_lists = dict(enumerate(self)) for target, source in [ (idx_a, idx_b) for (idx_a, seglist_a), ( idx_b, seglist_b) in iterutils.choices(segment_lists.items(), 2) if seglist_a.valid == seglist_b.valid and seglist_a.active == seglist_b.active and seglist_a.name == seglist_b.name and seglist_a.version == seglist_b.version and seglist_a.comment == seglist_b.comment ]: try: source = segment_lists.pop(source) except KeyError: continue segment_lists[target].instruments |= source.instruments self.clear() self.update(segment_lists.values())
def component_offsetvectors(offsetvectors, n): """ Given an iterable of offset vectors, return the shortest list of the unique n-instrument offset vectors from which all the vectors in the input iterable can be constructed. This can be used to determine the minimal set of n-instrument coincs required to construct all of the coincs for all of the requested instrument and offset combinations in a set of offset vectors. It is assumed that the coincs for the vector {"H1": 0, "H2": 10, "L1": 20} can be constructed from the coincs for the vectors {"H1": 0, "H2": 10} and {"H2": 0, "L1": 10}, that is only the relative offsets are significant in determining if two events are coincident, not the absolute offsets. """ # # collect unique instrument set / deltas combinations # delta_sets = {} for vect in offsetvectors: for instruments in iterutils.choices(sorted(vect), n): # NOTE: the arithmetic used to construct the # offsets *must* match the arithmetic used by # offsetvector.deltas so that the results of the # two can be compared to each other without worry # of floating-point round off confusing things. delta_sets.setdefault(instruments, set()).add(tuple(vect[instrument] - vect[instruments[0]] for instrument in instruments)) # # translate into a list of normalized n-instrument offset vectors # return [offsetvector(zip(instruments, deltas)) for instruments, delta_set in delta_sets.items() for deltas in delta_set]
def ifo_combos(ifosegdict): ifos = [] combos = [] for ifo in ifosegdict.keys(): if ifosegdict[ifo]: ifos.append(ifo) ifos.sort() for i in range(2, len(ifos)+1): combos.extend([j for j in iterutils.choices(ifos,i)]) l = [i for i in combos] combos = [] for i in l: combos.append(",".join(i)) #FIXME assumes we don't look at H1H2 if 'H1,H2' in combos: combos.remove('H1,H2') return combos
def ifo_combos(ifosegdict): ifos = [] combos = [] for ifo in ifosegdict.keys(): if ifosegdict[ifo]: ifos.append(ifo) ifos.sort() for i in range(2, len(ifos) + 1): combos.extend([j for j in iterutils.choices(ifos, i)]) l = [i for i in combos] combos = [] for i in l: combos.append(",".join(i)) #FIXME assumes we don't look at H1H2 if 'H1,H2' in combos: combos.remove('H1,H2') return combos
def test_vote(self): """ Test vote(). """ for i in range(algebra_repeats): seglists = [] for j in range(random.randint(0, 10)): seglists.append( verifyutils.random_coalesced_list(algebra_listlength)) n = random.randint(0, len(seglists)) correct = reduce(lambda x, y: x | y, (votes and reduce(lambda a, b: a & b, votes) or segments.segmentlist() for votes in iterutils.choices(seglists, n)), segments.segmentlist()) self.assertEqual(correct, segmentsUtils.vote(seglists, n))
def break_up_coinc(coincRow, coincTable, coinc_stat, id_generator): """ This is the function that splits up the triples and quadruples into their respective subcoincidences """ ifostr,ifolist=coincRow.get_ifos() new_ifos=[ifos for n in range(2, len(ifolist) + 1) for ifos in iterutils.choices(ifolist, n)] n_subcombs=len(new_ifos) for comb in new_ifos: n_ifos_in_comb=len(comb) t=getattr(coincRow,comb[0]) event_id=t.event_id new_event_id=id_generator.new(t) myrow=CoincInspiralUtils.coincInspiralTable.row(new_event_id) for ifo in comb: myrow.add_trig(getattr(coincRow,ifo),coinc_stat) myt=getattr(myrow,comb[0]) coincTable.append(myrow)
def get_allifo_combos(ifo_keys, min_num_ifos): """ Returns two dictionaries, one of the on instruments and the other of the excluded instruments in a given coincident time type e.g. (H1,L1; H1,L1,V1). The dictionary keys are in the same format as the instruments column of the experiment table. @param ifo_keys: an sorted list of single ifos as strings @param min_num_ifos: the minimum number of ifos in a combination """ on_ifos = {} excluded_ifos = {} ifo_set = set(ifo_keys) for num_ifos in range(min_num_ifos, len(ifo_set)+1): for sub_combo in iterutils.choices( list(ifo_set), num_ifos): sorted_on_list = sorted(sub_combo) on_ifos[','.join(sorted_on_list)] = sorted_on_list excluded_ifos[','.join(sorted_on_list)] = list(ifo_set - set(sorted_on_list)) return on_ifos, excluded_ifos
def get_allifo_combos(ifo_keys, min_num_ifos): """ Returns two dictionaries, one of the on instruments and the other of the excluded instruments in a given coincident time type e.g. (H1,L1; H1,L1,V1). The dictionary keys are in the same format as the instruments column of the experiment table. @param ifo_keys: an sorted list of single ifos as strings @param min_num_ifos: the minimum number of ifos in a combination """ on_ifos = {} excluded_ifos = {} ifo_set = set(ifo_keys) for num_ifos in range(min_num_ifos, len(ifo_set) + 1): for sub_combo in iterutils.choices(list(ifo_set), num_ifos): sorted_on_list = sorted(sub_combo) on_ifos[",".join(sorted_on_list)] = sorted_on_list excluded_ifos[",".join(sorted_on_list)] = list(ifo_set - set(sorted_on_list)) return on_ifos, excluded_ifos
def get_coinc_window(connection, ifos): # determine the minimum time shift sqlquery = """ SELECT MIN(ABS(offset)) FROM time_slide WHERE offset != 0.0 """ shift = connection.execute( sqlquery ).fetchone()[0] # SQL query to get gps end-times for a given type of double sqlquery = """ SELECT DISTINCT si_ifo1.end_time, si_ifo2.end_time, si_ifo1.end_time_ns, si_ifo2.end_time_ns FROM coinc_inspiral JOIN sngl_inspiral AS si_ifo1, coinc_event_map AS cem_ifo1 ON ( coinc_inspiral.coinc_event_id == cem_ifo1.coinc_event_id AND cem_ifo1.event_id == si_ifo1.event_id AND si_ifo1.ifo == ?) JOIN sngl_inspiral AS si_ifo2, coinc_event_map AS cem_ifo2 ON ( coinc_inspiral.coinc_event_id == cem_ifo2.coinc_event_id AND cem_ifo2.event_id == si_ifo2.event_id AND si_ifo2.ifo == ?) """ # loop over pairs of instruments from the ifos list tau = {} for ifo_pair in iterutils.choices(ifos, 2): toa_diff = np.array([]) # determine the difference in trigger end-times after sliding the times for coinc in connection.execute( sqlquery, tuple(ifos) ): dT_sec = (coinc[0]-coinc[1]) + (coinc[2]-coinc[3])*1e-9 num_shifts = np.round(dT_sec/shift) toa_diff = np.append( toa_diff, dT_sec - num_shifts*shift ) # the max-min of time differences defines the size of the coinc window tau[','.join(ifos)] = np.max(toa_diff) - np.min(toa_diff) return tau
def get_coinc_window(connection, ifos): # determine the minimum time shift sqlquery = """ SELECT MIN(ABS(offset)) FROM time_slide WHERE offset != 0.0 """ shift = connection.execute(sqlquery).fetchone()[0] # SQL query to get gps end-times for a given type of double sqlquery = """ SELECT DISTINCT si_ifo1.end_time, si_ifo2.end_time, si_ifo1.end_time_ns, si_ifo2.end_time_ns FROM coinc_inspiral JOIN sngl_inspiral AS si_ifo1, coinc_event_map AS cem_ifo1 ON ( coinc_inspiral.coinc_event_id == cem_ifo1.coinc_event_id AND cem_ifo1.event_id == si_ifo1.event_id AND si_ifo1.ifo == ?) JOIN sngl_inspiral AS si_ifo2, coinc_event_map AS cem_ifo2 ON ( coinc_inspiral.coinc_event_id == cem_ifo2.coinc_event_id AND cem_ifo2.event_id == si_ifo2.event_id AND si_ifo2.ifo == ?) """ # loop over pairs of instruments from the ifos list tau = {} for ifo_pair in iterutils.choices(ifos, 2): toa_diff = np.array([]) # determine the difference in trigger end-times after sliding the times for coinc in connection.execute(sqlquery, tuple(ifos)): dT_sec = (coinc[0] - coinc[1]) + (coinc[2] - coinc[3]) * 1e-9 num_shifts = np.round(dT_sec / shift) toa_diff = np.append(toa_diff, dT_sec - num_shifts * shift) # the max-min of time differences defines the size of the coinc window tau[','.join(ifos)] = np.max(toa_diff) - np.min(toa_diff) return tau
def multi_ifo_compute_offsource_segment(analyzable_dict, on_source, **kwargs): """ Return the off-source segment determined for multiple IFO times along with the IFO combo that determined that segment. Calls compute_offsource_segment as necessary, passing all kwargs as necessary. """ # sieve down to relevant segments and IFOs; sort IFOs by sensitivity new_analyzable_dict = segments.segmentlistdict() for ifo, seglist in analyzable_dict.iteritems(): try: ind = seglist.find(on_source) except ValueError: continue new_analyzable_dict[ifo] = segments.segmentlist([seglist[ind]]) analyzable_ifos = new_analyzable_dict.keys() analyzable_ifos.sort(sensitivity_cmp) # now try getting off-source segments; start trying with all IFOs, then # work our way to smaller and smaller subsets; exclude single IFOs. test_combos = itertools.chain( \ *itertools.imap(lambda n: iterutils.choices(analyzable_ifos, n), xrange(len(analyzable_ifos), 1, -1))) off_source_segment = None the_ifo_combo = [] for ifo_combo in test_combos: trial_seglist = new_analyzable_dict.intersection(ifo_combo) temp_segment = compute_offsource_segment(trial_seglist, on_source, **kwargs) if temp_segment is not None: off_source_segment = temp_segment the_ifo_combo = list(ifo_combo) the_ifo_combo.sort() break return off_source_segment, the_ifo_combo
def get_rinca_livetimes(ring_sets, veto_segments, offset_vectors, verbose=False): # FIXME: somebody should document this livetimes = {} for available_instruments, rings in ring_sets.items(): for on_instruments in (combo for m in range(2, len(available_instruments) + 1) for combo in iterutils.choices( sorted(available_instruments), m)): if verbose: print >> sys.stderr, "%s/%s" % (",".join( on_instruments), ",".join(sorted(available_instruments))), on_instruments = frozenset(on_instruments) if on_instruments not in livetimes: livetimes[on_instruments] = [0.0] * len(offset_vectors) for i, livetime in enumerate( SnglInspiralUtils.compute_rinca_livetime( on_instruments, available_instruments - on_instruments, rings, veto_segments, offset_vectors)): livetimes[on_instruments][i] += livetime return livetimes
def coinc_params(events, offsetvector, triangulators): # # check for coincs that have been vetoed entirely # if len(events) < 2: return None # # Initialize the parameter dictionary, sort the events by # instrument name (the multi-instrument parameters are defined for # the instruments in this order and the triangulators are # constructed this way too), and retrieve the sorted instrument # names # params = {} events = tuple(sorted(events, key = lambda event: event.ifo)) instruments = tuple(event.ifo for event in events) # # zero-instrument parameters # ignored, ignored, ignored, rss_timing_residual = triangulators[instruments](tuple(event.peak + offsetvector[event.ifo] for event in events)) # FIXME: rss_timing_residual is forced to 0 to disable this # feature. all the code to compute it properly is still here and # given suitable initializations, the distribution data is still # two-dimensional and has a suitable filter applied to it, but all # events are forced into the RSS_{\Delta t} = 0 bin, in effect # removing that dimension from the data. We can look at this again # sometime in the future if we're curious why it didn't help. Just # delete the next line and you're back in business. rss_timing_residual = 0.0 params["instrumentgroup,rss_timing_residual"] = (frozenset(instruments), rss_timing_residual) # # one-instrument parameters # for event in events: prefix = "%s_" % event.ifo params["%ssnr2_chi2" % prefix] = (event.snr**2.0, event.chisq / event.chisq_dof) # # two-instrument parameters. note that events are sorted by # instrument # for event1, event2 in iterutils.choices(events, 2): assert event1.ifo != event2.ifo prefix = "%s_%s_" % (event1.ifo, event2.ifo) dt = float((event1.peak + offsetvector[event1.ifo]) - (event2.peak + offsetvector[event2.ifo])) params["%sdt" % prefix] = (dt,) dA = math.log10(abs(event1.amplitude / event2.amplitude)) params["%sdA" % prefix] = (dA,) # f_cut = central_freq + bandwidth/2 f_cut1 = event1.central_freq + event1.bandwidth / 2 f_cut2 = event2.central_freq + event2.bandwidth / 2 df = float((math.log10(f_cut1) - math.log10(f_cut2)) / (math.log10(f_cut1) + math.log10(f_cut2))) params["%sdf" % prefix] = (df,) # # done # return params
result_dbs_cache = lal.Cache().from_urls( non_sim_dbs ) for result_db in result_dbs_cache: # get tag and veto_cat print "Creating jobs for %s database..." % result_db.description tag = result_db.description.replace('_CLUSTERED_CBC_RESULTS', '') cat_num = get_veto_cat_from_tag( tag ) veto_cat = '_'.join([ 'CAT', str(cat_num), 'VETO' ]) # get all possible instruments_on in this database instruments = lsctables.instrument_set_from_ifos(result_db.observatory) distinct_instrument_sets = [instruments] distinct_instrument_sets.extend( set(sub_combo) for nn in range(2, len(instruments)) for sub_combo in iterutils.choices( list(instruments), nn ) ) # add the injection xmls to the FULL_DATA databases if 'FULL_DATA' in tag and veto_cat in sim_caches: # create a sqlite node to add the injetion results sim2fulldb_node = pipeline.LigolwSqliteNode( sql_fromcache_job ) sim2fulldb_node.set_category('ligolw_sqlite') sim2fulldb_node.set_input_cache( sim_caches[veto_cat] ) sim2fulldb_node.set_database( result_db.path ) sim2fulldb_node.set_tmp_space( tmp_space ) sim2fulldb_node.add_parent( injfind_nodes[veto_cat] ) sim2fulldb_node.add_parent( cluster_nodes[tag] ) dag.add_node( sim2fulldb_node )
def get_doubles(instruments): all_ifos = instruments.strip().split(',') ifo_combinations = list(iterutils.choices(all_ifos, 2)) for comb in ifo_combinations: comb = ','.join(comb) return comb
def optimize(self): """ Identifies segment lists that differ only in their instruments --- they have the same valid and active segments, the same name, version and the same comment --- and then deletes all but one of them, leaving just a single list having the union of the instruments. """ self.sort() segment_lists = dict(enumerate(self)) for target, source in [(idx_a, idx_b) for (idx_a, seglist_a), (idx_b, seglist_b) in iterutils.choices(segment_lists.items(), 2) if seglist_a.valid == seglist_b.valid and seglist_a.active == seglist_b.active and seglist_a.name == seglist_b.name and seglist_a.version == seglist_b.version and seglist_a.comment == seglist_b.comment]: try: source = segment_lists.pop(source) except KeyError: continue segment_lists[target].instruments |= source.instruments self.clear() self.update(segment_lists.values())
def populate_experiment_table( xmldoc, search_group, trigger_program, lars_id, instruments, comments = None, add_inst_subsets = False, verbose = False ): """ Populate the experiment table using the given entries. If add_inst_subsets is set to True, will write additional entries for every possible sub-combination of the given instrument set. Returns a dictionary of experiment_ids keyed by the instrument set. @xmldoc: xmldoc to get/write table to @lars_id: lars_id of the experiment @search_group: lsc group that performed the experiment (e.g., cbc) @trigger_program: name of the program that performed the analysis (e.g., inspiral, ringdown, etc.) @comments: any desired comments @add_inst_subsets: will write an entry for every possible subset of @instruments @verbose: be verbose """ if verbose: print >> sys.stderr, "\tPopulating the Experiment table..." # find the experiment table or create one if needed try: expr_table = table.get_table(xmldoc, lsctables.ExperimentTable.tableName) except ValueError: expr_table = xmldoc.childNodes[0].appendChild(lsctables.New(lsctables.ExperimentTable)) # determine experiment start and end times expr_start_time, expr_end_time = get_experiment_times(xmldoc) # write entry to the experiment table for the given instruments if it doesn't already exist experiment_ids = {} for nn in range(len(instruments), 1, -1): if not add_inst_subsets and (nn != len(instruments)): break # add every possible sub-combination of the instrument set if # they're not already in the table for sub_combo in iterutils.choices( list(instruments), nn ): if frozenset(sub_combo) not in experiment_ids: experiment_ids[frozenset(sub_combo)] = expr_table.write_new_expr_id( search_group, trigger_program, lars_id, sub_combo, expr_start_time, expr_end_time, comments = comments ) return experiment_ids
def coinc_params(events, offsetvector, triangulators): # # check for coincs that have been vetoed entirely # if len(events) < 2: return None # # Initialize the parameter dictionary, sort the events by # instrument name (the multi-instrument parameters are defined for # the instruments in this order and the triangulators are # constructed this way too), and retrieve the sorted instrument # names # params = {} events = tuple(sorted(events, key=lambda event: event.ifo)) instruments = tuple(event.ifo for event in events) # # zero-instrument parameters # ignored, ignored, ignored, rss_timing_residual = triangulators[ instruments](tuple(event.peak + offsetvector[event.ifo] for event in events)) # FIXME: rss_timing_residual is forced to 0 to disable this # feature. all the code to compute it properly is still here and # given suitable initializations, the distribution data is still # two-dimensional and has a suitable filter applied to it, but all # events are forced into the RSS_{\Delta t} = 0 bin, in effect # removing that dimension from the data. We can look at this again # sometime in the future if we're curious why it didn't help. Just # delete the next line and you're back in business. rss_timing_residual = 0.0 params["instrumentgroup,rss_timing_residual"] = ( frozenset(instruments), rss_timing_residual) # # one-instrument parameters # for event in events: prefix = "%s_" % event.ifo params["%ssnr2_chi2" % prefix] = (event.snr**2.0, event.chisq / event.chisq_dof) # # two-instrument parameters. note that events are sorted by # instrument # for event1, event2 in iterutils.choices(events, 2): assert event1.ifo != event2.ifo prefix = "%s_%s_" % (event1.ifo, event2.ifo) dt = float((event1.peak + offsetvector[event1.ifo]) - (event2.peak + offsetvector[event2.ifo])) params["%sdt" % prefix] = (dt, ) dA = math.log10(abs(event1.amplitude / event2.amplitude)) params["%sdA" % prefix] = (dA, ) # f_cut = central_freq + bandwidth/2 f_cut1 = event1.central_freq + event1.bandwidth / 2 f_cut2 = event2.central_freq + event2.bandwidth / 2 df = float((math.log10(f_cut1) - math.log10(f_cut2)) / (math.log10(f_cut1) + math.log10(f_cut2))) params["%sdf" % prefix] = (df, ) # # done # return params
def test_vote(self): """ Test vote(). """ for i in range(algebra_repeats): seglists = [] for j in range(random.randint(0, 10)): seglists.append(verifyutils.random_coalesced_list(algebra_listlength)) n = random.randint(0, len(seglists)) correct = reduce(lambda x, y: x | y, (votes and reduce(lambda a, b: a & b, votes) or segments.segmentlist() for votes in iterutils.choices(seglists, n)), segments.segmentlist()) self.assertEqual(correct, segmentsUtils.vote(seglists, n))
input_user_tag = tag + '_' # Copies the number of slides to the codes that use it num_slides = cp.get('pipeline','num-slides') if num_slides: for section in ['plotthinca', 'plotethinca', 'ploteffdistcut']: cp.set(section,'num-slides',num_slides) ############################### # Construct list based on ifos supplied at command line ifolist = [ifo for ifo in ('G1','H1', 'H2', 'L1', 'V1') \ if getattr(opts, "%s_data" % ifo.lower())] if opts.two_ifo: ifo_combo=list(iterutils.choices(ifolist,2)) if opts.three_ifo: ifo_combo=list(iterutils.choices(ifolist,2)) + list(iterutils.choices(ifolist,3)) if opts.four_ifo: ifo_combo=list(iterutils.choices(ifolist,2)) + list(iterutils.choices(ifolist,3)) + \ list(iterutils.choices(ifolist,4)) if opts.analyze_all: ifo_combo=CoincInspiralUtils.get_ifo_combos(ifolist) ############################################################################## # try to make a directory to store the cache files and job logs try: os.mkdir('logs') except: pass ##############################################################################
def get_doubles(instruments): all_ifos = instruments.strip().split(',') ifo_combinations = list(iterutils.choices(all_ifos,2)) for comb in ifo_combinations: comb=','.join(comb) return comb
input_user_tag = tag + '_' # Copies the number of slides to the codes that use it num_slides = cp.get('pipeline', 'num-slides') if num_slides: for section in ['plotthinca', 'plotethinca', 'ploteffdistcut']: cp.set(section, 'num-slides', num_slides) ############################### # Construct list based on ifos supplied at command line ifolist = [ifo for ifo in ('G1','H1', 'H2', 'L1', 'V1') \ if getattr(opts, "%s_data" % ifo.lower())] if opts.two_ifo: ifo_combo = list(iterutils.choices(ifolist, 2)) if opts.three_ifo: ifo_combo = list(iterutils.choices(ifolist, 2)) + list( iterutils.choices(ifolist, 3)) if opts.four_ifo: ifo_combo=list(iterutils.choices(ifolist,2)) + list(iterutils.choices(ifolist,3)) + \ list(iterutils.choices(ifolist,4)) if opts.analyze_all: ifo_combo = CoincInspiralUtils.get_ifo_combos(ifolist) ############################################################################## # try to make a directory to store the cache files and job logs try: os.mkdir('logs') except: pass
def get_thinca_livetimes(ring_sets, veto_segments, offset_vectors, verbose = False): # FIXME: somebody should document this livetimes = {} for available_instruments, rings in ring_sets.items(): for on_instruments in (combo for m in range(2, len(available_instruments) + 1) for combo in iterutils.choices(sorted(available_instruments), m)): if verbose: print >>sys.stderr, "%s/%s" % (",".join(on_instruments), ",".join(sorted(available_instruments))), on_instruments = frozenset(on_instruments) if on_instruments not in livetimes: livetimes[on_instruments] = [0.0] * len(offset_vectors) for i, livetime in enumerate(SnglInspiralUtils.compute_thinca_livetime(on_instruments, available_instruments - on_instruments, rings, veto_segments, offset_vectors)): livetimes[on_instruments][i] += livetime return livetimes
def round_robin(round_robin_cache_nodes, round_robin_cache): parents = list(iterutils.choices(round_robin_cache_nodes, len(round_robin_cache_nodes) - 1)) parents.reverse() parents = [reduce(lambda a, b: a | b, seq) for seq in parents] return zip(parents, [cache for (cache,) in iterutils.choices(round_robin_cache, 1)])
def exttrig_dataquery(grb_name, grb_time, grb_ra, grb_dec, offset, config_file, extend=False, useold=False, make_plots=False, make_xml=False): ''' Finds science time of all available IFOs. ''' ############################################################################## # get segment duration and minimum amount of science time ############################################################################## # read the configuration file cp = ConfigParser.ConfigParser() cp.read(config_file) # define hardcoded variables basic_ifolist = ifolist = ['H1','H2','L1','V1'] catlist = [1,2,3] sensitivity_dict = {"H1": 1, "L1": 2, "H2": 3, "V1": 4, "G1": 5} # get segment length from configuration file pad_data = int(cp.get('data','pad-data')) if cp.has_option('data','segment-duration'): blockDuration = segmentDuration = psdDuration = int(cp.get('data','segment-duration')) elif cp.has_option('data','segment-length'): blockDuration = segmentDuration = psdDuration = int(cp.get('data','segment-length')) /int(cp.get('data','sample-rate')) else: raise ValueError, "EXIT: Cannot find segment-duration in [data] section of configuration file!" # get sample rate if cp.has_option('data','sample-rate'): sampleRate = int(cp.get('data', 'sample-rate')) print ">> Sample rate has been set to: %d"%sampleRate print else: print ">> ERROR: Need to specify sample-rate in [data] section of configuration file in order to calculate inputs for downstream processes." sys.exit() # if not extend option then need to get block duration if not extend: if cp.has_option('data','block-duration'): blockDuration = int(cp.get('data','block-duration')) elif cp.has_option('data','segment-length'): s_length = int(cp.get('data', 'segment-length')) s_num = int(cp.get('data', 'number-of-segments')) s_rate = int(cp.get('data', 'sample-rate')) s_overlap = int(cp.get('inspiral', 'segment-overlap')) # calculate blockDuration blockDuration = ( s_length * s_num - ( s_num - 1 ) * s_overlap ) / s_rate else: raise ValueError, "EXIT: Cannot find block-duration in [data] section of configuration file! Either set block-duration or use --extend option." # calculate the minimum amount of science time need and how the length of quanta to be added on both ends of the analysis time minscilength = blockDuration + 2 * pad_data quanta = segmentDuration / 2 # if extend beyond minscilength; add segments of quanta length to each end of segment print ">> Minimum science segment length is: %ss"%minscilength print if extend: print ">> Will extend minimum science segment by quanta of: %ss"%quanta print ############################################################################## # get list of segments for each IFO and put in science txt file ############################################################################## if not useold: # external call to ligolw_segment_query query_start = int(grb_time - offset) query_end = int(grb_time + offset) for ifo in ifolist: if cp.has_option('segments','%s-segments'%ifo.lower()): segmentName = cp.get('segments','%s-segments'%ifo.lower()) check_segment_availability(grb_name, grb_time, query_start, query_end, offset, ifo, segmentName) ############################################################################## # get veto files ############################################################################## if not useold: # get and read veto definer file veto_file_url = cp.get('exttrig','cvs_veto_definer') veto_file_path,headers = urllib.urlretrieve(veto_file_url,os.path.basename(veto_file_url)) # do ligolw_segments_from_cats deltat = 500 args = {'start_time' : int(grb_time - offset - deltat), 'end_time' : int(grb_time + offset + deltat), 'veto_file_path' : veto_file_path} cmd = "ligolw_segments_from_cats --database --veto-file={veto_file_path} --separate-categories --gps-start-time {start_time} --gps-end-time {end_time} --output-dir=. --individual-results".format(**args) print '>>',cmd print process = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output,err = process.communicate() # Rename the veto files for easier handling veto_files = glob.glob('./*VETOTIME_CAT*{start_time}*xml'.format(**args)) for filename in veto_files: p = filename.split('-') newname = "%s-%s_grb%s.xml"%(p[0], p[1], grb_name) shutil.move(filename, newname) ############################################################################## # look in txt files and find segment with onsource and minscilength ############################################################################## # create segment that is +/- offset of GRB time onsource = [grb_time - int(cp.get('exttrig','onsource_left')),\ grb_time + int(cp.get('exttrig','onsource_right'))] onSourceSegment = segments.segment(onsource[0], onsource[1]) # get segments in science txt files; see if segments length at least minscilength # if no then discard them; if yes then put in segdict[ifo] and ifo in ifolist basic_segdict = segdict = segments.segmentlistdict() for ifo in ifolist: # check configuration file if not cp.has_option('segments','%s-segments' % ifo.lower()): continue # find segment with onsource and check it is at least minscilength ifo_segfile = '%s-science_grb%s.txt' %(ifo, grb_name) if os.path.exists(ifo_segfile): tmplist = segmentsUtils.fromsegwizard(open(ifo_segfile)) try: s = tmplist.find(onSourceSegment) except ValueError: # if onsource not in segments then move on to next IFO continue if abs(tmplist[s]) >= minscilength: segdict[ifo] = segments.segmentlist([tmplist[s]]) basic_segdict[ifo] = segments.segmentlist([s for s in tmplist]) ifolist = segdict.keys() if len(ifolist) < 2: print "EXIT: Less than 2 interferometers have available data!" sys.exit() ############################################################################## # apply vetoes ############################################################################## # apply print ">> Vetoes that overlap with science segments:" for ifo in ifolist: # flag; True if IFO not vetoed cat_flag = True for cat in catlist: # create list and check for overlaps xmlsegfile = "./%s-VETOTIME_CAT%s_grb%s.xml" %(ifo, cat, grb_name) if os.path.exists(xmlsegfile) and cat_flag: testseg = segments.segment([segdict[ifo][0][0],segdict[ifo][0][1]]) list_overlaps = [] # load the content of the veto-file xmldoc = utils.load_filename(xmlsegfile, gz = False, contenthandler = lsctables.use_in(ligolw.LIGOLWContentHandler)) segs = lsctables.SegmentTable.get_table(xmldoc) segdefs = lsctables.SegmentDefTable.get_table(xmldoc) # create a mapping between the segments and their definitions defdict = {} for segdef in segdefs: defdict[segdef.segment_def_id] = segdef.name # find veto segments that intersect science segment of IFO with onsource for seg in segs: s = segments.segment(seg.start_time, seg.end_time) if testseg.intersects(s): id = seg.segment_def_id list_overlaps.append([defdict[id], seg.start_time, seg.end_time]) # cut veto CAT1 segments out of science segment; CAT1,2,3 veto IFO if in onsource will veto IFO for name, segstart, segend in list_overlaps: print "CAT%s IFO %s, Start: %d End: %d because %s"%(cat, ifo, segstart, segend, name) s = segments.segment(segstart, segend) if onSourceSegment.intersects(s): segdict.pop(ifo, None) cat_flag = False break if cat == 1: vetoes = segments.segmentlist(segments.segment(s[1], s[2]) for s in list_overlaps) segdict[ifo] -= vetoes # get list of IFOs ifolist = segdict.keys() print if len(ifolist) < 2: print "EXIT: After vetoes, less than 2 interferometers have available data!" sys.exit() ############################################################################## # determine segment to be analyzed ############################################################################## # sort from most sensitive to least sensitive def sensitivity_cmp(ifo1, ifo2): return cmp(sensitivity_dict[ifo1], sensitivity_dict[ifo2]) ifolist.sort(sensitivity_cmp) # compares IFOs and finds the segment to analyze # now try getting off-source segments # start trying with all IFOs # work our way through subsets; beginning with most sensitive combinations test_combos = itertools.chain(*itertools.imap(lambda n: iterutils.choices(ifolist, n), xrange(len(ifolist), 1, -1))) off_source_segment = None the_ifo_combo = [] for ifo_combo in test_combos: # find conincident science time of IFOs trial_seglist = segdict.intersection(ifo_combo) if abs(trial_seglist) < minscilength: print "EXIT: IFOs do not overlap enough for minscilength",abs(trial_seglist) sys.exit() else: pass # find segment with grb_time inside try: super_seg = trial_seglist[trial_seglist.find(onSourceSegment)].contract(pad_data) except ValueError: print "EXIT: ValueError with super_seg" sys.exit() if onSourceSegment not in super_seg: print "EXIT: onsource not in super_seg" sys.exit() # find int division of onsource time intervals before and after grb tplus = (super_seg[1] - onSourceSegment[1]) tminus = (onSourceSegment[0] - super_seg[0]) # get minimum number of onsource time intervals in offsource tmin = ( minscilength - 2*pad_data - abs(onSourceSegment) ) # cut to get minscilength if tplus + tminus > tmin: half_max = tmin // 2 if tplus < half_max: print ">> Left sticks out so cut it." remainder = tmin - tplus tminus = min(remainder, tminus) elif tminus < half_max: print ">> Right sticks out so cut it." remainder = tmin - tminus tplus = min(remainder, tplus) else: print ">> Both sides stick out so cut as symmetrically as possible." tminus = half_max tplus = tmin - half_max # odd trial sticks out on right if tplus + tminus < tmin: offsource = None temp_segment = segments.segment((onSourceSegment[0] - tminus - pad_data, onSourceSegment[1] + tplus + pad_data)) if temp_segment is not None: offsource = temp_segment ifolist = list(ifo_combo) if extend: # extend as many adjacent 128 second blocks as possible begin_time = offsource[0] - quanta * (abs(super_seg[0]-offsource[0])//quanta) end_time = offsource[1] + quanta * (abs(super_seg[1]-offsource[1])//quanta) offsource = segments.segment((begin_time,end_time)) break print # check length at least minscilength if abs(offsource) < minscilength: print abs(offsource),minscilength print "EXIT: Calculated offsource segment but less than minscilength!" sys.exit() # check if no detectors can be used then exit if len(ifolist) < 2: print "EXIT: Calculated offsource segment but less than two IFOs!" sys.exit() # check edge case if abs(offsource[0]-onsource[0]) < pad_data or abs(offsource[1]-onsource[1]) < pad_data: print "WARNING: GRB time close to edge of offsource. Its within the padding time." # concatenate "H1L1V1", etc. ifolist.sort() ifotag = "".join(ifolist) print ">> Offsource segment for %s GRB is:"%ifotag print "Start:",offsource[0],"End:",offsource[1],"Duration:",offsource[1]-offsource[0],"Left:",grb_time-offsource[0],"Right:",offsource[1]-grb_time print ############################################################################## # output ############################################################################## # write analyse txt files for ifo in basic_ifolist: if ifo in ifolist: analysisFP = open('%s-analyse_grb%s.txt' %(ifo,grb_name),'w') analysisFP.write('# seg\t start \t stop \t duration\n') analysisFP.write('0\t %d\t %d\t %d\n' %(offsource[0],offsource[1],offsource[1]-offsource[0])) else: analysisFP = open('%s-analyse_grb%s.txt' %(ifo,grb_name),'w') analysisFP.write('# seg\t start \t stop \t duration\n') # calculate blockDuration blockDuration = int(abs(offsource[0]-offsource[1])) - 2 * pad_data # calculate psdDuration # gets largest power of two such that blockDuration/psdDuration = psdRatio # could have done a binary & operator that is faster but this is more user-friendly I believe min_psdDuration = int(cp.get('exttrig', 'min-psd-length')) psdRatio = int(cp.get('exttrig', 'psd-ratio')) psdDuration = 2**int(numpy.log2(blockDuration/psdRatio)) if psdDuration < min_psdDuration: print "EXIT: PSD segment duration is too short. It is %ds but needs to be at least %ds in length."%(psdDuration,min_psdDuration) sys.exit() # some downstream processes (e.g. lalapps_tmpltbank) cannot handle these inputs if cp.has_option('data', 'segment-duration'): cp.remove_option('data', 'segment-duration') cp.remove_option('data', 'block-duration') # some downstream processes (e.g. lalapps_tmpltbank) requires these options to run print ">> Using sample rate of %d to calculate inputs for downstream processes."%sampleRate print segmentLength = segmentDuration*sampleRate segmentCount = blockDuration/(segmentDuration/2) - 1 # subtract 1 because one segment length is overlapped segmentOverlap = segmentLength/2 cp.set('data', 'segment-length', segmentLength) cp.set('data', 'number-of-segments', segmentCount) cp.set('inspiral', 'segment-overlap', segmentOverlap) # set values for [coh_PTF_inspral] section in configuration file cp.set('coh_PTF_inspiral', 'block-duration', blockDuration) cp.set('coh_PTF_inspiral', 'segment-duration', segmentDuration) cp.set('coh_PTF_inspiral', 'psd-segment-duration', psdDuration) cp.set('coh_PTF_inspiral', 'pad-data', pad_data) f = open('grb%s.ini'%grb_name,'w') cp.write(f) f.close() print ">> The [data] section of the configuration file has been edited with the following values:" print "sample-rate=",sampleRate print "segment-length=",segmentLength print "number-of-segments=",segmentCount print "segment-overlap=",segmentOverlap print print ">> The [coh_PTF_inspiral] section of the configuration file has been edited with the following values:" print "block-duration =",blockDuration print "segment-duration =",segmentDuration print "psd-segment-duration =",psdDuration print "pad-data =",pad_data print # plot segments offSourceSegment = segments.segment(offsource[0], offsource[1]) plot_window = segments.segment(grb_time-offset, grb_time+offset) plot_segments(basic_segdict, onSourceSegment, offSourceSegment, grb_time, plot_window, "segment_plot_%s.png"%grb_name, grb_name) # make xml file if make_xml: # create a new xml document with an ExtTriggers Table xmldoc = ligolw.Document() xmldoc.appendChild(ligolw.LIGO_LW()) tbl = lsctables.New(lsctables.ExtTriggersTable) xmldoc.childNodes[-1].appendChild(tbl) # set the values we need row = lsctables.ExtTriggersTable() row.process_id = None row.det_alts = None row.det_band = None row.det_fluence = None row.det_fluence_int = None row.det_name = None row.det_peak = None row.det_peak_int = None row.det_snr = '' row.email_time = 0 row.event_dec = float(grb_dec) row.event_dec_err = 0.0 row.event_epoch = '' row.event_err_type = '' row.event_ra = float(grb_ra) row.event_ra_err = 0.0 row.start_time = grb_time row.start_time_ns = 0 row.event_type = '' row.event_z = 0.0 row.event_z_err = 0.0 row.notice_comments = '' row.notice_id = '' row.notice_sequence = '' row.notice_time = 0 row.notice_type = '' row.notice_url = '' row.obs_fov_dec = 0.0 row.obs_fov_dec_width = 0.0 row.obs_fov_ra = 0.0 row.obs_fov_ra_width = 0.0 row.obs_loc_ele = 0.0 row.obs_loc_lat = 0.0 row.obs_loc_long = 0.0 row.ligo_fave_lho = 0.0 row.ligo_fave_llo = 0.0 row.ligo_delay = 0.0 row.event_number_gcn = 9999 row.event_number_grb = grb_name row.event_status = 0 # insert into the table and write file tbl.extend([row]) filename = 'grb%s.xml' % grb_name utils.write_filename(xmldoc, filename) # plot all vetoes if make_plots: vetodict = segments.segmentlistdict() for cat in catlist: for ifo in ifolist: vetofile = "%s-VETOTIME_CAT%s_grb%s.xml" % (ifo, cat, grb_name) xmldoc = utils.load_filename(vetofile, gz = False, contenthandler = lsctables.use_in(ligolw.LIGOLWContentHandler)) segs = lsctables.SegmentTable.get_table(xmldoc) segdefs = lsctables.SegmentDefTable.get_table(xmldoc) vetodict[ifo] = segments.segmentlist(segments.segment(s.start_time, s.end_time) for s in segs) if vetodict: plot_segments(vetodict, onSourceSegment, offSourceSegment, grb_time, plot_window, "veto_plot_CAT%s_%s.png"%(cat,grb_name), "%s CAT%s"%(grb_name, cat)) # return return 'grb%s.ini'%grb_name, ifolist, onSourceSegment, offSourceSegment
def coinc_params(events, offsetvector): # # check for coincs that have been vetoed entirely # if len(events) < 2: return None params = {} # the "time" is the ms_snr squared weighted average of the # peak times neglecting light-travel times. because # LIGOTimeGPS objects have overflow problems in this sort # of a calculation, the first event's peak time is used as # an epoch and the calculations are done w.r.t. that time. # FIXME: this time is available as the peak_time in the # multi_burst table, and it should be retrieved from that # table instead of being recomputed events = tuple(events) t = events[0].peak t += sum( float(event.peak - t) * event.ms_snr**2.0 for event in events) / sum(event.ms_snr**2.0 for event in events) gmst = date.XLALGreenwichMeanSiderealTime(t) % (2 * math.pi) for event1, event2 in iterutils.choices( sorted(events, lambda a, b: cmp(a.ifo, b.ifo)), 2): if event1.ifo == event2.ifo: # a coincidence is parameterized only by # inter-instrument deltas continue prefix = "%s_%s_" % (event1.ifo, event2.ifo) # in each of the following, if the list of events contains # more than one event from a given instrument, the smallest # deltas are recorded dt = float(event1.peak + offsetvector[event1.ifo] - event2.peak - offsetvector[event2.ifo]) name = "%sdt" % prefix if name not in params or abs(params[name][0]) > abs(dt): #params[name] = (dt,) params[name] = (dt, gmst) df = (event1.peak_frequency - event2.peak_frequency) / ( (event1.peak_frequency + event2.peak_frequency) / 2) name = "%sdf" % prefix if name not in params or abs(params[name][0]) > abs(df): #params[name] = (df,) params[name] = (df, gmst) dh = (event1.ms_hrss - event2.ms_hrss) / ( (event1.ms_hrss + event2.ms_hrss) / 2) name = "%sdh" % prefix if name not in params or abs(params[name][0]) > abs(dh): #params[name] = (dh,) params[name] = (dh, gmst) dband = (event1.ms_bandwidth - event2.ms_bandwidth) / ( (event1.ms_bandwidth + event2.ms_bandwidth) / 2) name = "%sdband" % prefix if name not in params or abs(params[name][0]) > abs(dband): #params[name] = (dband,) params[name] = (dband, gmst) ddur = (event1.ms_duration - event2.ms_duration) / ( (event1.ms_duration + event2.ms_duration) / 2) name = "%sddur" % prefix if name not in params or abs(params[name][0]) > abs(ddur): #params[name] = (ddur,) params[name] = (ddur, gmst) return params
def coinc_params(events, offsetvector): # # check for coincs that have been vetoed entirely # if len(events) < 2: return None params = {} # the "time" is the ms_snr squared weighted average of the # peak times neglecting light-travel times. because # LIGOTimeGPS objects have overflow problems in this sort # of a calculation, the first event's peak time is used as # an epoch and the calculations are done w.r.t. that time. # FIXME: this time is available as the peak_time in the # multi_burst table, and it should be retrieved from that # table instead of being recomputed events = tuple(events) t = events[0].peak t += sum(float(event.peak - t) * event.ms_snr**2.0 for event in events) / sum(event.ms_snr**2.0 for event in events) gmst = date.XLALGreenwichMeanSiderealTime(t) % (2 * math.pi) for event1, event2 in iterutils.choices(sorted(events, lambda a, b: cmp(a.ifo, b.ifo)), 2): if event1.ifo == event2.ifo: # a coincidence is parameterized only by # inter-instrument deltas continue prefix = "%s_%s_" % (event1.ifo, event2.ifo) # in each of the following, if the list of events contains # more than one event from a given instrument, the smallest # deltas are recorded dt = float(event1.peak + offsetvector[event1.ifo] - event2.peak - offsetvector[event2.ifo]) name = "%sdt" % prefix if name not in params or abs(params[name][0]) > abs(dt): #params[name] = (dt,) params[name] = (dt, gmst) df = (event1.peak_frequency - event2.peak_frequency) / ((event1.peak_frequency + event2.peak_frequency) / 2) name = "%sdf" % prefix if name not in params or abs(params[name][0]) > abs(df): #params[name] = (df,) params[name] = (df, gmst) dh = (event1.ms_hrss - event2.ms_hrss) / ((event1.ms_hrss + event2.ms_hrss) / 2) name = "%sdh" % prefix if name not in params or abs(params[name][0]) > abs(dh): #params[name] = (dh,) params[name] = (dh, gmst) dband = (event1.ms_bandwidth - event2.ms_bandwidth) / ((event1.ms_bandwidth + event2.ms_bandwidth) / 2) name = "%sdband" % prefix if name not in params or abs(params[name][0]) > abs(dband): #params[name] = (dband,) params[name] = (dband, gmst) ddur = (event1.ms_duration - event2.ms_duration) / ((event1.ms_duration + event2.ms_duration) / 2) name = "%sddur" % prefix if name not in params or abs(params[name][0]) > abs(ddur): #params[name] = (ddur,) params[name] = (ddur, gmst) return params
def exttrig_dataquery(grb_name, grb_time, grb_ra, grb_dec, offset, config_file, extend=False, useold=False, make_plots=False, make_xml=False): ''' Finds science time of all available IFOs. ''' ############################################################################## # get segment duration and minimum amount of science time ############################################################################## # read the configuration file cp = ConfigParser.ConfigParser() cp.read(config_file) # define hardcoded variables basic_ifolist = ifolist = ['H1', 'H2', 'L1', 'V1'] catlist = [1, 2, 3] sensitivity_dict = {"H1": 1, "L1": 2, "H2": 3, "V1": 4, "G1": 5} # get segment length from configuration file pad_data = int(cp.get('data', 'pad-data')) if cp.has_option('data', 'segment-duration'): blockDuration = segmentDuration = psdDuration = int( cp.get('data', 'segment-duration')) elif cp.has_option('data', 'segment-length'): blockDuration = segmentDuration = psdDuration = int( cp.get('data', 'segment-length')) / int( cp.get('data', 'sample-rate')) else: raise ValueError, "EXIT: Cannot find segment-duration in [data] section of configuration file!" # get sample rate if cp.has_option('data', 'sample-rate'): sampleRate = int(cp.get('data', 'sample-rate')) print ">> Sample rate has been set to: %d" % sampleRate print else: print ">> ERROR: Need to specify sample-rate in [data] section of configuration file in order to calculate inputs for downstream processes." sys.exit() # if not extend option then need to get block duration if not extend: if cp.has_option('data', 'block-duration'): blockDuration = int(cp.get('data', 'block-duration')) elif cp.has_option('data', 'segment-length'): s_length = int(cp.get('data', 'segment-length')) s_num = int(cp.get('data', 'number-of-segments')) s_rate = int(cp.get('data', 'sample-rate')) s_overlap = int(cp.get('inspiral', 'segment-overlap')) # calculate blockDuration blockDuration = (s_length * s_num - (s_num - 1) * s_overlap) / s_rate else: raise ValueError, "EXIT: Cannot find block-duration in [data] section of configuration file! Either set block-duration or use --extend option." # calculate the minimum amount of science time need and how the length of quanta to be added on both ends of the analysis time minscilength = blockDuration + 2 * pad_data quanta = segmentDuration / 2 # if extend beyond minscilength; add segments of quanta length to each end of segment print ">> Minimum science segment length is: %ss" % minscilength print if extend: print ">> Will extend minimum science segment by quanta of: %ss" % quanta print ############################################################################## # get list of segments for each IFO and put in science txt file ############################################################################## if not useold: # external call to ligolw_segment_query query_start = int(grb_time - offset) query_end = int(grb_time + offset) for ifo in ifolist: if cp.has_option('segments', '%s-segments' % ifo.lower()): segmentName = cp.get('segments', '%s-segments' % ifo.lower()) check_segment_availability(grb_name, grb_time, query_start, query_end, offset, ifo, segmentName) ############################################################################## # get veto files ############################################################################## if not useold: # get and read veto definer file veto_file_url = cp.get('exttrig', 'cvs_veto_definer') veto_file_path, headers = urllib.urlretrieve( veto_file_url, os.path.basename(veto_file_url)) # do ligolw_segments_from_cats deltat = 500 args = { 'start_time': int(grb_time - offset - deltat), 'end_time': int(grb_time + offset + deltat), 'veto_file_path': veto_file_path } cmd = "ligolw_segments_from_cats --database --veto-file={veto_file_path} --separate-categories --gps-start-time {start_time} --gps-end-time {end_time} --output-dir=. --individual-results".format( **args) print '>>', cmd print process = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, err = process.communicate() # Rename the veto files for easier handling veto_files = glob.glob( './*VETOTIME_CAT*{start_time}*xml'.format(**args)) for filename in veto_files: p = filename.split('-') newname = "%s-%s_grb%s.xml" % (p[0], p[1], grb_name) shutil.move(filename, newname) ############################################################################## # look in txt files and find segment with onsource and minscilength ############################################################################## # create segment that is +/- offset of GRB time onsource = [grb_time - int(cp.get('exttrig','onsource_left')),\ grb_time + int(cp.get('exttrig','onsource_right'))] onSourceSegment = segments.segment(onsource[0], onsource[1]) # get segments in science txt files; see if segments length at least minscilength # if no then discard them; if yes then put in segdict[ifo] and ifo in ifolist basic_segdict = segdict = segments.segmentlistdict() for ifo in ifolist: # check configuration file if not cp.has_option('segments', '%s-segments' % ifo.lower()): continue # find segment with onsource and check it is at least minscilength ifo_segfile = '%s-science_grb%s.txt' % (ifo, grb_name) if os.path.exists(ifo_segfile): tmplist = segmentsUtils.fromsegwizard(open(ifo_segfile)) try: s = tmplist.find(onSourceSegment) except ValueError: # if onsource not in segments then move on to next IFO continue if abs(tmplist[s]) >= minscilength: segdict[ifo] = segments.segmentlist([tmplist[s]]) basic_segdict[ifo] = segments.segmentlist([s for s in tmplist]) ifolist = segdict.keys() if len(ifolist) < 2: print "EXIT: Less than 2 interferometers have available data!" sys.exit() ############################################################################## # apply vetoes ############################################################################## # apply print ">> Vetoes that overlap with science segments:" for ifo in ifolist: # flag; True if IFO not vetoed cat_flag = True for cat in catlist: # create list and check for overlaps xmlsegfile = "./%s-VETOTIME_CAT%s_grb%s.xml" % (ifo, cat, grb_name) if os.path.exists(xmlsegfile) and cat_flag: testseg = segments.segment( [segdict[ifo][0][0], segdict[ifo][0][1]]) list_overlaps = [] # load the content of the veto-file xmldoc = utils.load_filename(xmlsegfile, gz=False, contenthandler=lsctables.use_in( ligolw.LIGOLWContentHandler)) segs = table.get_table(xmldoc, lsctables.SegmentTable.tableName) segdefs = table.get_table(xmldoc, lsctables.SegmentDefTable.tableName) # create a mapping between the segments and their definitions defdict = {} for segdef in segdefs: defdict[segdef.segment_def_id] = segdef.name # find veto segments that intersect science segment of IFO with onsource for seg in segs: s = segments.segment(seg.start_time, seg.end_time) if testseg.intersects(s): id = seg.segment_def_id list_overlaps.append( [defdict[id], seg.start_time, seg.end_time]) # cut veto CAT1 segments out of science segment; CAT1,2,3 veto IFO if in onsource will veto IFO for name, segstart, segend in list_overlaps: print "CAT%s IFO %s, Start: %d End: %d because %s" % ( cat, ifo, segstart, segend, name) s = segments.segment(segstart, segend) if onSourceSegment.intersects(s): segdict.pop(ifo, None) cat_flag = False break if cat == 1: vetoes = segments.segmentlist( segments.segment(s[1], s[2]) for s in list_overlaps) segdict[ifo] -= vetoes # get list of IFOs ifolist = segdict.keys() print if len(ifolist) < 2: print "EXIT: After vetoes, less than 2 interferometers have available data!" sys.exit() ############################################################################## # determine segment to be analyzed ############################################################################## # sort from most sensitive to least sensitive def sensitivity_cmp(ifo1, ifo2): return cmp(sensitivity_dict[ifo1], sensitivity_dict[ifo2]) ifolist.sort(sensitivity_cmp) # compares IFOs and finds the segment to analyze # now try getting off-source segments # start trying with all IFOs # work our way through subsets; beginning with most sensitive combinations test_combos = itertools.chain(*itertools.imap( lambda n: iterutils.choices(ifolist, n), xrange(len(ifolist), 1, -1))) off_source_segment = None the_ifo_combo = [] for ifo_combo in test_combos: # find conincident science time of IFOs trial_seglist = segdict.intersection(ifo_combo) if abs(trial_seglist) < minscilength: print "EXIT: IFOs do not overlap enough for minscilength", abs( trial_seglist) sys.exit() else: pass # find segment with grb_time inside try: super_seg = trial_seglist[trial_seglist.find( onSourceSegment)].contract(pad_data) except ValueError: print "EXIT: ValueError with super_seg" sys.exit() if onSourceSegment not in super_seg: print "EXIT: onsource not in super_seg" sys.exit() # find int division of onsource time intervals before and after grb tplus = (super_seg[1] - onSourceSegment[1]) tminus = (onSourceSegment[0] - super_seg[0]) # get minimum number of onsource time intervals in offsource tmin = (minscilength - 2 * pad_data - abs(onSourceSegment)) # cut to get minscilength if tplus + tminus > tmin: half_max = tmin // 2 if tplus < half_max: print ">> Left sticks out so cut it." remainder = tmin - tplus tminus = min(remainder, tminus) elif tminus < half_max: print ">> Right sticks out so cut it." remainder = tmin - tminus tplus = min(remainder, tplus) else: print ">> Both sides stick out so cut as symmetrically as possible." tminus = half_max tplus = tmin - half_max # odd trial sticks out on right if tplus + tminus < tmin: offsource = None temp_segment = segments.segment( (onSourceSegment[0] - tminus - pad_data, onSourceSegment[1] + tplus + pad_data)) if temp_segment is not None: offsource = temp_segment ifolist = list(ifo_combo) if extend: # extend as many adjacent 128 second blocks as possible begin_time = offsource[0] - quanta * ( abs(super_seg[0] - offsource[0]) // quanta) end_time = offsource[1] + quanta * ( abs(super_seg[1] - offsource[1]) // quanta) offsource = segments.segment((begin_time, end_time)) break print # check length at least minscilength if abs(offsource) < minscilength: print abs(offsource), minscilength print "EXIT: Calculated offsource segment but less than minscilength!" sys.exit() # check if no detectors can be used then exit if len(ifolist) < 2: print "EXIT: Calculated offsource segment but less than two IFOs!" sys.exit() # check edge case if abs(offsource[0] - onsource[0]) < pad_data or abs(offsource[1] - onsource[1]) < pad_data: print "WARNING: GRB time close to edge of offsource. Its within the padding time." # concatenate "H1L1V1", etc. ifolist.sort() ifotag = "".join(ifolist) print ">> Offsource segment for %s GRB is:" % ifotag print "Start:", offsource[0], "End:", offsource[1], "Duration:", offsource[ 1] - offsource[0], "Left:", grb_time - offsource[ 0], "Right:", offsource[1] - grb_time print ############################################################################## # output ############################################################################## # write analyse txt files for ifo in basic_ifolist: if ifo in ifolist: analysisFP = open('%s-analyse_grb%s.txt' % (ifo, grb_name), 'w') analysisFP.write('# seg\t start \t stop \t duration\n') analysisFP.write( '0\t %d\t %d\t %d\n' % (offsource[0], offsource[1], offsource[1] - offsource[0])) else: analysisFP = open('%s-analyse_grb%s.txt' % (ifo, grb_name), 'w') analysisFP.write('# seg\t start \t stop \t duration\n') # calculate blockDuration blockDuration = int(abs(offsource[0] - offsource[1])) - 2 * pad_data # calculate psdDuration # gets largest power of two such that blockDuration/psdDuration = psdRatio # could have done a binary & operator that is faster but this is more user-friendly I believe min_psdDuration = int(cp.get('exttrig', 'min-psd-length')) psdRatio = int(cp.get('exttrig', 'psd-ratio')) psdDuration = 2**int(numpy.log2(blockDuration / psdRatio)) if psdDuration < min_psdDuration: print "EXIT: PSD segment duration is too short. It is %ds but needs to be at least %ds in length." % ( psdDuration, min_psdDuration) sys.exit() # some downstream processes (e.g. lalapps_tmpltbank) cannot handle these inputs if cp.has_option('data', 'segment-duration'): cp.remove_option('data', 'segment-duration') cp.remove_option('data', 'block-duration') # some downstream processes (e.g. lalapps_tmpltbank) requires these options to run print ">> Using sample rate of %d to calculate inputs for downstream processes." % sampleRate print segmentLength = segmentDuration * sampleRate segmentCount = blockDuration / ( segmentDuration / 2) - 1 # subtract 1 because one segment length is overlapped segmentOverlap = segmentLength / 2 cp.set('data', 'segment-length', segmentLength) cp.set('data', 'number-of-segments', segmentCount) cp.set('inspiral', 'segment-overlap', segmentOverlap) # set values for [coh_PTF_inspral] section in configuration file cp.set('coh_PTF_inspiral', 'block-duration', blockDuration) cp.set('coh_PTF_inspiral', 'segment-duration', segmentDuration) cp.set('coh_PTF_inspiral', 'psd-segment-duration', psdDuration) cp.set('coh_PTF_inspiral', 'pad-data', pad_data) f = open('grb%s.ini' % grb_name, 'w') cp.write(f) f.close() print ">> The [data] section of the configuration file has been edited with the following values:" print "sample-rate=", sampleRate print "segment-length=", segmentLength print "number-of-segments=", segmentCount print "segment-overlap=", segmentOverlap print print ">> The [coh_PTF_inspiral] section of the configuration file has been edited with the following values:" print "block-duration =", blockDuration print "segment-duration =", segmentDuration print "psd-segment-duration =", psdDuration print "pad-data =", pad_data print # plot segments offSourceSegment = segments.segment(offsource[0], offsource[1]) plot_window = segments.segment(grb_time - offset, grb_time + offset) plot_segments(basic_segdict, onSourceSegment, offSourceSegment, grb_time, plot_window, "segment_plot_%s.png" % grb_name, grb_name) # make xml file if make_xml: # create a new xml document with an ExtTriggers Table xmldoc = ligolw.Document() xmldoc.appendChild(ligolw.LIGO_LW()) tbl = lsctables.New(lsctables.ExtTriggersTable) xmldoc.childNodes[-1].appendChild(tbl) # set the values we need row = lsctables.ExtTriggersTable() row.process_id = None row.det_alts = None row.det_band = None row.det_fluence = None row.det_fluence_int = None row.det_name = None row.det_peak = None row.det_peak_int = None row.det_snr = '' row.email_time = 0 row.event_dec = float(grb_dec) row.event_dec_err = 0.0 row.event_epoch = '' row.event_err_type = '' row.event_ra = float(grb_ra) row.event_ra_err = 0.0 row.start_time = grb_time row.start_time_ns = 0 row.event_type = '' row.event_z = 0.0 row.event_z_err = 0.0 row.notice_comments = '' row.notice_id = '' row.notice_sequence = '' row.notice_time = 0 row.notice_type = '' row.notice_url = '' row.obs_fov_dec = 0.0 row.obs_fov_dec_width = 0.0 row.obs_fov_ra = 0.0 row.obs_fov_ra_width = 0.0 row.obs_loc_ele = 0.0 row.obs_loc_lat = 0.0 row.obs_loc_long = 0.0 row.ligo_fave_lho = 0.0 row.ligo_fave_llo = 0.0 row.ligo_delay = 0.0 row.event_number_gcn = 9999 row.event_number_grb = grb_name row.event_status = 0 # insert into the table and write file tbl.extend([row]) filename = 'grb%s.xml' % grb_name utils.write_filename(xmldoc, filename) # plot all vetoes if make_plots: vetodict = segments.segmentlistdict() for cat in catlist: for ifo in ifolist: vetofile = "%s-VETOTIME_CAT%s_grb%s.xml" % (ifo, cat, grb_name) xmldoc = utils.load_filename(vetofile, gz=False, contenthandler=lsctables.use_in( ligolw.LIGOLWContentHandler)) segs = table.get_table(xmldoc, lsctables.SegmentTable.tableName) segdefs = table.get_table(xmldoc, lsctables.SegmentDefTable.tableName) vetodict[ifo] = segments.segmentlist( segments.segment(s.start_time, s.end_time) for s in segs) if vetodict: plot_segments(vetodict, onSourceSegment, offSourceSegment, grb_time, plot_window, "veto_plot_CAT%s_%s.png" % (cat, grb_name), "%s CAT%s" % (grb_name, cat)) # return return 'grb%s.ini' % grb_name, ifolist, onSourceSegment, offSourceSegment