def ligolw_bucut(xmldoc, options, burst_test_func, veto_segments=segments.segmentlistdict(), del_non_coincs=False, del_skipped_injections=False, program=None, verbose=False): contents = DocContents(xmldoc, program) process = append_process(xmldoc, options) apply_filters(contents, burst_test_func, veto_segments, del_non_coincs=del_non_coincs, del_skipped_injections=del_skipped_injections, verbose=verbose) ligolw_process.set_process_end_time(process) seg = contents.outsegs.extent_all() ligolw_search_summary.append_search_summary(xmldoc, process, inseg=seg, outseg=seg, nevents=len( contents.snglbursttable)) return xmldoc
def gen_likelihood_control(coinc_params_distributions, seglists, name=u"ligolw_burca_tailor", comment=u""): xmldoc = ligolw.Document() node = xmldoc.appendChild(ligolw.LIGO_LW()) process = ligolw_process.register_to_xmldoc(xmldoc, program=process_program_name, paramdict={}, version=__version__, cvs_repository="lscsoft", cvs_entry_time=__date__, comment=comment) coinc_params_distributions.process_id = process.process_id ligolw_search_summary.append_search_summary(xmldoc, process, ifos=seglists.keys(), inseg=seglists.extent_all(), outseg=seglists.extent_all()) node.appendChild(coinc_params_distributions.to_xml(name)) ligolw_process.set_process_end_time(process) return xmldoc
def gen_likelihood_control(coinc_params_distributions, seglists, name = u"ligolw_burca_tailor", comment = u""): xmldoc = ligolw.Document() node = xmldoc.appendChild(ligolw.LIGO_LW()) process = ligolw_process.register_to_xmldoc(xmldoc, program = process_program_name, paramdict = {}, version = __version__, cvs_repository = "lscsoft", cvs_entry_time = __date__, comment = comment) coinc_params_distributions.process_id = process.process_id ligolw_search_summary.append_search_summary(xmldoc, process, ifos = seglists.keys(), inseg = seglists.extent_all(), outseg = seglists.extent_all()) node.appendChild(coinc_params_distributions.to_xml(name)) ligolw_process.set_process_end_time(process) return xmldoc
def ligolw_bucut(xmldoc, options, burst_test_func, veto_segments = segments.segmentlistdict(), del_non_coincs = False, del_skipped_injections = False, program = None, verbose = False): contents = DocContents(xmldoc, program) process = append_process(xmldoc, options) apply_filters(contents, burst_test_func, veto_segments, del_non_coincs = del_non_coincs, del_skipped_injections = del_skipped_injections, verbose = verbose) ligolw_process.set_process_end_time(process) seg = contents.outsegs.extent_all() ligolw_search_summary.append_search_summary(xmldoc, process, inseg = seg, outseg = seg, nevents = len(contents.snglbursttable)) return xmldoc
def ligolw_sicluster(doc, **kwargs): # Extract segments and tables inseg, outseg, snglinspiraltable = get_tables(doc) # Add process information try: process = append_process(doc, **kwargs) except ValueError: process = None # Delete all triggers below threshold if kwargs["snr_threshold"] > 0: thresh = float(kwargs["snr_threshold"]) if kwargs["verbose"]: print >>sys.stderr, "discarding triggers with snr < %f ..." % \ kwargs["snr_threshold"] for i in range(len(snglinspiraltable) - 1, -1, -1): if snglinspiraltable[i].snr <= thresh: del snglinspiraltable[i] # Cluster snglcluster.cluster_events( snglinspiraltable, testfunc=lambda a, b: SnglInspiralUtils.CompareSnglInspiral( a, b, twindow=kwargs["cluster_window"]), clusterfunc=SnglInspiralCluster, sortfunc=SnglInspiralUtils.CompareSnglInspiralByEndTime, bailoutfunc=lambda a, b: SnglInspiralUtils.CompareSnglInspiral( a, b, twindow=kwargs["cluster_window"]), verbose=kwargs["verbose"]) # Sort by signal-to-noise ratio if kwargs["sort_ascending_snr"] or kwargs["sort_descending_snr"]: if kwargs["verbose"]: print >> sys.stderr, "sorting by snr ..." snglinspiraltable.sort(SnglInspiralUtils.CompareSnglInspiralBySnr) if kwargs["sort_descending_snr"]: snglinspiraltable.reverse() # Add search summary information if process and inseg and outseg: ligolw_search_summary.append_search_summary( doc, process, inseg=inseg, outseg=outseg, nevents=len(snglinspiraltable)) if process: ligolw_process.set_process_end_time(process) return doc
def ligolw_sicluster(doc, **kwargs): # Extract segments and tables inseg, outseg, snglinspiraltable = get_tables(doc) # Add process information try: process = append_process(doc, **kwargs) except ValueError: process = None # Delete all triggers below threshold if kwargs["snr_threshold"] > 0: thresh = float(kwargs["snr_threshold"]) if kwargs["verbose"]: print >>sys.stderr, "discarding triggers with snr < %f ..." % \ kwargs["snr_threshold"] for i in range(len(snglinspiraltable) - 1, -1, -1): if snglinspiraltable[i].snr <= thresh: del snglinspiraltable[i] # Cluster snglcluster.cluster_events( snglinspiraltable, testfunc = lambda a, b: SnglInspiralUtils.CompareSnglInspiral(a, b, twindow = kwargs["cluster_window"]), clusterfunc = SnglInspiralCluster, sortfunc = SnglInspiralUtils.CompareSnglInspiralByEndTime, bailoutfunc = lambda a, b: SnglInspiralUtils.CompareSnglInspiral(a, b, twindow = kwargs["cluster_window"]), verbose = kwargs["verbose"] ) # Sort by signal-to-noise ratio if kwargs["sort_ascending_snr"] or kwargs["sort_descending_snr"]: if kwargs["verbose"]: print >>sys.stderr, "sorting by snr ..." snglinspiraltable.sort(SnglInspiralUtils.CompareSnglInspiralBySnr) if kwargs["sort_descending_snr"]: snglinspiraltable.reverse() # Add search summary information if process and inseg and outseg: ligolw_search_summary.append_search_summary(doc, process, inseg = inseg, outseg = outseg, nevents = len(snglinspiraltable)) if process: ligolw_process.set_process_end_time(process) return doc
def create_xml(ts_data,psd_segment_length,window_fraction,event_list,station,setname="MagneticFields"): __program__ = 'pyburst_excesspower' start_time = LIGOTimeGPS(int(ts_data.start_time)) end_time = LIGOTimeGPS(int(ts_data.end_time)) inseg = segment(start_time,end_time) xmldoc = ligolw.Document() xmldoc.appendChild(ligolw.LIGO_LW()) ifo = 'H1'#channel_name.split(":")[0] straindict = psd.insert_psd_option_group.__dict__ proc_row = register_to_xmldoc(xmldoc, __program__,straindict, ifos=[ifo],version=git_version.id, cvs_repository=git_version.branch, cvs_entry_time=git_version.date) outseg = determine_output_segment(inseg, psd_segment_length, ts_data.sample_rate, window_fraction) ss = append_search_summary(xmldoc, proc_row, ifos=(station,), inseg=inseg, outseg=outseg) for sb in event_list: sb.process_id = proc_row.process_id sb.search = proc_row.program sb.ifo, sb.channel = station, setname xmldoc.childNodes[0].appendChild(event_list) fname = make_filename(station, inseg) utils.write_filename(xmldoc, fname, gz=fname.endswith("gz"))
# Write process metadata to output file. process = command.register_to_xmldoc(out_xmldoc, parser, opts, ifos=opts.detector, comment="Simulated coincidences") # Add search summary to output file. all_time = segments.segment( [glue.lal.LIGOTimeGPS(0), glue.lal.LIGOTimeGPS(2e9)]) search_summary_table = lsctables.New(lsctables.SearchSummaryTable) out_xmldoc.childNodes[0].appendChild(search_summary_table) summary = ligolw_search_summary.append_search_summary(out_xmldoc, process, inseg=all_time, outseg=all_time) # Read PSDs. progress.update(-1, 'reading ' + opts.reference_psd.name) xmldoc, _ = ligolw_utils.load_fileobj( opts.reference_psd, contenthandler=lal.series.PSDContentHandler) psds = lal.series.read_psd_xmldoc(xmldoc, root_name=None) psds = { key: timing.InterpolatedPSD(filter.abscissa(psd), psd.data.data) for key, psd in psds.items() if psd is not None } # Read injection file. progress.update(-1, 'reading ' + opts.input.name) xmldoc, _ = ligolw_utils.load_fileobj(
def ligolw_bucluster( xmldoc, program, process, prefunc, postfunc, testfunc, clusterfunc, sortfunc = None, bailoutfunc = None, verbose = False ): """ Run the clustering algorithm on the list of burst candidates. The return value is the tuple (xmldoc, changed), where xmldoc is the input document, and changed is a boolean that is True if the contents of the sngl_burst table were altered, and False if the triggers were not modified by the clustering process. If the document does not contain a sngl_burst table, then the document is not modified (including no modifications to the process metadata tables). """ # # Extract live time segment and sngl_burst table # try: sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc) except ValueError: # no-op: document does not contain a sngl_burst table if verbose: print >>sys.stderr, "document does not contain a sngl_burst table, skipping ..." return xmldoc, False seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = program).coalesce() # # Remove all H2 triggers intersecting the frequency band # 1138.6 Hz -- 1216.0 Hz # # FIXME: put this into the excess power pipeline, correctly # #bad_band = segments.segment(1138.586956521739, 1216.0326086956522) #for i in xrange(len(sngl_burst_table) - 1, -1, -1): # a = sngl_burst_table[i] # if a.ifo == "H2" and a.band.intersects(bad_band): # del sngl_burst_table[i] # # Preprocess candidates # if verbose: print >>sys.stderr, "pre-processing ..." preprocess_output = prefunc(sngl_burst_table) # # Cluster # table_changed = snglcluster.cluster_events(sngl_burst_table, testfunc, clusterfunc, sortfunc = sortfunc, bailoutfunc = bailoutfunc, verbose = verbose) # # Postprocess candidates # if verbose: print >>sys.stderr, "post-processing ..." postfunc(sngl_burst_table, preprocess_output) # # Update instrument list in process table and add search summary # information # process.instruments = seglists.keys() ligolw_search_summary.append_search_summary(xmldoc, process, inseg = seglists.extent_all(), outseg = seglists.extent_all(), nevents = len(sngl_burst_table)) # # Done # return xmldoc, table_changed
distributions.numerator.increment(params, weight = weight_func(sim)) # # Clean up. # contents.xmldoc.unlink() connection.close() dbtables.discard_connection_filename(filename, working_filename, verbose = options.verbose) # # Output. # ligolw_search_summary.append_search_summary(xmldoc, process, ifos = segs.keys(), inseg = segs.extent_all(), outseg = segs.extent_all()) xmldoc.childNodes[-1].appendChild(distributions.to_xml(u"string_cusp_likelihood")) ligolw_process.set_process_end_time(process) def T010150_basename(instruments, description, seg): start = int(math.floor(seg[0])) duration = int(math.ceil(seg[1] - start)) return "%s-%s-%d-%d" % ("+".join(sorted(instruments)), description, start, duration) if options.T010150: filename = "%s.xml.gz" % T010150_basename(segs.keys(), options.T010150, segs.extent_all()) else: filename = options.output ligolw_utils.write_filename(xmldoc, filename, verbose = verbose, gz = (filename or "stdout").endswith(".gz"))
def fake_trigger_generator(instrument='H1'): """ Generate fake trigger maps. Parameters ---------- instrument : str Instrument name """ xmldoc = ligolw.Document() xmldoc.appendChild(ligolw.LIGO_LW()) # Process information proc = process.append_process(xmldoc, "fake_search") process.append_process_params(xmldoc, proc, {}) t0 = 1e9 ntrig = 1000 ifo = instrument inseg = segment(LIGOTimeGPS(t0), LIGOTimeGPS(t0 + ntrig / 10)) outseg = segment(LIGOTimeGPS(t0), LIGOTimeGPS(t0 + ntrig / 10)) # Search summary search_summary.append_search_summary(xmldoc, proc, comment="Fake triggers", ifos=(ifo, ), inseg=inseg, outseg=outseg) columns = [ 'chisq_dof', 'bandwidth', 'central_freq', 'confidence', 'peak_time_ns', 'start_time', 'process_id', 'fhigh', 'stop_time_ns', 'channel', 'ifo', 'duration', 'event_id', 'hrss', 'stop_time', 'peak_time', 'snr', 'search', 'start_time_ns', 'flow', 'amplitude' ] table = lsctables.New(lsctables.SnglBurstTable, columns) # Generate uniformly distributed trigger times with approximate rate of 10 s times = t0 + uniform.rvs(0, ntrig / 10., ntrig) for t in times: row = table.RowType() # time frequency position and extent row.chisq_dof = int(2 + expon.rvs(2)) row.duration = 1. / 2**int(uniform.rvs(0, 7)) row.bandwidth = row.chisq_dof / row.duration / 2 row.central_freq = uniform.rvs(16, 2048) row.flow = max(row.central_freq - row.bandwidth, 0) row.fhigh = min(row.central_freq + row.bandwidth, 2048) ns, sec = math.modf(t) ns = int("%09d" % (ns * 1e9)) row.peak_time, row.peak_time_ns = int(sec), ns ns, sec = math.modf(t - row.duration / 2) ns = int("%09d" % (ns * 1e9)) row.start_time, row.start_time_ns = int(sec), ns ns, sec = math.modf(t + row.duration / 2) ns = int("%09d" % (ns * 1e9)) row.stop_time, row.stop_time_ns = int(sec), ns # TODO: Correlate some triggers, an upward fluctuation often triggers a few # tiles ontop of each other # SNR and confidence row.snr = 5. while row.snr < 2 * row.chisq_dof: row.snr = chi2.rvs(row.chisq_dof) row.confidence = chi2.sf(row.snr, row.chisq_dof) row.snr = math.sqrt(row.snr / row.chisq_dof - 1) row.hrss = row.amplitude = 1e-21 # metadata row.search = "fake_search" row.channel = "FAKE" row.ifo = ifo row.event_id = table.get_next_id() row.process_id = proc.process_id table.append(row) xmldoc.childNodes[0].appendChild(table) utils.write_filename(xmldoc, "%s-FAKE_SEARCH-%d-%d.xml.gz" % (ifo, int(t0), 10000), gz=True)
data_length = sample_rate * data_duration # data length in samples # Open output file. out_xmldoc = ligolw.Document() out_xmldoc.appendChild(ligolw.LIGO_LW()) # Write process metadata to output file. process = command.register_to_xmldoc( out_xmldoc, parser, opts, ifos=opts.detector, comment="Little hope!") # Add search summary to output file. all_time = segments.segment([glue.lal.LIGOTimeGPS(0), glue.lal.LIGOTimeGPS(2e9)]) search_summary_table = lsctables.New(lsctables.SearchSummaryTable) out_xmldoc.childNodes[0].appendChild(search_summary_table) summary = ligolw_search_summary.append_search_summary(out_xmldoc, process, inseg=all_time, outseg=all_time) # Read template bank file. progress.update(-1, 'reading ' + opts.template_bank.name) xmldoc, _ = ligolw_utils.load_fileobj( opts.template_bank, contenthandler=ligolw_bayestar.LSCTablesContentHandler) # Determine the low frequency cutoff from the template bank file. template_bank_f_low = ligolw_bayestar.get_template_bank_f_low(xmldoc) template_bank = ligolw_table.get_table(xmldoc, lsctables.SnglInspiralTable.tableName) # Read injection file. progress.update(-1, 'reading ' + opts.input.name) xmldoc, _ = ligolw_utils.load_fileobj(
def main(args=None): p = parser() opts = p.parse_args(args) # LIGO-LW XML imports. from glue.ligolw import ligolw from glue.ligolw.param import Param from glue.ligolw.utils import process as ligolw_process from glue.ligolw.utils.search_summary import append_search_summary from glue.ligolw import utils as ligolw_utils from glue.ligolw.lsctables import (New, CoincDefTable, CoincID, CoincInspiralTable, CoincMapTable, CoincTable, ProcessParamsTable, ProcessTable, SimInspiralTable, SnglInspiralTable, TimeSlideTable) # glue, LAL and pylal imports. from ligo import segments import glue.lal import lal.series import lalsimulation from lalinspiral.inspinjfind import InspiralSCExactCoincDef from lalinspiral.thinca import InspiralCoincDef from tqdm import tqdm # FIXME: disable progress bar monitor thread. # # I was getting error messages that look like this: # # Traceback (most recent call last): # File "/tqdm/_tqdm.py", line 885, in __del__ # self.close() # File "/tqdm/_tqdm.py", line 1090, in close # self._decr_instances(self) # File "/tqdm/_tqdm.py", line 454, in _decr_instances # cls.monitor.exit() # File "/tqdm/_monitor.py", line 52, in exit # self.join() # File "/usr/lib64/python3.6/threading.py", line 1053, in join # raise RuntimeError("cannot join current thread") # RuntimeError: cannot join current thread # # I don't know what causes this... maybe a race condition in tqdm's cleanup # code. Anyway, this should disable the tqdm monitor thread entirely. tqdm.monitor_interval = 0 # BAYESTAR imports. from ..io.events.ligolw import ContentHandler from ..bayestar import filter # Read PSDs. xmldoc, _ = ligolw_utils.load_fileobj( opts.reference_psd, contenthandler=lal.series.PSDContentHandler) psds = lal.series.read_psd_xmldoc(xmldoc, root_name=None) psds = { key: filter.InterpolatedPSD(filter.abscissa(psd), psd.data.data) for key, psd in psds.items() if psd is not None } psds = [psds[ifo] for ifo in opts.detector] # Extract simulation table from injection file. inj_xmldoc, _ = ligolw_utils.load_fileobj(opts.input, contenthandler=ContentHandler) orig_sim_inspiral_table = SimInspiralTable.get_table(inj_xmldoc) # Prune injections that are outside distance limits. orig_sim_inspiral_table[:] = [ row for row in orig_sim_inspiral_table if opts.min_distance <= row.distance <= opts.max_distance ] # Open output file. xmldoc = ligolw.Document() xmlroot = xmldoc.appendChild(ligolw.LIGO_LW()) # Create tables. Process and ProcessParams tables are copied from the # injection file. coinc_def_table = xmlroot.appendChild(New(CoincDefTable)) coinc_inspiral_table = xmlroot.appendChild(New(CoincInspiralTable)) coinc_map_table = xmlroot.appendChild(New(CoincMapTable)) coinc_table = xmlroot.appendChild(New(CoincTable)) xmlroot.appendChild(ProcessParamsTable.get_table(inj_xmldoc)) xmlroot.appendChild(ProcessTable.get_table(inj_xmldoc)) sim_inspiral_table = xmlroot.appendChild(New(SimInspiralTable)) sngl_inspiral_table = xmlroot.appendChild(New(SnglInspiralTable)) time_slide_table = xmlroot.appendChild(New(TimeSlideTable)) # Write process metadata to output file. process = register_to_xmldoc(xmldoc, p, opts, ifos=opts.detector, comment="Simulated coincidences") # Add search summary to output file. all_time = segments.segment( [glue.lal.LIGOTimeGPS(0), glue.lal.LIGOTimeGPS(2e9)]) append_search_summary(xmldoc, process, inseg=all_time, outseg=all_time) # Create a time slide entry. Needed for coinc_event rows. time_slide_id = time_slide_table.get_time_slide_id( {ifo: 0 for ifo in opts.detector}, create_new=process) # Populate CoincDef table. inspiral_coinc_def = copy.copy(InspiralCoincDef) inspiral_coinc_def.coinc_def_id = coinc_def_table.get_next_id() coinc_def_table.append(inspiral_coinc_def) found_coinc_def = copy.copy(InspiralSCExactCoincDef) found_coinc_def.coinc_def_id = coinc_def_table.get_next_id() coinc_def_table.append(found_coinc_def) # Precompute values that are common to all simulations. detectors = [ lalsimulation.DetectorPrefixToLALDetector(ifo) for ifo in opts.detector ] responses = [det.response for det in detectors] locations = [det.location for det in detectors] if opts.jobs == 1: pool_map = map else: from .. import omp from multiprocessing import Pool omp.num_threads = 1 # disable OpenMP parallelism pool_map = Pool(opts.jobs).imap func = functools.partial(simulate, psds=psds, responses=responses, locations=locations, measurement_error=opts.measurement_error, f_low=opts.f_low, waveform=opts.waveform) # Make sure that each thread gets a different random number state. # We start by drawing a random integer s in the main thread, and # then the i'th subprocess will seed itself with the integer i + s. # # The seed must be an unsigned 32-bit integer, so if there are n # threads, then s must be drawn from the interval [0, 2**32 - n). # # Note that *we* are thread 0, so there are a total of # n=1+len(sim_inspiral_table) threads. seed = np.random.randint(0, 2**32 - len(sim_inspiral_table) - 1) np.random.seed(seed) count_coincs = 0 with tqdm(total=len(orig_sim_inspiral_table)) as progress: for sim_inspiral, simulation in zip( orig_sim_inspiral_table, pool_map( func, zip( np.arange(len(orig_sim_inspiral_table)) + seed + 1, orig_sim_inspiral_table))): progress.update() sngl_inspirals = [] used_snr_series = [] net_snr = 0.0 count_triggers = 0 # Loop over individual detectors and create SnglInspiral entries. for ifo, (horizon, abs_snr, arg_snr, toa, series) \ in zip(opts.detector, simulation): if np.random.uniform() > opts.duty_cycle: continue elif abs_snr >= opts.snr_threshold: # If SNR < threshold, then the injection is not found. # Skip it. count_triggers += 1 net_snr += np.square(abs_snr) elif not opts.keep_subthreshold: continue # Create SnglInspiral entry. used_snr_series.append(series) sngl_inspirals.append( sngl_inspiral_table.RowType(**dict( dict.fromkeys(sngl_inspiral_table.validcolumns, None), process_id=process.process_id, ifo=ifo, mass1=sim_inspiral.mass1, mass2=sim_inspiral.mass2, spin1x=sim_inspiral.spin1x, spin1y=sim_inspiral.spin1y, spin1z=sim_inspiral.spin1z, spin2x=sim_inspiral.spin2x, spin2y=sim_inspiral.spin2y, spin2z=sim_inspiral.spin2z, end=toa, snr=abs_snr, coa_phase=arg_snr, eff_distance=horizon / abs_snr))) net_snr = np.sqrt(net_snr) # If too few triggers were found, then skip this event. if count_triggers < opts.min_triggers: continue # If network SNR < threshold, then the injection is not found. # Skip it. if net_snr < opts.net_snr_threshold: continue # Add Coinc table entry. coinc = coinc_table.appendRow( coinc_event_id=coinc_table.get_next_id(), process_id=process.process_id, coinc_def_id=inspiral_coinc_def.coinc_def_id, time_slide_id=time_slide_id, insts=opts.detector, nevents=len(opts.detector), likelihood=None) # Add CoincInspiral table entry. coinc_inspiral_table.appendRow( coinc_event_id=coinc.coinc_event_id, instruments=[ sngl_inspiral.ifo for sngl_inspiral in sngl_inspirals ], end=lal.LIGOTimeGPS(1e-9 * np.mean([ sngl_inspiral.end.ns() for sngl_inspiral in sngl_inspirals if sngl_inspiral.end is not None ])), mass=sim_inspiral.mass1 + sim_inspiral.mass2, mchirp=sim_inspiral.mchirp, combined_far=0.0, # Not provided false_alarm_rate=0.0, # Not provided minimum_duration=None, # Not provided snr=net_snr) # Record all sngl_inspiral records and associate them with coincs. for sngl_inspiral, series in zip(sngl_inspirals, used_snr_series): # Give this sngl_inspiral record an id and add it to the table. sngl_inspiral.event_id = sngl_inspiral_table.get_next_id() sngl_inspiral_table.append(sngl_inspiral) if opts.enable_snr_series: elem = lal.series.build_COMPLEX8TimeSeries(series) elem.appendChild( Param.from_pyvalue(u'event_id', sngl_inspiral.event_id)) xmlroot.appendChild(elem) # Add CoincMap entry. coinc_map_table.appendRow( coinc_event_id=coinc.coinc_event_id, table_name=sngl_inspiral_table.tableName, event_id=sngl_inspiral.event_id) # Record injection if not opts.preserve_ids: sim_inspiral.simulation_id = sim_inspiral_table.get_next_id() sim_inspiral_table.append(sim_inspiral) count_coincs += 1 progress.set_postfix(saved=count_coincs) # Record coincidence associating injections with events. for i, sim_inspiral in enumerate(sim_inspiral_table): coinc = coinc_table.appendRow( coinc_event_id=coinc_table.get_next_id(), process_id=process.process_id, coinc_def_id=found_coinc_def.coinc_def_id, time_slide_id=time_slide_id, instruments=None, nevents=None, likelihood=None) coinc_map_table.appendRow(coinc_event_id=coinc.coinc_event_id, table_name=sim_inspiral_table.tableName, event_id=sim_inspiral.simulation_id) coinc_map_table.appendRow(coinc_event_id=coinc.coinc_event_id, table_name=coinc_table.tableName, event_id=CoincID(i)) # Record process end time. ligolw_process.set_process_end_time(process) # Write output file. write_fileobj(xmldoc, opts.output)
def bucluster( xmldoc, program, process, prefunc, postfunc, testfunc, clusterfunc, sortfunc = None, bailoutfunc = None, verbose = False ): """ Run the clustering algorithm on the list of burst candidates. The return value is the tuple (xmldoc, changed), where xmldoc is the input document, and changed is a boolean that is True if the contents of the sngl_burst table were altered, and False if the triggers were not modified by the clustering process. If the document does not contain a sngl_burst table, then the document is not modified (including no modifications to the process metadata tables). """ # # Extract live time segment and sngl_burst table # try: sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc) except ValueError: # no-op: document does not contain a sngl_burst table if verbose: print >>sys.stderr, "document does not contain a sngl_burst table, skipping ..." return xmldoc, False seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = program).coalesce() # # Preprocess candidates # if verbose: print >>sys.stderr, "pre-processing ..." preprocess_output = prefunc(sngl_burst_table) # # Cluster # table_changed = snglcluster.cluster_events(sngl_burst_table, testfunc, clusterfunc, sortfunc = sortfunc, bailoutfunc = bailoutfunc, verbose = verbose) # # Postprocess candidates # if verbose: print >>sys.stderr, "post-processing ..." postfunc(sngl_burst_table, preprocess_output) # # Update instrument list in process table and add search summary # information # process.instruments = seglists.keys() ligolw_search_summary.append_search_summary(xmldoc, process, inseg = seglists.extent_all(), outseg = seglists.extent_all(), nevents = len(sngl_burst_table)) # # Done # return xmldoc, table_changed
def bucluster(xmldoc, program, process, prefunc, postfunc, testfunc, clusterfunc, sortfunc=None, bailoutfunc=None, verbose=False): """ Run the clustering algorithm on the list of burst candidates. The return value is the tuple (xmldoc, changed), where xmldoc is the input document, and changed is a boolean that is True if the contents of the sngl_burst table were altered, and False if the triggers were not modified by the clustering process. If the document does not contain a sngl_burst table, then the document is not modified (including no modifications to the process metadata tables). """ # # Extract live time segment and sngl_burst table # try: sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc) except ValueError: # no-op: document does not contain a sngl_burst table if verbose: print >> sys.stderr, "document does not contain a sngl_burst table, skipping ..." return xmldoc, False seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary( xmldoc, program=program).coalesce() # # Preprocess candidates # if verbose: print >> sys.stderr, "pre-processing ..." preprocess_output = prefunc(sngl_burst_table) # # Cluster # table_changed = snglcluster.cluster_events(sngl_burst_table, testfunc, clusterfunc, sortfunc=sortfunc, bailoutfunc=bailoutfunc, verbose=verbose) # # Postprocess candidates # if verbose: print >> sys.stderr, "post-processing ..." postfunc(sngl_burst_table, preprocess_output) # # Update instrument list in process table and add search summary # information # process.instruments = seglists.keys() ligolw_search_summary.append_search_summary(xmldoc, process, inseg=seglists.extent_all(), outseg=seglists.extent_all(), nevents=len(sngl_burst_table)) # # Done # return xmldoc, table_changed
def excess_power2( ts_data, # Time series from magnetic field data psd_segment_length, # Length of each segment in seconds psd_segment_stride, # Separation between 2 consecutive segments in seconds psd_estimation, # Average method window_fraction, # Withening window fraction tile_fap, # Tile false alarm probability threshold in Gaussian noise. station, # Station nchans=None, # Total number of channels band=None, # Channel bandwidth fmin=0, # Lowest frequency of the filter bank. fmax=None, # Highest frequency of the filter bank. max_duration=None, # Maximum duration of the tile wtype='tukey'): # Whitening type, can tukey or hann """ Perform excess-power search analysis on magnetic field data. This method will produce a bunch of time-frequency plots for every tile duration and bandwidth analysed as well as a XML file identifying all the triggers found in the selected data within the user-defined time range. Parameters ---------- ts_data : TimeSeries Time Series from magnetic field data psd_segment_length : float Length of each segment in seconds psd_segment_stride : float Separation between 2 consecutive segments in seconds psd_estimation : string Average method window_fraction : float Withening window fraction tile_fap : float Tile false alarm probability threshold in Gaussian noise. nchans : int Total number of channels band : float Channel bandwidth fmin : float Lowest frequency of the filter bank. fmax : float Highest frequency of the filter bank """ # Determine sampling rate based on extracted time series sample_rate = ts_data.sample_rate # Check if tile maximum frequency is not defined if fmax is None or fmax > sample_rate / 2.: # Set the tile maximum frequency equal to the Nyquist frequency # (i.e. half the sampling rate) fmax = sample_rate / 2.0 # Check whether or not tile bandwidth and channel are defined if band is None and nchans is None: # Exit program with error message exit("Either bandwidth or number of channels must be specified...") else: # Check if tile maximum frequency larger than its minimum frequency assert fmax >= fmin # Define spectral band of data data_band = fmax - fmin # Check whether tile bandwidth or channel is defined if band is not None: # Define number of possible filter bands nchans = int(data_band / band) - 1 elif nchans is not None: # Define filter bandwidth band = data_band / nchans nchans = nchans - 1 # Check if number of channels is superior than unity assert nchans > 1 # Print segment information print '|- Estimating PSD from segments of time', print '%.2f s in length, with %.2f s stride...' % (psd_segment_length, psd_segment_stride) # Convert time series as array of float data = ts_data.astype(numpy.float64) # Define segment length for PSD estimation in sample unit seg_len = int(psd_segment_length * sample_rate) # Define separation between consecutive segments in sample unit seg_stride = int(psd_segment_stride * sample_rate) # Calculate the overall PSD from individual PSD segments fd_psd = psd.welch(data, avg_method=psd_estimation, seg_len=seg_len, seg_stride=seg_stride) # We need this for the SWIG functions... lal_psd = fd_psd.lal() # Plot the power spectral density plot_spectrum(fd_psd) # Create whitening window print "|- Whitening window and spectral correlation..." if wtype == 'hann': window = lal.CreateHannREAL8Window(seg_len) elif wtype == 'tukey': window = lal.CreateTukeyREAL8Window(seg_len, window_fraction) else: raise ValueError("Can't handle window type %s" % wtype) # Create FFT plan fft_plan = lal.CreateForwardREAL8FFTPlan(len(window.data.data), 1) # Perform two point spectral correlation spec_corr = lal.REAL8WindowTwoPointSpectralCorrelation(window, fft_plan) # Initialise filter bank print "|- Create filter..." filter_bank, fdb = [], [] # Loop for each channels for i in range(nchans): channel_flow = fmin + band / 2 + i * band channel_width = band # Create excess power filter lal_filter = lalburst.CreateExcessPowerFilter(channel_flow, channel_width, lal_psd, spec_corr) filter_bank.append(lal_filter) fdb.append(Spectrum.from_lal(lal_filter)) # Calculate the minimum bandwidth min_band = (len(filter_bank[0].data.data) - 1) * filter_bank[0].deltaF / 2 # Plot filter bank plot_bank(fdb) # Convert filter bank from frequency to time domain print "|- Convert all the frequency domain to the time domain..." tdb = [] # Loop for each filter's spectrum for fdt in fdb: zero_padded = numpy.zeros(int((fdt.f0 / fdt.df).value) + len(fdt)) st = int((fdt.f0 / fdt.df).value) zero_padded[st:st + len(fdt)] = numpy.real_if_close(fdt.value) n_freq = int(sample_rate / 2 / fdt.df.value) * 2 tdt = numpy.fft.irfft(zero_padded, n_freq) * math.sqrt(sample_rate) tdt = numpy.roll(tdt, len(tdt) / 2) tdt = TimeSeries(tdt, name="", epoch=fdt.epoch, sample_rate=sample_rate) tdb.append(tdt) # Plot time series filter plot_filters(tdb, fmin, band) # Compute the renormalization for the base filters up to a given bandwidth. mu_sq_dict = {} # Loop through powers of 2 up to number of channels for nc_sum in range(0, int(math.log(nchans, 2))): nc_sum = 2**nc_sum - 1 print "|- Calculating renormalization for resolution level containing %d %fHz channels" % ( nc_sum + 1, min_band) mu_sq = (nc_sum + 1) * numpy.array([ lalburst.ExcessPowerFilterInnerProduct(f, f, spec_corr, None) for f in filter_bank ]) # Uncomment to get all possible frequency renormalizations #for n in xrange(nc_sum, nchans): # channel position index for n in xrange(nc_sum, nchans, nc_sum + 1): # channel position index for k in xrange(0, nc_sum): # channel sum index # FIXME: We've precomputed this, so use it instead mu_sq[n] += 2 * lalburst.ExcessPowerFilterInnerProduct( filter_bank[n - k], filter_bank[n - 1 - k], spec_corr, None) #print mu_sq[nc_sum::nc_sum+1] mu_sq_dict[nc_sum] = mu_sq # Create an event list where all the triggers will be stored event_list = lsctables.New(lsctables.SnglBurstTable, [ 'start_time', 'start_time_ns', 'peak_time', 'peak_time_ns', 'duration', 'bandwidth', 'central_freq', 'chisq_dof', 'confidence', 'snr', 'amplitude', 'channel', 'ifo', 'process_id', 'event_id', 'search', 'stop_time', 'stop_time_ns' ]) # Create repositories to save TF and time series plots os.system('mkdir -p segments/time-frequency') os.system('mkdir -p segments/time-series') # Define time edges t_idx_min, t_idx_max = 0, seg_len while t_idx_max <= len(ts_data): # Define starting and ending time of the segment in seconds start_time = ts_data.start_time + t_idx_min / float( ts_data.sample_rate) end_time = ts_data.start_time + t_idx_max / float(ts_data.sample_rate) print "\n|-- Analyzing block %i to %i (%.2f percent)" % ( start_time, end_time, 100 * float(t_idx_max) / len(ts_data)) # Model a withen time series for the block tmp_ts_data = types.TimeSeries(ts_data[t_idx_min:t_idx_max] * window.data.data, delta_t=1. / ts_data.sample_rate, epoch=start_time) # Save time series in relevant repository segfolder = 'segments/%i-%i' % (start_time, end_time) os.system('mkdir -p ' + segfolder) plot_ts(tmp_ts_data, fname='segments/time-series/%i-%i.png' % (start_time, end_time)) # Convert times series to frequency series fs_data = tmp_ts_data.to_frequencyseries() print "|-- Frequency series data has variance: %s" % fs_data.data.std( )**2 # Whitening (FIXME: Whiten the filters, not the data) fs_data.data /= numpy.sqrt(fd_psd) / numpy.sqrt(2 * fd_psd.delta_f) print "|-- Whitened frequency series data has variance: %s" % fs_data.data.std( )**2 print "|-- Create time-frequency plane for current block" # Return the complex snr, along with its associated normalization of the template, # matched filtered against the data #filter.matched_filter_core(types.FrequencySeries(tmp_filter_bank,delta_f=fd_psd.delta_f), # fs_data,h_norm=1,psd=fd_psd,low_frequency_cutoff=filter_bank[0].f0, # high_frequency_cutoff=filter_bank[0].f0+2*band) print "|-- Filtering all %d channels..." % nchans # Initialise 2D zero array tmp_filter_bank = numpy.zeros(len(fd_psd), dtype=numpy.complex128) # Initialise 2D zero array for time-frequency map tf_map = numpy.zeros((nchans, seg_len), dtype=numpy.complex128) # Loop over all the channels for i in range(nchans): # Reset filter bank series tmp_filter_bank *= 0.0 # Index of starting frequency f1 = int(filter_bank[i].f0 / fd_psd.delta_f) # Index of ending frequency f2 = int((filter_bank[i].f0 + 2 * band) / fd_psd.delta_f) + 1 # (FIXME: Why is there a factor of 2 here?) tmp_filter_bank[f1:f2] = filter_bank[i].data.data * 2 # Define the template to filter the frequency series with template = types.FrequencySeries(tmp_filter_bank, delta_f=fd_psd.delta_f, copy=False) # Create filtered series filtered_series = filter.matched_filter_core( template, fs_data, h_norm=None, psd=None, low_frequency_cutoff=filter_bank[i].f0, high_frequency_cutoff=filter_bank[i].f0 + 2 * band) # Include filtered series in the map tf_map[i, :] = filtered_series[0].numpy() # Plot spectrogram plot_spectrogram(numpy.abs(tf_map).T, tmp_ts_data.delta_t, band, ts_data.sample_rate, start_time, end_time, fname='segments/time-frequency/%i-%i.png' % (start_time, end_time)) # Loop through all summed channels for nc_sum in range(0, int(math.log(nchans, 2)))[::-1]: nc_sum = 2**nc_sum - 1 mu_sq = mu_sq_dict[nc_sum] # Clip the boundaries to remove window corruption clip_samples = int(psd_segment_length * window_fraction * ts_data.sample_rate / 2) # Constructing tile and calculate their energy print "\n|--- Constructing tile with %d summed channels..." % ( nc_sum + 1) # Current bandwidth of the time-frequency map tiles df = band * (nc_sum + 1) dt = 1.0 / (2 * df) # How much each "step" is in the time domain -- under sampling rate us_rate = int(round(dt / ts_data.delta_t)) print "|--- Undersampling rate for this level: %f" % ( ts_data.sample_rate / us_rate) print "|--- Calculating tiles..." # Making independent tiles # because [0:-0] does not give the full array tf_map_temp = tf_map[:,clip_samples:-clip_samples:us_rate] \ if clip_samples > 0 else tf_map[:,::us_rate] tiles = tf_map_temp.copy() # Here's the deal: we're going to keep only the valid output and # it's *always* going to exist in the lowest available indices stride = nc_sum + 1 for i in xrange(tiles.shape[0] / stride): numpy.absolute(tiles[stride * i:stride * (i + 1)].sum(axis=0), tiles[stride * (i + 1) - 1]) tiles = tiles[nc_sum::nc_sum + 1].real**2 / mu_sq[nc_sum::nc_sum + 1].reshape( -1, 1) print "|--- TF-plane is %dx%s samples" % tiles.shape print "|--- Tile energy mean %f, var %f" % (numpy.mean(tiles), numpy.var(tiles)) # Define maximum number of degrees of freedom and check it larger or equal to 2 max_dof = 32 if max_duration == None else 2 * max_duration * df assert max_dof >= 2 # Loop through multiple degrees of freedom for j in [2**l for l in xrange(0, int(math.log(max_dof, 2)))]: # Duration is fixed by the NDOF and bandwidth duration = j * dt print "\n|----- Explore signal duration of %f s..." % duration print "|----- Summing DOF = %d ..." % (2 * j) tlen = tiles.shape[1] - 2 * j + 1 + 1 dof_tiles = numpy.zeros((tiles.shape[0], tlen)) sum_filter = numpy.array([1, 0] * (j - 1) + [1]) for f in range(tiles.shape[0]): # Sum and drop correlate tiles dof_tiles[f] = fftconvolve(tiles[f], sum_filter, 'valid') print "|----- Summed tile energy mean: %f, var %f" % ( numpy.mean(dof_tiles), numpy.var(dof_tiles)) plot_spectrogram( dof_tiles.T, dt, df, ts_data.sample_rate, start_time, end_time, fname='segments/%i-%i/tf_%02ichans_%02idof.png' % (start_time, end_time, nc_sum + 1, 2 * j)) threshold = scipy.stats.chi2.isf(tile_fap, j) print "|------ Threshold for this level: %f" % threshold spant, spanf = dof_tiles.shape[1] * dt, dof_tiles.shape[0] * df print "|------ Processing %.2fx%.2f time-frequency map." % ( spant, spanf) # Since we clip the data, the start time needs to be adjusted accordingly window_offset_epoch = fs_data.epoch + psd_segment_length * window_fraction / 2 window_offset_epoch = LIGOTimeGPS(float(window_offset_epoch)) for i, j in zip(*numpy.where(dof_tiles > threshold)): event = event_list.RowType() # The points are summed forward in time and thus a `summed point' is the # sum of the previous N points. If this point is above threshold, it # corresponds to a tile which spans the previous N points. However, the # 0th point (due to the convolution specifier 'valid') is actually # already a duration from the start time. All of this means, the + # duration and the - duration cancels, and the tile 'start' is, by # definition, the start of the time frequency map if j = 0 # FIXME: I think this needs a + dt/2 to center the tile properly event.set_start(window_offset_epoch + float(j * dt)) event.set_stop(window_offset_epoch + float(j * dt) + duration) event.set_peak(event.get_start() + duration / 2) event.central_freq = filter_bank[ 0].f0 + band / 2 + i * df + 0.5 * df event.duration = duration event.bandwidth = df event.chisq_dof = 2 * duration * df event.snr = math.sqrt(dof_tiles[i, j] / event.chisq_dof - 1) # FIXME: Magic number 0.62 should be determine empircally event.confidence = -lal.LogChisqCCDF( event.snr * 0.62, event.chisq_dof * 0.62) event.amplitude = None event.process_id = None event.event_id = event_list.get_next_id() event_list.append(event) for event in event_list[::-1]: if event.amplitude != None: continue etime_min_idx = float(event.get_start()) - float( fs_data.epoch) etime_min_idx = int(etime_min_idx / tmp_ts_data.delta_t) etime_max_idx = float(event.get_start()) - float( fs_data.epoch) + event.duration etime_max_idx = int(etime_max_idx / tmp_ts_data.delta_t) # (band / 2) to account for sin^2 wings from finest filters flow_idx = int((event.central_freq - event.bandwidth / 2 - (df / 2) - fmin) / df) fhigh_idx = int((event.central_freq + event.bandwidth / 2 + (df / 2) - fmin) / df) # TODO: Check that the undersampling rate is always commensurate # with the indexing: that is to say that # mod(etime_min_idx, us_rate) == 0 always z_j_b = tf_map[flow_idx:fhigh_idx, etime_min_idx:etime_max_idx:us_rate] event.amplitude = 0 print "|------ Total number of events: %d" % len(event_list) t_idx_min += int(seg_len * (1 - window_fraction)) t_idx_max += int(seg_len * (1 - window_fraction)) setname = "MagneticFields" __program__ = 'pyburst_excesspower' start_time = LIGOTimeGPS(int(ts_data.start_time)) end_time = LIGOTimeGPS(int(ts_data.end_time)) inseg = segment(start_time, end_time) xmldoc = ligolw.Document() xmldoc.appendChild(ligolw.LIGO_LW()) ifo = 'H1' #channel_name.split(":")[0] straindict = psd.insert_psd_option_group.__dict__ proc_row = register_to_xmldoc(xmldoc, __program__, straindict, ifos=[ifo], version=git_version.id, cvs_repository=git_version.branch, cvs_entry_time=git_version.date) dt_stride = psd_segment_length sample_rate = ts_data.sample_rate # Amount to overlap successive blocks so as not to lose data window_overlap_samples = window_fraction * sample_rate outseg = inseg.contract(window_fraction * dt_stride / 2) # With a given dt_stride, we cannot process the remainder of this data remainder = math.fmod(abs(outseg), dt_stride * (1 - window_fraction)) # ...so make an accounting of it outseg = segment(outseg[0], outseg[1] - remainder) ss = append_search_summary(xmldoc, proc_row, ifos=(station, ), inseg=inseg, outseg=outseg) for sb in event_list: sb.process_id = proc_row.process_id sb.search = proc_row.program sb.ifo, sb.channel = station, setname xmldoc.childNodes[0].appendChild(event_list) fname = 'excesspower.xml.gz' utils.write_filename(xmldoc, fname, gz=fname.endswith("gz"))
def ligolw_bucluster(xmldoc, program, process, prefunc, postfunc, testfunc, clusterfunc, sortfunc=None, bailoutfunc=None, verbose=False): """ Run the clustering algorithm on the list of burst candidates. The return value is the tuple (xmldoc, changed), where xmldoc is the input document, and changed is a boolean that is True if the contents of the sngl_burst table were altered, and False if the triggers were not modified by the clustering process. If the document does not contain a sngl_burst table, then the document is not modified (including no modifications to the process metadata tables). """ # # Extract live time segment and sngl_burst table # try: sngl_burst_table = lsctables.SnglBurstTable.get_table(xmldoc) except ValueError: # no-op: document does not contain a sngl_burst table if verbose: print >> sys.stderr, "document does not contain a sngl_burst table, skipping ..." return xmldoc, False seglists = ligolw_search_summary.segmentlistdict_fromsearchsummary( xmldoc, program=program).coalesce() # # Remove all H2 triggers intersecting the frequency band # 1138.6 Hz -- 1216.0 Hz # # FIXME: put this into the excess power pipeline, correctly # #bad_band = segments.segment(1138.586956521739, 1216.0326086956522) #for i in xrange(len(sngl_burst_table) - 1, -1, -1): # a = sngl_burst_table[i] # if a.ifo == "H2" and a.band.intersects(bad_band): # del sngl_burst_table[i] # # Preprocess candidates # if verbose: print >> sys.stderr, "pre-processing ..." preprocess_output = prefunc(sngl_burst_table) # # Cluster # table_changed = snglcluster.cluster_events(sngl_burst_table, testfunc, clusterfunc, sortfunc=sortfunc, bailoutfunc=bailoutfunc, verbose=verbose) # # Postprocess candidates # if verbose: print >> sys.stderr, "post-processing ..." postfunc(sngl_burst_table, preprocess_output) # # Update instrument list in process table and add search summary # information # process.instruments = seglists.keys() ligolw_search_summary.append_search_summary(xmldoc, process, inseg=seglists.extent_all(), outseg=seglists.extent_all(), nevents=len(sngl_burst_table)) # # Done # return xmldoc, table_changed
# Clean up. # contents.xmldoc.unlink() connection.close() dbtables.discard_connection_filename(filename, working_filename, verbose=options.verbose) # # Output. # ligolw_search_summary.append_search_summary(xmldoc, process, ifos=segs.keys(), inseg=segs.extent_all(), outseg=segs.extent_all()) xmldoc.childNodes[-1].appendChild( distributions.to_xml(u"string_cusp_likelihood")) ligolw_process.set_process_end_time(process) def T010150_basename(instruments, description, seg): start = int(math.floor(seg[0])) duration = int(math.ceil(seg[1] - start)) return "%s-%s-%d-%d" % ("+".join( sorted(instruments)), description, start, duration) if options.T010150:
def excess_power( ts_data, # Time series from magnetic field data band=None, # Channel bandwidth channel_name='channel-name', # Channel name fmin=0, # Lowest frequency of the filter bank. fmax=None, # Highest frequency of the filter bank. impulse=False, # Impulse response make_plot=True, # Condition to produce plots max_duration=None, # Maximum duration of the tile nchans=256, # Total number of channels psd_estimation='median-mean', # Average method psd_segment_length=60, # Length of each segment in seconds psd_segment_stride=30, # Separation between 2 consecutive segments in seconds station='station-name', # Station name tile_fap=1e-7, # Tile false alarm probability threshold in Gaussian noise. verbose=True, # Print details window_fraction=0, # Withening window fraction wtype='tukey'): # Whitening type, can tukey or hann ''' Perform excess-power search analysis on magnetic field data. This method will produce a bunch of time-frequency plots for every tile duration and bandwidth analysed as well as a XML file identifying all the triggers found in the selected data within the user-defined time range. Parameters ---------- ts_data : TimeSeries Time Series from magnetic field data psd_segment_length : float Length of each segment in seconds psd_segment_stride : float Separation between 2 consecutive segments in seconds psd_estimation : string Average method window_fraction : float Withening window fraction tile_fap : float Tile false alarm probability threshold in Gaussian noise. nchans : int Total number of channels band : float Channel bandwidth fmin : float Lowest frequency of the filter bank. fmax : float Highest frequency of the filter bank Examples -------- The program can be ran as an executable by using the ``excesspower`` command line as follows:: excesspower --station "mainz01" \\ --start-time "2017-04-15-17-1" \\ --end-time "2017-04-15-18" \\ --rep "/Users/vincent/ASTRO/data/GNOME/GNOMEDrive/gnome/serverdata/" \\ --resample 512 \\ --verbose ''' # Determine sampling rate based on extracted time series sample_rate = ts_data.sample_rate # Check if tile maximum frequency is not defined if fmax is None or fmax > sample_rate / 2.: # Set the tile maximum frequency equal to the Nyquist frequency # (i.e. half the sampling rate) fmax = sample_rate / 2.0 # Check whether or not tile bandwidth and channel are defined if band is None and nchans is None: # Exit program with error message exit("Either bandwidth or number of channels must be specified...") else: # Check if tile maximum frequency larger than its minimum frequency assert fmax >= fmin # Define spectral band of data data_band = fmax - fmin # Check whether tile bandwidth or channel is defined if band is not None: # Define number of possible filter bands nchans = int(data_band / band) elif nchans is not None: # Define filter bandwidth band = data_band / nchans nchans -= 1 # Check if number of channels is superior than unity assert nchans > 1 # Print segment information if verbose: print '|- Estimating PSD from segments of', if verbose: print '%.2f s, with %.2f s stride...' % (psd_segment_length, psd_segment_stride) # Convert time series as array of float data = ts_data.astype(numpy.float64) # Define segment length for PSD estimation in sample unit seg_len = int(psd_segment_length * sample_rate) # Define separation between consecutive segments in sample unit seg_stride = int(psd_segment_stride * sample_rate) # Minimum frequency of detectable signal in a segment delta_f = 1. / psd_segment_length # Calculate PSD length counting the zero frequency element fd_len = fmax / delta_f + 1 # Calculate the overall PSD from individual PSD segments if impulse: # Produce flat data flat_data = numpy.ones(int(fd_len)) * 2. / fd_len # Create PSD frequency series fd_psd = types.FrequencySeries(flat_data, 1. / psd_segment_length, ts_data.start_time) else: # Create overall PSD using Welch's method fd_psd = psd.welch(data, avg_method=psd_estimation, seg_len=seg_len, seg_stride=seg_stride) if make_plot: # Plot the power spectral density plot_spectrum(fd_psd) # We need this for the SWIG functions lal_psd = fd_psd.lal() # Create whitening window if verbose: print "|- Whitening window and spectral correlation..." if wtype == 'hann': window = lal.CreateHannREAL8Window(seg_len) elif wtype == 'tukey': window = lal.CreateTukeyREAL8Window(seg_len, window_fraction) else: raise ValueError("Can't handle window type %s" % wtype) # Create FFT plan fft_plan = lal.CreateForwardREAL8FFTPlan(len(window.data.data), 1) # Perform two point spectral correlation spec_corr = lal.REAL8WindowTwoPointSpectralCorrelation(window, fft_plan) # Determine length of individual filters filter_length = int(2 * band / fd_psd.delta_f) + 1 # Initialise filter bank if verbose: print "|- Create bank of %i filters of %i Hz bandwidth..." % ( nchans, filter_length) # Initialise array to store filter's frequency series and metadata lal_filters = [] # Initialise array to store filter's time series fdb = [] # Loop over the channels for i in range(nchans): # Define central position of the filter freq = fmin + band / 2 + i * band # Create excess power filter lal_filter = lalburst.CreateExcessPowerFilter(freq, band, lal_psd, spec_corr) # Testing spectral correlation on filter #print lalburst.ExcessPowerFilterInnerProduct(lal_filter, lal_filter, spec_corr, None) # Append entire filter structure lal_filters.append(lal_filter) # Append filter's spectrum fdb.append(FrequencySeries.from_lal(lal_filter)) #print fdb[0].frequencies #print fdb[0] if make_plot: # Plot filter bank plot_bank(fdb) # Convert filter bank from frequency to time domain if verbose: print "|- Convert all the frequency domain to the time domain..." tdb = [] # Loop for each filter's spectrum for fdt in fdb: zero_padded = numpy.zeros(int((fdt.f0 / fdt.df).value) + len(fdt)) st = int((fdt.f0 / fdt.df).value) zero_padded[st:st + len(fdt)] = numpy.real_if_close(fdt.value) n_freq = int(sample_rate / 2 / fdt.df.value) * 2 tdt = numpy.fft.irfft(zero_padded, n_freq) * math.sqrt(sample_rate) tdt = numpy.roll(tdt, len(tdt) / 2) tdt = TimeSeries(tdt, name="", epoch=fdt.epoch, sample_rate=sample_rate) tdb.append(tdt) # Plot time series filter plot_filters(tdb, fmin, band) # Computer whitened inner products of input filters with themselves #white_filter_ip = numpy.array([lalburst.ExcessPowerFilterInnerProduct(f, f, spec_corr, None) for f in lal_filters]) # Computer unwhitened inner products of input filters with themselves #unwhite_filter_ip = numpy.array([lalburst.ExcessPowerFilterInnerProduct(f, f, spec_corr, lal_psd) for f in lal_filters]) # Computer whitened filter inner products between input adjacent filters #white_ss_ip = numpy.array([lalburst.ExcessPowerFilterInnerProduct(f1, f2, spec_corr, None) for f1, f2 in zip(lal_filters[:-1], lal_filters[1:])]) # Computer unwhitened filter inner products between input adjacent filters #unwhite_ss_ip = numpy.array([lalburst.ExcessPowerFilterInnerProduct(f1, f2, spec_corr, lal_psd) for f1, f2 in zip(lal_filters[:-1], lal_filters[1:])]) # Check filter's bandwidth is equal to user defined channel bandwidth min_band = (len(lal_filters[0].data.data) - 1) * lal_filters[0].deltaF / 2 assert min_band == band # Create an event list where all the triggers will be stored event_list = lsctables.New(lsctables.SnglBurstTable, [ 'start_time', 'start_time_ns', 'peak_time', 'peak_time_ns', 'duration', 'bandwidth', 'central_freq', 'chisq_dof', 'confidence', 'snr', 'amplitude', 'channel', 'ifo', 'process_id', 'event_id', 'search', 'stop_time', 'stop_time_ns' ]) # Create repositories to save TF and time series plots os.system('mkdir -p segments/time-frequency') os.system('mkdir -p segments/time-series') # Define time edges t_idx_min, t_idx_max = 0, seg_len # Loop over each segment while t_idx_max <= len(ts_data): # Define first and last timestamps of the block start_time = ts_data.start_time + t_idx_min / float( ts_data.sample_rate) end_time = ts_data.start_time + t_idx_max / float(ts_data.sample_rate) if verbose: print "\n|- Analyzing block %i to %i (%.2f percent)" % ( start_time, end_time, 100 * float(t_idx_max) / len(ts_data)) # Debug for impulse response if impulse: for i in range(t_idx_min, t_idx_max): ts_data[i] = 1000. if i == (t_idx_max + t_idx_min) / 2 else 0. # Model a withen time series for the block tmp_ts_data = types.TimeSeries(ts_data[t_idx_min:t_idx_max] * window.data.data, delta_t=1. / ts_data.sample_rate, epoch=start_time) # Save time series in relevant repository os.system('mkdir -p segments/%i-%i' % (start_time, end_time)) if make_plot: # Plot time series plot_ts(tmp_ts_data, fname='segments/time-series/%i-%i.png' % (start_time, end_time)) # Convert times series to frequency series fs_data = tmp_ts_data.to_frequencyseries() if verbose: print "|- Frequency series data has variance: %s" % fs_data.data.std( )**2 # Whitening (FIXME: Whiten the filters, not the data) fs_data.data /= numpy.sqrt(fd_psd) / numpy.sqrt(2 * fd_psd.delta_f) if verbose: print "|- Whitened frequency series data has variance: %s" % fs_data.data.std( )**2 if verbose: print "|- Create time-frequency plane for current block" # Return the complex snr, along with its associated normalization of the template, # matched filtered against the data #filter.matched_filter_core(types.FrequencySeries(tmp_filter_bank,delta_f=fd_psd.delta_f), # fs_data,h_norm=1,psd=fd_psd,low_frequency_cutoff=lal_filters[0].f0, # high_frequency_cutoff=lal_filters[0].f0+2*band) if verbose: print "|- Filtering all %d channels...\n" % nchans, # Initialise 2D zero array tmp_filter_bank = numpy.zeros(len(fd_psd), dtype=numpy.complex128) # Initialise 2D zero array for time-frequency map tf_map = numpy.zeros((nchans, seg_len), dtype=numpy.complex128) # Loop over all the channels for i in range(nchans): # Reset filter bank series tmp_filter_bank *= 0.0 # Index of starting frequency f1 = int(lal_filters[i].f0 / fd_psd.delta_f) # Index of last frequency bin f2 = int((lal_filters[i].f0 + 2 * band) / fd_psd.delta_f) + 1 # (FIXME: Why is there a factor of 2 here?) tmp_filter_bank[f1:f2] = lal_filters[i].data.data * 2 # Define the template to filter the frequency series with template = types.FrequencySeries(tmp_filter_bank, delta_f=fd_psd.delta_f, copy=False) # Create filtered series filtered_series = filter.matched_filter_core( template, fs_data, h_norm=None, psd=None, low_frequency_cutoff=lal_filters[i].f0, high_frequency_cutoff=lal_filters[i].f0 + 2 * band) # Include filtered series in the map tf_map[i, :] = filtered_series[0].numpy() if make_plot: # Plot spectrogram plot_spectrogram(numpy.abs(tf_map).T, dt=tmp_ts_data.delta_t, df=band, ymax=ts_data.sample_rate / 2., t0=start_time, t1=end_time, fname='segments/time-frequency/%i-%i.png' % (start_time, end_time)) plot_tiles_ts(numpy.abs(tf_map), 2, 1, sample_rate=ts_data.sample_rate, t0=start_time, t1=end_time, fname='segments/%i-%i/ts.png' % (start_time, end_time)) #plot_tiles_tf(numpy.abs(tf_map),2,1,ymax=ts_data.sample_rate/2, # sample_rate=ts_data.sample_rate,t0=start_time,t1=end_time, # fname='segments/%i-%i/tf.png'%(start_time,end_time)) # Loop through powers of 2 up to number of channels for nc_sum in range(0, int(math.log(nchans, 2)))[::-1]: # Calculate total number of summed channels nc_sum = 2**nc_sum if verbose: print "\n\t|- Contructing tiles containing %d narrow band channels" % nc_sum # Compute full bandwidth of virtual channel df = band * nc_sum # Compute minimal signal's duration in virtual channel dt = 1.0 / (2 * df) # Compute under sampling rate us_rate = int(round(dt / ts_data.delta_t)) if verbose: print "\t|- Undersampling rate for this level: %f" % ( ts_data.sample_rate / us_rate) if verbose: print "\t|- Calculating tiles..." # Clip the boundaries to remove window corruption clip_samples = int(psd_segment_length * window_fraction * ts_data.sample_rate / 2) # Undersample narrow band channel's time series # Apply clipping condition because [0:-0] does not give the full array tf_map_temp = tf_map[:,clip_samples:-clip_samples:us_rate] \ if clip_samples > 0 else tf_map[:,::us_rate] # Initialise final tile time-frequency map tiles = numpy.zeros(((nchans + 1) / nc_sum, tf_map_temp.shape[1])) # Loop over tile index for i in xrange(len(tiles)): # Sum all inner narrow band channels ts_tile = numpy.absolute(tf_map_temp[nc_sum * i:nc_sum * (i + 1)].sum(axis=0)) # Define index of last narrow band channel for given tile n = (i + 1) * nc_sum - 1 n = n - 1 if n == len(lal_filters) else n # Computer withened inner products of each input filter with itself mu_sq = nc_sum * lalburst.ExcessPowerFilterInnerProduct( lal_filters[n], lal_filters[n], spec_corr, None) #kmax = nc_sum-1 if n==len(lal_filters) else nc_sum-2 # Loop over the inner narrow band channels for k in xrange(0, nc_sum - 1): # Computer whitened filter inner products between input adjacent filters mu_sq += 2 * lalburst.ExcessPowerFilterInnerProduct( lal_filters[n - k], lal_filters[n - 1 - k], spec_corr, None) # Normalise tile's time series tiles[i] = ts_tile.real**2 / mu_sq if verbose: print "\t|- TF-plane is %dx%s samples" % tiles.shape if verbose: print "\t|- Tile energy mean %f, var %f" % (numpy.mean(tiles), numpy.var(tiles)) # Define maximum number of degrees of freedom and check it larger or equal to 2 max_dof = 32 if max_duration == None else int(max_duration / dt) assert max_dof >= 2 # Loop through multiple degrees of freedom for j in [2**l for l in xrange(0, int(math.log(max_dof, 2)))]: # Duration is fixed by the NDOF and bandwidth duration = j * dt if verbose: print "\n\t\t|- Summing DOF = %d ..." % (2 * j) if verbose: print "\t\t|- Explore signal duration of %f s..." % duration # Construct filter sum_filter = numpy.array([1, 0] * (j - 1) + [1]) # Calculate length of filtered time series tlen = tiles.shape[1] - sum_filter.shape[0] + 1 # Initialise filtered time series array dof_tiles = numpy.zeros((tiles.shape[0], tlen)) # Loop over tiles for f in range(tiles.shape[0]): # Sum and drop correlate tiles dof_tiles[f] = fftconvolve(tiles[f], sum_filter, 'valid') if verbose: print "\t\t|- Summed tile energy mean: %f" % ( numpy.mean(dof_tiles)) if verbose: print "\t\t|- Variance tile energy: %f" % ( numpy.var(dof_tiles)) if make_plot: plot_spectrogram( dof_tiles.T, dt, df, ymax=ts_data.sample_rate / 2, t0=start_time, t1=end_time, fname='segments/%i-%i/%02ichans_%02idof.png' % (start_time, end_time, nc_sum, 2 * j)) plot_tiles_ts( dof_tiles, 2 * j, df, sample_rate=ts_data.sample_rate / us_rate, t0=start_time, t1=end_time, fname='segments/%i-%i/%02ichans_%02idof_ts.png' % (start_time, end_time, nc_sum, 2 * j)) plot_tiles_tf( dof_tiles, 2 * j, df, ymax=ts_data.sample_rate / 2, sample_rate=ts_data.sample_rate / us_rate, t0=start_time, t1=end_time, fname='segments/%i-%i/%02ichans_%02idof_tf.png' % (start_time, end_time, nc_sum, 2 * j)) threshold = scipy.stats.chi2.isf(tile_fap, j) if verbose: print "\t\t|- Threshold for this level: %f" % threshold spant, spanf = dof_tiles.shape[1] * dt, dof_tiles.shape[0] * df if verbose: print "\t\t|- Processing %.2fx%.2f time-frequency map." % ( spant, spanf) # Since we clip the data, the start time needs to be adjusted accordingly window_offset_epoch = fs_data.epoch + psd_segment_length * window_fraction / 2 window_offset_epoch = LIGOTimeGPS(float(window_offset_epoch)) for i, j in zip(*numpy.where(dof_tiles > threshold)): event = event_list.RowType() # The points are summed forward in time and thus a `summed point' is the # sum of the previous N points. If this point is above threshold, it # corresponds to a tile which spans the previous N points. However, the # 0th point (due to the convolution specifier 'valid') is actually # already a duration from the start time. All of this means, the + # duration and the - duration cancels, and the tile 'start' is, by # definition, the start of the time frequency map if j = 0 # FIXME: I think this needs a + dt/2 to center the tile properly event.set_start(window_offset_epoch + float(j * dt)) event.set_stop(window_offset_epoch + float(j * dt) + duration) event.set_peak(event.get_start() + duration / 2) event.central_freq = lal_filters[ 0].f0 + band / 2 + i * df + 0.5 * df event.duration = duration event.bandwidth = df event.chisq_dof = 2 * duration * df event.snr = math.sqrt(dof_tiles[i, j] / event.chisq_dof - 1) # FIXME: Magic number 0.62 should be determine empircally event.confidence = -lal.LogChisqCCDF( event.snr * 0.62, event.chisq_dof * 0.62) event.amplitude = None event.process_id = None event.event_id = event_list.get_next_id() event_list.append(event) for event in event_list[::-1]: if event.amplitude != None: continue etime_min_idx = float(event.get_start()) - float( fs_data.epoch) etime_min_idx = int(etime_min_idx / tmp_ts_data.delta_t) etime_max_idx = float(event.get_start()) - float( fs_data.epoch) + event.duration etime_max_idx = int(etime_max_idx / tmp_ts_data.delta_t) # (band / 2) to account for sin^2 wings from finest filters flow_idx = int((event.central_freq - event.bandwidth / 2 - (df / 2) - fmin) / df) fhigh_idx = int((event.central_freq + event.bandwidth / 2 + (df / 2) - fmin) / df) # TODO: Check that the undersampling rate is always commensurate # with the indexing: that is to say that # mod(etime_min_idx, us_rate) == 0 always z_j_b = tf_map[flow_idx:fhigh_idx, etime_min_idx:etime_max_idx:us_rate] # FIXME: Deal with negative hrss^2 -- e.g. remove the event try: event.amplitude = measure_hrss( z_j_b, unwhite_filter_ip[flow_idx:fhigh_idx], unwhite_ss_ip[flow_idx:fhigh_idx - 1], white_ss_ip[flow_idx:fhigh_idx - 1], fd_psd.delta_f, tmp_ts_data.delta_t, len(lal_filters[0].data.data), event.chisq_dof) except ValueError: event.amplitude = 0 if verbose: print "\t\t|- Total number of events: %d" % len(event_list) t_idx_min += int(seg_len * (1 - window_fraction)) t_idx_max += int(seg_len * (1 - window_fraction)) setname = "MagneticFields" __program__ = 'pyburst_excesspower_gnome' start_time = LIGOTimeGPS(int(ts_data.start_time)) end_time = LIGOTimeGPS(int(ts_data.end_time)) inseg = segment(start_time, end_time) xmldoc = ligolw.Document() xmldoc.appendChild(ligolw.LIGO_LW()) ifo = channel_name.split(":")[0] straindict = psd.insert_psd_option_group.__dict__ proc_row = register_to_xmldoc(xmldoc, __program__, straindict, ifos=[ifo], version=git_version.id, cvs_repository=git_version.branch, cvs_entry_time=git_version.date) dt_stride = psd_segment_length sample_rate = ts_data.sample_rate # Amount to overlap successive blocks so as not to lose data window_overlap_samples = window_fraction * sample_rate outseg = inseg.contract(window_fraction * dt_stride / 2) # With a given dt_stride, we cannot process the remainder of this data remainder = math.fmod(abs(outseg), dt_stride * (1 - window_fraction)) # ...so make an accounting of it outseg = segment(outseg[0], outseg[1] - remainder) ss = append_search_summary(xmldoc, proc_row, ifos=(station, ), inseg=inseg, outseg=outseg) for sb in event_list: sb.process_id = proc_row.process_id sb.search = proc_row.program sb.ifo, sb.channel = station, setname xmldoc.childNodes[0].appendChild(event_list) ifostr = ifo if isinstance(ifo, str) else "".join(ifo) st_rnd, end_rnd = int(math.floor(inseg[0])), int(math.ceil(inseg[1])) dur = end_rnd - st_rnd fname = "%s-excesspower-%d-%d.xml.gz" % (ifostr, st_rnd, dur) utils.write_filename(xmldoc, fname, gz=fname.endswith("gz")) plot_triggers(fname)