def event_rate(self, nevents=10): """ Calculate the Poissonian significance of the 'on source' trial set for up to the loudest nevents. """ offtime = float(abs(segments.segmentlist(self.offsource.keys()))) offsource = sorted(chain(*self.offsource.values()), key=lambda sb: -sb.snr) offrate = zip(offsource, map(lambda i: i / offtime, range(1, len(offsource) + 1))) offrate = offrate[::-1] offsource = offsource[::-1] offsnr = [sb.snr for sb in offsource] ontime = float(abs(segments.segmentlist(self.onsource.keys()))) if ontime == 0: return [] onsource = sorted(chain(*self.onsource.values()), key=lambda sb: -sb.snr) onsnr = [sb.snr for sb in onsource] onrate = [] for snr in onsnr: try: onrate.append(offrate[bisect_left(offsnr, snr)][1]) except IndexError: # on SNR > max off SNR onrate.append(0) return onrate
def generated_vdb_ascii(json_str,filepath): res_dict=json.loads(json_str) active_list=res_dict['active'] active_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in active_list]) known_list=res_dict['known'] known_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in known_list]) query_start=res_dict['query_information']['start'] query_stop=res_dict['query_information']['end'] if query_start!=0 and query_stop!=0: requested_span=segments.segmentlist([segments.segment(query_start,query_stop)]) else: requested_span=segments.segmentlist([segments.segment(0,9999999999)]) active_segments_string=',1 \n'.join([str(i[0])+","+str(i[1]) for i in active_segments])+",1 \n" unknown_segments=requested_span-known_segments unknown_segments_string=',-1 \n'.join([str(i[0])+","+str(i[1]) for i in unknown_segments])+",-1 \n" known_not_active_segments=known_segments-active_segments known_not_active_segments_string=',0 \n'.join([str(i[0])+","+str(i[1]) for i in known_not_active_segments])+",0 \n" output_fileh=open(filepath,'w+') query_info_string=json.dumps(res_dict['query_information'], indent=1) output_fileh.writelines(query_info_string) output_fileh.write('\n') output_fileh.writelines(active_segments_string) output_fileh.writelines(unknown_segments_string) output_fileh.writelines(known_not_active_segments_string) output_fileh.close() return filepath
def generated_vdb_ascii(json_dict,filepath): #res_dict=json.loads(json_str) res_dict=json_dict active_list=res_dict['active'] active_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in active_list]) active_segments.coalesce() known_list=res_dict['known'] known_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in known_list]) known_segments.coalesce() query_start=res_dict['query_information']['start'] query_stop=res_dict['query_information']['end'] if query_start!=0 and query_stop!=0: requested_span=segments.segmentlist([segments.segment(query_start,query_stop)]) else: requested_span=segments.segmentlist([segments.segment(0,9999999999)]) active_segments_string=',1 \n'.join([str(i[0])+","+str(i[1]) for i in active_segments])+",1 \n" unknown_segments=requested_span-known_segments unknown_segments_string=',-1 \n'.join([str(i[0])+","+str(i[1]) for i in unknown_segments])+",-1 \n" known_not_active_segments=known_segments-active_segments known_not_active_segments_string=',0 \n'.join([str(i[0])+","+str(i[1]) for i in known_not_active_segments])+",0 \n" output_fileh=open(filepath,'a') query_info_string=json.dumps(res_dict['query_information'], indent=1) output_fileh.writelines(query_info_string) output_fileh.write('\n') output_fileh.writelines(active_segments_string) output_fileh.writelines(unknown_segments_string) output_fileh.writelines(known_not_active_segments_string) output_fileh.close() return filepath
def setup_psd_calculate(workflow, frame_files, ifo, segments, segment_name, out_dir, tags=None): make_analysis_dir(out_dir) tags = [] if not tags else tags if workflow.cp.has_option_tags('workflow-psd', 'parallelization-factor', tags=tags): num_parts = int(workflow.cp.get_opt_tags('workflow-psd', 'parallelization-factor', tags=tags)) else: num_parts = 1 # get rid of duplicate segments which happen when splitting the bank segments = segmentlist(frozenset(segments)) segment_lists = list(chunks(segments, num_parts)) psd_files = FileList([]) for i, segs in enumerate(segment_lists): seg_file = SegFile.from_segment_list('%s_%s' %(segment_name, i), segmentlist(segs), segment_name, ifo, valid_segment=workflow.analysis_time, extension='xml', directory=out_dir) psd_files += [make_psd_file(workflow, frame_files, seg_file, segment_name, out_dir, tags=tags + ['PART%s' % i])] if num_parts > 1: return merge_psds(workflow, psd_files, ifo, out_dir, tags=tags) else: return psd_files[0]
def normalize(self): """ Redistribute events to offsource and onsource based on current time span. """ all_segs = segments.segmentlist(self.onsource.keys()) if len(all_segs) == 0: return if len(self.offsource.keys()) > 0: all_segs += segments.segmentlist(self.offsource.keys()) all_segs.coalesce() begin, end = all_segs[0][0], all_segs[-1][1] span = float(end - begin) if span < self.onsource_interval: # Not much we can do. return if span > self.offsource_interval + self.onsource_interval: begin = end - (self.offsource_interval + self.onsource_interval) onsource_seg = segments.segment(end - self.onsource_interval, end) offsource_seg = segments.segment(begin, end - self.onsource_interval) for seg, sbt in self.offsource.items(): try: seg & offsource_seg except ValueError: # offsource segment is out of the current window del self.offsource[seg] continue newseg = seg & offsource_seg if seg != newseg: del self.offsource[seg] self.offsource[newseg] = filter( lambda sb: (sb.peak_time + 1e-9 * sb.peak_time_ns) in newseg, sbt) for seg, sbt in self.onsource.items(): if seg in onsource_seg: continue elif offsource_seg.disjoint(seg) == 1: # segment ran off the span since last check del self.onsource[seg] continue offseg = seg & offsource_seg del self.onsource[seg] try: onseg = seg & onsource_seg self.onsource[onseg] = filter( lambda sb: (sb.peak_time + 1e-9 * sb.peak_time_ns) in onseg, sbt) except ValueError: # onsource segment completely out of new segment pass self.offsource[offseg] = filter( lambda sb: (sb.peak_time + 1e-9 * sb.peak_time_ns) in offseg, sbt)
def __init__(self,ifo,name,version): self.known=segments.segmentlist([]) self.active=segments.segmentlist([]) # self.metadata={} self.flagDict={} self.ifo=ifo self.name=name self.version=version
def __init__(self, instrument): self.fig, self.axes = SnglBurstUtils.make_burst_plot("%s Confidence" % instrument, "Coincident Event Rate (Hz)") self.instrument = instrument self.foreground = [] self.background = [] self.foreground_segs = segments.segmentlist() self.background_segs = segments.segmentlist() self.axes.loglog()
def __init__(self, instrument): self.fig, self.axes = SnglBurstUtils.make_burst_plot( "%s Confidence" % instrument, "Coincident Event Rate (Hz)") self.instrument = instrument self.foreground = [] self.background = [] self.foreground_segs = segments.segmentlist() self.background_segs = segments.segmentlist() self.axes.loglog()
def get_moon_segments(config_struct, segmentlist, observer, fxdbdy, radec): if "moon_constraint" in config_struct: moon_constraint = float(config_struct["moon_constraint"]) else: moon_constraint = 20.0 moonsegmentlist = segments.segmentlist() dt = 1.0 / 24.0 tt = np.arange(segmentlist[0][0], segmentlist[-1][1] + dt, dt) ra2 = radec.ra.radian d2 = radec.dec.radian # Where is the moon? moon = ephem.Moon() for ii in range(len(tt) - 1): observer.date = ephem.Date(Time(tt[ii], format='mjd', scale='utc').iso) moon.compute(observer) fxdbdy.compute(observer) alt_target = float(repr(fxdbdy.alt)) * (360 / (2 * np.pi)) az_target = float(repr(fxdbdy.az)) * (360 / (2 * np.pi)) #print("Altitude / Azimuth of target: %.5f / %.5f"%(alt_target,az_target)) alt_moon = float(repr(moon.alt)) * (360 / (2 * np.pi)) az_moon = float(repr(moon.az)) * (360 / (2 * np.pi)) #print("Altitude / Azimuth of moon: %.5f / %.5f"%(alt_moon,az_moon)) ra_moon = (180 / np.pi) * float(repr(moon.ra)) dec_moon = (180 / np.pi) * float(repr(moon.dec)) # Coverting both target and moon ra and dec to radians ra1 = float(repr(moon.ra)) d1 = float(repr(moon.dec)) # Calculate angle between target and moon cosA = np.sin(d1) * np.sin(d2) + np.cos(d1) * np.cos(d2) * np.cos(ra1 - ra2) angle = np.arccos(cosA) * (360 / (2 * np.pi)) #print("Angle between moon and target: %.5f"%(angle)) #if angle >= 50.0*moon.moon_phase**2: if angle >= moon_constraint: segment = segments.segment(tt[ii], tt[ii + 1]) moonsegmentlist = moonsegmentlist + segments.segmentlist([segment]) moonsegmentlist.coalesce() moonsegmentlistdic = segments.segmentlistdict() moonsegmentlistdic["observations"] = segmentlist moonsegmentlistdic["moon"] = moonsegmentlist moonsegmentlist = moonsegmentlistdic.intersection(["observations", "moon"]) moonsegmentlist.coalesce() return moonsegmentlist
def load_segments_from_xml(xml_doc, return_dict=False, select_id=None): """Read a ligo.segments.segmentlist from the file object file containing an xml segment table. Parameters ---------- xml_doc: name of segment xml file Keyword Arguments: return_dict : [ True | False ] return a ligo.segments.segmentlistdict containing coalesced ligo.segments.segmentlists keyed by seg_def.name for each entry in the contained segment_def_table. Default False select_id : int return a ligo.segments.segmentlist object containing only those segments matching the given segment_def_id integer """ # Load SegmentDefTable and SegmentTable seg_def_table = load_xml_table(xml_doc, glsctables.SegmentDefTable.tableName) seg_table = load_xml_table(xml_doc, glsctables.SegmentTable.tableName) if return_dict: segs = segments.segmentlistdict() else: segs = segments.segmentlist() seg_id = {} for seg_def in seg_def_table: seg_id[int(seg_def.segment_def_id)] = str(seg_def.name) if return_dict: segs[str(seg_def.name)] = segments.segmentlist() for seg in seg_table: if return_dict: segs[seg_id[int(seg.segment_def_id)]]\ .append(segments.segment(seg.start_time, seg.end_time)) continue if select_id and int(seg.segment_def_id) == select_id: segs.append(segments.segment(seg.start_time, seg.end_time)) continue segs.append(segments.segment(seg.start_time, seg.end_time)) if return_dict: for seg_name in seg_id.values(): segs[seg_name] = segs[seg_name].coalesce() else: segs = segs.coalesce() return segs
def get_segments(params, config_struct): gpstime = params["gpstime"] event_mjd = Time(gpstime, format='gps', scale='utc').mjd segmentlist = segments.segmentlist() n_windows = len(params["Tobs"]) // 2 start_segments = event_mjd + params["Tobs"][::2] end_segments = event_mjd + params["Tobs"][1::2] for start_segment, end_segment in zip(start_segments, end_segments): segmentlist.append(segments.segment(start_segment, end_segment)) observer = ephem.Observer() observer.lat = str(config_struct["latitude"]) observer.lon = str(config_struct["longitude"]) observer.horizon = str(-12.0) observer.elevation = config_struct["elevation"] date_start = ephem.Date( Time(segmentlist[0][0], format='mjd', scale='utc').iso) date_end = ephem.Date( Time(segmentlist[-1][1], format='mjd', scale='utc').iso) observer.date = ephem.Date( Time(segmentlist[0][0], format='mjd', scale='utc').iso) sun = ephem.Sun() nightsegmentlist = segments.segmentlist() while date_start < date_end: date_rise = observer.next_rising(sun, start=date_start) date_set = observer.next_setting(sun, start=date_start) if date_set > date_rise: date_set = observer.previous_setting(sun, start=date_start) astropy_rise = Time(date_rise.datetime(), scale='utc').mjd astropy_set = Time(date_set.datetime(), scale='utc').mjd segment = segments.segment(astropy_set, astropy_rise) nightsegmentlist = nightsegmentlist + segments.segmentlist([segment]) nightsegmentlist.coalesce() date_start = date_rise observer.date = date_rise segmentlistdic = segments.segmentlistdict() segmentlistdic["observations"] = segmentlist segmentlistdic["night"] = nightsegmentlist segmentlist = segmentlistdic.intersection(["observations", "night"]) segmentlist.coalesce() return segmentlist
def __init__(self,ifo,name,version,hackDec11=False): self.known=segments.segmentlist([]) self.active=segments.segmentlist([]) # self.metadata={} self.flagDict={} self.ifo=ifo self.name=name self.version=version self.temp_process_ids={} # Used to hold the data # # associated with a process_id if hackDec11: self.insert_history={} else: self.insert_history=[] # holds the process_metadatas and insertion_metadatas # Note that this assumes that proper dictionaries are appended to this list
def setup_psd_calculate(workflow, frame_files, ifo, segments, segment_name, out_dir, tags=None): make_analysis_dir(out_dir) tags = [] if not tags else tags if workflow.cp.has_option_tags('workflow-psd', 'parallelization-factor', tags=tags): num_parts = int( workflow.cp.get_opt_tags('workflow-psd', 'parallelization-factor', tags=tags)) else: num_parts = 1 # get rid of duplicate segments which happen when splitting the bank segments = segmentlist(frozenset(segments)) segment_lists = list(chunks(segments, num_parts)) psd_files = FileList([]) for i, segs in enumerate(segment_lists): seg_file = SegFile.from_segment_list( '%s_%s' % (segment_name, i), segmentlist(segs), segment_name, ifo, valid_segment=workflow.analysis_time, extension='xml', directory=out_dir) psd_files += [ make_psd_file(workflow, frame_files, seg_file, segment_name, out_dir, tags=tags + ['PART%s' % i]) ] if num_parts > 1: return merge_psds(workflow, psd_files, ifo, out_dir, tags=tags) else: return psd_files[0]
def construct_trials(seg_files, seg_dict, ifos, slide_dict, vetoes): """Constructs trials from triggers, timeslides, segments and vetoes""" trial_dict = {} # Get segments segs = read_seg_files(seg_files) # Separate segments trial_time = abs(segs['on']) for slide_id in slide_dict: # These can only *reduce* the analysis time curr_seg_list = seg_dict[slide_id] # Construct the buffer segment list seg_buffer = segments.segmentlist() for ifo in ifos: slide_offset = slide_dict[slide_id][ifo] seg_buffer.append( segments.segment(segs['buffer'][0] - slide_offset, segs['buffer'][1] - slide_offset)) seg_buffer.coalesce() # Construct the ifo-indexed dictionary of slid veteoes slid_vetoes = slide_vetoes(vetoes, slide_dict, slide_id) # Construct trial list and check against buffer trial_dict[slide_id] = segments.segmentlist() for curr_seg in curr_seg_list: iter_int = 1 while 1: trial_end = curr_seg[0] + trial_time * iter_int if trial_end > curr_seg[1]: break curr_trial = segments.segment(trial_end - trial_time, trial_end) if not seg_buffer.intersects_segment(curr_trial): intersect = numpy.any([ slid_vetoes[ifo].intersects_segment(curr_trial) for ifo in ifos ]) if not intersect: trial_dict[slide_id].append(curr_trial) iter_int += 1 return trial_dict
def get_summary(basedir, ifo, cluster, cat, start_time, end_time): all_sum = segmentlist([]) cur_time = start_time while cur_time < end_time: tstring = os.popen('tconvert -f %Y%m/%Y%m%d ' + str(cur_time)).readlines()[0].strip() infile = open('%s/%s/%s-0-SUMMARY_%s.csv' % (basedir, tstring, ifo, cluster)) lines = [l.strip().split(',') for l in infile.readlines()] summary = segmentlist([segment(int(l[0]), int(l[1])) for l in lines]).coalesce() all_sum = all_sum + summary cur_time += 60 * 60 * 24 all_sum = all_sum & segmentlist([segment(start_time, end_time)]) return all_sum
def split_segment(seg, min_segment_length, pad, overlap, short_segment_duration, max_job_length): # avoid infinite loop if min_segment_length + 2 * pad <= overlap: raise ValueError( "infinite loop: min_segment_length + 2 * pad must be > overlap") # clip max_job_length down to an allowed size max_job_length = clip_segment_length(max_job_length, pad, short_segment_duration) seglist = segments.segmentlist() while abs(seg) >= min_segment_length + 2 * pad: # try to use max_job_length each time if abs(seg) >= max_job_length: seglist.append(segments.segment(seg[0], seg[0] + max_job_length)) else: seglist.append( segments.segment( seg[0], seg[0] + clip_segment_length( abs(seg), pad, short_segment_duration))) assert abs(seglist[-1]) != 0 # safety-check for no-op # bounds must be integers if abs((int(seglist[-1][0]) - seglist[-1][0]) / seglist[-1][0]) > 1e-14 or abs( (int(seglist[-1][1]) - seglist[-1][1]) / seglist[-1][1]) > 1e-14: raise ValueError("segment %s does not have integer boundaries" % str(seglist[-1])) # advance segment seg = segments.segment(seglist[-1][1] - overlap, seg[1]) if not seglist: raise ValueError("unable to use segment %s" % str(seg)) return seglist
def convert_json_list_to_segmentlist(jsonlist): """ Helper function used to convert json list of lists type object to a segmentlist object """ segment_list=segments.segmentlist([segments.segment(x[0],x[1]) for x in jsonlist]) return segment_list
def segmentlistdict(self): """ A segmentlistdict object describing the instruments and time spanned by this CacheEntry. A new object is constructed each time this attribute is accessed (segments are immutable so there is no reason to try to share a reference to the CacheEntry's internal segment; modifications of one would not be reflected in the other anyway). Example: >>> c = CacheEntry("H1 S5 815901601 576.5 file://localhost/home/kipp/tmp/1/H1-815901601-576.xml") >>> c.segmentlistdict['H1'] [segment(LIGOTimeGPS(815901601, 0), LIGOTimeGPS(815902177, 500000000))] The \"observatory\" column of the cache entry, which is frequently used to store instrument names, is parsed into instrument names for the dictionary keys using the same rules as glue.ligolw.lsctables.instrumentsproperty.get(). Example: >>> c = CacheEntry("H1H2, S5 815901601 576.5 file://localhost/home/kipp/tmp/1/H1H2-815901601-576.xml") >>> c.segmentlistdict['H1H2'] [segment(LIGOTimeGPS(815901601, 0), LIGOTimeGPS(815902177, 500000000))] """ # the import has to be done here to break the cyclic # dependancy from glue.ligolw.lsctables import instrumentsproperty instruments = instrumentsproperty.get(self.observatory) or (None, ) return segments.segmentlistdict( (instrument, segments.segmentlist(self.segment is not None and [self.segment] or [])) for instrument in instruments)
def get_science_segs_from_datafind_outs(datafindcaches): """ This function will calculate the science segments that are covered in the OutGroupList containing the frame files returned by various calls to the datafind server. This can then be used to check whether this list covers what it is expected to cover. Parameters ---------- datafindcaches : OutGroupList List of all the datafind output files. Returns -------- newScienceSegs : Dictionary of ifo keyed glue.segment.segmentlist instances The times covered by the frames found in datafindOuts. """ newScienceSegs = {} for cache in datafindcaches: if len(cache) > 0: groupSegs = segments.segmentlist(e.segment for e in cache).coalesce() ifo = cache.ifo if ifo not in newScienceSegs: newScienceSegs[ifo] = groupSegs else: newScienceSegs[ifo].extend(groupSegs) newScienceSegs[ifo].coalesce() return newScienceSegs
def get_science_segs_from_datafind_outs(datafindcaches): """ This function will calculate the science segments that are covered in the OutGroupList containing the frame files returned by various calls to the datafind server. This can then be used to check whether this list covers what it is expected to cover. Parameters ---------- datafindcaches : OutGroupList List of all the datafind output files. Returns -------- newScienceSegs : Dictionary of ifo keyed ligo.segments.segmentlist instances The times covered by the frames found in datafindOuts. """ newScienceSegs = {} for cache in datafindcaches: if len(cache) > 0: groupSegs = segments.segmentlist(e.segment for e in cache).coalesce() ifo = cache.ifo if ifo not in newScienceSegs: newScienceSegs[ifo] = groupSegs else: newScienceSegs[ifo].extend(groupSegs) newScienceSegs[ifo].coalesce() return newScienceSegs
def add_events(self, sbtable, inseg=None): """ Add a trial to the current running tally. If segment is provided, then the key in the trial table is set to be this. Otherwise, the segment is determined from the peak times of the snglbursts """ # If no events are provided and no segment is indicated, there is no # operation to map this into a trial, so we do nothing if len(sbtable) == 0 and inseg is None: return if inseg is None: inseg = [] for sb in sbtable: start = sb.start_time + 1e-9 * sb.start_time_ns stop = sb.start_time + sb.duration inseg.append(segments.segment(start, stop)) inseg = segments.segmentlist(inseg).coalesce() inseg = segments.segment(inseg[0][0], inseg[-1][1]) oldsegs = filter(lambda s: s.intersects(inseg), self.onsource.keys()) # FIXME: Is it possible for this to be > 1? # Yes, but the reorganization logic is tricky. # Call normalize often (like everytime you add a new segment). if len(oldsegs) == 1: oldseg = oldsegs[0] sbtable += self.onsource[oldseg] del self.onsource[oldseg] inseg = oldseg | inseg self.onsource[inseg] = sbtable
def get_exposures(params, config_struct, segmentlist): ''' Convert the availability times to a list segments with the length of telescope exposures. segmentlist: the segments that the telescope can do the follow-up. ''' exposurelist = segments.segmentlist() if "overhead_per_exposure" in config_struct.keys(): overhead = config_struct["overhead_per_exposure"] else: overhead = 0.0 # add the filter change time to the total overheads for integrated if not params["doAlternatingFilters"]: overhead = overhead + config_struct["filt_change_time"] exposure_time = np.max(params["exposuretimes"]) for ii in range(len(segmentlist)): start_segment, end_segment = segmentlist[ii][0], segmentlist[ii][1] exposures = np.arange(start_segment, end_segment, (overhead + exposure_time) / 86400.0) for jj in range(len(exposures)): exposurelist.append( segments.segment(exposures[jj], exposures[jj] + exposure_time / 86400.0)) return exposurelist
def __init__(self, ifo): self.fig, self.axes = SnglBurstUtils.make_burst_plot("Confidence", "Trigger Rate (Hz)") self.ifo = ifo self.nevents = 0 self.x = [] self.seglist = segments.segmentlist() self.axes.loglog()
def indices_within_segments(times, segment_files, ifo=None, segment_name=None): """ Return the list of indices that should be vetoed by the segments in the list of veto_files. Parameters ---------- times: numpy.ndarray of integer type Array of gps start times segment_files: string or list of strings A string or list of strings that contain the path to xml files that contain a segment table ifo: string, optional The ifo to retrieve segments for from the segment files segment_name: str, optional name of segment Returns ------- indices: numpy.ndarray The array of index values within the segments segmentlist: The segment list corresponding to the selected time. """ veto_segs = segmentlist([]) indices = numpy.array([], dtype=numpy.uint32) for veto_file in segment_files: veto_segs += select_segments_by_definer(veto_file, segment_name, ifo) veto_segs.coalesce() start, end = segments_to_start_end(veto_segs) if len(start) > 0: idx = indices_within_times(times, start, end) indices = numpy.union1d(indices, idx) return indices, veto_segs.coalesce()
def get_ha_segments(config_struct,segmentlist,observer,fxdbdy,radec): if "ha_constraint" in config_struct: ha_constraint = config_struct["ha_constraint"].split(",") ha_min = float(ha_constraint[0]) ha_max = float(ha_constraint[1]) else: ha_min, ha_max = -24.0, 24.0 if config_struct["telescope"] == "DECam": if radec.dec.deg <= -30.0: ha_min, ha_max = -5.2, 5.2 else: ha_min, ha_max = -0.644981*np.sqrt(35.0-radec.dec.deg), 0.644981*np.sqrt(35.0-radec.dec.deg) location = astropy.coordinates.EarthLocation(config_struct["longitude"], config_struct["latitude"], config_struct["elevation"]) halist = segments.segmentlist() for seg in segmentlist: mjds = np.linspace(seg[0], seg[1], 100) tt = Time(mjds, format='mjd', scale='utc', location=location) lst = tt.sidereal_time('mean') ha = (lst - radec.ra).hour idx = np.where((ha >= ha_min) & (ha <= ha_max))[0] if len(idx) >= 2: halist.append(segments.segment(mjds[idx[0]],mjds[idx[-1]])) return halist
def __init__(self, ifo): self.fig, self.axes = SnglBurstUtils.make_burst_plot("GPS Time (s)", "Confidence") self.ifo = ifo self.nevents = 0 self.x = [] self.y = [] self.seglist = segments.segmentlist() self.axes.semilogy()
def finish(self): self.axes.plot(self.injected_x, self.injected_y, "k+") if not options.made_only: self.axes.plot(self.missed_x, self.missed_y, "rx") for seg in ~self.seglist & segments.segmentlist([segments.segment(self.axes.get_xlim())]): self.axes.axvspan(float(seg[0]), float(seg[1]), facecolor = "k", alpha = 0.2) self.axes.set_ylim([min(self.injected_y), max(self.injected_y)]) self.axes.set_title("Injection Locations\n(%d Injections)" % self.num_injections)
def cleanlist(seglist, min_length): removals = segments.segmentlist() for seg in seglist: if seg.duration() < min_length: removals.append(seg) seglist = seglist - removals return seglist
def cleanlist(seglist, min_length): removals = segments.segmentlist() for seg in seglist: if seg.__abs__() < min_length: removals.append(seg) seglist = seglist - removals return seglist
def convert_json_list_to_segmentlist(jsonlist): """ Helper function used to convert JSON list of lists-type object to a segmentlist object- * Utility method, ripped from jsonhelper.py in dqsegdb package, until we can use it as a dependency. """ return segments.segmentlist([segments.segment(x[0],x[1]) for x in jsonlist])
def get_segments_tile(config_struct, observatory, radec, segmentlist): observer = astroplan.Observer(location=observatory) fxdbdy = astroplan.FixedTarget(coord=radec) date_start = Time(segmentlist[0][0], format='mjd', scale='utc') date_end = Time(segmentlist[-1][1], format='mjd', scale='utc') tilesegmentlist = segments.segmentlist() while date_start.mjd < date_end.mjd: date_rise = observer.target_rise_time(date_start, fxdbdy) date_set = observer.target_set_time(date_start, fxdbdy) print(date_rise.mjd, date_set.mjd) if (date_rise.mjd < 0) and (date_set.mjd < 0): break print(date_rise.mjd, date_set.mjd) if date_rise > date_set: date_rise = observer.target_rise_time( date_start - TimeDelta(24 * u.hour), fxdbdy) print(date_rise.mjd, date_set.mjd) segment = segments.segment(date_rise.mjd, date_set.mjd) tilesegmentlist = tilesegmentlist + segments.segmentlist([segment]) tilesegmentlist.coalesce() date_start = date_set + TimeDelta(24 * u.hour) #moonsegmentlist = get_skybrightness(\ # config_struct,segmentlist,observer,fxdbdy,radec) moonsegmentlist = get_moon_segments(\ config_struct,segmentlist,observer,fxdbdy,radec) tilesegmentlistdic = segments.segmentlistdict() tilesegmentlistdic["observations"] = segmentlist tilesegmentlistdic["tile"] = tilesegmentlist tilesegmentlistdic["moon"] = moonsegmentlist tilesegmentlist = tilesegmentlistdic.intersection( ["observations", "tile", "moon"]) tilesegmentlist.coalesce() return tilesegmentlist
def generated_ascii(json_str,filepath): res_dict=json.loads(json_str) active_list=res_dict['active'] active_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in active_list]) active_segments_string='\n'.join([str(i[0])+","+str(i[1]) for i in active_segments]) output_fileh=open(filepath,'w+') output_fileh.writelines(active_segments_string) output_fileh.close() return filepath
def finish(self): self.axes.set_title("Time-Frequency Plane\n(%d Triggers)" % self.nevents) for seg in ~self.seglist & segments.segmentlist( [segments.segment(self.axes.get_xlim())]): self.axes.axvspan(float(seg[0]), float(seg[1]), facecolor="k", alpha=0.2)
def finish(self): self.axes.set_title("Trigger Confidence vs. Time\n(%d Triggers)" % self.nevents) self.axes.plot(self.x, self.y, "k+") for seg in ~self.seglist & segments.segmentlist( [segments.segment(self.axes.get_xlim())]): self.axes.axvspan(float(seg[0]), float(seg[1]), facecolor="k", alpha=0.2)
def __init__(self, instrument): self.fig, self.axes = SnglBurstUtils.make_burst_plot("GPS Time (s)", "Frequency (Hz)") self.axes.semilogy() self.instrument = instrument self.num_injections = 0 self.injected_x = [] self.injected_y = [] self.missed_x = [] self.missed_y = [] self.seglist = segments.segmentlist()
def schedule_alternating(params, config_struct, telescope, map_struct, tile_struct): if "filt_change_time" in config_struct.keys(): filt_change_time = config_struct["filt_change_time"] else: filt_change_time = 0 filters, exposuretimes = params["filters"], params["exposuretimes"] coverage_structs = [] maxidx = 0 for i in range(len(exposuretimes)): params["filters"] = [filters[i]] params["exposuretimes"] = [exposuretimes[i]] config_struct["exposurelist"] = segments.segmentlist(config_struct["exposurelist"][maxidx:]) total_nexps = len(config_struct["exposurelist"]) # if the duration of a single block is less than 30 min, shift by additional time to add up to 30 min if i > 0: start = Time(coverage_struct["data"][0][2], format='mjd') end = Time(coverage_struct["data"][-1][2], format='mjd') delta = end - start delta.format = 'sec' duration = delta.value + exposuretimes[i] + filt_change_time extra_time = (30 * 60) - duration if extra_time > 0: extra_time = extra_time + filt_change_time elif extra_time <= 0: extra_time = filt_change_time config_struct["exposurelist"] = config_struct["exposurelist"].shift(extra_time / 86400.) prob = {} for key in tile_struct.keys(): if tile_struct[key]['prob']==0.0: prob[key]=0.0 if not params["tilesType"] == "galaxy": tile_struct = gwemopt.tiles.powerlaw_tiles_struct(params, config_struct, telescope, map_struct, tile_struct) if params["doBalanceExposure"]: for key in prob: #re-assigns 0 prob to tiles w/ unbalanced observations in case they were overwritten tile_struct[key]['prob'] = 0.0 coverage_struct = gwemopt.scheduler.scheduler(params, config_struct, tile_struct) if params["doMaxTiles"]: tile_struct,doReschedule = gwemopt.utils.slice_number_tiles(params, telescope, tile_struct, coverage_struct) if doReschedule: coverage_struct = gwemopt.scheduler.scheduler(params, config_struct, tile_struct) if len(coverage_struct["exposureused"]) > 0: maxidx = int(coverage_struct["exposureused"][-1]) deltaL = total_nexps - maxidx elif len(coverage_struct["exposureused"]) == 0: deltaL = 0 coverage_structs.append(coverage_struct) if deltaL <= 1: break params["filters"], params["exposuretimes"] = filters, exposuretimes return gwemopt.coverage.combine_coverage_structs(coverage_structs),tile_struct
def event_significance(self, nevents=10, rank_fcn=None): """ Calculate the Poissonian significance of the 'on source' trial set for up to the loudest nevents. """ if rank_fcn is None: rank_fcn = lambda e: e.snr offtime = float(abs(segments.segmentlist(self.offsource.keys()))) offsource = sorted(chain(*self.offsource.values()), key=lambda sb: -sb.snr) offrate = zip(offsource, map(lambda i: i / offtime, range(1, len(offsource) + 1))) offrate = offrate[::-1] offsource = offsource[::-1] offsnr = map(rank_fcn, offsource) ontime = float(abs(segments.segmentlist(self.onsource.keys()))) if ontime == 0: return [] onsource = sorted(chain(*self.onsource.values()), key=lambda sb: -sb.snr) onsnr = map(rank_fcn, onsource) onrate = [] for snr in onsnr: try: onrate.append(offrate[bisect_left(offsnr, snr)][1]) except IndexError: # on SNR > max off SNR onrate.append(0) onsource_sig = [] for i, sb in enumerate(onsource[:nevents]): # From Gaussian #exp_num = chi2.cdf(sb.chisq_dof, sb.snr)*len(onsource) # From off-source exp_num = onrate[i] * ontime # FIXME: requires scipy >= 0.10 #onsource_sig.append([sb.snr, -poisson.logsf(i, exp_num)]) onsource_sig.append( [rank_fcn(sb), -numpy.log(poisson.sf(i, exp_num))]) return onsource_sig
def get_segment_summary_times(scienceFile, segmentName): """ This function will find the times for which the segment_summary is set for the flag given by segmentName. Parameters ----------- scienceFile : SegFile The segment file that we want to use to determine this. segmentName : string The DQ flag to search for times in the segment_summary table. Returns --------- summSegList : ligo.segments.segmentlist The times that are covered in the segment summary table. """ # Parse the segmentName segmentName = segmentName.split(':') if not len(segmentName) in [2,3]: raise ValueError("Invalid channel name %s." %(segmentName)) ifo = segmentName[0] channel = segmentName[1] version = '' if len(segmentName) == 3: version = int(segmentName[2]) # Load the filename xmldoc = utils.load_filename(scienceFile.cache_entry.path, gz=scienceFile.cache_entry.path.endswith("gz"), contenthandler=ContentHandler) # Get the segment_def_id for the segmentName segmentDefTable = table.get_table(xmldoc, "segment_definer") for entry in segmentDefTable: if (entry.ifos == ifo) and (entry.name == channel): if len(segmentName) == 2 or (entry.version==version): segDefID = entry.segment_def_id break else: raise ValueError("Cannot find channel %s in segment_definer table."\ %(segmentName)) # Get the segmentlist corresponding to this segmentName in segment_summary segmentSummTable = table.get_table(xmldoc, "segment_summary") summSegList = segments.segmentlist([]) for entry in segmentSummTable: if entry.segment_def_id == segDefID: segment = segments.segment(entry.start_time, entry.end_time) summSegList.append(segment) summSegList.coalesce() return summSegList
def coalesceResultDictionary(result_dict): """ Takes a dictionary as returned by QueryTimes or QueryTimeless and converts the lists of tuples into actual segment lists (and coalesces them). Parameters ---------- result_dict : `dict` This is the input result dictionary from the other api calls out_result_dict : `dict` This is the output result dictionary with actual segment lists (and coalesced results). """ import copy out_result_dict=copy.deepcopy(result_dict) active_seg_python_list=[segments.segment(i[0],i[1]) for i in result_dict['active']] active_seg_list=segments.segmentlist(active_seg_python_list) active_seg_list.coalesce() out_result_dict['active']=active_seg_list known_seg_python_list=[segments.segment(i[0],i[1]) for i in result_dict['known']] known_seg_list=segments.segmentlist(known_seg_python_list) known_seg_list.coalesce() out_result_dict['known']=known_seg_list return out_result_dict
def get_valid_segments(segment_url, base_dir, ifo, science_flag, start_time, end_time): print("Finding valid analysis times for %s, please hold..." % ifo) cmd = 'ligolw_segment_query --query-segments --segment-url %s --include-segments %s --gps-start-time %d --gps-end-time %d | ligolw_print -t segment -c start_time -c end_time' % (segment_url, science_flag, start_time, end_time) pipe = os.popen(cmd) print(cmd) results = [x.strip().split(',') for x in pipe] science = segments.segmentlist([segments.segment(int(x[0]), int(x[1])) for x in results]) science.coalesce() print("Science: ") for s in science: print(s[0], s[1]) framedir = base_dir + '/' + ifo[0] + '1' chunks = [f.split('.')[0].split('-') for f in get_all_files_in_range(framedir, start_time, end_time)] available = segments.segmentlist([ segments.segment( int(x[-2]), int(x[-2]) + int(x[-1]) ) for x in chunks if len(x) == 6 ]) available.coalesce() print("Available:") for s in available: print(s[0], s[1]) result = science & available result.coalesce() print("Result:") for s in result: print(s[0], s[1]) print("done.") return result
def select_segments_by_definer(segment_file, segment_name=None, ifo=None): """ Return the list of segments that match the segment name Parameters ---------- segment_file: str path to segment xml file segment_name: str Name of segment ifo: str, optional Returns ------- seg: list of segments """ from glue.ligolw.ligolw import LIGOLWContentHandler as h; lsctables.use_in(h) indoc = ligolw_utils.load_filename(segment_file, False, contenthandler=h) segment_table = table.get_table(indoc, 'segment') seg_def_table = table.get_table(indoc, 'segment_definer') def_ifos = seg_def_table.getColumnByName('ifos') def_names = seg_def_table.getColumnByName('name') def_ids = seg_def_table.getColumnByName('segment_def_id') valid_id = [] for def_ifo, def_name, def_id in zip(def_ifos, def_names, def_ids): if ifo and ifo != def_ifo: continue if segment_name and segment_name != def_name: continue valid_id += [def_id] start = numpy.array(segment_table.getColumnByName('start_time')) start_ns = numpy.array(segment_table.getColumnByName('start_time_ns')) end = numpy.array(segment_table.getColumnByName('end_time')) end_ns = numpy.array(segment_table.getColumnByName('end_time_ns')) start, end = start + 1e-9 * start_ns, end + 1e-9 * end_ns did = segment_table.getColumnByName('segment_def_id') keep = numpy.array([d in valid_id for d in did]) if sum(keep) > 0: return start_end_to_segments(start[keep], end[keep]) else: return segmentlist([])
def get_missing_segs_from_frame_file_cache(datafindcaches): """ This function will use os.path.isfile to determine if all the frame files returned by the local datafind server actually exist on the disk. This can then be used to update the science times if needed. Parameters ----------- datafindcaches : OutGroupList List of all the datafind output files. Returns -------- missingFrameSegs : Dict. of ifo keyed glue.segment.segmentlist instances The times corresponding to missing frames found in datafindOuts. missingFrames: Dict. of ifo keyed lal.Cache instances The list of missing frames """ missingFrameSegs = {} missingFrames = {} for cache in datafindcaches: if len(cache) > 0: # Don't bother if these are not file:// urls, assume all urls in # one cache file must be the same type if not cache[0].scheme == 'file': warn_msg = "We have %s entries in the " %(cache[0].scheme,) warn_msg += "cache file. I do not check if these exist." logging.info(warn_msg) continue _, currMissingFrames = cache.checkfilesexist(on_missing="warn") missingSegs = segments.segmentlist(e.segment \ for e in currMissingFrames).coalesce() ifo = cache.ifo if ifo not in missingFrameSegs: missingFrameSegs[ifo] = missingSegs missingFrames[ifo] = lal.Cache(currMissingFrames) else: missingFrameSegs[ifo].extend(missingSegs) # NOTE: This .coalesce probably isn't needed as the segments # should be disjoint. If speed becomes an issue maybe remove it? missingFrameSegs[ifo].coalesce() missingFrames[ifo].extend(currMissingFrames) return missingFrameSegs, missingFrames
def run_show_types(doc, connection, engine, gps_start_time, gps_end_time, included_segments_string, excluded_segments_string): resulttable = lsctables.New(ShowTypesResultTable) doc.childNodes[0].appendChild(resulttable) sql = """SELECT segment_definer.ifos, segment_definer.name, segment_definer.version, (CASE WHEN segment_definer.comment IS NULL THEN '-' WHEN segment_definer.comment IS NOT NULL THEN segment_definer.comment END), segment_summary.start_time, segment_summary.end_time, (CASE WHEN segment_summary.comment IS NULL THEN '-' WHEN segment_summary.comment IS NOT NULL THEN segment_summary.comment END) FROM segment_definer, segment_summary WHERE segment_definer.segment_def_id = segment_summary.segment_def_id AND NOT (segment_summary.start_time > %d OR %d > segment_summary.end_time) """ % (gps_end_time, gps_start_time) rows = engine.query(sql) seg_dict = {} for row in rows: ifos, name, version, segment_definer_comment, segment_summary_start_time, segment_summary_end_time, segment_summary_comment = row key = (ifos, name, version, segment_definer_comment, segment_summary_comment) if key not in seg_dict: seg_dict[key] = [] seg_dict[key].append(segments.segment(segment_summary_start_time, segment_summary_end_time)) for key, value in seg_dict.iteritems(): segmentlist = segments.segmentlist(value) segmentlist.coalesce() for segment in segmentlist: result = ShowTypesResult() result.ifos, result.name, result.version, result.segment_definer_comment, result.segment_summary_comment = key result.segment_summary_start_time, result.segment_summary_end_time = segment result.ifos = result.ifos.strip() resulttable.append(result) engine.close()
def split_segment(timing_params, segment, psds_per_job): """ Split the data segment into correctly-overlaping segments. We try to have the numbers of PSDs in each segment be equal to psds_per_job, but with a short segment at the end if needed. """ # in seconds joblength = job_length_from_psds(timing_params, psds_per_job) # in samples joboverlap = 2 * timing_params.filter_corruption + (timing_params.psd_length - timing_params.psd_shift) # in seconds joboverlap /= timing_params.resample_rate segs = segments.segmentlist() t = segment[0] while t + joblength <= segment[1]: segs.append(segments.segment(t, t + joblength) & segment) t += joblength - joboverlap extra_psds = int(psds_from_job_length(timing_params, float(segment[1] - t))) if extra_psds: segs.append(segments.segment(t, t + job_length_from_psds(timing_params, extra_psds))) return segs
def split_segment(seg, min_segment_length, pad, overlap, short_segment_duration, max_job_length): # avoid infinite loop if min_segment_length + 2 * pad <= overlap: raise ValueError("infinite loop: min_segment_length + 2 * pad must be > overlap") # clip max_job_length down to an allowed size max_job_length = clip_segment_length(max_job_length, pad, short_segment_duration) seglist = segments.segmentlist() while abs(seg) >= min_segment_length + 2 * pad: # try to use max_job_length each time if abs(seg) >= max_job_length: seglist.append(segments.segment(seg[0], seg[0] + max_job_length)) else: seglist.append(segments.segment(seg[0], seg[0] + clip_segment_length(abs(seg), pad, short_segment_duration))) assert abs(seglist[-1]) != 0 # safety-check for no-op # bounds must be integers if abs((int(seglist[-1][0]) - seglist[-1][0]) / seglist[-1][0]) > 1e-14 or abs((int(seglist[-1][1]) - seglist[-1][1]) / seglist[-1][1]) > 1e-14: raise ValueError("segment %s does not have integer boundaries" % str(seglist[-1])) # advance segment seg = segments.segment(seglist[-1][1] - overlap, seg[1]) if not seglist: raise ValueError("unable to use segment %s" % str(seg)) return seglist
def segmentlistdict(self): """ A segmentlistdict object describing the instruments and time spanned by this CacheEntry. A new object is constructed each time this attribute is accessed (segments are immutable so there is no reason to try to share a reference to the CacheEntry's internal segment; modifications of one would not be reflected in the other anyway). Example: >>> c = CacheEntry("H1 S5 815901601 576.5 file://localhost/home/kipp/tmp/1/H1-815901601-576.xml") >>> c.segmentlistdict['H1'] [segment(LIGOTimeGPS(815901601, 0), LIGOTimeGPS(815902177, 500000000))] The \"observatory\" column of the cache entry, which is frequently used to store instrument names, is parsed into instrument names for the dictionary keys using the same rules as ligo.lw.lsctables.instrumentsproperty.get(). Example: >>> c = CacheEntry("H1H2, S5 815901601 576.5 file://localhost/home/kipp/tmp/1/H1H2-815901601-576.xml") >>> c.segmentlistdict['H1H2'] [segment(LIGOTimeGPS(815901601, 0), LIGOTimeGPS(815902177, 500000000))] """ # the import has to be done here to break the cyclic # dependancy try: from ligo.lw.lsctables import instrumentsproperty except ImportError: # FIXME: remove when we can rely on ligo.lw being installed # (why isn't it!?) from glue.ligolw.lsctables import instrumentsproperty instruments = instrumentsproperty.get(self.observatory) or (None,) return segments.segmentlistdict((instrument, segments.segmentlist(self.segment is not None and [self.segment] or [])) for instrument in instruments)
############################################################################## # get the pad and chunk lengths from the values in the ini file paddata = int(cp.get('data', 'pad-data')) n = int(cp.get('data', 'segment-length')) s = int(cp.get('data', 'number-of-segments')) r = int(cp.get('data', 'sample-rate')) o = int(cp.get('inspiral', 'segment-overlap')) length = ( n * s - ( s - 1 ) * o ) / r overlap = o / r minsciseg = length + 2 * paddata ############################################################################## # Based on the start and end time, generate a list of epochs to # analyze. An entire hipe dag will be run for each of these epochs. search_epochs = segments.segmentlist() istart = opts.start_time while ( istart < opts.end_time ): iend = istart + opts.interval if iend > opts.end_time: iend = opts.end_time search_epochs.append(segments.segment(istart,iend)) istart += opts.interval # FIXME: the writing out of the segments should be done at the end so # that successfully generated dags, etc can be maintained from run to # run segmentsUtils.tosegwizard(file("multi_hipe_selectedsegs.txt",'w'),search_epochs) ############################################################################## # Read in all the segment lists ifolist = []
############################################################################## # set up the segment including the off-source segment grb_ifolist.sort() ifo_times = "".join(grb_ifolist) if offSourceSegment is None: print("Warning: insufficient multi-IFO data to construct an off-source segment for GRB %s; skipping" % grb.event_number_grb, file=sys.stderr) continue elif opts.verbose: print("Sufficient off-source data has been found in", ifo_times, "time.") # write out the segment list to a segwizard file offsource_segfile = idirectory + "/offSourceSeg.txt" segmentsUtils.tosegwizard(open(offsource_segfile, "w"), segments.segmentlist([offSourceSegment])) onsource_segfile = idirectory + "/onSourceSeg.txt" segmentsUtils.tosegwizard(file(onsource_segfile, "w"), segments.segmentlist([onSourceSegment])) segLen = abs( onSourceSegment ) bufferSegment = segments.segment( onSourceSegment[0]-opts.number_buffer_left*segLen,\ onSourceSegment[1]+opts.number_buffer_right*segLen) buffer_segfile = idirectory + "/bufferSeg.txt" segmentsUtils.tosegwizard(file(buffer_segfile, "w"), segments.segmentlist([bufferSegment])) if opts.verbose: print("on-source segment: ", onSourceSegment) print("off-source segment: ", offSourceSegment) ############################################################################
def getSegments ( seglistin, interval): seglistout = segments.segmentlist([s for s in seglistin \ if (s[1] > interval[0] and s[0] < interval[1]) ]) return seglistout
def calculate_combined_result(includedJSON,excludedJSON,startTime,endTime,ifo): """ Calculate the result of the union of the active times for the included flag less the intersection of that result with the union of the excluded flags Inputs are 2 lists of python dictionaries representing the JSON (already have run json.loads() on the JSON), a start time, and end time, and the ifo name (it does not make sense to include/exclude across multiple ifos) Parameters ---------- startTime : `int` Ex: 999999999 endTime : `int` Ex: 999999999 ifo : `string` Ex: 'L1' """ total_active_list=segments.segmentlist([]) for flag in includedJSON: #result=json.loads(flag) #flagDict=result['flags'][0] active_list=flag['active'] active_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in active_list]) total_active_list=total_active_list+active_segments total_active_list.coalesce() for flag in excludedJSON: #result=json.loads(flag) #flagDict=result['flags'][0] active_list=flag['active'] active_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in active_list]) total_active_list=total_active_list-active_segments total_active_list.coalesce() # Now, total_active_list contains a segmentlist object with segments spanning the expected result # includedJSON and excludedJSON contain lists of JSON text blobs (not parsed with json.loads yet) ## Note: About known segments for the result: We just report the start and end time of the period queried! If you wanted to report the actual validity of multiple segments, it's somewhat undefined if the excluded ones and/or some of the included flags aren't known about for a time when the included ones are; Technically since exclusion trumps all inclusions, if an excluded segment is known and active at any given time, the result is known for that time explicitly. result_known_segment_list=segments.segmentlist([segments.segment(startTime,endTime)]) ## Now we have to build the JSON for this flag # JSON flag objects looks like this (each is a dictionary!): # { # "ifo" : "ifo", # "name" : "flag", # "version" : n, # "comment" : "description", # "provenance_url" : "aLog URL", # "deactivated" : false|true, # "active_indicates_ifo_badness" : true|false|null, # // known segments returned for both /active and /known URIs, no segments are returned for the /metadata or /report/flags queries # // aka S6 summary segments # "known" : [ [ts,te], [ts,te], ... ] # // active segments returned only for /active URI: # "active" : [ [ts,te], [ts,te], ... ] # // \textcolor{red}{Comment: or "segment" : [ [ts,te,value], [ts,te,value], ...] (where value can be -1,0 or +1)} # // inactive == (known - active) # // unknown == (all_time - known) # }, ## Make the json-ready flag dictionary for the combined result: ifo=ifo # replicating old behavoir from ligolw_segment_query # Note: This just uses the ifo of the last excluded flag! name='RESULT' version=1 known_segments=result_known_segment_list active_segments=total_active_list result_flag=jsonhelper.buildFlagDict(ifo,name,version,known_segments,active_segments) return result_flag
def run_query_types(doc, proc_id, connection, engine, gps_start_time, gps_end_time, included_segments): query_segment = segments.segmentlist([segments.segment(gps_start_time, gps_end_time)]) sql = """SELECT segment_definer.ifos, segment_definer.name,segment_definer.version, (CASE WHEN segment_definer.comment IS NULL THEN '-' WHEN segment_definer.comment IS NOT NULL THEN segment_definer.comment END), segment_summary.start_time, segment_summary.end_time, (CASE WHEN segment_summary.comment IS NULL THEN '-' WHEN segment_summary.comment IS NOT NULL THEN segment_summary.comment END) FROM segment_definer, segment_summary WHERE segment_definer.segment_def_id = segment_summary.segment_def_id AND NOT(%d > segment_summary.end_time OR segment_summary.start_time > %d) """ % (gps_start_time, gps_end_time) type_clauses = map(seg_spec_to_sql, included_segments.split(',')) if type_clauses != []: sql += " AND (" + "OR ".join(type_clauses) + ")" segment_types = {} for row in engine.query(sql): sd_ifo, sd_name, sd_vers, sd_comment, ss_start, ss_end, ss_comment = row key = (sd_ifo, sd_name, sd_vers, sd_comment, ss_comment) if key not in segment_types: segment_types[key] = segments.segmentlist([]) segment_types[key] |= segments.segmentlist([segments.segment(ss_start, ss_end)]) engine.close() # Create segment definer and segment_summary tables seg_def_table = lsctables.New(lsctables.SegmentDefTable, columns = ["process_id", "segment_def_id", "ifos", "name", "version", "comment"]) doc.childNodes[0].appendChild(seg_def_table) seg_sum_table = lsctables.New(lsctables.SegmentSumTable, columns = ["process_id", "segment_sum_id", "start_time", "start_time_ns", "end_time", "end_time_ns", "comment", "segment_def_id"]) doc.childNodes[0].appendChild(seg_sum_table) for key in segment_types: # Make sure the intervals fall within the query window and coalesce segment_types[key].coalesce() segment_types[key] &= query_segment seg_def_id = seg_def_table.get_next_id() segment_definer = lsctables.SegmentDef() segment_definer.process_id = proc_id segment_definer.segment_def_id = seg_def_id segment_definer.ifos = key[0] segment_definer.name = key[1] segment_definer.version = key[2] segment_definer.comment = key[3] seg_def_table.append(segment_definer) # add each segment summary to the segment_summary_table for seg in segment_types[key]: segment_sum = lsctables.SegmentSum() segment_sum.comment = key[4] segment_sum.process_id = proc_id segment_sum.segment_def_id = seg_def_id segment_sum.segment_sum_id = seg_sum_table.get_next_id() segment_sum.start_time = seg[0] segment_sum.start_time_ns = 0 segment_sum.end_time = seg[1] segment_sum.end_time_ns = 0 seg_sum_table.append(segment_sum)
def calculate_versionless_result(jsonResults,startTime,endTime,ifo_input=None): """ Construct output segments lists from multiple JSON objects. The jsonResults input is a list of json ojbects and are expected to be in order of decreasing versions. """ debug=False active_results={} segment_known_results={} affected_results={} total_active_list=segments.segmentlist([]) total_query_time=segments.segmentlist([segments.segment(startTime,endTime)]) total_known_list=segments.segmentlist([]) for resultin in jsonResults: #result=json.loads(resultin) result=resultin # old : flagDict=result['flags'][0] # Our queries above each return 1 flag version = int(result['version']) deactivated_state=result['metadata']['deactivated'] if str(deactivated_state) in ["False","false"]: known_list=result['known'] known_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in known_list]) # make a segment list object to do arithmetic known_segments.coalesce() segment_known_results[version]=known_segments active_list=result['active'] active_segments=segments.segmentlist([segments.segment(x[0],x[1]) for x in active_list]) # make a segment list object to do arithmetic active_segments.coalesce() active_results[version]=active_segments if debug: print("Active results for version %d" % version) print(active_results[version]) # Now I have 2 dictionaries of known and active segments with versions as keys in case I want/need them later... # This next step might seem a bit confusing: # We need to take the active segments for this version only during times that were not known by higher segment versions # Thus we need to take the intersection (& operator) of the unknown segments across all higher versions with the known segments for this version, then take the intersection of that result with the active segments for this version, and then add that to the current list of total active segments... phew: total_active_list |= (total_query_time-total_known_list)&known_segments&active_segments if debug: import pdb print("Running pdb to see what is in total_active_list") pdb.set_trace() total_active_list.coalesce() # The S6 clients want to know about the range of times affected by a given version explicitly, so those are calculated here: affected_results[version]=(total_query_time-total_known_list)&known_segments # Note that the order matters here! we use the total_known_list from the previous iteration of the loop step to figure out which active segments to use in this iteration of the loop, so the above line must come before the next total_known_list |= known_segments total_known_list.coalesce() if ifo_input==None: if len(jsonResults)==0: import exceptions exceptions.RuntimeError("No versions for flag in versionless query") else: ifo=result['ifo'] else: #Only use ifo_input if we can't extract the ifo from the json result (usually because json result is empty) try: ifo=result['ifo'] except: ifo=ifo_input name='RESULT' # Fix!!! Executive decision to make this clear that this is not a specific IFO:FLAG:VERSION resource, but rather a contrived result version=1 # Fix!!! Executive decision to make this match what old clients expect # I would prefer that this is more clear that this is not a specific IFO:FLAG:VERSION resource, but rather a contrived result, possibly by making it version 0 total_active_list.coalesce() total_known_list.coalesce() result_flag=jsonhelper.buildFlagDict(ifo,name,version,total_known_list,total_active_list) return result_flag,affected_results