def generate_all_state(start, end, register=True, **kwargs): """Build a new `SummaryState` for the given [start, end) interval. Parameters ---------- start : `~gwpy.time.LIGOTimeGPS`, float the GPS start time of the current analysis end : `~gwpy.time.LIGOTimeGPS`, float the GPS end time of the current analysis register : `bool`, optional should the new `SummaryState` be registered, default `True` **kwargs other keyword arguments passed to the `SummaryState` constructor Returns ------- allstate : `SummaryState` the newly created 'All' `SummaryState` """ now = min(end, NOW) all_ = SummaryState(ALLSTATE, known=SegmentList([Segment(start, end)]), active=SegmentList([Segment(start, now)]), **kwargs) all_.ready = True if register: register_state(all_) return all_
def test_add_state_segments(self): fig, ax = self.new() # mock up some segments and add them as 'state' segments segs = SegmentList([Segment(1, 2), Segment(4, 5)]) segax = fig.add_state_segments(segs) # check that the new axes aligns with the parent utils.assert_array_equal(segax.get_position().intervalx, ax.get_position().intervalx) coll = segax.collections[0] for seg, path in zip(segs, coll.get_paths()): utils.assert_array_equal(path.vertices, [(seg[0], -.4), (seg[1], -.4), (seg[1], .4), (seg[0], .4), (seg[0], -.4)]) with pytest.raises(ValueError): fig.add_state_segments(segs, location='left') # test that this doesn't work with non-timeseries axes fig = self.FIGURE_CLASS() ax = fig.gca(projection='rectilinear') with pytest.raises(ValueError) as exc: fig.add_state_segments(segs) assert str(exc.value) == ("No 'timeseries' Axes found, cannot anchor " "new segment Axes.")
def __init__(self, parent=None, tssb=None, conc=0.1): super(DataNode, self).__init__(parent=parent, tssb=tssb) # pi is a first-class citizen self.pi = 0.0 self.param = 0.0 self.param1 = 0.0 self.pi1 = 0.0 # used in MH to store old state self.path = None # set of nodes from root to this node self.ht = 0.0 if parent is None: self._conc = conc self.pi = 1.0 self.param = 1.0 else: self.pi = rand(1) * parent.pi parent.pi = parent.pi - self.pi self.param = self.pi # 此处初始化应该设置为SegmentList self.varphiR = SegmentList([Segment(0, 1)]) self.piR = SegmentList([Segment(0, 1)]) self.epsilon = "" # tssb node self.tNode = None
def _read(q, pstart, pend): try: # don't go beyond the requested limits pstart = float(max(start, pstart)) pend = float(min(end, pend)) # if resampling TimeSeries, pad by 8 seconds inside cache limits if cls not in (StateVector, StateVectorDict) and resample: cstart = float(max(cspan[0], pstart - 8)) subcache = cache.sieve(segment=Segment(cstart, pend)) out = cls.read(subcache, channel, format=format, start=cstart, end=pend, resample=None, **kwargs) out = out.resample(resample) q.put(out.crop(pstart, pend)) else: subcache = cache.sieve(segment=Segment(pstart, pend)) q.put( cls.read(subcache, channel, format=format, start=pstart, end=pend, resample=resample, **kwargs)) except Exception as e: q.put(e)
def test_get_frame_segments(find): assert segments.get_frame_segments("X", "X1_R", 0, 100) == SegmentList( [Segment(0, 10), Segment(20, 30)]) assert segments.get_frame_segments( "X", "X1_R", 25, 100, ) == SegmentList([Segment(25, 30)])
def test_query_dqsegdb_multi(self): segs = SegmentList([Segment(0, 2), Segment(8, 10)]) result = query_dqsegdb(self.TEST_CLASS.query_dqsegdb, QUERY_FLAGS[0], segs) RESULT = QUERY_RESULTC[QUERY_FLAGS[0]] assert isinstance(result, self.TEST_CLASS) utils.assert_segmentlist_equal(result.known, RESULT.known & segs) utils.assert_segmentlist_equal(result.active, RESULT.active & segs)
def test_query_dqsegdb_multi(self): querymid = int(QUERY_START + (QUERY_END - QUERY_START) / 2.) segs = SegmentList( [Segment(QUERY_START, querymid), Segment(querymid, QUERY_END)]) flag = QUERY_FLAGS[0] result = self._query(DataQualityFlag.query, flag, segs, url=QUERY_URL) self.assertEqual(result.known, QUERY_RESULT[flag].known) self.assertEqual(result.active, QUERY_RESULT[flag].active)
def test_segments_from_sngl_burst(): tab = io_ligolw.sngl_burst_from_times([1, 4, 7, 10], channel='test') segs = io_ligolw.segments_from_sngl_burst(tab, 1) assert_segmentlist_equal( segs['test'].active, SegmentList([ Segment(0, 2), Segment(3, 5), Segment(6, 8), Segment(9, 11), ]))
def test_query_dqsegdb_multi(self): querymid = int(QUERY_START + (QUERY_END - QUERY_START) / 2.) segs = SegmentList( [Segment(QUERY_START, querymid), Segment(querymid, QUERY_END)]) flag = self._query(DataQualityFlag.query_dqsegdb, QUERY_FLAG, segs, url=QUERY_URL) self.assertEqual(flag.known, QUERY_KNOWN) self.assertEqual(flag.active, QUERY_ACTIVE)
def test_get_triggers(self): # test that trigfind raises a warning if the channel-level directory # doesn't exist with pytest.warns(UserWarning): out = triggers.get_triggers('X1:DOES_NOT_EXIST', 'omicron', SegmentList([Segment(0, 100)])) # check output type and columns self.assertIsInstance(out, numpy.ndarray) for col in ['time', 'frequency', 'snr']: self.assertIn(col, out.dtype.fields) # test that unknown ETG raises KeyError self.assertRaises(KeyError, triggers.get_triggers, 'X1:DOES_NOT_EXIST', 'fake-etg', SegmentList([Segment(0, 100)]))
def find_all_coincidences(triggers, channel, snrs, windows): """Find the number of coincs between each auxiliary channel and the primary Parameters ---------- primary : `numpy.ndarray` an array of times for the primary channel auxiliary : `numpy.recarray` an array of triggers for a set of auxiliary channels snrs : `list` of `float` the SNR thresholds to use window : `list` of `float` the time windows to use """ triggers.sort(order='time') windows = sorted(windows, reverse=True) snrs = sorted(snrs) coincs = dict((p, {}) for p in itertools.product(windows, snrs)) for i, x in enumerate(triggers): if x['channel'] != channel: continue t = x['time'] channels = dict((key, set()) for key in coincs) j = i - 1 segs = [Segment(t-dt/2., t+dt/2.) for dt in windows] # define coincidence test def add_if_coinc(event): if event['channel'] == channel: return in_seg = filter(lambda s: s[0] <= event['time'] <= s[1], segs) if not in_seg: # no triggers in window return for k, w in enumerate(in_seg): for snr in filter(lambda s: event['snr'] >= s, snrs): channels[(windows[k], snr)].add(event['channel']) return 1 # search left half-window while j >= 0: if not add_if_coinc(triggers[j]): break j -= 1 j = i + 1 # search right half-window while j < triggers.shape[0]: if not add_if_coinc(triggers[j]): break j += 1 # count 'em up for p, cset in channels.items(): for c in cset: try: coincs[p][c] += 1 except KeyError: coincs[p][c] = 1 return coincs
def segments_from_array(array): """Convert a 2-dimensional `numpy.ndarray` to a `SegmentList` """ out = SegmentList() for row in array: out.append(Segment(*row)) return out
def divide_segmentlist(start,end,bins=4096,write=True,**kwargs): ''' Divide given period to segmenlist Parameters ---------- start : `int` GPS start time of given period end : `int` GPS end time of given period bins : `int`, optional The number of bins. Unit is second. Default value is 4096 =(2**12). Returns ------- segmentlist : `gwpy.segment.SegmentList` Divided segmentlist ''' if ((end-start) % bins) != 0: raise ValueError('Not divisible!') _start = range(start ,end ,bins) _end = range(start+bins,end+bins,bins) segmentlist = SegmentList([Segment(s,e) for s,e in zip(_start,_end)]) log.debug(segmentlist[0]) log.debug(segmentlist[-1]) if write: segmentlist.write('./segmentlist/total.txt') return segmentlist
def find_kw(channel, start, end, base=None): """Find KW trigger XML files """ span = Segment(to_gps(start), to_gps(end)) channel = get_channel(channel) ifo = channel.ifo if base is None and channel.name.split(':', 1)[-1] == 'GDS-CALIB_STRAIN': tag = '%s-KW_HOFT' % ifo[0].upper() base = '/gds-%s/dmt/triggers/%s' % (ifo.lower(), tag) elif base is None: tag = '%s-KW_TRIGGERS' % ifo[0].upper() base = '/gds-%s/dmt/triggers/%s' % (ifo.lower(), tag) gps5 = int('%.5s' % start) end5 = int('%.5s' % end) out = Cache() append = out.append while gps5 <= end5: trigglob = os.path.join( base, '%s-%d' % (tag, gps5), '%s-*-*.xml' % tag) found = glob.glob(trigglob) for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) gps5 += 1 out.sort(key=lambda e: e.path) vprint(" Found %d files for %s (KW)\n" % (len(out), channel.ndsname)) return out
def find_dmt_omega(channel, start, end, base=None): """Find DMT-Omega trigger XML files """ span = Segment(to_gps(start), to_gps(end)) channel = get_channel(channel) ifo = channel.ifo if base is None and channel.name.split(':', 1)[-1] == 'GDS-CALIB_STRAIN': base = '/gds-%s/dmt/triggers/%s-HOFT_Omega' % ( ifo.lower(), ifo[0].upper()) elif base is None: raise NotImplementedError("This method doesn't know how to locate DMT " "Omega trigger files for %r" % str(channel)) gps5 = int('%.5s' % start) end5 = int('%.5s' % end) out = Cache() append = out.append while gps5 <= end5: trigglob = os.path.join( base, str(gps5), '%s-%s_%s_%s_OmegaC-*-*.xml' % ( ifo, channel.system, channel.subsystem, channel.signal)) found = glob.glob(trigglob) for f in found: ce = CacheEntry.from_T050017(f) if ce.segment.intersects(span): append(ce) gps5 += 1 out.sort(key=lambda e: e.path) vprint(" Found %d files for %s (DMT-Omega)\n" % (len(out), channel.ndsname)) return out
def fetch(self, config=GWSummConfigParser(), segdb_error='raise', datafind_error='raise', **kwargs): """Finalise this state by fetching its defining segments, either from global memory, or from the segment database """ # check we haven't done this before if self.ready: return self # fetch data if self.definition: match = re.search('(%s)' % '|'.join(MATHOPS.keys()), self.definition) else: match = None if self.filename: self._read_segments(self.filename) elif match: channel, thresh = self.definition.split(match.groups()[0]) channel = channel.rstrip() thresh = float(thresh.strip()) self._fetch_data(channel, thresh, match.groups()[0], config=config, datafind_error=datafind_error, **kwargs) # fetch segments elif self.definition: self._fetch_segments(config=config, segdb_error=segdb_error, **kwargs) # fetch null else: start = config.getfloat(DEFAULTSECT, 'gps-start-time') end = config.getfloat(DEFAULTSECT, 'gps-end-time') self.known = [(start, end)] self.active = self.known # restrict to given hours if self.hours: segs_ = SegmentList() # get start day d = Time(float(self.start), format='gps', scale='utc').datetime d.replace(hour=0, minute=0, second=0, microsecond=0) end_ = Time(float(self.end), format='gps', scale='utc').datetime while d < end_: # get GPS of day t = to_gps(d) # for each [start, end) hour pair, build a segment for h0, h1 in self.hours: segs_.append(Segment(t + h0 * 3600, t + h1 * 3600)) # increment and return d += datetime.timedelta(1) self.known &= segs_ self.active &= segs_ # FIXME self.ready = True return self
def find_best_frames(ifo, frametype, start, end, **kwargs): """Find frames for the given type, replacing with a better type if needed """ # find cache for this frametype cache = find_frames(ifo, frametype, start, end, **kwargs) # check for gaps in current cache span = SegmentList([Segment(start, end)]) gaps = span - cache_segments(cache) # if gaps and using aggregated h(t), check short files if abs(gaps) and frametype in SHORT_HOFT_TYPES: f2 = SHORT_HOFT_TYPES[frametype] vprint(" Gaps discovered in aggregated h(t) type " "%s, checking %s\n" % (frametype, f2)) kwargs['gaps'] = 'ignore' cache.extend( filter(lambda e: file_segment(e) in gaps, find_frames(ifo, f2, start, end, **kwargs))) new = int(abs(gaps - cache_segments(cache))) if new: vprint(" %ss extra coverage with frametype %s\n" % (new, f2)) else: vprint(" No extra coverage with frametype %s\n" % f2) return cache, frametype
def check_flag(flag, gpstime, duration, pad): """Check that a state flag is active during an entire analysis segment Parameters ---------- flag : `str` state flag to check gpstime : `float` GPS time of required data duration : `float` duration (in seconds) of required data pad : `float` amount of extra data to read in at the start and end for filtering Returns ------- check : `bool` Boolean switch to pass (`True`) or fail (`False`) depending on whether the given flag is active """ # set GPS start and end time start = gpstime - duration / 2. - pad end = gpstime + duration / 2. + pad seg = Segment(start, end) # query for state segments active = DataQualityFlag.query(flag, start, end, url=DEFAULT_SEGMENT_SERVER).active # check that state flag is active during the entire analysis if (not active.intersects_segment(seg)) or (abs(active[0]) < abs(seg)): return False return True
def img_qtransform(data, To=2, frange=(10, 2048), qrange=(4, 64), vmin=0, qsplit=False, dT=2.0): """ this funciton performs the q-transform on the strain data and returns it as an rgb image inputs: data - TimeSeries - the strain data to be transformed frange - tuple - frequency range of the q-transform qrange - tuple - q factor range of the q-transform vmin - scalar - the value range for the colormap to cover. if None the full range will be covered. if vmax='auto' the range will be set automatically according to the q-transform values qsplit - if True split the qtransform to separate images for better resolution output: qimg - ndarray - array with the q-transform image """ # qt = data.q_transform(frange=frange, qrange=qrange, whiten=True, tres=0.002) # compute the q-transform with the built in function of gwpy # qt = qt[int(To/2.0 / qt.dt.value):-int(To/2.0 / qt.dt.value)] # crop the q-transform to remove half the overlap time at each end # vmax = qt.max() # print(vmax.value) vmax = 25.5 # I think this is the value used in gravityspy (if I understand correctly) if qsplit: qt = data.q_transform(frange=frange, qrange=qrange, whiten=True, tres=0.002) # compute the q-transform with the built in function of gwpy qt = qt[int(To/2.0 / qt.dt.value):-int(To/2.0 / qt.dt.value)] # crop the q-transform to remove half the overlap time at each end qimg = qimg_split(qt, dT=dT, vmin=vmin, vmax=vmax) else: t_center = data.times[int(len(data.times)/2)].value outseg = Segment(t_center - dT/2, t_center + dT/2) qt = data.q_transform(frange=frange, qrange=qrange, whiten=True, tres=0.002, gps=t_center, search=0.5, fres=0.5, outseg=outseg) qt = qt.crop(t_center - dT/2, t_center + dT/2) qimg = qimg_draw(qt, vmin, vmax) return qimg
def run(args, config): """Execute the flag study """ if (args.label == 'Vetoes' and len(args.flag) == 1 and not os.path.isfile(args.flag[0])): args.label = args.flag[0] vprint("\n-------------------------------------------------\n") vprint("Processing %s\n" % args.label) span = Segment(args.gps_start_time, args.gps_end_time) # format analysis state if args.analysis_flag: state = SummaryState(args.analysis_flag, definition=args.analysis_flag, known=[span]) state.fetch(config=config) else: state = generate_all_state(*span) tab = FlagTab(args.label, args.gps_start_time, args.gps_end_time, args.flag, states=[state], metrics=args.metrics, channel=args.channel, etg=args.trigger_format, intersection=args.intersection) tab.index = 'index.html' tab.process(config=config) tab.write_html(ifo='VET')
def use_segmentlist(f, arg1, segments, *args, **kwargs): """Decorator a method to convert incoming segments into a `SegmentList` """ if isinstance(segments, DataQualityFlag): segments = segments.active elif not isinstance(segments, segmentlist): segments = SegmentList([Segment(*x) for x in segments]) return f(arg1, segments, *args, **kwargs)
def segmentlist_from_tree(tree, coalesce=False): """Read a `~ligo.segments.segmentlist` from a 'segments' `ROOT.Tree` """ segs = SegmentList() for i in range(tree.GetEntries()): tree.GetEntry(i) segs.append(Segment(tree.start, tree.end)) return segs
def test_prepend(self): a = self.read() a.crop(968654552, 968654552.5) b = self.read() b.crop(968654552.5, 968654553, copy=True) b.prepend(a) for key in b: self.assertEqual(b[key].span, Segment(968654552, 968654553))
def test_append(self): a = self.read() a.crop(968654552, 968654552.5, copy=True) b = self.read() b.crop(968654552.5, 968654553) a.append(b) for key in a: self.assertEqual(a[key].span, Segment(968654552, 968654553))
def make_cache(): segs = SegmentList() cache = Cache() for seg in [(0, 1), (1, 2), (4, 5)]: d = seg[1] - seg[0] f = 'A-B-%d-%d.tmp' % (seg[0], d) cache.append(CacheEntry.from_T050017(f)) segs.append(Segment(*seg)) return cache, segs
def test_get_triggers(): # test that trigfind raises a warning if the channel-level directory # doesn't exist with pytest.warns(UserWarning): out = triggers.get_triggers('X1:DOES_NOT_EXIST', 'omicron', SegmentList([Segment(0, 100)])) # check output type and columns assert isinstance(out, Table) for col in ['time', 'frequency', 'snr']: assert col in out.dtype.names
def make_cache(): segs = SegmentList() cache = Cache() for seg in [(0, 1), (1, 2), (4, 5)]: d = seg[1] - seg[0] _, f = tempfile.mkstemp(prefix='A-', suffix='-%d-%d.tmp' % (seg[0], d)) cache.append(CacheEntry.from_T050017(f)) segs.append(Segment(*seg)) return cache, segs
def removeLeft(self, boundary): slRm = SegmentList([Segment(self._minU, boundary)]) self._supportiveRanges = self._supportiveRanges - slRm self._supportiveRanges.coalesce() if 0 < len(self._supportiveRanges): self._minU = self._supportiveRanges[0][0] self._cumLens = self._get_cumulative_lens() else: self._minU = self._maxU self._cumLens = np.array([])
def Seg_Split(start, end, frame): """Return a list of time segments that combine to form the period between start and end, with information gaps excluded. Arguments: start -- All times in all output segments are at or after this time end -- All times in all output segments are at or before this time frame -- String such as 'L1_R' informed by desired observatory and type of frame.. Returns: segs -- a list of tuples, each representing a usable time segment.""" empties = [] obsruns = List_Runs(start, end) runends = open('Obsrun_Endtimes.txt', 'r').readlines() if len(obsruns) == 0: return [] if len(obsruns) > 1: for i in range(len(obsruns) - 1): empties.append( Segment(int(runends[obsruns[i] - 1][16:26]), int(runends[obsruns[i + 1] - 1][1:11]))) for run in obsruns: segstrings = open('Gaps/O{}-{}_Gaps.txt'.format(str(run), frame), 'r').readlines() for i in range(len(segstrings)): segstrings[i] = Segment(int(segstrings[i][1:11]), int(segstrings[i][16:26])) empties = empties + segstrings starts = [start] + [seg[1] for seg in empties] ends = [end] + [seg[0] for seg in empties] bookends = starts + ends bookends.sort() if bookends[-1] == end: bookends = bookends[bookends.index(start):] else: bookends = bookends[bookends.index(start):bookends.index(end) + 1] if bookends[1] in starts: bookends = bookends[1:] if bookends[-2] in ends: bookends = bookends[:-1] return Pair_Up(bookends, 0)
def extent(self): """The enclosing segment during which data have been fetched .. warning:: Thie `extent` does not guarantee that all data in the middle have been fetched, gaps may be present depending on which segments were used :type: `~gwpy.segments.Segment` """ return Segment(*self.segments.extent())
def read_cache(cache, channel, start=None, end=None, resample=None, gap=None, pad=None, nproc=1, format=None, **kwargs): """Read a `TimeSeries` from a cache of data files using multiprocessing. The inner-workings are agnostic of data-type, but can only handle a single data type at a time. Parameters ---------- cache : :class:`glue.lal.Cache`, `str` cache of GWF frame files, or path to a LAL-format cache file on disk channel : :class:`~gwpy.detector.channel.Channel`, `str` data channel to read from frames start : `Time`, `~gwpy.time.LIGOTimeGPS`, optional start GPS time of desired data end : `Time`, `~gwpy.time.LIGOTimeGPS`, optional end GPS time of desired data resample : `float`, optional rate (samples per second) to resample format : `str`, optional name of data file format, e.g. ``gwf`` or ``hdf``. nproc : `int`, default: ``1`` maximum number of independent frame reading processes, default is set to single-process file reading. gap : `str`, optional how to handle gaps in the cache, one of - 'ignore': do nothing, let the undelying reader method handle it - 'warn': do nothing except print a warning to the screen - 'raise': raise an exception upon finding a gap (default) - 'pad': insert a value to fill the gaps pad : `float`, optional value with which to fill gaps in the source data, only used if gap is not given, or `gap='pad'` is given Notes ----- The number of independent processes spawned by this function can be calculated as ``min(maxprocesses, len(cache)//minprocesssize)``. Returns ------- data : :class:`~gwpy.timeseries.TimeSeries` a new `TimeSeries` containing the data read from disk """ from gwpy.segments import (Segment, SegmentList) cls = kwargs.pop('target', TimeSeries) # open cache from file if given if isinstance(cache, (unicode, str, file)): cache = open_cache(cache) # fudge empty cache if len(cache) == 0: return cls([], channel=channel, epoch=start) # use cache to get start end times cache.sort(key=lambda ce: ce.segment[0]) if start is None: start = cache[0].segment[0] if end is None: end = cache[-1].segment[1] # get span span = Segment(start, end) if cls not in (StateVector, StateVectorDict) and resample: cache = cache.sieve(segment=span.protract(8)) else: cache = cache.sieve(segment=span) cspan = Segment(cache[0].segment[0], cache[-1].segment[1]) # check for gaps if gap is None and pad is not None: gap = 'pad' elif gap is None: gap = 'raise' segs = cache_segments(cache, on_missing='ignore') & SegmentList([span]) if len(segs) != 1 and gap.lower() == 'ignore' or gap.lower() == 'pad': pass elif len(segs) != 1: gaps = SegmentList([cspan]) - segs msg = ("The cache given to %s.read has gaps in it in the " "following segments:\n %s" % (cls.__name__, '\n '.join(map(str, gaps)))) if gap.lower() == 'warn': warnings.warn(msg) else: raise ValueError(msg) segs = type(segs)([span]) # if reading a small number of channels, try to use lalframe, its faster if format is None and ( isinstance(channel, str) or (isinstance(channel, (list, tuple)) and len(channel) <= MAX_LALFRAME_CHANNELS)): try: from lalframe import frread except ImportError: format = 'gwf' else: kwargs.pop('type', None) format = 'lalframe' # otherwise use the file extension as the format elif format is None: format = os.path.splitext(cache[0].path)[1][1:] # -- process multiple cache segments -------- # this entry point loops this method for each segment if len(segs) > 1: out = None for seg in segs: new = read_cache(cache, channel, start=seg[0], end=seg[1], resample=resample, nproc=nproc, format=format, target=cls, **kwargs) if out is None: out = new.copy() else: out.append(new, gap='pad', pad=pad) return out # -- process single cache segment # force one frame per process minimum nproc = min(nproc, len(cache)) # single-process if nproc <= 1: return cls.read(cache, channel, format=format, start=start, end=end, resample=resample, **kwargs) # define how to read each frame def _read(q, pstart, pend): try: # don't go beyond the requested limits pstart = float(max(start, pstart)) pend = float(min(end, pend)) # if resampling TimeSeries, pad by 8 seconds inside cache limits if cls not in (StateVector, StateVectorDict) and resample: cstart = float(max(cspan[0], pstart - 8)) subcache = cache.sieve(segment=Segment(cstart, pend)) out = cls.read(subcache, channel, format=format, start=cstart, end=pend, resample=None, **kwargs) out = out.resample(resample) q.put(out.crop(pstart, pend)) else: subcache = cache.sieve(segment=Segment(pstart, pend)) q.put(cls.read(subcache, channel, format=format, start=pstart, end=pend, resample=resample, **kwargs)) except Exception as e: q.put(e) # separate cache into parts fperproc = int(ceil(len(cache) / nproc)) subcaches = [Cache(cache[i:i+fperproc]) for i in range(0, len(cache), fperproc)] subsegments = SegmentList([Segment(c[0].segment[0], c[-1].segment[1]) for c in subcaches]) # start all processes queue = ProcessQueue(nproc) proclist = [] for subseg in subsegments: process = Process(target=_read, args=(queue, subseg[0], subseg[1])) process.daemon = True proclist.append(process) process.start() # get data and block data = [queue.get() for p in proclist] for result in data: process.join() if isinstance(result, Exception): raise result # format and return if issubclass(cls, dict): try: data.sort(key=lambda tsd: tsd.values()[0].epoch.gps) except IndexError: pass out = cls() while len(data): tsd = data.pop(0) out.append(tsd) del tsd return out else: if cls in (TimeSeries, TimeSeriesDict): out = TimeSeriesList(*data) else: out = StateVectorList(*data) out.sort(key=lambda ts: ts.epoch.gps) ts = out.join(gap=gap) return ts