def test_1d_two_channels_roundtrip(self): """ roundtrip test call with two channels in a frame """ a = Fr.frgetvect1d("./test.dat", "Adc1") Fr.frputvect('writetest.gwf', [{ 'name': 'Adc1', 'data': a[0], 'start': a[1], 'dx': a[3], 'kind': 'ADC', 'x_unit': a[4], 'y_unit': a[5] }, { 'name': 'reverse', 'data': a[0][::-1], 'start': a[1], 'dx': a[3], 'kind': 'ADC', 'x_unit': a[4], 'y_unit': a[5] }]) b = Fr.frgetvect1d("writetest.gwf", "Adc1") self.assert_(numpy.alltrue(a[0] == b[0])) self.assert_(numpy.alltrue(a[1:] == b[1:])) c = Fr.frgetvect1d("writetest.gwf", "reverse") self.assert_(numpy.alltrue(a[0][::-1] == c[0])) self.assert_(numpy.alltrue(a[1:] == c[1:])) os.remove("writetest.gwf")
def toframefile(filename, channel, data, start, dx, **frargs): """ Write numpy array data to GWF frame file using the given arguments. Arguments: filename : string name of file to write channel : string name of channel to write data : numpy.array array of data to write start : float GPS start time (s) or minimum frequency (Hz) dx : float GPS time step (s) or frequency step (Hz) Unnamed arguments are held in frargs. For usage, see documentation for pylal.Fr.frputvect. """ datadict = frargs datadict['name'] = channel datadict['data'] = data datadict['start'] = start datadict['dx'] = dx Fr.frputvect(filename, [datadict], verbose=False)
def add_noise_frames_to_signal_frames(noise_frame_files, noise_frame_channel, inj_frame_file, inj_frame_channel, ifo, outfile): """ Take noise frame and time-sorted injection frames, and write a single output frame containing both channels between start and stop of noise frame """ #Load in injection frame and create value and time arrays inj_value_array, inj_start_time, __, inj_dt, __, __ = Fr.frgetvect1d(filename=inj_frame_file,channel='%s:%s'%(ifo,inj_frame_channel)) len_inj_array = len(inj_value_array) inj_time_array = np.arange(inj_start_time, inj_start_time + inj_dt*len_inj_array, inj_dt) #Initialize noise times to actually add noise_final_array = np.zeros(len(inj_value_array)) #Load in inj frames and create value and time arrays for i,noise_frame_file in enumerate(noise_frame_files): #Load in noise information for this frame noise_value_array, noise_start_time, __, noise_dt, __, __ = Fr.frgetvect1d(filename=noise_frame_file,channel='%s:%s'%(ifo,noise_frame_channel)) len_noise_array = len(noise_value_array) noise_time_array = np.arange(noise_start_time, noise_start_time + noise_dt*len_noise_array, noise_dt) #Keep on injection information contained within the noise frame tmp_truth_array = (noise_time_array >= inj_time_array[0]) * (noise_time_array <= inj_time_array[-1]) tmp_value_array = noise_value_array[tmp_truth_array] tmp_time_array = noise_time_array[tmp_truth_array] noise_final_array[ (inj_time_array >= tmp_time_array[0]) * (inj_time_array <= tmp_time_array[-1]) ] += tmp_value_array #Save the final noise + inj array frames_dic = {} frames_dic['noise'] = dict(name='%s:%s'%(ifo,noise_frame_channel), data=noise_final_array, start=inj_start_time, dx=inj_dt, type=1) frames_dic['inj'] = dict(name='%s:%s'%(ifo,inj_frame_channel), data=inj_value_array, start=inj_start_time, dx=inj_dt, type=1) Fr.frputvect(outfile, frames_dic.values())
def test_1d_default_roundtrip(self): """ roundtrip test call with default values """ a = Fr.frgetvect1d("./test.dat","Adc1") Fr.frputvect('writetest.gwf', [{'name':'Adc1', 'data':a[0], 'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4], 'y_unit':a[5]}]) b = Fr.frgetvect1d("writetest.gwf", "Adc1") self.assert_(numpy.alltrue(a[0] == b[0])) self.assert_(numpy.alltrue(a[1:] == b[1:])) os.remove("writetest.gwf")
def test_1d_two_channels_roundtrip(self): """ roundtrip test call with two channels in a frame """ a = Fr.frgetvect1d("./test.dat","Adc1") Fr.frputvect('writetest.gwf', [{'name':'Adc1', 'data':a[0], 'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4], 'y_unit':a[5]},{'name':'reverse', 'data':a[0][::-1], 'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4], 'y_unit': a[5]}]) b = Fr.frgetvect1d("writetest.gwf", "Adc1") self.assert_(numpy.alltrue(a[0] == b[0])) self.assert_(numpy.alltrue(a[1:] == b[1:])) c = Fr.frgetvect1d("writetest.gwf", "reverse") self.assert_(numpy.alltrue(a[0][::-1] == c[0])) self.assert_(numpy.alltrue(a[1:] == c[1:])) os.remove("writetest.gwf")
def test_1d_keywords_roundtrip(self): """ roundtrip test call with keyword arguments """ a = Fr.frgetvect1d("./test.dat", "Adc1", span=1) Fr.frputvect('writetest.gwf', [{ 'name': 'Adc1', 'data': a[0], 'start': a[1], 'dx': a[3], 'kind': 'ADC', 'x_unit': a[4], 'y_unit': a[5] }]) b = Fr.frgetvect1d("writetest.gwf", "Adc1") self.assert_(numpy.alltrue(a[0] == b[0])) self.assert_(numpy.alltrue(a[1:] == b[1:])) os.remove("writetest.gwf")
def getRawData(channel, gps, dt): """Read data from RAW file: ch = channel name gps = gps time dt = duration """ global files # get the list of frame files obs = channel[0:2] if len(files) == 0: files = find_data_path(obs, gps, gps + dt) # read data from all files data = array([]) for f in files: # get the frame file start GPS and duration from the name gps0 = int(f.split('-')[-2]) gps1 = gps0 + int(f.split('-')[-1].split('.')[-2]) # find the right time segment to load from this file gps0 = max(gps0, gps) gps1 = min(gps1, gps + dt) # read data and append x = Fr.frgetvect(f, channel, gps0, gps1 - gps0) data = concatenate([data, x[0]]) return data, int(1 / x[3][0])
def fromframefile(filename, channel, start=None, end=None): """ Extract 1D data for given channel from the GWF format frame filename. Returns (x, data) pair of numpy arrays (x is array of times or frequencies. Arguments: filename : string path to GWF frame file channel : string channel name to extract Keyword arguments: start : float GPS start time (s) or minimum frequency (Hz) to return end : float GPS end time (s) or maximum frequency (Hz) to return """ # try to extract data from frame y, fstart, offset, dt = Fr.frgetvect1d(filename, str(channel))[:4] x = fstart+dt*numpy.arange(len(y))+offset # apply constraint on x-axis if start or end: if not start: start=-numpy.infty if not end: end=numpy.infty condition = (x>=start) & (x<end) y = y[condition] x = x[condition] return x,y
def frames2vect(frames, channel, start=-np.infty, stop=np.infty): """ digs out the correct data from a list of frames assumes contiguous frames, constant dt, etc. returns vect, time """ frames = [(extract_start_dur(frame), frame) for frame in frames] frames.sort(key=lambda l: l[0][0]) v = np.array([]) t = np.array([]) for (s, d), frame in frames: frame_start = max(s, start) frame_stop = min(s + d, stop) frame_span = frame_stop - frame_start vect, s, _, dt, _, _ = Fr.frgetvect1d(frame, channel, start=frame_start, span=frame_span) N = len(vect) v = np.concatenate((v, vect)) t = np.concatenate((t, np.arange(s, s + dt * N, dt))) if not len(v): raise ValueError("no Data found!") truth = (start <= t) * (t <= stop) t = t[truth] return v[truth], t[0], t[1] - t[0]
def write_frame(TimeSeries, ifo, usertag, outdir): """ Write a frame """ # Construct name site=ifo.strip('1') frame_name = '{site}-{ifo}_{usertag}-{epoch}-{datalen}.gwf'.format( site=site, ifo=ifo, usertag=usertag, epoch=str(int(TimeSeries.epoch)), datalen=str(int(TimeSeries.data.length * TimeSeries.deltaT))) channel_list = [ {'name':'%s:STRAIN'%ifo, 'data':np.array(TimeSeries.data.data), 'start':TimeSeries.epoch, 'dx':TimeSeries.deltaT, 'kind':'SIM'}, ] print 'writing frame %s...'%frame_name frame_out_path = '%s/%s'%(os.path.abspath(outdir), frame_name) Fr.frputvect(frame_out_path, channel_list) # # Generate a cache file # # setup url path, filename = os.path.split(frame_out_path.strip()) url = "file://localhost%s" % os.path.abspath(os.path.join(path, filename)) # create cache entry c=gluelal.CacheEntry.from_T050017(url) # write to file cache_file = frame_out_path.replace('gwf','lcf') f=open(cache_file,'w') f.writelines('%s\n'%str(c)) f.close() return frame_out_path,cache_file
def extract_scisegs(frames, channel, bitmask, start, stride): """ extract scisegs from channel in frames using bitmask """ if not frames: ### empty list, so no segments return [] ### extract vectors and build segments segset = [] for frame in frames: ### extract the vector from the frame vect, s, ds, dt, xunit, yunit = Fr.frgetvect1d(frame, channel) n = len(vect) ### build time vector add starting time t = np.arange(0, dt*n, dt) + s+ds ### determine whether state acceptable ### add "False" buffers to set up the computation of start and end time # state = np.concatenate( ([False], vect == bitmask, [False])) ### exact integer match state = np.concatenate( ([False], (vect >> bitmask) & 1, [False])) ### bitwise operation ### determine beginning of segments ### i=False i+1 = True strip the trailing buffer b = ( (1-state[:-1])*(state[1:]) )[:-1].astype(bool) b = t[b] ### select out times ### determine end of segments ### i=True i+1=False strip the leading buffer e = ( (state[:-1])*(1-state[1:]) )[1:].astype(bool) e = t[e] + dt ### select out times ### extra dt moves these markers to the end of segments ### stitch together start and end times, append to global list segset += list( np.transpose( np.array( [b, e] ) ) ) if not segset: ### empty list return [] ### clean up segs! segs = [] seg1 = segset[0] for seg2 in segset[1:]: if seg1[1] == seg2[0]: seg1[1] = seg2[1] ### join the segments else: ### check segment for sanity append_seg = check_seg( seg1, (start, start+stride) ) if append_seg: segs.append( append_seg ) seg1 = seg2 append_seg = check_seg( seg1, (start, start+stride) ) if append_seg: segs.append( append_seg ) ### return final list of lists! return segs
def read_frame(filename, channels, start=None, end=None): if start is not None and end is not None: if type(start) is _lal.LIGOTimeGPS: if start.gpsNanoSeconds != 0: raise ValueError('start and end times must be integer valued') else: start = start.gpsSeconds else: if int(start) != start: raise ValueError('start and end times must be integer valued') else: start = int(start) if type(end) is _lal.LIGOTimeGPS: if end.gpsNanoSeconds != 0: raise ValueError('start and end times must be integer valued') else: end = end.gpsSeconds else: if int(end) != end: raise ValueError('start and end times must be integer valued') else: end = int(end) span = end - start if span <= 0: raise ValueError('beginning must be before end') else: start = -1 span = -1 if type(channels) is list: ts = [] for channel in channels: frdata = Fr.frgetvect1d(filename, channel, start, span) ts.append(pycbc.types.TimeSeries(initial_array=frdata[0], delta_t=frdata[3], epoch=_lal.LIGOTimeGPS(frdata[1]), copy=False)) else: frdata = Fr.frgetvect1d(filename, channels, start, span) ts = pycbc.types.TimeSeries(initial_array=frdata[0], delta_t=frdata[3], epoch=_lal.LIGOTimeGPS(frdata[1]), copy=False) return ts
def vec_from_frames(frames, start, stop, verbose=False): """ returns a numpy array of the data inculded in frames between start and stop CURRENTLY ASSUME CONTIGUOUS DATA, but we should check this meant to be used with files_from_cache """ vecs = [] dt = 0 for frame, strt, dur in frames: if verbose: print frame s = max(strt, start) d = min(start+dur,stop) - s vec, gpstart, offset, dt, _, _ = Fr.frgetvect1d(frame, ifo_chan, start=s, span=d) vecs.append( vec ) vec = np.concatenate(vecs) return vec, dt
def checkDQ(framecache_file, chname, abs_start, abs_stop, ifo, bitmask): """ Takes a file containing the data quality state vetor and converts it to segments of a desired bitmask, returning injection status """ #Initialize data-quality veto status as False (will change if 'No veto' bits are off) DQV_flag = False #Open framecache file cache = open(framecache_file, 'rt') #Loop over frames in the cache for line in cache: #Get frame_start, frame_stride, and frame_file words = line.split() frame_start = int(words[2]) frame_stride = int(words[3]) frame_file = words[4].split('file://localhost')[1] #Open state vector as array state_array = Fr.frgetvect1d(filename=frame_file,channel='%s:%s'%(ifo,chname))[0] #Calculate sample rate samp_rate = len(state_array)/float(frame_stride) #Loop over state vetor for i, value in enumerate(state_array): #Check to make sure we've passed the absolute start if (frame_start + i/float(samp_rate)) < abs_start: continue #Check to make sure we haven't passed the absolute stop elif (frame_start + i/float(samp_rate)) > abs_stop: break #Check if DQ vector denotes that a veto is present if ((int(value) & bitmask) != bitmask) or (int(value)<0): #(e.g., we might want bits ??, ??, or ?? to be on if there are no data-quality vetoes) DQV_flag = True cache.close() #Return injection flag return DQV_flag
def test_frgetevent_known(self): """ test that we can pull the known contents from a test MBTA file """ a = Fr.frgetevent("MbtaFake-930909680-16.gwf") self.assertEqual(len(a), 1) self.assertEqual(a[0]["name"], "MbtaHLV_Chi2OK") self.assertAlmostEqual(a[0]["H1:end_time"], 930493014.980663, 6) self.assertAlmostEqual(a[0]["H1:SNR"], 9.0) self.assertAlmostEqual(a[0]["H1:mass1"], 3.14696, 5) self.assertAlmostEqual(a[0]["H1:mass2"], 10.2256, 4) self.assertAlmostEqual(a[0]["H1:eff_distance"], 230.718, 3) self.assertAlmostEqual(a[0]["L1:end_time"], 930493014.984246, 6) self.assertAlmostEqual(a[0]["L1:SNR"], 8.0) self.assertAlmostEqual(a[0]["L1:mass1"], 3.44696, 5) self.assertAlmostEqual(a[0]["L1:mass2"], 10.5256, 4) self.assertAlmostEqual(a[0]["L1:eff_distance"], 582.025, 3) self.assertAlmostEqual(a[0]["V1:end_time"], 930493014.989135, 6) self.assertAlmostEqual(a[0]["V1:SNR"], 7.0) self.assertAlmostEqual(a[0]["V1:mass1"], 2.84696, 5) self.assertAlmostEqual(a[0]["V1:mass2"], 9.92556, 5) self.assertAlmostEqual(a[0]["V1:eff_distance"], 200.113, 3)
def write_frame(det_data, ifo, seed, epoch, datalen, outdir): """ Write a frame """ # Construct name site = ifo.strip('1') frame_name = '{site}-{ifo}_{wf_name}_{seed}-{epoch}-{datalen}.gwf'.format( site=site, ifo=ifo, wf_name=det_data.waveform_name, seed=seed, epoch=str(int(epoch)), datalen=str(int(datalen))) channel_list = [ { 'name': '%s:STRAIN' % ifo, 'data': np.array(det_data.td_response.data), 'start': epoch, 'dx': 1.0 / 16384, 'kind': 'SIM' }, { 'name': '%s:SIGNAL' % ifo, 'data': np.array(det_data.td_signal.data), 'start': epoch, 'dx': 1.0 / 16384, 'kind': 'SIM' }, { 'name': '%s:NOISE' % ifo, 'data': np.array(det_data.td_noise.data), 'start': epoch, 'dx': 1.0 / 16384, 'kind': 'SIM' }, ] print 'writing frame %s...' % frame_name frame_out_path = '%s/%s' % (os.path.abspath(outdir), frame_name) Fr.frputvect(frame_out_path, channel_list) # # Generate a cache file # # setup url path, filename = os.path.split(frame_out_path.strip()) url = "file://localhost%s" % os.path.abspath(os.path.join(path, filename)) # create cache entry c = gluelal.CacheEntry.from_T050017(url) # write to file cache_file = frame_out_path.replace('gwf', 'lcf') f = open(cache_file, 'w') f.writelines('%s\n' % str(c)) f.close() return frame_out_path, cache_file
virgo_fs = 20000.0 f, ax = pl.subplots(nrows=2,ncols=1) for s,site in enumerate(sites): start=float(trigger_time)-0.5*datalen if site=='V': delta_t = 1/virgo_fs else: delta_t = 1/ligo_fs # Stick these in pycbc time series and we'll have an easy time computing # matches etc later bmdc_data[site] = pycbc.types.TimeSeries(Fr.frgetvect(burstmdc_frame, burstmdc_channels[s], start=start, span=datalen)[0], delta_t=delta_t, epoch=start) lalsim_data[site] = pycbc.types.TimeSeries(Fr.frgetvect(lalsim_frames[s], lalsim_channels[s], start=start, span=datalen)[0], delta_t=1/ligo_fs, epoch=start) # Plot burstMDC channels ax[0].plot(bmdc_data[site].sample_times-float(trigger_time), bmdc_data[site], label=burstmdc_channels[s]) ax[0].set_xlabel('Seconds after %s'%trigger_time) ax[0].legend() ax[0].set_title('BurstMDC') # Plot lalsim channels
# # Are we looking outside the range rather than inside? # if min_threshold is not None and max_threshold is not None: invert = min_threshold >= max_threshold else: invert = False if opts.verbose: print "Inverted? %s"% str(invert) seglist = segmentlist([]) for path in cache.pfnlist(): # # Read data # data, start, _, dt, _, _ = Fr.frgetvect1d(path, channel) # # Apply conditions and transform samples to segments # if equals is not None: seglist.extend(dqsegs.equality_data_to_seglist(data, start, dt, equality=equals)) if bit_mask is not None: seglist.extend(dqsegs.mask_data_to_seglist(data, start, dt, mask_on=bit_mask)) else: seglist.extend(dqsegs.threshold_data_to_seglist(data, start, dt, min_threshold=min_threshold, max_threshold=max_threshold, invert=invert)) seglist.coalesce() all_segs["%s: %s" % (channel, str(opthresholds))] = seglist if opts.verbose:
def test_frame(self): # This is a file in the temp directory that will be deleted when it is garbage collected frmfile = tempfile.NamedTemporaryFile() filename = frmfile.name # Now we will create a frame file, specifiying that it is a timeseries Fr.frputvect(filename,[{'name':'channel1', 'data':self.data1, 'start':int(self.epoch), 'dx':self.delta_t,'type':1}, {'name':'channel2', 'data':self.data2, 'start':int(self.epoch), 'dx':self.delta_t,'type':1}]) with self.context: if _options['scheme'] == 'cpu': # Reading just one channel first ts1 = pycbc.frame.read_frame(filename,'channel1') # Chacking all values self.checkCurrentState((ts1,),(self.data1,),self.places) # Now checking the start time self.assertTrue(ts1.start_time == self.epoch) # And the duration self.assertTrue(ts1.end_time-ts1.start_time == self.size*self.delta_t) # Now reading multiple channels ts2 = pycbc.frame.read_frame(filename,['channel1','channel2']) # We should get back a list self.assertTrue(type(ts2) is list) self.checkCurrentState(ts2, (self.data1,self.data2), self.places) self.assertTrue(ts2[0].start_time == self.epoch) self.assertTrue(ts2[1].start_time == self.epoch) self.assertTrue(ts2[0].end_time-ts2[0].start_time == self.size*self.delta_t) self.assertTrue(ts2[1].end_time-ts2[1].start_time == self.size*self.delta_t) # These are the times and indices for the segment we will try to read start = self.epoch+10 end = self.epoch+50 startind = int(10/self.delta_t) endind = int(50/self.delta_t) # Now reading in a specific segment with an integer ts3 = pycbc.frame.read_frame(filename, 'channel1', start=int(start), end=int(end)) # The same, but with a LIGOTimeGPS for the start and end times ts4 = pycbc.frame.read_frame(filename, 'channel1', start=start, end=end) # Now we will check those two TimeSeries self.checkCurrentState((ts3,ts4), (self.data1[startind:endind],self.data1[startind:endind]), self.places) self.assertTrue(40 - (float(ts3.end_time)-float(ts3.start_time)) < self.delta_t) self.assertTrue(ts3.start_time == start) self.assertTrue(40 - (float(ts4.end_time)-float(ts4.start_time)) < self.delta_t) self.assertTrue(ts4.start_time == start) # And now some cases that should raise errors # There must be a span grater than 0 self.assertRaises(ValueError, pycbc.frame.read_frame,filename,'channel1', start=self.epoch,end=self.epoch) # The start must be before the end self.assertRaises(ValueError, pycbc.frame.read_frame,filename,'channel1', start=self.epoch+1,end=self.epoch) # Non integer times should also raise an error badtime = lal.LIGOTimeGPS(int(self.epoch)+5,1000) self.assertRaises(ValueError, pycbc.frame.read_frame,filename,'channel1', start=self.epoch,end=badtime) self.assertRaises(ValueError, pycbc.frame.read_frame,filename,'channel1', start=float(self.epoch),end=float(badtime))
def framecache2segs(framecache_file, chname, abs_start, abs_stop, outdir, ifo, run_bitmask, inj_bitmask): """ Takes a file containing the data quality state vetor and converts it to segments of a desired bitmask, returning injection status """ #Initialize injection status as False (will change if 'No injection' bits are off) inj_flag = False #Open framecache file and segment file to write to cache = open(framecache_file, 'rt') segfile = open(outdir+'/%s_%s_%s.seg'%(ifo,abs_start,abs_stop),'wt') #Define start and stop of current segment current_start = None current_stop = None #Loop over frames in the cache for line in cache: #Get frame_start, frame_stride, and frame_file words = line.split() frame_start = int(words[2]) frame_stride = int(words[3]) frame_file = words[4].split('file://localhost')[1] #Open state vector as array state_array = Fr.frgetvect1d(filename=frame_file,channel='%s:%s'%(ifo,chname))[0] #Calculate sample rate samp_rate = len(state_array)/float(frame_stride) #Loop over state vetor for i, value in enumerate(state_array): #Check to make sure we've passed the absolute start if (frame_start + i/float(samp_rate)) < abs_start: continue #Check to make sure we haven't passed the absolute stop elif (frame_start + i/float(samp_rate)) > abs_stop: break #Check if state vector corresponds to desired bitmask elif ((int(value) & run_bitmask) == run_bitmask) and (int(value)>=0): #(e.g., 0b00011 = 3 and we want bits 0 and 1 to be on, so we do & with 3) #Data is good, start new seg if needed if not current_start: current_start = int(np.ceil(frame_start + i/float(samp_rate) )) #data good starting at ith sample, use ceiling so don't underestimate start else: #Data not good, end current seg if needed if current_start: current_stop = int(np.floor(frame_start + i/float(samp_rate) )) #data goes bad at ith sample but good until then, use floor so don't overestimate stop if current_start < current_stop: segfile.write('%s %s\n'%(current_start, current_stop)) #Wait to start next segment until find good data current_start = None current_stop = None #Check if state vector denotes that an injection is present if ((int(value) & inj_bitmask) != inj_bitmask): #(e.g., we might want bits 5, 6, 7, or 8 to be on if there are no HW injections) inj_flag = True #Write final segment for this frame if needed if current_start: if (current_start < int(frame_start+frame_stride)) and (int(frame_start+frame_stride) < abs_stop): segfile.write('%s %s\n'%(current_start, int(frame_start+frame_stride))) elif (current_start < int(frame_start+frame_stride)) and (current_start < abs_stop): segfile.write('%s %s\n'%(current_start, int(abs_stop))) #Wait to start next segment until find good data current_start = None current_stop = None cache.close() segfile.close() #Return injection flag return inj_flag
def readframedata(frameCache, channelName, frameType, startTime, stopTime, allowRedundantFlag, debugLevel): #READFRAMEDATA Read a single channel of data from frame files # #READFRAMEDATA finds and retrieves the requested time series data from a #set of frame files. The data is specified by the frame file type, #channel name, start time, and duration or stop time. The necessary frame #files are located using a file name caching scheme. #% #usage: [data, sampleFrequency, time] = ... #readframedata(frameCache, channelName, frameType, ... #startTime, stopTime, allowRedundantFlag, ... #debugLevel); #% #frameCache file name cache #channelName channel name #frameType frame file type #startTime GPS start time #stopTime GPS stop time (or duration) #allowRedundantFlag permit redundant frame data #debugLevel verboseness of debug output #% #data data vector #sampleFrequency sample frequency [Hz] #time time vector #% #READFRAMEDATA expects frame cache information in the format produced by #LOADFRAMECACHE which contains the site, frame type, duration, time, and #location of available frame data files. #% #The requested site designator is determined from the first character of #the requested channel name. The frame file type may contain wildcards. #Unless redundant frame data is permitted, it is an error if the same #frame file appears more than once in the frame cache file. If redundant #frame data is permitted, the first matching frame file from the cache is #used. It is always an error if two frame files overlap but do not cover #the same time range or differ in type. By default, redundant frame data #is permitted. #% #READFRAMEDATA retrieves data from the requested start time up to, but not #including, the requested stop time, such that stop minus start seconds #are retrieved. Alternatively, the desired duration in seconds may be #specified instead of the GPS stop time parameter. #% #The resulting time series is returned as two row vectors containing the #data sequence and the corresponding GPS timestamps, as well as the scalar #sample frequency. To protect against roundoff error, an integer sample #frequency is assumed. #% #If it is unable to load the requested data, READFRAMEDATA returns empty #result vectors and zero sample frequency as well as a warning if #debugLevel is set to 1 or higher. By default, a debugLevel of unity is #assumed. #% #READFRAMEDATA is built on top of the FRGETVECT function from the FrameL #library, which is available from the following URL. #% # #% # # #Shourov K. Chatterji <*****@*****.**> #Jameson Rollins <*****@*****.**> # #$Id: readframedata.m 2326 2009-09-21 08:37:42Z jrollins $ # # Rewritten in Python by Sudarshan Ghonge <*****@*****.**> # 2015-09-04 # # if specified stop time precedes start time, if(stopTime<startTime): # treat stop time as a duration stopTime = startTime + stopTime # determine site designator from channel name site = channelName[0] #if specified frame cache is invalid if(frameCache==None): if(debugLevel>=1): # Issue warning print 'Warning: Invalid frame cache' data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] # return empty results # Identifying matching segments from frame cache # find overlap of cache segments with requested data segmentStartTimes = np.maximum(startTime, frameCache.startTimes) segmentStopTimes = np.minimum(stopTime, frameCache.stopTimes) # identify cache segments which overlap requested times segments = np.where(segmentStopTimes > segmentStartTimes)[0] # if no segments overlap with requested times if(len(segments)==0): if debugLevel>=1: print 'Warning: No data available for [%d, %d]' %(startTime, stopTime) data = [] time = [] sampleFrequency = 0 # return empty results return [data, sampleFrequency] # otherwise, find overlapping segments else: # identify cache segments with requested site and frame type siteMatches = [] sitesScanned=0 for iSite in frameCache.sites[segments]: if site in iSite: siteMatches.append(sitesScanned) sitesScanned+=1 siteMatches = np.asarray(siteMatches) frameTypeMatches = [] frameTypesScanned=0 for iType in frameCache.frameTypes[segments]: if frameType in iType: frameTypeMatches.append(frameTypesScanned) frameTypesScanned+=1 frameTypeMatches = np.asarray(frameTypeMatches) segIdx = np.intersect1d(siteMatches, frameTypeMatches) if(len(segIdx)==0): segments = [] else: segments = segments[segIdx] # Identify available frame files # initialize list of available frame files frameFilePaths = [] frameFileTypes = [] frameFileStartTimes = [] frameFileStopTimes = [] # lopp over the matching segments for segment in segments: # frame type of the frame files in segment frameFileType = frameCache.frameTypes[segment] firstFrameFileStartTime = frameCache.startTimes[segment] + frameCache.durations[segment]*np.floor((segmentStartTimes[segment]-frameCache.startTimes[segment])/frameCache.durations[segment]) lastFrameFileStartTime = frameCache.startTimes[segment] + frameCache.durations[segment]*np.ceil((segmentStopTimes[segment] - frameCache.startTimes[segment])/frameCache.durations[segment] - 1) for frameFileStartTime in np.arange(firstFrameFileStartTime, lastFrameFileStartTime + frameCache.durations[segment], frameCache.durations[segment]): frameFileStopTime = frameFileStartTime + frameCache.durations[segment] frameFilePath = frameCache.directories[segment] + '/' + frameCache.sites[segment] + '-' + frameCache.frameTypes[segment] + '-' + '%09d' %(frameFileStartTime) + '-' + '%d' %(frameCache.durations[segment]) + '.gwf' # In case the frame file durations start becoming variable. The ls <name> with a '*' in place of the duration field # returns the file name. we then use that file name import os.path if(not os.path.isfile(frameFilePath)): frameFilePath = frameCache.directories[segment] + '/' + frameCache.sites[segment] +'-' + frameCache.frameTypes[segment] + '-' + '%010d' %(frameFileStartTime) + '-' + '*'+ '.gwf' frameFilePath = os.popen('ls %s'%(frameFilePath)).readlines()[0].split('\n')[0] if(os.path.isfile(frameFilePath)): frameFilePaths.append(frameFilePath) frameFileTypes.append(frameFileType) frameFileStartTimes.append(frameFileStartTime) frameFileStopTimes.append(frameFileStopTime) frameFilePaths = np.asarray(frameFilePaths) frameFileTypes = np.asarray(frameFileTypes) frameFileStartTimes = np.asarray(frameFileStartTimes) frameFileStopTimes = np.asarray(frameFileStopTimes) numberOfFrameFiles = len(frameFilePaths) keepFrameFileNumbers = [] for frameFileNumber in range(numberOfFrameFiles): keepFrameFileFlag = True for previousFrameFileNumber in range(frameFileNumber): overlapStartTime = np.maximum(frameFileStartTimes[frameFileNumber], frameFileStartTimes[previousFrameFileNumber]) overlapStoptime = np.minimum(frameFileStopTimes[frameFileNumber], frameFileStopTimes[previousFrameFileNumber]) if (overlapStartTime < overlapStoptime): if(allowRedundantFlag): if((frameFileStartTimes[frameFileNumber]==frameFileStartTimes[previousFrameFileNumber]) & (frameFileStopTimes[frameFileNumber] == frameFileStopTimes[previousFrameFileNumber]) & (frameFileTypes[frameFileNumber]==frameFileTypes[previousFrameFileNumber])): keepFrameFileFlag = False continue else: if(debugLevel>=1): print 'Warning: Overlapping but dissimilar frame files %s and %s.' %(frameFilePaths[frameFileNumber], frameFilePaths[previousFrameFileNumber]) data = [] time = [] sampleFrequency=0 return [data, sampleFrequency] else: if(debugLevel>=1): print 'Warning: Redundant frame files %s and %s.' %(frameFilePaths[frameFileNumber], frameFilePaths[previousFrameFileNumber]) data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] if(keepFrameFileFlag): keepFrameFileNumbers.append(frameFileNumber) keepFrameFileNumbers = np.asarray(keepFrameFileNumbers) frameFilePaths = frameFilePaths[keepFrameFileNumbers] frameFileTypes = frameFileTypes[keepFrameFileNumbers] frameFileStartTimes = frameFileStartTimes[keepFrameFileNumbers] frameFileStopTimes = frameFileStopTimes[keepFrameFileNumbers] sortedIndices = np.argsort(frameFileStartTimes) frameFilePaths = frameFilePaths[sortedIndices] frameFiletypes = frameFileTypes[sortedIndices] frameFileStartTimes = frameFileStartTimes[sortedIndices] frameFileStopTimes = frameFileStopTimes[sortedIndices] continuityStartTimes = np.append(np.maximum(startTime, frameFileStartTimes), stopTime) continuityStopTimes = np.append(startTime, np.minimum(stopTime, frameFileStopTimes)) discontinuities = np.where(continuityStartTimes!=continuityStopTimes)[0] if(len(discontinuities)>0): if(debugLevel >=1): print 'Warning: Missing %s '%(channelName), frameType[1:len(frameType)-1], ' data at ' , '%d'%(np.round(continuityStopTimes[discontinuities[0]])), '.' data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] data = np.array([]) time = np.array([]) sampleFrequency = None numberOfFrameFiles = len(frameFilePaths) for frameFileNumber in range(numberOfFrameFiles): frameFilePath = frameFilePaths[frameFileNumber] if(debugLevel>=2): print 'Reading %s...\n' %(frameFilePath) frameFileStartTime = frameFileStartTimes[frameFileNumber] frameFileStopTime = frameFileStopTimes[frameFileNumber] readData = [] readStartTime = np.maximum(startTime, frameFileStartTime) readDuration = np.minimum(stopTime, frameFileStopTime) - readStartTime realChannelName = channelName try: outputStruct = Fr.frgetvect(frameFilePath, realChannelName, readStartTime, readDuration, False) readData = outputStruct[0] readTime = outputStruct[2] readSampleFrequency = 1.0/outputStruct[3][0] readTimeStep = outputStruct[3][0] readGPS = outputStruct[1] except Exception as inst: if(debugLevel>=2): print inst.message if((len(readData)==0) | np.any(np.isnan(readData))): if(debugLevel>=1): print 'Warning: Error reading %s from %s.' %(channelName, frameFilePath) data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] if(sampleFrequency==None): sampleFrequency = readSampleFrequency elif(sampleFrequency!=readSampleFrequency): if(debugLevel>=1): print 'Warning: Inconsistent sample frequency for %s in frameFilePath.' %(channelname, frameFilePath) data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] data = np.append(data, readData) return [data, sampleFrequency]
import pmns_utils import pmns_simsig import matplotlib matplotlib.rc('xtick', labelsize=16) matplotlib.rc('ytick', labelsize=16) matplotlib.rc('font', size=16) from matplotlib import pyplot as pl import matplotlib.cm as cm from mpl_toolkits.axes_grid1 import make_axes_locatable # # Load frame data # h1data = Fr.frgetvect('H-H1-871727590-513.gwf', 'H1:LSC-STRAIN', span=500) # # Put frame data in lal TimeSeries # noise = lal.CreateREAL8TimeSeries('H1', lal.LIGOTimeGPS(h1data[1]), 0, h1data[3][0], lal.StrainUnit, len(h1data[0])) noise.data.data = h1data[0] nonoise = lal.CreateREAL8TimeSeries('H1', lal.LIGOTimeGPS(h1data[1]), 0, h1data[3][0], lal.StrainUnit, len(h1data[0])) nonoise.data.data = np.zeros(len(h1data[0])) # # Generate a post-merger signal #
#!/usr/bin/env python from pylal import Fr h_frame = 'H-H1_magA_tapered_1105199095-946076460-512.gwf' l_frame = 'L-L1_magA_tapered_1105199095-946076460-512.gwf' gps_start = 946076563.480737805 ra = 1.39 dec = -0.93 h_data = Fr.frgetvect(h_frame, 'H1:STRAIN', start=gps_start, span=1) l_data = Fr.frgetvect(l_frame, 'L1:STRAIN', start=gps_start, span=1)
def generate_gwf(self, mdc, directory, channel="SCIENCE", force=False): """ Produce the gwf file which corresponds to the MDC set over the period of this frame. Parameters ---------- mdc : MDCSet object The MDC set which should be used to produce this frame. directory : str The root directory where all of the frames are to be stored, for example "/home/albert.einstein/data/mdc/frames/" would cause the SineGaussian injections to be made in the directories under "/home/albert.einstein/data/mdc/frames/sg" channel : str The name of the channel which the injections should be made into. This is prepended by the initials for each interferometer, so there will be a channel for each interferometer in the gwf. force : bool If true this forces the recreation of a GWF file even if it already exists. Outputs ------- gwf The GWF file for this frame. """ ifosstr = "".join(set(ifo[0] for ifo in self.ifos)) family = mdc.waveforms[0].waveform filename = "{}-{}-{}-{}.gwf".format(ifosstr, family, self.start, self.duration) head_date = str(self.start)[:5] frameloc = directory+"/"+mdc.directory_path()+"/"+head_date+"/" #print frameloc, filename if not os.path.isfile(frameloc + filename) or force: data = [] # Define the start point of the time series top be generated for the injection epoch = lal.LIGOTimeGPS(self.start) # Loop through each interferometer for ifo in self.ifos: # Calculate the number of samples in the timeseries nsamp = (self.end-self.start)*16384 # Make the timeseries h_resp = lal.CreateREAL8TimeSeries("inj time series", epoch, 0, 1.0/16384, lal.StrainUnit, nsamp) # Loop over all of the injections corresponding to this frame rowlist = self.get_rowlist(mdc) if len(rowlist)==0: return for row in rowlist: sim_burst = mdc.waveforms[row] # Produce the time domain waveform for this injection hp, hx = lalburst.GenerateSimBurst(sim_burst, 1.0/16384); # Apply detector response det = lalsimulation.DetectorPrefixToLALDetector(ifo) # Produce the total strains h_tot = lalsimulation.SimDetectorStrainREAL8TimeSeries(hp, hx, sim_burst.ra, sim_burst.dec, sim_burst.psi, det) # Inject the waveform into the overall timeseries lalsimulation.SimAddInjectionREAL8TimeSeries(h_resp, h_tot, None) # Write out the data to the list which will eventually become our frame data.append({"name": "%s:%s" % (ifo, channel), "data": h_resp.data.data, "start": float(epoch), "dx": h_resp.deltaT, "kind": "SIM"}) # Make the directory in which to store the files # if it doesn't exist already mkdir(frameloc) # Write out the frame file Fr.frputvect(frameloc+filename, data)
def test_cache(self): # Knowing the middle of our array will be helpful, because we will put half on one frame, # and half on hte other. We will also need this to read in a segment of hte cache that # crosses this seam. half = int((self.size/2)*self.delta_t) # These need to be named so that lalapps_path2cache can turn them into a single cache file. frmfile1 = tempfile.NamedTemporaryFile(prefix='H-frame-'+str(int(self.epoch))+'-'+str(half)+'.') frmfile2 = tempfile.NamedTemporaryFile(prefix='H-frame-'+str(int(self.epoch+half))+'-'+str(half)+'.') frmfile3 = tempfile.NamedTemporaryFile(prefix='H-frame-'+str(int(self.epoch+half+16))+'-'+str(half-16)+'.') # We will need access to the actual filenames. frmname1 = frmfile1.name frmname2 = frmfile2.name frmname3 = frmfile3.name firsthalf1 = self.data1[0:(self.size/2)] secondhalf1 = self.data1[(self.size/2):] # This third piece will be paired up with the first one to create a cache file with a gap # of 16 seconds after the half-way point gaphalf1 = self.data1[(self.size/2 + 16/self.delta_t):] # The same is done to the second dataset firsthalf2 = self.data2[0:(self.size/2)] secondhalf2 = self.data2[(self.size/2):] gaphalf2 = self.data2[(self.size/2 + 16/self.delta_t):] # Now we will create a frame file, this will hold the first half of our data Fr.frputvect(frmname1,[{'name':'channel1', 'data':firsthalf1, 'start':int(self.epoch), 'dx':self.delta_t,'type':1}, {'name':'channel2', 'data':firsthalf2, 'start':int(self.epoch), 'dx':self.delta_t,'type':1}]) # This will hold the second half Fr.frputvect(frmname2,[{'name':'channel1', 'data':secondhalf1, 'start':int(self.epoch+half), 'dx':self.delta_t,'type':1}, {'name':'channel2', 'data':secondhalf2, 'start':int(self.epoch+half), 'dx':self.delta_t,'type':1}]) # This third one will hold the second half, but without the first 16 seconds, so we can check importing from a cache with holes. Fr.frputvect(frmname3,[{'name':'channel1', 'data':gaphalf1, 'start':int(self.epoch + half + 16), 'dx':self.delta_t,'type':1}, {'name':'channel2', 'data':gaphalf2, 'start':int(self.epoch + half + 16), 'dx':self.delta_t,'type':1}]) # These files are what path2cache will actually read, they will hold a list of frames frmlist1 = tempfile.NamedTemporaryFile() frmlist2 = tempfile.NamedTemporaryFile() listname1 = frmlist1.name listname2 = frmlist2.name # The first one will contain the complete set, split over two frames with open(listname1, 'w') as f1: f1.write(frmname1+'\n') f1.write(frmname2) # This second one will use the gap frame for the second half, so there # will be a 16 second gap in the middle of this cache with open(listname2, 'w') as f2: f2.write(frmname1+'\n') f2.write(frmname3) cache1 = tempfile.NamedTemporaryFile() cache2 = tempfile.NamedTemporaryFile() cachename1 = cache1.name cachename2 = cache2.name # Now we can actually make the caches from the list of frames subprocess.call(['lalapps_path2cache','-i',listname1,'-o',cachename1]) subprocess.call(['lalapps_path2cache','-i',listname2,'-o',cachename2]) with self.context: if _options['scheme'] == 'cpu': # Reading just one channel first ts = pycbc.frame.read_cache(cachename1,'channel1',self.epoch,self.epoch+self.size*self.delta_t) self.checkCurrentState([ts],[self.data1],self.places) self.assertTrue(ts.start_time == self.epoch) self.assertTrue(ts.end_time-ts.start_time == self.size*self.delta_t) # Now reading multiple channels ts = pycbc.frame.read_cache(cachename1,['channel1','channel2'],self.epoch,self.epoch+self.size*self.delta_t) self.assertTrue(type(ts) is list) self.checkCurrentState(ts, [self.data1,self.data2], self.places) self.assertTrue(ts[0].start_time == self.epoch) self.assertTrue(ts[1].start_time == self.epoch) self.assertTrue(ts[0].end_time-ts[0].start_time == self.size*self.delta_t) self.assertTrue(ts[1].end_time-ts[1].start_time == self.size*self.delta_t) # Now reading in a specific segment with an integer start = self.epoch + 10 end = self.epoch + half + 50 startind = 10/self.delta_t endind = (half + 50) / self.delta_t ts = pycbc.frame.read_cache(cachename1, 'channel1', start=int(start), end=int(end)) # Now we'll check all the values self.checkCurrentState((ts,), (self.data1[startind:endind],), self.places) # The duration self.assertTrue((40+half) - (float(ts.end_time)-float(ts.start_time)) < self.delta_t) # And the start self.assertTrue(ts.start_time == self.epoch+10) # The same, but with a LIGOTimeGPS for the start and end times ts = pycbc.frame.read_cache(cachename1, 'channel1', start=start, end=end) # Now we'll check all the values self.checkCurrentState((ts,), (self.data1[startind:endind],), self.places) # The duration self.assertTrue((40+half) - (float(ts.end_time)-float(ts.start_time)) < self.delta_t) # And the start self.assertTrue(ts.start_time == self.epoch+10) # And now some cases that should raise errors # There should be an error if there are gaps in the data requested self.assertRaises(ValueError, pycbc.frame.read_cache,cachename2,'channel1', self.epoch,self.epoch+self.size*self.delta_t) # There must be a span grater than 0 self.assertRaises(ValueError, pycbc.frame.read_cache,cachename1,'channel1', start=self.epoch,end=self.epoch) # The start must be before the end self.assertRaises(ValueError, pycbc.frame.read_cache,cachename1,'channel1', start=self.epoch+1,end=self.epoch) # Non integer times should also raise an error badtime = lal.LIGOTimeGPS(int(self.epoch)+5,1000) self.assertRaises(ValueError, pycbc.frame.read_cache,cachename1,'channel1', start=self.epoch,end=badtime) self.assertRaises(ValueError, pycbc.frame.read_cache,cachename1,'channel1', start=float(self.epoch),end=float(badtime))
def readframedata(frameCache, channelName, frameType, startTime, stopTime, allowRedundantFlag, debugLevel): #READFRAMEDATA Read a single channel of data from frame files # #READFRAMEDATA finds and retrieves the requested time series data from a #set of frame files. The data is specified by the frame file type, #channel name, start time, and duration or stop time. The necessary frame #files are located using a file name caching scheme. #% #usage: [data, sampleFrequency, time] = ... #readframedata(frameCache, channelName, frameType, ... #startTime, stopTime, allowRedundantFlag, ... #debugLevel); #% #frameCache file name cache #channelName channel name #frameType frame file type #startTime GPS start time #stopTime GPS stop time (or duration) #allowRedundantFlag permit redundant frame data #debugLevel verboseness of debug output #% #data data vector #sampleFrequency sample frequency [Hz] #time time vector #% #READFRAMEDATA expects frame cache information in the format produced by #LOADFRAMECACHE which contains the site, frame type, duration, time, and #location of available frame data files. #% #The requested site designator is determined from the first character of #the requested channel name. The frame file type may contain wildcards. #Unless redundant frame data is permitted, it is an error if the same #frame file appears more than once in the frame cache file. If redundant #frame data is permitted, the first matching frame file from the cache is #used. It is always an error if two frame files overlap but do not cover #the same time range or differ in type. By default, redundant frame data #is permitted. #% #READFRAMEDATA retrieves data from the requested start time up to, but not #including, the requested stop time, such that stop minus start seconds #are retrieved. Alternatively, the desired duration in seconds may be #specified instead of the GPS stop time parameter. #% #The resulting time series is returned as two row vectors containing the #data sequence and the corresponding GPS timestamps, as well as the scalar #sample frequency. To protect against roundoff error, an integer sample #frequency is assumed. #% #If it is unable to load the requested data, READFRAMEDATA returns empty #result vectors and zero sample frequency as well as a warning if #debugLevel is set to 1 or higher. By default, a debugLevel of unity is #assumed. #% #READFRAMEDATA is built on top of the FRGETVECT function from the FrameL #library, which is available from the following URL. #% # #% # # #Shourov K. Chatterji <*****@*****.**> #Jameson Rollins <*****@*****.**> # #$Id: readframedata.m 2326 2009-09-21 08:37:42Z jrollins $ # # Rewritten in Python by Sudarshan Ghonge <*****@*****.**> # 2015-09-04 # # if specified stop time precedes start time, if (stopTime < startTime): # treat stop time as a duration stopTime = startTime + stopTime # determine site designator from channel name site = channelName[0] #if specified frame cache is invalid if (frameCache == None): if (debugLevel >= 1): # Issue warning print 'Warning: Invalid frame cache' data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] # return empty results # Identifying matching segments from frame cache # find overlap of cache segments with requested data segmentStartTimes = np.maximum(startTime, frameCache.startTimes) segmentStopTimes = np.minimum(stopTime, frameCache.stopTimes) # identify cache segments which overlap requested times segments = np.where(segmentStopTimes > segmentStartTimes)[0] # if no segments overlap with requested times if (len(segments) == 0): if debugLevel >= 1: print 'Warning: No data available for [%d, %d]' % (startTime, stopTime) data = [] time = [] sampleFrequency = 0 # return empty results return [data, sampleFrequency] # otherwise, find overlapping segments else: # identify cache segments with requested site and frame type siteMatches = [] sitesScanned = 0 for iSite in frameCache.sites[segments]: if site in iSite: siteMatches.append(sitesScanned) sitesScanned += 1 siteMatches = np.asarray(siteMatches) frameTypeMatches = [] frameTypesScanned = 0 for iType in frameCache.frameTypes[segments]: if frameType in iType: frameTypeMatches.append(frameTypesScanned) frameTypesScanned += 1 frameTypeMatches = np.asarray(frameTypeMatches) segIdx = np.intersect1d(siteMatches, frameTypeMatches) if (len(segIdx) == 0): segments = [] else: segments = segments[segIdx] # Identify available frame files # initialize list of available frame files frameFilePaths = [] frameFileTypes = [] frameFileStartTimes = [] frameFileStopTimes = [] # lopp over the matching segments for segment in segments: # frame type of the frame files in segment frameFileType = frameCache.frameTypes[segment] firstFrameFileStartTime = frameCache.startTimes[ segment] + frameCache.durations[segment] * np.floor( (segmentStartTimes[segment] - frameCache.startTimes[segment]) / frameCache.durations[segment]) lastFrameFileStartTime = frameCache.startTimes[ segment] + frameCache.durations[segment] * np.ceil( (segmentStopTimes[segment] - frameCache.startTimes[segment]) / frameCache.durations[segment] - 1) for frameFileStartTime in np.arange( firstFrameFileStartTime, lastFrameFileStartTime + frameCache.durations[segment], frameCache.durations[segment]): frameFileStopTime = frameFileStartTime + frameCache.durations[ segment] frameFilePath = frameCache.directories[ segment] + '/' + frameCache.sites[ segment] + '-' + frameCache.frameTypes[ segment] + '-' + '%09d' % ( frameFileStartTime) + '-' + '%d' % ( frameCache.durations[segment]) + '.gwf' # In case the frame file durations start becoming variable. The ls <name> with a '*' in place of the duration field # returns the file name. we then use that file name import os.path if (not os.path.isfile(frameFilePath)): frameFilePath = frameCache.directories[ segment] + '/' + frameCache.sites[ segment] + '-' + frameCache.frameTypes[ segment] + '-' + '%010d' % ( frameFileStartTime) + '-' + '*' + '.gwf' frameFilePath = os.popen( 'ls %s' % (frameFilePath)).readlines()[0].split('\n')[0] if (os.path.isfile(frameFilePath)): frameFilePaths.append(frameFilePath) frameFileTypes.append(frameFileType) frameFileStartTimes.append(frameFileStartTime) frameFileStopTimes.append(frameFileStopTime) frameFilePaths = np.asarray(frameFilePaths) frameFileTypes = np.asarray(frameFileTypes) frameFileStartTimes = np.asarray(frameFileStartTimes) frameFileStopTimes = np.asarray(frameFileStopTimes) numberOfFrameFiles = len(frameFilePaths) keepFrameFileNumbers = [] for frameFileNumber in range(numberOfFrameFiles): keepFrameFileFlag = True for previousFrameFileNumber in range(frameFileNumber): overlapStartTime = np.maximum( frameFileStartTimes[frameFileNumber], frameFileStartTimes[previousFrameFileNumber]) overlapStoptime = np.minimum( frameFileStopTimes[frameFileNumber], frameFileStopTimes[previousFrameFileNumber]) if (overlapStartTime < overlapStoptime): if (allowRedundantFlag): if ((frameFileStartTimes[frameFileNumber] == frameFileStartTimes[previousFrameFileNumber]) & (frameFileStopTimes[frameFileNumber] == frameFileStopTimes[previousFrameFileNumber]) & (frameFileTypes[frameFileNumber] == frameFileTypes[previousFrameFileNumber])): keepFrameFileFlag = False continue else: if (debugLevel >= 1): print 'Warning: Overlapping but dissimilar frame files %s and %s.' % ( frameFilePaths[frameFileNumber], frameFilePaths[previousFrameFileNumber]) data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] else: if (debugLevel >= 1): print 'Warning: Redundant frame files %s and %s.' % ( frameFilePaths[frameFileNumber], frameFilePaths[previousFrameFileNumber]) data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] if (keepFrameFileFlag): keepFrameFileNumbers.append(frameFileNumber) keepFrameFileNumbers = np.asarray(keepFrameFileNumbers) frameFilePaths = frameFilePaths[keepFrameFileNumbers] frameFileTypes = frameFileTypes[keepFrameFileNumbers] frameFileStartTimes = frameFileStartTimes[keepFrameFileNumbers] frameFileStopTimes = frameFileStopTimes[keepFrameFileNumbers] sortedIndices = np.argsort(frameFileStartTimes) frameFilePaths = frameFilePaths[sortedIndices] frameFiletypes = frameFileTypes[sortedIndices] frameFileStartTimes = frameFileStartTimes[sortedIndices] frameFileStopTimes = frameFileStopTimes[sortedIndices] continuityStartTimes = np.append( np.maximum(startTime, frameFileStartTimes), stopTime) continuityStopTimes = np.append(startTime, np.minimum(stopTime, frameFileStopTimes)) discontinuities = np.where(continuityStartTimes != continuityStopTimes)[0] if (len(discontinuities) > 0): if (debugLevel >= 1): print 'Warning: Missing %s ' % (channelName), frameType[ 1:len(frameType) - 1], ' data at ', '%d' % (np.round( continuityStopTimes[discontinuities[0]])), '.' data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] data = np.array([]) time = np.array([]) sampleFrequency = None numberOfFrameFiles = len(frameFilePaths) for frameFileNumber in range(numberOfFrameFiles): frameFilePath = frameFilePaths[frameFileNumber] if (debugLevel >= 2): print 'Reading %s...\n' % (frameFilePath) frameFileStartTime = frameFileStartTimes[frameFileNumber] frameFileStopTime = frameFileStopTimes[frameFileNumber] readData = [] readStartTime = np.maximum(startTime, frameFileStartTime) readDuration = np.minimum(stopTime, frameFileStopTime) - readStartTime realChannelName = channelName try: outputStruct = Fr.frgetvect(frameFilePath, realChannelName, readStartTime, readDuration, False) readData = outputStruct[0] readTime = outputStruct[2] readSampleFrequency = 1.0 / outputStruct[3][0] readTimeStep = outputStruct[3][0] readGPS = outputStruct[1] except Exception as inst: if (debugLevel >= 2): print inst.message if ((len(readData) == 0) | np.any(np.isnan(readData))): if (debugLevel >= 1): print 'Warning: Error reading %s from %s.' % (channelName, frameFilePath) data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] if (sampleFrequency == None): sampleFrequency = readSampleFrequency elif (sampleFrequency != readSampleFrequency): if (debugLevel >= 1): print 'Warning: Inconsistent sample frequency for %s in frameFilePath.' % ( channelname, frameFilePath) data = [] time = [] sampleFrequency = 0 return [data, sampleFrequency] data = np.append(data, readData) return [data, sampleFrequency]