def test_1d_two_channels_roundtrip(self): """ roundtrip test call with two channels in a frame """ a = Fr.frgetvect1d("./test.dat", "Adc1") Fr.frputvect('writetest.gwf', [{ 'name': 'Adc1', 'data': a[0], 'start': a[1], 'dx': a[3], 'kind': 'ADC', 'x_unit': a[4], 'y_unit': a[5] }, { 'name': 'reverse', 'data': a[0][::-1], 'start': a[1], 'dx': a[3], 'kind': 'ADC', 'x_unit': a[4], 'y_unit': a[5] }]) b = Fr.frgetvect1d("writetest.gwf", "Adc1") self.assert_(numpy.alltrue(a[0] == b[0])) self.assert_(numpy.alltrue(a[1:] == b[1:])) c = Fr.frgetvect1d("writetest.gwf", "reverse") self.assert_(numpy.alltrue(a[0][::-1] == c[0])) self.assert_(numpy.alltrue(a[1:] == c[1:])) os.remove("writetest.gwf")
def add_noise_frames_to_signal_frames(noise_frame_files, noise_frame_channel, inj_frame_file, inj_frame_channel, ifo, outfile): """ Take noise frame and time-sorted injection frames, and write a single output frame containing both channels between start and stop of noise frame """ #Load in injection frame and create value and time arrays inj_value_array, inj_start_time, __, inj_dt, __, __ = Fr.frgetvect1d(filename=inj_frame_file,channel='%s:%s'%(ifo,inj_frame_channel)) len_inj_array = len(inj_value_array) inj_time_array = np.arange(inj_start_time, inj_start_time + inj_dt*len_inj_array, inj_dt) #Initialize noise times to actually add noise_final_array = np.zeros(len(inj_value_array)) #Load in inj frames and create value and time arrays for i,noise_frame_file in enumerate(noise_frame_files): #Load in noise information for this frame noise_value_array, noise_start_time, __, noise_dt, __, __ = Fr.frgetvect1d(filename=noise_frame_file,channel='%s:%s'%(ifo,noise_frame_channel)) len_noise_array = len(noise_value_array) noise_time_array = np.arange(noise_start_time, noise_start_time + noise_dt*len_noise_array, noise_dt) #Keep on injection information contained within the noise frame tmp_truth_array = (noise_time_array >= inj_time_array[0]) * (noise_time_array <= inj_time_array[-1]) tmp_value_array = noise_value_array[tmp_truth_array] tmp_time_array = noise_time_array[tmp_truth_array] noise_final_array[ (inj_time_array >= tmp_time_array[0]) * (inj_time_array <= tmp_time_array[-1]) ] += tmp_value_array #Save the final noise + inj array frames_dic = {} frames_dic['noise'] = dict(name='%s:%s'%(ifo,noise_frame_channel), data=noise_final_array, start=inj_start_time, dx=inj_dt, type=1) frames_dic['inj'] = dict(name='%s:%s'%(ifo,inj_frame_channel), data=inj_value_array, start=inj_start_time, dx=inj_dt, type=1) Fr.frputvect(outfile, frames_dic.values())
def test_1d_default_roundtrip(self): """ roundtrip test call with default values """ a = Fr.frgetvect1d("./test.dat","Adc1") Fr.frputvect('writetest.gwf', [{'name':'Adc1', 'data':a[0], 'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4], 'y_unit':a[5]}]) b = Fr.frgetvect1d("writetest.gwf", "Adc1") self.assert_(numpy.alltrue(a[0] == b[0])) self.assert_(numpy.alltrue(a[1:] == b[1:])) os.remove("writetest.gwf")
def test_1d_two_channels_roundtrip(self): """ roundtrip test call with two channels in a frame """ a = Fr.frgetvect1d("./test.dat","Adc1") Fr.frputvect('writetest.gwf', [{'name':'Adc1', 'data':a[0], 'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4], 'y_unit':a[5]},{'name':'reverse', 'data':a[0][::-1], 'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4], 'y_unit': a[5]}]) b = Fr.frgetvect1d("writetest.gwf", "Adc1") self.assert_(numpy.alltrue(a[0] == b[0])) self.assert_(numpy.alltrue(a[1:] == b[1:])) c = Fr.frgetvect1d("writetest.gwf", "reverse") self.assert_(numpy.alltrue(a[0][::-1] == c[0])) self.assert_(numpy.alltrue(a[1:] == c[1:])) os.remove("writetest.gwf")
def test_1d_keywords_roundtrip(self): """ roundtrip test call with keyword arguments """ a = Fr.frgetvect1d("./test.dat", "Adc1", span=1) Fr.frputvect('writetest.gwf', [{ 'name': 'Adc1', 'data': a[0], 'start': a[1], 'dx': a[3], 'kind': 'ADC', 'x_unit': a[4], 'y_unit': a[5] }]) b = Fr.frgetvect1d("writetest.gwf", "Adc1") self.assert_(numpy.alltrue(a[0] == b[0])) self.assert_(numpy.alltrue(a[1:] == b[1:])) os.remove("writetest.gwf")
def frames2vect(frames, channel, start=-np.infty, stop=np.infty): """ digs out the correct data from a list of frames assumes contiguous frames, constant dt, etc. returns vect, time """ frames = [(extract_start_dur(frame), frame) for frame in frames] frames.sort(key=lambda l: l[0][0]) v = np.array([]) t = np.array([]) for (s, d), frame in frames: frame_start = max(s, start) frame_stop = min(s + d, stop) frame_span = frame_stop - frame_start vect, s, _, dt, _, _ = Fr.frgetvect1d(frame, channel, start=frame_start, span=frame_span) N = len(vect) v = np.concatenate((v, vect)) t = np.concatenate((t, np.arange(s, s + dt * N, dt))) if not len(v): raise ValueError("no Data found!") truth = (start <= t) * (t <= stop) t = t[truth] return v[truth], t[0], t[1] - t[0]
def fromframefile(filename, channel, start=None, end=None): """ Extract 1D data for given channel from the GWF format frame filename. Returns (x, data) pair of numpy arrays (x is array of times or frequencies. Arguments: filename : string path to GWF frame file channel : string channel name to extract Keyword arguments: start : float GPS start time (s) or minimum frequency (Hz) to return end : float GPS end time (s) or maximum frequency (Hz) to return """ # try to extract data from frame y, fstart, offset, dt = Fr.frgetvect1d(filename, str(channel))[:4] x = fstart+dt*numpy.arange(len(y))+offset # apply constraint on x-axis if start or end: if not start: start=-numpy.infty if not end: end=numpy.infty condition = (x>=start) & (x<end) y = y[condition] x = x[condition] return x,y
def extract_scisegs(frames, channel, bitmask, start, stride): """ extract scisegs from channel in frames using bitmask """ if not frames: ### empty list, so no segments return [] ### extract vectors and build segments segset = [] for frame in frames: ### extract the vector from the frame vect, s, ds, dt, xunit, yunit = Fr.frgetvect1d(frame, channel) n = len(vect) ### build time vector add starting time t = np.arange(0, dt*n, dt) + s+ds ### determine whether state acceptable ### add "False" buffers to set up the computation of start and end time # state = np.concatenate( ([False], vect == bitmask, [False])) ### exact integer match state = np.concatenate( ([False], (vect >> bitmask) & 1, [False])) ### bitwise operation ### determine beginning of segments ### i=False i+1 = True strip the trailing buffer b = ( (1-state[:-1])*(state[1:]) )[:-1].astype(bool) b = t[b] ### select out times ### determine end of segments ### i=True i+1=False strip the leading buffer e = ( (state[:-1])*(1-state[1:]) )[1:].astype(bool) e = t[e] + dt ### select out times ### extra dt moves these markers to the end of segments ### stitch together start and end times, append to global list segset += list( np.transpose( np.array( [b, e] ) ) ) if not segset: ### empty list return [] ### clean up segs! segs = [] seg1 = segset[0] for seg2 in segset[1:]: if seg1[1] == seg2[0]: seg1[1] = seg2[1] ### join the segments else: ### check segment for sanity append_seg = check_seg( seg1, (start, start+stride) ) if append_seg: segs.append( append_seg ) seg1 = seg2 append_seg = check_seg( seg1, (start, start+stride) ) if append_seg: segs.append( append_seg ) ### return final list of lists! return segs
def read_frame(filename, channels, start=None, end=None): if start is not None and end is not None: if type(start) is _lal.LIGOTimeGPS: if start.gpsNanoSeconds != 0: raise ValueError('start and end times must be integer valued') else: start = start.gpsSeconds else: if int(start) != start: raise ValueError('start and end times must be integer valued') else: start = int(start) if type(end) is _lal.LIGOTimeGPS: if end.gpsNanoSeconds != 0: raise ValueError('start and end times must be integer valued') else: end = end.gpsSeconds else: if int(end) != end: raise ValueError('start and end times must be integer valued') else: end = int(end) span = end - start if span <= 0: raise ValueError('beginning must be before end') else: start = -1 span = -1 if type(channels) is list: ts = [] for channel in channels: frdata = Fr.frgetvect1d(filename, channel, start, span) ts.append(pycbc.types.TimeSeries(initial_array=frdata[0], delta_t=frdata[3], epoch=_lal.LIGOTimeGPS(frdata[1]), copy=False)) else: frdata = Fr.frgetvect1d(filename, channels, start, span) ts = pycbc.types.TimeSeries(initial_array=frdata[0], delta_t=frdata[3], epoch=_lal.LIGOTimeGPS(frdata[1]), copy=False) return ts
def vec_from_frames(frames, start, stop, verbose=False): """ returns a numpy array of the data inculded in frames between start and stop CURRENTLY ASSUME CONTIGUOUS DATA, but we should check this meant to be used with files_from_cache """ vecs = [] dt = 0 for frame, strt, dur in frames: if verbose: print frame s = max(strt, start) d = min(start+dur,stop) - s vec, gpstart, offset, dt, _, _ = Fr.frgetvect1d(frame, ifo_chan, start=s, span=d) vecs.append( vec ) vec = np.concatenate(vecs) return vec, dt
def checkDQ(framecache_file, chname, abs_start, abs_stop, ifo, bitmask): """ Takes a file containing the data quality state vetor and converts it to segments of a desired bitmask, returning injection status """ #Initialize data-quality veto status as False (will change if 'No veto' bits are off) DQV_flag = False #Open framecache file cache = open(framecache_file, 'rt') #Loop over frames in the cache for line in cache: #Get frame_start, frame_stride, and frame_file words = line.split() frame_start = int(words[2]) frame_stride = int(words[3]) frame_file = words[4].split('file://localhost')[1] #Open state vector as array state_array = Fr.frgetvect1d(filename=frame_file,channel='%s:%s'%(ifo,chname))[0] #Calculate sample rate samp_rate = len(state_array)/float(frame_stride) #Loop over state vetor for i, value in enumerate(state_array): #Check to make sure we've passed the absolute start if (frame_start + i/float(samp_rate)) < abs_start: continue #Check to make sure we haven't passed the absolute stop elif (frame_start + i/float(samp_rate)) > abs_stop: break #Check if DQ vector denotes that a veto is present if ((int(value) & bitmask) != bitmask) or (int(value)<0): #(e.g., we might want bits ??, ??, or ?? to be on if there are no data-quality vetoes) DQV_flag = True cache.close() #Return injection flag return DQV_flag
# # Are we looking outside the range rather than inside? # if min_threshold is not None and max_threshold is not None: invert = min_threshold >= max_threshold else: invert = False if opts.verbose: print "Inverted? %s"% str(invert) seglist = segmentlist([]) for path in cache.pfnlist(): # # Read data # data, start, _, dt, _, _ = Fr.frgetvect1d(path, channel) # # Apply conditions and transform samples to segments # if equals is not None: seglist.extend(dqsegs.equality_data_to_seglist(data, start, dt, equality=equals)) if bit_mask is not None: seglist.extend(dqsegs.mask_data_to_seglist(data, start, dt, mask_on=bit_mask)) else: seglist.extend(dqsegs.threshold_data_to_seglist(data, start, dt, min_threshold=min_threshold, max_threshold=max_threshold, invert=invert)) seglist.coalesce() all_segs["%s: %s" % (channel, str(opthresholds))] = seglist if opts.verbose:
def framecache2segs(framecache_file, chname, abs_start, abs_stop, outdir, ifo, run_bitmask, inj_bitmask): """ Takes a file containing the data quality state vetor and converts it to segments of a desired bitmask, returning injection status """ #Initialize injection status as False (will change if 'No injection' bits are off) inj_flag = False #Open framecache file and segment file to write to cache = open(framecache_file, 'rt') segfile = open(outdir+'/%s_%s_%s.seg'%(ifo,abs_start,abs_stop),'wt') #Define start and stop of current segment current_start = None current_stop = None #Loop over frames in the cache for line in cache: #Get frame_start, frame_stride, and frame_file words = line.split() frame_start = int(words[2]) frame_stride = int(words[3]) frame_file = words[4].split('file://localhost')[1] #Open state vector as array state_array = Fr.frgetvect1d(filename=frame_file,channel='%s:%s'%(ifo,chname))[0] #Calculate sample rate samp_rate = len(state_array)/float(frame_stride) #Loop over state vetor for i, value in enumerate(state_array): #Check to make sure we've passed the absolute start if (frame_start + i/float(samp_rate)) < abs_start: continue #Check to make sure we haven't passed the absolute stop elif (frame_start + i/float(samp_rate)) > abs_stop: break #Check if state vector corresponds to desired bitmask elif ((int(value) & run_bitmask) == run_bitmask) and (int(value)>=0): #(e.g., 0b00011 = 3 and we want bits 0 and 1 to be on, so we do & with 3) #Data is good, start new seg if needed if not current_start: current_start = int(np.ceil(frame_start + i/float(samp_rate) )) #data good starting at ith sample, use ceiling so don't underestimate start else: #Data not good, end current seg if needed if current_start: current_stop = int(np.floor(frame_start + i/float(samp_rate) )) #data goes bad at ith sample but good until then, use floor so don't overestimate stop if current_start < current_stop: segfile.write('%s %s\n'%(current_start, current_stop)) #Wait to start next segment until find good data current_start = None current_stop = None #Check if state vector denotes that an injection is present if ((int(value) & inj_bitmask) != inj_bitmask): #(e.g., we might want bits 5, 6, 7, or 8 to be on if there are no HW injections) inj_flag = True #Write final segment for this frame if needed if current_start: if (current_start < int(frame_start+frame_stride)) and (int(frame_start+frame_stride) < abs_stop): segfile.write('%s %s\n'%(current_start, int(frame_start+frame_stride))) elif (current_start < int(frame_start+frame_stride)) and (current_start < abs_stop): segfile.write('%s %s\n'%(current_start, int(abs_stop))) #Wait to start next segment until find good data current_start = None current_stop = None cache.close() segfile.close() #Return injection flag return inj_flag