def test_1d_two_channels_roundtrip(self):
""" roundtrip test call with two channels in a frame """
a = Fr.frgetvect1d("./test.dat", "Adc1")
Fr.frputvect('writetest.gwf', [{
'name': 'Adc1',
'data': a[0],
'start': a[1],
'dx': a[3],
'kind': 'ADC',
'x_unit': a[4],
'y_unit': a[5]
}, {
'name': 'reverse',
'data': a[0][::-1],
'start': a[1],
'dx': a[3],
'kind': 'ADC',
'x_unit': a[4],
'y_unit': a[5]
}])
b = Fr.frgetvect1d("writetest.gwf", "Adc1")
self.assert_(numpy.alltrue(a[0] == b[0]))
self.assert_(numpy.alltrue(a[1:] == b[1:]))
c = Fr.frgetvect1d("writetest.gwf", "reverse")
self.assert_(numpy.alltrue(a[0][::-1] == c[0]))
self.assert_(numpy.alltrue(a[1:] == c[1:]))
os.remove("writetest.gwf")
def add_noise_frames_to_signal_frames(noise_frame_files, noise_frame_channel, inj_frame_file, inj_frame_channel, ifo, outfile):
"""
Take noise frame and time-sorted injection frames, and write a single output frame containing both channels between start and stop of noise frame
"""
#Load in injection frame and create value and time arrays
inj_value_array, inj_start_time, __, inj_dt, __, __ = Fr.frgetvect1d(filename=inj_frame_file,channel='%s:%s'%(ifo,inj_frame_channel))
len_inj_array = len(inj_value_array)
inj_time_array = np.arange(inj_start_time, inj_start_time + inj_dt*len_inj_array, inj_dt)
#Initialize noise times to actually add
noise_final_array = np.zeros(len(inj_value_array))
#Load in inj frames and create value and time arrays
for i,noise_frame_file in enumerate(noise_frame_files):
#Load in noise information for this frame
noise_value_array, noise_start_time, __, noise_dt, __, __ = Fr.frgetvect1d(filename=noise_frame_file,channel='%s:%s'%(ifo,noise_frame_channel))
len_noise_array = len(noise_value_array)
noise_time_array = np.arange(noise_start_time, noise_start_time + noise_dt*len_noise_array, noise_dt)
#Keep on injection information contained within the noise frame
tmp_truth_array = (noise_time_array >= inj_time_array[0]) * (noise_time_array <= inj_time_array[-1])
tmp_value_array = noise_value_array[tmp_truth_array]
tmp_time_array = noise_time_array[tmp_truth_array]
noise_final_array[ (inj_time_array >= tmp_time_array[0]) * (inj_time_array <= tmp_time_array[-1]) ] += tmp_value_array
#Save the final noise + inj array
frames_dic = {}
frames_dic['noise'] = dict(name='%s:%s'%(ifo,noise_frame_channel), data=noise_final_array, start=inj_start_time, dx=inj_dt, type=1)
frames_dic['inj'] = dict(name='%s:%s'%(ifo,inj_frame_channel), data=inj_value_array, start=inj_start_time, dx=inj_dt, type=1)
Fr.frputvect(outfile, frames_dic.values())
def test_1d_default_roundtrip(self):
""" roundtrip test call with default values """
a = Fr.frgetvect1d("./test.dat","Adc1")
Fr.frputvect('writetest.gwf', [{'name':'Adc1', 'data':a[0],
'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4],
'y_unit':a[5]}])
b = Fr.frgetvect1d("writetest.gwf", "Adc1")
self.assert_(numpy.alltrue(a[0] == b[0]))
self.assert_(numpy.alltrue(a[1:] == b[1:]))
os.remove("writetest.gwf")
def test_1d_two_channels_roundtrip(self):
""" roundtrip test call with two channels in a frame """
a = Fr.frgetvect1d("./test.dat","Adc1")
Fr.frputvect('writetest.gwf', [{'name':'Adc1', 'data':a[0],
'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4],
'y_unit':a[5]},{'name':'reverse', 'data':a[0][::-1], 'start':a[1],
'dx':a[3], 'kind':'ADC', 'x_unit':a[4], 'y_unit': a[5]}])
b = Fr.frgetvect1d("writetest.gwf", "Adc1")
self.assert_(numpy.alltrue(a[0] == b[0]))
self.assert_(numpy.alltrue(a[1:] == b[1:]))
c = Fr.frgetvect1d("writetest.gwf", "reverse")
self.assert_(numpy.alltrue(a[0][::-1] == c[0]))
self.assert_(numpy.alltrue(a[1:] == c[1:]))
os.remove("writetest.gwf")
def test_1d_keywords_roundtrip(self):
""" roundtrip test call with keyword arguments """
a = Fr.frgetvect1d("./test.dat", "Adc1", span=1)
Fr.frputvect('writetest.gwf', [{
'name': 'Adc1',
'data': a[0],
'start': a[1],
'dx': a[3],
'kind': 'ADC',
'x_unit': a[4],
'y_unit': a[5]
}])
b = Fr.frgetvect1d("writetest.gwf", "Adc1")
self.assert_(numpy.alltrue(a[0] == b[0]))
self.assert_(numpy.alltrue(a[1:] == b[1:]))
os.remove("writetest.gwf")
def frames2vect(frames, channel, start=-np.infty, stop=np.infty):
"""
digs out the correct data from a list of frames
assumes contiguous frames, constant dt, etc.
returns vect, time
"""
frames = [(extract_start_dur(frame), frame) for frame in frames]
frames.sort(key=lambda l: l[0][0])
v = np.array([])
t = np.array([])
for (s, d), frame in frames:
frame_start = max(s, start)
frame_stop = min(s + d, stop)
frame_span = frame_stop - frame_start
vect, s, _, dt, _, _ = Fr.frgetvect1d(frame, channel, start=frame_start, span=frame_span)
N = len(vect)
v = np.concatenate((v, vect))
t = np.concatenate((t, np.arange(s, s + dt * N, dt)))
if not len(v):
raise ValueError("no Data found!")
truth = (start <= t) * (t <= stop)
t = t[truth]
return v[truth], t[0], t[1] - t[0]
def fromframefile(filename, channel, start=None, end=None):
"""
Extract 1D data for given channel from the GWF format frame filename.
Returns (x, data) pair of numpy arrays (x is array of times or frequencies.
Arguments:
filename : string
path to GWF frame file
channel : string
channel name to extract
Keyword arguments:
start : float
GPS start time (s) or minimum frequency (Hz) to return
end : float
GPS end time (s) or maximum frequency (Hz) to return
"""
# try to extract data from frame
y, fstart, offset, dt = Fr.frgetvect1d(filename, str(channel))[:4]
x = fstart+dt*numpy.arange(len(y))+offset
# apply constraint on x-axis
if start or end:
if not start: start=-numpy.infty
if not end: end=numpy.infty
condition = (x>=start) & (x<end)
y = y[condition]
x = x[condition]
return x,y
def extract_scisegs(frames, channel, bitmask, start, stride):
"""
extract scisegs from channel in frames using bitmask
"""
if not frames: ### empty list, so no segments
return []
### extract vectors and build segments
segset = []
for frame in frames:
### extract the vector from the frame
vect, s, ds, dt, xunit, yunit = Fr.frgetvect1d(frame, channel)
n = len(vect)
### build time vector add starting time
t = np.arange(0, dt*n, dt) + s+ds
### determine whether state acceptable
### add "False" buffers to set up the computation of start and end time
# state = np.concatenate( ([False], vect == bitmask, [False])) ### exact integer match
state = np.concatenate( ([False], (vect >> bitmask) & 1, [False])) ### bitwise operation
### determine beginning of segments
### i=False i+1 = True strip the trailing buffer
b = ( (1-state[:-1])*(state[1:]) )[:-1].astype(bool)
b = t[b] ### select out times
### determine end of segments
### i=True i+1=False strip the leading buffer
e = ( (state[:-1])*(1-state[1:]) )[1:].astype(bool)
e = t[e] + dt ### select out times
### extra dt moves these markers to the end of segments
### stitch together start and end times, append to global list
segset += list( np.transpose( np.array( [b, e] ) ) )
if not segset: ### empty list
return []
### clean up segs!
segs = []
seg1 = segset[0]
for seg2 in segset[1:]:
if seg1[1] == seg2[0]:
seg1[1] = seg2[1] ### join the segments
else:
### check segment for sanity
append_seg = check_seg( seg1, (start, start+stride) )
if append_seg:
segs.append( append_seg )
seg1 = seg2
append_seg = check_seg( seg1, (start, start+stride) )
if append_seg:
segs.append( append_seg )
### return final list of lists!
return segs
def read_frame(filename, channels, start=None, end=None):
if start is not None and end is not None:
if type(start) is _lal.LIGOTimeGPS:
if start.gpsNanoSeconds != 0:
raise ValueError('start and end times must be integer valued')
else:
start = start.gpsSeconds
else:
if int(start) != start:
raise ValueError('start and end times must be integer valued')
else:
start = int(start)
if type(end) is _lal.LIGOTimeGPS:
if end.gpsNanoSeconds != 0:
raise ValueError('start and end times must be integer valued')
else:
end = end.gpsSeconds
else:
if int(end) != end:
raise ValueError('start and end times must be integer valued')
else:
end = int(end)
span = end - start
if span <= 0:
raise ValueError('beginning must be before end')
else:
start = -1
span = -1
if type(channels) is list:
ts = []
for channel in channels:
frdata = Fr.frgetvect1d(filename, channel, start, span)
ts.append(pycbc.types.TimeSeries(initial_array=frdata[0],
delta_t=frdata[3],
epoch=_lal.LIGOTimeGPS(frdata[1]),
copy=False))
else:
frdata = Fr.frgetvect1d(filename, channels, start, span)
ts = pycbc.types.TimeSeries(initial_array=frdata[0],
delta_t=frdata[3],
epoch=_lal.LIGOTimeGPS(frdata[1]),
copy=False)
return ts
def vec_from_frames(frames, start, stop, verbose=False): """ returns a numpy array of the data inculded in frames between start and stop CURRENTLY ASSUME CONTIGUOUS DATA, but we should check this meant to be used with files_from_cache """ vecs = [] dt = 0 for frame, strt, dur in frames: if verbose: print frame s = max(strt, start) d = min(start+dur,stop) - s vec, gpstart, offset, dt, _, _ = Fr.frgetvect1d(frame, ifo_chan, start=s, span=d) vecs.append( vec ) vec = np.concatenate(vecs) return vec, dt
def checkDQ(framecache_file, chname, abs_start, abs_stop, ifo, bitmask):
"""
Takes a file containing the data quality state vetor and converts it to segments of a desired bitmask, returning injection status
"""
#Initialize data-quality veto status as False (will change if 'No veto' bits are off)
DQV_flag = False
#Open framecache file
cache = open(framecache_file, 'rt')
#Loop over frames in the cache
for line in cache:
#Get frame_start, frame_stride, and frame_file
words = line.split()
frame_start = int(words[2])
frame_stride = int(words[3])
frame_file = words[4].split('file://localhost')[1]
#Open state vector as array
state_array = Fr.frgetvect1d(filename=frame_file,channel='%s:%s'%(ifo,chname))[0]
#Calculate sample rate
samp_rate = len(state_array)/float(frame_stride)
#Loop over state vetor
for i, value in enumerate(state_array):
#Check to make sure we've passed the absolute start
if (frame_start + i/float(samp_rate)) < abs_start:
continue
#Check to make sure we haven't passed the absolute stop
elif (frame_start + i/float(samp_rate)) > abs_stop:
break
#Check if DQ vector denotes that a veto is present
if ((int(value) & bitmask) != bitmask) or (int(value)<0): #(e.g., we might want bits ??, ??, or ?? to be on if there are no data-quality vetoes)
DQV_flag = True
cache.close()
#Return injection flag
return DQV_flag
# # Are we looking outside the range rather than inside? # if min_threshold is not None and max_threshold is not None: invert = min_threshold >= max_threshold else: invert = False if opts.verbose: print "Inverted? %s"% str(invert) seglist = segmentlist([]) for path in cache.pfnlist(): # # Read data # data, start, _, dt, _, _ = Fr.frgetvect1d(path, channel) # # Apply conditions and transform samples to segments # if equals is not None: seglist.extend(dqsegs.equality_data_to_seglist(data, start, dt, equality=equals)) if bit_mask is not None: seglist.extend(dqsegs.mask_data_to_seglist(data, start, dt, mask_on=bit_mask)) else: seglist.extend(dqsegs.threshold_data_to_seglist(data, start, dt, min_threshold=min_threshold, max_threshold=max_threshold, invert=invert)) seglist.coalesce() all_segs["%s: %s" % (channel, str(opthresholds))] = seglist if opts.verbose:
def framecache2segs(framecache_file, chname, abs_start, abs_stop, outdir, ifo, run_bitmask, inj_bitmask):
"""
Takes a file containing the data quality state vetor and converts it to segments of a desired bitmask, returning injection status
"""
#Initialize injection status as False (will change if 'No injection' bits are off)
inj_flag = False
#Open framecache file and segment file to write to
cache = open(framecache_file, 'rt')
segfile = open(outdir+'/%s_%s_%s.seg'%(ifo,abs_start,abs_stop),'wt')
#Define start and stop of current segment
current_start = None
current_stop = None
#Loop over frames in the cache
for line in cache:
#Get frame_start, frame_stride, and frame_file
words = line.split()
frame_start = int(words[2])
frame_stride = int(words[3])
frame_file = words[4].split('file://localhost')[1]
#Open state vector as array
state_array = Fr.frgetvect1d(filename=frame_file,channel='%s:%s'%(ifo,chname))[0]
#Calculate sample rate
samp_rate = len(state_array)/float(frame_stride)
#Loop over state vetor
for i, value in enumerate(state_array):
#Check to make sure we've passed the absolute start
if (frame_start + i/float(samp_rate)) < abs_start:
continue
#Check to make sure we haven't passed the absolute stop
elif (frame_start + i/float(samp_rate)) > abs_stop:
break
#Check if state vector corresponds to desired bitmask
elif ((int(value) & run_bitmask) == run_bitmask) and (int(value)>=0): #(e.g., 0b00011 = 3 and we want bits 0 and 1 to be on, so we do & with 3)
#Data is good, start new seg if needed
if not current_start:
current_start = int(np.ceil(frame_start + i/float(samp_rate) )) #data good starting at ith sample, use ceiling so don't underestimate start
else:
#Data not good, end current seg if needed
if current_start:
current_stop = int(np.floor(frame_start + i/float(samp_rate) )) #data goes bad at ith sample but good until then, use floor so don't overestimate stop
if current_start < current_stop:
segfile.write('%s %s\n'%(current_start, current_stop))
#Wait to start next segment until find good data
current_start = None
current_stop = None
#Check if state vector denotes that an injection is present
if ((int(value) & inj_bitmask) != inj_bitmask): #(e.g., we might want bits 5, 6, 7, or 8 to be on if there are no HW injections)
inj_flag = True
#Write final segment for this frame if needed
if current_start:
if (current_start < int(frame_start+frame_stride)) and (int(frame_start+frame_stride) < abs_stop):
segfile.write('%s %s\n'%(current_start, int(frame_start+frame_stride)))
elif (current_start < int(frame_start+frame_stride)) and (current_start < abs_stop):
segfile.write('%s %s\n'%(current_start, int(abs_stop)))
#Wait to start next segment until find good data
current_start = None
current_stop = None
cache.close()
segfile.close()
#Return injection flag
return inj_flag
