def test_1d_two_channels_roundtrip(self):
""" roundtrip test call with two channels in a frame """
a = Fr.frgetvect1d("./test.dat", "Adc1")
Fr.frputvect('writetest.gwf', [{
'name': 'Adc1',
'data': a[0],
'start': a[1],
'dx': a[3],
'kind': 'ADC',
'x_unit': a[4],
'y_unit': a[5]
}, {
'name': 'reverse',
'data': a[0][::-1],
'start': a[1],
'dx': a[3],
'kind': 'ADC',
'x_unit': a[4],
'y_unit': a[5]
}])
b = Fr.frgetvect1d("writetest.gwf", "Adc1")
self.assert_(numpy.alltrue(a[0] == b[0]))
self.assert_(numpy.alltrue(a[1:] == b[1:]))
c = Fr.frgetvect1d("writetest.gwf", "reverse")
self.assert_(numpy.alltrue(a[0][::-1] == c[0]))
self.assert_(numpy.alltrue(a[1:] == c[1:]))
os.remove("writetest.gwf")
def toframefile(filename, channel, data, start, dx, **frargs):
"""
Write numpy array data to GWF frame file using the given arguments.
Arguments:
filename : string
name of file to write
channel : string
name of channel to write
data : numpy.array
array of data to write
start : float
GPS start time (s) or minimum frequency (Hz)
dx : float
GPS time step (s) or frequency step (Hz)
Unnamed arguments are held in frargs. For usage, see documentation for
pylal.Fr.frputvect.
"""
datadict = frargs
datadict['name'] = channel
datadict['data'] = data
datadict['start'] = start
datadict['dx'] = dx
Fr.frputvect(filename, [datadict], verbose=False)
def add_noise_frames_to_signal_frames(noise_frame_files, noise_frame_channel, inj_frame_file, inj_frame_channel, ifo, outfile):
"""
Take noise frame and time-sorted injection frames, and write a single output frame containing both channels between start and stop of noise frame
"""
#Load in injection frame and create value and time arrays
inj_value_array, inj_start_time, __, inj_dt, __, __ = Fr.frgetvect1d(filename=inj_frame_file,channel='%s:%s'%(ifo,inj_frame_channel))
len_inj_array = len(inj_value_array)
inj_time_array = np.arange(inj_start_time, inj_start_time + inj_dt*len_inj_array, inj_dt)
#Initialize noise times to actually add
noise_final_array = np.zeros(len(inj_value_array))
#Load in inj frames and create value and time arrays
for i,noise_frame_file in enumerate(noise_frame_files):
#Load in noise information for this frame
noise_value_array, noise_start_time, __, noise_dt, __, __ = Fr.frgetvect1d(filename=noise_frame_file,channel='%s:%s'%(ifo,noise_frame_channel))
len_noise_array = len(noise_value_array)
noise_time_array = np.arange(noise_start_time, noise_start_time + noise_dt*len_noise_array, noise_dt)
#Keep on injection information contained within the noise frame
tmp_truth_array = (noise_time_array >= inj_time_array[0]) * (noise_time_array <= inj_time_array[-1])
tmp_value_array = noise_value_array[tmp_truth_array]
tmp_time_array = noise_time_array[tmp_truth_array]
noise_final_array[ (inj_time_array >= tmp_time_array[0]) * (inj_time_array <= tmp_time_array[-1]) ] += tmp_value_array
#Save the final noise + inj array
frames_dic = {}
frames_dic['noise'] = dict(name='%s:%s'%(ifo,noise_frame_channel), data=noise_final_array, start=inj_start_time, dx=inj_dt, type=1)
frames_dic['inj'] = dict(name='%s:%s'%(ifo,inj_frame_channel), data=inj_value_array, start=inj_start_time, dx=inj_dt, type=1)
Fr.frputvect(outfile, frames_dic.values())
def test_1d_default_roundtrip(self):
""" roundtrip test call with default values """
a = Fr.frgetvect1d("./test.dat","Adc1")
Fr.frputvect('writetest.gwf', [{'name':'Adc1', 'data':a[0],
'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4],
'y_unit':a[5]}])
b = Fr.frgetvect1d("writetest.gwf", "Adc1")
self.assert_(numpy.alltrue(a[0] == b[0]))
self.assert_(numpy.alltrue(a[1:] == b[1:]))
os.remove("writetest.gwf")
def test_1d_two_channels_roundtrip(self):
""" roundtrip test call with two channels in a frame """
a = Fr.frgetvect1d("./test.dat","Adc1")
Fr.frputvect('writetest.gwf', [{'name':'Adc1', 'data':a[0],
'start':a[1], 'dx':a[3], 'kind':'ADC', 'x_unit':a[4],
'y_unit':a[5]},{'name':'reverse', 'data':a[0][::-1], 'start':a[1],
'dx':a[3], 'kind':'ADC', 'x_unit':a[4], 'y_unit': a[5]}])
b = Fr.frgetvect1d("writetest.gwf", "Adc1")
self.assert_(numpy.alltrue(a[0] == b[0]))
self.assert_(numpy.alltrue(a[1:] == b[1:]))
c = Fr.frgetvect1d("writetest.gwf", "reverse")
self.assert_(numpy.alltrue(a[0][::-1] == c[0]))
self.assert_(numpy.alltrue(a[1:] == c[1:]))
os.remove("writetest.gwf")
def test_1d_keywords_roundtrip(self):
""" roundtrip test call with keyword arguments """
a = Fr.frgetvect1d("./test.dat", "Adc1", span=1)
Fr.frputvect('writetest.gwf', [{
'name': 'Adc1',
'data': a[0],
'start': a[1],
'dx': a[3],
'kind': 'ADC',
'x_unit': a[4],
'y_unit': a[5]
}])
b = Fr.frgetvect1d("writetest.gwf", "Adc1")
self.assert_(numpy.alltrue(a[0] == b[0]))
self.assert_(numpy.alltrue(a[1:] == b[1:]))
os.remove("writetest.gwf")
def getRawData(channel, gps, dt):
"""Read data from RAW file:
ch = channel name
gps = gps time
dt = duration
"""
global files
# get the list of frame files
obs = channel[0:2]
if len(files) == 0:
files = find_data_path(obs, gps, gps + dt)
# read data from all files
data = array([])
for f in files:
# get the frame file start GPS and duration from the name
gps0 = int(f.split('-')[-2])
gps1 = gps0 + int(f.split('-')[-1].split('.')[-2])
# find the right time segment to load from this file
gps0 = max(gps0, gps)
gps1 = min(gps1, gps + dt)
# read data and append
x = Fr.frgetvect(f, channel, gps0, gps1 - gps0)
data = concatenate([data, x[0]])
return data, int(1 / x[3][0])
def fromframefile(filename, channel, start=None, end=None):
"""
Extract 1D data for given channel from the GWF format frame filename.
Returns (x, data) pair of numpy arrays (x is array of times or frequencies.
Arguments:
filename : string
path to GWF frame file
channel : string
channel name to extract
Keyword arguments:
start : float
GPS start time (s) or minimum frequency (Hz) to return
end : float
GPS end time (s) or maximum frequency (Hz) to return
"""
# try to extract data from frame
y, fstart, offset, dt = Fr.frgetvect1d(filename, str(channel))[:4]
x = fstart+dt*numpy.arange(len(y))+offset
# apply constraint on x-axis
if start or end:
if not start: start=-numpy.infty
if not end: end=numpy.infty
condition = (x>=start) & (x<end)
y = y[condition]
x = x[condition]
return x,y
def frames2vect(frames, channel, start=-np.infty, stop=np.infty):
"""
digs out the correct data from a list of frames
assumes contiguous frames, constant dt, etc.
returns vect, time
"""
frames = [(extract_start_dur(frame), frame) for frame in frames]
frames.sort(key=lambda l: l[0][0])
v = np.array([])
t = np.array([])
for (s, d), frame in frames:
frame_start = max(s, start)
frame_stop = min(s + d, stop)
frame_span = frame_stop - frame_start
vect, s, _, dt, _, _ = Fr.frgetvect1d(frame, channel, start=frame_start, span=frame_span)
N = len(vect)
v = np.concatenate((v, vect))
t = np.concatenate((t, np.arange(s, s + dt * N, dt)))
if not len(v):
raise ValueError("no Data found!")
truth = (start <= t) * (t <= stop)
t = t[truth]
return v[truth], t[0], t[1] - t[0]
def write_frame(TimeSeries, ifo, usertag, outdir):
"""
Write a frame
"""
# Construct name
site=ifo.strip('1')
frame_name = '{site}-{ifo}_{usertag}-{epoch}-{datalen}.gwf'.format(
site=site, ifo=ifo, usertag=usertag,
epoch=str(int(TimeSeries.epoch)),
datalen=str(int(TimeSeries.data.length * TimeSeries.deltaT)))
channel_list = [
{'name':'%s:STRAIN'%ifo,
'data':np.array(TimeSeries.data.data),
'start':TimeSeries.epoch,
'dx':TimeSeries.deltaT,
'kind':'SIM'},
]
print 'writing frame %s...'%frame_name
frame_out_path = '%s/%s'%(os.path.abspath(outdir), frame_name)
Fr.frputvect(frame_out_path, channel_list)
#
# Generate a cache file
#
# setup url
path, filename = os.path.split(frame_out_path.strip())
url = "file://localhost%s" % os.path.abspath(os.path.join(path, filename))
# create cache entry
c=gluelal.CacheEntry.from_T050017(url)
# write to file
cache_file = frame_out_path.replace('gwf','lcf')
f=open(cache_file,'w')
f.writelines('%s\n'%str(c))
f.close()
return frame_out_path,cache_file
def extract_scisegs(frames, channel, bitmask, start, stride):
"""
extract scisegs from channel in frames using bitmask
"""
if not frames: ### empty list, so no segments
return []
### extract vectors and build segments
segset = []
for frame in frames:
### extract the vector from the frame
vect, s, ds, dt, xunit, yunit = Fr.frgetvect1d(frame, channel)
n = len(vect)
### build time vector add starting time
t = np.arange(0, dt*n, dt) + s+ds
### determine whether state acceptable
### add "False" buffers to set up the computation of start and end time
# state = np.concatenate( ([False], vect == bitmask, [False])) ### exact integer match
state = np.concatenate( ([False], (vect >> bitmask) & 1, [False])) ### bitwise operation
### determine beginning of segments
### i=False i+1 = True strip the trailing buffer
b = ( (1-state[:-1])*(state[1:]) )[:-1].astype(bool)
b = t[b] ### select out times
### determine end of segments
### i=True i+1=False strip the leading buffer
e = ( (state[:-1])*(1-state[1:]) )[1:].astype(bool)
e = t[e] + dt ### select out times
### extra dt moves these markers to the end of segments
### stitch together start and end times, append to global list
segset += list( np.transpose( np.array( [b, e] ) ) )
if not segset: ### empty list
return []
### clean up segs!
segs = []
seg1 = segset[0]
for seg2 in segset[1:]:
if seg1[1] == seg2[0]:
seg1[1] = seg2[1] ### join the segments
else:
### check segment for sanity
append_seg = check_seg( seg1, (start, start+stride) )
if append_seg:
segs.append( append_seg )
seg1 = seg2
append_seg = check_seg( seg1, (start, start+stride) )
if append_seg:
segs.append( append_seg )
### return final list of lists!
return segs
def read_frame(filename, channels, start=None, end=None):
if start is not None and end is not None:
if type(start) is _lal.LIGOTimeGPS:
if start.gpsNanoSeconds != 0:
raise ValueError('start and end times must be integer valued')
else:
start = start.gpsSeconds
else:
if int(start) != start:
raise ValueError('start and end times must be integer valued')
else:
start = int(start)
if type(end) is _lal.LIGOTimeGPS:
if end.gpsNanoSeconds != 0:
raise ValueError('start and end times must be integer valued')
else:
end = end.gpsSeconds
else:
if int(end) != end:
raise ValueError('start and end times must be integer valued')
else:
end = int(end)
span = end - start
if span <= 0:
raise ValueError('beginning must be before end')
else:
start = -1
span = -1
if type(channels) is list:
ts = []
for channel in channels:
frdata = Fr.frgetvect1d(filename, channel, start, span)
ts.append(pycbc.types.TimeSeries(initial_array=frdata[0],
delta_t=frdata[3],
epoch=_lal.LIGOTimeGPS(frdata[1]),
copy=False))
else:
frdata = Fr.frgetvect1d(filename, channels, start, span)
ts = pycbc.types.TimeSeries(initial_array=frdata[0],
delta_t=frdata[3],
epoch=_lal.LIGOTimeGPS(frdata[1]),
copy=False)
return ts
def vec_from_frames(frames, start, stop, verbose=False): """ returns a numpy array of the data inculded in frames between start and stop CURRENTLY ASSUME CONTIGUOUS DATA, but we should check this meant to be used with files_from_cache """ vecs = [] dt = 0 for frame, strt, dur in frames: if verbose: print frame s = max(strt, start) d = min(start+dur,stop) - s vec, gpstart, offset, dt, _, _ = Fr.frgetvect1d(frame, ifo_chan, start=s, span=d) vecs.append( vec ) vec = np.concatenate(vecs) return vec, dt
def checkDQ(framecache_file, chname, abs_start, abs_stop, ifo, bitmask):
"""
Takes a file containing the data quality state vetor and converts it to segments of a desired bitmask, returning injection status
"""
#Initialize data-quality veto status as False (will change if 'No veto' bits are off)
DQV_flag = False
#Open framecache file
cache = open(framecache_file, 'rt')
#Loop over frames in the cache
for line in cache:
#Get frame_start, frame_stride, and frame_file
words = line.split()
frame_start = int(words[2])
frame_stride = int(words[3])
frame_file = words[4].split('file://localhost')[1]
#Open state vector as array
state_array = Fr.frgetvect1d(filename=frame_file,channel='%s:%s'%(ifo,chname))[0]
#Calculate sample rate
samp_rate = len(state_array)/float(frame_stride)
#Loop over state vetor
for i, value in enumerate(state_array):
#Check to make sure we've passed the absolute start
if (frame_start + i/float(samp_rate)) < abs_start:
continue
#Check to make sure we haven't passed the absolute stop
elif (frame_start + i/float(samp_rate)) > abs_stop:
break
#Check if DQ vector denotes that a veto is present
if ((int(value) & bitmask) != bitmask) or (int(value)<0): #(e.g., we might want bits ??, ??, or ?? to be on if there are no data-quality vetoes)
DQV_flag = True
cache.close()
#Return injection flag
return DQV_flag
def test_frgetevent_known(self):
""" test that we can pull the known contents from a test MBTA file """
a = Fr.frgetevent("MbtaFake-930909680-16.gwf")
self.assertEqual(len(a), 1)
self.assertEqual(a[0]["name"], "MbtaHLV_Chi2OK")
self.assertAlmostEqual(a[0]["H1:end_time"], 930493014.980663, 6)
self.assertAlmostEqual(a[0]["H1:SNR"], 9.0)
self.assertAlmostEqual(a[0]["H1:mass1"], 3.14696, 5)
self.assertAlmostEqual(a[0]["H1:mass2"], 10.2256, 4)
self.assertAlmostEqual(a[0]["H1:eff_distance"], 230.718, 3)
self.assertAlmostEqual(a[0]["L1:end_time"], 930493014.984246, 6)
self.assertAlmostEqual(a[0]["L1:SNR"], 8.0)
self.assertAlmostEqual(a[0]["L1:mass1"], 3.44696, 5)
self.assertAlmostEqual(a[0]["L1:mass2"], 10.5256, 4)
self.assertAlmostEqual(a[0]["L1:eff_distance"], 582.025, 3)
self.assertAlmostEqual(a[0]["V1:end_time"], 930493014.989135, 6)
self.assertAlmostEqual(a[0]["V1:SNR"], 7.0)
self.assertAlmostEqual(a[0]["V1:mass1"], 2.84696, 5)
self.assertAlmostEqual(a[0]["V1:mass2"], 9.92556, 5)
self.assertAlmostEqual(a[0]["V1:eff_distance"], 200.113, 3)
def test_frgetevent_known(self):
""" test that we can pull the known contents from a test MBTA file """
a = Fr.frgetevent("MbtaFake-930909680-16.gwf")
self.assertEqual(len(a), 1)
self.assertEqual(a[0]["name"], "MbtaHLV_Chi2OK")
self.assertAlmostEqual(a[0]["H1:end_time"], 930493014.980663, 6)
self.assertAlmostEqual(a[0]["H1:SNR"], 9.0)
self.assertAlmostEqual(a[0]["H1:mass1"], 3.14696, 5)
self.assertAlmostEqual(a[0]["H1:mass2"], 10.2256, 4)
self.assertAlmostEqual(a[0]["H1:eff_distance"], 230.718, 3)
self.assertAlmostEqual(a[0]["L1:end_time"], 930493014.984246, 6)
self.assertAlmostEqual(a[0]["L1:SNR"], 8.0)
self.assertAlmostEqual(a[0]["L1:mass1"], 3.44696, 5)
self.assertAlmostEqual(a[0]["L1:mass2"], 10.5256, 4)
self.assertAlmostEqual(a[0]["L1:eff_distance"], 582.025, 3)
self.assertAlmostEqual(a[0]["V1:end_time"], 930493014.989135, 6)
self.assertAlmostEqual(a[0]["V1:SNR"], 7.0)
self.assertAlmostEqual(a[0]["V1:mass1"], 2.84696, 5)
self.assertAlmostEqual(a[0]["V1:mass2"], 9.92556, 5)
self.assertAlmostEqual(a[0]["V1:eff_distance"], 200.113, 3)
def write_frame(det_data, ifo, seed, epoch, datalen, outdir):
"""
Write a frame
"""
# Construct name
site = ifo.strip('1')
frame_name = '{site}-{ifo}_{wf_name}_{seed}-{epoch}-{datalen}.gwf'.format(
site=site,
ifo=ifo,
wf_name=det_data.waveform_name,
seed=seed,
epoch=str(int(epoch)),
datalen=str(int(datalen)))
channel_list = [
{
'name': '%s:STRAIN' % ifo,
'data': np.array(det_data.td_response.data),
'start': epoch,
'dx': 1.0 / 16384,
'kind': 'SIM'
},
{
'name': '%s:SIGNAL' % ifo,
'data': np.array(det_data.td_signal.data),
'start': epoch,
'dx': 1.0 / 16384,
'kind': 'SIM'
},
{
'name': '%s:NOISE' % ifo,
'data': np.array(det_data.td_noise.data),
'start': epoch,
'dx': 1.0 / 16384,
'kind': 'SIM'
},
]
print 'writing frame %s...' % frame_name
frame_out_path = '%s/%s' % (os.path.abspath(outdir), frame_name)
Fr.frputvect(frame_out_path, channel_list)
#
# Generate a cache file
#
# setup url
path, filename = os.path.split(frame_out_path.strip())
url = "file://localhost%s" % os.path.abspath(os.path.join(path, filename))
# create cache entry
c = gluelal.CacheEntry.from_T050017(url)
# write to file
cache_file = frame_out_path.replace('gwf', 'lcf')
f = open(cache_file, 'w')
f.writelines('%s\n' % str(c))
f.close()
return frame_out_path, cache_file
virgo_fs = 20000.0
f, ax = pl.subplots(nrows=2,ncols=1)
for s,site in enumerate(sites):
start=float(trigger_time)-0.5*datalen
if site=='V':
delta_t = 1/virgo_fs
else:
delta_t = 1/ligo_fs
# Stick these in pycbc time series and we'll have an easy time computing
# matches etc later
bmdc_data[site] = pycbc.types.TimeSeries(Fr.frgetvect(burstmdc_frame,
burstmdc_channels[s], start=start, span=datalen)[0],
delta_t=delta_t, epoch=start)
lalsim_data[site] = pycbc.types.TimeSeries(Fr.frgetvect(lalsim_frames[s],
lalsim_channels[s], start=start, span=datalen)[0],
delta_t=1/ligo_fs, epoch=start)
# Plot burstMDC channels
ax[0].plot(bmdc_data[site].sample_times-float(trigger_time), bmdc_data[site],
label=burstmdc_channels[s])
ax[0].set_xlabel('Seconds after %s'%trigger_time)
ax[0].legend()
ax[0].set_title('BurstMDC')
# Plot lalsim channels
# # Are we looking outside the range rather than inside? # if min_threshold is not None and max_threshold is not None: invert = min_threshold >= max_threshold else: invert = False if opts.verbose: print "Inverted? %s"% str(invert) seglist = segmentlist([]) for path in cache.pfnlist(): # # Read data # data, start, _, dt, _, _ = Fr.frgetvect1d(path, channel) # # Apply conditions and transform samples to segments # if equals is not None: seglist.extend(dqsegs.equality_data_to_seglist(data, start, dt, equality=equals)) if bit_mask is not None: seglist.extend(dqsegs.mask_data_to_seglist(data, start, dt, mask_on=bit_mask)) else: seglist.extend(dqsegs.threshold_data_to_seglist(data, start, dt, min_threshold=min_threshold, max_threshold=max_threshold, invert=invert)) seglist.coalesce() all_segs["%s: %s" % (channel, str(opthresholds))] = seglist if opts.verbose:
def test_frame(self):
# This is a file in the temp directory that will be deleted when it is garbage collected
frmfile = tempfile.NamedTemporaryFile()
filename = frmfile.name
# Now we will create a frame file, specifiying that it is a timeseries
Fr.frputvect(filename,[{'name':'channel1', 'data':self.data1, 'start':int(self.epoch), 'dx':self.delta_t,'type':1},
{'name':'channel2', 'data':self.data2, 'start':int(self.epoch), 'dx':self.delta_t,'type':1}])
with self.context:
if _options['scheme'] == 'cpu':
# Reading just one channel first
ts1 = pycbc.frame.read_frame(filename,'channel1')
# Chacking all values
self.checkCurrentState((ts1,),(self.data1,),self.places)
# Now checking the start time
self.assertTrue(ts1.start_time == self.epoch)
# And the duration
self.assertTrue(ts1.end_time-ts1.start_time == self.size*self.delta_t)
# Now reading multiple channels
ts2 = pycbc.frame.read_frame(filename,['channel1','channel2'])
# We should get back a list
self.assertTrue(type(ts2) is list)
self.checkCurrentState(ts2, (self.data1,self.data2), self.places)
self.assertTrue(ts2[0].start_time == self.epoch)
self.assertTrue(ts2[1].start_time == self.epoch)
self.assertTrue(ts2[0].end_time-ts2[0].start_time == self.size*self.delta_t)
self.assertTrue(ts2[1].end_time-ts2[1].start_time == self.size*self.delta_t)
# These are the times and indices for the segment we will try to read
start = self.epoch+10
end = self.epoch+50
startind = int(10/self.delta_t)
endind = int(50/self.delta_t)
# Now reading in a specific segment with an integer
ts3 = pycbc.frame.read_frame(filename, 'channel1', start=int(start), end=int(end))
# The same, but with a LIGOTimeGPS for the start and end times
ts4 = pycbc.frame.read_frame(filename, 'channel1', start=start, end=end)
# Now we will check those two TimeSeries
self.checkCurrentState((ts3,ts4), (self.data1[startind:endind],self.data1[startind:endind]), self.places)
self.assertTrue(40 - (float(ts3.end_time)-float(ts3.start_time)) < self.delta_t)
self.assertTrue(ts3.start_time == start)
self.assertTrue(40 - (float(ts4.end_time)-float(ts4.start_time)) < self.delta_t)
self.assertTrue(ts4.start_time == start)
# And now some cases that should raise errors
# There must be a span grater than 0
self.assertRaises(ValueError, pycbc.frame.read_frame,filename,'channel1',
start=self.epoch,end=self.epoch)
# The start must be before the end
self.assertRaises(ValueError, pycbc.frame.read_frame,filename,'channel1',
start=self.epoch+1,end=self.epoch)
# Non integer times should also raise an error
badtime = lal.LIGOTimeGPS(int(self.epoch)+5,1000)
self.assertRaises(ValueError, pycbc.frame.read_frame,filename,'channel1',
start=self.epoch,end=badtime)
self.assertRaises(ValueError, pycbc.frame.read_frame,filename,'channel1',
start=float(self.epoch),end=float(badtime))
def framecache2segs(framecache_file, chname, abs_start, abs_stop, outdir, ifo, run_bitmask, inj_bitmask):
"""
Takes a file containing the data quality state vetor and converts it to segments of a desired bitmask, returning injection status
"""
#Initialize injection status as False (will change if 'No injection' bits are off)
inj_flag = False
#Open framecache file and segment file to write to
cache = open(framecache_file, 'rt')
segfile = open(outdir+'/%s_%s_%s.seg'%(ifo,abs_start,abs_stop),'wt')
#Define start and stop of current segment
current_start = None
current_stop = None
#Loop over frames in the cache
for line in cache:
#Get frame_start, frame_stride, and frame_file
words = line.split()
frame_start = int(words[2])
frame_stride = int(words[3])
frame_file = words[4].split('file://localhost')[1]
#Open state vector as array
state_array = Fr.frgetvect1d(filename=frame_file,channel='%s:%s'%(ifo,chname))[0]
#Calculate sample rate
samp_rate = len(state_array)/float(frame_stride)
#Loop over state vetor
for i, value in enumerate(state_array):
#Check to make sure we've passed the absolute start
if (frame_start + i/float(samp_rate)) < abs_start:
continue
#Check to make sure we haven't passed the absolute stop
elif (frame_start + i/float(samp_rate)) > abs_stop:
break
#Check if state vector corresponds to desired bitmask
elif ((int(value) & run_bitmask) == run_bitmask) and (int(value)>=0): #(e.g., 0b00011 = 3 and we want bits 0 and 1 to be on, so we do & with 3)
#Data is good, start new seg if needed
if not current_start:
current_start = int(np.ceil(frame_start + i/float(samp_rate) )) #data good starting at ith sample, use ceiling so don't underestimate start
else:
#Data not good, end current seg if needed
if current_start:
current_stop = int(np.floor(frame_start + i/float(samp_rate) )) #data goes bad at ith sample but good until then, use floor so don't overestimate stop
if current_start < current_stop:
segfile.write('%s %s\n'%(current_start, current_stop))
#Wait to start next segment until find good data
current_start = None
current_stop = None
#Check if state vector denotes that an injection is present
if ((int(value) & inj_bitmask) != inj_bitmask): #(e.g., we might want bits 5, 6, 7, or 8 to be on if there are no HW injections)
inj_flag = True
#Write final segment for this frame if needed
if current_start:
if (current_start < int(frame_start+frame_stride)) and (int(frame_start+frame_stride) < abs_stop):
segfile.write('%s %s\n'%(current_start, int(frame_start+frame_stride)))
elif (current_start < int(frame_start+frame_stride)) and (current_start < abs_stop):
segfile.write('%s %s\n'%(current_start, int(abs_stop)))
#Wait to start next segment until find good data
current_start = None
current_stop = None
cache.close()
segfile.close()
#Return injection flag
return inj_flag
def readframedata(frameCache, channelName, frameType, startTime, stopTime,
allowRedundantFlag, debugLevel):
#READFRAMEDATA Read a single channel of data from frame files
#
#READFRAMEDATA finds and retrieves the requested time series data from a
#set of frame files. The data is specified by the frame file type,
#channel name, start time, and duration or stop time. The necessary frame
#files are located using a file name caching scheme.
#%
#usage: [data, sampleFrequency, time] = ...
#readframedata(frameCache, channelName, frameType, ...
#startTime, stopTime, allowRedundantFlag, ...
#debugLevel);
#%
#frameCache file name cache
#channelName channel name
#frameType frame file type
#startTime GPS start time
#stopTime GPS stop time (or duration)
#allowRedundantFlag permit redundant frame data
#debugLevel verboseness of debug output
#%
#data data vector
#sampleFrequency sample frequency [Hz]
#time time vector
#%
#READFRAMEDATA expects frame cache information in the format produced by
#LOADFRAMECACHE which contains the site, frame type, duration, time, and
#location of available frame data files.
#%
#The requested site designator is determined from the first character of
#the requested channel name. The frame file type may contain wildcards.
#Unless redundant frame data is permitted, it is an error if the same
#frame file appears more than once in the frame cache file. If redundant
#frame data is permitted, the first matching frame file from the cache is
#used. It is always an error if two frame files overlap but do not cover
#the same time range or differ in type. By default, redundant frame data
#is permitted.
#%
#READFRAMEDATA retrieves data from the requested start time up to, but not
#including, the requested stop time, such that stop minus start seconds
#are retrieved. Alternatively, the desired duration in seconds may be
#specified instead of the GPS stop time parameter.
#%
#The resulting time series is returned as two row vectors containing the
#data sequence and the corresponding GPS timestamps, as well as the scalar
#sample frequency. To protect against roundoff error, an integer sample
#frequency is assumed.
#%
#If it is unable to load the requested data, READFRAMEDATA returns empty
#result vectors and zero sample frequency as well as a warning if
#debugLevel is set to 1 or higher. By default, a debugLevel of unity is
#assumed.
#%
#READFRAMEDATA is built on top of the FRGETVECT function from the FrameL
#library, which is available from the following URL.
#%
#
#%
#
#
#Shourov K. Chatterji <*****@*****.**>
#Jameson Rollins <*****@*****.**>
#
#$Id: readframedata.m 2326 2009-09-21 08:37:42Z jrollins $
#
# Rewritten in Python by Sudarshan Ghonge <*****@*****.**>
# 2015-09-04
#
# if specified stop time precedes start time,
if(stopTime<startTime):
# treat stop time as a duration
stopTime = startTime + stopTime
# determine site designator from channel name
site = channelName[0]
#if specified frame cache is invalid
if(frameCache==None):
if(debugLevel>=1):
# Issue warning
print 'Warning: Invalid frame cache'
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
# return empty results
# Identifying matching segments from frame cache
# find overlap of cache segments with requested data
segmentStartTimes = np.maximum(startTime, frameCache.startTimes)
segmentStopTimes = np.minimum(stopTime, frameCache.stopTimes)
# identify cache segments which overlap requested times
segments = np.where(segmentStopTimes > segmentStartTimes)[0]
# if no segments overlap with requested times
if(len(segments)==0):
if debugLevel>=1:
print 'Warning: No data available for [%d, %d]' %(startTime, stopTime)
data = []
time = []
sampleFrequency = 0
# return empty results
return [data, sampleFrequency]
# otherwise, find overlapping segments
else:
# identify cache segments with requested site and frame type
siteMatches = []
sitesScanned=0
for iSite in frameCache.sites[segments]:
if site in iSite:
siteMatches.append(sitesScanned)
sitesScanned+=1
siteMatches = np.asarray(siteMatches)
frameTypeMatches = []
frameTypesScanned=0
for iType in frameCache.frameTypes[segments]:
if frameType in iType:
frameTypeMatches.append(frameTypesScanned)
frameTypesScanned+=1
frameTypeMatches = np.asarray(frameTypeMatches)
segIdx = np.intersect1d(siteMatches, frameTypeMatches)
if(len(segIdx)==0):
segments = []
else:
segments = segments[segIdx]
# Identify available frame files
# initialize list of available frame files
frameFilePaths = []
frameFileTypes = []
frameFileStartTimes = []
frameFileStopTimes = []
# lopp over the matching segments
for segment in segments:
# frame type of the frame files in segment
frameFileType = frameCache.frameTypes[segment]
firstFrameFileStartTime = frameCache.startTimes[segment] + frameCache.durations[segment]*np.floor((segmentStartTimes[segment]-frameCache.startTimes[segment])/frameCache.durations[segment])
lastFrameFileStartTime = frameCache.startTimes[segment] + frameCache.durations[segment]*np.ceil((segmentStopTimes[segment] - frameCache.startTimes[segment])/frameCache.durations[segment] - 1)
for frameFileStartTime in np.arange(firstFrameFileStartTime,
lastFrameFileStartTime + frameCache.durations[segment],
frameCache.durations[segment]):
frameFileStopTime = frameFileStartTime + frameCache.durations[segment]
frameFilePath = frameCache.directories[segment] + '/' + frameCache.sites[segment] + '-' + frameCache.frameTypes[segment] + '-' + '%09d' %(frameFileStartTime) + '-' + '%d' %(frameCache.durations[segment]) + '.gwf'
# In case the frame file durations start becoming variable. The ls <name> with a '*' in place of the duration field
# returns the file name. we then use that file name
import os.path
if(not os.path.isfile(frameFilePath)):
frameFilePath = frameCache.directories[segment] + '/' + frameCache.sites[segment] +'-' + frameCache.frameTypes[segment] + '-' + '%010d' %(frameFileStartTime) + '-' + '*'+ '.gwf'
frameFilePath = os.popen('ls %s'%(frameFilePath)).readlines()[0].split('\n')[0]
if(os.path.isfile(frameFilePath)):
frameFilePaths.append(frameFilePath)
frameFileTypes.append(frameFileType)
frameFileStartTimes.append(frameFileStartTime)
frameFileStopTimes.append(frameFileStopTime)
frameFilePaths = np.asarray(frameFilePaths)
frameFileTypes = np.asarray(frameFileTypes)
frameFileStartTimes = np.asarray(frameFileStartTimes)
frameFileStopTimes = np.asarray(frameFileStopTimes)
numberOfFrameFiles = len(frameFilePaths)
keepFrameFileNumbers = []
for frameFileNumber in range(numberOfFrameFiles):
keepFrameFileFlag = True
for previousFrameFileNumber in range(frameFileNumber):
overlapStartTime = np.maximum(frameFileStartTimes[frameFileNumber],
frameFileStartTimes[previousFrameFileNumber])
overlapStoptime = np.minimum(frameFileStopTimes[frameFileNumber],
frameFileStopTimes[previousFrameFileNumber])
if (overlapStartTime < overlapStoptime):
if(allowRedundantFlag):
if((frameFileStartTimes[frameFileNumber]==frameFileStartTimes[previousFrameFileNumber]) & (frameFileStopTimes[frameFileNumber] == frameFileStopTimes[previousFrameFileNumber])
& (frameFileTypes[frameFileNumber]==frameFileTypes[previousFrameFileNumber])):
keepFrameFileFlag = False
continue
else:
if(debugLevel>=1):
print 'Warning: Overlapping but dissimilar frame files %s and %s.' %(frameFilePaths[frameFileNumber], frameFilePaths[previousFrameFileNumber])
data = []
time = []
sampleFrequency=0
return [data, sampleFrequency]
else:
if(debugLevel>=1):
print 'Warning: Redundant frame files %s and %s.' %(frameFilePaths[frameFileNumber], frameFilePaths[previousFrameFileNumber])
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
if(keepFrameFileFlag):
keepFrameFileNumbers.append(frameFileNumber)
keepFrameFileNumbers = np.asarray(keepFrameFileNumbers)
frameFilePaths = frameFilePaths[keepFrameFileNumbers]
frameFileTypes = frameFileTypes[keepFrameFileNumbers]
frameFileStartTimes = frameFileStartTimes[keepFrameFileNumbers]
frameFileStopTimes = frameFileStopTimes[keepFrameFileNumbers]
sortedIndices = np.argsort(frameFileStartTimes)
frameFilePaths = frameFilePaths[sortedIndices]
frameFiletypes = frameFileTypes[sortedIndices]
frameFileStartTimes = frameFileStartTimes[sortedIndices]
frameFileStopTimes = frameFileStopTimes[sortedIndices]
continuityStartTimes = np.append(np.maximum(startTime, frameFileStartTimes), stopTime)
continuityStopTimes = np.append(startTime, np.minimum(stopTime, frameFileStopTimes))
discontinuities = np.where(continuityStartTimes!=continuityStopTimes)[0]
if(len(discontinuities)>0):
if(debugLevel >=1):
print 'Warning: Missing %s '%(channelName), frameType[1:len(frameType)-1], ' data at ' , '%d'%(np.round(continuityStopTimes[discontinuities[0]])), '.'
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
data = np.array([])
time = np.array([])
sampleFrequency = None
numberOfFrameFiles = len(frameFilePaths)
for frameFileNumber in range(numberOfFrameFiles):
frameFilePath = frameFilePaths[frameFileNumber]
if(debugLevel>=2):
print 'Reading %s...\n' %(frameFilePath)
frameFileStartTime = frameFileStartTimes[frameFileNumber]
frameFileStopTime = frameFileStopTimes[frameFileNumber]
readData = []
readStartTime = np.maximum(startTime, frameFileStartTime)
readDuration = np.minimum(stopTime, frameFileStopTime) - readStartTime
realChannelName = channelName
try:
outputStruct = Fr.frgetvect(frameFilePath, realChannelName, readStartTime, readDuration, False)
readData = outputStruct[0]
readTime = outputStruct[2]
readSampleFrequency = 1.0/outputStruct[3][0]
readTimeStep = outputStruct[3][0]
readGPS = outputStruct[1]
except Exception as inst:
if(debugLevel>=2):
print inst.message
if((len(readData)==0) | np.any(np.isnan(readData))):
if(debugLevel>=1):
print 'Warning: Error reading %s from %s.' %(channelName, frameFilePath)
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
if(sampleFrequency==None):
sampleFrequency = readSampleFrequency
elif(sampleFrequency!=readSampleFrequency):
if(debugLevel>=1):
print 'Warning: Inconsistent sample frequency for %s in frameFilePath.' %(channelname, frameFilePath)
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
data = np.append(data, readData)
return [data, sampleFrequency]
import pmns_utils
import pmns_simsig
import matplotlib
matplotlib.rc('xtick', labelsize=16)
matplotlib.rc('ytick', labelsize=16)
matplotlib.rc('font', size=16)
from matplotlib import pyplot as pl
import matplotlib.cm as cm
from mpl_toolkits.axes_grid1 import make_axes_locatable
#
# Load frame data
#
h1data = Fr.frgetvect('H-H1-871727590-513.gwf', 'H1:LSC-STRAIN', span=500)
#
# Put frame data in lal TimeSeries
#
noise = lal.CreateREAL8TimeSeries('H1', lal.LIGOTimeGPS(h1data[1]), 0,
h1data[3][0], lal.StrainUnit, len(h1data[0]))
noise.data.data = h1data[0]
nonoise = lal.CreateREAL8TimeSeries('H1', lal.LIGOTimeGPS(h1data[1]), 0,
h1data[3][0], lal.StrainUnit, len(h1data[0]))
nonoise.data.data = np.zeros(len(h1data[0]))
#
# Generate a post-merger signal
#
#!/usr/bin/env python from pylal import Fr h_frame = 'H-H1_magA_tapered_1105199095-946076460-512.gwf' l_frame = 'L-L1_magA_tapered_1105199095-946076460-512.gwf' gps_start = 946076563.480737805 ra = 1.39 dec = -0.93 h_data = Fr.frgetvect(h_frame, 'H1:STRAIN', start=gps_start, span=1) l_data = Fr.frgetvect(l_frame, 'L1:STRAIN', start=gps_start, span=1)
def generate_gwf(self, mdc, directory, channel="SCIENCE", force=False):
"""
Produce the gwf file which corresponds to the MDC set over the period of this frame.
Parameters
----------
mdc : MDCSet object
The MDC set which should be used to produce this frame.
directory : str
The root directory where all of the frames are to be stored, for example
"/home/albert.einstein/data/mdc/frames/"
would cause the SineGaussian injections to be made in the directories under
"/home/albert.einstein/data/mdc/frames/sg"
channel : str
The name of the channel which the injections should be made into. This is prepended by the initials
for each interferometer, so there will be a channel for each interferometer in the gwf.
force : bool
If true this forces the recreation of a GWF file even if it already exists.
Outputs
-------
gwf
The GWF file for this frame.
"""
ifosstr = "".join(set(ifo[0] for ifo in self.ifos))
family = mdc.waveforms[0].waveform
filename = "{}-{}-{}-{}.gwf".format(ifosstr, family, self.start, self.duration)
head_date = str(self.start)[:5]
frameloc = directory+"/"+mdc.directory_path()+"/"+head_date+"/"
#print frameloc, filename
if not os.path.isfile(frameloc + filename) or force:
data = []
# Define the start point of the time series top be generated for the injection
epoch = lal.LIGOTimeGPS(self.start)
# Loop through each interferometer
for ifo in self.ifos:
# Calculate the number of samples in the timeseries
nsamp = (self.end-self.start)*16384
# Make the timeseries
h_resp = lal.CreateREAL8TimeSeries("inj time series", epoch, 0, 1.0/16384, lal.StrainUnit, nsamp)
# Loop over all of the injections corresponding to this frame
rowlist = self.get_rowlist(mdc)
if len(rowlist)==0: return
for row in rowlist:
sim_burst = mdc.waveforms[row]
# Produce the time domain waveform for this injection
hp, hx = lalburst.GenerateSimBurst(sim_burst, 1.0/16384);
# Apply detector response
det = lalsimulation.DetectorPrefixToLALDetector(ifo)
# Produce the total strains
h_tot = lalsimulation.SimDetectorStrainREAL8TimeSeries(hp, hx,
sim_burst.ra, sim_burst.dec, sim_burst.psi, det)
# Inject the waveform into the overall timeseries
lalsimulation.SimAddInjectionREAL8TimeSeries(h_resp, h_tot, None)
# Write out the data to the list which will eventually become our frame
data.append({"name": "%s:%s" % (ifo, channel),
"data": h_resp.data.data,
"start": float(epoch),
"dx": h_resp.deltaT,
"kind": "SIM"})
# Make the directory in which to store the files
# if it doesn't exist already
mkdir(frameloc)
# Write out the frame file
Fr.frputvect(frameloc+filename, data)
def test_cache(self):
# Knowing the middle of our array will be helpful, because we will put half on one frame,
# and half on hte other. We will also need this to read in a segment of hte cache that
# crosses this seam.
half = int((self.size/2)*self.delta_t)
# These need to be named so that lalapps_path2cache can turn them into a single cache file.
frmfile1 = tempfile.NamedTemporaryFile(prefix='H-frame-'+str(int(self.epoch))+'-'+str(half)+'.')
frmfile2 = tempfile.NamedTemporaryFile(prefix='H-frame-'+str(int(self.epoch+half))+'-'+str(half)+'.')
frmfile3 = tempfile.NamedTemporaryFile(prefix='H-frame-'+str(int(self.epoch+half+16))+'-'+str(half-16)+'.')
# We will need access to the actual filenames.
frmname1 = frmfile1.name
frmname2 = frmfile2.name
frmname3 = frmfile3.name
firsthalf1 = self.data1[0:(self.size/2)]
secondhalf1 = self.data1[(self.size/2):]
# This third piece will be paired up with the first one to create a cache file with a gap
# of 16 seconds after the half-way point
gaphalf1 = self.data1[(self.size/2 + 16/self.delta_t):]
# The same is done to the second dataset
firsthalf2 = self.data2[0:(self.size/2)]
secondhalf2 = self.data2[(self.size/2):]
gaphalf2 = self.data2[(self.size/2 + 16/self.delta_t):]
# Now we will create a frame file, this will hold the first half of our data
Fr.frputvect(frmname1,[{'name':'channel1', 'data':firsthalf1, 'start':int(self.epoch), 'dx':self.delta_t,'type':1},
{'name':'channel2', 'data':firsthalf2, 'start':int(self.epoch), 'dx':self.delta_t,'type':1}])
# This will hold the second half
Fr.frputvect(frmname2,[{'name':'channel1', 'data':secondhalf1, 'start':int(self.epoch+half), 'dx':self.delta_t,'type':1},
{'name':'channel2', 'data':secondhalf2, 'start':int(self.epoch+half), 'dx':self.delta_t,'type':1}])
# This third one will hold the second half, but without the first 16 seconds, so we can check importing from a cache with holes.
Fr.frputvect(frmname3,[{'name':'channel1', 'data':gaphalf1, 'start':int(self.epoch + half + 16), 'dx':self.delta_t,'type':1},
{'name':'channel2', 'data':gaphalf2, 'start':int(self.epoch + half + 16), 'dx':self.delta_t,'type':1}])
# These files are what path2cache will actually read, they will hold a list of frames
frmlist1 = tempfile.NamedTemporaryFile()
frmlist2 = tempfile.NamedTemporaryFile()
listname1 = frmlist1.name
listname2 = frmlist2.name
# The first one will contain the complete set, split over two frames
with open(listname1, 'w') as f1:
f1.write(frmname1+'\n')
f1.write(frmname2)
# This second one will use the gap frame for the second half, so there
# will be a 16 second gap in the middle of this cache
with open(listname2, 'w') as f2:
f2.write(frmname1+'\n')
f2.write(frmname3)
cache1 = tempfile.NamedTemporaryFile()
cache2 = tempfile.NamedTemporaryFile()
cachename1 = cache1.name
cachename2 = cache2.name
# Now we can actually make the caches from the list of frames
subprocess.call(['lalapps_path2cache','-i',listname1,'-o',cachename1])
subprocess.call(['lalapps_path2cache','-i',listname2,'-o',cachename2])
with self.context:
if _options['scheme'] == 'cpu':
# Reading just one channel first
ts = pycbc.frame.read_cache(cachename1,'channel1',self.epoch,self.epoch+self.size*self.delta_t)
self.checkCurrentState([ts],[self.data1],self.places)
self.assertTrue(ts.start_time == self.epoch)
self.assertTrue(ts.end_time-ts.start_time == self.size*self.delta_t)
# Now reading multiple channels
ts = pycbc.frame.read_cache(cachename1,['channel1','channel2'],self.epoch,self.epoch+self.size*self.delta_t)
self.assertTrue(type(ts) is list)
self.checkCurrentState(ts, [self.data1,self.data2], self.places)
self.assertTrue(ts[0].start_time == self.epoch)
self.assertTrue(ts[1].start_time == self.epoch)
self.assertTrue(ts[0].end_time-ts[0].start_time == self.size*self.delta_t)
self.assertTrue(ts[1].end_time-ts[1].start_time == self.size*self.delta_t)
# Now reading in a specific segment with an integer
start = self.epoch + 10
end = self.epoch + half + 50
startind = 10/self.delta_t
endind = (half + 50) / self.delta_t
ts = pycbc.frame.read_cache(cachename1, 'channel1', start=int(start), end=int(end))
# Now we'll check all the values
self.checkCurrentState((ts,), (self.data1[startind:endind],), self.places)
# The duration
self.assertTrue((40+half) - (float(ts.end_time)-float(ts.start_time)) < self.delta_t)
# And the start
self.assertTrue(ts.start_time == self.epoch+10)
# The same, but with a LIGOTimeGPS for the start and end times
ts = pycbc.frame.read_cache(cachename1, 'channel1', start=start, end=end)
# Now we'll check all the values
self.checkCurrentState((ts,), (self.data1[startind:endind],), self.places)
# The duration
self.assertTrue((40+half) - (float(ts.end_time)-float(ts.start_time)) < self.delta_t)
# And the start
self.assertTrue(ts.start_time == self.epoch+10)
# And now some cases that should raise errors
# There should be an error if there are gaps in the data requested
self.assertRaises(ValueError, pycbc.frame.read_cache,cachename2,'channel1',
self.epoch,self.epoch+self.size*self.delta_t)
# There must be a span grater than 0
self.assertRaises(ValueError, pycbc.frame.read_cache,cachename1,'channel1',
start=self.epoch,end=self.epoch)
# The start must be before the end
self.assertRaises(ValueError, pycbc.frame.read_cache,cachename1,'channel1',
start=self.epoch+1,end=self.epoch)
# Non integer times should also raise an error
badtime = lal.LIGOTimeGPS(int(self.epoch)+5,1000)
self.assertRaises(ValueError, pycbc.frame.read_cache,cachename1,'channel1',
start=self.epoch,end=badtime)
self.assertRaises(ValueError, pycbc.frame.read_cache,cachename1,'channel1',
start=float(self.epoch),end=float(badtime))
def readframedata(frameCache, channelName, frameType, startTime, stopTime,
allowRedundantFlag, debugLevel):
#READFRAMEDATA Read a single channel of data from frame files
#
#READFRAMEDATA finds and retrieves the requested time series data from a
#set of frame files. The data is specified by the frame file type,
#channel name, start time, and duration or stop time. The necessary frame
#files are located using a file name caching scheme.
#%
#usage: [data, sampleFrequency, time] = ...
#readframedata(frameCache, channelName, frameType, ...
#startTime, stopTime, allowRedundantFlag, ...
#debugLevel);
#%
#frameCache file name cache
#channelName channel name
#frameType frame file type
#startTime GPS start time
#stopTime GPS stop time (or duration)
#allowRedundantFlag permit redundant frame data
#debugLevel verboseness of debug output
#%
#data data vector
#sampleFrequency sample frequency [Hz]
#time time vector
#%
#READFRAMEDATA expects frame cache information in the format produced by
#LOADFRAMECACHE which contains the site, frame type, duration, time, and
#location of available frame data files.
#%
#The requested site designator is determined from the first character of
#the requested channel name. The frame file type may contain wildcards.
#Unless redundant frame data is permitted, it is an error if the same
#frame file appears more than once in the frame cache file. If redundant
#frame data is permitted, the first matching frame file from the cache is
#used. It is always an error if two frame files overlap but do not cover
#the same time range or differ in type. By default, redundant frame data
#is permitted.
#%
#READFRAMEDATA retrieves data from the requested start time up to, but not
#including, the requested stop time, such that stop minus start seconds
#are retrieved. Alternatively, the desired duration in seconds may be
#specified instead of the GPS stop time parameter.
#%
#The resulting time series is returned as two row vectors containing the
#data sequence and the corresponding GPS timestamps, as well as the scalar
#sample frequency. To protect against roundoff error, an integer sample
#frequency is assumed.
#%
#If it is unable to load the requested data, READFRAMEDATA returns empty
#result vectors and zero sample frequency as well as a warning if
#debugLevel is set to 1 or higher. By default, a debugLevel of unity is
#assumed.
#%
#READFRAMEDATA is built on top of the FRGETVECT function from the FrameL
#library, which is available from the following URL.
#%
#
#%
#
#
#Shourov K. Chatterji <*****@*****.**>
#Jameson Rollins <*****@*****.**>
#
#$Id: readframedata.m 2326 2009-09-21 08:37:42Z jrollins $
#
# Rewritten in Python by Sudarshan Ghonge <*****@*****.**>
# 2015-09-04
#
# if specified stop time precedes start time,
if (stopTime < startTime):
# treat stop time as a duration
stopTime = startTime + stopTime
# determine site designator from channel name
site = channelName[0]
#if specified frame cache is invalid
if (frameCache == None):
if (debugLevel >= 1):
# Issue warning
print 'Warning: Invalid frame cache'
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
# return empty results
# Identifying matching segments from frame cache
# find overlap of cache segments with requested data
segmentStartTimes = np.maximum(startTime, frameCache.startTimes)
segmentStopTimes = np.minimum(stopTime, frameCache.stopTimes)
# identify cache segments which overlap requested times
segments = np.where(segmentStopTimes > segmentStartTimes)[0]
# if no segments overlap with requested times
if (len(segments) == 0):
if debugLevel >= 1:
print 'Warning: No data available for [%d, %d]' % (startTime,
stopTime)
data = []
time = []
sampleFrequency = 0
# return empty results
return [data, sampleFrequency]
# otherwise, find overlapping segments
else:
# identify cache segments with requested site and frame type
siteMatches = []
sitesScanned = 0
for iSite in frameCache.sites[segments]:
if site in iSite:
siteMatches.append(sitesScanned)
sitesScanned += 1
siteMatches = np.asarray(siteMatches)
frameTypeMatches = []
frameTypesScanned = 0
for iType in frameCache.frameTypes[segments]:
if frameType in iType:
frameTypeMatches.append(frameTypesScanned)
frameTypesScanned += 1
frameTypeMatches = np.asarray(frameTypeMatches)
segIdx = np.intersect1d(siteMatches, frameTypeMatches)
if (len(segIdx) == 0):
segments = []
else:
segments = segments[segIdx]
# Identify available frame files
# initialize list of available frame files
frameFilePaths = []
frameFileTypes = []
frameFileStartTimes = []
frameFileStopTimes = []
# lopp over the matching segments
for segment in segments:
# frame type of the frame files in segment
frameFileType = frameCache.frameTypes[segment]
firstFrameFileStartTime = frameCache.startTimes[
segment] + frameCache.durations[segment] * np.floor(
(segmentStartTimes[segment] - frameCache.startTimes[segment]) /
frameCache.durations[segment])
lastFrameFileStartTime = frameCache.startTimes[
segment] + frameCache.durations[segment] * np.ceil(
(segmentStopTimes[segment] - frameCache.startTimes[segment]) /
frameCache.durations[segment] - 1)
for frameFileStartTime in np.arange(
firstFrameFileStartTime,
lastFrameFileStartTime + frameCache.durations[segment],
frameCache.durations[segment]):
frameFileStopTime = frameFileStartTime + frameCache.durations[
segment]
frameFilePath = frameCache.directories[
segment] + '/' + frameCache.sites[
segment] + '-' + frameCache.frameTypes[
segment] + '-' + '%09d' % (
frameFileStartTime) + '-' + '%d' % (
frameCache.durations[segment]) + '.gwf'
# In case the frame file durations start becoming variable. The ls <name> with a '*' in place of the duration field
# returns the file name. we then use that file name
import os.path
if (not os.path.isfile(frameFilePath)):
frameFilePath = frameCache.directories[
segment] + '/' + frameCache.sites[
segment] + '-' + frameCache.frameTypes[
segment] + '-' + '%010d' % (
frameFileStartTime) + '-' + '*' + '.gwf'
frameFilePath = os.popen(
'ls %s' % (frameFilePath)).readlines()[0].split('\n')[0]
if (os.path.isfile(frameFilePath)):
frameFilePaths.append(frameFilePath)
frameFileTypes.append(frameFileType)
frameFileStartTimes.append(frameFileStartTime)
frameFileStopTimes.append(frameFileStopTime)
frameFilePaths = np.asarray(frameFilePaths)
frameFileTypes = np.asarray(frameFileTypes)
frameFileStartTimes = np.asarray(frameFileStartTimes)
frameFileStopTimes = np.asarray(frameFileStopTimes)
numberOfFrameFiles = len(frameFilePaths)
keepFrameFileNumbers = []
for frameFileNumber in range(numberOfFrameFiles):
keepFrameFileFlag = True
for previousFrameFileNumber in range(frameFileNumber):
overlapStartTime = np.maximum(
frameFileStartTimes[frameFileNumber],
frameFileStartTimes[previousFrameFileNumber])
overlapStoptime = np.minimum(
frameFileStopTimes[frameFileNumber],
frameFileStopTimes[previousFrameFileNumber])
if (overlapStartTime < overlapStoptime):
if (allowRedundantFlag):
if ((frameFileStartTimes[frameFileNumber]
== frameFileStartTimes[previousFrameFileNumber]) &
(frameFileStopTimes[frameFileNumber]
== frameFileStopTimes[previousFrameFileNumber])
& (frameFileTypes[frameFileNumber]
== frameFileTypes[previousFrameFileNumber])):
keepFrameFileFlag = False
continue
else:
if (debugLevel >= 1):
print 'Warning: Overlapping but dissimilar frame files %s and %s.' % (
frameFilePaths[frameFileNumber],
frameFilePaths[previousFrameFileNumber])
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
else:
if (debugLevel >= 1):
print 'Warning: Redundant frame files %s and %s.' % (
frameFilePaths[frameFileNumber],
frameFilePaths[previousFrameFileNumber])
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
if (keepFrameFileFlag):
keepFrameFileNumbers.append(frameFileNumber)
keepFrameFileNumbers = np.asarray(keepFrameFileNumbers)
frameFilePaths = frameFilePaths[keepFrameFileNumbers]
frameFileTypes = frameFileTypes[keepFrameFileNumbers]
frameFileStartTimes = frameFileStartTimes[keepFrameFileNumbers]
frameFileStopTimes = frameFileStopTimes[keepFrameFileNumbers]
sortedIndices = np.argsort(frameFileStartTimes)
frameFilePaths = frameFilePaths[sortedIndices]
frameFiletypes = frameFileTypes[sortedIndices]
frameFileStartTimes = frameFileStartTimes[sortedIndices]
frameFileStopTimes = frameFileStopTimes[sortedIndices]
continuityStartTimes = np.append(
np.maximum(startTime, frameFileStartTimes), stopTime)
continuityStopTimes = np.append(startTime,
np.minimum(stopTime, frameFileStopTimes))
discontinuities = np.where(continuityStartTimes != continuityStopTimes)[0]
if (len(discontinuities) > 0):
if (debugLevel >= 1):
print 'Warning: Missing %s ' % (channelName), frameType[
1:len(frameType) - 1], ' data at ', '%d' % (np.round(
continuityStopTimes[discontinuities[0]])), '.'
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
data = np.array([])
time = np.array([])
sampleFrequency = None
numberOfFrameFiles = len(frameFilePaths)
for frameFileNumber in range(numberOfFrameFiles):
frameFilePath = frameFilePaths[frameFileNumber]
if (debugLevel >= 2):
print 'Reading %s...\n' % (frameFilePath)
frameFileStartTime = frameFileStartTimes[frameFileNumber]
frameFileStopTime = frameFileStopTimes[frameFileNumber]
readData = []
readStartTime = np.maximum(startTime, frameFileStartTime)
readDuration = np.minimum(stopTime, frameFileStopTime) - readStartTime
realChannelName = channelName
try:
outputStruct = Fr.frgetvect(frameFilePath, realChannelName,
readStartTime, readDuration, False)
readData = outputStruct[0]
readTime = outputStruct[2]
readSampleFrequency = 1.0 / outputStruct[3][0]
readTimeStep = outputStruct[3][0]
readGPS = outputStruct[1]
except Exception as inst:
if (debugLevel >= 2):
print inst.message
if ((len(readData) == 0) | np.any(np.isnan(readData))):
if (debugLevel >= 1):
print 'Warning: Error reading %s from %s.' % (channelName,
frameFilePath)
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
if (sampleFrequency == None):
sampleFrequency = readSampleFrequency
elif (sampleFrequency != readSampleFrequency):
if (debugLevel >= 1):
print 'Warning: Inconsistent sample frequency for %s in frameFilePath.' % (
channelname, frameFilePath)
data = []
time = []
sampleFrequency = 0
return [data, sampleFrequency]
data = np.append(data, readData)
return [data, sampleFrequency]
