def resampleado(strain,hh,grafica,fs,fs_ligo):
template = types.TimeSeries(initial_array=hh, delta_t=1.0/fs , epoch=0)
# Downsampling the data to 4096Hz || 8192Hz || 16KHz
strain = resample_to_delta_t(strain, 1.0/fs)
if grafica == 1:
pylab.figure('Fig_strain')
pylab.title('Strain resampled')
pylab.plot(strain.sample_times, strain)
pylab.xlabel('Time (s)')
pylab.grid()
pylab.show()
pylab.figure('Fig_template')
pylab.title('Template resampled')
pylab.plot(template.sample_times, template)
pylab.grid()
pylab.xlabel('Time (s)')
pylab.show()
return template, strain
def IFFT_to_TD(b, beta, sp, sc, delta_t):
hp = ppE_to_h(b, beta, sp, sc)[0]
hc = ppE_to_h(b, beta, sp, sc)[1]
tlen_sp = int(1.0 / delta_t / sp.delta_f)
tlen_sc = int(1.0 / delta_t / sc.delta_f)
hp.resize(tlen_sp / 2 + 1)
hc.resize(tlen_sc / 2 + 1)
tp, tc = types.TimeSeries(types.zeros(tlen), delta_t)
fft.ifft(hp, tp)
fft.ifft(hc, tc)
return tp, tc
def make(strain,template,shift,grafica):
template.resize(len(strain))
injection = template
# Getting parameters for the injection
start = strain.start_time
delta = strain.delta_t
# Seting the parameters in the injection
data_i = template.data
delta_i = delta
inicio = start
# Creating the injection vector
injection = types.TimeSeries(initial_array=data_i,delta_t=delta_i,epoch=inicio)
injection_shift = injection.cyclic_time_shift(shift)
maximo = abs(injection_shift).numpy().argmax()
injmax = injection_shift[maximo]
tmax = injection.sample_times[maximo]
# Printing the time where the amplitud is maximum
strain.data = injection_shift.data + strain.data
if grafica == 1:
print 'tiempo template maximo'
print tmax
pylab.figure('Fig_injc_shifted')
pylab.plot(injection.sample_times,injection,label='Original resized template')
pylab.plot(injection.sample_times,injection_shift,label='Injecion resized and shifted')
pylab.title('Injection resized and shifted')
pylab.grid()
pylab.legend()
pylab.show()
#print 'valor máximo'
#print injmax
return strain,tmax,injection_shift
def Read(templatename, fs, doplot):
# ---------------------------------------------
# READ THE TEMPLATE (RAW DATA)
t, hp, hx, h = GW_CCSNeFromCatalog.Read(templatename, 0)
# ---------------------------------------------
# COMPUTE STRAIN h(t) = F+ h+ + Fx hx
# ---------------------------------------------
# RESAMPLING TEMPLATE AT FS
# Make sure that initial time is zero
t = t - t[0]
# Create the new time vector for the new sampling frequency
tend = t[len(t) - 1]
tnew = np.arange(0, tend, (1.0 / fs))
# Do de interpolation
hs = CubicSpline(t, h)
hh = hs(tnew)
# ---------------------------------------------
# CREATE TIME SERIES OBJECT
template = types.TimeSeries(initial_array=hh, delta_t=1.0 / fs, epoch=0)
# ---------------------------------------------
# DO PLOT
if doplot == 1:
pylab.figure()
pylab.plot(t, h, 'r', label='Original')
pylab.plot(tnew, hh, 'b', label='Resampled')
pylab.xlabel('Time (s)', fontsize=18)
pylab.ylabel('Strain', fontsize=18)
pylab.grid(True)
pylab.legend()
pylab.show()
pylab.figure()
pylab.plot(template.sample_times, template)
pylab.xlabel('Time (s)', fontsize=18, color='black')
pylab.ylabel('Strain', fontsize=18, color='black')
pylab.grid(True)
pylab.show()
# ---------------------------------------------
# RETURN OUTPUT DATA
return template
def impulse_data(epoch=1153742417.0,sample_rate=512,psd_segment_length=60):
"""
Create fake time series data. The flux data is generated using a
random Gaussian distribution.
Parameters
----------
sample_rate : int
Sampling rate of fake data
psd_segment_length : int
Length of each segment in seconds
"""
ts_data = numpy.zeros(sample_rate * psd_segment_length)
ts_data = types.TimeSeries(ts_data, delta_t=1.0/sample_rate, epoch=epoch)
return ts_data
def fake_data(sample_rate, psd_segment_length):
"""
Create fake time series data. The flux data is generated using a
random Gaussian distribution.
Parameters
----------
sample_rate : int
Sampling rate of fake data
psd_segment_length : int
Length of each segment in seconds
"""
epoch = 1153742447.0 - 10
ts_data = numpy.random.normal(0, 1, sample_rate * psd_segment_length * 16)
ts_data = types.TimeSeries(ts_data, delta_t=1.0 / sample_rate, epoch=epoch)
return ts_data
def resampleado(strain, hh, grafica):
template = types.TimeSeries(initial_array=hh, delta_t=1.0 / 8192, epoch=0)
# Downsampling the data to 4096Hz || 8192Hz || 16KHz
strain = resample_to_delta_t(strain, 1.0 / 8192)
template = resample_to_delta_t(template, 1.0 / 8192)
if grafica == 1:
pylab.figure(figsize=[15, 5])
pylab.title('strain resampled')
pylab.plot(strain.sample_times, strain)
pylab.xlabel('Time (s)')
pylab.show()
def fake_data(epoch=1153742417.0,sample_rate=512,psd_segment_length=60,nsegs=16):
"""
Create fake time series data. The flux data is generated using a
random Gaussian distribution.
Parameters
----------
sample_rate : int
Sampling rate of fake data
psd_segment_length : int
Length of each segment in seconds
nsegs : int
Number of segments present in time series
"""
ts_data = numpy.random.normal(0,1,sample_rate*psd_segment_length*nsegs)
ts_data = types.TimeSeries(ts_data,delta_t=1.0/sample_rate,epoch=epoch)
return ts_data
def identify_block(ts_data,fd_psd,window,t_idx_min,t_idx_max):
"""
Get frequency series of the current block
Parameters
----------
ts_data : TimeSeries
Time series of magnetic field data
fd_psd :
Power Spectrum Density
window :
t_idx_min : float
Index in time series of first data point
t_idx_max : float
Index in time series of last data point
Return
------
start_time : float
Starting time of the block
end_time : float
Ending time of the block
tmp_ts_data : TimeSeries
Time series magnetic data of the block
fs_data : FrequencySeries
Frequency series magnetic data of the block
"""
# Define starting and ending time of the segment in seconds
start_time = ts_data.start_time + t_idx_min/float(ts_data.sample_rate)
end_time = ts_data.start_time + t_idx_max/float(ts_data.sample_rate)
print "|-- Analyzing block %i to %i (%.2f percent)"%(start_time,end_time,100*float(t_idx_max)/len(ts_data))
# Model a withen time series for the block
tmp_ts_data = types.TimeSeries(ts_data[t_idx_min:t_idx_max]*window,delta_t=1./ts_data.sample_rate,epoch=start_time)
# Save time series in relevant repository
segfolder = 'segments/%i-%i'%(start_time,end_time)
os.system('mkdir -p '+segfolder)
plot_ts(tmp_ts_data,fname='segments/time-series/%i-%i.png'%(start_time,end_time))
# Convert times series to frequency series
fs_data = tmp_ts_data.to_frequencyseries()
print "|-- Frequency series data has variance: %s" % fs_data.data.std()**2
# Whitening (FIXME: Whiten the filters, not the data)
fs_data.data /= numpy.sqrt(fd_psd) / numpy.sqrt(2 * fd_psd.delta_f)
print "|-- Whitened frequency series data has variance: %s" % fs_data.data.std()**2
return start_time, end_time, tmp_ts_data, fs_data
def to_pycbc(self, copy=True):
"""Convert this `TimeSeries` into a PyCBC
`~pycbc.types.timeseries.TimeSeries`
Parameters
----------
copy : `bool`, optional, default: `True`
if `True`, copy these data to a new array
Returns
-------
timeseries : `~pycbc.types.timeseries.TimeSeries`
a PyCBC representation of this `TimeSeries`
"""
from pycbc import types
return types.TimeSeries(self.value,
delta_t=self.dt.to('s').value,
epoch=self.epoch.gps,
copy=copy)
def gaussian(total_time, fs):
# Number of points for the time vector
N = total_time * fs
# Number of seconds for the injection to be shifted
shift = randi.randrange(5, 27, 1)
a = []
for i in range(N):
ran = (random.gauss(0, 1E-21))
a.append(ran)
strain = types.TimeSeries(initial_array=a, delta_t=1.0 / fs, epoch=0)
return strain, shift
def excess_power2(
ts_data, # Time series from magnetic field data
psd_segment_length, # Length of each segment in seconds
psd_segment_stride, # Separation between 2 consecutive segments in seconds
psd_estimation, # Average method
window_fraction, # Withening window fraction
tile_fap, # Tile false alarm probability threshold in Gaussian noise.
station, # Station
nchans=None, # Total number of channels
band=None, # Channel bandwidth
fmin=0, # Lowest frequency of the filter bank.
fmax=None, # Highest frequency of the filter bank.
max_duration=None, # Maximum duration of the tile
wtype='tukey'): # Whitening type, can tukey or hann
"""
Perform excess-power search analysis on magnetic field data.
This method will produce a bunch of time-frequency plots for every
tile duration and bandwidth analysed as well as a XML file identifying
all the triggers found in the selected data within the user-defined
time range.
Parameters
----------
ts_data : TimeSeries
Time Series from magnetic field data
psd_segment_length : float
Length of each segment in seconds
psd_segment_stride : float
Separation between 2 consecutive segments in seconds
psd_estimation : string
Average method
window_fraction : float
Withening window fraction
tile_fap : float
Tile false alarm probability threshold in Gaussian noise.
nchans : int
Total number of channels
band : float
Channel bandwidth
fmin : float
Lowest frequency of the filter bank.
fmax : float
Highest frequency of the filter bank
"""
# Determine sampling rate based on extracted time series
sample_rate = ts_data.sample_rate
# Check if tile maximum frequency is not defined
if fmax is None or fmax > sample_rate / 2.:
# Set the tile maximum frequency equal to the Nyquist frequency
# (i.e. half the sampling rate)
fmax = sample_rate / 2.0
# Check whether or not tile bandwidth and channel are defined
if band is None and nchans is None:
# Exit program with error message
exit("Either bandwidth or number of channels must be specified...")
else:
# Check if tile maximum frequency larger than its minimum frequency
assert fmax >= fmin
# Define spectral band of data
data_band = fmax - fmin
# Check whether tile bandwidth or channel is defined
if band is not None:
# Define number of possible filter bands
nchans = int(data_band / band) - 1
elif nchans is not None:
# Define filter bandwidth
band = data_band / nchans
nchans = nchans - 1
# Check if number of channels is superior than unity
assert nchans > 1
# Print segment information
print '|- Estimating PSD from segments of time',
print '%.2f s in length, with %.2f s stride...' % (psd_segment_length,
psd_segment_stride)
# Convert time series as array of float
data = ts_data.astype(numpy.float64)
# Define segment length for PSD estimation in sample unit
seg_len = int(psd_segment_length * sample_rate)
# Define separation between consecutive segments in sample unit
seg_stride = int(psd_segment_stride * sample_rate)
# Calculate the overall PSD from individual PSD segments
fd_psd = psd.welch(data,
avg_method=psd_estimation,
seg_len=seg_len,
seg_stride=seg_stride)
# We need this for the SWIG functions...
lal_psd = fd_psd.lal()
# Plot the power spectral density
plot_spectrum(fd_psd)
# Create whitening window
print "|- Whitening window and spectral correlation..."
if wtype == 'hann':
window = lal.CreateHannREAL8Window(seg_len)
elif wtype == 'tukey':
window = lal.CreateTukeyREAL8Window(seg_len, window_fraction)
else:
raise ValueError("Can't handle window type %s" % wtype)
# Create FFT plan
fft_plan = lal.CreateForwardREAL8FFTPlan(len(window.data.data), 1)
# Perform two point spectral correlation
spec_corr = lal.REAL8WindowTwoPointSpectralCorrelation(window, fft_plan)
# Initialise filter bank
print "|- Create filter..."
filter_bank, fdb = [], []
# Loop for each channels
for i in range(nchans):
channel_flow = fmin + band / 2 + i * band
channel_width = band
# Create excess power filter
lal_filter = lalburst.CreateExcessPowerFilter(channel_flow,
channel_width, lal_psd,
spec_corr)
filter_bank.append(lal_filter)
fdb.append(Spectrum.from_lal(lal_filter))
# Calculate the minimum bandwidth
min_band = (len(filter_bank[0].data.data) - 1) * filter_bank[0].deltaF / 2
# Plot filter bank
plot_bank(fdb)
# Convert filter bank from frequency to time domain
print "|- Convert all the frequency domain to the time domain..."
tdb = []
# Loop for each filter's spectrum
for fdt in fdb:
zero_padded = numpy.zeros(int((fdt.f0 / fdt.df).value) + len(fdt))
st = int((fdt.f0 / fdt.df).value)
zero_padded[st:st + len(fdt)] = numpy.real_if_close(fdt.value)
n_freq = int(sample_rate / 2 / fdt.df.value) * 2
tdt = numpy.fft.irfft(zero_padded, n_freq) * math.sqrt(sample_rate)
tdt = numpy.roll(tdt, len(tdt) / 2)
tdt = TimeSeries(tdt,
name="",
epoch=fdt.epoch,
sample_rate=sample_rate)
tdb.append(tdt)
# Plot time series filter
plot_filters(tdb, fmin, band)
# Compute the renormalization for the base filters up to a given bandwidth.
mu_sq_dict = {}
# Loop through powers of 2 up to number of channels
for nc_sum in range(0, int(math.log(nchans, 2))):
nc_sum = 2**nc_sum - 1
print "|- Calculating renormalization for resolution level containing %d %fHz channels" % (
nc_sum + 1, min_band)
mu_sq = (nc_sum + 1) * numpy.array([
lalburst.ExcessPowerFilterInnerProduct(f, f, spec_corr, None)
for f in filter_bank
])
# Uncomment to get all possible frequency renormalizations
#for n in xrange(nc_sum, nchans): # channel position index
for n in xrange(nc_sum, nchans, nc_sum + 1): # channel position index
for k in xrange(0, nc_sum): # channel sum index
# FIXME: We've precomputed this, so use it instead
mu_sq[n] += 2 * lalburst.ExcessPowerFilterInnerProduct(
filter_bank[n - k], filter_bank[n - 1 - k], spec_corr,
None)
#print mu_sq[nc_sum::nc_sum+1]
mu_sq_dict[nc_sum] = mu_sq
# Create an event list where all the triggers will be stored
event_list = lsctables.New(lsctables.SnglBurstTable, [
'start_time', 'start_time_ns', 'peak_time', 'peak_time_ns', 'duration',
'bandwidth', 'central_freq', 'chisq_dof', 'confidence', 'snr',
'amplitude', 'channel', 'ifo', 'process_id', 'event_id', 'search',
'stop_time', 'stop_time_ns'
])
# Create repositories to save TF and time series plots
os.system('mkdir -p segments/time-frequency')
os.system('mkdir -p segments/time-series')
# Define time edges
t_idx_min, t_idx_max = 0, seg_len
while t_idx_max <= len(ts_data):
# Define starting and ending time of the segment in seconds
start_time = ts_data.start_time + t_idx_min / float(
ts_data.sample_rate)
end_time = ts_data.start_time + t_idx_max / float(ts_data.sample_rate)
print "\n|-- Analyzing block %i to %i (%.2f percent)" % (
start_time, end_time, 100 * float(t_idx_max) / len(ts_data))
# Model a withen time series for the block
tmp_ts_data = types.TimeSeries(ts_data[t_idx_min:t_idx_max] *
window.data.data,
delta_t=1. / ts_data.sample_rate,
epoch=start_time)
# Save time series in relevant repository
segfolder = 'segments/%i-%i' % (start_time, end_time)
os.system('mkdir -p ' + segfolder)
plot_ts(tmp_ts_data,
fname='segments/time-series/%i-%i.png' %
(start_time, end_time))
# Convert times series to frequency series
fs_data = tmp_ts_data.to_frequencyseries()
print "|-- Frequency series data has variance: %s" % fs_data.data.std(
)**2
# Whitening (FIXME: Whiten the filters, not the data)
fs_data.data /= numpy.sqrt(fd_psd) / numpy.sqrt(2 * fd_psd.delta_f)
print "|-- Whitened frequency series data has variance: %s" % fs_data.data.std(
)**2
print "|-- Create time-frequency plane for current block"
# Return the complex snr, along with its associated normalization of the template,
# matched filtered against the data
#filter.matched_filter_core(types.FrequencySeries(tmp_filter_bank,delta_f=fd_psd.delta_f),
# fs_data,h_norm=1,psd=fd_psd,low_frequency_cutoff=filter_bank[0].f0,
# high_frequency_cutoff=filter_bank[0].f0+2*band)
print "|-- Filtering all %d channels..." % nchans
# Initialise 2D zero array
tmp_filter_bank = numpy.zeros(len(fd_psd), dtype=numpy.complex128)
# Initialise 2D zero array for time-frequency map
tf_map = numpy.zeros((nchans, seg_len), dtype=numpy.complex128)
# Loop over all the channels
for i in range(nchans):
# Reset filter bank series
tmp_filter_bank *= 0.0
# Index of starting frequency
f1 = int(filter_bank[i].f0 / fd_psd.delta_f)
# Index of ending frequency
f2 = int((filter_bank[i].f0 + 2 * band) / fd_psd.delta_f) + 1
# (FIXME: Why is there a factor of 2 here?)
tmp_filter_bank[f1:f2] = filter_bank[i].data.data * 2
# Define the template to filter the frequency series with
template = types.FrequencySeries(tmp_filter_bank,
delta_f=fd_psd.delta_f,
copy=False)
# Create filtered series
filtered_series = filter.matched_filter_core(
template,
fs_data,
h_norm=None,
psd=None,
low_frequency_cutoff=filter_bank[i].f0,
high_frequency_cutoff=filter_bank[i].f0 + 2 * band)
# Include filtered series in the map
tf_map[i, :] = filtered_series[0].numpy()
# Plot spectrogram
plot_spectrogram(numpy.abs(tf_map).T,
tmp_ts_data.delta_t,
band,
ts_data.sample_rate,
start_time,
end_time,
fname='segments/time-frequency/%i-%i.png' %
(start_time, end_time))
# Loop through all summed channels
for nc_sum in range(0, int(math.log(nchans, 2)))[::-1]:
nc_sum = 2**nc_sum - 1
mu_sq = mu_sq_dict[nc_sum]
# Clip the boundaries to remove window corruption
clip_samples = int(psd_segment_length * window_fraction *
ts_data.sample_rate / 2)
# Constructing tile and calculate their energy
print "\n|--- Constructing tile with %d summed channels..." % (
nc_sum + 1)
# Current bandwidth of the time-frequency map tiles
df = band * (nc_sum + 1)
dt = 1.0 / (2 * df)
# How much each "step" is in the time domain -- under sampling rate
us_rate = int(round(dt / ts_data.delta_t))
print "|--- Undersampling rate for this level: %f" % (
ts_data.sample_rate / us_rate)
print "|--- Calculating tiles..."
# Making independent tiles
# because [0:-0] does not give the full array
tf_map_temp = tf_map[:,clip_samples:-clip_samples:us_rate] \
if clip_samples > 0 else tf_map[:,::us_rate]
tiles = tf_map_temp.copy()
# Here's the deal: we're going to keep only the valid output and
# it's *always* going to exist in the lowest available indices
stride = nc_sum + 1
for i in xrange(tiles.shape[0] / stride):
numpy.absolute(tiles[stride * i:stride * (i + 1)].sum(axis=0),
tiles[stride * (i + 1) - 1])
tiles = tiles[nc_sum::nc_sum + 1].real**2 / mu_sq[nc_sum::nc_sum +
1].reshape(
-1, 1)
print "|--- TF-plane is %dx%s samples" % tiles.shape
print "|--- Tile energy mean %f, var %f" % (numpy.mean(tiles),
numpy.var(tiles))
# Define maximum number of degrees of freedom and check it larger or equal to 2
max_dof = 32 if max_duration == None else 2 * max_duration * df
assert max_dof >= 2
# Loop through multiple degrees of freedom
for j in [2**l for l in xrange(0, int(math.log(max_dof, 2)))]:
# Duration is fixed by the NDOF and bandwidth
duration = j * dt
print "\n|----- Explore signal duration of %f s..." % duration
print "|----- Summing DOF = %d ..." % (2 * j)
tlen = tiles.shape[1] - 2 * j + 1 + 1
dof_tiles = numpy.zeros((tiles.shape[0], tlen))
sum_filter = numpy.array([1, 0] * (j - 1) + [1])
for f in range(tiles.shape[0]):
# Sum and drop correlate tiles
dof_tiles[f] = fftconvolve(tiles[f], sum_filter, 'valid')
print "|----- Summed tile energy mean: %f, var %f" % (
numpy.mean(dof_tiles), numpy.var(dof_tiles))
plot_spectrogram(
dof_tiles.T,
dt,
df,
ts_data.sample_rate,
start_time,
end_time,
fname='segments/%i-%i/tf_%02ichans_%02idof.png' %
(start_time, end_time, nc_sum + 1, 2 * j))
threshold = scipy.stats.chi2.isf(tile_fap, j)
print "|------ Threshold for this level: %f" % threshold
spant, spanf = dof_tiles.shape[1] * dt, dof_tiles.shape[0] * df
print "|------ Processing %.2fx%.2f time-frequency map." % (
spant, spanf)
# Since we clip the data, the start time needs to be adjusted accordingly
window_offset_epoch = fs_data.epoch + psd_segment_length * window_fraction / 2
window_offset_epoch = LIGOTimeGPS(float(window_offset_epoch))
for i, j in zip(*numpy.where(dof_tiles > threshold)):
event = event_list.RowType()
# The points are summed forward in time and thus a `summed point' is the
# sum of the previous N points. If this point is above threshold, it
# corresponds to a tile which spans the previous N points. However, the
# 0th point (due to the convolution specifier 'valid') is actually
# already a duration from the start time. All of this means, the +
# duration and the - duration cancels, and the tile 'start' is, by
# definition, the start of the time frequency map if j = 0
# FIXME: I think this needs a + dt/2 to center the tile properly
event.set_start(window_offset_epoch + float(j * dt))
event.set_stop(window_offset_epoch + float(j * dt) +
duration)
event.set_peak(event.get_start() + duration / 2)
event.central_freq = filter_bank[
0].f0 + band / 2 + i * df + 0.5 * df
event.duration = duration
event.bandwidth = df
event.chisq_dof = 2 * duration * df
event.snr = math.sqrt(dof_tiles[i, j] / event.chisq_dof -
1)
# FIXME: Magic number 0.62 should be determine empircally
event.confidence = -lal.LogChisqCCDF(
event.snr * 0.62, event.chisq_dof * 0.62)
event.amplitude = None
event.process_id = None
event.event_id = event_list.get_next_id()
event_list.append(event)
for event in event_list[::-1]:
if event.amplitude != None:
continue
etime_min_idx = float(event.get_start()) - float(
fs_data.epoch)
etime_min_idx = int(etime_min_idx / tmp_ts_data.delta_t)
etime_max_idx = float(event.get_start()) - float(
fs_data.epoch) + event.duration
etime_max_idx = int(etime_max_idx / tmp_ts_data.delta_t)
# (band / 2) to account for sin^2 wings from finest filters
flow_idx = int((event.central_freq - event.bandwidth / 2 -
(df / 2) - fmin) / df)
fhigh_idx = int((event.central_freq + event.bandwidth / 2 +
(df / 2) - fmin) / df)
# TODO: Check that the undersampling rate is always commensurate
# with the indexing: that is to say that
# mod(etime_min_idx, us_rate) == 0 always
z_j_b = tf_map[flow_idx:fhigh_idx,
etime_min_idx:etime_max_idx:us_rate]
event.amplitude = 0
print "|------ Total number of events: %d" % len(event_list)
t_idx_min += int(seg_len * (1 - window_fraction))
t_idx_max += int(seg_len * (1 - window_fraction))
setname = "MagneticFields"
__program__ = 'pyburst_excesspower'
start_time = LIGOTimeGPS(int(ts_data.start_time))
end_time = LIGOTimeGPS(int(ts_data.end_time))
inseg = segment(start_time, end_time)
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
ifo = 'H1' #channel_name.split(":")[0]
straindict = psd.insert_psd_option_group.__dict__
proc_row = register_to_xmldoc(xmldoc,
__program__,
straindict,
ifos=[ifo],
version=git_version.id,
cvs_repository=git_version.branch,
cvs_entry_time=git_version.date)
dt_stride = psd_segment_length
sample_rate = ts_data.sample_rate
# Amount to overlap successive blocks so as not to lose data
window_overlap_samples = window_fraction * sample_rate
outseg = inseg.contract(window_fraction * dt_stride / 2)
# With a given dt_stride, we cannot process the remainder of this data
remainder = math.fmod(abs(outseg), dt_stride * (1 - window_fraction))
# ...so make an accounting of it
outseg = segment(outseg[0], outseg[1] - remainder)
ss = append_search_summary(xmldoc,
proc_row,
ifos=(station, ),
inseg=inseg,
outseg=outseg)
for sb in event_list:
sb.process_id = proc_row.process_id
sb.search = proc_row.program
sb.ifo, sb.channel = station, setname
xmldoc.childNodes[0].appendChild(event_list)
fname = 'excesspower.xml.gz'
utils.write_filename(xmldoc, fname, gz=fname.endswith("gz"))
dt = np.loadtxt('And1815fr1kpc_equ.txt', usecols=(0))
hp = np.loadtxt('And1815fr1kpc_equ.txt', usecols=(1))
hc = np.loadtxt('And1815fr1kpc_equ.txt', usecols=(2))
h = hc + hp
fs = 8192
pylab.figure('Original')
pylab.plot(dt, h)
pylab.grid()
pylab.show()
template = types.TimeSeries(initial_array=h, delta_t=1.0 / 2048, epoch=0)
pylab.figure('Original')
pylab.plot(template.sample_times, template)
pylab.grid()
pylab.show()
template = resample_to_delta_t(template, 1.0 / 1024)
pylab.figure('Original')
pylab.plot(template.sample_times, template)
pylab.grid()
pylab.show()
interpolacion(h, dt, 1, fs)
def principal(runs,archivo_nombre,local,ifo,hilo):
# Loading our modules
from modulos import gw_resample, gw_injection, gw_matched_filter
from modulos import gw_detection, gw_expected, gw_qtransform, gw_data, gw_chisqv2, gw_readtemplatev
from Noise_Sources import gw_load_noise, load
import numpy as np
import random as randi
from pycbc import frame, types
import urllib
grafica = 0
SNRthr = 5.5
NSegmnt = 128
#initial distance
i_d = 1
#final distance
f_d = 10
#number of slices
N_d = 128
#dic = np.linspace(i_d,f_d,N_d)
#di = []
#di = [1,5,10,15,20,25,30,35,40,45,50]
#di.extend(dic)
if local == False:
url = 'https://www.gw-openscience.org/archive/data/O1_16KHZ/1126170624/'+ifo+'-'+ifo+'1_LOSC_16_V1-'
urllib.urlretrieve(url+str(archivo_nombre)+'-4096.gwf', str(archivo_nombre)+'.gwf')
#-------------------------------------------------------------------------
#temporal stuff
aprox = 'NEW'
numero = 1
total_time = 32
fs = 8192 # Sampling frequency
fs_ligo = 16384
#---------------------------------------------------------------------------#
# loading files #
#---------------------------------------------------------------------------#
archivo = str(archivo_nombre)+'.gwf'
strain_initial = frame.read_frame(archivo, 'H1:GWOSC-16KHZ_R1_STRAIN')
strain_n = strain_initial.data
del strain_initial
strain_light = []
for i in range(NSegmnt):
a = i * total_time * fs_ligo
b = (i + 1) * total_time * fs_ligo
strain_light.append(strain_n[a:b])
###############################################################################
#
# vectores de resultados
#
###############################################################################
SNRrec = []
SNRexp = []
trigger_time = []
injection_time = []
total_flag = []
numero = 20
for i in range(numero):
for dat in range(len(strain_light)):
print dat, ' indice de segmento'
strain = strain_light[dat]
strain = types.TimeSeries(initial_array=strain, delta_t=1.0/fs_ligo , epoch=0)
rec = []
exp = []
flag = []
tt = []
ti = []
assert isinstance(runs, object)
# for d in range(len(di)):
distancia = 3#di[d]
n=numero
print distancia, ' distancia'
# Reading template
h,dt,nombre = gw_readtemplatev.lectura(n,aprox)
# Distance
hh, dts,dtf = gw_resample.interpolacion(h,dt,grafica,fs)
hh = hh/distancia
shift = 28
template, noise = gw_resample.resampleado(strain,hh,grafica,fs,fs_ligo)
# Making uniform the distribution of the dt in the template
# These function returns the resampled strain,
#Resampling and resizing data
# Making the injection of the template into the data
strain_inj,tmax,injection_shift = gw_injection.make(noise,template,shift,grafica)
# The limit values for the frequency in the filters
fc = 15
hc = 1200
mc = 733
# Making the matched filter
# This function give us the timeseries object of the snr over the time, the maximum value of the snr and
# the time that it occurs
snr,psd,index_snr_trigger,time,rho,psd_o,rho_0 = gw_matched_filter.make(strain_inj,template,fc,mc,hc,grafica)
# Getting the Chi^2 value
snr2,HSNR2_time,nsnr = gw_chisqv2.xis(snr,template,strain_inj,psd_o,fc,grafica,time,dtf,hc,shift,index_snr_trigger)
# Getting the theoretical value of snr
#-------------------------------------------------------------------------------------------------------------------------
# Detection part
desicion = gw_detection.decision(rho,SNRthr)
#--------------------------------------------------------------------------------------------------------------------------
# The time, amplitude, and phase of the SNR peak tell us how to align
# our proposed signal with the data.
rec.append(abs(rho))
exp.append(abs(rho_0))
flag.append(desicion)
tt.append(shift)
ti.append(time)
SNRrec.extend(rec)
SNRexp.extend(exp)
total_flag.extend(flag)
trigger_time.extend(ti)
injection_time.extend(tt)
rec = []
exp = []
flag = []
tt = []
ti = []
np.savetxt('Total'+str(archivo_nombre)+ifo+'Hilo'+hilo+'.out',(SNRrec,SNRexp,total_flag,trigger_time,injection_time),fmt='%3.2f')
return
def burst_inject(ts_data,
loc=0.5,
duration=0.1,
hrss=0.0275,
amp=100.,
plot=True,
sine=False):
'''
Inject burst signal in time series data
Parameters
----------
ts_data : TimeSeries
Time series magnetic field data
duration : float
Duration of the burst
hrss : float
hrss
Return
------
ts_data : TimeSeries
New time series data with injected burst
hp : TimeSeries
Time series data of the burst signal only
'''
# Define sampling rate
sample_rate = float(ts_data.sample_rate)
# Define time period from sampling rate
delta_t = 1.0 / sample_rate
# Define start time of the time series
t0 = float(ts_data.start_time)
# Define final time of the time series
t1 = t0 + len(ts_data) / sample_rate
if sine:
# First method to create sine gaussian burst
f_0 = 18
filter_band = 4
q = math.sqrt(2) * f_0 / filter_band * 2
# Create sine gaussian burst
hp, hx = SimBurstSineGaussian(q * 2, f_0, hrss, 1, 0, delta_t)
else:
# Create gaussian burst
hp, hx = SimBurstGaussian(duration, hrss, delta_t)
hp = TimeSeries.from_lal(hp)
hx = TimeSeries.from_lal(hx)
# We rescale the amplitude to hide or expose it in the data a bit better
hp *= amp
if plot:
# Plot fake burst signal
pyplot.plot(hp.times, hp, 'k-')
#pyplot.xlim([-0.5, 0.5])
#pyplot.ylim([-0.1, 0.1]);
pyplot.xlabel('Time (s)')
pyplot.ylabel('Magnitude')
pyplot.savefig('fakesignal.png')
pyplot.close()
# Define burst epoch
hp.epoch = int(t0 + loc * (t1 - t0))
# Define burst first timestamp
st = int((hp.epoch.value - t0) * sample_rate - len(hp) / 2)
# Define burst final timestamp
en = st + len(hp)
# Convert time series into gwpy.timeseries.TimeSeries format
ts_data = TimeSeries(ts_data, sample_rate=sample_rate, epoch=t0)
# Include burst in data
ts_data[st:en] += hp
# Convert back to pycbc.types.TimeSeries format
ts_data = types.TimeSeries(ts_data.value,
delta_t=1.0 / sample_rate,
epoch=t0)
# Return time series with burst included
return ts_data, hp
strainMag = np.sqrt((strainVal1[i])**2 +
(strainVal2[i])**2)
else:
print "dataFormat is incorrect or is not specified. Edit the metadata file and try again."
if strainMag > dmax:
maxloc = i
dmax = strainMag
timeAdjust = times[maxloc]
for i in range(0, len(times)):
times[i] = (times[i] - timeAdjust)
# leaving this here for now; sometimes I've needed to test with
# different values/equations
massMpc = 1
strainVal1 = types.TimeSeries(strainVal1 / massMpc,
delta_t=delta_t)
strainVal2 = types.TimeSeries(strainVal2, delta_t=delta_t)
# run romSpline to convert into reduced order spline, then assign final .h5 values
# and write all data to .h5 file
# handled independently for Magnitude/Argument vs. Pluss/Cross data, based on
# unique needs for each format
if (dataFormat == "MagArg"):
strainAmp = np.array(strainVal1)
strainPhase = np.array(strainVal2)
print 'fitting spline...'
sAmpH = romSpline.ReducedOrderSpline(times,
strainAmp,
rel=True,
verbose=False)
def get_data(station,
starttime,
endtime,
activity=False,
rep='/GNOMEDrive/gnome/serverdata/',
resample=None):
"""
Glob all files withing user-defined period and extract data.
Parameters
----------
station : str
Name of the station to be analysed
t0 : int
GPS timestamp of the first required magnetic field data
t1 : int
GPS timestamp of the last required magnetic field data
Return
------
ts_data, ts_list, activity : TimeSeries, dictionary, list
Time series data for selected time period, list of time series
for each segment, sampling rate of the retrieved data
"""
setname = "MagneticFields"
dstr = ['%Y', '%m', '%d', '%H', '%M']
dsplit = '-'.join(dstr[:starttime.count('-') + 1])
start = datetime.strptime(starttime, dsplit)
starttime = construct_utc_from_metadata(start.strftime("%Y/%m/%d"),
start.strftime("%H:%M:%S.%d"))
dsplit = '-'.join(dstr[:endtime.count('-') + 1])
end = datetime.strptime(endtime, dsplit)
endtime = construct_utc_from_metadata(end.strftime("%Y/%m/%d"),
end.strftime("%H:%M:%S.%d"))
dataset = []
for date in numpy.arange(start, end, timedelta(minutes=1)):
date = date.astype(datetime)
path1 = rep + station + '/' + date.strftime("%Y/%m/%d/")
path2 = station + '_' + date.strftime("%Y%m%d_%H%M*.hdf5")
fullpath = os.path.join(path1, path2)
dataset += glob.glob(fullpath)
if len(dataset) == 0:
print "ERROR: No data files were found..."
quit()
file_order, data_order = {}, {}
for fname in dataset:
hfile = h5py.File(fname, "r")
segfile = file_to_segment(hfile, setname)
file_order[segfile] = fname
data_order[segfile] = hfile
# Extract sample rate from metadata of last read data file
sample_rate = hfile[setname].attrs["SamplingRate(Hz)"]
# Estimate full segment activity list
activity = create_activity_list(station, data_order)
# Generate an ASCII representation of the GPS timestamped
# segments of time covered by the input data
seglist = segmentlist(data_order.keys())
# Sort the segment list
seglist.sort()
# Create list of time series from every segment
ts_list = generate_timeseries(file_order, setname)
# Retrieve channel data for all the segments
full_data = numpy.hstack(
[retrieve_channel_data(data_order[seg], setname) for seg in seglist])
new_sample_rate = sample_rate if resample == None else resample
new_data_length = len(full_data) / float(sample_rate) * new_sample_rate
full_data = scipy.signal.resample(full_data, int(new_data_length))
# Models a time series consisting of uniformly sampled scalar values
ts_data = types.TimeSeries(full_data,
delta_t=1. / new_sample_rate,
epoch=seglist[0][0])
for v in data_order.values():
v.close()
return ts_data, ts_list, activity, int(starttime), int(endtime)
em = pieces[2]
# EX: Rh_l2_m0_r00400.txt
# pull time and strain data from appropriate file(s) and run appropriate conversions
times = h_group[key][:, 0]
strainVal1 = h_group[key][:, 1]
strainVal2 = h_group[key][:, 2]
for i in range(0, len(times)):
times[i] = (times[i] - timeAdjust)
strainVal1, strainVal2 = convert_strain(
strainFormat, strainVal1, strainVal2, total_grav_mass)
times = convert_time(timeFormat, times, total_grav_mass)
times = np.array(types.TimeSeries(times, delta_t=delta_t))
strainVal1 = types.TimeSeries(strainVal1, delta_t=delta_t)
strainVal2 = types.TimeSeries(strainVal2, delta_t=delta_t)
strain2neg = types.TimeSeries(0 - strainVal2,
delta_t=delta_t)
strainAmp = []
strainPhase = []
strainAmp2 = []
strainPhase2 = []
# run romSpline to convert into reduced order spline, then assign final .h5 values
# and write all data to .h5 file
# handled independently for Magnitude/Argument vs. Pluss/Cross data, based on
# unique needs for each format
if (dataFormat == "MagArg"):
def get_waveform(approximant,
phase_order,
amplitude_order,
spin_order,
template_params,
start_frequency,
sample_rate,
length,
datafile=None,
verbose=False):
delta_t = 1. / sample_rate
delta_f = 1. / length
filter_N = int(length)
filter_n = filter_N / 2 + 1
if approximant in waveform.fd_approximants(
) and 'Eccentric' not in approximant:
print("NORMAL FD WAVEFORM for", approximant)
delta_f = sample_rate / length
hplus, hcross = waveform.get_fd_waveform(
template_params,
approximant=approximant,
spin_order=spin_order,
phase_order=phase_order,
delta_f=delta_f,
f_lower=start_frequency,
amplitude_order=amplitude_order)
elif approximant in waveform.td_approximants(
) and 'Eccentric' not in approximant:
print("NORMAL TD WAVEFORM for", approximant)
hplus, hcross = waveform.get_td_waveform(
template_params,
approximant=approximant,
spin_order=spin_order,
phase_order=phase_order,
delta_t=1.0 / sample_rate,
f_lower=start_frequency,
amplitude_order=amplitude_order)
elif 'EccentricIMR' in approximant:
# {{{
# Legacy support
import sys
sys.path.append('/home/kuma/grav/kuma/src/Eccentric_IMR/Codes/Python/')
import EccentricIMR as Ecc
try:
mass1 = getattr(template_params, 'mass1')
mass2 = getattr(template_params, 'mass2')
except:
raise RuntimeError("template_params does not have mass1 or mass2!")
try:
ecc = getattr(template_params, 'alpha1')
if 'E0' in approximant:
ecc = 0
anom = getattr(template_params, 'alpha2')
inc = getattr(template_params, 'inclination')
rtrans = getattr(template_params, 'alpha')
beta = 0
except:
raise RuntimeError(
"template_params does not have alpha{,1,2} or inclination")
tol = 1.e-16
fmin = start_frequency
sample_rate = sample_rate
#
print(" Using phase order: %d" % phase_order, file=sys.stdout)
sys.stdout.flush()
hplus, hcross = Ecc.generate_eccentric_waveform(
mass1,
mass2,
ecc,
anom,
inc,
beta,
tol,
r_transition=rtrans,
phase_order=phase_order,
fmin=fmin,
sample_rate=sample_rate,
inspiral_only=False)
# }}}
elif 'EccentricInspiral' in approximant:
# {{{
# Legacy support
import sys
sys.path.append('/home/kuma/grav/kuma/src/Eccentric_IMR/Codes/Python/')
import EccentricIMR as Ecc
try:
mass1 = getattr(template_params, 'mass1')
mass2 = getattr(template_params, 'mass2')
except:
raise RuntimeError("template_params does not have mass1 or mass2!")
try:
ecc = getattr(template_params, 'alpha1')
if 'E0' in approximant:
ecc = 0
anom = getattr(template_params, 'alpha2')
inc = getattr(template_params, 'inclination')
beta = getattr(template_params, 'alpha')
except:
raise RuntimeError(
"template_params does not have alpha{,1,2} or inclination")
tol = 1.e-16
fmin = start_frequency
sample_rate = sample_rate
#
hplus, hcross = Ecc.generate_eccentric_waveform(
mass1,
mass2,
ecc,
anom,
inc,
beta,
tol,
phase_order=phase_order,
fmin=fmin,
sample_rate=sample_rate,
inspiral_only=True)
# }}}
elif 'EccentricFD' in approximant:
# {{{
# Legacy support
import lalsimulation as ls
import lal
delta_f = sample_rate / length
try:
mass1 = getattr(template_params, 'mass1')
mass2 = getattr(template_params, 'mass2')
except:
raise RuntimeError("template_params does not have mass1 or mass2!")
try:
ecc = getattr(template_params, 'alpha1')
if 'E0' in approximant:
ecc = 0
anom = getattr(template_params, 'alpha2')
inc = getattr(template_params, 'inclination')
except:
raise RuntimeError(
"template_params does not have alpha{1,2} or inclination")
eccPar = ls.SimInspiralCreateTestGRParam("inclination_azimuth", inc)
ls.SimInspiralAddTestGRParam(eccPar, "e_min", ecc)
fmin = start_frequency
fmax = sample_rate / 2
#
thp, thc = ls.SimInspiralChooseFDWaveform(
0, delta_f, mass1 * lal.MSUN_SI, mass2 * lal.MSUN_SI, 0, 0, 0, 0,
0, 0, fmin, fmax, 0, 1.e6 * lal.PC_SI, inc, 0, 0, None, eccPar, -1,
7, ls.EccentricFD)
hplus = types.FrequencySeries(thp.data.data[:],
delta_f=thp.deltaF,
epoch=thp.epoch)
hcross = types.FrequencySeries(thc.data.data[:],
delta_f=thc.deltaF,
epoch=thc.epoch)
# }}}
elif 'FromDataFile' in approximant:
# {{{
# Legacy support
if not os.path.exists(datafile):
raise IOError("File %s not found!" % datafile)
if verbose:
print("Reading from data file %s" % datafile)
# Figure out waveform parameters from filename
#q_value, M_value, w_value, _, _ = EA.get_q_m_e_pn_o_from_filename(datafile)
q_value, M_value, w_value = EA.get_q_m_e_from_filename(datafile)
# Read data, down-sample (assume data file is more finely sampled than
# needed, i.e. interpolation is NOT supported, nor will be)
data = np.loadtxt(datafile)
dt = data[1, 0] - data[0, 0]
delta_t = 1. / sample_rate
downsample_ratio = delta_t / dt
if not approx_equal(downsample_ratio, np.int(downsample_ratio)):
raise RuntimeError(
"Cannot handling resampling at a fractional factor = %e" %
downsample_ratio)
elif verbose:
print("Downsampling by a factor of %d" % int(downsample_ratio))
h_real = types.TimeSeries(data[::int(downsample_ratio), 1] /
DYN_RANGE_FAC,
delta_t=delta_t)
h_imag = types.TimeSeries(data[::int(downsample_ratio), 2] /
DYN_RANGE_FAC,
delta_t=delta_t)
if verbose:
print("max, min,len of h_real = ", max(h_real.data),
min(h_real.data), len(h_real.data))
# Compute Strain
tmplt_pars = template_params
wav = generate_detector_strain(tmplt_pars, h_real, h_imag)
wav = extend_waveform_TimeSeries(wav, filter_N)
# Return TimeSeries with (m1, m2, w_value)
m1, m2 = mtotal_eta_to_mass1_mass2(M_value,
q_value / (1. + q_value)**2)
htilde = make_frequency_series(wav)
htilde = extend_waveform_FrequencySeries(htilde, filter_n)
if verbose:
print("ISNAN(htilde from file) = ", np.any(np.isnan(htilde.data)))
return htilde, [m1, m2, w_value, dt]
# }}}
else:
raise IOError(".. APPROXIMANT %s not found.." % approximant)
##
hvec = hplus
htilde = make_frequency_series(hvec)
htilde = extend_waveform_FrequencySeries(htilde, filter_n)
if any(isnan(hplus.data)) or any(isnan(hcross.data)):
print("..### %s hplus or hcross have NANS!!" % approximant)
if any(isinf(hplus.data)) or any(isinf(hcross.data)):
print("..### %s hplus or hcross have INFS!!" % approximant)
if any(isnan(htilde.data)):
print("..### %s Fourier transform htilde has NANS!!" % approximant)
if any(isinf(htilde.data)):
print("..### %s Fourier transform htilde has INFS!!" % approximant)
return htilde
def excess_power(
ts_data, # Time series from magnetic field data
band=None, # Channel bandwidth
channel_name='channel-name', # Channel name
fmin=0, # Lowest frequency of the filter bank.
fmax=None, # Highest frequency of the filter bank.
impulse=False, # Impulse response
make_plot=True, # Condition to produce plots
max_duration=None, # Maximum duration of the tile
nchans=256, # Total number of channels
psd_estimation='median-mean', # Average method
psd_segment_length=60, # Length of each segment in seconds
psd_segment_stride=30, # Separation between 2 consecutive segments in seconds
station='station-name', # Station name
tile_fap=1e-7, # Tile false alarm probability threshold in Gaussian noise.
verbose=True, # Print details
window_fraction=0, # Withening window fraction
wtype='tukey'): # Whitening type, can tukey or hann
'''
Perform excess-power search analysis on magnetic field data.
This method will produce a bunch of time-frequency plots for every
tile duration and bandwidth analysed as well as a XML file identifying
all the triggers found in the selected data within the user-defined
time range.
Parameters
----------
ts_data : TimeSeries
Time Series from magnetic field data
psd_segment_length : float
Length of each segment in seconds
psd_segment_stride : float
Separation between 2 consecutive segments in seconds
psd_estimation : string
Average method
window_fraction : float
Withening window fraction
tile_fap : float
Tile false alarm probability threshold in Gaussian noise.
nchans : int
Total number of channels
band : float
Channel bandwidth
fmin : float
Lowest frequency of the filter bank.
fmax : float
Highest frequency of the filter bank
Examples
--------
The program can be ran as an executable by using the ``excesspower`` command
line as follows::
excesspower --station "mainz01" \\
--start-time "2017-04-15-17-1" \\
--end-time "2017-04-15-18" \\
--rep "/Users/vincent/ASTRO/data/GNOME/GNOMEDrive/gnome/serverdata/" \\
--resample 512 \\
--verbose
'''
# Determine sampling rate based on extracted time series
sample_rate = ts_data.sample_rate
# Check if tile maximum frequency is not defined
if fmax is None or fmax > sample_rate / 2.:
# Set the tile maximum frequency equal to the Nyquist frequency
# (i.e. half the sampling rate)
fmax = sample_rate / 2.0
# Check whether or not tile bandwidth and channel are defined
if band is None and nchans is None:
# Exit program with error message
exit("Either bandwidth or number of channels must be specified...")
else:
# Check if tile maximum frequency larger than its minimum frequency
assert fmax >= fmin
# Define spectral band of data
data_band = fmax - fmin
# Check whether tile bandwidth or channel is defined
if band is not None:
# Define number of possible filter bands
nchans = int(data_band / band)
elif nchans is not None:
# Define filter bandwidth
band = data_band / nchans
nchans -= 1
# Check if number of channels is superior than unity
assert nchans > 1
# Print segment information
if verbose: print '|- Estimating PSD from segments of',
if verbose:
print '%.2f s, with %.2f s stride...' % (psd_segment_length,
psd_segment_stride)
# Convert time series as array of float
data = ts_data.astype(numpy.float64)
# Define segment length for PSD estimation in sample unit
seg_len = int(psd_segment_length * sample_rate)
# Define separation between consecutive segments in sample unit
seg_stride = int(psd_segment_stride * sample_rate)
# Minimum frequency of detectable signal in a segment
delta_f = 1. / psd_segment_length
# Calculate PSD length counting the zero frequency element
fd_len = fmax / delta_f + 1
# Calculate the overall PSD from individual PSD segments
if impulse:
# Produce flat data
flat_data = numpy.ones(int(fd_len)) * 2. / fd_len
# Create PSD frequency series
fd_psd = types.FrequencySeries(flat_data, 1. / psd_segment_length,
ts_data.start_time)
else:
# Create overall PSD using Welch's method
fd_psd = psd.welch(data,
avg_method=psd_estimation,
seg_len=seg_len,
seg_stride=seg_stride)
if make_plot:
# Plot the power spectral density
plot_spectrum(fd_psd)
# We need this for the SWIG functions
lal_psd = fd_psd.lal()
# Create whitening window
if verbose: print "|- Whitening window and spectral correlation..."
if wtype == 'hann': window = lal.CreateHannREAL8Window(seg_len)
elif wtype == 'tukey':
window = lal.CreateTukeyREAL8Window(seg_len, window_fraction)
else:
raise ValueError("Can't handle window type %s" % wtype)
# Create FFT plan
fft_plan = lal.CreateForwardREAL8FFTPlan(len(window.data.data), 1)
# Perform two point spectral correlation
spec_corr = lal.REAL8WindowTwoPointSpectralCorrelation(window, fft_plan)
# Determine length of individual filters
filter_length = int(2 * band / fd_psd.delta_f) + 1
# Initialise filter bank
if verbose:
print "|- Create bank of %i filters of %i Hz bandwidth..." % (
nchans, filter_length)
# Initialise array to store filter's frequency series and metadata
lal_filters = []
# Initialise array to store filter's time series
fdb = []
# Loop over the channels
for i in range(nchans):
# Define central position of the filter
freq = fmin + band / 2 + i * band
# Create excess power filter
lal_filter = lalburst.CreateExcessPowerFilter(freq, band, lal_psd,
spec_corr)
# Testing spectral correlation on filter
#print lalburst.ExcessPowerFilterInnerProduct(lal_filter, lal_filter, spec_corr, None)
# Append entire filter structure
lal_filters.append(lal_filter)
# Append filter's spectrum
fdb.append(FrequencySeries.from_lal(lal_filter))
#print fdb[0].frequencies
#print fdb[0]
if make_plot:
# Plot filter bank
plot_bank(fdb)
# Convert filter bank from frequency to time domain
if verbose:
print "|- Convert all the frequency domain to the time domain..."
tdb = []
# Loop for each filter's spectrum
for fdt in fdb:
zero_padded = numpy.zeros(int((fdt.f0 / fdt.df).value) + len(fdt))
st = int((fdt.f0 / fdt.df).value)
zero_padded[st:st + len(fdt)] = numpy.real_if_close(fdt.value)
n_freq = int(sample_rate / 2 / fdt.df.value) * 2
tdt = numpy.fft.irfft(zero_padded, n_freq) * math.sqrt(sample_rate)
tdt = numpy.roll(tdt, len(tdt) / 2)
tdt = TimeSeries(tdt,
name="",
epoch=fdt.epoch,
sample_rate=sample_rate)
tdb.append(tdt)
# Plot time series filter
plot_filters(tdb, fmin, band)
# Computer whitened inner products of input filters with themselves
#white_filter_ip = numpy.array([lalburst.ExcessPowerFilterInnerProduct(f, f, spec_corr, None) for f in lal_filters])
# Computer unwhitened inner products of input filters with themselves
#unwhite_filter_ip = numpy.array([lalburst.ExcessPowerFilterInnerProduct(f, f, spec_corr, lal_psd) for f in lal_filters])
# Computer whitened filter inner products between input adjacent filters
#white_ss_ip = numpy.array([lalburst.ExcessPowerFilterInnerProduct(f1, f2, spec_corr, None) for f1, f2 in zip(lal_filters[:-1], lal_filters[1:])])
# Computer unwhitened filter inner products between input adjacent filters
#unwhite_ss_ip = numpy.array([lalburst.ExcessPowerFilterInnerProduct(f1, f2, spec_corr, lal_psd) for f1, f2 in zip(lal_filters[:-1], lal_filters[1:])])
# Check filter's bandwidth is equal to user defined channel bandwidth
min_band = (len(lal_filters[0].data.data) - 1) * lal_filters[0].deltaF / 2
assert min_band == band
# Create an event list where all the triggers will be stored
event_list = lsctables.New(lsctables.SnglBurstTable, [
'start_time', 'start_time_ns', 'peak_time', 'peak_time_ns', 'duration',
'bandwidth', 'central_freq', 'chisq_dof', 'confidence', 'snr',
'amplitude', 'channel', 'ifo', 'process_id', 'event_id', 'search',
'stop_time', 'stop_time_ns'
])
# Create repositories to save TF and time series plots
os.system('mkdir -p segments/time-frequency')
os.system('mkdir -p segments/time-series')
# Define time edges
t_idx_min, t_idx_max = 0, seg_len
# Loop over each segment
while t_idx_max <= len(ts_data):
# Define first and last timestamps of the block
start_time = ts_data.start_time + t_idx_min / float(
ts_data.sample_rate)
end_time = ts_data.start_time + t_idx_max / float(ts_data.sample_rate)
if verbose:
print "\n|- Analyzing block %i to %i (%.2f percent)" % (
start_time, end_time, 100 * float(t_idx_max) / len(ts_data))
# Debug for impulse response
if impulse:
for i in range(t_idx_min, t_idx_max):
ts_data[i] = 1000. if i == (t_idx_max + t_idx_min) / 2 else 0.
# Model a withen time series for the block
tmp_ts_data = types.TimeSeries(ts_data[t_idx_min:t_idx_max] *
window.data.data,
delta_t=1. / ts_data.sample_rate,
epoch=start_time)
# Save time series in relevant repository
os.system('mkdir -p segments/%i-%i' % (start_time, end_time))
if make_plot:
# Plot time series
plot_ts(tmp_ts_data,
fname='segments/time-series/%i-%i.png' %
(start_time, end_time))
# Convert times series to frequency series
fs_data = tmp_ts_data.to_frequencyseries()
if verbose:
print "|- Frequency series data has variance: %s" % fs_data.data.std(
)**2
# Whitening (FIXME: Whiten the filters, not the data)
fs_data.data /= numpy.sqrt(fd_psd) / numpy.sqrt(2 * fd_psd.delta_f)
if verbose:
print "|- Whitened frequency series data has variance: %s" % fs_data.data.std(
)**2
if verbose: print "|- Create time-frequency plane for current block"
# Return the complex snr, along with its associated normalization of the template,
# matched filtered against the data
#filter.matched_filter_core(types.FrequencySeries(tmp_filter_bank,delta_f=fd_psd.delta_f),
# fs_data,h_norm=1,psd=fd_psd,low_frequency_cutoff=lal_filters[0].f0,
# high_frequency_cutoff=lal_filters[0].f0+2*band)
if verbose: print "|- Filtering all %d channels...\n" % nchans,
# Initialise 2D zero array
tmp_filter_bank = numpy.zeros(len(fd_psd), dtype=numpy.complex128)
# Initialise 2D zero array for time-frequency map
tf_map = numpy.zeros((nchans, seg_len), dtype=numpy.complex128)
# Loop over all the channels
for i in range(nchans):
# Reset filter bank series
tmp_filter_bank *= 0.0
# Index of starting frequency
f1 = int(lal_filters[i].f0 / fd_psd.delta_f)
# Index of last frequency bin
f2 = int((lal_filters[i].f0 + 2 * band) / fd_psd.delta_f) + 1
# (FIXME: Why is there a factor of 2 here?)
tmp_filter_bank[f1:f2] = lal_filters[i].data.data * 2
# Define the template to filter the frequency series with
template = types.FrequencySeries(tmp_filter_bank,
delta_f=fd_psd.delta_f,
copy=False)
# Create filtered series
filtered_series = filter.matched_filter_core(
template,
fs_data,
h_norm=None,
psd=None,
low_frequency_cutoff=lal_filters[i].f0,
high_frequency_cutoff=lal_filters[i].f0 + 2 * band)
# Include filtered series in the map
tf_map[i, :] = filtered_series[0].numpy()
if make_plot:
# Plot spectrogram
plot_spectrogram(numpy.abs(tf_map).T,
dt=tmp_ts_data.delta_t,
df=band,
ymax=ts_data.sample_rate / 2.,
t0=start_time,
t1=end_time,
fname='segments/time-frequency/%i-%i.png' %
(start_time, end_time))
plot_tiles_ts(numpy.abs(tf_map),
2,
1,
sample_rate=ts_data.sample_rate,
t0=start_time,
t1=end_time,
fname='segments/%i-%i/ts.png' %
(start_time, end_time))
#plot_tiles_tf(numpy.abs(tf_map),2,1,ymax=ts_data.sample_rate/2,
# sample_rate=ts_data.sample_rate,t0=start_time,t1=end_time,
# fname='segments/%i-%i/tf.png'%(start_time,end_time))
# Loop through powers of 2 up to number of channels
for nc_sum in range(0, int(math.log(nchans, 2)))[::-1]:
# Calculate total number of summed channels
nc_sum = 2**nc_sum
if verbose:
print "\n\t|- Contructing tiles containing %d narrow band channels" % nc_sum
# Compute full bandwidth of virtual channel
df = band * nc_sum
# Compute minimal signal's duration in virtual channel
dt = 1.0 / (2 * df)
# Compute under sampling rate
us_rate = int(round(dt / ts_data.delta_t))
if verbose:
print "\t|- Undersampling rate for this level: %f" % (
ts_data.sample_rate / us_rate)
if verbose: print "\t|- Calculating tiles..."
# Clip the boundaries to remove window corruption
clip_samples = int(psd_segment_length * window_fraction *
ts_data.sample_rate / 2)
# Undersample narrow band channel's time series
# Apply clipping condition because [0:-0] does not give the full array
tf_map_temp = tf_map[:,clip_samples:-clip_samples:us_rate] \
if clip_samples > 0 else tf_map[:,::us_rate]
# Initialise final tile time-frequency map
tiles = numpy.zeros(((nchans + 1) / nc_sum, tf_map_temp.shape[1]))
# Loop over tile index
for i in xrange(len(tiles)):
# Sum all inner narrow band channels
ts_tile = numpy.absolute(tf_map_temp[nc_sum * i:nc_sum *
(i + 1)].sum(axis=0))
# Define index of last narrow band channel for given tile
n = (i + 1) * nc_sum - 1
n = n - 1 if n == len(lal_filters) else n
# Computer withened inner products of each input filter with itself
mu_sq = nc_sum * lalburst.ExcessPowerFilterInnerProduct(
lal_filters[n], lal_filters[n], spec_corr, None)
#kmax = nc_sum-1 if n==len(lal_filters) else nc_sum-2
# Loop over the inner narrow band channels
for k in xrange(0, nc_sum - 1):
# Computer whitened filter inner products between input adjacent filters
mu_sq += 2 * lalburst.ExcessPowerFilterInnerProduct(
lal_filters[n - k], lal_filters[n - 1 - k], spec_corr,
None)
# Normalise tile's time series
tiles[i] = ts_tile.real**2 / mu_sq
if verbose: print "\t|- TF-plane is %dx%s samples" % tiles.shape
if verbose:
print "\t|- Tile energy mean %f, var %f" % (numpy.mean(tiles),
numpy.var(tiles))
# Define maximum number of degrees of freedom and check it larger or equal to 2
max_dof = 32 if max_duration == None else int(max_duration / dt)
assert max_dof >= 2
# Loop through multiple degrees of freedom
for j in [2**l for l in xrange(0, int(math.log(max_dof, 2)))]:
# Duration is fixed by the NDOF and bandwidth
duration = j * dt
if verbose: print "\n\t\t|- Summing DOF = %d ..." % (2 * j)
if verbose:
print "\t\t|- Explore signal duration of %f s..." % duration
# Construct filter
sum_filter = numpy.array([1, 0] * (j - 1) + [1])
# Calculate length of filtered time series
tlen = tiles.shape[1] - sum_filter.shape[0] + 1
# Initialise filtered time series array
dof_tiles = numpy.zeros((tiles.shape[0], tlen))
# Loop over tiles
for f in range(tiles.shape[0]):
# Sum and drop correlate tiles
dof_tiles[f] = fftconvolve(tiles[f], sum_filter, 'valid')
if verbose:
print "\t\t|- Summed tile energy mean: %f" % (
numpy.mean(dof_tiles))
if verbose:
print "\t\t|- Variance tile energy: %f" % (
numpy.var(dof_tiles))
if make_plot:
plot_spectrogram(
dof_tiles.T,
dt,
df,
ymax=ts_data.sample_rate / 2,
t0=start_time,
t1=end_time,
fname='segments/%i-%i/%02ichans_%02idof.png' %
(start_time, end_time, nc_sum, 2 * j))
plot_tiles_ts(
dof_tiles,
2 * j,
df,
sample_rate=ts_data.sample_rate / us_rate,
t0=start_time,
t1=end_time,
fname='segments/%i-%i/%02ichans_%02idof_ts.png' %
(start_time, end_time, nc_sum, 2 * j))
plot_tiles_tf(
dof_tiles,
2 * j,
df,
ymax=ts_data.sample_rate / 2,
sample_rate=ts_data.sample_rate / us_rate,
t0=start_time,
t1=end_time,
fname='segments/%i-%i/%02ichans_%02idof_tf.png' %
(start_time, end_time, nc_sum, 2 * j))
threshold = scipy.stats.chi2.isf(tile_fap, j)
if verbose:
print "\t\t|- Threshold for this level: %f" % threshold
spant, spanf = dof_tiles.shape[1] * dt, dof_tiles.shape[0] * df
if verbose:
print "\t\t|- Processing %.2fx%.2f time-frequency map." % (
spant, spanf)
# Since we clip the data, the start time needs to be adjusted accordingly
window_offset_epoch = fs_data.epoch + psd_segment_length * window_fraction / 2
window_offset_epoch = LIGOTimeGPS(float(window_offset_epoch))
for i, j in zip(*numpy.where(dof_tiles > threshold)):
event = event_list.RowType()
# The points are summed forward in time and thus a `summed point' is the
# sum of the previous N points. If this point is above threshold, it
# corresponds to a tile which spans the previous N points. However, the
# 0th point (due to the convolution specifier 'valid') is actually
# already a duration from the start time. All of this means, the +
# duration and the - duration cancels, and the tile 'start' is, by
# definition, the start of the time frequency map if j = 0
# FIXME: I think this needs a + dt/2 to center the tile properly
event.set_start(window_offset_epoch + float(j * dt))
event.set_stop(window_offset_epoch + float(j * dt) +
duration)
event.set_peak(event.get_start() + duration / 2)
event.central_freq = lal_filters[
0].f0 + band / 2 + i * df + 0.5 * df
event.duration = duration
event.bandwidth = df
event.chisq_dof = 2 * duration * df
event.snr = math.sqrt(dof_tiles[i, j] / event.chisq_dof -
1)
# FIXME: Magic number 0.62 should be determine empircally
event.confidence = -lal.LogChisqCCDF(
event.snr * 0.62, event.chisq_dof * 0.62)
event.amplitude = None
event.process_id = None
event.event_id = event_list.get_next_id()
event_list.append(event)
for event in event_list[::-1]:
if event.amplitude != None:
continue
etime_min_idx = float(event.get_start()) - float(
fs_data.epoch)
etime_min_idx = int(etime_min_idx / tmp_ts_data.delta_t)
etime_max_idx = float(event.get_start()) - float(
fs_data.epoch) + event.duration
etime_max_idx = int(etime_max_idx / tmp_ts_data.delta_t)
# (band / 2) to account for sin^2 wings from finest filters
flow_idx = int((event.central_freq - event.bandwidth / 2 -
(df / 2) - fmin) / df)
fhigh_idx = int((event.central_freq + event.bandwidth / 2 +
(df / 2) - fmin) / df)
# TODO: Check that the undersampling rate is always commensurate
# with the indexing: that is to say that
# mod(etime_min_idx, us_rate) == 0 always
z_j_b = tf_map[flow_idx:fhigh_idx,
etime_min_idx:etime_max_idx:us_rate]
# FIXME: Deal with negative hrss^2 -- e.g. remove the event
try:
event.amplitude = measure_hrss(
z_j_b, unwhite_filter_ip[flow_idx:fhigh_idx],
unwhite_ss_ip[flow_idx:fhigh_idx - 1],
white_ss_ip[flow_idx:fhigh_idx - 1],
fd_psd.delta_f, tmp_ts_data.delta_t,
len(lal_filters[0].data.data), event.chisq_dof)
except ValueError:
event.amplitude = 0
if verbose:
print "\t\t|- Total number of events: %d" % len(event_list)
t_idx_min += int(seg_len * (1 - window_fraction))
t_idx_max += int(seg_len * (1 - window_fraction))
setname = "MagneticFields"
__program__ = 'pyburst_excesspower_gnome'
start_time = LIGOTimeGPS(int(ts_data.start_time))
end_time = LIGOTimeGPS(int(ts_data.end_time))
inseg = segment(start_time, end_time)
xmldoc = ligolw.Document()
xmldoc.appendChild(ligolw.LIGO_LW())
ifo = channel_name.split(":")[0]
straindict = psd.insert_psd_option_group.__dict__
proc_row = register_to_xmldoc(xmldoc,
__program__,
straindict,
ifos=[ifo],
version=git_version.id,
cvs_repository=git_version.branch,
cvs_entry_time=git_version.date)
dt_stride = psd_segment_length
sample_rate = ts_data.sample_rate
# Amount to overlap successive blocks so as not to lose data
window_overlap_samples = window_fraction * sample_rate
outseg = inseg.contract(window_fraction * dt_stride / 2)
# With a given dt_stride, we cannot process the remainder of this data
remainder = math.fmod(abs(outseg), dt_stride * (1 - window_fraction))
# ...so make an accounting of it
outseg = segment(outseg[0], outseg[1] - remainder)
ss = append_search_summary(xmldoc,
proc_row,
ifos=(station, ),
inseg=inseg,
outseg=outseg)
for sb in event_list:
sb.process_id = proc_row.process_id
sb.search = proc_row.program
sb.ifo, sb.channel = station, setname
xmldoc.childNodes[0].appendChild(event_list)
ifostr = ifo if isinstance(ifo, str) else "".join(ifo)
st_rnd, end_rnd = int(math.floor(inseg[0])), int(math.ceil(inseg[1]))
dur = end_rnd - st_rnd
fname = "%s-excesspower-%d-%d.xml.gz" % (ifostr, st_rnd, dur)
utils.write_filename(xmldoc, fname, gz=fname.endswith("gz"))
plot_triggers(fname)
import pylab
from pycbc import types, fft, waveform
# Get a time domain waveform
hp, hc = waveform.get_td_waveform(approximant="EOBNRv2",
mass1=6,
mass2=6,
delta_t=1.0 / 4096,
f_lower=40)
# Get a frequency domain waveform
sptilde, sctilde = waveform.get_fd_waveform(approximant="TaylorF2",
mass1=6,
mass2=6,
delta_f=1.0 / 4,
f_lower=40)
# FFT it to the time-domain
tlen = 1.0 / hp.delta_t / sptilde.delta_f
sptilde.resize(tlen / 2 + 1)
sp = types.TimeSeries(types.zeros(tlen), delta_t=hp.delta_t)
fft.ifft(sptilde, sp)
pylab.plot(sp.sample_times, sp, label="TaylorF2 (IFFT)")
pylab.plot(hp.sample_times, hp, label="EOBNRv2")
pylab.ylabel('Strain')
pylab.xlabel('Time (s)')
pylab.legend()
pylab.show()
def get_data(station,start_time,end_time,rep='/GNOMEDrive/gnome/serverdata/',
resample=None,activity=False,unit='V',output='all',segtxt=False,
channel='MagneticFields'):
"""
Glob all files withing user-defined period and extract data.
Parameters
----------
station : str
Name of the station to be analysed
start_time : int
GPS timestamp of the first required magnetic field data
end_time : int
GPS timestamp of the last required magnetic field data
rep : str
Data repository. Default is the GNOME server repository.
resample : int
New sampling rate
activity : bool
Output the activity of data
unit : str
Output unit format (V for voltage, pT for magnetic field)
output : str
Output data to be extracted. If output is equal to 'ts',
only the time series will be given.
Returns
-------
ts_data : pycbc.types.TimeSeries
Time series data for selected time period.
ts_list : dictionary
List of time series.
activity : gwpy.segments.DataQualityDict
List all the segment of data retrieved
t0 : astropy.time.Time
First timestamp
t1 : astropy.time.Time
Last timestamp
"""
if start_time==None or end_time==None:
print "ERROR: No start or end date given..."
quit()
# Define data attribute to be extracted from HDF5 files
setname = channel
dstr = ['%Y','%m','%d','%H','%M','%S','%f']
dsplit = '-'.join(dstr[:start_time.count('-')+1])
start = datetime.strptime(start_time,dsplit)
dsplit = '-'.join(dstr[:end_time.count('-')+1])
end = datetime.strptime(end_time,dsplit)
dataset = []
for date in numpy.arange(start,end,timedelta(minutes=1)):
date = date.astype(datetime)
path1 = rep+station+'/'+date.strftime("%Y/%m/%d/")
path2 = station+'_'+date.strftime("%Y%m%d_%H%M*.hdf5")
fullpath = os.path.join(path1,path2)
dataset += glob.glob(fullpath)
if len(dataset)==0:
print "ERROR: No data files were found..."
quit()
file_order,data_order = {},{}
for fname in dataset:
hfile = h5py.File(fname, "r")
# Extract all atributes from the data
attrs = hfile[setname].attrs
# Define each attribute
dstr, t0, t1 = attrs["Date"], attrs["t0"], attrs["t1"]
# Construct GPS starting time from data
start_utc = construct_utc_from_metadata(dstr, t0)
# Construct GPS ending time from data
end_utc = construct_utc_from_metadata(dstr, t1)
# Represent the range of times in the semi-open interval
segfile = segment(start_utc,end_utc)
file_order[segfile] = fname
data_order[segfile] = hfile
# Create list of time series from every segment
ts_list = TimeSeriesList()
for seg in sorted(file_order):
hfile = h5py.File(file_order[seg], "r")
dset = hfile[setname]
sample_rate = dset.attrs["SamplingRate(Hz)"]
gps_epoch = construct_utc_from_metadata(dset.attrs["Date"], dset.attrs["t0"])
data = dset[:]
if unit=='pT':
data = eval(dset.attrs['MagFieldEq'].replace('MagneticFields','data').replace('[pT]',''))
ts_data = TimeSeries(data, sample_rate=sample_rate, epoch=gps_epoch)
ts_list.append(ts_data)
hfile.close()
# Generate an ASCII representation of the GPS timestamped segments of time covered by the input data
seglist = segmentlist(data_order.keys())
# Sort the segment list
seglist.sort()
# Initialise dictionary for segment information
activity = DataQualityDict()
if segtxt:
# Save time span for each segment in ASCII file
with open("segments.txt", "w") as fout:
for seg in seglist:
print >>fout, "%10.9f %10.9f" % seg
# FIXME: Active should be masked from the sanity channel
activity[station] = DataQualityFlag(station,active=seglist.coalesce(),known=seglist.coalesce())
# Generate an ASCII representation of the GPS timestamped segments of time covered by the input data
seglist = segmentlist(data_order.keys())
# Sort the segment list
seglist.sort()
# Retrieve channel data for all the segments
if unit=='V':
full_data = numpy.hstack([data_order[seg][setname][:] for seg in seglist])
if unit=='pT':
full_data = []
for seg in seglist:
dset = data_order[seg][setname]
data = dset[:]
data = eval(dset.attrs['MagFieldEq'].replace('MagneticFields','data').replace('[pT]',''))
full_data = numpy.hstack((full_data,data))
for v in data_order.values():
v.close()
new_sample_rate = float(sample_rate) if resample==None else float(resample)
new_data_length = len(full_data)*new_sample_rate/float(sample_rate)
full_data = scipy.signal.resample(full_data,int(new_data_length))
# Models a time series consisting of uniformly sampled scalar values
ts_data = types.TimeSeries(full_data,delta_t=1./new_sample_rate,epoch=seglist[0][0])
if output=='ts':
return ts_data
t0,t1 = time_convert(start_time,end_time)
return ts_data,ts_list,activity,t0,t1
