def get_range_spectrogram(channel, segments, config=None, cache=None,
query=True, nds=None, return_=True, nproc=1,
datafind_error='raise', frametype=None, stride=60,
fftlength=None, overlap=None, method=None,
**rangekwargs):
"""Estimate the spectral contribution to sensitive distance for a given
strain channel
"""
channel, key = _metadata(channel, **rangekwargs)
# get new segments
havesegs = globalv.SPECTROGRAMS.get(key, SpectrogramList()).segments
new, query = _segments_diff(segments, havesegs, query)
if query: # calculate new data
spectrograms = get_spectrogram(
channel, new, config=config, cache=cache, nds=nds, format='psd',
frametype=frametype, nproc=nproc, datafind_error=datafind_error,
stride=stride, fftlength=fftlength, overlap=overlap, method=method)
for sg in spectrograms: # get contribution from each spectrogram
outspec = astro.range_spectrogram(sg, **rangekwargs)
outspec.channel = key
add_spectrogram(outspec if 'energy' in rangekwargs else
outspec**(1/2.), key=key)
if return_:
return globalv.SPECTROGRAMS.get(key, SpectrogramList())
def get_spectrogram(channel, segments, config=ConfigParser(), cache=None,
query=True, nds='guess', format='power', return_=True,
frametype=None, multiprocess=True, datafind_error='raise',
**fftparams):
"""Retrieve the time-series and generate a spectrogram of the given
channel
"""
channel = get_channel(channel)
# read data for all sub-channels
specs = []
channels = re_channel.findall(channel.ndsname)
for c in channels:
specs.append(_get_spectrogram(c, segments, config=config, cache=cache,
query=query, nds=nds, format=format,
return_=return_, frametype=frametype,
multiprocess=multiprocess,
datafind_error=datafind_error,
**fftparams))
if return_ and len(channels) == 1:
return specs[0]
elif return_:
# get union of segments for all sub-channels
datasegs = reduce(operator.and_, [sgl.segments for sgl in specs])
# build meta-spectrogram for all interseceted segments
out = SpectrogramList()
operators = [channel.name[m.span()[1]] for m in
list(re_channel.finditer(channel.ndsname))[:-1]]
for seg in datasegs:
sg = _get_spectrogram(channels[0], SegmentList([seg]),
config=config, query=False, format=format,
return_=True)[0]
sg.name = str(channel)
for op, ch in zip(operators, channels[1:]):
try:
op = OPERATOR[op]
except KeyError as e:
e.args = ('Cannot parse math operator %r' % op,)
raise
data = _get_spectrogram(ch, SegmentList([seg]),
config=config, query=False,
format=format, return_=True)
try:
sg = op(sg, data[0])
except ValueError as e:
if 'could not be broadcast together' in str(e):
s = min(sg.shape[0], data[0].shape[0])
sg = op(sg[:s], data[0][:s])
else:
raise
out.append(sg)
return out
def get_spectrograms(channels,
segments,
config=None,
cache=None,
query=True,
nds=None,
format='power',
return_=True,
frametype=None,
multiprocess=True,
datafind_error='raise',
**fftparams):
"""Get spectrograms for multiple channels
"""
channels = map(get_channel, channels)
# get timeseries data in bulk
if query:
qchannels = map(
get_channel,
set([
c for group in map(split_channel_combination, channels)
for c in group
]))
keys = []
for channel in qchannels:
fftparams_ = get_fftparams(channel, **fftparams)
keys.append(make_globalv_key(channel, fftparams_))
havesegs = reduce(
operator.and_,
(globalv.SPECTROGRAMS.get(key, SpectrogramList()).segments
for key in keys))
new = segments - havesegs
strides = set([getattr(c, 'stride', 0) for c in qchannels])
if len(strides) == 1:
stride = strides.pop()
new = type(new)([s for s in new if abs(s) >= stride])
get_timeseries_dict(qchannels,
new,
config=config,
cache=cache,
multiprocess=multiprocess,
frametype=frametype,
datafind_error=datafind_error,
nds=nds,
return_=False)
# loop over channels and generate spectrograms
out = OrderedDict()
for channel in channels:
out[channel] = get_spectrogram(channel,
segments,
config=config,
cache=cache,
query=query,
nds=nds,
format=format,
multiprocess=multiprocess,
return_=return_,
datafind_error=datafind_error,
**fftparams)
return out
def add_spectrogram(specgram, key=None, coalesce=True):
"""Add a `Spectrogram` to the global memory cache
"""
if key is None:
key = specgram.name or str(specgram.channel)
globalv.SPECTROGRAMS.setdefault(key, SpectrogramList())
globalv.SPECTROGRAMS[key].append(specgram)
if coalesce:
globalv.SPECTROGRAMS[key].coalesce()
def add_coherence_component_spectrogram(specgram, key=None, coalesce=True):
"""Add a `Coherence spectrogram` to the global memory cache
"""
if key is None:
key = specgram.name or str(specgram.channel)
globalv.COHERENCE_COMPONENTS.setdefault(key, SpectrogramList())
globalv.COHERENCE_COMPONENTS[key].append(specgram)
if coalesce:
globalv.COHERENCE_COMPONENTS[key].coalesce()
def get_coherence_spectrograms(channel_pairs, segments, config=None,
cache=None, query=True, nds=None,
return_=True, frametype=None, nproc=1,
datafind_error='raise', **fftparams):
"""Get coherence spectrograms for multiple channels
"""
if fftparams.get('method', 'welch') != 'welch':
raise ValueError("Cannot process coherence data with method=%r"
% fftparams.get('method'))
fftparams['method'] = 'welch'
channels = list(map(get_channel, channel_pairs))
pairs = list(zip(channels[0::2], channels[1::2]))
# get timeseries data in bulk
if query:
qchannels = []
havesegs = []
for c1, c2 in pairs:
c1 = get_channel(c1)
c2 = get_channel(c2)
fftparams_ = get_fftparams(c1, **fftparams)
key = make_globalv_key((c1, c2), fftparams_)
qchannels.extend((c1, c2))
havesegs.append(globalv.SPECTROGRAMS.get(
key, SpectrogramList()).segments)
havesegs = reduce(operator.and_, havesegs)
new = segments - havesegs
strides = set([getattr(c, 'stride', 0) for c in qchannels])
if len(strides) == 1:
stride = strides.pop()
new = type(new)([s for s in new if abs(s) >= stride])
get_timeseries_dict(qchannels, new, config=config, cache=cache,
nproc=nproc, frametype=frametype,
datafind_error=datafind_error, nds=nds,
return_=False)
# loop over channels and generate spectrograms
out = OrderedDict()
for channel_pair in pairs:
out[channel_pair] = get_coherence_spectrogram(
channel_pair, segments, config=config, cache=cache, query=query,
nds=nds, nproc=nproc, return_=return_,
datafind_error=datafind_error, **fftparams)
return out
def get_spectrograms(channels, segments, config=ConfigParser(), cache=None,
query=True, nds='guess', format='power', return_=True,
method='median-mean', frametype=None, multiprocess=True,
datafind_error='raise', **fftparams):
"""Get spectrograms for multiple channels
"""
channels = map(get_channel, channels)
# get timeseries data in bulk
if query:
qchannels = map(
get_channel, set(c2 for c in
map(lambda x: re_channel.findall(x.ndsname), channels)
for c2 in c))
if format in ['rayleigh']:
method_ = format
else:
method_ = method
keys = ['%s,%s' % (channel.ndsname, method_) for channel in qchannels]
havesegs = reduce(operator.and_, (globalv.SPECTROGRAMS.get(
key, SpectrogramList()).segments for key in keys))
new = segments - havesegs
strides = set([getattr(c, 'stride', 0) for c in qchannels])
if len(strides) == 1:
stride = strides.pop()
new = type(new)([s for s in new if abs(s) >= stride])
get_timeseries_dict(qchannels, new, config=config, cache=cache,
multiprocess=multiprocess, frametype=frametype,
datafind_error=datafind_error, nds=nds,
return_=False)
# loop over channels and generate spectrograms
out = OrderedDict()
for channel in channels:
out[channel] = get_spectrogram(
channel, segments, config=config, cache=cache, query=query,
nds=nds, format=format, multiprocess=multiprocess,
return_=return_, method=method, datafind_error=datafind_error,
**fftparams)
return out
def _get_coherence_spectrogram(channel_pair, segments, config=None,
cache=None, query=True, nds=None,
return_=True, frametype=None, nproc=1,
datafind_error='raise', return_components=False,
**fftparams):
channel1 = get_channel(channel_pair[0])
channel2 = get_channel(channel_pair[1])
# clean fftparams dict using channel 1 default values
fftparams.setdefault('method', 'welch')
fftparams = get_fftparams(channel1, **fftparams)
# key used to store the coherence spectrogram in globalv
key = make_globalv_key(channel_pair, fftparams)
# keys used to store component spectrograms in globalv
components = ('Cxy', 'Cxx', 'Cyy')
ckeys = [
make_globalv_key([channel1, channel2], fftparams),
make_globalv_key(channel1, fftparams),
make_globalv_key(channel2, fftparams),
]
# convert fftparams to regular dict
fftparams = fftparams.dict()
# work out what new segments are needed
# need to truncate to segments of integer numbers of strides
stride = float(fftparams.pop('stride'))
overlap = float(fftparams['overlap'])
new = type(segments)()
for seg in segments - globalv.SPECTROGRAMS.get(
key, SpectrogramList()).segments:
dur = float(abs(seg)) // stride * stride
if dur < stride + overlap:
continue
new.append(type(seg)(seg[0], seg[0]+dur))
# extract FFT params for TimeSeries.spectrogram
spec_fftparams = fftparams.copy()
for fftkey in ('method', 'scheme',):
fftparams.pop(fftkey, None)
# if there are no existing spectrogram, initialize as a list
globalv.SPECTROGRAMS.setdefault(key, SpectrogramList())
# XXX HACK: use dummy timeseries to find lower sampling rate
if len(segments) > 0:
s = segments[0].start
dts1 = get_timeseries(channel1, SegmentList([Segment(s, s+1)]),
config=config, cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error, nds=nds)
dts2 = get_timeseries(channel2, SegmentList([Segment(s, s+1)]),
config=config, cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error, nds=nds)
sampling = min(dts1[0].sample_rate.value, dts2[0].sample_rate.value)
else:
sampling = None
# initialize component lists if they don't exist yet
for ck in ckeys:
globalv.COHERENCE_COMPONENTS.setdefault(ck, SpectrogramList())
# get data if query=True or if there are new segments
query &= abs(new) != 0
if query:
# the intersecting segments will be calculated when needed
intersection = None
# loop over components needed to calculate coherence
for comp in components:
# key used to store this component in globalv (incl sample rate)
ckey = ckeys[components.index(comp)]
try:
filter_ = channel1.frequency_response
except AttributeError:
filter_ = None
else:
if isinstance(filter_, str):
filter_ = safe_eval(filter_, strict=True)
# check how much of this component still needs to be calculated
req = new - globalv.COHERENCE_COMPONENTS.get(
ckey, SpectrogramList()).segments
# get data if there are new segments
if abs(req) != 0:
# calculate intersection of segments lazily
# this should occur on first pass (Cxy)
if intersection is None:
total1 = get_timeseries(
channel1, req, config=config,
cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error,
nds=nds)
total2 = get_timeseries(
channel2, req, config=config,
cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error,
nds=nds)
intersection = total1.segments & total2.segments
# get required timeseries data (using intersection)
tslist1, tslist2 = [], []
if comp in ('Cxy', 'Cxx'):
tslist1 = get_timeseries(
channel1, intersection, config=config,
cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error,
nds=nds)
if comp in ('Cxy', 'Cyy'):
tslist2 = get_timeseries(
channel2, intersection, config=config,
cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error,
nds=nds)
# calculate component
if len(tslist1) + len(tslist2):
vprint(" Calculating component %s for coherence "
"spectrogram for %s and %s @ %d Hz" % (
comp, str(channel1), str(channel2), sampling))
for ts1, ts2 in zip_longest(tslist1, tslist2):
# ensure there is enough data to do something with
if comp in ('Cxx', 'Cxy') and abs(ts1.span) < stride:
continue
elif comp in ('Cyy', 'Cxy') and abs(ts2.span) < stride:
continue
# downsample if necessary
if ts1 is not None and ts1.sample_rate.value != sampling:
ts1 = ts1.resample(sampling)
if ts2 is not None and ts2.sample_rate.value != sampling:
ts2 = ts2.resample(sampling)
# ignore units when calculating coherence
if ts1 is not None:
ts1._unit = units.Unit('count')
if ts2 is not None:
ts2._unit = units.Unit('count')
# calculate the component spectrogram
if comp == 'Cxy':
specgram = ts1.csd_spectrogram(
ts2, stride, nproc=nproc, **fftparams)
elif comp == 'Cxx':
specgram = ts1.spectrogram(stride, nproc=nproc,
**spec_fftparams)
elif comp == 'Cyy':
specgram = ts2.spectrogram(stride, nproc=nproc,
**spec_fftparams)
if filter_:
specgram = (specgram**(1/2.)).filter(*filter_,
inplace=True) ** 2
add_coherence_component_spectrogram(specgram, key=ckey)
vprint('.')
if len(tslist1) + len(tslist2):
vprint('\n')
# calculate coherence from the components and store in globalv
for seg in new:
cxy, cxx, cyy = [
_get_from_list(globalv.COHERENCE_COMPONENTS[ck], seg) for
ck in ckeys]
csg = abs(cxy)**2 / cxx / cyy
globalv.SPECTROGRAMS[key].append(csg)
globalv.SPECTROGRAMS[key].coalesce()
if not return_:
return
elif return_components:
# return list of component spectrogram lists
out = [SpectrogramList(), SpectrogramList(), SpectrogramList()]
for comp in components:
index = components.index(comp)
ckey = ckeys[index]
for specgram in globalv.COHERENCE_COMPONENTS[ckey]:
for seg in segments:
if abs(seg) < specgram.dt.value:
continue
if specgram.span.intersects(seg):
common = specgram.span & type(seg)(
seg[0], seg[1] + specgram.dt.value)
s = specgram.crop(*common)
if s.shape[0]:
out[index].append(s)
out[index] = out[index].coalesce()
return out
else:
# return list of coherence spectrograms
out = SpectrogramList()
for specgram in globalv.SPECTROGRAMS[key]:
for seg in segments:
if abs(seg) < specgram.dt.value:
continue
if specgram.span.intersects(seg):
common = specgram.span & type(seg)(
seg[0], seg[1] + specgram.dt.value)
s = specgram.crop(*common)
if s.shape[0]:
out.append(s)
return out.coalesce()
def get_spectrograms(channels,
segments,
config=None,
cache=None,
query=True,
nds=None,
format='power',
return_=True,
frametype=None,
nproc=1,
datafind_error='raise',
**fftparams):
"""Get spectrograms for multiple channels
"""
channels = list(map(get_channel, channels))
# get timeseries data in bulk
if query:
# get underlying list of data channels to read
qchannels = list(
map(
get_channel,
set([
c for group in map(split_channel_combination, channels)
for c in group
])))
# work out FFT params and storage keys for each data channel
keys = []
for channel in qchannels:
fftparams_ = get_fftparams(channel, **fftparams)
keys.append(make_globalv_key(channel, fftparams_))
# restrict segments to those big enough to hold >= 1 stride
strides = set([getattr(c, 'stride', 0) for c in qchannels])
if len(strides) == 1:
stride = strides.pop()
segments = type(segments)(s for s in segments if abs(s) >= stride)
# work out new segments for which to read data
havesegs = reduce(
operator.and_,
(globalv.SPECTROGRAMS.get(key, SpectrogramList()).segments
for key in keys))
new = segments - havesegs
# read data for new segments
get_timeseries_dict(qchannels,
new,
config=config,
cache=cache,
nproc=nproc,
frametype=frametype,
datafind_error=datafind_error,
nds=nds,
return_=False)
# loop over channels and generate spectrograms
out = OrderedDict()
for channel in channels:
out[channel] = get_spectrogram(channel,
segments,
config=config,
cache=cache,
query=query,
nds=nds,
format=format,
nproc=nproc,
return_=return_,
datafind_error=datafind_error,
**fftparams)
return out
def _get_spectrogram(channel,
segments,
config=None,
cache=None,
query=True,
nds=None,
format='power',
return_=True,
frametype=None,
nproc=1,
datafind_error='raise',
**fftparams):
channel = get_channel(channel)
# if we aren't given a method, check to see whether data have already
# been processed, if so, choose that one
if fftparams.get('method', None) is None:
methods = set([
key.split(';')[1] for key in globalv.SPECTROGRAMS
if key.startswith('%s;' % channel.ndsname)
])
try:
fftparams['method'] = list(methods)[0]
except IndexError:
fftparams['method'] = 'welch'
# clean fftparams dict using channel default values
fftparams = get_fftparams(channel, **fftparams)
# override special-case methods
if format in ['rayleigh']:
fftparams.method = format
# key used to store the coherence spectrogram in globalv
key = make_globalv_key(channel, fftparams)
# keep FftParams as a dict for convenience
fftparams = fftparams.dict()
# read segments from global memory
havesegs = globalv.SPECTROGRAMS.get(key, SpectrogramList()).segments
new = segments - havesegs
query &= abs(new) != 0
globalv.SPECTROGRAMS.setdefault(key, SpectrogramList())
if query:
# extract spectrogram stride from dict
try:
stride = float(fftparams.pop('stride'))
except (TypeError, KeyError) as e:
msg = ('cannot parse a spectrogram stride from the kwargs '
'given, please give some or all of fftlength, overlap, '
'stride')
if isinstance(e, TypeError):
e.args = (msg, )
raise
raise TypeError(msg)
# read channel information
try:
filter_ = channel.frequency_response
except AttributeError:
filter_ = None
else:
if isinstance(filter_, str) and os.path.isfile(filter_):
filter_ = io.read_frequencyseries(filter_)
elif isinstance(filter_, str):
filter_ = safe_eval(filter_, strict=True)
# get time-series data
timeserieslist = get_timeseries(channel,
new,
config=config,
cache=cache,
frametype=frametype,
nproc=nproc,
query=query,
datafind_error=datafind_error,
nds=nds)
# calculate spectrograms
if len(timeserieslist):
vprint(" Calculating (%s) spectrograms for %s" %
(fftparams['method'], str(channel)))
for ts in timeserieslist:
# if too short for a single segment, continue
if abs(ts.span) < (stride + fftparams.get('overlap', 0)):
continue
# truncate timeseries to integer number of strides
d = size_for_spectrogram(
ts.duration.to('s').value, stride, fftparams['fftlength'],
fftparams.get('overlap', 0))
ts = ts.crop(ts.span[0], ts.span[0] + d, copy=False)
# calculate spectrogram
try:
# rayleigh spectrogram has its own instance method
if fftparams.get('method', None) == 'rayleigh':
spec_kw = fftparams.copy()
for fftkey in (
'method',
'scheme',
): # remove ASD keys
spec_kw.pop(fftkey, None)
spec_func = ts.rayleigh_spectrogram
else:
spec_kw = fftparams
spec_func = ts.spectrogram
specgram = spec_func(stride, nproc=nproc, **spec_kw)
except ZeroDivisionError:
if stride == 0:
raise ZeroDivisionError("Spectrogram stride is 0")
elif fftparams['fftlength'] == 0:
raise ZeroDivisionError("FFT length is 0")
else:
raise
except ValueError as e:
if 'has no unit' in str(e):
unit = ts.unit
ts._unit = units.Unit('count')
specgram = ts.spectrogram(stride, nproc=nproc, **fftparams)
specgram._unit = unit**2 / units.Hertz
else:
raise
if isinstance(filter_, FrequencySeries) and (fftparams['method']
not in ['rayleigh']):
specgram = apply_transfer_function_series(specgram, filter_)
elif filter_ and fftparams['method'] not in ['rayleigh']:
# manually setting x0 is a hack against precision error
# somewhere inside the **(1/2.) operation (Quantity)
x0 = specgram.x0.value
specgram = (specgram**(1 / 2.)).filter(*filter_,
inplace=True)**2
specgram.x0 = x0
if specgram.unit is None:
specgram._unit = channel.unit
elif len(globalv.SPECTROGRAMS[key]):
specgram._unit = globalv.SPECTROGRAMS[key][-1].unit
add_spectrogram(specgram, key=key)
vprint('.')
if len(timeserieslist):
vprint('\n')
if not return_:
return
# return correct data
out = SpectrogramList()
for specgram in globalv.SPECTROGRAMS[key]:
for seg in segments:
if abs(seg) < specgram.dt.value:
continue
if specgram.span.intersects(seg):
common = specgram.span & type(seg)(seg[0],
seg[1] + specgram.dt.value)
s = specgram.crop(*common)
if format in ['amplitude', 'asd']:
s = s**(1 / 2.)
elif format in ['rayleigh']:
# XXX FIXME: this corrects the bias offset in Rayleigh
med = numpy.median(s.value)
s /= med
if s.shape[0]:
out.append(s)
return out.coalesce()
def _get_coherence_spectrogram(channel_pair, segments, config=None,
cache=None, query=True, nds=None,
return_=True, frametype=None, nproc=1,
datafind_error='raise', return_components=False,
**fftparams):
channel1 = get_channel(channel_pair[0])
channel2 = get_channel(channel_pair[1])
# clean fftparams dict using channel 1 default values
fftparams.setdefault('method', 'welch')
fftparams = get_fftparams(channel1, **fftparams)
# key used to store the coherence spectrogram in globalv
key = make_globalv_key(channel_pair, fftparams)
# keys used to store component spectrograms in globalv
components = ('Cxy', 'Cxx', 'Cyy')
ckeys = [
make_globalv_key([channel1, channel2], fftparams),
make_globalv_key(channel1, fftparams),
make_globalv_key(channel2, fftparams),
]
# convert fftparams to regular dict
fftparams = fftparams.dict()
# work out what new segments are needed
# need to truncate to segments of integer numbers of strides
stride = float(fftparams.pop('stride'))
overlap = float(fftparams['overlap'])
new = type(segments)()
for seg in segments - globalv.SPECTROGRAMS.get(
key, SpectrogramList()).segments:
dur = float(abs(seg)) // stride * stride
if dur < stride + overlap:
continue
new.append(type(seg)(seg[0], seg[0]+dur))
# extract FFT params for TimeSeries.spectrogram
spec_fftparams = fftparams.copy()
for fftkey in ('method', 'scheme',):
fftparams.pop(fftkey, None)
# if there are no existing spectrogram, initialize as a list
globalv.SPECTROGRAMS.setdefault(key, SpectrogramList())
# XXX HACK: use dummy timeseries to find lower sampling rate
if len(segments) > 0:
s = segments[0].start
dts1 = get_timeseries(channel1, SegmentList([Segment(s, s+1)]),
config=config, cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error, nds=nds)
dts2 = get_timeseries(channel2, SegmentList([Segment(s, s+1)]),
config=config, cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error, nds=nds)
try:
sampling = min(dts1[0].sample_rate.value,
dts2[0].sample_rate.value)
except IndexError:
sampling = None
else:
sampling = None
# initialize component lists if they don't exist yet
for ck in ckeys:
globalv.COHERENCE_COMPONENTS.setdefault(ck, SpectrogramList())
# get data if query=True or if there are new segments
query &= abs(new) != 0
if query:
# the intersecting segments will be calculated when needed
intersection = None
# loop over components needed to calculate coherence
for comp in components:
# key used to store this component in globalv (incl sample rate)
ckey = ckeys[components.index(comp)]
try:
filter_ = channel1.frequency_response
except AttributeError:
filter_ = None
else:
if isinstance(filter_, str):
filter_ = safe_eval(filter_, strict=True)
# check how much of this component still needs to be calculated
req = new - globalv.COHERENCE_COMPONENTS.get(
ckey, SpectrogramList()).segments
# get data if there are new segments
if abs(req) != 0:
# calculate intersection of segments lazily
# this should occur on first pass (Cxy)
if intersection is None:
total1 = get_timeseries(
channel1, req, config=config,
cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error,
nds=nds)
total2 = get_timeseries(
channel2, req, config=config,
cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error,
nds=nds)
intersection = total1.segments & total2.segments
# get required timeseries data (using intersection)
tslist1, tslist2 = [], []
if comp in ('Cxy', 'Cxx'):
tslist1 = get_timeseries(
channel1, intersection, config=config,
cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error,
nds=nds)
if comp in ('Cxy', 'Cyy'):
tslist2 = get_timeseries(
channel2, intersection, config=config,
cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error,
nds=nds)
# calculate component
if len(tslist1) + len(tslist2):
vprint(" Calculating component %s for coherence "
"spectrogram for %s and %s @ %d Hz" % (
comp, str(channel1), str(channel2), sampling))
for ts1, ts2 in zip_longest(tslist1, tslist2):
# ensure there is enough data to do something with
if comp in ('Cxx', 'Cxy') and abs(ts1.span) < stride:
continue
elif comp in ('Cyy', 'Cxy') and abs(ts2.span) < stride:
continue
# downsample if necessary
if ts1 is not None and ts1.sample_rate.value != sampling:
ts1 = ts1.resample(sampling)
if ts2 is not None and ts2.sample_rate.value != sampling:
ts2 = ts2.resample(sampling)
# ignore units when calculating coherence
if ts1 is not None:
ts1._unit = units.Unit('count')
if ts2 is not None:
ts2._unit = units.Unit('count')
# calculate the component spectrogram
if comp == 'Cxy':
specgram = ts1.csd_spectrogram(
ts2, stride, nproc=nproc, **fftparams)
elif comp == 'Cxx':
specgram = ts1.spectrogram(stride, nproc=nproc,
**spec_fftparams)
elif comp == 'Cyy':
specgram = ts2.spectrogram(stride, nproc=nproc,
**spec_fftparams)
if filter_:
specgram = (specgram**(1/2.)).filter(*filter_,
inplace=True) ** 2
add_coherence_component_spectrogram(specgram, key=ckey)
vprint('.')
if len(tslist1) + len(tslist2):
vprint('\n')
spans = [SegmentList([ # record spectrogaram spans
spec.span for spec in globalv.COHERENCE_COMPONENTS[ck]
]) for ck in ckeys]
new = reduce(operator.and_, spans, new).coalesce()
for seg in new: # compute coherence from components
cxy, cxx, cyy = [_get_from_list(
globalv.COHERENCE_COMPONENTS[ck], seg) for ck in ckeys]
csg = abs(cxy)**2 / cxx / cyy
globalv.SPECTROGRAMS[key].append(csg)
globalv.SPECTROGRAMS[key].coalesce()
if not return_:
return
elif return_components:
# return list of component spectrogram lists
out = [SpectrogramList(), SpectrogramList(), SpectrogramList()]
for comp in components:
index = components.index(comp)
ckey = ckeys[index]
for specgram in globalv.COHERENCE_COMPONENTS[ckey]:
for seg in segments:
if abs(seg) < specgram.dt.value:
continue
if specgram.span.intersects(seg):
common = specgram.span & type(seg)(
seg[0], seg[1] + specgram.dt.value)
s = specgram.crop(*common)
if s.shape[0]:
out[index].append(s)
out[index] = out[index].coalesce()
return out
else:
# return list of coherence spectrograms
out = SpectrogramList()
for specgram in globalv.SPECTROGRAMS[key]:
for seg in segments:
if abs(seg) < specgram.dt.value:
continue
if specgram.span.intersects(seg):
common = specgram.span & type(seg)(
seg[0], seg[1] + specgram.dt.value)
s = specgram.crop(*common)
if s.shape[0]:
out.append(s)
return out.coalesce()
def _get_spectrogram(channel, segments, config=ConfigParser(), cache=None,
query=True, nds='guess', format='power', return_=True,
frametype=None, multiprocess=True, method=None,
datafind_error='raise', **fftparams):
channel = get_channel(channel)
if method is None:
methods = set([key.split(',', 1)[1] for key in globalv.SPECTROGRAMS
if key.startswith('%s,' % channel.ndsname)])
if len(methods) == 1:
method = list(methods)[0]
else:
method = 'median-mean'
if format in ['rayleigh']:
method = format
key = '%s,%s' % (channel.ndsname, method)
# read segments from global memory
havesegs = globalv.SPECTROGRAMS.get(key,
SpectrogramList()).segments
new = segments - havesegs
# get processes
if multiprocess is True:
nproc = count_free_cores()
elif multiprocess is False:
nproc = 1
else:
nproc = multiprocess
globalv.SPECTROGRAMS.setdefault(key, SpectrogramList())
query &= abs(new) != 0
if query:
# read channel information
try:
filter_ = channel.frequency_response
except AttributeError:
filter_ = None
else:
if isinstance(filter_, str):
filter_ = eval(filter_)
# read FFT params
fftparams = fftparams.copy()
for param in ['fftlength', 'overlap']:
if hasattr(channel, param):
fftparams[param] = float(getattr(channel, param))
elif param in fftparams:
fftparams[param] = float(fftparams[param])
try:
stride = float(fftparams.pop('stride'))
except KeyError:
stride = None
if hasattr(channel, 'stride'):
stride = float(channel.stride)
if hasattr(channel, 'window'):
fftparams['window'] = channel.window
# get time-series data
if stride is not None:
tmp = type(new)()
for s in new:
if abs(s) < stride:
continue
else:
d = float(abs(s))
tmp.append(type(s)(s[0], s[0] + d//stride * stride))
new = tmp
timeserieslist = get_timeseries(channel, new, config=config,
cache=cache, frametype=frametype,
multiprocess=nproc, query=query,
datafind_error=datafind_error, nds=nds)
# calculate spectrograms
if len(timeserieslist):
vprint(" Calculating spectrograms for %s" % str(channel))
for ts in timeserieslist:
fftparams.setdefault('fftlength', 1)
fftparams.setdefault('overlap', 0.5)
if not stride and fftparams['overlap'] != 0:
stride = ceil(fftparams['fftlength'] * 1.5)
elif not stride:
stride = fftparams['fftlength']
if abs(ts.span) < stride:
continue
try:
specgram = ts.spectrogram(stride, nproc=nproc, method=method,
**fftparams)
except ZeroDivisionError:
if stride == 0:
raise ZeroDivisionError("Spectrogram stride is 0")
elif fftparams['fftlength'] == 0:
raise ZeroDivisionError("FFT length is 0")
else:
raise
except ValueError as e:
if 'has no unit' in str(e):
unit = ts.unit
ts._unit = units.Unit('count')
specgram = ts.spectrogram(stride, nproc=nproc, **fftparams)
specgram._unit = unit ** 2 / units.Hertz
else:
raise
if filter_ and method not in ['rayleigh']:
specgram = (specgram ** (1/2.)).filter(*filter_, inplace=True) ** 2
if specgram.unit is None:
specgram._unit = channel.unit
elif len(globalv.SPECTROGRAMS[key]):
specgram._unit = globalv.SPECTROGRAMS[key][-1].unit
globalv.SPECTROGRAMS[key].append(specgram)
globalv.SPECTROGRAMS[key].coalesce()
vprint('.')
if len(timeserieslist):
vprint('\n')
if not return_:
return
# return correct data
out = SpectrogramList()
for specgram in globalv.SPECTROGRAMS[key]:
for seg in segments:
if abs(seg) < specgram.dt.value:
continue
if specgram.span.intersects(seg):
common = specgram.span & type(seg)(seg[0],
seg[1] + specgram.dt.value)
s = specgram.crop(*common)
if format in ['amplitude', 'asd']:
s = s**(1/2.)
elif format in ['rayleigh']:
# XXX FIXME: this corrects the bias offset in Rayleigh
med = numpy.median(s.value)
s /= med
if s.shape[0]:
out.append(s)
return out.coalesce()
def _get_spectrogram(channel, segments, config=None, cache=None,
query=True, nds=None, format='power', return_=True,
frametype=None, nproc=1,
datafind_error='raise', **fftparams):
channel = get_channel(channel)
# if we aren't given a method, check to see whether data have already
# been processed, if so, choose that one
if fftparams.get('method', None) is None:
methods = set([key.split(';')[1] for key in globalv.SPECTROGRAMS
if key.startswith('%s;' % channel.ndsname)])
try:
fftparams['method'] = list(methods)[0]
except IndexError:
fftparams['method'] = 'welch'
# clean fftparams dict using channel default values
fftparams = get_fftparams(channel, **fftparams)
# override special-case methods
if format in ['rayleigh']:
fftparams.method = format
# key used to store the coherence spectrogram in globalv
key = make_globalv_key(channel, fftparams)
# keep FftParams as a dict for convenience
fftparams = fftparams.dict()
# read segments from global memory
havesegs = globalv.SPECTROGRAMS.get(key, SpectrogramList()).segments
new = segments - havesegs
query &= abs(new) != 0
globalv.SPECTROGRAMS.setdefault(key, SpectrogramList())
if query:
# extract spectrogram stride from dict
try:
stride = float(fftparams.pop('stride'))
except (TypeError, KeyError) as e:
msg = ('cannot parse a spectrogram stride from the kwargs '
'given, please give some or all of fftlength, overlap, '
'stride')
if isinstance(e, TypeError):
e.args = (msg,)
raise
raise TypeError(msg)
# read channel information
try:
filter_ = channel.frequency_response
except AttributeError:
filter_ = None
else:
if isinstance(filter_, str) and os.path.isfile(filter_):
filter_ = io.read_frequencyseries(filter_)
elif isinstance(filter_, str):
filter_ = safe_eval(filter_, strict=True)
# get time-series data
timeserieslist = get_timeseries(channel, new, config=config,
cache=cache, frametype=frametype,
nproc=nproc, query=query,
datafind_error=datafind_error, nds=nds)
# calculate spectrograms
if len(timeserieslist):
vprint(" Calculating (%s) spectrograms for %s"
% (fftparams['method'], str(channel)))
for ts in timeserieslist:
# if too short for a single segment, continue
if abs(ts.span) < (stride + fftparams.get('overlap', 0)):
continue
# truncate timeseries to integer number of strides
d = size_for_spectrogram(ts.duration.to('s').value, stride,
fftparams['fftlength'],
fftparams.get('overlap', 0))
ts = ts.crop(ts.span[0], ts.span[0] + d, copy=False)
# calculate spectrogram
try:
# rayleigh spectrogram has its own instance method
if fftparams.get('method', None) == 'rayleigh':
spec_kw = fftparams.copy()
for fftkey in ('method', 'scheme',): # remove ASD keys
spec_kw.pop(fftkey, None)
spec_func = ts.rayleigh_spectrogram
else:
spec_kw = fftparams
spec_func = ts.spectrogram
specgram = spec_func(stride, nproc=nproc, **spec_kw)
except ZeroDivisionError:
if stride == 0:
raise ZeroDivisionError("Spectrogram stride is 0")
elif fftparams['fftlength'] == 0:
raise ZeroDivisionError("FFT length is 0")
else:
raise
except ValueError as e:
if 'has no unit' in str(e):
unit = ts.unit
ts._unit = units.Unit('count')
specgram = ts.spectrogram(stride, nproc=nproc, **fftparams)
specgram._unit = unit ** 2 / units.Hertz
else:
raise
if isinstance(filter_, FrequencySeries) and (
fftparams['method'] not in ['rayleigh']):
specgram = apply_transfer_function_series(specgram, filter_)
elif filter_ and fftparams['method'] not in ['rayleigh']:
# manually setting x0 is a hack against precision error
# somewhere inside the **(1/2.) operation (Quantity)
x0 = specgram.x0.value
specgram = (specgram ** (1/2.)).filter(*filter_,
inplace=True) ** 2
specgram.x0 = x0
if specgram.unit is None:
specgram._unit = channel.unit
elif len(globalv.SPECTROGRAMS[key]):
specgram._unit = globalv.SPECTROGRAMS[key][-1].unit
add_spectrogram(specgram, key=key)
vprint('.')
if len(timeserieslist):
vprint('\n')
if not return_:
return
# return correct data
out = SpectrogramList()
for specgram in globalv.SPECTROGRAMS[key]:
for seg in segments:
if abs(seg) < specgram.dt.value:
continue
if specgram.span.intersects(seg):
common = specgram.span & type(seg)(seg[0],
seg[1] + specgram.dt.value)
s = specgram.crop(*common)
if format in ['amplitude', 'asd']:
s = s**(1/2.)
elif format in ['rayleigh']:
# XXX FIXME: this corrects the bias offset in Rayleigh
med = numpy.median(s.value)
s /= med
if s.shape[0]:
out.append(s)
return out.coalesce()