def main():
stations = 'PB03 PB04'
component = 'Z'
t1 = UTC('2006-01-01')
t2 = UTC()
# t1 = UTC('2007-01-01')
# t2 = UTC('2007-01-03')
method1 = 'filter0.01-1_water_env2_whitening_1bit'
method2 = 'filter0.01-1_water_env2_whitening_1bit_fft'
data1 = IPOC(xcorr_append='/Tocopilla/' + method1, use_local_LVC=True)
data2 = IPOC(xcorr_append='/Tocopilla/' + method2, use_local_LVC=True)
for station in stations.split():
for day in daygen(t1, t2):
try:
stream = data1.getStream(day, station, component)
except:
log.warning('Could not read stream for day %s station %s' % (day, station))
else:
if len(stream) != 1:
log.warning('Stream for day %s station %s has wrong length %d' % (day, station, len(stream)))
elif stream[0].stats.npts / stream[0].stats.sampling_rate < 24 * 3600 * 0.5:
log.warning('Stream for day %s station %s has only a coverage of %f -> discard' % (day, station, 1. * stream[0].stats.npts / stream[0].stats.sampling_rate / 24 / 3600))
else:
stream.fft()
stream.write(data2.getDay(station, day), 'Q')
def analyze():
#stations = 'PB01 PB02 PB03 PB04 PB05 PB06 PB07 PB08 PB09 PB10 PB11 PB12 PB13 PB14 PB15 PB16 HMBCX MNMCX PATCX PSGCX LVC TAIQ'
stations = 'PB01 PB02 PATCX'
stations = 'PATCX'
t1 = UTC('2009-01-01')
t2 = UTC('2010-01-01')
data = IPOC()
for station in stations.split():
hours = [[] for i in range(24)]
times = []
levels = []
for t_day in daygen(t1, t2):
try:
stream = data.getRawStreamFromClient(t_day, t_day + 24 * 3600, station, component='Z')
except ValueError:
continue
for tr in stream:
tr.stats.filter = ''
stream.demean()
stream.detrend()
stream.filter2(4, 6)
stream.downsample2(5)
stream.merge()
tr = stream[0]
startt = tr.stats.starttime
endt = tr.stats.endtime
if endt - startt < 12 * 3600:
continue
tr.data = obspy.signal.cpxtrace.envelope(tr.data)[1][:len(tr.data)]
for hour in range(24):
tr2 = tr.slice(t_day + hour * 3600, t_day + (hour + 1) * 3600)
if tr2.stats.endtime - tr2.stats.starttime < 1800:
continue
num_stds = 60 # =^ every minute
len_parts = len(tr2.data) // 60 # =^ 1min
len_stds = len_parts // 6 # =^ 10s
stds = np.array([np.std(tr2.data[i:i + len_stds]) for i in np.arange(num_stds) * len_parts])
stds = stds[stds != 0.]
num_stds = len(stds)
if num_stds < 50:
continue
stds = np.sort(stds)[num_stds // 5:-num_stds // 5]
stds = stds[stds < np.min(stds) * 2.]
val = np.mean(stds)
levels.append(val)
times.append(date2num(t_day + (0.5 + hour) * 3600))
hours[hour].append(val)
errors = np.array([np.std(hours[i], ddof=1) / len(hours[i]) ** 0.5 for i in range(24)])
hours = np.array([np.mean(hours[i]) for i in range(24)])
times = np.array(times)
levels = np.array(levels)
np.savez('/home/richter/Results/IPOC/xcorr/noise_apmlitudes_%s_4-6Hz.npz' % station,
hours=hours, errors=errors, times=times, levels=levels)
def main():
stations = 'PB03 PB04'
component = 'Z'
t1 = UTC('2006-01-01')
t2 = UTC()
# t1 = UTC('2007-01-01')
# t2 = UTC('2007-01-03')
method1 = 'filter0.01-1_water_env2_whitening_1bit'
method2 = 'filter0.01-1_water_env2_whitening_1bit_fft'
data1 = IPOC(xcorr_append='/Tocopilla/' + method1, use_local_LVC=True)
data2 = IPOC(xcorr_append='/Tocopilla/' + method2, use_local_LVC=True)
for station in stations.split():
for day in daygen(t1, t2):
try:
stream = data1.getStream(day, station, component)
except:
log.warning('Could not read stream for day %s station %s' %
(day, station))
else:
if len(stream) != 1:
log.warning(
'Stream for day %s station %s has wrong length %d' %
(day, station, len(stream)))
elif stream[0].stats.npts / stream[
0].stats.sampling_rate < 24 * 3600 * 0.5:
log.warning(
'Stream for day %s station %s has only a coverage of %f -> discard'
% (day, station, 1. * stream[0].stats.npts /
stream[0].stats.sampling_rate / 24 / 3600))
else:
stream.fft()
stream.write(data2.getDay(station, day), 'Q')
def analyze():
#stations = 'PB01 PB02 PB03 PB04 PB05 PB06 PB07 PB08 PB09 PB10 PB11 PB12 PB13 PB14 PB15 PB16 HMBCX MNMCX PATCX PSGCX LVC TAIQ'
stations = 'PB01 PB02 PATCX'
stations = 'PATCX'
t1 = UTC('2009-01-01')
t2 = UTC('2010-01-01')
data = IPOC()
for station in stations.split():
hours = [[] for i in range(24)]
times = []
levels = []
for t_day in daygen(t1, t2):
try:
stream = data.getRawStreamFromClient(t_day,
t_day + 24 * 3600,
station,
component='Z')
except ValueError:
continue
for tr in stream:
tr.stats.filter = ''
stream.demean()
stream.detrend()
stream.filter2(4, 6)
stream.downsample2(5)
stream.merge()
tr = stream[0]
startt = tr.stats.starttime
endt = tr.stats.endtime
if endt - startt < 12 * 3600:
continue
tr.data = obspy.signal.cpxtrace.envelope(tr.data)[1][:len(tr.data)]
for hour in range(24):
tr2 = tr.slice(t_day + hour * 3600, t_day + (hour + 1) * 3600)
if tr2.stats.endtime - tr2.stats.starttime < 1800:
continue
num_stds = 60 # =^ every minute
len_parts = len(tr2.data) // 60 # =^ 1min
len_stds = len_parts // 6 # =^ 10s
stds = np.array([
np.std(tr2.data[i:i + len_stds])
for i in np.arange(num_stds) * len_parts
])
stds = stds[stds != 0.]
num_stds = len(stds)
if num_stds < 50:
continue
stds = np.sort(stds)[num_stds // 5:-num_stds // 5]
stds = stds[stds < np.min(stds) * 2.]
val = np.mean(stds)
levels.append(val)
times.append(date2num(t_day + (0.5 + hour) * 3600))
hours[hour].append(val)
errors = np.array(
[np.std(hours[i], ddof=1) / len(hours[i])**0.5 for i in range(24)])
hours = np.array([np.mean(hours[i]) for i in range(24)])
times = np.array(times)
levels = np.array(levels)
np.savez(
'/home/richter/Results/IPOC/xcorr/noise_apmlitudes_%s_4-6Hz.npz' %
station,
hours=hours,
errors=errors,
times=times,
levels=levels)
def prepare(data, stations, t1, t2, component='all', use_floating_stream=True,
use_client=True, #arclink_client_for_LVC=None,
pool=None, max_preload=5, **kwargs):
"""
Prepare data for cross correlation.
Day files of raw data are loaded (from files data.raw), and the prepared
data is again written to day files (to files data.raw).
@param data: sito.data object with attributes (raw, x_prep) set
@param stations: tuple of stations
@param t1: UTC object with date of first day
@param t2: UTC object with date of last day
@param filter_: tuple of min and max frequencies (or None) for filtering
@param downsample: downsample to this sampling rate (or None)
@param component: string out of ('Z', 'N', 'E', 'all')
@param normalize: method for normalizing the data in time domain (or None)
see xcorr.timeNorm
@param param_norm: parameter passed to xcorr.timeNorm()
@param whitening: bool, apply spectral whitening, default: False
@type use_floating_stream: bool
@param use_floating_stream: You should set this to True, if you have day
files which do not start/end exactly at midnight
default: False
@param reserve: parameter passed to FloatingStream()
should be the time between the end/beginning of your data and
midnight
Her is the documentation string of xcorr.timeNorm
"""
# filter=(None, None), downsample=None, #@ReservedAssignment
# eventremoval=None, param_removal=None,
# normalize=None, param_norm=None,
# whitening=None, filter_before_whitening=True, use_this_filter_after_whitening=None,
# freq_domain=False, discard=0.5):
# kwargs = dict(filter=filter, downsample=downsample, #@ReservedAssignment
# eventremoval=eventremoval, param_removal=param_removal,
# normalize=normalize, param_norm=param_norm,
# whitening=whitening, filter_before_whitening=filter_before_whitening,
# use_this_filter_after_whitening=use_this_filter_after_whitening,
# freq_domain=freq_domain, discard=discard)
log.info('Prepare data for noise correlation: %s' % util.parameters())
print 'Prepare data for noise correlation...'
if pool:
async_results = []
if use_client:
if 'trim' not in kwargs:
kwargs['trim'] = 'day'
for station in ProgressBar()(stations):
for t_day in daygen(t1, t2):
#for (station, t_day) in ProgressBar()(itertools.product(stations, daygen(t1, t2))):
try:
stream = data.getRawStreamFromClient(t_day - 60, t_day + 24 * 3600 + 60, station, component=component)
except Exception as ex:
log.info('Error loading station %s day %s: %s' % (str(station), t_day.date, ex))
continue
# if station != 'LVC' or not arclink_client_for_LVC:
# continue
# try:
# stream = arclink_client_for_LVC.getWaveform('GE', 'LVC', '10', 'BH' + component,
# t_day - 60, t_day + 24 * 3600 + 60)
# except:
# continue
# else:
# log.info('Found data on GEOFON.')
if len(stream) == 0:
log.info('Stream length 0 station %s day %s' % (str(station), t_day.date))
continue
## one day
def _callback(msg):
if msg:
log.info(msg % ((stream[0].stats.npts / stream[0].stats.sampling_rate / 24 / 3600),
str(station), t_day.date))
args = (stream, data.getDay(station, t_day, data.x_prep))
if pool:
if len(async_results) >= max_preload:
async_results[-max_preload].wait()
async_results.append(
pool.apply_async(_prepare_stream, args, kwargs, callback=_callback))
else:
_callback(_prepare_stream(*args, **kwargs))
else:
if not use_floating_stream:
data2 = data
for station in stations:
if use_floating_stream:
data2 = FloatingStream(data, t1, station, 0, component=component, use_get_raw=True)
for t_day in daygen(t1, t2):
try:
stream = data2.getRawStream(t_day, station, component=component)
except ValueError:
log.info('Error loading station %s day %s' % (str(station), t_day.date))
continue
if len(stream) == 0:
log.info('Stream length 0 station %s day %s' % (str(station), t_day.date))
continue
## one day
def _callback(msg):
if msg:
log.info(msg % ((stream[0].stats.npts / stream[0].stats.sampling_rate / 24 / 3600),
str(station), t_day.date))
args = (stream, data.getDay(station, t_day, data.x_prep))
if pool:
if len(async_results) >= max_preload:
async_results[-max_preload].wait()
async_results.append(
pool.apply_async(_prepare_stream, args, kwargs, callback=_callback))
else:
_callback(_prepare_stream(*args, **kwargs))
if pool:
for res in async_results:
res.wait()
