def __init__(self, ntracks=6, use_opengl=False, panel_parent=None, follow=20):
source_pile = pile.make_pile(['Demodataset_new.mseed'])
p = hamster_pile.HamsterPile()
p.set_fixation_length(20.)
pile_viewer.PileViewer.__init__(self, p, ntracks_shown_max=ntracks,
use_opengl=use_opengl, panel_parent=panel_parent)
self._tlast = time.time()
self._tlast_stalta = None
self._tmin = source_pile.tmin + self._tlast % (source_pile.tmax - source_pile.tmin)
v = self.get_view()
v.follow(float(follow))
self._source_pile = source_pile
self._timer = QTimer( self )
self.connect( self._timer, SIGNAL("timeout()"), self.periodical )
self._timer.setInterval(1000)
self._timer.start()
self._detectiontimer = QTimer( self )
self.connect( self._detectiontimer, SIGNAL("timeout()"), self.stalta )
self._detectiontimer.setInterval(3000)
self._detectiontimer.start()
def read_file(f, l):
s = False
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
try:
s = read(f, format="MSEED")
except Exception as e:
try:
l.write(
str(e) +
": obspy error messages\n===> NOW Trying toswitch to pyrocko decoding\n"
)
except:
print("Error writing Log file during obspy alternative")
try:
s = pile.make_pile([f]).to_obspy_stream()
except Exception as p:
if len(w):
for wm in w:
try:
l.write(
str(wm.message) +
": general read warning messages\n")
except:
pass
try:
l.write(str(p) + ": pyrocko error messages\n")
except:
print("Error writing Log file during pyrocko alternative")
l.close()
sys.exit(1)
return s
def __init__(self, files, scale=1., station_mapping={}, network_mapping={},
channel_mapping={}, location_mapping={}):
self.scale = scale
self.nslc_mapping = {}
self.data_pile = pile.make_pile(files)
self.noise = self.dictify_noise(self.data_pile, network_mapping, channel_mapping,
station_mapping, location_mapping)
self.merge_fader = trace.CosFader(xfrac=0.1)
def snuffle(config):
global _lassie_config
_lassie_config = copy.deepcopy(config)
for _ifc in _lassie_config.image_function_contributions:
_ifc.setup(config)
def load_snuffling(win):
s = LassieSnuffling()
s.config = _lassie_config
s.setup()
win.pile_viewer.viewer.add_snuffling(s, reloaded=True)
win.pile_viewer.viewer.add_blacklist_pattern('*.SMAX.i.*')
for bl in _lassie_config.blacklist:
win.pile_viewer.viewer.add_blacklist_pattern('%s.*' % bl)
detections_path = _lassie_config.get_detections_path()
if os.path.exists(detections_path):
s.detections = detections_to_event_markers(detections_path)
s.add_markers(s.detections)
for _ifc in s.config.image_function_contributions:
if isinstance(_ifc, ifc.ManualPickIFC):
markers_path_extra = _ifc.picks_path
elif isinstance(_ifc, ifc.TemplateMatchingIFC):
markers_path_extra = _ifc.template_markers_path
else:
continue
if os.path.exists(markers_path_extra):
s.add_markers(pmarker.load_markers(markers_path_extra))
else:
logger.warn(
'No such file: %s (referenced in %s, named %s)' %
(markers_path_extra, _ifc.__class__.__name__, _ifc.name))
receivers = config.get_receivers()
stations = set()
lats, lons = geo.points_coords(receivers, system='latlon')
for ir, (lat, lon) in enumerate(zip(lats, lons)):
n, s, l = receivers[ir].codes[:3]
stations.add(
model.Station(lat=lat, lon=lon, network=n, station=s, location=l))
paths = config.expand_path(config.data_paths)
paths.append(config.get_ifm_dir_path())
p = pile.make_pile(paths=paths, fileformat='detect')
meta = {'tabu': True}
for tr in p.iter_traces(trace_selector=lambda x: x.station == 'SMAX'):
if tr.meta:
tr.meta.update(meta)
else:
tr.meta = meta
snuffler.snuffle(p, stations=stations, launch_hook=load_snuffling)
def setup(self):
self.data_pile = pile.make_pile(
self.data_paths, fileformat=self.data_format)
if self.data_pile.is_empty():
sys.exit('Data pile is empty!')
self.deltat_want = self.config.deltat_want or \
min(self.data_pile.deltats.keys())
self.n_samples = int(
(self.config.sample_length + self.config.tpad) / self.deltat_want)
logger.debug('loading marker file %s' % self.fn_markers)
# loads just plain markers:
markers = marker.load_markers(self.fn_markers)
if self.fn_events:
markers.extend(
[marker.EventMarker(e) for e in
load_events(self.fn_events)])
if self.sort_markers:
logger.info('sorting markers!')
markers.sort(key=lambda x: x.tmin)
marker.associate_phases_to_events(markers)
markers_by_nsl = {}
for m in markers:
if not m.match_nsl(self.config.reference_target.codes[:3]):
continue
if m.get_phasename().upper() != self.align_phase:
continue
markers_by_nsl.setdefault(m.one_nslc()[:3], []).append(m)
assert(len(markers_by_nsl) == 1)
# filter markers that do not have an event assigned:
self.markers = list(markers_by_nsl.values())[0]
if not self.labeled:
dummy_event = Event(lat=0., lon=0., depth=0.)
for m in self.markers:
if not m.get_event():
m.set_event(dummy_event)
self.markers = [m for m in self.markers if m.get_event() is not None]
if not len(self.markers):
raise Exception('No markers left in dataset')
self.config.channels = list(self.data_pile.nslc_ids.keys())
self.config.channels.sort()
def make_pile(self):
'''Create a pile.
To be overloaded in subclass. The default implementation just calls
:py:func:`pyrocko.pile.make_pile` to create a pile from command line arguments.
'''
cachedirname = '/tmp/snuffle_cache_%s' % os.environ['USER']
sources = self._cli_params.get('sources', sys.argv[1:])
return pile.make_pile(sources, cachedirname=cachedirname, regex=self._cli_params['regex'], fileformat=self._cli_params['format'])
def load_eqt_folder(data_paths,
tinc,
path,
tmin="2021-05-26 06:20:03.800",
tmax="2016-02-12 06:20:03.800",
minlat=49.1379,
maxlat=49.1879,
minlon=8.1223,
maxlon=8.1723,
channels=["EH" + "[ZNE]"],
client_list=["BGR"],
download=True,
seiger=True,
selection=None,
path_waveforms=None,
sds=None,
stream=False,
data_format='mseed',
deltat_want=100,
tstart=None,
tstop=None,
hf=8,
lf=2,
models=[]):
data_pile = pile.make_pile(data_paths,
fileformat=data_format,
show_progress=False)
iter_chunked(tinc,
path,
data_pile,
tmin=tmin,
tmax=tmax,
minlat=minlat,
maxlat=maxlat,
minlon=minlon,
maxlon=maxlon,
channels=channels,
client_list=client_list,
download=download,
seiger=seiger,
selection=selection,
path_waveforms=path_waveforms,
stream=stream,
sds=sds,
reject_blacklisted=None,
tpad=0,
tstart=None,
tstop=None,
hf=hf,
lf=lf,
models=models)
def setup(self):
self.data_pile = pile.make_pile(
self.data_paths, fileformat=self.data_format)
if self.data_pile.is_empty():
sys.exit('Data pile is empty!')
self.deltat_want = self.config.deltat_want or \
min(self.data_pile.deltats.keys())
self.n_samples = int(
(self.config.sample_length + self.config.tpad) / self.deltat_want)
logger.debug('loading marker file %s' % self.fn_markers)
# loads just plain markers:
markers = marker.load_markers(self.fn_markers)
if self.fn_events:
markers.extend(
[marker.EventMarker(e) for e in
load_events(self.fn_events)])
marker.associate_phases_to_events(markers)
markers = [m for m in markers if isinstance(m, marker.PhaseMarker)]
markers_dict = defaultdict(list)
for m in markers:
if m.get_phasename().upper() != self.align_phase:
continue
markers_dict[m.get_event()].append(m)
self.markers = []
for e, _markers in markers_dict.items():
first = min(_markers, key=lambda x: x.tmin)
self.markers.append(first)
if not self.labeled:
dummy_event = Event(lat=0., lon=0., depth=0.)
for m in self.markers:
if not m.get_event():
m.set_event(dummy_event)
self.markers = [m for m in self.markers if m.get_event() is not None]
if not len(self.markers):
raise Exception('No markers left in dataset')
self.config.channels = list(self.data_pile.nslc_ids.keys())
self.config.channels.sort()
def testMisfitOfSameTracesDtDifferentShifted(self):
"""
Tests:
Different length
Different delta t
Shifted
L2-Norm
L1-Norm
time- and frequency-domain
"""
test_file = os.path.join(os.path.dirname(__file__),
'../examples/1989.072.evt.mseed')
p = pile.make_pile(test_file, show_progress=False)
rt = p.all()[0]
tt = rt.copy()
# make downsampled, chopped copies:
deltats = [0.5, 1.0, 2.0]
tt.chop(tmin=rt.tmin + 10, tmax=rt.tmax - 15)
tts = [tt.copy() for i in range(len(deltats))]
[t.downsample_to(deltats[i]) for i, t in enumerate(tts)]
# shift traces:
t_shifts = [1.0, 0.49999, 0.5]
for ts in t_shifts:
tts_shifted = [t.copy() for t in tts]
map(lambda x: x.shift(ts), tts_shifted)
tts.extend(tts_shifted)
a = rt.tmin
d = rt.tmax
b = a + (d - a) / 10
c = d - (d - a) / 10
taper = trace.CosTaper(a, b, c, d)
fresponse = trace.FrequencyResponse()
norms = [1, 2]
domains = ['time_domain', 'frequency_domain', 'envelope', 'absolute']
setups = [
trace.MisfitSetup(norm=n,
taper=taper,
domain=domain,
filter=fresponse) for domain in domains
for n in norms
]
for cand in tts:
for setup in setups:
m, n = rt.misfit(candidate=cand, setup=setup)
self.assertNotEqual(m, None, 'misfit\'s m is None')
def testMisfitOfSameTracesDtDifferentShifted(self):
"""
Tests:
Different length
Different delta t
Shifted
L2-Norm
L1-Norm
time- and frequency-domain
"""
test_file = os.path.join(
os.path.dirname(__file__),
'../examples/1989.072.evt.mseed')
p = pile.make_pile(test_file, show_progress=False)
rt = p.all()[0]
tt = rt.copy()
# make downsampled, chopped copies:
deltats = [0.5, 1.0, 2.0]
tt.chop(tmin=rt.tmin+10, tmax=rt.tmax-15)
tts = [tt.copy() for i in range(len(deltats))]
[t.downsample_to(deltats[i]) for i, t in enumerate(tts)]
# shift traces:
t_shifts = [1.0, 0.49999, 0.5]
for ts in t_shifts:
tts_shifted = [t.copy() for t in tts]
map(lambda x: x.shift(ts), tts_shifted)
tts.extend(tts_shifted)
a = rt.tmin
d = rt.tmax
b = a+(d-a)/10
c = d-(d-a)/10
taper = trace.CosTaper(a, b, c, d)
fresponse = trace.FrequencyResponse()
norms = [1, 2]
domains = ['time_domain', 'frequency_domain', 'envelope', 'absolute']
setups = [trace.MisfitSetup(
norm=n,
taper=taper,
domain=domain,
filter=fresponse) for domain in domains for n in norms]
for cand in tts:
for setup in setups:
m, n = rt.misfit(candidate=cand, setup=setup)
self.assertNotEqual(m, None, 'misfit\'s m is None')
def setUpClass(cls):
'''
Create a reusable snuffler instance for all tests cases.
'''
super(GUITest, cls).setUpClass()
cls.snuffler = Snuffler() # noqa
fpath = common.test_data_file('test2.mseed')
p = make_pile(fpath, show_progress=False)
cls.win = SnufflerWindow(pile=p)
cls.pile_viewer = cls.win.pile_viewer
pv = cls.pile_viewer
cls.main_control_defaults = dict(
highpass_control=pv.highpass_control.get_value(),
lowpass_control=pv.lowpass_control.get_value(),
gain_control=pv.gain_control.get_value(),
rot_control=pv.rot_control.get_value())
def setUpClass(cls):
'''
Create a reusable snuffler instance for all tests cases.
'''
super(GUITest, cls).setUpClass()
if no_gui: # nosetests runs this even when class is has @skip
return
cls.snuffler = Snuffler() # noqa
fpath = common.test_data_file('test2.mseed')
p = make_pile(fpath, show_progress=False)
cls.win = SnufflerWindow(pile=p)
cls.pile_viewer = cls.win.pile_viewer
cls.viewer = cls.win.pile_viewer.viewer
pv = cls.pile_viewer
cls.main_control_defaults = dict(
highpass_control=pv.highpass_control.get_value(),
lowpass_control=pv.lowpass_control.get_value(),
gain_control=pv.gain_control.get_value(),
rot_control=pv.rot_control.get_value())
def setUpClass(cls):
'''
Create a reusable snuffler instance for all tests cases.
'''
super(GUITest, cls).setUpClass()
if no_gui: # nosetests runs this even when class is has @skip
return
from pyrocko.gui import snuffler as sm
cls.snuffler = sm.get_snuffler_instance()
fpath = common.test_data_file('test2.mseed')
p = make_pile(fpath, show_progress=False)
cls.win = SnufflerWindow(pile=p)
cls.pile_viewer = cls.win.pile_viewer
cls.viewer = cls.win.pile_viewer.viewer
pv = cls.pile_viewer
cls.main_control_defaults = dict(
highpass_control=pv.highpass_control.get_value(),
lowpass_control=pv.lowpass_control.get_value(),
gain_control=pv.gain_control.get_value(),
rot_control=pv.rot_control.get_value())
def search(config,
override_tmin=None,
override_tmax=None,
show_detections=False,
show_movie=False,
show_window_traces=False,
force=False,
stop_after_first=False,
nparallel=6,
save_imax=False,
bark=False):
fp = config.expand_path
run_path = fp(config.run_path)
# if op.exists(run_path):
# if force:
# shutil.rmtree(run_path)
# else:
# raise common.LassieError(
# 'run directory already exists: %s' %
# run_path)
util.ensuredir(run_path)
write_config(config, op.join(run_path, 'config.yaml'))
ifm_path_template = config.get_ifm_path_template()
detections_path = config.get_detections_path()
events_path = config.get_events_path()
figures_path_template = config.get_figures_path_template()
config.setup_image_function_contributions()
ifcs = config.image_function_contributions
grid = config.get_grid()
receivers = config.get_receivers()
norm_map = gridmod.geometrical_normalization(grid, receivers)
data_paths = fp(config.data_paths)
for data_path in fp(data_paths):
if not op.exists(data_path):
pass
p = pile.make_pile(data_paths, fileformat='detect')
if p.is_empty():
raise common.LassieError('no usable waveforms found')
for ifc in ifcs:
ifc.prescan(p)
shift_tables = []
tshift_minmaxs = []
for ifc in ifcs:
shift_tables.append(ifc.get_table(grid, receivers))
tshift_minmaxs.append(num.nanmin(shift_tables[-1]))
tshift_minmaxs.append(num.nanmax(shift_tables[-1]))
fsmooth_min = min(ifc.get_fsmooth() for ifc in ifcs)
tshift_min = min(tshift_minmaxs)
tshift_max = max(tshift_minmaxs)
if config.detector_tpeaksearch is not None:
tpeaksearch = config.detector_tpeaksearch
else:
tpeaksearch = (tshift_max - tshift_min) + 1.0 / fsmooth_min
tpad = max(ifc.get_tpad() for ifc in ifcs) + \
(tshift_max - tshift_min) + tpeaksearch
tinc = (tshift_max - tshift_min) * 10. + 3.0 * tpad
tavail = p.tmax - p.tmin
tinc = min(tinc, tavail - 2.0 * tpad)
if tinc <= 0:
raise common.LassieError('available waveforms too short \n'
'required: %g s\n'
'available: %g s\n' % (2. * tpad, tavail))
blacklist = set(tuple(s.split('.')) for s in config.blacklist)
whitelist = set(tuple(s.split('.')) for s in config.whitelist)
distances = grid.distances(receivers)
distances_to_grid = num.min(distances, axis=0)
distance_min = num.min(distances)
distance_max = num.max(distances)
station_index = dict(
(rec.codes, i) for (i, rec) in enumerate(receivers)
if rec.codes not in blacklist and (
not whitelist or rec.codes in whitelist) and (
config.distance_max is None
or distances_to_grid[i] <= config.distance_max))
check_data_consistency(p, config)
deltat_cf = max(p.deltats.keys())
assert deltat_cf > 0.0
while True:
if not all(ifc.deltat_cf_is_available(deltat_cf * 2) for ifc in ifcs):
break
deltat_cf *= 2
logger.info('CF lassie sampling interval (rate): %g s (%g Hz)' %
(deltat_cf, 1.0 / deltat_cf))
ngridpoints = grid.size()
logger.info('number of grid points: %i' % ngridpoints)
logger.info('minimum source-receiver distance: %g m' % distance_min)
logger.info('maximum source-receiver distance: %g m' % distance_max)
logger.info('minimum travel-time: %g s' % tshift_min)
logger.info('maximum travel-time: %g s' % tshift_max)
idetection = 0
tmin = override_tmin or config.tmin or p.tmin + tpad
tmax = override_tmax or config.tmax or p.tmax - tpad
events = config.get_events()
twindows = []
if events is not None:
for ev in events:
if tmin <= ev.time <= tmax:
twindows.append(
(ev.time + tshift_min - (tshift_max - tshift_min) *
config.event_time_window_factor,
ev.time + tshift_min + (tshift_max - tshift_min) *
config.event_time_window_factor))
else:
twindows.append((tmin, tmax))
for iwindow_group, (tmin_win, tmax_win) in enumerate(twindows):
nwin = int(math.ceil((tmax_win - tmin_win) / tinc))
logger.info('start processing time window group %i/%i: %s - %s' %
(iwindow_group + 1, len(twindows),
util.time_to_str(tmin_win), util.time_to_str(tmax_win)))
logger.info('number of time windows: %i' % nwin)
logger.info('time window length: %g s' % (tinc + 2.0 * tpad))
logger.info('time window payload: %g s' % tinc)
logger.info('time window padding: 2 x %g s' % tpad)
logger.info('time window overlap: %g%%' % (100.0 * 2.0 * tpad /
(tinc + 2.0 * tpad)))
iwin = -1
for trs in p.chopper(
tmin=tmin_win,
tmax=tmax_win,
tinc=tinc,
tpad=tpad,
want_incomplete=config.fill_incomplete_with_zeros,
trace_selector=lambda tr: tr.nslc_id[:3] in station_index):
iwin += 1
trs_ok = []
for tr in trs:
if tr.ydata.size == 0:
logger.warn('skipping empty trace: %s.%s.%s.%s' %
tr.nslc_id)
continue
if not num.all(num.isfinite(tr.ydata)):
logger.warn('skipping trace because of invalid values: '
'%s.%s.%s.%s' % tr.nslc_id)
continue
trs_ok.append(tr)
trs = trs_ok
if not trs:
continue
logger.info('processing time window %i/%i: %s - %s' %
(iwin + 1, nwin, util.time_to_str(
trs[0].wmin), util.time_to_str(trs[0].wmax)))
wmin = trs[0].wmin
wmax = trs[0].wmax
if config.fill_incomplete_with_zeros:
trs = zero_fill(trs, wmin - tpad, wmax + tpad)
t0 = math.floor(wmin / deltat_cf) * deltat_cf
iwmin = int(round((wmin - tpeaksearch - t0) / deltat_cf))
iwmax = int(round((wmax + tpeaksearch - t0) / deltat_cf))
lengthout = iwmax - iwmin + 1
pdata = []
trs_debug = []
parstack_params = []
for iifc, ifc in enumerate(ifcs):
dataset = ifc.preprocess(trs, wmin - tpeaksearch,
wmax + tpeaksearch,
tshift_max - tshift_min, deltat_cf)
if not dataset:
continue
nstations_selected = len(dataset)
nsls_selected, trs_selected = zip(*dataset)
for tr in trs_selected:
tr.meta = {'tabu': True}
trs_debug.extend(trs + list(trs_selected))
istations_selected = num.array(
[station_index[nsl] for nsl in nsls_selected],
dtype=num.int)
arrays = [tr.ydata.astype(num.float) for tr in trs_selected]
offsets = num.array([
int(round((tr.tmin - t0) / deltat_cf))
for tr in trs_selected
],
dtype=num.int32)
w = ifc.get_weights(nsls_selected)
weights = num.ones((ngridpoints, nstations_selected))
weights *= w[num.newaxis, :]
weights *= ifc.weight
shift_table = shift_tables[iifc]
ok = num.isfinite(shift_table[:, istations_selected])
bad = num.logical_not(ok)
shifts = -num.round(shift_table[:, istations_selected] /
deltat_cf).astype(num.int32)
weights[bad] = 0.0
shifts[bad] = num.max(shifts[ok])
pdata.append((list(trs_selected), shift_table, ifc))
parstack_params.append((arrays, offsets, shifts, weights))
if config.stacking_blocksize is not None:
ipstep = config.stacking_blocksize
frames = None
else:
ipstep = lengthout
frames = num.zeros((ngridpoints, lengthout))
twall_start = time.time()
frame_maxs = num.zeros(lengthout)
frame_argmaxs = num.zeros(lengthout, dtype=num.int)
ipmin = iwmin
while ipmin < iwmin + lengthout:
ipsize = min(ipstep, iwmin + lengthout - ipmin)
if ipstep == lengthout:
frames_p = frames
else:
frames_p = num.zeros((ngridpoints, ipsize))
for (arrays, offsets, shifts, weights) in parstack_params:
frames_p, _ = parstack(arrays,
offsets,
shifts,
weights,
0,
offsetout=ipmin,
lengthout=ipsize,
result=frames_p,
nparallel=nparallel,
impl='openmp')
if config.sharpness_normalization:
frame_p_maxs = frames_p.max(axis=0)
frame_p_means = num.abs(frames_p).mean(axis=0)
frames_p *= (frame_p_maxs / frame_p_means)[num.newaxis, :]
frames_p *= norm_map[:, num.newaxis]
if config.ifc_count_normalization:
frames_p *= 1.0 / len(ifcs)
frame_maxs[ipmin-iwmin:ipmin-iwmin+ipsize] = \
frames_p.max(axis=0)
frame_argmaxs[ipmin-iwmin:ipmin-iwmin+ipsize] = \
pargmax(frames_p)
ipmin += ipstep
del frames_p
twall_end = time.time()
logger.info('wallclock time for stacking: %g s' %
(twall_end - twall_start))
tmin_frames = t0 + iwmin * deltat_cf
tr_stackmax = trace.Trace('',
'SMAX',
'',
'',
tmin=tmin_frames,
deltat=deltat_cf,
ydata=frame_maxs)
tr_stackmax.meta = {'tabu': True}
trs_debug.append(tr_stackmax)
if show_window_traces:
trace.snuffle(trs_debug)
ydata_window = tr_stackmax.chop(wmin, wmax,
inplace=False).get_ydata()
logger.info('CF stats: min %g, max %g, median %g' %
(num.min(ydata_window), num.max(ydata_window),
num.median(ydata_window)))
if nstations_selected != 17:
logger.info(
'Warning, station outage detected! Nr of station operable: %s'
% nstations_selected)
detector_threshold_seiger = config.detector_threshold - (
(17 - nstations_selected) * 4
) # 17 is maximum number of seiger stations, 4 is a mean baseline for noise
if nstations_selected != 17:
logger.info(
'Warning, station outage detected! Nr of station operable: %s, threshold now: %s'
% (nstations_selected, detector_threshold_seiger))
tpeaks, apeaks = list(
zip(*[(tpeak, apeak) for (tpeak, apeak) in zip(
*tr_stackmax.peaks(detector_threshold_seiger, tpeaksearch))
if wmin <= tpeak and tpeak < wmax])) or ([], [])
tr_stackmax_indx = tr_stackmax.copy(data=False)
tr_stackmax_indx.set_ydata(frame_argmaxs.astype(num.int32))
tr_stackmax_indx.set_location('i')
for (tpeak, apeak) in zip(tpeaks, apeaks):
iframe = int(round((tpeak - tmin_frames) / deltat_cf))
imax = frame_argmaxs[iframe]
latpeak, lonpeak, xpeak, ypeak, zpeak = \
grid.index_to_location(imax)
idetection += 1
detection = Detection(id='%06i' % idetection,
time=tpeak,
location=geo.Point(lat=float(latpeak),
lon=float(lonpeak),
x=float(xpeak),
y=float(ypeak),
z=float(zpeak)),
ifm=float(apeak))
if bark:
common.bark()
logger.info('detection found: %s' % str(detection))
f = open(detections_path, 'a')
f.write(
'%06i %s %g %g %g %g %g %g\n' %
(idetection,
util.time_to_str(tpeak, format='%Y-%m-%d %H:%M:%S.6FRAC'),
apeak, latpeak, lonpeak, xpeak, ypeak, zpeak))
f.close()
ev = detection.get_event()
f = open(events_path, 'a')
model.dump_events([ev], stream=f)
f.close()
if show_detections or config.save_figures:
fmin = min(ifc.fmin for ifc in ifcs)
fmax = min(ifc.fmax for ifc in ifcs)
fn = figures_path_template % {
'id': util.tts(t0).replace(" ", "T"),
'format': 'png'
}
util.ensuredirs(fn)
if frames is not None:
frames_p = frames
tmin_frames_p = tmin_frames
iframe_p = iframe
else:
iframe_min = max(
0, int(round(iframe - tpeaksearch / deltat_cf)))
iframe_max = min(
lengthout - 1,
int(round(iframe + tpeaksearch / deltat_cf)))
ipsize = iframe_max - iframe_min + 1
frames_p = num.zeros((ngridpoints, ipsize))
tmin_frames_p = tmin_frames + iframe_min * deltat_cf
iframe_p = iframe - iframe_min
for (arrays, offsets, shifts, weights) \
in parstack_params:
frames_p, _ = parstack(arrays,
offsets,
shifts,
weights,
0,
offsetout=iwmin +
iframe_min,
lengthout=ipsize,
result=frames_p,
nparallel=nparallel,
impl='openmp')
if config.sharpness_normalization:
frame_p_maxs = frames_p.max(axis=0)
frame_p_means = num.abs(frames_p).mean(axis=0)
frames_p *= (frame_p_maxs /
frame_p_means)[num.newaxis, :]
frames_p *= norm_map[:, num.newaxis]
if config.ifc_count_normalization:
frames_p *= 1.0 / len(ifcs)
try:
plot.plot_detection(grid,
receivers,
frames_p,
tmin_frames_p,
deltat_cf,
imax,
iframe_p,
xpeak,
ypeak,
zpeak,
tr_stackmax,
tpeaks,
apeaks,
detector_threshold_seiger,
wmin,
wmax,
pdata,
trs,
fmin,
fmax,
idetection,
tpeaksearch,
movie=show_movie,
show=show_detections,
save_filename=fn,
event=ev)
except:
pass
del frames_p
if stop_after_first:
return
tr_stackmax.chop(wmin, wmax)
tr_stackmax_indx.chop(wmin, wmax)
if save_imax is True:
io.save([tr_stackmax, tr_stackmax_indx], ifm_path_template)
del frames
logger.info('end processing time window group: %s - %s' %
(util.time_to_str(tmin_win), util.time_to_str(tmax_win)))
cat = Catalog()
files = glob("%s/../figures/*qml*" % run_path)
files.sort(key=os.path.getmtime)
for file in files:
cat_read = read_events(file)
for event in cat_read:
cat.append(event)
cat.write("%s/../all_events_stacking.qml" % run_path, format="QUAKEML")
from pyrocko import io, trace, pile
from pyrocko.example import get_example_data
get_example_data('test.mseed')
traces = io.load('test.mseed')
traces[0].snuffle() # look at a single trace
trace.snuffle(traces) # look at a bunch of traces
# do something with the traces:
new_traces = []
for tr in traces:
new = tr.copy()
new.whiten()
# to allow the viewer to distinguish the traces
new.set_location('whitened')
new_traces.append(new)
trace.snuffle(traces + new_traces)
# it is also possible to 'snuffle' a pile:
p = pile.make_pile(['test.mseed'])
p.snuffle()
#fbands.apppend([4.0, 10.])
phases = LocalEngine(store_superdirs=['/data/stores'],
default_store_id='globalttt').get_store()
#filenames = glob.glob('data/*.mseed')
#filenames = glob.glob('/data/webnet/waveform_R/2008/*.mseed')
#datapath = '/data/webnet/mseed/2008'
#datapath = '/data/webnet/waveform_R/2008'
#datapath = '/data/share/Res_all_NKC'
datapath = '/media/usb0/Res_all_NKC_taper'
#datapath = '/media/usb0/restituted_pyrocko'
stations = model.load_stations('../data/stations.pf')
reference_id = 'NKC'
references = {}
data_pile = pile.make_pile(datapath, selector='rest_*')
fband = {'order': 4, 'corner_hp': 1.0, 'corner_lp': 4.}
window = StaticLengthWindow(static_length=30., phase_position=0.5)
taper = trace.CosFader(xfrac=0.25)
#event_selector = EventSelector(distmin=1000*km,
# distmax=20000*km,
# depthmin=2*km,
# depthmax=600*km,
# magmin=4.9)
candidate_fn = '../candidates2013.pf'
candidates = [
m.get_event() for m in gui_util.Marker.load_markers(candidate_fn)
def call(events,
data_paths,
fmin=1,
fmax=5,
show_restituded_traces=False,
stations=None,
needs_restitution=True,
make_markers=True,
show_plot=True,
modify_inplace=True):
events.sort(key=lambda ev: ev.time)
stations_dict = dict((s.nsl(), s) for s in stations)
markers = []
local_magnitudes = []
p = pile.make_pile(data_paths, fileformat="mseed", show_progress=False)
for event in events:
mags = defaultdict(list)
tpad = 2. / fmin
def trace_selector(tr):
c = tr.channel.upper()
return c.endswith('E') or c.endswith('N') or \
tr.location.endswith('_rest')
distances = {}
rest_traces = []
event2 = copy.deepcopy(event)
for tr in get_traces(p, event, stations_dict.values(), trace_selector,
tpad):
nslc = tr.nslc_id
try:
tr.highpass(4, fmin, nyquist_exception=True)
tr.lowpass(4, fmax, nyquist_exception=True)
except:
pass
try:
station = stations_dict[nslc[:3]]
except KeyError as e:
print(e)
continue
if needs_restitution is True:
resp = get_response(nslc)
try:
tr_vel = tr.transfer(tfade=tpad,
freqlimits=(fmin * 0.5, fmin, fmax,
fmax * 2.0),
transfer_function=resp,
invert=True)
except trace.TraceTooShort as e:
continue
else:
try:
tr_vel = tr.transfer(
tfade=tpad,
freqlimits=(fmin * 0.5, fmin, fmax, fmax * 2.0),
transfer_function=wood_anderson_response,
invert=False)
except trace.TraceTooShort as e:
continue
distance = orthodrome.distance_accurate50m(event, station)
tr_vel.set_codes(location=tr_vel.location + '_rest')
tr_vel.meta = dict(tabu=True)
t_of_max, amplitude = tr_vel.absmax()
if show_restituded_traces:
rest_traces.append(tr_vel)
m_nslc = tr_vel.nslc_id
else:
m_nslc = tr.nslc_id
mag = local_magnitude(distance, amplitude)
if make_markers is True:
markers.append(
PhaseMarker([m_nslc],
t_of_max,
t_of_max,
1,
phasename='%3.1f' % mag,
event=event2))
mags[nslc[:2]].append(mag)
distances[nslc[:2]] = distance
if not mags:
continue
for k in mags:
mags[k] = max(mags[k])
local_magnitude = round(num.median(list(mags.values())), 1)
if show_plot is True:
data = []
for k in mags:
data.append((distances[k], mags[k]))
dists, mags_arr = num.array(data).T
dists /= km
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
axes.plot(dists, mags_arr, 'o', color=to01(graph_colors[0]))
for x, y, label in zip(dists, mags_arr, mags.keys()):
axes.text(x, y, '.'.join(label))
axes.axhline(local_magnitude, color=to01(graph_colors[0]))
mag_std = num.std(list(mags.values()))
msg = 'local magnitude: %s, std: %s' % \
(round(local_magnitude, 1),
round(mag_std, 1))
axes.text(max(dists),
local_magnitude,
msg,
verticalalignment='bottom',
horizontalalignment='right')
axes.axhspan(local_magnitude - mag_std,
local_magnitude + mag_std,
alpha=0.1)
axes.set_xlabel('Distance [km]')
axes.set_ylabel('Local Magnitude')
plt.savefig("ml_%s.png" % event.name)
local_magnitudes.append(local_magnitude)
if modify_inplace is True:
event.magnitude = local_magnitude
def main():
parser = OptionParser(usage=usage, description=description)
parser.add_option('--force',
dest='force',
action='store_true',
default=False,
help='allow recreation of output <directory>')
parser.add_option('--debug',
dest='debug',
action='store_true',
default=False,
help='print debugging information to stderr')
parser.add_option('--dry-run',
dest='dry_run',
action='store_true',
default=False,
help='show available stations/channels and exit '
'(do not download waveforms)')
parser.add_option('--continue',
dest='continue_',
action='store_true',
default=False,
help='continue download after a accident')
parser.add_option('--local-data',
dest='local_data',
action='append',
help='add file/directory with local data')
parser.add_option('--local-stations',
dest='local_stations',
action='append',
help='add local stations file')
parser.add_option('--selection',
dest='selection_file',
action='append',
help='add local stations file')
parser.add_option(
'--local-responses-resp',
dest='local_responses_resp',
action='append',
help='add file/directory with local responses in RESP format')
parser.add_option('--local-responses-pz',
dest='local_responses_pz',
action='append',
help='add file/directory with local pole-zero responses')
parser.add_option(
'--local-responses-stationxml',
dest='local_responses_stationxml',
help='add file with local response information in StationXML format')
parser.add_option(
'--window',
dest='window',
default='full',
help='set time window to choose [full, p, "<time-start>,<time-end>"'
'] (time format is YYYY-MM-DD HH:MM:SS)')
parser.add_option(
'--out-components',
choices=['enu', 'rtu'],
dest='out_components',
default='rtu',
help='set output component orientations to radial-transverse-up [rtu] '
'(default) or east-north-up [enu]')
parser.add_option('--out-units',
choices=['M', 'M/S', 'M/S**2'],
dest='output_units',
default='M',
help='set output units to displacement "M" (default),'
' velocity "M/S" or acceleration "M/S**2"')
parser.add_option(
'--padding-factor',
type=float,
default=3.0,
dest='padding_factor',
help='extend time window on either side, in multiples of 1/<fmin_hz> '
'(default: 5)')
parser.add_option(
'--zero-padding',
dest='zero_pad',
action='store_true',
default=False,
help='Extend traces by zero-padding if clean restitution requires'
'longer windows')
parser.add_option(
'--credentials',
dest='user_credentials',
action='append',
default=[],
metavar='SITE,USER,PASSWD',
help='user credentials for specific site to access restricted data '
'(this option can be repeated)')
parser.add_option(
'--token',
dest='auth_tokens',
metavar='SITE,FILENAME',
action='append',
default=[],
help='user authentication token for specific site to access '
'restricted data (this option can be repeated)')
parser.add_option(
'--sites',
dest='sites',
metavar='SITE1,SITE2,...',
default='geofon,iris,orfeus',
help='sites to query (available: %s, default: "%%default"' %
', '.join(g_sites_available))
parser.add_option(
'--band-codes',
dest='priority_band_code',
metavar='V,L,M,B,H,S,E,...',
default='B,H',
help='select and prioritize band codes (default: %default)')
parser.add_option(
'--instrument-codes',
dest='priority_instrument_code',
metavar='H,L,G,...',
default='H,L',
help='select and prioritize instrument codes (default: %default)')
parser.add_option('--radius-min',
dest='radius_min',
metavar='VALUE',
default=0.0,
type=float,
help='minimum radius [km]')
parser.add_option('--nstations-wanted',
dest='nstations_wanted',
metavar='N',
type=int,
help='number of stations to select initially')
(options, args) = parser.parse_args(sys.argv[1:])
print('Parsed arguments:', args)
if len(args) not in (10, 7, 6):
parser.print_help()
sys.exit(1)
if options.debug:
util.setup_logging(program_name, 'debug')
else:
util.setup_logging(program_name, 'info')
if options.local_responses_pz and options.local_responses_resp:
logger.critical('cannot use local responses in PZ and RESP '
'format at the same time')
sys.exit(1)
n_resp_opt = 0
for resp_opt in (options.local_responses_pz, options.local_responses_resp,
options.local_responses_stationxml):
if resp_opt:
n_resp_opt += 1
if n_resp_opt > 1:
logger.critical('can only handle local responses from either PZ or '
'RESP or StationXML. Cannot yet merge different '
'response formats.')
sys.exit(1)
if options.local_responses_resp and not options.local_stations:
logger.critical('--local-responses-resp can only be used '
'when --stations is also given.')
sys.exit(1)
try:
ename = ''
magnitude = None
mt = None
if len(args) == 10:
time = util.str_to_time(args[1] + ' ' + args[2])
lat = float(args[3])
lon = float(args[4])
depth = float(args[5]) * km
iarg = 6
elif len(args) == 7:
if args[2].find(':') == -1:
sname_or_date = None
lat = float(args[1])
lon = float(args[2])
event = None
time = None
else:
sname_or_date = args[1] + ' ' + args[2]
iarg = 3
elif len(args) == 6:
sname_or_date = args[1]
iarg = 2
if len(args) in (7, 6) and sname_or_date is not None:
events = get_events_by_name_or_date([sname_or_date],
catalog=geofon)
if len(events) == 0:
logger.critical('no event found')
sys.exit(1)
elif len(events) > 1:
logger.critical('more than one event found')
sys.exit(1)
event = events[0]
time = event.time
lat = event.lat
lon = event.lon
depth = event.depth
ename = event.name
magnitude = event.magnitude
mt = event.moment_tensor
radius = float(args[iarg]) * km
fmin = float(args[iarg + 1])
sample_rate = float(args[iarg + 2])
eventname = args[iarg + 3]
cwd = str(sys.argv[1])
event_dir = op.join(cwd, 'data', 'events', eventname)
output_dir = op.join(event_dir, 'waveforms')
except:
raise
parser.print_help()
sys.exit(1)
if options.force and op.isdir(event_dir):
if not options.continue_:
shutil.rmtree(event_dir)
if op.exists(event_dir) and not options.continue_:
logger.critical(
'directory "%s" exists. Delete it first or use the --force option'
% event_dir)
sys.exit(1)
util.ensuredir(output_dir)
if time is not None:
event = model.Event(time=time,
lat=lat,
lon=lon,
depth=depth,
name=ename,
magnitude=magnitude,
moment_tensor=mt)
if options.window == 'full':
if event is None:
logger.critical('need event for --window=full')
sys.exit(1)
low_velocity = 1500.
timewindow = VelocityWindow(low_velocity,
tpad=options.padding_factor / fmin)
tmin, tmax = timewindow(time, radius, depth)
elif options.window == 'p':
if event is None:
logger.critical('need event for --window=p')
sys.exit(1)
phases = list(map(cake.PhaseDef, 'P p'.split()))
emod = cake.load_model()
tpad = options.padding_factor / fmin
timewindow = PhaseWindow(emod, phases, -tpad, tpad)
arrivaltimes = []
for dist in num.linspace(0, radius, 20):
try:
arrivaltimes.extend(timewindow(time, dist, depth))
except NoArrival:
pass
if not arrivaltimes:
logger.error('required phase arrival not found')
sys.exit(1)
tmin = min(arrivaltimes)
tmax = max(arrivaltimes)
else:
try:
stmin, stmax = options.window.split(',')
tmin = util.str_to_time(stmin.strip())
tmax = util.str_to_time(stmax.strip())
timewindow = FixedWindow(tmin, tmax)
except ValueError:
logger.critical('invalid argument to --window: "%s"' %
options.window)
sys.exit(1)
if event is not None:
event.name = eventname
tfade = tfade_factor / fmin
tpad = tfade
tmin -= tpad
tmax += tpad
tinc = None
priority_band_code = options.priority_band_code.split(',')
for s in priority_band_code:
if len(s) != 1:
logger.critical('invalid band code: %s' % s)
priority_instrument_code = options.priority_instrument_code.split(',')
for s in priority_instrument_code:
if len(s) != 1:
logger.critical('invalid instrument code: %s' % s)
station_query_conf = dict(latitude=lat,
longitude=lon,
minradius=options.radius_min * km * cake.m2d,
maxradius=radius * cake.m2d,
channel=','.join('%s??' % s
for s in priority_band_code))
target_sample_rate = sample_rate
fmax = target_sample_rate
# target_sample_rate = None
# priority_band_code = ['H', 'B', 'M', 'L', 'V', 'E', 'S']
priority_units = ['M/S', 'M', 'M/S**2']
# output_units = 'M'
sites = [x.strip() for x in options.sites.split(',') if x.strip()]
for site in sites:
if site not in g_sites_available:
logger.critical('unknown FDSN site: %s' % site)
sys.exit(1)
for s in options.user_credentials:
try:
site, user, passwd = s.split(',')
g_user_credentials[site] = user, passwd
except ValueError:
logger.critical('invalid format for user credentials: "%s"' % s)
sys.exit(1)
for s in options.auth_tokens:
try:
site, token_filename = s.split(',')
with open(token_filename, 'r') as f:
g_auth_tokens[site] = f.read()
except (ValueError, OSError, IOError):
logger.critical('cannot get token from file: %s' % token_filename)
sys.exit(1)
fn_template0 = \
'data_%(network)s.%(station)s.%(location)s.%(channel)s_%(tmin)s.mseed'
fn_template_raw = op.join(output_dir, 'raw', fn_template0)
fn_stations_raw = op.join(output_dir, 'stations.raw.txt')
fn_template_rest = op.join(output_dir, 'rest', fn_template0)
fn_commandline = op.join(output_dir, 'beatdown.command')
ftap = (ffade_factors[0] * fmin, fmin, fmax, ffade_factors[1] * fmax)
# chapter 1: download
sxs = []
for site in sites:
try:
extra_args = {
'iris': dict(matchtimeseries=True),
}.get(site, {})
extra_args.update(station_query_conf)
if site == 'geonet':
extra_args.update(starttime=tmin, endtime=tmax)
else:
extra_args.update(startbefore=tmax,
endafter=tmin,
includerestricted=(site in g_user_credentials
or site in g_auth_tokens))
logger.info('downloading channel information (%s)' % site)
sx = fdsn.station(site=site,
format='text',
level='channel',
**extra_args)
except fdsn.EmptyResult:
logger.error('No stations matching given criteria. (%s)' % site)
sx = None
if sx is not None:
sxs.append(sx)
if all(sx is None for sx in sxs) and not options.local_data:
sys.exit(1)
nsl_to_sites = defaultdict(list)
nsl_to_station = {}
if options.selection_file:
logger.info('using stations from stations file!')
stations = []
for fn in options.selection_file:
stations.extend(model.load_stations(fn))
nsls_selected = set(s.nsl() for s in stations)
else:
nsls_selected = None
for sx, site in zip(sxs, sites):
site_stations = sx.get_pyrocko_stations()
for s in site_stations:
nsl = s.nsl()
nsl_to_sites[nsl].append(site)
if nsl not in nsl_to_station:
if nsls_selected:
if nsl in nsls_selected:
nsl_to_station[nsl] = s
else:
nsl_to_station[
nsl] = s # using first site with this station
logger.info('number of stations found: %i' % len(nsl_to_station))
# station weeding
if options.nstations_wanted:
nsls_selected = None
stations_all = [
nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())
]
for s in stations_all:
s.set_event_relative_data(event)
stations_selected = weeding.weed_stations(stations_all,
options.nstations_wanted)[0]
nsls_selected = set(s.nsl() for s in stations_selected)
logger.info('number of stations selected: %i' % len(nsls_selected))
if tinc is None:
tinc = 3600.
have_data = set()
if options.continue_:
fns = glob.glob(fn_template_raw % starfill())
p = pile.make_pile(fns)
else:
fns = []
have_data_site = {}
could_have_data_site = {}
for site in sites:
have_data_site[site] = set()
could_have_data_site[site] = set()
available_through = defaultdict(set)
it = 0
nt = int(math.ceil((tmax - tmin) / tinc))
for it in range(nt):
tmin_win = tmin + it * tinc
tmax_win = min(tmin + (it + 1) * tinc, tmax)
logger.info('time window %i/%i (%s - %s)' %
(it + 1, nt, util.tts(tmin_win), util.tts(tmax_win)))
have_data_this_window = set()
if options.continue_:
trs_avail = p.all(tmin=tmin_win, tmax=tmax_win, load_data=False)
for tr in trs_avail:
have_data_this_window.add(tr.nslc_id)
for site, sx in zip(sites, sxs):
if sx is None:
continue
selection = []
channels = sx.choose_channels(
target_sample_rate=target_sample_rate,
priority_band_code=priority_band_code,
priority_units=priority_units,
priority_instrument_code=priority_instrument_code,
timespan=(tmin_win, tmax_win))
for nslc in sorted(channels.keys()):
if nsls_selected is not None and nslc[:3] not in nsls_selected:
continue
could_have_data_site[site].add(nslc)
if nslc not in have_data_this_window:
channel = channels[nslc]
if event:
lat_, lon_ = event.lat, event.lon
else:
lat_, lon_ = lat, lon
try:
dist = orthodrome.distance_accurate50m_numpy(
lat_, lon_, channel.latitude.value,
channel.longitude.value)
except:
dist = orthodrome.distance_accurate50m_numpy(
lat_, lon_, channel.latitude, channel.longitude)
if event:
depth_ = event.depth
time_ = event.time
else:
depth_ = None
time_ = None
tmin_, tmax_ = timewindow(time_, dist, depth_)
tmin_this = tmin_ - tpad
tmax_this = float(tmax_ + tpad)
tmin_req = max(tmin_win, tmin_this)
tmax_req = min(tmax_win, tmax_this)
if channel.sample_rate:
try:
deltat = 1.0 / int(channel.sample_rate.value)
except:
deltat = 1.0 / int(channel.sample_rate)
else:
deltat = 1.0
if tmin_req < tmax_req:
logger.debug('deltat %f' % deltat)
# extend time window by some samples because otherwise
# sometimes gaps are produced
# apparently the WS are only sensitive to full seconds
# round to avoid gaps, increase safetiy window
selection.append(nslc +
(math.floor(tmin_req - deltat * 20.0),
math.ceil(tmax_req + deltat * 20.0)))
if options.dry_run:
for (net, sta, loc, cha, tmin, tmax) in selection:
available_through[net, sta, loc, cha].add(site)
else:
neach = 100
i = 0
nbatches = ((len(selection) - 1) // neach) + 1
while i < len(selection):
selection_now = selection[i:i + neach]
f = tempfile.NamedTemporaryFile()
try:
sbatch = ''
if nbatches > 1:
sbatch = ' (batch %i/%i)' % (
(i // neach) + 1, nbatches)
logger.info('downloading data (%s)%s' % (site, sbatch))
data = fdsn.dataselect(site=site,
selection=selection_now,
**get_user_credentials(site))
while True:
buf = data.read(1024)
if not buf:
break
f.write(buf)
f.flush()
trs = io.load(f.name)
for tr in trs:
tr.fix_deltat_rounding_errors()
logger.debug('cutting window: %f - %f' %
(tmin_win, tmax_win))
logger.debug(
'available window: %f - %f, nsamples: %g' %
(tr.tmin, tr.tmax, tr.ydata.size))
try:
logger.debug('tmin before snap %f' % tr.tmin)
tr.snap(interpolate=True)
logger.debug('tmin after snap %f' % tr.tmin)
tr.chop(tmin_win,
tmax_win,
snap=(math.floor, math.ceil),
include_last=True)
logger.debug(
'cut window: %f - %f, nsamles: %g' %
(tr.tmin, tr.tmax, tr.ydata.size))
have_data.add(tr.nslc_id)
have_data_site[site].add(tr.nslc_id)
except trace.NoData:
pass
fns2 = io.save(trs, fn_template_raw)
for fn in fns2:
if fn in fns:
logger.warn('overwriting file %s', fn)
fns.extend(fns2)
except fdsn.EmptyResult:
pass
except HTTPError:
logger.warn('an error occurred while downloading data '
'for channels \n %s' %
'\n '.join('.'.join(x[:4])
for x in selection_now))
f.close()
i += neach
if options.dry_run:
nslcs = sorted(available_through.keys())
all_channels = defaultdict(set)
all_stations = defaultdict(set)
def plural_s(x):
return '' if x == 1 else 's'
for nslc in nslcs:
sites = tuple(sorted(available_through[nslc]))
logger.info('selected: %s.%s.%s.%s from site%s %s' %
(nslc + (plural_s(len(sites)), '+'.join(sites))))
all_channels[sites].add(nslc)
all_stations[sites].add(nslc[:3])
nchannels_all = 0
nstations_all = 0
for sites in sorted(all_channels.keys(),
key=lambda sites: (-len(sites), sites)):
nchannels = len(all_channels[sites])
nstations = len(all_stations[sites])
nchannels_all += nchannels
nstations_all += nstations
logger.info('selected (%s): %i channel%s (%i station%s)' %
('+'.join(sites), nchannels, plural_s(nchannels),
nstations, plural_s(nstations)))
logger.info('selected total: %i channel%s (%i station%s)' %
(nchannels_all, plural_s(nchannels_all), nstations_all,
plural_s(nstations_all)))
logger.info('dry run done.')
sys.exit(0)
for nslc in have_data:
# if we are in continue mode, we have to guess where the data came from
if not any(nslc in have_data_site[site] for site in sites):
for site in sites:
if nslc in could_have_data_site[site]:
have_data_site[site].add(nslc)
sxs = {}
for site in sites:
selection = []
for nslc in sorted(have_data_site[site]):
selection.append(nslc + (tmin - tpad, tmax + tpad))
if selection:
logger.info('downloading response information (%s)' % site)
sxs[site] = fdsn.station(site=site,
level='response',
selection=selection)
sxs[site].dump_xml(filename=op.join(output_dir, 'stations.%s.xml' %
site))
# chapter 1.5: inject local data
if options.local_data:
have_data_site['local'] = set()
plocal = pile.make_pile(options.local_data, fileformat='detect')
logger.info(
'Importing local data from %s between %s (%f) and %s (%f)' %
(options.local_data, util.time_to_str(tmin), tmin,
util.time_to_str(tmax), tmax))
for traces in plocal.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=tmin,
tmax=tmax,
tinc=tinc):
for tr in traces:
if tr.nslc_id not in have_data:
fns.extend(io.save(traces, fn_template_raw))
have_data_site['local'].add(tr.nslc_id)
have_data.add(tr.nslc_id)
sites.append('local')
if options.local_responses_pz:
sxs['local'] = epz.make_stationxml(
epz.iload(options.local_responses_pz))
if options.local_responses_resp:
local_stations = []
for fn in options.local_stations:
local_stations.extend(model.load_stations(fn))
sxs['local'] = resp.make_stationxml(
local_stations, resp.iload(options.local_responses_resp))
if options.local_responses_stationxml:
sxs['local'] = stationxml.load_xml(
filename=options.local_responses_stationxml)
# chapter 1.6: dump raw data stations file
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())]
util.ensuredirs(fn_stations_raw)
model.dump_stations(stations, fn_stations_raw)
dump_commandline(sys.argv, fn_commandline)
# chapter 2: restitution
if not fns:
logger.error('no data available')
sys.exit(1)
p = pile.make_pile(fns, show_progress=False)
p.get_deltatmin()
otinc = None
if otinc is None:
otinc = nice_seconds_floor(p.get_deltatmin() * 500000.)
otinc = 3600.
otmin = math.floor(p.tmin / otinc) * otinc
otmax = math.ceil(p.tmax / otinc) * otinc
otpad = tpad * 2
fns = []
rest_traces_b = []
win_b = None
for traces_a in p.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=otmin,
tmax=otmax,
tinc=otinc,
tpad=otpad):
rest_traces_a = []
win_a = None
for tr in traces_a:
win_a = tr.wmin, tr.wmax
if win_b and win_b[0] >= win_a[0]:
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
rest_traces_b = []
win_b = None
response = None
failure = []
for site in sites:
try:
if site not in sxs:
continue
logger.debug('Getting response for %s' % tr.__str__())
response = sxs[site].get_pyrocko_response(
tr.nslc_id,
timespan=(tr.tmin, tr.tmax),
fake_input_units=options.output_units)
break
except stationxml.NoResponseInformation:
failure.append('%s: no response information' % site)
except stationxml.MultipleResponseInformation:
failure.append('%s: multiple response information' % site)
if response is None:
failure = ', '.join(failure)
else:
failure = ''
try:
if tr.tmin > tmin and options.zero_pad:
logger.warning(
'Trace too short for clean restitution in '
'desired frequency band -> zero-padding!')
tr.extend(tr.tmin - tfade, tr.tmax + tfade, 'repeat')
rest_tr = tr.transfer(tfade, ftap, response, invert=True)
rest_traces_a.append(rest_tr)
except (trace.TraceTooShort, trace.NoData):
failure = 'trace too short'
if failure:
logger.warn('failed to restitute trace %s.%s.%s.%s (%s)' %
(tr.nslc_id + (failure, )))
if rest_traces_b:
rest_traces = trace.degapper(rest_traces_b + rest_traces_a,
deoverlap='crossfade_cos')
fns.extend(cut_n_dump(rest_traces, win_b, fn_template_rest))
rest_traces_a = []
if win_a:
for tr in rest_traces:
try:
rest_traces_a.append(
tr.chop(win_a[0], win_a[1] + otpad, inplace=False))
except trace.NoData:
pass
rest_traces_b = rest_traces_a
win_b = win_a
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
# chapter 3: rotated restituted traces for inspection
if not event:
sys.exit(0)
fn_template1 = \
'DISPL.%(network)s.%(station)s.%(location)s.%(channel)s'
fn_waveforms = op.join(output_dir, 'prepared', fn_template1)
fn_stations = op.join(output_dir, 'stations.prepared.txt')
fn_event = op.join(event_dir, 'event.txt')
fn_event_yaml = op.join(event_dir, 'event.yaml')
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
p = pile.make_pile(fns, show_progress=False)
deltat = None
if sample_rate is not None:
deltat = 1.0 / sample_rate
traces_beat = []
used_stations = []
for nsl, s in nsl_to_station.items():
s.set_event_relative_data(event)
traces = p.all(trace_selector=lambda tr: tr.nslc_id[:3] == nsl)
if options.out_components == 'rtu':
pios = s.guess_projections_to_rtu(out_channels=('R', 'T', 'Z'))
elif options.out_components == 'enu':
pios = s.guess_projections_to_enu(out_channels=('E', 'N', 'Z'))
else:
assert False
for (proj, in_channels, out_channels) in pios:
proc = trace.project(traces, proj, in_channels, out_channels)
for tr in proc:
tr_beat = heart.SeismicDataset.from_pyrocko_trace(tr)
traces_beat.append(tr_beat)
for ch in out_channels:
if ch.name == tr.channel:
s.add_channel(ch)
if proc:
io.save(proc, fn_waveforms)
used_stations.append(s)
stations = list(used_stations)
util.ensuredirs(fn_stations)
model.dump_stations(stations, fn_stations)
model.dump_events([event], fn_event)
from pyrocko.guts import dump
dump([event], filename=fn_event_yaml)
utility.dump_objects(op.join(cwd, 'seismic_data.pkl'),
outlist=[stations, traces_beat])
logger.info('prepared waveforms from %i stations' % len(stations))
from pyrocko import trace
from pyrocko import model
from pyrocko import pile
from pyrocko import gui_util
from pyrocko.gf import LocalEngine
from autogain import autogain, util_optic
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
km = 1000.
if __name__ == '__main__':
# Where is your data stored?:
datapath = '/media/usb/webnet/pole_zero/restituted_displacement/2013Mar'
data_pile = pile.make_pile(datapath)
# And the stations:
stations = model.load_stations('/media/usb/webnet/meta/stations.pf')
# Station code of the trace you want to scale agains:
reference_id ='NKC'
# Frequency band to use:
fband = {'order':4, 'corner_hp':1.0, 'corner_lp':4.}
# And a taper to avoid filtering artefacts.
taper = trace.CosFader(xfrac=0.25)
# Define a window to chop traces. In this case a static length of 20
# seconds will be used and the synthetic phase arrival will be in the
def load_data_archieve(validation_data,
gf_freq,
duration=4,
wanted_start=None,
wanted_end=None):
folder = validation_data
pathlist = Path(folder).glob('day*')
waveforms = []
stations = []
if wanted_start is not None:
try:
wanted_start = util.stt(wanted_start)
wanted_end = util.stt(wanted_end)
except:
pass
from pyrocko import pile
paths = []
safecon = 0
for path in sorted(pathlist):
path = str(path)
d2 = float(str(path)[-12:])
d1 = float(str(path)[-25:-13])
if wanted_start is not None:
do_safety_files = False
if (d1 >= wanted_start
and d2 <= wanted_end) or (d2 - wanted_end < 86400.
and d2 - wanted_end > 0.
and safecon == 0):
st = model.load_stations(path + "/waveforms/stations.raw.txt")
d_diff = d2 - d1
tr_packages = int(d_diff / duration)
#for tr in traces:
# tr.downsample_to(gf_freq)
# if safecon == 0:
pathlist_waveform_files = Path(path + "/waveforms/rest/").glob(
'*.mseed')
wanted_start_str = util.tts(wanted_start)[14:16]
diff_to_full = float(wanted_start_str)
max_diff = 55.
min_diff = 5.
if diff_to_full > max_diff or diff_to_full < min_diff:
do_safety_files = True
for path_wave in sorted(pathlist_waveform_files):
path_wave = str(path_wave)
p1 = path_wave[-25:-15]
p2 = path_wave[-14:-12]
p3 = path_wave[-11:-9]
p4 = path_wave[-8:-6]
try:
file_time = util.stt(p1 + " " + p2 + ":" + p3 + ":" +
p4)
tdiff = file_time - wanted_start
if do_safety_files is True:
if float(p2) - float(
util.tts(wanted_start)[11:13]) == 0:
paths.append(str(path_wave))
if diff_to_full > max_diff and float(p2) - float(
util.tts(wanted_start)[11:13]) == 1.:
paths.append(str(path_wave))
if diff_to_full < min_diff and float(p2) - float(
util.tts(wanted_start)[11:13]) == -1.:
paths.append(str(path_wave))
else:
if float(p2) - float(
util.tts(wanted_start)[11:13]) == 0:
paths.append(str(path_wave))
except:
pass
safecon += 1
p = pile.make_pile(paths)
for traces in p.chopper(tmin=wanted_start, tinc=duration):
if traces:
if traces[0].tmax < wanted_end:
# for i in range(0, tr_packages):
# traces = traces
#for tr in traces:
# tr.chop(tr.tmin+i*duration,
# tr.tmin+i*duration+duration)
#tr.downsample_to(gf_freq)
waveforms.append(traces)
stations.append(st)
# else:
# traces = io.load(path+"/waveforms/rest/traces.mseed")
# st = model.load_stations(path+"/waveforms/stations.raw.txt")
# for tr in traces:
# tr.downsample_to(gf_freq)
# waveforms.append(traces)
# stations.append(st)
return waveforms, stations
def iris_pull(options, conf, event_names):
conf = conf['iris_pull_config']
if not event_names:
sys.exit('need event name')
preparator = InvResponsePreparator(conf.begin_phase, conf.end_phase,
conf.inv_response_frequencyband)
for event_name in event_names:
conf.event_name = event_name
event = _get_event_infos(conf)
tevent = event.time
station_query_save_path = conf.path('station_query_save_path')
if os.path.exists(station_query_save_path):
logger.info('Using stored station query.')
all_stations = pload(station_query_save_path)
else:
logger.info('Querying for stations...')
all_stations = combi_get_stations(
lat=event.lat,
lon=event.lon,
rmin=conf.query_rmin,
rmax=conf.query_rmax,
tmin=tevent,
tmax=tevent + 3600.,
channel_pattern=conf.query_channel_pattern)
util.ensuredirs(station_query_save_path)
pdump(all_stations, station_query_save_path)
nstations = len(set(
(sta.network, sta.station) for sta in all_stations))
logger.info('Station query returned %i station%s' %
(nstations, plural_s(nstations)))
preferred_ns = set(
iris_get_vnets(conf.preferred_virtual_networks,
tmin=tevent,
tmax=tevent + 3600.))
preferred_n = set(conf.preferred_networks)
for station in all_stations:
station.set_event_relative_data(event)
raw_trace_path = conf.path('raw_trace_path')
nsl_all = set(get_nsl(s) for s in all_stations)
state_save_path = conf.path('state_save_path')
if os.path.exists(state_save_path):
nsl_ok, nsl_blacklist, nsl_use = pload(state_save_path)
else:
nsl_ok = set()
nsl_blacklist = set()
nsl_use = set()
manual_blacklist_path = conf.path('manual_blacklist_path')
nsl_blacklist.update(read_manual_blacklist(manual_blacklist_path))
nsl_selected = set()
nwanted = conf.get_or_none('nwanted')
assert len(nwanted) == 2
while True:
if nwanted:
selected = select_stations(all_stations,
nwanted[1],
preferred_n=preferred_n,
preferred_ns=preferred_ns,
preferred_nsl=nsl_ok,
blacklist_nsl=nsl_blacklist)
else:
selected = all_stations
nsl_selected = set(get_nsl(s) for s in selected)
download = [s for s in selected if get_nsl(s) not in nsl_ok]
nsl_download = set(get_nsl(s) for s in download)
combi_get_responses(download, tevent, conf.path('resp_path'))
tmin = preparator.get_tmin_limit(event, download)
tmax = preparator.get_tmax_limit(event, download)
logger.info('Loading data for event %s:\n%s' %
(event_name, str_nsl_selection(nsl_download)))
try:
fns = combi_get_data(
download,
tmin,
tmax,
raw_trace_path,
neach=conf.query_nstations_per_datarequest)
p = pile.make_pile(fns, show_progress=False)
prepared_trace_path = conf.path('prepared_trace_path')
for traces in preparator.iter_prepare(p, event, download):
for tr in traces:
nsl_ok.add(get_nsl(tr))
io.save(traces, prepared_trace_path)
except iris_ws.NotFound:
pass
logger.info('Blacklisting:\n%s' %
str_nsl_selection(nsl_download - nsl_ok))
nsl_blacklist.update(nsl_download - nsl_ok)
preferred_ns.update(set(nsl[:2] for nsl in nsl_ok))
nsl_use = nsl_ok & nsl_selected
logger.info('Have %i stations with data.' % len(nsl_use))
if not nwanted:
break
else:
if len(nsl_selected) == len(nsl_all):
break
if len(nsl_use) > nwanted[0]:
break
pdump((nsl_ok, nsl_blacklist, nsl_use), state_save_path)
if nwanted and len(nsl_use) >= nwanted[1]:
selected = select_stations(selected,
nwanted[1],
preferred_n=preferred_n,
preferred_ns=preferred_ns,
blacklist_nsl=nsl_blacklist)
nsl_selected = set(get_nsl(s) for s in selected)
nsl_use = nsl_ok & nsl_selected
stations = [s for s in all_stations if get_nsl(s) in nsl_use]
model.dump_stations(stations, conf.path('stations_ok_path'))
model.dump_stations(all_stations, conf.path('stations_all_path'))
if nsl_use:
logger.info('Data available for event %s:\n%s' %
(event_name, str_nsl_selection(nsl_use)))
else:
logger.info('No data availabe for event %s' % event_name)
def dump_pile(self, fn='test_dumped_seismograms.mseed'):
pile.make_pile(seismograms.values(), fn=fn)
"""
Extrahiere miniseed-Daten um Events, die aus Katalog ausgelesen werden
"""
# mit sys.argv kann man alles abfragen, was hinter dem Programmaufruf steht.
dirs = sys.argv[1:]
# lade das Stationsfile:
stats = model.load_stations('/scratch/local1/doctar/meta/stations.txt')
# lese Events:
events = model.load_events('/home/zmaw/u254061/master/event_marker_IPMA.txt')
# Erstelle ein pile aus allen miniseed in den Ordnern, die man hinter dem Programmaufruf angegeben hat:
outpile = pile.make_pile(dirs)
f=open('/home/zmaw/u254061/master/event_marker_IPMA.txt')
for line in f:
if line.lstrip().startswith('#'):
continue
toks = line.split()
timedate, timetime =toks[1], toks[2]
gtime = util.str_to_time(str(timedate+' '+timetime))
trange = [gtime-100, gtime+1000]
new_pile = []
for traces in outpile.chopper(trange[0], trange[1], load_data=True, degap=False):
if traces:
window_start = traces[0].wmin
timestring = util.time_to_str(window_start, format='%Y-%m-%d_%H')
#fbands.apppend([4.0, 10.])
phases = LocalEngine(store_superdirs=['/data/stores'],
default_store_id='globalttt').get_store()
#filenames = glob.glob('data/*.mseed')
#filenames = glob.glob('/data/webnet/waveform_R/2008/*.mseed')
#datapath = '/data/webnet/mseed/2008'
#datapath = '/data/webnet/waveform_R/2008'
#datapath = '/data/share/Res_all_NKC'
datapath = '/media/usb0/Res_all_NKC_taper'
#datapath = '/media/usb0/restituted_pyrocko'
stations = model.load_stations('../data/stations.pf')
reference_id ='NKC'
references = {}
data_pile = pile.make_pile(datapath, selector='rest_*')
fband = {'order':4, 'corner_hp':1.0, 'corner_lp':4.}
window = StaticLengthWindow(static_length=30.,
phase_position=0.5)
taper = trace.CosFader(xfrac=0.25)
#event_selector = EventSelector(distmin=1000*km,
# distmax=20000*km,
# depthmin=2*km,
# depthmax=600*km,
# magmin=4.9)
candidate_fn = '../candidates2013.pf'
def load(self, inv):
# load the data as a pyrocko pile and reform them into an array of traces
data = pile.make_pile([self.wdir + self.reduction])
self.traces = data.all()
# load station file
fname = self.wdir + self.network
stations_list = model.load_stations(fname)
for s in stations_list:
s.set_channels_by_name(*self.component.split())
self.targets = []
self.tmin, self.tmax = [], []
self.arrivals = []
self.names = []
for station, tr in zip(stations_list,
self.traces): # iterate over all stations
# print station.lat, station.lon
target = Target(
lat=np.float(station.lat), # station lat.
lon=np.float(station.lon), # station lon.
store_id=inv.store, # The gf-store to be used for this target,
# we can also employ different gf-stores for different targets.
interpolation='multilinear', # interp. method between gf cells
quantity='displacement', # wanted retrieved quantity
codes=station.nsl() +
('BH' + self.component, )) # Station and network code
# Next we extract the expected arrival time for this station from the the store,
# so we can use this later to define a cut-out window for the optimization:
self.targets.append(target)
self.names.append(station.nsl()[1])
# print len(self.traces), len(self.targets)
for station, tr, target in zip(stations_list, self.traces,
self.targets):
engine = LocalEngine(store_superdirs=inv.store_path)
store = engine.get_store(inv.store)
# trace.snuffle(tr, events=self.events)
arrival = store.t(self.phase, self.base_source,
target) # expected P-wave arrival
# print arrival
tmin = self.base_source.time + arrival - 15 # start 15s before theor. arrival
tmax = self.base_source.time + arrival + 15 # end 15s after theor. arrival
# # print self.tmin,self.tmax
tr.chop(tmin=tmin, tmax=tmax)
self.tmin.append(tmin)
self.tmax.append(tmax)
self.arrivals.append(self.base_source.time + arrival)
self.Npoints = len(self.targets)
# data vector
self.d = []
self.d.append(map((lambda x: getattr(x, 'ydata')), self.traces))
self.d = flatten(self.d)
# time vector
t = []
for i in xrange(self.Npoints):
t.append(self.traces[i].get_xdata())
# self.t.append(map((lambda x: getattr(x,'get_xdata()')),self.traces))
# convert time
self.t = time2dec(map(util.time_to_str, flatten(t)))
# print self.t
self.N = len(self.d)
def prep_psd_fct(i_st,
st,
l,
subset_catalog,
dir_f,
arrT_array,
arrT_R_array,
datapath,
syndatapath,
tinc,
tpad,
dt_s,
dt_e,
n,
fac_norm,
f_ign,
plot_psds=False,
plot_ratio_extra=False,
plot_m_rat=False,
plot_flat_ranges=False,
plot_neighb_ranges=False):
"""
Preparing the PSD calculations and plotting, e.g. making data-piles
and calling next function if both, synthetic and recorded data is
in piles.
:param i_st: number of station - must fit arrival time table!
:param st: pyrocko station object (current station)
:param subset_catalog: Catalog to be used (list of pyrocko events)
:param dir_f: directory to store PSD results at
:param arrT_array: Arrival times
:param datapath: Path to restituted (,rotated and downsampled) data
:param syndatapath: Path to sythetic data
:param plot_ratio_extra: Should ratios of PSDs be plotted to extra figure?
They are displayed within the PSD plot anyway,
default is False.
:returns freq_rat_list: List containing freuqncy ranges at which the
ratio between synthetic and observed psd is
constant
:returns nslc_list: nslc list for current station
"""
st_name = st.station
st_data_pile = pile.make_pile(datapath,
regex='%s_%s_' % (st.network, st_name),
show_progress=False)
st_syn_data_pile = pile.make_pile(syndatapath,
regex='%s_%s_' % (st.network, st_name),
show_progress=False)
freq_rat_list = []
freq_rat_list_y = []
nslc_list = []
# flat_by_next = []
# flat_by_next_y = []
if st_data_pile.tmin is not None and st_data_pile.tmax is not None and\
st_syn_data_pile.tmin is not None\
and st_syn_data_pile.tmax is not None:
nsl = st.nsl()
for cha in ['Z', 'R', 'T']:
outs = calc_plot_psds(subset_catalog,
st_data_pile,
st_syn_data_pile,
cha,
l,
dir_f,
arrT_array,
arrT_R_array,
nsl,
i_st,
tinc,
tpad,
dt_s,
dt_e,
n,
fac_norm,
f_ign,
plot_psds=plot_psds,
plot_ratio_extra=plot_ratio_extra,
plot_m_rat=plot_m_rat,
plot_flat_ranges=plot_flat_ranges,
plot_neighb_ranges=plot_neighb_ranges)
ratio_npar = outs[-1]
f_syn_keep = outs[-2]
if plot_psds is True and outs[0] and outs[1]:
plot_psd_from_dict(outs[0], outs[1], st, l, cha,
subset_catalog, dir_f)
if plot_ratio_extra is True and outs[2]:
plot_psdratio_from_dict(outs[2], st, l, cha,
subset_catalog, dir_f)
elif plot_psds is not True and plot_ratio_extra is True and outs[2]:
plot_psdratio_from_dict(outs[2], st, l, cha, subset_catalog,
dir_f)
if ratio_npar.shape != (1, ):
m_rat = calc_median_ratio(ratio_npar)
if plot_m_rat is True:
plot_m_ratio(m_rat, f_syn_keep, nsl, l, dir_f, cha)
f, y = const_psd_rat(m_rat,
cha,
st,
l,
f_syn_keep,
n,
fac_norm,
f_ign,
plot_flat_range=plot_flat_ranges,
dir_f=dir_f)
freq_rat_list.append(f)
freq_rat_list_y.append(y)
nslc_list.append('%s.%s.%s.%s' % (nsl[0], nsl[1], l, cha))
'''
f2, y2 = flat_by_neighbor_comp(
m_rat, cha, st, f_syn_keep, dir_f,
n, fac_norm, f_ign,
plot_psd_neighbcomp=plot_neighb_ranges)
flat_by_next.append(f2)
flat_by_next_y.append(y2)
'''
return freq_rat_list, freq_rat_list_y, nslc_list # ,\
from pyrocko import pile, io, util
import time, calendar
# when pile.make_pile() is called without any arguments, the command line
# parameters given to the script are searched for waveform files and directories
p = pile.make_pile()
# get timestamp for full hour before first data sample in all selected traces
tmin = calendar.timegm(time.gmtime(p.tmin)[:4] + (0, 0))
tinc = 3600.0
tpad = 10.0
target_deltat = 0.1
# iterate over the data, with a window length of one hour and 2x10 seconds of
# overlap
for traces in p.chopper(tmin=tmin, tinc=tinc, tpad=tpad):
if traces: # the list could be empty due to gaps
for tr in traces:
tr.downsample_to(target_deltat, snap=True, demean=False)
# remove overlapping
tr.chop(tr.wmin, tr.wmax)
window_start = traces[0].wmin
timestring = util.time_to_str(window_start, format="%Y-%m-%d_%H")
filepath = "downsampled/%(station)s_%(channel)s_%(mytimestring)s.mseed"
io.save(traces, filepath, additional={"mytimestring": timestring})
def snuffle(pile=None, **kwargs):
'''View pile in a snuffler window.
:param pile: :py:class:`pile.Pile` object to be visualized
:param stations: list of `pyrocko.model.Station` objects or ``None``
:param events: list of `pyrocko.model.Event` objects or ``None``
:param markers: list of `pyrocko.gui.util.Marker` objects or ``None``
:param ntracks: float, number of tracks to be shown initially (default: 12)
:param follow: time interval (in seconds) for real time follow mode or
``None``
:param controls: bool, whether to show the main controls (default:
``True``)
:param opengl: bool, whether to use opengl (default: ``False``)
:param paths: list of files and directories to search for trace files
:param pattern: regex which filenames must match
:param format: format of input files
:param cache_dir: cache directory with trace meta information
:param force_cache: bool, whether to use the cache when attribute spoofing
is active
:param store_path: filename template, where to store trace data from input
streams
:param store_interval: float, time interval (in seconds) between stream
buffer dumps
:param want_markers: bool, whether markers should be returned
:param launch_hook: callback function called before snuffler window is
shown
'''
from .snuffler_app import Snuffler, SnufflerWindow, \
setup_acquisition_sources, PollInjector
if pile is None:
pile = pile.make_pile()
global app
if app is None:
import locale
locale.setlocale(locale.LC_ALL, 'C')
app = Snuffler()
kwargs_load = {}
for k in ('paths', 'regex', 'format', 'cache_dir', 'force_cache'):
try:
kwargs_load[k] = kwargs.pop(k)
except KeyError:
pass
store_path = kwargs.pop('store_path', None)
store_interval = kwargs.pop('store_interval', 600)
want_markers = kwargs.pop('want_markers', False)
launch_hook = kwargs.pop('launch_hook', None)
win = SnufflerWindow(pile, **kwargs)
if launch_hook:
if not isinstance(launch_hook, list):
launch_hook = [launch_hook]
for hook in launch_hook:
hook(win)
sources = []
pollinjector = None
tempdir = None
if 'paths' in kwargs_load:
sources.extend(setup_acquisition_sources(kwargs_load['paths']))
if sources:
if store_path is None:
tempdir = tempfile.mkdtemp('', 'snuffler-tmp-')
store_path = pjoin(
tempdir,
'trace-%(network)s.%(station)s.%(location)s.%(channel)s.'
'%(tmin)s.mseed')
elif os.path.isdir(store_path):
store_path = pjoin(
store_path,
'trace-%(network)s.%(station)s.%(location)s.%(channel)s.'
'%(tmin)s.mseed')
pollinjector = PollInjector(
pile,
fixation_length=store_interval,
path=store_path)
for source in sources:
source.start()
pollinjector.add_source(source)
win.get_view().load(**kwargs_load)
if not win.is_closing():
app.install_sigint_handler()
app.exec_()
app.uninstall_sigint_handler()
for source in sources:
source.stop()
if pollinjector:
pollinjector.fixate_all()
ret = win.return_tag()
if want_markers:
markers = win.get_view().get_markers()
del win
gc.collect()
if tempdir:
shutil.rmtree(tempdir)
if want_markers:
return ret, markers
else:
return ret
class TestFSU(unittest.TestCase):
tpile_1 = list(
pile.make_pile(pjoin(mseeds, 'testFiles/'),
show_progress=False).iter_all())
tpile_2 = list(
pile.make_pile(pjoin(mseeds, 'testFiles_2/'),
show_progress=False).iter_all())
random_pile_test = list(
pile.make_pile(pjoin(mseeds, 'testRandomTestFiles/'),
show_progress=False).iter_all())
random_pile_reference = list(
pile.make_pile(pjoin(mseeds, 'referenceRandomTestFiles/'),
show_progress=False).iter_all())
t_min = util.str_to_time('2010-01-01 22:00:00')
data1 = np.array([0, 0, 0, 0, 1, 0, 0, 0])
data2 = np.array([0, 0, 0, 0, -1, 0, 0, 0])
ttrace_1 = trace.Trace(network='1',
station='TEST',
channel='z',
deltat=0.5,
ydata=data1)
ttrace_2 = trace.Trace(network='2',
station='TEST',
channel='z',
deltat=0.5,
ydata=data2)
def test_misfit_by_samples(self):
data = np.random.random(100)
ttrace = trace.Trace(network='1',
station='T1',
channel='z',
deltat=0.5,
ydata=data)
equal_traces_list = [ttrace.get_ydata(), ttrace.get_ydata()]
self.assertEqual(misfit_by_samples(equal_traces_list), 0,
'misfit of same traces is zero')
def test_misfit_by_samples_squared(self):
equal_neg_traces_list = [
self.ttrace_1.get_ydata(),
self.ttrace_2.get_ydata()
]
self.assertEqual(
misfit_by_samples(equal_neg_traces_list, square=True), 0,
'misfit of squared traces is zero if one trace is negative of the other'
)
def test_find_matching_traces(self):
self.assertEqual(
len(
find_matching_traces([list(self.tpile_1)],
test_list=self.tpile_2)), 12,
'did not find all 12 of 12 pairs of traces')
def test_time_domain_misfit_equal_piles(self):
self.assertEqual(
time_domain_misfit(reference_pile=[self.tpile_1],
test_list=self.tpile_2), 0,
'Time Domain Misfit of equal piles is not 0!')
def test_frequency_domain_misfit_equal_piles(self):
self.assertEqual(
frequency_domain_misfit(reference_pile=[self.tpile_1],
test_list=self.tpile_2), 0,
'Frequency Domain Misfit of equal test_piles is NOT 0')
def test_time_domain_misfit_UNequal_piles(self):
self.assertNotEqual(
time_domain_misfit(reference_pile=[self.random_pile_reference],
test_list=self.random_pile_test), 0,
'Time Domain Misfit of UNequal traces is not 0')
def test_frequency_domain_misfit_UNequal_piles(self):
self.assertNotEqual(
frequency_domain_misfit(
reference_pile=[self.random_pile_reference],
test_list=self.random_pile_test), 0,
'Freq Dom Misfit of unequal piles is 0 but should not be 0')
def test_equalize_sampling_rate(self):
'''
:return:
'''
data1 = np.random.random(100)
data2 = np.random.random(100)
ttrace_1 = trace.Trace(network='1',
station='T1',
channel='z',
deltat=0.5,
ydata=data1)
ttrace_2 = trace.Trace(network='1',
station='T2',
channel='z',
deltat=0.01,
ydata=data2)
downsample_if_needed([[ttrace_1, ttrace_2]])
self.assertEqual(ttrace_1.deltat, ttrace_2.deltat,
'Equalization of sampling rates unsuccessful!')
self.assertEqual(ttrace_1.deltat, 0.6,
'new sampling rate of ttrace_1 wrong')
self.assertEqual(ttrace_2.deltat, 0.6,
'new sampling rate of ttrace_2 wrong')
def test_chop_to_same_sample_length(self):
data1 = np.random.random(100)
data2 = np.random.random(80)
[data1, data2] = chop_longer_samples([data1, data2])
self.assertEqual(np.shape(data1), np.shape(data2),
'shape after chopping not equal')
def main():
parser = OptionParser(usage=usage, description=description)
parser.add_option(
"--force",
dest="force",
action="store_true",
default=False,
help="allow recreation of output <directory>",
)
parser.add_option(
"--debug",
dest="debug",
action="store_true",
default=False,
help="print debugging information to stderr",
)
parser.add_option(
"--dry-run",
dest="dry_run",
action="store_true",
default=False,
help="show available stations/channels and exit "
"(do not download waveforms)",
)
parser.add_option(
"--continue",
dest="continue_",
action="store_true",
default=False,
help="continue download after a accident",
)
parser.add_option(
"--local-data",
dest="local_data",
action="append",
help="add file/directory with local data",
)
parser.add_option(
"--local-stations",
dest="local_stations",
action="append",
help="add local stations file",
)
parser.add_option(
"--local-responses-resp",
dest="local_responses_resp",
action="append",
help="add file/directory with local responses in RESP format",
)
parser.add_option(
"--local-responses-pz",
dest="local_responses_pz",
action="append",
help="add file/directory with local pole-zero responses",
)
parser.add_option(
"--local-responses-stationxml",
dest="local_responses_stationxml",
help="add file with local response information in StationXML format",
)
parser.add_option(
"--window",
dest="window",
default="full",
help='set time window to choose [full, p, "<time-start>,<time-end>"'
"] (time format is YYYY-MM-DD HH:MM:SS)",
)
parser.add_option(
"--out-components",
choices=["enu", "rtu"],
dest="out_components",
default="rtu",
help="set output component orientations to radial-transverse-up [rtu] "
"(default) or east-north-up [enu]",
)
parser.add_option(
"--padding-factor",
type=float,
default=3.0,
dest="padding_factor",
help="extend time window on either side, in multiples of 1/<fmin_hz> "
"(default: 5)",
)
parser.add_option(
"--credentials",
dest="user_credentials",
action="append",
default=[],
metavar="SITE,USER,PASSWD",
help="user credentials for specific site to access restricted data "
"(this option can be repeated)",
)
parser.add_option(
"--token",
dest="auth_tokens",
metavar="SITE,FILENAME",
action="append",
default=[],
help="user authentication token for specific site to access "
"restricted data (this option can be repeated)",
)
parser.add_option(
"--sites",
dest="sites",
metavar="SITE1,SITE2,...",
# default='bgr',
default="http://ws.gpi.kit.edu,bgr,http://188.246.25.142:8080",
help='sites to query (available: %s, default: "%%default"' %
", ".join(g_sites_available),
)
parser.add_option(
"--band-codes",
dest="priority_band_code",
metavar="V,L,M,B,H,S,E,...",
default="V,L,M,B,H,E",
help="select and prioritize band codes (default: %default)",
)
parser.add_option(
"--instrument-codes",
dest="priority_instrument_code",
metavar="H,L,G,...",
default="H,L,O,",
help="select and prioritize instrument codes (default: %default)",
)
parser.add_option(
"--radius-min",
dest="radius_min",
metavar="VALUE",
default=0.0,
type=float,
help="minimum radius [km]",
)
parser.add_option(
"--tinc",
dest="tinc",
metavar="VALUE",
default=3600.0 * 12.0,
type=float,
help="length of seperate saved files in s",
)
parser.add_option(
"--nstations-wanted",
dest="nstations_wanted",
metavar="N",
type=int,
help="number of stations to select initially",
)
(options, args) = parser.parse_args(sys.argv[1:])
if len(args) not in (9, 6, 5):
parser.print_help()
sys.exit(1)
if options.debug:
util.setup_logging(program_name, "debug")
else:
util.setup_logging(program_name, "info")
if options.local_responses_pz and options.local_responses_resp:
logger.critical("cannot use local responses in PZ and RESP "
"format at the same time")
sys.exit(1)
n_resp_opt = 0
for resp_opt in (
options.local_responses_pz,
options.local_responses_resp,
options.local_responses_stationxml,
):
if resp_opt:
n_resp_opt += 1
if n_resp_opt > 1:
logger.critical("can only handle local responses from either PZ or "
"RESP or StationXML. Cannot yet merge different "
"response formats.")
sys.exit(1)
if options.local_responses_resp and not options.local_stations:
logger.critical("--local-responses-resp can only be used "
"when --stations is also given.")
sys.exit(1)
try:
ename = ""
magnitude = None
mt = None
if len(args) == 9:
time = util.str_to_time(args[0] + " " + args[1])
lat = float(args[2])
lon = float(args[3])
depth = float(args[4]) * km
iarg = 5
elif len(args) == 6:
if args[1].find(":") == -1:
sname_or_date = None
lat = float(args[0])
lon = float(args[1])
event = None
time = None
else:
sname_or_date = args[0] + " " + args[1]
iarg = 2
elif len(args) == 5:
sname_or_date = args[0]
iarg = 1
if len(args) in (6, 5) and sname_or_date is not None:
events = get_events_by_name_or_date([sname_or_date],
catalog=geofon)
if len(events) == 0:
logger.critical("no event found")
sys.exit(1)
elif len(events) > 1:
logger.critical("more than one event found")
sys.exit(1)
event = events[0]
time = event.time
lat = event.lat
lon = event.lon
depth = event.depth
ename = event.name
magnitude = event.magnitude
mt = event.moment_tensor
radius = float(args[iarg]) * km
fmin = float(args[iarg + 1])
sample_rate = float(args[iarg + 2])
eventname = args[iarg + 3]
event_dir = op.join("data", "events", eventname)
output_dir = op.join(event_dir, "waveforms")
except:
raise
parser.print_help()
sys.exit(1)
if options.force and op.isdir(event_dir):
if not options.continue_:
shutil.rmtree(event_dir)
if op.exists(event_dir) and not options.continue_:
logger.critical(
'directory "%s" exists. Delete it first or use the --force option'
% event_dir)
sys.exit(1)
util.ensuredir(output_dir)
if time is not None:
event = model.Event(
time=time,
lat=lat,
lon=lon,
depth=depth,
name=ename,
magnitude=magnitude,
moment_tensor=mt,
)
if options.window == "full":
if event is None:
logger.critical("need event for --window=full")
sys.exit(1)
low_velocity = 1500.0
timewindow = VelocityWindow(low_velocity,
tpad=options.padding_factor / fmin)
tmin, tmax = timewindow(time, radius, depth)
elif options.window == "p":
if event is None:
logger.critical("need event for --window=p")
sys.exit(1)
phases = list(map(cake.PhaseDef, "P p".split()))
emod = cake.load_model()
tpad = options.padding_factor / fmin
timewindow = PhaseWindow(emod, phases, -tpad, tpad)
arrivaltimes = []
for dist in num.linspace(0, radius, 20):
try:
arrivaltimes.extend(timewindow(time, dist, depth))
except NoArrival:
pass
if not arrivaltimes:
logger.error("required phase arrival not found")
sys.exit(1)
tmin = min(arrivaltimes)
tmax = max(arrivaltimes)
else:
try:
stmin, stmax = options.window.split(",")
tmin = util.str_to_time(stmin.strip())
tmax = util.str_to_time(stmax.strip())
timewindow = FixedWindow(tmin, tmax)
except ValueError:
logger.critical('invalid argument to --window: "%s"' %
options.window)
sys.exit(1)
if event is not None:
event.name = eventname
tlen = tmax - tmin
tfade = tfade_factor / fmin
tpad = tfade
tmin -= tpad
tmax += tpad
priority_band_code = options.priority_band_code.split(",")
for s in priority_band_code:
if len(s) != 1:
logger.critical("invalid band code: %s" % s)
priority_instrument_code = options.priority_instrument_code.split(",")
for s in priority_instrument_code:
if len(s) != 1:
logger.critical("invalid instrument code: %s" % s)
station_query_conf = dict(
latitude=lat,
longitude=lon,
minradius=options.radius_min * km * cake.m2d,
maxradius=radius * cake.m2d,
channel=",".join("?%s?" % s for s in priority_band_code),
)
target_sample_rate = sample_rate
fmax = target_sample_rate
# target_sample_rate = None
# priority_band_code = ['H', 'B', 'M', 'L', 'V', 'E', 'S']
priority_units = ["M/S", "M", "M/S**2"]
output_units = "M"
sites = [x.strip() for x in options.sites.split(",") if x.strip()]
tinc = options.tinc
# for site in sites:
# if site not in g_sites_available:
# logger.critical('unknown FDSN site: %s' % site)
# sys.exit(1)
for s in options.user_credentials:
try:
site, user, passwd = s.split(",")
g_user_credentials[site] = user, passwd
except ValueError:
logger.critical('invalid format for user credentials: "%s"' % s)
sys.exit(1)
for s in options.auth_tokens:
try:
site, token_filename = s.split(",")
with open(token_filename, "r") as f:
g_auth_tokens[site] = f.read()
except (ValueError, OSError, IOError):
logger.critical("cannot get token from file: %s" % token_filename)
sys.exit(1)
fn_template0 = (
"data_%(network)s.%(station)s.%(location)s.%(channel)s_%(tmin)s.mseed")
fn_template_raw = op.join(output_dir, "raw", fn_template0)
fn_template_raw_folder = op.join(output_dir, "raw/", "traces.mseed")
fn_stations_raw = op.join(output_dir, "stations.raw.txt")
fn_template_rest = op.join(output_dir, "rest", fn_template0)
fn_commandline = op.join(output_dir, "seigerdown.command")
ftap = (ffade_factors[0] * fmin, fmin, fmax, ffade_factors[1] * fmax)
# chapter 1: download
sxs = []
for site in sites:
try:
extra_args = {
"iris": dict(matchtimeseries=True),
}.get(site, {})
extra_args.update(station_query_conf)
if site == "geonet":
extra_args.update(starttime=tmin, endtime=tmax)
else:
extra_args.update(
startbefore=tmax,
endafter=tmin,
includerestricted=(site in g_user_credentials
or site in g_auth_tokens),
)
logger.info("downloading channel information (%s)" % site)
sx = fdsn.station(site=site,
format="text",
level="channel",
**extra_args)
except fdsn.EmptyResult:
logger.error("No stations matching given criteria. (%s)" % site)
sx = None
if sx is not None:
sxs.append(sx)
if all(sx is None for sx in sxs) and not options.local_data:
sys.exit(1)
nsl_to_sites = defaultdict(list)
nsl_to_station = {}
for sx, site in zip(sxs, sites):
site_stations = sx.get_pyrocko_stations()
for s in site_stations:
nsl = s.nsl()
nsl_to_sites[nsl].append(site)
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s # using first site with this station
logger.info("number of stations found: %i" % len(nsl_to_station))
# station weeding
nsls_selected = None
if options.nstations_wanted:
stations_all = [
nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())
]
for s in stations_all:
s.set_event_relative_data(event)
stations_selected = weeding.weed_stations(stations_all,
options.nstations_wanted)[0]
nsls_selected = set(s.nsl() for s in stations_selected)
logger.info("number of stations selected: %i" % len(nsls_selected))
have_data = set()
if options.continue_:
fns = glob.glob(fn_template_raw % starfill())
p = pile.make_pile(fns)
else:
fns = []
have_data_site = {}
could_have_data_site = {}
for site in sites:
have_data_site[site] = set()
could_have_data_site[site] = set()
available_through = defaultdict(set)
it = 0
nt = int(math.ceil((tmax - tmin) / tinc))
for it in range(nt):
tmin_win = tmin + it * tinc
tmax_win = min(tmin + (it + 1) * tinc, tmax)
logger.info("time window %i/%i (%s - %s)" %
(it + 1, nt, util.tts(tmin_win), util.tts(tmax_win)))
have_data_this_window = set()
if options.continue_:
trs_avail = p.all(tmin=tmin_win, tmax=tmax_win, load_data=False)
for tr in trs_avail:
have_data_this_window.add(tr.nslc_id)
for site, sx in zip(sites, sxs):
if sx is None:
continue
selection = []
channels = sx.choose_channels(
target_sample_rate=target_sample_rate,
priority_band_code=priority_band_code,
priority_units=priority_units,
priority_instrument_code=priority_instrument_code,
timespan=(tmin_win, tmax_win),
)
for nslc in sorted(channels.keys()):
if nsls_selected is not None and nslc[:3] not in nsls_selected:
continue
could_have_data_site[site].add(nslc)
if nslc not in have_data_this_window:
channel = channels[nslc]
if event:
lat_, lon_ = event.lat, event.lon
else:
lat_, lon_ = lat, lon
dist = orthodrome.distance_accurate50m_numpy(
lat_, lon_, channel.latitude.value,
channel.longitude.value)
if event:
depth_ = event.depth
time_ = event.time
else:
depth_ = None
time_ = None
tmin_, tmax_ = timewindow(time_, dist, depth_)
tmin_this = tmin_ - tpad
tmax_this = tmax_ + tpad
tmin_req = max(tmin_win, tmin_this)
tmax_req = min(tmax_win, tmax_this)
if channel.sample_rate:
deltat = 1.0 / channel.sample_rate.value
else:
deltat = 1.0
if tmin_req < tmax_req:
# extend time window by some samples because otherwise
# sometimes gaps are produced
selection.append(nslc + (tmin_req - deltat * 10.0,
tmax_req + deltat * 10.0))
if options.dry_run:
for (net, sta, loc, cha, tmin, tmax) in selection:
available_through[net, sta, loc, cha].add(site)
else:
neach = 100
i = 0
nbatches = ((len(selection) - 1) // neach) + 1
while i < len(selection):
selection_now = selection[i:i + neach]
f = tempfile.NamedTemporaryFile()
try:
sbatch = ""
if nbatches > 1:
sbatch = " (batch %i/%i)" % (
(i // neach) + 1, nbatches)
logger.info("downloading data (%s)%s" % (site, sbatch))
data = fdsn.dataselect(site=site,
selection=selection_now,
**get_user_credentials(site))
while True:
buf = data.read(1024)
if not buf:
break
f.write(buf)
f.flush()
trs = io.load(f.name)
for tr in trs:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
try:
tr.chop(tmin_win, tmax_win)
have_data.add(tr.nslc_id)
have_data_site[site].add(tr.nslc_id)
except trace.NoData:
pass
fns2 = io.save(trs, fn_template_raw)
io.save(trs, fn_template_raw_folder)
for fn in fns2:
if fn in fns:
logger.warn("overwriting file %s", fn)
fns.extend(fns2)
except fdsn.EmptyResult:
pass
except HTTPError:
logger.warn("an error occurred while downloading data "
"for channels \n %s" %
"\n ".join(".".join(x[:4])
for x in selection_now))
f.close()
i += neach
if options.dry_run:
nslcs = sorted(available_through.keys())
all_channels = defaultdict(set)
all_stations = defaultdict(set)
def plural_s(x):
return "" if x == 1 else "s"
for nslc in nslcs:
sites = tuple(sorted(available_through[nslc]))
logger.info("selected: %s.%s.%s.%s from site%s %s" %
(nslc + (plural_s(len(sites)), "+".join(sites))))
all_channels[sites].add(nslc)
all_stations[sites].add(nslc[:3])
nchannels_all = 0
nstations_all = 0
for sites in sorted(all_channels.keys(),
key=lambda sites: (-len(sites), sites)):
nchannels = len(all_channels[sites])
nstations = len(all_stations[sites])
nchannels_all += nchannels
nstations_all += nstations
logger.info("selected (%s): %i channel%s (%i station%s)" % (
"+".join(sites),
nchannels,
plural_s(nchannels),
nstations,
plural_s(nstations),
))
logger.info("selected total: %i channel%s (%i station%s)" % (
nchannels_all,
plural_s(nchannels_all),
nstations_all,
plural_s(nstations_all),
))
logger.info("dry run done.")
sys.exit(0)
for nslc in have_data:
# if we are in continue mode, we have to guess where the data came from
if not any(nslc in have_data_site[site] for site in sites):
for site in sites:
if nslc in could_have_data_site[site]:
have_data_site[site].add(nslc)
sxs = {}
for site in sites:
selection = []
for nslc in sorted(have_data_site[site]):
selection.append(nslc + (tmin - tpad, tmax + tpad))
if selection:
logger.info("downloading response information (%s)" % site)
sxs[site] = fdsn.station(site=site,
level="response",
selection=selection)
sited = site
if site == "http://192.168.11.220:8080":
sited = "bgr_internal"
elif site == "http://ws.gpi.kit.edu":
sited = "kit"
if site == "http://188.246.25.142:8080":
sited = "moer"
sxs[site].dump_xml(filename=op.join(output_dir, "stations.%s.xml" %
sited))
# chapter 1.5: inject local data
if options.local_data:
have_data_site["local"] = set()
plocal = pile.make_pile(options.local_data, fileformat="detect")
for traces in plocal.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=tmin,
tmax=tmax,
tinc=tinc):
for tr in traces:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
if tr.nslc_id not in have_data:
fns.extend(io.save(traces, fn_template_raw))
have_data_site["local"].add(tr.nslc_id)
have_data.add(tr.nslc_id)
sites.append("local")
if options.local_responses_pz:
sxs["local"] = epz.make_stationxml(
epz.iload(options.local_responses_pz))
if options.local_responses_resp:
local_stations = []
for fn in options.local_stations:
local_stations.extend(model.load_stations(fn))
sxs["local"] = resp.make_stationxml(
local_stations, resp.iload(options.local_responses_resp))
if options.local_responses_stationxml:
sxs["local"] = stationxml.load_xml(
filename=options.local_responses_stationxml)
# chapter 1.6: dump raw data stations file
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())]
util.ensuredirs(fn_stations_raw)
model.dump_stations(stations, fn_stations_raw)
dump_commandline(sys.argv, fn_commandline)
# chapter 2: restitution
if not fns:
logger.error("no data available")
sys.exit(1)
p = pile.make_pile(fns, show_progress=False)
p.get_deltatmin()
otinc = None
if otinc is None:
otinc = nice_seconds_floor(p.get_deltatmin() * 500000.0)
otinc = 3600.0
otmin = math.floor(p.tmin / otinc) * otinc
otmax = math.ceil(p.tmax / otinc) * otinc
otpad = tpad * 2
fns = []
rest_traces_b = []
win_b = None
for traces_a in p.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=otmin,
tmax=otmax,
tinc=otinc,
tpad=otpad):
rest_traces_a = []
win_a = None
for tr in traces_a:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
win_a = tr.wmin, tr.wmax
if win_b and win_b[0] >= win_a[0]:
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
rest_traces_b = []
win_b = None
response = None
failure = []
for site in sites:
try:
if site not in sxs:
continue
response = sxs[site].get_pyrocko_response(
tr.nslc_id,
timespan=(tr.tmin, tr.tmax),
fake_input_units=output_units,
)
break
except stationxml.NoResponseInformation:
failure.append("%s: no response information" % site)
except stationxml.MultipleResponseInformation:
failure.append("%s: multiple response information" % site)
if response is None:
failure = ", ".join(failure)
else:
failure = ""
try:
rest_tr = tr.transfer(tfade, ftap, response, invert=True)
rest_traces_a.append(rest_tr)
except (trace.TraceTooShort, trace.NoData):
failure = "trace too short"
if failure:
logger.warn("failed to restitute trace %s.%s.%s.%s (%s)" %
(tr.nslc_id + (failure, )))
if rest_traces_b:
rest_traces = trace.degapper(rest_traces_b + rest_traces_a,
deoverlap="crossfade_cos")
fns.extend(cut_n_dump(rest_traces, win_b, fn_template_rest))
rest_traces_a = []
if win_a:
for tr in rest_traces:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
try:
rest_traces_a.append(
tr.chop(win_a[0], win_a[1] + otpad, inplace=False))
except trace.NoData:
pass
rest_traces_b = rest_traces_a
win_b = win_a
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
# chapter 3: rotated restituted traces for inspection
if not event:
sys.exit(0)
fn_template1 = "DISPL.%(network)s.%(station)s.%(location)s.%(channel)s"
fn_waveforms = op.join(output_dir, "prepared", fn_template1)
fn_stations = op.join(output_dir, "stations.prepared.txt")
fn_event = op.join(event_dir, "event.txt")
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
p = pile.make_pile(fns, show_progress=False)
deltat = None
if sample_rate is not None:
deltat = 1.0 / sample_rate
used_stations = []
for nsl, s in nsl_to_station.items():
s.set_event_relative_data(event)
traces = p.all(trace_selector=lambda tr: tr.nslc_id[:3] == nsl)
keep = []
for tr in traces:
if deltat is not None:
try:
tr.downsample_to(deltat, snap=True, allow_upsample_max=5)
keep.append(tr)
except util.UnavailableDecimation as e:
logger.warn("Cannot downsample %s.%s.%s.%s: %s" %
(tr.nslc_id + (e, )))
continue
if options.out_components == "rtu":
pios = s.guess_projections_to_rtu(out_channels=("R", "T", "Z"))
elif options.out_components == "enu":
pios = s.guess_projections_to_enu(out_channels=("E", "N", "Z"))
else:
assert False
for (proj, in_channels, out_channels) in pios:
proc = trace.project(traces, proj, in_channels, out_channels)
for tr in proc:
for ch in out_channels:
if ch.name == tr.channel:
s.add_channel(ch)
if proc:
io.save(proc, fn_waveforms)
used_stations.append(s)
stations = list(used_stations)
util.ensuredirs(fn_stations)
model.dump_stations(stations, fn_stations)
model.dump_events([event], fn_event)
logger.info("prepared waveforms from %i stations" % len(stations))
s.set_event_relative_data(event)
stations_selected = weeding.weed_stations(stations_all,
options.nstations_wanted)[0]
nsls_selected = set(s.nsl() for s in stations_selected)
logger.info('number of stations selected: %i' % len(nsls_selected))
if tinc is None:
tinc = 3600.
have_data = set()
if options.continue_:
fns = glob.glob(fn_template_raw % starfill())
p = pile.make_pile(fns)
else:
fns = []
have_data_site = {}
could_have_data_site = {}
for site in sites:
have_data_site[site] = set()
could_have_data_site[site] = set()
available_through = defaultdict(set)
it = 0
nt = int(math.ceil((tmax - tmin) / tinc))
for it in range(nt):
tmin_win = tmin + it * tinc
tmax_win = min(tmin + (it + 1) * tinc, tmax)
def run_cont_inversion(cfg_file,tmin_data):
#read in config file
#including setup of path structure
cfg = AT.read_in_config_file(cfg_file)
cfg['datetime'] = util.time_to_str(tmin_data,format='%Y-%m-%dT%H:%M:%S')
event_count = 0
cfg['event_count'] = event_count
filter_flag = int( float( cfg.get('filter_flag',0) ) )
bp_hp_freq = float(cfg.get('bp_lower_corner'))
pad_length = 1./bp_hp_freq*2
if filter_flag == 0:
pad_length = 10
#------------------------------------------
# set files
#
cont_temp_dir =op.realpath(op.abspath(op.join(cfg['temporary_directory'],'cont_run')))
if not op.isdir(cont_temp_dir):
os.makedirs(cont_temp_dir)
cfg['temporary_directory'] = cont_temp_dir
# set file for 'time - VR' history
VR_history_file = op.realpath(op.abspath(op.join(cfg['temporary_directory'],'VR_over_time.txt')))
open( VR_history_file,'w').close()
cfg['vr_history_file'] = VR_history_file
# set file for logging
log_file = op.realpath(op.abspath(op.join(cfg['temporary_directory'],'arctic_log_file.log')))
open(log_file,'w').close()
cfg['log_file'] = log_file
# set file for storing last config dictionary for debugging
cfg_pickle_file = op.realpath(op.abspath(op.join(cfg['temporary_directory'],'cfg_pickle.pp')))
open(cfg_pickle_file,'w').close()
cfg['cfg_pickle_file'] = cfg_pickle_file
# set file for storing last solution dictionary for debugging
solution_pickle_file = op.realpath(op.abspath(op.join(cfg['temporary_directory'],'cfg_pickle.pp')))
open(solution_pickle_file,'w').close()
cfg['solution_pickle_file'] = solution_pickle_file
#------------------------------------------ # set time parameters
#
#safety time delay in seconds
#data are analysed maximal to the current time minus the latency
# allows small buffer for incoming data in real data test case
latency = 5.
# length of analysis window
wlen = float(cfg['time_window_length'])
#minimum window step size, avoiding too high cpu load
minstep = float(cfg['window_steps_in_s'])
# inintialise window end time
tlast = None
#------------------------------------------
# set other parameters
#
print 'setting parameters (time, files, coordinates, etc.)...'
if not AT.set_general_parameters(cfg):
print 'ERROR! could not set up parameters'
raise
print '...done \n '
print 'setting GF... \n'
if not AT.read_in_GF(cfg):
print 'ERROR! could not set up GF array'
exit()
print '...GF ready \n'
if not AT.setup_A(cfg):
print 'ERROR! could not set up correlation matrix A'
raise
if not AT.calc_inv_A(cfg):
print 'ERROR! could not set up inverted matrix inv_A'
raise
print '...done \n'
#------------------------------------------
#
#define data pile from given file location
p = pile.make_pile(cfg['data_dir'])
# read in GFs
if not AT.read_in_GF(cfg):
exit('ERROR in reading GFs')
#------------------------------------------
#d_break(locals(), 'last exit before while loop')
#------------------------------------------
#set current real time as basis time
tmin_real = time.time()
#------------------------------------------
#start infinite run
while True:
# check, if enough time has pased since last loop run
while True:
# define internal time
tfake = ( time.time() - (tmin_real-tmin_data))
# either beginning of analysis or minimal time for window movement has passed
if tlast is None or tfake - tlast > minstep:
break
#wait time, needed to fill minumum window step length
time.sleep(minstep-(tfake-tlast))
print 'real_time: %s -- data_time: %s'%(util.time_to_str(time.time()),util.time_to_str(tfake) )
#set temporary analysis window
tmin = tfake - latency - wlen
tmax = tfake - latency + p.get_deltats()[0]
cfg['current_window_starttime'] = tmin
print 'analysing from %s -- %s'%(util.time_to_str(tmin),util.time_to_str(tmax))
#read in all data traces, available for this window
try:
traces = p.all( tmin=tmin, tmax=tmax, want_incomplete=False ,tpad = pad_length)
for t in traces:
restitute_t(t,cfg,bp_hp_freq)
except Exception, e:
sys.stderr.write( '%s' % e )
pass
#d_break(locals(),'%s'%(util.time_to_str(tfake)))
# analyse the data
#try:
analysis_4_given_window(traces,cfg)
#except Exception, e:
# sys.stderr.write( '%s' % e )
#save current time stamp for next loop run
tlast = tfake
def _get_pile(conf, pathconf):
fn_pattern = conf.path(pathconf)
fns = glob.glob(fn_pattern % Stars())
return pile.make_pile(fns, show_progress=False)
from pyrocko import trace
from pyrocko import model
from pyrocko import pile
from pyrocko import gui_util
from pyrocko.gf import LocalEngine
from autogain import autogain, util_optic
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
km = 1000.
if __name__ == '__main__':
# Where is your data stored?:
datapath = '/media/usb/webnet/pole_zero/restituted_displacement/2013Mar'
data_pile = pile.make_pile(datapath)
# And the stations:
stations = model.load_stations('/media/usb/webnet/meta/stations.pf')
# Station code of the trace you want to scale agains:
reference_id = 'NKC'
# Frequency band to use:
fband = {'order': 4, 'corner_hp': 1.0, 'corner_lp': 4.}
# And a taper to avoid filtering artefacts.
taper = trace.CosFader(xfrac=0.25)
# Define a window to chop traces. In this case a static length of 20
# seconds will be used and the synthetic phase arrival will be in the
def snuffle(pile=None, **kwargs):
'''View pile in a snuffler window.
:param pile: :py:class:`pile.Pile` object to be visualized
:param stations: list of `pyrocko.model.Station` objects or ``None``
:param events: list of `pyrocko.model.Event` objects or ``None``
:param markers: list of `pyrocko.gui.util.Marker` objects or ``None``
:param ntracks: float, number of tracks to be shown initially (default: 12)
:param follow: time interval (in seconds) for real time follow mode or
``None``
:param controls: bool, whether to show the main controls (default:
``True``)
:param opengl: bool, whether to use opengl (default: ``False``)
:param paths: list of files and directories to search for trace files
:param pattern: regex which filenames must match
:param format: format of input files
:param cache_dir: cache directory with trace meta information
:param force_cache: bool, whether to use the cache when attribute spoofing
is active
:param store_path: filename template, where to store trace data from input
streams
:param store_interval: float, time interval (in seconds) between stream
buffer dumps
:param want_markers: bool, whether markers should be returned
:param launch_hook: callback function called before snuffler window is
shown
'''
from .snuffler_app import Snuffler, SnufflerWindow, \
setup_acquisition_sources, PollInjector
if pile is None:
pile = pile.make_pile()
global app
if app is None:
import locale
locale.setlocale(locale.LC_ALL, 'C')
app = Snuffler()
kwargs_load = {}
for k in ('paths', 'regex', 'format', 'cache_dir', 'force_cache'):
try:
kwargs_load[k] = kwargs.pop(k)
except KeyError:
pass
store_path = kwargs.pop('store_path', None)
store_interval = kwargs.pop('store_interval', 600)
want_markers = kwargs.pop('want_markers', False)
launch_hook = kwargs.pop('launch_hook', None)
win = SnufflerWindow(pile, **kwargs)
if launch_hook:
launch_hook(win)
sources = []
pollinjector = None
tempdir = None
if 'paths' in kwargs_load:
sources.extend(setup_acquisition_sources(kwargs_load['paths']))
if sources:
if store_path is None:
tempdir = tempfile.mkdtemp('', 'snuffler-tmp-')
store_path = pjoin(
tempdir,
'trace-%(network)s.%(station)s.%(location)s.%(channel)s.'
'%(tmin)s.mseed')
elif os.path.isdir(store_path):
store_path = pjoin(
store_path,
'trace-%(network)s.%(station)s.%(location)s.%(channel)s.'
'%(tmin)s.mseed')
pollinjector = PollInjector(
pile,
fixation_length=store_interval,
path=store_path)
for source in sources:
source.start()
pollinjector.add_source(source)
win.get_view().load(**kwargs_load)
if not win.is_closing():
app.install_sigint_handler()
app.exec_()
app.uninstall_sigint_handler()
for source in sources:
source.stop()
if pollinjector:
pollinjector.fixate_all()
ret = win.return_tag()
if want_markers:
markers = win.get_view().get_markers()
del win
gc.collect()
if tempdir:
shutil.rmtree(tempdir)
if want_markers:
return ret, markers
else:
return ret
def prep_orient(datapath,
st,
loc,
catalog,
dir_ro,
v_rayleigh,
bp,
dt_start,
dt_stop,
ccmin=0.80,
plot_heatmap=False,
plot_distr=False,
debug=False):
"""
Perform orientation analysis using Rayleigh waves, main function.
time wdw: 20s before 4.0km/s arrival and 600 s afterwards
(Stachnik et al. 2012)
- compute radial component for back values of 0 to 360 deg
- for each c-c of hilbert(R) with Z comp.
- call plotting functions and/or write results to file
:param datapath: path to rrd data
:param st: current station (pyrocko station object)
:param catalog: list of pyrocko events used for analysis
:param dir_ro: output directory
:param plot_heatmap: bool, optional
:param plot_distr: bool, optional
"""
logs = logging.getLogger('prep_orient')
st_data_pile = pile.make_pile(datapath,
regex='%s_%s_' % (st.network, st.station),
show_progress=False)
n_ev = len(catalog)
if st_data_pile.tmin is not None and st_data_pile.tmax is not None:
# calculate dist between all events and current station
r_arr_by_ev = num.empty(n_ev)
ev_lats = num.asarray([ev.lat for ev in catalog])
ev_lons = num.asarray([ev.lon for ev in catalog])
dists = distance_accurate50m_numpy(a_lats=ev_lats,
a_lons=ev_lons,
b_lats=st.lat,
b_lons=st.lon,
implementation='c')
r_arr_by_ev = (dists / 1000.) / v_rayleigh
cc_i_ev_vs_rota = num.empty((n_ev, 360))
rot_angles = range(-180, 180, 1)
for i_ev, ev in enumerate(catalog):
arrT = ev.time + r_arr_by_ev[i_ev]
start_twd1 = ev.time
end_twd1 = arrT + 1800
trZ = get_tr_by_cha(st_data_pile, start_twd1, end_twd1, loc, 'Z')
trR = get_tr_by_cha(st_data_pile, start_twd1, end_twd1, loc, 'R')
trT = get_tr_by_cha(st_data_pile, start_twd1, end_twd1, loc, 'T')
start_twd2 = ev.time + r_arr_by_ev[i_ev] - dt_start
end_twd2 = arrT + dt_stop
if len(trZ) == 1 and len(trR) == 1 and len(trT) == 1:
trZ = trZ[0]
trR = trR[0]
trT = trT[0]
# debugging - window selection:
if debug is True:
trace.snuffle([trZ, trR, trT],
markers=[
pm.Marker(nslc_ids=[
trZ.nslc_id, trR.nslc_id, trT.nslc_id
],
tmin=start_twd2,
tmax=end_twd2),
pm.Marker(nslc_ids=[
trZ.nslc_id, trR.nslc_id, trT.nslc_id
],
tmin=arrT,
tmax=arrT + 3)
])
else:
cc_i_ev_vs_rota[i_ev, :] = num.nan
continue
try:
trZ.bandpass(bp[0], bp[1], bp[2])
trZ.chop(tmin=start_twd2, tmax=end_twd2)
except trace.NoData:
logs.warning('no data %s %s %s' % (trZ, trR, trT))
continue
for i_r, r in enumerate(rot_angles):
print('rotation angle [deg]: %5d' % r, end='\r')
rot_2, rot_3 = trace.rotate(traces=[trR, trT],
azimuth=r,
in_channels=['R', 'T'],
out_channels=['2', '3'])
rot_2_y = rot_2.ydata
rot_2_hilb = num.imag(trace.hilbert(rot_2_y, len(rot_2_y)))
rot_2_hilb_tr = trace.Trace(deltat=rot_2.deltat,
ydata=rot_2_hilb,
tmin=rot_2.tmin)
# problem: rot_2 and rot_2_hilb look exactly the same!
# --> no phase shift. why? should be num.imag!!!
# trace.snuffle([rot_2, rot_2_hilb_tr])
rot_2_hilb_tr.bandpass(bp[0], bp[1], bp[2])
rot_2_hilb_tr.chop(tmin=start_twd2, tmax=end_twd2)
# if st.station == 'RORO' and r == 0:
# trace.snuffle([rot_2_hilb_tr, trZ])
# normalize traces
trZ.ydata /= abs(max(trZ.ydata))
rot_2_hilb_tr.ydata /= abs(max(rot_2_hilb_tr.ydata))
c = trace.correlate(trZ,
rot_2_hilb_tr,
mode='valid',
normalization='normal')
t, coef = c.max()
t2, coef2 = max_or_min(c)
'''
if st.station == 'MATE' and r == 0:
print(i_ev, ev.name, ev.depth)
print(r, t, coef, t2, coef2)
trace.snuffle([trZ, trR, rot_2_hilb_tr])
'''
cc_i_ev_vs_rota[i_ev, i_r] = coef
'''
if st.station == 'MATE':
for i_ev in range(n_ev):
print(num.argmax(cc_i_ev_vs_rota[i_ev,:]),
num.max(cc_i_ev_vs_rota[i_ev,:]))
'''
if plot_heatmap is True:
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8, 2))
cax = ax.imshow(cc_i_ev_vs_rota,
interpolation='nearest',
vmin=-1.0,
vmax=1.0,
aspect='auto',
extent=[-180, 180, n_ev, 0],
cmap='binary')
ax.set_ylabel('i_ev')
ax.set_xlabel('Correction angle (deg)')
ax.set_title('%s %s' % (st.network, st.station))
cbar = fig.colorbar(cax,
ticks=[0, 0.5, 1.0],
orientation='horizontal',
fraction=0.05,
pad=0.5)
cbar.ax.set_xticklabels(['0', '0.5', '1.0'])
plt.tight_layout()
# plt.show(fig)
fig.savefig(
os.path.join(
dir_ro, '%s_%s_%s_rot_cc_heatmap.png' %
(st.network, st.station, loc)))
plt.close()
if plot_distr is True:
plot_ccdistr_each_event(cc_i_ev_vs_rota, catalog, rot_angles, st,
loc, dir_ro)
median_a, mean_a, std_a, switched, n_ev =\
get_m_angle_switched(cc_i_ev_vs_rota, catalog, st, ccmin)
dict_ev_angle = get_m_angle_all(cc_i_ev_vs_rota, catalog, st, ccmin)
return median_a, mean_a, std_a, switched, dict_ev_angle, n_ev
# Download example data
get_example_data('data_conversion', recursive=True)
input_path = 'data_conversion/mseed'
output_path = 'data_conversion/sac/' \
'%(dirhz)s/%(station)s_%(channel)s_%(tmin)s.sac'
fn_stations = 'data_conversion/stations.txt'
stations_list = model.load_stations(fn_stations)
stations = {}
for s in stations_list:
stations[s.network, s.station, s.location] = s
s.set_channels_by_name(*'BHN BHE BHZ BLN BLE BLZ'.split())
p = pile.make_pile(input_path)
h = 3600.
tinc = 1 * h
tmin = util.day_start(p.tmin)
for traces in p.chopper_grouped(tmin=tmin,
tinc=tinc,
gather=lambda tr: tr.nslc_id):
for tr in traces:
dirhz = '%ihz' % int(round(1. / tr.deltat))
io.save([tr],
output_path,
format='sac',
additional={'dirhz': dirhz},
stations=stations)
from pyrocko import pile, io, util
import time, calendar
# when pile.make_pile() is called without any arguments, the command line
# parameters given to the script are searched for waveform files and directories
p = pile.make_pile()
# get timestamp for full hour before first data sample in all selected traces
tmin = calendar.timegm( time.gmtime(p.tmin)[:4] + ( 0, 0 ) )
tinc = 3600.
tpad = 10.
target_deltat = 0.1
# iterate over the data, with a window length of one hour and 2x10 seconds of
# overlap
for traces in p.chopper(tmin=tmin, tinc=tinc, tpad=tpad):
if traces: # the list could be empty due to gaps
for tr in traces:
tr.downsample_to(target_deltat, snap=True, demean=False)
# remove overlapping
tr.chop(tr.wmin, tr.wmax)
window_start = traces[0].wmin
timestring = util.time_to_str(window_start, format='%Y-%m-%d_%H')
filepath = 'downsampled/%(station)s_%(channel)s_%(mytimestring)s.mseed'
io.save(traces, filepath, additional={'mytimestring': timestring})
from pyrocko import pile, io, util
import time
import calendar
''' Chope a pile of waveform traces into segments '''
p = pile.make_pile(['test.mseed'])
# get timestamp for full hour before first data sample in all selected traces
tmin = calendar.timegm(time.gmtime(p.tmin)[:4] + (0, 0))
# iterate over the data, with a window length of one hour
for traces in p.chopper(tmin=tmin, tinc=3600):
if traces: # the list could be empty due to gaps
window_start = traces[0].wmin
timestring = util.time_to_str(window_start, format='%Y-%m-%d_%H')
filepath = 'test_hourfiles/hourfile-%s.mseed' % timestring
io.save(traces, filepath)
# Download example data
get_example_data('data_conversion', recursive=True)
input_path = 'data_conversion/mseed'
output_path = 'data_conversion/sac/' \
'%(dirhz)s/%(station)s_%(channel)s_%(tmin)s.sac'
fn_stations = 'data_conversion/stations.txt'
stations_list = model.load_stations(fn_stations)
stations = {}
for s in stations_list:
stations[s.network, s.station, s.location] = s
s.set_channels_by_name(*'BHN BHE BHZ BLN BLE BLZ'.split())
p = pile.make_pile(input_path)
h = 3600.
tinc = 1*h
tmin = util.day_start(p.tmin)
for traces in p.chopper_grouped(tmin=tmin, tinc=tinc,
gather=lambda tr: tr.nslc_id):
for tr in traces:
dirhz = '%ihz' % int(round(1./tr.deltat))
io.save(
[tr], output_path,
format='sac',
additional={'dirhz': dirhz},
stations=stations
)
