def __init__(self):
# Set up receiver configuration.
tab = '''
HH1 58.500 12.5000 0
HH2 48.500 12.5000 0
HH3 48.500 3.5000 0
HH4 58.500 3.5000 0
'''.strip()
receivers = []
for line_tab in tab.split('\n'):
station, lat, lon, depth = line_tab.split()
r = receiver.Receiver(lat, lon, components='neu', name='.%s.' % station)
receivers.append(r)
stations = receivers_to_stations(receivers)
model.dump_stations(stations, 'reference_stations.txt')
# Composition of the source
self.olat, self.olon = 52.0000, 9.00000
self.otime = util.str_to_time('1986-08-22 07:00:00')
# The gfdb can be chosen within snuffler.
# This refers to the 'add_parameter' method.
db = gfdb.Gfdb('fomostos/local1/local1')
seis = seismosizer.Seismosizer(hosts=['localhost'])
seis.set_database(db)
seis.set_effective_dt(db.dt)
seis.set_local_interpolation('bilinear')
seis.set_receivers(receivers)
seis.set_source_location(self.olat, self.olon, self.otime)
seis.set_source_constraints(0, 0, 0, 0, 0, -1)
self.seis = seis
def save(self):
if not self.current_stuff:
self.fail('Nothing to save.')
data_fn = self.output_filename(caption='Save Data', dir='data-%(network)s-%(station)s-%(location)s-%(channel)s-%(tmin)s.mseed')
stations_fn = self.output_filename(caption='Save Stations File', dir='stations.txt')
all_traces, stations = self.current_stuff
io.save(all_traces, data_fn)
model.dump_stations(stations, stations_fn)
def save(self):
if not self.current_stuff:
self.fail('Nothing to save.')
data_fn = self.output_filename(caption='Save Data', dir='data-%(network)s-%(station)s-%(location)s-%(channel)s-%(tmin)s.mseed')
stations_fn = self.output_filename(caption='Save Stations File', dir='stations.txt')
all_traces, stations = self.current_stuff
io.save(all_traces, data_fn)
model.dump_stations(stations, stations_fn)
def save(synthetic_traces, event, stations, savedir, noise_events=False):
model.dump_stations(stations, savedir + 'model.txt')
io.save(synthetic_traces, savedir + 'traces.mseed')
model.dump_events(event, savedir + 'event.txt')
model.dump_stations(stations, savedir + 'stations.pf')
st_xml = stationxml.FDSNStationXML.from_pyrocko_stations(
stations, add_flat_responses_from='M')
st_xml.dump_xml(filename=savedir + 'stations.xml')
if noise_events is not False:
model.dump_events(noise_events, savedir + 'events_noise.txt')
def call(self):
self.cleanup()
if self.tinc is not None:
template = \
'trace_%n.%s.%l.%c_%(tmin_us)s'
else:
template = 'trace_%n.%s.%l.%c'
if self.format == 'text':
default_output_filename = template + '.dat'
else:
default_output_filename = template + '.' + self.format
out_filename = self.output_filename('Template for output files',
default_output_filename)
viewer = self.get_viewer()
for trs in self.chopper_selected_traces(fallback=True, tinc=self.tinc):
trs2save = []
for tr in trs:
if self.format == 'mseed':
if len(tr.network) > 2:
tr.set_network(tr.network[:2])
if len(tr.station) > 5:
tr.set_station(tr.station[:5])
if len(tr.location) > 2:
tr.set_location(tr.location[:2])
if len(tr.channel) > 3:
tr.set_channel(tr.channel[:3])
if viewer.lowpass:
if viewer.lowpass < 0.5/tr.deltat:
tr.lowpass(4, viewer.lowpass, demean=False)
if viewer.highpass:
if viewer.highpass < 0.5/tr.deltat:
tr.highpass(4, viewer.highpass, demean=False)
trs2save.append(tr)
try:
io.save(
trs2save, out_filename,
format=self.format,
overwrite=True)
except io.io_common.FileSaveError as e:
self.fail(str(e))
if self.save_stations:
stations = viewer.stations.values()
fn = self.output_filename('Save Stations', 'stations.pf')
model.dump_stations(list(stations), fn)
def testIOStations(self):
tempdir = tempfile.mkdtemp()
fn = pjoin(tempdir, 'stations.txt')
ne = model.Channel('NE', azimuth=45., dip=0.)
se = model.Channel('SE', azimuth=135., dip=0.)
stations = [ model.Station('', sta,'', 0.,0., 0., channels=[ne,se]) for sta in ['STA1', 'STA2'] ]
model.dump_stations(stations, fn)
stations = model.load_stations(fn)
shutil.rmtree(tempdir)
def call(self):
self.cleanup()
if self.tinc is not None:
template = \
'trace_%n.%s.%l.%c_%(tmin_us)s'
else:
template = 'trace_%n.%s.%l.%c'
if self.format == 'text':
default_output_filename = template + '.dat'
else:
default_output_filename = template + '.' + self.format
out_filename = self.output_filename('Template for output files',
default_output_filename)
viewer = self.get_viewer()
for trs in self.chopper_selected_traces(fallback=True, tinc=self.tinc):
trs2save = []
for tr in trs:
if self.format == 'mseed':
if len(tr.network) > 2:
tr.set_network(tr.network[:2])
if len(tr.station) > 5:
tr.set_station(tr.station[:5])
if len(tr.location) > 2:
tr.set_location(tr.location[:2])
if len(tr.channel) > 3:
tr.set_channel(tr.channel[:3])
if viewer.lowpass:
if viewer.lowpass < 0.5 / tr.deltat:
tr.lowpass(4, viewer.lowpass, demean=False)
if viewer.highpass:
if viewer.highpass < 0.5 / tr.deltat:
tr.highpass(4, viewer.highpass, demean=False)
trs2save.append(tr)
try:
io.save(trs2save,
out_filename,
format=self.format,
overwrite=True)
except io.io_common.FileSaveError as e:
self.fail(str(e))
if self.save_stations:
stations = viewer.stations.values()
fn = self.output_filename('Save Stations', 'stations.pf')
model.dump_stations(list(stations), fn)
def testIOStations(self):
tempdir = self.make_tempdir()
fn = pjoin(tempdir, 'stations.txt')
ne = model.Channel('NE', azimuth=45., dip=0.)
se = model.Channel('SE', azimuth=135., dip=0.)
stations = [
model.Station('', sta, '', 0., 0., 0., channels=[ne, se])
for sta in ['STA1', 'STA2']]
model.dump_stations(stations, fn)
stations = model.load_stations(fn)
def save(self):
if not self.current_stuff:
self.fail("Nothing to save.")
data_fn = self.output_filename(
caption="Save Data", dir="data-%(network)s-%(station)s-%(location)s-%(channel)s-%(tmin)s.mseed"
)
stations_fn = self.output_filename(caption="Save Stations File", dir="stations.txt")
all_traces, stations = self.current_stuff
io.save(all_traces, data_fn)
model.dump_stations(stations, stations_fn)
def testIOStations(self):
tempdir = tempfile.mkdtemp(prefix='pyrocko-model')
fn = pjoin(tempdir, 'stations.txt')
ne = model.Channel('NE', azimuth=45., dip=0.)
se = model.Channel('SE', azimuth=135., dip=0.)
stations = [
model.Station('', sta, '', 0., 0., 0., channels=[ne, se])
for sta in ['STA1', 'STA2']
]
model.dump_stations(stations, fn)
stations = model.load_stations(fn)
shutil.rmtree(tempdir)
def call(self):
self.cleanup()
if self.combine and not self.format=='mseed':
self.fail('"Combine" only possible when exporting "mseed"')
if self.combine:
template = 'traces_export'
else:
template = 'trace_%(network)s.%(station)s.%(location)s.%(channel)s'
try:
if self.format == 'text':
default_output_filename = template + '.dat'
else:
default_output_filename = template + '.' + self.format
out_filename = self.output_filename('Template for output files',
default_output_filename)
except NoViewerSet:
out_filename = self.out_filename
traces = self.chopper_selected_traces(fallback=True)
trs2save = []
for trs in traces:
for tr in trs:
if self.format == 'mseed':
if len(tr.network) > 2:
tr.set_network(tr.network[:2])
if len(tr.station) > 5:
tr.set_station(tr.station[:5])
if len(tr.location) > 2:
tr.set_location(tr.location[:2])
if len(tr.channel) > 3:
tr.set_channel(tr.channel[:3])
if self.combine:
trs2save.append(tr)
else:
io.save(tr, out_filename, format=self.format)
if self.combine:
io.save(trs2save, out_filename, format=self.format)
if self.save_stations:
stations = self.get_viewer().stations.values()
fn = self.output_filename('Save Stations', 'stations.pf')
model.dump_stations(stations, fn)
def __init__(self):
# Set up receiver configuration.
tab = '''
HH1 58.500 12.5000 0
HH2 48.500 12.5000 0
HH3 48.500 3.5000 0
HH4 58.500 3.5000 0
'''.strip()
receivers = []
for line_tab in tab.split('\n'):
station, lat, lon, depth = line_tab.split()
r = receiver.Receiver(lat,
lon,
components='neu',
name='.%s.' % station)
receivers.append(r)
stations = receivers_to_stations(receivers)
model.dump_stations(stations, 'reference_stations.txt')
# Composition of the source
self.olat, self.olon = 52.0000, 9.00000
self.otime = util.str_to_time('1986-08-22 07:00:00')
# The gfdb can be chosen within snuffler.
# This refers to the 'add_parameter' method.
db = gfdb.Gfdb('fomostos/local1/local1')
seis = seismosizer.Seismosizer(hosts=['localhost'])
seis.set_database(db)
seis.set_effective_dt(db.dt)
seis.set_local_interpolation('bilinear')
seis.set_receivers(receivers)
seis.set_source_location(self.olat, self.olon, self.otime)
seis.set_source_constraints(0, 0, 0, 0, 0, -1)
self.seis = seis
from pyrocko.rdseed import SeedVolumeAccess from pyrocko import model import sys '''Extract pyrocko stations from dataless seed volume. This requires rdseed. ''' fn = sys.argv[1] print fn dseedfile = SeedVolumeAccess(fn) stations = dseedfile._get_stations_from_file() model.dump_stations(stations, 'stations_from_rdseed.pf')
# chapter 1.6: dump raw data stations file
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())]
util.ensuredirs(fn_stations_raw)
model.dump_stations(stations, fn_stations_raw)
dump_commandline(sys.argv, fn_commandline)
# chapter 2: restitution
if not fns:
logger.error('no data available')
sys.exit(1)
p = pile.make_pile(fns, show_progress=False)
p.get_deltatmin()
otinc = None
if otinc is None:
otinc = nice_seconds_floor(p.get_deltatmin() * 500000.)
otinc = 3600.
def download(self, event, directory='array_data', timing=None, length=None,
want='all', force=False, prefix=False, dump_config=False,
get_responses=False):
""":param want: either 'all' or ID as string or list of IDs as strings
"""
use = []
#ts = {}
unit = 'M'
if all([timing, length]) is None:
raise Exception('Define one of "timing" and "length"')
prefix = prefix or ''
directory = pjoin(prefix, directory)
if not os.path.isdir(directory):
os.mkdir(directory)
pzresponses = {}
logger.info('download data: %s at %sN %sE' % (
event.name, event.lat, event.lon))
for site, array_data_provder in self.providers.items():
logger.info('requesting data from site %s' % site)
for array_id, codes in array_data_provder.items():
if array_id not in want and want != ['all']:
continue
sub_directory = pjoin(directory, array_id)
logger.info("%s" % array_id)
codes = array_data_provder[array_id]
if not isinstance(codes, list):
codes = [codes]
selection = [
c + tuple((event.time, event.time+1000.)) for c in codes]
logger.debug('selection: %s' % selection)
try:
# if site=='bgr':
# st = ws.station(url='http://eida.bgr.de/', selection=selection)
# else:
# st = ws.station(site=site, selection=selection)
st = ws.station(site=site, selection=selection)
except ws.EmptyResult as e:
logging.error('No results: %s %s. skip' % (e, array_id))
continue
except ValueError as e:
logger.error(e)
logger.error('...skipping...')
continue
stations = st.get_pyrocko_stations()
min_dist = min(
[ortho.distance_accurate50m(s, event) for s in stations])
max_dist = max(
[ortho.distance_accurate50m(s, event) for s in stations])
mod = cake.load_model(crust2_profile=(event.lat, event.lon))
if length:
tstart = 0.
tend = length
elif timing:
tstart = timing[0].t(mod, (event.depth, min_dist))
tend = timing[1].t(mod, (event.depth, max_dist))
selection = [
c + tuple((event.time + tstart, event.time + tend)
) for c in codes]
try:
d = ws.dataselect(site=site, selection=selection)
store.remake_dir(sub_directory, force)
store.remake_dir(pjoin(sub_directory, 'responses'), force)
fn = pjoin(sub_directory, 'traces.mseed')
with open(fn, 'w') as f:
f.write(d.read())
f.close()
if get_responses:
trs = io.load(fn, getdata=False)
logger.info('Request responses from %s' % site)
if progressbar:
pb = progressbar.ProgressBar(maxval=len(trs)).start()
for i_tr, tr in enumerate(trs):
try:
st = ws.station(
site=site, selection=selection, level='response')
pzresponse = st.get_pyrocko_response(
nslc=tr.nslc_id,
timespan=(tr.tmin, tr.tmax),
fake_input_units=unit)
pzresponse.regularize()
except fdsnstation.NoResponseInformation as e:
logger.warn("no response information: %s" % e)
pzresponse = None
pass
except fdsnstation.MultipleResponseInformation as e:
logger.warn("MultipleResponseInformation: %s" % e)
pzresponse = None
pass
pzresponses[tr.nslc_id] = pzresponse
pzresponses[tr.nslc_id].dump(filename=pjoin(
sub_directory,
'responses',
'resp_%s.yaml' % '.'.join(tr.nslc_id)))
if progressbar:
pb.update(i_tr)
if progressbar:
pb.finish()
model.dump_stations(
stations, pjoin(sub_directory, 'stations.pf'))
if timing:
t = Timings(list(timing))
self.timings[array_id] = t
if array_id not in use and array_id not in self.use:
use.append(array_id)
except ws.EmptyResult as e:
logging.error('%s on %s' % (e, array_id))
self.use.extend(use)
def iris_pull(options, conf, event_names):
conf = conf['iris_pull_config']
if not event_names:
sys.exit('need event name')
preparator = InvResponsePreparator(conf.begin_phase, conf.end_phase,
conf.inv_response_frequencyband)
for event_name in event_names:
conf.event_name = event_name
event = _get_event_infos(conf)
tevent = event.time
station_query_save_path = conf.path('station_query_save_path')
if os.path.exists(station_query_save_path):
logger.info('Using stored station query.')
all_stations = pload(station_query_save_path)
else:
logger.info('Querying for stations...')
all_stations = combi_get_stations(
lat=event.lat,
lon=event.lon,
rmin=conf.query_rmin,
rmax=conf.query_rmax,
tmin=tevent,
tmax=tevent + 3600.,
channel_pattern=conf.query_channel_pattern)
util.ensuredirs(station_query_save_path)
pdump(all_stations, station_query_save_path)
nstations = len(set(
(sta.network, sta.station) for sta in all_stations))
logger.info('Station query returned %i station%s' %
(nstations, plural_s(nstations)))
preferred_ns = set(
iris_get_vnets(conf.preferred_virtual_networks,
tmin=tevent,
tmax=tevent + 3600.))
preferred_n = set(conf.preferred_networks)
for station in all_stations:
station.set_event_relative_data(event)
raw_trace_path = conf.path('raw_trace_path')
nsl_all = set(get_nsl(s) for s in all_stations)
state_save_path = conf.path('state_save_path')
if os.path.exists(state_save_path):
nsl_ok, nsl_blacklist, nsl_use = pload(state_save_path)
else:
nsl_ok = set()
nsl_blacklist = set()
nsl_use = set()
manual_blacklist_path = conf.path('manual_blacklist_path')
nsl_blacklist.update(read_manual_blacklist(manual_blacklist_path))
nsl_selected = set()
nwanted = conf.get_or_none('nwanted')
assert len(nwanted) == 2
while True:
if nwanted:
selected = select_stations(all_stations,
nwanted[1],
preferred_n=preferred_n,
preferred_ns=preferred_ns,
preferred_nsl=nsl_ok,
blacklist_nsl=nsl_blacklist)
else:
selected = all_stations
nsl_selected = set(get_nsl(s) for s in selected)
download = [s for s in selected if get_nsl(s) not in nsl_ok]
nsl_download = set(get_nsl(s) for s in download)
combi_get_responses(download, tevent, conf.path('resp_path'))
tmin = preparator.get_tmin_limit(event, download)
tmax = preparator.get_tmax_limit(event, download)
logger.info('Loading data for event %s:\n%s' %
(event_name, str_nsl_selection(nsl_download)))
try:
fns = combi_get_data(
download,
tmin,
tmax,
raw_trace_path,
neach=conf.query_nstations_per_datarequest)
p = pile.make_pile(fns, show_progress=False)
prepared_trace_path = conf.path('prepared_trace_path')
for traces in preparator.iter_prepare(p, event, download):
for tr in traces:
nsl_ok.add(get_nsl(tr))
io.save(traces, prepared_trace_path)
except iris_ws.NotFound:
pass
logger.info('Blacklisting:\n%s' %
str_nsl_selection(nsl_download - nsl_ok))
nsl_blacklist.update(nsl_download - nsl_ok)
preferred_ns.update(set(nsl[:2] for nsl in nsl_ok))
nsl_use = nsl_ok & nsl_selected
logger.info('Have %i stations with data.' % len(nsl_use))
if not nwanted:
break
else:
if len(nsl_selected) == len(nsl_all):
break
if len(nsl_use) > nwanted[0]:
break
pdump((nsl_ok, nsl_blacklist, nsl_use), state_save_path)
if nwanted and len(nsl_use) >= nwanted[1]:
selected = select_stations(selected,
nwanted[1],
preferred_n=preferred_n,
preferred_ns=preferred_ns,
blacklist_nsl=nsl_blacklist)
nsl_selected = set(get_nsl(s) for s in selected)
nsl_use = nsl_ok & nsl_selected
stations = [s for s in all_stations if get_nsl(s) in nsl_use]
model.dump_stations(stations, conf.path('stations_ok_path'))
model.dump_stations(all_stations, conf.path('stations_all_path'))
if nsl_use:
logger.info('Data available for event %s:\n%s' %
(event_name, str_nsl_selection(nsl_use)))
else:
logger.info('No data availabe for event %s' % event_name)
# chapter 1.6: dump raw data stations file
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())]
util.ensuredirs(fn_stations_raw)
model.dump_stations(stations, fn_stations_raw)
dump_commandline(sys.argv, fn_commandline)
# chapter 2: restitution
if not fns:
logger.error('no data available')
sys.exit(1)
p = pile.make_pile(fns, show_progress=False)
p.get_deltatmin()
otinc = None
if otinc is None:
otinc = nice_seconds_floor(p.get_deltatmin() * 500000.)
otinc = 3600.
def save_station(self):
fn = self.output_filename('Save Station')
dump_stations([self.station_c], fn)
def download(self,
event,
directory='array_data',
timing=None,
length=None,
want='all',
force=False,
prefix=False,
dump_config=False,
get_responses=False):
""":param want: either 'all' or ID as string or list of IDs as strings
"""
use = []
#ts = {}
unit = 'M'
if all([timing, length]) is None:
raise Exception('Define one of "timing" and "length"')
prefix = prefix or ''
directory = pjoin(prefix, directory)
if not os.path.isdir(directory):
os.mkdir(directory)
pzresponses = {}
logger.info('download data: %s at %sN %sE' %
(event.name, event.lat, event.lon))
for site, array_data_provder in self.providers.items():
logger.info('requesting data from site %s' % site)
for array_id, codes in array_data_provder.items():
if array_id not in want and want != ['all']:
continue
sub_directory = pjoin(directory, array_id)
logger.info("%s" % array_id)
codes = array_data_provder[array_id]
if not isinstance(codes, list):
codes = [codes]
selection = [
c + tuple((event.time, event.time + 1000.)) for c in codes
]
logger.debug('selection: %s' % selection)
try:
# if site=='bgr':
# st = ws.station(url='http://eida.bgr.de/', selection=selection)
# else:
# st = ws.station(site=site, selection=selection)
st = ws.station(site=site, selection=selection)
except ws.EmptyResult as e:
logging.error('No results: %s %s. skip' % (e, array_id))
continue
except ValueError as e:
logger.error(e)
logger.error('...skipping...')
continue
stations = st.get_pyrocko_stations()
min_dist = min(
[ortho.distance_accurate50m(s, event) for s in stations])
max_dist = max(
[ortho.distance_accurate50m(s, event) for s in stations])
mod = cake.load_model(crust2_profile=(event.lat, event.lon))
if length:
tstart = 0.
tend = length
elif timing:
tstart = timing[0].t(mod, (event.depth, min_dist))
tend = timing[1].t(mod, (event.depth, max_dist))
selection = [
c + tuple((event.time + tstart, event.time + tend))
for c in codes
]
try:
d = ws.dataselect(site=site, selection=selection)
store.remake_dir(sub_directory, force)
store.remake_dir(pjoin(sub_directory, 'responses'), force)
fn = pjoin(sub_directory, 'traces.mseed')
with open(fn, 'w') as f:
f.write(d.read())
f.close()
if get_responses:
trs = io.load(fn, getdata=False)
logger.info('Request responses from %s' % site)
if progressbar:
pb = progressbar.ProgressBar(
maxval=len(trs)).start()
for i_tr, tr in enumerate(trs):
try:
st = ws.station(site=site,
selection=selection,
level='response')
pzresponse = st.get_pyrocko_response(
nslc=tr.nslc_id,
timespan=(tr.tmin, tr.tmax),
fake_input_units=unit)
pzresponse.regularize()
except fdsnstation.NoResponseInformation as e:
logger.warn("no response information: %s" % e)
pzresponse = None
pass
except fdsnstation.MultipleResponseInformation as e:
logger.warn("MultipleResponseInformation: %s" %
e)
pzresponse = None
pass
pzresponses[tr.nslc_id] = pzresponse
pzresponses[tr.nslc_id].dump(filename=pjoin(
sub_directory, 'responses', 'resp_%s.yaml' %
'.'.join(tr.nslc_id)))
if progressbar:
pb.update(i_tr)
if progressbar:
pb.finish()
model.dump_stations(stations,
pjoin(sub_directory, 'stations.pf'))
if timing:
t = Timings(list(timing))
self.timings[array_id] = t
if array_id not in use and array_id not in self.use:
use.append(array_id)
except ws.EmptyResult as e:
logging.error('%s on %s' % (e, array_id))
self.use.extend(use)
def save_station(self, fn):
dump_stations([self.station_c], fn)
def main():
parser = OptionParser(usage=usage, description=description)
parser.add_option(
"--force",
dest="force",
action="store_true",
default=False,
help="allow recreation of output <directory>",
)
parser.add_option(
"--debug",
dest="debug",
action="store_true",
default=False,
help="print debugging information to stderr",
)
parser.add_option(
"--dry-run",
dest="dry_run",
action="store_true",
default=False,
help="show available stations/channels and exit "
"(do not download waveforms)",
)
parser.add_option(
"--continue",
dest="continue_",
action="store_true",
default=False,
help="continue download after a accident",
)
parser.add_option(
"--local-data",
dest="local_data",
action="append",
help="add file/directory with local data",
)
parser.add_option(
"--local-stations",
dest="local_stations",
action="append",
help="add local stations file",
)
parser.add_option(
"--local-responses-resp",
dest="local_responses_resp",
action="append",
help="add file/directory with local responses in RESP format",
)
parser.add_option(
"--local-responses-pz",
dest="local_responses_pz",
action="append",
help="add file/directory with local pole-zero responses",
)
parser.add_option(
"--local-responses-stationxml",
dest="local_responses_stationxml",
help="add file with local response information in StationXML format",
)
parser.add_option(
"--window",
dest="window",
default="full",
help='set time window to choose [full, p, "<time-start>,<time-end>"'
"] (time format is YYYY-MM-DD HH:MM:SS)",
)
parser.add_option(
"--out-components",
choices=["enu", "rtu"],
dest="out_components",
default="rtu",
help="set output component orientations to radial-transverse-up [rtu] "
"(default) or east-north-up [enu]",
)
parser.add_option(
"--padding-factor",
type=float,
default=3.0,
dest="padding_factor",
help="extend time window on either side, in multiples of 1/<fmin_hz> "
"(default: 5)",
)
parser.add_option(
"--credentials",
dest="user_credentials",
action="append",
default=[],
metavar="SITE,USER,PASSWD",
help="user credentials for specific site to access restricted data "
"(this option can be repeated)",
)
parser.add_option(
"--token",
dest="auth_tokens",
metavar="SITE,FILENAME",
action="append",
default=[],
help="user authentication token for specific site to access "
"restricted data (this option can be repeated)",
)
parser.add_option(
"--sites",
dest="sites",
metavar="SITE1,SITE2,...",
# default='bgr',
default="http://ws.gpi.kit.edu,bgr,http://188.246.25.142:8080",
help='sites to query (available: %s, default: "%%default"' %
", ".join(g_sites_available),
)
parser.add_option(
"--band-codes",
dest="priority_band_code",
metavar="V,L,M,B,H,S,E,...",
default="V,L,M,B,H,E",
help="select and prioritize band codes (default: %default)",
)
parser.add_option(
"--instrument-codes",
dest="priority_instrument_code",
metavar="H,L,G,...",
default="H,L,O,",
help="select and prioritize instrument codes (default: %default)",
)
parser.add_option(
"--radius-min",
dest="radius_min",
metavar="VALUE",
default=0.0,
type=float,
help="minimum radius [km]",
)
parser.add_option(
"--tinc",
dest="tinc",
metavar="VALUE",
default=3600.0 * 12.0,
type=float,
help="length of seperate saved files in s",
)
parser.add_option(
"--nstations-wanted",
dest="nstations_wanted",
metavar="N",
type=int,
help="number of stations to select initially",
)
(options, args) = parser.parse_args(sys.argv[1:])
if len(args) not in (9, 6, 5):
parser.print_help()
sys.exit(1)
if options.debug:
util.setup_logging(program_name, "debug")
else:
util.setup_logging(program_name, "info")
if options.local_responses_pz and options.local_responses_resp:
logger.critical("cannot use local responses in PZ and RESP "
"format at the same time")
sys.exit(1)
n_resp_opt = 0
for resp_opt in (
options.local_responses_pz,
options.local_responses_resp,
options.local_responses_stationxml,
):
if resp_opt:
n_resp_opt += 1
if n_resp_opt > 1:
logger.critical("can only handle local responses from either PZ or "
"RESP or StationXML. Cannot yet merge different "
"response formats.")
sys.exit(1)
if options.local_responses_resp and not options.local_stations:
logger.critical("--local-responses-resp can only be used "
"when --stations is also given.")
sys.exit(1)
try:
ename = ""
magnitude = None
mt = None
if len(args) == 9:
time = util.str_to_time(args[0] + " " + args[1])
lat = float(args[2])
lon = float(args[3])
depth = float(args[4]) * km
iarg = 5
elif len(args) == 6:
if args[1].find(":") == -1:
sname_or_date = None
lat = float(args[0])
lon = float(args[1])
event = None
time = None
else:
sname_or_date = args[0] + " " + args[1]
iarg = 2
elif len(args) == 5:
sname_or_date = args[0]
iarg = 1
if len(args) in (6, 5) and sname_or_date is not None:
events = get_events_by_name_or_date([sname_or_date],
catalog=geofon)
if len(events) == 0:
logger.critical("no event found")
sys.exit(1)
elif len(events) > 1:
logger.critical("more than one event found")
sys.exit(1)
event = events[0]
time = event.time
lat = event.lat
lon = event.lon
depth = event.depth
ename = event.name
magnitude = event.magnitude
mt = event.moment_tensor
radius = float(args[iarg]) * km
fmin = float(args[iarg + 1])
sample_rate = float(args[iarg + 2])
eventname = args[iarg + 3]
event_dir = op.join("data", "events", eventname)
output_dir = op.join(event_dir, "waveforms")
except:
raise
parser.print_help()
sys.exit(1)
if options.force and op.isdir(event_dir):
if not options.continue_:
shutil.rmtree(event_dir)
if op.exists(event_dir) and not options.continue_:
logger.critical(
'directory "%s" exists. Delete it first or use the --force option'
% event_dir)
sys.exit(1)
util.ensuredir(output_dir)
if time is not None:
event = model.Event(
time=time,
lat=lat,
lon=lon,
depth=depth,
name=ename,
magnitude=magnitude,
moment_tensor=mt,
)
if options.window == "full":
if event is None:
logger.critical("need event for --window=full")
sys.exit(1)
low_velocity = 1500.0
timewindow = VelocityWindow(low_velocity,
tpad=options.padding_factor / fmin)
tmin, tmax = timewindow(time, radius, depth)
elif options.window == "p":
if event is None:
logger.critical("need event for --window=p")
sys.exit(1)
phases = list(map(cake.PhaseDef, "P p".split()))
emod = cake.load_model()
tpad = options.padding_factor / fmin
timewindow = PhaseWindow(emod, phases, -tpad, tpad)
arrivaltimes = []
for dist in num.linspace(0, radius, 20):
try:
arrivaltimes.extend(timewindow(time, dist, depth))
except NoArrival:
pass
if not arrivaltimes:
logger.error("required phase arrival not found")
sys.exit(1)
tmin = min(arrivaltimes)
tmax = max(arrivaltimes)
else:
try:
stmin, stmax = options.window.split(",")
tmin = util.str_to_time(stmin.strip())
tmax = util.str_to_time(stmax.strip())
timewindow = FixedWindow(tmin, tmax)
except ValueError:
logger.critical('invalid argument to --window: "%s"' %
options.window)
sys.exit(1)
if event is not None:
event.name = eventname
tlen = tmax - tmin
tfade = tfade_factor / fmin
tpad = tfade
tmin -= tpad
tmax += tpad
priority_band_code = options.priority_band_code.split(",")
for s in priority_band_code:
if len(s) != 1:
logger.critical("invalid band code: %s" % s)
priority_instrument_code = options.priority_instrument_code.split(",")
for s in priority_instrument_code:
if len(s) != 1:
logger.critical("invalid instrument code: %s" % s)
station_query_conf = dict(
latitude=lat,
longitude=lon,
minradius=options.radius_min * km * cake.m2d,
maxradius=radius * cake.m2d,
channel=",".join("?%s?" % s for s in priority_band_code),
)
target_sample_rate = sample_rate
fmax = target_sample_rate
# target_sample_rate = None
# priority_band_code = ['H', 'B', 'M', 'L', 'V', 'E', 'S']
priority_units = ["M/S", "M", "M/S**2"]
output_units = "M"
sites = [x.strip() for x in options.sites.split(",") if x.strip()]
tinc = options.tinc
# for site in sites:
# if site not in g_sites_available:
# logger.critical('unknown FDSN site: %s' % site)
# sys.exit(1)
for s in options.user_credentials:
try:
site, user, passwd = s.split(",")
g_user_credentials[site] = user, passwd
except ValueError:
logger.critical('invalid format for user credentials: "%s"' % s)
sys.exit(1)
for s in options.auth_tokens:
try:
site, token_filename = s.split(",")
with open(token_filename, "r") as f:
g_auth_tokens[site] = f.read()
except (ValueError, OSError, IOError):
logger.critical("cannot get token from file: %s" % token_filename)
sys.exit(1)
fn_template0 = (
"data_%(network)s.%(station)s.%(location)s.%(channel)s_%(tmin)s.mseed")
fn_template_raw = op.join(output_dir, "raw", fn_template0)
fn_template_raw_folder = op.join(output_dir, "raw/", "traces.mseed")
fn_stations_raw = op.join(output_dir, "stations.raw.txt")
fn_template_rest = op.join(output_dir, "rest", fn_template0)
fn_commandline = op.join(output_dir, "seigerdown.command")
ftap = (ffade_factors[0] * fmin, fmin, fmax, ffade_factors[1] * fmax)
# chapter 1: download
sxs = []
for site in sites:
try:
extra_args = {
"iris": dict(matchtimeseries=True),
}.get(site, {})
extra_args.update(station_query_conf)
if site == "geonet":
extra_args.update(starttime=tmin, endtime=tmax)
else:
extra_args.update(
startbefore=tmax,
endafter=tmin,
includerestricted=(site in g_user_credentials
or site in g_auth_tokens),
)
logger.info("downloading channel information (%s)" % site)
sx = fdsn.station(site=site,
format="text",
level="channel",
**extra_args)
except fdsn.EmptyResult:
logger.error("No stations matching given criteria. (%s)" % site)
sx = None
if sx is not None:
sxs.append(sx)
if all(sx is None for sx in sxs) and not options.local_data:
sys.exit(1)
nsl_to_sites = defaultdict(list)
nsl_to_station = {}
for sx, site in zip(sxs, sites):
site_stations = sx.get_pyrocko_stations()
for s in site_stations:
nsl = s.nsl()
nsl_to_sites[nsl].append(site)
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s # using first site with this station
logger.info("number of stations found: %i" % len(nsl_to_station))
# station weeding
nsls_selected = None
if options.nstations_wanted:
stations_all = [
nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())
]
for s in stations_all:
s.set_event_relative_data(event)
stations_selected = weeding.weed_stations(stations_all,
options.nstations_wanted)[0]
nsls_selected = set(s.nsl() for s in stations_selected)
logger.info("number of stations selected: %i" % len(nsls_selected))
have_data = set()
if options.continue_:
fns = glob.glob(fn_template_raw % starfill())
p = pile.make_pile(fns)
else:
fns = []
have_data_site = {}
could_have_data_site = {}
for site in sites:
have_data_site[site] = set()
could_have_data_site[site] = set()
available_through = defaultdict(set)
it = 0
nt = int(math.ceil((tmax - tmin) / tinc))
for it in range(nt):
tmin_win = tmin + it * tinc
tmax_win = min(tmin + (it + 1) * tinc, tmax)
logger.info("time window %i/%i (%s - %s)" %
(it + 1, nt, util.tts(tmin_win), util.tts(tmax_win)))
have_data_this_window = set()
if options.continue_:
trs_avail = p.all(tmin=tmin_win, tmax=tmax_win, load_data=False)
for tr in trs_avail:
have_data_this_window.add(tr.nslc_id)
for site, sx in zip(sites, sxs):
if sx is None:
continue
selection = []
channels = sx.choose_channels(
target_sample_rate=target_sample_rate,
priority_band_code=priority_band_code,
priority_units=priority_units,
priority_instrument_code=priority_instrument_code,
timespan=(tmin_win, tmax_win),
)
for nslc in sorted(channels.keys()):
if nsls_selected is not None and nslc[:3] not in nsls_selected:
continue
could_have_data_site[site].add(nslc)
if nslc not in have_data_this_window:
channel = channels[nslc]
if event:
lat_, lon_ = event.lat, event.lon
else:
lat_, lon_ = lat, lon
dist = orthodrome.distance_accurate50m_numpy(
lat_, lon_, channel.latitude.value,
channel.longitude.value)
if event:
depth_ = event.depth
time_ = event.time
else:
depth_ = None
time_ = None
tmin_, tmax_ = timewindow(time_, dist, depth_)
tmin_this = tmin_ - tpad
tmax_this = tmax_ + tpad
tmin_req = max(tmin_win, tmin_this)
tmax_req = min(tmax_win, tmax_this)
if channel.sample_rate:
deltat = 1.0 / channel.sample_rate.value
else:
deltat = 1.0
if tmin_req < tmax_req:
# extend time window by some samples because otherwise
# sometimes gaps are produced
selection.append(nslc + (tmin_req - deltat * 10.0,
tmax_req + deltat * 10.0))
if options.dry_run:
for (net, sta, loc, cha, tmin, tmax) in selection:
available_through[net, sta, loc, cha].add(site)
else:
neach = 100
i = 0
nbatches = ((len(selection) - 1) // neach) + 1
while i < len(selection):
selection_now = selection[i:i + neach]
f = tempfile.NamedTemporaryFile()
try:
sbatch = ""
if nbatches > 1:
sbatch = " (batch %i/%i)" % (
(i // neach) + 1, nbatches)
logger.info("downloading data (%s)%s" % (site, sbatch))
data = fdsn.dataselect(site=site,
selection=selection_now,
**get_user_credentials(site))
while True:
buf = data.read(1024)
if not buf:
break
f.write(buf)
f.flush()
trs = io.load(f.name)
for tr in trs:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
try:
tr.chop(tmin_win, tmax_win)
have_data.add(tr.nslc_id)
have_data_site[site].add(tr.nslc_id)
except trace.NoData:
pass
fns2 = io.save(trs, fn_template_raw)
io.save(trs, fn_template_raw_folder)
for fn in fns2:
if fn in fns:
logger.warn("overwriting file %s", fn)
fns.extend(fns2)
except fdsn.EmptyResult:
pass
except HTTPError:
logger.warn("an error occurred while downloading data "
"for channels \n %s" %
"\n ".join(".".join(x[:4])
for x in selection_now))
f.close()
i += neach
if options.dry_run:
nslcs = sorted(available_through.keys())
all_channels = defaultdict(set)
all_stations = defaultdict(set)
def plural_s(x):
return "" if x == 1 else "s"
for nslc in nslcs:
sites = tuple(sorted(available_through[nslc]))
logger.info("selected: %s.%s.%s.%s from site%s %s" %
(nslc + (plural_s(len(sites)), "+".join(sites))))
all_channels[sites].add(nslc)
all_stations[sites].add(nslc[:3])
nchannels_all = 0
nstations_all = 0
for sites in sorted(all_channels.keys(),
key=lambda sites: (-len(sites), sites)):
nchannels = len(all_channels[sites])
nstations = len(all_stations[sites])
nchannels_all += nchannels
nstations_all += nstations
logger.info("selected (%s): %i channel%s (%i station%s)" % (
"+".join(sites),
nchannels,
plural_s(nchannels),
nstations,
plural_s(nstations),
))
logger.info("selected total: %i channel%s (%i station%s)" % (
nchannels_all,
plural_s(nchannels_all),
nstations_all,
plural_s(nstations_all),
))
logger.info("dry run done.")
sys.exit(0)
for nslc in have_data:
# if we are in continue mode, we have to guess where the data came from
if not any(nslc in have_data_site[site] for site in sites):
for site in sites:
if nslc in could_have_data_site[site]:
have_data_site[site].add(nslc)
sxs = {}
for site in sites:
selection = []
for nslc in sorted(have_data_site[site]):
selection.append(nslc + (tmin - tpad, tmax + tpad))
if selection:
logger.info("downloading response information (%s)" % site)
sxs[site] = fdsn.station(site=site,
level="response",
selection=selection)
sited = site
if site == "http://192.168.11.220:8080":
sited = "bgr_internal"
elif site == "http://ws.gpi.kit.edu":
sited = "kit"
if site == "http://188.246.25.142:8080":
sited = "moer"
sxs[site].dump_xml(filename=op.join(output_dir, "stations.%s.xml" %
sited))
# chapter 1.5: inject local data
if options.local_data:
have_data_site["local"] = set()
plocal = pile.make_pile(options.local_data, fileformat="detect")
for traces in plocal.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=tmin,
tmax=tmax,
tinc=tinc):
for tr in traces:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
if tr.nslc_id not in have_data:
fns.extend(io.save(traces, fn_template_raw))
have_data_site["local"].add(tr.nslc_id)
have_data.add(tr.nslc_id)
sites.append("local")
if options.local_responses_pz:
sxs["local"] = epz.make_stationxml(
epz.iload(options.local_responses_pz))
if options.local_responses_resp:
local_stations = []
for fn in options.local_stations:
local_stations.extend(model.load_stations(fn))
sxs["local"] = resp.make_stationxml(
local_stations, resp.iload(options.local_responses_resp))
if options.local_responses_stationxml:
sxs["local"] = stationxml.load_xml(
filename=options.local_responses_stationxml)
# chapter 1.6: dump raw data stations file
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())]
util.ensuredirs(fn_stations_raw)
model.dump_stations(stations, fn_stations_raw)
dump_commandline(sys.argv, fn_commandline)
# chapter 2: restitution
if not fns:
logger.error("no data available")
sys.exit(1)
p = pile.make_pile(fns, show_progress=False)
p.get_deltatmin()
otinc = None
if otinc is None:
otinc = nice_seconds_floor(p.get_deltatmin() * 500000.0)
otinc = 3600.0
otmin = math.floor(p.tmin / otinc) * otinc
otmax = math.ceil(p.tmax / otinc) * otinc
otpad = tpad * 2
fns = []
rest_traces_b = []
win_b = None
for traces_a in p.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=otmin,
tmax=otmax,
tinc=otinc,
tpad=otpad):
rest_traces_a = []
win_a = None
for tr in traces_a:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
win_a = tr.wmin, tr.wmax
if win_b and win_b[0] >= win_a[0]:
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
rest_traces_b = []
win_b = None
response = None
failure = []
for site in sites:
try:
if site not in sxs:
continue
response = sxs[site].get_pyrocko_response(
tr.nslc_id,
timespan=(tr.tmin, tr.tmax),
fake_input_units=output_units,
)
break
except stationxml.NoResponseInformation:
failure.append("%s: no response information" % site)
except stationxml.MultipleResponseInformation:
failure.append("%s: multiple response information" % site)
if response is None:
failure = ", ".join(failure)
else:
failure = ""
try:
rest_tr = tr.transfer(tfade, ftap, response, invert=True)
rest_traces_a.append(rest_tr)
except (trace.TraceTooShort, trace.NoData):
failure = "trace too short"
if failure:
logger.warn("failed to restitute trace %s.%s.%s.%s (%s)" %
(tr.nslc_id + (failure, )))
if rest_traces_b:
rest_traces = trace.degapper(rest_traces_b + rest_traces_a,
deoverlap="crossfade_cos")
fns.extend(cut_n_dump(rest_traces, win_b, fn_template_rest))
rest_traces_a = []
if win_a:
for tr in rest_traces:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
try:
rest_traces_a.append(
tr.chop(win_a[0], win_a[1] + otpad, inplace=False))
except trace.NoData:
pass
rest_traces_b = rest_traces_a
win_b = win_a
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
# chapter 3: rotated restituted traces for inspection
if not event:
sys.exit(0)
fn_template1 = "DISPL.%(network)s.%(station)s.%(location)s.%(channel)s"
fn_waveforms = op.join(output_dir, "prepared", fn_template1)
fn_stations = op.join(output_dir, "stations.prepared.txt")
fn_event = op.join(event_dir, "event.txt")
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
p = pile.make_pile(fns, show_progress=False)
deltat = None
if sample_rate is not None:
deltat = 1.0 / sample_rate
used_stations = []
for nsl, s in nsl_to_station.items():
s.set_event_relative_data(event)
traces = p.all(trace_selector=lambda tr: tr.nslc_id[:3] == nsl)
keep = []
for tr in traces:
if deltat is not None:
try:
tr.downsample_to(deltat, snap=True, allow_upsample_max=5)
keep.append(tr)
except util.UnavailableDecimation as e:
logger.warn("Cannot downsample %s.%s.%s.%s: %s" %
(tr.nslc_id + (e, )))
continue
if options.out_components == "rtu":
pios = s.guess_projections_to_rtu(out_channels=("R", "T", "Z"))
elif options.out_components == "enu":
pios = s.guess_projections_to_enu(out_channels=("E", "N", "Z"))
else:
assert False
for (proj, in_channels, out_channels) in pios:
proc = trace.project(traces, proj, in_channels, out_channels)
for tr in proc:
for ch in out_channels:
if ch.name == tr.channel:
s.add_channel(ch)
if proc:
io.save(proc, fn_waveforms)
used_stations.append(s)
stations = list(used_stations)
util.ensuredirs(fn_stations)
model.dump_stations(stations, fn_stations)
model.dump_events([event], fn_event)
logger.info("prepared waveforms from %i stations" % len(stations))
def call(self):
self.cleanup()
if self.tinc is not None:
template = \
'trace_%n.%s.%l.%c_%(tmin_us)s'
else:
template = 'trace_%n.%s.%l.%c'
if self.format == 'text':
default_output_filename = template + '.dat'
else:
default_output_filename = template + '.' + self.format
out_filename = self.output_filename('Template for output files',
default_output_filename)
viewer = self.get_viewer()
for trs in self.chopper_selected_traces(fallback=True, tinc=self.tinc):
traces_save = []
for tr in trs:
if self.format == 'mseed':
if len(tr.network) > 2:
tr.set_network(tr.network[:2])
if len(tr.station) > 5:
tr.set_station(tr.station[:5])
if len(tr.location) > 2:
tr.set_location(tr.location[:2])
if len(tr.channel) > 3:
tr.set_channel(tr.channel[:3])
if self.apply_filter:
if viewer.lowpass is not None and \
viewer.highpass is not None:
tr.bandpass(2, viewer.highpass, viewer.lowpass)
elif viewer.lowpass is not None:
if viewer.lowpass < 0.5 / tr.deltat:
tr.lowpass(4, viewer.lowpass, demean=False)
elif viewer.highpass is not None:
if viewer.highpass < 0.5 / tr.deltat:
tr.highpass(4, viewer.highpass, demean=False)
traces_save.append(tr)
if viewer.rotate != 0.0 and self.apply_filter:
phi = viewer.rotate / 180. * math.pi
cphi = math.cos(phi)
sphi = math.sin(phi)
for a in traces_save:
for b in traces_save:
if (a.network == b.network and a.station == b.station
and a.location == b.location and
((a.channel.lower().endswith('n')
and b.channel.lower().endswith('e')) or
(a.channel.endswith('1') and b.channel.endswith('2')))
and abs(a.deltat - b.deltat) < a.deltat * 0.001
and abs(a.tmin - b.tmin) < a.deltat * 0.01
and a.get_ydata().size == b.get_ydata().size):
aydata = a.get_ydata() * cphi + b.get_ydata() * sphi
bydata = -a.get_ydata() * sphi + b.get_ydata() * cphi
a.set_ydata(aydata)
b.set_ydata(bydata)
try:
io.save(traces_save,
out_filename,
format=self.format,
overwrite=True)
except io.io_common.FileSaveError as e:
self.fail(str(e))
logger.info('saved waveforms to %s', out_filename)
if self.save_stations:
stations = viewer.stations.values()
fn = self.output_filename('Save Stations', 'stations.pf')
model.dump_stations(list(stations), fn)
logger.info('saved stations to %s', fn)
sta = section.sta2
tr = section.pyrocko_trace()
return Station(lat=sta.lat,
lon=sta.lon,
depth=sta.depth,
elevation=sta.elevation,
network=tr.network,
station=tr.station)
with open(fn, 'r') as f:
for i, tr in enumerate(ims.iload_fh(f)):
try:
traces.append(tr.pyrocko_trace())
stations.append(get_pyrocko_station(tr))
except AttributeError as e:
print 'caught exception: ', e
except:
print 'some uncaught exception just happend which I am skipping...'
pass
for tr in traces:
tr.set_location('')
print traces
print stations
io.save(traces, 'extracted_from_gse2.mseed')
dump_stations(stations, 'stations_gse2.pf')
print '........................................'
print 'DONE - but check stations_gse2.pf, especially depth! Is it 0 or -999.?'
def save_station(self):
fn = self.output_filename('Save Station')
dump_stations([self.station_c], fn)
def save_kiwi_dataset(acc, stations, traces, event, config):
if config.has('data_dir'):
data_dir = config.path('data_dir')
if os.path.exists(data_dir):
shutil.rmtree(data_dir)
if config.has('skeleton_dir'):
copy_files(config.path('skeleton_dir'), config.path('main_dir'))
if config.has('raw_trace_path'):
trace_selector = None
if config.has('station_filter'):
trace_selector = lambda tr: get_nsl(tr) in stations and config.station_filter(stations[get_nsl(tr)])
for raw_traces in acc.iter_traces(trace_selector=trace_selector):
io.save(raw_traces, config.path('raw_trace_path'))
eventfn = config.path('event_info_path')
util.ensuredirs(eventfn)
save_event_info_file(eventfn, event)
dstations = stations.values()
dstations.sort( lambda a,b: cmp(a.dist_m, b.dist_m) )
# gather traces by station
dataset = []
used_channels = {}
for station in dstations:
station_traces = []
for tr in traces:
if get_nsl(tr) == get_nsl(station):
if tr.channel in config.wanted_channels:
station_traces.append(tr)
if get_nsl(tr) not in used_channels:
used_channels[get_nsl(tr)] = []
used_channels[get_nsl(tr)].append(station.get_channel(tr.channel))
station_traces.sort(lambda a,b: cmp(a.channel, b.channel))
kiwi_components = ''
for tr in station_traces:
kiwi_components += config.kiwi_component_map[tr.channel]
if station_traces:
dataset.append( (station, kiwi_components, station_traces) )
if config.has('stations_path'):
fpath = config.path('stations_path')
util.ensuredirs(fpath)
ddstations = copy.deepcopy(dstations)
for sta in ddstations:
if get_nsl(sta) in used_channels:
sta.set_channels(used_channels[get_nsl(sta)])
model.dump_stations(ddstations, fpath)
if config.has('receivers_path'):
fpath = config.path('receivers_path')
util.ensuredirs(fpath)
f = open(fpath, 'w')
iref = 1
nsets = 1
if config.has('nsets'):
nsets = config.nsets
for station, components, traces in dataset:
nsl = '.'.join((get_nsl(station)))
for i in range(nsets):
depth = 0.0
if station.depth is not None:
depth = station.depth
if config.has('receivers_path'):
f.write('%15.8e %15.8e %15.8e %3s %-15s\n' % (station.lat, station.lon, depth, components, nsl) )
for tr in traces:
tr = tr.copy()
if config.trace_time_zero == 'event':
tr.shift(-event.time)
ydata = tr.get_ydata()
ydata *= config.trace_factor
fn = config.path('displacement_trace_path', {
'ireceiver': iref,
'component': config.kiwi_component_map[tr.channel],
'network': tr.network,
'station': tr.station,
'location': tr.location,
'channel': tr.channel})
io.save([tr], fn)
iref += 1
if config.has('receivers_path'):
f.close()
if config.has('reference_time_path'):
fpath = config.path('reference_time_path')
f = open(fpath, 'w')
f.write('%i %s\n' % (event.time,
time.strftime('%Y/%m/%d %H:%M:%S',
time.gmtime(event.time))))
f.close()
if config.has('source_origin_path'):
fpath = config.path('source_origin_path')
f = open(fpath, 'w')
t = 0.
if config.trace_time_zero == 'system':
t = event.time
f.write('%e %e %f\n' % (event.lat, event.lon, t))
f.close()
def main():
parser = OptionParser(usage=usage, description=description)
parser.add_option('--force',
dest='force',
action='store_true',
default=False,
help='allow recreation of output <directory>')
parser.add_option('--debug',
dest='debug',
action='store_true',
default=False,
help='print debugging information to stderr')
parser.add_option('--dry-run',
dest='dry_run',
action='store_true',
default=False,
help='show available stations/channels and exit '
'(do not download waveforms)')
parser.add_option('--continue',
dest='continue_',
action='store_true',
default=False,
help='continue download after a accident')
parser.add_option('--local-data',
dest='local_data',
action='append',
help='add file/directory with local data')
parser.add_option('--local-stations',
dest='local_stations',
action='append',
help='add local stations file')
parser.add_option('--selection',
dest='selection_file',
action='append',
help='add local stations file')
parser.add_option(
'--local-responses-resp',
dest='local_responses_resp',
action='append',
help='add file/directory with local responses in RESP format')
parser.add_option('--local-responses-pz',
dest='local_responses_pz',
action='append',
help='add file/directory with local pole-zero responses')
parser.add_option(
'--local-responses-stationxml',
dest='local_responses_stationxml',
help='add file with local response information in StationXML format')
parser.add_option(
'--window',
dest='window',
default='full',
help='set time window to choose [full, p, "<time-start>,<time-end>"'
'] (time format is YYYY-MM-DD HH:MM:SS)')
parser.add_option(
'--out-components',
choices=['enu', 'rtu'],
dest='out_components',
default='rtu',
help='set output component orientations to radial-transverse-up [rtu] '
'(default) or east-north-up [enu]')
parser.add_option('--out-units',
choices=['M', 'M/S', 'M/S**2'],
dest='output_units',
default='M',
help='set output units to displacement "M" (default),'
' velocity "M/S" or acceleration "M/S**2"')
parser.add_option(
'--padding-factor',
type=float,
default=3.0,
dest='padding_factor',
help='extend time window on either side, in multiples of 1/<fmin_hz> '
'(default: 5)')
parser.add_option(
'--zero-padding',
dest='zero_pad',
action='store_true',
default=False,
help='Extend traces by zero-padding if clean restitution requires'
'longer windows')
parser.add_option(
'--credentials',
dest='user_credentials',
action='append',
default=[],
metavar='SITE,USER,PASSWD',
help='user credentials for specific site to access restricted data '
'(this option can be repeated)')
parser.add_option(
'--token',
dest='auth_tokens',
metavar='SITE,FILENAME',
action='append',
default=[],
help='user authentication token for specific site to access '
'restricted data (this option can be repeated)')
parser.add_option(
'--sites',
dest='sites',
metavar='SITE1,SITE2,...',
default='geofon,iris,orfeus',
help='sites to query (available: %s, default: "%%default"' %
', '.join(g_sites_available))
parser.add_option(
'--band-codes',
dest='priority_band_code',
metavar='V,L,M,B,H,S,E,...',
default='B,H',
help='select and prioritize band codes (default: %default)')
parser.add_option(
'--instrument-codes',
dest='priority_instrument_code',
metavar='H,L,G,...',
default='H,L',
help='select and prioritize instrument codes (default: %default)')
parser.add_option('--radius-min',
dest='radius_min',
metavar='VALUE',
default=0.0,
type=float,
help='minimum radius [km]')
parser.add_option('--nstations-wanted',
dest='nstations_wanted',
metavar='N',
type=int,
help='number of stations to select initially')
(options, args) = parser.parse_args(sys.argv[1:])
print('Parsed arguments:', args)
if len(args) not in (10, 7, 6):
parser.print_help()
sys.exit(1)
if options.debug:
util.setup_logging(program_name, 'debug')
else:
util.setup_logging(program_name, 'info')
if options.local_responses_pz and options.local_responses_resp:
logger.critical('cannot use local responses in PZ and RESP '
'format at the same time')
sys.exit(1)
n_resp_opt = 0
for resp_opt in (options.local_responses_pz, options.local_responses_resp,
options.local_responses_stationxml):
if resp_opt:
n_resp_opt += 1
if n_resp_opt > 1:
logger.critical('can only handle local responses from either PZ or '
'RESP or StationXML. Cannot yet merge different '
'response formats.')
sys.exit(1)
if options.local_responses_resp and not options.local_stations:
logger.critical('--local-responses-resp can only be used '
'when --stations is also given.')
sys.exit(1)
try:
ename = ''
magnitude = None
mt = None
if len(args) == 10:
time = util.str_to_time(args[1] + ' ' + args[2])
lat = float(args[3])
lon = float(args[4])
depth = float(args[5]) * km
iarg = 6
elif len(args) == 7:
if args[2].find(':') == -1:
sname_or_date = None
lat = float(args[1])
lon = float(args[2])
event = None
time = None
else:
sname_or_date = args[1] + ' ' + args[2]
iarg = 3
elif len(args) == 6:
sname_or_date = args[1]
iarg = 2
if len(args) in (7, 6) and sname_or_date is not None:
events = get_events_by_name_or_date([sname_or_date],
catalog=geofon)
if len(events) == 0:
logger.critical('no event found')
sys.exit(1)
elif len(events) > 1:
logger.critical('more than one event found')
sys.exit(1)
event = events[0]
time = event.time
lat = event.lat
lon = event.lon
depth = event.depth
ename = event.name
magnitude = event.magnitude
mt = event.moment_tensor
radius = float(args[iarg]) * km
fmin = float(args[iarg + 1])
sample_rate = float(args[iarg + 2])
eventname = args[iarg + 3]
cwd = str(sys.argv[1])
event_dir = op.join(cwd, 'data', 'events', eventname)
output_dir = op.join(event_dir, 'waveforms')
except:
raise
parser.print_help()
sys.exit(1)
if options.force and op.isdir(event_dir):
if not options.continue_:
shutil.rmtree(event_dir)
if op.exists(event_dir) and not options.continue_:
logger.critical(
'directory "%s" exists. Delete it first or use the --force option'
% event_dir)
sys.exit(1)
util.ensuredir(output_dir)
if time is not None:
event = model.Event(time=time,
lat=lat,
lon=lon,
depth=depth,
name=ename,
magnitude=magnitude,
moment_tensor=mt)
if options.window == 'full':
if event is None:
logger.critical('need event for --window=full')
sys.exit(1)
low_velocity = 1500.
timewindow = VelocityWindow(low_velocity,
tpad=options.padding_factor / fmin)
tmin, tmax = timewindow(time, radius, depth)
elif options.window == 'p':
if event is None:
logger.critical('need event for --window=p')
sys.exit(1)
phases = list(map(cake.PhaseDef, 'P p'.split()))
emod = cake.load_model()
tpad = options.padding_factor / fmin
timewindow = PhaseWindow(emod, phases, -tpad, tpad)
arrivaltimes = []
for dist in num.linspace(0, radius, 20):
try:
arrivaltimes.extend(timewindow(time, dist, depth))
except NoArrival:
pass
if not arrivaltimes:
logger.error('required phase arrival not found')
sys.exit(1)
tmin = min(arrivaltimes)
tmax = max(arrivaltimes)
else:
try:
stmin, stmax = options.window.split(',')
tmin = util.str_to_time(stmin.strip())
tmax = util.str_to_time(stmax.strip())
timewindow = FixedWindow(tmin, tmax)
except ValueError:
logger.critical('invalid argument to --window: "%s"' %
options.window)
sys.exit(1)
if event is not None:
event.name = eventname
tfade = tfade_factor / fmin
tpad = tfade
tmin -= tpad
tmax += tpad
tinc = None
priority_band_code = options.priority_band_code.split(',')
for s in priority_band_code:
if len(s) != 1:
logger.critical('invalid band code: %s' % s)
priority_instrument_code = options.priority_instrument_code.split(',')
for s in priority_instrument_code:
if len(s) != 1:
logger.critical('invalid instrument code: %s' % s)
station_query_conf = dict(latitude=lat,
longitude=lon,
minradius=options.radius_min * km * cake.m2d,
maxradius=radius * cake.m2d,
channel=','.join('%s??' % s
for s in priority_band_code))
target_sample_rate = sample_rate
fmax = target_sample_rate
# target_sample_rate = None
# priority_band_code = ['H', 'B', 'M', 'L', 'V', 'E', 'S']
priority_units = ['M/S', 'M', 'M/S**2']
# output_units = 'M'
sites = [x.strip() for x in options.sites.split(',') if x.strip()]
for site in sites:
if site not in g_sites_available:
logger.critical('unknown FDSN site: %s' % site)
sys.exit(1)
for s in options.user_credentials:
try:
site, user, passwd = s.split(',')
g_user_credentials[site] = user, passwd
except ValueError:
logger.critical('invalid format for user credentials: "%s"' % s)
sys.exit(1)
for s in options.auth_tokens:
try:
site, token_filename = s.split(',')
with open(token_filename, 'r') as f:
g_auth_tokens[site] = f.read()
except (ValueError, OSError, IOError):
logger.critical('cannot get token from file: %s' % token_filename)
sys.exit(1)
fn_template0 = \
'data_%(network)s.%(station)s.%(location)s.%(channel)s_%(tmin)s.mseed'
fn_template_raw = op.join(output_dir, 'raw', fn_template0)
fn_stations_raw = op.join(output_dir, 'stations.raw.txt')
fn_template_rest = op.join(output_dir, 'rest', fn_template0)
fn_commandline = op.join(output_dir, 'beatdown.command')
ftap = (ffade_factors[0] * fmin, fmin, fmax, ffade_factors[1] * fmax)
# chapter 1: download
sxs = []
for site in sites:
try:
extra_args = {
'iris': dict(matchtimeseries=True),
}.get(site, {})
extra_args.update(station_query_conf)
if site == 'geonet':
extra_args.update(starttime=tmin, endtime=tmax)
else:
extra_args.update(startbefore=tmax,
endafter=tmin,
includerestricted=(site in g_user_credentials
or site in g_auth_tokens))
logger.info('downloading channel information (%s)' % site)
sx = fdsn.station(site=site,
format='text',
level='channel',
**extra_args)
except fdsn.EmptyResult:
logger.error('No stations matching given criteria. (%s)' % site)
sx = None
if sx is not None:
sxs.append(sx)
if all(sx is None for sx in sxs) and not options.local_data:
sys.exit(1)
nsl_to_sites = defaultdict(list)
nsl_to_station = {}
if options.selection_file:
logger.info('using stations from stations file!')
stations = []
for fn in options.selection_file:
stations.extend(model.load_stations(fn))
nsls_selected = set(s.nsl() for s in stations)
else:
nsls_selected = None
for sx, site in zip(sxs, sites):
site_stations = sx.get_pyrocko_stations()
for s in site_stations:
nsl = s.nsl()
nsl_to_sites[nsl].append(site)
if nsl not in nsl_to_station:
if nsls_selected:
if nsl in nsls_selected:
nsl_to_station[nsl] = s
else:
nsl_to_station[
nsl] = s # using first site with this station
logger.info('number of stations found: %i' % len(nsl_to_station))
# station weeding
if options.nstations_wanted:
nsls_selected = None
stations_all = [
nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())
]
for s in stations_all:
s.set_event_relative_data(event)
stations_selected = weeding.weed_stations(stations_all,
options.nstations_wanted)[0]
nsls_selected = set(s.nsl() for s in stations_selected)
logger.info('number of stations selected: %i' % len(nsls_selected))
if tinc is None:
tinc = 3600.
have_data = set()
if options.continue_:
fns = glob.glob(fn_template_raw % starfill())
p = pile.make_pile(fns)
else:
fns = []
have_data_site = {}
could_have_data_site = {}
for site in sites:
have_data_site[site] = set()
could_have_data_site[site] = set()
available_through = defaultdict(set)
it = 0
nt = int(math.ceil((tmax - tmin) / tinc))
for it in range(nt):
tmin_win = tmin + it * tinc
tmax_win = min(tmin + (it + 1) * tinc, tmax)
logger.info('time window %i/%i (%s - %s)' %
(it + 1, nt, util.tts(tmin_win), util.tts(tmax_win)))
have_data_this_window = set()
if options.continue_:
trs_avail = p.all(tmin=tmin_win, tmax=tmax_win, load_data=False)
for tr in trs_avail:
have_data_this_window.add(tr.nslc_id)
for site, sx in zip(sites, sxs):
if sx is None:
continue
selection = []
channels = sx.choose_channels(
target_sample_rate=target_sample_rate,
priority_band_code=priority_band_code,
priority_units=priority_units,
priority_instrument_code=priority_instrument_code,
timespan=(tmin_win, tmax_win))
for nslc in sorted(channels.keys()):
if nsls_selected is not None and nslc[:3] not in nsls_selected:
continue
could_have_data_site[site].add(nslc)
if nslc not in have_data_this_window:
channel = channels[nslc]
if event:
lat_, lon_ = event.lat, event.lon
else:
lat_, lon_ = lat, lon
try:
dist = orthodrome.distance_accurate50m_numpy(
lat_, lon_, channel.latitude.value,
channel.longitude.value)
except:
dist = orthodrome.distance_accurate50m_numpy(
lat_, lon_, channel.latitude, channel.longitude)
if event:
depth_ = event.depth
time_ = event.time
else:
depth_ = None
time_ = None
tmin_, tmax_ = timewindow(time_, dist, depth_)
tmin_this = tmin_ - tpad
tmax_this = float(tmax_ + tpad)
tmin_req = max(tmin_win, tmin_this)
tmax_req = min(tmax_win, tmax_this)
if channel.sample_rate:
try:
deltat = 1.0 / int(channel.sample_rate.value)
except:
deltat = 1.0 / int(channel.sample_rate)
else:
deltat = 1.0
if tmin_req < tmax_req:
logger.debug('deltat %f' % deltat)
# extend time window by some samples because otherwise
# sometimes gaps are produced
# apparently the WS are only sensitive to full seconds
# round to avoid gaps, increase safetiy window
selection.append(nslc +
(math.floor(tmin_req - deltat * 20.0),
math.ceil(tmax_req + deltat * 20.0)))
if options.dry_run:
for (net, sta, loc, cha, tmin, tmax) in selection:
available_through[net, sta, loc, cha].add(site)
else:
neach = 100
i = 0
nbatches = ((len(selection) - 1) // neach) + 1
while i < len(selection):
selection_now = selection[i:i + neach]
f = tempfile.NamedTemporaryFile()
try:
sbatch = ''
if nbatches > 1:
sbatch = ' (batch %i/%i)' % (
(i // neach) + 1, nbatches)
logger.info('downloading data (%s)%s' % (site, sbatch))
data = fdsn.dataselect(site=site,
selection=selection_now,
**get_user_credentials(site))
while True:
buf = data.read(1024)
if not buf:
break
f.write(buf)
f.flush()
trs = io.load(f.name)
for tr in trs:
tr.fix_deltat_rounding_errors()
logger.debug('cutting window: %f - %f' %
(tmin_win, tmax_win))
logger.debug(
'available window: %f - %f, nsamples: %g' %
(tr.tmin, tr.tmax, tr.ydata.size))
try:
logger.debug('tmin before snap %f' % tr.tmin)
tr.snap(interpolate=True)
logger.debug('tmin after snap %f' % tr.tmin)
tr.chop(tmin_win,
tmax_win,
snap=(math.floor, math.ceil),
include_last=True)
logger.debug(
'cut window: %f - %f, nsamles: %g' %
(tr.tmin, tr.tmax, tr.ydata.size))
have_data.add(tr.nslc_id)
have_data_site[site].add(tr.nslc_id)
except trace.NoData:
pass
fns2 = io.save(trs, fn_template_raw)
for fn in fns2:
if fn in fns:
logger.warn('overwriting file %s', fn)
fns.extend(fns2)
except fdsn.EmptyResult:
pass
except HTTPError:
logger.warn('an error occurred while downloading data '
'for channels \n %s' %
'\n '.join('.'.join(x[:4])
for x in selection_now))
f.close()
i += neach
if options.dry_run:
nslcs = sorted(available_through.keys())
all_channels = defaultdict(set)
all_stations = defaultdict(set)
def plural_s(x):
return '' if x == 1 else 's'
for nslc in nslcs:
sites = tuple(sorted(available_through[nslc]))
logger.info('selected: %s.%s.%s.%s from site%s %s' %
(nslc + (plural_s(len(sites)), '+'.join(sites))))
all_channels[sites].add(nslc)
all_stations[sites].add(nslc[:3])
nchannels_all = 0
nstations_all = 0
for sites in sorted(all_channels.keys(),
key=lambda sites: (-len(sites), sites)):
nchannels = len(all_channels[sites])
nstations = len(all_stations[sites])
nchannels_all += nchannels
nstations_all += nstations
logger.info('selected (%s): %i channel%s (%i station%s)' %
('+'.join(sites), nchannels, plural_s(nchannels),
nstations, plural_s(nstations)))
logger.info('selected total: %i channel%s (%i station%s)' %
(nchannels_all, plural_s(nchannels_all), nstations_all,
plural_s(nstations_all)))
logger.info('dry run done.')
sys.exit(0)
for nslc in have_data:
# if we are in continue mode, we have to guess where the data came from
if not any(nslc in have_data_site[site] for site in sites):
for site in sites:
if nslc in could_have_data_site[site]:
have_data_site[site].add(nslc)
sxs = {}
for site in sites:
selection = []
for nslc in sorted(have_data_site[site]):
selection.append(nslc + (tmin - tpad, tmax + tpad))
if selection:
logger.info('downloading response information (%s)' % site)
sxs[site] = fdsn.station(site=site,
level='response',
selection=selection)
sxs[site].dump_xml(filename=op.join(output_dir, 'stations.%s.xml' %
site))
# chapter 1.5: inject local data
if options.local_data:
have_data_site['local'] = set()
plocal = pile.make_pile(options.local_data, fileformat='detect')
logger.info(
'Importing local data from %s between %s (%f) and %s (%f)' %
(options.local_data, util.time_to_str(tmin), tmin,
util.time_to_str(tmax), tmax))
for traces in plocal.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=tmin,
tmax=tmax,
tinc=tinc):
for tr in traces:
if tr.nslc_id not in have_data:
fns.extend(io.save(traces, fn_template_raw))
have_data_site['local'].add(tr.nslc_id)
have_data.add(tr.nslc_id)
sites.append('local')
if options.local_responses_pz:
sxs['local'] = epz.make_stationxml(
epz.iload(options.local_responses_pz))
if options.local_responses_resp:
local_stations = []
for fn in options.local_stations:
local_stations.extend(model.load_stations(fn))
sxs['local'] = resp.make_stationxml(
local_stations, resp.iload(options.local_responses_resp))
if options.local_responses_stationxml:
sxs['local'] = stationxml.load_xml(
filename=options.local_responses_stationxml)
# chapter 1.6: dump raw data stations file
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())]
util.ensuredirs(fn_stations_raw)
model.dump_stations(stations, fn_stations_raw)
dump_commandline(sys.argv, fn_commandline)
# chapter 2: restitution
if not fns:
logger.error('no data available')
sys.exit(1)
p = pile.make_pile(fns, show_progress=False)
p.get_deltatmin()
otinc = None
if otinc is None:
otinc = nice_seconds_floor(p.get_deltatmin() * 500000.)
otinc = 3600.
otmin = math.floor(p.tmin / otinc) * otinc
otmax = math.ceil(p.tmax / otinc) * otinc
otpad = tpad * 2
fns = []
rest_traces_b = []
win_b = None
for traces_a in p.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=otmin,
tmax=otmax,
tinc=otinc,
tpad=otpad):
rest_traces_a = []
win_a = None
for tr in traces_a:
win_a = tr.wmin, tr.wmax
if win_b and win_b[0] >= win_a[0]:
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
rest_traces_b = []
win_b = None
response = None
failure = []
for site in sites:
try:
if site not in sxs:
continue
logger.debug('Getting response for %s' % tr.__str__())
response = sxs[site].get_pyrocko_response(
tr.nslc_id,
timespan=(tr.tmin, tr.tmax),
fake_input_units=options.output_units)
break
except stationxml.NoResponseInformation:
failure.append('%s: no response information' % site)
except stationxml.MultipleResponseInformation:
failure.append('%s: multiple response information' % site)
if response is None:
failure = ', '.join(failure)
else:
failure = ''
try:
if tr.tmin > tmin and options.zero_pad:
logger.warning(
'Trace too short for clean restitution in '
'desired frequency band -> zero-padding!')
tr.extend(tr.tmin - tfade, tr.tmax + tfade, 'repeat')
rest_tr = tr.transfer(tfade, ftap, response, invert=True)
rest_traces_a.append(rest_tr)
except (trace.TraceTooShort, trace.NoData):
failure = 'trace too short'
if failure:
logger.warn('failed to restitute trace %s.%s.%s.%s (%s)' %
(tr.nslc_id + (failure, )))
if rest_traces_b:
rest_traces = trace.degapper(rest_traces_b + rest_traces_a,
deoverlap='crossfade_cos')
fns.extend(cut_n_dump(rest_traces, win_b, fn_template_rest))
rest_traces_a = []
if win_a:
for tr in rest_traces:
try:
rest_traces_a.append(
tr.chop(win_a[0], win_a[1] + otpad, inplace=False))
except trace.NoData:
pass
rest_traces_b = rest_traces_a
win_b = win_a
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
# chapter 3: rotated restituted traces for inspection
if not event:
sys.exit(0)
fn_template1 = \
'DISPL.%(network)s.%(station)s.%(location)s.%(channel)s'
fn_waveforms = op.join(output_dir, 'prepared', fn_template1)
fn_stations = op.join(output_dir, 'stations.prepared.txt')
fn_event = op.join(event_dir, 'event.txt')
fn_event_yaml = op.join(event_dir, 'event.yaml')
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
p = pile.make_pile(fns, show_progress=False)
deltat = None
if sample_rate is not None:
deltat = 1.0 / sample_rate
traces_beat = []
used_stations = []
for nsl, s in nsl_to_station.items():
s.set_event_relative_data(event)
traces = p.all(trace_selector=lambda tr: tr.nslc_id[:3] == nsl)
if options.out_components == 'rtu':
pios = s.guess_projections_to_rtu(out_channels=('R', 'T', 'Z'))
elif options.out_components == 'enu':
pios = s.guess_projections_to_enu(out_channels=('E', 'N', 'Z'))
else:
assert False
for (proj, in_channels, out_channels) in pios:
proc = trace.project(traces, proj, in_channels, out_channels)
for tr in proc:
tr_beat = heart.SeismicDataset.from_pyrocko_trace(tr)
traces_beat.append(tr_beat)
for ch in out_channels:
if ch.name == tr.channel:
s.add_channel(ch)
if proc:
io.save(proc, fn_waveforms)
used_stations.append(s)
stations = list(used_stations)
util.ensuredirs(fn_stations)
model.dump_stations(stations, fn_stations)
model.dump_events([event], fn_event)
from pyrocko.guts import dump
dump([event], filename=fn_event_yaml)
utility.dump_objects(op.join(cwd, 'seismic_data.pkl'),
outlist=[stations, traces_beat])
logger.info('prepared waveforms from %i stations' % len(stations))
# request meta data
traces_iris = io.load(os.path.join(sdspath,'traces_iris.mseed'))
stations_real_iris = []
gaps= []
for tr in traces_iris:
for st in stations_iris:
if tr.station == st.station and tr.location == st.location:
stations_real_iris.append(st)
gaps.append(st.station)
remove =[x for x in gaps if gaps.count(x) > 1]
for re in remove:
for st in stations_real_iris:
if st.station == re:
stations_real_iris.remove(st)
model.dump_stations(stations_real_iris, os.path.join(sdspath,'stations_iris.txt'))
request_response = fdsn.station(
site=site, selection=selection_iris, level='response')
# save the response in YAML and StationXML format
request_response.dump(filename=os.path.join(sdspath,'responses_iris.yml'))
request_response.dump_xml(filename=os.path.join(sdspath,'responses_iris.xml'))
sx = stationxml.load_xml(filename=os.path.join(sdspath,'responses_iris.xml'))
pyrocko_stations = sx.get_pyrocko_stations()
#model.dump_stations(stations_real, os.path.join(sdspath,'stations2.txt'))
# Loop through retrieved waveforms and request meta information
# for each trace
event_origin = gf.Source(
lat=event.lat,
lon=event.lon)
def check(
config,
event_names=None,
target_string_ids=None,
show_waveforms=False,
n_random_synthetics=10,
stations_used_path=None):
markers = []
stations_used = {}
erroneous = []
for ievent, event_name in enumerate(event_names):
ds = config.get_dataset(event_name)
event = ds.get_event()
trs_all = []
try:
problem = config.get_problem(event)
_, nfamilies = problem.get_family_mask()
logger.info('Problem: %s' % problem.name)
logger.info('Number of target families: %i' % nfamilies)
logger.info('Number of targets (total): %i' % len(problem.targets))
if target_string_ids:
problem.targets = [
target for target in problem.targets
if util.match_nslc(target_string_ids, target.string_id())]
logger.info(
'Number of targets (selected): %i' % len(problem.targets))
check_problem(problem)
results_list = []
sources = []
if n_random_synthetics == 0:
x = problem.get_reference_model()
sources.append(problem.base_source)
results = problem.evaluate(x)
results_list.append(results)
else:
for i in range(n_random_synthetics):
x = problem.get_random_model()
sources.append(problem.get_source(x))
results = problem.evaluate(x)
results_list.append(results)
if show_waveforms:
engine = config.engine_config.get_engine()
times = []
tdata = []
for target in problem.targets:
tobs_shift_group = []
tcuts = []
for source in sources:
tmin_fit, tmax_fit, tfade, tfade_taper = \
target.get_taper_params(engine, source)
times.extend((tmin_fit-tfade*2., tmax_fit+tfade*2.))
tobs, tsyn = target.get_pick_shift(engine, source)
if None not in (tobs, tsyn):
tobs_shift = tobs - tsyn
else:
tobs_shift = 0.0
tcuts.append(target.get_cutout_timespan(
tmin_fit+tobs_shift, tmax_fit+tobs_shift, tfade))
tobs_shift_group.append(tobs_shift)
tcuts = num.array(tcuts, dtype=num.float)
tdata.append((
tfade,
num.mean(tobs_shift_group),
(num.min(tcuts[:, 0]), num.max(tcuts[:, 1]))))
tmin = min(times)
tmax = max(times)
tmax += (tmax-tmin)*2
for (tfade, tobs_shift, tcut), target in zip(
tdata, problem.targets):
store = engine.get_store(target.store_id)
deltat = store.config.deltat
freqlimits = list(target.get_freqlimits())
freqlimits[2] = 0.45/deltat
freqlimits[3] = 0.5/deltat
freqlimits = tuple(freqlimits)
try:
trs_projected, trs_restituted, trs_raw, _ = \
ds.get_waveform(
target.codes,
tmin=tmin+tobs_shift,
tmax=tmax+tobs_shift,
tfade=tfade,
freqlimits=freqlimits,
deltat=deltat,
backazimuth=target.
get_backazimuth_for_waveform(),
debug=True)
except NotFound as e:
logger.warn(str(e))
continue
trs_projected = copy.deepcopy(trs_projected)
trs_restituted = copy.deepcopy(trs_restituted)
trs_raw = copy.deepcopy(trs_raw)
for trx in trs_projected + trs_restituted + trs_raw:
trx.shift(-tobs_shift)
trx.set_codes(
network='',
station=target.string_id(),
location='')
for trx in trs_projected:
trx.set_codes(location=trx.location + '2_proj')
for trx in trs_restituted:
trx.set_codes(location=trx.location + '1_rest')
for trx in trs_raw:
trx.set_codes(location=trx.location + '0_raw')
trs_all.extend(trs_projected)
trs_all.extend(trs_restituted)
trs_all.extend(trs_raw)
for source in sources:
tmin_fit, tmax_fit, tfade, tfade_taper = \
target.get_taper_params(engine, source)
markers.append(pmarker.Marker(
nslc_ids=[('', target.string_id(), '*_proj', '*')],
tmin=tmin_fit, tmax=tmax_fit))
markers.append(pmarker.Marker(
nslc_ids=[('', target.string_id(), '*_raw', '*')],
tmin=tcut[0]-tobs_shift, tmax=tcut[1]-tobs_shift,
kind=1))
else:
for itarget, target in enumerate(problem.targets):
nok = 0
for results in results_list:
result = results[itarget]
if not isinstance(result, gf.SeismosizerError):
nok += 1
if nok == 0:
sok = 'not used'
elif nok == len(results_list):
sok = 'ok'
try:
s = ds.get_station(target)
stations_used[s.nsl()] = s
except (NotFound, InvalidObject):
pass
else:
sok = 'not used (%i/%i ok)' % (nok, len(results_list))
logger.info('%-40s %s' % (
(target.string_id() + ':', sok)))
except GrondError as e:
logger.error('Event %i, "%s": %s' % (
ievent,
event.name or util.time_to_str(event.time),
str(e)))
erroneous.append(event)
if show_waveforms:
trace.snuffle(trs_all, stations=ds.get_stations(), markers=markers)
if stations_used_path:
stations = list(stations_used.values())
stations.sort(key=lambda s: s.nsl())
model.dump_stations(stations, stations_used_path)
if erroneous:
raise GrondError(
'Check failed for events: %s'
% ', '.join(ev.name for ev in erroneous))
def save_station(self, fn):
dump_stations([self.station_c], fn)
