def testProjection(self):
s2 = math.sqrt(2.)
ndata = num.array([s2,s2], dtype=num.float)
edata = num.array([s2,0.], dtype=num.float)
ddata = num.array([1.,-1.], dtype=num.float)
dt = 1.0
n = trace.Trace(deltat=dt, ydata=ndata, tmin=100, channel='N')
e = trace.Trace(deltat=dt, ydata=edata, tmin=100, channel='E')
d = trace.Trace(deltat=dt, ydata=ddata, tmin=100, channel='D')
azi = 45.
cazi = math.cos(azi*d2r)
sazi = math.sin(azi*d2r)
rot45 = num.array([[cazi, sazi, 0],[-sazi,cazi, 0], [0,0,-1]], dtype=num.float)
C = lambda x: model.Channel(x)
rotated = trace.project([n,e,d], rot45, [C('N'),C('E'),C('D')], [C('R'),C('T'),C('U')])
for tr in rotated:
if tr.channel == 'R':
r = tr
if tr.channel == 'T':
t = tr
if tr.channel == 'U':
u = tr
assert( num.all(r.get_ydata() - num.array([ 2., 1. ]) < 1.0e-6 ) )
assert( num.all(t.get_ydata() - num.array([ 0., -1 ]) < 1.0e-6 ) )
assert( num.all(u.get_ydata() - num.array([ -1., 1. ]) < 1.0e-6 ) )
deps = trace.project_dependencies(rot45, [C('N'),C('E'),C('D')], [C('R'),C('T'),C('U')])
assert( set(['N','E']) == set(deps['R']) and set(['N', 'E']) == set(deps['T']) and
set(['D']) == set(deps['U']) )
# should work though no horizontals given
projected = trace.project([d], rot45, [C('N'),C('E'),C('D')], [C('R'),C('T'),C('U')])
if tr.channel == 'U': u = tr
assert( num.all(u.get_ydata() - num.array([ -1., 1. ]) < 1.0e-6 ) )
def testProjection(self):
s2 = math.sqrt(2.)
ndata = num.array([s2,s2], dtype=num.float)
edata = num.array([s2,0.], dtype=num.float)
ddata = num.array([1.,-1.], dtype=num.float)
dt = 1.0
n = trace.Trace(deltat=dt, ydata=ndata, tmin=100, channel='N')
e = trace.Trace(deltat=dt, ydata=edata, tmin=100, channel='E')
d = trace.Trace(deltat=dt, ydata=ddata, tmin=100, channel='D')
azi = 45.
cazi = math.cos(azi*d2r)
sazi = math.sin(azi*d2r)
rot45 = num.array([[cazi, sazi, 0],[-sazi,cazi, 0], [0,0,-1]], dtype=num.float)
C = lambda x: model.Channel(x)
rotated = trace.project([n,e,d], rot45, [C('N'),C('E'),C('D')], [C('R'),C('T'),C('U')])
for tr in rotated:
if tr.channel == 'R':
r = tr
if tr.channel == 'T':
t = tr
if tr.channel == 'U':
u = tr
assert( num.all(r.get_ydata() - num.array([ 2., 1. ]) < 1.0e-6 ) )
assert( num.all(t.get_ydata() - num.array([ 0., -1 ]) < 1.0e-6 ) )
assert( num.all(u.get_ydata() - num.array([ -1., 1. ]) < 1.0e-6 ) )
deps = trace.project_dependencies(rot45, [C('N'),C('E'),C('D')], [C('R'),C('T'),C('U')])
assert( set(['N','E']) == set(deps['R']) and set(['N', 'E']) == set(deps['T']) and
set(['D']) == set(deps['U']) )
# should work though no horizontals given
projected = trace.project([d], rot45, [C('N'),C('E'),C('D')], [C('R'),C('T'),C('U')])
if tr.channel == 'U': u = tr
assert( num.all(u.get_ydata() - num.array([ -1., 1. ]) < 1.0e-6 ) )
def testProjectionsZOnly(self):
km = 1000.
ev = model.Event(lat=-10, lon=150., depth=0.0)
for azi in num.linspace(0., 360., 37):
lat, lon = orthodrome.ne_to_latlon(ev.lat, ev.lon,
10. * km * math.cos(azi),
10. * km * math.sin(azi))
sta = model.Station(lat=lat, lon=lon)
sta.set_event_relative_data(ev)
sta.set_channels_by_name('BHZ', 'BHN', 'BHE')
traces = [
trace.Trace(channel='BHZ', ydata=num.array([1.0])),
]
projected = []
for m, in_channels, out_channels in sta.guess_projections_to_enu():
projected.extend(
trace.project(traces, m, in_channels, out_channels))
def g(traces, cha):
for tr in traces:
if tr.channel == cha:
return tr
z = g(projected, 'U')
assert (near(z.ydata[0], 1.0, 0.001))
def testProjections(self):
km = 1000.
ev = model.Event(lat=-10, lon=150., depth=0.0)
for azi in num.linspace(0., 360., 37):
lat,lon = orthodrome.ne_to_latlon(ev.lat, ev.lon, 10.*km * math.cos(azi), 10.*km * math.sin(azi) )
sta = model.Station(lat=lat, lon=lon)
sta.set_event_relative_data(ev)
sta.set_channels_by_name('BHZ', 'BHE', 'BHN')
r = 1.
t = 1.
traces = [
trace.Trace(channel='BHE', ydata=num.array([math.sin(azi)*r+math.cos(azi)*t])),
trace.Trace(channel='BHN', ydata=num.array([math.cos(azi)*r-math.sin(azi)*t])),
]
for m, in_channels, out_channels in sta.guess_projections_to_rtu():
projected = trace.project(traces, m, in_channels, out_channels)
def g(traces, cha):
for tr in traces:
if tr.channel == cha:
return tr
r = g(projected, 'R')
t = g(projected, 'T')
assert( near(r.ydata[0], 1.0, 0.001) )
assert( near(t.ydata[0], 1.0, 0.001) )
def testProjectionsZOnly(self):
km = 1000.
ev = model.Event(lat=-10, lon=150., depth=0.0)
for azi in num.linspace(0., 360., 37):
lat, lon = orthodrome.ne_to_latlon(
ev.lat, ev.lon, 10.*km * math.cos(azi), 10.*km * math.sin(azi))
sta = model.Station(lat=lat, lon=lon)
sta.set_event_relative_data(ev)
sta.set_channels_by_name('BHZ', 'BHN', 'BHE')
traces = [
trace.Trace(
channel='BHZ',
ydata=num.array([1.0])),
]
projected = []
for m, in_channels, out_channels in sta.guess_projections_to_enu():
projected.extend(trace.project(
traces, m, in_channels, out_channels))
def g(traces, cha):
for tr in traces:
if tr.channel == cha:
return tr
z = g(projected, 'U')
assert(near(z.ydata[0], 1.0, 0.001))
def rotate_traces_and_stations(datatraces, stations, event):
"""
Rotate traces and stations into RTZ with respect to the event.
Updates channels of stations in place!
Parameters
---------
datatraces: list
of :class:`pyrocko.trace.Trace`
stations: list
of :class:`pyrocko.model.Station`
event: :class:`pyrocko.model.Event`
Returns
-------
rotated traces to RTZ
"""
from pyrocko import trace
station2traces = utility.gather(datatraces, lambda t: t.station)
trs_projected = []
for station in stations:
station.set_event_relative_data(event)
projections = station.guess_projections_to_rtu(out_channels=('R', 'T',
'Z'))
try:
traces = station2traces[station.station]
except (KeyError):
logger.warning('Did not find data traces for station "%s"' %
stations.station)
continue
ntraces = len(traces)
if ntraces < 3:
logger.warn('Only found %i component(s) for station %s' %
(ntraces, station.station))
for matrix, in_channels, out_channels in projections:
proc = trace.project(traces, matrix, in_channels, out_channels)
for tr in proc:
logger.debug('Outtrace: \n %s' % tr.__str__())
for ch in out_channels:
if ch.name == tr.channel:
station.add_channel(ch)
if proc:
logger.debug('Updated station: \n %s' % station.__str__())
trs_projected.extend(proc)
return trs_projected
def testProjections(self):
km = 1000.
ev = model.Event(lat=-10, lon=150., depth=0.0)
for azi in num.linspace(0., 360., 37):
lat, lon = orthodrome.ne_to_latlon(ev.lat, ev.lon,
10. * km * math.cos(azi),
10. * km * math.sin(azi))
sta = model.Station(lat=lat, lon=lon)
sta.set_event_relative_data(ev)
sta.set_channels_by_name('BHZ', 'BHE', 'BHN')
r = 1.
t = 1.
traces = [
trace.Trace(channel='BHE',
ydata=num.array(
[math.sin(azi) * r + math.cos(azi) * t])),
trace.Trace(channel='BHN',
ydata=num.array(
[math.cos(azi) * r - math.sin(azi) * t])),
]
for m, in_channels, out_channels in sta.guess_projections_to_rtu():
projected = trace.project(traces, m, in_channels, out_channels)
def g(traces, cha):
for tr in traces:
if tr.channel == cha:
return tr
r = g(projected, 'R')
t = g(projected, 'T')
assert (near(r.ydata[0], 1.0, 0.001))
assert (near(t.ydata[0], 1.0, 0.001))
def main():
parser = OptionParser(usage=usage, description=description)
parser.add_option('--force',
dest='force',
action='store_true',
default=False,
help='allow recreation of output <directory>')
parser.add_option('--debug',
dest='debug',
action='store_true',
default=False,
help='print debugging information to stderr')
parser.add_option('--dry-run',
dest='dry_run',
action='store_true',
default=False,
help='show available stations/channels and exit '
'(do not download waveforms)')
parser.add_option('--continue',
dest='continue_',
action='store_true',
default=False,
help='continue download after a accident')
parser.add_option('--local-data',
dest='local_data',
action='append',
help='add file/directory with local data')
parser.add_option('--local-stations',
dest='local_stations',
action='append',
help='add local stations file')
parser.add_option('--selection',
dest='selection_file',
action='append',
help='add local stations file')
parser.add_option(
'--local-responses-resp',
dest='local_responses_resp',
action='append',
help='add file/directory with local responses in RESP format')
parser.add_option('--local-responses-pz',
dest='local_responses_pz',
action='append',
help='add file/directory with local pole-zero responses')
parser.add_option(
'--local-responses-stationxml',
dest='local_responses_stationxml',
help='add file with local response information in StationXML format')
parser.add_option(
'--window',
dest='window',
default='full',
help='set time window to choose [full, p, "<time-start>,<time-end>"'
'] (time format is YYYY-MM-DD HH:MM:SS)')
parser.add_option(
'--out-components',
choices=['enu', 'rtu'],
dest='out_components',
default='rtu',
help='set output component orientations to radial-transverse-up [rtu] '
'(default) or east-north-up [enu]')
parser.add_option('--out-units',
choices=['M', 'M/S', 'M/S**2'],
dest='output_units',
default='M',
help='set output units to displacement "M" (default),'
' velocity "M/S" or acceleration "M/S**2"')
parser.add_option(
'--padding-factor',
type=float,
default=3.0,
dest='padding_factor',
help='extend time window on either side, in multiples of 1/<fmin_hz> '
'(default: 5)')
parser.add_option(
'--zero-padding',
dest='zero_pad',
action='store_true',
default=False,
help='Extend traces by zero-padding if clean restitution requires'
'longer windows')
parser.add_option(
'--credentials',
dest='user_credentials',
action='append',
default=[],
metavar='SITE,USER,PASSWD',
help='user credentials for specific site to access restricted data '
'(this option can be repeated)')
parser.add_option(
'--token',
dest='auth_tokens',
metavar='SITE,FILENAME',
action='append',
default=[],
help='user authentication token for specific site to access '
'restricted data (this option can be repeated)')
parser.add_option(
'--sites',
dest='sites',
metavar='SITE1,SITE2,...',
default='geofon,iris,orfeus',
help='sites to query (available: %s, default: "%%default"' %
', '.join(g_sites_available))
parser.add_option(
'--band-codes',
dest='priority_band_code',
metavar='V,L,M,B,H,S,E,...',
default='B,H',
help='select and prioritize band codes (default: %default)')
parser.add_option(
'--instrument-codes',
dest='priority_instrument_code',
metavar='H,L,G,...',
default='H,L',
help='select and prioritize instrument codes (default: %default)')
parser.add_option('--radius-min',
dest='radius_min',
metavar='VALUE',
default=0.0,
type=float,
help='minimum radius [km]')
parser.add_option('--nstations-wanted',
dest='nstations_wanted',
metavar='N',
type=int,
help='number of stations to select initially')
(options, args) = parser.parse_args(sys.argv[1:])
print('Parsed arguments:', args)
if len(args) not in (10, 7, 6):
parser.print_help()
sys.exit(1)
if options.debug:
util.setup_logging(program_name, 'debug')
else:
util.setup_logging(program_name, 'info')
if options.local_responses_pz and options.local_responses_resp:
logger.critical('cannot use local responses in PZ and RESP '
'format at the same time')
sys.exit(1)
n_resp_opt = 0
for resp_opt in (options.local_responses_pz, options.local_responses_resp,
options.local_responses_stationxml):
if resp_opt:
n_resp_opt += 1
if n_resp_opt > 1:
logger.critical('can only handle local responses from either PZ or '
'RESP or StationXML. Cannot yet merge different '
'response formats.')
sys.exit(1)
if options.local_responses_resp and not options.local_stations:
logger.critical('--local-responses-resp can only be used '
'when --stations is also given.')
sys.exit(1)
try:
ename = ''
magnitude = None
mt = None
if len(args) == 10:
time = util.str_to_time(args[1] + ' ' + args[2])
lat = float(args[3])
lon = float(args[4])
depth = float(args[5]) * km
iarg = 6
elif len(args) == 7:
if args[2].find(':') == -1:
sname_or_date = None
lat = float(args[1])
lon = float(args[2])
event = None
time = None
else:
sname_or_date = args[1] + ' ' + args[2]
iarg = 3
elif len(args) == 6:
sname_or_date = args[1]
iarg = 2
if len(args) in (7, 6) and sname_or_date is not None:
events = get_events_by_name_or_date([sname_or_date],
catalog=geofon)
if len(events) == 0:
logger.critical('no event found')
sys.exit(1)
elif len(events) > 1:
logger.critical('more than one event found')
sys.exit(1)
event = events[0]
time = event.time
lat = event.lat
lon = event.lon
depth = event.depth
ename = event.name
magnitude = event.magnitude
mt = event.moment_tensor
radius = float(args[iarg]) * km
fmin = float(args[iarg + 1])
sample_rate = float(args[iarg + 2])
eventname = args[iarg + 3]
cwd = str(sys.argv[1])
event_dir = op.join(cwd, 'data', 'events', eventname)
output_dir = op.join(event_dir, 'waveforms')
except:
raise
parser.print_help()
sys.exit(1)
if options.force and op.isdir(event_dir):
if not options.continue_:
shutil.rmtree(event_dir)
if op.exists(event_dir) and not options.continue_:
logger.critical(
'directory "%s" exists. Delete it first or use the --force option'
% event_dir)
sys.exit(1)
util.ensuredir(output_dir)
if time is not None:
event = model.Event(time=time,
lat=lat,
lon=lon,
depth=depth,
name=ename,
magnitude=magnitude,
moment_tensor=mt)
if options.window == 'full':
if event is None:
logger.critical('need event for --window=full')
sys.exit(1)
low_velocity = 1500.
timewindow = VelocityWindow(low_velocity,
tpad=options.padding_factor / fmin)
tmin, tmax = timewindow(time, radius, depth)
elif options.window == 'p':
if event is None:
logger.critical('need event for --window=p')
sys.exit(1)
phases = list(map(cake.PhaseDef, 'P p'.split()))
emod = cake.load_model()
tpad = options.padding_factor / fmin
timewindow = PhaseWindow(emod, phases, -tpad, tpad)
arrivaltimes = []
for dist in num.linspace(0, radius, 20):
try:
arrivaltimes.extend(timewindow(time, dist, depth))
except NoArrival:
pass
if not arrivaltimes:
logger.error('required phase arrival not found')
sys.exit(1)
tmin = min(arrivaltimes)
tmax = max(arrivaltimes)
else:
try:
stmin, stmax = options.window.split(',')
tmin = util.str_to_time(stmin.strip())
tmax = util.str_to_time(stmax.strip())
timewindow = FixedWindow(tmin, tmax)
except ValueError:
logger.critical('invalid argument to --window: "%s"' %
options.window)
sys.exit(1)
if event is not None:
event.name = eventname
tfade = tfade_factor / fmin
tpad = tfade
tmin -= tpad
tmax += tpad
tinc = None
priority_band_code = options.priority_band_code.split(',')
for s in priority_band_code:
if len(s) != 1:
logger.critical('invalid band code: %s' % s)
priority_instrument_code = options.priority_instrument_code.split(',')
for s in priority_instrument_code:
if len(s) != 1:
logger.critical('invalid instrument code: %s' % s)
station_query_conf = dict(latitude=lat,
longitude=lon,
minradius=options.radius_min * km * cake.m2d,
maxradius=radius * cake.m2d,
channel=','.join('%s??' % s
for s in priority_band_code))
target_sample_rate = sample_rate
fmax = target_sample_rate
# target_sample_rate = None
# priority_band_code = ['H', 'B', 'M', 'L', 'V', 'E', 'S']
priority_units = ['M/S', 'M', 'M/S**2']
# output_units = 'M'
sites = [x.strip() for x in options.sites.split(',') if x.strip()]
for site in sites:
if site not in g_sites_available:
logger.critical('unknown FDSN site: %s' % site)
sys.exit(1)
for s in options.user_credentials:
try:
site, user, passwd = s.split(',')
g_user_credentials[site] = user, passwd
except ValueError:
logger.critical('invalid format for user credentials: "%s"' % s)
sys.exit(1)
for s in options.auth_tokens:
try:
site, token_filename = s.split(',')
with open(token_filename, 'r') as f:
g_auth_tokens[site] = f.read()
except (ValueError, OSError, IOError):
logger.critical('cannot get token from file: %s' % token_filename)
sys.exit(1)
fn_template0 = \
'data_%(network)s.%(station)s.%(location)s.%(channel)s_%(tmin)s.mseed'
fn_template_raw = op.join(output_dir, 'raw', fn_template0)
fn_stations_raw = op.join(output_dir, 'stations.raw.txt')
fn_template_rest = op.join(output_dir, 'rest', fn_template0)
fn_commandline = op.join(output_dir, 'beatdown.command')
ftap = (ffade_factors[0] * fmin, fmin, fmax, ffade_factors[1] * fmax)
# chapter 1: download
sxs = []
for site in sites:
try:
extra_args = {
'iris': dict(matchtimeseries=True),
}.get(site, {})
extra_args.update(station_query_conf)
if site == 'geonet':
extra_args.update(starttime=tmin, endtime=tmax)
else:
extra_args.update(startbefore=tmax,
endafter=tmin,
includerestricted=(site in g_user_credentials
or site in g_auth_tokens))
logger.info('downloading channel information (%s)' % site)
sx = fdsn.station(site=site,
format='text',
level='channel',
**extra_args)
except fdsn.EmptyResult:
logger.error('No stations matching given criteria. (%s)' % site)
sx = None
if sx is not None:
sxs.append(sx)
if all(sx is None for sx in sxs) and not options.local_data:
sys.exit(1)
nsl_to_sites = defaultdict(list)
nsl_to_station = {}
if options.selection_file:
logger.info('using stations from stations file!')
stations = []
for fn in options.selection_file:
stations.extend(model.load_stations(fn))
nsls_selected = set(s.nsl() for s in stations)
else:
nsls_selected = None
for sx, site in zip(sxs, sites):
site_stations = sx.get_pyrocko_stations()
for s in site_stations:
nsl = s.nsl()
nsl_to_sites[nsl].append(site)
if nsl not in nsl_to_station:
if nsls_selected:
if nsl in nsls_selected:
nsl_to_station[nsl] = s
else:
nsl_to_station[
nsl] = s # using first site with this station
logger.info('number of stations found: %i' % len(nsl_to_station))
# station weeding
if options.nstations_wanted:
nsls_selected = None
stations_all = [
nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())
]
for s in stations_all:
s.set_event_relative_data(event)
stations_selected = weeding.weed_stations(stations_all,
options.nstations_wanted)[0]
nsls_selected = set(s.nsl() for s in stations_selected)
logger.info('number of stations selected: %i' % len(nsls_selected))
if tinc is None:
tinc = 3600.
have_data = set()
if options.continue_:
fns = glob.glob(fn_template_raw % starfill())
p = pile.make_pile(fns)
else:
fns = []
have_data_site = {}
could_have_data_site = {}
for site in sites:
have_data_site[site] = set()
could_have_data_site[site] = set()
available_through = defaultdict(set)
it = 0
nt = int(math.ceil((tmax - tmin) / tinc))
for it in range(nt):
tmin_win = tmin + it * tinc
tmax_win = min(tmin + (it + 1) * tinc, tmax)
logger.info('time window %i/%i (%s - %s)' %
(it + 1, nt, util.tts(tmin_win), util.tts(tmax_win)))
have_data_this_window = set()
if options.continue_:
trs_avail = p.all(tmin=tmin_win, tmax=tmax_win, load_data=False)
for tr in trs_avail:
have_data_this_window.add(tr.nslc_id)
for site, sx in zip(sites, sxs):
if sx is None:
continue
selection = []
channels = sx.choose_channels(
target_sample_rate=target_sample_rate,
priority_band_code=priority_band_code,
priority_units=priority_units,
priority_instrument_code=priority_instrument_code,
timespan=(tmin_win, tmax_win))
for nslc in sorted(channels.keys()):
if nsls_selected is not None and nslc[:3] not in nsls_selected:
continue
could_have_data_site[site].add(nslc)
if nslc not in have_data_this_window:
channel = channels[nslc]
if event:
lat_, lon_ = event.lat, event.lon
else:
lat_, lon_ = lat, lon
try:
dist = orthodrome.distance_accurate50m_numpy(
lat_, lon_, channel.latitude.value,
channel.longitude.value)
except:
dist = orthodrome.distance_accurate50m_numpy(
lat_, lon_, channel.latitude, channel.longitude)
if event:
depth_ = event.depth
time_ = event.time
else:
depth_ = None
time_ = None
tmin_, tmax_ = timewindow(time_, dist, depth_)
tmin_this = tmin_ - tpad
tmax_this = float(tmax_ + tpad)
tmin_req = max(tmin_win, tmin_this)
tmax_req = min(tmax_win, tmax_this)
if channel.sample_rate:
try:
deltat = 1.0 / int(channel.sample_rate.value)
except:
deltat = 1.0 / int(channel.sample_rate)
else:
deltat = 1.0
if tmin_req < tmax_req:
logger.debug('deltat %f' % deltat)
# extend time window by some samples because otherwise
# sometimes gaps are produced
# apparently the WS are only sensitive to full seconds
# round to avoid gaps, increase safetiy window
selection.append(nslc +
(math.floor(tmin_req - deltat * 20.0),
math.ceil(tmax_req + deltat * 20.0)))
if options.dry_run:
for (net, sta, loc, cha, tmin, tmax) in selection:
available_through[net, sta, loc, cha].add(site)
else:
neach = 100
i = 0
nbatches = ((len(selection) - 1) // neach) + 1
while i < len(selection):
selection_now = selection[i:i + neach]
f = tempfile.NamedTemporaryFile()
try:
sbatch = ''
if nbatches > 1:
sbatch = ' (batch %i/%i)' % (
(i // neach) + 1, nbatches)
logger.info('downloading data (%s)%s' % (site, sbatch))
data = fdsn.dataselect(site=site,
selection=selection_now,
**get_user_credentials(site))
while True:
buf = data.read(1024)
if not buf:
break
f.write(buf)
f.flush()
trs = io.load(f.name)
for tr in trs:
tr.fix_deltat_rounding_errors()
logger.debug('cutting window: %f - %f' %
(tmin_win, tmax_win))
logger.debug(
'available window: %f - %f, nsamples: %g' %
(tr.tmin, tr.tmax, tr.ydata.size))
try:
logger.debug('tmin before snap %f' % tr.tmin)
tr.snap(interpolate=True)
logger.debug('tmin after snap %f' % tr.tmin)
tr.chop(tmin_win,
tmax_win,
snap=(math.floor, math.ceil),
include_last=True)
logger.debug(
'cut window: %f - %f, nsamles: %g' %
(tr.tmin, tr.tmax, tr.ydata.size))
have_data.add(tr.nslc_id)
have_data_site[site].add(tr.nslc_id)
except trace.NoData:
pass
fns2 = io.save(trs, fn_template_raw)
for fn in fns2:
if fn in fns:
logger.warn('overwriting file %s', fn)
fns.extend(fns2)
except fdsn.EmptyResult:
pass
except HTTPError:
logger.warn('an error occurred while downloading data '
'for channels \n %s' %
'\n '.join('.'.join(x[:4])
for x in selection_now))
f.close()
i += neach
if options.dry_run:
nslcs = sorted(available_through.keys())
all_channels = defaultdict(set)
all_stations = defaultdict(set)
def plural_s(x):
return '' if x == 1 else 's'
for nslc in nslcs:
sites = tuple(sorted(available_through[nslc]))
logger.info('selected: %s.%s.%s.%s from site%s %s' %
(nslc + (plural_s(len(sites)), '+'.join(sites))))
all_channels[sites].add(nslc)
all_stations[sites].add(nslc[:3])
nchannels_all = 0
nstations_all = 0
for sites in sorted(all_channels.keys(),
key=lambda sites: (-len(sites), sites)):
nchannels = len(all_channels[sites])
nstations = len(all_stations[sites])
nchannels_all += nchannels
nstations_all += nstations
logger.info('selected (%s): %i channel%s (%i station%s)' %
('+'.join(sites), nchannels, plural_s(nchannels),
nstations, plural_s(nstations)))
logger.info('selected total: %i channel%s (%i station%s)' %
(nchannels_all, plural_s(nchannels_all), nstations_all,
plural_s(nstations_all)))
logger.info('dry run done.')
sys.exit(0)
for nslc in have_data:
# if we are in continue mode, we have to guess where the data came from
if not any(nslc in have_data_site[site] for site in sites):
for site in sites:
if nslc in could_have_data_site[site]:
have_data_site[site].add(nslc)
sxs = {}
for site in sites:
selection = []
for nslc in sorted(have_data_site[site]):
selection.append(nslc + (tmin - tpad, tmax + tpad))
if selection:
logger.info('downloading response information (%s)' % site)
sxs[site] = fdsn.station(site=site,
level='response',
selection=selection)
sxs[site].dump_xml(filename=op.join(output_dir, 'stations.%s.xml' %
site))
# chapter 1.5: inject local data
if options.local_data:
have_data_site['local'] = set()
plocal = pile.make_pile(options.local_data, fileformat='detect')
logger.info(
'Importing local data from %s between %s (%f) and %s (%f)' %
(options.local_data, util.time_to_str(tmin), tmin,
util.time_to_str(tmax), tmax))
for traces in plocal.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=tmin,
tmax=tmax,
tinc=tinc):
for tr in traces:
if tr.nslc_id not in have_data:
fns.extend(io.save(traces, fn_template_raw))
have_data_site['local'].add(tr.nslc_id)
have_data.add(tr.nslc_id)
sites.append('local')
if options.local_responses_pz:
sxs['local'] = epz.make_stationxml(
epz.iload(options.local_responses_pz))
if options.local_responses_resp:
local_stations = []
for fn in options.local_stations:
local_stations.extend(model.load_stations(fn))
sxs['local'] = resp.make_stationxml(
local_stations, resp.iload(options.local_responses_resp))
if options.local_responses_stationxml:
sxs['local'] = stationxml.load_xml(
filename=options.local_responses_stationxml)
# chapter 1.6: dump raw data stations file
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())]
util.ensuredirs(fn_stations_raw)
model.dump_stations(stations, fn_stations_raw)
dump_commandline(sys.argv, fn_commandline)
# chapter 2: restitution
if not fns:
logger.error('no data available')
sys.exit(1)
p = pile.make_pile(fns, show_progress=False)
p.get_deltatmin()
otinc = None
if otinc is None:
otinc = nice_seconds_floor(p.get_deltatmin() * 500000.)
otinc = 3600.
otmin = math.floor(p.tmin / otinc) * otinc
otmax = math.ceil(p.tmax / otinc) * otinc
otpad = tpad * 2
fns = []
rest_traces_b = []
win_b = None
for traces_a in p.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=otmin,
tmax=otmax,
tinc=otinc,
tpad=otpad):
rest_traces_a = []
win_a = None
for tr in traces_a:
win_a = tr.wmin, tr.wmax
if win_b and win_b[0] >= win_a[0]:
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
rest_traces_b = []
win_b = None
response = None
failure = []
for site in sites:
try:
if site not in sxs:
continue
logger.debug('Getting response for %s' % tr.__str__())
response = sxs[site].get_pyrocko_response(
tr.nslc_id,
timespan=(tr.tmin, tr.tmax),
fake_input_units=options.output_units)
break
except stationxml.NoResponseInformation:
failure.append('%s: no response information' % site)
except stationxml.MultipleResponseInformation:
failure.append('%s: multiple response information' % site)
if response is None:
failure = ', '.join(failure)
else:
failure = ''
try:
if tr.tmin > tmin and options.zero_pad:
logger.warning(
'Trace too short for clean restitution in '
'desired frequency band -> zero-padding!')
tr.extend(tr.tmin - tfade, tr.tmax + tfade, 'repeat')
rest_tr = tr.transfer(tfade, ftap, response, invert=True)
rest_traces_a.append(rest_tr)
except (trace.TraceTooShort, trace.NoData):
failure = 'trace too short'
if failure:
logger.warn('failed to restitute trace %s.%s.%s.%s (%s)' %
(tr.nslc_id + (failure, )))
if rest_traces_b:
rest_traces = trace.degapper(rest_traces_b + rest_traces_a,
deoverlap='crossfade_cos')
fns.extend(cut_n_dump(rest_traces, win_b, fn_template_rest))
rest_traces_a = []
if win_a:
for tr in rest_traces:
try:
rest_traces_a.append(
tr.chop(win_a[0], win_a[1] + otpad, inplace=False))
except trace.NoData:
pass
rest_traces_b = rest_traces_a
win_b = win_a
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
# chapter 3: rotated restituted traces for inspection
if not event:
sys.exit(0)
fn_template1 = \
'DISPL.%(network)s.%(station)s.%(location)s.%(channel)s'
fn_waveforms = op.join(output_dir, 'prepared', fn_template1)
fn_stations = op.join(output_dir, 'stations.prepared.txt')
fn_event = op.join(event_dir, 'event.txt')
fn_event_yaml = op.join(event_dir, 'event.yaml')
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
p = pile.make_pile(fns, show_progress=False)
deltat = None
if sample_rate is not None:
deltat = 1.0 / sample_rate
traces_beat = []
used_stations = []
for nsl, s in nsl_to_station.items():
s.set_event_relative_data(event)
traces = p.all(trace_selector=lambda tr: tr.nslc_id[:3] == nsl)
if options.out_components == 'rtu':
pios = s.guess_projections_to_rtu(out_channels=('R', 'T', 'Z'))
elif options.out_components == 'enu':
pios = s.guess_projections_to_enu(out_channels=('E', 'N', 'Z'))
else:
assert False
for (proj, in_channels, out_channels) in pios:
proc = trace.project(traces, proj, in_channels, out_channels)
for tr in proc:
tr_beat = heart.SeismicDataset.from_pyrocko_trace(tr)
traces_beat.append(tr_beat)
for ch in out_channels:
if ch.name == tr.channel:
s.add_channel(ch)
if proc:
io.save(proc, fn_waveforms)
used_stations.append(s)
stations = list(used_stations)
util.ensuredirs(fn_stations)
model.dump_stations(stations, fn_stations)
model.dump_events([event], fn_event)
from pyrocko.guts import dump
dump([event], filename=fn_event_yaml)
utility.dump_objects(op.join(cwd, 'seismic_data.pkl'),
outlist=[stations, traces_beat])
logger.info('prepared waveforms from %i stations' % len(stations))
keep.append(tr)
except util.UnavailableDecimation as e:
logger.warn('Cannot downsample %s.%s.%s.%s: %s' %
(tr.nslc_id + (e, )))
continue
if options.out_components == 'rtu':
pios = s.guess_projections_to_rtu(out_channels=('R', 'T', 'Z'))
elif options.out_components == 'enu':
pios = s.guess_projections_to_enu(out_channels=('E', 'N', 'Z'))
else:
assert False
for (proj, in_channels, out_channels) in pios:
proc = trace.project(traces, proj, in_channels, out_channels)
for tr in proc:
for ch in out_channels:
if ch.name == tr.channel:
s.add_channel(ch)
if proc:
io.save(proc, fn_waveforms)
used_stations.append(s)
stations = list(used_stations)
util.ensuredirs(fn_stations)
model.dump_stations(stations, fn_stations)
model.dump_events([event], fn_event)
logger.info('prepared waveforms from %i stations' % len(stations))
def _get_waveform(self,
obj,
quantity='displacement',
tmin=None,
tmax=None,
tpad=0.,
tfade=0.,
freqlimits=None,
deltat=None,
cache=None,
backazimuth=None,
source=None,
target=None,
debug=False):
assert not debug or (debug and cache is None)
if cache is True:
cache = self._cache
_, _, _, channel = self.get_nslc(obj)
station = self.get_station(self.get_nsl(obj))
nslc = station.nsl() + (channel, )
if self.is_blacklisted(nslc):
raise NotFound('Waveform is blacklisted:', nslc)
if not self.is_whitelisted(nslc):
raise NotFound('Waveform is not on whitelist:', nslc)
assert tmin is not None
assert tmax is not None
tmin = float(tmin)
tmax = float(tmax)
nslc = tuple(nslc)
cache_k = nslc + (tmin, tmax, tuple(freqlimits), tfade, deltat, tpad,
quantity)
if cache is not None and (nslc + cache_k) in cache:
obj = cache[nslc + cache_k]
if isinstance(obj, Exception):
raise obj
elif obj is None:
raise NotFound('Waveform not found!', nslc)
else:
return obj
syn_test = self.synthetic_test
toffset_noise_extract = 0.0
if syn_test:
if not syn_test.respect_data_availability:
if syn_test.real_noise_scale != 0.0:
raise DatasetError(
'respect_data_availability=False and '
'addition of real noise cannot be combined.')
tr = syn_test.get_waveform(nslc,
tmin,
tmax,
tfade=tfade,
freqlimits=freqlimits)
if cache is not None:
cache[tr.nslc_id + cache_k] = tr
if debug:
return [], [], []
else:
return tr
if syn_test.real_noise_scale != 0.0:
toffset_noise_extract = syn_test.real_noise_toffset
abs_delays = []
for ocha in 'ENZRT':
sc = self.station_corrections.get(station.nsl() + (channel, ),
None)
if sc:
abs_delays.append(abs(sc.delay))
if abs_delays:
abs_delay_max = max(abs_delays)
else:
abs_delay_max = 0.0
projections = self._get_projections(station, backazimuth, source,
target, tmin, tmax)
try:
trs_projected = []
trs_restituted = []
trs_raw = []
exceptions = []
for matrix, in_channels, out_channels in projections:
deps = trace.project_dependencies(matrix, in_channels,
out_channels)
try:
trs_restituted_group = []
trs_raw_group = []
if channel in deps:
for cha in deps[channel]:
trs_restituted_this, trs_raw_this = \
self.get_waveform_restituted(
station.nsl() + (cha,),
quantity=quantity,
tmin=tmin, tmax=tmax,
tpad=tpad+abs_delay_max,
toffset_noise_extract=toffset_noise_extract, # noqa
tfade=tfade,
freqlimits=freqlimits,
deltat=deltat,
want_incomplete=debug,
extend_incomplete=self.extend_incomplete)
trs_restituted_group.extend(trs_restituted_this)
trs_raw_group.extend(trs_raw_this)
trs_projected.extend(
trace.project(trs_restituted_group, matrix,
in_channels, out_channels))
trs_restituted.extend(trs_restituted_group)
trs_raw.extend(trs_raw_group)
except NotFound as e:
exceptions.append((in_channels, out_channels, e))
if not trs_projected:
err = []
for (in_channels, out_channels, e) in exceptions:
sin = ', '.join(c.name for c in in_channels)
sout = ', '.join(c.name for c in out_channels)
err.append('(%s) -> (%s): %s' % (sin, sout, e))
raise NotFound('\n'.join(err))
for tr in trs_projected:
sc = self.station_corrections.get(tr.nslc_id, None)
if sc:
if self.apply_correction_factors:
tr.ydata /= sc.factor
if self.apply_correction_delays:
tr.shift(-sc.delay)
if tmin is not None and tmax is not None:
tr.chop(tmin, tmax)
if syn_test:
trs_projected_synthetic = []
for tr in trs_projected:
if tr.channel == channel:
tr_syn = syn_test.get_waveform(tr.nslc_id,
tmin,
tmax,
tfade=tfade,
freqlimits=freqlimits)
if tr_syn:
if syn_test.real_noise_scale != 0.0:
tr_syn = tr_syn.copy()
tr_noise = tr.copy()
tr_noise.set_ydata(tr_noise.get_ydata() *
syn_test.real_noise_scale)
tr_syn.add(tr_noise)
trs_projected_synthetic.append(tr_syn)
trs_projected = trs_projected_synthetic
if cache is not None:
for tr in trs_projected:
cache[tr.nslc_id + cache_k] = tr
tr_return = None
for tr in trs_projected:
if tr.channel == channel:
tr_return = tr
if debug:
return trs_projected, trs_restituted, trs_raw, tr_return
elif tr_return:
return tr_return
else:
raise NotFound('waveform not available',
station.nsl() + (channel, ))
except NotFound:
if cache is not None:
cache[nslc + cache_k] = None
raise
def main():
parser = OptionParser(usage=usage, description=description)
parser.add_option(
"--force",
dest="force",
action="store_true",
default=False,
help="allow recreation of output <directory>",
)
parser.add_option(
"--debug",
dest="debug",
action="store_true",
default=False,
help="print debugging information to stderr",
)
parser.add_option(
"--dry-run",
dest="dry_run",
action="store_true",
default=False,
help="show available stations/channels and exit "
"(do not download waveforms)",
)
parser.add_option(
"--continue",
dest="continue_",
action="store_true",
default=False,
help="continue download after a accident",
)
parser.add_option(
"--local-data",
dest="local_data",
action="append",
help="add file/directory with local data",
)
parser.add_option(
"--local-stations",
dest="local_stations",
action="append",
help="add local stations file",
)
parser.add_option(
"--local-responses-resp",
dest="local_responses_resp",
action="append",
help="add file/directory with local responses in RESP format",
)
parser.add_option(
"--local-responses-pz",
dest="local_responses_pz",
action="append",
help="add file/directory with local pole-zero responses",
)
parser.add_option(
"--local-responses-stationxml",
dest="local_responses_stationxml",
help="add file with local response information in StationXML format",
)
parser.add_option(
"--window",
dest="window",
default="full",
help='set time window to choose [full, p, "<time-start>,<time-end>"'
"] (time format is YYYY-MM-DD HH:MM:SS)",
)
parser.add_option(
"--out-components",
choices=["enu", "rtu"],
dest="out_components",
default="rtu",
help="set output component orientations to radial-transverse-up [rtu] "
"(default) or east-north-up [enu]",
)
parser.add_option(
"--padding-factor",
type=float,
default=3.0,
dest="padding_factor",
help="extend time window on either side, in multiples of 1/<fmin_hz> "
"(default: 5)",
)
parser.add_option(
"--credentials",
dest="user_credentials",
action="append",
default=[],
metavar="SITE,USER,PASSWD",
help="user credentials for specific site to access restricted data "
"(this option can be repeated)",
)
parser.add_option(
"--token",
dest="auth_tokens",
metavar="SITE,FILENAME",
action="append",
default=[],
help="user authentication token for specific site to access "
"restricted data (this option can be repeated)",
)
parser.add_option(
"--sites",
dest="sites",
metavar="SITE1,SITE2,...",
# default='bgr',
default="http://ws.gpi.kit.edu,bgr,http://188.246.25.142:8080",
help='sites to query (available: %s, default: "%%default"' %
", ".join(g_sites_available),
)
parser.add_option(
"--band-codes",
dest="priority_band_code",
metavar="V,L,M,B,H,S,E,...",
default="V,L,M,B,H,E",
help="select and prioritize band codes (default: %default)",
)
parser.add_option(
"--instrument-codes",
dest="priority_instrument_code",
metavar="H,L,G,...",
default="H,L,O,",
help="select and prioritize instrument codes (default: %default)",
)
parser.add_option(
"--radius-min",
dest="radius_min",
metavar="VALUE",
default=0.0,
type=float,
help="minimum radius [km]",
)
parser.add_option(
"--tinc",
dest="tinc",
metavar="VALUE",
default=3600.0 * 12.0,
type=float,
help="length of seperate saved files in s",
)
parser.add_option(
"--nstations-wanted",
dest="nstations_wanted",
metavar="N",
type=int,
help="number of stations to select initially",
)
(options, args) = parser.parse_args(sys.argv[1:])
if len(args) not in (9, 6, 5):
parser.print_help()
sys.exit(1)
if options.debug:
util.setup_logging(program_name, "debug")
else:
util.setup_logging(program_name, "info")
if options.local_responses_pz and options.local_responses_resp:
logger.critical("cannot use local responses in PZ and RESP "
"format at the same time")
sys.exit(1)
n_resp_opt = 0
for resp_opt in (
options.local_responses_pz,
options.local_responses_resp,
options.local_responses_stationxml,
):
if resp_opt:
n_resp_opt += 1
if n_resp_opt > 1:
logger.critical("can only handle local responses from either PZ or "
"RESP or StationXML. Cannot yet merge different "
"response formats.")
sys.exit(1)
if options.local_responses_resp and not options.local_stations:
logger.critical("--local-responses-resp can only be used "
"when --stations is also given.")
sys.exit(1)
try:
ename = ""
magnitude = None
mt = None
if len(args) == 9:
time = util.str_to_time(args[0] + " " + args[1])
lat = float(args[2])
lon = float(args[3])
depth = float(args[4]) * km
iarg = 5
elif len(args) == 6:
if args[1].find(":") == -1:
sname_or_date = None
lat = float(args[0])
lon = float(args[1])
event = None
time = None
else:
sname_or_date = args[0] + " " + args[1]
iarg = 2
elif len(args) == 5:
sname_or_date = args[0]
iarg = 1
if len(args) in (6, 5) and sname_or_date is not None:
events = get_events_by_name_or_date([sname_or_date],
catalog=geofon)
if len(events) == 0:
logger.critical("no event found")
sys.exit(1)
elif len(events) > 1:
logger.critical("more than one event found")
sys.exit(1)
event = events[0]
time = event.time
lat = event.lat
lon = event.lon
depth = event.depth
ename = event.name
magnitude = event.magnitude
mt = event.moment_tensor
radius = float(args[iarg]) * km
fmin = float(args[iarg + 1])
sample_rate = float(args[iarg + 2])
eventname = args[iarg + 3]
event_dir = op.join("data", "events", eventname)
output_dir = op.join(event_dir, "waveforms")
except:
raise
parser.print_help()
sys.exit(1)
if options.force and op.isdir(event_dir):
if not options.continue_:
shutil.rmtree(event_dir)
if op.exists(event_dir) and not options.continue_:
logger.critical(
'directory "%s" exists. Delete it first or use the --force option'
% event_dir)
sys.exit(1)
util.ensuredir(output_dir)
if time is not None:
event = model.Event(
time=time,
lat=lat,
lon=lon,
depth=depth,
name=ename,
magnitude=magnitude,
moment_tensor=mt,
)
if options.window == "full":
if event is None:
logger.critical("need event for --window=full")
sys.exit(1)
low_velocity = 1500.0
timewindow = VelocityWindow(low_velocity,
tpad=options.padding_factor / fmin)
tmin, tmax = timewindow(time, radius, depth)
elif options.window == "p":
if event is None:
logger.critical("need event for --window=p")
sys.exit(1)
phases = list(map(cake.PhaseDef, "P p".split()))
emod = cake.load_model()
tpad = options.padding_factor / fmin
timewindow = PhaseWindow(emod, phases, -tpad, tpad)
arrivaltimes = []
for dist in num.linspace(0, radius, 20):
try:
arrivaltimes.extend(timewindow(time, dist, depth))
except NoArrival:
pass
if not arrivaltimes:
logger.error("required phase arrival not found")
sys.exit(1)
tmin = min(arrivaltimes)
tmax = max(arrivaltimes)
else:
try:
stmin, stmax = options.window.split(",")
tmin = util.str_to_time(stmin.strip())
tmax = util.str_to_time(stmax.strip())
timewindow = FixedWindow(tmin, tmax)
except ValueError:
logger.critical('invalid argument to --window: "%s"' %
options.window)
sys.exit(1)
if event is not None:
event.name = eventname
tlen = tmax - tmin
tfade = tfade_factor / fmin
tpad = tfade
tmin -= tpad
tmax += tpad
priority_band_code = options.priority_band_code.split(",")
for s in priority_band_code:
if len(s) != 1:
logger.critical("invalid band code: %s" % s)
priority_instrument_code = options.priority_instrument_code.split(",")
for s in priority_instrument_code:
if len(s) != 1:
logger.critical("invalid instrument code: %s" % s)
station_query_conf = dict(
latitude=lat,
longitude=lon,
minradius=options.radius_min * km * cake.m2d,
maxradius=radius * cake.m2d,
channel=",".join("?%s?" % s for s in priority_band_code),
)
target_sample_rate = sample_rate
fmax = target_sample_rate
# target_sample_rate = None
# priority_band_code = ['H', 'B', 'M', 'L', 'V', 'E', 'S']
priority_units = ["M/S", "M", "M/S**2"]
output_units = "M"
sites = [x.strip() for x in options.sites.split(",") if x.strip()]
tinc = options.tinc
# for site in sites:
# if site not in g_sites_available:
# logger.critical('unknown FDSN site: %s' % site)
# sys.exit(1)
for s in options.user_credentials:
try:
site, user, passwd = s.split(",")
g_user_credentials[site] = user, passwd
except ValueError:
logger.critical('invalid format for user credentials: "%s"' % s)
sys.exit(1)
for s in options.auth_tokens:
try:
site, token_filename = s.split(",")
with open(token_filename, "r") as f:
g_auth_tokens[site] = f.read()
except (ValueError, OSError, IOError):
logger.critical("cannot get token from file: %s" % token_filename)
sys.exit(1)
fn_template0 = (
"data_%(network)s.%(station)s.%(location)s.%(channel)s_%(tmin)s.mseed")
fn_template_raw = op.join(output_dir, "raw", fn_template0)
fn_template_raw_folder = op.join(output_dir, "raw/", "traces.mseed")
fn_stations_raw = op.join(output_dir, "stations.raw.txt")
fn_template_rest = op.join(output_dir, "rest", fn_template0)
fn_commandline = op.join(output_dir, "seigerdown.command")
ftap = (ffade_factors[0] * fmin, fmin, fmax, ffade_factors[1] * fmax)
# chapter 1: download
sxs = []
for site in sites:
try:
extra_args = {
"iris": dict(matchtimeseries=True),
}.get(site, {})
extra_args.update(station_query_conf)
if site == "geonet":
extra_args.update(starttime=tmin, endtime=tmax)
else:
extra_args.update(
startbefore=tmax,
endafter=tmin,
includerestricted=(site in g_user_credentials
or site in g_auth_tokens),
)
logger.info("downloading channel information (%s)" % site)
sx = fdsn.station(site=site,
format="text",
level="channel",
**extra_args)
except fdsn.EmptyResult:
logger.error("No stations matching given criteria. (%s)" % site)
sx = None
if sx is not None:
sxs.append(sx)
if all(sx is None for sx in sxs) and not options.local_data:
sys.exit(1)
nsl_to_sites = defaultdict(list)
nsl_to_station = {}
for sx, site in zip(sxs, sites):
site_stations = sx.get_pyrocko_stations()
for s in site_stations:
nsl = s.nsl()
nsl_to_sites[nsl].append(site)
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s # using first site with this station
logger.info("number of stations found: %i" % len(nsl_to_station))
# station weeding
nsls_selected = None
if options.nstations_wanted:
stations_all = [
nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())
]
for s in stations_all:
s.set_event_relative_data(event)
stations_selected = weeding.weed_stations(stations_all,
options.nstations_wanted)[0]
nsls_selected = set(s.nsl() for s in stations_selected)
logger.info("number of stations selected: %i" % len(nsls_selected))
have_data = set()
if options.continue_:
fns = glob.glob(fn_template_raw % starfill())
p = pile.make_pile(fns)
else:
fns = []
have_data_site = {}
could_have_data_site = {}
for site in sites:
have_data_site[site] = set()
could_have_data_site[site] = set()
available_through = defaultdict(set)
it = 0
nt = int(math.ceil((tmax - tmin) / tinc))
for it in range(nt):
tmin_win = tmin + it * tinc
tmax_win = min(tmin + (it + 1) * tinc, tmax)
logger.info("time window %i/%i (%s - %s)" %
(it + 1, nt, util.tts(tmin_win), util.tts(tmax_win)))
have_data_this_window = set()
if options.continue_:
trs_avail = p.all(tmin=tmin_win, tmax=tmax_win, load_data=False)
for tr in trs_avail:
have_data_this_window.add(tr.nslc_id)
for site, sx in zip(sites, sxs):
if sx is None:
continue
selection = []
channels = sx.choose_channels(
target_sample_rate=target_sample_rate,
priority_band_code=priority_band_code,
priority_units=priority_units,
priority_instrument_code=priority_instrument_code,
timespan=(tmin_win, tmax_win),
)
for nslc in sorted(channels.keys()):
if nsls_selected is not None and nslc[:3] not in nsls_selected:
continue
could_have_data_site[site].add(nslc)
if nslc not in have_data_this_window:
channel = channels[nslc]
if event:
lat_, lon_ = event.lat, event.lon
else:
lat_, lon_ = lat, lon
dist = orthodrome.distance_accurate50m_numpy(
lat_, lon_, channel.latitude.value,
channel.longitude.value)
if event:
depth_ = event.depth
time_ = event.time
else:
depth_ = None
time_ = None
tmin_, tmax_ = timewindow(time_, dist, depth_)
tmin_this = tmin_ - tpad
tmax_this = tmax_ + tpad
tmin_req = max(tmin_win, tmin_this)
tmax_req = min(tmax_win, tmax_this)
if channel.sample_rate:
deltat = 1.0 / channel.sample_rate.value
else:
deltat = 1.0
if tmin_req < tmax_req:
# extend time window by some samples because otherwise
# sometimes gaps are produced
selection.append(nslc + (tmin_req - deltat * 10.0,
tmax_req + deltat * 10.0))
if options.dry_run:
for (net, sta, loc, cha, tmin, tmax) in selection:
available_through[net, sta, loc, cha].add(site)
else:
neach = 100
i = 0
nbatches = ((len(selection) - 1) // neach) + 1
while i < len(selection):
selection_now = selection[i:i + neach]
f = tempfile.NamedTemporaryFile()
try:
sbatch = ""
if nbatches > 1:
sbatch = " (batch %i/%i)" % (
(i // neach) + 1, nbatches)
logger.info("downloading data (%s)%s" % (site, sbatch))
data = fdsn.dataselect(site=site,
selection=selection_now,
**get_user_credentials(site))
while True:
buf = data.read(1024)
if not buf:
break
f.write(buf)
f.flush()
trs = io.load(f.name)
for tr in trs:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
try:
tr.chop(tmin_win, tmax_win)
have_data.add(tr.nslc_id)
have_data_site[site].add(tr.nslc_id)
except trace.NoData:
pass
fns2 = io.save(trs, fn_template_raw)
io.save(trs, fn_template_raw_folder)
for fn in fns2:
if fn in fns:
logger.warn("overwriting file %s", fn)
fns.extend(fns2)
except fdsn.EmptyResult:
pass
except HTTPError:
logger.warn("an error occurred while downloading data "
"for channels \n %s" %
"\n ".join(".".join(x[:4])
for x in selection_now))
f.close()
i += neach
if options.dry_run:
nslcs = sorted(available_through.keys())
all_channels = defaultdict(set)
all_stations = defaultdict(set)
def plural_s(x):
return "" if x == 1 else "s"
for nslc in nslcs:
sites = tuple(sorted(available_through[nslc]))
logger.info("selected: %s.%s.%s.%s from site%s %s" %
(nslc + (plural_s(len(sites)), "+".join(sites))))
all_channels[sites].add(nslc)
all_stations[sites].add(nslc[:3])
nchannels_all = 0
nstations_all = 0
for sites in sorted(all_channels.keys(),
key=lambda sites: (-len(sites), sites)):
nchannels = len(all_channels[sites])
nstations = len(all_stations[sites])
nchannels_all += nchannels
nstations_all += nstations
logger.info("selected (%s): %i channel%s (%i station%s)" % (
"+".join(sites),
nchannels,
plural_s(nchannels),
nstations,
plural_s(nstations),
))
logger.info("selected total: %i channel%s (%i station%s)" % (
nchannels_all,
plural_s(nchannels_all),
nstations_all,
plural_s(nstations_all),
))
logger.info("dry run done.")
sys.exit(0)
for nslc in have_data:
# if we are in continue mode, we have to guess where the data came from
if not any(nslc in have_data_site[site] for site in sites):
for site in sites:
if nslc in could_have_data_site[site]:
have_data_site[site].add(nslc)
sxs = {}
for site in sites:
selection = []
for nslc in sorted(have_data_site[site]):
selection.append(nslc + (tmin - tpad, tmax + tpad))
if selection:
logger.info("downloading response information (%s)" % site)
sxs[site] = fdsn.station(site=site,
level="response",
selection=selection)
sited = site
if site == "http://192.168.11.220:8080":
sited = "bgr_internal"
elif site == "http://ws.gpi.kit.edu":
sited = "kit"
if site == "http://188.246.25.142:8080":
sited = "moer"
sxs[site].dump_xml(filename=op.join(output_dir, "stations.%s.xml" %
sited))
# chapter 1.5: inject local data
if options.local_data:
have_data_site["local"] = set()
plocal = pile.make_pile(options.local_data, fileformat="detect")
for traces in plocal.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=tmin,
tmax=tmax,
tinc=tinc):
for tr in traces:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
if tr.nslc_id not in have_data:
fns.extend(io.save(traces, fn_template_raw))
have_data_site["local"].add(tr.nslc_id)
have_data.add(tr.nslc_id)
sites.append("local")
if options.local_responses_pz:
sxs["local"] = epz.make_stationxml(
epz.iload(options.local_responses_pz))
if options.local_responses_resp:
local_stations = []
for fn in options.local_stations:
local_stations.extend(model.load_stations(fn))
sxs["local"] = resp.make_stationxml(
local_stations, resp.iload(options.local_responses_resp))
if options.local_responses_stationxml:
sxs["local"] = stationxml.load_xml(
filename=options.local_responses_stationxml)
# chapter 1.6: dump raw data stations file
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
stations = [nsl_to_station[nsl_] for nsl_ in sorted(nsl_to_station.keys())]
util.ensuredirs(fn_stations_raw)
model.dump_stations(stations, fn_stations_raw)
dump_commandline(sys.argv, fn_commandline)
# chapter 2: restitution
if not fns:
logger.error("no data available")
sys.exit(1)
p = pile.make_pile(fns, show_progress=False)
p.get_deltatmin()
otinc = None
if otinc is None:
otinc = nice_seconds_floor(p.get_deltatmin() * 500000.0)
otinc = 3600.0
otmin = math.floor(p.tmin / otinc) * otinc
otmax = math.ceil(p.tmax / otinc) * otinc
otpad = tpad * 2
fns = []
rest_traces_b = []
win_b = None
for traces_a in p.chopper_grouped(gather=lambda tr: tr.nslc_id,
tmin=otmin,
tmax=otmax,
tinc=otinc,
tpad=otpad):
rest_traces_a = []
win_a = None
for tr in traces_a:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
win_a = tr.wmin, tr.wmax
if win_b and win_b[0] >= win_a[0]:
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
rest_traces_b = []
win_b = None
response = None
failure = []
for site in sites:
try:
if site not in sxs:
continue
response = sxs[site].get_pyrocko_response(
tr.nslc_id,
timespan=(tr.tmin, tr.tmax),
fake_input_units=output_units,
)
break
except stationxml.NoResponseInformation:
failure.append("%s: no response information" % site)
except stationxml.MultipleResponseInformation:
failure.append("%s: multiple response information" % site)
if response is None:
failure = ", ".join(failure)
else:
failure = ""
try:
rest_tr = tr.transfer(tfade, ftap, response, invert=True)
rest_traces_a.append(rest_tr)
except (trace.TraceTooShort, trace.NoData):
failure = "trace too short"
if failure:
logger.warn("failed to restitute trace %s.%s.%s.%s (%s)" %
(tr.nslc_id + (failure, )))
if rest_traces_b:
rest_traces = trace.degapper(rest_traces_b + rest_traces_a,
deoverlap="crossfade_cos")
fns.extend(cut_n_dump(rest_traces, win_b, fn_template_rest))
rest_traces_a = []
if win_a:
for tr in rest_traces:
if tr.station == "7869":
tr.station = "MOER"
tr.network = "LE"
tr.location = ""
try:
rest_traces_a.append(
tr.chop(win_a[0], win_a[1] + otpad, inplace=False))
except trace.NoData:
pass
rest_traces_b = rest_traces_a
win_b = win_a
fns.extend(cut_n_dump(rest_traces_b, win_b, fn_template_rest))
# chapter 3: rotated restituted traces for inspection
if not event:
sys.exit(0)
fn_template1 = "DISPL.%(network)s.%(station)s.%(location)s.%(channel)s"
fn_waveforms = op.join(output_dir, "prepared", fn_template1)
fn_stations = op.join(output_dir, "stations.prepared.txt")
fn_event = op.join(event_dir, "event.txt")
nsl_to_station = {}
for site in sites:
if site in sxs:
stations = sxs[site].get_pyrocko_stations(timespan=(tmin, tmax))
for s in stations:
nsl = s.nsl()
if nsl not in nsl_to_station:
nsl_to_station[nsl] = s
p = pile.make_pile(fns, show_progress=False)
deltat = None
if sample_rate is not None:
deltat = 1.0 / sample_rate
used_stations = []
for nsl, s in nsl_to_station.items():
s.set_event_relative_data(event)
traces = p.all(trace_selector=lambda tr: tr.nslc_id[:3] == nsl)
keep = []
for tr in traces:
if deltat is not None:
try:
tr.downsample_to(deltat, snap=True, allow_upsample_max=5)
keep.append(tr)
except util.UnavailableDecimation as e:
logger.warn("Cannot downsample %s.%s.%s.%s: %s" %
(tr.nslc_id + (e, )))
continue
if options.out_components == "rtu":
pios = s.guess_projections_to_rtu(out_channels=("R", "T", "Z"))
elif options.out_components == "enu":
pios = s.guess_projections_to_enu(out_channels=("E", "N", "Z"))
else:
assert False
for (proj, in_channels, out_channels) in pios:
proc = trace.project(traces, proj, in_channels, out_channels)
for tr in proc:
for ch in out_channels:
if ch.name == tr.channel:
s.add_channel(ch)
if proc:
io.save(proc, fn_waveforms)
used_stations.append(s)
stations = list(used_stations)
util.ensuredirs(fn_stations)
model.dump_stations(stations, fn_stations)
model.dump_events([event], fn_event)
logger.info("prepared waveforms from %i stations" % len(stations))
except trace.TraceTooShort, e:
self.problems().add('gappy', tr.full_id)
logger.warn( '%s' % e )
continue
displacements.append(displacement)
if nsl not in out_stations:
out_stations[nsl] = copy.deepcopy(station)
out_station = out_stations[nsl]
if displacements:
if projections:
for project in projections:
matrix, in_channels, out_channels = project(out_station)
projected = trace.project(displacements, matrix, in_channels, out_channels)
displacements.extend(projected)
for tr in projected:
for ch in out_channels:
if ch.name == tr.channel:
out_station.add_channel(ch)
if rotations:
for rotate in rotations:
angle, in_channels, out_channels = rotate(out_station)
rotated = trace.rotate(displacements, angle, in_channels, out_channels)
displacements.extend(rotated)
for tr in rotated:
for ch in out_channels:
if ch.name == tr.channel:
out_station.add_channel(ch)
def get_waveform(
self,
obj, quantity='displacement',
tmin=None, tmax=None, tpad=0.,
tfade=0., freqlimits=None, deltat=None, cache=None,
backazimuth=None,
source=None,
target=None,
debug=False):
assert not debug or (debug and cache is None)
if cache is True:
cache = self._cache
_, _, _, channel = self.get_nslc(obj)
station = self.get_station(self.get_nsl(obj))
nslc = station.nsl() + (channel,)
if self.is_blacklisted(nslc):
raise NotFound(
'waveform is blacklisted', nslc)
if not self.is_whitelisted(nslc):
raise NotFound(
'waveform is not on whitelist', nslc)
if tmin is not None:
tmin = float(tmin)
if tmax is not None:
tmax = float(tmax)
if cache is not None and (nslc, tmin, tmax) in cache:
obj = cache[nslc, tmin, tmax]
if isinstance(obj, Exception):
raise obj
else:
return obj
abs_delays = []
for ocha in 'ENZRT':
sc = self.station_corrections.get(station.nsl() + (channel,), None)
if sc:
abs_delays.append(abs(sc.delay))
if abs_delays:
abs_delay_max = max(abs_delays)
else:
abs_delay_max = 0.0
projections = self._get_projections(
station, backazimuth, source, target, tmin, tmax)
try:
trs_projected = []
trs_restituted = []
trs_raw = []
for matrix, in_channels, out_channels in projections:
deps = trace.project_dependencies(
matrix, in_channels, out_channels)
trs_restituted_group = []
trs_raw_group = []
if channel in deps:
for cha in deps[channel]:
trs_restituted_this, trs_raw_this = \
self.get_waveform_restituted(
station.nsl() + (cha,),
quantity=quantity,
tmin=tmin, tmax=tmax, tpad=tpad+abs_delay_max,
toffset_noise_extract=0.0,
tfade=tfade,
freqlimits=freqlimits,
deltat=deltat,
want_incomplete=debug,
extend_incomplete=self.extend_incomplete)
trs_restituted_group.extend(trs_restituted_this)
trs_raw_group.extend(trs_raw_this)
trs_projected.extend(
trace.project(
trs_restituted_group, matrix,
in_channels, out_channels))
trs_restituted.extend(trs_restituted_group)
trs_raw.extend(trs_raw_group)
for tr in trs_projected:
sc = self.station_corrections.get(tr.nslc_id, None)
if sc:
if self.apply_correction_factors:
tr.ydata /= sc.factor
if self.apply_correction_delays:
tr.shift(-sc.delay)
if tmin is not None and tmax is not None:
tr.chop(tmin, tmax)
if cache is not None:
for tr in trs_projected:
cache[tr.nslc_id, tmin, tmax] = tr
if debug:
return trs_projected, trs_restituted, trs_raw
for tr in trs_projected:
if tr.channel == channel:
return tr
raise NotFound(
'waveform not available', station.nsl() + (channel,))
except NotFound as e:
if cache is not None:
cache[nslc, tmin, tmax] = e
raise
def kiwi_setup(options, config, event_names):
from tunguska import prepare, gfdb
conf = config['iris_pull_config']
kiwi_conf = config['kiwi_config']
if kiwi_conf.has('gfdb_path'):
db = gfdb.Gfdb(kiwi_conf.path('gfdb_path'))
deltat = db.dt
else:
if kiwi_conf.has('deltat'):
deltat = kiwi_conf.deltat
else:
deltat = None
db = None
if not event_names:
sys.exit('need event name')
for event_name in event_names:
conf.event_name = event_name
kiwi_conf.event_name = event_name
event = _get_event_infos(conf)
stations = _get_stations(conf)
for station in stations:
station.set_event_relative_data(event)
traces = _get_prepared_traces(conf, stations)
raw_traces = _get_raw_traces(conf, stations)
p = pile.Pile()
buf = pile.MemTracesFile(None, traces)
p.add_file(buf)
processed = []
for station in stations:
tt1 = kiwi_conf.cut_span[0](station.dist_m, event.depth)
tt2 = kiwi_conf.cut_span[1](station.dist_m, event.depth)
if None in (tt1, tt2):
continue
tmin = tt1 + event.time
tmax = tt2 + event.time
traces = p.all(
tmin=tmin,
tmax=tmax,
want_incomplete=False,
trace_selector=lambda tr: get_nsl(tr) == get_nsl(station))
for proj, in_channels, out_channels in station.guess_projections_to_rtu(
out_channels=('R', 'T', 'Z')):
proc = trace.project(traces, proj, in_channels, out_channels)
processed.extend(proc)
for tr in proc:
for ch in out_channels:
if ch.name == tr.channel:
station.add_channel(ch)
for tr in processed:
if deltat is not None:
try:
tr.downsample_to(deltat, snap=True, allow_upsample_max=5)
except util.UnavailableDecimation, e:
logger.warn('Cannot downsample %s.%s.%s.%s: %s' %
(tr.nslc_id + (e, )))
continue
stations_by_nsl = dict((get_nsl(s), s) for s in stations)
acc = DummyAcc(raw_traces)
prepare.save_kiwi_dataset(acc, stations_by_nsl, processed, event,
kiwi_conf)
