class A(Object):
m = List.T(Int.T())
xmltagname = 'a'
class X(Object):
y = Int.T(default=1, xmlstyle='attribute')
z = List.T(Int.T())
a = Int.T(xmlstyle='content')
class A(Object):
x = Any.T()
lst = List.T(Any.T())
class Polarity_switch(Object):
switched_by_stat = List.T(Event_sw.T())
class Y(Object):
xs = List.T(X.T(), default=[X(a=2)])
class WaveformGenerator(TargetGenerator):
station_generator = StationGenerator.T(
default=RandomStationGenerator.D(),
help='The StationGenerator for creating the stations.')
noise_generator = WaveformNoiseGenerator.T(
default=WhiteNoiseGenerator.D(),
help='Add Synthetic noise on the waveforms.')
store_id = gf.StringID.T(
default=DEFAULT_STORE_ID,
help='The GF store to use for forward-calculations.')
seismogram_quantity = StringChoice.T(
choices=['displacement', 'velocity', 'acceleration', 'counts'],
default='displacement')
vmin_cut = Float.T(
default=2000.,
help='Minimum velocity to seismic velicty to consider in the model.')
vmax_cut = Float.T(
default=8000.,
help='Maximum velocity to seismic velicty to consider in the model.')
fmin = Float.T(default=0.01,
help='Minimum frequency/wavelength to resolve in the'
' synthetic waveforms.')
tabulated_phases = List.T(gf.meta.TPDef.T(),
optional=True,
help='Define seismic phases to be calculated.')
taper = trace.Taper.T(
optional=True,
help='Time domain taper applied to synthetic waveforms.')
compensate_synthetic_offsets = Bool.T(
default=False,
help='Center synthetic trace amplitudes using mean of waveform tips.')
tinc = Float.T(optional=True, help='Time increment of waveforms.')
continuous = Bool.T(default=True,
help='Only produce traces that intersect with events.')
def __init__(self, *args, **kwargs):
super(WaveformGenerator, self).__init__(*args, **kwargs)
self._targets = []
def get_stations(self):
return self.station_generator.get_stations()
def get_targets(self):
if self._targets:
return self._targets
for station in self.get_stations():
channel_data = []
channels = station.get_channels()
if channels:
for channel in channels:
channel_data.append(
[channel.name, channel.azimuth, channel.dip])
else:
for c_name in ['BHZ', 'BHE', 'BHN']:
channel_data.append([
c_name,
model.guess_azimuth_from_name(c_name),
model.guess_dip_from_name(c_name)
])
for c_name, c_azi, c_dip in channel_data:
target = gf.Target(codes=(station.network, station.station,
station.location, c_name),
quantity='displacement',
lat=station.lat,
lon=station.lon,
depth=station.depth,
store_id=self.store_id,
optimization='enable',
interpolation='nearest_neighbor',
azimuth=c_azi,
dip=c_dip)
self._targets.append(target)
return self._targets
def get_time_range(self, sources):
dmin, dmax = self.station_generator.get_distance_range(sources)
times = num.array([source.time for source in sources], dtype=num.float)
tmin_events = num.min(times)
tmax_events = num.max(times)
tmin = tmin_events + dmin / self.vmax_cut - 10.0 / self.fmin
tmax = tmax_events + dmax / self.vmin_cut + 10.0 / self.fmin
return tmin, tmax
def get_codes_to_deltat(self, engine, sources):
deltats = {}
targets = self.get_targets()
for source in sources:
for target in targets:
deltats[target.codes] = engine.get_store(
target.store_id).config.deltat
return deltats
def get_useful_time_increment(self, engine, sources):
_, dmax = self.station_generator.get_distance_range(sources)
tinc = dmax / self.vmin_cut + 2.0 / self.fmin
deltats = set(self.get_codes_to_deltat(engine, sources).values())
deltat = reduce(util.lcm, deltats)
tinc = int(round(tinc / deltat)) * deltat
return tinc
def get_relevant_sources(self, sources, tmin, tmax):
dmin, dmax = self.station_generator.get_distance_range(sources)
trange = tmax - tmin
tmax_pad = trange + tmax + dmin / self.vmax_cut
tmin_pad = tmin - (dmax / self.vmin_cut + trange)
return [s for s in sources if s.time < tmax_pad and s.time > tmin_pad]
def get_waveforms(self, engine, sources, tmin, tmax):
sources_relevant = self.get_relevant_sources(sources, tmin, tmax)
if not (self.continuous or sources_relevant):
return []
trs = {}
tts = util.time_to_str
for nslc, deltat in self.get_codes_to_deltat(engine, sources).items():
tr_tmin = int(round(tmin / deltat)) * deltat
tr_tmax = (int(round(tmax / deltat)) - 1) * deltat
nsamples = int(round((tr_tmax - tr_tmin) / deltat)) + 1
tr = trace.Trace(*nslc,
tmin=tr_tmin,
ydata=num.zeros(nsamples),
deltat=deltat)
self.noise_generator.add_noise(tr)
trs[nslc] = tr
logger.debug('Forward modelling waveforms between %s - %s...' %
(tts(tmin, format='%Y-%m-%d_%H-%M-%S'),
tts(tmax, format='%Y-%m-%d_%H-%M-%S')))
if not sources_relevant:
return list(trs.values())
targets = self.get_targets()
response = engine.process(sources_relevant, targets)
for source, target, res in response.iter_results(get='results'):
if isinstance(res, gf.SeismosizerError):
logger.warning(
'Out of bounds! \nTarget: %s\nSource: %s\n' %
('.'.join(target.codes)), source)
continue
tr = res.trace.pyrocko_trace()
candidate = trs[target.codes]
if not candidate.overlaps(tr.tmin, tr.tmax):
continue
if self.compensate_synthetic_offsets:
tr.ydata -= (num.mean(tr.ydata[-3:-1]) +
num.mean(tr.ydata[1:3])) / 2.
if self.taper:
tr.taper(self.taper)
resp = self.get_transfer_function(target.codes)
if resp:
tr = tr.transfer(transfer_function=resp)
candidate.add(tr)
trs[target.codes] = candidate
return list(trs.values())
def get_onsets(self, engine, sources, *args, **kwargs):
if not self.tabulated_phases:
return []
targets = {t.codes[:3]: t for t in self.get_targets()}
phase_markers = []
for nsl, target in targets.items():
store = engine.get_store(target.store_id)
for source in sources:
d = source.distance_to(target)
for phase in self.tabulated_phases:
t = store.t(phase.definition, (source.depth, d))
if not t:
continue
t += source.time
phase_markers.append(
PhaseMarker(phasename=phase.id,
tmin=t,
tmax=t,
event=source.pyrocko_event(),
nslc_ids=(nsl + ('*', ), )))
return phase_markers
def get_transfer_function(self, codes):
if self.seismogram_quantity == 'displacement':
return None
elif self.seismogram_quantity == 'velocity':
return trace.DifferentiationResponse(1)
elif self.seismogram_quantity == 'acceleration':
return trace.DifferentiationResponse(2)
elif self.seismogram_quantity == 'counts':
raise NotImplemented()
def dump_data(self,
engine,
sources,
path,
tmin=None,
tmax=None,
overwrite=False):
fns = []
fns.extend(
self.dump_waveforms(engine, sources, path, tmin, tmax, overwrite))
fns.extend(self.dump_responses(path))
return fns
def dump_waveforms(self,
engine,
sources,
path,
tmin=None,
tmax=None,
overwrite=False):
path_waveforms = op.join(path, 'waveforms')
# gf.store.remake_dir(path_waveforms, force=overwrite)
path_traces = op.join(
path_waveforms, '%(wmin_year)s', '%(wmin_month)s', '%(wmin_day)s',
'waveform_%(network)s_%(station)s_' +
'%(location)s_%(channel)s_%(tmin)s_%(tmax)s.mseed')
tmin_all, tmax_all = self.get_time_range(sources)
tmin = tmin if tmin is not None else tmin_all
tmax = tmax if tmax is not None else tmax_all
tts = util.time_to_str
tinc = self.tinc or self.get_useful_time_increment(engine, sources)
tmin = math.floor(tmin / tinc) * tinc
tmax = math.ceil(tmax / tinc) * tinc
nwin = int(round((tmax - tmin) / tinc))
pbar = util.progressbar('Generating waveforms', nwin)
for iwin in range(nwin):
pbar.update(iwin)
tmin_win = max(tmin, tmin + iwin * tinc)
tmax_win = min(tmax, tmin + (iwin + 1) * tinc)
if tmax_win <= tmin_win:
continue
trs = self.get_waveforms(engine, sources, tmin_win, tmax_win)
try:
io.save(trs,
path_traces,
additional=dict(wmin_year=tts(tmin_win, format='%Y'),
wmin_month=tts(tmin_win, format='%m'),
wmin_day=tts(tmin_win, format='%d'),
wmin=tts(tmin_win,
format='%Y-%m-%d_%H-%M-%S'),
wmax_year=tts(tmax_win, format='%Y'),
wmax_month=tts(tmax_win, format='%m'),
wmax_day=tts(tmax_win, format='%d'),
wmax=tts(tmax_win,
format='%Y-%m-%d_%H-%M-%S')),
overwrite=overwrite)
except FileSaveError as e:
logger.debug('Waveform exists %s' % e)
pbar.finish()
return [path_waveforms]
def dump_responses(self, path):
from pyrocko.io import stationxml
logger.debug('Writing out StationXML...')
path_responses = op.join(path, 'meta')
util.ensuredir(path_responses)
fn_stationxml = op.join(path_responses, 'stations.xml')
stations = self.station_generator.get_stations()
sxml = stationxml.FDSNStationXML.from_pyrocko_stations(stations)
sunit = {
'displacement': 'M',
'velocity': 'M/S',
'acceleration': 'M/S**2',
'counts': 'COUNTS'
}[self.seismogram_quantity]
response = stationxml.Response(
instrument_sensitivity=stationxml.Sensitivity(
value=1.,
frequency=1.,
input_units=stationxml.Units(sunit),
output_units=stationxml.Units('COUNTS')),
stage_list=[])
for net, station, channel in sxml.iter_network_station_channels():
channel.response = response
sxml.dump_xml(filename=fn_stationxml)
return [path_responses]
def add_map_artists(self, engine, sources, automap):
automap.add_stations(self.get_stations())
class dict_stats_all_rota(Object):
dict_stats_all = List.T(rota_ev_by_stat.T())
class ResultStats(Object):
problem = Problem.T()
parameter_stats_list = List.T(ParameterStats.T())
class DataGeneratorBase(Object):
'''This is the base class for all generators.
This class to dump and load data to/from all subclasses into
TFRecordDatasets.
'''
fn_tfrecord = String.T(optional=True)
noise = Noise.T(optional=True, help='Add noise to feature')
station_dropout_rate = Float.T(default=0.,
help='Rate by which to mask all channels of station')
station_dropout_distribution = Bool.T(default=True,
help='If *true*, station dropout will be drawn from a uniform '
'distribution limited by this station_dropout.')
nmax = Int.T(optional=True)
labeled = Bool.T(default=True)
blacklist = List.T(optional=True, help='List of indices to ignore.')
random_seed = Int.T(default=0)
def __init__(self, *args, **kwargs):
self.config = kwargs.pop('config', None)
super(DataGeneratorBase, self).__init__(*args, **kwargs)
self.blacklist = set() if not self.blacklist else set(self.blacklist)
self.evolution = 0
def normalize_label(self, label):
if self.labeled:
return self.config.normalize_label(label)
return label
def set_config(self, pinky_config):
self.config = pinky_config
self.setup()
def setup(self):
...
def reset(self):
self.evolution = 0
@property
def tensor_shape(self):
return self.config.tensor_shape
@property
def n_samples(self):
return self.config._n_samples
@n_samples.setter
def n_samples(self, v):
self.config._n_samples = v
@property
@lru_cache(maxsize=1)
def nsl_to_indices(self):
''' Returns a dictionary which maps nsl codes to indexing arrays.'''
indices = OrderedDict()
for nslc, index in self.nslc_to_index.items():
key = nslc[:3]
_v = indices.get(key, [])
_v.append(index)
indices[key] = _v
for k in indices.keys():
indices[k] = num.array(indices[k])
return indices
@property
@lru_cache(maxsize=1)
def nsl_indices(self):
''' Returns a 2D array of indices of channels belonging to one station.'''
return [v for v in self.nsl_to_indices.values()]
@property
def nslc_to_index(self):
''' Returns a dictionary which maps nslc codes to trace indices.'''
d = OrderedDict()
idx = 0
for nslc in self.config.channels:
if not util.match_nslc(self.config.blacklist, nslc):
d[nslc] = idx
idx += 1
return d
def reject_blacklisted(self, tr):
'''returns `False` if nslc codes of `tr` match any of the blacklisting
patters. Otherwise returns `True`'''
return not util.match_nslc(self.config.blacklist, tr.nslc_id)
def filter_iter(self, iterator):
'''Apply *blacklist*ing by example indices
:param iterator: producing iterator
'''
for i, item in enumerate(iterator):
if i not in self.blacklist:
yield i, item
@property
def generate_output_types(self):
'''Return data types of features and labels'''
return tf.float32, tf.float32
def unpack_examples(self, record_iterator):
'''Parse examples stored in TFRecordData to `tf.train.Example`'''
for string_record in record_iterator:
example = tf.train.Example()
example.ParseFromString(string_record)
chunk = example.features.feature['data'].bytes_list.value[0]
label = example.features.feature['label'].bytes_list.value[0]
chunk = num.fromstring(chunk, dtype=num.float32)
chunk = chunk.reshape((self.config.n_channels, -1))
label = num.fromstring(label, dtype=num.float32)
yield chunk, label
@property
def tstart_data(self):
return None
def iter_chunked(self, tinc):
# if data has been written to tf records:
return self.iter_examples_and_labels()
def iter_examples_and_labels(self):
'''Subclass this method!
Yields: feature, label
Chunks that are all NAN will be skipped.
'''
record_iterator = tf.python_io.tf_record_iterator(
path=self.fn_tfrecord)
for chunk, label in self.unpack_examples(record_iterator):
if all_NAN(chunk):
logger.debug('all NAN. skipping...')
continue
yield chunk, label
def generate_chunked(self, tinc=1):
'''Takes the output of `iter_examples_and_labels` and applies post
processing (see: `process_chunk`).
'''
for i, (chunk, label) in self.filter_iter(self.iter_chunked(tinc)):
yield self.process_chunk(chunk), self.normalize_label(label)
def generate(self, return_gaps=False):
'''Takes the output of `iter_examples_and_labels` and applies post
processing (see: `process_chunk`).
'''
self.evolution += 1
num.random.seed(self.random_seed + self.evolution)
for i, (chunk, label) in self.filter_iter(
self.iter_examples_and_labels()):
yield self.process_chunk(chunk, return_gaps=return_gaps), self.normalize_label(label)
def extract_labels(self):
'''Overwrite this method!'''
if not self.labeled:
return UNLABELED
def iter_labels(self):
'''Iterate through labels.'''
for i, (_, label) in self.filter_iter(
self.iter_examples_and_labels()):
yield label
@property
def text_labels(self):
'''Returns a list of strings to identify the labels.
Overwrite this method for more meaningfull identifiers.'''
return ['%i' % (i) for i, d in
self.filter_iter(self.iter_examples_and_labels())]
def gaps(self):
'''Returns a list containing the gaps of each example'''
gaps = []
for (_, gap), _ in self.generate(return_gaps=True):
gaps.append(gap)
return gaps
def snrs(self, split_factor):
snrs = []
for chunk, _ in self.generate():
snrs.append(snr(chunk, split_factor))
return snrs
@property
def output_shapes(self):
return (self.config.output_shapes)
def get_dataset(self):
return tf.data.Dataset.from_generator(
self.generate,
self.generate_output_types,
output_shapes=self.output_shapes)
def get_chunked_dataset(self, tinc=1.):
gen = partial(self.generate_chunked, tinc=tinc)
return tf.data.Dataset.from_generator(
gen,
self.generate_output_types,
output_shapes=self.output_shapes)
def get_raw_data_chunk(self, shape):
'''Return an array of size (Nchannels x Nsamples_max) filled with
NANs.'''
empty_array = num.empty(shape, dtype=num.float32)
empty_array.fill(num.nan)
return empty_array
def pack_examples(self):
'''Serialize Examples to strings.'''
for ydata, label in self.iter_examples_and_labels():
yield tf.train.Example(
features=tf.train.Features(
feature={
'data': _BytesFeature(ydata.tobytes()),
'label': _BytesFeature(num.array(
label, dtype=num.float32).tobytes()),
}))
def mask(self, chunk, rate):
'''For data augmentation: Mask traces in chunks with NaNs.
NaNs will be filled by the imputation method provided by the config
file.
:param rate: probability with which traces are NaN-ed
'''
# print(rate)
indices = self.nsl_indices
a = num.random.random(len(indices))
i = num.where(a < rate)[0]
for ii in i:
chunk[indices[ii], :] = num.nan
def random_trim(self, chunk, margin):
'''For data augmentation: Randomly trim examples in time domain with
*margin* seconds.'''
sample_margin = int(margin / self.config.effective_deltat)
nstart = num.random.randint(low=0, high=sample_margin)
_, n_samples = self.config.tensor_shape
nstop = nstart + n_samples
chunk[:, :nstart] = 0.
chunk[:, nstop:] = 0.
def process_chunk(self, chunk, return_gaps=False):
'''Performs preprocessing of data chunks.'''
if self.config.t_translation_max:
self.random_trim(chunk, self.config.t_translation_max)
# add noise
if self.noise:
self.noise(chunk)
# apply normalization
self.config.normalization(chunk)
# apply station dropout
if self.station_dropout_rate:
if self.station_dropout_distribution:
self.mask(chunk, num.random.uniform(
high=self.station_dropout_rate))
else:
self.mask(chunk, self.station_dropout_rate)
# fill gaps
if self.config.imputation:
gaps = num.isnan(chunk)
chunk[gaps] = self.config.imputation(chunk)
if not return_gaps:
return chunk
else:
return chunk, gaps
def write(self, directory):
'''Write example data to TFRecordDataset using `self.writer`.'''
logger.debug('writing TFRecordDataset: %s' % directory)
writer = tf.python_io.TFRecordWriter(directory)
for ex in self.pack_examples():
writer.write(ex.SerializeToString())
def cleanup(self):
'''Remove remaining folders'''
delete_if_exists(self.fn_tfrecord)
class Config(HasPaths):
stations_path = Path.T(optional=True,
help='stations file in Pyrocko format')
stations_stationxml_path = Path.T(
optional=True, help='stations file in StationXML format')
receivers = List.T(receiver.Receiver.T(),
help='receiver coordinates if not read from file')
data_paths = List.T(Path.T(),
help='list of directories paths to search for data')
events_path = Path.T(
optional=True,
help='limit processing to time windows around given events')
event_time_window_factor = Float.T(
default=2.,
help='controls length of time windows for event-wise processing')
blacklist = List.T(
String.T(),
help='codes in the form NET.STA.LOC of receivers to be excluded')
whitelist = List.T(
String.T(),
help='codes in the form NET.STA.LOC of receivers to be included')
distance_max = Float.T(optional=True,
help='receiver maximum distance from grid')
tmin = Timestamp.T(optional=True,
help='beginning of time interval to be processed')
tmax = Timestamp.T(optional=True,
help='end of time interval to be processed')
run_path = Path.T(optional=True, help='output is saved to this directory')
save_figures = Bool.T(default=False,
help='flag to activate saving of detection figures')
grid = grid.Grid.T(
optional=True,
help='definition of search grid, if not given, a default grid is '
'chosen')
autogrid_radius_factor = Float.T(
default=1.5,
help='size factor to use when automatically choosing a grid size')
autogrid_density_factor = Float.T(
default=10.0,
help='grid density factor used when automatically choosing a grid '
'spacing')
image_function_contributions = List.T(ifc.IFC.T(),
help='image function contributions')
sharpness_normalization = Bool.T(
default=True,
help='whether to divide image function frames by their mean value')
ifc_count_normalization = Bool.T(
default=False,
help='whether to divide image function by number of contributors')
detector_threshold = Float.T(default=70.,
help='threshold on detector function')
detector_tpeaksearch = Float.T(optional=True,
help='search time span for peak detection')
fill_incomplete_with_zeros = Bool.T(
default=True,
help='fill incomplete trace time windows with zeros '
'(and let edge effects ruin your day)')
earthmodels = List.T(Earthmodel.T(),
help='list of earthmodels usable in shifters')
tabulated_phases = List.T(
TPDef.T(), help='list of tabulated phase definitions usable shifters')
cache_path = Path.T(
default='lassie_phases.cache',
help='directory where lassie stores tabulated phases etc.')
stacking_blocksize = Int.T(
optional=True,
help='enable chunked stacking to reduce memory usage. Setting this to '
'e.g. 64 will use ngridpoints * 64 * 8 bytes of memory to hold '
'the stacking results, instead of computing the whole processing '
'time window in one shot. Setting this to a very small number '
'may lead to bad performance. If this is enabled together with '
'plotting, the cutout of the image function seen in the map '
'image must be stacked again just for plotting (redundantly and '
'memory greedy) because it may intersect more than one '
'processing chunk.')
def __init__(self, *args, **kwargs):
HasPaths.__init__(self, *args, **kwargs)
self._receivers = None
self._grid = None
self._events = None
self._config_name = 'untitled'
def setup_image_function_contributions(self):
'''
Post-init setup of image function contributors.
'''
for ifc_ in self.image_function_contributions:
ifc_.setup(self)
def set_config_name(self, config_name):
self._config_name = config_name
def expand_path(self, path):
def extra(path):
return expand_template(path, dict(config_name=self._config_name))
return HasPaths.expand_path(self, path, extra=extra)
def get_events_path(self):
run_path = self.expand_path(self.run_path)
return op.join(run_path, 'events.list')
def get_ifm_dir_path(self):
run_path = self.expand_path(self.run_path)
return op.join(run_path, 'ifm')
def get_ifm_path_template(self):
return op.join(self.get_ifm_dir_path(),
'%(station)s_%(tmin_ms)s.mseed')
def get_detections_path(self):
run_path = self.expand_path(self.run_path)
return op.join(run_path, 'detections.list')
def get_figures_path_template(self):
run_path = self.expand_path(self.run_path)
return op.join(run_path, 'figures', '%(id)s.%(format)s')
def get_receivers(self):
'''Aggregate receivers from different sources.'''
fp = self.expand_path
if self._receivers is None:
self._receivers = list(self.receivers)
if self.stations_path:
for station in model.load_stations(fp(self.stations_path)):
self._receivers.append(
receiver.Receiver(codes=station.nsl(),
lat=station.lat,
lon=station.lon,
z=station.depth))
if self.stations_stationxml_path:
sx = stationxml.load_xml(
filename=fp(self.stations_stationxml_path))
for station in sx.get_pyrocko_stations():
self._receivers.append(
receiver.Receiver(codes=station.nsl(),
lat=station.lat,
lon=station.lon,
z=station.depth))
return self._receivers
def get_events(self):
if self.events_path is None:
return None
if self._events is None:
self._events = model.load_events(self.expand_path(
self.events_path))
return self._events
def get_grid(self):
'''Get grid or make default grid.'''
self.setup_image_function_contributions()
if self._grid is None:
if not self.grid:
receivers = self.get_receivers()
fsmooth_max = max(ifc.get_fsmooth()
for ifc in self.image_function_contributions)
vmin = min(ifc.shifter.get_vmin()
for ifc in self.image_function_contributions)
spacing = vmin / fsmooth_max / self.autogrid_density_factor
lat0, lon0, north, east, depth = geo.bounding_box_square(
*geo.points_coords(receivers),
scale=self.autogrid_radius_factor)
self._grid = grid.Carthesian3DGrid(lat=lat0,
lon=lon0,
xmin=north[0],
xmax=north[1],
dx=spacing,
ymin=east[0],
ymax=east[1],
dy=spacing,
zmin=depth[0],
zmax=depth[1],
dz=spacing)
logger.info('automatic grid:\n%s' % self._grid)
else:
self._grid = self.grid
self._grid.update()
return self._grid
class QSSPConfigFull(QSSPConfig):
time_window = Float.T(default=900.0)
receiver_depth = Float.T(default=0.0)
sampling_interval = Float.T(default=5.0)
output_filename = String.T(default='receivers')
output_format = Int.T(default=1)
output_time_window = Float.T(optional=True)
gf_directory = String.T(default='qssp_green')
greens_functions = List.T(QSSPGreen.T())
sources = List.T(QSSPSource.T())
receivers = List.T(QSSPReceiver.T())
earthmodel_1d = gf.meta.Earthmodel1D.T(optional=True)
@staticmethod
def example():
conf = QSSPConfigFull()
conf.sources.append(QSSPSourceMT())
lats = [20.]
conf.receivers.extend(QSSPReceiver(lat=lat) for lat in lats)
conf.greens_functions.append(QSSPGreen())
return conf
@property
def components(self):
return qssp_components[self.output_format]
def get_output_filenames(self, rundir):
return [
pjoin(rundir, self.output_filename+'.'+c) for c in self.components]
def ensure_gf_directory(self):
util.ensuredir(self.gf_directory)
def string_for_config(self):
def aggregate(l):
return len(l), '\n'.join(x.string_for_config() for x in l)
assert len(self.greens_functions) > 0
assert len(self.sources) > 0
assert len(self.receivers) > 0
d = self.__dict__.copy()
if self.output_time_window is None:
d['output_time_window'] = self.time_window
if self.output_slowness_max is None:
d['output_slowness_max'] = self.slowness_max
if self.frequency_max is None:
d['frequency_max'] = 0.5/self.sampling_interval
d['gf_directory'] = os.path.abspath(self.gf_directory) + '/'
d['n_receiver_lines'], d['receiver_lines'] = aggregate(self.receivers)
d['n_source_lines'], d['source_lines'] = aggregate(self.sources)
d['n_gf_lines'], d['gf_lines'] = aggregate(self.greens_functions)
model_str, nlines = cake_model_to_config(self.earthmodel_1d)
d['n_model_lines'] = nlines
d['model_lines'] = model_str
if len(self.sources) == 0 or isinstance(self.sources[0], QSSPSourceMT):
d['point_source_type'] = 1
else:
d['point_source_type'] = 2
if self.qssp_version == '2010beta':
d['scutoff_doc'] = '''
# (SH waves), and cutoff harmonic degree for static deformation
'''.strip()
d['scutoff'] = '%i' % self.cutoff_harmonic_degree_sd
d['sfilter_doc'] = '''
# 3. selection of order of Butterworth low-pass filter (if <= 0, then no
# filtering), corner frequency (smaller than the cut-off frequency defined
# above)
'''.strip()
if self.bandpass_order != 0:
raise QSSPError(
'this version of qssp does not support bandpass '
'settings, use lowpass instead')
d['sfilter'] = '%i %f' % (
self.lowpass_order,
self.lowpass_corner)
elif self.qssp_version == '2010':
d['scutoff_doc'] = '''
# (SH waves), minimum and maximum cutoff harmonic degrees
# Note: if the near-field static displacement is desired, the minimum
# cutoff harmonic degree should not be smaller than, e.g., 2000.
'''.strip()
d['scutoff'] = '%i %i' % (
self.cutoff_harmonic_degree_min,
self.cutoff_harmonic_degree_max)
d['sfilter_doc'] = '''
# 3. selection of order of Butterworth bandpass filter (if <= 0, then no
# filtering), lower and upper corner frequencies (smaller than the cut-off
# frequency defined above)
'''.strip()
if self.lowpass_order != 0:
raise QSSPError(
'this version of qssp does not support lowpass settings, '
'use bandpass instead')
d['sfilter'] = '%i %f %f' % (
self.bandpass_order,
self.bandpass_corner_low,
self.bandpass_corner_high)
template = '''# autogenerated QSSP input by qssp.py
#
# This is the input file of FORTRAN77 program "qssp2010" for calculating
# synthetic seismograms of a self-gravitating, spherically symmetric,
# isotropic and viscoelastic earth.
#
# by
# Rongjiang Wang <*****@*****.**>
# Helmholtz-Centre Potsdam
# GFZ German Reseach Centre for Geosciences
# Telegrafenberg, D-14473 Potsdam, Germany
#
# Last modified: Potsdam, July, 2010
#
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
# If not specified, SI Unit System is used overall!
#
# Coordinate systems:
# spherical (r,t,p) with r = radial,
# t = co-latitude,
# p = east longitude.
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
#
# UNIFORM RECEIVER DEPTH
# ======================
# 1. uniform receiver depth [km]
#-------------------------------------------------------------------------------------------
%(receiver_depth)e
#-------------------------------------------------------------------------------------------
#
# TIME (FREQUENCY) SAMPLING
# =========================
# 1. time window [sec], sampling interval [sec]
# 2. max. frequency [Hz] of Green's functions
# 3. max. slowness [s/km] of Green's functions
# Note: if the near-field static displacement is desired, the maximum slowness should not
# be smaller than the S wave slowness in the receiver layer
# 4. anti-aliasing factor (> 0 & < 1), if it is <= 0 or >= 1/e (~ 0.4), then
# default value of 1/e is used (e.g., 0.1 = alias phases will be suppressed
# to 10%% of their original amplitude)
#
# Note: The computation effort increases linearly the time window and
# quadratically with the cut-off frequency.
#-------------------------------------------------------------------------------------------
%(time_window)e %(sampling_interval)e
%(frequency_max)e
%(slowness_max)e
%(antialiasing_factor)e
#-------------------------------------------------------------------------------------------
#
# SELF-GRAVITATING EFFECT
# =======================
# 1. the critical frequency [Hz] and the critical harmonic degree, below which
# the self-gravitating effect should be included
#-------------------------------------------------------------------------------------------
%(crit_frequency_sge)e %(crit_harmonic_degree_sge)i
#-------------------------------------------------------------------------------------------
#
# WAVE TYPES
# ==========
# 1. selection (1/0 = yes/no) of speroidal modes (P-SV waves), selection of toroidal modes
%(scutoff_doc)s
#-------------------------------------------------------------------------------------------
%(spheroidal_modes)i %(toroidal_modes)i %(scutoff)s
#-------------------------------------------------------------------------------------------
# GREEN'S FUNCTION FILES
# ======================
# 1. number of discrete source depths, estimated radius of each source patch [km] and
# directory for Green's functions
# 2. list of the source depths [km], the respective file names of the Green's
# functions (spectra) and the switch number (0/1) (0 = do not calculate
# this Green's function because it exists already, 1 = calculate or update
# this Green's function. Note: update is required if any of the above
# parameters is changed)
#-------------------------------------------------------------------------------------------
%(n_gf_lines)i %(source_patch_radius)e '%(gf_directory)s'
%(gf_lines)s
#-------------------------------------------------------------------------------------------
#
# MULTI-EVENT SOURCE PARAMETERS
# =============================
# 1. number of discrete point sources and selection of the source data format
# (1 or 2)
# 2. list of the multi-event sources
# Format 1:
# M-Unit Mrr Mtt Mpp Mrt Mrp Mtp Lat Lon Depth T_origin T_rise
# [Nm] [deg] [deg] [km] [sec] [sec]
# Format 2:
# Moment Strike Dip Rake Lat Lon Depth T_origin T_rise
# [Nm] [deg] [deg] [deg] [deg] [deg] [km] [sec] [sec]
#-------------------------------------------------------------------------------------------
%(n_source_lines)i %(point_source_type)i
%(source_lines)s
#-------------------------------------------------------------------------------------------
#
# RECEIVER PARAMETERS
# ===================
# 1. output file name and and selection of output format:
# 1 = cartesian: vertical(z)/north(n)/east(e);
# 2 = spherical: radial(r)/theta(t)/phi(p)
# (Note: if output format 2 is selected, the epicenter (T_origin = 0)
# 2. output time window [sec] (<= Green's function time window)
%(sfilter_doc)s
# 4. lower and upper slowness cut-off [s/km] (slowness band-pass filter)
# 5. number of receiver
# 6. list of the station parameters
# Format:
# Lat Lon Name Time_reduction
# [deg] [deg] [sec]
# (Note: Time_reduction = start time of the time window)
#-------------------------------------------------------------------------------------------
'%(output_filename)s' %(output_format)i
%(output_time_window)e
%(sfilter)s
%(output_slowness_min)e %(output_slowness_max)e
%(n_receiver_lines)i
%(receiver_lines)s
#-------------------------------------------------------------------------------------------
#
# LAYERED EARTH MODEL (IASP91)
# ============================
# 1. number of data lines of the layered model and selection for including
# the physical dispersion according Kamamori & Anderson (1977)
#-------------------------------------------------------------------------------------------
%(n_model_lines)i %(include_physical_dispersion)i
#-------------------------------------------------------------------------------------------
#
# MULTILAYERED MODEL PARAMETERS (source site)
# ===========================================
# no depth[km] vp[km/s] vs[km/s] ro[g/cm^3] qp qs
#-------------------------------------------------------------------------------------------
%(model_lines)s
#---------------------------------end of all inputs-----------------------------------------
''' # noqa
return (template % d).encode('ascii')
class DatasetConfig(HasPaths):
''' Configuration for a Grond `Dataset` object. '''
stations_path = Path.T(
optional=True,
help='List of files with station coordinates in Pyrocko format.')
stations_stationxml_paths = List.T(
Path.T(),
optional=True,
help='List of files with station coordinates in StationXML format.')
events_path = Path.T(optional=True,
help='File with hypocenter information and possibly'
' reference solution')
waveform_paths = List.T(Path.T(),
optional=True,
help='List of directories with raw waveform data')
clippings_path = Path.T(optional=True)
responses_sacpz_path = Path.T(
optional=True,
help='List of SACPZ response files for restitution of'
' the raw waveform data.')
responses_stationxml_paths = List.T(
Path.T(),
optional=True,
help='List of StationXML response files for restitution of'
' the raw waveform data.')
station_corrections_path = Path.T(
optional=True, help='File containing station correction informations.')
apply_correction_factors = Bool.T(
optional=True,
default=True,
help='Apply correction factors from station corrections.')
apply_correction_delays = Bool.T(
optional=True,
default=True,
help='Apply correction delays from station corrections.')
apply_displaced_sampling_workaround = Bool.T(
optional=True,
default=False,
help='Work around displaced sampling issues.')
extend_incomplete = Bool.T(default=False,
help='Extend incomplete seismic traces.')
picks_paths = List.T(Path.T())
blacklist_paths = List.T(
Path.T(), help='List of text files with blacklisted stations.')
blacklist = List.T(
String.T(),
help='Stations/components to be excluded according to their STA, '
'NET.STA, NET.STA.LOC, or NET.STA.LOC.CHA codes.')
whitelist_paths = List.T(
Path.T(), help='List of text files with whitelisted stations.')
whitelist = List.T(
String.T(),
optional=True,
help='If not None, list of stations/components to include according '
'to their STA, NET.STA, NET.STA.LOC, or NET.STA.LOC.CHA codes. '
'Note: '
'when whitelisting on channel level, both, the raw and '
'the processed channel codes have to be listed.')
synthetic_test = SyntheticTest.T(optional=True)
kite_scene_paths = List.T(Path.T(),
optional=True,
help='List of directories for the InSAR scenes.')
gnss_campaign_paths = List.T(
Path.T(),
optional=True,
help='List of directories for the GNSS campaign data.')
def __init__(self, *args, **kwargs):
HasPaths.__init__(self, *args, **kwargs)
self._ds = {}
def get_event_names(self):
logger.info('Loading events ...')
def extra(path):
return expand_template(path, dict(event_name='*'))
def fp(path):
return self.expand_path(path, extra=extra)
def check_events(events, fn):
for ev in events:
if not ev.name:
logger.warning('Event in %s has no name!', fn)
return
if not ev.lat or not ev.lon:
logger.warning('Event %s has inconsistent coordinates!',
ev.name)
if not ev.depth:
logger.warning('Event %s has no depth!', ev.name)
if not ev.time:
logger.warning('Event %s has no time!', ev.name)
events = []
events_path = fp(self.events_path)
fns = glob.glob(events_path)
if not fns:
raise DatasetError('No event files matching "%s".' % events_path)
for fn in fns:
logger.debug('Loading from file %s' % fn)
ev = model.load_events(filename=fn)
check_events(ev, fn)
events.extend(ev)
event_names = [ev_.name for ev_ in events]
event_names.sort()
return event_names
def get_dataset(self, event_name):
if event_name not in self._ds:
def extra(path):
return expand_template(path, dict(event_name=event_name))
def fp(path):
p = self.expand_path(path, extra=extra)
if p is None:
return None
if isinstance(p, list):
for path in p:
if not op.exists(path):
logger.warn('Path %s does not exist.' % path)
else:
if not op.exists(p):
logger.warn('Path %s does not exist.' % p)
return p
ds = Dataset(event_name)
try:
ds.add_events(filename=fp(self.events_path))
ds.add_stations(
pyrocko_stations_filename=fp(self.stations_path),
stationxml_filenames=fp(self.stations_stationxml_paths))
if self.waveform_paths:
ds.add_waveforms(paths=fp(self.waveform_paths))
if self.kite_scene_paths:
ds.add_kite_scenes(paths=fp(self.kite_scene_paths))
if self.gnss_campaign_paths:
ds.add_gnss_campaigns(paths=fp(self.gnss_campaign_paths))
if self.clippings_path:
ds.add_clippings(markers_filename=fp(self.clippings_path))
if self.responses_sacpz_path:
ds.add_responses(
sacpz_dirname=fp(self.responses_sacpz_path))
if self.responses_stationxml_paths:
ds.add_responses(stationxml_filenames=fp(
self.responses_stationxml_paths))
if self.station_corrections_path:
ds.add_station_corrections(
filename=fp(self.station_corrections_path))
ds.apply_correction_factors = self.apply_correction_factors
ds.apply_correction_delays = self.apply_correction_delays
ds.apply_displaced_sampling_workaround = \
self.apply_displaced_sampling_workaround
ds.extend_incomplete = self.extend_incomplete
for picks_path in self.picks_paths:
ds.add_picks(filename=fp(picks_path))
ds.add_blacklist(self.blacklist)
ds.add_blacklist(filenames=fp(self.blacklist_paths))
if self.whitelist:
ds.add_whitelist(self.whitelist)
if self.whitelist_paths:
ds.add_whitelist(filenames=fp(self.whitelist_paths))
ds.set_synthetic_test(copy.deepcopy(self.synthetic_test))
self._ds[event_name] = ds
except (FileLoadError, OSError) as e:
raise DatasetError(str(e))
return self._ds[event_name]
