def get(self, event, iteration):
"""
Returns the window manager instance for a given event and iteration.
:param event_name: The name of the event.
:param iteration_name: The name of the iteration.
"""
event = self.comm.events.get(event)
iteration = self.comm.iterations.get(iteration)
event_name = event["event_name"]
iteration_name = iteration.name
if not self.comm.events.has_event(event_name):
msg = "Event '%s' not known." % event_name
raise LASIFNotFoundError(msg)
if not self.comm.iterations.has_iteration(iteration_name):
msg = "Iteration '%s' not known." % iteration_name
raise LASIFNotFoundError(msg)
iteration = self.comm.iterations.get(iteration_name)
if event_name not in iteration.events:
msg = "Event '%s' not part of iteration '%s'." % (event_name,
iteration_name)
raise ValueError(msg)
folder = os.path.join(
self._folder, event_name,
self.comm.iterations.get_long_iteration_name(iteration_name))
return WindowGroupManager(folder,
iteration_name,
event_name,
comm=self.comm)
def get_project_function(self, fct_type):
"""
Helper importing the project specific function.
:param fct_type: The desired function.
"""
# Cache to avoid repeated imports.
if fct_type in self.__project_function_cache:
return self.__project_function_cache[fct_type]
# type / filename map
fct_type_map = {
"window_picking_function": "window_picking_function.py",
"preprocessing_function": "preprocessing_function.py",
"data_svd_selection": "data_svd_selection.py",
"process_synthetics": "process_synthetics.py",
"source_time_function": "source_time_function.py",
"instaseis_synthetics_function":
"instaseis_synthetics_function.py",
"stf_deconvolution": "stf_deconvolution.py",
}
if fct_type not in fct_type:
msg = "Function '%s' not found. Available types: %s" % (
fct_type, str(list(fct_type_map.keys())))
raise LASIFNotFoundError(msg)
filename = os.path.join(self.paths["functions"],
fct_type_map[fct_type])
if not os.path.exists(filename):
msg = "No file '%s' in existence." % filename
raise LASIFNotFoundError(msg)
fct_template = imp.load_source("_lasif_fct_template", filename)
try:
fct = getattr(fct_template, fct_type)
except AttributeError:
raise LASIFNotFoundError(
"Could not find function %s in file '%s'" %
(fct_type, filename))
if not callable(fct):
raise LASIFError("Attribute %s in file '%s' is not a function." %
(fct_type, filename))
# Add to cache.
self.__project_function_cache[fct_type] = fct
return fct
def __init__(self, comm, iteration, event):
self.comm = comm
self.event = self.comm.events.get(event)
self.iteration = self.comm.iterations.get(iteration)
self.event_name = self.event["event_name"]
if self.event_name not in self.iteration.events:
msg = "Event '%s' not part of iteration '%s'." % (
self.event_name, self.iteration.name)
raise LASIFNotFoundError(msg)
# Get all stations defined for the given iteration and event.
stations = set(
self.iteration.events[self.event_name]["stations"].keys())
# Only use those stations that actually have processed and synthetic
# data available! Especially synthetics might not always be available.
processed = comm.waveforms.get_metadata_processed(
self.event_name, self.iteration.processing_tag)
synthetics = comm.waveforms.get_metadata_synthetic(
self.event_name, self.iteration)
processed = set(
["%s.%s" % (_i["network"], _i["station"]) for _i in processed])
synthetics = set(
["%s.%s" % (_i["network"], _i["station"]) for _i in synthetics])
self.stations = tuple(
sorted(stations.intersection(processed).intersection(synthetics)))
self._current_index = -1
def get_available_synthetics(self, event_name):
"""
Returns the available synthetics for a given event.
:param event_name: The event name.
"""
data_dir = os.path.join(self._synthetics_folder, event_name)
if not os.path.exists(data_dir):
raise LASIFNotFoundError("No synthetic data for event '%s'." %
event_name)
iterations = []
for folder in os.listdir(data_dir):
if not os.path.isdir(os.path.join(
self._synthetics_folder, event_name, folder)) \
or not fnmatch.fnmatch(folder, "ITERATION_*"):
continue
iterations.append(folder)
# Make sure the iterations also contain the event and the stations.
its = []
for iteration in iterations:
try:
it = self.comm.iterations.get(iteration)
except LASIFNotFoundError:
continue
if event_name not in it.events:
continue
its.append(it.name)
return its
def __what_is_this_station_file(self, file_path, filetype):
ft_map = {
"dataless_seed": "SEED",
"resp": "RESP",
"stationxml": "StationXML"
}
info = self.comm.stations.get_details_for_filename(file_path)
if info is None:
raise LASIFNotFoundError("File '%s' is not valid station file." %
file_path)
info = [
"\t%s | %s - %s | Lat/Lng/Ele/Dep: %s/%s/%s/%s" %
(_i["channel_id"], str(_i["start_date"]),
str(_i["end_date"]) if _i["end_date"] else "--",
"%.2f" % _i["latitude"] if _i["latitude"] is not None else "--",
"%.2f" % _i["longitude"] if _i["longitude"] is not None else "--",
"%.2f" % _i["elevation_in_m"] if _i["elevation_in_m"] is not None
else "--", "%.2f" % _i["local_depth_in_m"]
if _i["local_depth_in_m"] is not None else "--") for _i in info
]
return ("The %s file contains information about %i channel%s:\n%s" %
(ft_map[filetype], len(info), "s" if len(info) > 1 else "",
"\n".join(info)))
def get_channel_filename(self, channel_id, time):
"""
Returns the absolute path of the file storing the information for the
given channel and time combination.
:param channel_id: The id of the channel.
:type channel_id: str
:param time: The time at which to retrieve the information.
:type time: ``int`` or :class:`~obspy.core.utcdatetime.UTCDateTime`
>>> import obspy
>>> comm = getfixture('stations_comm')
It works with :class:`~obspy.core.utcdatetime.UTCDateTime` objects
>>> comm.stations.get_channel_filename( # doctest: +ELLIPSIS
... "IU.ANMO.10.BHZ", obspy.UTCDateTime(2012, 3, 14))
u'/.../IRIS_single_channel_with_response.xml'
and timestamps.
>>> comm.stations.get_channel_filename( # doctest: +ELLIPSIS
... "IU.ANMO.10.BHZ", 1331683200)
u'/.../IRIS_single_channel_with_response.xml'
"""
filename = self._station_cache.get_station_filename(channel_id, time)
if filename is None:
raise LASIFNotFoundError(
"Could not find a station file for channel '%s' at %s." % (
channel_id, str(time)))
return filename
def _get_waveforms(self,
event_name,
station_id,
data_type,
tag_or_iteration=None):
waveform_cache = self.get_waveform_cache(event_name, data_type,
tag_or_iteration)
network, station = station_id.split(".")
files = waveform_cache.get_files_for_station(network, station)
if len(files) == 0:
raise LASIFNotFoundError("No '%s' waveform data found for event "
"'%s' and station '%s'." %
(data_type, event_name, station_id))
if data_type in ["raw", "processed"]:
# Sort files by location.
locations = {
key: list(value)
for key, value in itertools.groupby(
files, key=lambda x: x["location"])
}
keys = sorted(locations.keys())
if len(keys) != 1:
msg = ("Found %s waveform data from %i locations for event "
"'%s' and station '%s': %s. Will only use data from "
"location '%s'." %
(data_type, len(keys), event_name, station_id,
", ".join(["'%s'" % _i for _i in keys]), keys[0]))
warnings.warn(msg, LASIFWarning)
files = locations[keys[0]]
st = obspy.Stream()
for single_file in files:
st += obspy.read(single_file["filename"])
st.sort()
return st
def get_metadata_raw(self, event_name):
"""
Returns the available metadata at the channel level for the raw
waveforms and the given event_name.
:param event_name: The name of the event.
:returns: A list of dictionaries, each describing channel level data
at a particular point in time.
In most instances, the ``latitude``, ``longitude``,
``elevation_in_m``, and ``local_depth_in_m`` values will be
``None`` as station information is usually retrieved from the
station metadata file. SAC files are an exception.
>>> import pprint
>>> comm = getfixture('waveforms_comm')
>>> pprint.pprint(sorted(comm.waveforms.get_metadata_raw(
... "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11"),
... key=lambda x: x["channel_id"])) \
# doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
[{'channel': u'BHE',
'channel_id': u'HL.ARG..BHE',
'elevation_in_m': None,
'endtime': UTCDateTime(2010, 3, 24, 15, 11, 30, 974999),
'filename': u'/.../HL.ARG..BHE.mseed',
'latitude': None,
'local_depth_in_m': None,
'location': u'',
'longitude': None,
'network': u'HL',
'starttime': UTCDateTime(2010, 3, 24, 14, 6, 31, 24999),
'station': u'ARG'},
{...},
...]
Each dictionary will have the following keys.
>>> sorted(comm.waveforms.get_metadata_raw(
... "GCMT_event_TURKEY_Mag_5.1_2010-3-24-14-11")[0].keys()) \
# doctest: +NORMALIZE_WHITESPACE
['channel', 'channel_id', 'elevation_in_m', 'endtime',
'filename', 'latitude', 'local_depth_in_m', 'location',
'longitude', 'network', 'starttime', 'station']
A :class:`~lasif.LASIFNotFoundError` will be raised, if no raw
waveform data is found for the specified event.
>>> comm.waveforms.get_metadata_raw("RandomEvent")
Traceback (most recent call last):
...
LASIFNotFoundError: ...
"""
waveform_cache = self.get_waveform_cache(event_name,
data_type="raw")
values = waveform_cache.get_values()
if not values:
msg = "No data for event '%s' found." % event_name
raise LASIFNotFoundError(msg)
return self._convert_timestamps(waveform_cache.get_values())
def get(self, iteration_name):
"""
Returns an iteration object.
:param iteration_name: The name of the iteration to retrieve.
>>> comm = getfixture('iterations_comm')
>>> comm.iterations.get("1") # doctest: +ELLIPSIS
<lasif.iteration_xml.Iteration object at ...>
>>> print comm.iterations.get("1") \
# doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
LASIF Iteration
Name: 1
...
A :class:`~lasif.LASIFNotFoundError` will be raised, if the
iteration is not known.
>>> comm.iterations.get("99")
Traceback (most recent call last):
...
LASIFNotFoundError: ...
It also works with the long iteration name and event an existing
iteration object. This makes it simple to use, one path for all
possibilities.
>>> it = comm.iterations.get("ITERATION_1")
>>> it # doctest: +ELLIPSIS
<lasif.iteration_xml.Iteration object at ...>
>>> comm.iterations.get(it)
<lasif.iteration_xml.Iteration object at ...>
"""
# Make it work with both the long and short version of the iteration
# name, and existing iteration object.
try:
iteration_name = str(iteration_name.iteration_name)
except AttributeError:
iteration_name = str(iteration_name)
iteration_name = iteration_name.lstrip("ITERATION_")
# Access cache.
if iteration_name in self.__cached_iterations:
return self.__cached_iterations[iteration_name]
it_dict = self.get_iteration_dict()
if iteration_name not in it_dict:
msg = "Iteration '%s' not found." % iteration_name
raise LASIFNotFoundError(msg)
from lasif.iteration_xml import Iteration
it = Iteration(it_dict[iteration_name],
stf_fct=self.comm.project.get_project_function(
"source_time_function"))
# Store in cache.
self.__cached_iterations[iteration_name] = it
return it
def get(self, iteration, event):
iteration = self.comm.iterations.get(iteration)
kernel_dir = os.path.join(self._folder, iteration.long_name, event)
if not os.path.exists(kernel_dir) or not os.path.isdir(kernel_dir):
raise LASIFNotFoundError("Kernel for iteration %s and event %s "
"not found" %
(iteration.long_name, event))
return kernel_dir
def select_component_from_stream(st, component):
"""
Helper function selecting a component from a Stream an raising the proper
error if not found.
This is a bit more flexible then stream.select() as it works with single
letter channels and lowercase channels.
"""
component = component.upper()
component = [tr for tr in st if tr.stats.channel[-1].upper() == component]
if not component:
raise LASIFNotFoundError("Component %s not found in Stream." %
component)
elif len(component) > 1:
raise LASIFNotFoundError("More than 1 Trace with component %s found "
"in Stream." % component)
return component[0]
def get_waveform_stf(self, event_name, tag_or_iteration, component="Z"):
"""
Gets the stf waveform for the given event and iteration and component as a
:class:`~obspy.core.stream.Stream` object.
:param event_name: The name of the event.
:param tag: The processing tag
:param component: The stf component 'Z', 'E' or 'N'
"""
import glob
if (component != 'Z') and (component != 'E') and (component != 'N'):
msg = (
"Invalid data component '%s'. Component should be E, N, or Z." %
component)
raise ValueError(msg)
data_type = "stf"
tag_or_iteration = \
self.comm.iterations.get(tag_or_iteration).long_name
data_path = self.get_waveform_folder(event_name, data_type,
tag_or_iteration)
if not tag_or_iteration:
msg = "Long iteration name must be given for stf data."
raise ValueError(msg)
if not os.path.exists(data_path):
msg = ("No stf data for event '%s' and iteration '%s' "
"found." % (event_name, tag_or_iteration))
raise LASIFNotFoundError(msg)
list_file = glob.glob(os.path.join(data_path, "stf_%s*" % component))
if len(list_file) > 1:
msg = (
"More than one stf data file for event '%s' and iteration '%s' and component %s "
"found." %
(event_name, tag_or_iteration, component))
raise LASIFNotFoundError(msg)
else:
try:
stf = obspy.read(list_file[0])
return stf
except BaseException:
msg = ("Cannot read stf file %s" % list_file[0])
raise LASIFNotFoundError(msg)
def get(self, event_name):
"""
Get information about one event.
This function uses multiple cache layers and is thus very cheap to
call.
:type event_name: str
:param event_name: The name of the event.
:rtype: dict
>>> comm = getfixture('events_comm')
>>> comm.events.get('GCMT_event_TURKEY_Mag_5.9_2011-5-19-20-15') \
# doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
{'m_tp': -2.17e+17, 'm_tt': 8.92e+17, 'depth_in_km': 7.0,
'event_name': u'GCMT_event_TURKEY_Mag_5.9_2011-5-19-20-15',
'region': u'TURKEY', 'longitude': 29.1,
'filename': u'/.../GCMT_event_TURKEY_Mag_5.9_2011-5-19-20-15.xml',
'm_rr': -8.07e+17, 'magnitude': 5.9, 'magnitude_type': u'Mwc',
'latitude': 39.15,
'origin_time': UTCDateTime(2011, 5, 19, 20, 15, 22, 900000),
'm_rp': -5.3e+16, 'm_pp': -8.5e+16, 'm_rt': 2.8e+16}
The moment tensor components are in ``Nm``. The dictionary will
contain the following keys:
>>> sorted(comm.events.get(
... 'GCMT_event_TURKEY_Mag_5.9_2011-5-19-20-15').keys()) \
# doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
['depth_in_km', 'event_name', 'filename', 'latitude', 'longitude',
'm_pp', 'm_rp', 'm_rr', 'm_rt', 'm_tp', 'm_tt', 'magnitude',
'magnitude_type', 'origin_time', 'region']
It also works with an existing event dictionary. This eases calling
the function under certain circumstances.
>>> ev = comm.events.get('GCMT_event_TURKEY_Mag_5.9_2011-5-19-20-15')
>>> ev == comm.events.get(ev)
True
"""
# Make sure it also works with existing event dictionaries. This
# has the potential to simplify lots of code.
try:
event_name = event_name["event_name"]
except (KeyError, TypeError):
pass
if event_name not in self.__event_info_cache:
raise LASIFNotFoundError("Event '%s' not known to LASIF." %
event_name)
return self.__event_info_cache[event_name]
def assert_has_boxfile(self, iteration, event):
"""
Makes sure the kernel in question has a boxfile. Otherwise it will
search all models until it finds one with the same dimensions and
copy the boxfile.
:param iteration: The iteration.
:param event: The event name.
"""
kernel_dir = self.get(iteration=iteration, event=event)
boxfile = os.path.join(kernel_dir, "boxfile")
if os.path.exists(boxfile):
return
boxfile_found = False
# Find all vp gradients.
vp_gradients = glob.glob(os.path.join(kernel_dir, "grad_csv_*"))
file_count = len(vp_gradients)
filesize = list(set([os.path.getsize(_i) for _i in vp_gradients]))
if len(filesize) != 1:
msg = ("The grad_cp_*_* files in '%s' do not all have "
"identical size.") % kernel_dir
raise ValueError(msg)
filesize = filesize[0]
# Now loop over all model directories until a fitting one if found.
for model_name in self.comm.models.list():
model_dir = self.comm.models.get(model_name)
# Use the lambda parameter files. One could also use any of the
# others.
lambda_files = glob.glob(os.path.join(model_dir, "lambda*"))
if len(lambda_files) != file_count:
continue
l_filesize = list(set([os.path.getsize(_i)
for _i in lambda_files]))
if len(l_filesize) != 1 or l_filesize[0] != filesize:
continue
model_boxfile = os.path.join(model_dir, "boxfile")
if not os.path.exists(model_boxfile):
continue
boxfile_found = True
boxfile = model_boxfile
break
if boxfile_found is not True:
msg = (
"Could not find a suitable boxfile for the kernel stored "
"in '%s'. Please either copy a suitable one to this "
"directory or add a model with the same dimension to LASIF. "
"LASIF will then be able to figure out the dimensions of it.")
raise LASIFNotFoundError(msg)
print("Copied boxfile from '%s' to '%s'." % (model_dir, kernel_dir))
shutil.copyfile(boxfile, os.path.join(kernel_dir, "boxfile"))
def list_for_event(self, event_name):
"""
List of all iterations with windows for an event.
:param event_name: The name of the event.
"""
event_folder = os.path.join(self._folder, event_name)
if not os.path.exists(event_folder):
msg = "No windows for event '%s'." % event_name
raise LASIFNotFoundError(msg)
return sorted([_i.lstrip("ITERATION_")
for _i in os.listdir(event_folder) if
os.path.isdir(os.path.join(event_folder, _i)) and
_i.startswith("ITERATION_")])
def get_metadata_synthetic(self, event_name, long_iteration_name):
"""
Get the synthetics metadata.
:param event_name: The name of the event.
:param long_iteration_name: The long form of the iteration name.
:param station_id: The id of the station in the form ``NET.STA``.
"""
# Assure the iteration actually contains the event.
it = self.comm.iterations.get(long_iteration_name)
if event_name not in it.events:
raise LASIFNotFoundError(
"Iteration '%s' does not contain event '%s'." %
(it.name, event_name))
waveform_cache = self.get_waveform_cache(
event_name,
data_type="synthetic",
tag_or_iteration=long_iteration_name)
values = waveform_cache.get_values()
if not values:
msg = ("No synthetic data for event '%s' and iteration '%s' "
"found." % (event_name, long_iteration_name))
raise LASIFNotFoundError(msg)
return self._convert_timestamps(waveform_cache.get_values())
def get_coordinates_for_station(self, event_name, station_id):
"""
Get the coordinates for one station.
Must be in sync with :meth:`~.get_all_stations_for_event`.
"""
event = self.comm.events.get(event_name)
# Collect information from all the different places.
waveform = self.comm.waveforms.get_metadata_raw_for_station(
event_name, station_id)[0]
station_coordinates = self.comm.stations.get_all_channels_at_time(
event["origin_time"])
try:
stat_coords = station_coordinates[waveform["channel_id"]]
except KeyError:
# No station file for channel.
raise LASIFNotFoundError("Station '%s' has no available "
"station file." % station_id)
# First attempt to retrieve from the station files.
if stat_coords["latitude"] is not None:
return stat_coords
# Then from the waveform metadata in the case of a sac file.
elif waveform["latitude"] is not None:
return waveform
# The last resort is a new query via the inventory database.
coords = self.comm.inventory_db.get_coordinates(station_id)
if coords["latitude"]:
return coords
else:
# No station file for channel.
raise LASIFNotFoundError("Coordinates could not be deduced for "
"station '%s' and event '%s'." %
(station_id, event_name))
def get_metadata_processed(self, event_name, tag):
"""
Get the processed metadata.
:param event_name: The name of the event.
:param tag: The processing tag.
"""
waveform_cache = self.get_waveform_cache(event_name,
data_type="processed",
tag_or_iteration=tag)
values = waveform_cache.get_values()
if not values:
msg = "No data for event '%s' and processing tag '%s' found." % \
(event_name, tag)
raise LASIFNotFoundError(msg)
return self._convert_timestamps(waveform_cache.get_values())
def get_available_processing_tags(self, event_name):
"""
Returns the available processing tags for a given event.
:param event_name: The event name.
"""
data_dir = os.path.join(self._data_folder, event_name)
if not os.path.exists(data_dir):
raise LASIFNotFoundError("No data for event '%s'." % event_name)
tags = []
for tag in os.listdir(data_dir):
# Only interested in preprocessed data.
if not tag.startswith("preprocessed") or \
tag.endswith("_cache.sqlite"):
continue
tags.append(tag)
return tags
def get_metadata_raw_for_station(self, event_name, station_id):
"""
Returns the available metadata at the channel level for the raw
waveforms and the given event_name at a certain station.
Same as :meth:`~.get_metadata_raw` just for only a single station.
:param event_name: The name of the event.
:param station_id: The station id.
:returns: A list of dictionaries, each describing channel level data
at a particular point in time.
"""
waveform_cache = self.get_waveform_cache(event_name, data_type="raw")
network_id, station_id = station_id.split(".")
values = waveform_cache.get_files_for_station(network_id, station_id)
if not values:
msg = "No data for event '%s' and station '%s' found." % (
event_name, station_id)
raise LASIFNotFoundError(msg)
return self._convert_timestamps(values)
def get_metadata_processed_for_station(self, event_name, tag, station_id):
"""
Get the processed metadata for a single station.
Same as :meth:`~.get_metadata_processed` but for a single station.
:param event_name: The name of the event.
:param tag: The processing tag.
:param station_id: The id of the station in the form ``NET.STA``.
"""
waveform_cache = self.get_waveform_cache(event_name,
data_type="processed",
tag_or_iteration=tag)
network_id, station_id = station_id.split(".")
values = waveform_cache.get_files_for_station(network_id, station_id)
if not values:
msg = "No processed data for event '%s', tag '%s', and station " \
"'%s' found." % (event_name, tag, station_id)
raise LASIFNotFoundError(msg)
return self._convert_timestamps(values)
def get_metadata_synthetic_for_station(self, event_name,
long_iteration_name, station_id):
"""
Get the synthetics metadata for a single station.
Same as :meth:`~.get_metadata_synthetic` but for a single station.
:param event_name: The name of the event.
:param long_iteration_name: The long form of the iteration name.
:param station_id: The id of the station in the form ``NET.STA``.
"""
waveform_cache = self.get_waveform_cache(
event_name, data_type="synthetic",
tag_or_iteration=long_iteration_name)
network_id, station_id = station_id.split(".")
values = waveform_cache.get_files_for_station(network_id, station_id)
if not values:
msg = "No synthetic data for event '%s', iteration '%s', " \
"and station '%s' found." % (event_name, long_iteration_name,
station_id)
raise LASIFNotFoundError(msg)
return self._convert_timestamps(values)
def what_is(self, path):
"""
Debug function returning a string with information about the file.
Useful as a debug function and to figure out what LASIF is doing.
:param path: The path to the file.
"""
path = os.path.normpath(os.path.abspath(path))
# File does not exist.
if not os.path.exists(path):
raise LASIFNotFoundError("Path '%s' does not exist." % path)
# File not part of the project.
if os.path.commonprefix([path, self.comm.project.paths["root"]]) \
!= self.comm.project.paths["root"]:
raise LASIFError("File '%s' is not part of the LASIF project." %
path)
# Split in dir an folder to ease the rest.
if os.path.isdir(path):
return self.__what_is_this_folder(path)
else:
return self.__what_is_this_file(path)
def get_waveform_cache(self,
event_name,
data_type,
tag_or_iteration=None,
dont_update=False):
"""
:param event_name: The name of the event.
:param data_type: The data type.
:param tag_or_iteration: If processed data, the tag, if synthetic
data, the iteration.
:param dont_update: If True, an existing cache will not be updated
but returned as is. If it does not exist, it will be updated
regardless.
"""
if data_type == "synthetic":
tag_or_iteration = \
self.comm.iterations.get(tag_or_iteration).long_name
data_path = self.get_waveform_folder(event_name, data_type,
tag_or_iteration)
if data_type == "raw":
if not os.path.exists(data_path):
msg = "No data for event '%s' found." % event_name
raise LASIFNotFoundError(msg)
label = "Raw"
elif data_type == "processed":
if not tag_or_iteration:
msg = "Tag must be given for processed data."
raise ValueError(msg)
if not os.path.exists(data_path):
msg = ("No data for event '%s' and processing tag '%s' "
"found." % (event_name, tag_or_iteration))
raise LASIFNotFoundError(msg)
label = "Processed %s" % event_name
elif data_type == "synthetic":
if not tag_or_iteration:
msg = "Long iteration name must be given for synthetic data."
raise ValueError(msg)
if not os.path.exists(data_path):
msg = ("No synthetic data for event '%s' and iteration '%s' "
"found." % (event_name, tag_or_iteration))
raise LASIFNotFoundError(msg)
label = "Synthetic %s" % event_name
else:
raise ValueError("Invalid data type '%s'." % data_type)
waveform_db_file = data_path + "_cache" + os.path.extsep + "sqlite"
if waveform_db_file in self.__cache:
return self.__cache[waveform_db_file]
if dont_update is True and os.path.exists(waveform_db_file):
cache = WaveformCache(cache_db_file=waveform_db_file,
root_folder=self.comm.project.paths["root"],
waveform_folder=data_path,
pretty_name="%s Waveform Cache" % label,
read_only=True)
elif data_type == "synthetic" \
and not os.path.exists(waveform_db_file) \
and os.listdir(data_path):
# If it is synthetic, read a file and assume all other files
# have the same length. This has the huge advantage that the
# files no longer have to be opened but only the filename has to
# be parsed. Only works for SES3D files.
files = sorted(os.listdir(data_path))
filename = os.path.join(data_path, files[len(files) // 2])
tr = obspy.read(filename)[0]
synthetic_info = {
"starttime_timestamp": tr.stats.starttime.timestamp,
"endtime_timestamp": tr.stats.endtime.timestamp
}
cache = WaveformCache(cache_db_file=waveform_db_file,
root_folder=self.comm.project.paths["root"],
waveform_folder=data_path,
pretty_name="%s Waveform Cache" % label,
read_only=self.comm.project.read_only_caches,
synthetic_info=synthetic_info)
else:
cache = WaveformCache(cache_db_file=waveform_db_file,
root_folder=self.comm.project.paths["root"],
waveform_folder=data_path,
pretty_name="%s Waveform Cache" % label,
read_only=self.comm.project.read_only_caches)
self.__cache[waveform_db_file] = cache
return cache
def create_successive_iteration(self,
existing_iteration_name,
new_iteration_name,
create_folders=True):
"""
Create an iteration based on an existing one.
It will take all settings in one iteration and transfers them to
another iteration. Any comments will be deleted.
:param existing_iteration_name: Name of the iteration to be used as
a template.
:param new_iteration_name: Name of the new iteration.
:param create_folders: Create the folders for the next iteration's
synthetic waveforms
Note that the ``create_folders=False`` argument is only used here
for testing purposes. In most cases you will want this to be ``True``.
>>> comm = getfixture('iterations_comm')
>>> comm.iterations.has_iteration("3")
False
>>> comm.iterations.create_successive_iteration("1", "3",
... create_folders=False)
>>> comm.iterations.has_iteration("3")
True
Comments of an iteration will be stripped.
>>> comm.iterations.get("1").comments
['Some', 'random comments']
>>> comm.iterations.get("3").comments
[]
>>> os.remove(comm.iterations.get_iteration_dict()["3"])
If the iteration template does not exist, a
:class:`~lasif.LASIFNotFoundError` will be raised.
>>> comm.iterations.create_successive_iteration("99", "100")
Traceback (most recent call last):
...
LASIFNotFoundError: ...
A ``ValueError`` will be raised if the new iteration already exists.
>>> comm.iterations.create_successive_iteration("1", "2")
Traceback (most recent call last):
...
ValueError: ...
"""
it_dict = self.get_iteration_dict()
if existing_iteration_name not in it_dict:
msg = "Iteration %s does not exists." % existing_iteration_name
raise LASIFNotFoundError(msg)
if new_iteration_name in it_dict:
msg = "Iteration %s already exists." % new_iteration_name
raise ValueError(msg)
from lasif.iteration_xml import Iteration
existing_iteration = Iteration(
it_dict[existing_iteration_name],
stf_fct=self.comm.project.get_project_function(
"source_time_function"))
# Clone the old iteration, delete any comments and change the name.
existing_iteration.comments = []
existing_iteration.iteration_name = new_iteration_name
self.save_iteration(existing_iteration)
if create_folders:
self.create_synthetics_folder_for_iteration(new_iteration_name)
self.create_stf_folder_for_iteration(new_iteration_name)
def calculate_adjoint_source(self, event_name, iteration_name, channel_id,
starttime, endtime, taper, taper_percentage,
ad_src_type):
"""
Calculates an adjoint source for a single window.
:param event_name: The name of the event.
:param iteration_name: The name of the iteration.
:param channel_id: The channel id in the form NET.STA.NET.CHA.
:param starttime: The starttime of the window.
:param endtime: The endtime of the window.
:param taper: How to taper the window.
:param taper_percentage: The taper percentage at one end as a
decimal number ranging from 0.0 to 0.5 for a full width taper.
:param ad_src_type: The type of adjoint source. Currently supported
are ``"TimeFrequencyPhaseMisfitFichtner2008"`` and ``"L2Norm"``.
"""
iteration = self.comm.iterations.get(iteration_name)
iteration_name = iteration.long_name
event = self.comm.events.get(event_name)
event_name = event["event_name"]
folder = os.path.join(self._folder, event_name, iteration_name)
if not os.path.exists(folder):
os.makedirs(folder)
filename = os.path.join(
folder, "%s_%s_%s_%s_%.2f_%s" %
(channel_id, str(starttime), str(endtime), str(taper),
taper_percentage, ad_src_type))
if os.path.exists(filename):
adsrc = joblib.load(filename)
if not self._validate_return_value(adsrc):
os.remove(filename)
else:
return adsrc
if ad_src_type not in MISFIT_MAPPING:
raise LASIFAdjointSourceCalculationError(
"Adjoint source type '%s' not supported. Supported types: %s" %
(ad_src_type, ", ".join(MISFIT_MAPPING.keys())))
waveforms = self.comm.query.get_matching_waveforms(
event=event_name,
iteration=iteration_name,
station_or_channel_id=channel_id)
data = waveforms.data
synth = waveforms.synthetics
if len(data) != 1:
raise LASIFNotFoundError(
"Data not found for event '%s', iteration '%s', and channel "
"'%s'." % (event_name, iteration_name, channel_id))
if len(synth) != 1:
raise LASIFNotFoundError(
"Synthetics not found for event '%s', iteration '%s', "
"and channel '%s'." % (event_name, iteration_name, channel_id))
data = data[0]
synth = synth[0]
# Make sure they are equal enough.
if abs(data.stats.starttime - synth.stats.starttime) > 0.1:
raise LASIFAdjointSourceCalculationError(
"Starttime not similar enough")
if data.stats.npts != synth.stats.npts:
raise LASIFAdjointSourceCalculationError(
"Differing number of samples")
if abs(data.stats.delta - synth.stats.delta) / data.stats.delta > \
0.01:
raise LASIFAdjointSourceCalculationError(
"Sampling rate not similar enough.")
original_stats = copy.deepcopy(data.stats)
for trace in [data, synth]:
trace.trim(starttime, endtime)
trace.taper(type=taper.lower(), max_percentage=taper_percentage)
trace.trim(original_stats.starttime,
original_stats.endtime,
pad=True,
fill_value=0.0)
# make time axis
t = np.linspace(0, (original_stats.npts - 1) * original_stats.delta,
original_stats.npts)
# set data and synthetics, compute actual misfit
t = np.require(t, dtype="float64", requirements="C")
data_d = np.require(data.data, dtype="float64", requirements="C")
synth_d = np.require(synth.data, dtype="float64", requirements="C")
process_parameters = iteration.get_process_params()
# compute misfit and adjoint source
adsrc = MISFIT_MAPPING[ad_src_type](
t, data_d, synth_d, 1.0 / process_parameters["lowpass"],
1.0 / process_parameters["highpass"])
# Recreate dictionary for clarity.
ret_val = {
"adjoint_source": adsrc["adjoint_source"],
"misfit_value": adsrc["misfit_value"],
"details": adsrc["details"]
}
if not self._validate_return_value(ret_val):
raise LASIFAdjointSourceCalculationError(
"Could not calculate adjoint source due to mismatching types.")
joblib.dump(ret_val, filename)
return ret_val
def discover_available_data(self, event_name, station_id):
"""
Discovers the available data for one event at a certain station.
Will raise a :exc:`~lasif.LASIFNotFoundError` if no raw data is
found for the given event and station combination.
:type event_name: str
:param event_name: The name of the event.
:type station_id: str
:param station_id: The id of the station in question.
:rtype: dict
:returns: Return a dictionary with "processed" and "synthetic" keys.
Both values will be a list of strings. In the case of "processed"
it will be a list of all available preprocessing tags. In the case
of the synthetics it will be a list of all iterations for which
synthetics are available.
"""
if not self.comm.events.has_event(event_name):
msg = "Event '%s' not found in project." % event_name
raise LASIFNotFoundError(msg)
# Attempt to get the station coordinates. This ensures availability
# of the raw data.
self.get_coordinates_for_station(event_name, station_id)
def get_components(waveform_cache):
return sorted([_i["channel"][-1] for _i in waveform_cache],
reverse=True)
raw = self.comm.waveforms.get_metadata_raw_for_station(
event_name, station_id)
raw_comps = get_components(raw)
# Collect all tags and iteration names.
all_files = {
"raw": {
"raw": raw_comps
},
"processed": {},
"synthetic": {}
}
# Get the available synthetic and processing tags.
proc_tags = self.comm.waveforms.get_available_processing_tags(
event_name)
for tag in proc_tags:
try:
procs = self.comm.waveforms.get_metadata_processed_for_station(
event_name, tag, station_id)
except LASIFNotFoundError:
continue
comps = get_components(procs)
if not comps:
continue
all_files["processed"][tag] = comps
iterations = self.comm.waveforms.get_available_synthetics(event_name)
synthetic_coordinates_mapping = {
"X": "N",
"Y": "E",
"Z": "Z",
"N": "N",
"E": "E"
}
for it in iterations:
try:
its = self.comm.waveforms.get_metadata_synthetic_for_station(
event_name, it, station_id)
except LASIFNotFoundError:
continue
comps = get_components(its)
if not comps:
continue
comps = [synthetic_coordinates_mapping[i] for i in comps]
all_files["synthetic"][it] = sorted(comps, reverse=True)
return all_files
def get(self, model_name):
model_dir = os.path.join(self._folder, model_name)
if not os.path.exists(model_dir) or not os.path.isdir(model_dir):
raise LASIFNotFoundError("Model '%s' not known." % model_name)
return model_dir
def select_windows_for_station(self, event, iteration, station, **kwargs):
"""
Selects windows for the given event, iteration, and station. Will
delete any previously existing windows for that station if any.
:param event: The event.
:param iteration: The iteration.
:param station: The station id in the form NET.STA.
"""
from lasif.utils import select_component_from_stream
# Load project specific window selection function.
select_windows = self.comm.project.get_project_function(
"window_picking_function")
event = self.comm.events.get(event)
iteration = self.comm.iterations.get(iteration)
data = self.comm.query.get_matching_waveforms(event, iteration,
station)
process_params = iteration.get_process_params()
minimum_period = 1.0 / process_params["lowpass"]
maximum_period = 1.0 / process_params["highpass"]
window_group_manager = self.comm.windows.get(event, iteration)
# Delete the windows for this stations.
window_group_manager.delete_windows_for_station(station)
found_something = False
for component in ["E", "N", "Z"]:
try:
data_tr = select_component_from_stream(data.data, component)
synth_tr = select_component_from_stream(
data.synthetics, component)
except LASIFNotFoundError:
continue
found_something = True
windows = select_windows(data_tr,
synth_tr,
event["latitude"],
event["longitude"],
event["depth_in_km"],
data.coordinates["latitude"],
data.coordinates["longitude"],
minimum_period=minimum_period,
maximum_period=maximum_period,
iteration=iteration,
**kwargs)
if not windows:
continue
window_group = window_group_manager.get(data_tr.id)
for starttime, endtime in windows:
window_group.add_window(starttime=starttime, endtime=endtime)
window_group.write()
if found_something is False:
raise LASIFNotFoundError(
"No matching data found for event '%s', iteration '%s', and "
"station '%s'." %
(event["event_name"], iteration.name, station))