def get_events(self, **kwargs) -> obspy.Catalog:
"""
Read events from bank.
Parameters
----------
{get_events_params}
"""
ind = self.read_index(**kwargs)
eids = ind["event_id"]
file_paths = ind["path"]
paths = str(self.bank_path) + _natify_paths(file_paths)
paths.drop_duplicates(inplace=True)
read_func = partial(try_read_catalog, format=self.format)
# Divide work evenly between workers, with a min chunksize of 1.
chunksize = len(paths) // self._max_workers or 1
map_kwargs = dict(chunksize=chunksize)
try:
mapped_values = self._map(read_func, paths.values, **map_kwargs)
non_none_values = (x for x in mapped_values if x is not None)
cat = reduce(add, non_none_values)
except TypeError: # empty events
cat = obspy.Catalog()
# Make sure only the events of interest are included
events = [eve for eve in cat if eve.resource_id.id in eids.values]
return obspy.Catalog(events=events)
def update_index(self, bar: Optional[ProgressBar] = None) -> "EventBank":
"""
Iterate files in bank and add any modified since last update to index.
Parameters
----------
{bar_parameter_description}
"""
def func(path):
""" Function to yield events, update_time and paths. """
cat = try_read_catalog(path, format=self.format)
update_time = getmtime(path)
path = path.replace(self.bank_path, "")
return cat, update_time, path
self._enforce_min_version() # delete index if schema has changed
# create iterator and lists for storing output
update_time = time.time()
iterator = self._measured_unindexed_iterator(bar)
events, update_times, paths = [], [], []
for cat, mtime, path in self._map(func, iterator):
if cat is None:
continue
for event in cat:
events.append(event)
update_times.append(mtime)
paths.append(path)
# add new events to database
df = obsplus.events.pd._default_cat_to_df(obspy.Catalog(events=events))
df["updated"] = update_times
df["path"] = paths
if len(df):
df_to_write = self._prepare_dataframe(df, EVENT_TYPES_INPUT)
self._write_update(df_to_write, update_time)
return self
def json_cat_from_disk(self, cat_from_json):
""" save the json events to disk and read it again into memory """
tf = tempfile.mkstemp()
cat_from_json = obspy.Catalog(cat_from_json)
cat_from_json.write(tf[1], "quakeml")
cat = obspy.read_events(tf[1])
return cat
def update_index(self, bar: Optional[ProgressBar] = None) -> "EventBank":
"""
Iterate files in bank and add any modified since last update to index.
Parameters
----------
{bar_parameter_description}
"""
self._enforce_min_version() # delete index if schema has changed
bar = self.get_progress_bar(bar) # get ProgressBar or None
# loop over un-index files and update
events, update_time, paths = [], [], []
for num, fi in enumerate(self._unindexed_file_iterator()):
cat = try_read_catalog(fi, format=self.format)
if cat is None:
continue
for event in cat:
events.append(event)
update_time.append(getmtime(fi))
paths.append(fi.replace(self.bank_path, ""))
# update progress bar
if bar is not None and num % self._bar_update_interval == 0:
bar.update(num)
# add new events to database
df = obsplus.events.pd._default_cat_to_df(obspy.Catalog(events=events))
df["updated"] = update_time
df["path"] = paths
if len(df):
self._write_update(self._clean_dataframe(df))
getattr(bar, "finish", lambda: None)() # call finish if bar exists
return self
def _tables_to_catalog(df_dict: Dict[str, pd.DataFrame]) -> obspy.Catalog:
""" convert a dict of dataframes back to an obspy events """
try:
events = df_dict["Event"]
except KeyError: # empty events
return obspy.Catalog()
func = partial(_construct_object, df_dict=df_dict, cls=ev.Event)
return events.apply(func, axis=1).values.tolist()
def catalog(self, bingham_dataset, new_time):
"""
Assemble a events to test yield_event_waveforms with an event
that was not initiated from the start.
"""
# get first event, create new origin to slightly add some time.
ori = Origin(time=new_time, latitude=47.1, longitude=-100.22)
event = Event(origins=[ori])
return obspy.Catalog(events=[event])
def catalog_directory(self):
""" return a directory of catalogs """
cat = obspy.read_events()
with tempfile.TemporaryDirectory() as tempdir:
for num, eve in enumerate(cat.events):
new_cat = obspy.Catalog(events=[eve])
file_name = f"{num}.xml"
write_path = join(self.nest_directly(num, tempdir), file_name)
new_cat.write(write_path, "quakeml")
yield tempdir
def simple_catalog_to_merge(self, bingham_catalog):
"""
Create a simple catalog to merge into bingham_cat using only one event.
"""
cat = obspy.Catalog(events=bingham_catalog[:2]).copy()
# drop first pick
cat[0].picks = cat[0].picks[1:]
# modify the picks to whole seconds, reset pick IDS
for pick, _, _ in yield_obj_parent_attr(cat, ev.Pick):
pick.time -= (pick.time.timestamp) % 1
pick.id = ev.ResourceIdentifier(referred_object=pick)
return cat
def simple_catalog_to_merge(self, bingham_catalog):
"""
Create a simple catalog to merge into bingham_cat using only two event.
"""
events = sorted(bingham_catalog, key=get_reference_time)
cat = obspy.Catalog(events=events[:2]).copy()
# drop first pick
cat[0].picks = cat[0].picks[1:]
# modify the picks to whole seconds, reset pick IDS
for pick, _, _ in yield_obj_parent_attr(cat, ev.Pick):
nearest_second = np.round(pick.time.timestamp)
pick.time = obspy.UTCDateTime(nearest_second)
# pick.resource_id = ev.ResourceIdentifier(referred_object=pick)
return cat
def get_events(self, **kwargs) -> obspy.Catalog:
"""
Read events from bank.
Parameters
----------
{get_events_params}
"""
paths = self.bank_path + self.read_index(columns="path", **kwargs).path
cats = (obspy.read_events(x) for x in paths)
try:
return reduce(add, cats)
except TypeError: # empty events
return obspy.Catalog()
def json_to_cat(cjson: Union[str, dict]) -> obspy.Catalog:
"""
Load json str or dict into a catalog object.
Parameters
----------
cjson
A str or dict produced by cat_to_dict or cat_to_json.
"""
# load json to dict
if isinstance(cjson, str):
cjson = json.loads(cjson)
assert isinstance(cjson, dict)
return obspy.Catalog(**_parse_dict_class(cjson))
def get_events(self, **kwargs) -> obspy.Catalog:
"""
Read events from bank.
Parameters
----------
{get_events_params}
"""
paths = self.bank_path + self.read_index(columns="path", **kwargs).path
read_func = partial(try_read_catalog, format=self.format)
map_kwargs = dict(chunksize=len(paths) // self._max_workers)
try:
mapped_values = self._map(read_func, paths.values, **map_kwargs)
return reduce(add, mapped_values)
except TypeError: # empty events
return obspy.Catalog()
def get_events(cat: obspy.Catalog, **kwargs) -> obspy.Catalog:
"""
Return a subset of a events filtered on input parameters.
See obspy.core.fdsn.Client.get_events for supported arguments.
"""
# if empty just return events
if not kwargs:
return cat
# make sure all inputs are supported
if not set(kwargs).issubset(SUPPORTED_PARAMS):
bad_params = set(kwargs) - SUPPORTED_PARAMS
msg = f"{bad_params} are not supported get_events parameters"
raise TypeError(msg)
event_ids = _get_ids(obsplus.events_to_df(cat), kwargs)
events = [eve for eve in cat if str(eve.resource_id) in event_ids]
return obspy.Catalog(events=events)
def get_events(cat: obspy.Catalog, **kwargs) -> obspy.Catalog:
"""
Return a subset of a events filtered on input parameters.
Parameters
----------
{get_event_parameters}
"""
# If not kwargs are passed just return all events
if not kwargs:
return cat
# Make sure all inputs are supported
if not set(kwargs).issubset(SUPPORTED_PARAMS):
bad_params = set(kwargs) - SUPPORTED_PARAMS
msg = f"{bad_params} are not supported get_events parameters"
raise TypeError(msg)
# Ensure all times are numpy datetimes
kwargs = _dict_times_to_npdatetimes(kwargs)
event_ids = _get_ids(obsplus.events_to_df(cat), kwargs)
events = [eve for eve in cat if str(eve.resource_id) in event_ids]
return obspy.Catalog(events=events)
def catalog(self):
""" assemble a events to test yield_event_waveforms with an event
that was not initiated from the start """
ori = Origin(time=self.t1, latitude=47.1, longitude=-100.22)
event = Event(origins=[ori])
return obspy.Catalog(events=[event])
def refine_picks(catalog, stream_dict, pre_pick, post_pick, shift_len,
cc_thresh, master=None, lowcut=1.0, highcut=20.0,
plotvar=False):
r"""Function to refine picks in a catalog based upon either a pre-chosen\
master event or the event in the catalog with the highest amplitude.
:type catalog: class: obspy.Catalog
:param catalog: Catalog of events which we want to adjust picks for
:type stream_dict: dict
:param stream_dict: Dictionary with key:value pairing of event\
ResourceID:obspy.Stream for each event in catalog.
:type pre_pick: float
:param pre_pick: Time before the pick to start the correlation window
:type post_pick: float
:param post_pick: Time after the pick to start the correlation window
:type shift_len: float
:param shift_len: Time to allow pick to vary
:type master: bool or str
:param master: If 'None', master event defaults to the event with the\
highest SNR. Otherwise, must specify a valid event resoure_id\
from the catalog.
:type lowcut: float
:param lowcut: Lowcut in Hz - default=1.0
:type highcut: float
:param highcut: Highcut in Hz - deafult=10.0
:returns: class: obspy.Catalog
"""
import obspy
if int(obspy.__version__.split('.')[0]) > 0:
from obspy.signal.cross_correlation import xcorr_pick_correction
else:
from obspy.signal.cross_correlation import xcorrPickCorrection \
as xcorr_pick_correction
# Establish master template if not specified
if master:
master_id = obspy.ResourceIdentifier(master)
else:
# Find event with highest SNR to be master
avg_snr = {}
for event in catalog:
avg_snr[event.resource_id] =\
sum([x.snr for x in event.amplitudes]) / len(event.amplitudes)
master_id = max(avg_snr.iterkeys(), key=(lambda key: avg_snr[key]))
# Loop back through catalog and extract master event (there better way?)
master_event = [x for x in catalog if x.resource_id == master_id][0]
master_stream = stream_dict[master_id]
new_catalog = obspy.Catalog()
# Figure total number of picks
tot_pks = 0
for event in catalog:
for cnt_pick in event:
tot_pks += 1
refined_num = 0
# Now loop the master through all events in catalog
for slave_event in catalog:
# Copy old slave event and reset the picks (keep the rest of the info)
# new_event = obspy.core.event.Event()
new_event = slave_event.copy()
new_event.picks = []
slave_stream = stream_dict[slave_event.resource_id]
# Find UNcommon picks between slave and master
mismatches = uncommon_picks(slave_event, master_event)
# Append them to new event (otherwise they get missed)
for uncom_pick in mismatches:
new_event.picks.append(uncom_pick)
for pick in master_event.picks:
# Find station, phase pairs
# Added by Carolin
slave_matches = [p for p in slave_event.picks
if p.phase_hint == pick.phase_hint
and p.waveform_id.station_code ==
pick.waveform_id.station_code]
if master_stream.select(station=pick.waveform_id.station_code,
channel='*' +
pick.waveform_id.channel_code[-1]):
mastertr = master_stream.\
select(station=pick.waveform_id.station_code,
channel='*' +
pick.waveform_id.channel_code[-1])[0]
else:
print('No waveform data for ' +
pick.waveform_id.station_code + '.' +
pick.waveform_id.channel_code)
break
for slave_pick in slave_matches:
if slave_stream.select(station=slave_pick.waveform_id.
station_code,
channel='*'+slave_pick.waveform_id.
channel_code[-1]):
slavetr = slave_stream.\
select(station=slave_pick.waveform_id.station_code,
channel='*'+slave_pick.waveform_id.
channel_code[-1])[0]
else:
print('No slave data for ' +
slave_pick.waveform_id.station_code + '.' +
slave_pick.waveform_id.channel_code)
break
try:
correction, cc =\
xcorr_pick_correction(pick.time, mastertr,
slave_pick.time,
slavetr, pre_pick, post_pick,
shift_len, filter="bandpass",
filter_options={'freqmin':
lowcut,
'freqmax':
highcut},
plot=plotvar)
if abs(correction) > shift_len:
warnings.warn('Shift correction too large, ' +
'will not use')
new_event.picks.append(slave_pick)
continue
if cc > cc_thresh:
print('Threshold exceeded')
new_pick_time = slave_pick.time + correction
new_pick = slave_pick.copy()
new_pick.time = new_pick_time
new_pick.creation_info.agency_id = 'VUW'
new_pick.creation_info.author = 'eqcorrscan.refine_picks()'
new_pick.creation_info.creation_time = obspy.UTCDateTime.now()
new_event.picks.append(new_pick)
refined_num += 1
else:
# new_event.picks.append(slave_pick)
print('Correlation not good enough to correct pick')
new_event.picks.append(slave_pick)
except:
# Should warn here
msg = "Couldn't compute correlation correction"
warnings.warn(msg)
new_event.picks.append(slave_pick)
continue
new_catalog += new_event
print('Refined %d of %d picks' % (refined_num, tot_pks))
return new_catalog
def miss_merge_catalog(self, bingham_catalog):
"""
Create a catalog whose events are too far in time to be merged.
"""
cat = obspy.Catalog(events=bingham_catalog[2:]).copy()
return cat
def bingham_cat_only_picks(self, bingham_dataset):
""" return bingham_test catalog with everything but picks removed """
events = []
for eve in bingham_dataset.event_client.get_events().copy():
events.append(ev.Event(picks=eve.picks))
return obspy.Catalog(events=events)
q = qdict[[f for f in fields if f[0] in 'Qx'][0]]
tw[id_] = [q, qc[id_], trtw]
text = collapse_json(json.dumps(tw, indent=2), 6)
with open(TWFILE, 'w') as f:
f.write(text)
return tw
def ev2id(event):
return str(event.resource_id).split('/')[-1]
def select_events(events, tw, qclevel):
events = [ev for ev in events if tw.get(ev2id(ev), (0, ))[0] >= qclevel]
events = [ev for ev in events if 'NOWEBNET' not in tw[ev2id(ev)][1]]
return obspy.Catalog(events)
if __name__ == '__main__':
print('load events')
allevents = get_events()
print('finished loading')
allevents = obspy.Catalog(
sorted(allevents, key=lambda ev: ev.origins[0].time))
PICKS, RELPICKS, _, EV2EV = get_picks(allevents)
events = allevents.filter(f'magnitude > {MAG}')
# create_event_qc_list(events)
tw = tw_from_qc_file(events)
for stream, event in iter_data(events, alldata=False):
single_plot(stream, event, tw=tw)
def _event_to_catalog(event):
return get_event_client(obspy.Catalog(events=[event]))
else:
trig_type = 1
triggers.append(obspy.UTCDateTime(time) - obspy.UTCDateTime(0))
trig_meta.append((net, sta, phase, obspy.UTCDateTime(time),
trig_type))
idx = np.argsort(triggers)
triggers = np.array([triggers[x] for x in idx])
trig_meta = [trig_meta[x] for x in idx]
print("Now building catalog")
#nll_summary_file = "%s/%s" % \
# (params['nlloc_loc_path'], params['nlloc_sum_file'])
#cat = obspy.io.nlloc.core.read_nlloc_hyp(nll_summary_file)
nll_files = glob.glob("%s/*.*.*.*.*.hyp" % params['nlloc_loc_path'])
cat = obspy.Catalog()
for fname in nll_files:
try:
cat += obspy.read_events(fname)
except:
continue
random.shuffle(nll_files)
for event in cat:
print(event.preferred_origin().time)
print(cat)
print()
if params['plot_seismicity']:
plot_seismicity(cat, params)
def get_subset_of_events(comm, count, events, existing_events=None):
"""
This function gets an optimally distributed set of events,
NO QA.
:param comm: LASIF communicator
:param count: number of events to choose.
:param events: list of event_names, from which to choose from. These
events must be known to LASIF
:param existing_events: list of events, that have been chosen already
and should thus be excluded from the selected options, but are also
taken into account when ensuring a good spatial distribution. The
function assumes that there are no common occurences between
events and existing events
:return: a list of chosen events.
"""
available_events = comm.events.list()
if len(events) < count:
raise LASIFError("Insufficient amount of events specified.")
if not type(count) == int:
raise ValueError("count should be an integer value.")
if count < 1:
raise ValueError("count should be at least 1.")
for event in events:
if event not in available_events:
raise LASIFNotFoundError(f"event : {event} not known to LASIF.")
if existing_events is None:
existing_events = []
else:
for event in events:
if event in existing_events:
raise LASIFError(f"event: {event} was existing already,"
f"but still supplied to choose from.")
cat = obspy.Catalog()
for event in events:
event_file_name = comm.waveforms.get_asdf_filename(event,
data_type="raw")
with pyasdf.ASDFDataSet(event_file_name, mode="r") as ds:
ev = ds.events[0]
# append event_name to comments, such that it can later be
# retrieved
ev.comments.append(event)
cat += ev
# Coordinates and the Catalog will have the same order!
coordinates = []
for event in cat:
org = event.preferred_origin() or event.origins[0]
coordinates.append((org.latitude, org.longitude))
chosen_events = []
existing_coordinates = []
for event in existing_events:
ev = comm.events.get(event)
existing_coordinates.append((ev["latitude"], ev["longitude"]))
# randomly start with one of the specified events
if not existing_coordinates:
idx = random.randint(0, len(cat) - 1)
chosen_events.append(cat[idx])
del cat.events[idx]
existing_coordinates.append(coordinates[idx])
del coordinates[idx]
count -= 1
while count:
if not coordinates:
print("\tNo events left to select from. Stopping here.")
break
# Build kdtree and query for the point furthest away from any other
# point.
kdtree = SphericalNearestNeighbour(np.array(existing_coordinates))
distances = kdtree.query(np.array(coordinates), k=1)[0]
idx = np.argmax(distances)
event = cat[idx]
coods = coordinates[idx]
del cat.events[idx]
del coordinates[idx]
chosen_events.append(event)
existing_coordinates.append(coods)
count -= 1
list_of_chosen_events = []
for ev in chosen_events:
list_of_chosen_events.append(ev.comments.pop())
if len(list_of_chosen_events) < count:
raise ValueError("Could not select a sufficient amount of events")
return list_of_chosen_events
def empty_catalog():
""" run an empty events through the picks_to_df function """
event = ev.Event()
cat = obspy.Catalog(events=[event])
return picks_to_dataframe(cat)
def select_events(events, tw, qclevel):
events = [ev for ev in events if tw.get(ev2id(ev), (0, ))[0] >= qclevel]
events = [ev for ev in events if 'NOWEBNET' not in tw[ev2id(ev)][1]]
return obspy.Catalog(events)
