def data_timing_quality_obspy_daily(path="",
key="",
scale=lambda x: x,
**kwargs):
"""
Extract timing quality flags from given miniseed file. One value per
file, which means one value per day.
:type path: str
:param path: path to miniseed file
:type key: str
:param key: key available with :func:`~obspy.io.mseed.util.get_flags` and
``timinig_quality`` key.
:type scale: func, optional
:param scale: defaults to identity function,
scaling function applied to data values
:rtype: list or None
:return: list of lists containing timestamp (as
:class:`~obspy.core.utcdatetime.UTCDateTime` instance) and data value.
"""
dtq = get_flags(path)["timing_quality"]
if key not in dtq:
return None
data_value = scale(dtq[key])
st = fileutils.get_stream(path)
st.sort(keys=["starttime"])
time_ = st[-1].stats.endtime
return [[time_, data_value]]
def _extract_mseed_flags(self):
flags = get_flags(self.files,
starttime=self.starttime,
endtime=self.endtime)
data_quality_flags_seconds = flags["data_quality_flags_percentages"]
activity_flags_seconds = flags["activity_flags_percentages"]
io_and_clock_flags_seconds = flags["io_and_clock_flags_percentages"]
timing_correction = flags["timing_correction"]
# Only calculate the timing quality statistics if each files has the
# timing quality set. This should usually be the case. Otherwise we
# would created tinted statistics. There is still a chance that some
# records in a file have timing qualities set and others not but
# that should be small.
if flags["timing_quality"]:
tq = flags["timing_quality"]
timing_quality_mean = tq["mean"]
timing_quality_min = tq["min"]
timing_quality_max = tq["max"]
timing_quality_median = tq["median"]
timing_quality_lower_quartile = tq["lower_quartile"]
timing_quality_upper_quartile = tq["upper_quartile"]
else:
timing_quality_mean = None
timing_quality_min = None
timing_quality_max = None
timing_quality_median = None
timing_quality_lower_quartile = None
timing_quality_upper_quartile = None
meta = self.meta
meta['num_records'] = flags['record_count']
# Set MiniSEED header counts
meta['miniseed_header_counts'] = {}
ref = meta['miniseed_header_counts']
ref['timing_correction'] = flags['timing_correction_count']
ref['activity_flags'] = flags['activity_flags_counts']
ref['io_and_clock_flags'] = flags['io_and_clock_flags_counts']
ref['data_quality_flags'] = flags['data_quality_flags_counts']
# Set MiniSEED header percentages
meta['miniseed_header_percentages'] = {}
ref = meta['miniseed_header_percentages']
ref['timing_correction'] = timing_correction
ref['timing_quality_mean'] = timing_quality_mean
ref['timing_quality_min'] = timing_quality_min
ref['timing_quality_max'] = timing_quality_max
ref['timing_quality_median'] = timing_quality_median
ref['timing_quality_lower_quartile'] = timing_quality_lower_quartile
ref['timing_quality_upper_quartile'] = timing_quality_upper_quartile
# According to schema @ maybe refactor this to less verbose flag
# names. Sets MiniSEED header flag percentages
ref['activity_flags'] = activity_flags_seconds
ref['data_quality_flags'] = data_quality_flags_seconds
ref['io_and_clock_flags'] = io_and_clock_flags_seconds
def _extract_mseed_flags(self):
flags = get_flags(self.files, starttime=self.starttime,
endtime=self.endtime)
data_quality_flags_seconds = flags["data_quality_flags_percentages"]
activity_flags_seconds = flags["activity_flags_percentages"]
io_and_clock_flags_seconds = flags["io_and_clock_flags_percentages"]
timing_correction = flags["timing_correction"]
# Only calculate the timing quality statistics if each files has the
# timing quality set. This should usually be the case. Otherwise we
# would created tinted statistics. There is still a chance that some
# records in a file have timing qualities set and others not but
# that should be small.
if flags["timing_quality"]:
tq = flags["timing_quality"]
timing_quality_mean = tq["mean"]
timing_quality_min = tq["min"]
timing_quality_max = tq["max"]
timing_quality_median = tq["median"]
timing_quality_lower_quartile = tq["lower_quartile"]
timing_quality_upper_quartile = tq["upper_quartile"]
else:
timing_quality_mean = None
timing_quality_min = None
timing_quality_max = None
timing_quality_median = None
timing_quality_lower_quartile = None
timing_quality_upper_quartile = None
meta = self.meta
meta['num_records'] = flags['record_count']
# Set MiniSEED header counts
meta['miniseed_header_counts'] = {}
ref = meta['miniseed_header_counts']
ref['timing_correction'] = flags['timing_correction_count']
ref['activity_flags'] = flags['activity_flags_counts']
ref['io_and_clock_flags'] = flags['io_and_clock_flags_counts']
ref['data_quality_flags'] = flags['data_quality_flags_counts']
# Set MiniSEED header percentages
meta['miniseed_header_percentages'] = {}
ref = meta['miniseed_header_percentages']
ref['timing_correction'] = timing_correction
ref['timing_quality_mean'] = timing_quality_mean
ref['timing_quality_min'] = timing_quality_min
ref['timing_quality_max'] = timing_quality_max
ref['timing_quality_median'] = timing_quality_median
ref['timing_quality_lower_quartile'] = timing_quality_lower_quartile
ref['timing_quality_upper_quartile'] = timing_quality_upper_quartile
# According to schema @ maybe refactor this to less verbose flag
# names. Sets MiniSEED header flag percentages
ref['activity_flags'] = activity_flags_seconds
ref['data_quality_flags'] = data_quality_flags_seconds
ref['io_and_clock_flags'] = io_and_clock_flags_seconds
def toph5(self, file_tuple):
"""
Takes a tuple (file_name or obspy stream, type)
and loads it into ph5_object
:type tuple
:param file_tuple containing
file_handle or obspy stream and file type as str
:return:
"""
index_t = list()
time_corrected = False
correction = False
current_mini = None
in_type = None
das_station_map = self.get_das_station_map()
existing_minis = self.get_minis(self.ph5_path)
if not das_station_map:
err = "Array metadata must exist before loading data"
LOGGER.error(err)
return "stop", index_t
# gets mapping of whats dases each minifile contains
minis = self.mini_map(existing_minis)
# check if we are opening a file or have an obspy stream
if isinstance(file_tuple[0], str):
st = reader(file_tuple[0], format=file_tuple[1])
in_type = "file"
if file_tuple[1] == 'MSEED':
try:
flags = get_flags(file_tuple[0])
if flags['activity_flags_counts'][
'time_correction_applied'] > 0:
LOGGER.info("Timing correction has been applied")
time_corrected = True
if flags["timing_correction"] != 0.0:
LOGGER.info('Timing Correction found')
correction = True
except BaseException:
pass
# is this an obspy stream?
elif isinstance(file_tuple[0], Stream):
st = file_tuple[0]
in_type = 'stream'
# is this an obspy trace?
elif isinstance(file_tuple[0], Trace):
st = Stream(traces=[file_tuple[0]])
in_type = 'trace'
# Loop through data and load it in to PH5
LOGGER.info('Processing {0} traces in stream for {1}'.format(
len(st), file_tuple[0]))
count = 1
for trace in st:
if self.verbose:
LOGGER.info('Processing trace {0} in {1}'.format(
trace.stats.channel, trace.stats.station))
if not trace.stats.channel == 'LOG':
if not trace.data.any():
LOGGER.info("No data for trace {0}...skipping".format(
trace.stats.channel))
continue
if not existing_minis:
current_mini = self.first_mini
else:
current_mini = None
for mini in minis:
for entry in das_station_map:
if (entry['serial'] in mini[1]
and entry['station'] == trace.stats.station):
current_mini = mini[0]
if not current_mini:
largest = 0
for x in minis:
if x[0] >= largest:
largest = x[0]
if (self.get_size_mini(largest) < self.mini_size_max):
current_mini = largest
else:
current_mini = largest + 1
# iterate through das_station_map
for entry in das_station_map:
time_t = {}
das = {}
index_t_entry = {}
# only load data if it matches
if trace.stats.station == entry['station']:
# open mini file
mini_handle, mini_name = self.openmini(current_mini)
# get node reference or create new node
d = mini_handle.ph5_g_receivers.getdas_g(entry['serial'])
if not d:
d, t, r, ti = mini_handle.ph5_g_receivers.newdas(
entry['serial'])
# start populating das table and data arrays
das['time/ascii_s'] = trace.stats.starttime
index_t_entry['start_time/ascii_s'] = (
trace.stats.starttime.isoformat())
time = timedoy.fdsn2epoch(
trace.stats.starttime.isoformat(), fepoch=True)
microsecond = (time % 1) * 1000000
das['time/epoch_l'] = (int(time))
das['time/micro_seconds_i'] = microsecond
das['time/type_s'] = 'BOTH'
index_t_entry['start_time/epoch_l'] = (int(time))
index_t_entry['start_time/micro_seconds_i'] = (microsecond)
index_t_entry['start_time/type_s'] = 'BOTH'
time = timedoy.fdsn2epoch(trace.stats.endtime.isoformat(),
fepoch=True)
microsecond = (time % 1) * 1000000
index_t_entry['end_time/ascii_s'] = (
trace.stats.endtime.isoformat())
index_t_entry['end_time/epoch_l'] = (int(time))
index_t_entry['end_time/micro_seconds_i'] = (microsecond)
index_t_entry['end_time/type_s'] = 'BOTH'
now = UTCDateTime.now()
index_t_entry['time_stamp/ascii_s'] = (now.isoformat())
time = timedoy.fdsn2epoch(now.isoformat(), fepoch=True)
microsecond = (time % 1) * 1000000
index_t_entry['time_stamp/epoch_l'] = (int(time))
index_t_entry['time_stamp/micro_seconds_i'] = (
int(microsecond))
index_t_entry['time_stamp/type_s'] = 'BOTH'
if correction or time_corrected:
time_t['das/serial_number_s'] = entry['serial']
if in_type == 'file':
time_t['description_s'] = file_tuple[0]
else:
time_t['description_s'] = (
str(trace.stats.station) +
str(trace.stats.channel))
# SEED time correction
# units are 0.0001 seconds per unit
time_t['offset_d'] = \
flags["timing_correction"] * 0.0001
time_t['start_time/epoch_l'] =\
index_t_entry['start_time/epoch_l']
time_t['start_time/micro_seconds_i'] =\
index_t_entry['start_time/micro_seconds_i']
time_t['end_time/epoch_l'] =\
index_t_entry['end_time/epoch_l']
time_t['end_time/micro_seconds_i'] =\
index_t_entry['end_time/micro_seconds_i']
length = trace.stats.npts * trace.stats.delta
if length != 0:
time_t['slope_d'] = time_t['offset_d'] / length
else:
time_t['slope_d'] = 0
if time_corrected:
time_t['corrected_i'] = 1
if time_t:
self.time_t.append(time_t)
if (trace.stats.sampling_rate >= 1
or trace.stats.sampling_rate == 0):
das['sample_rate_i'] = trace.stats.sampling_rate
das['sample_rate_multiplier_i'] = 1
else:
das['sample_rate_i'] = 0
das['sample_rate_multiplier_i'] = (
1 / trace.stats.sampling_rate)
channel_list = list(trace.stats.channel)
if channel_list[2] in ({'3', 'Z', 'z'}):
das['channel_number_i'] = 3
elif channel_list[2] in ({'2', 'E', 'e'}):
das['channel_number_i'] = 2
elif channel_list[2] in ({'1', 'N', 'n'}):
das['channel_number_i'] = 1
elif channel_list[2].isdigit():
das['channel_number_i'] = channel_list[2]
elif trace.stats.channel == 'LOG':
das['channel_number_i'] = -2
das['sample_rate_i'] = 0
das['sample_rate_multiplier_i'] = 1
else:
das['channel_number_i'] = -5
if in_type == 'file':
das['raw_file_name_s'] = file_tuple[0]
else:
das['raw_file_name_s'] = 'obspy_stream'
if trace.stats.channel == 'LOG':
das['sample_count_i'] = 0
else:
das['sample_count_i'] = trace.stats.npts
# figure out receiver and response n_i
for array_entry in self.arrays:
if (array_entry['sample_rate_i']
== trace.stats.sampling_rate
and array_entry['channel_number_i']
== das['channel_number_i'] and
array_entry['id_s'] == trace.stats.station):
das['receiver_table_n_i'] =\
array_entry['receiver_table_n_i']
das['response_table_n_i'] =\
array_entry['response_table_n_i']
# Make sure we aren't overwriting a data array
while True:
next_ = str(count).zfill(5)
das['array_name_data_a'] = "Data_a_{0}".format(next_)
node = mini_handle.ph5_g_receivers.find_trace_ref(
das['array_name_data_a'])
if not node:
break
count = count + 1
continue
mini_handle.ph5_g_receivers.setcurrent(d)
data = array(trace.data)
if trace.stats.channel == 'LOG':
mini_handle.ph5_g_receivers.newarray(
das['array_name_data_a'],
data,
dtype='|S1',
description=None)
else:
data_type = data[0].__class__.__name__
mini_handle.ph5_g_receivers.newarray(
das['array_name_data_a'],
data,
dtype=data_type,
description=None)
mini_handle.ph5_g_receivers.populateDas_t(das)
index_t_entry['external_file_name_s'] = "./{}".format(
mini_name)
das_path = "/Experiment_g/Receivers_g/" \
"Das_g_{0}".format(entry['serial'])
index_t_entry['hdf5_path_s'] = das_path
index_t_entry['serial_number_s'] = entry['serial']
index_t.append(index_t_entry)
# Don't forget to close minifile
mini_handle.ph5close()
LOGGER.info('Finished processing {0}'.format(file_tuple[0]))
# last thing is to return the index table so far.
# index_t will be populated in main() after all
# files are loaded
return "done", index_t