def read_nsmn(filename):
"""Read the Turkish NSMN strong motion data format.
Args:
filename (str): path to NSMN data file.
Returns:
list: Sequence of one StationStream object containing 3 StationTrace objects.
"""
header = _read_header(filename)
header1 = copy.deepcopy(header)
header2 = copy.deepcopy(header)
header3 = copy.deepcopy(header)
header1['standard']['horizontal_orientation'] = 0.0
header1['channel'] = get_channel_name(header['sampling_rate'], True, False,
True)
header2['standard']['horizontal_orientation'] = 90.0
header2['channel'] = get_channel_name(header['sampling_rate'], True, False,
False)
header3['standard']['horizontal_orientation'] = 0.0
header3['channel'] = get_channel_name(header['sampling_rate'], True, True,
False)
# three columns of NS, EW, UD
# data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS,
# delimiter=[COLWIDTH] * NCOLS, encoding=ENCODING)
data = np.loadtxt(filename, skiprows=TEXT_HDR_ROWS, encoding=ENCODING)
data1 = data[:, 0]
data2 = data[:, 1]
data3 = data[:, 2]
trace1 = StationTrace(data=data1, header=header1)
trace2 = StationTrace(data=data2, header=header2)
trace3 = StationTrace(data=data3, header=header3)
stream = StationStream(traces=[trace1, trace2, trace3])
return [stream]
def test_arias():
ddir = os.path.join('data', 'testdata')
datadir = pkg_resources.resource_filename('gmprocess', ddir)
data_file = os.path.join(datadir, 'arias_data.json')
with open(data_file, 'rt') as f:
jdict = json.load(f)
time = np.array(jdict['time'])
# input output is m/s/s
acc = np.array(jdict['acc']) / 100
target_IA = jdict['ia']
delta = time[2] - time[1]
sr = 1 / delta
header = {
'delta': delta,
'sampling_rate': sr,
'npts': len(acc),
'units': 'm/s/s',
'channel': 'HN1',
'standard': {
'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'source_file': '',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'structure_type': '',
'sensor_serial_number': '',
'process_level': 'raw counts',
'process_time': '',
'horizontal_orientation': np.nan,
'units': 'acc',
'instrument_damping': np.nan
}
}
# input is cm/s/s output is m/s/s
trace = StationTrace(data=acc * 100, header=header)
trace2 = trace.copy()
trace2.stats.channel = 'HN2'
stream = StationStream([trace, trace2])
station = StationSummary.from_stream(stream, ['ARITHMETIC_MEAN'],
['arias'])
pgms = station.pgms
Ia = pgms[(pgms.IMT == 'ARIAS')
& (pgms.IMC == 'ARITHMETIC_MEAN')].Result.tolist()[0]
# the target has only one decimal place and is in cm/s/s
Ia = Ia * 100
np.testing.assert_almost_equal(Ia, target_IA, decimal=1)
# Test other components
data_files, _ = read_data_dir('cwb', 'us1000chhc', '2-ECU.dat')
stream = read_data(data_files[0])[0]
station = StationSummary.from_stream(stream, [
'channels', 'gmrotd', 'rotd50', 'greater_of_two_horizontals',
'ARITHMETIC_MEAN'
], ['arias'])
stream = StationSummary.from_stream(stream, ['gmrotd50'], ['arias'])
assert stream.pgms.Result.tolist() == []
def read_bhrc(filename):
"""Read the Iran BHRC strong motion data format.
Args:
filename (str): path to BHRC data file.
Returns:
list: Sequence of one StationStream object containing 3
StationTrace objects.
"""
header1, offset = _read_header_lines(filename, 0)
data1, offset = _read_data(filename, offset, header1)
header2, offset = _read_header_lines(filename, offset)
data2, offset = _read_data(filename, offset, header2)
header3, offset = _read_header_lines(filename, offset)
data3, offset = _read_data(filename, offset, header3)
trace1 = StationTrace(data1, header1)
trace2 = StationTrace(data2, header2)
trace3 = StationTrace(data3, header3)
stream = StationStream([trace1, trace2, trace3])
for tr in stream:
if tr.stats.standard.process_level != PROCESS_LEVELS['V0']:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
tr.setProvenance('remove_response', response)
return [stream]
def test_arias():
ddir = os.path.join('data', 'testdata')
datadir = pkg_resources.resource_filename('gmprocess', ddir)
data_file = os.path.join(datadir, 'arias_data.json')
with open(data_file, 'rt') as f:
jdict = json.load(f)
time = np.array(jdict['time'])
# input output is m/s/s
acc = np.array(jdict['acc']) / 100
target_IA = jdict['ia']
delta = time[2] - time[1]
sr = 1 / delta
header = {
'delta': delta,
'sampling_rate': sr,
'npts': len(acc),
'units': 'm/s/s',
'channel': 'HN1',
'standard': {'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'source_file': '',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'structure_type': '',
'sensor_serial_number': '',
'process_level': 'raw counts',
'process_time': '',
'horizontal_orientation': np.nan,
'units': 'acc',
'instrument_damping': np.nan}
}
# input is cm/s/s output is m/s/s
trace = StationTrace(data=acc * 100, header=header)
trace2 = trace.copy()
trace2.stats.channel = 'HN2'
stream = StationStream([trace, trace2])
station = StationSummary.from_stream(stream, ['ARITHMETIC_MEAN'], ['arias'])
pgms = station.pgms
Ia = pgms[(pgms.IMT == 'ARIAS') & (pgms.IMC == 'ARITHMETIC_MEAN')].Result.tolist()[0]
# the target has only one decimal place and is in cm/s/s
Ia = Ia * 100
np.testing.assert_almost_equal(Ia, target_IA, decimal=1)
# Test other components
data_files, _ = read_data_dir('cwb', 'us1000chhc', '2-ECU.dat')
stream = read_data(data_files[0])[0]
station = StationSummary.from_stream(stream,
['channels', 'gmrotd', 'rotd50',
'greater_of_two_horizontals', 'ARITHMETIC_MEAN'],
['arias'])
stream = StationSummary.from_stream(stream, ['gmrotd50'], ['arias'])
assert stream.pgms.Result.tolist() == []
def read_fdsn(filename):
"""Read Obspy data file (SAC, MiniSEED, etc).
Args:
filename (str):
Path to data file.
kwargs (ref):
Other arguments will be ignored.
Returns:
Stream: StationStream object.
"""
logging.debug("Starting read_fdsn.")
if not is_fdsn(filename):
raise Exception('%s is not a valid Obspy file format.' % filename)
streams = []
tstream = read(filename)
xmlfile = _get_station_file(filename, tstream)
inventory = read_inventory(xmlfile)
traces = []
for ttrace in tstream:
trace = StationTrace(data=ttrace.data,
header=ttrace.stats,
inventory=inventory)
location = ttrace.stats.location
trace.stats.channel = get_channel_name(
trace.stats.sampling_rate, trace.stats.channel[1] == 'N',
inventory.get_orientation(trace.id)['dip'] in [90, -90]
or trace.stats.channel[2] == 'Z',
is_channel_north(inventory.get_orientation(trace.id)['azimuth']))
if trace.stats.location == '':
trace.stats.location = '--'
network = ttrace.stats.network
if network in LOCATION_CODES:
codes = LOCATION_CODES[network]
if location in codes:
sdict = codes[location]
if sdict['free_field']:
trace.stats.standard.structure_type = 'free_field'
else:
trace.stats.standard.structure_type = sdict['description']
head, tail = os.path.split(filename)
trace.stats['standard']['source_file'] = tail or os.path.basename(head)
traces.append(trace)
stream = StationStream(traces=traces)
streams.append(stream)
return streams
def read_unam(filename):
"""Read the Mexican UNAM strong motion data format.
Args:
filename (str): path to UNAM data file.
Returns:
list: Sequence of one StationStream object containing 3
StationTrace objects.
"""
channels = _read_header(filename)
npts = channels[0]['npts']
all_data = np.genfromtxt(filename, skip_header=ALL_HEADERS, max_rows=npts)
trace1 = StationTrace(data=all_data[:, 0], header=channels[0])
trace2 = StationTrace(data=all_data[:, 1], header=channels[1])
trace3 = StationTrace(data=all_data[:, 2], header=channels[2])
stream = StationStream(traces=[trace1, trace2, trace3])
return [stream]
def read_bhrc(filename):
"""Read the Iran BHRC strong motion data format.
Args:
filename (str): path to BHRC data file.
Returns:
list: Sequence of one StationStream object containing 3 StationTrace objects.
"""
header1, offset = _read_header_lines(filename, 0)
data1, offset = _read_data(filename, offset, header1)
header2, offset = _read_header_lines(filename, offset)
data2, offset = _read_data(filename, offset, header2)
header3, offset = _read_header_lines(filename, offset)
data3, offset = _read_data(filename, offset, header3)
trace1 = StationTrace(data1, header1)
trace2 = StationTrace(data2, header2)
trace3 = StationTrace(data3, header3)
stream = StationStream([trace1, trace2, trace3])
return [stream]
def read_fdsn(filename):
"""Read Obspy data file (SAC, MiniSEED, etc).
Args:
filename (str):
Path to data file.
kwargs (ref):
Other arguments will be ignored.
Returns:
Stream: StationStream object.
"""
logging.debug("Starting read_fdsn.")
if not is_fdsn(filename):
raise Exception('%s is not a valid Obspy file format.' % filename)
streams = []
tstream = read(filename)
xmlfile = _get_station_file(filename, tstream)
inventory = read_inventory(xmlfile)
traces = []
for ttrace in tstream:
trace = StationTrace(data=ttrace.data,
header=ttrace.stats,
inventory=inventory)
location = ttrace.stats.location
network = ttrace.stats.network
if network in LOCATION_CODES:
codes = LOCATION_CODES[network]
if location in codes:
sdict = codes[location]
if sdict['free_field']:
trace.stats.standard.structure_type = 'free_field'
else:
trace.stats.standard.structure_type = sdict['description']
head, tail = os.path.split(filename)
trace.stats['standard']['source_file'] = tail or os.path.basename(head)
traces.append(trace)
stream = StationStream(traces=traces)
streams.append(stream)
return streams
def get_integral(self):
"""
Calculated the integral of each trace's data.
Returns:
stream: StationStream with the integrated data.
"""
stream = StationStream([])
for trace in self.transform_data:
integrated_trace = trace.integrate()
integrated_trace.stats['units'] = 'veloc'
strace = StationTrace(data=integrated_trace.data,
header=integrated_trace.stats)
stream.append(strace)
return stream
def get_derivative(self):
"""
Calculated the derivative of each trace's data.
Returns:
stream: StationStream with the differentiated data.
"""
stream = StationStream([])
for trace in self.transform_data:
integrated_trace = trace.differentiate()
integrated_trace.stats['units'] = 'acc'
strace = StationTrace(data=integrated_trace.data,
header=integrated_trace.stats)
stream.append(strace)
return stream
def get_arias(self):
"""
Performs calculation of arias intensity.
Returns:
arias_intensities: Dictionary of arias intensity for each channel.
"""
arias_intensities = {}
arias_stream = StationStream([])
for trace in self.reduction_data:
dt = trace.stats['delta']
# convert from cm/s/s to m/s/s
acc = trace.data * 0.01
# Calculate Arias Intensity
integrated_acc2 = integrate.cumtrapz(acc * acc, dx=dt)
arias_intensity = integrated_acc2 * np.pi * GAL_TO_PCTG / 2
channel = trace.stats.channel
trace.stats.standard.units = 'veloc'
trace.stats.npts = len(arias_intensity)
arias_stream.append(StationTrace(arias_intensity, trace.stats))
arias_intensities[channel] = np.abs(np.max(arias_intensity))
self.arias_stream = arias_stream
return arias_intensities
def read_esm(filename):
"""Read European ESM strong motion file.
Args:
filename (str): Path to possible ESM data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing one channels of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_esm.")
if not is_esm(filename):
raise Exception('%s is not a valid ESM file' % filename)
# Parse the header portion of the file
header = {}
with open(filename, 'rt') as f:
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
for line in lines:
parts = line.split(':')
key = parts[0].strip()
value = ':'.join(parts[1:]).strip()
header[key] = value
stats = {}
standard = {}
format_specific = {}
coordinates = {}
# fill in all known stats header fields
stats['network'] = header['NETWORK']
stats['station'] = header['STATION_CODE']
stats['channel'] = header['STREAM']
stats['location'] = '--'
stats['delta'] = float(header['SAMPLING_INTERVAL_S'])
stats['sampling_rate'] = 1 / stats['delta']
stats['calib'] = 1.0
stats['npts'] = int(header['NDATA'])
stimestr = header['DATE_TIME_FIRST_SAMPLE_YYYYMMDD_HHMMSS']
stats['starttime'] = datetime.strptime(stimestr, TIMEFMT)
# fill in standard fields
head, tail = os.path.split(filename)
standard['source_file'] = tail or os.path.basename(head)
standard['source'] = SRC
standard['source_format'] = FORMAT
standard['horizontal_orientation'] = np.nan
standard['station_name'] = header['STATION_NAME']
try:
standard['instrument_period'] = 1 / \
float(header['INSTRUMENTAL_FREQUENCY_HZ'])
except ValueError:
standard['instrument_period'] = np.nan
try:
standard['instrument_damping'] = 1 / \
float(header['INSTRUMENTAL_DAMPING'])
except ValueError:
standard['instrument_damping'] = np.nan
ptimestr = header['DATA_TIMESTAMP_YYYYMMDD_HHMMSS']
ptime = datetime.strptime(ptimestr, TIMEFMT).strftime(TIMEFMT2)
standard['process_time'] = ptime
standard['process_level'] = PROCESS_LEVELS['V1']
instr_str = header['INSTRUMENT']
parts = instr_str.split('|')
sensor_str = parts[0].split('=')[1].strip()
standard['sensor_serial_number'] = ''
standard['instrument'] = sensor_str
standard['comments'] = ''
standard['structure_type'] = ''
standard['units'] = 'cm/s^2'
standard['units_type'] = 'acc'
standard['instrument_sensitivity'] = np.nan
standard['corner_frequency'] = np.nan
coordinates['latitude'] = float(header['STATION_LATITUDE_DEGREE'])
coordinates['longitude'] = float(header['STATION_LONGITUDE_DEGREE'])
coordinates['elevation'] = float(header['STATION_ELEVATION_M'])
# read in the data
data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS)
# create a Trace from the data and metadata
stats['standard'] = standard
stats['coordinates'] = coordinates
trace = StationTrace(data.copy(), Stats(stats.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
ftype = header['FILTER_TYPE'].capitalize()
forder = int(header['FILTER_ORDER'])
lowfreq = float(header['LOW_CUT_FREQUENCY_HZ'])
highfreq = float(header['LOW_CUT_FREQUENCY_HZ'])
filter_att = {'bandpass_filter':
{'filter_type': ftype,
'lower_corner_frequency': lowfreq,
'higher_corner_frequency': highfreq,
'filter_order': forder}}
trace.setProvenance('lowpass_filter', filter_att)
detrend_att = {'detrend': {'detrending_method': 'baseline'}}
trace.setProvenance('detrend', detrend_att)
stream = StationStream(traces=[trace])
return [stream]
def read_at2(dfile, horient=0.0):
# This is a conveneince method so we can read in these specific data for
# testing, it is not a general purpose reader since this format does not
# contain a lot of metadata that is generally required for it to be useful.
skiprows = 4
datafile = open(dfile, 'r')
datareader = csv.reader(datafile)
data = []
header = []
# for i in range(skiprows):
# next(datareader)
# header.append(datareader.readlines())
count = 0
for row in datareader:
if count < skiprows:
header.append(row)
else:
data.extend([float(e) for e in row[0].split()])
count += 1
datafile.close()
hdr = {}
hdr['network'] = ''
hdr['station'] = ''
if horient == 0:
hdr['channel'] = 'BH1'
else:
hdr['channel'] = 'BH2'
hdr['location'] = '--'
dt = float(header[3][1].split('=')[1].strip().lower().replace('sec', ''))
hdr['npts'] = len(data)
hdr['sampling_rate'] = 1 / dt
hdr['duration'] = (hdr['npts'] - 1) / hdr['sampling_rate']
hdr['starttime'] = 0
# There is no lat/lon...
hdr['coordinates'] = {'latitude': 0.0, 'longitude': 0.0, 'elevation': 0.0}
standard = {}
standard['units'] = 'acc'
standard['units_type'] = 'acc'
standard['horizontal_orientation'] = horient
standard['vertical_orientation'] = np.nan
standard['source_file'] = dfile
standard['station_name'] = ''
standard['corner_frequency'] = 30.0
standard['structure_type'] = ''
standard['comments'] = ''
standard['instrument'] = ''
standard['instrument_period'] = 1.0
standard['instrument_sensitivity'] = 1.0
standard['source'] = 'PEER'
standard['instrument_damping'] = 0.1
standard['sensor_serial_number'] = ''
standard['process_level'] = 'corrected physical units'
standard['source_format'] = 'AT2'
standard['process_time'] = ''
hdr['standard'] = standard
# convert data from g to cm/s^2
g_to_cmss = 980.665
tr = StationTrace(np.array(data.copy()) * g_to_cmss, Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
tr.setProvenance('remove_response', response)
return tr
def test_trace():
data = np.random.rand(1000)
header = {'sampling_rate': 1,
'npts': len(data),
'network': 'US',
'location': '11',
'station': 'ABCD',
'channel': 'HN1',
'starttime': UTCDateTime(2010, 1, 1, 0, 0, 0)}
inventory = get_inventory()
invtrace = StationTrace(data=data,
header=header, inventory=inventory)
invtrace.setProvenance('detrend', {'detrending_method': 'demean'})
invtrace.setParameter('failed', True)
invtrace.setParameter('corner_frequencies', [1, 2, 3])
invtrace.setParameter('metadata', {'name': 'Fred'})
assert invtrace.getProvenance('detrend')[0] == {
'detrending_method': 'demean'}
assert invtrace.getParameter('failed')
assert invtrace.getParameter('corner_frequencies') == [1, 2, 3]
assert invtrace.getParameter('metadata') == {'name': 'Fred'}
def _read_volume_two(filename, line_offset, location='', units='acc'):
"""Read channel data from DMG text file.
Args:
filename (str): Input DMG V2 filename.
line_offset (int): Line offset to beginning of channel text block.
units (str): units to get
Returns:
tuple: (list of obspy Trace, int line offset)
"""
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(V2_TEXT_HDR_ROWS)]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skip_rows = V2_TEXT_HDR_ROWS + line_offset
int_data = _read_lines(skip_rows, V2_INT_HDR_ROWS, V2_INT_FMT, filename)
int_data = int_data[0:100].astype(np.int32)
# read in lines of float data
skip_rows += V2_INT_HDR_ROWS
flt_data = _read_lines(skip_rows, V2_REAL_HDR_ROWS, V2_REAL_FMT, filename)
flt_data = flt_data[:100]
skip_rows += V2_REAL_HDR_ROWS
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including csmip/dmg here, don't always provide this. We'll flag it as
# "--".
hdr = _get_header_info(int_data, flt_data, lines, 'V2', location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
traces = []
# read acceleration data
if hdr['npts'] > 0:
acc_rows, acc_fmt, unit = _get_data_format(filename, skip_rows,
hdr['npts'])
acc_data = _read_lines(skip_rows + 1, acc_rows, acc_fmt, filename)
acc_data = acc_data[:hdr['npts']]
if unit in UNIT_CONVERSIONS:
acc_data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('DMG: %s is not a supported unit.' % unit)
acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
acc_trace.setProvenance('remove_response', response)
if units == 'acc':
traces += [acc_trace]
skip_rows += int(acc_rows) + 1
# -------------------------------------------------------------------------
# NOTE: The way we were initially reading velocity and displacement data was
# not correct. I'm deleting it for now since we don't need it. If/when we
# revisit this we need to be more careful about how this is handled.
# -------------------------------------------------------------------------
# read velocity data
vel_hdr = hdr.copy()
vel_hdr['standard']['units'] = 'vel'
vel_hdr['npts'] = int_data[63]
if vel_hdr['npts'] > 0:
vel_rows, vel_fmt, unit = _get_data_format(filename, skip_rows,
vel_hdr['npts'])
vel_data = _read_lines(skip_rows + 1, vel_rows, vel_fmt, filename)
vel_data = vel_data[:vel_hdr['npts']]
skip_rows += int(vel_rows) + 1
# read displacement data
disp_hdr = hdr.copy()
disp_hdr['standard']['units'] = 'disp'
disp_hdr['npts'] = int_data[65]
if disp_hdr['npts'] > 0:
disp_rows, disp_fmt, unit = _get_data_format(filename, skip_rows,
disp_hdr['npts'])
disp_data = _read_lines(skip_rows + 1, disp_rows, disp_fmt, filename)
disp_data = disp_data[:disp_hdr['npts']]
skip_rows += int(disp_rows) + 1
new_offset = skip_rows + 1 # there is an 'end of record' line after the data]
return (traces, new_offset)
def read_smc(filename, **kwargs):
"""Read SMC strong motion file.
Args:
filename (str): Path to possible SMC data file.
kwargs (ref):
any_structure (bool): Read data from any type of structure,
raise Exception if False and structure type is not free-field.
accept_flagged (bool): accept problem flagged data.
set_location (str): Two character code for location.
Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing one channel of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_smc.")
any_structure = kwargs.get('any_structure', False)
accept_flagged = kwargs.get('accept_flagged', False)
location = kwargs.get('location', '')
if not is_smc(filename):
raise Exception('%s is not a valid SMC file' % filename)
with open(filename, 'rt') as f:
line = f.readline().strip()
if 'DISPLACEMENT' in line:
raise GMProcessException(
'SMC: Diplacement records are not supported: '
'%s.' % filename)
elif 'VELOCITY' in line:
raise GMProcessException(
'SMC: Velocity records are not supported: '
'%s.' % filename)
elif line == "*":
raise GMProcessException(
'SMC: No record volume specified in file: '
'%s.' % filename)
stats, num_comments = _get_header_info(filename,
any_structure=any_structure,
accept_flagged=accept_flagged,
location=location)
skip = ASCII_HEADER_LINES + INTEGER_HEADER_LINES + \
num_comments + FLOAT_HEADER_LINES
# read float data (8 columns per line)
nrows = int(np.floor(stats['npts'] / DATA_COLUMNS))
data = np.genfromtxt(filename,
max_rows=nrows,
skip_header=skip,
delimiter=FLOAT_DATA_WIDTHS)
data = data.flatten()
if stats['npts'] % DATA_COLUMNS:
lastrow = np.genfromtxt(filename,
max_rows=1,
skip_header=skip + nrows,
delimiter=FLOAT_DATA_WIDTHS)
data = np.append(data, lastrow)
data = data[0:stats['npts']]
trace = StationTrace(data, header=stats)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
stream = StationStream(traces=[trace])
return [stream]
def _read_channel(filename, line_offset, location=''):
"""Read channel data from COSMOS V1/V2 text file.
Args:
filename (str): Input COSMOS V1/V2 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station, location, and process level from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# read in lines of integer data
skiprows = line_offset + TEXT_HDR_ROWS
int_lines, int_data = _read_lines(skiprows, filename)
int_data = int_data.astype(np.int32)
# read in lines of float data
skiprows += int_lines + 1
flt_lines, flt_data = _read_lines(skiprows, filename)
# read in comment lines
skiprows += flt_lines + 1
cmt_lines, cmt_data = _read_lines(skiprows, filename)
skiprows += cmt_lines + 1
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including cosmos here, don't provide this. We'll flag it as "--".
hdr = _get_header_info(int_data, flt_data, lines,
cmt_data, location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# read in the data
nrows, data = _read_lines(skiprows, filename)
# check units
unit = hdr['format_specific']['physical_units']
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('COSMOS: %s is not a supported unit.' % unit)
if hdr['standard']['units'] != 'acc':
raise GMProcessException('COSMOS: Only acceleration data accepted.')
trace = StationTrace(data.copy(), Stats(hdr.copy()))
# record that this data has been converted to g, if it has
if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def test_rotd():
ddir = os.path.join('data', 'testdata', 'process')
datadir = pkg_resources.resource_filename('gmprocess', ddir)
# Create a stream and station summary, convert from m/s^2 to cm/s^2 (GAL)
osc1_data = np.genfromtxt(datadir + '/ALCTENE.UW..sac.acc.final.txt')
osc2_data = np.genfromtxt(datadir + '/ALCTENN.UW..sac.acc.final.txt')
osc1_data = osc1_data.T[1] * 100
osc2_data = osc2_data.T[1] * 100
tr1 = StationTrace(data=osc1_data, header={
'channel': 'HN1', 'delta': 0.01,
'npts': 24001,
'standard': {
'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'source_file': '',
'structure_type': '',
'horizontal_orientation': np.nan,
'vertical_orientation': np.nan,
'sensor_serial_number': '',
'process_level': 'corrected physical units',
'process_time': '',
'units': 'acc',
'units_type': 'acc',
'instrument_sensitivity': np.nan,
'instrument_damping': np.nan
}
})
tr2 = StationTrace(data=osc2_data, header={
'channel': 'HN2', 'delta': 0.01,
'npts': 24001, 'standard': {
'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'structure_type': '',
'source_file': '',
'horizontal_orientation': np.nan,
'vertical_orientation': np.nan,
'sensor_serial_number': '',
'process_level': 'corrected physical units',
'process_time': '',
'units': 'acc',
'units_type': 'acc',
'instrument_sensitivity': np.nan,
'instrument_damping': np.nan
}
})
st = StationStream([tr1, tr2])
for tr in st:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
tr.setProvenance('remove_response', response)
target_pga50 = 4.12528265306
target_sa1050 = 10.7362857143
target_pgv50 = 6.239364
target_sa0350 = 10.1434159021
target_sa3050 = 1.12614169215
station = StationSummary.from_stream(
st, ['rotd50'],
['pga', 'pgv', 'sa0.3', 'sa1.0', 'sa3.0']
)
pgms = station.pgms
pga = pgms.loc['PGA', 'ROTD(50.0)'].Result
pgv = pgms.loc['PGV', 'ROTD(50.0)'].Result
SA10 = pgms.loc['SA(1.000)', 'ROTD(50.0)'].Result
SA03 = pgms.loc['SA(0.300)', 'ROTD(50.0)'].Result
SA30 = pgms.loc['SA(3.000)', 'ROTD(50.0)'].Result
np.testing.assert_allclose(pga, target_pga50, atol=0.1)
np.testing.assert_allclose(SA10, target_sa1050, atol=0.1)
np.testing.assert_allclose(pgv, target_pgv50, atol=0.1)
np.testing.assert_allclose(SA03, target_sa0350, atol=0.1)
np.testing.assert_allclose(SA30, target_sa3050, atol=0.1)
def test_radial_transverse():
origin = Origin(latitude=47.149, longitude=-122.7266667)
st = read(os.path.join(datadir, 'resp_cor', 'UW.ALCT.--.*.MSEED'))
st[0].stats.standard = {}
st[0].stats.standard['horizontal_orientation'] = 0.
st[0].stats['channel'] = 'HN1'
st[1].stats.standard = {}
st[1].stats.standard['horizontal_orientation'] = 90.
st[1].stats['channel'] = 'HN2'
st[2].stats.standard = {}
st[2].stats.standard['horizontal_orientation'] = np.nan
st[2].stats['channel'] = 'HNZ'
inv = read_inventory(os.path.join(datadir, 'inventory.xml'))
stalat, stalon = inv[0][0][0].latitude, inv[0][0][0].longitude
for i, tr in enumerate(st):
tr.stats['coordinates'] = {'latitude': stalat}
tr.stats['coordinates']['longitude'] = stalon
tr.stats['standard'].update({'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'instrument': '',
'instrument_period': np.nan,
'vertical_orientation': np.nan,
'source_format': 'json',
'comments': '',
'structure_type': '',
'source_file': '',
'sensor_serial_number': '',
'process_level': 'raw counts',
'process_time': '',
'units': 'acc',
'units_type': 'acc',
'instrument_sensitivity': np.nan,
'instrument_damping': np.nan})
baz = gps2dist_azimuth(stalat, stalon,
origin.latitude, origin.longitude)[1]
st1 = st.copy()
st1[0].stats.channel = st1[0].stats.channel[:-1] + 'N'
st1[1].stats.channel = st1[1].stats.channel[:-1] + 'E'
st1.rotate(method='NE->RT', back_azimuth=baz)
pgms = np.abs(st1.max())
st2 = StationStream([])
for t in st:
st2.append(StationTrace(t.data, t.stats))
for tr in st2:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
tr.setProvenance('remove_response', response)
summary = StationSummary.from_stream(
st2, ['radial_transverse'], ['pga'], origin)
pgmdf = summary.pgms
R = pgmdf.loc['PGA', 'HNR'].Result
T = pgmdf.loc['PGA', 'HNT'].Result
np.testing.assert_almost_equal(
pgms[0], sp.g * R)
np.testing.assert_almost_equal(
pgms[1], sp.g * T)
# Test with a station whose channels are not aligned to E-N
SEW_st = read(os.path.join(datadir, 'resp_cor', 'GS.SEW.*.mseed'))
SEW_inv = read_inventory(os.path.join(datadir, 'inventory_sew.xml'))
stalat, stalon = inv[0][0][0].latitude, inv[0][0][0].longitude
# This needs to be checked. The target data doesn't appear to be
# correct. This can be updated when a tolerance is added to the rotate
# method.
"""traces = []
for tr in SEW_st:
tr.stats.coordinates = {'latitude': stalat,
'longitude': stalon}
tr.stats.standard = {'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'structure_type': '',
'sensor_serial_number': '',
'process_level': 'raw counts',
'process_time': '',
'horizontal_orientation':
SEW_inv.get_channel_metadata(tr.get_id())['azimuth'],
'units': 'acc',
'instrument_damping': np.nan}
traces += [StationTrace(tr.data, tr.stats)]
baz = gps2dist_azimuth(stalat, stalon,
origin.latitude, origin.longitude)[1]
SEW_st_copy = StationStream(traces)
SEW_st_copy.rotate(method='->NE', inventory=SEW_inv)
SEW_st_copy.rotate(method='NE->RT', back_azimuth=baz)
pgms = np.abs(SEW_st_copy.max())
summary = StationSummary.from_stream(
SEW_st, ['radial_transverse'], ['pga'], origin)
np.testing.assert_almost_equal(
pgms[1], sp.g * summary.pgms['PGA']['R'])
np.testing.assert_almost_equal(
pgms[2], sp.g * summary.pgms['PGA']['T'])"""
# Test failure case without two horizontal channels
copy1 = st2.copy()
copy1[0].stats.channel = copy1[0].stats.channel[:-1] + '3'
pgms = StationSummary.from_stream(
copy1, ['radial_transverse'], ['pga'], origin).pgms
assert np.isnan(pgms.loc['PGA', 'HNR'].Result)
assert np.isnan(pgms.loc['PGA', 'HNT'].Result)
# Test failure case when channels are not orthogonal
copy3 = st2.copy()
copy3[0].stats.standard.horizontal_orientation = 100
pgms = StationSummary.from_stream(
copy3, ['radial_transverse'], ['pga'], origin).pgms
assert np.isnan(pgms.loc['PGA', 'HNR'].Result)
assert np.isnan(pgms.loc['PGA', 'HNT'].Result)
def _read_channel(filename, line_offset, location=''):
"""Read channel data from COSMOS V1/V2 text file.
Args:
filename (str): Input COSMOS V1/V2 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station, location, and process level from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# read in lines of integer data
skiprows = line_offset + TEXT_HDR_ROWS
int_lines, int_data = _read_lines(skiprows, filename)
int_data = int_data.astype(np.int32)
# read in lines of float data
skiprows += int_lines + 1
flt_lines, flt_data = _read_lines(skiprows, filename)
# read in comment lines
skiprows += flt_lines + 1
cmt_lines, cmt_data = _read_lines(skiprows, filename)
skiprows += cmt_lines + 1
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including cosmos here, don't provide this. We'll flag it as "--".
hdr = _get_header_info(int_data, flt_data, lines,
cmt_data, location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# read in the data
nrows, data = _read_lines(skiprows, filename)
# Check for "off-by-one" problem that sometimes occurs with cosmos data
# Notes:
# - We cannot do this check inside _get_header_info because we don't
# have the data there.
# - That method is written to set npts from the header as documented in
# the spec ("lenght" == npts*dt) but it appears that sometimes a
# different convention is used where the "length" of the record is
# actually is actuation (npts-1)*dt. In this case, we need to
# recompute duration and npts
if hdr['npts'] == (len(data) - 1):
hdr['npts'] = len(data)
hdr['duration'] = (hdr['npts'] - 1) * hdr['delta']
# check units
unit = hdr['format_specific']['physical_units']
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
if unit != 'counts':
raise GMProcessException(
'COSMOS: %s is not a supported unit.' % unit)
if hdr['standard']['units'] != 'acc':
raise GMProcessException('COSMOS: Only acceleration data accepted.')
trace = StationTrace(data.copy(), Stats(hdr.copy()))
# record that this data has been converted to g, if it has
if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def read_knet(filename):
"""Read Japanese KNET strong motion file.
Args:
filename (str): Path to possible KNET data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_knet.")
if not is_knet(filename):
raise Exception('%s is not a valid KNET file' % filename)
# Parse the header portion of the file
with open(filename, 'rt') as f:
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
hdr = {}
coordinates = {}
standard = {}
hdr['network'] = 'BO'
hdr['station'] = lines[5].split()[2]
logging.debug('station: %s' % hdr['station'])
standard['station_name'] = ''
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including KNET here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
coordinates['latitude'] = float(lines[6].split()[2])
coordinates['longitude'] = float(lines[7].split()[2])
coordinates['elevation'] = float(lines[8].split()[2])
hdr['sampling_rate'] = float(
re.search('\\d+', lines[10].split()[2]).group())
hdr['delta'] = 1 / hdr['sampling_rate']
standard['units'] = 'acc'
dir_string = lines[12].split()[1].strip()
# knet files have directions listed as N-S, E-W, or U-D,
# whereas in kiknet those directions are '4', '5', or '6'.
if dir_string in ['N-S', '1', '4']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
elif dir_string in ['E-W', '2', '5']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
elif dir_string in ['U-D', '3', '6']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
else:
raise Exception('KNET: Could not parse direction %s' %
lines[12].split()[1])
logging.debug('channel: %s' % hdr['channel'])
scalestr = lines[13].split()[2]
parts = scalestr.split('/')
num = float(parts[0].replace('(gal)', ''))
den = float(parts[1])
calib = num / den
hdr['calib'] = calib
duration = float(lines[11].split()[2])
hdr['npts'] = int(duration * hdr['sampling_rate'])
timestr = ' '.join(lines[9].split()[2:4])
# The K-NET and KiK-Net data logger adds a 15s time delay
# this is removed here
sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0)
# Shift the time to utc (Japanese time is 9 hours ahead)
sttime = sttime - timedelta(seconds=9 * 3600.)
hdr['starttime'] = sttime
# read in the data - there is a max of 8 columns per line
# the code below handles the case when last line has
# less than 8 columns
if hdr['npts'] % COLS_PER_LINE != 0:
nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE))
nrows2 = 1
else:
nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE))
nrows2 = 0
data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS,
max_rows=nrows, filling_values=np.nan)
data = data.flatten()
if nrows2:
skip_header = TEXT_HDR_ROWS + nrows
data2 = np.genfromtxt(filename, skip_header=skip_header,
max_rows=nrows2, filling_values=np.nan)
data = np.hstack((data, data2))
nrows += nrows2
# apply the correction factor we're given in the header
data *= calib
# fill out the rest of the standard dictionary
standard['horizontal_orientation'] = np.nan
standard['instrument_period'] = np.nan
standard['instrument_damping'] = np.nan
standard['process_time'] = ''
standard['process_level'] = PROCESS_LEVELS['V1']
standard['sensor_serial_number'] = ''
standard['instrument'] = ''
standard['comments'] = ''
standard['structure_type'] = ''
if dir_string in ['1', '2', '3']:
standard['structure_type'] = 'borehole'
standard['corner_frequency'] = np.nan
standard['units'] = 'acc'
standard['source'] = SRC
standard['source_format'] = 'knet'
head, tail = os.path.split(filename)
standard['source_file'] = tail or os.path.basename(head)
hdr['coordinates'] = coordinates
hdr['standard'] = standard
# create a Trace from the data and metadata
trace = StationTrace(data.copy(), Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
stream = StationStream(traces=[trace])
return [stream]
def test_fas():
"""
Testing based upon the work provided in
https://github.com/arkottke/notebooks/blob/master/effective_amp_spectrum.ipynb
"""
ddir = os.path.join('data', 'testdata')
datadir = pkg_resources.resource_filename('gmprocess', ddir)
fas_file = os.path.join(datadir, 'fas_greater_of_two_horizontals.pkl')
p1 = os.path.join(datadir, 'peer', 'RSN763_LOMAP_GIL067.AT2')
p2 = os.path.join(datadir, 'peer', 'RSN763_LOMAP_GIL337.AT2')
stream = StationStream([])
for idx, fpath in enumerate([p1, p2]):
with open(fpath) as file_obj:
for _ in range(3):
next(file_obj)
meta = re.findall(r'[.0-9]+', next(file_obj))
dt = float(meta[1])
accels = np.array(
[col for line in file_obj for col in line.split()])
trace = StationTrace(data=accels,
header={
'channel': 'H' + str(idx),
'delta': dt,
'units': 'acc',
'standard': {
'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'structure_type': '',
'sensor_serial_number': '',
'source_file': '',
'process_level': 'raw counts',
'process_time': '',
'horizontal_orientation': np.nan,
'vertical_orientation': np.nan,
'units': 'acc',
'units_type': 'acc',
'instrument_sensitivity': np.nan,
'instrument_damping': np.nan
}
})
stream.append(trace)
for tr in stream:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
tr.setProvenance('remove_response', response)
target_df = pd.read_pickle(fas_file)
ind_vals = target_df.index.values
per = np.unique(
[float(i[0].split(')')[0].split('(')[1]) for i in ind_vals])
freqs = 1 / per
imts = ['fas' + str(p) for p in per]
summary = StationSummary.from_stream(stream,
['greater_of_two_horizontals'],
imts,
bandwidth=30)
pgms = summary.pgms
# pgms.to_pickle(fas_file)
for idx, f in enumerate(freqs):
fstr = 'FAS(%.3f)' % (1 / f)
fval1 = pgms.loc[fstr, 'GREATER_OF_TWO_HORIZONTALS'].Result
fval2 = target_df.loc[fstr, 'GREATER_OF_TWO_HORIZONTALS'].Result
np.testing.assert_allclose(fval1, fval2, rtol=1e-5, atol=1e-5)
def _read_channel(filename, line_offset, volume, location='', alternate=False):
"""Read channel data from USC V1 text file.
Args:
filename (str): Input USC V1 filename.
line_offset (int): Line offset to beginning of channel text block.
volume (dictionary): Dictionary of formatting information
Returns:
tuple: (obspy Trace, int line offset)
"""
if alternate:
int_rows = 5
int_fmt = 20 * [4]
data_cols = 8
else:
int_rows = volume['INT_HDR_ROWS']
int_fmt = volume['INT_FMT']
data_cols = 10
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skiprows = line_offset + volume['TEXT_HDR_ROWS']
int_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=int_rows, dtype=np.int32,
delimiter=int_fmt).flatten()
# read in lines of float data
skiprows += int_rows
flt_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64,
delimiter=volume['FLT_FMT']).flatten()
hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
skiprows += volume['FLT_HDR_ROWS']
# read in the data
nrows = int(np.floor(hdr['npts'] * 2 / data_cols))
all_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=nrows, dtype=np.float64,
delimiter=volume['COL_FMT'])
data = all_data.flatten()[1::2]
times = all_data.flatten()[0::2]
frac = hdr['format_specific']['fractional_unit']
if frac > 0:
data *= UNIT_CONVERSIONS['g'] * frac
logging.debug('Data converted from g * %s to cm/s/s' % (frac))
else:
unit = _get_units(lines[11])
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('USC: %s is not a supported unit.' % unit)
# Put file name into dictionary
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
trace = StationTrace(data.copy(), Stats(hdr.copy()))
if not is_evenly_spaced(times):
trace = resample_uneven_trace(trace, times, data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def _read_channel(filename, line_offset):
"""Read channel data from GNS V1 text file.
Args:
filename (str): Input GNS V1 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station and location strings from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# this code supports V1 and V2 format files. Which one is this?
data_format = 'V2'
if lines[0].lower().find('uncorrected') >= 0:
data_format = 'V1'
# parse out the station code, name, and component string
# from text header
station = lines[1].split()[1]
logging.debug('station: %s' % station)
name = lines[2].replace(' ', '_').strip()
component = lines[12].split()[1]
# parse the instrument type from the text header
instrument = lines[3].split()[1]
# parse the sensor resolution from the text header
resolution_str = lines[4].split()[1]
resolution = int(re.search('\d+', resolution_str).group())
# read floating point header array
skip_header = line_offset + TEXT_HDR_ROWS
hdr_data = np.genfromtxt(filename, skip_header=skip_header,
max_rows=FP_HDR_ROWS)
# parse header dictionary from float header array
hdr = _read_header(hdr_data, station, name,
component, data_format,
instrument, resolution)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including GeoNet here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
skip_header2 = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS
widths = [8] * COLS_PER_ROW
nrows = int(np.ceil(hdr['npts'] / COLS_PER_ROW))
data = np.genfromtxt(filename, skip_header=skip_header2,
max_rows=nrows, filling_values=np.nan,
delimiter=widths)
data = data.flatten()
data = data[0:hdr['npts']]
# for debugging, read in the velocity data
nvel = hdr_data[3, 4]
if nvel:
if nvel % COLS_PER_ROW != 0:
nvel_rows = int(np.floor(nvel / COLS_PER_ROW))
nvel_rows2 = 1
else:
nvel_rows = int(np.ceil(nvel / COLS_PER_ROW))
nvel_rows2 = 0
skip_header_vel = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS + nrows
widths = [8] * COLS_PER_ROW
velocity = np.genfromtxt(filename, skip_header=skip_header_vel,
max_rows=nvel_rows, filling_values=np.nan,
delimiter=widths)
velocity = velocity.flatten()
velocity *= MMPS_TO_CMPS
else:
velocity = np.array([])
# for V2 files, there are extra blocks of data we need to skip containing
# velocity and displacement data
if data_format == 'V2':
velrows = int(np.ceil(hdr_data[3, 4] / COLS_PER_ROW))
disrows = int(np.ceil(hdr_data[3, 5] / COLS_PER_ROW))
nrows = nrows + velrows + disrows
data *= MMPS_TO_CMPS # convert to cm/s**2
trace = StationTrace(data, Stats(hdr))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
offset = skip_header2 + nrows
return (trace, offset, velocity)
def getStreams(self, eventid, stations=None, labels=None):
"""Get Stream from ASDF file given event id and input tags.
Args:
eventid (str):
Event ID corresponding to an Event in the workspace.
stations (list):
List of stations to search for.
labels (list):
List of processing labels to search for.
Returns:
StreamCollection: Object containing list of organized
StationStreams.
"""
trace_auxholder = []
stream_auxholder = []
if 'TraceProcessingParameters' in self.dataset.auxiliary_data:
trace_auxholder = self.dataset.auxiliary_data.TraceProcessingParameters
if 'StreamProcessingParameters' in self.dataset.auxiliary_data:
stream_auxholder = self.dataset.auxiliary_data.StreamProcessingParameters
streams = []
if stations is None:
stations = self.getStations(eventid)
if labels is None:
labels = self.getLabels()
# tried doing a query here using dataset ifilter on event,
# but it didn't work...
for waveform in self.dataset.waveforms:
tags = waveform.get_waveform_tags()
for tag in tags:
parts = tag.split('_')
if len(parts) > 2:
tlabel = parts[-1]
teventid = '_'.join(parts[0:-1])
else:
teventid, tlabel = tag.split('_')
if eventid != teventid:
continue
if tlabel not in labels:
continue
stream_name = list(
waveform.get_waveform_attributes().keys())[0]
parts = stream_name.split('.')
tstation = parts[1]
if tstation not in stations:
continue
tstream = waveform[tag]
inventory = waveform['StationXML']
for ttrace in tstream:
trace = StationTrace(data=ttrace.data,
header=ttrace.stats,
inventory=inventory)
# get the provenance information
provname = format_nslct(trace.stats, tag)
if provname in self.dataset.provenance.list():
provdoc = self.dataset.provenance[provname]
trace.setProvenanceDocument(provdoc)
# get the trace processing parameters
top = format_netsta(trace.stats)
trace_path = format_nslct(trace.stats, tag)
if top in trace_auxholder:
root_auxholder = trace_auxholder[top]
if trace_path in root_auxholder:
bytelist = root_auxholder[
trace_path].data[:].tolist()
jsonstr = ''.join([chr(b) for b in bytelist])
jdict = json.loads(jsonstr)
for key, value in jdict.items():
trace.setParameter(key, value)
# get the trace spectra arrays from auxiliary,
# repack into stationtrace object
spectra = {}
if 'Cache' in self.dataset.auxiliary_data:
for aux in self.dataset.auxiliary_data['Cache'].list():
auxarray = self.dataset.auxiliary_data['Cache'][aux]
if top not in auxarray.list():
continue
auxarray_top = auxarray[top]
if trace_path in auxarray_top:
specparts = camel_case_split(aux)
array_name = specparts[-1].lower()
specname = '_'.join(specparts[:-1]).lower()
specarray = auxarray_top[trace_path].data.value
if specname in spectra:
spectra[specname][array_name] = specarray
else:
spectra[specname] = {array_name: specarray}
for key, value in spectra.items():
trace.setCached(key, value)
stream = StationStream(traces=[trace])
stream.tag = tag # testing this out
# get the stream processing parameters
stream_path = format_nslit(trace.stats, stream.get_inst(), tag)
if top in stream_auxholder:
top_auxholder = stream_auxholder[top]
if stream_path in top_auxholder:
auxarray = top_auxholder[stream_path]
bytelist = auxarray.data[:].tolist()
jsonstr = ''.join([chr(b) for b in bytelist])
jdict = json.loads(jsonstr)
for key, value in jdict.items():
stream.setStreamParam(key, value)
streams.append(stream)
streams = StreamCollection(streams)
return streams
def getStreams(self, eventid, stations=None, labels=None):
"""Get Stream from ASDF file given event id and input tags.
Args:
eventid (str):
Event ID corresponding to an Event in the workspace.
stations (list):
List of stations to search for.
labels (list):
List of processing labels to search for.
Returns:
StreamCollection: Object containing list of organized
StationStreams.
"""
trace_auxholder = []
stream_auxholder = []
if 'TraceProcessingParameters' in self.dataset.auxiliary_data:
trace_auxholder = self.dataset.auxiliary_data.TraceProcessingParameters
if 'StreamProcessingParameters' in self.dataset.auxiliary_data:
stream_auxholder = self.dataset.auxiliary_data.StreamProcessingParameters
streams = []
all_tags = []
if stations is None:
stations = self.getStations(eventid)
if labels is None:
labels = self.getLabels()
for station in stations:
for label in labels:
all_tags.append('%s_%s' % (station.lower(), label))
for waveform in self.dataset.waveforms:
ttags = waveform.get_waveform_tags()
wtags = []
if not len(all_tags):
wtags = ttags
else:
wtags = list(set(all_tags).intersection(set(ttags)))
for tag in wtags:
if eventid in waveform[tag][0].stats.asdf.event_ids:
tstream = waveform[tag].copy()
inventory = waveform['StationXML']
traces = []
for ttrace in tstream:
trace = StationTrace(data=ttrace.data,
header=ttrace.stats,
inventory=inventory)
tpl = (trace.stats.network.lower(),
trace.stats.station.lower(),
trace.stats.channel.lower())
channel = '%s_%s_%s' % tpl
channel_tag = '%s_%s' % (tag, channel)
if channel_tag in self.dataset.provenance.list():
provdoc = self.dataset.provenance[channel_tag]
trace.setProvenanceDocument(provdoc)
trace_path = '%s_%s_%s' % (eventid,
tag,
trace.stats.channel)
if trace_path in trace_auxholder:
bytelist = trace_auxholder[trace_path].data[:].tolist(
)
jsonstr = ''.join([chr(b) for b in bytelist])
jdict = json.loads(jsonstr)
# jdict = unstringify_dict(jdict)
for key, value in jdict.items():
trace.setParameter(key, value)
traces.append(trace)
stream = StationStream(traces=traces)
stream.tag = tag # testing this out
# look for stream-based metadata
stream_path = '%s_%s' % (eventid, tag)
if stream_path in stream_auxholder:
bytelist = stream_auxholder[stream_path].data[:].tolist(
)
jsonstr = ''.join([chr(b) for b in bytelist])
jdict = json.loads(jsonstr)
# jdict = unstringify_dict(jdict)
for key, value in jdict.items():
stream.setStreamParam(key, value)
streams.append(stream)
streams = StreamCollection(streams)
return streams
def _read_channel(filename, line_offset, location=''):
"""Read channel data from COSMOS V1/V2 text file.
Args:
filename (str): Input COSMOS V1/V2 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station, location, and process level from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# read in lines of integer data
skiprows = line_offset + TEXT_HDR_ROWS
int_lines, int_data = _read_lines(skiprows, filename)
int_data = int_data.astype(np.int32)
# read in lines of float data
skiprows += int_lines + 1
flt_lines, flt_data = _read_lines(skiprows, filename)
# read in comment lines
skiprows += flt_lines + 1
cmt_lines, cmt_data = _read_lines(skiprows, filename)
skiprows += cmt_lines + 1
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including cosmos here, don't provide this. We'll flag it as "--".
hdr = _get_header_info(int_data,
flt_data,
lines,
cmt_data,
location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# read in the data
nrows, data = _read_lines(skiprows, filename)
# check units
unit = hdr['format_specific']['physical_units']
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('COSMOS: %s is not a supported unit.' % unit)
if hdr['standard']['units'] != 'acc':
raise GMProcessException('COSMOS: Only acceleration data accepted.')
trace = StationTrace(data.copy(), Stats(hdr.copy()))
# record that this data has been converted to g, if it has
if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def read_knet(filename):
"""Read Japanese KNET strong motion file.
Args:
filename (str): Path to possible KNET data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_knet.")
if not is_knet(filename):
raise Exception('%s is not a valid KNET file' % filename)
# Parse the header portion of the file
with open(filename, 'rt') as f:
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
hdr = {}
coordinates = {}
standard = {}
hdr['network'] = 'BO'
hdr['station'] = lines[5].split()[2]
logging.debug('station: %s' % hdr['station'])
standard['station_name'] = ''
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including KNET here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
coordinates['latitude'] = float(lines[6].split()[2])
coordinates['longitude'] = float(lines[7].split()[2])
coordinates['elevation'] = float(lines[8].split()[2])
hdr['sampling_rate'] = float(
re.search('\\d+', lines[10].split()[2]).group())
hdr['delta'] = 1 / hdr['sampling_rate']
standard['units'] = 'acc'
dir_string = lines[12].split()[1].strip()
# knet files have directions listed as N-S, E-W, or U-D,
# whereas in kiknet those directions are '4', '5', or '6'.
if dir_string in ['N-S', '1', '4']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
elif dir_string in ['E-W', '2', '5']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
elif dir_string in ['U-D', '3', '6']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
else:
raise Exception('KNET: Could not parse direction %s' %
lines[12].split()[1])
logging.debug('channel: %s' % hdr['channel'])
scalestr = lines[13].split()[2]
parts = scalestr.split('/')
num = float(parts[0].replace('(gal)', ''))
den = float(parts[1])
calib = num / den
hdr['calib'] = calib
duration = float(lines[11].split()[2])
hdr['npts'] = int(duration * hdr['sampling_rate'])
timestr = ' '.join(lines[9].split()[2:4])
# The K-NET and KiK-Net data logger adds a 15s time delay
# this is removed here
sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0)
# Shift the time to utc (Japanese time is 9 hours ahead)
sttime = sttime - timedelta(seconds=9 * 3600.)
hdr['starttime'] = sttime
# read in the data - there is a max of 8 columns per line
# the code below handles the case when last line has
# less than 8 columns
if hdr['npts'] % COLS_PER_LINE != 0:
nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE))
nrows2 = 1
else:
nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE))
nrows2 = 0
data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS,
max_rows=nrows, filling_values=np.nan)
data = data.flatten()
if nrows2:
skip_header = TEXT_HDR_ROWS + nrows
data2 = np.genfromtxt(filename, skip_header=skip_header,
max_rows=nrows2, filling_values=np.nan)
data = np.hstack((data, data2))
nrows += nrows2
# apply the correction factor we're given in the header
data *= calib
# fill out the rest of the standard dictionary
standard['horizontal_orientation'] = np.nan
standard['instrument_period'] = np.nan
standard['instrument_damping'] = np.nan
standard['process_time'] = ''
standard['process_level'] = PROCESS_LEVELS['V1']
standard['sensor_serial_number'] = ''
standard['instrument'] = ''
standard['comments'] = ''
standard['structure_type'] = ''
if dir_string in ['1', '2', '3']:
standard['structure_type'] = 'borehole'
standard['corner_frequency'] = np.nan
standard['units'] = 'acc'
standard['source'] = SRC
standard['source_format'] = 'knet'
hdr['coordinates'] = coordinates
hdr['standard'] = standard
# create a Trace from the data and metadata
trace = StationTrace(data.copy(), Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
stream = StationStream(traces=[trace])
return [stream]
def _read_channel(filename, line_offset):
"""Read channel data from GNS V1 text file.
Args:
filename (str): Input GNS V1 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station and location strings from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# this code supports V1 and V2 format files. Which one is this?
data_format = 'V2'
if lines[0].lower().find('uncorrected') >= 0:
data_format = 'V1'
# parse out the station code, name, and component string
# from text header
station = lines[1].split()[1]
logging.debug('station: %s' % station)
name = lines[2].replace(' ', '_').strip()
component = lines[12].split()[1]
# parse the instrument type from the text header
instrument = lines[3].split()[1]
# parse the sensor resolution from the text header
resolution_str = lines[4].split()[1]
resolution = int(re.search('\d+', resolution_str).group())
# read floating point header array
skip_header = line_offset + TEXT_HDR_ROWS
hdr_data = np.genfromtxt(filename,
skip_header=skip_header,
max_rows=FP_HDR_ROWS)
# parse header dictionary from float header array
hdr = _read_header(hdr_data, station, name, component, data_format,
instrument, resolution)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including GeoNet here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
skip_header2 = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS
widths = [8] * COLS_PER_ROW
nrows = int(np.ceil(hdr['npts'] / COLS_PER_ROW))
data = np.genfromtxt(filename,
skip_header=skip_header2,
max_rows=nrows,
filling_values=np.nan,
delimiter=widths)
data = data.flatten()
data = data[0:hdr['npts']]
# for debugging, read in the velocity data
nvel = hdr_data[3, 4]
if nvel:
if nvel % COLS_PER_ROW != 0:
nvel_rows = int(np.floor(nvel / COLS_PER_ROW))
nvel_rows2 = 1
else:
nvel_rows = int(np.ceil(nvel / COLS_PER_ROW))
nvel_rows2 = 0
skip_header_vel = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS + nrows
widths = [8] * COLS_PER_ROW
velocity = np.genfromtxt(filename,
skip_header=skip_header_vel,
max_rows=nvel_rows,
filling_values=np.nan,
delimiter=widths)
velocity = velocity.flatten()
velocity *= MMPS_TO_CMPS
else:
velocity = np.array([])
# for V2 files, there are extra blocks of data we need to skip containing
# velocity and displacement data
if data_format == 'V2':
velrows = int(np.ceil(hdr_data[3, 4] / COLS_PER_ROW))
disrows = int(np.ceil(hdr_data[3, 5] / COLS_PER_ROW))
nrows = nrows + velrows + disrows
data *= MMPS_TO_CMPS # convert to cm/s**2
trace = StationTrace(data, Stats(hdr))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
offset = skip_header2 + nrows
return (trace, offset, velocity)
def read_cwb(filename, **kwargs):
"""Read Taiwan Central Weather Bureau strong motion file.
Args:
filename (str): Path to possible CWB data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration
data (cm/s**2).
"""
logging.debug("Starting read_cwb.")
if not is_cwb(filename):
raise Exception('%s is not a valid CWB strong motion data file.'
% filename)
f = open(filename, 'rt')
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including CWB here, don't provide this. We'll flag it as "--".
data = np.genfromtxt(filename, skip_header=HDR_ROWS,
delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW
hdr = _get_header_info(f, data)
f.close()
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
hdr_z = hdr.copy()
hdr_z['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
hdr_z['standard']['horizontal_orientation'] = np.nan
hdr_h1 = hdr.copy()
hdr_h1['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
hdr_h1['standard']['horizontal_orientation'] = np.nan
hdr_h2 = hdr.copy()
hdr_h2['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
hdr_h2['standard']['horizontal_orientation'] = np.nan
stats_z = Stats(hdr_z)
stats_h1 = Stats(hdr_h1)
stats_h2 = Stats(hdr_h2)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace_z = StationTrace(data=data[:, 1], header=stats_z)
trace_z.setProvenance('remove_response', response)
trace_h1 = StationTrace(data=data[:, 2], header=stats_h1)
trace_h1.setProvenance('remove_response', response)
trace_h2 = StationTrace(data=data[:, 3], header=stats_h2)
trace_h2.setProvenance('remove_response', response)
stream = StationStream([trace_z, trace_h1, trace_h2])
return [stream]
def getStreams(self, eventid, stations=None, labels=None):
"""Get Stream from ASDF file given event id and input tags.
Args:
eventid (str):
Event ID corresponding to an Event in the workspace.
stations (list):
List of stations to search for.
labels (list):
List of processing labels to search for.
Returns:
StreamCollection: Object containing list of organized
StationStreams.
"""
trace_auxholder = []
stream_auxholder = []
if 'TraceProcessingParameters' in self.dataset.auxiliary_data:
trace_auxholder = self.dataset.auxiliary_data.TraceProcessingParameters
if 'StreamProcessingParameters' in self.dataset.auxiliary_data:
stream_auxholder = self.dataset.auxiliary_data.StreamProcessingParameters
streams = []
all_tags = []
if stations is None:
stations = self.getStations(eventid)
if labels is None:
labels = self.getLabels()
for station in stations:
for label in labels:
all_tags.append('%s_%s_%s' % (eventid, station.lower(), label))
for waveform in self.dataset.waveforms:
ttags = waveform.get_waveform_tags()
wtags = []
if not len(all_tags):
wtags = ttags
else:
wtags = list(set(all_tags).intersection(set(ttags)))
for tag in wtags:
if eventid in waveform[tag][0].stats.asdf.event_ids:
tstream = waveform[tag].copy()
inventory = waveform['StationXML']
for ttrace in tstream:
trace = StationTrace(data=ttrace.data,
header=ttrace.stats,
inventory=inventory)
tpl = (trace.stats.network.lower(),
trace.stats.station.lower(),
trace.stats.channel.lower())
channel = '%s_%s_%s' % tpl
channel_tag = '%s_%s' % (tag, channel)
if channel_tag in self.dataset.provenance.list():
provdoc = self.dataset.provenance[channel_tag]
trace.setProvenanceDocument(provdoc)
trace_path = '%s_%s' % (
tag, trace.stats.channel)
if trace_path in trace_auxholder:
bytelist = trace_auxholder[
trace_path].data[:].tolist()
jsonstr = ''.join([chr(b) for b in bytelist])
jdict = json.loads(jsonstr)
for key, value in jdict.items():
trace.setParameter(key, value)
stream = StationStream(traces=[trace])
stream.tag = tag # testing this out
# look for stream-based metadata
if tag in stream_auxholder:
bytelist = stream_auxholder[
tag].data[:].tolist()
jsonstr = ''.join([chr(b) for b in bytelist])
jdict = json.loads(jsonstr)
for key, value in jdict.items():
stream.setStreamParam(key, value)
streams.append(stream)
streams = StreamCollection(streams)
return streams
def test_fas():
"""
Testing based upon the work provided in
https://github.com/arkottke/notebooks/blob/master/effective_amp_spectrum.ipynb
"""
ddir = os.path.join('data', 'testdata')
datadir = pkg_resources.resource_filename('gmprocess', ddir)
fas_file = os.path.join(datadir, 'fas_results.txt')
p1 = os.path.join(datadir, 'peer', 'RSN763_LOMAP_GIL067.AT2')
p2 = os.path.join(datadir, 'peer', 'RSN763_LOMAP_GIL337.AT2')
stream = StationStream([])
for idx, fpath in enumerate([p1, p2]):
with open(fpath) as file_obj:
for _ in range(3):
next(file_obj)
meta = re.findall(r'[.0-9]+', next(file_obj))
count = int(meta[0])
dt = float(meta[1])
accels = np.array(
[col for line in file_obj for col in line.split()])
trace = StationTrace(data=accels, header={
'channel': 'H' + str(idx),
'delta': dt,
'units': 'acc',
'standard': {'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'structure_type': '',
'sensor_serial_number': '',
'source_file': '',
'process_level': 'raw counts',
'process_time': '',
'horizontal_orientation': np.nan,
'units': 'acc',
'units_type': 'acc',
'instrument_sensitivity': np.nan,
'instrument_damping': np.nan}})
stream.append(trace)
for tr in stream:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
tr.setProvenance('remove_response', response)
freqs, fas = np.loadtxt(fas_file, unpack=True,
usecols=(0, 1), delimiter=',')
# scaling required on the test data as it was not accounted for originally
imts = ['fas' + str(1 / p) for p in freqs]
summary = StationSummary.from_stream(stream, ['quadratic_mean'], imts,
bandwidth=30)
pgms = summary.pgms
for idx, f in enumerate(freqs):
fstr = 'FAS(' + str(1 / f) + ')'
fval = pgms[pgms.IMT == fstr].Result.tolist()[0]
np.testing.assert_array_almost_equal(
fval, fas[idx] / len(stream[0].data))
# test exceptions
failed = False
try:
fas_dict = calculate_fas(
stream, '', 1 / freqs, 'some other smoothing', 30)
except Exception as e:
failed = True
assert(failed == True)
failed = False
invalid_channels_stream = stream
invalid_channels_stream[0].stats.channel = 'Z'
try:
fas_dict = calculate_fas(
invalid_channels_stream, '', 1 / freqs, 'konno_ohmachi', 30)
except Exception as e:
failed = True
assert(failed == True)
failed = False
invalid_units_stream = stream
invalid_units_stream[0].stats.units = 'other'
try:
fas_dict = calculate_fas(invalid_units_stream,
'', 1 / freqs, 'konno_ohmachi', 30)
except Exception as e:
failed = True
assert(failed == True)
def _read_volume_one(filename, line_offset, location='', units='acc'):
"""Read channel data from DMG Volume 1 text file.
Args:
filename (str): Input DMG V1 filename.
line_offset (int): Line offset to beginning of channel text block.
units (str): units to get
Returns:
tuple: (list of obspy Trace, int line offset)
"""
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
unit = _get_units(lines[11])
# read in lines of integer data
skip_rows = V1_TEXT_HDR_ROWS + line_offset
int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename)
int_data = int_data[0:100].astype(np.int32)
# read in lines of float data
skip_rows += V1_INT_HDR_ROWS
flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename)
skip_rows += V1_REAL_HDR_ROWS
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including csmip/dmg here, don't always provide this. We'll flag it as
# "--".
hdr = _get_header_info_v1(int_data,
flt_data,
lines,
'V1',
location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# sometimes (??) a line of text is inserted in between the float header and
# the beginning of the data. Let's check for this...
with open(filename, 'rt') as f:
for _ in range(skip_rows):
next(f)
test_line = f.readline()
has_text = re.search('[A-Z]+|[a-z]+', test_line) is not None
if has_text:
skip_rows += 1
widths = [9] * 8
max_rows = int(np.ceil(hdr['npts'] / 8))
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[:hdr['npts']]
evenly_spaced = True
# Sometimes, npts is incrrectly specified, leading to nans
# in the resulting data. Fix that here
if np.any(np.isnan(acc_data)):
while np.isnan(acc_data[-1]):
acc_data = acc_data[:-1]
hdr['npts'] = len(acc_data)
else:
# acceleration data is interleaved between time data
max_rows = int(np.ceil(hdr['npts'] / 5))
widths = [7] * 10
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[1::2][:hdr['npts']]
times = data[0::2][:hdr['npts']]
evenly_spaced = is_evenly_spaced(times)
if unit in UNIT_CONVERSIONS:
acc_data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('DMG: %s is not a supported unit.' % unit)
acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()))
# Check if the times were included in the file but were not evenly spaced
if not evenly_spaced:
acc_trace = resample_uneven_trace(acc_trace, times, acc_data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
acc_trace.setProvenance('remove_response', response)
traces = [acc_trace]
new_offset = skip_rows + max_rows + 1 # there is an end of record line
return (traces, new_offset)
def test_duration():
ddir = os.path.join('data', 'testdata')
datadir = pkg_resources.resource_filename('gmprocess', ddir)
data_file = os.path.join(datadir, 'duration_data.json')
with open(data_file, 'rt') as f:
jdict = json.load(f)
time = np.array(jdict['time'])
# input output is m/s/s
acc = np.array(jdict['acc']) / 100
target_d595 = jdict['d595']
delta = time[2] - time[1]
sr = 1 / delta
header = {
'delta': delta,
'sampling_rate': sr,
'npts': len(acc),
'units': 'm/s/s',
'channel': 'HN1',
'standard': {
'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'source_file': '',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'structure_type': '',
'sensor_serial_number': '',
'process_level': 'raw counts',
'process_time': '',
'horizontal_orientation': np.nan,
'units': 'acc',
'units_type': 'acc',
'instrument_sensitivity': np.nan,
'instrument_damping': np.nan
}
}
# input is cm/s/s output is m/s/s
trace = StationTrace(data=acc * 100, header=header)
trace2 = trace.copy()
trace2.stats.channel = 'HN2'
stream = StationStream([trace, trace2])
for tr in stream:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
tr.setProvenance('remove_response', response)
station = StationSummary.from_stream(stream, ['ARITHMETIC_MEAN'],
['duration'])
pgms = station.pgms
d595 = pgms[(pgms.IMT == 'DURATION')
& (pgms.IMC == 'ARITHMETIC_MEAN')].Result.tolist()[0]
np.testing.assert_allclose(d595, target_d595, atol=1e-4, rtol=1e-4)
# Test other components
data_files, _ = read_data_dir('cwb', 'us1000chhc', '2-ECU.dat')
stream = read_data(data_files[0])[0]
station = StationSummary.from_stream(stream, [
'channels', 'gmrotd', 'rotd50', 'greater_of_two_horizontals',
'ARITHMETIC_MEAN', 'geometric_mean'
], ['duration'])
# Currently disallowed
assert 'gmrotd' not in station.pgms['IMC']
assert 'rotd50' not in station.pgms['IMC']
print(station)
def read_renadic(filename):
"""Read the Chilean RENADIC strong motion data format.
Args:
filename (str): path to RENADIC data file.
Returns:
list: Sequence of one StationStream object containing 3
StationTrace objects.
"""
# This network does not include station coordinates in the data files,
# but they did provide a PDF table with information about each station,
# including structure type (free field or something else) and the
# coordinates
data_dir = pkg_resources.resource_filename('gmprocess',
'data')
tablefile = os.path.join(data_dir, 'station_coordinates.xlsx')
table = pd.read_excel(tablefile)
with open(filename, 'rt', encoding=ENCODING) as f:
lines1 = [next(f) for x in range(TEXT_HDR_ROWS)]
header1 = _read_header(lines1, filename, table)
ndata_rows = int(np.ceil((header1['npts'] * 2) / NCOLS))
skip_rows = TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS
data1 = _read_data(filename, skip_rows, header1['npts'])
skip_rows += ndata_rows + 1
with open(filename, 'rt', encoding=ENCODING) as f:
[next(f) for x in range(skip_rows)]
lines2 = [next(f) for x in range(TEXT_HDR_ROWS)]
header2 = _read_header(lines2, filename, table)
skip_rows += TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS
data2 = _read_data(filename, skip_rows, header1['npts'])
skip_rows += ndata_rows + 1
with open(filename, 'rt', encoding=ENCODING) as f:
[next(f) for x in range(skip_rows)]
lines3 = [next(f) for x in range(TEXT_HDR_ROWS)]
header3 = _read_header(lines3, filename, table)
skip_rows += TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS
data3 = _read_data(filename, skip_rows, header1['npts'])
trace1 = StationTrace(data=data1, header=header1)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace1.setProvenance('remove_response', response)
trace2 = StationTrace(data=data2, header=header2)
trace2.setProvenance('remove_response', response)
trace3 = StationTrace(data=data3, header=header3)
trace3.setProvenance('remove_response', response)
stream = StationStream(traces=[trace1, trace2, trace3])
return [stream]
def test_rotation():
# Create a stream and station summary, convert from m/s^2 to cm/s^2 (GAL)
osc1_data = np.genfromtxt(datadir + '/ALCTENE.UW..sac.acc.final.txt')
osc2_data = np.genfromtxt(datadir + '/ALCTENN.UW..sac.acc.final.txt')
osc1_data = osc1_data.T[1] * 100
osc2_data = osc2_data.T[1] * 100
tr1 = StationTrace(data=osc1_data,
header={
'channel': 'HN1',
'delta': 0.01,
'npts': len(osc1_data),
'standard': {
'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'structure_type': '',
'sensor_serial_number': '',
'process_level': 'raw counts',
'process_time': '',
'source_file': '',
'horizontal_orientation': np.nan,
'units': 'acc',
'instrument_damping': np.nan
}
})
tr2 = StationTrace(data=osc2_data,
header={
'channel': 'HN2',
'delta': 0.01,
'npts': len(osc2_data),
'standard': {
'corner_frequency': np.nan,
'station_name': '',
'source': 'json',
'instrument': '',
'instrument_period': np.nan,
'source_format': 'json',
'comments': '',
'source_file': '',
'structure_type': '',
'sensor_serial_number': '',
'process_level': 'raw counts',
'process_time': '',
'horizontal_orientation': np.nan,
'units': 'acc',
'instrument_damping': np.nan
}
})
st = StationStream([tr1, tr2])
rotation_class = Rotation(st)
# Test that GM, AM, and MAX work as expected with simple 1D datasets
osc1 = np.asarray([0.0, 1.0, 2.0, 3.0])
osc2 = np.asarray([4.0, 5.0, 6.0, 7.0])
max_gm = rotation_class.get_max(osc1, 'gm', osc2)
np.testing.assert_allclose(max_gm, 4.5826, atol=0.0001)
max_am = rotation_class.get_max(osc1, 'am', osc2)
np.testing.assert_allclose(max_am, 5.0, atol=0.0001)
max_max = rotation_class.get_max(osc1, 'max', osc2)
np.testing.assert_allclose(max_max, 7.0, atol=0.0001)
# Test max for 1 1d Array
osc1 = np.array([0.0, 1.0, 2.0])
max_val = rotation_class.get_max(osc1, 'max')
assert max_val == 2.0
# Test arithmetic mean with 2D input
osc1 = np.array([[0.0, 1.0], [2.0, 3.0]])
osc2 = np.array([[4.0, 5.0], [6.0, 7.0]])
means = rotation_class.get_max(osc1, 'am', osc2)[0]
assert (means[0] == 3.0 and means[1] == 5.0)
# Test greater of two horizontals
maxs = rotation_class.get_max(osc1, 'max', osc2)[0]
assert (maxs[0] == 5.0 and maxs[1] == 7.0)
# Invalid dimensions
osc1 = np.zeros((2, 3, 2))
try:
rotation_class.get_max(osc1, 'gm')
success = True
except Exception:
success = False
assert success is False
# dimensions don't match
osc1 = np.array([1.0, 2.0])
osc2 = np.array([[1.0, 2.0], [3.0, 4.0]])
try:
rotation_class.get_max(osc1, 'gm', osc2)
success = True
except Exception:
success = False
assert success is False
# Both invalid dimensions
osc1 = np.zeros((2, 3, 2))
osc2 = np.zeros((2, 3, 2))
try:
rotation_class.get_max(osc1, 'gm', osc2)
success = True
except Exception:
success = False
assert success is False
# Invalid method pick
try:
osc1 = np.array([0.0, 1.0, 2.0])
rotation_class.get_max(osc1, 'foo')
success = True
except Exception:
success = False
assert success is False
def read_cwb(filename, **kwargs):
"""Read Taiwan Central Weather Bureau strong motion file.
Args:
filename (str): Path to possible CWB data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration
data (cm/s**2).
"""
logging.debug("Starting read_cwb.")
if not is_cwb(filename):
raise Exception('%s is not a valid CWB strong motion data file.' %
filename)
f = open(filename, 'rt')
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including CWB here, don't provide this. We'll flag it as "--".
data = np.genfromtxt(filename,
skip_header=HDR_ROWS,
delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW
hdr = _get_header_info(f, data)
f.close()
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
hdr_z = hdr.copy()
hdr_z['channel'] = get_channel_name(hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
hdr_z['standard']['horizontal_orientation'] = np.nan
hdr_z['standard']['vertical_orientation'] = np.nan
hdr_z['standard']['units_type'] = get_units_type(hdr_z['channel'])
hdr_h1 = hdr.copy()
hdr_h1['channel'] = get_channel_name(hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
hdr_h1['standard']['horizontal_orientation'] = np.nan
hdr_h1['standard']['vertical_orientation'] = np.nan
hdr_h1['standard']['units_type'] = get_units_type(hdr_h1['channel'])
hdr_h2 = hdr.copy()
hdr_h2['channel'] = get_channel_name(hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
hdr_h2['standard']['horizontal_orientation'] = np.nan
hdr_h2['standard']['vertical_orientation'] = np.nan
hdr_h2['standard']['units_type'] = get_units_type(hdr_h2['channel'])
stats_z = Stats(hdr_z)
stats_h1 = Stats(hdr_h1)
stats_h2 = Stats(hdr_h2)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace_z = StationTrace(data=data[:, 1], header=stats_z)
trace_z.setProvenance('remove_response', response)
trace_h1 = StationTrace(data=data[:, 2], header=stats_h1)
trace_h1.setProvenance('remove_response', response)
trace_h2 = StationTrace(data=data[:, 3], header=stats_h2)
trace_h2.setProvenance('remove_response', response)
stream = StationStream([trace_z, trace_h1, trace_h2])
return [stream]
def read_unam(filename):
"""Read the Mexican UNAM strong motion data format.
Args:
filename (str): path to UNAM data file.
Returns:
list: Sequence of one StationStream object containing 3
StationTrace objects.
"""
channels = _read_header(filename)
npts = channels[0]['npts']
all_data = np.genfromtxt(filename, skip_header=ALL_HEADERS, max_rows=npts)
trace1 = StationTrace(data=all_data[:, 0], header=channels[0])
trace2 = StationTrace(data=all_data[:, 1], header=channels[1])
trace3 = StationTrace(data=all_data[:, 2], header=channels[2])
# tell the trace that data has already been converted to physical units
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace1.setProvenance('remove_response', response)
trace2.setProvenance('remove_response', response)
trace3.setProvenance('remove_response', response)
stream = StationStream(traces=[trace1, trace2, trace3])
return [stream]
def _read_channel(filename, line_offset, volume, location='', alternate=False):
"""Read channel data from USC V1 text file.
Args:
filename (str): Input USC V1 filename.
line_offset (int): Line offset to beginning of channel text block.
volume (dictionary): Dictionary of formatting information
Returns:
tuple: (obspy Trace, int line offset)
"""
if alternate:
int_rows = 5
int_fmt = 20 * [4]
data_cols = 8
else:
int_rows = volume['INT_HDR_ROWS']
int_fmt = volume['INT_FMT']
data_cols = 10
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skiprows = line_offset + volume['TEXT_HDR_ROWS']
int_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=int_rows, dtype=np.int32,
delimiter=int_fmt).flatten()
# read in lines of float data
skiprows += int_rows
flt_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64,
delimiter=volume['FLT_FMT']).flatten()
hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
skiprows += volume['FLT_HDR_ROWS']
# read in the data
nrows = int(np.floor(hdr['npts'] * 2 / data_cols))
all_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=nrows, dtype=np.float64,
delimiter=volume['COL_FMT'])
data = all_data.flatten()[1::2]
times = all_data.flatten()[0::2]
frac = hdr['format_specific']['fractional_unit']
if frac > 0:
data *= UNIT_CONVERSIONS['g'] * frac
logging.debug('Data converted from g * %s to cm/s/s' % (frac))
else:
unit = _get_units(lines[11])
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('USC: %s is not a supported unit.' % unit)
# Put file name into dictionary
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
trace = StationTrace(data.copy(), Stats(hdr.copy()))
if not is_evenly_spaced(times):
trace = resample_uneven_trace(trace, times, data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def read_smc(filename, **kwargs):
"""Read SMC strong motion file.
Args:
filename (str): Path to possible SMC data file.
kwargs (ref):
any_structure (bool): Read data from any type of structure,
raise Exception if False and structure type is not free-field.
accept_flagged (bool): accept problem flagged data.
set_location (str): Two character code for location.
Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing one channel of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_smc.")
any_structure = kwargs.get('any_structure', False)
accept_flagged = kwargs.get('accept_flagged', False)
location = kwargs.get('location', '')
if not is_smc(filename):
raise Exception('%s is not a valid SMC file' % filename)
with open(filename, 'rt') as f:
line = f.readline().strip()
if 'DISPLACEMENT' in line:
raise GMProcessException('SMC: Diplacement records are not supported: '
'%s.' % filename)
elif 'VELOCITY' in line:
raise GMProcessException('SMC: Velocity records are not supported: '
'%s.' % filename)
elif line == "*":
raise GMProcessException('SMC: No record volume specified in file: '
'%s.' % filename)
stats, num_comments = _get_header_info(
filename, any_structure=any_structure,
accept_flagged=accept_flagged, location=location)
skip = ASCII_HEADER_LINES + INTEGER_HEADER_LINES + \
num_comments + FLOAT_HEADER_LINES
# read float data (8 columns per line)
nrows = int(np.floor(stats['npts'] / DATA_COLUMNS))
data = np.genfromtxt(filename,
max_rows=nrows,
skip_header=skip,
delimiter=FLOAT_DATA_WIDTHS)
data = data.flatten()
if stats['npts'] % DATA_COLUMNS:
lastrow = np.genfromtxt(filename, max_rows=1,
skip_header=skip + nrows,
delimiter=FLOAT_DATA_WIDTHS)
data = np.append(data, lastrow)
data = data[0:stats['npts']]
trace = StationTrace(data, header=stats)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
stream = StationStream(traces=[trace])
return [stream]
