def read_unam(filename):
"""Read the Mexican UNAM strong motion data format.
Args:
filename (str): path to UNAM data file.
Returns:
list: Sequence of one StationStream object containing 3
StationTrace objects.
"""
channels = _read_header(filename)
npts = channels[0]['npts']
all_data = np.genfromtxt(filename, skip_header=ALL_HEADERS, max_rows=npts)
trace1 = StationTrace(data=all_data[:, 0], header=channels[0])
trace2 = StationTrace(data=all_data[:, 1], header=channels[1])
trace3 = StationTrace(data=all_data[:, 2], header=channels[2])
# tell the trace that data has already been converted to physical units
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace1.setProvenance('remove_response', response)
trace2.setProvenance('remove_response', response)
trace3.setProvenance('remove_response', response)
stream = StationStream(traces=[trace1, trace2, trace3])
return [stream]
def test_trace():
data = np.random.rand(1000)
header = {'sampling_rate': 1,
'npts': len(data),
'network': 'US',
'location': '11',
'station': 'ABCD',
'channel': 'HN1',
'starttime': UTCDateTime(2010, 1, 1, 0, 0, 0)}
inventory = get_inventory()
invtrace = StationTrace(data=data,
header=header, inventory=inventory)
invtrace.setProvenance('detrend', {'detrending_method': 'demean'})
invtrace.setParameter('failed', True)
invtrace.setParameter('corner_frequencies', [1, 2, 3])
invtrace.setParameter('metadata', {'name': 'Fred'})
assert invtrace.getProvenance('detrend')[0] == {
'detrending_method': 'demean'}
assert invtrace.getParameter('failed')
assert invtrace.getParameter('corner_frequencies') == [1, 2, 3]
assert invtrace.getParameter('metadata') == {'name': 'Fred'}
def test_trace():
data = np.random.rand(1000)
header = {
'sampling_rate': 1,
'npts': len(data),
'network': 'US',
'location': '11',
'station': 'ABCD',
'channel': 'HN1',
'starttime': UTCDateTime(2010, 1, 1, 0, 0, 0)
}
inventory = get_inventory()
invtrace = StationTrace(data=data, header=header, inventory=inventory)
invtrace.setProvenance('detrend', {'detrending_method': 'demean'})
invtrace.setParameter('failed', True)
invtrace.setParameter('corner_frequencies', [1, 2, 3])
invtrace.setParameter('metadata', {'name': 'Fred'})
assert invtrace.getProvenance('detrend')[0] == {
'detrending_method': 'demean'
}
assert invtrace.getParameter('failed')
assert invtrace.getParameter('corner_frequencies') == [1, 2, 3]
assert invtrace.getParameter('metadata') == {'name': 'Fred'}
def read_renadic(filename):
"""Read the Chilean RENADIC strong motion data format.
Args:
filename (str): path to RENADIC data file.
Returns:
list: Sequence of one StationStream object containing 3
StationTrace objects.
"""
# This network does not include station coordinates in the data files,
# but they did provide a PDF table with information about each station,
# including structure type (free field or something else) and the
# coordinates
data_dir = pkg_resources.resource_filename('gmprocess',
'data')
tablefile = os.path.join(data_dir, 'station_coordinates.xlsx')
table = pd.read_excel(tablefile)
with open(filename, 'rt', encoding=ENCODING) as f:
lines1 = [next(f) for x in range(TEXT_HDR_ROWS)]
header1 = _read_header(lines1, filename, table)
ndata_rows = int(np.ceil((header1['npts'] * 2) / NCOLS))
skip_rows = TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS
data1 = _read_data(filename, skip_rows, header1['npts'])
skip_rows += ndata_rows + 1
with open(filename, 'rt', encoding=ENCODING) as f:
[next(f) for x in range(skip_rows)]
lines2 = [next(f) for x in range(TEXT_HDR_ROWS)]
header2 = _read_header(lines2, filename, table)
skip_rows += TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS
data2 = _read_data(filename, skip_rows, header1['npts'])
skip_rows += ndata_rows + 1
with open(filename, 'rt', encoding=ENCODING) as f:
[next(f) for x in range(skip_rows)]
lines3 = [next(f) for x in range(TEXT_HDR_ROWS)]
header3 = _read_header(lines3, filename, table)
skip_rows += TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS
data3 = _read_data(filename, skip_rows, header1['npts'])
trace1 = StationTrace(data=data1, header=header1)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace1.setProvenance('remove_response', response)
trace2 = StationTrace(data=data2, header=header2)
trace2.setProvenance('remove_response', response)
trace3 = StationTrace(data=data3, header=header3)
trace3.setProvenance('remove_response', response)
stream = StationStream(traces=[trace1, trace2, trace3])
return [stream]
def read_nsmn(filename):
"""Read the Turkish NSMN strong motion data format.
Args:
filename (str): path to NSMN data file.
Returns:
list: Sequence of one StationStream object containing 3
StationTrace objects.
"""
header = _read_header(filename)
header1 = copy.deepcopy(header)
header2 = copy.deepcopy(header)
header3 = copy.deepcopy(header)
header1['standard']['horizontal_orientation'] = 0.0
header1['standard']['vertical_orientation'] = np.nan
header1['channel'] = get_channel_name(header['sampling_rate'], True, False,
True)
header1['standard']['units_type'] = get_units_type(header1['channel'])
header2['standard']['horizontal_orientation'] = 90.0
header2['standard']['vertical_orientation'] = np.nan
header2['channel'] = get_channel_name(header['sampling_rate'], True, False,
False)
header2['standard']['units_type'] = get_units_type(header2['channel'])
header3['standard']['horizontal_orientation'] = 0.0
header3['standard']['vertical_orientation'] = np.nan
header3['channel'] = get_channel_name(header['sampling_rate'], True, True,
False)
header3['standard']['units_type'] = get_units_type(header3['channel'])
# three columns of NS, EW, UD
# data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS,
# delimiter=[COLWIDTH] * NCOLS, encoding=ENCODING)
data = np.loadtxt(filename, skiprows=TEXT_HDR_ROWS, encoding=ENCODING)
data1 = data[:, 0]
data2 = data[:, 1]
data3 = data[:, 2]
trace1 = StationTrace(data=data1, header=header1)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace1.setProvenance('remove_response', response)
trace2 = StationTrace(data=data2, header=header2)
trace2.setProvenance('remove_response', response)
trace3 = StationTrace(data=data3, header=header3)
trace3.setProvenance('remove_response', response)
stream = StationStream(traces=[trace1, trace2, trace3])
return [stream]
def _read_volume_two(filename, line_offset, location='', units='acc'):
"""Read channel data from DMG text file.
Args:
filename (str): Input DMG V2 filename.
line_offset (int): Line offset to beginning of channel text block.
units (str): units to get
Returns:
tuple: (list of obspy Trace, int line offset)
"""
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(V2_TEXT_HDR_ROWS)]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skip_rows = V2_TEXT_HDR_ROWS + line_offset
int_data = _read_lines(skip_rows, V2_INT_HDR_ROWS, V2_INT_FMT, filename)
int_data = int_data[0:100].astype(np.int32)
# read in lines of float data
skip_rows += V2_INT_HDR_ROWS
flt_data = _read_lines(skip_rows, V2_REAL_HDR_ROWS, V2_REAL_FMT, filename)
flt_data = flt_data[:100]
skip_rows += V2_REAL_HDR_ROWS
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including csmip/dmg here, don't always provide this. We'll flag it as
# "--".
hdr = _get_header_info(int_data, flt_data, lines, 'V2', location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
traces = []
# read acceleration data
if hdr['npts'] > 0:
acc_rows, acc_fmt, unit = _get_data_format(filename, skip_rows,
hdr['npts'])
acc_data = _read_lines(skip_rows + 1, acc_rows, acc_fmt, filename)
acc_data = acc_data[:hdr['npts']]
if unit in UNIT_CONVERSIONS:
acc_data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('DMG: %s is not a supported unit.' % unit)
acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
acc_trace.setProvenance('remove_response', response)
if units == 'acc':
traces += [acc_trace]
skip_rows += int(acc_rows) + 1
# -------------------------------------------------------------------------
# NOTE: The way we were initially reading velocity and displacement data was
# not correct. I'm deleting it for now since we don't need it. If/when we
# revisit this we need to be more careful about how this is handled.
# -------------------------------------------------------------------------
# read velocity data
vel_hdr = hdr.copy()
vel_hdr['standard']['units'] = 'vel'
vel_hdr['npts'] = int_data[63]
if vel_hdr['npts'] > 0:
vel_rows, vel_fmt, unit = _get_data_format(filename, skip_rows,
vel_hdr['npts'])
vel_data = _read_lines(skip_rows + 1, vel_rows, vel_fmt, filename)
vel_data = vel_data[:vel_hdr['npts']]
skip_rows += int(vel_rows) + 1
# read displacement data
disp_hdr = hdr.copy()
disp_hdr['standard']['units'] = 'disp'
disp_hdr['npts'] = int_data[65]
if disp_hdr['npts'] > 0:
disp_rows, disp_fmt, unit = _get_data_format(filename, skip_rows,
disp_hdr['npts'])
disp_data = _read_lines(skip_rows + 1, disp_rows, disp_fmt, filename)
disp_data = disp_data[:disp_hdr['npts']]
skip_rows += int(disp_rows) + 1
new_offset = skip_rows + 1 # there is an 'end of record' line after the data]
return (traces, new_offset)
def _read_channel(filename, line_offset, location=''):
"""Read channel data from COSMOS V1/V2 text file.
Args:
filename (str): Input COSMOS V1/V2 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station, location, and process level from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# read in lines of integer data
skiprows = line_offset + TEXT_HDR_ROWS
int_lines, int_data = _read_lines(skiprows, filename)
int_data = int_data.astype(np.int32)
# read in lines of float data
skiprows += int_lines + 1
flt_lines, flt_data = _read_lines(skiprows, filename)
# read in comment lines
skiprows += flt_lines + 1
cmt_lines, cmt_data = _read_lines(skiprows, filename)
skiprows += cmt_lines + 1
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including cosmos here, don't provide this. We'll flag it as "--".
hdr = _get_header_info(int_data, flt_data, lines,
cmt_data, location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# read in the data
nrows, data = _read_lines(skiprows, filename)
# Check for "off-by-one" problem that sometimes occurs with cosmos data
# Notes:
# - We cannot do this check inside _get_header_info because we don't
# have the data there.
# - That method is written to set npts from the header as documented in
# the spec ("lenght" == npts*dt) but it appears that sometimes a
# different convention is used where the "length" of the record is
# actually is actuation (npts-1)*dt. In this case, we need to
# recompute duration and npts
if hdr['npts'] == (len(data) - 1):
hdr['npts'] = len(data)
hdr['duration'] = (hdr['npts'] - 1) * hdr['delta']
# check units
unit = hdr['format_specific']['physical_units']
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
if unit != 'counts':
raise GMProcessException(
'COSMOS: %s is not a supported unit.' % unit)
if hdr['standard']['units'] != 'acc':
raise GMProcessException('COSMOS: Only acceleration data accepted.')
trace = StationTrace(data.copy(), Stats(hdr.copy()))
# record that this data has been converted to g, if it has
if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def read_smc(filename, **kwargs):
"""Read SMC strong motion file.
Args:
filename (str): Path to possible SMC data file.
kwargs (ref):
any_structure (bool): Read data from any type of structure,
raise Exception if False and structure type is not free-field.
accept_flagged (bool): accept problem flagged data.
set_location (str): Two character code for location.
Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing one channel of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_smc.")
any_structure = kwargs.get('any_structure', False)
accept_flagged = kwargs.get('accept_flagged', False)
location = kwargs.get('location', '')
if not is_smc(filename):
raise Exception('%s is not a valid SMC file' % filename)
with open(filename, 'rt') as f:
line = f.readline().strip()
if 'DISPLACEMENT' in line:
raise GMProcessException(
'SMC: Diplacement records are not supported: '
'%s.' % filename)
elif 'VELOCITY' in line:
raise GMProcessException(
'SMC: Velocity records are not supported: '
'%s.' % filename)
elif line == "*":
raise GMProcessException(
'SMC: No record volume specified in file: '
'%s.' % filename)
stats, num_comments = _get_header_info(filename,
any_structure=any_structure,
accept_flagged=accept_flagged,
location=location)
skip = ASCII_HEADER_LINES + INTEGER_HEADER_LINES + \
num_comments + FLOAT_HEADER_LINES
# read float data (8 columns per line)
nrows = int(np.floor(stats['npts'] / DATA_COLUMNS))
data = np.genfromtxt(filename,
max_rows=nrows,
skip_header=skip,
delimiter=FLOAT_DATA_WIDTHS)
data = data.flatten()
if stats['npts'] % DATA_COLUMNS:
lastrow = np.genfromtxt(filename,
max_rows=1,
skip_header=skip + nrows,
delimiter=FLOAT_DATA_WIDTHS)
data = np.append(data, lastrow)
data = data[0:stats['npts']]
trace = StationTrace(data, header=stats)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
stream = StationStream(traces=[trace])
return [stream]
def _read_volume_one(filename, line_offset, location='', units='acc'):
"""Read channel data from DMG Volume 1 text file.
Args:
filename (str): Input DMG V1 filename.
line_offset (int): Line offset to beginning of channel text block.
units (str): units to get
Returns:
tuple: (list of obspy Trace, int line offset)
"""
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
unit = _get_units(lines[11])
# read in lines of integer data
skip_rows = V1_TEXT_HDR_ROWS + line_offset
int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename)
int_data = int_data[0:100].astype(np.int32)
# read in lines of float data
skip_rows += V1_INT_HDR_ROWS
flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename)
skip_rows += V1_REAL_HDR_ROWS
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including csmip/dmg here, don't always provide this. We'll flag it as
# "--".
hdr = _get_header_info_v1(int_data,
flt_data,
lines,
'V1',
location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# sometimes (??) a line of text is inserted in between the float header and
# the beginning of the data. Let's check for this...
with open(filename, 'rt') as f:
for _ in range(skip_rows):
next(f)
test_line = f.readline()
has_text = re.search('[A-Z]+|[a-z]+', test_line) is not None
if has_text:
skip_rows += 1
widths = [9] * 8
max_rows = int(np.ceil(hdr['npts'] / 8))
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[:hdr['npts']]
evenly_spaced = True
# Sometimes, npts is incrrectly specified, leading to nans
# in the resulting data. Fix that here
if np.any(np.isnan(acc_data)):
while np.isnan(acc_data[-1]):
acc_data = acc_data[:-1]
hdr['npts'] = len(acc_data)
else:
# acceleration data is interleaved between time data
max_rows = int(np.ceil(hdr['npts'] / 5))
widths = [7] * 10
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[1::2][:hdr['npts']]
times = data[0::2][:hdr['npts']]
evenly_spaced = is_evenly_spaced(times)
if unit in UNIT_CONVERSIONS:
acc_data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('DMG: %s is not a supported unit.' % unit)
acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()))
# Check if the times were included in the file but were not evenly spaced
if not evenly_spaced:
acc_trace = resample_uneven_trace(acc_trace, times, acc_data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
acc_trace.setProvenance('remove_response', response)
traces = [acc_trace]
new_offset = skip_rows + max_rows + 1 # there is an end of record line
return (traces, new_offset)
def _read_channel(filename, line_offset, location=''):
"""Read channel data from COSMOS V1/V2 text file.
Args:
filename (str): Input COSMOS V1/V2 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station, location, and process level from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# read in lines of integer data
skiprows = line_offset + TEXT_HDR_ROWS
int_lines, int_data = _read_lines(skiprows, filename)
int_data = int_data.astype(np.int32)
# read in lines of float data
skiprows += int_lines + 1
flt_lines, flt_data = _read_lines(skiprows, filename)
# read in comment lines
skiprows += flt_lines + 1
cmt_lines, cmt_data = _read_lines(skiprows, filename)
skiprows += cmt_lines + 1
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including cosmos here, don't provide this. We'll flag it as "--".
hdr = _get_header_info(int_data,
flt_data,
lines,
cmt_data,
location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# read in the data
nrows, data = _read_lines(skiprows, filename)
# check units
unit = hdr['format_specific']['physical_units']
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('COSMOS: %s is not a supported unit.' % unit)
if hdr['standard']['units'] != 'acc':
raise GMProcessException('COSMOS: Only acceleration data accepted.')
trace = StationTrace(data.copy(), Stats(hdr.copy()))
# record that this data has been converted to g, if it has
if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def _read_channel(filename, line_offset):
"""Read channel data from GNS V1 text file.
Args:
filename (str): Input GNS V1 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station and location strings from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# this code supports V1 and V2 format files. Which one is this?
data_format = 'V2'
if lines[0].lower().find('uncorrected') >= 0:
data_format = 'V1'
# parse out the station code, name, and component string
# from text header
station = lines[1].split()[1]
logging.debug('station: %s' % station)
name = lines[2].replace(' ', '_').strip()
component = lines[12].split()[1]
# parse the instrument type from the text header
instrument = lines[3].split()[1]
# parse the sensor resolution from the text header
resolution_str = lines[4].split()[1]
resolution = int(re.search('\d+', resolution_str).group())
# read floating point header array
skip_header = line_offset + TEXT_HDR_ROWS
hdr_data = np.genfromtxt(filename,
skip_header=skip_header,
max_rows=FP_HDR_ROWS)
# parse header dictionary from float header array
hdr = _read_header(hdr_data, station, name, component, data_format,
instrument, resolution)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including GeoNet here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
skip_header2 = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS
widths = [8] * COLS_PER_ROW
nrows = int(np.ceil(hdr['npts'] / COLS_PER_ROW))
data = np.genfromtxt(filename,
skip_header=skip_header2,
max_rows=nrows,
filling_values=np.nan,
delimiter=widths)
data = data.flatten()
data = data[0:hdr['npts']]
# for debugging, read in the velocity data
nvel = hdr_data[3, 4]
if nvel:
if nvel % COLS_PER_ROW != 0:
nvel_rows = int(np.floor(nvel / COLS_PER_ROW))
nvel_rows2 = 1
else:
nvel_rows = int(np.ceil(nvel / COLS_PER_ROW))
nvel_rows2 = 0
skip_header_vel = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS + nrows
widths = [8] * COLS_PER_ROW
velocity = np.genfromtxt(filename,
skip_header=skip_header_vel,
max_rows=nvel_rows,
filling_values=np.nan,
delimiter=widths)
velocity = velocity.flatten()
velocity *= MMPS_TO_CMPS
else:
velocity = np.array([])
# for V2 files, there are extra blocks of data we need to skip containing
# velocity and displacement data
if data_format == 'V2':
velrows = int(np.ceil(hdr_data[3, 4] / COLS_PER_ROW))
disrows = int(np.ceil(hdr_data[3, 5] / COLS_PER_ROW))
nrows = nrows + velrows + disrows
data *= MMPS_TO_CMPS # convert to cm/s**2
trace = StationTrace(data, Stats(hdr))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
offset = skip_header2 + nrows
return (trace, offset, velocity)
def read_knet(filename):
"""Read Japanese KNET strong motion file.
Args:
filename (str): Path to possible KNET data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_knet.")
if not is_knet(filename):
raise Exception('%s is not a valid KNET file' % filename)
# Parse the header portion of the file
with open(filename, 'rt') as f:
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
hdr = {}
coordinates = {}
standard = {}
hdr['network'] = 'BO'
hdr['station'] = lines[5].split()[2]
logging.debug('station: %s' % hdr['station'])
standard['station_name'] = ''
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including KNET here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
coordinates['latitude'] = float(lines[6].split()[2])
coordinates['longitude'] = float(lines[7].split()[2])
coordinates['elevation'] = float(lines[8].split()[2])
hdr['sampling_rate'] = float(
re.search('\\d+', lines[10].split()[2]).group())
hdr['delta'] = 1 / hdr['sampling_rate']
standard['units'] = 'acc'
dir_string = lines[12].split()[1].strip()
# knet files have directions listed as N-S, E-W, or U-D,
# whereas in kiknet those directions are '4', '5', or '6'.
if dir_string in ['N-S', '1', '4']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
elif dir_string in ['E-W', '2', '5']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
elif dir_string in ['U-D', '3', '6']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
else:
raise Exception('KNET: Could not parse direction %s' %
lines[12].split()[1])
logging.debug('channel: %s' % hdr['channel'])
scalestr = lines[13].split()[2]
parts = scalestr.split('/')
num = float(parts[0].replace('(gal)', ''))
den = float(parts[1])
calib = num / den
hdr['calib'] = calib
duration = float(lines[11].split()[2])
hdr['npts'] = int(duration * hdr['sampling_rate'])
timestr = ' '.join(lines[9].split()[2:4])
# The K-NET and KiK-Net data logger adds a 15s time delay
# this is removed here
sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0)
# Shift the time to utc (Japanese time is 9 hours ahead)
sttime = sttime - timedelta(seconds=9 * 3600.)
hdr['starttime'] = sttime
# read in the data - there is a max of 8 columns per line
# the code below handles the case when last line has
# less than 8 columns
if hdr['npts'] % COLS_PER_LINE != 0:
nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE))
nrows2 = 1
else:
nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE))
nrows2 = 0
data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS,
max_rows=nrows, filling_values=np.nan)
data = data.flatten()
if nrows2:
skip_header = TEXT_HDR_ROWS + nrows
data2 = np.genfromtxt(filename, skip_header=skip_header,
max_rows=nrows2, filling_values=np.nan)
data = np.hstack((data, data2))
nrows += nrows2
# apply the correction factor we're given in the header
data *= calib
# fill out the rest of the standard dictionary
standard['horizontal_orientation'] = np.nan
standard['instrument_period'] = np.nan
standard['instrument_damping'] = np.nan
standard['process_time'] = ''
standard['process_level'] = PROCESS_LEVELS['V1']
standard['sensor_serial_number'] = ''
standard['instrument'] = ''
standard['comments'] = ''
standard['structure_type'] = ''
if dir_string in ['1', '2', '3']:
standard['structure_type'] = 'borehole'
standard['corner_frequency'] = np.nan
standard['units'] = 'acc'
standard['source'] = SRC
standard['source_format'] = 'knet'
hdr['coordinates'] = coordinates
hdr['standard'] = standard
# create a Trace from the data and metadata
trace = StationTrace(data.copy(), Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
stream = StationStream(traces=[trace])
return [stream]
def read_cwb(filename, **kwargs):
"""Read Taiwan Central Weather Bureau strong motion file.
Args:
filename (str): Path to possible CWB data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration
data (cm/s**2).
"""
logging.debug("Starting read_cwb.")
if not is_cwb(filename):
raise Exception('%s is not a valid CWB strong motion data file.'
% filename)
f = open(filename, 'rt')
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including CWB here, don't provide this. We'll flag it as "--".
data = np.genfromtxt(filename, skip_header=HDR_ROWS,
delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW
hdr = _get_header_info(f, data)
f.close()
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
hdr_z = hdr.copy()
hdr_z['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
hdr_z['standard']['horizontal_orientation'] = np.nan
hdr_h1 = hdr.copy()
hdr_h1['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
hdr_h1['standard']['horizontal_orientation'] = np.nan
hdr_h2 = hdr.copy()
hdr_h2['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
hdr_h2['standard']['horizontal_orientation'] = np.nan
stats_z = Stats(hdr_z)
stats_h1 = Stats(hdr_h1)
stats_h2 = Stats(hdr_h2)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace_z = StationTrace(data=data[:, 1], header=stats_z)
trace_z.setProvenance('remove_response', response)
trace_h1 = StationTrace(data=data[:, 2], header=stats_h1)
trace_h1.setProvenance('remove_response', response)
trace_h2 = StationTrace(data=data[:, 3], header=stats_h2)
trace_h2.setProvenance('remove_response', response)
stream = StationStream([trace_z, trace_h1, trace_h2])
return [stream]
def _read_channel(filename, line_offset, location=''):
"""Read channel data from COSMOS V1/V2 text file.
Args:
filename (str): Input COSMOS V1/V2 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station, location, and process level from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# read in lines of integer data
skiprows = line_offset + TEXT_HDR_ROWS
int_lines, int_data = _read_lines(skiprows, filename)
int_data = int_data.astype(np.int32)
# read in lines of float data
skiprows += int_lines + 1
flt_lines, flt_data = _read_lines(skiprows, filename)
# read in comment lines
skiprows += flt_lines + 1
cmt_lines, cmt_data = _read_lines(skiprows, filename)
skiprows += cmt_lines + 1
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including cosmos here, don't provide this. We'll flag it as "--".
hdr = _get_header_info(int_data, flt_data, lines,
cmt_data, location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# read in the data
nrows, data = _read_lines(skiprows, filename)
# check units
unit = hdr['format_specific']['physical_units']
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('COSMOS: %s is not a supported unit.' % unit)
if hdr['standard']['units'] != 'acc':
raise GMProcessException('COSMOS: Only acceleration data accepted.')
trace = StationTrace(data.copy(), Stats(hdr.copy()))
# record that this data has been converted to g, if it has
if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def read_knet(filename):
"""Read Japanese KNET strong motion file.
Args:
filename (str): Path to possible KNET data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_knet.")
if not is_knet(filename):
raise Exception('%s is not a valid KNET file' % filename)
# Parse the header portion of the file
with open(filename, 'rt') as f:
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
hdr = {}
coordinates = {}
standard = {}
hdr['network'] = 'BO'
hdr['station'] = lines[5].split()[2]
logging.debug('station: %s' % hdr['station'])
standard['station_name'] = ''
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including KNET here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
coordinates['latitude'] = float(lines[6].split()[2])
coordinates['longitude'] = float(lines[7].split()[2])
coordinates['elevation'] = float(lines[8].split()[2])
hdr['sampling_rate'] = float(
re.search('\\d+', lines[10].split()[2]).group())
hdr['delta'] = 1 / hdr['sampling_rate']
standard['units'] = 'acc'
dir_string = lines[12].split()[1].strip()
# knet files have directions listed as N-S, E-W, or U-D,
# whereas in kiknet those directions are '4', '5', or '6'.
if dir_string in ['N-S', '1', '4']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
elif dir_string in ['E-W', '2', '5']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
elif dir_string in ['U-D', '3', '6']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
else:
raise Exception('KNET: Could not parse direction %s' %
lines[12].split()[1])
logging.debug('channel: %s' % hdr['channel'])
scalestr = lines[13].split()[2]
parts = scalestr.split('/')
num = float(parts[0].replace('(gal)', ''))
den = float(parts[1])
calib = num / den
hdr['calib'] = calib
duration = float(lines[11].split()[2])
hdr['npts'] = int(duration * hdr['sampling_rate'])
timestr = ' '.join(lines[9].split()[2:4])
# The K-NET and KiK-Net data logger adds a 15s time delay
# this is removed here
sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0)
# Shift the time to utc (Japanese time is 9 hours ahead)
sttime = sttime - timedelta(seconds=9 * 3600.)
hdr['starttime'] = sttime
# read in the data - there is a max of 8 columns per line
# the code below handles the case when last line has
# less than 8 columns
if hdr['npts'] % COLS_PER_LINE != 0:
nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE))
nrows2 = 1
else:
nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE))
nrows2 = 0
data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS,
max_rows=nrows, filling_values=np.nan)
data = data.flatten()
if nrows2:
skip_header = TEXT_HDR_ROWS + nrows
data2 = np.genfromtxt(filename, skip_header=skip_header,
max_rows=nrows2, filling_values=np.nan)
data = np.hstack((data, data2))
nrows += nrows2
# apply the correction factor we're given in the header
data *= calib
# fill out the rest of the standard dictionary
standard['horizontal_orientation'] = np.nan
standard['instrument_period'] = np.nan
standard['instrument_damping'] = np.nan
standard['process_time'] = ''
standard['process_level'] = PROCESS_LEVELS['V1']
standard['sensor_serial_number'] = ''
standard['instrument'] = ''
standard['comments'] = ''
standard['structure_type'] = ''
if dir_string in ['1', '2', '3']:
standard['structure_type'] = 'borehole'
standard['corner_frequency'] = np.nan
standard['units'] = 'acc'
standard['source'] = SRC
standard['source_format'] = 'knet'
head, tail = os.path.split(filename)
standard['source_file'] = tail or os.path.basename(head)
hdr['coordinates'] = coordinates
hdr['standard'] = standard
# create a Trace from the data and metadata
trace = StationTrace(data.copy(), Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
stream = StationStream(traces=[trace])
return [stream]
def _read_channel(filename, line_offset, volume, location='', alternate=False):
"""Read channel data from USC V1 text file.
Args:
filename (str): Input USC V1 filename.
line_offset (int): Line offset to beginning of channel text block.
volume (dictionary): Dictionary of formatting information
Returns:
tuple: (obspy Trace, int line offset)
"""
if alternate:
int_rows = 5
int_fmt = 20 * [4]
data_cols = 8
else:
int_rows = volume['INT_HDR_ROWS']
int_fmt = volume['INT_FMT']
data_cols = 10
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skiprows = line_offset + volume['TEXT_HDR_ROWS']
int_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=int_rows, dtype=np.int32,
delimiter=int_fmt).flatten()
# read in lines of float data
skiprows += int_rows
flt_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64,
delimiter=volume['FLT_FMT']).flatten()
hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
skiprows += volume['FLT_HDR_ROWS']
# read in the data
nrows = int(np.floor(hdr['npts'] * 2 / data_cols))
all_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=nrows, dtype=np.float64,
delimiter=volume['COL_FMT'])
data = all_data.flatten()[1::2]
times = all_data.flatten()[0::2]
frac = hdr['format_specific']['fractional_unit']
if frac > 0:
data *= UNIT_CONVERSIONS['g'] * frac
logging.debug('Data converted from g * %s to cm/s/s' % (frac))
else:
unit = _get_units(lines[11])
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('USC: %s is not a supported unit.' % unit)
# Put file name into dictionary
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
trace = StationTrace(data.copy(), Stats(hdr.copy()))
if not is_evenly_spaced(times):
trace = resample_uneven_trace(trace, times, data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def _read_channel(filename, line_offset):
"""Read channel data from GNS V1 text file.
Args:
filename (str): Input GNS V1 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station and location strings from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# this code supports V1 and V2 format files. Which one is this?
data_format = 'V2'
if lines[0].lower().find('uncorrected') >= 0:
data_format = 'V1'
# parse out the station code, name, and component string
# from text header
station = lines[1].split()[1]
logging.debug('station: %s' % station)
name = lines[2].replace(' ', '_').strip()
component = lines[12].split()[1]
# parse the instrument type from the text header
instrument = lines[3].split()[1]
# parse the sensor resolution from the text header
resolution_str = lines[4].split()[1]
resolution = int(re.search('\d+', resolution_str).group())
# read floating point header array
skip_header = line_offset + TEXT_HDR_ROWS
hdr_data = np.genfromtxt(filename, skip_header=skip_header,
max_rows=FP_HDR_ROWS)
# parse header dictionary from float header array
hdr = _read_header(hdr_data, station, name,
component, data_format,
instrument, resolution)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including GeoNet here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
skip_header2 = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS
widths = [8] * COLS_PER_ROW
nrows = int(np.ceil(hdr['npts'] / COLS_PER_ROW))
data = np.genfromtxt(filename, skip_header=skip_header2,
max_rows=nrows, filling_values=np.nan,
delimiter=widths)
data = data.flatten()
data = data[0:hdr['npts']]
# for debugging, read in the velocity data
nvel = hdr_data[3, 4]
if nvel:
if nvel % COLS_PER_ROW != 0:
nvel_rows = int(np.floor(nvel / COLS_PER_ROW))
nvel_rows2 = 1
else:
nvel_rows = int(np.ceil(nvel / COLS_PER_ROW))
nvel_rows2 = 0
skip_header_vel = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS + nrows
widths = [8] * COLS_PER_ROW
velocity = np.genfromtxt(filename, skip_header=skip_header_vel,
max_rows=nvel_rows, filling_values=np.nan,
delimiter=widths)
velocity = velocity.flatten()
velocity *= MMPS_TO_CMPS
else:
velocity = np.array([])
# for V2 files, there are extra blocks of data we need to skip containing
# velocity and displacement data
if data_format == 'V2':
velrows = int(np.ceil(hdr_data[3, 4] / COLS_PER_ROW))
disrows = int(np.ceil(hdr_data[3, 5] / COLS_PER_ROW))
nrows = nrows + velrows + disrows
data *= MMPS_TO_CMPS # convert to cm/s**2
trace = StationTrace(data, Stats(hdr))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
offset = skip_header2 + nrows
return (trace, offset, velocity)
def read_at2(dfile, horient=0.0):
# This is a conveneince method so we can read in these specific data for
# testing, it is not a general purpose reader since this format does not
# contain a lot of metadata that is generally required for it to be useful.
skiprows = 4
datafile = open(dfile, 'r')
datareader = csv.reader(datafile)
data = []
header = []
# for i in range(skiprows):
# next(datareader)
# header.append(datareader.readlines())
count = 0
for row in datareader:
if count < skiprows:
header.append(row)
else:
data.extend([float(e) for e in row[0].split()])
count += 1
datafile.close()
hdr = {}
hdr['network'] = ''
hdr['station'] = ''
if horient == 0:
hdr['channel'] = 'BH1'
else:
hdr['channel'] = 'BH2'
hdr['location'] = '--'
dt = float(header[3][1].split('=')[1].strip().lower().replace('sec', ''))
hdr['npts'] = len(data)
hdr['sampling_rate'] = 1 / dt
hdr['duration'] = (hdr['npts'] - 1) / hdr['sampling_rate']
hdr['starttime'] = 0
# There is no lat/lon...
hdr['coordinates'] = {'latitude': 0.0, 'longitude': 0.0, 'elevation': 0.0}
standard = {}
standard['units'] = 'acc'
standard['units_type'] = 'acc'
standard['horizontal_orientation'] = horient
standard['vertical_orientation'] = np.nan
standard['source_file'] = dfile
standard['station_name'] = ''
standard['corner_frequency'] = 30.0
standard['structure_type'] = ''
standard['comments'] = ''
standard['instrument'] = ''
standard['instrument_period'] = 1.0
standard['instrument_sensitivity'] = 1.0
standard['source'] = 'PEER'
standard['instrument_damping'] = 0.1
standard['sensor_serial_number'] = ''
standard['process_level'] = 'corrected physical units'
standard['source_format'] = 'AT2'
standard['process_time'] = ''
hdr['standard'] = standard
# convert data from g to cm/s^2
g_to_cmss = 980.665
tr = StationTrace(np.array(data.copy()) * g_to_cmss, Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
tr.setProvenance('remove_response', response)
return tr
def read_cwb(filename, **kwargs):
"""Read Taiwan Central Weather Bureau strong motion file.
Args:
filename (str): Path to possible CWB data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration
data (cm/s**2).
"""
logging.debug("Starting read_cwb.")
if not is_cwb(filename):
raise Exception('%s is not a valid CWB strong motion data file.' %
filename)
f = open(filename, 'rt')
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including CWB here, don't provide this. We'll flag it as "--".
data = np.genfromtxt(filename,
skip_header=HDR_ROWS,
delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW
hdr = _get_header_info(f, data)
f.close()
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
hdr_z = hdr.copy()
hdr_z['channel'] = get_channel_name(hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
hdr_z['standard']['horizontal_orientation'] = np.nan
hdr_z['standard']['vertical_orientation'] = np.nan
hdr_z['standard']['units_type'] = get_units_type(hdr_z['channel'])
hdr_h1 = hdr.copy()
hdr_h1['channel'] = get_channel_name(hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
hdr_h1['standard']['horizontal_orientation'] = np.nan
hdr_h1['standard']['vertical_orientation'] = np.nan
hdr_h1['standard']['units_type'] = get_units_type(hdr_h1['channel'])
hdr_h2 = hdr.copy()
hdr_h2['channel'] = get_channel_name(hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
hdr_h2['standard']['horizontal_orientation'] = np.nan
hdr_h2['standard']['vertical_orientation'] = np.nan
hdr_h2['standard']['units_type'] = get_units_type(hdr_h2['channel'])
stats_z = Stats(hdr_z)
stats_h1 = Stats(hdr_h1)
stats_h2 = Stats(hdr_h2)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace_z = StationTrace(data=data[:, 1], header=stats_z)
trace_z.setProvenance('remove_response', response)
trace_h1 = StationTrace(data=data[:, 2], header=stats_h1)
trace_h1.setProvenance('remove_response', response)
trace_h2 = StationTrace(data=data[:, 3], header=stats_h2)
trace_h2.setProvenance('remove_response', response)
stream = StationStream([trace_z, trace_h1, trace_h2])
return [stream]
def read_esm(filename):
"""Read European ESM strong motion file.
Args:
filename (str): Path to possible ESM data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing one channels of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_esm.")
if not is_esm(filename):
raise Exception('%s is not a valid ESM file' % filename)
# Parse the header portion of the file
header = {}
with open(filename, 'rt') as f:
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
for line in lines:
parts = line.split(':')
key = parts[0].strip()
value = ':'.join(parts[1:]).strip()
header[key] = value
stats = {}
standard = {}
format_specific = {}
coordinates = {}
# fill in all known stats header fields
stats['network'] = header['NETWORK']
stats['station'] = header['STATION_CODE']
stats['channel'] = header['STREAM']
stats['location'] = '--'
stats['delta'] = float(header['SAMPLING_INTERVAL_S'])
stats['sampling_rate'] = 1 / stats['delta']
stats['calib'] = 1.0
stats['npts'] = int(header['NDATA'])
stimestr = header['DATE_TIME_FIRST_SAMPLE_YYYYMMDD_HHMMSS']
stats['starttime'] = datetime.strptime(stimestr, TIMEFMT)
# fill in standard fields
head, tail = os.path.split(filename)
standard['source_file'] = tail or os.path.basename(head)
standard['source'] = SRC
standard['source_format'] = FORMAT
standard['horizontal_orientation'] = np.nan
standard['station_name'] = header['STATION_NAME']
try:
standard['instrument_period'] = 1 / \
float(header['INSTRUMENTAL_FREQUENCY_HZ'])
except ValueError:
standard['instrument_period'] = np.nan
try:
standard['instrument_damping'] = 1 / \
float(header['INSTRUMENTAL_DAMPING'])
except ValueError:
standard['instrument_damping'] = np.nan
ptimestr = header['DATA_TIMESTAMP_YYYYMMDD_HHMMSS']
ptime = datetime.strptime(ptimestr, TIMEFMT).strftime(TIMEFMT2)
standard['process_time'] = ptime
standard['process_level'] = PROCESS_LEVELS['V1']
instr_str = header['INSTRUMENT']
parts = instr_str.split('|')
sensor_str = parts[0].split('=')[1].strip()
standard['sensor_serial_number'] = ''
standard['instrument'] = sensor_str
standard['comments'] = ''
standard['structure_type'] = ''
standard['units'] = 'cm/s^2'
standard['units_type'] = 'acc'
standard['instrument_sensitivity'] = np.nan
standard['corner_frequency'] = np.nan
coordinates['latitude'] = float(header['STATION_LATITUDE_DEGREE'])
coordinates['longitude'] = float(header['STATION_LONGITUDE_DEGREE'])
coordinates['elevation'] = float(header['STATION_ELEVATION_M'])
# read in the data
data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS)
# create a Trace from the data and metadata
stats['standard'] = standard
stats['coordinates'] = coordinates
trace = StationTrace(data.copy(), Stats(stats.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
ftype = header['FILTER_TYPE'].capitalize()
forder = int(header['FILTER_ORDER'])
lowfreq = float(header['LOW_CUT_FREQUENCY_HZ'])
highfreq = float(header['LOW_CUT_FREQUENCY_HZ'])
filter_att = {'bandpass_filter':
{'filter_type': ftype,
'lower_corner_frequency': lowfreq,
'higher_corner_frequency': highfreq,
'filter_order': forder}}
trace.setProvenance('lowpass_filter', filter_att)
detrend_att = {'detrend': {'detrending_method': 'baseline'}}
trace.setProvenance('detrend', detrend_att)
stream = StationStream(traces=[trace])
return [stream]
def _read_channel(filename, line_offset, volume, location='', alternate=False):
"""Read channel data from USC V1 text file.
Args:
filename (str): Input USC V1 filename.
line_offset (int): Line offset to beginning of channel text block.
volume (dictionary): Dictionary of formatting information
Returns:
tuple: (obspy Trace, int line offset)
"""
if alternate:
int_rows = 5
int_fmt = 20 * [4]
data_cols = 8
else:
int_rows = volume['INT_HDR_ROWS']
int_fmt = volume['INT_FMT']
data_cols = 10
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skiprows = line_offset + volume['TEXT_HDR_ROWS']
int_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=int_rows, dtype=np.int32,
delimiter=int_fmt).flatten()
# read in lines of float data
skiprows += int_rows
flt_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64,
delimiter=volume['FLT_FMT']).flatten()
hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
skiprows += volume['FLT_HDR_ROWS']
# read in the data
nrows = int(np.floor(hdr['npts'] * 2 / data_cols))
all_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=nrows, dtype=np.float64,
delimiter=volume['COL_FMT'])
data = all_data.flatten()[1::2]
times = all_data.flatten()[0::2]
frac = hdr['format_specific']['fractional_unit']
if frac > 0:
data *= UNIT_CONVERSIONS['g'] * frac
logging.debug('Data converted from g * %s to cm/s/s' % (frac))
else:
unit = _get_units(lines[11])
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('USC: %s is not a supported unit.' % unit)
# Put file name into dictionary
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
trace = StationTrace(data.copy(), Stats(hdr.copy()))
if not is_evenly_spaced(times):
trace = resample_uneven_trace(trace, times, data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def read_smc(filename, **kwargs):
"""Read SMC strong motion file.
Args:
filename (str): Path to possible SMC data file.
kwargs (ref):
any_structure (bool): Read data from any type of structure,
raise Exception if False and structure type is not free-field.
accept_flagged (bool): accept problem flagged data.
set_location (str): Two character code for location.
Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing one channel of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_smc.")
any_structure = kwargs.get('any_structure', False)
accept_flagged = kwargs.get('accept_flagged', False)
location = kwargs.get('location', '')
if not is_smc(filename):
raise Exception('%s is not a valid SMC file' % filename)
with open(filename, 'rt') as f:
line = f.readline().strip()
if 'DISPLACEMENT' in line:
raise GMProcessException('SMC: Diplacement records are not supported: '
'%s.' % filename)
elif 'VELOCITY' in line:
raise GMProcessException('SMC: Velocity records are not supported: '
'%s.' % filename)
elif line == "*":
raise GMProcessException('SMC: No record volume specified in file: '
'%s.' % filename)
stats, num_comments = _get_header_info(
filename, any_structure=any_structure,
accept_flagged=accept_flagged, location=location)
skip = ASCII_HEADER_LINES + INTEGER_HEADER_LINES + \
num_comments + FLOAT_HEADER_LINES
# read float data (8 columns per line)
nrows = int(np.floor(stats['npts'] / DATA_COLUMNS))
data = np.genfromtxt(filename,
max_rows=nrows,
skip_header=skip,
delimiter=FLOAT_DATA_WIDTHS)
data = data.flatten()
if stats['npts'] % DATA_COLUMNS:
lastrow = np.genfromtxt(filename, max_rows=1,
skip_header=skip + nrows,
delimiter=FLOAT_DATA_WIDTHS)
data = np.append(data, lastrow)
data = data[0:stats['npts']]
trace = StationTrace(data, header=stats)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
stream = StationStream(traces=[trace])
return [stream]