def _read_channel(filename, line_offset, volume, location='', alternate=False):
"""Read channel data from USC V1 text file.
Args:
filename (str): Input USC V1 filename.
line_offset (int): Line offset to beginning of channel text block.
volume (dictionary): Dictionary of formatting information
Returns:
tuple: (obspy Trace, int line offset)
"""
if alternate:
int_rows = 5
int_fmt = 20 * [4]
data_cols = 8
else:
int_rows = volume['INT_HDR_ROWS']
int_fmt = volume['INT_FMT']
data_cols = 10
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skiprows = line_offset + volume['TEXT_HDR_ROWS']
int_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=int_rows, dtype=np.int32,
delimiter=int_fmt).flatten()
# read in lines of float data
skiprows += int_rows
flt_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64,
delimiter=volume['FLT_FMT']).flatten()
hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
skiprows += volume['FLT_HDR_ROWS']
# read in the data
nrows = int(np.floor(hdr['npts'] * 2 / data_cols))
all_data = np.genfromtxt(filename, skip_header=skiprows,
max_rows=nrows, dtype=np.float64,
delimiter=volume['COL_FMT'])
data = all_data.flatten()[1::2]
times = all_data.flatten()[0::2]
frac = hdr['format_specific']['fractional_unit']
if frac > 0:
data *= UNIT_CONVERSIONS['g'] * frac
logging.debug('Data converted from g * %s to cm/s/s' % (frac))
else:
unit = _get_units(lines[11])
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('USC: %s is not a supported unit.' % unit)
# Put file name into dictionary
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
trace = StationTrace(data.copy(), Stats(hdr.copy()))
if not is_evenly_spaced(times):
trace = resample_uneven_trace(trace, times, data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def _read_channel(filename, line_offset, volume, location='', alternate=False):
"""Read channel data from USC V1 text file.
Args:
filename (str):
Input USC V1 filename.
line_offset (int):
Line offset to beginning of channel text block.
volume (dictionary):
Dictionary of formatting information.
Returns:
tuple: (obspy Trace, int line offset)
"""
if alternate:
int_rows = 5
int_fmt = 20 * [4]
data_cols = 8
else:
int_rows = volume['INT_HDR_ROWS']
int_fmt = volume['INT_FMT']
data_cols = 10
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skiprows = line_offset + volume['TEXT_HDR_ROWS']
int_data = np.genfromtxt(filename,
skip_header=skiprows,
max_rows=int_rows,
dtype=np.int32,
delimiter=int_fmt).flatten()
# read in lines of float data
skiprows += int_rows
flt_data = np.genfromtxt(filename,
skip_header=skiprows,
max_rows=volume['FLT_HDR_ROWS'],
dtype=np.float64,
delimiter=volume['FLT_FMT']).flatten()
hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
skiprows += volume['FLT_HDR_ROWS']
# read in the data
nrows = int(np.floor(hdr['npts'] * 2 / data_cols))
all_data = np.genfromtxt(filename,
skip_header=skiprows,
max_rows=nrows,
dtype=np.float64,
delimiter=volume['COL_FMT'])
data = all_data.flatten()[1::2]
times = all_data.flatten()[0::2]
frac = hdr['format_specific']['fractional_unit']
if frac > 0:
data *= UNIT_CONVERSIONS['g'] * frac
logging.debug('Data converted from g * %s to cm/s/s' % (frac))
else:
unit = _get_units(lines[11])
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise ValueError('USC: %s is not a supported unit.' % unit)
# Put file name into dictionary
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
trace = StationTrace(data.copy(), Stats(hdr.copy()))
if not is_evenly_spaced(times):
trace = resample_uneven_trace(trace, times, data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def _read_volume_one(filename, line_offset, location='', units='acc'):
"""Read channel data from DMG Volume 1 text file.
Args:
filename (str):
Input DMG V1 filename.
line_offset (int):
Line offset to beginning of channel text block.
units (str):
units to get.
Returns:
tuple: (list of obspy Trace, int line offset)
"""
# Parse the header portion of the file
try:
with open(filename, 'rt', encoding='utf-8') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
unit = _get_units(lines[11])
# read in lines of integer data
skip_rows = V1_TEXT_HDR_ROWS + line_offset
int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename)
int_data = int_data[0:100].astype(np.int32)
# read in lines of float data
skip_rows += V1_INT_HDR_ROWS
flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename)
skip_rows += V1_REAL_HDR_ROWS
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including csmip/dmg here, don't always provide this. We'll flag it as
# "--".
hdr = _get_header_info_v1(
int_data, flt_data, lines, 'V1', location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# sometimes (??) a line of text is inserted in between the float header and
# the beginning of the data. Let's check for this...
with open(filename, 'rt', encoding='utf-8') as f:
for _ in range(skip_rows):
next(f)
test_line = f.readline()
has_text = re.search('[A-Z]+|[a-z]+', test_line) is not None
if has_text:
skip_rows += 1
widths = [9] * 8
max_rows = int(np.ceil(hdr['npts'] / 8))
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[:hdr['npts']]
evenly_spaced = True
# Sometimes, npts is incrrectly specified, leading to nans
# in the resulting data. Fix that here
if np.any(np.isnan(acc_data)):
while np.isnan(acc_data[-1]):
acc_data = acc_data[:-1]
hdr['npts'] = len(acc_data)
else:
# acceleration data is interleaved between time data
max_rows = int(np.ceil(hdr['npts'] / 5))
widths = [7] * 10
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[1::2][:hdr['npts']]
times = data[0::2][:hdr['npts']]
evenly_spaced = is_evenly_spaced(times)
if unit in UNIT_CONVERSIONS:
acc_data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise ValueError('DMG: %s is not a supported unit.' % unit)
acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()))
# Check if the times were included in the file but were not evenly spaced
if not evenly_spaced:
acc_trace = resample_uneven_trace(acc_trace, times, acc_data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
acc_trace.setProvenance('remove_response', response)
traces = [acc_trace]
new_offset = skip_rows + max_rows + 1 # there is an end of record line
return (traces, new_offset)
def _read_volume_one(filename, line_offset, location="", units="acc", config=None):
"""Read channel data from DMG Volume 1 text file.
Args:
filename (str):
Input DMG V1 filename.
line_offset (int):
Line offset to beginning of channel text block.
units (str):
units to get.
config (dict):
Configuration options.
Returns:
tuple: (list of obspy Trace, int line offset)
"""
# Parse the header portion of the file
try:
with open(filename, "rt", encoding="utf-8") as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
unit = _get_units(lines[11])
# read in lines of integer data
skip_rows = V1_TEXT_HDR_ROWS + line_offset
int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename)
int_data = int_data[0:100].astype(np.int32)
# read in lines of float data
skip_rows += V1_INT_HDR_ROWS
flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename)
skip_rows += V1_REAL_HDR_ROWS
# ------------------------------------------------------------------------------- #
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including csmip/dmg here, don't always provide this. We'll flag it as
# "--".
# Update Feb 4, 2022
# In order to be able to store structural/geotech array data in the ASDF file, we
# HAVE to use location code to uniquely identify the different instruments. The
# arrays can only be read in as StreamArrays and not StreamCollections. In such
# cases we will use the channel number as location code so that the data can be
# saved.
# ------------------------------------------------------------------------------- #
hdr = _get_header_info_v1(
int_data, flt_data, lines, "V1", location=location, config=config
)
head, tail = os.path.split(filename)
hdr["standard"]["source_file"] = tail or os.path.basename(head)
# sometimes (??) a line of text is inserted in between the float header and
# the beginning of the data. Let's check for this...
with open(filename, "rt", encoding="utf-8") as f:
for _ in range(skip_rows):
next(f)
test_line = f.readline()
has_text = re.search("[A-Z]+|[a-z]+", test_line) is not None
if has_text:
skip_rows += 1
widths = [9] * 8
max_rows = int(np.ceil(hdr["npts"] / 8))
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[: hdr["npts"]]
evenly_spaced = True
# Sometimes, npts is incrrectly specified, leading to nans
# in the resulting data. Fix that here
if np.any(np.isnan(acc_data)):
while np.isnan(acc_data[-1]):
acc_data = acc_data[:-1]
hdr["npts"] = len(acc_data)
else:
# acceleration data is interleaved between time data
max_rows = int(np.ceil(hdr["npts"] / 5))
widths = [7] * 10
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[1::2][: hdr["npts"]]
times = data[0::2][: hdr["npts"]]
evenly_spaced = is_evenly_spaced(times)
if unit in UNIT_CONVERSIONS:
acc_data *= UNIT_CONVERSIONS[unit]
logging.debug(f"Data converted from {unit} to cm/s/s")
else:
raise ValueError(f"DMG: {unit} is not a supported unit.")
acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()), config=config)
# Check if the times were included in the file but were not evenly spaced
if not evenly_spaced:
acc_trace = resample_uneven_trace(acc_trace, times, acc_data, config=config)
response = {"input_units": "counts", "output_units": "cm/s^2"}
acc_trace.setProvenance("remove_response", response)
traces = [acc_trace]
new_offset = skip_rows + max_rows + 1 # there is an end of record line
return (traces, new_offset)