def _read_channel(filename, line_offset, volume, location='', alternate=False): """Read channel data from USC V1 text file. Args: filename (str): Input USC V1 filename. line_offset (int): Line offset to beginning of channel text block. volume (dictionary): Dictionary of formatting information Returns: tuple: (obspy Trace, int line offset) """ if alternate: int_rows = 5 int_fmt = 20 * [4] data_cols = 8 else: int_rows = volume['INT_HDR_ROWS'] int_fmt = volume['INT_FMT'] data_cols = 10 # Parse the header portion of the file try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) # read in lines of integer data skiprows = line_offset + volume['TEXT_HDR_ROWS'] int_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=int_rows, dtype=np.int32, delimiter=int_fmt).flatten() # read in lines of float data skiprows += int_rows flt_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64, delimiter=volume['FLT_FMT']).flatten() hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location) skiprows += volume['FLT_HDR_ROWS'] # read in the data nrows = int(np.floor(hdr['npts'] * 2 / data_cols)) all_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=nrows, dtype=np.float64, delimiter=volume['COL_FMT']) data = all_data.flatten()[1::2] times = all_data.flatten()[0::2] frac = hdr['format_specific']['fractional_unit'] if frac > 0: data *= UNIT_CONVERSIONS['g'] * frac logging.debug('Data converted from g * %s to cm/s/s' % (frac)) else: unit = _get_units(lines[11]) if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('USC: %s is not a supported unit.' % unit) # Put file name into dictionary head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) trace = StationTrace(data.copy(), Stats(hdr.copy())) if not is_evenly_spaced(times): trace = resample_uneven_trace(trace, times, data) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def _read_channel(filename, line_offset, volume, location='', alternate=False): """Read channel data from USC V1 text file. Args: filename (str): Input USC V1 filename. line_offset (int): Line offset to beginning of channel text block. volume (dictionary): Dictionary of formatting information. Returns: tuple: (obspy Trace, int line offset) """ if alternate: int_rows = 5 int_fmt = 20 * [4] data_cols = 8 else: int_rows = volume['INT_HDR_ROWS'] int_fmt = volume['INT_FMT'] data_cols = 10 # Parse the header portion of the file try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) # read in lines of integer data skiprows = line_offset + volume['TEXT_HDR_ROWS'] int_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=int_rows, dtype=np.int32, delimiter=int_fmt).flatten() # read in lines of float data skiprows += int_rows flt_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64, delimiter=volume['FLT_FMT']).flatten() hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location) skiprows += volume['FLT_HDR_ROWS'] # read in the data nrows = int(np.floor(hdr['npts'] * 2 / data_cols)) all_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=nrows, dtype=np.float64, delimiter=volume['COL_FMT']) data = all_data.flatten()[1::2] times = all_data.flatten()[0::2] frac = hdr['format_specific']['fractional_unit'] if frac > 0: data *= UNIT_CONVERSIONS['g'] * frac logging.debug('Data converted from g * %s to cm/s/s' % (frac)) else: unit = _get_units(lines[11]) if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise ValueError('USC: %s is not a supported unit.' % unit) # Put file name into dictionary head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) trace = StationTrace(data.copy(), Stats(hdr.copy())) if not is_evenly_spaced(times): trace = resample_uneven_trace(trace, times, data) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def _read_volume_one(filename, line_offset, location='', units='acc'): """Read channel data from DMG Volume 1 text file. Args: filename (str): Input DMG V1 filename. line_offset (int): Line offset to beginning of channel text block. units (str): units to get. Returns: tuple: (list of obspy Trace, int line offset) """ # Parse the header portion of the file try: with open(filename, 'rt', encoding='utf-8') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) unit = _get_units(lines[11]) # read in lines of integer data skip_rows = V1_TEXT_HDR_ROWS + line_offset int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename) int_data = int_data[0:100].astype(np.int32) # read in lines of float data skip_rows += V1_INT_HDR_ROWS flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename) skip_rows += V1_REAL_HDR_ROWS # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including csmip/dmg here, don't always provide this. We'll flag it as # "--". hdr = _get_header_info_v1( int_data, flt_data, lines, 'V1', location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # sometimes (??) a line of text is inserted in between the float header and # the beginning of the data. Let's check for this... with open(filename, 'rt', encoding='utf-8') as f: for _ in range(skip_rows): next(f) test_line = f.readline() has_text = re.search('[A-Z]+|[a-z]+', test_line) is not None if has_text: skip_rows += 1 widths = [9] * 8 max_rows = int(np.ceil(hdr['npts'] / 8)) data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[:hdr['npts']] evenly_spaced = True # Sometimes, npts is incrrectly specified, leading to nans # in the resulting data. Fix that here if np.any(np.isnan(acc_data)): while np.isnan(acc_data[-1]): acc_data = acc_data[:-1] hdr['npts'] = len(acc_data) else: # acceleration data is interleaved between time data max_rows = int(np.ceil(hdr['npts'] / 5)) widths = [7] * 10 data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[1::2][:hdr['npts']] times = data[0::2][:hdr['npts']] evenly_spaced = is_evenly_spaced(times) if unit in UNIT_CONVERSIONS: acc_data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise ValueError('DMG: %s is not a supported unit.' % unit) acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy())) # Check if the times were included in the file but were not evenly spaced if not evenly_spaced: acc_trace = resample_uneven_trace(acc_trace, times, acc_data) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} acc_trace.setProvenance('remove_response', response) traces = [acc_trace] new_offset = skip_rows + max_rows + 1 # there is an end of record line return (traces, new_offset)
def _read_volume_one(filename, line_offset, location="", units="acc", config=None): """Read channel data from DMG Volume 1 text file. Args: filename (str): Input DMG V1 filename. line_offset (int): Line offset to beginning of channel text block. units (str): units to get. config (dict): Configuration options. Returns: tuple: (list of obspy Trace, int line offset) """ # Parse the header portion of the file try: with open(filename, "rt", encoding="utf-8") as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) unit = _get_units(lines[11]) # read in lines of integer data skip_rows = V1_TEXT_HDR_ROWS + line_offset int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename) int_data = int_data[0:100].astype(np.int32) # read in lines of float data skip_rows += V1_INT_HDR_ROWS flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename) skip_rows += V1_REAL_HDR_ROWS # ------------------------------------------------------------------------------- # # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including csmip/dmg here, don't always provide this. We'll flag it as # "--". # Update Feb 4, 2022 # In order to be able to store structural/geotech array data in the ASDF file, we # HAVE to use location code to uniquely identify the different instruments. The # arrays can only be read in as StreamArrays and not StreamCollections. In such # cases we will use the channel number as location code so that the data can be # saved. # ------------------------------------------------------------------------------- # hdr = _get_header_info_v1( int_data, flt_data, lines, "V1", location=location, config=config ) head, tail = os.path.split(filename) hdr["standard"]["source_file"] = tail or os.path.basename(head) # sometimes (??) a line of text is inserted in between the float header and # the beginning of the data. Let's check for this... with open(filename, "rt", encoding="utf-8") as f: for _ in range(skip_rows): next(f) test_line = f.readline() has_text = re.search("[A-Z]+|[a-z]+", test_line) is not None if has_text: skip_rows += 1 widths = [9] * 8 max_rows = int(np.ceil(hdr["npts"] / 8)) data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[: hdr["npts"]] evenly_spaced = True # Sometimes, npts is incrrectly specified, leading to nans # in the resulting data. Fix that here if np.any(np.isnan(acc_data)): while np.isnan(acc_data[-1]): acc_data = acc_data[:-1] hdr["npts"] = len(acc_data) else: # acceleration data is interleaved between time data max_rows = int(np.ceil(hdr["npts"] / 5)) widths = [7] * 10 data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[1::2][: hdr["npts"]] times = data[0::2][: hdr["npts"]] evenly_spaced = is_evenly_spaced(times) if unit in UNIT_CONVERSIONS: acc_data *= UNIT_CONVERSIONS[unit] logging.debug(f"Data converted from {unit} to cm/s/s") else: raise ValueError(f"DMG: {unit} is not a supported unit.") acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()), config=config) # Check if the times were included in the file but were not evenly spaced if not evenly_spaced: acc_trace = resample_uneven_trace(acc_trace, times, acc_data, config=config) response = {"input_units": "counts", "output_units": "cm/s^2"} acc_trace.setProvenance("remove_response", response) traces = [acc_trace] new_offset = skip_rows + max_rows + 1 # there is an end of record line return (traces, new_offset)