def _read_channel(filename, line_offset, volume, location='', alternate=False):
    """Read channel data from USC V1 text file.

    Args:
        filename (str): Input USC V1 filename.
        line_offset (int): Line offset to beginning of channel text block.
        volume (dictionary): Dictionary of formatting information
    Returns:
        tuple: (obspy Trace, int line offset)
    """
    if alternate:
        int_rows = 5
        int_fmt = 20 * [4]
        data_cols = 8
    else:
        int_rows = volume['INT_HDR_ROWS']
        int_fmt = volume['INT_FMT']
        data_cols = 10
    # Parse the header portion of the file
    try:
        with open(filename, 'rt') as f:
            for _ in range(line_offset):
                next(f)
            lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
    # Accounts for blank lines at end of files
    except StopIteration:
        return (None, 1 + line_offset)
    # read in lines of integer data
    skiprows = line_offset + volume['TEXT_HDR_ROWS']
    int_data = np.genfromtxt(filename, skip_header=skiprows,
                             max_rows=int_rows, dtype=np.int32,
                             delimiter=int_fmt).flatten()

    # read in lines of float data
    skiprows += int_rows
    flt_data = np.genfromtxt(filename, skip_header=skiprows,
                             max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64,
                             delimiter=volume['FLT_FMT']).flatten()
    hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
    skiprows += volume['FLT_HDR_ROWS']
    # read in the data
    nrows = int(np.floor(hdr['npts'] * 2 / data_cols))
    all_data = np.genfromtxt(filename, skip_header=skiprows,
                             max_rows=nrows, dtype=np.float64,
                             delimiter=volume['COL_FMT'])
    data = all_data.flatten()[1::2]
    times = all_data.flatten()[0::2]

    frac = hdr['format_specific']['fractional_unit']
    if frac > 0:
        data *= UNIT_CONVERSIONS['g'] * frac
        logging.debug('Data converted from g * %s to cm/s/s' % (frac))
    else:
        unit = _get_units(lines[11])
        if unit in UNIT_CONVERSIONS:
            data *= UNIT_CONVERSIONS[unit]
            logging.debug('Data converted from %s to cm/s/s' % (unit))
        else:
            raise GMProcessException('USC: %s is not a supported unit.' % unit)

    # Put file name into dictionary
    head, tail = os.path.split(filename)
    hdr['standard']['source_file'] = tail or os.path.basename(head)

    trace = StationTrace(data.copy(), Stats(hdr.copy()))
    if not is_evenly_spaced(times):
        trace = resample_uneven_trace(trace, times, data)

    response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
    trace.setProvenance('remove_response', response)

    # set new offset
    new_offset = skiprows + nrows
    new_offset += 1  # there is an 'end of record' line after the data

    return (trace, new_offset)
Esempio n. 2
0
def _read_channel(filename, line_offset, volume, location='', alternate=False):
    """Read channel data from USC V1 text file.

    Args:
        filename (str):
            Input USC V1 filename.
        line_offset (int):
            Line offset to beginning of channel text block.
        volume (dictionary):
            Dictionary of formatting information.

    Returns:
        tuple: (obspy Trace, int line offset)
    """
    if alternate:
        int_rows = 5
        int_fmt = 20 * [4]
        data_cols = 8
    else:
        int_rows = volume['INT_HDR_ROWS']
        int_fmt = volume['INT_FMT']
        data_cols = 10
    # Parse the header portion of the file
    try:
        with open(filename, 'rt') as f:
            for _ in range(line_offset):
                next(f)
            lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
    # Accounts for blank lines at end of files
    except StopIteration:
        return (None, 1 + line_offset)
    # read in lines of integer data
    skiprows = line_offset + volume['TEXT_HDR_ROWS']
    int_data = np.genfromtxt(filename,
                             skip_header=skiprows,
                             max_rows=int_rows,
                             dtype=np.int32,
                             delimiter=int_fmt).flatten()

    # read in lines of float data
    skiprows += int_rows
    flt_data = np.genfromtxt(filename,
                             skip_header=skiprows,
                             max_rows=volume['FLT_HDR_ROWS'],
                             dtype=np.float64,
                             delimiter=volume['FLT_FMT']).flatten()
    hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
    skiprows += volume['FLT_HDR_ROWS']
    # read in the data
    nrows = int(np.floor(hdr['npts'] * 2 / data_cols))
    all_data = np.genfromtxt(filename,
                             skip_header=skiprows,
                             max_rows=nrows,
                             dtype=np.float64,
                             delimiter=volume['COL_FMT'])
    data = all_data.flatten()[1::2]
    times = all_data.flatten()[0::2]

    frac = hdr['format_specific']['fractional_unit']
    if frac > 0:
        data *= UNIT_CONVERSIONS['g'] * frac
        logging.debug('Data converted from g * %s to cm/s/s' % (frac))
    else:
        unit = _get_units(lines[11])
        if unit in UNIT_CONVERSIONS:
            data *= UNIT_CONVERSIONS[unit]
            logging.debug('Data converted from %s to cm/s/s' % (unit))
        else:
            raise ValueError('USC: %s is not a supported unit.' % unit)

    # Put file name into dictionary
    head, tail = os.path.split(filename)
    hdr['standard']['source_file'] = tail or os.path.basename(head)

    trace = StationTrace(data.copy(), Stats(hdr.copy()))
    if not is_evenly_spaced(times):
        trace = resample_uneven_trace(trace, times, data)

    response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
    trace.setProvenance('remove_response', response)

    # set new offset
    new_offset = skiprows + nrows
    new_offset += 1  # there is an 'end of record' line after the data

    return (trace, new_offset)
Esempio n. 3
0
def _read_volume_one(filename, line_offset, location='', units='acc'):
    """Read channel data from DMG Volume 1 text file.

    Args:
        filename (str):
            Input DMG V1 filename.
        line_offset (int):
            Line offset to beginning of channel text block.
        units (str):
            units to get.

    Returns:
        tuple: (list of obspy Trace, int line offset)
    """
    # Parse the header portion of the file
    try:
        with open(filename, 'rt', encoding='utf-8') as f:
            for _ in range(line_offset):
                next(f)
            lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)]
    # Accounts for blank lines at end of files
    except StopIteration:
        return (None, 1 + line_offset)

    unit = _get_units(lines[11])
    # read in lines of integer data
    skip_rows = V1_TEXT_HDR_ROWS + line_offset
    int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename)
    int_data = int_data[0:100].astype(np.int32)

    # read in lines of float data
    skip_rows += V1_INT_HDR_ROWS
    flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename)
    skip_rows += V1_REAL_HDR_ROWS

    # according to the powers that defined the Network.Station.Channel.Location
    # "standard", Location is a two character field.  Most data providers,
    # including csmip/dmg here, don't always provide this.  We'll flag it as
    # "--".

    hdr = _get_header_info_v1(
        int_data, flt_data, lines, 'V1', location=location)
    head, tail = os.path.split(filename)
    hdr['standard']['source_file'] = tail or os.path.basename(head)

    # sometimes (??) a line of text is inserted in between the float header and
    # the beginning of the data. Let's check for this...
    with open(filename, 'rt', encoding='utf-8') as f:
        for _ in range(skip_rows):
            next(f)
        test_line = f.readline()

    has_text = re.search('[A-Z]+|[a-z]+', test_line) is not None
    if has_text:
        skip_rows += 1
        widths = [9] * 8
        max_rows = int(np.ceil(hdr['npts'] / 8))
        data = _read_lines(skip_rows, max_rows, widths, filename)
        acc_data = data[:hdr['npts']]
        evenly_spaced = True
        # Sometimes, npts is incrrectly specified, leading to nans
        # in the resulting data. Fix that here
        if np.any(np.isnan(acc_data)):
            while np.isnan(acc_data[-1]):
                acc_data = acc_data[:-1]
            hdr['npts'] = len(acc_data)
    else:
        # acceleration data is interleaved between time data
        max_rows = int(np.ceil(hdr['npts'] / 5))
        widths = [7] * 10
        data = _read_lines(skip_rows, max_rows, widths, filename)
        acc_data = data[1::2][:hdr['npts']]
        times = data[0::2][:hdr['npts']]
        evenly_spaced = is_evenly_spaced(times)

    if unit in UNIT_CONVERSIONS:
        acc_data *= UNIT_CONVERSIONS[unit]
        logging.debug('Data converted from %s to cm/s/s' % (unit))
    else:
        raise ValueError('DMG: %s is not a supported unit.' % unit)

    acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()))

    # Check if the times were included in the file but were not evenly spaced
    if not evenly_spaced:
        acc_trace = resample_uneven_trace(acc_trace, times, acc_data)

    response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
    acc_trace.setProvenance('remove_response', response)

    traces = [acc_trace]
    new_offset = skip_rows + max_rows + 1  # there is an end of record line
    return (traces, new_offset)
Esempio n. 4
0
def _read_volume_one(filename, line_offset, location="", units="acc", config=None):
    """Read channel data from DMG Volume 1 text file.

    Args:
        filename (str):
            Input DMG V1 filename.
        line_offset (int):
            Line offset to beginning of channel text block.
        units (str):
            units to get.
        config (dict):
            Configuration options.

    Returns:
        tuple: (list of obspy Trace, int line offset)
    """
    # Parse the header portion of the file
    try:
        with open(filename, "rt", encoding="utf-8") as f:
            for _ in range(line_offset):
                next(f)
            lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)]
    # Accounts for blank lines at end of files
    except StopIteration:
        return (None, 1 + line_offset)

    unit = _get_units(lines[11])
    # read in lines of integer data
    skip_rows = V1_TEXT_HDR_ROWS + line_offset
    int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename)
    int_data = int_data[0:100].astype(np.int32)

    # read in lines of float data
    skip_rows += V1_INT_HDR_ROWS
    flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename)
    skip_rows += V1_REAL_HDR_ROWS

    # ------------------------------------------------------------------------------- #
    # according to the powers that defined the Network.Station.Channel.Location
    # "standard", Location is a two character field.  Most data providers,
    # including csmip/dmg here, don't always provide this.  We'll flag it as
    # "--".

    # Update Feb 4, 2022
    # In order to be able to store structural/geotech array data in the ASDF file, we
    # HAVE to use location code to uniquely identify the different instruments. The
    # arrays can only be read in as StreamArrays and not StreamCollections. In such
    # cases we will use the channel number as location code so that the data can be
    # saved.
    # ------------------------------------------------------------------------------- #

    hdr = _get_header_info_v1(
        int_data, flt_data, lines, "V1", location=location, config=config
    )
    head, tail = os.path.split(filename)
    hdr["standard"]["source_file"] = tail or os.path.basename(head)

    # sometimes (??) a line of text is inserted in between the float header and
    # the beginning of the data. Let's check for this...
    with open(filename, "rt", encoding="utf-8") as f:
        for _ in range(skip_rows):
            next(f)
        test_line = f.readline()

    has_text = re.search("[A-Z]+|[a-z]+", test_line) is not None
    if has_text:
        skip_rows += 1
        widths = [9] * 8
        max_rows = int(np.ceil(hdr["npts"] / 8))
        data = _read_lines(skip_rows, max_rows, widths, filename)
        acc_data = data[: hdr["npts"]]
        evenly_spaced = True
        # Sometimes, npts is incrrectly specified, leading to nans
        # in the resulting data. Fix that here
        if np.any(np.isnan(acc_data)):
            while np.isnan(acc_data[-1]):
                acc_data = acc_data[:-1]
            hdr["npts"] = len(acc_data)
    else:
        # acceleration data is interleaved between time data
        max_rows = int(np.ceil(hdr["npts"] / 5))
        widths = [7] * 10
        data = _read_lines(skip_rows, max_rows, widths, filename)
        acc_data = data[1::2][: hdr["npts"]]
        times = data[0::2][: hdr["npts"]]
        evenly_spaced = is_evenly_spaced(times)

    if unit in UNIT_CONVERSIONS:
        acc_data *= UNIT_CONVERSIONS[unit]
        logging.debug(f"Data converted from {unit} to cm/s/s")
    else:
        raise ValueError(f"DMG: {unit} is not a supported unit.")

    acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()), config=config)

    # Check if the times were included in the file but were not evenly spaced
    if not evenly_spaced:
        acc_trace = resample_uneven_trace(acc_trace, times, acc_data, config=config)

    response = {"input_units": "counts", "output_units": "cm/s^2"}
    acc_trace.setProvenance("remove_response", response)

    traces = [acc_trace]
    new_offset = skip_rows + max_rows + 1  # there is an end of record line
    return (traces, new_offset)