def read_unam(filename): """Read the Mexican UNAM strong motion data format. Args: filename (str): path to UNAM data file. Returns: list: Sequence of one StationStream object containing 3 StationTrace objects. """ channels = _read_header(filename) npts = channels[0]['npts'] all_data = np.genfromtxt(filename, skip_header=ALL_HEADERS, max_rows=npts) trace1 = StationTrace(data=all_data[:, 0], header=channels[0]) trace2 = StationTrace(data=all_data[:, 1], header=channels[1]) trace3 = StationTrace(data=all_data[:, 2], header=channels[2]) # tell the trace that data has already been converted to physical units response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace1.setProvenance('remove_response', response) trace2.setProvenance('remove_response', response) trace3.setProvenance('remove_response', response) stream = StationStream(traces=[trace1, trace2, trace3]) return [stream]
def test_trace(): data = np.random.rand(1000) header = {'sampling_rate': 1, 'npts': len(data), 'network': 'US', 'location': '11', 'station': 'ABCD', 'channel': 'HN1', 'starttime': UTCDateTime(2010, 1, 1, 0, 0, 0)} inventory = get_inventory() invtrace = StationTrace(data=data, header=header, inventory=inventory) invtrace.setProvenance('detrend', {'detrending_method': 'demean'}) invtrace.setParameter('failed', True) invtrace.setParameter('corner_frequencies', [1, 2, 3]) invtrace.setParameter('metadata', {'name': 'Fred'}) assert invtrace.getProvenance('detrend')[0] == { 'detrending_method': 'demean'} assert invtrace.getParameter('failed') assert invtrace.getParameter('corner_frequencies') == [1, 2, 3] assert invtrace.getParameter('metadata') == {'name': 'Fred'}
def test_trace(): data = np.random.rand(1000) header = { 'sampling_rate': 1, 'npts': len(data), 'network': 'US', 'location': '11', 'station': 'ABCD', 'channel': 'HN1', 'starttime': UTCDateTime(2010, 1, 1, 0, 0, 0) } inventory = get_inventory() invtrace = StationTrace(data=data, header=header, inventory=inventory) invtrace.setProvenance('detrend', {'detrending_method': 'demean'}) invtrace.setParameter('failed', True) invtrace.setParameter('corner_frequencies', [1, 2, 3]) invtrace.setParameter('metadata', {'name': 'Fred'}) assert invtrace.getProvenance('detrend')[0] == { 'detrending_method': 'demean' } assert invtrace.getParameter('failed') assert invtrace.getParameter('corner_frequencies') == [1, 2, 3] assert invtrace.getParameter('metadata') == {'name': 'Fred'}
def read_renadic(filename): """Read the Chilean RENADIC strong motion data format. Args: filename (str): path to RENADIC data file. Returns: list: Sequence of one StationStream object containing 3 StationTrace objects. """ # This network does not include station coordinates in the data files, # but they did provide a PDF table with information about each station, # including structure type (free field or something else) and the # coordinates data_dir = pkg_resources.resource_filename('gmprocess', 'data') tablefile = os.path.join(data_dir, 'station_coordinates.xlsx') table = pd.read_excel(tablefile) with open(filename, 'rt', encoding=ENCODING) as f: lines1 = [next(f) for x in range(TEXT_HDR_ROWS)] header1 = _read_header(lines1, filename, table) ndata_rows = int(np.ceil((header1['npts'] * 2) / NCOLS)) skip_rows = TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS data1 = _read_data(filename, skip_rows, header1['npts']) skip_rows += ndata_rows + 1 with open(filename, 'rt', encoding=ENCODING) as f: [next(f) for x in range(skip_rows)] lines2 = [next(f) for x in range(TEXT_HDR_ROWS)] header2 = _read_header(lines2, filename, table) skip_rows += TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS data2 = _read_data(filename, skip_rows, header1['npts']) skip_rows += ndata_rows + 1 with open(filename, 'rt', encoding=ENCODING) as f: [next(f) for x in range(skip_rows)] lines3 = [next(f) for x in range(TEXT_HDR_ROWS)] header3 = _read_header(lines3, filename, table) skip_rows += TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS data3 = _read_data(filename, skip_rows, header1['npts']) trace1 = StationTrace(data=data1, header=header1) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace1.setProvenance('remove_response', response) trace2 = StationTrace(data=data2, header=header2) trace2.setProvenance('remove_response', response) trace3 = StationTrace(data=data3, header=header3) trace3.setProvenance('remove_response', response) stream = StationStream(traces=[trace1, trace2, trace3]) return [stream]
def read_nsmn(filename): """Read the Turkish NSMN strong motion data format. Args: filename (str): path to NSMN data file. Returns: list: Sequence of one StationStream object containing 3 StationTrace objects. """ header = _read_header(filename) header1 = copy.deepcopy(header) header2 = copy.deepcopy(header) header3 = copy.deepcopy(header) header1['standard']['horizontal_orientation'] = 0.0 header1['standard']['vertical_orientation'] = np.nan header1['channel'] = get_channel_name(header['sampling_rate'], True, False, True) header1['standard']['units_type'] = get_units_type(header1['channel']) header2['standard']['horizontal_orientation'] = 90.0 header2['standard']['vertical_orientation'] = np.nan header2['channel'] = get_channel_name(header['sampling_rate'], True, False, False) header2['standard']['units_type'] = get_units_type(header2['channel']) header3['standard']['horizontal_orientation'] = 0.0 header3['standard']['vertical_orientation'] = np.nan header3['channel'] = get_channel_name(header['sampling_rate'], True, True, False) header3['standard']['units_type'] = get_units_type(header3['channel']) # three columns of NS, EW, UD # data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS, # delimiter=[COLWIDTH] * NCOLS, encoding=ENCODING) data = np.loadtxt(filename, skiprows=TEXT_HDR_ROWS, encoding=ENCODING) data1 = data[:, 0] data2 = data[:, 1] data3 = data[:, 2] trace1 = StationTrace(data=data1, header=header1) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace1.setProvenance('remove_response', response) trace2 = StationTrace(data=data2, header=header2) trace2.setProvenance('remove_response', response) trace3 = StationTrace(data=data3, header=header3) trace3.setProvenance('remove_response', response) stream = StationStream(traces=[trace1, trace2, trace3]) return [stream]
def _read_volume_two(filename, line_offset, location='', units='acc'): """Read channel data from DMG text file. Args: filename (str): Input DMG V2 filename. line_offset (int): Line offset to beginning of channel text block. units (str): units to get Returns: tuple: (list of obspy Trace, int line offset) """ try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(V2_TEXT_HDR_ROWS)] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) # read in lines of integer data skip_rows = V2_TEXT_HDR_ROWS + line_offset int_data = _read_lines(skip_rows, V2_INT_HDR_ROWS, V2_INT_FMT, filename) int_data = int_data[0:100].astype(np.int32) # read in lines of float data skip_rows += V2_INT_HDR_ROWS flt_data = _read_lines(skip_rows, V2_REAL_HDR_ROWS, V2_REAL_FMT, filename) flt_data = flt_data[:100] skip_rows += V2_REAL_HDR_ROWS # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including csmip/dmg here, don't always provide this. We'll flag it as # "--". hdr = _get_header_info(int_data, flt_data, lines, 'V2', location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) traces = [] # read acceleration data if hdr['npts'] > 0: acc_rows, acc_fmt, unit = _get_data_format(filename, skip_rows, hdr['npts']) acc_data = _read_lines(skip_rows + 1, acc_rows, acc_fmt, filename) acc_data = acc_data[:hdr['npts']] if unit in UNIT_CONVERSIONS: acc_data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('DMG: %s is not a supported unit.' % unit) acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} acc_trace.setProvenance('remove_response', response) if units == 'acc': traces += [acc_trace] skip_rows += int(acc_rows) + 1 # ------------------------------------------------------------------------- # NOTE: The way we were initially reading velocity and displacement data was # not correct. I'm deleting it for now since we don't need it. If/when we # revisit this we need to be more careful about how this is handled. # ------------------------------------------------------------------------- # read velocity data vel_hdr = hdr.copy() vel_hdr['standard']['units'] = 'vel' vel_hdr['npts'] = int_data[63] if vel_hdr['npts'] > 0: vel_rows, vel_fmt, unit = _get_data_format(filename, skip_rows, vel_hdr['npts']) vel_data = _read_lines(skip_rows + 1, vel_rows, vel_fmt, filename) vel_data = vel_data[:vel_hdr['npts']] skip_rows += int(vel_rows) + 1 # read displacement data disp_hdr = hdr.copy() disp_hdr['standard']['units'] = 'disp' disp_hdr['npts'] = int_data[65] if disp_hdr['npts'] > 0: disp_rows, disp_fmt, unit = _get_data_format(filename, skip_rows, disp_hdr['npts']) disp_data = _read_lines(skip_rows + 1, disp_rows, disp_fmt, filename) disp_data = disp_data[:disp_hdr['npts']] skip_rows += int(disp_rows) + 1 new_offset = skip_rows + 1 # there is an 'end of record' line after the data] return (traces, new_offset)
def _read_channel(filename, line_offset, location=''): """Read channel data from COSMOS V1/V2 text file. Args: filename (str): Input COSMOS V1/V2 filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (obspy Trace, int line offset) """ # read station, location, and process level from text header with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(TEXT_HDR_ROWS)] # read in lines of integer data skiprows = line_offset + TEXT_HDR_ROWS int_lines, int_data = _read_lines(skiprows, filename) int_data = int_data.astype(np.int32) # read in lines of float data skiprows += int_lines + 1 flt_lines, flt_data = _read_lines(skiprows, filename) # read in comment lines skiprows += flt_lines + 1 cmt_lines, cmt_data = _read_lines(skiprows, filename) skiprows += cmt_lines + 1 # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including cosmos here, don't provide this. We'll flag it as "--". hdr = _get_header_info(int_data, flt_data, lines, cmt_data, location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # read in the data nrows, data = _read_lines(skiprows, filename) # Check for "off-by-one" problem that sometimes occurs with cosmos data # Notes: # - We cannot do this check inside _get_header_info because we don't # have the data there. # - That method is written to set npts from the header as documented in # the spec ("lenght" == npts*dt) but it appears that sometimes a # different convention is used where the "length" of the record is # actually is actuation (npts-1)*dt. In this case, we need to # recompute duration and npts if hdr['npts'] == (len(data) - 1): hdr['npts'] = len(data) hdr['duration'] = (hdr['npts'] - 1) * hdr['delta'] # check units unit = hdr['format_specific']['physical_units'] if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: if unit != 'counts': raise GMProcessException( 'COSMOS: %s is not a supported unit.' % unit) if hdr['standard']['units'] != 'acc': raise GMProcessException('COSMOS: Only acceleration data accepted.') trace = StationTrace(data.copy(), Stats(hdr.copy())) # record that this data has been converted to g, if it has if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def read_smc(filename, **kwargs): """Read SMC strong motion file. Args: filename (str): Path to possible SMC data file. kwargs (ref): any_structure (bool): Read data from any type of structure, raise Exception if False and structure type is not free-field. accept_flagged (bool): accept problem flagged data. set_location (str): Two character code for location. Other arguments will be ignored. Returns: Stream: Obspy Stream containing one channel of acceleration data (cm/s**2). """ logging.debug("Starting read_smc.") any_structure = kwargs.get('any_structure', False) accept_flagged = kwargs.get('accept_flagged', False) location = kwargs.get('location', '') if not is_smc(filename): raise Exception('%s is not a valid SMC file' % filename) with open(filename, 'rt') as f: line = f.readline().strip() if 'DISPLACEMENT' in line: raise GMProcessException( 'SMC: Diplacement records are not supported: ' '%s.' % filename) elif 'VELOCITY' in line: raise GMProcessException( 'SMC: Velocity records are not supported: ' '%s.' % filename) elif line == "*": raise GMProcessException( 'SMC: No record volume specified in file: ' '%s.' % filename) stats, num_comments = _get_header_info(filename, any_structure=any_structure, accept_flagged=accept_flagged, location=location) skip = ASCII_HEADER_LINES + INTEGER_HEADER_LINES + \ num_comments + FLOAT_HEADER_LINES # read float data (8 columns per line) nrows = int(np.floor(stats['npts'] / DATA_COLUMNS)) data = np.genfromtxt(filename, max_rows=nrows, skip_header=skip, delimiter=FLOAT_DATA_WIDTHS) data = data.flatten() if stats['npts'] % DATA_COLUMNS: lastrow = np.genfromtxt(filename, max_rows=1, skip_header=skip + nrows, delimiter=FLOAT_DATA_WIDTHS) data = np.append(data, lastrow) data = data[0:stats['npts']] trace = StationTrace(data, header=stats) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) stream = StationStream(traces=[trace]) return [stream]
def _read_volume_one(filename, line_offset, location='', units='acc'): """Read channel data from DMG Volume 1 text file. Args: filename (str): Input DMG V1 filename. line_offset (int): Line offset to beginning of channel text block. units (str): units to get Returns: tuple: (list of obspy Trace, int line offset) """ # Parse the header portion of the file try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) unit = _get_units(lines[11]) # read in lines of integer data skip_rows = V1_TEXT_HDR_ROWS + line_offset int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename) int_data = int_data[0:100].astype(np.int32) # read in lines of float data skip_rows += V1_INT_HDR_ROWS flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename) skip_rows += V1_REAL_HDR_ROWS # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including csmip/dmg here, don't always provide this. We'll flag it as # "--". hdr = _get_header_info_v1(int_data, flt_data, lines, 'V1', location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # sometimes (??) a line of text is inserted in between the float header and # the beginning of the data. Let's check for this... with open(filename, 'rt') as f: for _ in range(skip_rows): next(f) test_line = f.readline() has_text = re.search('[A-Z]+|[a-z]+', test_line) is not None if has_text: skip_rows += 1 widths = [9] * 8 max_rows = int(np.ceil(hdr['npts'] / 8)) data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[:hdr['npts']] evenly_spaced = True # Sometimes, npts is incrrectly specified, leading to nans # in the resulting data. Fix that here if np.any(np.isnan(acc_data)): while np.isnan(acc_data[-1]): acc_data = acc_data[:-1] hdr['npts'] = len(acc_data) else: # acceleration data is interleaved between time data max_rows = int(np.ceil(hdr['npts'] / 5)) widths = [7] * 10 data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[1::2][:hdr['npts']] times = data[0::2][:hdr['npts']] evenly_spaced = is_evenly_spaced(times) if unit in UNIT_CONVERSIONS: acc_data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('DMG: %s is not a supported unit.' % unit) acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy())) # Check if the times were included in the file but were not evenly spaced if not evenly_spaced: acc_trace = resample_uneven_trace(acc_trace, times, acc_data) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} acc_trace.setProvenance('remove_response', response) traces = [acc_trace] new_offset = skip_rows + max_rows + 1 # there is an end of record line return (traces, new_offset)
def _read_channel(filename, line_offset, location=''): """Read channel data from COSMOS V1/V2 text file. Args: filename (str): Input COSMOS V1/V2 filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (obspy Trace, int line offset) """ # read station, location, and process level from text header with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(TEXT_HDR_ROWS)] # read in lines of integer data skiprows = line_offset + TEXT_HDR_ROWS int_lines, int_data = _read_lines(skiprows, filename) int_data = int_data.astype(np.int32) # read in lines of float data skiprows += int_lines + 1 flt_lines, flt_data = _read_lines(skiprows, filename) # read in comment lines skiprows += flt_lines + 1 cmt_lines, cmt_data = _read_lines(skiprows, filename) skiprows += cmt_lines + 1 # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including cosmos here, don't provide this. We'll flag it as "--". hdr = _get_header_info(int_data, flt_data, lines, cmt_data, location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # read in the data nrows, data = _read_lines(skiprows, filename) # check units unit = hdr['format_specific']['physical_units'] if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('COSMOS: %s is not a supported unit.' % unit) if hdr['standard']['units'] != 'acc': raise GMProcessException('COSMOS: Only acceleration data accepted.') trace = StationTrace(data.copy(), Stats(hdr.copy())) # record that this data has been converted to g, if it has if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def _read_channel(filename, line_offset): """Read channel data from GNS V1 text file. Args: filename (str): Input GNS V1 filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (obspy Trace, int line offset) """ # read station and location strings from text header with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(TEXT_HDR_ROWS)] # this code supports V1 and V2 format files. Which one is this? data_format = 'V2' if lines[0].lower().find('uncorrected') >= 0: data_format = 'V1' # parse out the station code, name, and component string # from text header station = lines[1].split()[1] logging.debug('station: %s' % station) name = lines[2].replace(' ', '_').strip() component = lines[12].split()[1] # parse the instrument type from the text header instrument = lines[3].split()[1] # parse the sensor resolution from the text header resolution_str = lines[4].split()[1] resolution = int(re.search('\d+', resolution_str).group()) # read floating point header array skip_header = line_offset + TEXT_HDR_ROWS hdr_data = np.genfromtxt(filename, skip_header=skip_header, max_rows=FP_HDR_ROWS) # parse header dictionary from float header array hdr = _read_header(hdr_data, station, name, component, data_format, instrument, resolution) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including GeoNet here, don't provide this. We'll flag it as "--". hdr['location'] = '--' skip_header2 = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS widths = [8] * COLS_PER_ROW nrows = int(np.ceil(hdr['npts'] / COLS_PER_ROW)) data = np.genfromtxt(filename, skip_header=skip_header2, max_rows=nrows, filling_values=np.nan, delimiter=widths) data = data.flatten() data = data[0:hdr['npts']] # for debugging, read in the velocity data nvel = hdr_data[3, 4] if nvel: if nvel % COLS_PER_ROW != 0: nvel_rows = int(np.floor(nvel / COLS_PER_ROW)) nvel_rows2 = 1 else: nvel_rows = int(np.ceil(nvel / COLS_PER_ROW)) nvel_rows2 = 0 skip_header_vel = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS + nrows widths = [8] * COLS_PER_ROW velocity = np.genfromtxt(filename, skip_header=skip_header_vel, max_rows=nvel_rows, filling_values=np.nan, delimiter=widths) velocity = velocity.flatten() velocity *= MMPS_TO_CMPS else: velocity = np.array([]) # for V2 files, there are extra blocks of data we need to skip containing # velocity and displacement data if data_format == 'V2': velrows = int(np.ceil(hdr_data[3, 4] / COLS_PER_ROW)) disrows = int(np.ceil(hdr_data[3, 5] / COLS_PER_ROW)) nrows = nrows + velrows + disrows data *= MMPS_TO_CMPS # convert to cm/s**2 trace = StationTrace(data, Stats(hdr)) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) offset = skip_header2 + nrows return (trace, offset, velocity)
def read_knet(filename): """Read Japanese KNET strong motion file. Args: filename (str): Path to possible KNET data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ logging.debug("Starting read_knet.") if not is_knet(filename): raise Exception('%s is not a valid KNET file' % filename) # Parse the header portion of the file with open(filename, 'rt') as f: lines = [next(f) for x in range(TEXT_HDR_ROWS)] hdr = {} coordinates = {} standard = {} hdr['network'] = 'BO' hdr['station'] = lines[5].split()[2] logging.debug('station: %s' % hdr['station']) standard['station_name'] = '' # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including KNET here, don't provide this. We'll flag it as "--". hdr['location'] = '--' coordinates['latitude'] = float(lines[6].split()[2]) coordinates['longitude'] = float(lines[7].split()[2]) coordinates['elevation'] = float(lines[8].split()[2]) hdr['sampling_rate'] = float( re.search('\\d+', lines[10].split()[2]).group()) hdr['delta'] = 1 / hdr['sampling_rate'] standard['units'] = 'acc' dir_string = lines[12].split()[1].strip() # knet files have directions listed as N-S, E-W, or U-D, # whereas in kiknet those directions are '4', '5', or '6'. if dir_string in ['N-S', '1', '4']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) elif dir_string in ['E-W', '2', '5']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) elif dir_string in ['U-D', '3', '6']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) else: raise Exception('KNET: Could not parse direction %s' % lines[12].split()[1]) logging.debug('channel: %s' % hdr['channel']) scalestr = lines[13].split()[2] parts = scalestr.split('/') num = float(parts[0].replace('(gal)', '')) den = float(parts[1]) calib = num / den hdr['calib'] = calib duration = float(lines[11].split()[2]) hdr['npts'] = int(duration * hdr['sampling_rate']) timestr = ' '.join(lines[9].split()[2:4]) # The K-NET and KiK-Net data logger adds a 15s time delay # this is removed here sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0) # Shift the time to utc (Japanese time is 9 hours ahead) sttime = sttime - timedelta(seconds=9 * 3600.) hdr['starttime'] = sttime # read in the data - there is a max of 8 columns per line # the code below handles the case when last line has # less than 8 columns if hdr['npts'] % COLS_PER_LINE != 0: nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE)) nrows2 = 1 else: nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE)) nrows2 = 0 data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS, max_rows=nrows, filling_values=np.nan) data = data.flatten() if nrows2: skip_header = TEXT_HDR_ROWS + nrows data2 = np.genfromtxt(filename, skip_header=skip_header, max_rows=nrows2, filling_values=np.nan) data = np.hstack((data, data2)) nrows += nrows2 # apply the correction factor we're given in the header data *= calib # fill out the rest of the standard dictionary standard['horizontal_orientation'] = np.nan standard['instrument_period'] = np.nan standard['instrument_damping'] = np.nan standard['process_time'] = '' standard['process_level'] = PROCESS_LEVELS['V1'] standard['sensor_serial_number'] = '' standard['instrument'] = '' standard['comments'] = '' standard['structure_type'] = '' if dir_string in ['1', '2', '3']: standard['structure_type'] = 'borehole' standard['corner_frequency'] = np.nan standard['units'] = 'acc' standard['source'] = SRC standard['source_format'] = 'knet' hdr['coordinates'] = coordinates hdr['standard'] = standard # create a Trace from the data and metadata trace = StationTrace(data.copy(), Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) stream = StationStream(traces=[trace]) return [stream]
def read_cwb(filename, **kwargs): """Read Taiwan Central Weather Bureau strong motion file. Args: filename (str): Path to possible CWB data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ logging.debug("Starting read_cwb.") if not is_cwb(filename): raise Exception('%s is not a valid CWB strong motion data file.' % filename) f = open(filename, 'rt') # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including CWB here, don't provide this. We'll flag it as "--". data = np.genfromtxt(filename, skip_header=HDR_ROWS, delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW hdr = _get_header_info(f, data) f.close() head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) hdr_z = hdr.copy() hdr_z['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) hdr_z['standard']['horizontal_orientation'] = np.nan hdr_h1 = hdr.copy() hdr_h1['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) hdr_h1['standard']['horizontal_orientation'] = np.nan hdr_h2 = hdr.copy() hdr_h2['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) hdr_h2['standard']['horizontal_orientation'] = np.nan stats_z = Stats(hdr_z) stats_h1 = Stats(hdr_h1) stats_h2 = Stats(hdr_h2) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace_z = StationTrace(data=data[:, 1], header=stats_z) trace_z.setProvenance('remove_response', response) trace_h1 = StationTrace(data=data[:, 2], header=stats_h1) trace_h1.setProvenance('remove_response', response) trace_h2 = StationTrace(data=data[:, 3], header=stats_h2) trace_h2.setProvenance('remove_response', response) stream = StationStream([trace_z, trace_h1, trace_h2]) return [stream]
def read_knet(filename): """Read Japanese KNET strong motion file. Args: filename (str): Path to possible KNET data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ logging.debug("Starting read_knet.") if not is_knet(filename): raise Exception('%s is not a valid KNET file' % filename) # Parse the header portion of the file with open(filename, 'rt') as f: lines = [next(f) for x in range(TEXT_HDR_ROWS)] hdr = {} coordinates = {} standard = {} hdr['network'] = 'BO' hdr['station'] = lines[5].split()[2] logging.debug('station: %s' % hdr['station']) standard['station_name'] = '' # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including KNET here, don't provide this. We'll flag it as "--". hdr['location'] = '--' coordinates['latitude'] = float(lines[6].split()[2]) coordinates['longitude'] = float(lines[7].split()[2]) coordinates['elevation'] = float(lines[8].split()[2]) hdr['sampling_rate'] = float( re.search('\\d+', lines[10].split()[2]).group()) hdr['delta'] = 1 / hdr['sampling_rate'] standard['units'] = 'acc' dir_string = lines[12].split()[1].strip() # knet files have directions listed as N-S, E-W, or U-D, # whereas in kiknet those directions are '4', '5', or '6'. if dir_string in ['N-S', '1', '4']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) elif dir_string in ['E-W', '2', '5']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) elif dir_string in ['U-D', '3', '6']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) else: raise Exception('KNET: Could not parse direction %s' % lines[12].split()[1]) logging.debug('channel: %s' % hdr['channel']) scalestr = lines[13].split()[2] parts = scalestr.split('/') num = float(parts[0].replace('(gal)', '')) den = float(parts[1]) calib = num / den hdr['calib'] = calib duration = float(lines[11].split()[2]) hdr['npts'] = int(duration * hdr['sampling_rate']) timestr = ' '.join(lines[9].split()[2:4]) # The K-NET and KiK-Net data logger adds a 15s time delay # this is removed here sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0) # Shift the time to utc (Japanese time is 9 hours ahead) sttime = sttime - timedelta(seconds=9 * 3600.) hdr['starttime'] = sttime # read in the data - there is a max of 8 columns per line # the code below handles the case when last line has # less than 8 columns if hdr['npts'] % COLS_PER_LINE != 0: nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE)) nrows2 = 1 else: nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE)) nrows2 = 0 data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS, max_rows=nrows, filling_values=np.nan) data = data.flatten() if nrows2: skip_header = TEXT_HDR_ROWS + nrows data2 = np.genfromtxt(filename, skip_header=skip_header, max_rows=nrows2, filling_values=np.nan) data = np.hstack((data, data2)) nrows += nrows2 # apply the correction factor we're given in the header data *= calib # fill out the rest of the standard dictionary standard['horizontal_orientation'] = np.nan standard['instrument_period'] = np.nan standard['instrument_damping'] = np.nan standard['process_time'] = '' standard['process_level'] = PROCESS_LEVELS['V1'] standard['sensor_serial_number'] = '' standard['instrument'] = '' standard['comments'] = '' standard['structure_type'] = '' if dir_string in ['1', '2', '3']: standard['structure_type'] = 'borehole' standard['corner_frequency'] = np.nan standard['units'] = 'acc' standard['source'] = SRC standard['source_format'] = 'knet' head, tail = os.path.split(filename) standard['source_file'] = tail or os.path.basename(head) hdr['coordinates'] = coordinates hdr['standard'] = standard # create a Trace from the data and metadata trace = StationTrace(data.copy(), Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) stream = StationStream(traces=[trace]) return [stream]
def _read_channel(filename, line_offset, volume, location='', alternate=False): """Read channel data from USC V1 text file. Args: filename (str): Input USC V1 filename. line_offset (int): Line offset to beginning of channel text block. volume (dictionary): Dictionary of formatting information Returns: tuple: (obspy Trace, int line offset) """ if alternate: int_rows = 5 int_fmt = 20 * [4] data_cols = 8 else: int_rows = volume['INT_HDR_ROWS'] int_fmt = volume['INT_FMT'] data_cols = 10 # Parse the header portion of the file try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) # read in lines of integer data skiprows = line_offset + volume['TEXT_HDR_ROWS'] int_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=int_rows, dtype=np.int32, delimiter=int_fmt).flatten() # read in lines of float data skiprows += int_rows flt_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64, delimiter=volume['FLT_FMT']).flatten() hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location) skiprows += volume['FLT_HDR_ROWS'] # read in the data nrows = int(np.floor(hdr['npts'] * 2 / data_cols)) all_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=nrows, dtype=np.float64, delimiter=volume['COL_FMT']) data = all_data.flatten()[1::2] times = all_data.flatten()[0::2] frac = hdr['format_specific']['fractional_unit'] if frac > 0: data *= UNIT_CONVERSIONS['g'] * frac logging.debug('Data converted from g * %s to cm/s/s' % (frac)) else: unit = _get_units(lines[11]) if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('USC: %s is not a supported unit.' % unit) # Put file name into dictionary head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) trace = StationTrace(data.copy(), Stats(hdr.copy())) if not is_evenly_spaced(times): trace = resample_uneven_trace(trace, times, data) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def read_at2(dfile, horient=0.0): # This is a conveneince method so we can read in these specific data for # testing, it is not a general purpose reader since this format does not # contain a lot of metadata that is generally required for it to be useful. skiprows = 4 datafile = open(dfile, 'r') datareader = csv.reader(datafile) data = [] header = [] # for i in range(skiprows): # next(datareader) # header.append(datareader.readlines()) count = 0 for row in datareader: if count < skiprows: header.append(row) else: data.extend([float(e) for e in row[0].split()]) count += 1 datafile.close() hdr = {} hdr['network'] = '' hdr['station'] = '' if horient == 0: hdr['channel'] = 'BH1' else: hdr['channel'] = 'BH2' hdr['location'] = '--' dt = float(header[3][1].split('=')[1].strip().lower().replace('sec', '')) hdr['npts'] = len(data) hdr['sampling_rate'] = 1 / dt hdr['duration'] = (hdr['npts'] - 1) / hdr['sampling_rate'] hdr['starttime'] = 0 # There is no lat/lon... hdr['coordinates'] = {'latitude': 0.0, 'longitude': 0.0, 'elevation': 0.0} standard = {} standard['units'] = 'acc' standard['units_type'] = 'acc' standard['horizontal_orientation'] = horient standard['vertical_orientation'] = np.nan standard['source_file'] = dfile standard['station_name'] = '' standard['corner_frequency'] = 30.0 standard['structure_type'] = '' standard['comments'] = '' standard['instrument'] = '' standard['instrument_period'] = 1.0 standard['instrument_sensitivity'] = 1.0 standard['source'] = 'PEER' standard['instrument_damping'] = 0.1 standard['sensor_serial_number'] = '' standard['process_level'] = 'corrected physical units' standard['source_format'] = 'AT2' standard['process_time'] = '' hdr['standard'] = standard # convert data from g to cm/s^2 g_to_cmss = 980.665 tr = StationTrace(np.array(data.copy()) * g_to_cmss, Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} tr.setProvenance('remove_response', response) return tr
def read_cwb(filename, **kwargs): """Read Taiwan Central Weather Bureau strong motion file. Args: filename (str): Path to possible CWB data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ logging.debug("Starting read_cwb.") if not is_cwb(filename): raise Exception('%s is not a valid CWB strong motion data file.' % filename) f = open(filename, 'rt') # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including CWB here, don't provide this. We'll flag it as "--". data = np.genfromtxt(filename, skip_header=HDR_ROWS, delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW hdr = _get_header_info(f, data) f.close() head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) hdr_z = hdr.copy() hdr_z['channel'] = get_channel_name(hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) hdr_z['standard']['horizontal_orientation'] = np.nan hdr_z['standard']['vertical_orientation'] = np.nan hdr_z['standard']['units_type'] = get_units_type(hdr_z['channel']) hdr_h1 = hdr.copy() hdr_h1['channel'] = get_channel_name(hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) hdr_h1['standard']['horizontal_orientation'] = np.nan hdr_h1['standard']['vertical_orientation'] = np.nan hdr_h1['standard']['units_type'] = get_units_type(hdr_h1['channel']) hdr_h2 = hdr.copy() hdr_h2['channel'] = get_channel_name(hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) hdr_h2['standard']['horizontal_orientation'] = np.nan hdr_h2['standard']['vertical_orientation'] = np.nan hdr_h2['standard']['units_type'] = get_units_type(hdr_h2['channel']) stats_z = Stats(hdr_z) stats_h1 = Stats(hdr_h1) stats_h2 = Stats(hdr_h2) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace_z = StationTrace(data=data[:, 1], header=stats_z) trace_z.setProvenance('remove_response', response) trace_h1 = StationTrace(data=data[:, 2], header=stats_h1) trace_h1.setProvenance('remove_response', response) trace_h2 = StationTrace(data=data[:, 3], header=stats_h2) trace_h2.setProvenance('remove_response', response) stream = StationStream([trace_z, trace_h1, trace_h2]) return [stream]
def read_esm(filename): """Read European ESM strong motion file. Args: filename (str): Path to possible ESM data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing one channels of acceleration data (cm/s**2). """ logging.debug("Starting read_esm.") if not is_esm(filename): raise Exception('%s is not a valid ESM file' % filename) # Parse the header portion of the file header = {} with open(filename, 'rt') as f: lines = [next(f) for x in range(TEXT_HDR_ROWS)] for line in lines: parts = line.split(':') key = parts[0].strip() value = ':'.join(parts[1:]).strip() header[key] = value stats = {} standard = {} format_specific = {} coordinates = {} # fill in all known stats header fields stats['network'] = header['NETWORK'] stats['station'] = header['STATION_CODE'] stats['channel'] = header['STREAM'] stats['location'] = '--' stats['delta'] = float(header['SAMPLING_INTERVAL_S']) stats['sampling_rate'] = 1 / stats['delta'] stats['calib'] = 1.0 stats['npts'] = int(header['NDATA']) stimestr = header['DATE_TIME_FIRST_SAMPLE_YYYYMMDD_HHMMSS'] stats['starttime'] = datetime.strptime(stimestr, TIMEFMT) # fill in standard fields head, tail = os.path.split(filename) standard['source_file'] = tail or os.path.basename(head) standard['source'] = SRC standard['source_format'] = FORMAT standard['horizontal_orientation'] = np.nan standard['station_name'] = header['STATION_NAME'] try: standard['instrument_period'] = 1 / \ float(header['INSTRUMENTAL_FREQUENCY_HZ']) except ValueError: standard['instrument_period'] = np.nan try: standard['instrument_damping'] = 1 / \ float(header['INSTRUMENTAL_DAMPING']) except ValueError: standard['instrument_damping'] = np.nan ptimestr = header['DATA_TIMESTAMP_YYYYMMDD_HHMMSS'] ptime = datetime.strptime(ptimestr, TIMEFMT).strftime(TIMEFMT2) standard['process_time'] = ptime standard['process_level'] = PROCESS_LEVELS['V1'] instr_str = header['INSTRUMENT'] parts = instr_str.split('|') sensor_str = parts[0].split('=')[1].strip() standard['sensor_serial_number'] = '' standard['instrument'] = sensor_str standard['comments'] = '' standard['structure_type'] = '' standard['units'] = 'cm/s^2' standard['units_type'] = 'acc' standard['instrument_sensitivity'] = np.nan standard['corner_frequency'] = np.nan coordinates['latitude'] = float(header['STATION_LATITUDE_DEGREE']) coordinates['longitude'] = float(header['STATION_LONGITUDE_DEGREE']) coordinates['elevation'] = float(header['STATION_ELEVATION_M']) # read in the data data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS) # create a Trace from the data and metadata stats['standard'] = standard stats['coordinates'] = coordinates trace = StationTrace(data.copy(), Stats(stats.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) ftype = header['FILTER_TYPE'].capitalize() forder = int(header['FILTER_ORDER']) lowfreq = float(header['LOW_CUT_FREQUENCY_HZ']) highfreq = float(header['LOW_CUT_FREQUENCY_HZ']) filter_att = {'bandpass_filter': {'filter_type': ftype, 'lower_corner_frequency': lowfreq, 'higher_corner_frequency': highfreq, 'filter_order': forder}} trace.setProvenance('lowpass_filter', filter_att) detrend_att = {'detrend': {'detrending_method': 'baseline'}} trace.setProvenance('detrend', detrend_att) stream = StationStream(traces=[trace]) return [stream]
def read_smc(filename, **kwargs): """Read SMC strong motion file. Args: filename (str): Path to possible SMC data file. kwargs (ref): any_structure (bool): Read data from any type of structure, raise Exception if False and structure type is not free-field. accept_flagged (bool): accept problem flagged data. set_location (str): Two character code for location. Other arguments will be ignored. Returns: Stream: Obspy Stream containing one channel of acceleration data (cm/s**2). """ logging.debug("Starting read_smc.") any_structure = kwargs.get('any_structure', False) accept_flagged = kwargs.get('accept_flagged', False) location = kwargs.get('location', '') if not is_smc(filename): raise Exception('%s is not a valid SMC file' % filename) with open(filename, 'rt') as f: line = f.readline().strip() if 'DISPLACEMENT' in line: raise GMProcessException('SMC: Diplacement records are not supported: ' '%s.' % filename) elif 'VELOCITY' in line: raise GMProcessException('SMC: Velocity records are not supported: ' '%s.' % filename) elif line == "*": raise GMProcessException('SMC: No record volume specified in file: ' '%s.' % filename) stats, num_comments = _get_header_info( filename, any_structure=any_structure, accept_flagged=accept_flagged, location=location) skip = ASCII_HEADER_LINES + INTEGER_HEADER_LINES + \ num_comments + FLOAT_HEADER_LINES # read float data (8 columns per line) nrows = int(np.floor(stats['npts'] / DATA_COLUMNS)) data = np.genfromtxt(filename, max_rows=nrows, skip_header=skip, delimiter=FLOAT_DATA_WIDTHS) data = data.flatten() if stats['npts'] % DATA_COLUMNS: lastrow = np.genfromtxt(filename, max_rows=1, skip_header=skip + nrows, delimiter=FLOAT_DATA_WIDTHS) data = np.append(data, lastrow) data = data[0:stats['npts']] trace = StationTrace(data, header=stats) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) stream = StationStream(traces=[trace]) return [stream]