def read_nsmn(filename): """Read the Turkish NSMN strong motion data format. Args: filename (str): path to NSMN data file. Returns: list: Sequence of one StationStream object containing 3 StationTrace objects. """ header = _read_header(filename) header1 = copy.deepcopy(header) header2 = copy.deepcopy(header) header3 = copy.deepcopy(header) header1['standard']['horizontal_orientation'] = 0.0 header1['channel'] = get_channel_name(header['sampling_rate'], True, False, True) header2['standard']['horizontal_orientation'] = 90.0 header2['channel'] = get_channel_name(header['sampling_rate'], True, False, False) header3['standard']['horizontal_orientation'] = 0.0 header3['channel'] = get_channel_name(header['sampling_rate'], True, True, False) # three columns of NS, EW, UD # data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS, # delimiter=[COLWIDTH] * NCOLS, encoding=ENCODING) data = np.loadtxt(filename, skiprows=TEXT_HDR_ROWS, encoding=ENCODING) data1 = data[:, 0] data2 = data[:, 1] data3 = data[:, 2] trace1 = StationTrace(data=data1, header=header1) trace2 = StationTrace(data=data2, header=header2) trace3 = StationTrace(data=data3, header=header3) stream = StationStream(traces=[trace1, trace2, trace3]) return [stream]
def test_arias(): ddir = os.path.join('data', 'testdata') datadir = pkg_resources.resource_filename('gmprocess', ddir) data_file = os.path.join(datadir, 'arias_data.json') with open(data_file, 'rt') as f: jdict = json.load(f) time = np.array(jdict['time']) # input output is m/s/s acc = np.array(jdict['acc']) / 100 target_IA = jdict['ia'] delta = time[2] - time[1] sr = 1 / delta header = { 'delta': delta, 'sampling_rate': sr, 'npts': len(acc), 'units': 'm/s/s', 'channel': 'HN1', 'standard': { 'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'source_file': '', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'structure_type': '', 'sensor_serial_number': '', 'process_level': 'raw counts', 'process_time': '', 'horizontal_orientation': np.nan, 'units': 'acc', 'instrument_damping': np.nan } } # input is cm/s/s output is m/s/s trace = StationTrace(data=acc * 100, header=header) trace2 = trace.copy() trace2.stats.channel = 'HN2' stream = StationStream([trace, trace2]) station = StationSummary.from_stream(stream, ['ARITHMETIC_MEAN'], ['arias']) pgms = station.pgms Ia = pgms[(pgms.IMT == 'ARIAS') & (pgms.IMC == 'ARITHMETIC_MEAN')].Result.tolist()[0] # the target has only one decimal place and is in cm/s/s Ia = Ia * 100 np.testing.assert_almost_equal(Ia, target_IA, decimal=1) # Test other components data_files, _ = read_data_dir('cwb', 'us1000chhc', '2-ECU.dat') stream = read_data(data_files[0])[0] station = StationSummary.from_stream(stream, [ 'channels', 'gmrotd', 'rotd50', 'greater_of_two_horizontals', 'ARITHMETIC_MEAN' ], ['arias']) stream = StationSummary.from_stream(stream, ['gmrotd50'], ['arias']) assert stream.pgms.Result.tolist() == []
def read_bhrc(filename): """Read the Iran BHRC strong motion data format. Args: filename (str): path to BHRC data file. Returns: list: Sequence of one StationStream object containing 3 StationTrace objects. """ header1, offset = _read_header_lines(filename, 0) data1, offset = _read_data(filename, offset, header1) header2, offset = _read_header_lines(filename, offset) data2, offset = _read_data(filename, offset, header2) header3, offset = _read_header_lines(filename, offset) data3, offset = _read_data(filename, offset, header3) trace1 = StationTrace(data1, header1) trace2 = StationTrace(data2, header2) trace3 = StationTrace(data3, header3) stream = StationStream([trace1, trace2, trace3]) for tr in stream: if tr.stats.standard.process_level != PROCESS_LEVELS['V0']: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} tr.setProvenance('remove_response', response) return [stream]
def test_arias(): ddir = os.path.join('data', 'testdata') datadir = pkg_resources.resource_filename('gmprocess', ddir) data_file = os.path.join(datadir, 'arias_data.json') with open(data_file, 'rt') as f: jdict = json.load(f) time = np.array(jdict['time']) # input output is m/s/s acc = np.array(jdict['acc']) / 100 target_IA = jdict['ia'] delta = time[2] - time[1] sr = 1 / delta header = { 'delta': delta, 'sampling_rate': sr, 'npts': len(acc), 'units': 'm/s/s', 'channel': 'HN1', 'standard': {'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'source_file': '', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'structure_type': '', 'sensor_serial_number': '', 'process_level': 'raw counts', 'process_time': '', 'horizontal_orientation': np.nan, 'units': 'acc', 'instrument_damping': np.nan} } # input is cm/s/s output is m/s/s trace = StationTrace(data=acc * 100, header=header) trace2 = trace.copy() trace2.stats.channel = 'HN2' stream = StationStream([trace, trace2]) station = StationSummary.from_stream(stream, ['ARITHMETIC_MEAN'], ['arias']) pgms = station.pgms Ia = pgms[(pgms.IMT == 'ARIAS') & (pgms.IMC == 'ARITHMETIC_MEAN')].Result.tolist()[0] # the target has only one decimal place and is in cm/s/s Ia = Ia * 100 np.testing.assert_almost_equal(Ia, target_IA, decimal=1) # Test other components data_files, _ = read_data_dir('cwb', 'us1000chhc', '2-ECU.dat') stream = read_data(data_files[0])[0] station = StationSummary.from_stream(stream, ['channels', 'gmrotd', 'rotd50', 'greater_of_two_horizontals', 'ARITHMETIC_MEAN'], ['arias']) stream = StationSummary.from_stream(stream, ['gmrotd50'], ['arias']) assert stream.pgms.Result.tolist() == []
def read_fdsn(filename): """Read Obspy data file (SAC, MiniSEED, etc). Args: filename (str): Path to data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: StationStream object. """ logging.debug("Starting read_fdsn.") if not is_fdsn(filename): raise Exception('%s is not a valid Obspy file format.' % filename) streams = [] tstream = read(filename) xmlfile = _get_station_file(filename, tstream) inventory = read_inventory(xmlfile) traces = [] for ttrace in tstream: trace = StationTrace(data=ttrace.data, header=ttrace.stats, inventory=inventory) location = ttrace.stats.location trace.stats.channel = get_channel_name( trace.stats.sampling_rate, trace.stats.channel[1] == 'N', inventory.get_orientation(trace.id)['dip'] in [90, -90] or trace.stats.channel[2] == 'Z', is_channel_north(inventory.get_orientation(trace.id)['azimuth'])) if trace.stats.location == '': trace.stats.location = '--' network = ttrace.stats.network if network in LOCATION_CODES: codes = LOCATION_CODES[network] if location in codes: sdict = codes[location] if sdict['free_field']: trace.stats.standard.structure_type = 'free_field' else: trace.stats.standard.structure_type = sdict['description'] head, tail = os.path.split(filename) trace.stats['standard']['source_file'] = tail or os.path.basename(head) traces.append(trace) stream = StationStream(traces=traces) streams.append(stream) return streams
def read_unam(filename): """Read the Mexican UNAM strong motion data format. Args: filename (str): path to UNAM data file. Returns: list: Sequence of one StationStream object containing 3 StationTrace objects. """ channels = _read_header(filename) npts = channels[0]['npts'] all_data = np.genfromtxt(filename, skip_header=ALL_HEADERS, max_rows=npts) trace1 = StationTrace(data=all_data[:, 0], header=channels[0]) trace2 = StationTrace(data=all_data[:, 1], header=channels[1]) trace3 = StationTrace(data=all_data[:, 2], header=channels[2]) stream = StationStream(traces=[trace1, trace2, trace3]) return [stream]
def read_bhrc(filename): """Read the Iran BHRC strong motion data format. Args: filename (str): path to BHRC data file. Returns: list: Sequence of one StationStream object containing 3 StationTrace objects. """ header1, offset = _read_header_lines(filename, 0) data1, offset = _read_data(filename, offset, header1) header2, offset = _read_header_lines(filename, offset) data2, offset = _read_data(filename, offset, header2) header3, offset = _read_header_lines(filename, offset) data3, offset = _read_data(filename, offset, header3) trace1 = StationTrace(data1, header1) trace2 = StationTrace(data2, header2) trace3 = StationTrace(data3, header3) stream = StationStream([trace1, trace2, trace3]) return [stream]
def read_fdsn(filename): """Read Obspy data file (SAC, MiniSEED, etc). Args: filename (str): Path to data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: StationStream object. """ logging.debug("Starting read_fdsn.") if not is_fdsn(filename): raise Exception('%s is not a valid Obspy file format.' % filename) streams = [] tstream = read(filename) xmlfile = _get_station_file(filename, tstream) inventory = read_inventory(xmlfile) traces = [] for ttrace in tstream: trace = StationTrace(data=ttrace.data, header=ttrace.stats, inventory=inventory) location = ttrace.stats.location network = ttrace.stats.network if network in LOCATION_CODES: codes = LOCATION_CODES[network] if location in codes: sdict = codes[location] if sdict['free_field']: trace.stats.standard.structure_type = 'free_field' else: trace.stats.standard.structure_type = sdict['description'] head, tail = os.path.split(filename) trace.stats['standard']['source_file'] = tail or os.path.basename(head) traces.append(trace) stream = StationStream(traces=traces) streams.append(stream) return streams
def get_integral(self): """ Calculated the integral of each trace's data. Returns: stream: StationStream with the integrated data. """ stream = StationStream([]) for trace in self.transform_data: integrated_trace = trace.integrate() integrated_trace.stats['units'] = 'veloc' strace = StationTrace(data=integrated_trace.data, header=integrated_trace.stats) stream.append(strace) return stream
def get_derivative(self): """ Calculated the derivative of each trace's data. Returns: stream: StationStream with the differentiated data. """ stream = StationStream([]) for trace in self.transform_data: integrated_trace = trace.differentiate() integrated_trace.stats['units'] = 'acc' strace = StationTrace(data=integrated_trace.data, header=integrated_trace.stats) stream.append(strace) return stream
def get_arias(self): """ Performs calculation of arias intensity. Returns: arias_intensities: Dictionary of arias intensity for each channel. """ arias_intensities = {} arias_stream = StationStream([]) for trace in self.reduction_data: dt = trace.stats['delta'] # convert from cm/s/s to m/s/s acc = trace.data * 0.01 # Calculate Arias Intensity integrated_acc2 = integrate.cumtrapz(acc * acc, dx=dt) arias_intensity = integrated_acc2 * np.pi * GAL_TO_PCTG / 2 channel = trace.stats.channel trace.stats.standard.units = 'veloc' trace.stats.npts = len(arias_intensity) arias_stream.append(StationTrace(arias_intensity, trace.stats)) arias_intensities[channel] = np.abs(np.max(arias_intensity)) self.arias_stream = arias_stream return arias_intensities
def read_esm(filename): """Read European ESM strong motion file. Args: filename (str): Path to possible ESM data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing one channels of acceleration data (cm/s**2). """ logging.debug("Starting read_esm.") if not is_esm(filename): raise Exception('%s is not a valid ESM file' % filename) # Parse the header portion of the file header = {} with open(filename, 'rt') as f: lines = [next(f) for x in range(TEXT_HDR_ROWS)] for line in lines: parts = line.split(':') key = parts[0].strip() value = ':'.join(parts[1:]).strip() header[key] = value stats = {} standard = {} format_specific = {} coordinates = {} # fill in all known stats header fields stats['network'] = header['NETWORK'] stats['station'] = header['STATION_CODE'] stats['channel'] = header['STREAM'] stats['location'] = '--' stats['delta'] = float(header['SAMPLING_INTERVAL_S']) stats['sampling_rate'] = 1 / stats['delta'] stats['calib'] = 1.0 stats['npts'] = int(header['NDATA']) stimestr = header['DATE_TIME_FIRST_SAMPLE_YYYYMMDD_HHMMSS'] stats['starttime'] = datetime.strptime(stimestr, TIMEFMT) # fill in standard fields head, tail = os.path.split(filename) standard['source_file'] = tail or os.path.basename(head) standard['source'] = SRC standard['source_format'] = FORMAT standard['horizontal_orientation'] = np.nan standard['station_name'] = header['STATION_NAME'] try: standard['instrument_period'] = 1 / \ float(header['INSTRUMENTAL_FREQUENCY_HZ']) except ValueError: standard['instrument_period'] = np.nan try: standard['instrument_damping'] = 1 / \ float(header['INSTRUMENTAL_DAMPING']) except ValueError: standard['instrument_damping'] = np.nan ptimestr = header['DATA_TIMESTAMP_YYYYMMDD_HHMMSS'] ptime = datetime.strptime(ptimestr, TIMEFMT).strftime(TIMEFMT2) standard['process_time'] = ptime standard['process_level'] = PROCESS_LEVELS['V1'] instr_str = header['INSTRUMENT'] parts = instr_str.split('|') sensor_str = parts[0].split('=')[1].strip() standard['sensor_serial_number'] = '' standard['instrument'] = sensor_str standard['comments'] = '' standard['structure_type'] = '' standard['units'] = 'cm/s^2' standard['units_type'] = 'acc' standard['instrument_sensitivity'] = np.nan standard['corner_frequency'] = np.nan coordinates['latitude'] = float(header['STATION_LATITUDE_DEGREE']) coordinates['longitude'] = float(header['STATION_LONGITUDE_DEGREE']) coordinates['elevation'] = float(header['STATION_ELEVATION_M']) # read in the data data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS) # create a Trace from the data and metadata stats['standard'] = standard stats['coordinates'] = coordinates trace = StationTrace(data.copy(), Stats(stats.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) ftype = header['FILTER_TYPE'].capitalize() forder = int(header['FILTER_ORDER']) lowfreq = float(header['LOW_CUT_FREQUENCY_HZ']) highfreq = float(header['LOW_CUT_FREQUENCY_HZ']) filter_att = {'bandpass_filter': {'filter_type': ftype, 'lower_corner_frequency': lowfreq, 'higher_corner_frequency': highfreq, 'filter_order': forder}} trace.setProvenance('lowpass_filter', filter_att) detrend_att = {'detrend': {'detrending_method': 'baseline'}} trace.setProvenance('detrend', detrend_att) stream = StationStream(traces=[trace]) return [stream]
def read_at2(dfile, horient=0.0): # This is a conveneince method so we can read in these specific data for # testing, it is not a general purpose reader since this format does not # contain a lot of metadata that is generally required for it to be useful. skiprows = 4 datafile = open(dfile, 'r') datareader = csv.reader(datafile) data = [] header = [] # for i in range(skiprows): # next(datareader) # header.append(datareader.readlines()) count = 0 for row in datareader: if count < skiprows: header.append(row) else: data.extend([float(e) for e in row[0].split()]) count += 1 datafile.close() hdr = {} hdr['network'] = '' hdr['station'] = '' if horient == 0: hdr['channel'] = 'BH1' else: hdr['channel'] = 'BH2' hdr['location'] = '--' dt = float(header[3][1].split('=')[1].strip().lower().replace('sec', '')) hdr['npts'] = len(data) hdr['sampling_rate'] = 1 / dt hdr['duration'] = (hdr['npts'] - 1) / hdr['sampling_rate'] hdr['starttime'] = 0 # There is no lat/lon... hdr['coordinates'] = {'latitude': 0.0, 'longitude': 0.0, 'elevation': 0.0} standard = {} standard['units'] = 'acc' standard['units_type'] = 'acc' standard['horizontal_orientation'] = horient standard['vertical_orientation'] = np.nan standard['source_file'] = dfile standard['station_name'] = '' standard['corner_frequency'] = 30.0 standard['structure_type'] = '' standard['comments'] = '' standard['instrument'] = '' standard['instrument_period'] = 1.0 standard['instrument_sensitivity'] = 1.0 standard['source'] = 'PEER' standard['instrument_damping'] = 0.1 standard['sensor_serial_number'] = '' standard['process_level'] = 'corrected physical units' standard['source_format'] = 'AT2' standard['process_time'] = '' hdr['standard'] = standard # convert data from g to cm/s^2 g_to_cmss = 980.665 tr = StationTrace(np.array(data.copy()) * g_to_cmss, Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} tr.setProvenance('remove_response', response) return tr
def test_trace(): data = np.random.rand(1000) header = {'sampling_rate': 1, 'npts': len(data), 'network': 'US', 'location': '11', 'station': 'ABCD', 'channel': 'HN1', 'starttime': UTCDateTime(2010, 1, 1, 0, 0, 0)} inventory = get_inventory() invtrace = StationTrace(data=data, header=header, inventory=inventory) invtrace.setProvenance('detrend', {'detrending_method': 'demean'}) invtrace.setParameter('failed', True) invtrace.setParameter('corner_frequencies', [1, 2, 3]) invtrace.setParameter('metadata', {'name': 'Fred'}) assert invtrace.getProvenance('detrend')[0] == { 'detrending_method': 'demean'} assert invtrace.getParameter('failed') assert invtrace.getParameter('corner_frequencies') == [1, 2, 3] assert invtrace.getParameter('metadata') == {'name': 'Fred'}
def _read_volume_two(filename, line_offset, location='', units='acc'): """Read channel data from DMG text file. Args: filename (str): Input DMG V2 filename. line_offset (int): Line offset to beginning of channel text block. units (str): units to get Returns: tuple: (list of obspy Trace, int line offset) """ try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(V2_TEXT_HDR_ROWS)] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) # read in lines of integer data skip_rows = V2_TEXT_HDR_ROWS + line_offset int_data = _read_lines(skip_rows, V2_INT_HDR_ROWS, V2_INT_FMT, filename) int_data = int_data[0:100].astype(np.int32) # read in lines of float data skip_rows += V2_INT_HDR_ROWS flt_data = _read_lines(skip_rows, V2_REAL_HDR_ROWS, V2_REAL_FMT, filename) flt_data = flt_data[:100] skip_rows += V2_REAL_HDR_ROWS # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including csmip/dmg here, don't always provide this. We'll flag it as # "--". hdr = _get_header_info(int_data, flt_data, lines, 'V2', location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) traces = [] # read acceleration data if hdr['npts'] > 0: acc_rows, acc_fmt, unit = _get_data_format(filename, skip_rows, hdr['npts']) acc_data = _read_lines(skip_rows + 1, acc_rows, acc_fmt, filename) acc_data = acc_data[:hdr['npts']] if unit in UNIT_CONVERSIONS: acc_data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('DMG: %s is not a supported unit.' % unit) acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} acc_trace.setProvenance('remove_response', response) if units == 'acc': traces += [acc_trace] skip_rows += int(acc_rows) + 1 # ------------------------------------------------------------------------- # NOTE: The way we were initially reading velocity and displacement data was # not correct. I'm deleting it for now since we don't need it. If/when we # revisit this we need to be more careful about how this is handled. # ------------------------------------------------------------------------- # read velocity data vel_hdr = hdr.copy() vel_hdr['standard']['units'] = 'vel' vel_hdr['npts'] = int_data[63] if vel_hdr['npts'] > 0: vel_rows, vel_fmt, unit = _get_data_format(filename, skip_rows, vel_hdr['npts']) vel_data = _read_lines(skip_rows + 1, vel_rows, vel_fmt, filename) vel_data = vel_data[:vel_hdr['npts']] skip_rows += int(vel_rows) + 1 # read displacement data disp_hdr = hdr.copy() disp_hdr['standard']['units'] = 'disp' disp_hdr['npts'] = int_data[65] if disp_hdr['npts'] > 0: disp_rows, disp_fmt, unit = _get_data_format(filename, skip_rows, disp_hdr['npts']) disp_data = _read_lines(skip_rows + 1, disp_rows, disp_fmt, filename) disp_data = disp_data[:disp_hdr['npts']] skip_rows += int(disp_rows) + 1 new_offset = skip_rows + 1 # there is an 'end of record' line after the data] return (traces, new_offset)
def read_smc(filename, **kwargs): """Read SMC strong motion file. Args: filename (str): Path to possible SMC data file. kwargs (ref): any_structure (bool): Read data from any type of structure, raise Exception if False and structure type is not free-field. accept_flagged (bool): accept problem flagged data. set_location (str): Two character code for location. Other arguments will be ignored. Returns: Stream: Obspy Stream containing one channel of acceleration data (cm/s**2). """ logging.debug("Starting read_smc.") any_structure = kwargs.get('any_structure', False) accept_flagged = kwargs.get('accept_flagged', False) location = kwargs.get('location', '') if not is_smc(filename): raise Exception('%s is not a valid SMC file' % filename) with open(filename, 'rt') as f: line = f.readline().strip() if 'DISPLACEMENT' in line: raise GMProcessException( 'SMC: Diplacement records are not supported: ' '%s.' % filename) elif 'VELOCITY' in line: raise GMProcessException( 'SMC: Velocity records are not supported: ' '%s.' % filename) elif line == "*": raise GMProcessException( 'SMC: No record volume specified in file: ' '%s.' % filename) stats, num_comments = _get_header_info(filename, any_structure=any_structure, accept_flagged=accept_flagged, location=location) skip = ASCII_HEADER_LINES + INTEGER_HEADER_LINES + \ num_comments + FLOAT_HEADER_LINES # read float data (8 columns per line) nrows = int(np.floor(stats['npts'] / DATA_COLUMNS)) data = np.genfromtxt(filename, max_rows=nrows, skip_header=skip, delimiter=FLOAT_DATA_WIDTHS) data = data.flatten() if stats['npts'] % DATA_COLUMNS: lastrow = np.genfromtxt(filename, max_rows=1, skip_header=skip + nrows, delimiter=FLOAT_DATA_WIDTHS) data = np.append(data, lastrow) data = data[0:stats['npts']] trace = StationTrace(data, header=stats) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) stream = StationStream(traces=[trace]) return [stream]
def _read_channel(filename, line_offset, location=''): """Read channel data from COSMOS V1/V2 text file. Args: filename (str): Input COSMOS V1/V2 filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (obspy Trace, int line offset) """ # read station, location, and process level from text header with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(TEXT_HDR_ROWS)] # read in lines of integer data skiprows = line_offset + TEXT_HDR_ROWS int_lines, int_data = _read_lines(skiprows, filename) int_data = int_data.astype(np.int32) # read in lines of float data skiprows += int_lines + 1 flt_lines, flt_data = _read_lines(skiprows, filename) # read in comment lines skiprows += flt_lines + 1 cmt_lines, cmt_data = _read_lines(skiprows, filename) skiprows += cmt_lines + 1 # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including cosmos here, don't provide this. We'll flag it as "--". hdr = _get_header_info(int_data, flt_data, lines, cmt_data, location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # read in the data nrows, data = _read_lines(skiprows, filename) # check units unit = hdr['format_specific']['physical_units'] if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('COSMOS: %s is not a supported unit.' % unit) if hdr['standard']['units'] != 'acc': raise GMProcessException('COSMOS: Only acceleration data accepted.') trace = StationTrace(data.copy(), Stats(hdr.copy())) # record that this data has been converted to g, if it has if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def test_rotd(): ddir = os.path.join('data', 'testdata', 'process') datadir = pkg_resources.resource_filename('gmprocess', ddir) # Create a stream and station summary, convert from m/s^2 to cm/s^2 (GAL) osc1_data = np.genfromtxt(datadir + '/ALCTENE.UW..sac.acc.final.txt') osc2_data = np.genfromtxt(datadir + '/ALCTENN.UW..sac.acc.final.txt') osc1_data = osc1_data.T[1] * 100 osc2_data = osc2_data.T[1] * 100 tr1 = StationTrace(data=osc1_data, header={ 'channel': 'HN1', 'delta': 0.01, 'npts': 24001, 'standard': { 'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'source_file': '', 'structure_type': '', 'horizontal_orientation': np.nan, 'vertical_orientation': np.nan, 'sensor_serial_number': '', 'process_level': 'corrected physical units', 'process_time': '', 'units': 'acc', 'units_type': 'acc', 'instrument_sensitivity': np.nan, 'instrument_damping': np.nan } }) tr2 = StationTrace(data=osc2_data, header={ 'channel': 'HN2', 'delta': 0.01, 'npts': 24001, 'standard': { 'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'structure_type': '', 'source_file': '', 'horizontal_orientation': np.nan, 'vertical_orientation': np.nan, 'sensor_serial_number': '', 'process_level': 'corrected physical units', 'process_time': '', 'units': 'acc', 'units_type': 'acc', 'instrument_sensitivity': np.nan, 'instrument_damping': np.nan } }) st = StationStream([tr1, tr2]) for tr in st: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} tr.setProvenance('remove_response', response) target_pga50 = 4.12528265306 target_sa1050 = 10.7362857143 target_pgv50 = 6.239364 target_sa0350 = 10.1434159021 target_sa3050 = 1.12614169215 station = StationSummary.from_stream( st, ['rotd50'], ['pga', 'pgv', 'sa0.3', 'sa1.0', 'sa3.0'] ) pgms = station.pgms pga = pgms.loc['PGA', 'ROTD(50.0)'].Result pgv = pgms.loc['PGV', 'ROTD(50.0)'].Result SA10 = pgms.loc['SA(1.000)', 'ROTD(50.0)'].Result SA03 = pgms.loc['SA(0.300)', 'ROTD(50.0)'].Result SA30 = pgms.loc['SA(3.000)', 'ROTD(50.0)'].Result np.testing.assert_allclose(pga, target_pga50, atol=0.1) np.testing.assert_allclose(SA10, target_sa1050, atol=0.1) np.testing.assert_allclose(pgv, target_pgv50, atol=0.1) np.testing.assert_allclose(SA03, target_sa0350, atol=0.1) np.testing.assert_allclose(SA30, target_sa3050, atol=0.1)
def test_radial_transverse(): origin = Origin(latitude=47.149, longitude=-122.7266667) st = read(os.path.join(datadir, 'resp_cor', 'UW.ALCT.--.*.MSEED')) st[0].stats.standard = {} st[0].stats.standard['horizontal_orientation'] = 0. st[0].stats['channel'] = 'HN1' st[1].stats.standard = {} st[1].stats.standard['horizontal_orientation'] = 90. st[1].stats['channel'] = 'HN2' st[2].stats.standard = {} st[2].stats.standard['horizontal_orientation'] = np.nan st[2].stats['channel'] = 'HNZ' inv = read_inventory(os.path.join(datadir, 'inventory.xml')) stalat, stalon = inv[0][0][0].latitude, inv[0][0][0].longitude for i, tr in enumerate(st): tr.stats['coordinates'] = {'latitude': stalat} tr.stats['coordinates']['longitude'] = stalon tr.stats['standard'].update({'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'instrument': '', 'instrument_period': np.nan, 'vertical_orientation': np.nan, 'source_format': 'json', 'comments': '', 'structure_type': '', 'source_file': '', 'sensor_serial_number': '', 'process_level': 'raw counts', 'process_time': '', 'units': 'acc', 'units_type': 'acc', 'instrument_sensitivity': np.nan, 'instrument_damping': np.nan}) baz = gps2dist_azimuth(stalat, stalon, origin.latitude, origin.longitude)[1] st1 = st.copy() st1[0].stats.channel = st1[0].stats.channel[:-1] + 'N' st1[1].stats.channel = st1[1].stats.channel[:-1] + 'E' st1.rotate(method='NE->RT', back_azimuth=baz) pgms = np.abs(st1.max()) st2 = StationStream([]) for t in st: st2.append(StationTrace(t.data, t.stats)) for tr in st2: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} tr.setProvenance('remove_response', response) summary = StationSummary.from_stream( st2, ['radial_transverse'], ['pga'], origin) pgmdf = summary.pgms R = pgmdf.loc['PGA', 'HNR'].Result T = pgmdf.loc['PGA', 'HNT'].Result np.testing.assert_almost_equal( pgms[0], sp.g * R) np.testing.assert_almost_equal( pgms[1], sp.g * T) # Test with a station whose channels are not aligned to E-N SEW_st = read(os.path.join(datadir, 'resp_cor', 'GS.SEW.*.mseed')) SEW_inv = read_inventory(os.path.join(datadir, 'inventory_sew.xml')) stalat, stalon = inv[0][0][0].latitude, inv[0][0][0].longitude # This needs to be checked. The target data doesn't appear to be # correct. This can be updated when a tolerance is added to the rotate # method. """traces = [] for tr in SEW_st: tr.stats.coordinates = {'latitude': stalat, 'longitude': stalon} tr.stats.standard = {'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'structure_type': '', 'sensor_serial_number': '', 'process_level': 'raw counts', 'process_time': '', 'horizontal_orientation': SEW_inv.get_channel_metadata(tr.get_id())['azimuth'], 'units': 'acc', 'instrument_damping': np.nan} traces += [StationTrace(tr.data, tr.stats)] baz = gps2dist_azimuth(stalat, stalon, origin.latitude, origin.longitude)[1] SEW_st_copy = StationStream(traces) SEW_st_copy.rotate(method='->NE', inventory=SEW_inv) SEW_st_copy.rotate(method='NE->RT', back_azimuth=baz) pgms = np.abs(SEW_st_copy.max()) summary = StationSummary.from_stream( SEW_st, ['radial_transverse'], ['pga'], origin) np.testing.assert_almost_equal( pgms[1], sp.g * summary.pgms['PGA']['R']) np.testing.assert_almost_equal( pgms[2], sp.g * summary.pgms['PGA']['T'])""" # Test failure case without two horizontal channels copy1 = st2.copy() copy1[0].stats.channel = copy1[0].stats.channel[:-1] + '3' pgms = StationSummary.from_stream( copy1, ['radial_transverse'], ['pga'], origin).pgms assert np.isnan(pgms.loc['PGA', 'HNR'].Result) assert np.isnan(pgms.loc['PGA', 'HNT'].Result) # Test failure case when channels are not orthogonal copy3 = st2.copy() copy3[0].stats.standard.horizontal_orientation = 100 pgms = StationSummary.from_stream( copy3, ['radial_transverse'], ['pga'], origin).pgms assert np.isnan(pgms.loc['PGA', 'HNR'].Result) assert np.isnan(pgms.loc['PGA', 'HNT'].Result)
def _read_channel(filename, line_offset, location=''): """Read channel data from COSMOS V1/V2 text file. Args: filename (str): Input COSMOS V1/V2 filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (obspy Trace, int line offset) """ # read station, location, and process level from text header with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(TEXT_HDR_ROWS)] # read in lines of integer data skiprows = line_offset + TEXT_HDR_ROWS int_lines, int_data = _read_lines(skiprows, filename) int_data = int_data.astype(np.int32) # read in lines of float data skiprows += int_lines + 1 flt_lines, flt_data = _read_lines(skiprows, filename) # read in comment lines skiprows += flt_lines + 1 cmt_lines, cmt_data = _read_lines(skiprows, filename) skiprows += cmt_lines + 1 # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including cosmos here, don't provide this. We'll flag it as "--". hdr = _get_header_info(int_data, flt_data, lines, cmt_data, location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # read in the data nrows, data = _read_lines(skiprows, filename) # Check for "off-by-one" problem that sometimes occurs with cosmos data # Notes: # - We cannot do this check inside _get_header_info because we don't # have the data there. # - That method is written to set npts from the header as documented in # the spec ("lenght" == npts*dt) but it appears that sometimes a # different convention is used where the "length" of the record is # actually is actuation (npts-1)*dt. In this case, we need to # recompute duration and npts if hdr['npts'] == (len(data) - 1): hdr['npts'] = len(data) hdr['duration'] = (hdr['npts'] - 1) * hdr['delta'] # check units unit = hdr['format_specific']['physical_units'] if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: if unit != 'counts': raise GMProcessException( 'COSMOS: %s is not a supported unit.' % unit) if hdr['standard']['units'] != 'acc': raise GMProcessException('COSMOS: Only acceleration data accepted.') trace = StationTrace(data.copy(), Stats(hdr.copy())) # record that this data has been converted to g, if it has if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def read_knet(filename): """Read Japanese KNET strong motion file. Args: filename (str): Path to possible KNET data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ logging.debug("Starting read_knet.") if not is_knet(filename): raise Exception('%s is not a valid KNET file' % filename) # Parse the header portion of the file with open(filename, 'rt') as f: lines = [next(f) for x in range(TEXT_HDR_ROWS)] hdr = {} coordinates = {} standard = {} hdr['network'] = 'BO' hdr['station'] = lines[5].split()[2] logging.debug('station: %s' % hdr['station']) standard['station_name'] = '' # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including KNET here, don't provide this. We'll flag it as "--". hdr['location'] = '--' coordinates['latitude'] = float(lines[6].split()[2]) coordinates['longitude'] = float(lines[7].split()[2]) coordinates['elevation'] = float(lines[8].split()[2]) hdr['sampling_rate'] = float( re.search('\\d+', lines[10].split()[2]).group()) hdr['delta'] = 1 / hdr['sampling_rate'] standard['units'] = 'acc' dir_string = lines[12].split()[1].strip() # knet files have directions listed as N-S, E-W, or U-D, # whereas in kiknet those directions are '4', '5', or '6'. if dir_string in ['N-S', '1', '4']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) elif dir_string in ['E-W', '2', '5']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) elif dir_string in ['U-D', '3', '6']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) else: raise Exception('KNET: Could not parse direction %s' % lines[12].split()[1]) logging.debug('channel: %s' % hdr['channel']) scalestr = lines[13].split()[2] parts = scalestr.split('/') num = float(parts[0].replace('(gal)', '')) den = float(parts[1]) calib = num / den hdr['calib'] = calib duration = float(lines[11].split()[2]) hdr['npts'] = int(duration * hdr['sampling_rate']) timestr = ' '.join(lines[9].split()[2:4]) # The K-NET and KiK-Net data logger adds a 15s time delay # this is removed here sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0) # Shift the time to utc (Japanese time is 9 hours ahead) sttime = sttime - timedelta(seconds=9 * 3600.) hdr['starttime'] = sttime # read in the data - there is a max of 8 columns per line # the code below handles the case when last line has # less than 8 columns if hdr['npts'] % COLS_PER_LINE != 0: nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE)) nrows2 = 1 else: nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE)) nrows2 = 0 data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS, max_rows=nrows, filling_values=np.nan) data = data.flatten() if nrows2: skip_header = TEXT_HDR_ROWS + nrows data2 = np.genfromtxt(filename, skip_header=skip_header, max_rows=nrows2, filling_values=np.nan) data = np.hstack((data, data2)) nrows += nrows2 # apply the correction factor we're given in the header data *= calib # fill out the rest of the standard dictionary standard['horizontal_orientation'] = np.nan standard['instrument_period'] = np.nan standard['instrument_damping'] = np.nan standard['process_time'] = '' standard['process_level'] = PROCESS_LEVELS['V1'] standard['sensor_serial_number'] = '' standard['instrument'] = '' standard['comments'] = '' standard['structure_type'] = '' if dir_string in ['1', '2', '3']: standard['structure_type'] = 'borehole' standard['corner_frequency'] = np.nan standard['units'] = 'acc' standard['source'] = SRC standard['source_format'] = 'knet' head, tail = os.path.split(filename) standard['source_file'] = tail or os.path.basename(head) hdr['coordinates'] = coordinates hdr['standard'] = standard # create a Trace from the data and metadata trace = StationTrace(data.copy(), Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) stream = StationStream(traces=[trace]) return [stream]
def test_fas(): """ Testing based upon the work provided in https://github.com/arkottke/notebooks/blob/master/effective_amp_spectrum.ipynb """ ddir = os.path.join('data', 'testdata') datadir = pkg_resources.resource_filename('gmprocess', ddir) fas_file = os.path.join(datadir, 'fas_greater_of_two_horizontals.pkl') p1 = os.path.join(datadir, 'peer', 'RSN763_LOMAP_GIL067.AT2') p2 = os.path.join(datadir, 'peer', 'RSN763_LOMAP_GIL337.AT2') stream = StationStream([]) for idx, fpath in enumerate([p1, p2]): with open(fpath) as file_obj: for _ in range(3): next(file_obj) meta = re.findall(r'[.0-9]+', next(file_obj)) dt = float(meta[1]) accels = np.array( [col for line in file_obj for col in line.split()]) trace = StationTrace(data=accels, header={ 'channel': 'H' + str(idx), 'delta': dt, 'units': 'acc', 'standard': { 'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'structure_type': '', 'sensor_serial_number': '', 'source_file': '', 'process_level': 'raw counts', 'process_time': '', 'horizontal_orientation': np.nan, 'vertical_orientation': np.nan, 'units': 'acc', 'units_type': 'acc', 'instrument_sensitivity': np.nan, 'instrument_damping': np.nan } }) stream.append(trace) for tr in stream: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} tr.setProvenance('remove_response', response) target_df = pd.read_pickle(fas_file) ind_vals = target_df.index.values per = np.unique( [float(i[0].split(')')[0].split('(')[1]) for i in ind_vals]) freqs = 1 / per imts = ['fas' + str(p) for p in per] summary = StationSummary.from_stream(stream, ['greater_of_two_horizontals'], imts, bandwidth=30) pgms = summary.pgms # pgms.to_pickle(fas_file) for idx, f in enumerate(freqs): fstr = 'FAS(%.3f)' % (1 / f) fval1 = pgms.loc[fstr, 'GREATER_OF_TWO_HORIZONTALS'].Result fval2 = target_df.loc[fstr, 'GREATER_OF_TWO_HORIZONTALS'].Result np.testing.assert_allclose(fval1, fval2, rtol=1e-5, atol=1e-5)
def _read_channel(filename, line_offset, volume, location='', alternate=False): """Read channel data from USC V1 text file. Args: filename (str): Input USC V1 filename. line_offset (int): Line offset to beginning of channel text block. volume (dictionary): Dictionary of formatting information Returns: tuple: (obspy Trace, int line offset) """ if alternate: int_rows = 5 int_fmt = 20 * [4] data_cols = 8 else: int_rows = volume['INT_HDR_ROWS'] int_fmt = volume['INT_FMT'] data_cols = 10 # Parse the header portion of the file try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) # read in lines of integer data skiprows = line_offset + volume['TEXT_HDR_ROWS'] int_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=int_rows, dtype=np.int32, delimiter=int_fmt).flatten() # read in lines of float data skiprows += int_rows flt_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64, delimiter=volume['FLT_FMT']).flatten() hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location) skiprows += volume['FLT_HDR_ROWS'] # read in the data nrows = int(np.floor(hdr['npts'] * 2 / data_cols)) all_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=nrows, dtype=np.float64, delimiter=volume['COL_FMT']) data = all_data.flatten()[1::2] times = all_data.flatten()[0::2] frac = hdr['format_specific']['fractional_unit'] if frac > 0: data *= UNIT_CONVERSIONS['g'] * frac logging.debug('Data converted from g * %s to cm/s/s' % (frac)) else: unit = _get_units(lines[11]) if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('USC: %s is not a supported unit.' % unit) # Put file name into dictionary head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) trace = StationTrace(data.copy(), Stats(hdr.copy())) if not is_evenly_spaced(times): trace = resample_uneven_trace(trace, times, data) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def _read_channel(filename, line_offset): """Read channel data from GNS V1 text file. Args: filename (str): Input GNS V1 filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (obspy Trace, int line offset) """ # read station and location strings from text header with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(TEXT_HDR_ROWS)] # this code supports V1 and V2 format files. Which one is this? data_format = 'V2' if lines[0].lower().find('uncorrected') >= 0: data_format = 'V1' # parse out the station code, name, and component string # from text header station = lines[1].split()[1] logging.debug('station: %s' % station) name = lines[2].replace(' ', '_').strip() component = lines[12].split()[1] # parse the instrument type from the text header instrument = lines[3].split()[1] # parse the sensor resolution from the text header resolution_str = lines[4].split()[1] resolution = int(re.search('\d+', resolution_str).group()) # read floating point header array skip_header = line_offset + TEXT_HDR_ROWS hdr_data = np.genfromtxt(filename, skip_header=skip_header, max_rows=FP_HDR_ROWS) # parse header dictionary from float header array hdr = _read_header(hdr_data, station, name, component, data_format, instrument, resolution) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including GeoNet here, don't provide this. We'll flag it as "--". hdr['location'] = '--' skip_header2 = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS widths = [8] * COLS_PER_ROW nrows = int(np.ceil(hdr['npts'] / COLS_PER_ROW)) data = np.genfromtxt(filename, skip_header=skip_header2, max_rows=nrows, filling_values=np.nan, delimiter=widths) data = data.flatten() data = data[0:hdr['npts']] # for debugging, read in the velocity data nvel = hdr_data[3, 4] if nvel: if nvel % COLS_PER_ROW != 0: nvel_rows = int(np.floor(nvel / COLS_PER_ROW)) nvel_rows2 = 1 else: nvel_rows = int(np.ceil(nvel / COLS_PER_ROW)) nvel_rows2 = 0 skip_header_vel = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS + nrows widths = [8] * COLS_PER_ROW velocity = np.genfromtxt(filename, skip_header=skip_header_vel, max_rows=nvel_rows, filling_values=np.nan, delimiter=widths) velocity = velocity.flatten() velocity *= MMPS_TO_CMPS else: velocity = np.array([]) # for V2 files, there are extra blocks of data we need to skip containing # velocity and displacement data if data_format == 'V2': velrows = int(np.ceil(hdr_data[3, 4] / COLS_PER_ROW)) disrows = int(np.ceil(hdr_data[3, 5] / COLS_PER_ROW)) nrows = nrows + velrows + disrows data *= MMPS_TO_CMPS # convert to cm/s**2 trace = StationTrace(data, Stats(hdr)) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) offset = skip_header2 + nrows return (trace, offset, velocity)
def getStreams(self, eventid, stations=None, labels=None): """Get Stream from ASDF file given event id and input tags. Args: eventid (str): Event ID corresponding to an Event in the workspace. stations (list): List of stations to search for. labels (list): List of processing labels to search for. Returns: StreamCollection: Object containing list of organized StationStreams. """ trace_auxholder = [] stream_auxholder = [] if 'TraceProcessingParameters' in self.dataset.auxiliary_data: trace_auxholder = self.dataset.auxiliary_data.TraceProcessingParameters if 'StreamProcessingParameters' in self.dataset.auxiliary_data: stream_auxholder = self.dataset.auxiliary_data.StreamProcessingParameters streams = [] if stations is None: stations = self.getStations(eventid) if labels is None: labels = self.getLabels() # tried doing a query here using dataset ifilter on event, # but it didn't work... for waveform in self.dataset.waveforms: tags = waveform.get_waveform_tags() for tag in tags: parts = tag.split('_') if len(parts) > 2: tlabel = parts[-1] teventid = '_'.join(parts[0:-1]) else: teventid, tlabel = tag.split('_') if eventid != teventid: continue if tlabel not in labels: continue stream_name = list( waveform.get_waveform_attributes().keys())[0] parts = stream_name.split('.') tstation = parts[1] if tstation not in stations: continue tstream = waveform[tag] inventory = waveform['StationXML'] for ttrace in tstream: trace = StationTrace(data=ttrace.data, header=ttrace.stats, inventory=inventory) # get the provenance information provname = format_nslct(trace.stats, tag) if provname in self.dataset.provenance.list(): provdoc = self.dataset.provenance[provname] trace.setProvenanceDocument(provdoc) # get the trace processing parameters top = format_netsta(trace.stats) trace_path = format_nslct(trace.stats, tag) if top in trace_auxholder: root_auxholder = trace_auxholder[top] if trace_path in root_auxholder: bytelist = root_auxholder[ trace_path].data[:].tolist() jsonstr = ''.join([chr(b) for b in bytelist]) jdict = json.loads(jsonstr) for key, value in jdict.items(): trace.setParameter(key, value) # get the trace spectra arrays from auxiliary, # repack into stationtrace object spectra = {} if 'Cache' in self.dataset.auxiliary_data: for aux in self.dataset.auxiliary_data['Cache'].list(): auxarray = self.dataset.auxiliary_data['Cache'][aux] if top not in auxarray.list(): continue auxarray_top = auxarray[top] if trace_path in auxarray_top: specparts = camel_case_split(aux) array_name = specparts[-1].lower() specname = '_'.join(specparts[:-1]).lower() specarray = auxarray_top[trace_path].data.value if specname in spectra: spectra[specname][array_name] = specarray else: spectra[specname] = {array_name: specarray} for key, value in spectra.items(): trace.setCached(key, value) stream = StationStream(traces=[trace]) stream.tag = tag # testing this out # get the stream processing parameters stream_path = format_nslit(trace.stats, stream.get_inst(), tag) if top in stream_auxholder: top_auxholder = stream_auxholder[top] if stream_path in top_auxholder: auxarray = top_auxholder[stream_path] bytelist = auxarray.data[:].tolist() jsonstr = ''.join([chr(b) for b in bytelist]) jdict = json.loads(jsonstr) for key, value in jdict.items(): stream.setStreamParam(key, value) streams.append(stream) streams = StreamCollection(streams) return streams
def getStreams(self, eventid, stations=None, labels=None): """Get Stream from ASDF file given event id and input tags. Args: eventid (str): Event ID corresponding to an Event in the workspace. stations (list): List of stations to search for. labels (list): List of processing labels to search for. Returns: StreamCollection: Object containing list of organized StationStreams. """ trace_auxholder = [] stream_auxholder = [] if 'TraceProcessingParameters' in self.dataset.auxiliary_data: trace_auxholder = self.dataset.auxiliary_data.TraceProcessingParameters if 'StreamProcessingParameters' in self.dataset.auxiliary_data: stream_auxholder = self.dataset.auxiliary_data.StreamProcessingParameters streams = [] all_tags = [] if stations is None: stations = self.getStations(eventid) if labels is None: labels = self.getLabels() for station in stations: for label in labels: all_tags.append('%s_%s' % (station.lower(), label)) for waveform in self.dataset.waveforms: ttags = waveform.get_waveform_tags() wtags = [] if not len(all_tags): wtags = ttags else: wtags = list(set(all_tags).intersection(set(ttags))) for tag in wtags: if eventid in waveform[tag][0].stats.asdf.event_ids: tstream = waveform[tag].copy() inventory = waveform['StationXML'] traces = [] for ttrace in tstream: trace = StationTrace(data=ttrace.data, header=ttrace.stats, inventory=inventory) tpl = (trace.stats.network.lower(), trace.stats.station.lower(), trace.stats.channel.lower()) channel = '%s_%s_%s' % tpl channel_tag = '%s_%s' % (tag, channel) if channel_tag in self.dataset.provenance.list(): provdoc = self.dataset.provenance[channel_tag] trace.setProvenanceDocument(provdoc) trace_path = '%s_%s_%s' % (eventid, tag, trace.stats.channel) if trace_path in trace_auxholder: bytelist = trace_auxholder[trace_path].data[:].tolist( ) jsonstr = ''.join([chr(b) for b in bytelist]) jdict = json.loads(jsonstr) # jdict = unstringify_dict(jdict) for key, value in jdict.items(): trace.setParameter(key, value) traces.append(trace) stream = StationStream(traces=traces) stream.tag = tag # testing this out # look for stream-based metadata stream_path = '%s_%s' % (eventid, tag) if stream_path in stream_auxholder: bytelist = stream_auxholder[stream_path].data[:].tolist( ) jsonstr = ''.join([chr(b) for b in bytelist]) jdict = json.loads(jsonstr) # jdict = unstringify_dict(jdict) for key, value in jdict.items(): stream.setStreamParam(key, value) streams.append(stream) streams = StreamCollection(streams) return streams
def read_knet(filename): """Read Japanese KNET strong motion file. Args: filename (str): Path to possible KNET data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ logging.debug("Starting read_knet.") if not is_knet(filename): raise Exception('%s is not a valid KNET file' % filename) # Parse the header portion of the file with open(filename, 'rt') as f: lines = [next(f) for x in range(TEXT_HDR_ROWS)] hdr = {} coordinates = {} standard = {} hdr['network'] = 'BO' hdr['station'] = lines[5].split()[2] logging.debug('station: %s' % hdr['station']) standard['station_name'] = '' # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including KNET here, don't provide this. We'll flag it as "--". hdr['location'] = '--' coordinates['latitude'] = float(lines[6].split()[2]) coordinates['longitude'] = float(lines[7].split()[2]) coordinates['elevation'] = float(lines[8].split()[2]) hdr['sampling_rate'] = float( re.search('\\d+', lines[10].split()[2]).group()) hdr['delta'] = 1 / hdr['sampling_rate'] standard['units'] = 'acc' dir_string = lines[12].split()[1].strip() # knet files have directions listed as N-S, E-W, or U-D, # whereas in kiknet those directions are '4', '5', or '6'. if dir_string in ['N-S', '1', '4']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) elif dir_string in ['E-W', '2', '5']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) elif dir_string in ['U-D', '3', '6']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) else: raise Exception('KNET: Could not parse direction %s' % lines[12].split()[1]) logging.debug('channel: %s' % hdr['channel']) scalestr = lines[13].split()[2] parts = scalestr.split('/') num = float(parts[0].replace('(gal)', '')) den = float(parts[1]) calib = num / den hdr['calib'] = calib duration = float(lines[11].split()[2]) hdr['npts'] = int(duration * hdr['sampling_rate']) timestr = ' '.join(lines[9].split()[2:4]) # The K-NET and KiK-Net data logger adds a 15s time delay # this is removed here sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0) # Shift the time to utc (Japanese time is 9 hours ahead) sttime = sttime - timedelta(seconds=9 * 3600.) hdr['starttime'] = sttime # read in the data - there is a max of 8 columns per line # the code below handles the case when last line has # less than 8 columns if hdr['npts'] % COLS_PER_LINE != 0: nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE)) nrows2 = 1 else: nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE)) nrows2 = 0 data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS, max_rows=nrows, filling_values=np.nan) data = data.flatten() if nrows2: skip_header = TEXT_HDR_ROWS + nrows data2 = np.genfromtxt(filename, skip_header=skip_header, max_rows=nrows2, filling_values=np.nan) data = np.hstack((data, data2)) nrows += nrows2 # apply the correction factor we're given in the header data *= calib # fill out the rest of the standard dictionary standard['horizontal_orientation'] = np.nan standard['instrument_period'] = np.nan standard['instrument_damping'] = np.nan standard['process_time'] = '' standard['process_level'] = PROCESS_LEVELS['V1'] standard['sensor_serial_number'] = '' standard['instrument'] = '' standard['comments'] = '' standard['structure_type'] = '' if dir_string in ['1', '2', '3']: standard['structure_type'] = 'borehole' standard['corner_frequency'] = np.nan standard['units'] = 'acc' standard['source'] = SRC standard['source_format'] = 'knet' hdr['coordinates'] = coordinates hdr['standard'] = standard # create a Trace from the data and metadata trace = StationTrace(data.copy(), Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) stream = StationStream(traces=[trace]) return [stream]
def read_cwb(filename, **kwargs): """Read Taiwan Central Weather Bureau strong motion file. Args: filename (str): Path to possible CWB data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ logging.debug("Starting read_cwb.") if not is_cwb(filename): raise Exception('%s is not a valid CWB strong motion data file.' % filename) f = open(filename, 'rt') # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including CWB here, don't provide this. We'll flag it as "--". data = np.genfromtxt(filename, skip_header=HDR_ROWS, delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW hdr = _get_header_info(f, data) f.close() head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) hdr_z = hdr.copy() hdr_z['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) hdr_z['standard']['horizontal_orientation'] = np.nan hdr_h1 = hdr.copy() hdr_h1['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) hdr_h1['standard']['horizontal_orientation'] = np.nan hdr_h2 = hdr.copy() hdr_h2['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) hdr_h2['standard']['horizontal_orientation'] = np.nan stats_z = Stats(hdr_z) stats_h1 = Stats(hdr_h1) stats_h2 = Stats(hdr_h2) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace_z = StationTrace(data=data[:, 1], header=stats_z) trace_z.setProvenance('remove_response', response) trace_h1 = StationTrace(data=data[:, 2], header=stats_h1) trace_h1.setProvenance('remove_response', response) trace_h2 = StationTrace(data=data[:, 3], header=stats_h2) trace_h2.setProvenance('remove_response', response) stream = StationStream([trace_z, trace_h1, trace_h2]) return [stream]
def getStreams(self, eventid, stations=None, labels=None): """Get Stream from ASDF file given event id and input tags. Args: eventid (str): Event ID corresponding to an Event in the workspace. stations (list): List of stations to search for. labels (list): List of processing labels to search for. Returns: StreamCollection: Object containing list of organized StationStreams. """ trace_auxholder = [] stream_auxholder = [] if 'TraceProcessingParameters' in self.dataset.auxiliary_data: trace_auxholder = self.dataset.auxiliary_data.TraceProcessingParameters if 'StreamProcessingParameters' in self.dataset.auxiliary_data: stream_auxholder = self.dataset.auxiliary_data.StreamProcessingParameters streams = [] all_tags = [] if stations is None: stations = self.getStations(eventid) if labels is None: labels = self.getLabels() for station in stations: for label in labels: all_tags.append('%s_%s_%s' % (eventid, station.lower(), label)) for waveform in self.dataset.waveforms: ttags = waveform.get_waveform_tags() wtags = [] if not len(all_tags): wtags = ttags else: wtags = list(set(all_tags).intersection(set(ttags))) for tag in wtags: if eventid in waveform[tag][0].stats.asdf.event_ids: tstream = waveform[tag].copy() inventory = waveform['StationXML'] for ttrace in tstream: trace = StationTrace(data=ttrace.data, header=ttrace.stats, inventory=inventory) tpl = (trace.stats.network.lower(), trace.stats.station.lower(), trace.stats.channel.lower()) channel = '%s_%s_%s' % tpl channel_tag = '%s_%s' % (tag, channel) if channel_tag in self.dataset.provenance.list(): provdoc = self.dataset.provenance[channel_tag] trace.setProvenanceDocument(provdoc) trace_path = '%s_%s' % ( tag, trace.stats.channel) if trace_path in trace_auxholder: bytelist = trace_auxholder[ trace_path].data[:].tolist() jsonstr = ''.join([chr(b) for b in bytelist]) jdict = json.loads(jsonstr) for key, value in jdict.items(): trace.setParameter(key, value) stream = StationStream(traces=[trace]) stream.tag = tag # testing this out # look for stream-based metadata if tag in stream_auxholder: bytelist = stream_auxholder[ tag].data[:].tolist() jsonstr = ''.join([chr(b) for b in bytelist]) jdict = json.loads(jsonstr) for key, value in jdict.items(): stream.setStreamParam(key, value) streams.append(stream) streams = StreamCollection(streams) return streams
def test_fas(): """ Testing based upon the work provided in https://github.com/arkottke/notebooks/blob/master/effective_amp_spectrum.ipynb """ ddir = os.path.join('data', 'testdata') datadir = pkg_resources.resource_filename('gmprocess', ddir) fas_file = os.path.join(datadir, 'fas_results.txt') p1 = os.path.join(datadir, 'peer', 'RSN763_LOMAP_GIL067.AT2') p2 = os.path.join(datadir, 'peer', 'RSN763_LOMAP_GIL337.AT2') stream = StationStream([]) for idx, fpath in enumerate([p1, p2]): with open(fpath) as file_obj: for _ in range(3): next(file_obj) meta = re.findall(r'[.0-9]+', next(file_obj)) count = int(meta[0]) dt = float(meta[1]) accels = np.array( [col for line in file_obj for col in line.split()]) trace = StationTrace(data=accels, header={ 'channel': 'H' + str(idx), 'delta': dt, 'units': 'acc', 'standard': {'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'structure_type': '', 'sensor_serial_number': '', 'source_file': '', 'process_level': 'raw counts', 'process_time': '', 'horizontal_orientation': np.nan, 'units': 'acc', 'units_type': 'acc', 'instrument_sensitivity': np.nan, 'instrument_damping': np.nan}}) stream.append(trace) for tr in stream: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} tr.setProvenance('remove_response', response) freqs, fas = np.loadtxt(fas_file, unpack=True, usecols=(0, 1), delimiter=',') # scaling required on the test data as it was not accounted for originally imts = ['fas' + str(1 / p) for p in freqs] summary = StationSummary.from_stream(stream, ['quadratic_mean'], imts, bandwidth=30) pgms = summary.pgms for idx, f in enumerate(freqs): fstr = 'FAS(' + str(1 / f) + ')' fval = pgms[pgms.IMT == fstr].Result.tolist()[0] np.testing.assert_array_almost_equal( fval, fas[idx] / len(stream[0].data)) # test exceptions failed = False try: fas_dict = calculate_fas( stream, '', 1 / freqs, 'some other smoothing', 30) except Exception as e: failed = True assert(failed == True) failed = False invalid_channels_stream = stream invalid_channels_stream[0].stats.channel = 'Z' try: fas_dict = calculate_fas( invalid_channels_stream, '', 1 / freqs, 'konno_ohmachi', 30) except Exception as e: failed = True assert(failed == True) failed = False invalid_units_stream = stream invalid_units_stream[0].stats.units = 'other' try: fas_dict = calculate_fas(invalid_units_stream, '', 1 / freqs, 'konno_ohmachi', 30) except Exception as e: failed = True assert(failed == True)
def _read_volume_one(filename, line_offset, location='', units='acc'): """Read channel data from DMG Volume 1 text file. Args: filename (str): Input DMG V1 filename. line_offset (int): Line offset to beginning of channel text block. units (str): units to get Returns: tuple: (list of obspy Trace, int line offset) """ # Parse the header portion of the file try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) unit = _get_units(lines[11]) # read in lines of integer data skip_rows = V1_TEXT_HDR_ROWS + line_offset int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename) int_data = int_data[0:100].astype(np.int32) # read in lines of float data skip_rows += V1_INT_HDR_ROWS flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename) skip_rows += V1_REAL_HDR_ROWS # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including csmip/dmg here, don't always provide this. We'll flag it as # "--". hdr = _get_header_info_v1(int_data, flt_data, lines, 'V1', location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # sometimes (??) a line of text is inserted in between the float header and # the beginning of the data. Let's check for this... with open(filename, 'rt') as f: for _ in range(skip_rows): next(f) test_line = f.readline() has_text = re.search('[A-Z]+|[a-z]+', test_line) is not None if has_text: skip_rows += 1 widths = [9] * 8 max_rows = int(np.ceil(hdr['npts'] / 8)) data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[:hdr['npts']] evenly_spaced = True # Sometimes, npts is incrrectly specified, leading to nans # in the resulting data. Fix that here if np.any(np.isnan(acc_data)): while np.isnan(acc_data[-1]): acc_data = acc_data[:-1] hdr['npts'] = len(acc_data) else: # acceleration data is interleaved between time data max_rows = int(np.ceil(hdr['npts'] / 5)) widths = [7] * 10 data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[1::2][:hdr['npts']] times = data[0::2][:hdr['npts']] evenly_spaced = is_evenly_spaced(times) if unit in UNIT_CONVERSIONS: acc_data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('DMG: %s is not a supported unit.' % unit) acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy())) # Check if the times were included in the file but were not evenly spaced if not evenly_spaced: acc_trace = resample_uneven_trace(acc_trace, times, acc_data) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} acc_trace.setProvenance('remove_response', response) traces = [acc_trace] new_offset = skip_rows + max_rows + 1 # there is an end of record line return (traces, new_offset)
def test_duration(): ddir = os.path.join('data', 'testdata') datadir = pkg_resources.resource_filename('gmprocess', ddir) data_file = os.path.join(datadir, 'duration_data.json') with open(data_file, 'rt') as f: jdict = json.load(f) time = np.array(jdict['time']) # input output is m/s/s acc = np.array(jdict['acc']) / 100 target_d595 = jdict['d595'] delta = time[2] - time[1] sr = 1 / delta header = { 'delta': delta, 'sampling_rate': sr, 'npts': len(acc), 'units': 'm/s/s', 'channel': 'HN1', 'standard': { 'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'source_file': '', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'structure_type': '', 'sensor_serial_number': '', 'process_level': 'raw counts', 'process_time': '', 'horizontal_orientation': np.nan, 'units': 'acc', 'units_type': 'acc', 'instrument_sensitivity': np.nan, 'instrument_damping': np.nan } } # input is cm/s/s output is m/s/s trace = StationTrace(data=acc * 100, header=header) trace2 = trace.copy() trace2.stats.channel = 'HN2' stream = StationStream([trace, trace2]) for tr in stream: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} tr.setProvenance('remove_response', response) station = StationSummary.from_stream(stream, ['ARITHMETIC_MEAN'], ['duration']) pgms = station.pgms d595 = pgms[(pgms.IMT == 'DURATION') & (pgms.IMC == 'ARITHMETIC_MEAN')].Result.tolist()[0] np.testing.assert_allclose(d595, target_d595, atol=1e-4, rtol=1e-4) # Test other components data_files, _ = read_data_dir('cwb', 'us1000chhc', '2-ECU.dat') stream = read_data(data_files[0])[0] station = StationSummary.from_stream(stream, [ 'channels', 'gmrotd', 'rotd50', 'greater_of_two_horizontals', 'ARITHMETIC_MEAN', 'geometric_mean' ], ['duration']) # Currently disallowed assert 'gmrotd' not in station.pgms['IMC'] assert 'rotd50' not in station.pgms['IMC'] print(station)
def read_renadic(filename): """Read the Chilean RENADIC strong motion data format. Args: filename (str): path to RENADIC data file. Returns: list: Sequence of one StationStream object containing 3 StationTrace objects. """ # This network does not include station coordinates in the data files, # but they did provide a PDF table with information about each station, # including structure type (free field or something else) and the # coordinates data_dir = pkg_resources.resource_filename('gmprocess', 'data') tablefile = os.path.join(data_dir, 'station_coordinates.xlsx') table = pd.read_excel(tablefile) with open(filename, 'rt', encoding=ENCODING) as f: lines1 = [next(f) for x in range(TEXT_HDR_ROWS)] header1 = _read_header(lines1, filename, table) ndata_rows = int(np.ceil((header1['npts'] * 2) / NCOLS)) skip_rows = TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS data1 = _read_data(filename, skip_rows, header1['npts']) skip_rows += ndata_rows + 1 with open(filename, 'rt', encoding=ENCODING) as f: [next(f) for x in range(skip_rows)] lines2 = [next(f) for x in range(TEXT_HDR_ROWS)] header2 = _read_header(lines2, filename, table) skip_rows += TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS data2 = _read_data(filename, skip_rows, header1['npts']) skip_rows += ndata_rows + 1 with open(filename, 'rt', encoding=ENCODING) as f: [next(f) for x in range(skip_rows)] lines3 = [next(f) for x in range(TEXT_HDR_ROWS)] header3 = _read_header(lines3, filename, table) skip_rows += TEXT_HDR_ROWS + INT_HEADER_ROWS + FLOAT_HEADER_ROWS data3 = _read_data(filename, skip_rows, header1['npts']) trace1 = StationTrace(data=data1, header=header1) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace1.setProvenance('remove_response', response) trace2 = StationTrace(data=data2, header=header2) trace2.setProvenance('remove_response', response) trace3 = StationTrace(data=data3, header=header3) trace3.setProvenance('remove_response', response) stream = StationStream(traces=[trace1, trace2, trace3]) return [stream]
def test_rotation(): # Create a stream and station summary, convert from m/s^2 to cm/s^2 (GAL) osc1_data = np.genfromtxt(datadir + '/ALCTENE.UW..sac.acc.final.txt') osc2_data = np.genfromtxt(datadir + '/ALCTENN.UW..sac.acc.final.txt') osc1_data = osc1_data.T[1] * 100 osc2_data = osc2_data.T[1] * 100 tr1 = StationTrace(data=osc1_data, header={ 'channel': 'HN1', 'delta': 0.01, 'npts': len(osc1_data), 'standard': { 'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'structure_type': '', 'sensor_serial_number': '', 'process_level': 'raw counts', 'process_time': '', 'source_file': '', 'horizontal_orientation': np.nan, 'units': 'acc', 'instrument_damping': np.nan } }) tr2 = StationTrace(data=osc2_data, header={ 'channel': 'HN2', 'delta': 0.01, 'npts': len(osc2_data), 'standard': { 'corner_frequency': np.nan, 'station_name': '', 'source': 'json', 'instrument': '', 'instrument_period': np.nan, 'source_format': 'json', 'comments': '', 'source_file': '', 'structure_type': '', 'sensor_serial_number': '', 'process_level': 'raw counts', 'process_time': '', 'horizontal_orientation': np.nan, 'units': 'acc', 'instrument_damping': np.nan } }) st = StationStream([tr1, tr2]) rotation_class = Rotation(st) # Test that GM, AM, and MAX work as expected with simple 1D datasets osc1 = np.asarray([0.0, 1.0, 2.0, 3.0]) osc2 = np.asarray([4.0, 5.0, 6.0, 7.0]) max_gm = rotation_class.get_max(osc1, 'gm', osc2) np.testing.assert_allclose(max_gm, 4.5826, atol=0.0001) max_am = rotation_class.get_max(osc1, 'am', osc2) np.testing.assert_allclose(max_am, 5.0, atol=0.0001) max_max = rotation_class.get_max(osc1, 'max', osc2) np.testing.assert_allclose(max_max, 7.0, atol=0.0001) # Test max for 1 1d Array osc1 = np.array([0.0, 1.0, 2.0]) max_val = rotation_class.get_max(osc1, 'max') assert max_val == 2.0 # Test arithmetic mean with 2D input osc1 = np.array([[0.0, 1.0], [2.0, 3.0]]) osc2 = np.array([[4.0, 5.0], [6.0, 7.0]]) means = rotation_class.get_max(osc1, 'am', osc2)[0] assert (means[0] == 3.0 and means[1] == 5.0) # Test greater of two horizontals maxs = rotation_class.get_max(osc1, 'max', osc2)[0] assert (maxs[0] == 5.0 and maxs[1] == 7.0) # Invalid dimensions osc1 = np.zeros((2, 3, 2)) try: rotation_class.get_max(osc1, 'gm') success = True except Exception: success = False assert success is False # dimensions don't match osc1 = np.array([1.0, 2.0]) osc2 = np.array([[1.0, 2.0], [3.0, 4.0]]) try: rotation_class.get_max(osc1, 'gm', osc2) success = True except Exception: success = False assert success is False # Both invalid dimensions osc1 = np.zeros((2, 3, 2)) osc2 = np.zeros((2, 3, 2)) try: rotation_class.get_max(osc1, 'gm', osc2) success = True except Exception: success = False assert success is False # Invalid method pick try: osc1 = np.array([0.0, 1.0, 2.0]) rotation_class.get_max(osc1, 'foo') success = True except Exception: success = False assert success is False
def read_cwb(filename, **kwargs): """Read Taiwan Central Weather Bureau strong motion file. Args: filename (str): Path to possible CWB data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ logging.debug("Starting read_cwb.") if not is_cwb(filename): raise Exception('%s is not a valid CWB strong motion data file.' % filename) f = open(filename, 'rt') # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including CWB here, don't provide this. We'll flag it as "--". data = np.genfromtxt(filename, skip_header=HDR_ROWS, delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW hdr = _get_header_info(f, data) f.close() head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) hdr_z = hdr.copy() hdr_z['channel'] = get_channel_name(hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) hdr_z['standard']['horizontal_orientation'] = np.nan hdr_z['standard']['vertical_orientation'] = np.nan hdr_z['standard']['units_type'] = get_units_type(hdr_z['channel']) hdr_h1 = hdr.copy() hdr_h1['channel'] = get_channel_name(hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) hdr_h1['standard']['horizontal_orientation'] = np.nan hdr_h1['standard']['vertical_orientation'] = np.nan hdr_h1['standard']['units_type'] = get_units_type(hdr_h1['channel']) hdr_h2 = hdr.copy() hdr_h2['channel'] = get_channel_name(hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) hdr_h2['standard']['horizontal_orientation'] = np.nan hdr_h2['standard']['vertical_orientation'] = np.nan hdr_h2['standard']['units_type'] = get_units_type(hdr_h2['channel']) stats_z = Stats(hdr_z) stats_h1 = Stats(hdr_h1) stats_h2 = Stats(hdr_h2) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace_z = StationTrace(data=data[:, 1], header=stats_z) trace_z.setProvenance('remove_response', response) trace_h1 = StationTrace(data=data[:, 2], header=stats_h1) trace_h1.setProvenance('remove_response', response) trace_h2 = StationTrace(data=data[:, 3], header=stats_h2) trace_h2.setProvenance('remove_response', response) stream = StationStream([trace_z, trace_h1, trace_h2]) return [stream]
def read_unam(filename): """Read the Mexican UNAM strong motion data format. Args: filename (str): path to UNAM data file. Returns: list: Sequence of one StationStream object containing 3 StationTrace objects. """ channels = _read_header(filename) npts = channels[0]['npts'] all_data = np.genfromtxt(filename, skip_header=ALL_HEADERS, max_rows=npts) trace1 = StationTrace(data=all_data[:, 0], header=channels[0]) trace2 = StationTrace(data=all_data[:, 1], header=channels[1]) trace3 = StationTrace(data=all_data[:, 2], header=channels[2]) # tell the trace that data has already been converted to physical units response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace1.setProvenance('remove_response', response) trace2.setProvenance('remove_response', response) trace3.setProvenance('remove_response', response) stream = StationStream(traces=[trace1, trace2, trace3]) return [stream]
def read_smc(filename, **kwargs): """Read SMC strong motion file. Args: filename (str): Path to possible SMC data file. kwargs (ref): any_structure (bool): Read data from any type of structure, raise Exception if False and structure type is not free-field. accept_flagged (bool): accept problem flagged data. set_location (str): Two character code for location. Other arguments will be ignored. Returns: Stream: Obspy Stream containing one channel of acceleration data (cm/s**2). """ logging.debug("Starting read_smc.") any_structure = kwargs.get('any_structure', False) accept_flagged = kwargs.get('accept_flagged', False) location = kwargs.get('location', '') if not is_smc(filename): raise Exception('%s is not a valid SMC file' % filename) with open(filename, 'rt') as f: line = f.readline().strip() if 'DISPLACEMENT' in line: raise GMProcessException('SMC: Diplacement records are not supported: ' '%s.' % filename) elif 'VELOCITY' in line: raise GMProcessException('SMC: Velocity records are not supported: ' '%s.' % filename) elif line == "*": raise GMProcessException('SMC: No record volume specified in file: ' '%s.' % filename) stats, num_comments = _get_header_info( filename, any_structure=any_structure, accept_flagged=accept_flagged, location=location) skip = ASCII_HEADER_LINES + INTEGER_HEADER_LINES + \ num_comments + FLOAT_HEADER_LINES # read float data (8 columns per line) nrows = int(np.floor(stats['npts'] / DATA_COLUMNS)) data = np.genfromtxt(filename, max_rows=nrows, skip_header=skip, delimiter=FLOAT_DATA_WIDTHS) data = data.flatten() if stats['npts'] % DATA_COLUMNS: lastrow = np.genfromtxt(filename, max_rows=1, skip_header=skip + nrows, delimiter=FLOAT_DATA_WIDTHS) data = np.append(data, lastrow) data = data[0:stats['npts']] trace = StationTrace(data, header=stats) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) stream = StationStream(traces=[trace]) return [stream]