def _init_header(self, path): config = _load_config(path) metadata = config['metadata'] header = Stats() config_key = self._config['header_sampling_rate_key'] try: header.sampling_rate = float(metadata[config_key]) except KeyError: raise KeyError("The following key was not found in the metadata: " + "{}. Did you set the correct ".format(config_key) + "'sample rate metadata key' in PAL H5 Output module?") header.npts = int(metadata[self._config['header_samples_per_record_key']]) - 1 try: if self._config['dd_300']: header.calib = metadata['dd_300_calibration'] elif self._config['dd_900']: header.calib = metadata['dd_900_calibration'] elif self._config['vd_08']: header.calib = metadata['vd_08_calibration'] elif self._config['vd_09']: header.calib = metadata['vd_09_calibration'] except KeyError: pass header.comments = str(config['comments']) header.place = metadata if self._config['header_extra1_name'] != '' and self._config['header_extra1_val'] != '': header[self._config['header_extra1_name']] = self._config['header_extra1_val'] if self._config['header_extra2_name'] != '' and self._config['header_extra2_val'] != '': header[self._config['header_extra2_name']] = self._config['header_extra2_val'] return header
def read_cwb(filename, **kwargs): """Read Taiwan Central Weather Bureau strong motion file. Args: filename (str): Path to possible CWB data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ if not is_cwb(filename): raise ValueError('%s is not a valid CWB strong motion data file.') f = open(filename, 'rt') # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including CWB here, don't provide this. We'll flag it as "--". data = np.genfromtxt(filename, skip_header=HDR_ROWS, delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW hdr = _get_header_info(f, data) f.close() hdr_z = hdr.copy() hdr_z['channel'] = get_channel_name(hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) hdr_z['standard']['horizontal_orientation'] = np.nan hdr_h1 = hdr.copy() hdr_h1['channel'] = get_channel_name(hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) hdr_h1['standard']['horizontal_orientation'] = np.nan hdr_h2 = hdr.copy() hdr_h2['channel'] = get_channel_name(hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) hdr_h2['standard']['horizontal_orientation'] = np.nan stats_z = Stats(hdr_z) stats_h1 = Stats(hdr_h1) stats_h2 = Stats(hdr_h2) trace_z = Trace(data=data[:, 1], header=stats_z) trace_h1 = Trace(data=data[:, 2], header=stats_h1) trace_h2 = Trace(data=data[:, 3], header=stats_h2) stream = Stream([trace_z, trace_h1, trace_h2]) return stream
def read_IMS_ASCII(path, net='', **kwargs): """ read a IMS_ASCII seismogram from a single station :param path: path to file :return: uquake.core.Stream """ data = np.loadtxt(path, delimiter=',', skiprows=1) stats = Stats() with open(path) as fid: field = fid.readline().split(',') stats.sampling_rate = float(field[1]) timetmp = datetime.fromtimestamp(float(field[5])) \ + timedelta( seconds=float(field[6]) / 1e6) # trigger time in second trgtime_UTC = UTCDateTime(timetmp) stats.starttime = trgtime_UTC - float(field[10]) / stats.sampling_rate stats.npts = len(data) stats.station = field[8] stats.network = net traces = [] component = np.array(['X', 'Y', 'Z']) std = np.std(data, axis=0) mstd = np.max(std) for k, dt in enumerate(data.T): stats.channel = '%s' % (component[k]) traces.append(Trace(data=np.array(dt), header=stats)) return Stream(traces=traces)
def convert_to_obspy(signal, fsp): # type: (np.array, float) -> obspy.core.trace """ Function to convert a numpy array from any infrasound reading into Obspy format. This could help us to take advantage of Obspy optimization routines. :param signal: A numpy array containg the signal we want to compute. :param fsp: The sampling frequency of the signal. :return: """ # we substract the mean stats = Stats() stats.sampling_rate = float(fsp) stats.npts = signal.shape[0] return Trace(data=signal.reshape(signal.shape[0], ), header=stats)
def _init_header(self, path): config = _load_config(path) metadata = config['metadata'] header = Stats() if 'ATS9440' in config['plugins'].keys(): ats_config = config['plugins']['ATS9440']['config'] elif 'ATS660' in config['plugins'].keys(): ats_config = config['plugins']['ATS660']['config'] else: raise KeyError('Cannot locate trace config data') if 'Polytec' in config['plugins'].keys(): polytec_config = config['plugins']['Polytec']['config'] else: raise KeyError('Cannot locate vibrometer config data') header.sampling_rate = _calc_sampling_rate(ats_config['sample_rate']) header.npts = int(ats_config['pre_trigger_samples'] + ats_config['post_trigger_samples']) - 1 if polytec_config['dd_300']: header.calib = float( re.findall(_NUMBER, polytec_config['dd_300_range'])[0]) elif polytec_config['dd_900']: header.calib = float( re.findall(_NUMBER, polytec_config['dd_900_range'])[0]) elif polytec_config['vd_08']: header.calib = float( re.findall(_NUMBER, polytec_config['vd_08_range'])[0]) elif polytec_config['vd_09']: header.calib = float( re.findall(_NUMBER, polytec_config['vd_09_range'])[0]) else: raise KeyError('Cannot locate vibrometer calibration data') header.comments = str(config['comments']) header.place = metadata if self._config['header_extra1_name'] != '' and self._config[ 'header_extra1_val'] != '': header[self._config['header_extra1_name']] = self._config[ 'header_extra1_val'] if self._config['header_extra2_name'] != '' and self._config[ 'header_extra2_val'] != '': header[self._config['header_extra2_name']] = self._config[ 'header_extra2_val'] return header
def get_Stream(self): """ Return an obspy stream """ stas = [ 'RMNW.GC01..CHZ', 'RMNW.GC01..CHX', 'RMNW.GC01..CHY', 'RMNW.GC02..CHZ', 'RMNW.GC02..CHX', 'RMNW.GC02..CHY', 'RMNW.GC03..CHZ', 'RMNW.GC03..CHX', 'RMNW.GC03..CHY', 'RMNW.GC04..CHZ', 'RMNW.GC04..CHX', 'RMNW.GC04..CHY' ] st = Stream() delta = self.sample_period / 1e6 time_v, data_mat = self.get_all_data() for col in range(data_mat.shape[1]): seed = stas[col].split('.') tr = Trace(data=data_mat[:, col], header=Stats( dict(starttime=time_v[0], npts=data_mat.shape[0], delta=delta, network=seed[0], station=seed[1], location=seed[2], channel=seed[3]))) st.traces.append(tr) return st
def readgeonet(geonetfile): """ Read strong motion data from a GeoNet data file @param geonetfile: Path to a valid GeoNet data file. @return: List of ObsPy Trace objects, containing accelerometer data in m/s. """ f = open(geonetfile, 'rt') tracelist = [] headerlist = [] try: hdrlines = readheaderlines(f) except: pass while len(hdrlines[-1]): hdrdict = readheader(hdrlines) numlines = int(np.ceil(hdrdict['npts'] / 10.0)) data = [] for i in range(0, numlines): line = f.readline() parts = line.strip().split() mdata = [float(p) for p in parts] data = data + mdata data = np.array(data) header = hdrdict.copy() stats = Stats(hdrdict) trace = Trace(data, header=stats) #apply the calibration and convert from mm/s^2 to m/s^2 trace.data = trace.data * trace.stats[ 'calib'] * 0.001 #convert to m/s^2 tracelist.append(trace.copy()) headerlist.append(header.copy()) hdrlines = readheaderlines(f) f.close() return (tracelist, headerlist)
def ReadStnPara(self): """ 读取redis中的台站参数信息,台站信息存入一维数组station_list,通道信息存入二维数组channel_list """ stn_para_key = 'StnPara' self.station_number = int(self.r.hget(stn_para_key, "station_number")) self.trace_list = [[Trace() for col in range(3)] for row in range(self.station_number)] print('++++') print(type(self.trace_list[0][0])) self.channel_list = [[0 for col in range(3)] for row in range(self.station_number)] self.station_list = [0 for col in range(self.station_number)] # multilist = [[0 for col in range(5)] for row in range(3)] #二维数组 coordinates = AttribDict() channel_name = [0 for col in range(3)] for key in ['latitude', 'longitude', 'elevation']: coordinates[key] = 0 chnNum = 0 for key in ['Z', 'E', 'N']: channel_name[chnNum] = key chnNum = chnNum + 1 for staNo in range(0, self.station_number): stn_para_field = '{0:0>4}'.format(staNo) stn_para_res = self.r.hget(stn_para_key, stn_para_field) stn_par = stn_para_res.decode('utf-8').split() # print(str(staNo) + str(stn_par)) stn_para_defaults = AttribDict() stn_para_defaults['coordinates'] = AttribDict() stn_para_defaults['coordinates'] = coordinates stn_para_defaults['network'] = stn_par[1] stn_para_defaults['station'] = stn_par[2] stn_para_defaults['channel'] = stn_par[4] stn_para_defaults['location'] = stn_par[5] stn_para_defaults['latitude'] = float(stn_par[6]) stn_para_defaults['longitude'] = float(stn_par[7]) stn_para_defaults['elevation'] = float(stn_par[8]) stn_para_defaults['channelNum'] = int(stn_par[9]) stn_para_defaults['sampling_rate'] = int(stn_par[10]) self.station_list[staNo] = stn_para_defaults for chnNo in range(0, self.station_list[staNo]['channelNum']): chn_para_defaults = Stats(AttribDict()) chn_para_defaults['sampling_rate'] = self.station_list[staNo][ 'sampling_rate'] chn_para_defaults['delta'] = 1.0 chn_para_defaults['calib'] = 1.0 chn_para_defaults['starttime'] = UTCDateTime(0) chn_para_defaults['npts'] = 0 chn_para_defaults['network'] = stn_para_defaults['network'] chn_para_defaults['station'] = stn_para_defaults['station'] chn_para_defaults['channel'] = stn_para_defaults[ 'channel'] + channel_name[chnNo] chn_para_defaults['location'] = stn_para_defaults['location'] chn_para_defaults['response'] = float(stn_par[11 + chnNo]) self.channel_list[staNo][chnNo] = chn_para_defaults
def readiran(iranfile, doRotation=True): """ Read strong motion data from a Iran data file @param iranfile: Path to a valid Iran data file. @keyword doRotation: Apply back-azimuth rotation of L & T channels to NS and EW. @return: List of ObsPy Trace objects, containing accelerometer data in m/s. """ f = open(iranfile, 'rt') tracelist = [] headerlist = [] try: hdrlines = readheaderlines(f) except: pass while len(hdrlines[-1]): hdrdict = readheader(hdrlines) numlines = int(np.ceil(hdrdict['npts'] / 10.0)) data = [] for i in range(0, numlines): line = f.readline() parts = line.strip().split() mdata = [float(p) for p in parts] data = data + mdata data = np.array(data) header = hdrdict.copy() stats = Stats(hdrdict) trace = Trace(data, header=stats) #apply the calibration and convert from mm/s^2 to m/s^2 trace.data = trace.data * trace.stats[ 'calib'] * 0.98 #convert to m/s^2 from g/10 tracelist.append(trace.copy()) headerlist.append(header.copy()) endblock = f.readline() hdrlines = readheaderlines(f) f.close() #data from Iran may be rotated so that one channel is aligned in the direction between the earthquake #epicenter and the station. We want to rotate the data back so that we have what are presumably the #original NS and EW channels. We presume that the "L" channel will rotate back to become NS, and #"T" will become EW. #First, find the channel called L* if doRotation: channels = [h['channel'][0:1] for h in headerlist] lidx = channels.index('L') tidx = channels.index('T') ldata = tracelist[lidx].data tdata = tracelist[tidx].data backaz = headerlist[0]['rotation']['L'] ndata, edata = rotate.rotate_RT_NE(ldata, tdata, backaz) tracelist[lidx].data = ndata.copy() tracelist[lidx].stats[ 'channel'] = 'H1' #most probably NS, but we're being cautious tracelist[tidx].data = edata.copy() tracelist[tidx].stats[ 'channel'] = 'H2' #most probably EW, but we're being cautious return (tracelist, headerlist)
def _remove_processing(self, st): """ copy stream and remove processing""" from obspy.core.trace import Stats st = st.copy() for tr in st: tr.stats = Stats({x: tr.stats[x] for x in Stats.defaults}) # tr.stats.pop("processing", None) return st
def _init_header(self, path): config = _load_config(path) metadata = config['metadata'] header = Stats() if 'ATS9440' in config['plugins'].keys(): ats_config = config['plugins']['ATS9440']['config'] elif 'ATS660' in config['plugins'].keys(): ats_config = config['plugins']['ATS660']['config'] else: raise KeyError('Cannot locate trace config data') if 'Polytec' in config['plugins'].keys(): polytec_config = config['plugins']['Polytec']['config'] else: raise KeyError('Cannot locate vibrometer config data') header.sampling_rate = _calc_sampling_rate(ats_config['sample_rate']) header.npts = int(ats_config['pre_trigger_samples'] + ats_config['post_trigger_samples']) - 1 if polytec_config['dd_300']: header.calib = float(re.findall( _NUMBER, polytec_config['dd_300_range'])[0]) elif polytec_config['dd_900']: header.calib = float(re.findall( _NUMBER, polytec_config['dd_900_range'])[0]) elif polytec_config['vd_08']: header.calib = float(re.findall( _NUMBER, polytec_config['vd_08_range'])[0]) elif polytec_config['vd_09']: header.calib = float(re.findall( _NUMBER, polytec_config['vd_09_range'])[0]) else: raise KeyError('Cannot locate vibrometer calibration data') header.comments = str(config['comments']) header.place = metadata if self._config['header_extra1_name'] != '' and self._config['header_extra1_val'] != '': header[self._config['header_extra1_name'] ] = self._config['header_extra1_val'] if self._config['header_extra2_name'] != '' and self._config['header_extra2_val'] != '': header[self._config['header_extra2_name'] ] = self._config['header_extra2_val'] return header
def test_imfv122_filename(): ''' Test generating IAGA2002 filename ''' stats = Stats( header={ 'network': 'C2', 'station': 'OTT', 'location': 'R0', 'channel': 'UFX', 'starttime': UTCDateTime(2020, 1, 10), 'detla': 60 }) assert pygeomag.data.formats.imfv122.get_filename(stats) == 'JAN1020.OTT'
def _read_channel(filename, line_offset, volume, location=''): """Read channel data from USC V1 text file. Args: filename (str): Input USC V1 filename. line_offset (int): Line offset to beginning of channel text block. volume (dictionary): Dictionary of formatting information Returns: tuple: (obspy Trace, int line offset) """ # Parse the header portion of the file try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) # read in lines of integer data skiprows = line_offset + volume['TEXT_HDR_ROWS'] int_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=volume['INT_HDR_ROWS'], dtype=np.int32, delimiter=volume['INT_FMT']).flatten() # read in lines of float data skiprows += volume['INT_HDR_ROWS'] + 1 flt_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64, delimiter=volume['FLT_FMT']).flatten() hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location) skiprows += volume['FLT_HDR_ROWS'] # read in the data nrows = int(np.floor(hdr['npts'] * 2 / 10)) data = np.genfromtxt(filename, skip_header=skiprows, max_rows=nrows, dtype=np.float64, delimiter=volume['COL_FMT']).flatten()[1::2] trace = Trace(data.copy(), Stats(hdr.copy())) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def _read_channel(filename, line_offset, location=''): """Read channel data from COSMOS V1/V2 text file. Args: filename (str): Input COSMOS V1/V2 filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (obspy Trace, int line offset) """ # read station, location, and process level from text header with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(TEXT_HDR_ROWS)] # read in lines of integer data skiprows = line_offset + TEXT_HDR_ROWS int_lines, int_data = _read_lines(skiprows, filename) int_data = int_data.astype(np.int32) # read in lines of float data skiprows += int_lines + 1 flt_lines, flt_data = _read_lines(skiprows, filename) # read in comment lines skiprows += flt_lines + 1 cmt_lines, cmt_data = _read_lines(skiprows, filename) skiprows += cmt_lines + 1 # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including cosmos here, don't provide this. We'll flag it as "--". hdr = _get_header_info(int_data, flt_data, lines, cmt_data, location=location) # read in the data nrows, data = _read_lines(skiprows, filename) trace = Trace(data.copy(), Stats(hdr.copy())) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def readknet(knetfilename): """ Read a KNet ASCII file, and return an ObsPy Trace object, plus a dictionary of header values. @param knetfilename: String path to valid KNet ASCII file, as described here: http://www.kyoshin.bosai.go.jp/kyoshin/man/knetform_en.html @return: ObsPy Trace object, and a dictionary of some of the header values found in the input file. """ data = [] hdrdict = {} f = open(knetfilename,'rt') dataOn = False headerlines = [] for line in f.readlines(): if line.startswith('Memo'): hdrdict = readheader(headerlines) dataOn = True continue if not dataOn: headerlines.append(line) continue if dataOn: parts = line.strip().split() mdata = [float(p) for p in parts] data = data + mdata f.close() #fill in the values usually expected in Stats as best we can hdrdict['npts'] = len(data) elapsed = float(hdrdict['npts'])/float(hdrdict['sampling_rate']) hdrdict['endtime'] = hdrdict['starttime'] + elapsed hdrdict['network'] = 'NIED' hdrdict['location'] = '' #The Stats constructor appears to modify the fields in the input dictionary - let's save #a copy header = hdrdict.copy() data = np.array(data) stats = Stats(hdrdict) trace = Trace(data,header=stats) #apply the calibration and convert to m/s^2 trace.data = trace.data * trace.stats['calib'] * 0.01 #convert to m/s^2 return (trace,header)
def _internal_read_knet_ascii(buf, **kwargs): """ Reads a K-NET/KiK-net ASCII file and returns an ObsPy Stream object. .. warning:: This function should NOT be called directly, it registers via the ObsPy :func:`~obspy.core.stream.read` function, call this instead. :param buf: File to read. :type buf: Open file or open file like object. """ data = [] hdrdict = {} cur_pos = buf.tell() buf.seek(0, 2) size = buf.tell() buf.seek(cur_pos, 0) # First read the headerlines headerlines = [] while buf.tell() < size: line = buf.readline().decode() headerlines.append(line) if line.startswith('Memo'): hdrdict = _read_knet_hdr(headerlines, **kwargs) break while buf.tell() < size: line = buf.readline() parts = line.strip().split() data += [float(p) for p in parts] hdrdict['npts'] = len(data) # The FDSN network code for the National Research Institute for Earth # Science and Disaster Prevention (NEID JAPAN) is BO (Bosai-Ken Network) hdrdict['network'] = 'BO' data = np.array(data) stats = Stats(hdrdict) trace = Trace(data, header=stats) return Stream([trace])
def readgeonet(geonetfile): """ Read strong motion data from a GeoNet data file @param geonetfile: Path to a valid GeoNet data file. @return: List of ObsPy Trace objects, containing accelerometer data in m/s. """ #notes on implementation: # originally I had written code to read each line of data manually, just # as I was reading the header lines. However, this became VERY slow for large # files. I discovered that numpy's genfromtxt function was much faster. However, # I could not get that function to work when I passed it a file object instead of a file name. # Consequently, the only way I could keep the genfromtxt() method and file pointer in sync was # to read and discard all of the lines of data that genfromtxt() parsed. While annoying, the combination # of these two seems to still be at least an order of magnitude faster than manually reading the file. f = open(geonetfile,'rt') tracelist = [] headerlist = [] totlines = 0 hdrlines = readheaderlines(f) while len(hdrlines[-1]): totlines += len(hdrlines) hdrdict = readheader(hdrlines) numlines = int(np.ceil(hdrdict['npts']/10.0)) data = np.genfromtxt(geonetfile,skip_header=totlines,max_rows=numlines) totlines += numlines #now we need to set the file position to where we just ended for i in range(0,numlines): f.readline() data = data.flatten() header = hdrdict.copy() stats = Stats(hdrdict) trace = Trace(data,header=stats) #apply the calibration and convert from mm/s^2 to m/s^2 trace.data = trace.data * trace.stats['calib'] * 0.001 #convert to m/s^2 tracelist.append(trace.copy()) headerlist.append(header.copy()) hdrlines = readheaderlines(f) f.close() return (tracelist,headerlist)
def readitaly(datafile): f = open(datafile, 'rt') #header needs: station,channel,location,npts,starttime,sampling_rate,delta,calib,lat,lon,height,duration,endtime,maxacc,network data = [] hdrdict = {} for line in f.readlines(): if not len(line.strip()): continue if not line.find(':') > -1: data.append(float(line.strip())) continue key, value = line.split(':') key = key.strip() value = value.strip() if key not in list(HEADERS.keys()): continue hdrkey = HEADERS[key] if hdrkey == 'starttime': value = UTCDateTime(datetime.datetime.strptime(value, TIMEFMT)) elif hdrkey not in ['station', 'channel', 'location', 'network']: value = float(value) hdrdict[hdrkey] = value f.close() hdrdict['sampling_rate'] = 1 / hdrdict['delta'] hdrdict['endtime'] = hdrdict['starttime'] + hdrdict['duration'] hdrdict['npts'] = int(hdrdict['npts']) hdrdict['calib'] = 1.0 hdrdict['units'] = 'acc' data = np.array(data) header = hdrdict.copy() stats = Stats(hdrdict) trace = Trace(data, header=stats) #apply the calibration and convert from mm/s^2 to m/s^2 trace.data = trace.data * trace.stats['calib'] * 0.01 #convert to m/s^2 return trace
def to_sac_and_mseed(self, export_path, station_number, force_without_loc): # Check if file exist export_path_sac = export_path + self.get_export_file_name() + ".sac" export_path_msd = export_path + self.get_export_file_name() + ".mseed" #export_path_wav = export_path + self.get_export_file_name() + ".wav" if os.path.exists(export_path_sac) and os.path.exists(export_path_msd): return # Check if the station location have been calculated if self.station_loc is None and not force_without_loc: print self.get_export_file_name() + ": Skip sac/mseed generation, wait the next ascent to compute location" return # Fill header info stats = Stats() stats.sampling_rate = self.decimated_fs stats.network = "MH" stats.station = station_number stats.starttime = self.date stats.sac = dict() if not force_without_loc: stats.sac["stla"] = self.station_loc.latitude stats.sac["stlo"] = self.station_loc.longitude stats.sac["stdp"] = self.depth stats.sac["user0"] = self.snr stats.sac["user1"] = self.criterion stats.sac["iztype"] = 9 # 9 == IB in sac format # Save data into a Stream object trace = Trace() trace.stats = stats trace.data = self.data stream = Stream(traces=[trace]) # Save stream object print export_path_sac stream.write(export_path_sac, format='SAC') print export_path_msd stream.write(export_path_msd, format='MSEED')
def read_esm(filename, config=None, **kwargs): """Read European ESM strong motion file. Args: filename (str): Path to possible ESM data file. config (dict): Dictionary containing configuration. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing one channels of acceleration data (cm/s**2). """ logging.debug("Starting read_esm.") if not is_esm(filename, config): raise Exception(f"{filename} is not a valid ESM file") # Parse the header portion of the file header = {} with open(filename, "rt") as f: lines = [next(f) for x in range(TEXT_HDR_ROWS)] for line in lines: parts = line.split(":") key = parts[0].strip() value = ":".join(parts[1:]).strip() header[key] = value stats = {} standard = {} coordinates = {} # fill in all known stats header fields stats["network"] = header["NETWORK"] stats["station"] = header["STATION_CODE"] stats["channel"] = header["STREAM"] stats["location"] = "--" stats["delta"] = float(header["SAMPLING_INTERVAL_S"]) stats["sampling_rate"] = 1 / stats["delta"] stats["calib"] = 1.0 stats["npts"] = int(header["NDATA"]) stimestr = header["DATE_TIME_FIRST_SAMPLE_YYYYMMDD_HHMMSS"] stats["starttime"] = datetime.strptime(stimestr, TIMEFMT) # fill in standard fields head, tail = os.path.split(filename) standard["source_file"] = tail or os.path.basename(head) standard["source"] = SRC standard["source_format"] = FORMAT standard["horizontal_orientation"] = np.nan standard["vertical_orientation"] = np.nan standard["station_name"] = header["STATION_NAME"] try: standard["instrument_period"] = 1 / float( header["INSTRUMENTAL_FREQUENCY_HZ"]) except ValueError: standard["instrument_period"] = np.nan try: standard["instrument_damping"] = 1 / float( header["INSTRUMENTAL_DAMPING"]) except ValueError: standard["instrument_damping"] = np.nan ptimestr = header["DATA_TIMESTAMP_YYYYMMDD_HHMMSS"] ptime = datetime.strptime(ptimestr, TIMEFMT).strftime(TIMEFMT2) standard["process_time"] = ptime standard["process_level"] = PROCESS_LEVELS["V1"] instr_str = header["INSTRUMENT"] parts = instr_str.split("|") sensor_str = parts[0].split("=")[1].strip() standard["sensor_serial_number"] = "" standard["instrument"] = sensor_str standard["comments"] = "" standard["structure_type"] = "" standard["units"] = "cm/s^2" standard["units_type"] = "acc" standard["instrument_sensitivity"] = np.nan standard["corner_frequency"] = np.nan coordinates["latitude"] = float(header["STATION_LATITUDE_DEGREE"]) coordinates["longitude"] = float(header["STATION_LONGITUDE_DEGREE"]) coordinates["elevation"] = float(header["STATION_ELEVATION_M"]) # read in the data data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS) # create a Trace from the data and metadata stats["standard"] = standard stats["coordinates"] = coordinates trace = StationTrace(data.copy(), Stats(stats.copy())) response = {"input_units": "counts", "output_units": "cm/s^2"} trace.setProvenance("remove_response", response) ftype = header["FILTER_TYPE"].capitalize() try: forder = int(header["FILTER_ORDER"]) except ValueError: forder = 0 try: lowfreq = float(header["LOW_CUT_FREQUENCY_HZ"]) except ValueError: lowfreq = np.nan try: highfreq = float(header["LOW_CUT_FREQUENCY_HZ"]) except ValueError: highfreq = np.nan if not np.isnan(lowfreq) and not np.isnan(lowfreq): filter_att = { "bandpass_filter": { "filter_type": ftype, "lower_corner_frequency": lowfreq, "higher_corner_frequency": highfreq, "filter_order": forder, } } trace.setProvenance("lowpass_filter", filter_att) detrend_att = {"detrend": {"detrending_method": "baseline"}} if "NOT REMOVED" not in header["BASELINE_CORRECTION"]: trace.setProvenance("detrend", detrend_att) stream = StationStream(traces=[trace]) return [stream]
def _read_volume_two(filename, line_offset, location='', units='acc'): """Read channel data from DMG text file. Args: filename (str): Input DMG V2 filename. line_offset (int): Line offset to beginning of channel text block. units (str): Units to get. Returns: tuple: (list of obspy Trace, int line offset) """ try: with open(filename, 'rt', encoding='utf-8') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(V2_TEXT_HDR_ROWS)] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) # read in lines of integer data skip_rows = V2_TEXT_HDR_ROWS + line_offset int_data = _read_lines(skip_rows, V2_INT_HDR_ROWS, V2_INT_FMT, filename) int_data = int_data[0:100].astype(np.int32) # read in lines of float data skip_rows += V2_INT_HDR_ROWS flt_data = _read_lines(skip_rows, V2_REAL_HDR_ROWS, V2_REAL_FMT, filename) flt_data = flt_data[:100] skip_rows += V2_REAL_HDR_ROWS # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including csmip/dmg here, don't always provide this. We'll flag it as # "--". hdr = _get_header_info(int_data, flt_data, lines, 'V2', location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) traces = [] # read acceleration data if hdr['npts'] > 0: acc_rows, acc_fmt, unit = _get_data_format( filename, skip_rows, hdr['npts']) acc_data = _read_lines(skip_rows + 1, acc_rows, acc_fmt, filename) acc_data = acc_data[:hdr['npts']] if unit in UNIT_CONVERSIONS: acc_data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise ValueError('DMG: %s is not a supported unit.' % unit) acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} acc_trace.setProvenance('remove_response', response) if units == 'acc': traces += [acc_trace] skip_rows += int(acc_rows) + 1 # ------------------------------------------------------------------------- # NOTE: The way we were initially reading velocity and displacement data # was not correct. I'm deleting it for now since we don't need it. If/when # we revisit this we need to be more careful about how this is handled. # ------------------------------------------------------------------------- # read velocity data vel_hdr = hdr.copy() vel_hdr['standard']['units'] = 'vel' vel_hdr['npts'] = int_data[63] if vel_hdr['npts'] > 0: vel_rows, vel_fmt, unit = _get_data_format( filename, skip_rows, vel_hdr['npts']) vel_data = _read_lines(skip_rows + 1, vel_rows, vel_fmt, filename) vel_data = vel_data[:vel_hdr['npts']] skip_rows += int(vel_rows) + 1 # read displacement data disp_hdr = hdr.copy() disp_hdr['standard']['units'] = 'disp' disp_hdr['npts'] = int_data[65] if disp_hdr['npts'] > 0: disp_rows, disp_fmt, unit = _get_data_format( filename, skip_rows, disp_hdr['npts']) disp_data = _read_lines(skip_rows + 1, disp_rows, disp_fmt, filename) disp_data = disp_data[:disp_hdr['npts']] skip_rows += int(disp_rows) + 1 # there is an 'end of record' line after the data] new_offset = skip_rows + 1 return (traces, new_offset)
def _read_volume(filename, line_offset): """Read channel data from text file. Args: filename (str): Input filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (list of obspy Trace, int line offset) """ # read station, location, and process level from text header with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(V2_TEXT_HDR_ROWS)] # parse out the station name, location, and process level hdr = {} # check that filename matches network and station station = lines[5][12:17].replace(' ', '') hdr['station'] = station hdr['process_level'] = 'V2' # read in lines of integer data skip_rows = V2_TEXT_HDR_ROWS + line_offset int_data = _read_lines(skip_rows, V2_INT_HDR_ROWS, V2_INT_FMT, filename) int_data = int_data[0:100].astype(np.int32) # read in lines of float data skip_rows += V2_INT_HDR_ROWS flt_data = _read_lines(skip_rows, V2_REAL_HDR_ROWS, V2_REAL_FMT, filename) flt_data = flt_data[:100] skip_rows += V2_REAL_HDR_ROWS # Parse name and code information name_length = int_data[29] name = re.sub(' +', ' ', lines[6][:name_length]).strip().replace(' ', '_') hdr['name'] = name hdr['network'] = 'Unknown' # set statistics hdr['units'] = 'acc' lat = lines[5][21:27].replace(' ', '') if lat[-1].upper() == 'S': lat = -1 * float(lat[0:-1]) lon = lines[5][30:37].replace(' ', '') if lon[-1].upper() == 'W': lon = -1 * float(lon[0:-1]) hdr['lat'] = lat hdr['lon'] = lon hdr['location'] = '--' hdr['delta'] = flt_data[60] hdr['sampling_rate'] = 1 / hdr['delta'] hdr['npts'] = int_data[52] hdr['source'] = hdr['network'] angle = int_data[26] if angle == 500 or angle == 600: hdr['channel'] = 'Z' elif angle > 315 or angle < 45 or (angle > 135 and angle < 225): hdr['channel'] = 'H1' else: hdr['channel'] = 'H2' traces = [] # read acceleration data if hdr['npts'] > 0: acc_rows, acc_fmt = _get_data_format(filename, skip_rows, hdr['npts']) acc_data = _read_lines(skip_rows + 1, acc_rows, acc_fmt, filename) acc_data = acc_data[:hdr['npts']] acc_trace = Trace(acc_data.copy(), Stats(hdr.copy())) traces += [acc_trace] skip_rows += int(acc_rows) + 1 # read acceleration data vel_hdr = hdr.copy() vel_hdr['units'] = 'vel' vel_hdr['npts'] = hdr['npts'] if vel_hdr['npts'] > 0: vel_rows, vel_fmt = _get_data_format(filename, skip_rows, vel_hdr['npts']) vel_data = _read_lines(skip_rows + 1, vel_rows, vel_fmt, filename) vel_data = vel_data[:vel_hdr['npts']] vel_trace = Trace(vel_data.copy(), Stats(vel_hdr.copy())) traces += [vel_trace] skip_rows += int(vel_rows) + 1 # read displacement data disp_hdr = hdr.copy() disp_hdr['units'] = 'disp' disp_hdr['npts'] = hdr['npts'] if disp_hdr['npts'] > 0: disp_rows, disp_fmt = _get_data_format(filename, skip_rows, disp_hdr['npts']) disp_data = _read_lines(skip_rows + 1, disp_rows, disp_fmt, filename) disp_data = disp_data[:disp_hdr['npts']] disp_trace = Trace(disp_data.copy(), Stats(disp_hdr.copy())) traces += [disp_trace] skip_rows += int(disp_rows) + 1 new_offset = skip_rows + 1 # there is an 'end of record' line after the data] return (traces, new_offset)
def dorange (self): # load batches print "mkms: loading batches.." self.bdatas = [] for i in self.ids: d = Dat () d.read (os.path.join (self.root, str(i) + '.DAT')) self.bdatas.append (d.bdata) # set up datastream print "mkms: setting up stream for %s.." % self.station, self.st = Stream () for bd in self.bdatas: for b in bd.batches: s = Stats () s.sampling_rate = self.sampling_rate s.npts = b.length s.network = self.network s.location = self.location s.station = self.station s.channel = self.channel s.starttime = UTCDateTime ((b.ref / 1000000.0)) t = Trace (data = numpy.array (b.samples_i, dtype = numpy.int32), header = s) self.st.append (t) print "done." # generate file name self.name = self.st[0].id.replace ('.', '_') self.start = self.st[0].stats.starttime self.name = self.start.strftime ("%Y-%m-%d-%H%M-%S") + '.' + self.name if self.optplot: self.plot () if not self.optnowrite: print "mkms: writing %s.mseed.." % self.name, if not os.path.exists (self.destdir): os.makedirs (self.destdir) self.st.write (os.path.join (self.destdir, self.name + '.mseed'), format = 'MSEED', encoding = 'INT32', byteorder = 1, flush = 1, verbose = 0) print "done." # write ids and refs idsf = open (os.path.join (self.destdir, self.name + '.ids'), 'w') refsf = open (os.path.join (self.destdir, self.name + '.refs'), 'w') for bd in self.bdatas: idsf.write ("%d,%d\n" % (bd.id, 1 if bd.e_sdlag else 0)) for b in bd.batches: refsf.write ("%d,%d,%d,%d,%s,%s,%s,%s,%s,%d\n" % (bd.id, b.no, b.ref, b.status, b.latitude[:-2], b.latitude[-2:], b.longitude[:-2], b.longitude[-2:], b.checksum, 1 if b.checksum_pass else 0)) idsf.close () refsf.close () return (self.name + '.mseed', idsf, refsf) else: print "mkms: would write %s.mseed (disabled)." % os.path.join (self.destdir, self.name) return None
def _read_volume_one(filename, line_offset, location='', units='acc'): """Read channel data from DMG Volume 1 text file. Args: filename (str): Input DMG V1 filename. line_offset (int): Line offset to beginning of channel text block. units (str): units to get. Returns: tuple: (list of obspy Trace, int line offset) """ # Parse the header portion of the file try: with open(filename, 'rt', encoding='utf-8') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) unit = _get_units(lines[11]) # read in lines of integer data skip_rows = V1_TEXT_HDR_ROWS + line_offset int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename) int_data = int_data[0:100].astype(np.int32) # read in lines of float data skip_rows += V1_INT_HDR_ROWS flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename) skip_rows += V1_REAL_HDR_ROWS # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including csmip/dmg here, don't always provide this. We'll flag it as # "--". hdr = _get_header_info_v1( int_data, flt_data, lines, 'V1', location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # sometimes (??) a line of text is inserted in between the float header and # the beginning of the data. Let's check for this... with open(filename, 'rt', encoding='utf-8') as f: for _ in range(skip_rows): next(f) test_line = f.readline() has_text = re.search('[A-Z]+|[a-z]+', test_line) is not None if has_text: skip_rows += 1 widths = [9] * 8 max_rows = int(np.ceil(hdr['npts'] / 8)) data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[:hdr['npts']] evenly_spaced = True # Sometimes, npts is incrrectly specified, leading to nans # in the resulting data. Fix that here if np.any(np.isnan(acc_data)): while np.isnan(acc_data[-1]): acc_data = acc_data[:-1] hdr['npts'] = len(acc_data) else: # acceleration data is interleaved between time data max_rows = int(np.ceil(hdr['npts'] / 5)) widths = [7] * 10 data = _read_lines(skip_rows, max_rows, widths, filename) acc_data = data[1::2][:hdr['npts']] times = data[0::2][:hdr['npts']] evenly_spaced = is_evenly_spaced(times) if unit in UNIT_CONVERSIONS: acc_data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise ValueError('DMG: %s is not a supported unit.' % unit) acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy())) # Check if the times were included in the file but were not evenly spaced if not evenly_spaced: acc_trace = resample_uneven_trace(acc_trace, times, acc_data) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} acc_trace.setProvenance('remove_response', response) traces = [acc_trace] new_offset = skip_rows + max_rows + 1 # there is an end of record line return (traces, new_offset)
def _read_channel(filename, line_offset): """Read channel data from GNS V1 text file. Args: filename (str): Input GNS V1 filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (obspy Trace, int line offset) """ # read station and location strings from text header with open(filename, 'rt', encoding='utf-8') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(TEXT_HDR_ROWS)] # this code supports V1 and V2 format files. Which one is this? data_format = 'V2' if lines[0].lower().find('uncorrected') >= 0: data_format = 'V1' # parse out the station code, name, and component string # from text header station = lines[1].split()[1] logging.debug('station: %s' % station) name = lines[2].replace(' ', '_').strip() component = lines[12].split()[1] # parse the instrument type from the text header instrument = lines[3].split()[1] # parse the sensor resolution from the text header resolution_str = lines[4].split()[1] resolution = int(re.search(r"\d+", resolution_str).group()) # read floating point header array skip_header = line_offset + TEXT_HDR_ROWS hdr_data = np.genfromtxt(filename, skip_header=skip_header, max_rows=FP_HDR_ROWS) # parse header dictionary from float header array hdr = _read_header(hdr_data, station, name, component, data_format, instrument, resolution) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including GeoNet here, don't provide this. We'll flag it as "--". hdr['location'] = '--' skip_header2 = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS widths = [8] * COLS_PER_ROW nrows = int(np.ceil(hdr['npts'] / COLS_PER_ROW)) data = np.genfromtxt(filename, skip_header=skip_header2, max_rows=nrows, filling_values=np.nan, delimiter=widths) data = data.flatten() data = data[0:hdr['npts']] # for debugging, read in the velocity data nvel = hdr_data[3, 4] if nvel: if nvel % COLS_PER_ROW != 0: nvel_rows = int(np.floor(nvel / COLS_PER_ROW)) else: nvel_rows = int(np.ceil(nvel / COLS_PER_ROW)) skip_header_vel = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS + nrows widths = [8] * COLS_PER_ROW velocity = np.genfromtxt(filename, skip_header=skip_header_vel, max_rows=nvel_rows, filling_values=np.nan, delimiter=widths) velocity = velocity.flatten() velocity *= MMPS_TO_CMPS else: velocity = np.array([]) # for V2 files, there are extra blocks of data we need to skip containing # velocity and displacement data if data_format == 'V2': velrows = int(np.ceil(hdr_data[3, 4] / COLS_PER_ROW)) disrows = int(np.ceil(hdr_data[3, 5] / COLS_PER_ROW)) nrows = nrows + velrows + disrows data *= MMPS_TO_CMPS # convert to cm/s**2 trace = StationTrace(data, Stats(hdr)) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) offset = skip_header2 + nrows return (trace, offset, velocity)
def python2obspy(self): from obspy.core.trace import Stats, Trace from obspy.core.utcdatetime import UTCDateTime s = Stats() s.network = self.network s.station = self.station s.location = self.location s.channel = self.channel s.sampling_rate = self.sampling_rate s.starttime = UTCDateTime(self.starttime) s.npts = len(self.data) misc_fields = dict() if 'CALIB' in self.misc_fields: s.calib = self.misc_fields.pop('CALIB') s.update(self.misc_fields) return Trace(self.data[:], header=s)
def _read_channel(filename, line_offset, volume, location='', alternate=False): """Read channel data from USC V1 text file. Args: filename (str): Input USC V1 filename. line_offset (int): Line offset to beginning of channel text block. volume (dictionary): Dictionary of formatting information. Returns: tuple: (obspy Trace, int line offset) """ if alternate: int_rows = 5 int_fmt = 20 * [4] data_cols = 8 else: int_rows = volume['INT_HDR_ROWS'] int_fmt = volume['INT_FMT'] data_cols = 10 # Parse the header portion of the file try: with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])] # Accounts for blank lines at end of files except StopIteration: return (None, 1 + line_offset) # read in lines of integer data skiprows = line_offset + volume['TEXT_HDR_ROWS'] int_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=int_rows, dtype=np.int32, delimiter=int_fmt).flatten() # read in lines of float data skiprows += int_rows flt_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=volume['FLT_HDR_ROWS'], dtype=np.float64, delimiter=volume['FLT_FMT']).flatten() hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location) skiprows += volume['FLT_HDR_ROWS'] # read in the data nrows = int(np.floor(hdr['npts'] * 2 / data_cols)) all_data = np.genfromtxt(filename, skip_header=skiprows, max_rows=nrows, dtype=np.float64, delimiter=volume['COL_FMT']) data = all_data.flatten()[1::2] times = all_data.flatten()[0::2] frac = hdr['format_specific']['fractional_unit'] if frac > 0: data *= UNIT_CONVERSIONS['g'] * frac logging.debug('Data converted from g * %s to cm/s/s' % (frac)) else: unit = _get_units(lines[11]) if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise ValueError('USC: %s is not a supported unit.' % unit) # Put file name into dictionary head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) trace = StationTrace(data.copy(), Stats(hdr.copy())) if not is_evenly_spaced(times): trace = resample_uneven_trace(trace, times, data) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)
def _prep_data_for_correlation(stream, templates, template_names=None, force_stream_epoch=True): """ Check that all channels are the same length and that all channels have data for both template and stream. Works in place on data - will cut to shortest length :param stream: Stream to compare data to :param templates: List of streams that will be forced to have the same channels as stream :param template_names: List of strings same length as templates :type force_stream_epoch: bool :param force_stream_epoch: Whether to force all channels in stream to cover the same time period :return: stream, templates, template_names (if template_names given) """ n_templates = len(templates) template_samp_rates = { tr.stats.sampling_rate for template in templates for tr in template} stream_samp_rates = {tr.stats.sampling_rate for tr in stream} samp_rates = template_samp_rates.union(stream_samp_rates) assert len(samp_rates) == 1, "Sampling rates differ" samp_rate = samp_rates.pop() out_stream = Stream() named = True if template_names is None: named = False template_names = range(n_templates) # Work out shapes. stream_start = min([tr.stats.starttime for tr in stream]) stream_end = max([tr.stats.endtime for tr in stream]) if force_stream_epoch: stream_length = int(samp_rate * (stream_end - stream_start)) + 1 else: stream_length = max([tr.stats.npts for tr in stream]) template_length = { tr.stats.npts for template in templates for tr in template} assert len(template_length) == 1, "Template traces not all the same length" template_length = template_length.pop() stream_ids = {tr.id for tr in stream} # Need to ensure that a channel can be in the template multiple times. template_ids = {stream_id: [] for stream_id in stream_ids} for template in templates: # Only include those in the stream. channels_in_template = { tr.id for tr in template}.intersection(stream_ids) for channel in channels_in_template: template_ids[channel].append(len(template.select(id=channel))) template_ids = {key: max(value) for key, value in template_ids.items() if len(value) > 0} seed_ids = sorted( [key.split('.') + [i] for key, value in template_ids.items() for i in range(value)]) seed_ids = [('.'.join(seed_id[0:-1]), seed_id[-1]) for seed_id in seed_ids] for channel_number, seed_id in enumerate(template_ids.keys()): stream_data = np.zeros(stream_length, dtype=np.float32) stream_channel = stream.select(id=seed_id) if len(stream_channel) > 1: raise NotImplementedError( "Multiple channels in continuous data for {0}".format(seed_id)) stream_channel = stream_channel[0] if stream_channel.stats.npts == stream_length: stream_data = stream_channel.data else: Logger.info('Data for {0} is not as long as needed, ' 'padding'.format(stream_channel.id)) if force_stream_epoch: start_pad = int(samp_rate * ( stream_channel.stats.starttime - stream_start)) end_pad = stream_length - ( start_pad + stream_channel.stats.npts) # In some cases there will be one sample missing when sampling # time-stamps are not set consistently between channels, this # results in start_pad and end_pad being len==0 if start_pad == 0 and end_pad == 0: Logger.debug("Start and end pad are both zero, padding " "at one end") if (stream_channel.stats.starttime - stream_start) > ( stream_end - stream_channel.stats.endtime): start_pad = int( stream_length - stream_channel.stats.npts) else: end_pad = int( stream_length - stream_channel.stats.npts) stream_channel.stats.starttime -= (start_pad / samp_rate) else: start_pad = 0 end_pad = stream_length - stream_channel.stats.npts if end_pad == 0: stream_data[start_pad:] = stream_channel.data else: stream_data[start_pad:-end_pad] = stream_channel.data header = stream_channel.stats.copy() header.npts = stream_length out_stream += Trace(data=stream_data, header=header) # Initialize nan template for speed. nan_channel = np.full(template_length, np.nan, dtype=np.float32) nan_template = Stream() for _seed_id in seed_ids: net, sta, loc, chan = _seed_id[0].split('.') nan_template += Trace(header=Stats({ 'network': net, 'station': sta, 'location': loc, 'channel': chan, 'starttime': UTCDateTime(), 'npts': template_length, 'sampling_rate': samp_rate})) # Remove templates with no matching channels filt = np.ones(len(template_names)).astype(bool) for i, template in enumerate(templates): template_ids = {tr.id for tr in template} if len(template_ids.intersection(stream_ids)) == 0: filt[i] = 0 _out = dict(zip( [_tn for _tn, _filt in zip(template_names, filt) if _filt], [_t for _t, _filt in zip(templates, filt) if _filt])) if len(_out) != len(templates): Logger.debug("Some templates not used due to no matching channels") # Fill out the templates with nan channels for template_name, template in _out.items(): template_starttime = min([tr.stats.starttime for tr in template]) out_template = nan_template.copy() for channel_number, _seed_id in enumerate(seed_ids): seed_id, channel_index = _seed_id template_channel = template.select(id=seed_id) if len(template_channel) <= channel_index: out_template[channel_number].data = nan_channel out_template[channel_number].stats.starttime = \ template_starttime else: out_template[channel_number] = template_channel[channel_index] _out.update({template_name: out_template}) out_templates = list(_out.values()) out_template_names = list(_out.keys()) if named: return out_stream, out_templates, out_template_names return out_stream, out_templates
def read_knet(filename): """Read Japanese KNET strong motion file. Args: filename (str): Path to possible KNET data file. kwargs (ref): Other arguments will be ignored. Returns: Stream: Obspy Stream containing three channels of acceleration data (cm/s**2). """ logging.debug("Starting read_knet.") if not is_knet(filename): raise Exception('%s is not a valid KNET file' % filename) # Parse the header portion of the file with open(filename, 'rt') as f: lines = [next(f) for x in range(TEXT_HDR_ROWS)] hdr = {} coordinates = {} standard = {} hdr['network'] = 'BO' hdr['station'] = lines[5].split()[2] logging.debug('station: %s' % hdr['station']) standard['station_name'] = '' # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including KNET here, don't provide this. We'll flag it as "--". hdr['location'] = '--' coordinates['latitude'] = float(lines[6].split()[2]) coordinates['longitude'] = float(lines[7].split()[2]) coordinates['elevation'] = float(lines[8].split()[2]) hdr['sampling_rate'] = float( re.search('\\d+', lines[10].split()[2]).group()) hdr['delta'] = 1 / hdr['sampling_rate'] standard['units'] = 'acc' dir_string = lines[12].split()[1].strip() # knet files have directions listed as N-S, E-W, or U-D, # whereas in kiknet those directions are '4', '5', or '6'. if dir_string in ['N-S', '1', '4']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=True) elif dir_string in ['E-W', '2', '5']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=False, is_north=False) elif dir_string in ['U-D', '3', '6']: hdr['channel'] = get_channel_name( hdr['sampling_rate'], is_acceleration=True, is_vertical=True, is_north=False) else: raise Exception('KNET: Could not parse direction %s' % lines[12].split()[1]) logging.debug('channel: %s' % hdr['channel']) scalestr = lines[13].split()[2] parts = scalestr.split('/') num = float(parts[0].replace('(gal)', '')) den = float(parts[1]) calib = num / den hdr['calib'] = calib duration = float(lines[11].split()[2]) hdr['npts'] = int(duration * hdr['sampling_rate']) timestr = ' '.join(lines[9].split()[2:4]) # The K-NET and KiK-Net data logger adds a 15s time delay # this is removed here sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0) # Shift the time to utc (Japanese time is 9 hours ahead) sttime = sttime - timedelta(seconds=9 * 3600.) hdr['starttime'] = sttime # read in the data - there is a max of 8 columns per line # the code below handles the case when last line has # less than 8 columns if hdr['npts'] % COLS_PER_LINE != 0: nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE)) nrows2 = 1 else: nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE)) nrows2 = 0 data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS, max_rows=nrows, filling_values=np.nan) data = data.flatten() if nrows2: skip_header = TEXT_HDR_ROWS + nrows data2 = np.genfromtxt(filename, skip_header=skip_header, max_rows=nrows2, filling_values=np.nan) data = np.hstack((data, data2)) nrows += nrows2 # apply the correction factor we're given in the header data *= calib # fill out the rest of the standard dictionary standard['horizontal_orientation'] = np.nan standard['instrument_period'] = np.nan standard['instrument_damping'] = np.nan standard['process_time'] = '' standard['process_level'] = PROCESS_LEVELS['V1'] standard['sensor_serial_number'] = '' standard['instrument'] = '' standard['comments'] = '' standard['structure_type'] = '' if dir_string in ['1', '2', '3']: standard['structure_type'] = 'borehole' standard['corner_frequency'] = np.nan standard['units'] = 'acc' standard['source'] = SRC standard['source_format'] = 'knet' hdr['coordinates'] = coordinates hdr['standard'] = standard # create a Trace from the data and metadata trace = StationTrace(data.copy(), Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) stream = StationStream(traces=[trace]) return [stream]
def read_at2(dfile, horient=0.0): # This is a conveneince method so we can read in these specific data for # testing, it is not a general purpose reader since this format does not # contain a lot of metadata that is generally required for it to be useful. skiprows = 4 datafile = open(dfile, 'r', encoding='utf-8') datareader = csv.reader(datafile) data = [] header = [] # for i in range(skiprows): # next(datareader) # header.append(datareader.readlines()) count = 0 for row in datareader: if count < skiprows: header.append(row) else: data.extend([float(e) for e in row[0].split()]) count += 1 datafile.close() hdr = {} hdr['network'] = '' hdr['station'] = '' if horient == 0: hdr['channel'] = 'BH1' else: hdr['channel'] = 'BH2' hdr['location'] = '--' dt = float(header[3][1].split('=')[1].strip().lower().replace('sec', '')) hdr['npts'] = len(data) hdr['sampling_rate'] = 1 / dt hdr['duration'] = (hdr['npts'] - 1) / hdr['sampling_rate'] hdr['starttime'] = 0 # There is no lat/lon... hdr['coordinates'] = {'latitude': 0.0, 'longitude': 0.0, 'elevation': 0.0} standard = {} standard['units'] = 'acc' standard['units_type'] = 'acc' standard['horizontal_orientation'] = horient standard['vertical_orientation'] = np.nan standard['source_file'] = dfile standard['station_name'] = '' standard['corner_frequency'] = 30.0 standard['structure_type'] = '' standard['comments'] = '' standard['instrument'] = '' standard['instrument_period'] = 1.0 standard['instrument_sensitivity'] = 1.0 standard['source'] = 'PEER' standard['instrument_damping'] = 0.1 standard['sensor_serial_number'] = '' standard['process_level'] = 'corrected physical units' standard['source_format'] = 'AT2' standard['process_time'] = '' hdr['standard'] = standard # convert data from g to cm/s^2 g_to_cmss = 980.665 tr = StationTrace(np.array(data.copy()) * g_to_cmss, Stats(hdr.copy())) response = {'input_units': 'counts', 'output_units': 'cm/s^2'} tr.setProvenance('remove_response', response) return tr
def synthetic_seismogram(Mw, duration=0.1, sampling_rate=10000, vp=5000.0, vs=3500.0, rho=2400, SSD=1, pwave=True): """ Create a synthetic displacement pulse at the source seismogram based on the brune model (Brune 1970). This model is extensively used and generally agrees with observations from many different setting and over a large range of magnitude. The displacement time function, u(t), is expressed as follows: u(t) = A_0 x t x omega_0 x H(t) * exp(-t x omega_0) , where t the time, omega_0, the angular frequency and H(t) is the heavyside function. Note that the angular frequency is calculated from the static stress drop (SSD). A0 is given by the following equation: A0 = M0 / (4 * pi * rho * v ** 3) References for further reading: - Routine data processing in earthquake seismology - Relating Peak Particle Velocity and Acceleration to Moment Magnitude in Passive (Micro-) Seismic Monitoring (www.bcengineers.com/images/BCE_Technical_Note_3.pdf) :param Mw: the moment magnitude of the seismic event (default: -1), this value determine the wave amplitude and the frequency content :type Mw: float :param noise_level: level of gaussian noise to add to the synthetic seismogram (default: 1e-5) :type noise_level: float :param duration: duration of the seismogram in seconds (default: 0.1), the pulse is centered at zero :type duration: float :param sampling_rate: sampling rate in Hz of the generated time series ( default: 10000) :type sampling_rate: int :param vp: P-wave velocity of the material at the source (default: 5000 m/s) :type vp: float :param vs: S-wave velocity of the material at the source (default: 3500 m/s) :type vs: float :param rho: density of the material at the source in kg/m**3 (default: 2400 kg/m**3) :param SSD: Static stress drop in "bar" (default: 1 bar) :type SSD: float :param pwave: Return P-wave displacement seismogram if True and S-wave displacement seismogram if false :rparam: tuple Obspy Trace containing the seismogram :rtype: Obspy Trace .. note:: The velocity and acceleration can easily be obtained by differentiating the trace using the Obspy Trace method differentiate. Example >>> tr = synthetic_seismogram(-1) >>> tr.differentiate() # this creates a velocity trace >>> tr.differentiate() # this creates an acceleration trace This operation is performed in place on the actual data arrays. The raw data is not accessible anymore afterwards. To keep your original data, use :meth:`~obspy.core.trace.Trace.copy` to create a copy of your trace object. This also makes an entry with information on the applied processing in ``stats.processing`` of this trace. """ M0 = Mw2M0(Mw) (f0p, f0s) = corner_frequency(Mw, vp, vs, SSD) # duration = 5 / f0p npts = duration * sampling_rate t = np.arange(npts) / sampling_rate if pwave: W0 = 2 * np.pi * f0p v = vp else: W0 = 2 * np.pi * f0s v = vs A0 = M0 / (4 * np.pi * rho * v**3) data = A0 * t * W0**2 * np.exp(-t * W0) data = np.roll(data, len(data) / 2) stats = Stats() stats.sampling_rate = sampling_rate stats.npts = 2 * npts - 1 from uquake.core.util.cepstrum import minimum_phase minphase_data = np.roll(minimum_phase(data, len(data)), len(data) / 2) return Trace(data=data, header=stats)
def _read_channel(filename, line_offset, location=''): """Read channel data from COSMOS V1/V2 text file. Args: filename (str): Input COSMOS V1/V2 filename. line_offset (int): Line offset to beginning of channel text block. Returns: tuple: (obspy Trace, int line offset) """ # read station, location, and process level from text header with open(filename, 'rt') as f: for _ in range(line_offset): next(f) lines = [next(f) for x in range(TEXT_HDR_ROWS)] # read in lines of integer data skiprows = line_offset + TEXT_HDR_ROWS int_lines, int_data = _read_lines(skiprows, filename) int_data = int_data.astype(np.int32) # read in lines of float data skiprows += int_lines + 1 flt_lines, flt_data = _read_lines(skiprows, filename) # read in comment lines skiprows += flt_lines + 1 cmt_lines, cmt_data = _read_lines(skiprows, filename) skiprows += cmt_lines + 1 # according to the powers that defined the Network.Station.Channel.Location # "standard", Location is a two character field. Most data providers, # including cosmos here, don't provide this. We'll flag it as "--". hdr = _get_header_info(int_data, flt_data, lines, cmt_data, location=location) head, tail = os.path.split(filename) hdr['standard']['source_file'] = tail or os.path.basename(head) # read in the data nrows, data = _read_lines(skiprows, filename) # check units unit = hdr['format_specific']['physical_units'] if unit in UNIT_CONVERSIONS: data *= UNIT_CONVERSIONS[unit] logging.debug('Data converted from %s to cm/s/s' % (unit)) else: raise GMProcessException('COSMOS: %s is not a supported unit.' % unit) if hdr['standard']['units'] != 'acc': raise GMProcessException('COSMOS: Only acceleration data accepted.') trace = StationTrace(data.copy(), Stats(hdr.copy())) # record that this data has been converted to g, if it has if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']: response = {'input_units': 'counts', 'output_units': 'cm/s^2'} trace.setProvenance('remove_response', response) # set new offset new_offset = skiprows + nrows new_offset += 1 # there is an 'end of record' line after the data return (trace, new_offset)