def _init_header(self, path):
config = _load_config(path)
metadata = config['metadata']
header = Stats()
config_key = self._config['header_sampling_rate_key']
try:
header.sampling_rate = float(metadata[config_key])
except KeyError:
raise KeyError("The following key was not found in the metadata: " +
"{}. Did you set the correct ".format(config_key) +
"'sample rate metadata key' in PAL H5 Output module?")
header.npts = int(metadata[self._config['header_samples_per_record_key']]) - 1
try:
if self._config['dd_300']:
header.calib = metadata['dd_300_calibration']
elif self._config['dd_900']:
header.calib = metadata['dd_900_calibration']
elif self._config['vd_08']:
header.calib = metadata['vd_08_calibration']
elif self._config['vd_09']:
header.calib = metadata['vd_09_calibration']
except KeyError:
pass
header.comments = str(config['comments'])
header.place = metadata
if self._config['header_extra1_name'] != '' and self._config['header_extra1_val'] != '':
header[self._config['header_extra1_name']] = self._config['header_extra1_val']
if self._config['header_extra2_name'] != '' and self._config['header_extra2_val'] != '':
header[self._config['header_extra2_name']] = self._config['header_extra2_val']
return header
def read_cwb(filename, **kwargs):
"""Read Taiwan Central Weather Bureau strong motion file.
Args:
filename (str): Path to possible CWB data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration data (cm/s**2).
"""
if not is_cwb(filename):
raise ValueError('%s is not a valid CWB strong motion data file.')
f = open(filename, 'rt')
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including CWB here, don't provide this. We'll flag it as "--".
data = np.genfromtxt(filename,
skip_header=HDR_ROWS,
delimiter=[COLWIDTH] * NCOLS) # time, Z, NS, EW
hdr = _get_header_info(f, data)
f.close()
hdr_z = hdr.copy()
hdr_z['channel'] = get_channel_name(hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
hdr_z['standard']['horizontal_orientation'] = np.nan
hdr_h1 = hdr.copy()
hdr_h1['channel'] = get_channel_name(hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
hdr_h1['standard']['horizontal_orientation'] = np.nan
hdr_h2 = hdr.copy()
hdr_h2['channel'] = get_channel_name(hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
hdr_h2['standard']['horizontal_orientation'] = np.nan
stats_z = Stats(hdr_z)
stats_h1 = Stats(hdr_h1)
stats_h2 = Stats(hdr_h2)
trace_z = Trace(data=data[:, 1], header=stats_z)
trace_h1 = Trace(data=data[:, 2], header=stats_h1)
trace_h2 = Trace(data=data[:, 3], header=stats_h2)
stream = Stream([trace_z, trace_h1, trace_h2])
return stream
def read_IMS_ASCII(path, net='', **kwargs):
"""
read a IMS_ASCII seismogram from a single station
:param path: path to file
:return: uquake.core.Stream
"""
data = np.loadtxt(path, delimiter=',', skiprows=1)
stats = Stats()
with open(path) as fid:
field = fid.readline().split(',')
stats.sampling_rate = float(field[1])
timetmp = datetime.fromtimestamp(float(field[5])) \
+ timedelta(
seconds=float(field[6]) / 1e6) # trigger time in second
trgtime_UTC = UTCDateTime(timetmp)
stats.starttime = trgtime_UTC - float(field[10]) / stats.sampling_rate
stats.npts = len(data)
stats.station = field[8]
stats.network = net
traces = []
component = np.array(['X', 'Y', 'Z'])
std = np.std(data, axis=0)
mstd = np.max(std)
for k, dt in enumerate(data.T):
stats.channel = '%s' % (component[k])
traces.append(Trace(data=np.array(dt), header=stats))
return Stream(traces=traces)
def convert_to_obspy(signal, fsp):
# type: (np.array, float) -> obspy.core.trace
"""
Function to convert a numpy array from any infrasound reading into Obspy format.
This could help us to take advantage of Obspy optimization routines.
:param signal: A numpy array containg the signal we want to compute.
:param fsp: The sampling frequency of the signal.
:return:
"""
# we substract the mean
stats = Stats()
stats.sampling_rate = float(fsp)
stats.npts = signal.shape[0]
return Trace(data=signal.reshape(signal.shape[0], ), header=stats)
def _init_header(self, path):
config = _load_config(path)
metadata = config['metadata']
header = Stats()
if 'ATS9440' in config['plugins'].keys():
ats_config = config['plugins']['ATS9440']['config']
elif 'ATS660' in config['plugins'].keys():
ats_config = config['plugins']['ATS660']['config']
else:
raise KeyError('Cannot locate trace config data')
if 'Polytec' in config['plugins'].keys():
polytec_config = config['plugins']['Polytec']['config']
else:
raise KeyError('Cannot locate vibrometer config data')
header.sampling_rate = _calc_sampling_rate(ats_config['sample_rate'])
header.npts = int(ats_config['pre_trigger_samples'] +
ats_config['post_trigger_samples']) - 1
if polytec_config['dd_300']:
header.calib = float(
re.findall(_NUMBER, polytec_config['dd_300_range'])[0])
elif polytec_config['dd_900']:
header.calib = float(
re.findall(_NUMBER, polytec_config['dd_900_range'])[0])
elif polytec_config['vd_08']:
header.calib = float(
re.findall(_NUMBER, polytec_config['vd_08_range'])[0])
elif polytec_config['vd_09']:
header.calib = float(
re.findall(_NUMBER, polytec_config['vd_09_range'])[0])
else:
raise KeyError('Cannot locate vibrometer calibration data')
header.comments = str(config['comments'])
header.place = metadata
if self._config['header_extra1_name'] != '' and self._config[
'header_extra1_val'] != '':
header[self._config['header_extra1_name']] = self._config[
'header_extra1_val']
if self._config['header_extra2_name'] != '' and self._config[
'header_extra2_val'] != '':
header[self._config['header_extra2_name']] = self._config[
'header_extra2_val']
return header
def get_Stream(self):
"""
Return an obspy stream
"""
stas = [
'RMNW.GC01..CHZ', 'RMNW.GC01..CHX', 'RMNW.GC01..CHY',
'RMNW.GC02..CHZ', 'RMNW.GC02..CHX', 'RMNW.GC02..CHY',
'RMNW.GC03..CHZ', 'RMNW.GC03..CHX', 'RMNW.GC03..CHY',
'RMNW.GC04..CHZ', 'RMNW.GC04..CHX', 'RMNW.GC04..CHY'
]
st = Stream()
delta = self.sample_period / 1e6
time_v, data_mat = self.get_all_data()
for col in range(data_mat.shape[1]):
seed = stas[col].split('.')
tr = Trace(data=data_mat[:, col],
header=Stats(
dict(starttime=time_v[0],
npts=data_mat.shape[0],
delta=delta,
network=seed[0],
station=seed[1],
location=seed[2],
channel=seed[3])))
st.traces.append(tr)
return st
def readgeonet(geonetfile):
"""
Read strong motion data from a GeoNet data file
@param geonetfile: Path to a valid GeoNet data file.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
f = open(geonetfile, 'rt')
tracelist = []
headerlist = []
try:
hdrlines = readheaderlines(f)
except:
pass
while len(hdrlines[-1]):
hdrdict = readheader(hdrlines)
numlines = int(np.ceil(hdrdict['npts'] / 10.0))
data = []
for i in range(0, numlines):
line = f.readline()
parts = line.strip().split()
mdata = [float(p) for p in parts]
data = data + mdata
data = np.array(data)
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data, header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats[
'calib'] * 0.001 #convert to m/s^2
tracelist.append(trace.copy())
headerlist.append(header.copy())
hdrlines = readheaderlines(f)
f.close()
return (tracelist, headerlist)
def ReadStnPara(self):
"""
读取redis中的台站参数信息，台站信息存入一维数组station_list，通道信息存入二维数组channel_list
"""
stn_para_key = 'StnPara'
self.station_number = int(self.r.hget(stn_para_key, "station_number"))
self.trace_list = [[Trace() for col in range(3)]
for row in range(self.station_number)]
print('++++')
print(type(self.trace_list[0][0]))
self.channel_list = [[0 for col in range(3)]
for row in range(self.station_number)]
self.station_list = [0 for col in range(self.station_number)]
# multilist = [[0 for col in range(5)] for row in range(3)] #二维数组
coordinates = AttribDict()
channel_name = [0 for col in range(3)]
for key in ['latitude', 'longitude', 'elevation']:
coordinates[key] = 0
chnNum = 0
for key in ['Z', 'E', 'N']:
channel_name[chnNum] = key
chnNum = chnNum + 1
for staNo in range(0, self.station_number):
stn_para_field = '{0:0>4}'.format(staNo)
stn_para_res = self.r.hget(stn_para_key, stn_para_field)
stn_par = stn_para_res.decode('utf-8').split()
# print(str(staNo) + str(stn_par))
stn_para_defaults = AttribDict()
stn_para_defaults['coordinates'] = AttribDict()
stn_para_defaults['coordinates'] = coordinates
stn_para_defaults['network'] = stn_par[1]
stn_para_defaults['station'] = stn_par[2]
stn_para_defaults['channel'] = stn_par[4]
stn_para_defaults['location'] = stn_par[5]
stn_para_defaults['latitude'] = float(stn_par[6])
stn_para_defaults['longitude'] = float(stn_par[7])
stn_para_defaults['elevation'] = float(stn_par[8])
stn_para_defaults['channelNum'] = int(stn_par[9])
stn_para_defaults['sampling_rate'] = int(stn_par[10])
self.station_list[staNo] = stn_para_defaults
for chnNo in range(0, self.station_list[staNo]['channelNum']):
chn_para_defaults = Stats(AttribDict())
chn_para_defaults['sampling_rate'] = self.station_list[staNo][
'sampling_rate']
chn_para_defaults['delta'] = 1.0
chn_para_defaults['calib'] = 1.0
chn_para_defaults['starttime'] = UTCDateTime(0)
chn_para_defaults['npts'] = 0
chn_para_defaults['network'] = stn_para_defaults['network']
chn_para_defaults['station'] = stn_para_defaults['station']
chn_para_defaults['channel'] = stn_para_defaults[
'channel'] + channel_name[chnNo]
chn_para_defaults['location'] = stn_para_defaults['location']
chn_para_defaults['response'] = float(stn_par[11 + chnNo])
self.channel_list[staNo][chnNo] = chn_para_defaults
def readiran(iranfile, doRotation=True):
"""
Read strong motion data from a Iran data file
@param iranfile: Path to a valid Iran data file.
@keyword doRotation: Apply back-azimuth rotation of L & T channels to NS and EW.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
f = open(iranfile, 'rt')
tracelist = []
headerlist = []
try:
hdrlines = readheaderlines(f)
except:
pass
while len(hdrlines[-1]):
hdrdict = readheader(hdrlines)
numlines = int(np.ceil(hdrdict['npts'] / 10.0))
data = []
for i in range(0, numlines):
line = f.readline()
parts = line.strip().split()
mdata = [float(p) for p in parts]
data = data + mdata
data = np.array(data)
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data, header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats[
'calib'] * 0.98 #convert to m/s^2 from g/10
tracelist.append(trace.copy())
headerlist.append(header.copy())
endblock = f.readline()
hdrlines = readheaderlines(f)
f.close()
#data from Iran may be rotated so that one channel is aligned in the direction between the earthquake
#epicenter and the station. We want to rotate the data back so that we have what are presumably the
#original NS and EW channels. We presume that the "L" channel will rotate back to become NS, and
#"T" will become EW.
#First, find the channel called L*
if doRotation:
channels = [h['channel'][0:1] for h in headerlist]
lidx = channels.index('L')
tidx = channels.index('T')
ldata = tracelist[lidx].data
tdata = tracelist[tidx].data
backaz = headerlist[0]['rotation']['L']
ndata, edata = rotate.rotate_RT_NE(ldata, tdata, backaz)
tracelist[lidx].data = ndata.copy()
tracelist[lidx].stats[
'channel'] = 'H1' #most probably NS, but we're being cautious
tracelist[tidx].data = edata.copy()
tracelist[tidx].stats[
'channel'] = 'H2' #most probably EW, but we're being cautious
return (tracelist, headerlist)
def _remove_processing(self, st):
""" copy stream and remove processing"""
from obspy.core.trace import Stats
st = st.copy()
for tr in st:
tr.stats = Stats({x: tr.stats[x] for x in Stats.defaults})
# tr.stats.pop("processing", None)
return st
def _init_header(self, path):
config = _load_config(path)
metadata = config['metadata']
header = Stats()
if 'ATS9440' in config['plugins'].keys():
ats_config = config['plugins']['ATS9440']['config']
elif 'ATS660' in config['plugins'].keys():
ats_config = config['plugins']['ATS660']['config']
else:
raise KeyError('Cannot locate trace config data')
if 'Polytec' in config['plugins'].keys():
polytec_config = config['plugins']['Polytec']['config']
else:
raise KeyError('Cannot locate vibrometer config data')
header.sampling_rate = _calc_sampling_rate(ats_config['sample_rate'])
header.npts = int(ats_config['pre_trigger_samples'] +
ats_config['post_trigger_samples']) - 1
if polytec_config['dd_300']:
header.calib = float(re.findall(
_NUMBER, polytec_config['dd_300_range'])[0])
elif polytec_config['dd_900']:
header.calib = float(re.findall(
_NUMBER, polytec_config['dd_900_range'])[0])
elif polytec_config['vd_08']:
header.calib = float(re.findall(
_NUMBER, polytec_config['vd_08_range'])[0])
elif polytec_config['vd_09']:
header.calib = float(re.findall(
_NUMBER, polytec_config['vd_09_range'])[0])
else:
raise KeyError('Cannot locate vibrometer calibration data')
header.comments = str(config['comments'])
header.place = metadata
if self._config['header_extra1_name'] != '' and self._config['header_extra1_val'] != '':
header[self._config['header_extra1_name']
] = self._config['header_extra1_val']
if self._config['header_extra2_name'] != '' and self._config['header_extra2_val'] != '':
header[self._config['header_extra2_name']
] = self._config['header_extra2_val']
return header
def test_imfv122_filename():
'''
Test generating IAGA2002 filename
'''
stats = Stats(
header={
'network': 'C2',
'station': 'OTT',
'location': 'R0',
'channel': 'UFX',
'starttime': UTCDateTime(2020, 1, 10),
'detla': 60
})
assert pygeomag.data.formats.imfv122.get_filename(stats) == 'JAN1020.OTT'
def _read_channel(filename, line_offset, volume, location=''):
"""Read channel data from USC V1 text file.
Args:
filename (str): Input USC V1 filename.
line_offset (int): Line offset to beginning of channel text block.
volume (dictionary): Dictionary of formatting information
Returns:
tuple: (obspy Trace, int line offset)
"""
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skiprows = line_offset + volume['TEXT_HDR_ROWS']
int_data = np.genfromtxt(filename,
skip_header=skiprows,
max_rows=volume['INT_HDR_ROWS'],
dtype=np.int32,
delimiter=volume['INT_FMT']).flatten()
# read in lines of float data
skiprows += volume['INT_HDR_ROWS'] + 1
flt_data = np.genfromtxt(filename,
skip_header=skiprows,
max_rows=volume['FLT_HDR_ROWS'],
dtype=np.float64,
delimiter=volume['FLT_FMT']).flatten()
hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
skiprows += volume['FLT_HDR_ROWS']
# read in the data
nrows = int(np.floor(hdr['npts'] * 2 / 10))
data = np.genfromtxt(filename,
skip_header=skiprows,
max_rows=nrows,
dtype=np.float64,
delimiter=volume['COL_FMT']).flatten()[1::2]
trace = Trace(data.copy(), Stats(hdr.copy()))
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def _read_channel(filename, line_offset, location=''):
"""Read channel data from COSMOS V1/V2 text file.
Args:
filename (str): Input COSMOS V1/V2 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station, location, and process level from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# read in lines of integer data
skiprows = line_offset + TEXT_HDR_ROWS
int_lines, int_data = _read_lines(skiprows, filename)
int_data = int_data.astype(np.int32)
# read in lines of float data
skiprows += int_lines + 1
flt_lines, flt_data = _read_lines(skiprows, filename)
# read in comment lines
skiprows += flt_lines + 1
cmt_lines, cmt_data = _read_lines(skiprows, filename)
skiprows += cmt_lines + 1
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including cosmos here, don't provide this. We'll flag it as "--".
hdr = _get_header_info(int_data,
flt_data,
lines,
cmt_data,
location=location)
# read in the data
nrows, data = _read_lines(skiprows, filename)
trace = Trace(data.copy(), Stats(hdr.copy()))
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def readknet(knetfilename):
"""
Read a KNet ASCII file, and return an ObsPy Trace object, plus a dictionary of header values.
@param knetfilename: String path to valid KNet ASCII file, as described here: http://www.kyoshin.bosai.go.jp/kyoshin/man/knetform_en.html
@return: ObsPy Trace object, and a dictionary of some of the header values found in the input file.
"""
data = []
hdrdict = {}
f = open(knetfilename,'rt')
dataOn = False
headerlines = []
for line in f.readlines():
if line.startswith('Memo'):
hdrdict = readheader(headerlines)
dataOn = True
continue
if not dataOn:
headerlines.append(line)
continue
if dataOn:
parts = line.strip().split()
mdata = [float(p) for p in parts]
data = data + mdata
f.close()
#fill in the values usually expected in Stats as best we can
hdrdict['npts'] = len(data)
elapsed = float(hdrdict['npts'])/float(hdrdict['sampling_rate'])
hdrdict['endtime'] = hdrdict['starttime'] + elapsed
hdrdict['network'] = 'NIED'
hdrdict['location'] = ''
#The Stats constructor appears to modify the fields in the input dictionary - let's save
#a copy
header = hdrdict.copy()
data = np.array(data)
stats = Stats(hdrdict)
trace = Trace(data,header=stats)
#apply the calibration and convert to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.01 #convert to m/s^2
return (trace,header)
def _internal_read_knet_ascii(buf, **kwargs):
"""
Reads a K-NET/KiK-net ASCII file and returns an ObsPy Stream object.
.. warning::
This function should NOT be called directly, it registers via the
ObsPy :func:`~obspy.core.stream.read` function, call this instead.
:param buf: File to read.
:type buf: Open file or open file like object.
"""
data = []
hdrdict = {}
cur_pos = buf.tell()
buf.seek(0, 2)
size = buf.tell()
buf.seek(cur_pos, 0)
# First read the headerlines
headerlines = []
while buf.tell() < size:
line = buf.readline().decode()
headerlines.append(line)
if line.startswith('Memo'):
hdrdict = _read_knet_hdr(headerlines, **kwargs)
break
while buf.tell() < size:
line = buf.readline()
parts = line.strip().split()
data += [float(p) for p in parts]
hdrdict['npts'] = len(data)
# The FDSN network code for the National Research Institute for Earth
# Science and Disaster Prevention (NEID JAPAN) is BO (Bosai-Ken Network)
hdrdict['network'] = 'BO'
data = np.array(data)
stats = Stats(hdrdict)
trace = Trace(data, header=stats)
return Stream([trace])
def readgeonet(geonetfile):
"""
Read strong motion data from a GeoNet data file
@param geonetfile: Path to a valid GeoNet data file.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
#notes on implementation:
# originally I had written code to read each line of data manually, just
# as I was reading the header lines. However, this became VERY slow for large
# files. I discovered that numpy's genfromtxt function was much faster. However,
# I could not get that function to work when I passed it a file object instead of a file name.
# Consequently, the only way I could keep the genfromtxt() method and file pointer in sync was
# to read and discard all of the lines of data that genfromtxt() parsed. While annoying, the combination
# of these two seems to still be at least an order of magnitude faster than manually reading the file.
f = open(geonetfile,'rt')
tracelist = []
headerlist = []
totlines = 0
hdrlines = readheaderlines(f)
while len(hdrlines[-1]):
totlines += len(hdrlines)
hdrdict = readheader(hdrlines)
numlines = int(np.ceil(hdrdict['npts']/10.0))
data = np.genfromtxt(geonetfile,skip_header=totlines,max_rows=numlines)
totlines += numlines
#now we need to set the file position to where we just ended
for i in range(0,numlines):
f.readline()
data = data.flatten()
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data,header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.001 #convert to m/s^2
tracelist.append(trace.copy())
headerlist.append(header.copy())
hdrlines = readheaderlines(f)
f.close()
return (tracelist,headerlist)
def readitaly(datafile):
f = open(datafile, 'rt')
#header needs: station,channel,location,npts,starttime,sampling_rate,delta,calib,lat,lon,height,duration,endtime,maxacc,network
data = []
hdrdict = {}
for line in f.readlines():
if not len(line.strip()):
continue
if not line.find(':') > -1:
data.append(float(line.strip()))
continue
key, value = line.split(':')
key = key.strip()
value = value.strip()
if key not in list(HEADERS.keys()):
continue
hdrkey = HEADERS[key]
if hdrkey == 'starttime':
value = UTCDateTime(datetime.datetime.strptime(value, TIMEFMT))
elif hdrkey not in ['station', 'channel', 'location', 'network']:
value = float(value)
hdrdict[hdrkey] = value
f.close()
hdrdict['sampling_rate'] = 1 / hdrdict['delta']
hdrdict['endtime'] = hdrdict['starttime'] + hdrdict['duration']
hdrdict['npts'] = int(hdrdict['npts'])
hdrdict['calib'] = 1.0
hdrdict['units'] = 'acc'
data = np.array(data)
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data, header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.01 #convert to m/s^2
return trace
def to_sac_and_mseed(self, export_path, station_number, force_without_loc):
# Check if file exist
export_path_sac = export_path + self.get_export_file_name() + ".sac"
export_path_msd = export_path + self.get_export_file_name() + ".mseed"
#export_path_wav = export_path + self.get_export_file_name() + ".wav"
if os.path.exists(export_path_sac) and os.path.exists(export_path_msd):
return
# Check if the station location have been calculated
if self.station_loc is None and not force_without_loc:
print self.get_export_file_name() + ": Skip sac/mseed generation, wait the next ascent to compute location"
return
# Fill header info
stats = Stats()
stats.sampling_rate = self.decimated_fs
stats.network = "MH"
stats.station = station_number
stats.starttime = self.date
stats.sac = dict()
if not force_without_loc:
stats.sac["stla"] = self.station_loc.latitude
stats.sac["stlo"] = self.station_loc.longitude
stats.sac["stdp"] = self.depth
stats.sac["user0"] = self.snr
stats.sac["user1"] = self.criterion
stats.sac["iztype"] = 9 # 9 == IB in sac format
# Save data into a Stream object
trace = Trace()
trace.stats = stats
trace.data = self.data
stream = Stream(traces=[trace])
# Save stream object
print export_path_sac
stream.write(export_path_sac, format='SAC')
print export_path_msd
stream.write(export_path_msd, format='MSEED')
def read_esm(filename, config=None, **kwargs):
"""Read European ESM strong motion file.
Args:
filename (str):
Path to possible ESM data file.
config (dict):
Dictionary containing configuration.
kwargs (ref):
Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing one channels of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_esm.")
if not is_esm(filename, config):
raise Exception(f"{filename} is not a valid ESM file")
# Parse the header portion of the file
header = {}
with open(filename, "rt") as f:
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
for line in lines:
parts = line.split(":")
key = parts[0].strip()
value = ":".join(parts[1:]).strip()
header[key] = value
stats = {}
standard = {}
coordinates = {}
# fill in all known stats header fields
stats["network"] = header["NETWORK"]
stats["station"] = header["STATION_CODE"]
stats["channel"] = header["STREAM"]
stats["location"] = "--"
stats["delta"] = float(header["SAMPLING_INTERVAL_S"])
stats["sampling_rate"] = 1 / stats["delta"]
stats["calib"] = 1.0
stats["npts"] = int(header["NDATA"])
stimestr = header["DATE_TIME_FIRST_SAMPLE_YYYYMMDD_HHMMSS"]
stats["starttime"] = datetime.strptime(stimestr, TIMEFMT)
# fill in standard fields
head, tail = os.path.split(filename)
standard["source_file"] = tail or os.path.basename(head)
standard["source"] = SRC
standard["source_format"] = FORMAT
standard["horizontal_orientation"] = np.nan
standard["vertical_orientation"] = np.nan
standard["station_name"] = header["STATION_NAME"]
try:
standard["instrument_period"] = 1 / float(
header["INSTRUMENTAL_FREQUENCY_HZ"])
except ValueError:
standard["instrument_period"] = np.nan
try:
standard["instrument_damping"] = 1 / float(
header["INSTRUMENTAL_DAMPING"])
except ValueError:
standard["instrument_damping"] = np.nan
ptimestr = header["DATA_TIMESTAMP_YYYYMMDD_HHMMSS"]
ptime = datetime.strptime(ptimestr, TIMEFMT).strftime(TIMEFMT2)
standard["process_time"] = ptime
standard["process_level"] = PROCESS_LEVELS["V1"]
instr_str = header["INSTRUMENT"]
parts = instr_str.split("|")
sensor_str = parts[0].split("=")[1].strip()
standard["sensor_serial_number"] = ""
standard["instrument"] = sensor_str
standard["comments"] = ""
standard["structure_type"] = ""
standard["units"] = "cm/s^2"
standard["units_type"] = "acc"
standard["instrument_sensitivity"] = np.nan
standard["corner_frequency"] = np.nan
coordinates["latitude"] = float(header["STATION_LATITUDE_DEGREE"])
coordinates["longitude"] = float(header["STATION_LONGITUDE_DEGREE"])
coordinates["elevation"] = float(header["STATION_ELEVATION_M"])
# read in the data
data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS)
# create a Trace from the data and metadata
stats["standard"] = standard
stats["coordinates"] = coordinates
trace = StationTrace(data.copy(), Stats(stats.copy()))
response = {"input_units": "counts", "output_units": "cm/s^2"}
trace.setProvenance("remove_response", response)
ftype = header["FILTER_TYPE"].capitalize()
try:
forder = int(header["FILTER_ORDER"])
except ValueError:
forder = 0
try:
lowfreq = float(header["LOW_CUT_FREQUENCY_HZ"])
except ValueError:
lowfreq = np.nan
try:
highfreq = float(header["LOW_CUT_FREQUENCY_HZ"])
except ValueError:
highfreq = np.nan
if not np.isnan(lowfreq) and not np.isnan(lowfreq):
filter_att = {
"bandpass_filter": {
"filter_type": ftype,
"lower_corner_frequency": lowfreq,
"higher_corner_frequency": highfreq,
"filter_order": forder,
}
}
trace.setProvenance("lowpass_filter", filter_att)
detrend_att = {"detrend": {"detrending_method": "baseline"}}
if "NOT REMOVED" not in header["BASELINE_CORRECTION"]:
trace.setProvenance("detrend", detrend_att)
stream = StationStream(traces=[trace])
return [stream]
def _read_volume_two(filename, line_offset, location='', units='acc'):
"""Read channel data from DMG text file.
Args:
filename (str):
Input DMG V2 filename.
line_offset (int):
Line offset to beginning of channel text block.
units (str):
Units to get.
Returns:
tuple: (list of obspy Trace, int line offset)
"""
try:
with open(filename, 'rt', encoding='utf-8') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(V2_TEXT_HDR_ROWS)]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skip_rows = V2_TEXT_HDR_ROWS + line_offset
int_data = _read_lines(skip_rows, V2_INT_HDR_ROWS, V2_INT_FMT, filename)
int_data = int_data[0:100].astype(np.int32)
# read in lines of float data
skip_rows += V2_INT_HDR_ROWS
flt_data = _read_lines(skip_rows, V2_REAL_HDR_ROWS, V2_REAL_FMT, filename)
flt_data = flt_data[:100]
skip_rows += V2_REAL_HDR_ROWS
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including csmip/dmg here, don't always provide this. We'll flag it as
# "--".
hdr = _get_header_info(int_data, flt_data, lines, 'V2', location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
traces = []
# read acceleration data
if hdr['npts'] > 0:
acc_rows, acc_fmt, unit = _get_data_format(
filename, skip_rows, hdr['npts'])
acc_data = _read_lines(skip_rows + 1, acc_rows, acc_fmt, filename)
acc_data = acc_data[:hdr['npts']]
if unit in UNIT_CONVERSIONS:
acc_data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise ValueError('DMG: %s is not a supported unit.' % unit)
acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
acc_trace.setProvenance('remove_response', response)
if units == 'acc':
traces += [acc_trace]
skip_rows += int(acc_rows) + 1
# -------------------------------------------------------------------------
# NOTE: The way we were initially reading velocity and displacement data
# was not correct. I'm deleting it for now since we don't need it. If/when
# we revisit this we need to be more careful about how this is handled.
# -------------------------------------------------------------------------
# read velocity data
vel_hdr = hdr.copy()
vel_hdr['standard']['units'] = 'vel'
vel_hdr['npts'] = int_data[63]
if vel_hdr['npts'] > 0:
vel_rows, vel_fmt, unit = _get_data_format(
filename, skip_rows, vel_hdr['npts'])
vel_data = _read_lines(skip_rows + 1, vel_rows, vel_fmt, filename)
vel_data = vel_data[:vel_hdr['npts']]
skip_rows += int(vel_rows) + 1
# read displacement data
disp_hdr = hdr.copy()
disp_hdr['standard']['units'] = 'disp'
disp_hdr['npts'] = int_data[65]
if disp_hdr['npts'] > 0:
disp_rows, disp_fmt, unit = _get_data_format(
filename, skip_rows, disp_hdr['npts'])
disp_data = _read_lines(skip_rows + 1, disp_rows, disp_fmt, filename)
disp_data = disp_data[:disp_hdr['npts']]
skip_rows += int(disp_rows) + 1
# there is an 'end of record' line after the data]
new_offset = skip_rows + 1
return (traces, new_offset)
def _read_volume(filename, line_offset):
"""Read channel data from text file.
Args:
filename (str): Input filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (list of obspy Trace, int line offset)
"""
# read station, location, and process level from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(V2_TEXT_HDR_ROWS)]
# parse out the station name, location, and process level
hdr = {}
# check that filename matches network and station
station = lines[5][12:17].replace(' ', '')
hdr['station'] = station
hdr['process_level'] = 'V2'
# read in lines of integer data
skip_rows = V2_TEXT_HDR_ROWS + line_offset
int_data = _read_lines(skip_rows, V2_INT_HDR_ROWS, V2_INT_FMT, filename)
int_data = int_data[0:100].astype(np.int32)
# read in lines of float data
skip_rows += V2_INT_HDR_ROWS
flt_data = _read_lines(skip_rows, V2_REAL_HDR_ROWS, V2_REAL_FMT, filename)
flt_data = flt_data[:100]
skip_rows += V2_REAL_HDR_ROWS
# Parse name and code information
name_length = int_data[29]
name = re.sub(' +', ' ', lines[6][:name_length]).strip().replace(' ', '_')
hdr['name'] = name
hdr['network'] = 'Unknown'
# set statistics
hdr['units'] = 'acc'
lat = lines[5][21:27].replace(' ', '')
if lat[-1].upper() == 'S':
lat = -1 * float(lat[0:-1])
lon = lines[5][30:37].replace(' ', '')
if lon[-1].upper() == 'W':
lon = -1 * float(lon[0:-1])
hdr['lat'] = lat
hdr['lon'] = lon
hdr['location'] = '--'
hdr['delta'] = flt_data[60]
hdr['sampling_rate'] = 1 / hdr['delta']
hdr['npts'] = int_data[52]
hdr['source'] = hdr['network']
angle = int_data[26]
if angle == 500 or angle == 600:
hdr['channel'] = 'Z'
elif angle > 315 or angle < 45 or (angle > 135 and angle < 225):
hdr['channel'] = 'H1'
else:
hdr['channel'] = 'H2'
traces = []
# read acceleration data
if hdr['npts'] > 0:
acc_rows, acc_fmt = _get_data_format(filename, skip_rows, hdr['npts'])
acc_data = _read_lines(skip_rows + 1, acc_rows, acc_fmt, filename)
acc_data = acc_data[:hdr['npts']]
acc_trace = Trace(acc_data.copy(), Stats(hdr.copy()))
traces += [acc_trace]
skip_rows += int(acc_rows) + 1
# read acceleration data
vel_hdr = hdr.copy()
vel_hdr['units'] = 'vel'
vel_hdr['npts'] = hdr['npts']
if vel_hdr['npts'] > 0:
vel_rows, vel_fmt = _get_data_format(filename, skip_rows,
vel_hdr['npts'])
vel_data = _read_lines(skip_rows + 1, vel_rows, vel_fmt, filename)
vel_data = vel_data[:vel_hdr['npts']]
vel_trace = Trace(vel_data.copy(), Stats(vel_hdr.copy()))
traces += [vel_trace]
skip_rows += int(vel_rows) + 1
# read displacement data
disp_hdr = hdr.copy()
disp_hdr['units'] = 'disp'
disp_hdr['npts'] = hdr['npts']
if disp_hdr['npts'] > 0:
disp_rows, disp_fmt = _get_data_format(filename, skip_rows,
disp_hdr['npts'])
disp_data = _read_lines(skip_rows + 1, disp_rows, disp_fmt, filename)
disp_data = disp_data[:disp_hdr['npts']]
disp_trace = Trace(disp_data.copy(), Stats(disp_hdr.copy()))
traces += [disp_trace]
skip_rows += int(disp_rows) + 1
new_offset = skip_rows + 1 # there is an 'end of record' line after the data]
return (traces, new_offset)
def dorange (self):
# load batches
print "mkms: loading batches.."
self.bdatas = []
for i in self.ids:
d = Dat ()
d.read (os.path.join (self.root, str(i) + '.DAT'))
self.bdatas.append (d.bdata)
# set up datastream
print "mkms: setting up stream for %s.." % self.station,
self.st = Stream ()
for bd in self.bdatas:
for b in bd.batches:
s = Stats ()
s.sampling_rate = self.sampling_rate
s.npts = b.length
s.network = self.network
s.location = self.location
s.station = self.station
s.channel = self.channel
s.starttime = UTCDateTime ((b.ref / 1000000.0))
t = Trace (data = numpy.array (b.samples_i, dtype = numpy.int32), header = s)
self.st.append (t)
print "done."
# generate file name
self.name = self.st[0].id.replace ('.', '_')
self.start = self.st[0].stats.starttime
self.name = self.start.strftime ("%Y-%m-%d-%H%M-%S") + '.' + self.name
if self.optplot:
self.plot ()
if not self.optnowrite:
print "mkms: writing %s.mseed.." % self.name,
if not os.path.exists (self.destdir):
os.makedirs (self.destdir)
self.st.write (os.path.join (self.destdir, self.name + '.mseed'), format = 'MSEED', encoding = 'INT32', byteorder = 1, flush = 1, verbose = 0)
print "done."
# write ids and refs
idsf = open (os.path.join (self.destdir, self.name + '.ids'), 'w')
refsf = open (os.path.join (self.destdir, self.name + '.refs'), 'w')
for bd in self.bdatas:
idsf.write ("%d,%d\n" % (bd.id, 1 if bd.e_sdlag else 0))
for b in bd.batches:
refsf.write ("%d,%d,%d,%d,%s,%s,%s,%s,%s,%d\n" % (bd.id, b.no, b.ref, b.status, b.latitude[:-2], b.latitude[-2:], b.longitude[:-2], b.longitude[-2:], b.checksum, 1 if b.checksum_pass else 0))
idsf.close ()
refsf.close ()
return (self.name + '.mseed', idsf, refsf)
else:
print "mkms: would write %s.mseed (disabled)." % os.path.join (self.destdir, self.name)
return None
def _read_volume_one(filename, line_offset, location='', units='acc'):
"""Read channel data from DMG Volume 1 text file.
Args:
filename (str):
Input DMG V1 filename.
line_offset (int):
Line offset to beginning of channel text block.
units (str):
units to get.
Returns:
tuple: (list of obspy Trace, int line offset)
"""
# Parse the header portion of the file
try:
with open(filename, 'rt', encoding='utf-8') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(V1_TEXT_HDR_ROWS)]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
unit = _get_units(lines[11])
# read in lines of integer data
skip_rows = V1_TEXT_HDR_ROWS + line_offset
int_data = _read_lines(skip_rows, V1_INT_HDR_ROWS, V2_INT_FMT, filename)
int_data = int_data[0:100].astype(np.int32)
# read in lines of float data
skip_rows += V1_INT_HDR_ROWS
flt_data = _read_lines(skip_rows, V1_REAL_HDR_ROWS, V2_REAL_FMT, filename)
skip_rows += V1_REAL_HDR_ROWS
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including csmip/dmg here, don't always provide this. We'll flag it as
# "--".
hdr = _get_header_info_v1(
int_data, flt_data, lines, 'V1', location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# sometimes (??) a line of text is inserted in between the float header and
# the beginning of the data. Let's check for this...
with open(filename, 'rt', encoding='utf-8') as f:
for _ in range(skip_rows):
next(f)
test_line = f.readline()
has_text = re.search('[A-Z]+|[a-z]+', test_line) is not None
if has_text:
skip_rows += 1
widths = [9] * 8
max_rows = int(np.ceil(hdr['npts'] / 8))
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[:hdr['npts']]
evenly_spaced = True
# Sometimes, npts is incrrectly specified, leading to nans
# in the resulting data. Fix that here
if np.any(np.isnan(acc_data)):
while np.isnan(acc_data[-1]):
acc_data = acc_data[:-1]
hdr['npts'] = len(acc_data)
else:
# acceleration data is interleaved between time data
max_rows = int(np.ceil(hdr['npts'] / 5))
widths = [7] * 10
data = _read_lines(skip_rows, max_rows, widths, filename)
acc_data = data[1::2][:hdr['npts']]
times = data[0::2][:hdr['npts']]
evenly_spaced = is_evenly_spaced(times)
if unit in UNIT_CONVERSIONS:
acc_data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise ValueError('DMG: %s is not a supported unit.' % unit)
acc_trace = StationTrace(acc_data.copy(), Stats(hdr.copy()))
# Check if the times were included in the file but were not evenly spaced
if not evenly_spaced:
acc_trace = resample_uneven_trace(acc_trace, times, acc_data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
acc_trace.setProvenance('remove_response', response)
traces = [acc_trace]
new_offset = skip_rows + max_rows + 1 # there is an end of record line
return (traces, new_offset)
def _read_channel(filename, line_offset):
"""Read channel data from GNS V1 text file.
Args:
filename (str):
Input GNS V1 filename.
line_offset (int):
Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station and location strings from text header
with open(filename, 'rt', encoding='utf-8') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# this code supports V1 and V2 format files. Which one is this?
data_format = 'V2'
if lines[0].lower().find('uncorrected') >= 0:
data_format = 'V1'
# parse out the station code, name, and component string
# from text header
station = lines[1].split()[1]
logging.debug('station: %s' % station)
name = lines[2].replace(' ', '_').strip()
component = lines[12].split()[1]
# parse the instrument type from the text header
instrument = lines[3].split()[1]
# parse the sensor resolution from the text header
resolution_str = lines[4].split()[1]
resolution = int(re.search(r"\d+", resolution_str).group())
# read floating point header array
skip_header = line_offset + TEXT_HDR_ROWS
hdr_data = np.genfromtxt(filename, skip_header=skip_header,
max_rows=FP_HDR_ROWS)
# parse header dictionary from float header array
hdr = _read_header(hdr_data, station, name,
component, data_format,
instrument, resolution)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including GeoNet here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
skip_header2 = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS
widths = [8] * COLS_PER_ROW
nrows = int(np.ceil(hdr['npts'] / COLS_PER_ROW))
data = np.genfromtxt(filename, skip_header=skip_header2,
max_rows=nrows, filling_values=np.nan,
delimiter=widths)
data = data.flatten()
data = data[0:hdr['npts']]
# for debugging, read in the velocity data
nvel = hdr_data[3, 4]
if nvel:
if nvel % COLS_PER_ROW != 0:
nvel_rows = int(np.floor(nvel / COLS_PER_ROW))
else:
nvel_rows = int(np.ceil(nvel / COLS_PER_ROW))
skip_header_vel = line_offset + TEXT_HDR_ROWS + FP_HDR_ROWS + nrows
widths = [8] * COLS_PER_ROW
velocity = np.genfromtxt(filename, skip_header=skip_header_vel,
max_rows=nvel_rows, filling_values=np.nan,
delimiter=widths)
velocity = velocity.flatten()
velocity *= MMPS_TO_CMPS
else:
velocity = np.array([])
# for V2 files, there are extra blocks of data we need to skip containing
# velocity and displacement data
if data_format == 'V2':
velrows = int(np.ceil(hdr_data[3, 4] / COLS_PER_ROW))
disrows = int(np.ceil(hdr_data[3, 5] / COLS_PER_ROW))
nrows = nrows + velrows + disrows
data *= MMPS_TO_CMPS # convert to cm/s**2
trace = StationTrace(data, Stats(hdr))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
offset = skip_header2 + nrows
return (trace, offset, velocity)
def python2obspy(self):
from obspy.core.trace import Stats, Trace
from obspy.core.utcdatetime import UTCDateTime
s = Stats()
s.network = self.network
s.station = self.station
s.location = self.location
s.channel = self.channel
s.sampling_rate = self.sampling_rate
s.starttime = UTCDateTime(self.starttime)
s.npts = len(self.data)
misc_fields = dict()
if 'CALIB' in self.misc_fields:
s.calib = self.misc_fields.pop('CALIB')
s.update(self.misc_fields)
return Trace(self.data[:], header=s)
def python2obspy(self):
from obspy.core.trace import Stats, Trace
from obspy.core.utcdatetime import UTCDateTime
s = Stats()
s.network = self.network
s.station = self.station
s.location = self.location
s.channel = self.channel
s.sampling_rate = self.sampling_rate
s.starttime = UTCDateTime(self.starttime)
s.npts = len(self.data)
misc_fields = dict()
if 'CALIB' in self.misc_fields:
s.calib = self.misc_fields.pop('CALIB')
s.update(self.misc_fields)
return Trace(self.data[:], header=s)
def _read_channel(filename, line_offset, volume, location='', alternate=False):
"""Read channel data from USC V1 text file.
Args:
filename (str):
Input USC V1 filename.
line_offset (int):
Line offset to beginning of channel text block.
volume (dictionary):
Dictionary of formatting information.
Returns:
tuple: (obspy Trace, int line offset)
"""
if alternate:
int_rows = 5
int_fmt = 20 * [4]
data_cols = 8
else:
int_rows = volume['INT_HDR_ROWS']
int_fmt = volume['INT_FMT']
data_cols = 10
# Parse the header portion of the file
try:
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(volume['TEXT_HDR_ROWS'])]
# Accounts for blank lines at end of files
except StopIteration:
return (None, 1 + line_offset)
# read in lines of integer data
skiprows = line_offset + volume['TEXT_HDR_ROWS']
int_data = np.genfromtxt(filename,
skip_header=skiprows,
max_rows=int_rows,
dtype=np.int32,
delimiter=int_fmt).flatten()
# read in lines of float data
skiprows += int_rows
flt_data = np.genfromtxt(filename,
skip_header=skiprows,
max_rows=volume['FLT_HDR_ROWS'],
dtype=np.float64,
delimiter=volume['FLT_FMT']).flatten()
hdr = _get_header_info(int_data, flt_data, lines, 'V1', location=location)
skiprows += volume['FLT_HDR_ROWS']
# read in the data
nrows = int(np.floor(hdr['npts'] * 2 / data_cols))
all_data = np.genfromtxt(filename,
skip_header=skiprows,
max_rows=nrows,
dtype=np.float64,
delimiter=volume['COL_FMT'])
data = all_data.flatten()[1::2]
times = all_data.flatten()[0::2]
frac = hdr['format_specific']['fractional_unit']
if frac > 0:
data *= UNIT_CONVERSIONS['g'] * frac
logging.debug('Data converted from g * %s to cm/s/s' % (frac))
else:
unit = _get_units(lines[11])
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise ValueError('USC: %s is not a supported unit.' % unit)
# Put file name into dictionary
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
trace = StationTrace(data.copy(), Stats(hdr.copy()))
if not is_evenly_spaced(times):
trace = resample_uneven_trace(trace, times, data)
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
def _prep_data_for_correlation(stream, templates, template_names=None,
force_stream_epoch=True):
"""
Check that all channels are the same length and that all channels have data
for both template and stream.
Works in place on data - will cut to shortest length
:param stream: Stream to compare data to
:param templates:
List of streams that will be forced to have the same channels as stream
:param template_names:
List of strings same length as templates
:type force_stream_epoch: bool
:param force_stream_epoch:
Whether to force all channels in stream to cover the same time period
:return: stream, templates, template_names (if template_names given)
"""
n_templates = len(templates)
template_samp_rates = {
tr.stats.sampling_rate for template in templates for tr in template}
stream_samp_rates = {tr.stats.sampling_rate for tr in stream}
samp_rates = template_samp_rates.union(stream_samp_rates)
assert len(samp_rates) == 1, "Sampling rates differ"
samp_rate = samp_rates.pop()
out_stream = Stream()
named = True
if template_names is None:
named = False
template_names = range(n_templates)
# Work out shapes.
stream_start = min([tr.stats.starttime for tr in stream])
stream_end = max([tr.stats.endtime for tr in stream])
if force_stream_epoch:
stream_length = int(samp_rate * (stream_end - stream_start)) + 1
else:
stream_length = max([tr.stats.npts for tr in stream])
template_length = {
tr.stats.npts for template in templates for tr in template}
assert len(template_length) == 1, "Template traces not all the same length"
template_length = template_length.pop()
stream_ids = {tr.id for tr in stream}
# Need to ensure that a channel can be in the template multiple times.
template_ids = {stream_id: [] for stream_id in stream_ids}
for template in templates:
# Only include those in the stream.
channels_in_template = {
tr.id for tr in template}.intersection(stream_ids)
for channel in channels_in_template:
template_ids[channel].append(len(template.select(id=channel)))
template_ids = {key: max(value) for key, value in template_ids.items()
if len(value) > 0}
seed_ids = sorted(
[key.split('.') + [i] for key, value in template_ids.items()
for i in range(value)])
seed_ids = [('.'.join(seed_id[0:-1]), seed_id[-1]) for seed_id in seed_ids]
for channel_number, seed_id in enumerate(template_ids.keys()):
stream_data = np.zeros(stream_length, dtype=np.float32)
stream_channel = stream.select(id=seed_id)
if len(stream_channel) > 1:
raise NotImplementedError(
"Multiple channels in continuous data for {0}".format(seed_id))
stream_channel = stream_channel[0]
if stream_channel.stats.npts == stream_length:
stream_data = stream_channel.data
else:
Logger.info('Data for {0} is not as long as needed, '
'padding'.format(stream_channel.id))
if force_stream_epoch:
start_pad = int(samp_rate * (
stream_channel.stats.starttime - stream_start))
end_pad = stream_length - (
start_pad + stream_channel.stats.npts)
# In some cases there will be one sample missing when sampling
# time-stamps are not set consistently between channels, this
# results in start_pad and end_pad being len==0
if start_pad == 0 and end_pad == 0:
Logger.debug("Start and end pad are both zero, padding "
"at one end")
if (stream_channel.stats.starttime - stream_start) > (
stream_end - stream_channel.stats.endtime):
start_pad = int(
stream_length - stream_channel.stats.npts)
else:
end_pad = int(
stream_length - stream_channel.stats.npts)
stream_channel.stats.starttime -= (start_pad / samp_rate)
else:
start_pad = 0
end_pad = stream_length - stream_channel.stats.npts
if end_pad == 0:
stream_data[start_pad:] = stream_channel.data
else:
stream_data[start_pad:-end_pad] = stream_channel.data
header = stream_channel.stats.copy()
header.npts = stream_length
out_stream += Trace(data=stream_data, header=header)
# Initialize nan template for speed.
nan_channel = np.full(template_length, np.nan, dtype=np.float32)
nan_template = Stream()
for _seed_id in seed_ids:
net, sta, loc, chan = _seed_id[0].split('.')
nan_template += Trace(header=Stats({
'network': net, 'station': sta, 'location': loc,
'channel': chan, 'starttime': UTCDateTime(),
'npts': template_length, 'sampling_rate': samp_rate}))
# Remove templates with no matching channels
filt = np.ones(len(template_names)).astype(bool)
for i, template in enumerate(templates):
template_ids = {tr.id for tr in template}
if len(template_ids.intersection(stream_ids)) == 0:
filt[i] = 0
_out = dict(zip(
[_tn for _tn, _filt in zip(template_names, filt) if _filt],
[_t for _t, _filt in zip(templates, filt) if _filt]))
if len(_out) != len(templates):
Logger.debug("Some templates not used due to no matching channels")
# Fill out the templates with nan channels
for template_name, template in _out.items():
template_starttime = min([tr.stats.starttime for tr in template])
out_template = nan_template.copy()
for channel_number, _seed_id in enumerate(seed_ids):
seed_id, channel_index = _seed_id
template_channel = template.select(id=seed_id)
if len(template_channel) <= channel_index:
out_template[channel_number].data = nan_channel
out_template[channel_number].stats.starttime = \
template_starttime
else:
out_template[channel_number] = template_channel[channel_index]
_out.update({template_name: out_template})
out_templates = list(_out.values())
out_template_names = list(_out.keys())
if named:
return out_stream, out_templates, out_template_names
return out_stream, out_templates
def read_knet(filename):
"""Read Japanese KNET strong motion file.
Args:
filename (str): Path to possible KNET data file.
kwargs (ref): Other arguments will be ignored.
Returns:
Stream: Obspy Stream containing three channels of acceleration data
(cm/s**2).
"""
logging.debug("Starting read_knet.")
if not is_knet(filename):
raise Exception('%s is not a valid KNET file' % filename)
# Parse the header portion of the file
with open(filename, 'rt') as f:
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
hdr = {}
coordinates = {}
standard = {}
hdr['network'] = 'BO'
hdr['station'] = lines[5].split()[2]
logging.debug('station: %s' % hdr['station'])
standard['station_name'] = ''
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including KNET here, don't provide this. We'll flag it as "--".
hdr['location'] = '--'
coordinates['latitude'] = float(lines[6].split()[2])
coordinates['longitude'] = float(lines[7].split()[2])
coordinates['elevation'] = float(lines[8].split()[2])
hdr['sampling_rate'] = float(
re.search('\\d+', lines[10].split()[2]).group())
hdr['delta'] = 1 / hdr['sampling_rate']
standard['units'] = 'acc'
dir_string = lines[12].split()[1].strip()
# knet files have directions listed as N-S, E-W, or U-D,
# whereas in kiknet those directions are '4', '5', or '6'.
if dir_string in ['N-S', '1', '4']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=True)
elif dir_string in ['E-W', '2', '5']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=False,
is_north=False)
elif dir_string in ['U-D', '3', '6']:
hdr['channel'] = get_channel_name(
hdr['sampling_rate'],
is_acceleration=True,
is_vertical=True,
is_north=False)
else:
raise Exception('KNET: Could not parse direction %s' %
lines[12].split()[1])
logging.debug('channel: %s' % hdr['channel'])
scalestr = lines[13].split()[2]
parts = scalestr.split('/')
num = float(parts[0].replace('(gal)', ''))
den = float(parts[1])
calib = num / den
hdr['calib'] = calib
duration = float(lines[11].split()[2])
hdr['npts'] = int(duration * hdr['sampling_rate'])
timestr = ' '.join(lines[9].split()[2:4])
# The K-NET and KiK-Net data logger adds a 15s time delay
# this is removed here
sttime = datetime.strptime(timestr, TIMEFMT) - timedelta(seconds=15.0)
# Shift the time to utc (Japanese time is 9 hours ahead)
sttime = sttime - timedelta(seconds=9 * 3600.)
hdr['starttime'] = sttime
# read in the data - there is a max of 8 columns per line
# the code below handles the case when last line has
# less than 8 columns
if hdr['npts'] % COLS_PER_LINE != 0:
nrows = int(np.floor(hdr['npts'] / COLS_PER_LINE))
nrows2 = 1
else:
nrows = int(np.ceil(hdr['npts'] / COLS_PER_LINE))
nrows2 = 0
data = np.genfromtxt(filename, skip_header=TEXT_HDR_ROWS,
max_rows=nrows, filling_values=np.nan)
data = data.flatten()
if nrows2:
skip_header = TEXT_HDR_ROWS + nrows
data2 = np.genfromtxt(filename, skip_header=skip_header,
max_rows=nrows2, filling_values=np.nan)
data = np.hstack((data, data2))
nrows += nrows2
# apply the correction factor we're given in the header
data *= calib
# fill out the rest of the standard dictionary
standard['horizontal_orientation'] = np.nan
standard['instrument_period'] = np.nan
standard['instrument_damping'] = np.nan
standard['process_time'] = ''
standard['process_level'] = PROCESS_LEVELS['V1']
standard['sensor_serial_number'] = ''
standard['instrument'] = ''
standard['comments'] = ''
standard['structure_type'] = ''
if dir_string in ['1', '2', '3']:
standard['structure_type'] = 'borehole'
standard['corner_frequency'] = np.nan
standard['units'] = 'acc'
standard['source'] = SRC
standard['source_format'] = 'knet'
hdr['coordinates'] = coordinates
hdr['standard'] = standard
# create a Trace from the data and metadata
trace = StationTrace(data.copy(), Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
stream = StationStream(traces=[trace])
return [stream]
def read_at2(dfile, horient=0.0):
# This is a conveneince method so we can read in these specific data for
# testing, it is not a general purpose reader since this format does not
# contain a lot of metadata that is generally required for it to be useful.
skiprows = 4
datafile = open(dfile, 'r', encoding='utf-8')
datareader = csv.reader(datafile)
data = []
header = []
# for i in range(skiprows):
# next(datareader)
# header.append(datareader.readlines())
count = 0
for row in datareader:
if count < skiprows:
header.append(row)
else:
data.extend([float(e) for e in row[0].split()])
count += 1
datafile.close()
hdr = {}
hdr['network'] = ''
hdr['station'] = ''
if horient == 0:
hdr['channel'] = 'BH1'
else:
hdr['channel'] = 'BH2'
hdr['location'] = '--'
dt = float(header[3][1].split('=')[1].strip().lower().replace('sec', ''))
hdr['npts'] = len(data)
hdr['sampling_rate'] = 1 / dt
hdr['duration'] = (hdr['npts'] - 1) / hdr['sampling_rate']
hdr['starttime'] = 0
# There is no lat/lon...
hdr['coordinates'] = {'latitude': 0.0, 'longitude': 0.0, 'elevation': 0.0}
standard = {}
standard['units'] = 'acc'
standard['units_type'] = 'acc'
standard['horizontal_orientation'] = horient
standard['vertical_orientation'] = np.nan
standard['source_file'] = dfile
standard['station_name'] = ''
standard['corner_frequency'] = 30.0
standard['structure_type'] = ''
standard['comments'] = ''
standard['instrument'] = ''
standard['instrument_period'] = 1.0
standard['instrument_sensitivity'] = 1.0
standard['source'] = 'PEER'
standard['instrument_damping'] = 0.1
standard['sensor_serial_number'] = ''
standard['process_level'] = 'corrected physical units'
standard['source_format'] = 'AT2'
standard['process_time'] = ''
hdr['standard'] = standard
# convert data from g to cm/s^2
g_to_cmss = 980.665
tr = StationTrace(np.array(data.copy()) * g_to_cmss, Stats(hdr.copy()))
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
tr.setProvenance('remove_response', response)
return tr
def synthetic_seismogram(Mw,
duration=0.1,
sampling_rate=10000,
vp=5000.0,
vs=3500.0,
rho=2400,
SSD=1,
pwave=True):
"""
Create a synthetic displacement pulse at the source seismogram based
on the brune model (Brune 1970).
This model is extensively used and generally agrees with observations from
many different setting and over a large range of magnitude.
The displacement time function, u(t), is expressed as follows:
u(t) = A_0 x t x omega_0 x H(t) * exp(-t x omega_0) ,
where t the time, omega_0, the angular frequency and H(t) is the
heavyside function. Note that the angular frequency is calculated from
the static stress drop (SSD). A0 is given by the following
equation:
A0 = M0 / (4 * pi * rho * v ** 3)
References for further reading:
- Routine data processing in earthquake seismology
- Relating Peak Particle Velocity and Acceleration to Moment Magnitude
in Passive (Micro-) Seismic Monitoring
(www.bcengineers.com/images/BCE_Technical_Note_3.pdf)
:param Mw: the moment magnitude of the seismic event (default: -1),
this value determine the wave amplitude and the
frequency content
:type Mw: float
:param noise_level: level of gaussian noise to add to the synthetic
seismogram (default: 1e-5)
:type noise_level: float
:param duration: duration of the seismogram in seconds (default: 0.1),
the pulse is centered at zero
:type duration: float
:param sampling_rate: sampling rate in Hz of the generated time series (
default: 10000)
:type sampling_rate: int
:param vp: P-wave velocity of the material at the source (default: 5000 m/s)
:type vp: float
:param vs: S-wave velocity of the material at the source (default: 3500 m/s)
:type vs: float
:param rho: density of the material at the source in kg/m**3 (default:
2400 kg/m**3)
:param SSD: Static stress drop in "bar" (default: 1 bar)
:type SSD: float
:param pwave: Return P-wave displacement seismogram if True and S-wave
displacement seismogram if false
:rparam: tuple Obspy Trace containing the seismogram
:rtype: Obspy Trace
.. note::
The velocity and acceleration can easily be obtained by
differentiating the trace using the Obspy Trace method
differentiate.
Example
>>> tr = synthetic_seismogram(-1)
>>> tr.differentiate() # this creates a velocity trace
>>> tr.differentiate() # this creates an acceleration trace
This operation is performed in place on the actual data arrays. The
raw data is not accessible anymore afterwards. To keep your
original data, use :meth:`~obspy.core.trace.Trace.copy` to create
a copy of your trace object.
This also makes an entry with information on the applied processing
in ``stats.processing`` of this trace.
"""
M0 = Mw2M0(Mw)
(f0p, f0s) = corner_frequency(Mw, vp, vs, SSD)
# duration = 5 / f0p
npts = duration * sampling_rate
t = np.arange(npts) / sampling_rate
if pwave:
W0 = 2 * np.pi * f0p
v = vp
else:
W0 = 2 * np.pi * f0s
v = vs
A0 = M0 / (4 * np.pi * rho * v**3)
data = A0 * t * W0**2 * np.exp(-t * W0)
data = np.roll(data, len(data) / 2)
stats = Stats()
stats.sampling_rate = sampling_rate
stats.npts = 2 * npts - 1
from uquake.core.util.cepstrum import minimum_phase
minphase_data = np.roll(minimum_phase(data, len(data)), len(data) / 2)
return Trace(data=data, header=stats)
def _read_channel(filename, line_offset, location=''):
"""Read channel data from COSMOS V1/V2 text file.
Args:
filename (str): Input COSMOS V1/V2 filename.
line_offset (int): Line offset to beginning of channel text block.
Returns:
tuple: (obspy Trace, int line offset)
"""
# read station, location, and process level from text header
with open(filename, 'rt') as f:
for _ in range(line_offset):
next(f)
lines = [next(f) for x in range(TEXT_HDR_ROWS)]
# read in lines of integer data
skiprows = line_offset + TEXT_HDR_ROWS
int_lines, int_data = _read_lines(skiprows, filename)
int_data = int_data.astype(np.int32)
# read in lines of float data
skiprows += int_lines + 1
flt_lines, flt_data = _read_lines(skiprows, filename)
# read in comment lines
skiprows += flt_lines + 1
cmt_lines, cmt_data = _read_lines(skiprows, filename)
skiprows += cmt_lines + 1
# according to the powers that defined the Network.Station.Channel.Location
# "standard", Location is a two character field. Most data providers,
# including cosmos here, don't provide this. We'll flag it as "--".
hdr = _get_header_info(int_data,
flt_data,
lines,
cmt_data,
location=location)
head, tail = os.path.split(filename)
hdr['standard']['source_file'] = tail or os.path.basename(head)
# read in the data
nrows, data = _read_lines(skiprows, filename)
# check units
unit = hdr['format_specific']['physical_units']
if unit in UNIT_CONVERSIONS:
data *= UNIT_CONVERSIONS[unit]
logging.debug('Data converted from %s to cm/s/s' % (unit))
else:
raise GMProcessException('COSMOS: %s is not a supported unit.' % unit)
if hdr['standard']['units'] != 'acc':
raise GMProcessException('COSMOS: Only acceleration data accepted.')
trace = StationTrace(data.copy(), Stats(hdr.copy()))
# record that this data has been converted to g, if it has
if hdr['standard']['process_level'] != PROCESS_LEVELS['V0']:
response = {'input_units': 'counts', 'output_units': 'cm/s^2'}
trace.setProvenance('remove_response', response)
# set new offset
new_offset = skiprows + nrows
new_offset += 1 # there is an 'end of record' line after the data
return (trace, new_offset)
