def stack(st, stats=None, kw_stack={}, kw_snr={}):
"""
Stack traces in stream.
"""
if st.count() == 0:
logger.error('No traces!')
return st
elif st.count() == 1:
return st
# Find lap
if kw_stack.get('sort', False):
st.sort(keys=['endtime'])
st.trim(
starttime=st[0].stats.starttime,
endtime=st[0].stats.endtime,
pad=True,
fill_value=0,
)
sk = Trace()
data = _data_stack(st, **kw_stack, **kw_snr)
if data is None:
return
else:
sk.data = data
sk.stats = _hdr_stack(st, stats, **kw_stack, **kw_snr)
key_snr = kw_stack.get('snr', 'user2')
weight = kw_stack.get('weight', 'None').lower()
if 'none' not in weight:
sk.stats.sac[key_snr] = snr(sk, **kw_snr)
return sk
def readgeonet(geonetfile):
"""
Read strong motion data from a GeoNet data file
@param geonetfile: Path to a valid GeoNet data file.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
f = open(geonetfile, 'rt')
tracelist = []
headerlist = []
try:
hdrlines = readheaderlines(f)
except:
pass
while len(hdrlines[-1]):
hdrdict = readheader(hdrlines)
numlines = int(np.ceil(hdrdict['npts'] / 10.0))
data = []
for i in range(0, numlines):
line = f.readline()
parts = line.strip().split()
mdata = [float(p) for p in parts]
data = data + mdata
data = np.array(data)
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data, header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats[
'calib'] * 0.001 #convert to m/s^2
tracelist.append(trace.copy())
headerlist.append(header.copy())
hdrlines = readheaderlines(f)
f.close()
return (tracelist, headerlist)
def readgeonet(geonetfile):
"""
Read strong motion data from a GeoNet data file
@param geonetfile: Path to a valid GeoNet data file.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
f = open(geonetfile,'rt')
tracelist = []
headerlist = []
try:
hdrlines = readheaderlines(f)
except:
pass
while len(hdrlines[-1]):
hdrdict = readheader(hdrlines)
numlines = int(np.ceil(hdrdict['npts']/10.0))
data = []
for i in range(0,numlines):
line = f.readline()
parts = line.strip().split()
mdata = [float(p) for p in parts]
data = data + mdata
data = np.array(data)
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data,header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.001 #convert to m/s^2
tracelist.append(trace.copy())
headerlist.append(header.copy())
hdrlines = readheaderlines(f)
f.close()
return (tracelist,headerlist)
def rand_stack(st, stats=None, kw_stack={}, kw_snr={}):
nsub = kw_stack.get('nsub', 4)
key_nsrc = kw_stack.get('key_nsrc', 'user0')
ntr = st.count()
if ntr < 2:
return st
sk = Trace()
sk.stats = _hdr_stack(st, stats, **kw_stack, **kw_snr)
if ntr <= nsub:
for tr in st:
yield tr
else:
ind = np.arange(ntr)
np.random.shuffle(ind)
for sub in np.array_split(ind, nsub):
data = 0
for i in sub:
data += st[i].data
n = sub.size
data /= n
sk.stats.sac[key_nsrc] = n
sk.data = data
yield sk
return
def readiran(iranfile, doRotation=True):
"""
Read strong motion data from a Iran data file
@param iranfile: Path to a valid Iran data file.
@keyword doRotation: Apply back-azimuth rotation of L & T channels to NS and EW.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
f = open(iranfile, 'rt')
tracelist = []
headerlist = []
try:
hdrlines = readheaderlines(f)
except:
pass
while len(hdrlines[-1]):
hdrdict = readheader(hdrlines)
numlines = int(np.ceil(hdrdict['npts'] / 10.0))
data = []
for i in range(0, numlines):
line = f.readline()
parts = line.strip().split()
mdata = [float(p) for p in parts]
data = data + mdata
data = np.array(data)
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data, header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats[
'calib'] * 0.98 #convert to m/s^2 from g/10
tracelist.append(trace.copy())
headerlist.append(header.copy())
endblock = f.readline()
hdrlines = readheaderlines(f)
f.close()
#data from Iran may be rotated so that one channel is aligned in the direction between the earthquake
#epicenter and the station. We want to rotate the data back so that we have what are presumably the
#original NS and EW channels. We presume that the "L" channel will rotate back to become NS, and
#"T" will become EW.
#First, find the channel called L*
if doRotation:
channels = [h['channel'][0:1] for h in headerlist]
lidx = channels.index('L')
tidx = channels.index('T')
ldata = tracelist[lidx].data
tdata = tracelist[tidx].data
backaz = headerlist[0]['rotation']['L']
ndata, edata = rotate.rotate_RT_NE(ldata, tdata, backaz)
tracelist[lidx].data = ndata.copy()
tracelist[lidx].stats[
'channel'] = 'H1' #most probably NS, but we're being cautious
tracelist[tidx].data = edata.copy()
tracelist[tidx].stats[
'channel'] = 'H2' #most probably EW, but we're being cautious
return (tracelist, headerlist)
def readiran(iranfile,doRotation=True):
"""
Read strong motion data from a Iran data file
@param iranfile: Path to a valid Iran data file.
@keyword doRotation: Apply back-azimuth rotation of L & T channels to NS and EW.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
f = open(iranfile,'rt')
tracelist = []
headerlist = []
try:
hdrlines = readheaderlines(f)
except:
pass
while len(hdrlines[-1]):
hdrdict = readheader(hdrlines)
numlines = int(np.ceil(hdrdict['npts']/10.0))
data = []
for i in range(0,numlines):
line = f.readline()
parts = line.strip().split()
mdata = [float(p) for p in parts]
data = data + mdata
data = np.array(data)
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data,header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.98 #convert to m/s^2 from g/10
tracelist.append(trace.copy())
headerlist.append(header.copy())
endblock = f.readline()
hdrlines = readheaderlines(f)
f.close()
#data from Iran may be rotated so that one channel is aligned in the direction between the earthquake
#epicenter and the station. We want to rotate the data back so that we have what are presumably the
#original NS and EW channels. We presume that the "L" channel will rotate back to become NS, and
#"T" will become EW.
#First, find the channel called L*
if doRotation:
channels = [h['channel'][0:1] for h in headerlist]
lidx = channels.index('L')
tidx = channels.index('T')
ldata = tracelist[lidx].data
tdata = tracelist[tidx].data
backaz = headerlist[0]['rotation']['L']
ndata,edata = rotate.rotate_RT_NE(ldata,tdata,backaz)
tracelist[lidx].data = ndata.copy()
tracelist[lidx].stats['channel'] = 'H1' #most probably NS, but we're being cautious
tracelist[tidx].data = edata.copy()
tracelist[tidx].stats['channel'] = 'H2' #most probably EW, but we're being cautious
return (tracelist,headerlist)
def create_trace(sta, T1, T2):
from obspy import Trace
from numpy import zeros
from numpy import empty, nan
tr = Trace()
tr.stats['station'] = sta.split('.')[0]
tr.stats['channel'] = sta.split('.')[1]
tr.stats['network'] = sta.split('.')[2]
tr.stats['location'] = sta.split('.')[3]
tr.stats['sampling_rate'] = 100
tr.stats['starttime'] = T1
#tr.data=zeros(int((T2-T1)*tr.stats['sampling_rate']))
tr.data = empty(int((T2 - T1) * tr.stats['sampling_rate']))
tr.data[:] = nan
return tr
def sym_xc(xc, noeven=False):
"""
Symmetrize a cross-correlation.
"""
pair = f'{xc.stats.sac.kevnm.split()[0]}-{xc.stats.station}'
if not _is_sym(xc):
logger.debug(f'Asymmetric {pair}')
end = min(abs(xc.stats.sac.b), xc.stats.sac.e)
xc = sliced(xc, -end, end)
if _even_npts(xc) and noeven:
logger.error(f'Even npts to symmexcize {pair}')
else:
sym = Trace()
sym.data = _data_sym(xc.data)
sym.stats = _hd_sym(xc.stats, npts=sym.data.size)
return sym
def get_stream(self,
processed_path,
kstnm,
kinst,
force_without_loc=False):
# Check if an interpolated station location exists
if self.station_loc is None and not force_without_loc:
return
# Save data into a Stream object
trace = Trace()
trace.stats = self.obspy_trace_stats
trace.data = self.processed_data
stream = Stream(traces=[trace])
return stream
def readknet(knetfilename):
"""
Read a KNet ASCII file, and return an ObsPy Trace object, plus a dictionary of header values.
@param knetfilename: String path to valid KNet ASCII file, as described here: http://www.kyoshin.bosai.go.jp/kyoshin/man/knetform_en.html
@return: ObsPy Trace object, and a dictionary of some of the header values found in the input file.
"""
data = []
hdrdict = {}
f = open(knetfilename,'rt')
dataOn = False
headerlines = []
for line in f.readlines():
if line.startswith('Memo'):
hdrdict = readheader(headerlines)
dataOn = True
continue
if not dataOn:
headerlines.append(line)
continue
if dataOn:
parts = line.strip().split()
mdata = [float(p) for p in parts]
data = data + mdata
f.close()
#fill in the values usually expected in Stats as best we can
hdrdict['npts'] = len(data)
elapsed = float(hdrdict['npts'])/float(hdrdict['sampling_rate'])
hdrdict['endtime'] = hdrdict['starttime'] + elapsed
hdrdict['network'] = 'NIED'
hdrdict['location'] = ''
#The Stats constructor appears to modify the fields in the input dictionary - let's save
#a copy
header = hdrdict.copy()
data = np.array(data)
stats = Stats(hdrdict)
trace = Trace(data,header=stats)
#apply the calibration and convert to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.01 #convert to m/s^2
return (trace,header)
def readknet(knetfilename):
"""
Read a KNet ASCII file, and return an ObsPy Trace object, plus a dictionary of header values.
@param knetfilename: String path to valid KNet ASCII file, as described here: http://www.kyoshin.bosai.go.jp/kyoshin/man/knetform_en.html
@return: ObsPy Trace object, and a dictionary of some of the header values found in the input file.
"""
data = []
hdrdict = {}
f = open(knetfilename,'rt')
dataOn = False
headerlines = []
for line in f.readlines():
if line.startswith('Memo'):
hdrdict = readheader(headerlines)
dataOn = True
continue
if not dataOn:
headerlines.append(line)
continue
if dataOn:
parts = line.strip().split()
mdata = [float(p) for p in parts]
data = data + mdata
f.close()
#fill in the values usually expected in Stats as best we can
hdrdict['npts'] = len(data)
elapsed = float(hdrdict['npts'])/float(hdrdict['sampling_rate'])
hdrdict['endtime'] = hdrdict['starttime'] + elapsed
hdrdict['network'] = 'NIED'
hdrdict['location'] = ''
#The Stats constructor appears to modify the fields in the input dictionary - let's save
#a copy
header = hdrdict.copy()
data = np.array(data)
stats = Stats(hdrdict)
trace = Trace(data,header=stats)
#apply the calibration and convert to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.01 #convert to m/s^2
return (trace,header)
def readgeonet(geonetfile):
"""
Read strong motion data from a GeoNet data file
@param geonetfile: Path to a valid GeoNet data file.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
#notes on implementation:
# originally I had written code to read each line of data manually, just
# as I was reading the header lines. However, this became VERY slow for large
# files. I discovered that numpy's genfromtxt function was much faster. However,
# I could not get that function to work when I passed it a file object instead of a file name.
# Consequently, the only way I could keep the genfromtxt() method and file pointer in sync was
# to read and discard all of the lines of data that genfromtxt() parsed. While annoying, the combination
# of these two seems to still be at least an order of magnitude faster than manually reading the file.
f = open(geonetfile,'rt')
tracelist = []
headerlist = []
totlines = 0
hdrlines = readheaderlines(f)
while len(hdrlines[-1]):
totlines += len(hdrlines)
hdrdict = readheader(hdrlines)
numlines = int(np.ceil(hdrdict['npts']/10.0))
data = np.genfromtxt(geonetfile,skip_header=totlines,max_rows=numlines)
totlines += numlines
#now we need to set the file position to where we just ended
for i in range(0,numlines):
f.readline()
data = data.flatten()
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data,header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.001 #convert to m/s^2
tracelist.append(trace.copy())
headerlist.append(header.copy())
hdrlines = readheaderlines(f)
f.close()
return (tracelist,headerlist)
def to_sac_and_mseed(self, export_path, station_number, force_without_loc):
# Check if file exist
export_path_sac = export_path + self.get_export_file_name() + ".sac"
export_path_msd = export_path + self.get_export_file_name() + ".mseed"
#export_path_wav = export_path + self.get_export_file_name() + ".wav"
if os.path.exists(export_path_sac) and os.path.exists(export_path_msd):
return
# Check if the station location have been calculated
if self.station_loc is None and not force_without_loc:
print self.get_export_file_name() + ": Skip sac/mseed generation, wait the next ascent to compute location"
return
# Fill header info
stats = Stats()
stats.sampling_rate = self.decimated_fs
stats.network = "MH"
stats.station = station_number
stats.starttime = self.date
stats.sac = dict()
if not force_without_loc:
stats.sac["stla"] = self.station_loc.latitude
stats.sac["stlo"] = self.station_loc.longitude
stats.sac["stdp"] = self.depth
stats.sac["user0"] = self.snr
stats.sac["user1"] = self.criterion
stats.sac["iztype"] = 9 # 9 == IB in sac format
# Save data into a Stream object
trace = Trace()
trace.stats = stats
trace.data = self.data
stream = Stream(traces=[trace])
# Save stream object
print export_path_sac
stream.write(export_path_sac, format='SAC')
print export_path_msd
stream.write(export_path_msd, format='MSEED')
def readitaly(datafile):
f = open(datafile, 'rt')
#header needs: station,channel,location,npts,starttime,sampling_rate,delta,calib,lat,lon,height,duration,endtime,maxacc,network
data = []
hdrdict = {}
for line in f.readlines():
if not len(line.strip()):
continue
if not line.find(':') > -1:
data.append(float(line.strip()))
continue
key, value = line.split(':')
key = key.strip()
value = value.strip()
if key not in list(HEADERS.keys()):
continue
hdrkey = HEADERS[key]
if hdrkey == 'starttime':
value = UTCDateTime(datetime.datetime.strptime(value, TIMEFMT))
elif hdrkey not in ['station', 'channel', 'location', 'network']:
value = float(value)
hdrdict[hdrkey] = value
f.close()
hdrdict['sampling_rate'] = 1 / hdrdict['delta']
hdrdict['endtime'] = hdrdict['starttime'] + hdrdict['duration']
hdrdict['npts'] = int(hdrdict['npts'])
hdrdict['calib'] = 1.0
hdrdict['units'] = 'acc'
data = np.array(data)
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data, header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.01 #convert to m/s^2
return trace
def readitaly(datafile):
f = open(datafile,'rt')
#header needs: station,channel,location,npts,starttime,sampling_rate,delta,calib,lat,lon,height,duration,endtime,maxacc,network
data = []
hdrdict = {}
for line in f.readlines():
if not len(line.strip()):
continue
if not line.find(':') > -1:
data.append(float(line.strip()))
continue
key,value = line.split(':')
key = key.strip()
value = value.strip()
if key not in list(HEADERS.keys()):
continue
hdrkey = HEADERS[key]
if hdrkey == 'starttime':
value = UTCDateTime(datetime.datetime.strptime(value,TIMEFMT))
elif hdrkey not in ['station','channel','location','network']:
value = float(value)
hdrdict[hdrkey] = value
f.close()
hdrdict['sampling_rate'] = 1/hdrdict['delta']
hdrdict['endtime'] = hdrdict['starttime'] + hdrdict['duration']
hdrdict['npts'] = int(hdrdict['npts'])
hdrdict['calib'] = 1.0
hdrdict['units'] = 'acc'
data = np.array(data)
header = hdrdict.copy()
stats = Stats(hdrdict)
trace = Trace(data,header=stats)
#apply the calibration and convert from mm/s^2 to m/s^2
trace.data = trace.data * trace.stats['calib'] * 0.01 #convert to m/s^2
return trace
def readturkey(turkeyfile):
"""
Read strong motion data from a Turkey data file
@param geonetfile: Path to a valid Turkey data file.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
f = open(turkeyfile,'rt')
dataOn = False
header = {}
nschannel = []
ewchannel = []
udchannel = []
for line in f.readlines():
if line.strip().startswith('STATION ID'):
parts = line.strip().split(':')
header['station'] = parts[1].strip()
continue
if line.strip().startswith('STATION COORD'):
parts = line.strip().split(':')
cstr = parts[1].strip()
parts = cstr.split('-')
header['lat'] = float(parts[0][0:-1])
header['lon'] = float(parts[1][0:-1])
continue
if line.strip().startswith('STATION ALT'):
parts = line.strip().split(':')
try:
header['height'] = float(parts[1])
except:
header['height'] = 0.0
continue
if line.strip().startswith('RECORD TIME'):
parts = line.strip().split(':')
timestr = ':'.join(parts[1:])
timestr = timestr.strip().replace('(GMT)','').strip()
dt = datetime.strptime(timestr[0:19],'%d/%m/%Y %H:%M:%S')
dt = dt.replace(microsecond=int(timestr[20:]))
header['starttime'] = UTCDateTime(dt)
continue
if line.strip().startswith('NUMBER OF DATA'):
parts = line.strip().split(':')
header['npts'] = int(parts[1])
continue
if line.strip().startswith('SAMPLING INTERVAL'):
parts = line.strip().split(':')
header['delta'] = float(parts[1])
header['sampling_rate'] = 1.0/header['delta']
continue
if line.strip().startswith('N-S'):
dataOn = True
continue
if dataOn:
parts = line.strip().split()
nschannel.append(float(parts[0]))
ewchannel.append(float(parts[1]))
udchannel.append(float(parts[2]))
f.close()
nschannel = np.array(nschannel)
ewchannel = np.array(ewchannel)
udchannel = np.array(udchannel)
header['network'] = 'TR'
header['units'] = 'acc'
nsheader = header.copy()
nsheader['channel'] = 'NS'
ewheader = header.copy()
ewheader['channel'] = 'EW'
udheader = header.copy()
udheader['channel'] = 'UD'
nsstats = Stats(nsheader)
nstrace = Trace(nschannel,header=nsstats)
ewstats = Stats(ewheader)
ewtrace = Trace(ewchannel,header=ewstats)
udstats = Stats(udheader)
udtrace = Trace(udchannel,header=udstats)
nstrace.data = nstrace.data * 0.01 #convert to m/s^2
ewtrace.data = ewtrace.data * 0.01 #convert to m/s^2
udtrace.data = udtrace.data * 0.01 #convert to m/s^2
tracelist = [nstrace,ewtrace,udtrace]
hdrlist = [nsheader,ewheader,udheader]
return (tracelist,hdrlist)
tsum = 0
for i in range(niter):
trace.set_data(a + 1)
start = time.time()
trace.envelope()
tsum = tsum + time.time() - start
avgEnvelope = tsum / niter
print("ObsPy test...")
from obspy.core.trace import Trace
from obspy.signal.filter import envelope
traceObspy = Trace(data=a)
traceObspy.detrend('demean')
tsum = 0
for i in range(niter):
traceObspy.data = a + 1
start = time.time()
traceObspy.detrend('demean')
tsum = tsum + time.time() - start
y = traceObspy.data
avgDemeanObspy = tsum / niter
#print("Obspy average time for demean %e (s)"%(avgDemeanObspy))
tsum = 0
for i in range(niter):
traceObspy.data = a + 1
start = time.time()
traceObspy.detrend('linear')
tsum = tsum + time.time() - start
y = traceObspy.data
avgDetrendObspy = tsum / niter
def readturkey(turkeyfile):
"""
Read strong motion data from a Turkey data file
@param geonetfile: Path to a valid Turkey data file.
@return: List of ObsPy Trace objects, containing accelerometer data in m/s.
"""
f = open(turkeyfile, 'rt')
dataOn = False
header = {}
nschannel = []
ewchannel = []
udchannel = []
for line in f.readlines():
if line.strip().startswith('STATION ID'):
parts = line.strip().split(':')
header['station'] = parts[1].strip()
continue
if line.strip().startswith('STATION COORD'):
parts = line.strip().split(':')
cstr = parts[1].strip()
parts = cstr.split('-')
header['lat'] = float(parts[0][0:-1])
header['lon'] = float(parts[1][0:-1])
continue
if line.strip().startswith('STATION ALT'):
parts = line.strip().split(':')
try:
header['height'] = float(parts[1])
except:
header['height'] = 0.0
continue
if line.strip().startswith('RECORD TIME'):
parts = line.strip().split(':')
timestr = ':'.join(parts[1:])
timestr = timestr.strip().replace('(GMT)', '').strip()
dt = datetime.strptime(timestr[0:19], '%d/%m/%Y %H:%M:%S')
dt = dt.replace(microsecond=int(timestr[20:]))
header['starttime'] = UTCDateTime(dt)
continue
if line.strip().startswith('NUMBER OF DATA'):
parts = line.strip().split(':')
header['npts'] = int(parts[1])
continue
if line.strip().startswith('SAMPLING INTERVAL'):
parts = line.strip().split(':')
header['delta'] = float(parts[1])
header['sampling_rate'] = 1.0 / header['delta']
continue
if line.strip().startswith('N-S'):
dataOn = True
continue
if dataOn:
parts = line.strip().split()
nschannel.append(float(parts[0]))
ewchannel.append(float(parts[1]))
udchannel.append(float(parts[2]))
f.close()
nschannel = np.array(nschannel)
ewchannel = np.array(ewchannel)
udchannel = np.array(udchannel)
header['network'] = 'TR'
header['units'] = 'acc'
nsheader = header.copy()
nsheader['channel'] = 'NS'
ewheader = header.copy()
ewheader['channel'] = 'EW'
udheader = header.copy()
udheader['channel'] = 'UD'
nsstats = Stats(nsheader)
nstrace = Trace(nschannel, header=nsstats)
ewstats = Stats(ewheader)
ewtrace = Trace(ewchannel, header=ewstats)
udstats = Stats(udheader)
udtrace = Trace(udchannel, header=udstats)
nstrace.data = nstrace.data * 0.01 #convert to m/s^2
ewtrace.data = ewtrace.data * 0.01 #convert to m/s^2
udtrace.data = udtrace.data * 0.01 #convert to m/s^2
tracelist = [nstrace, ewtrace, udtrace]
hdrlist = [nsheader, ewheader, udheader]
return (tracelist, hdrlist)
def detect_stalta(self):
STA_len = self.params["STA_len"]
LTA_len = self.params["LTA_len"]
STALTA_thresh = self.params["STALTA_thresh"]
det_off_win = self.params["no_det_win"]
try:
devices = self.traces.data["device_id"].unique()
except Exception as exception:
logging.error(exception)
devices = []
for device in devices:
for channel in ["x", "y", "z"]:
trace = self.traces.data[self.traces.data["device_id"] ==
device][channel].to_numpy()
time = self.traces.data[self.traces.data["device_id"] ==
device]["cloud_t"]
sr = self.traces.data[self.traces.data["device_id"] ==
device]["sr"].iloc[0]
if len(trace) > int(np.ceil(sr * (STA_len + LTA_len))):
# set new trace
tr = Trace()
tr.data = trace
tr.stats.delta = 1 / sr
tr.stats.channel = channel
tr.stats.station = device
# tr.filter("highpass", freq=0.2)
tr.detrend(type="constant")
tr_orig = tr.copy()
std = np.std(tr_orig.data[:int(STA_len * sr)])
tr.trigger(
"recstalta",
sta=self.params["STA_len"],
lta=self.params["LTA_len"],
)
(ind, ) = np.where(tr.data > STALTA_thresh)
if len(ind) > 0:
det_time = time.iloc[ind[0]]
past_detections = self.detections.data[
(self.detections.data["device_id"] == device)
& (self.detections.data["cloud_t"] - det_time +
det_off_win) > 0]
if (past_detections.shape[0]
== 0) & (std <= self.params["max_std"]):
# Get event ID
# timestamp
timestamp = datetime.utcfromtimestamp(det_time)
year = str(timestamp.year - 2000).zfill(2)
month = str(timestamp.month).zfill(2)
day = str(timestamp.day).zfill(2)
hour = str(timestamp.hour).zfill(2)
minute = str(timestamp.minute).zfill(2)
detection_id = "D_" + year + month + day + hour + minute
new_detection = pd.DataFrame(
{
"detection_id": detection_id,
"device_id": device,
"cloud_t": det_time,
"mag1": None,
"mag2": None,
"mag3": None,
"mag4": None,
"mag5": None,
"mag6": None,
"mag7": None,
"mag8": None,
"mag9": None,
"event_id": None,
},
index=[0],
)
# plot all detections and save in obj/detections folder
if self.params["plot_detection"]:
self.plot_detection(tr_orig, tr, device,
detection_id, std)
self.detections.update(new_detection)
def invert_raw():
######################################
# Binary
######################################
if mode == "Binary":
catch_files = []
files = glob.glob(file_path + "*")
for file in files:
catch = re.findall(
".*[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}_[0-9]{2}_[0-9]{2}\.[0-9]{6}",
file)
if len(catch) > 0:
catch_files.append(file)
######################################
# Freq file
######################################
freq_file = glob.glob(file_path + "*_freq")
if len(freq_file) > 1:
print "warning : more than one freq file in folder"
if len(freq_file) == 0:
print "warning no freq file discovered use :" + str(sampling_freq)
else:
content = "40.000000"
with open(freq_file[0], "r") as f:
content = f.read()
sampling_freq = float(content)
print "Sampling used : " + str(sampling_freq)
files_nb = len(catch_files)
file_offset = 1
for catch_file in catch_files:
print catch_file
print "File nb : " + str(file_offset) + "/" + str(files_nb)
date = UTCDateTime(
re.findall(
".*([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}_[0-9]{2}_[0-9]{2}\.[0-9]{6})",
catch_file)[0])
rawdata = numpy.fromfile(catch_file, numpy.int32)
######################################
# Plot plotly file
######################################
# Add acoustic values to the graph
#data_line = graph.Scattergl(x=[date + i/sampling_freq for i in range(0,len(rawdata))],
# y=rawdata,
# name="counts",
# line=dict(color='blue', width=2),
# mode='lines')
#plotlydata = [data_line]
#layout = graph.Layout(title="Plot",
# xaxis=dict(title='Date', titlefont=dict(size=18)),
# yaxis=dict(title='Counts', titlefont=dict(size=18)),
# hovermode='closest'
# )
#plotly.plot({'data': plotlydata, 'layout': layout},
# filename=catch_file + ".html",
# auto_open=False)
######################################
# Create SAC file
######################################
# Fill header info
stats = Stats()
stats.sampling_rate = sampling_freq
stats.network = "test"
stats.station = 0
stats.starttime = date
stats.sac = dict()
# Save data into a Stream object
trace = Trace()
trace.stats = stats
trace.data = rawdata
stream = Stream(traces=[trace])
# Save stream object
stream.write(catch_file + ".sac", format='SAC')
stream.write(catch_file + ".mseed", format='MSEED')
file_offset = file_offset + 1
else:
######################################
# Text
######################################
#filename = "tool_invert_raw/1553771378.490936"
#date = UTCDateTime(1553771378.490936)
# text
#f = open(filename, 'r')
#rawdata = numpy.array(f.read().rstrip('\n').split('\n'))
#f.close()
# binary
######################################
# Plot plotly file
######################################
# Add acoustic values to the graph
data_line = graph.Scattergl(
x=[date + i / sampling_freq for i in range(0, len(rawdata))],
y=rawdata,
name="counts",
line=dict(color='blue', width=2),
mode='lines')
plotlydata = [data_line]
layout = graph.Layout(title="Plot",
xaxis=dict(title='Date',
titlefont=dict(size=18)),
yaxis=dict(title='Counts',
titlefont=dict(size=18)),
hovermode='closest')
plotly.plot({
'data': plotlydata,
'layout': layout
},
filename=filename + ".html",
auto_open=False)
######################################
# Create SAC file
######################################
# Fill header info
stats = Stats()
stats.sampling_rate = sampling_freq
stats.network = "test"
stats.station = 0
stats.starttime = date
stats.sac = dict()
# Save data into a Stream object
trace = Trace()
trace.stats = stats
trace.data = rawdata
stream = Stream(traces=[trace])
# Save stream object
stream.write(filename + ".sac", format='SAC')
def _read_y(filename, headonly=False, **kwargs): # @UnusedVariable
"""
Reads a Nanometrics Y file and returns an ObsPy Stream object.
.. warning::
This function should NOT be called directly, it registers via the
ObsPy :func:`~obspy.core.stream.read` function, call this instead.
:type filename: str
:param filename: Nanometrics Y file to be read.
:type headonly: bool, optional
:param headonly: If set to True, read only the head. This is most useful
for scanning available data in huge (temporary) data sets.
:rtype: :class:`~obspy.core.stream.Stream`
:return: A ObsPy Stream object.
.. rubric:: Example
>>> from obspy import read
>>> st = read("/path/to/YAYT_BHZ_20021223.124800")
>>> st # doctest: +ELLIPSIS
<obspy.core.stream.Stream object at 0x...>
>>> print(st) # doctest: +ELLIPSIS
1 Trace(s) in Stream:
.AYT..BHZ | 2002-12-23T12:48:00.000100Z - ... | 100.0 Hz, 18000 samples
"""
# The first tag in a Y-file must be the TAG_Y_FILE (0) tag. This must be
# followed by the following tags, in any order:
# TAG_STATION_INFO (1)
# TAG_STATION_LOCATION (2)
# TAG_STATION_PARAMETERS (3)
# TAG_STATION_DATABASE (4)
# TAG_SERIES_INFO (5)
# TAG_SERIES_DATABASE (6)
# The following tag is optional:
# TAG_STATION_RESPONSE (26)
# The last tag in the file must be a TAG_DATA_INT32 (7) tag. This tag must
# be followed by an array of LONG's. The number of entries in the array
# must agree with what was described in the TAG_SERIES_INFO data.
with open(filename, "rb") as fh:
trace = Trace()
trace.stats.y = AttribDict()
count = -1
while True:
endian, tag_type, next_tag, _next_same = __parse_tag(fh)
if tag_type == 1:
# TAG_STATION_INFO
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# UCHAR Station[5] (BLANKPAD)
# Station is the five letter SEED format station
# identification.
# UCHAR Location[2] (BLANKPAD)
# Location Location is the two letter SEED format location
# identification.
# UCHAR Channel[3] (BLANKPAD)
# Channel Channel is the three letter SEED format channel
# identification.
# UCHAR NetworkID[51] (ASCIIZ)
# This is some descriptive text identifying the network.
# UCHAR SiteName[61] (ASCIIZ)
# SiteName is some text identifying the site.
# UCHAR Comment[31] (ASCIIZ)
# Comment is any comment for this station.
# UCHAR SensorType[51] (ASCIIZ)
# SensorType is some text describing the type of sensor used
# at the station.
# UCHAR DataFormat[7] (ASCIIZ)
# DataFormat is some text describing the data format recorded
# at the station.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(b"5s2s3s51z61z31z51z7z", data[8:])
trace.stats.station = parts[0]
trace.stats.location = parts[1]
trace.stats.channel = parts[2]
# extra
params = AttribDict()
params.network_id = parts[3]
params.side_name = parts[4]
params.comment = parts[5]
params.sensor_type = parts[6]
params.data_format = parts[7]
trace.stats.y.tag_station_info = params
elif tag_type == 2:
# TAG_STATION_LOCATION
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# FLOAT Latitude
# Latitude in degrees of the location of the station. The
# latitude should be between -90 (South) and +90 (North).
# FLOAT Longitude
# Longitude in degrees of the location of the station. The
# longitude should be between -180 (West) and +180 (East).
# FLOAT Elevation
# Elevation in meters above sea level of the station.
# FLOAT Depth
# Depth is the depth in meters of the sensor.
# FLOAT Azimuth
# Azimuth of the sensor in degrees clockwise.
# FLOAT Dip
# Dip is the dip of the sensor. 90 degrees is defined as
# vertical right way up.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(endian + b"ffffff", data[8:])
params = AttribDict()
params.latitude = parts[0]
params.longitude = parts[1]
params.elevation = parts[2]
params.depth = parts[3]
params.azimuth = parts[4]
params.dip = parts[5]
trace.stats.y.tag_station_location = params
elif tag_type == 3:
# TAG_STATION_PARAMETERS
# UCHAR Update[16]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# REALTIME StartValidTime
# Time that the information in these records became valid.
# REALTIME EndValidTime
# Time that the information in these records became invalid.
# FLOAT Sensitivity
# Sensitivity of the sensor in nanometers per bit.
# FLOAT SensFreq
# Frequency at which the sensitivity was measured.
# FLOAT SampleRate
# This is the number of samples per second. This value can be
# less than 1.0. (i.e. 0.1)
# FLOAT MaxClkDrift
# Maximum drift rate of the clock in seconds per sample.
# UCHAR SensUnits[24] (ASCIIZ)
# Some text indicating the units in which the sensitivity was
# measured.
# UCHAR CalibUnits[24] (ASCIIZ)
# Some text indicating the units in which calibration input
# was measured.
# UCHAR ChanFlags[27] (BLANKPAD)
# Text indicating the channel flags according to the SEED
# definition.
# UCHAR UpdateFlag
# This flag must be “N” or “U” according to the SEED
# definition.
# UCHAR Filler[4]
# Filler Pads out the record to satisfy the alignment
# restrictions for reading data on a SPARC processor.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(endian + b"ddffff24z24z27sc4s", data[16:])
trace.stats.sampling_rate = parts[4]
# extra
params = AttribDict()
params.start_valid_time = parts[0]
params.end_valid_time = parts[1]
params.sensitivity = parts[2]
params.sens_freq = parts[3]
params.sample_rate = parts[4]
params.max_clk_drift = parts[5]
params.sens_units = parts[6]
params.calib_units = parts[7]
params.chan_flags = parts[8]
params.update_flag = parts[9]
trace.stats.y.tag_station_parameters = params
elif tag_type == 4:
# TAG_STATION_DATABASE
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# REALTIME LoadDate
# Date the information was loaded into the database.
# UCHAR Key[16]
# Unique key that identifies this record in the database.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(endian + b"d16s", data[8:])
params = AttribDict()
params.load_date = parts[0]
params.key = parts[1]
trace.stats.y.tag_station_database = params
elif tag_type == 5:
# TAG_SERIES_INFO
# UCHAR Update[16]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# REALTIME StartTime
# This is start time of the data in this series.
# REALTIME EndTime
# This is end time of the data in this series.
# ULONG NumSamples
# This is the number of samples of data in this series.
# LONG DCOffset
# DCOffset is the DC offset of the data.
# LONG MaxAmplitude
# MaxAmplitude is the maximum amplitude of the data.
# LONG MinAmplitude
# MinAmplitude is the minimum amplitude of the data.
# UCHAR Format[8] (ASCIIZ)
# This is the format of the data. This should always be
# “YFILE”.
# UCHAR FormatVersion[8] (ASCIIZ)
# FormatVersion is the version of the format of the data.
# This should always be “5.0”
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(endian + b"ddLlll8z8z", data[16:])
trace.stats.starttime = UTCDateTime(parts[0])
count = parts[2]
# extra
params = AttribDict()
params.endtime = UTCDateTime(parts[1])
params.num_samples = parts[2]
params.dc_offset = parts[3]
params.max_amplitude = parts[4]
params.min_amplitude = parts[5]
params.format = parts[6]
params.format_version = parts[7]
trace.stats.y.tag_series_info = params
elif tag_type == 6:
# TAG_SERIES_DATABASE
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# REALTIME LoadDate
# Date the information was loaded into the database.
# UCHAR Key[16]
# Unique key that identifies this record in the database.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(endian + b"d16s", data[8:])
params = AttribDict()
params.load_date = parts[0]
params.key = parts[1]
trace.stats.y.tag_series_database = params
elif tag_type == 26:
# TAG_STATION_RESPONSE
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# UCHAR PathName[260]
# PathName is the full name of the file which contains the
# response information for this station.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(b"260s", data[8:])
params = AttribDict()
params.path_name = parts[0]
trace.stats.y.tag_station_response = params
elif tag_type == 7:
# TAG_DATA_INT32
trace.data = from_buffer(fh.read(np.dtype(np.int32).itemsize * count), dtype=np.int32)
# break loop as TAG_DATA_INT32 should be the last tag in file
break
else:
fh.seek(next_tag, 1)
return Stream([trace])
def _read_y(filename, headonly=False, **kwargs): # @UnusedVariable
"""
Reads a Nanometrics Y file and returns an ObsPy Stream object.
.. warning::
This function should NOT be called directly, it registers via the
ObsPy :func:`~obspy.core.stream.read` function, call this instead.
:type filename: str
:param filename: Nanometrics Y file to be read.
:type headonly: bool, optional
:param headonly: If set to True, read only the head. This is most useful
for scanning available data in huge (temporary) data sets.
:rtype: :class:`~obspy.core.stream.Stream`
:return: A ObsPy Stream object.
.. rubric:: Example
>>> from obspy import read
>>> st = read("/path/to/YAYT_BHZ_20021223.124800")
>>> st # doctest: +ELLIPSIS
<obspy.core.stream.Stream object at 0x...>
>>> print(st) # doctest: +ELLIPSIS
1 Trace(s) in Stream:
.AYT..BHZ | 2002-12-23T12:48:00.000100Z - ... | 100.0 Hz, 18000 samples
"""
# The first tag in a Y-file must be the TAG_Y_FILE (0) tag. This must be
# followed by the following tags, in any order:
# TAG_STATION_INFO (1)
# TAG_STATION_LOCATION (2)
# TAG_STATION_PARAMETERS (3)
# TAG_STATION_DATABASE (4)
# TAG_SERIES_INFO (5)
# TAG_SERIES_DATABASE (6)
# The following tag is optional:
# TAG_STATION_RESPONSE (26)
# The last tag in the file must be a TAG_DATA_INT32 (7) tag. This tag must
# be followed by an array of LONG's. The number of entries in the array
# must agree with what was described in the TAG_SERIES_INFO data.
with open(filename, 'rb') as fh:
trace = Trace()
trace.stats.y = AttribDict()
count = -1
while True:
endian, tag_type, next_tag, _next_same = _parse_tag(fh)
if tag_type == 1:
# TAG_STATION_INFO
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# UCHAR Station[5] (BLANKPAD)
# Station is the five letter SEED format station
# identification.
# UCHAR Location[2] (BLANKPAD)
# Location Location is the two letter SEED format location
# identification.
# UCHAR Channel[3] (BLANKPAD)
# Channel Channel is the three letter SEED format channel
# identification.
# UCHAR NetworkID[51] (ASCIIZ)
# This is some descriptive text identifying the network.
# UCHAR SiteName[61] (ASCIIZ)
# SiteName is some text identifying the site.
# UCHAR Comment[31] (ASCIIZ)
# Comment is any comment for this station.
# UCHAR SensorType[51] (ASCIIZ)
# SensorType is some text describing the type of sensor used
# at the station.
# UCHAR DataFormat[7] (ASCIIZ)
# DataFormat is some text describing the data format recorded
# at the station.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(b'5s2s3s51z61z31z51z7z',
data[8:])
trace.stats.station = parts[0]
trace.stats.location = parts[1]
trace.stats.channel = parts[2]
# extra
params = AttribDict()
params.network_id = parts[3]
params.site_name = parts[4]
params.comment = parts[5]
params.sensor_type = parts[6]
params.data_format = parts[7]
trace.stats.y.tag_station_info = params
elif tag_type == 2:
# TAG_STATION_LOCATION
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# FLOAT Latitude
# Latitude in degrees of the location of the station. The
# latitude should be between -90 (South) and +90 (North).
# FLOAT Longitude
# Longitude in degrees of the location of the station. The
# longitude should be between -180 (West) and +180 (East).
# FLOAT Elevation
# Elevation in meters above sea level of the station.
# FLOAT Depth
# Depth is the depth in meters of the sensor.
# FLOAT Azimuth
# Azimuth of the sensor in degrees clockwise.
# FLOAT Dip
# Dip is the dip of the sensor. 90 degrees is defined as
# vertical right way up.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(endian + b'ffffff',
data[8:])
params = AttribDict()
params.latitude = parts[0]
params.longitude = parts[1]
params.elevation = parts[2]
params.depth = parts[3]
params.azimuth = parts[4]
params.dip = parts[5]
trace.stats.y.tag_station_location = params
elif tag_type == 3:
# TAG_STATION_PARAMETERS
# UCHAR Update[16]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# REALTIME StartValidTime
# Time that the information in these records became valid.
# REALTIME EndValidTime
# Time that the information in these records became invalid.
# FLOAT Sensitivity
# Sensitivity of the sensor in nanometers per bit.
# FLOAT SensFreq
# Frequency at which the sensitivity was measured.
# FLOAT SampleRate
# This is the number of samples per second. This value can be
# less than 1.0. (i.e. 0.1)
# FLOAT MaxClkDrift
# Maximum drift rate of the clock in seconds per sample.
# UCHAR SensUnits[24] (ASCIIZ)
# Some text indicating the units in which the sensitivity was
# measured.
# UCHAR CalibUnits[24] (ASCIIZ)
# Some text indicating the units in which calibration input
# was measured.
# UCHAR ChanFlags[27] (BLANKPAD)
# Text indicating the channel flags according to the SEED
# definition.
# UCHAR UpdateFlag
# This flag must be “N” or “U” according to the SEED
# definition.
# UCHAR Filler[4]
# Filler Pads out the record to satisfy the alignment
# restrictions for reading data on a SPARC processor.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(
endian + b'ddffff24z24z27sc4s', data[16:])
trace.stats.sampling_rate = parts[4]
# extra
params = AttribDict()
params.start_valid_time = parts[0]
params.end_valid_time = parts[1]
params.sensitivity = parts[2]
params.sens_freq = parts[3]
params.sample_rate = parts[4]
params.max_clk_drift = parts[5]
params.sens_units = parts[6]
params.calib_units = parts[7]
params.chan_flags = parts[8]
params.update_flag = parts[9]
trace.stats.y.tag_station_parameters = params
elif tag_type == 4:
# TAG_STATION_DATABASE
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# REALTIME LoadDate
# Date the information was loaded into the database.
# UCHAR Key[16]
# Unique key that identifies this record in the database.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(endian + b'd16s',
data[8:])
params = AttribDict()
params.load_date = parts[0]
params.key = parts[1]
trace.stats.y.tag_station_database = params
elif tag_type == 5:
# TAG_SERIES_INFO
# UCHAR Update[16]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# REALTIME StartTime
# This is start time of the data in this series.
# REALTIME EndTime
# This is end time of the data in this series.
# ULONG NumSamples
# This is the number of samples of data in this series.
# LONG DCOffset
# DCOffset is the DC offset of the data.
# LONG MaxAmplitude
# MaxAmplitude is the maximum amplitude of the data.
# LONG MinAmplitude
# MinAmplitude is the minimum amplitude of the data.
# UCHAR Format[8] (ASCIIZ)
# This is the format of the data. This should always be
# “YFILE”.
# UCHAR FormatVersion[8] (ASCIIZ)
# FormatVersion is the version of the format of the data.
# This should always be “5.0”
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(endian + b'ddLlll8z8z',
data[16:])
trace.stats.starttime = UTCDateTime(parts[0])
count = parts[2]
# extra
params = AttribDict()
params.endtime = UTCDateTime(parts[1])
params.num_samples = parts[2]
params.dc_offset = parts[3]
params.max_amplitude = parts[4]
params.min_amplitude = parts[5]
params.format = parts[6]
params.format_version = parts[7]
trace.stats.y.tag_series_info = params
elif tag_type == 6:
# TAG_SERIES_DATABASE
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# REALTIME LoadDate
# Date the information was loaded into the database.
# UCHAR Key[16]
# Unique key that identifies this record in the database.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(endian + b'd16s',
data[8:])
params = AttribDict()
params.load_date = parts[0]
params.key = parts[1]
trace.stats.y.tag_series_database = params
elif tag_type == 26:
# TAG_STATION_RESPONSE
# UCHAR Update[8]
# This field is only used internally for administrative
# purposes. It should always be set to zeroes.
# UCHAR PathName[260]
# PathName is the full name of the file which contains the
# response information for this station.
data = fh.read(next_tag)
parts = _unpack_with_asciiz_and_decode(b'260s', data[8:])
params = AttribDict()
params.path_name = parts[0]
trace.stats.y.tag_station_response = params
elif tag_type == 7:
# TAG_DATA_INT32
trace.data = from_buffer(fh.read(
np.dtype(np.int32).itemsize * count),
dtype=np.int32)
# break loop as TAG_DATA_INT32 should be the last tag in file
break
else:
fh.seek(next_tag, 1)
return Stream([trace])
