def psd(station, parser):
data = IPOC()
ppsd_length = 6 * 3600
overlap = 0.5
dt = 3 * 24 * 3600
t1 = UTC('2006-01-01')
t2 = UTC('2013-11-01')
ppsd = None
print t1, t2
while t1 < t2:
try:
if station != 'LVC':
stream = data.client.getWaveform('CX', station, '', 'HHZ', t1, t1 + dt + overlap * ppsd_length)
else:
stream = data.client.getWaveform('GE', 'LVC', '00', 'BHZ', t1, t1 + dt + overlap * ppsd_length)
except:
t1 += dt
continue
if ppsd is None:
ppsd = PPSD(stream[0].stats, parser=parser, skip_on_gaps=True,
db_bins=(-200, -50, 0.5),
ppsd_length=ppsd_length, overlap=overlap)
print t1
ppsd.add(stream)
t1 += dt
if ppsd is not None:
print 'station %s: %d segments' % (station, len(ppsd.times))
ppsd.save("/home/richter/Results/IPOC/PPSD/ppsd_%s_6h.pkl.bz2" % station, compress=True)
return True
else:
return False
def ppsd(mseed_file,drop_value=9999):
'''
Get ppsd for one day file
'''
from obspy.signal import PPSD
from obspy import read
from numpy import where,mean,random
#get data
st=read(mseed_file)
# st[0].data=random.normal(0, 0.1, size=st[0].stats.npts)
#find gaps and not gaps
gaps=where(st[0].data==drop_value)[0]
not_gaps=where(st[0].data!=drop_value)[0]
#Find mean wthout taking gaps into account and remove it
bias=mean(st[0].data[not_gaps])
st[0].data=st[0].data-bias
#zero out gaps
st[0].data[gaps]=0
#define frequency response
paz = {'gain': 1.0,'sensitivity': 1.0,'poles': [1,],'zeros': [0j, 0j]}
#initalize ppsd object
ppsd = PPSD(st[0].stats, paz,db_bins=(-60, 10, 0.5),period_limits=[2,600],special_handling='ringlaser')
#add to ppsd
ppsd.add(st)
def plot_PPSD(trace, sta, start_time, interval=7200, filebase=None, show=True):
"""
Plot a Probabilistic Power Spectral Desnsity for the trace
trace = obspy Trace objet
sta = obspy Inventory/Station object corresponding to the trace
start_time = time at which to start spectra
interval=offset between PSDs (seconds, minimum=3600)
"""
now_time = trace.start_time
first_read = True
while now_time < trace.end_time - interval:
if first_read:
if trace.stats.component[1] == 'D':
ppsd = PPSD(trace.stats,
metadata=sta,
special_handling='hydrophone')
else:
ppsd = PPSD(trace.stats, metadata=sta)
first_read = False
ppsd.add(trace)
now_time += interval
ppsd.save_npz(f'{filebase}_PPSD.npz')
if filebase:
description = '{}.{}.{}.{}'.format(trace.stats.network,
trace.stats.station,
trace.stats.location,
trace.stats.channel)
ppsd.plot(filebase + '_' + description + '_PPSD.png')
if show:
plt.plot()
# ppsd.plot_temporal([0.1,1,10])
# ppsd.plot_spectrogram()
return 0
def get_power(trPower, inv, periodlist):
"""
Calculates the PSD using ObsPy PPSD.
trPower: input ObsPy trace that must be > 3600. sec long.
inv: ObsPy inventory. Must be level = 'response'
periodlist: a list of periods at which you want the power
returns: power in dB at the periods in periodlist
"""
from obspy.signal import PPSD
powers_at_periods = []
if (trPower.stats.delta * trPower.stats.npts > 3600.):
ppsd = PPSD(trPower.stats, metadata=inv)
ppsd.add(trPower)
if (len(ppsd._binned_psds) > 0):
psd_periods = ppsd._period_binning[2]
psd_power = []
psd_power = ppsd._binned_psds[0]
for i in range(0, len(periodlist)):
powers_at_periods.append(psd_power[np.argmin(
abs(psd_periods - periodlist[i]))])
else:
for i in range(0, len(periodlist)):
powers_at_periods.append(-1)
return powers_at_periods
def test_PPSD(self):
"""
Test PPSD routine with some real data. Data was downsampled to 100Hz
so the ppsd is a bit distorted which does not matter for the purpose
of testing.
"""
# load test file
file_data = os.path.join(self.path,
'BW.KW1._.EHZ.D.2011.090_downsampled.asc.gz')
file_histogram = os.path.join(self.path,
'BW.KW1._.EHZ.D.2011.090_downsampled__ppsd_hist_stack.npy')
file_binning = os.path.join(self.path,
'BW.KW1._.EHZ.D.2011.090_downsampled__ppsd_mixed.npz')
# parameters for the test
data = np.loadtxt(file_data)
stats = {'_format': 'MSEED',
'calib': 1.0,
'channel': 'EHZ',
'delta': 0.01,
'endtime': UTCDateTime(2011, 3, 31, 2, 36, 0, 180000),
'location': '',
'mseed': {'dataquality': 'D', 'record_length': 512,
'encoding': 'STEIM2', 'byteorder': '>'},
'network': 'BW',
'npts': 936001,
'sampling_rate': 100.0,
'starttime': UTCDateTime(2011, 3, 31, 0, 0, 0, 180000),
'station': 'KW1'}
tr = Trace(data, stats)
st = Stream([tr])
paz = {'gain': 60077000.0,
'poles': [(-0.037004 + 0.037016j), (-0.037004 - 0.037016j),
(-251.33 + 0j), (-131.04 - 467.29j),
(-131.04 + 467.29j)],
'sensitivity': 2516778400.0,
'zeros': [0j, 0j]}
ppsd = PPSD(tr.stats, paz)
ppsd.add(st)
# read results and compare
result_hist = np.load(file_histogram)
self.assertEqual(len(ppsd.times), 4)
self.assertEqual(ppsd.nfft, 65536)
self.assertEqual(ppsd.nlap, 49152)
np.testing.assert_array_equal(ppsd.hist_stack, result_hist)
# add the same data a second time (which should do nothing at all) and
# test again - but it will raise UserWarnings, which we omit for now
with warnings.catch_warnings(record=True):
warnings.simplefilter('ignore', UserWarning)
ppsd.add(st)
np.testing.assert_array_equal(ppsd.hist_stack, result_hist)
# test the binning arrays
binning = np.load(file_binning)
np.testing.assert_array_equal(ppsd.spec_bins, binning['spec_bins'])
np.testing.assert_array_equal(ppsd.period_bins, binning['period_bins'])
def psd_plot(station):
ppsd = PPSD.load("/home/richter/Results/IPOC/PPSD/ppsd_%s_6h.pkl.bz2" %
station)
ppsd.plot(show=False,
show_coverage=False,
show_percentiles=True,
period_lim=(0.05, 1000),
max_percentage=10)
fig = mpl.pyplot.figure(1)
ax = fig.axes[0]
ax.set_ylim((-200, -50))
ax.set_xticklabels(['%s' % t for t in ax.get_xticks()])
ax.set_xlabel('period (s)')
ax.set_ylabel('amplitude (dB)')
col = ax.collections[0]
bbox = col.colorbar.ax.get_position()
fig.delaxes(col.colorbar.ax)
if COLORBAR:
fig.colorbar(col,
cax=fig.add_axes(
[bbox.x0 + 0.1, bbox.y0, bbox.width, bbox.height]),
extend='max')
#col.colorbar.set_ticks(range(8))
col.colorbar.set_label('probability (%)')
bbox = ax.get_position()
ax.set_position(
[bbox.x0, bbox.y0, bbox.width + 0.2 - 0.1 * COLORBAR, bbox.height])
fig.savefig("/home/richter/Results/IPOC/PPSD/ppsd_%s_6h_cb.png" % station,
dpi=150,
bbox_inches='tight')
mpl.pyplot.close()
def correct_sensitivity(station):
ppsd = PPSD.load("/home/richter/Results/IPOC/PPSD/ppsd_%s_6h_wrong_gain.pkl" % station)
dB = 10 * np.log10(2516580000.0 / 629145000.0)
if station == 'PB14':
dB = 10 * np.log10(588000000.0 / 629145000.0)
ppsd.spec_bins = ppsd.spec_bins - dB
ppsd.yedges = ppsd.yedges - dB
ppsd.save("/home/richter/Results/IPOC/PPSD/ppsd_%s_6h.pkl" % station)
def correct_sensitivity(station):
ppsd = PPSD.load(
"/home/richter/Results/IPOC/PPSD/ppsd_%s_6h_wrong_gain.pkl" % station)
dB = 10 * np.log10(2516580000.0 / 629145000.0)
if station == 'PB14':
dB = 10 * np.log10(588000000.0 / 629145000.0)
ppsd.spec_bins = ppsd.spec_bins - dB
ppsd.yedges = ppsd.yedges - dB
ppsd.save("/home/richter/Results/IPOC/PPSD/ppsd_%s_6h.pkl" % station)
def get_all_values(self, nested_dictionary):
for key, value in nested_dictionary.items():
if self.check == False:
if type(value) is dict:
nested_dictionary[key] = self.get_all_values(value)
else:
files = []
process_list = []
if type(value[0]) == list:
for j in value[0]:
st = read(j)
files.append(st[0])
ppsd = value[1]
else:
for j in value:
st = read(j)
files.append(st[0])
try:
ppsd = PPSD(
files[0].stats,
metadata=self.metadata,
ppsd_length=self.length,
overlap=self.overlap,
period_smoothing_width_octaves=self.smoothing,
period_step_octaves=self.period)
except:
pass
for i, j in zip(files, value):
try:
if self.check == False:
ppsd.add(i)
self.processedFiles = self.processedFiles + 1
print(i, " processed")
self.fileProcessed.emit(self.processedFiles)
else:
process_list.append(j)
except:
process_list.append(j)
nested_dictionary[key] = [process_list, ppsd]
return nested_dictionary
def below_noise_model(station, data, inv, save_plot=False):
tr = df_to_trace(station, data)
ppsd = PPSD(tr.stats, metadata=inv)
ppsd.add(tr)
fig = ppsd.plot(show=False)
if save_plot:
julday = format_date_to_str(tr.stats.starttime.julday, 3)
fig.savefig(
f"plot_data/psd/{station}/{tr.stats.starttime.year}.{julday}.png",
dpi=300)
nlnm_t, nlnm_db = get_nlnm()
trace_t = ppsd.period_bin_centers.tolist()
interp_func = interpolate.interp1d(nlnm_t, nlnm_db, bounds_error=False)
interp_db = interp_func(trace_t)
traces_db = ppsd.psd_values
min_t = closest_index_of_list(trace_t, 2.5)
max_t = closest_index_of_list(trace_t, 10)
for t, trace_db in enumerate(traces_db):
diff = np.substract(trace_db[min_t:max_t + 1], interp_db[min_t,
max_t + 1])
for i, element in enumerate(diff):
if element < 0:
time_processed = ppsd.times_processed[t]
year = format_date_to_str(time_processed.year, 4)
month = format_date_to_str(time_processed.month, 2)
day = format_date_to_str(time_processed.day, 2)
hour = format_date_to_str(time_processed.hour, 2)
minute = format_date_to_str(time_processed.minute, 2)
second = format_date_to_str(time_processed.second, 2)
datetime = f'D{year}{month}{day}T{hour}{minute}{second}'
_id = station + '.' + datetime + '.1'
return datetime, f'{str(element)}dB', _id, 1, 'Below Low Noise Model', station
return None, f'OK. BelowLowNoiseModel of {station}', None, 0, None, None
def plot_by_path(NPZ_path):
NPZ = PPSD.load_npz(NPZ_path)
NPZ_name = f'{NPZ.network}.{NPZ.station}.{NPZ.location}.{NPZ.channel}.png'
print(f'graficando {NPZ_name}')
NPZ.plot(NPZ_name,
cmap=pqlx,
show_histogram=True,
show_percentiles=True,
percentiles=[90, 10],
period_lim=(0.1, 179))
return NPZ
def ppsd_NPZ(npz_paths):
npz_0 = PPSD.load_npz(npz_paths[0])
print(f'{npz_0.network}.{npz_0.station}.{npz_0.location}.{npz_0.channel}')
for npz in npz_paths[1:]:
npz_0.add_npz(npz)
# try: npz_0.add_npz(npz)
# except AssertionError as e: print(str(e))
print('saving')
save_NPZ(npz_0)
return npz_0
def Noise_plotting(station, channel, PAZ, datasource):
"""
Function to make use of obspy's PPSD functionality to read in data from
a single station and the poles-and-zeros for that station before plotting
the PPSD for this station. See McNamara(2004) for more details.
:type station: String
:param station: Station name as it is in the filenames in the database
:type channel: String
:param channel: Channel name as it is in the filenames in the database
:type PAZ: Dict
:param PAZ: Must contain, Poles, Zeros, Sensitivity, Gain
:type Poles: List of Complex
:type Zeros: List of Complex
:type Sensitivity: Float
:type Gain: Float
:type datasource: String
:param datasource: The directory in which data can be found, can contain
wildcards.
:returns: PPSD object
"""
from obspy.signal import PPSD
from obspy import read as obsread
import glob
stafiles=glob.glob(datasource+'/*'+station+'*'+channel+'*')
stafiles.sort()
# Initialize PPSD
st=obsread(stafiles[0])
ppsd = PPSD(st[0].stats, PAZ)
for stafile in stafiles[1:]:
print 'Adding waveform from: '+stafile
st=obsread(stafile)
# Add after read to conserve memory
ppsd.add(st)
# Plot the PPSD
ppsd.plot()
return ppsd
def ppsd(self,
fmin=1.,
fmax=100.,
special_handling=None,
filename=None,
save=False):
"""
Function that calculates the probabilistic power spectral density
of a given station-channel combination.
:type fmin: float
:param fmin: Minimum frequency to show in PPSD plot
:type fmax: float
:param fmax: Maximum frequency to show in PPSD plot
"""
# read list of files
files = np.genfromtxt(self.filist, dtype=str)
n = files.size
# if no paz information is given, divide by 1.0
if self.metadata == None:
self.metadata = {"sensitivity": 1.0}
# loop over files
for i in range(n):
st = read(self.path + files[i])
st.merge()
#st.decimate(self.dec_fact)
if len(st) > 1:
warnings.warn("more than one trace in st")
tr = st.select(station=self.stn, channel=self.chn)[0]
# at first run, initialize PPSD instance
if i == 0:
# "is_rotational_data" is set in order not to differentiate that data
inst = PPSD(tr.stats,
metadata=self.metadata,
special_handling=special_handling,
ppsd_length=1800.)
# add trace
print("add trace %s ..." % tr)
inst.add(tr)
print("number of psd segments:", len(inst.current_times_used))
inst.plot(show_noise_models=True,
xaxis_frequency=True,
period_lim=(fmin, fmax),
filename=filename)
if save:
inst.save_npz("ppsd_%s_%s.npz" % (self.stn, self.chn))
def multi_component_ppsd(comp,specific,avg='median'):
"""Return average arrays for each station in a given folder, choice of
component, can be N/E/Z/*
Choice of average available from the PPSD objects
"""
fidtmplt = "RD{s:0>2}.HH{c}*"
if specific == "ALL":
folder = 'FATHOM/*'
else:
folder = 'FATHOM/{}'.format(specific)
folderlist = glob.glob(os.path.join(pathnames()['ppsd'],folder))
averages,stations = [],[]
for F in folderlist:
print(F)
# check range of stations available using filenames
allfiles = glob.glob(os.path.join(F,'*HH*.npz'))
allfiles.sort()
lowrange = int(os.path.basename(allfiles[0]).split('.')[0][2:])
highrange = int(os.path.basename(allfiles[-1]).split('.')[0][2:])
# loop by station and inner loop by components per station
for i in range(lowrange,highrange+1):
files = glob.glob(os.path.join(F,fidtmplt.format(s=i,c=comp)))
if not files:
print('no files',fidtmplt.format(s=i,c=comp))
continue
inneraverages = []
for fid in files:
ppsd = PPSD.load_npz(fid)
if avg == 'mean':
average = ppsd.get_mean()
elif avg == 'mode':
average = ppsd.get_mode()
elif avg == 'median':
average = ppsd.get_percentile(percentile=50)
inneraverages.append(average[1])
outeraverage = np.mean(np.array(inneraverages),axis=0)
averages.append(outeraverage)
stations.append("RD{:0>2}".format(i))
periods = average[0]
if specific == "ALL":
stations,averages = collapse_stations(stations,averages)
return stations,periods,averages
def psd_plot(station):
ppsd = PPSD.load("/home/richter/Results/IPOC/PPSD/ppsd_%s_6h.pkl.bz2" % station)
ppsd.plot(show=False, show_coverage=False, show_percentiles=True,
period_lim=(0.05, 1000), max_percentage=10)
fig = mpl.pyplot.figure(1)
ax = fig.axes[0]
ax.set_ylim((-200, -50))
ax.set_xticklabels(['%s' % t for t in ax.get_xticks()])
ax.set_xlabel('period (s)')
ax.set_ylabel('amplitude (dB)')
col = ax.collections[0]
bbox = col.colorbar.ax.get_position()
fig.delaxes(col.colorbar.ax)
if COLORBAR:
fig.colorbar(col, cax=fig.add_axes([bbox.x0 + 0.1, bbox.y0, bbox.width, bbox.height]),
extend='max')
#col.colorbar.set_ticks(range(8))
col.colorbar.set_label('probability (%)')
bbox = ax.get_position()
ax.set_position([bbox.x0, bbox.y0, bbox.width + 0.2 - 0.1 * COLORBAR, bbox.height])
fig.savefig("/home/richter/Results/IPOC/PPSD/ppsd_%s_6h_cb.png" % station, dpi=150, bbox_inches='tight')
mpl.pyplot.close()
def _colormap_plot_ppsd(cmaps):
"""
Plot for illustrating colormaps: PPSD.
:param cmaps: list of :class:`~matplotlib.colors.Colormap`
:rtype: None
"""
import matplotlib.pyplot as plt
from obspy import read
from obspy.signal import PPSD
from obspy.io.xseed import Parser
st = read("https://examples.obspy.org/BW.KW1..EHZ.D.2011.037")
st += read("https://examples.obspy.org/BW.KW1..EHZ.D.2011.038")
parser = Parser("https://examples.obspy.org/dataless.seed.BW_KW1")
ppsd = PPSD(st[0].stats, metadata=parser)
ppsd.add(st)
for cmap in cmaps:
ppsd.plot(cmap=cmap, show=False)
plt.show()
def ppsd(self, fmin=1., fmax=100., special_handling=None, filename=None, save=False):
"""
Function that calculates the probabilistic power spectral density
of a given station-channel combination.
:type fmin: float
:param fmin: Minimum frequency to show in PPSD plot
:type fmax: float
:param fmax: Maximum frequency to show in PPSD plot
"""
# read list of files
files = np.genfromtxt(self.filist, dtype=str)
n = files.size
# if no paz information is given, divide by 1.0
if self.metadata == None:
self.metadata = {"sensitivity": 1.0}
# loop over files
for i in range(n):
st = read(files[i])
st.merge()
st.decimate(self.dec_fact)
if len(st) > 1:
warnings.warn("more than one trace in st")
tr = st.select(station=self.stn, channel=self.chn)[0]
# at first run, initialize PPSD instance
if i == 0:
# "is_rotational_data" is set in order not to differentiate that data
inst = PPSD(tr.stats, metadata=self.metadata, special_handling=special_handling)
# add trace
print("add trace %s ..." % tr)
inst.add(tr)
print("number of psd segments:", len(inst.current_times_used))
inst.plot(show_noise_models=True, xaxis_frequency=True, period_lim=(fmin, fmax), filename=filename)
if save:
inst.save_npz("ppsd_%s_%s.npz" % (self.stn, self.chn))
def psd(station, parser):
data = IPOC()
ppsd_length = 6 * 3600
overlap = 0.5
dt = 3 * 24 * 3600
t1 = UTC('2006-01-01')
t2 = UTC('2013-11-01')
ppsd = None
print t1, t2
while t1 < t2:
try:
if station != 'LVC':
stream = data.client.getWaveform(
'CX', station, '', 'HHZ', t1,
t1 + dt + overlap * ppsd_length)
else:
stream = data.client.getWaveform(
'GE', 'LVC', '00', 'BHZ', t1,
t1 + dt + overlap * ppsd_length)
except:
t1 += dt
continue
if ppsd is None:
ppsd = PPSD(stream[0].stats,
parser=parser,
skip_on_gaps=True,
db_bins=(-200, -50, 0.5),
ppsd_length=ppsd_length,
overlap=overlap)
print t1
ppsd.add(stream)
t1 += dt
if ppsd is not None:
print 'station %s: %d segments' % (station, len(ppsd.times))
ppsd.save("/home/richter/Results/IPOC/PPSD/ppsd_%s_6h.pkl.bz2" %
station,
compress=True)
return True
else:
return False
def Noise_plotting(station, channel, PAZ, datasource):
"""
Function to make use of obspy's PPSD functionality to read in data from
a single station and the poles-and-zeros for that station before plotting
the PPSD for this station. See McNamara(2004) for more details.
:type station: String
:param station: Station name as it is in the filenames in the database
:type channel: String
:param channel: Channel name as it is in the filenames in the database
:type PAZ: Dict
:param PAZ: Must contain, Poles, Zeros, Sensitivity, Gain
:type Poles: List of Complex
:type Zeros: List of Complex
:type Sensitivity: Float
:type Gain: Float
:type datasource: String
:param datasource: The directory in which data can be found, can contain
wildcards.
:returns: PPSD object
"""
from obspy.signal import PPSD
from obspy import read as obsread
import glob
stafiles = glob.glob(datasource + '/*' + station + '*' + channel + '*')
stafiles.sort()
# Initialize PPSD
st = obsread(stafiles[0])
ppsd = PPSD(st[0].stats, PAZ)
for stafile in stafiles[1:]:
print 'Adding waveform from: ' + stafile
st = obsread(stafile)
# Add after read to conserve memory
ppsd.add(st)
# Plot the PPSD
ppsd.plot()
return ppsd
def analyze_noise(data_files,response,decimateby=5):
"""run through data files and create PPSD objects using obsy
"""
data_files.sort()
print("++ {} data files".format(len(data_files)))
inv = read_inventory(response)
# initialize PPSD with first datafile
print("1/{} Initializing with data file: ".format(len(data_files)),
os.path.basename(data_files[0]),end='... ')
start = time.time()
st = read(data_files[0])
if decimateby != 0:
st.decimate(decimateby)
ppsd = PPSD(st[0].stats, metadata=inv)
ppsd.add(st)
year_start = st[0].stats.starttime.year
jday_start = st[0].stats.starttime.julday
end = time.time()
print("complete ({}s)".format(round(end-start,2)))
# loop over rest of datafiles and add to ppsd
for i,filename in enumerate(data_files[1:]):
print('{0}/{1} {2}'.format(i+2,len(data_files),
os.path.basename(filename)),end='... ')
try:
start = time.time()
st = read(filename)
if decimateby != 0:
st.decimate(decimateby)
ppsd.add(st)
end = time.time()
print("complete ({}s)".format(round(end-start,2)))
except Exception as e:
print(e)
pass
return ppsd
def _get_ppsd(self, st_info):
#Devuelve la ppsd del stream st dado en el parametro de la función
try:
st = st_info[0]
ppsd_dir = st_info[2]
print("HOLA ", ppsd_dir)
tr = st[0]
ppsd = PPSD(tr.stats,
metadata=self.parser,
skip_on_gaps=True,
overlapping=0.5)
ppsd.add(st)
ppsd.save_npz(ppsd_dir)
return [ppsd, ppsd_dir]
except Exception as e:
msg = (
f"Error getting ppsd from: {st_info[1]} for the next reason:\n"
"%s: %s\n"
"Skipping this stream.")
msg = msg % (e.__class__.__name__, str(e))
warnings.warn(msg)
pass
st = read('RAW/*', debug_headers=True)
k = len(st)
i = 0
while i < k:
tr = st[i]
print(tr.id, tr.stats.starttime.year, tr.stats.starttime.julday)
### differential Volcity to Accerletion
diftr = obspy.core.trace.Trace.differentiate(tr)
### Pole and Zero
chn = tr.stats.channel
sta = tr.stats.station
Ps = "PZs/" + "*" + sta + "*" + "HHZ" + "*"
HH_paz = glob.glob(Ps)
pz.attach_paz(diftr, HH_paz[0])
paz = dict(diftr.stats.paz)
### power spectrum density
ppsd = PPSD(diftr.stats, paz, ppsd_length=3600.0, overlap=0.95)
ppsd.add(diftr)
[t, amp] = ppsd.get_mode()
### Output
ts = str(tr.stats.starttime.year) + "." + str(tr.stats.starttime.julday)
txt = sta + "." + chn + "." + ts + ".txt"
with open(txt, mode="w") as f:
for j in range(len(t)):
f.write("%e %6.2f\n" % (t[j], amp[j]))
i += 1
import matplotlib.pyplot as plt
from obspy.signal.spectral_estimation import get_nlnm, get_nhnm
import matplotlib as mpl
mpl.rc('font', family='serif')
mpl.rc('font', serif='Times')
mpl.rc('text', usetex=True)
mpl.rc('font', size=18)
files = glob.glob("*.npz")
files = list(files)
files.sort()
fig = plt.figure(1, figsize=(12, 12))
for curfile in files:
ppsd = PPSD.load_npz(curfile)
#ppsd.plot()
pers, means = ppsd.get_mean()
if 'XX' in curfile:
stalab = 'GS 005 ' + curfile.split('.')[3].replace('PDF', '')
else:
stalab = curfile.split('.')[0] + ' ' + curfile.split(
'.')[1] + ' ' + curfile.split('.')[3].replace('PDF', '')
plt.semilogx(1. / pers, means, label=stalab, linewidth=2)
NHNMper, NHNMpower = get_nhnm()
NLNMper, NLNMpower = get_nlnm()
plt.semilogx(1. / NLNMper, NLNMpower, linewidth=3., color='k')
plt.semilogx(1. / NHNMper, NHNMpower, linewidth=3., color='k')
plt.xlabel('Frequency (Hz)')
plt.legend(loc=9, ncol=3)
ondeckdatafile = "datafiles/ELYS0.allseispress.dl0204.mseed"
# ondeckmetadata = "datafiles/ELYS0.dl0129.response.xml"
ondeckmetadata = "datafiles/ELYS0.all.dl0226.response.xml"
cMHdata = "datafiles/CRUI3.SP.mseed"
cMHmeta = "datafiles/CRUI3.xml"
cEHdata = "datafiles/CRUI1-2.mseed"
cEHmeta = "datafiles/CRUI1.xml"
#first get cruise ppsd info
print("Working on cruise data")
stMHc = read(cMHdata)
invMHc = read_inventory(cMHmeta)
stMHc_sel = stMHc.select(channel='MHW')
trc = stMHc_sel[0]
ppsdMHc = PPSD(trc.stats, metadata=invMHc, ppsd_length=600.0,
skip_on_gaps=True, period_limits=(0.02, 100.0),
db_bins=(-200, -50, 1.))
ppsdMHc.add(stMHc_sel)
(cMHpd, cMHpsd) = ppsdMHc.get_mode()
stEHc = read(cEHdata)
invEHc = read_inventory(cEHmeta)
stEHc_sel = stEHc.select(channel='EHW')
trc = stEHc_sel[0]
ppsdEHc = PPSD(trc.stats, metadata=invEHc, ppsd_length=200.0,
skip_on_gaps=True, period_limits=(0.02, 100.0),
db_bins=(-200, -50, 1.))
ppsdEHc.add(stEHc_sel)
(cEHpd, cEHpsd) = ppsdEHc.get_mode()
# For reference, earth low and high noise models
(nlnmpd, nlnmpsd) = get_nlnm()
def process(self, SDSFile):
"""
def PSDCollector::process
Processes a single SDSFile to extract PSDs and store them in the DB
"""
# Create an empty spectra list that can be preemptively
# returned to the calling procedure
spectra = list()
inventory = SDSFile.inventory
# Inventory could not be read
if inventory is None:
raise Exception("Inventory could not be read")
# And the prepared data
data = self.__prepareData(SDSFile)
# Data could not be read
if data is None:
raise Exception("Data could not be read")
# Try creating the PPSD
try:
# Set handling to hydrophone if using pressure data
# This is a bit hacky but the process should be the same for infrasound data
handling = "hydrophone" if SDSFile.isPressureChannel else None
ppsd = PPSD(data[0].stats,
inventory,
period_limits=self.PERIOD_LIMIT_TUPLE,
special_handling=handling)
# Add the waveform
ppsd.add(data)
except Exception as ex:
raise Exception("Error processing PPSD: '%s'" % (str(ex)))
for segment, time in zip(ppsd._binned_psds, SDSFile.psdBins):
# XXX NOTE:
# Modified /home/ubuntu/.local/lib/python2.7/site-packages/obspy/signal/spectral_estimation.py
# And /usr/local/lib/python3.5/dist-packages/obspy/signal/spectral_estimation.py
# To set ppsd.valid as a public attribute! We need this to determine the offset on the frequency axis
try:
psd_array = self.__getFrequencyOffset(
segment, ppsd.valid, SDSFile.isPressureChannel)
byteAmplitudes = self.__toByteArray(psd_array)
# This may fail in multiple ways.. try the next segment
except Exception as ex:
self.logger.warning(
"Failed processing PPSD for 1 segment: '%s'" % (str(ex)))
continue
# Add hash of the data & metadata (first 8 hex digits)
# Saving 64 bytes * 2 makes (checksums) our database pretty big and this should be sufficient to
# detect changes
psdObject = {
"fileId": SDSFile.filename,
"checksum": SDSFile.checksum,
"checksumInventory": self.__getResponseChecksum(inventory),
"net": SDSFile.net,
"sta": SDSFile.sta,
"loc": SDSFile.loc,
"cha": SDSFile.cha,
"quality": SDSFile.quality,
"ts": time.datetime,
"te": (time + timedelta(minutes=60)).datetime,
"bin": byteAmplitudes
}
spectra.append(psdObject)
return spectra
def addNetDemo(fSrcDir, static_path):
STATIC_PATH = static_path
sDenDir = 'networks'
fDenDir = os.path.join(STATIC_PATH, sDenDir)
mkfile(fDenDir, 0)
# updateSql() # 删除旧数据并更新数据库
all_files = []
all_paths = []
all_files, all_paths = show_path(fSrcDir, all_files, all_paths)
for i in range(0, len(all_files)):
file = all_files[i]
path = all_paths[i]
if file.count('.') >= 6:
dayCount = countDay_1OfYear(datetime.date.today())
(NetCode, StaCode, LocCode, ChCode, DataCode, nYear,
nDay) = file.split('.')
if (len(NetCode) <= 2 and len(StaCode) <= 5 and len(LocCode) <= 2
and len(ChCode) <= 3 and DataCode == 'D'
and len(nYear) <= 4 and len(nDay) <= 3
and int(nDay) == dayCount):
net = Network(NetCode, NetCode, fSrcDir, sDenDir,
3).get_or_create_Network()
sta = Station(net, StaCode, StaCode).get_or_create_Station()
cDigitizerInfo = DigitizerInfo('TDE-324', '10Vpp', '100Hz',
'Linear')
(bRet, AD, gain, rate,
filter) = cDigitizerInfo.getDigitizerInfo()
if not bRet:
print('Digitizer not found!')
continue
cSensorInfo = SensorInfo('TMA-33')
(bRet, sensor, sensorinfo) = cSensorInfo.getSensorInfo()
if not bRet:
print('Sensor not found!')
continue
adsensor = ADSensor(filter, sensorinfo).get_ADSensor()
sta_adsensor = Sta_ADSensor(
sta, adsensor).get_or_create_Sta_ADSensor()
ch = Channel(sta_adsensor, LocCode, ChCode).get_or_create_CH()
sDenDir2 = sDenDir + '/' + NetCode
fDenDir = os.path.join(STATIC_PATH, sDenDir2)
mkfile(fDenDir, 0)
sDenDir2 = sDenDir2 + '/' + StaCode
fDenDir = os.path.join(STATIC_PATH, sDenDir2)
mkfile(fDenDir, 0)
sDenDir2 = sDenDir2 + '/' + nYear
fDenDir = os.path.join(STATIC_PATH, sDenDir2)
mkfile(fDenDir, 0)
sDenDir2 = sDenDir2 + '/' + nDay
fDenDir = os.path.join(STATIC_PATH, sDenDir2)
mkfile(fDenDir, 0)
from obspy import read
# from obspy.io.xseed import Parser
from obspy.signal import PPSD
from obspy.imaging.cm import pqlx
try:
st = read(path)
except Exception as ex:
print('%s数据读取错误\n' % file, ex)
continue
ChName = NetCode + '.' + StaCode + '.' + LocCode + '.' + ChCode + '.' + nYear + '.' + nDay
outfile1 = fDenDir + '/' + ChName + '.day_wave.png'
outfile2 = fDenDir + '/' + ChName + '.day_wave.low_pass_0.2Hz.png'
outfile3 = fDenDir + '/' + ChName + '.day_wave.high_pass_0.2Hz.png'
outfile4 = fDenDir + '/' + ChName + '.ppsd.png'
outfile5 = fDenDir + '/' + ChName + '.spectrogram.png'
print(NetCode, StaCode, LocCode, ChCode, DataCode, nYear, nDay)
st.plot(size=(1600, 1200),
tick_format='%I:%M:%p',
type="dayplot",
interval=30,
right_vertical_labels=True,
vertical_scaling_range=st[0].data.std() * 20,
one_tick_per_line=True,
color=["r", "b", "g"],
show_y_UTC_label=True,
title=ChName,
time_offset=8,
outfile=outfile1)
st2 = st.copy()
st.filter("lowpass", freq=0.2, corners=2)
st.plot(size=(1600, 1200),
tick_format='%I:%M:%p',
type="dayplot",
interval=30,
right_vertical_labels=True,
vertical_scaling_range=st[0].data.std() * 20,
one_tick_per_line=True,
color=["r", "b", "g"],
show_y_UTC_label=True,
title=ChName + '.low_pass 0.2Hz',
time_offset=8,
outfile=outfile2)
st2.filter("highpass", freq=0.2)
st2.plot(
size=(1600, 1200),
tick_format='%I:%M:%p',
type="dayplot",
interval=30,
right_vertical_labels=True,
vertical_scaling_range=st2[0].data.std() * 20,
one_tick_per_line=True,
color=["r", "b", "g"],
show_y_UTC_label=True,
# events={"min_magnitude": 5},
title=ChName + '.high_pass 0.2Hz',
time_offset=8,
outfile=outfile3)
paz = {}
paz['zeros'] = []
paz['zeros'] = Zeros(sensorinfo).getZero()
paz['poles'] = []
paz['poles'] = Poles(sensorinfo).getPole()
if 2000 <= cSensorInfo.getField('IMainType', sensor) <= 3000:
paz['zeros'].append(complex(0., 0))
paz['gain'] = cSensorInfo.getField('IGainNormalization',
sensorinfo)
paz['sensitivity'] = cSensorInfo.getField('IGain', sensorinfo) \
* cDigitizerInfo.getField('sensitivity', filter)
print(paz)
st = read(path)
# print(st)
ppsd = PPSD(st[0].stats, paz)
ppsd.add(st)
# print(ppsd.times_data)
# print('len=',len(ppsd.times_data),ppsd.times_data[0][0],ppsd.times_data[0][1])
ppsd.plot(outfile4, xaxis_frequency=True, cmap=pqlx)
ppsd.plot_spectrogram(filename=outfile5, cmap='CMRmap_r')
if cSensorInfo.getField('IMainType', sensor) < 2000:
outfile6 = fDenDir + '/' + ChName + '.1-2s.sp.png'
ppsd.plot_temporal(1.414, filename=outfile6)
elif 2000 <= cSensorInfo.getField('IMainType',
sensor) < 3000: # 加速度模式)
outfile6 = fDenDir + '/' + ChName + '.1-2Hz.sp.png'
ppsd.plot_temporal(.707, filename=outfile6)
fBlankTime = 0.
for i in range(1, len(ppsd.times_data)): # 1个整时间段说明未丢数
dt = (ppsd.times_data[i][0] - ppsd.times_data[i - 1][1])
if dt < 0:
print(dt, ppsd.times_data[i][0],
ppsd.times_data[i - 1][1])
else:
fBlankTime += dt
runrate = 1.0 - fBlankTime / 86400.
date = datetime.date(ppsd.times_data[0][0].year,
ppsd.times_data[0][0].month,
ppsd.times_data[0][0].day)
DayData(ch, date, runrate).set_or_create_Day_data()
else:
print(file, "Name is error.")
channel=chan,
starttime=UTCDateTime('2004-001T00:00:00.0'),
endtime=day + secperday,
filename=respfilename(ch))
resp = irisclient.evalresp(network,
station,
loc,
chan,
filename="%s%s.png" % (qcfigs, ch),
output='plot')
except:
print("No response data for channel %s" % (ch))
data = {}
for ch in ids:
print(respfilename(ch))
stch = st.select(id=ch) # Just take the data for a single channel
calc_daily_stats(stch)
try:
ppsd = PPSD(stch[0].stats, metadata=str(respfilename(ch)))
ppsd.add(stch)
figname = "%s%d/%03d/%s.png" % (qcfigs, day.year, day.julday, ch)
path_verify(figname)
ppsd.plot(figname, cmap=pqlx)
data = ppsd.get_percentile(percentile=50)
fname = "%s%d/%03d/PPSDper50_%s.npz" % (qcdata, day.year,
day.julday, ch)
path_verify(fname)
np.savez(fname, data)
except:
print("Error with PPSD for %s check for response" % (ch))
from obspy import read
from obspy.signal import PPSD
from obspy.imaging.cm import pqlx
from obspy.io.xseed import Parser
st = read("IN.ZIRO..SHZ.D.2020.092.000051.SAC")
parser = Parser("ZIRODATALESS.SEED")
ppsd = PPSD(st[0].stats, metadata=parser)
ppsd.add(st)
st = read("IN.ZIRO..SHZ.D.2020.092.000051.SAC")
ppsd.add(st)
ppsd.plot(cmap=pqlx)
st.merge() except: 'Unable to read the data' sys.exit() if debug: for tr in st: print 'Here is the data stream: ' + str(tr) print 'Here is the window length of your PSDs: ' + str(parserval.len) print 'Here is the overlap: ' + str(parserval.overlap) #Make the PDF ppsd = PPSD(st[0].stats,paz=pazval,ppsd_length=parserval.len,overlap=parserval.overlap) for tr in st: ppsd.add(tr) if debug: for pdftime in ppsd.times: print 'Here is what is in the PDF: ' + str(pdftime) try: pdfstring = "PDF" + st[0].stats.station + st[0].stats.channel + str(st[0].stats.starttime.year)+ \ str(st[0].stats.starttime.julday).zfill(3) + ".jpg" medianstring = "MEDIAN" + st[0].stats.station + st[0].stats.channel + str(st[0].stats.starttime.year)+ \ str(st[0].stats.starttime.julday).zfill(3) if debug: print 'Saving the PDF to : ' + pdfstring print 'Saving the median to : ' + medianstring ppsd.plot(show_percentiles=True,percentiles=[50], filename=pdfstring,
from obspy.core import read
from obspy.xseed import Parser
from obspy.signal import PPSD
st = read("http://examples.obspy.org/BW.KW1..EHZ.D.2011.037")
tr = st.select(id="BW.KW1..EHZ")[0]
parser = Parser("http://examples.obspy.org/dataless.seed.BW_KW1")
paz = parser.getPAZ(tr.id)
ppsd = PPSD(tr.stats, paz)
ppsd.add(st)
st = read("http://examples.obspy.org/BW.KW1..EHZ.D.2011.038")
ppsd.add(st)
ppsd.plot()
def plotpowermagnitudeSpectrum(tr):
print('plotting magnitude spectrum....')
ppsd = PPSD(tr.stats, metadata=' ')
ppsd.add(tr)
ppsd.plot()
return
else:
sts = st.select(channel=channels[i], location=locations[i])
# Fix to remove overlaps, but not mask the data
sts = sts.merge()
sts = sts.split()
sts.sort(keys=['starttime', 'endtime', 'channel'])
print(sts)
for j, tr in enumerate(sts):
print("Working on trace {}".format(j))
print(tr)
length = tr.stats['endtime'] - tr.stats['starttime']
cumlen = cumlen + length
nevents_tr = nevents*length/secyear
ppsd = PPSD(tr.stats, metadata=inv, ppsd_length=200.0)
ppsd.add(Stream(tr))
psdmean = 0
for period in psdperiodrange:
psds = ppsd.extract_psd_values(period)[0]
psdmean = psdmean + math.pow(10.0, 0.05*np.mean(psds))
psdamp = psdmean/len(psdperiodrange)
threshold = psdamp*snr
print("{} Threshold: {}".format(j,threshold))
nev_tr = np.zeros_like(nevents)
for k, mag in enumerate(magarray):
idx = next((x for x, v in enumerate(amp_mag_dist[k][::-1])
if v>threshold), None)
if idx is not None:
idx = len(distarray)-idx-1
nev_tr[:, :, k] = afrac[idx]*nevents_tr[:, :, k]
def test_PPSD(self):
"""
Test PPSD routine with some real data. Data was downsampled to 100Hz
so the ppsd is a bit distorted which does not matter for the purpose
of testing.
"""
# load test file
file_data = os.path.join(self.path,
'BW.KW1._.EHZ.D.2011.090_downsampled.asc.gz')
file_histogram = os.path.join(
self.path,
'BW.KW1._.EHZ.D.2011.090_downsampled__ppsd_hist_stack.npy')
file_binning = os.path.join(
self.path, 'BW.KW1._.EHZ.D.2011.090_downsampled__ppsd_mixed.npz')
# parameters for the test
data = np.loadtxt(file_data)
stats = {
'_format': 'MSEED',
'calib': 1.0,
'channel': 'EHZ',
'delta': 0.01,
'endtime': UTCDateTime(2011, 3, 31, 2, 36, 0, 180000),
'location': '',
'mseed': {
'dataquality': 'D',
'record_length': 512,
'encoding': 'STEIM2',
'byteorder': '>'
},
'network': 'BW',
'npts': 936001,
'sampling_rate': 100.0,
'starttime': UTCDateTime(2011, 3, 31, 0, 0, 0, 180000),
'station': 'KW1'
}
tr = Trace(data, stats)
st = Stream([tr])
paz = {
'gain':
60077000.0,
'poles': [(-0.037004 + 0.037016j), (-0.037004 - 0.037016j),
(-251.33 + 0j), (-131.04 - 467.29j),
(-131.04 + 467.29j)],
'sensitivity':
2516778400.0,
'zeros': [0j, 0j]
}
ppsd = PPSD(tr.stats, paz)
ppsd.add(st)
# read results and compare
result_hist = np.load(file_histogram)
self.assertEqual(len(ppsd.times), 4)
self.assertEqual(ppsd.nfft, 65536)
self.assertEqual(ppsd.nlap, 49152)
np.testing.assert_array_equal(ppsd.hist_stack, result_hist)
# add the same data a second time (which should do nothing at all) and
# test again - but it will raise UserWarnings, which we omit for now
with warnings.catch_warnings(record=True):
warnings.simplefilter('ignore', UserWarning)
ppsd.add(st)
np.testing.assert_array_equal(ppsd.hist_stack, result_hist)
# test the binning arrays
binning = np.load(file_binning)
np.testing.assert_array_equal(ppsd.spec_bins, binning['spec_bins'])
np.testing.assert_array_equal(ppsd.period_bins, binning['period_bins'])
from obspy.core.util.base import get_example_file
from obspy.signal import PPSD
ppsd = PPSD.load_npz(get_example_file('ppsd_kw1_ehz.npz'))
ppsd.plot_temporal([0.1, 1, 10])
def get_ppsd(my_storage,
client,
inv,
ppsd_restrictions,
single_cha_contents,
starttime,
endtime,
plot_trace=False):
"""
Calculates the ppsd object according to starttime, endtime
and ppsd_restrictions parameters. It will be save in
my_storage/{network}.{station}.{location}.{channel}/ppsd
Parameters:
-----------
my_storage: str
Path to save all ppsd analyses
client: Client object from obspy
To use get_waveforms method
inv: Inventory object from obspy
To recognize the filtered stations that you want to
calculate the ppsd
ppsd_restrictions: PPSDRestrictions
Information about the PPSD parameters
single_cha_contents: 'str'
network.station.location.channel
starttime: UTCDateTime
Start time that will be used to calculate the ppsd.
endtime: UTCDateTime
End time that will be used to calculate the ppsd.
plot_trace: Boolean
Plot the stream (It consumes a little bit time)
"""
network, station, location, channel = single_cha_contents.split('.')
try:
st = client.get_waveforms(network=network,
station=station,
location=location,
channel=channel,
starttime=starttime,
endtime=endtime)
except:
strftime = "%Y%m%dT%H%M%SZ"
st_warn = (f"{network}."
f"{station}."
f"{location}."
f"{channel}"
f"__{starttime.strftime(strftime)}"
f"__{endtime.strftime(strftime)}")
st = None
now = dt.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
if st == None:
print_logs(job='load_trace',
content=single_cha_contents,
status='no',
path=st_warn)
return None
if plot_trace == True:
plotst_path = get_path(my_storage,
PLOT_TRACE_DIRNAME,
single_cha_contents,
starttime,
endtime,
extension_file='jpg')
filename = os.path.basename(plotst_path)
if os.path.isfile(plotst_path) == True:
print_logs(job='save_trace',
content=single_cha_contents,
status='exist',
path=filename)
else:
plotst_dir = os.path.dirname(plotst_path)
if os.path.isdir(plotst_dir) == False:
os.makedirs(plotst_dir)
st.plot(outfile=plotst_path)
print_logs(job='save_trace',
content=single_cha_contents,
status='ok',
path=filename)
now = dt.datetime.now().strftime("%Y/%m/%d %H:%M:%S")
try:
ppsd_path = get_path(my_storage,
PPSD_DIRNAME,
single_cha_contents,
starttime,
endtime,
extension_file='npz')
filename = os.path.basename(ppsd_path)
if os.path.isfile(ppsd_path) == True:
print_logs(job='save_ppsd',
content=single_cha_contents,
status='exist',
path=filename)
else:
ppsd_dir = os.path.dirname(ppsd_path)
if os.path.isdir(ppsd_dir) == False:
os.makedirs(ppsd_dir)
tr = st[0]
ppsd = PPSD(tr.stats, metadata=inv, **ppsd_restrictions.__dict__)
ppsd.add(st)
if ppsd_restrictions.time_of_weekday != None:
ppsd.calculate_histogram(
time_of_weekday=ppsd_restrictions.time_of_weekday)
ppsd.save_npz(ppsd_path)
print_logs(job='save_ppsd',
content=single_cha_contents,
status='ok',
path=filename)
except:
print_logs(job='save_ppsd',
content=single_cha_contents,
status='exist',
path=filename)
from obspy import read
from obspy.signal import PPSD
from obspy.io.xseed import Parser
st = read("https://examples.obspy.org/BW.KW1..EHZ.D.2011.037")
parser = Parser("https://examples.obspy.org/dataless.seed.BW_KW1")
ppsd = PPSD(st[0].stats, metadata=parser)
ppsd.add(st)
st = read("https://examples.obspy.org/BW.KW1..EHZ.D.2011.038")
ppsd.add(st)
ppsd.plot(cumulative=True)
def get_MassPPSD(my_storage, paths, dld_restrictions):
"""
parameters
----------
my_storage: str
Path to save all ppsd analyses
paths: list
list of paths that will be loaded or added in one Massive PPSD
dld_restrictions: DownloadRestrictions object
Class storing non-PPSD restrictions for a downloading
ppsd calculations
returns:
save MassPPSD in my_storage/{network}.{station}.{location}.{channel}/Mass_PPSD/
"""
content = paths[0].split('/')[-1].split('__')[0]
## to find the path for saving the MassPPSD file
network, station, location, channel = content.split('.')
_str = "{network}.{station}.{location}.{channel}__{starttime}__{endtime}"
filename = get_mseed_filename(_str, network, station, location, channel,
dld_restrictions.starttime,
dld_restrictions.endtime)
filename = filename + f'.npz'
path2save = os.path.join(my_storage, content, MASSPPSD_DIRNAME)
if os.path.isdir(path2save) == False:
os.makedirs(path2save)
else:
pass
path2save = os.path.join(path2save, filename)
####
if os.path.isfile(path2save) == True:
text = f"This file already exists: {path2save}"
print_logs(job='OK', content=content, status='exist', path=text)
else:
loaded = False
index = 0
while loaded == False:
try:
ppsd = PPSD.load_npz(paths[index])
loaded = True
print_logs(job='loaded_npz',
content=content,
status='ok',
path=paths[index])
except:
print_logs(job='loaded_npz',
content=content,
status='no',
path=paths[index])
if index == len(paths) - 1:
loaded = None
else:
pass
index += 1
if loaded == True:
for path in paths[index + 1::]:
if os.path.isfile(path) == True:
print_logs(job='to_added_npz',
content=content,
status='-',
path=path)
ppsd.add_npz(path)
else:
print_logs(job='to_added_npz',
content=content,
status='no',
path=path)
try:
ppsd.save_npz(path2save)
print_logs(job='MassPPSD',
content=content,
status='ok',
path=path2save)
except:
print_logs(job='MassPPSD',
content=content,
status='no',
path=path2save)
text = f"{content}"
print_logs(job='OK', content=content, status='ok', path=text)
else:
text = f"Not be able to load any npz in {content}"
print_logs(job='FAILED', content=content, status='no', path=text)
from obspy import read
from obspy.signal import PPSD
from obspy.imaging.cm import pqlx
from obspy.io.xseed import Parser
st = read("http://examples.obspy.org/BW.KW1..EHZ.D.2011.037")
parser = Parser("http://examples.obspy.org/dataless.seed.BW_KW1")
ppsd = PPSD(st[0].stats, metadata=parser)
ppsd.add(st)
st = read("http://examples.obspy.org/BW.KW1..EHZ.D.2011.038")
ppsd.add(st)
ppsd.plot(cmap=pqlx)
def read(
pastdays_select=None,
bufferdays_select=None,
#whitelists = ['CH.*..HG*','XE.*.HN*'],
whitelists=['SV.*.00.HN*', 'XE.*.00.HN*'],
slinktooldir='/home/sysop/slinktool/latencies/',
psddir='/home/sysop/msnoise/PSD/NPZ/',
src=None,
flights=None):
dfs = []
ppsds = []
buffersizes = {}
files = []
for w in whitelists:
parts = (slinktooldir, w)
filenames = '%s/*/*/*/*.D/%s*.????.???' % parts
files += glob.glob(filenames)
yeardays = list(set(['.'.join(f.split('.')[-2:]) for f in files]))
mseedids = list(
set(['.'.join(f.split('/')[-1].split('.')[:4]) for f in files]))
#print(files)#yeardays)
#print(mseedids)
now = datetime.now()
for n in range(int(pastdays_select.value[0]),
int(1 + pastdays_select.value[1])):
if len(list(buffersizes.keys())) >= maxnstation:
break
d = timedelta(days=n)
year = (now - d).year
day = (now - d).timetuple().tm_yday
if '%s.%03d' % (year, day) not in yeardays:
continue
parts = (slinktooldir, year, year, day)
dayfilenames = '%s/%d/*/*/*.D/*.%d.%03d' % parts
dayfilenames = dayfilenames.replace('//', '/')
#print(files)
#print(dayfilenames)
dayfilenames = [f for f in files if fnmatch.fnmatch(f, dayfilenames)]
if len(dayfilenames) == 0:
continue
#print(dayfilenames)
for whitelist in whitelists:
if len(list(buffersizes.keys())) >= maxnstation:
break
# Read in data from slinktool
parts = (slinktooldir, year, whitelist, year, day)
filenames = '%s/%d/*/*/*.D/%s.%d.%03d' % parts
filenames = filenames.replace('//', '/')
#print(filenames)
filenames = [
f for f in dayfilenames if fnmatch.fnmatch(f, filenames)
]
#print(filenames)
for filename in filenames[::-1]:
if len(list(buffersizes.keys())) >= maxnstation:
break
mseedid = '.'.join(filename.split('/')[-1].split('.')[:4])
if mseedid not in buffersizes:
buffersizes[mseedid] = 0
if buffersizes[mseedid] + 1 > np.ceil(bufferdays_select.value):
continue
print('loading %s (%s lines)' %
(filename, int(bufferdays_select.value * 24 * 60 * 60)))
buffersizes[mseedid] += 1
df = pd.read_csv(
filename,
nrows=int(bufferdays_select.value * 24 * 60 * 60),
sep=' ',
#XE_CH01_00_HNZ, 100 samples, 100 Hz, 2020,141,19:01:44.040000 (latency ~1.3802 sec)#
names=[
'name', 'reclen', 'unitlen', 'freq', 'unitfreq',
'time', '(latency', 'arr_delay', 'unitdelay'
],
index_col=False)[['arr_delay', 'time', 'name']]
# Trim
df['time'] = pd.to_datetime(df['time'],
format='%Y,%j,%H:%M:%S.%f')
earliest = max(
df['time']) - timedelta(days=bufferdays_select.value)
df = df.loc[df['time'] >= earliest]
# clean
df['name'] = df['name'].map(lambda x: x.rstrip(','))
df['name'] = df['name'].str.replace("_", ".")
# Convert
df['Delays'] = df['arr_delay'].map(
lambda x: x.lstrip('~')).astype(float)
# Compute reception time and latency
df['arr_time'] = pd.to_datetime(df['time']) + pd.to_timedelta(
df['Delays'], unit='s')
df['Latencies'] = df['arr_time'].diff() / np.timedelta64(
1, 's')
# Concatenate data
dfs.append(df[['name', 'arr_time', 'Latencies', 'Delays']])
# get psd
print(filename.replace(slinktooldir, psddir) + '.npz')
filename = glob.glob(
filename.replace(slinktooldir, psddir) + '.npz')
print(filename)
if len(filename):
print('loading', filename[0])
ppsd = PPSD.load_npz(filename[0])
rows = []
times = []
indexes = np.argsort(ppsd.times_processed)
for indexpsd in indexes:
time = ppsd.times_processed[indexpsd]
if time < min(df['time']):
continue
if time > max(df['time']):
break
times += [time.datetime]
rows += [[
mseedid, ppsd.period_bin_centers,
ppsd.psd_values[indexpsd]
]]
if len(times):
ppsds.append(
pd.DataFrame(
np.array(rows),
index=np.array(times),
columns=['name', 'PSD_periods', 'PSD']))
#print('OK')
# Concatenate all data into one DataFrame
flights = pd.concat(dfs, ignore_index=True)
ppsds = pd.concat(ppsds, ignore_index=True)
# Available carrier list
available_carriers = list(flights['name'].unique())
# Sort the list in-place (alphabetical order)
available_carriers.sort()
return [flights, ppsds], available_carriers
