def plot_by_path(NPZ_path):
NPZ = PPSD.load_npz(NPZ_path)
NPZ_name = f'{NPZ.network}.{NPZ.station}.{NPZ.location}.{NPZ.channel}.png'
print(f'graficando {NPZ_name}')
NPZ.plot(NPZ_name,
cmap=pqlx,
show_histogram=True,
show_percentiles=True,
percentiles=[90, 10],
period_lim=(0.1, 179))
return NPZ
def ppsd_NPZ(npz_paths):
npz_0 = PPSD.load_npz(npz_paths[0])
print(f'{npz_0.network}.{npz_0.station}.{npz_0.location}.{npz_0.channel}')
for npz in npz_paths[1:]:
npz_0.add_npz(npz)
# try: npz_0.add_npz(npz)
# except AssertionError as e: print(str(e))
print('saving')
save_NPZ(npz_0)
return npz_0
def multi_component_ppsd(comp,specific,avg='median'):
"""Return average arrays for each station in a given folder, choice of
component, can be N/E/Z/*
Choice of average available from the PPSD objects
"""
fidtmplt = "RD{s:0>2}.HH{c}*"
if specific == "ALL":
folder = 'FATHOM/*'
else:
folder = 'FATHOM/{}'.format(specific)
folderlist = glob.glob(os.path.join(pathnames()['ppsd'],folder))
averages,stations = [],[]
for F in folderlist:
print(F)
# check range of stations available using filenames
allfiles = glob.glob(os.path.join(F,'*HH*.npz'))
allfiles.sort()
lowrange = int(os.path.basename(allfiles[0]).split('.')[0][2:])
highrange = int(os.path.basename(allfiles[-1]).split('.')[0][2:])
# loop by station and inner loop by components per station
for i in range(lowrange,highrange+1):
files = glob.glob(os.path.join(F,fidtmplt.format(s=i,c=comp)))
if not files:
print('no files',fidtmplt.format(s=i,c=comp))
continue
inneraverages = []
for fid in files:
ppsd = PPSD.load_npz(fid)
if avg == 'mean':
average = ppsd.get_mean()
elif avg == 'mode':
average = ppsd.get_mode()
elif avg == 'median':
average = ppsd.get_percentile(percentile=50)
inneraverages.append(average[1])
outeraverage = np.mean(np.array(inneraverages),axis=0)
averages.append(outeraverage)
stations.append("RD{:0>2}".format(i))
periods = average[0]
if specific == "ALL":
stations,averages = collapse_stations(stations,averages)
return stations,periods,averages
def get_MassPPSD(my_storage, paths, dld_restrictions):
"""
parameters
----------
my_storage: str
Path to save all ppsd analyses
paths: list
list of paths that will be loaded or added in one Massive PPSD
dld_restrictions: DownloadRestrictions object
Class storing non-PPSD restrictions for a downloading
ppsd calculations
returns:
save MassPPSD in my_storage/{network}.{station}.{location}.{channel}/Mass_PPSD/
"""
content = paths[0].split('/')[-1].split('__')[0]
## to find the path for saving the MassPPSD file
network, station, location, channel = content.split('.')
_str = "{network}.{station}.{location}.{channel}__{starttime}__{endtime}"
filename = get_mseed_filename(_str, network, station, location, channel,
dld_restrictions.starttime,
dld_restrictions.endtime)
filename = filename + f'.npz'
path2save = os.path.join(my_storage, content, MASSPPSD_DIRNAME)
if os.path.isdir(path2save) == False:
os.makedirs(path2save)
else:
pass
path2save = os.path.join(path2save, filename)
####
if os.path.isfile(path2save) == True:
text = f"This file already exists: {path2save}"
print_logs(job='OK', content=content, status='exist', path=text)
else:
loaded = False
index = 0
while loaded == False:
try:
ppsd = PPSD.load_npz(paths[index])
loaded = True
print_logs(job='loaded_npz',
content=content,
status='ok',
path=paths[index])
except:
print_logs(job='loaded_npz',
content=content,
status='no',
path=paths[index])
if index == len(paths) - 1:
loaded = None
else:
pass
index += 1
if loaded == True:
for path in paths[index + 1::]:
if os.path.isfile(path) == True:
print_logs(job='to_added_npz',
content=content,
status='-',
path=path)
ppsd.add_npz(path)
else:
print_logs(job='to_added_npz',
content=content,
status='no',
path=path)
try:
ppsd.save_npz(path2save)
print_logs(job='MassPPSD',
content=content,
status='ok',
path=path2save)
except:
print_logs(job='MassPPSD',
content=content,
status='no',
path=path2save)
text = f"{content}"
print_logs(job='OK', content=content, status='ok', path=text)
else:
text = f"Not be able to load any npz in {content}"
print_logs(job='FAILED', content=content, status='no', path=text)
def read(
pastdays_select=None,
bufferdays_select=None,
#whitelists = ['CH.*..HG*','XE.*.HN*'],
whitelists=['SV.*.00.HN*', 'XE.*.00.HN*'],
slinktooldir='/home/sysop/slinktool/latencies/',
psddir='/home/sysop/msnoise/PSD/NPZ/',
src=None,
flights=None):
dfs = []
ppsds = []
buffersizes = {}
files = []
for w in whitelists:
parts = (slinktooldir, w)
filenames = '%s/*/*/*/*.D/%s*.????.???' % parts
files += glob.glob(filenames)
yeardays = list(set(['.'.join(f.split('.')[-2:]) for f in files]))
mseedids = list(
set(['.'.join(f.split('/')[-1].split('.')[:4]) for f in files]))
#print(files)#yeardays)
#print(mseedids)
now = datetime.now()
for n in range(int(pastdays_select.value[0]),
int(1 + pastdays_select.value[1])):
if len(list(buffersizes.keys())) >= maxnstation:
break
d = timedelta(days=n)
year = (now - d).year
day = (now - d).timetuple().tm_yday
if '%s.%03d' % (year, day) not in yeardays:
continue
parts = (slinktooldir, year, year, day)
dayfilenames = '%s/%d/*/*/*.D/*.%d.%03d' % parts
dayfilenames = dayfilenames.replace('//', '/')
#print(files)
#print(dayfilenames)
dayfilenames = [f for f in files if fnmatch.fnmatch(f, dayfilenames)]
if len(dayfilenames) == 0:
continue
#print(dayfilenames)
for whitelist in whitelists:
if len(list(buffersizes.keys())) >= maxnstation:
break
# Read in data from slinktool
parts = (slinktooldir, year, whitelist, year, day)
filenames = '%s/%d/*/*/*.D/%s.%d.%03d' % parts
filenames = filenames.replace('//', '/')
#print(filenames)
filenames = [
f for f in dayfilenames if fnmatch.fnmatch(f, filenames)
]
#print(filenames)
for filename in filenames[::-1]:
if len(list(buffersizes.keys())) >= maxnstation:
break
mseedid = '.'.join(filename.split('/')[-1].split('.')[:4])
if mseedid not in buffersizes:
buffersizes[mseedid] = 0
if buffersizes[mseedid] + 1 > np.ceil(bufferdays_select.value):
continue
print('loading %s (%s lines)' %
(filename, int(bufferdays_select.value * 24 * 60 * 60)))
buffersizes[mseedid] += 1
df = pd.read_csv(
filename,
nrows=int(bufferdays_select.value * 24 * 60 * 60),
sep=' ',
#XE_CH01_00_HNZ, 100 samples, 100 Hz, 2020,141,19:01:44.040000 (latency ~1.3802 sec)#
names=[
'name', 'reclen', 'unitlen', 'freq', 'unitfreq',
'time', '(latency', 'arr_delay', 'unitdelay'
],
index_col=False)[['arr_delay', 'time', 'name']]
# Trim
df['time'] = pd.to_datetime(df['time'],
format='%Y,%j,%H:%M:%S.%f')
earliest = max(
df['time']) - timedelta(days=bufferdays_select.value)
df = df.loc[df['time'] >= earliest]
# clean
df['name'] = df['name'].map(lambda x: x.rstrip(','))
df['name'] = df['name'].str.replace("_", ".")
# Convert
df['Delays'] = df['arr_delay'].map(
lambda x: x.lstrip('~')).astype(float)
# Compute reception time and latency
df['arr_time'] = pd.to_datetime(df['time']) + pd.to_timedelta(
df['Delays'], unit='s')
df['Latencies'] = df['arr_time'].diff() / np.timedelta64(
1, 's')
# Concatenate data
dfs.append(df[['name', 'arr_time', 'Latencies', 'Delays']])
# get psd
print(filename.replace(slinktooldir, psddir) + '.npz')
filename = glob.glob(
filename.replace(slinktooldir, psddir) + '.npz')
print(filename)
if len(filename):
print('loading', filename[0])
ppsd = PPSD.load_npz(filename[0])
rows = []
times = []
indexes = np.argsort(ppsd.times_processed)
for indexpsd in indexes:
time = ppsd.times_processed[indexpsd]
if time < min(df['time']):
continue
if time > max(df['time']):
break
times += [time.datetime]
rows += [[
mseedid, ppsd.period_bin_centers,
ppsd.psd_values[indexpsd]
]]
if len(times):
ppsds.append(
pd.DataFrame(
np.array(rows),
index=np.array(times),
columns=['name', 'PSD_periods', 'PSD']))
#print('OK')
# Concatenate all data into one DataFrame
flights = pd.concat(dfs, ignore_index=True)
ppsds = pd.concat(ppsds, ignore_index=True)
# Available carrier list
available_carriers = list(flights['name'].unique())
# Sort the list in-place (alphabetical order)
available_carriers.sort()
return [flights, ppsds], available_carriers
import matplotlib.pyplot as plt
from obspy.signal.spectral_estimation import get_nlnm, get_nhnm
import matplotlib as mpl
mpl.rc('font', family='serif')
mpl.rc('font', serif='Times')
mpl.rc('text', usetex=True)
mpl.rc('font', size=18)
files = glob.glob("*.npz")
files = list(files)
files.sort()
fig = plt.figure(1, figsize=(12, 12))
for curfile in files:
ppsd = PPSD.load_npz(curfile)
#ppsd.plot()
pers, means = ppsd.get_mean()
if 'XX' in curfile:
stalab = 'GS 005 ' + curfile.split('.')[3].replace('PDF', '')
else:
stalab = curfile.split('.')[0] + ' ' + curfile.split(
'.')[1] + ' ' + curfile.split('.')[3].replace('PDF', '')
plt.semilogx(1. / pers, means, label=stalab, linewidth=2)
NHNMper, NHNMpower = get_nhnm()
NLNMper, NLNMpower = get_nlnm()
plt.semilogx(1. / NLNMper, NLNMpower, linewidth=3., color='k')
plt.semilogx(1. / NHNMper, NHNMpower, linewidth=3., color='k')
plt.xlabel('Frequency (Hz)')
plt.legend(loc=9, ncol=3)
#st = read(curfile)
#if 'ppsd' not in vars():
#ppsd = PPSD(st[0].stats, inv, period_smoothing_width_octaves=0.5)
#if debug:
#print(curfile)
#ppsd.add(st)
#ppsd.save_npz(net + '_' + sta + '_' + chan + '.npz')
#del ppsd
##ppsd.plot()
per, nlnm = get_nlnm()
per, nhnm = get_nhnm()
fig = plt.figure(1, figsize=(12, 12))
for chan in ['LHZ', 'LHN', 'LHE']:
ppsd = PPSD.load_npz(net + '_' + sta + '_' + chan + '.npz')
#result = ppsd.psd_values
#print(result)
perLP, mean = ppsd.get_mean()
plt.semilogx(perLP, mean, label=chan + ' Mean', linewidth=2)
plt.xlim((2, 100.))
plt.semilogx(per, nhnm, 'C4', label='NHNM/NLNM', linewidth=3)
plt.semilogx(per, nlnm, 'C4', linewidth=3)
plt.xlabel('Period (s)')
#plt.xlim((1./20., 1000.))
ppsd = PPSD.load_npz(net + '_' + sta + '_SHZ.npz')
perLP, mean = ppsd.get_percentile(92.)
plt.semilogx(perLP, mean, label='SHZ 92nd Percentile', linewidth=2)
plt.ylabel('Power (dB rel. 1 $(m/s^2)^2/Hz)$)')
plt.legend(loc=9)
'gain':
60077000.0,
'poles': [
-0.037004 + 0.037016j, -0.037004 - 0.037016j, -251.33 + 0j,
-131.04 - 467.29j, -131.04 + 467.29j
],
'sensitivity':
2516778400.0,
'zeros': [0j, 0j]
}
ppsd = PPSD(tr.stats, paz)
print(ppsd.id)
print(ppsd.times_processed)
ppsd.add(st)
print(ppsd.times_processed)
ppsd.plot()
ppsd.save_npz("myfile.npz")
ppsd = PPSD.load_npz("myfile.npz")
#%% Try getting the NHNM and NLNM
nhnm = spec.get_nhnm()
plt.semilogx(nhnm[0], nhnm[1])
nlnm = spec.get_nlnm()
plt.semilogx(nlnm[0], nlnm[1])
# ---------------------------------
ppsds = {}
pbar = tqdm.tqdm(datelist)
for day in pbar:
YYYY = day.strftime("%Y")
MM = day.strftime("%m")
DD = day.strftime("%d")
datestr = day.strftime("%Y-%m-%d")
npzdatadir = "{}/npz/{}/{}/{}".format(DATADIR, YYYY, MM, DD)
fn_pattern = "{}/{}_{}.npz".format(npzdatadir, datestr, nslc)
pbar.set_description("Reading %s" % fn_pattern)
for fn in glob(fn_pattern):
mseedid = fn.replace(".npz", "").split("_")[-1]
if mseedid not in ppsds:
ppsds[mseedid] = PPSD.load_npz(fn) #, allow_pickle=True)
else:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
ppsds[mseedid].add_npz(fn) #, allow_pickle=True)
# ---------------------------------------------
# Reead rms files
# ---------------------------------------------
headers = ['rms', 'Datetime']
dtypes = {'rms': 'float', 'Datetime': 'str'}
parse_dates = ['Datetime']
print("%s/csv/*_%s_%s.csv" % (DATADIR, nslc, FREQ1))
#for fn in glob("%s/csv/*_%s_%s.csv" % (DATADIR,nslc,FREQ1)):
from obspy.core.util.base import get_example_file
from obspy.signal import PPSD
ppsd = PPSD.load_npz(get_example_file('ppsd_kw1_ehz.npz'))
ppsd.plot_temporal([0.1, 1, 10])
ppsd = PPSD(stats=tr.stats, metadata=inv)
ppsd.add(tr)
ppsd.plot()
# -
# Since longer term stacks would need too much waveform data and take way too long to compute, we prepared one year continuous data preprocessed for a single channel of station `FUR` to play with..
#
# * load long term pre-computed PPSD from file `PPSD_FUR_HHN.npz` using `PPSD`'s `load_npz()` staticmethod (i.e. it is called directly from the class, not an instance object of the class)
# * plot the PPSD (default is full time-range, depending on how much data and spread is in the data, adjust `max_percentage` option of `plot()` option) (might take a couple of minutes..!)
# * do a cumulative plot (which is good to judge non-exceedance percentage dB thresholds)
# +
from obspy.signal import PPSD
ppsd = PPSD.load_npz("data/PPSD_FUR_HHN.npz")
# -
ppsd.plot(max_percentage=10)
ppsd.plot(cumulative=True)
# * do different stacks of the data using the [`calculate_histogram()` (see docs!)](http://docs.obspy.org/packages/autogen/obspy.signal.spectral_estimation.PPSD.calculate_histogram.html) method of `PPSD` and visualize them
# * compare differences in different frequency bands qualitatively (anthropogenic vs. "natural" noise)..
# * nighttime stack, daytime stack
# * advanced exercise: Use the `callback` option and use some crazy custom callback function in `calculate_histogram()`, e.g. stack together all data from birthdays in your family.. or all German holidays + Sundays in the time span.. or from dates of some bands' concerts on a tour.. etc.
ppsd.calculate_histogram(time_of_weekday=[(-1, 0, 2), (-1, 22, 24)])
ppsd.plot(max_percentage=10)
ppsd.calculate_histogram(time_of_weekday=[(-1, 8, 16)])
ppsd.plot(max_percentage=10)
def main(net,
sta,
loc,
chan,
time_of_weekday=None,
period_lim=None,
show=False,
outfile=None,
cmap="viridis",
color_lim=None):
db = connect()
logging.debug('Preloading all instrument response')
response_format = get_config(db, 'response_format')
response_files = glob.glob(
os.path.join(get_config(db, 'response_path'),
"%s.%s.*.*" % (net, sta)))
start, end, datelist = build_movstack_datelist(db)
first = True
ppsd = None
for day in datelist:
jday = int(day.strftime("%j"))
toglob = os.path.join(
'PSD', 'NPZ', "%s" % day.year, net, sta, chan + ".D",
"%s.%s.%s.%s.D.%s.%03i.npz" %
(net, sta, loc, chan, day.year, jday))
files = glob.glob(toglob)
if not len(files):
print("No files found for %s.%s.%s.%s: %s" %
(net, sta, loc, chan, day))
continue
file = files[0]
if os.path.isfile(file):
if first:
ppsd = PPSD.load_npz(file)
first = False
else:
try:
ppsd.add_npz(file)
except:
pass
if not ppsd:
# f = open(os.path.join(os.path.split(__file__)[0], "nodata.png"), 'rb').read()
# outfile.write(f)
return
print("Calculate histogram...")
ppsd.calculate_histogram(time_of_weekday=time_of_weekday)
if len(ppsd._times_processed) < 2:
print("Not enough data for %s.%s - %s" % (net, sta, comp))
return
print("Plotting PPSD...")
fig = ppsd.plot(show_mean=True,
show=False,
period_lim=period_lim,
show_coverage=False)
print(fig)
print("Plotting spectrogram...")
gs = GridSpec(2,
2,
width_ratios=[10, 1],
hspace=0.3,
left=0.2,
right=0.85,
top=0.80,
bottom=0.2)
fig.set_size_inches(8.2, 11.6, forward=True)
fig.axes[0].set_subplotspec(gs[0])
fig.axes[0].set_position(gs[0].get_position(fig))
fig.axes[1].set_subplotspec(gs[1])
fig.axes[1].set_position(gs[1].get_position(fig))
fig.axes[0].set_title("")
ax2 = plt.subplot(gs[2])
print("Creating Time Index...")
print(
UTCDateTime(ppsd.times_processed[1]).datetime -
UTCDateTime(ppsd.times_processed[0]).datetime)
new_times = pd.date_range(UTCDateTime(ppsd.times_processed[0]).datetime,
UTCDateTime(ppsd.times_processed[-1]).datetime,
freq="1H")
ind_times = np.array(
[UTCDateTime(t).datetime for t in ppsd.current_times_used])
data = np.asarray(ppsd._binned_psds)
print("Creating DataFrame...")
data = pd.DataFrame(data, index=ind_times)
# TODO : why resample here ?
# data = data.resample("H").fillna(method="ffill")
if color_lim:
vmin, vmax = color_lim
plt.pcolormesh(data.index,
ppsd.period_bin_centers,
data.T,
cmap=cmap,
vmin=vmin,
vmax=vmax,
rasterized=True)
else:
plt.imshow(
data.T,
cmap=cmap,
aspect='auto',
origin='lower',
extent=(date2num(new_times[0].to_pydatetime()),
date2num(new_times[-1].to_pydatetime()),
ppsd.period_bin_centers[0], ppsd.period_bin_centers[-1]),
)
cb = plt.colorbar(cax=plt.subplot(gs[3]), use_gridspec=True)
plt.sca(ax2)
cb.set_label("Amplitude [dB]")
plt.ylabel("Period [s]")
plt.semilogy((UTCDateTime(ppsd.times_processed[0]).datetime), 0, alpha=0)
plt.xlim(
UTCDateTime(ppsd.times_processed[0]).datetime,
UTCDateTime(ppsd.times_processed[-1]).datetime)
plt.ylim(period_lim[0], period_lim[1])
plt.suptitle(ppsd._get_plot_title())
fig.autofmt_xdate()
xax = plt.gca().get_xaxis() # get the x-axis
adf = xax.get_major_formatter() # the the auto-formatter
adf.scaled[1. / 24] = '%Y-%m-%d %H:%M' # set the < 1d scale to H:M
adf.scaled[1.0] = '%Y-%m-%d' # set the > 1d < 1m scale to Y-m-d
adf.scaled[30.] = '%Y-%m' # set the > 1m < 1Y scale to Y-m
adf.scaled[365.] = '%Y' # set the > 1y scale to Y
if outfile:
print("output to:", outfile)
plt.savefig(outfile)
if show:
plt.show()
plt.gcf()
plt.gca()
plt.close(fig)
del fig
from obspy.core.util.base import get_example_file
from obspy.signal import PPSD
ppsd = PPSD.load_npz(get_example_file('ppsd_kw1_ehz.npz'), allow_pickle=True)
ppsd.plot_temporal([0.1, 1, 10])
from obspy.core.util.base import get_example_file
from obspy.signal import PPSD
ppsd = PPSD.load_npz(get_example_file('ppsd_kw1_ehz.npz'))
ppsd.plot_temporal([0.1, 1, 10])
n = read(nfile)
z = read(zfile)
#on the first time that data is found, check if the PPSD exists,
#if not then create it
if ppsd_init == False: #i.e. haven't checked for existence of ppsd
Eppsd_file = ppsd_dir + site + '.LXE.ppsd.npz'
Nppsd_file = ppsd_dir + site + '.LXN.ppsd.npz'
Zppsd_file = ppsd_dir + site + '.LXZ.ppsd.npz'
#Does the ppsd object exist?
if exists(Eppsd_file) == True:
#Load the previously saved PPSDs
Eppsd = PPSD.load_npz(Eppsd_file, metadata=paz)
Nppsd = PPSD.load_npz(Nppsd_file, metadata=paz)
Zppsd = PPSD.load_npz(Zppsd_file, metadata=paz)
#They don't exist yet, create new objects
else:
Eppsd = PPSD(e[0].stats,
paz,
db_bins=(-80, 20, 1.0),
period_limits=[2, 1200],
special_handling="ringlaser")
Nppsd = PPSD(n[0].stats,
paz,
db_bins=(-80, 20, 1.0),
period_limits=[2, 1200],
special_handling="ringlaser")
