def plot_by_path(NPZ_path): NPZ = PPSD.load_npz(NPZ_path) NPZ_name = f'{NPZ.network}.{NPZ.station}.{NPZ.location}.{NPZ.channel}.png' print(f'graficando {NPZ_name}') NPZ.plot(NPZ_name, cmap=pqlx, show_histogram=True, show_percentiles=True, percentiles=[90, 10], period_lim=(0.1, 179)) return NPZ
def ppsd_NPZ(npz_paths): npz_0 = PPSD.load_npz(npz_paths[0]) print(f'{npz_0.network}.{npz_0.station}.{npz_0.location}.{npz_0.channel}') for npz in npz_paths[1:]: npz_0.add_npz(npz) # try: npz_0.add_npz(npz) # except AssertionError as e: print(str(e)) print('saving') save_NPZ(npz_0) return npz_0
def multi_component_ppsd(comp,specific,avg='median'): """Return average arrays for each station in a given folder, choice of component, can be N/E/Z/* Choice of average available from the PPSD objects """ fidtmplt = "RD{s:0>2}.HH{c}*" if specific == "ALL": folder = 'FATHOM/*' else: folder = 'FATHOM/{}'.format(specific) folderlist = glob.glob(os.path.join(pathnames()['ppsd'],folder)) averages,stations = [],[] for F in folderlist: print(F) # check range of stations available using filenames allfiles = glob.glob(os.path.join(F,'*HH*.npz')) allfiles.sort() lowrange = int(os.path.basename(allfiles[0]).split('.')[0][2:]) highrange = int(os.path.basename(allfiles[-1]).split('.')[0][2:]) # loop by station and inner loop by components per station for i in range(lowrange,highrange+1): files = glob.glob(os.path.join(F,fidtmplt.format(s=i,c=comp))) if not files: print('no files',fidtmplt.format(s=i,c=comp)) continue inneraverages = [] for fid in files: ppsd = PPSD.load_npz(fid) if avg == 'mean': average = ppsd.get_mean() elif avg == 'mode': average = ppsd.get_mode() elif avg == 'median': average = ppsd.get_percentile(percentile=50) inneraverages.append(average[1]) outeraverage = np.mean(np.array(inneraverages),axis=0) averages.append(outeraverage) stations.append("RD{:0>2}".format(i)) periods = average[0] if specific == "ALL": stations,averages = collapse_stations(stations,averages) return stations,periods,averages
def get_MassPPSD(my_storage, paths, dld_restrictions): """ parameters ---------- my_storage: str Path to save all ppsd analyses paths: list list of paths that will be loaded or added in one Massive PPSD dld_restrictions: DownloadRestrictions object Class storing non-PPSD restrictions for a downloading ppsd calculations returns: save MassPPSD in my_storage/{network}.{station}.{location}.{channel}/Mass_PPSD/ """ content = paths[0].split('/')[-1].split('__')[0] ## to find the path for saving the MassPPSD file network, station, location, channel = content.split('.') _str = "{network}.{station}.{location}.{channel}__{starttime}__{endtime}" filename = get_mseed_filename(_str, network, station, location, channel, dld_restrictions.starttime, dld_restrictions.endtime) filename = filename + f'.npz' path2save = os.path.join(my_storage, content, MASSPPSD_DIRNAME) if os.path.isdir(path2save) == False: os.makedirs(path2save) else: pass path2save = os.path.join(path2save, filename) #### if os.path.isfile(path2save) == True: text = f"This file already exists: {path2save}" print_logs(job='OK', content=content, status='exist', path=text) else: loaded = False index = 0 while loaded == False: try: ppsd = PPSD.load_npz(paths[index]) loaded = True print_logs(job='loaded_npz', content=content, status='ok', path=paths[index]) except: print_logs(job='loaded_npz', content=content, status='no', path=paths[index]) if index == len(paths) - 1: loaded = None else: pass index += 1 if loaded == True: for path in paths[index + 1::]: if os.path.isfile(path) == True: print_logs(job='to_added_npz', content=content, status='-', path=path) ppsd.add_npz(path) else: print_logs(job='to_added_npz', content=content, status='no', path=path) try: ppsd.save_npz(path2save) print_logs(job='MassPPSD', content=content, status='ok', path=path2save) except: print_logs(job='MassPPSD', content=content, status='no', path=path2save) text = f"{content}" print_logs(job='OK', content=content, status='ok', path=text) else: text = f"Not be able to load any npz in {content}" print_logs(job='FAILED', content=content, status='no', path=text)
def read( pastdays_select=None, bufferdays_select=None, #whitelists = ['CH.*..HG*','XE.*.HN*'], whitelists=['SV.*.00.HN*', 'XE.*.00.HN*'], slinktooldir='/home/sysop/slinktool/latencies/', psddir='/home/sysop/msnoise/PSD/NPZ/', src=None, flights=None): dfs = [] ppsds = [] buffersizes = {} files = [] for w in whitelists: parts = (slinktooldir, w) filenames = '%s/*/*/*/*.D/%s*.????.???' % parts files += glob.glob(filenames) yeardays = list(set(['.'.join(f.split('.')[-2:]) for f in files])) mseedids = list( set(['.'.join(f.split('/')[-1].split('.')[:4]) for f in files])) #print(files)#yeardays) #print(mseedids) now = datetime.now() for n in range(int(pastdays_select.value[0]), int(1 + pastdays_select.value[1])): if len(list(buffersizes.keys())) >= maxnstation: break d = timedelta(days=n) year = (now - d).year day = (now - d).timetuple().tm_yday if '%s.%03d' % (year, day) not in yeardays: continue parts = (slinktooldir, year, year, day) dayfilenames = '%s/%d/*/*/*.D/*.%d.%03d' % parts dayfilenames = dayfilenames.replace('//', '/') #print(files) #print(dayfilenames) dayfilenames = [f for f in files if fnmatch.fnmatch(f, dayfilenames)] if len(dayfilenames) == 0: continue #print(dayfilenames) for whitelist in whitelists: if len(list(buffersizes.keys())) >= maxnstation: break # Read in data from slinktool parts = (slinktooldir, year, whitelist, year, day) filenames = '%s/%d/*/*/*.D/%s.%d.%03d' % parts filenames = filenames.replace('//', '/') #print(filenames) filenames = [ f for f in dayfilenames if fnmatch.fnmatch(f, filenames) ] #print(filenames) for filename in filenames[::-1]: if len(list(buffersizes.keys())) >= maxnstation: break mseedid = '.'.join(filename.split('/')[-1].split('.')[:4]) if mseedid not in buffersizes: buffersizes[mseedid] = 0 if buffersizes[mseedid] + 1 > np.ceil(bufferdays_select.value): continue print('loading %s (%s lines)' % (filename, int(bufferdays_select.value * 24 * 60 * 60))) buffersizes[mseedid] += 1 df = pd.read_csv( filename, nrows=int(bufferdays_select.value * 24 * 60 * 60), sep=' ', #XE_CH01_00_HNZ, 100 samples, 100 Hz, 2020,141,19:01:44.040000 (latency ~1.3802 sec)# names=[ 'name', 'reclen', 'unitlen', 'freq', 'unitfreq', 'time', '(latency', 'arr_delay', 'unitdelay' ], index_col=False)[['arr_delay', 'time', 'name']] # Trim df['time'] = pd.to_datetime(df['time'], format='%Y,%j,%H:%M:%S.%f') earliest = max( df['time']) - timedelta(days=bufferdays_select.value) df = df.loc[df['time'] >= earliest] # clean df['name'] = df['name'].map(lambda x: x.rstrip(',')) df['name'] = df['name'].str.replace("_", ".") # Convert df['Delays'] = df['arr_delay'].map( lambda x: x.lstrip('~')).astype(float) # Compute reception time and latency df['arr_time'] = pd.to_datetime(df['time']) + pd.to_timedelta( df['Delays'], unit='s') df['Latencies'] = df['arr_time'].diff() / np.timedelta64( 1, 's') # Concatenate data dfs.append(df[['name', 'arr_time', 'Latencies', 'Delays']]) # get psd print(filename.replace(slinktooldir, psddir) + '.npz') filename = glob.glob( filename.replace(slinktooldir, psddir) + '.npz') print(filename) if len(filename): print('loading', filename[0]) ppsd = PPSD.load_npz(filename[0]) rows = [] times = [] indexes = np.argsort(ppsd.times_processed) for indexpsd in indexes: time = ppsd.times_processed[indexpsd] if time < min(df['time']): continue if time > max(df['time']): break times += [time.datetime] rows += [[ mseedid, ppsd.period_bin_centers, ppsd.psd_values[indexpsd] ]] if len(times): ppsds.append( pd.DataFrame( np.array(rows), index=np.array(times), columns=['name', 'PSD_periods', 'PSD'])) #print('OK') # Concatenate all data into one DataFrame flights = pd.concat(dfs, ignore_index=True) ppsds = pd.concat(ppsds, ignore_index=True) # Available carrier list available_carriers = list(flights['name'].unique()) # Sort the list in-place (alphabetical order) available_carriers.sort() return [flights, ppsds], available_carriers
import matplotlib.pyplot as plt from obspy.signal.spectral_estimation import get_nlnm, get_nhnm import matplotlib as mpl mpl.rc('font', family='serif') mpl.rc('font', serif='Times') mpl.rc('text', usetex=True) mpl.rc('font', size=18) files = glob.glob("*.npz") files = list(files) files.sort() fig = plt.figure(1, figsize=(12, 12)) for curfile in files: ppsd = PPSD.load_npz(curfile) #ppsd.plot() pers, means = ppsd.get_mean() if 'XX' in curfile: stalab = 'GS 005 ' + curfile.split('.')[3].replace('PDF', '') else: stalab = curfile.split('.')[0] + ' ' + curfile.split( '.')[1] + ' ' + curfile.split('.')[3].replace('PDF', '') plt.semilogx(1. / pers, means, label=stalab, linewidth=2) NHNMper, NHNMpower = get_nhnm() NLNMper, NLNMpower = get_nlnm() plt.semilogx(1. / NLNMper, NLNMpower, linewidth=3., color='k') plt.semilogx(1. / NHNMper, NHNMpower, linewidth=3., color='k') plt.xlabel('Frequency (Hz)') plt.legend(loc=9, ncol=3)
#st = read(curfile) #if 'ppsd' not in vars(): #ppsd = PPSD(st[0].stats, inv, period_smoothing_width_octaves=0.5) #if debug: #print(curfile) #ppsd.add(st) #ppsd.save_npz(net + '_' + sta + '_' + chan + '.npz') #del ppsd ##ppsd.plot() per, nlnm = get_nlnm() per, nhnm = get_nhnm() fig = plt.figure(1, figsize=(12, 12)) for chan in ['LHZ', 'LHN', 'LHE']: ppsd = PPSD.load_npz(net + '_' + sta + '_' + chan + '.npz') #result = ppsd.psd_values #print(result) perLP, mean = ppsd.get_mean() plt.semilogx(perLP, mean, label=chan + ' Mean', linewidth=2) plt.xlim((2, 100.)) plt.semilogx(per, nhnm, 'C4', label='NHNM/NLNM', linewidth=3) plt.semilogx(per, nlnm, 'C4', linewidth=3) plt.xlabel('Period (s)') #plt.xlim((1./20., 1000.)) ppsd = PPSD.load_npz(net + '_' + sta + '_SHZ.npz') perLP, mean = ppsd.get_percentile(92.) plt.semilogx(perLP, mean, label='SHZ 92nd Percentile', linewidth=2) plt.ylabel('Power (dB rel. 1 $(m/s^2)^2/Hz)$)') plt.legend(loc=9)
'gain': 60077000.0, 'poles': [ -0.037004 + 0.037016j, -0.037004 - 0.037016j, -251.33 + 0j, -131.04 - 467.29j, -131.04 + 467.29j ], 'sensitivity': 2516778400.0, 'zeros': [0j, 0j] } ppsd = PPSD(tr.stats, paz) print(ppsd.id) print(ppsd.times_processed) ppsd.add(st) print(ppsd.times_processed) ppsd.plot() ppsd.save_npz("myfile.npz") ppsd = PPSD.load_npz("myfile.npz") #%% Try getting the NHNM and NLNM nhnm = spec.get_nhnm() plt.semilogx(nhnm[0], nhnm[1]) nlnm = spec.get_nlnm() plt.semilogx(nlnm[0], nlnm[1])
# --------------------------------- ppsds = {} pbar = tqdm.tqdm(datelist) for day in pbar: YYYY = day.strftime("%Y") MM = day.strftime("%m") DD = day.strftime("%d") datestr = day.strftime("%Y-%m-%d") npzdatadir = "{}/npz/{}/{}/{}".format(DATADIR, YYYY, MM, DD) fn_pattern = "{}/{}_{}.npz".format(npzdatadir, datestr, nslc) pbar.set_description("Reading %s" % fn_pattern) for fn in glob(fn_pattern): mseedid = fn.replace(".npz", "").split("_")[-1] if mseedid not in ppsds: ppsds[mseedid] = PPSD.load_npz(fn) #, allow_pickle=True) else: with warnings.catch_warnings(): warnings.simplefilter("ignore") ppsds[mseedid].add_npz(fn) #, allow_pickle=True) # --------------------------------------------- # Reead rms files # --------------------------------------------- headers = ['rms', 'Datetime'] dtypes = {'rms': 'float', 'Datetime': 'str'} parse_dates = ['Datetime'] print("%s/csv/*_%s_%s.csv" % (DATADIR, nslc, FREQ1)) #for fn in glob("%s/csv/*_%s_%s.csv" % (DATADIR,nslc,FREQ1)):
from obspy.core.util.base import get_example_file from obspy.signal import PPSD ppsd = PPSD.load_npz(get_example_file('ppsd_kw1_ehz.npz')) ppsd.plot_temporal([0.1, 1, 10])
ppsd = PPSD(stats=tr.stats, metadata=inv) ppsd.add(tr) ppsd.plot() # - # Since longer term stacks would need too much waveform data and take way too long to compute, we prepared one year continuous data preprocessed for a single channel of station `FUR` to play with.. # # * load long term pre-computed PPSD from file `PPSD_FUR_HHN.npz` using `PPSD`'s `load_npz()` staticmethod (i.e. it is called directly from the class, not an instance object of the class) # * plot the PPSD (default is full time-range, depending on how much data and spread is in the data, adjust `max_percentage` option of `plot()` option) (might take a couple of minutes..!) # * do a cumulative plot (which is good to judge non-exceedance percentage dB thresholds) # + from obspy.signal import PPSD ppsd = PPSD.load_npz("data/PPSD_FUR_HHN.npz") # - ppsd.plot(max_percentage=10) ppsd.plot(cumulative=True) # * do different stacks of the data using the [`calculate_histogram()` (see docs!)](http://docs.obspy.org/packages/autogen/obspy.signal.spectral_estimation.PPSD.calculate_histogram.html) method of `PPSD` and visualize them # * compare differences in different frequency bands qualitatively (anthropogenic vs. "natural" noise).. # * nighttime stack, daytime stack # * advanced exercise: Use the `callback` option and use some crazy custom callback function in `calculate_histogram()`, e.g. stack together all data from birthdays in your family.. or all German holidays + Sundays in the time span.. or from dates of some bands' concerts on a tour.. etc. ppsd.calculate_histogram(time_of_weekday=[(-1, 0, 2), (-1, 22, 24)]) ppsd.plot(max_percentage=10) ppsd.calculate_histogram(time_of_weekday=[(-1, 8, 16)]) ppsd.plot(max_percentage=10)
def main(net, sta, loc, chan, time_of_weekday=None, period_lim=None, show=False, outfile=None, cmap="viridis", color_lim=None): db = connect() logging.debug('Preloading all instrument response') response_format = get_config(db, 'response_format') response_files = glob.glob( os.path.join(get_config(db, 'response_path'), "%s.%s.*.*" % (net, sta))) start, end, datelist = build_movstack_datelist(db) first = True ppsd = None for day in datelist: jday = int(day.strftime("%j")) toglob = os.path.join( 'PSD', 'NPZ', "%s" % day.year, net, sta, chan + ".D", "%s.%s.%s.%s.D.%s.%03i.npz" % (net, sta, loc, chan, day.year, jday)) files = glob.glob(toglob) if not len(files): print("No files found for %s.%s.%s.%s: %s" % (net, sta, loc, chan, day)) continue file = files[0] if os.path.isfile(file): if first: ppsd = PPSD.load_npz(file) first = False else: try: ppsd.add_npz(file) except: pass if not ppsd: # f = open(os.path.join(os.path.split(__file__)[0], "nodata.png"), 'rb').read() # outfile.write(f) return print("Calculate histogram...") ppsd.calculate_histogram(time_of_weekday=time_of_weekday) if len(ppsd._times_processed) < 2: print("Not enough data for %s.%s - %s" % (net, sta, comp)) return print("Plotting PPSD...") fig = ppsd.plot(show_mean=True, show=False, period_lim=period_lim, show_coverage=False) print(fig) print("Plotting spectrogram...") gs = GridSpec(2, 2, width_ratios=[10, 1], hspace=0.3, left=0.2, right=0.85, top=0.80, bottom=0.2) fig.set_size_inches(8.2, 11.6, forward=True) fig.axes[0].set_subplotspec(gs[0]) fig.axes[0].set_position(gs[0].get_position(fig)) fig.axes[1].set_subplotspec(gs[1]) fig.axes[1].set_position(gs[1].get_position(fig)) fig.axes[0].set_title("") ax2 = plt.subplot(gs[2]) print("Creating Time Index...") print( UTCDateTime(ppsd.times_processed[1]).datetime - UTCDateTime(ppsd.times_processed[0]).datetime) new_times = pd.date_range(UTCDateTime(ppsd.times_processed[0]).datetime, UTCDateTime(ppsd.times_processed[-1]).datetime, freq="1H") ind_times = np.array( [UTCDateTime(t).datetime for t in ppsd.current_times_used]) data = np.asarray(ppsd._binned_psds) print("Creating DataFrame...") data = pd.DataFrame(data, index=ind_times) # TODO : why resample here ? # data = data.resample("H").fillna(method="ffill") if color_lim: vmin, vmax = color_lim plt.pcolormesh(data.index, ppsd.period_bin_centers, data.T, cmap=cmap, vmin=vmin, vmax=vmax, rasterized=True) else: plt.imshow( data.T, cmap=cmap, aspect='auto', origin='lower', extent=(date2num(new_times[0].to_pydatetime()), date2num(new_times[-1].to_pydatetime()), ppsd.period_bin_centers[0], ppsd.period_bin_centers[-1]), ) cb = plt.colorbar(cax=plt.subplot(gs[3]), use_gridspec=True) plt.sca(ax2) cb.set_label("Amplitude [dB]") plt.ylabel("Period [s]") plt.semilogy((UTCDateTime(ppsd.times_processed[0]).datetime), 0, alpha=0) plt.xlim( UTCDateTime(ppsd.times_processed[0]).datetime, UTCDateTime(ppsd.times_processed[-1]).datetime) plt.ylim(period_lim[0], period_lim[1]) plt.suptitle(ppsd._get_plot_title()) fig.autofmt_xdate() xax = plt.gca().get_xaxis() # get the x-axis adf = xax.get_major_formatter() # the the auto-formatter adf.scaled[1. / 24] = '%Y-%m-%d %H:%M' # set the < 1d scale to H:M adf.scaled[1.0] = '%Y-%m-%d' # set the > 1d < 1m scale to Y-m-d adf.scaled[30.] = '%Y-%m' # set the > 1m < 1Y scale to Y-m adf.scaled[365.] = '%Y' # set the > 1y scale to Y if outfile: print("output to:", outfile) plt.savefig(outfile) if show: plt.show() plt.gcf() plt.gca() plt.close(fig) del fig
from obspy.core.util.base import get_example_file from obspy.signal import PPSD ppsd = PPSD.load_npz(get_example_file('ppsd_kw1_ehz.npz'), allow_pickle=True) ppsd.plot_temporal([0.1, 1, 10])
from obspy.core.util.base import get_example_file from obspy.signal import PPSD ppsd = PPSD.load_npz(get_example_file('ppsd_kw1_ehz.npz')) ppsd.plot_temporal([0.1, 1, 10])
n = read(nfile) z = read(zfile) #on the first time that data is found, check if the PPSD exists, #if not then create it if ppsd_init == False: #i.e. haven't checked for existence of ppsd Eppsd_file = ppsd_dir + site + '.LXE.ppsd.npz' Nppsd_file = ppsd_dir + site + '.LXN.ppsd.npz' Zppsd_file = ppsd_dir + site + '.LXZ.ppsd.npz' #Does the ppsd object exist? if exists(Eppsd_file) == True: #Load the previously saved PPSDs Eppsd = PPSD.load_npz(Eppsd_file, metadata=paz) Nppsd = PPSD.load_npz(Nppsd_file, metadata=paz) Zppsd = PPSD.load_npz(Zppsd_file, metadata=paz) #They don't exist yet, create new objects else: Eppsd = PPSD(e[0].stats, paz, db_bins=(-80, 20, 1.0), period_limits=[2, 1200], special_handling="ringlaser") Nppsd = PPSD(n[0].stats, paz, db_bins=(-80, 20, 1.0), period_limits=[2, 1200], special_handling="ringlaser")