def test_gmean_gerr3(self):
"""Difference between biased and unbiased error is small for large N"""
x = np.linspace(0.5, 1.5, 100)
w = np.linspace(0.5, 1.5, 100)
error = (0.01, 0.01)
gerrwbias = np.array(gerr(x, weights=w, unbiased=False)[1:])
gerrwunbias = np.array(gerr(x, weights=w, unbiased=True)[1:])
gerrbias = np.array(gerr(x, unbiased=False)[1:])
gerrunbias = np.array(gerr(x, unbiased=True)[1:])
np.testing.assert_array_less(gerrwunbias - gerrwbias, error)
np.testing.assert_array_less(gerrunbias - gerrbias, error)
def test_gmean_gerr3(self):
"""Difference between biased and unbiased error is small for large N"""
x = np.linspace(0.5, 1.5, 100)
w = np.linspace(0.5, 1.5, 100)
error = (0.01, 0.01)
gerrwbias = np.array(gerr(x, weights=w, unbiased=False)[1:])
gerrwunbias = np.array(gerr(x, weights=w, unbiased=True)[1:])
gerrbias = np.array(gerr(x, unbiased=False)[1:])
gerrunbias = np.array(gerr(x, unbiased=True)[1:])
np.testing.assert_array_less(gerrwunbias - gerrwbias, error)
np.testing.assert_array_less(gerrunbias - gerrbias, error)
def test_gmean_gerr1(self):
"""Results are same for no weight and equal weights"""
x = np.array([3, 5, 8, 10, np.nan, 3, 4, -np.inf])
w = np.ones(len(x))
self.assertEqual(gmean(x), gmean(x, weights=w))
self.assertEqual(gerr(x), gerr(x, weights=w))
x = x.reshape((2, 4))
w = np.ones(x.shape)
self.assertEqual(len(gmean(x, axis=0)), x.shape[1])
np.testing.assert_array_equal(gmean(x, axis=0),
gmean(x, axis=0, weights=w))
np.testing.assert_array_equal(gerr(x, axis=0),
gerr(x, axis=0, weights=w))
def test_gmean_gerr1(self):
"""Results are same for no weight and equal weights"""
x = np.array([3, 5, 8, 10, np.nan, 3, 4, -np.inf])
w = np.ones(len(x))
self.assertEqual(gmean(x), gmean(x, weights=w))
self.assertEqual(gerr(x), gerr(x, weights=w))
x = x.reshape((2, 4))
w = np.ones(x.shape)
self.assertEqual(len(gmean(x, axis=0)), x.shape[1])
np.testing.assert_array_equal(gmean(x, axis=0),
gmean(x, axis=0, weights=w))
np.testing.assert_array_equal(gerr(x, axis=0),
gerr(x, axis=0, weights=w))
def test_robust(self):
x = [1, 5, 6, 6, 6, 10000]
m, err1, err2 = gerr(x, robust=True)
m_, err1_, err2_ = gerr(x)
self.assertLess(m, m_)
self.assertLess(err1, err1_)
self.assertLess(err2, err2_)
x = np.reshape([3, 4, 5, 4, 5, 100, 1, 5, 6, 6, 6, 10000], (2, 6))
m, err1, err2 = gerr(x, axis=1, robust=True)
m_, err1_, err2_ = gerr(x, axis=1)
np.testing.assert_array_less(m, m_)
np.testing.assert_array_less(err1, err1_)
np.testing.assert_array_less(err2, err2_)
def test_robust(self):
x = [1, 5, 6, 6, 6, 10000]
m, err1, err2 = gerr(x, robust=True)
m_, err1_, err2_ = gerr(x)
self.assertLess(m, m_)
self.assertLess(err1, err1_)
self.assertLess(err2, err2_)
x = np.reshape([3, 4, 5, 4, 5, 100, 1, 5, 6, 6, 6, 10000], (2, 6))
m, err1, err2 = gerr(x, axis=1, robust=True)
m_, err1_, err2_ = gerr(x, axis=1)
np.testing.assert_array_less(m, m_)
np.testing.assert_array_less(err1, err1_)
np.testing.assert_array_less(err2, err2_)
def plot_results(result,
v0=None,
fname=None,
title=None,
quantities=QUANTITIES,
mean=None,
llim=None,
Qlim=None,
figsize=None):
"""Plot results"""
freq = np.array(result['freq'])
N = len(quantities)
n = int(np.ceil(np.sqrt(N)))
fig = plt.figure(figsize=figsize)
gs = gridspec.GridSpec(n, n)
share = None
col = collect_results(result, only=('g0', 'b', 'error', 'v0'))
v0 = v0 or result['config'].get('v0') or col['v0']
result = col
result.pop('v0', None)
weights = 1 / np.array(result['error']) if mean == 'weighted' else None
robust = mean == 'robust'
for i, q in enumerate(quantities):
ax = plt.subplot(gs[i // n, i % n], sharex=share)
if q == 'nobs':
nobs = np.sum(~np.isnan(result['g0']), axis=0)
ax.bar(freq, nobs, width=0.1 * freq, color='gray')
else:
value = result[DEPMAP[q]]
value = calc_dependent(q, value, freq, v0)
freqs = np.repeat(freq[np.newaxis, :], value.shape[0], axis=0)
ax.loglog(freqs, value, 'o', ms=MS, color='gray', mec='gray')
means, err1, err2 = gerr(value,
axis=0,
weights=weights,
robust=robust)
errs = (err1, err2)
ax.errorbar(freq,
means,
yerr=errs,
marker='o',
mfc='k',
mec='k',
color='m',
ecolor='m')
ax.annotate(QLABELS[q], (1, 1), (-5, -5),
'axes fraction',
'offset points',
ha='right',
va='top')
_set_gridlabels(ax, i, n, n, N, ylabel=None)
if share is None:
share = ax
if q in ('Qsc', 'Qi') and Qlim:
ax.set_ylim(Qlim)
if q in ('lsc', 'li') and llim:
ax.set_ylim(llim)
ax.set_xlim(freqlim(freq))
_savefig(fig, fname=fname, title=title)
def plot_sites(result, fname=None, title=None, mean=None,
xlim=None, ylim=(1e-2, 1e2), nx=None, figsize=None):
"""Plot site amplification factors"""
freq = np.array(result['freq'])
col = collect_results(result, only=['R', 'error'])
R = col['R']
weights = 1 / np.array(col['error']) if mean == 'weighted' else None
robust = mean == 'robust'
max_nobs = np.max([np.sum(~np.isnan(r), axis=0) for r in R.values()])
N = max_nobs > 1
for station in sorted(R):
if not np.all(np.isnan(R[station])):
N = N + 1
# N = len(R) + (max_nobs > 1)
# for i
fig = plt.figure(figsize=figsize)
nx, ny, gs = _get_grid(N, nx=nx)
cmap = plt.get_cmap('hot_r', max_nobs)
norm = mpl.colors.Normalize(vmin=0.5, vmax=max_nobs + 0.5)
share = None
i = 0
for station in sorted(R):
if np.all(np.isnan(R[station])):
continue
ax = plt.subplot(gs[i // nx, i % nx], sharex=share, sharey=share)
means, err1, err2 = gerr(R[station], axis=0, weights=weights,
robust=robust)
nobs = 1. * np.sum(~np.isnan(R[station]), axis=0)
errs = (err1, err2)
freqs = np.repeat(freq[np.newaxis, :], R[station].shape[0], axis=0)
# if not np.all(np.isnan(R[station])):
if max_nobs == 1:
kwargs = {'c': 'k'}
else:
kwargs = {'c': nobs, 'norm': norm, 'cmap': cmap}
ax.loglog(freqs, R[station], 'o', ms=MS, color='gray', mec='gray')
ax.errorbar(freq, means, yerr=errs, marker=None,
color='m', ecolor='m')
sc = ax.scatter(freq, means, s=4 * MS ** 2,
marker='o', zorder=10,
linewidth=0.5,
**kwargs)
ax.annotate(station, (1, 1), (-5, -5), 'axes fraction',
'offset points', ha='right', va='top', size='x-small')
_set_gridlabels(ax, i, nx, ny, N, ylabel='amplification factor')
if share is None:
share = ax
i += 1
ax.set_xlim(xlim or freqlim(freq))
if ylim:
ax.set_ylim(ylim)
if max_nobs != 1:
ax = plt.subplot(gs[(N - 1) // nx, (N - 1) % nx])
ax.set_axis_off()
fig.colorbar(sc, ax=ax, shrink=0.9, format='%d', label='nobs',
ticks=np.arange(0, max_nobs + 1, max(1, max_nobs // 5)))
_savefig(fig, fname=fname, title=title)
def plot_sites(result, fname=None, title=None, mean=None,
xlim=None, ylim=(1e-2, 1e2), nx=None, figsize=None):
freq = np.array(result['freq'])
g0, b, error, R, _, _, _ = collect_results(result)
weights = 1 / np.array(error) if mean == 'weighted' else None
robust = mean == 'robust'
max_nobs = np.max([np.sum(~np.isnan(r), axis=0) for r in R.values()])
N = max_nobs > 1
for station in sorted(R):
if not np.all(np.isnan(R[station])):
N = N + 1
#N = len(R) + (max_nobs > 1)
#for i
fig = plt.figure(figsize=figsize)
nx, ny, gs = _get_grid(N, nx=nx)
cmap = plt.get_cmap('hot_r', max_nobs)
norm = mpl.colors.Normalize(vmin=0.5, vmax=max_nobs + 0.5)
share = None
i = 0
for station in sorted(R):
if np.all(np.isnan(R[station])):
continue
ax = plt.subplot(gs[i // nx, i % nx], sharex=share, sharey=share)
means, err1, err2 = gerr(R[station], axis=0, weights=weights,
robust=robust)
nobs = 1. * np.sum(~np.isnan(R[station]), axis=0)
errs = (err1, err2)
freqs = np.repeat(freq[np.newaxis, :], R[station].shape[0], axis=0)
#if not np.all(np.isnan(R[station])):
if max_nobs == 1:
kwargs = {'c': 'k'}
else:
kwargs = {'c': nobs, 'norm': norm, 'cmap': cmap}
ax.loglog(freqs, R[station], 'o', ms=MS, color='gray', mec='gray')
ax.errorbar(freq, means, yerr=errs, marker=None,
color='m', ecolor='m')
sc = ax.scatter(freq, means, s=4 * MS ** 2,
marker='o', zorder=10,
linewidth=0.5,
**kwargs)
ax.annotate(station, (1, 1), (-5, -5), 'axes fraction',
'offset points', ha='right', va='top', size='x-small')
_set_gridlabels(ax, i, nx, ny, N, ylabel='amplification factor')
if share is None:
share = ax
i += 1
ax.set_xlim(xlim or freqlim(freq))
if ylim:
ax.set_ylim(ylim)
if max_nobs != 1:
ax = plt.subplot(gs[(N - 1) // nx, (N - 1) % nx])
ax.set_axis_off()
fig.colorbar(sc, ax=ax, shrink=0.9, format='%d', label='nobs',
ticks=np.arange(0, max_nobs + 1, max(1, max_nobs // 5)))
_savefig(fig, fname=fname, title=title)
def test_gmean_gerr2(self):
"""Results are same for repeated elements and adapted weights"""
x1 = np.array([1, 2, 2, 5, 5, 5, 5, 5, np.inf])
x2 = np.array([1, 2, 5, np.inf])
w = np.array([1, 2, 5, 10])
self.assertEqual(gmean(x1), gmean(x2, weights=w))
# errors can only be compared for biased std
np.testing.assert_array_equal(gerr(x1, unbiased=False)[1:],
gerr(x2, weights=w, unbiased=False)[1:])
# weighted errors are bigger than unweighted
np.testing.assert_array_less(gerr(x1)[1:], gerr(x2, weights=w)[1:])
# biased errors are smaller than unbiased
np.testing.assert_array_less(gerr(x2, weights=w, unbiased=False)[1:],
gerr(x2, weights=w)[1:])
def plot_results(result, v0=None, fname=None, title=None,
quantities=QUANTITIES, mean=None,
llim=None, Qlim=None, figsize=None):
freq = np.array(result['freq'])
N = len(quantities)
n = int(np.ceil(np.sqrt(N)))
fig = plt.figure(figsize=figsize)
gs = gridspec.GridSpec(n, n)
share = None
g0, b, error, R, _, _, v02 = collect_results(result)
v0 = v0 or result['config'].get('v0') or v02
result = {'g0': g0, 'b': b, 'error': error, 'R': R}
weights = 1 / np.array(error) if mean == 'weighted' else None
robust = mean == 'robust'
for i, q in enumerate(quantities):
ax = plt.subplot(gs[i // n, i % n], sharex=share)
if q == 'nobs':
nobs = np.sum(~np.isnan(g0), axis=0)
ax.bar(freq, nobs, width=0.1 * freq, color='gray')
else:
value = result[DEPMAP[q]]
value = calc_dependent(q, value, freq, v0)
freqs = np.repeat(freq[np.newaxis, :], value.shape[0], axis=0)
ax.loglog(freqs, value, 'o', ms=MS, color='gray', mec='gray')
means, err1, err2 = gerr(
value, axis=0, weights=weights, robust=robust)
errs = (err1, err2)
ax.errorbar(freq, means, yerr=errs, marker='o',
mfc='k', mec='k', color='m', ecolor='m')
ax.annotate(QLABELS[q], (1, 1), (-5, -5), 'axes fraction',
'offset points', ha='right', va='top')
_set_gridlabels(ax, i, n, n, N, ylabel=None)
if share is None:
share = ax
if q in ('Qsc', 'Qi') and Qlim:
ax.set_ylim(Qlim)
if q in ('lsc', 'li') and llim:
ax.set_ylim(llim)
ax.set_xlim(freqlim(freq))
_savefig(fig, fname=fname, title=title)
def test_gmean_gerr2(self):
"""Results are same for repeated elements and adapted weights"""
x1 = np.array([1, 2, 2, 5, 5, 5, 5, 5, np.inf])
x2 = np.array([1, 2, 5, np.inf])
w = np.array([1, 2, 5, 10])
self.assertEqual(gmean(x1), gmean(x2, weights=w))
# errors can only be compared for biased std
np.testing.assert_array_equal(
gerr(x1, unbiased=False)[1:],
gerr(x2, weights=w, unbiased=False)[1:])
# weighted errors are bigger than unweighted
np.testing.assert_array_less(gerr(x1)[1:], gerr(x2, weights=w)[1:])
# biased errors are smaller than unbiased
np.testing.assert_array_less(
gerr(x2, weights=w, unbiased=False)[1:],
gerr(x2, weights=w)[1:])
def collect_results(correct=True, events=False):
with open(CONF_FNAME) as f:
conf = json.load(f, cls=ConfigJSONDecoder)
freq = get_freqs(**conf['freqs'])
fbands = list(freq.values())
freq = list(freq.keys())
fa = np.asarray(freq)
print('copy results from shove')
r = {'freq': freq, 'events': {}, 'config': conf}
if GENERATE_TEST_FILE:
nobs_station = defaultdict(int)
with JsonShove(SHOVE_FNAME, compress=True) as sh:
msg = '%d in shove / %d events processed'
print(msg % (len(sh), len(glob(EVENT_GLOB))))
for evid in sh:
if sh[evid] is None:
continue
if GENERATE_TEST_FILE:
stations = sh[evid]['R'].keys()
if min(nobs_station[s] for s in stations) >= MAX_NOBS:
continue
for s in stations:
nobs_station[s] = nobs_station[s] + 1
r['events'][evid] = sh[evid]
if GENERATE_TEST_FILE:
i = 6
for evid in r['events']:
for sta in r['events'][evid]['R']:
r['events'][evid]['R'][sta] = \
r['events'][evid]['R'][sta][i:i + 1]
r['events'][evid] = {
'W': r['events'][evid]['W'][i:i + 1],
'R': r['events'][evid]['R']
}
r['freq'] = r['freq'][i:i + 1]
del r['config']
with open(TMP + 'usarray_dataset.json', 'wb') as f:
json.dump(r, f)
return
if IGNORE_STATIONS is not None:
print('remove stations %s' % IGNORE_STATIONS)
sh = r['events']
for evid in sh.keys():
for station in sh[evid]['R'].keys()[:]:
if station in IGNORE_STATIONS:
sh[evid]['R'].pop(station)
if len(sh[evid]['R']) == 0:
sh.pop(evid)
Nres = len(r['events'])
print("number of events with results: %d" % Nres)
rorig = r
if correct:
rorig = deepcopy(r)
print('correct sensitivities')
# r = align_site_responses(r, station=BOREHOLE_STATIONS, response=0.25)
r = align_site_responses(r)
if events:
print('collect magnitudes')
eventresults2json(r)
print('collect results')
data = defaultdict(lambda: defaultdict(lambda: [[] for _ in freq]))
data['v0'] = defaultdict(list)
results = defaultdict(lambda: defaultdict(dict))
sh = r['events']
sh2 = rorig['events']
for evid in sh:
for station in sh2[evid]['R']:
v0 = sh2[evid]['v0']
data['v0'][station].append(v0)
for i in range(len(sh2[evid]['R'][station])):
R1 = sh[evid]['R'][station][i]
R2 = sh2[evid]['R'][station][i]
b = sh[evid]['b'][i]
g0 = sh[evid]['g0'][i]
if R2 is not None:
if R1 is not None:
data['R'][station][i].append(R1)
Qi = calc_dependent('Qi', b, freq[i], v0)
data['Qi'][station][i].append(Qi)
Qsc = calc_dependent('Qsc', g0, freq[i], v0)
data['Qsc'][station][i].append(Qsc)
print('calc mean, err and slope')
for what in ('v0', 'R', 'Qsc', 'Qi'):
for station in data[what]:
d = data[what][station]
if what == 'v0':
results[what][station] = np.mean(d)
continue
mean, err, nobs = [], [], []
for i in range(len(freq)):
da = np.asarray(d[i], dtype=np.float)
m, e1, e2 = gerr(da)
e = np.log10(m) - np.log10(e1)
m = np.log10(m)
mean.append(None if not np.isscalar(m) or np.isnan(m) else m)
err.append(None if not np.isscalar(e) or np.isnan(e) else e)
nobs.append(np.count_nonzero(np.logical_not(np.isnan(da))))
results[what][station]['mean'] = mean
results[what][station]['error'] = err
if what == 'Qsc':
results['nobs'][station] = nobs
mean = np.asarray(mean, dtype=np.float)
ind = np.logical_not(np.isnan(mean))
if what == 'R':
continue
elif np.count_nonzero(ind) < 3:
slope, intercept = None, None
else:
slope, intercept = linear_fit(mean[ind], np.log10(fa[ind]))
results[what][station]['slope'] = slope
results[what][station]['intercept'] = intercept
print('convert to OrderedDict and save json')
for key in results:
if key not in ('v0', 'nobs'):
for sta in results[key]:
results[key][sta] = sort_dict(results[key][sta], order=ORDER)
results[key] = OrderedDict(sorted(results[key].items()))
results['freqs'] = freq
results['freq_bands'] = fbands
results['stations'] = collect_stations(False) # get coordinates
results['title'] = TITLE
results['description'] = DESC
results['author'] = 'Tom Eulenfeld'
# results['copyright'] = COPYR
results['license'] = LICENSE
results = sort_dict(results, order=ORDER)
with open(RESULTSJSON, 'w') as f:
f.write(to_json(results, nl_after_str=True))
def plot_sites(result,
mean=None,
xlim=None,
ylim=(1e-2, 1e2),
nx=None,
cmap='viridis_r',
vmin=None,
vmax=None,
**kwargs):
"""Plot site amplification factors"""
freq = np.array(result['freq'])
# True for invert_events_simultaneously
single_inversion = 'R' not in list(result['events'].values())[0]
if single_inversion:
colres = result
R = copy(colres['R'])
for sta in R:
R[sta] = np.array(R[sta], dtype=float)
max_nobs = 1
else:
colres = collect_results(result, only=['R', 'error'])
R = colres['R']
max_nobs = np.max([np.sum(~np.isnan(r), axis=0) for r in R.values()])
weights = 1 / np.array(colres['error']) if mean == 'weighted' else None
robust = mean == 'robust'
N = max_nobs > 1
for station in sorted(R):
if not np.all(np.isnan(R[station])):
N = N + 1
# N = len(R) + (max_nobs > 1)
fig = plt.figure()
nx, ny, gs = _get_grid(N, nx=nx)
if cmap is None:
cmap = 'black'
cmap = plt.get_cmap(cmap, max_nobs)
if vmax is None:
vmax = max_nobs + 0.5
if vmin is None:
vmin = 0.5
norm = mpl.colors.Normalize(vmin=vmin, vmax=vmax)
share = None
i = 0
for station in sorted(R):
if np.all(np.isnan(R[station])):
continue
ax = plt.subplot(gs[i // nx, i % nx], sharex=share, sharey=share)
means, err1, err2 = gerr(R[station],
axis=0,
weights=weights,
robust=robust)
errs = (err1, err2)
# if not np.all(np.isnan(R[station])):
if max_nobs == 1:
kw = {'c': 'k'}
else:
nobs = 1. * np.sum(~np.isnan(R[station]), axis=0)
kw = {'c': nobs, 'norm': norm, 'cmap': cmap}
if not single_inversion:
freqs = np.repeat(freq[np.newaxis, :], R[station].shape[0], axis=0)
ax.plot(freqs, R[station], 'o', ms=MS, color='gray', mec='gray')
ax.errorbar(freq, means, yerr=errs, marker=None, color='m', ecolor='m')
sc = ax.scatter(freq,
means,
s=4 * MS**2,
marker='o',
zorder=10,
linewidth=0.5,
**kw)
ax.set_xscale('log')
ax.set_yscale('log')
ax.annotate(station, (1, 1), (-5, -5),
'axes fraction',
'offset points',
ha='right',
va='top',
size='x-small')
_set_gridlabels(ax, i, nx, ny, N, ylabel='amplification factor')
if share is None:
share = ax
i += 1
ax.set_xlim(xlim or freqlim(freq))
if ylim:
ax.set_ylim(ylim)
if max_nobs != 1:
ax = plt.subplot(gs[(N - 1) // nx, (N - 1) % nx])
ax.set_axis_off()
fig.colorbar(sc,
ax=ax,
shrink=0.9,
format='%d',
label='nobs',
ticks=np.arange(0, max_nobs + 1, max(1, max_nobs // 5)))
_savefig(fig, **kwargs)