def generate_artist(self):
container=self.get_container()
if self.isempty() is False:
return
x, y = self._eval_xy() # this handles "use_var"
lp=self.getp("loaded_property")
if True:
x, y = self.getp(("x", "y"))
if y is None: return
if x is None: return
if (x is not None and
y is not None):
self._data_extent=[np.nanmin(x), np.nanmax(x),
np.nanmin(y), np.nanmax(y)]
if len(y.shape) == 1:
kywds = self._var["kywds"]
args, self._tri = tri_args(x, y, self._tri)
kywds['mask'] = self.getp('mask')
kywds['linestyle'] = self.getp('linestyle')
kywds['linewidth'] = self.getp('linewidth')
kywds['color'] = self.getp('color')
a = triplot(container, *args, **kywds)
self.set_artist(a[0])
self._other_artists = a[1:]
if lp is not None:
for i in range(0, len(lp)):
self.set_artist_property(self._artists[i], lp[i])
self.delp("loaded_property")
self.set_rasterized()
def show_overlay(img3d, cc3d, ncc=10, s=85, xyz = 'xy',alpha=.8):
"""Shows the connected components overlayed over img3d
Input
======
img3d -- 3d array
cc3d -- 3d array ( preferably of same shape as img3d, use get_3d_cc(...) )
ncc -- where to cut off the color scale
s -- slice to show
xyz -- which projection to use in {'xy','xz','yz'}
"""
cc = get_slice(cc3d,s,xyz)
img = get_slice(img3d,s,xyz)
notcc = np.isnan(cc)
incc = np.not_equal(notcc,True)
img4 = plt.cm.gray(img/np.nanmax(img))
if ncc is not np.Inf:
cc = plt.cm.jet(cc/float(ncc))
else:
cc = plt.cm.jet(np.log(cc)/np.log(np.nanmax(cc)))
cc[notcc,:]=img4[notcc,:]
cc[incc,3] = 1-img[incc]/(2*np.nanmax(img))
plt.imshow(cc)
def tuning(x, y, err=None, smooth=None, ylabel=None, pal=None):
"""
Plot a tuning curve
"""
if smooth is not None:
xs, ys = smoothfit(x, y, smooth)
plt.plot(xs, ys, linewidth=4, color="black", zorder=1)
else:
ys = asarray([0])
if pal is None:
pal = sns.color_palette("husl", n_colors=len(x) + 6)
pal = pal[2 : 2 + len(x)][::-1]
plt.scatter(x, y, s=300, linewidth=0, color=pal, zorder=2)
if err is not None:
plt.errorbar(x, y, yerr=err, linestyle="None", ecolor="black", zorder=1)
plt.xlabel("Wall distance (mm)")
plt.ylabel(ylabel)
plt.xlim([-2.5, 32.5])
errTmp = err
errTmp[isnan(err)] = 0
rng = max([nanmax(ys), nanmax(y + errTmp)])
plt.ylim([0 - rng * 0.1, rng + rng * 0.1])
plt.yticks(linspace(0, rng, 3))
plt.xticks(range(0, 40, 10))
sns.despine()
return rng
def __call__(self, transform_xy, x1, y1, x2, y2):
"""
get extreme values.
x1, y1, x2, y2 in image coordinates (0-based)
nx, ny : number of divisions in each axis
"""
x_, y_ = np.linspace(x1, x2, self.nx), np.linspace(y1, y2, self.ny)
x, y = np.meshgrid(x_, y_)
lon, lat = transform_xy(np.ravel(x), np.ravel(y))
# iron out jumps, but algorithm should be improved.
# This is just naive way of doing and my fail for some cases.
# Consider replacing this with numpy.unwrap
# We are ignoring invalid warnings. They are triggered when
# comparing arrays with NaNs using > We are already handling
# that correctly using np.nanmin and np.nanmax
with np.errstate(invalid='ignore'):
if self.lon_cycle is not None:
lon0 = np.nanmin(lon)
lon -= 360. * ((lon - lon0) > 180.)
if self.lat_cycle is not None:
lat0 = np.nanmin(lat)
lat -= 360. * ((lat - lat0) > 180.)
lon_min, lon_max = np.nanmin(lon), np.nanmax(lon)
lat_min, lat_max = np.nanmin(lat), np.nanmax(lat)
lon_min, lon_max, lat_min, lat_max = \
self._adjust_extremes(lon_min, lon_max, lat_min, lat_max)
return lon_min, lon_max, lat_min, lat_max
def summary():
# read sonde data
for sites in [[0],[1],[2]]:
slist,snames=read_diff_events(sites=sites)
ecount = [len(s.einds) for s in slist]
mintp = [np.nanmin(s.tp) for s in slist]
meantp = [np.nanmean(s.tp) for s in slist]
maxtp = [np.nanmax(s.tp) for s in slist]
head="%9s"%slist[0].name
ecount = "events "
meantp = "mean tph "
minmax = "tph bound"
for sonde, sname in zip(slist,snames):
head=head+'| %16s'%sname
ecount=ecount+'| %16d'%len(sonde.einds)
meantp=meantp+'| %16.2f'%np.nanmean(sonde.tp)
minmax=minmax+'| %7.2f,%7.2f '%(np.nanmin(sonde.tp),np.nanmax(sonde.tp))
print("")
print(head)
print(ecount)
print(meantp)
print(minmax)
def normalizeFloatImage3(floatImage):
mn_0 = np.nanmin(np.nanmin(floatImage[:, :,1]))
mx_0 = np.nanmax(np.nanmax(floatImage[:, :, 1]))
rows = floatImage.shape[0]
cols = floatImage.shape[1]
mn_0 = 1000
mx_0 =-1000
for r in range(rows):
for c in range(cols):
if floatImage[r,c, 2] <= 0:
mn_0 = mn_0 if mn_0 <= floatImage[r,c, 1 ] else floatImage[r,c, 1 ]
mx_0 = mx_0 if mx_0 >= floatImage[r,c, 1 ] else floatImage[r,c, 1 ]
fctr_0 = 255.0/(mx_0 - mn_0)
for r in range(rows):
for c in range(cols):
if floatImage[r,c, 2] <= 0:
floatImage[r,c, 1] = (floatImage[r,c, 1] - mn_0) * fctr_0
else:
floatImage[r,c, 1]=0
print "mn_0: ", mn_0, " mx_0: ", mx_0
def _set_minmax(self):
data = self._get_fast_data()
try:
self.maxval = numpy.nanmax(data)
self.minval = numpy.nanmin(data)
except Exception:
self.maxval = 0
self.minval = 0
# TODO: see if there is a faster way to ignore infinity
try:
if numpy.isfinite(self.maxval):
self.maxval_noinf = self.maxval
else:
self.maxval_noinf = numpy.nanmax(data[numpy.isfinite(data)])
except:
self.maxval_noinf = self.maxval
try:
if numpy.isfinite(self.minval):
self.minval_noinf = self.minval
else:
self.minval_noinf = numpy.nanmin(data[numpy.isfinite(data)])
except:
self.minval_noinf = self.minval
def __call__(self, transform_xy, x1, y1, x2, y2):
"""
get extreme values.
x1, y1, x2, y2 in image coordinates (0-based)
nx, ny : number of divisions in each axis
"""
x_, y_ = np.linspace(x1, x2, self.nx), np.linspace(y1, y2, self.ny)
x, y = np.meshgrid(x_, y_)
lon, lat = transform_xy(np.ravel(x), np.ravel(y))
# iron out jumps, but algorithm should be improved.
# Tis is just naive way of doing and my fail for some cases.
if self.lon_cycle is not None:
lon0 = np.nanmin(lon)
lon -= 360.0 * ((lon - lon0) > 180.0)
if self.lat_cycle is not None:
lat0 = np.nanmin(lat)
lat -= 360.0 * ((lat - lat0) > 180.0)
lon_min, lon_max = np.nanmin(lon), np.nanmax(lon)
lat_min, lat_max = np.nanmin(lat), np.nanmax(lat)
lon_min, lon_max, lat_min, lat_max = self._adjust_extremes(lon_min, lon_max, lat_min, lat_max)
return lon_min, lon_max, lat_min, lat_max
def calc_norm_summary_tables(accuracy_tbl, time_tbl):
"""
Calculate normalized performance/ranking summary, as numpy
matrices as usual for convenience, and matrices of additional
statistics (min, max, percentiles, etc.)
Here normalized means relative to the best which gets a 1, all
others get the ratio resulting from dividing by the performance of
the best.
"""
# Min across all minimizers, i.e. for each fit problem what is the lowest chi-squared and the lowest time
min_sum_err_sq = np.nanmin(accuracy_tbl, 1)
min_runtime = np.nanmin(time_tbl, 1)
# create normalised tables
norm_acc_rankings = accuracy_tbl / min_sum_err_sq[:, None]
norm_runtimes = time_tbl / min_runtime[:, None]
summary_cells_acc = np.array([np.nanmin(norm_acc_rankings, 0),
np.nanmax(norm_acc_rankings, 0),
stats.nanmean(norm_acc_rankings, 0),
stats.nanmedian(norm_acc_rankings, 0)
])
summary_cells_runtime = np.array([np.nanmin(norm_runtimes, 0),
np.nanmax(norm_runtimes, 0),
stats.nanmean(norm_runtimes, 0),
stats.nanmedian(norm_runtimes, 0)
])
return norm_acc_rankings, norm_runtimes, summary_cells_acc, summary_cells_runtime
def test_derivative(self):
log= logging.getLogger( "test_J0023.derivative_test")
testp = tdu.get_derivative_params(self.modelJ0023)
delay = self.modelJ0023.delay(self.toasJ0023)
for p in testp.keys():
log.debug( "Runing derivative for %s", 'd_delay_d_'+p)
if p in ['EPS2', 'EPS1']:
testp[p] = 10
ndf = self.modelJ0023.d_phase_d_param_num(self.toasJ0023, p, testp[p])
adf = self.modelJ0023.d_phase_d_param(self.toasJ0023, delay, p)
diff = adf - ndf
if not np.all(diff.value) == 0.0:
mean_der = (adf+ndf)/2.0
relative_diff = np.abs(diff)/np.abs(mean_der)
#print "Diff Max is :", np.abs(diff).max()
msg = 'Derivative test failed at d_delay_d_%s with max relative difference %lf' % (p, np.nanmax(relative_diff).value)
if p in ['PMELONG', 'ELONG']:
tol = 2e-2
elif p in ['FB2', 'FB3']:
tol = 0.08
else:
tol = 1e-3
log.debug( "derivative relative diff for %s, %lf"%('d_delay_d_'+p, np.nanmax(relative_diff).value))
assert np.nanmax(relative_diff) < tol, msg
else:
continue
def _get_Tp_limits(self):
"""Get the limits for the graphs in temperature and pressure, based on
SI units: [Tmin, Tmax, pmin, pmax]"""
T_lo,T_hi,P_lo,P_hi = self.limits
Ts_lo,Ts_hi = self._get_sat_bounds(CoolProp.iT)
Ps_lo,Ps_hi = self._get_sat_bounds(CoolProp.iP)
if T_lo is None: T_lo = 0.0
elif T_lo < self.ID_FACTOR: T_lo *= Ts_lo
if T_hi is None: T_hi = 1e6
elif T_hi < self.ID_FACTOR: T_hi *= Ts_hi
if P_lo is None: P_lo = 0.0
elif P_lo < self.ID_FACTOR: P_lo *= Ps_lo
if P_hi is None: P_hi = 1e10
elif P_hi < self.ID_FACTOR: P_hi *= Ps_hi
try: T_lo = np.nanmax([T_lo, self._state.trivial_keyed_output(CoolProp.iT_min)])
except: pass
try: T_hi = np.nanmin([T_hi, self._state.trivial_keyed_output(CoolProp.iT_max)])
except: pass
try: P_lo = np.nanmax([P_lo, self._state.trivial_keyed_output(CoolProp.iP_min)])
except: pass
try: P_hi = np.nanmin([P_hi, self._state.trivial_keyed_output(CoolProp.iP_max)])
except: pass
return [T_lo,T_hi,P_lo,P_hi]
def draw_hmap_old(hmap, yvals, fname=None):
"""
Plot a matrix as a heat map and write an image file.
:param hmap: Heat map matrix.
:param yvals: Heat map Y labels (e.g. amino acid names).
:param fname: Destination image file.
"""
if np.nanmax(hmap) > abs(np.nanmin(hmap)):
vmax = np.nanmax(hmap)
vmin = -np.nanmax(hmap)
else:
vmax = abs(np.nanmin(hmap))
vmin = np.nanmin(hmap)
fig = plt.figure()
plt.figure(figsize=(20,10))
plt.imshow(hmap, cmap='RdBu', interpolation = 'nearest',aspect='auto',vmin = vmin ,vmax = vmax )
plt.xlim(0, hmap.shape[1])
plt.ylim(0, hmap.shape[0])
ax = plt.gca()
fig.set_facecolor('white')
ax.set_xlim((-0.5, hmap.shape[1] -0.5))
ax.set_ylim((-0.5, hmap.shape[0] -0.5))
ax.set_yticks([x for x in xrange(0, hmap.shape[0])])
ax.set_yticklabels(yvals)
ax.set_xticks(range(0,76,5))
ax.set_xticklabels(range(2,76,5)+['STOP'])
ax.set_ylabel('Residue')
ax.set_xlabel('Ub Sequence Position')
cb = plt.colorbar()
cb.set_clim(vmin=vmin, vmax=vmax)
cb.set_label('Relative Fitness')
if fname is not None:
plt.savefig(fname, bbox_inches='tight')
return fig
def show(self,**kwargs):
display = kwargs.get('display', True)
show_layers = kwargs.get('show_layers',self.layers)
try:
show_layers=sorted(show_layers)
except TypeError:
show_layers=[show_layers]
extent=kwargs.get('extent',
max_axis(*tuple(_image.axis for _image in self.image_sorted[self.layers[0]])))
vmin=kwargs.get('vmin')
vmax=kwargs.get('vmax')
fig = plt.figure(figsize=(8, 8*abs((extent[3]-extent[2])*1./(extent[1]-extent[0]))))
for layer in show_layers:
for image in self.image_sorted[layer]:
if layer==show_layers[0] and image==self.image_sorted[layer][0]:
if not vmin:
kwargs['vmin']=np.nanmin(image.image)
vmin=np.nanmin(image.image)
if not vmax:
kwargs['vmax']=np.nanmax(image.image)
vmax=np.nanmax(image.image)
image.show(hold=True,**kwargs)
else:
image.show(hold=True,vmin=vmin,vmax=vmax,scalebar='off',colorbar='off')
plt.xlim(extent[:2])
plt.ylim(extent[-2:])
if display:
plt.show()
else:
return fig
def plot_richness_scatter(gals, name, full_set):
log_counts_a, scatter_a = richness_scatter(gals[gals['ssfr'] < -11.0], full_set)
log_counts_p, scatter_p = richness_scatter(gals[gals['pred'] < -11.0], full_set)
#fig1 = plt.figure(figsize=(12,7))
#frame1=fig1.add_axes((.1,.3,.8,.6))
#plt.subplot(121)
plt.plot(log_counts_a, scatter_a, 'o', label='input', color='k', markersize=7)
plt.plot(log_counts_p, scatter_p, 'o', label='predicted', color=red_col, markersize=7)
#plt.title('Scatter in richness ' + name)
plt.xlabel('Log Number of red satellites')
plt.xlabel('$<log N_{red sat}>$')
plt.xlim(-.1,2.6)
plt.ylim(0, np.max([np.nanmax(scatter_a),np.nanmax(scatter_p)]) +.1)
plt.ylabel('Scatter in $M_{halo}$')
plt.legend(loc='best')
#plt.subplot(122)
#frame2=fig1.add_axes((.1,.1,.8,.2))
#series_a = pd.Series(scatter_a, index=counts_a)
#series_p = pd.Series(scatter_p, index=counts_p)
# scat_diff = (series_a - series_p)/series_a
#scat_ratio = series_p/series_a
#plt.plot(scat_diff.index, scat_diff.values, 'ob')
#plt.plot(scat_ratio.index, scat_ratio.values, 'ob')
#plt.title("Scatter ratios in richness for actual vs predicted")
#plt.axhline(0)
#plt.ylabel('Error')
#plt.xlabel('Number of red satellites')
return
def classify(request):
C = json.loads(request.POST["C"])
try:
features, labels = get_multi_features(request)
except ValueError as e:
return HttpResponse(json.dumps({"status": e.message}))
try:
kernel = get_kernel(request, features)
except ValueError as e:
return HttpResponse(json.dumps({"status": e.message}))
learn = "No"
values=[]
try:
domain = json.loads(request.POST['axis_domain'])
x, y, z = svm.classify_svm(sg.GMNPSVM, features, labels, kernel, domain, learn, values, C, False)
except Exception as e:
return HttpResponse(json.dumps({"status": repr(e)}))
# z = z + np.random.rand(*z.shape) * 0.01
z_max = np.nanmax(z)
z_min = np.nanmin(z)
z_delta = 0.1*(np.nanmax(z)-np.nanmin(z))
data = {"status": "ok",
"domain": [z_min-z_delta, z_max+z_delta],
"max": z_max+z_delta,
"min": z_min-z_delta,
"z": z.tolist()}
return HttpResponse(json.dumps(data))
def getRange(self, axis, depname, axrange):
"""Update axis range from data."""
s = self.settings
doc = self.document
if ( (depname == 'sx' and s.direction == 'horizontal') or
(depname == 'sy' and s.direction == 'vertical') ):
# update axis in direction of data
if s.calculate:
# update from values
values = s.get('values').getData(doc)
if values:
for v in values:
if len(v.data) > 0:
axrange[0] = min(axrange[0], N.nanmin(v.data))
axrange[1] = max(axrange[1], N.nanmax(v.data))
else:
# update from manual entries
drange = self.rangeManual()
axrange[0] = min(axrange[0], drange[0])
axrange[1] = max(axrange[1], drange[1])
else:
# update axis in direction of datasets
posns = self.getPosns()
if len(posns) > 0:
axrange[0] = min(axrange[0], N.nanmin(posns)-0.5)
axrange[1] = max(axrange[1], N.nanmax(posns)+0.5)
def acquire_data(self, var_name=None, slice_=()):
if var_name in self._variables:
vars = [var_name]
else:
vars = self._variables
if not isinstance(slice_, tuple): slice_ = (slice_,)
for vn in vars:
var = self._data_array[vn]
ndims = len(var.shape)
# Ensure the slice_ is the appropriate length
if len(slice_) < ndims:
slice_ += (slice(None),) * (ndims-len(slice_))
arri = ArrayIterator(var, self._block_size)[slice_]
for d in arri:
if d.dtype.char is "S":
# Obviously, we can't get the range of values for a string data type!
rng = None
elif isinstance(d, numpy.ma.masked_array):
# TODO: This is a temporary fix because numpy 'nanmin' and 'nanmax'
# are currently broken for masked_arrays:
# http://mail.scipy.org/pipermail/numpy-discussion/2011-July/057806.html
dc = d.compressed()
if dc.size == 0:
rng = None
else:
rng = (numpy.nanmin(dc), numpy.nanmax(dc))
else:
rng = (numpy.nanmin(d), numpy.nanmax(d))
yield vn, arri.curr_slice, rng, d
return
def stokes_plot(x_data, xlabel, I_data, Q_data, U_data, V_data,
filename):
"""Generate plot of 4 stokes parameters"""
fig, (ax1, ax2, ax3, ax4) = plt.subplots(4, sharex=True)
ax1.plot(x_data,I_data)
ax1.set_xlim(np.nanmin(x_data),np.nanmax(x_data))
ax1.set_ylim(np.nanmin(I_data[I_data.nonzero()]),
np.nanmax(I_data))
ax1.set_ylabel("Stokes I (K)")
ax2.plot(x_data,Q_data)
ax2.set_ylim(np.nanmin(Q_data),np.nanmax(Q_data))
ax2.set_ylabel("Stokes Q (K)")
ax3.plot(x_data,U_data)
ax3.set_ylim(np.nanmin(U_data),np.nanmax(U_data))
ax3.set_ylabel("Stokes U (K)")
ax4.plot(x_data,V_data)
ax4.set_ylim(np.nanmin(V_data),np.nanmax(V_data))
ax4.set_ylabel("Stokes V (K)")
ax4.set_xlabel(xlabel)
fig.subplots_adjust(hspace=0.1)
for ax in [ax1, ax2, ax3, ax4]:
# make the fontsize a bit smaller
for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] +
ax.get_xticklabels() + ax.get_yticklabels()):
item.set_fontsize(12)
plt.savefig(filename)
plt.close(fig)
def set_range(self, x_data, y_data):
min_x, max_x = np.nanmin(x_data), np.nanmax(x_data)
min_y, max_y = np.nanmin(y_data), np.nanmax(y_data)
self.plotview.setRange(
QRectF(min_x, min_y, max_x - min_x, max_y - min_y),
padding=0.025)
self.plotview.replot()
def _axes_domain(self, nx=None, ny=None, background_patch=None):
"""Returns x_range, y_range"""
DEBUG = False
transform = self._crs_transform()
ax_transform = self.axes.transAxes
desired_trans = ax_transform - transform
nx = nx or 30
ny = ny or 30
x = np.linspace(1e-9, 1 - 1e-9, nx)
y = np.linspace(1e-9, 1 - 1e-9, ny)
x, y = np.meshgrid(x, y)
coords = np.concatenate([x.flatten()[:, None], y.flatten()[:, None]], 1)
in_data = desired_trans.transform(coords)
ax_to_bkg_patch = self.axes.transAxes - background_patch.get_transform()
ok = np.zeros(in_data.shape[:-1], dtype=np.bool)
# XXX Vectorise contains_point
for i, val in enumerate(in_data):
# convert the coordinates of the data to the background
# patches coordinates
background_coord = ax_to_bkg_patch.transform(coords[i : i + 1, :])
bkg_patch_contains = background_patch.get_path().contains_point
if bkg_patch_contains(background_coord[0, :]):
color = "r"
ok[i] = True
else:
color = "b"
if DEBUG:
import matplotlib.pyplot as plt
plt.plot(coords[i, 0], coords[i, 1], "o" + color, clip_on=False, transform=ax_transform)
# plt.text(coords[i, 0], coords[i, 1], str(val), clip_on=False,
# transform=ax_transform, rotation=23,
# horizontalalignment='right')
inside = in_data[ok, :]
x_range = np.nanmin(inside[:, 0]), np.nanmax(inside[:, 0])
y_range = np.nanmin(inside[:, 1]), np.nanmax(inside[:, 1])
# XXX Cartopy specific thing. Perhaps make this bit a specialisation
# in a subclass...
crs = self.crs
if isinstance(crs, Projection):
x_range = np.clip(x_range, *crs.x_limits)
y_range = np.clip(y_range, *crs.y_limits)
# if the limit is >90 of the full x limit, then just use the full
# x limit (this makes circular handling better)
prct = np.abs(np.diff(x_range) / np.diff(crs.x_limits))
if prct > 0.9:
x_range = crs.x_limits
return x_range, y_range
def info(self):
"""
info()
Prints out a simple human-readable summary of the spectrum,
containing the name of the spectrum, the units on its axes,
and their limits. Also shows whether the spectrum has been
baselined or convolved yet.
Parameters
----------
None
Returns
-------
Nothing, but prints out a summary of the spectrum.
"""
print "---"
print "Summary for spectrum " + self.name
print "x unit: " + str(self.x.unit)
print "min(x): " + str(np.nanmin(self.x.value))
print "max(x): " + str(np.nanmax(self.x.value))
print "y unit: " + str(self.y.unit)
print "min(y): " + str(np.nanmin(self.y.value))
print "max(y): " + str(np.nanmax(self.y.value))
print "baselined: " + str(self.baselined)
print "convolved: " + str(self.convolved)
print "---"
def threshold(frame, threshold = 0.5, normalized_threshold = True, threshold_type = cv2.THRESH_BINARY, debug = False):
"""
tresholding an image: the input type has to be either np.uint8 or np.float32
returns a uint8 image
"""
fmin = np.double( np.nanmin(frame) )
fmax = np.double( np.nanmax(frame) )
if debug:
print "fmin: ", fmin
print "fmax: ", fmax
floatframe = ( (1.0 - 0.0) / (fmax - fmin) * (np.double(frame) - fmin) ).astype(np.float32)
if debug:
print "floatframe: ", floatframe.dtype
print "min: ", np.nanmin(floatframe)
print "max: ", np.nanmax(floatframe)
if not normalized_threshold:
threshold = 1.0 / (fmax - fmin) * (threshold - fmin)
if debug:
print "normalized threshold: ", threshold
retval, t = cv2.threshold(floatframe, thresh = threshold, maxval = 255, type = threshold_type)
return t
def bin_fit(x, y, buckets=3):
assert buckets in [3,25]
xstd=np.nanstd(x)
if buckets==3:
binlimits=[np.nanmin(x), -xstd/2.0,xstd/2.0 , np.nanmax(x)]
elif buckets==25:
steps=xstd/4.0
binlimits=np.arange(-xstd*3.0, xstd*3.0, steps)
binlimits=[np.nanmin(x)]+list(binlimits)+[np.nanmax(x)]
fit_y=[]
err_y=[]
x_values_to_plot=[]
for binidx in range(len(binlimits))[1:]:
lower_bin_x=binlimits[binidx-1]
upper_bin_x=binlimits[binidx]
x_values_to_plot.append(np.mean([lower_bin_x, upper_bin_x]))
y_in_bin=[y[idx] for idx in range(len(y)) if x[idx]>=lower_bin_x and x[idx]<upper_bin_x]
fit_y.append(np.nanmedian(y_in_bin))
err_y.append(np.nanstd(y_in_bin))
## no zeros
return (binlimits, x_values_to_plot, fit_y, err_y)
def TestPlot(fig=None):
A = numpy.array([1,2,3,4,2,5,8,3,2,3,5,6])
B = numpy.array([8,7,3,6,4,numpy.nan,9,3,7,numpy.nan,2,4])
C = numpy.array([6,3,4,7,2,1,1,7,8,4,3,2])
D = numpy.array([5,2,4,5,3,8,2,5,3,5,6,8])
# A work around to get the histograms overplotted with each other to overlap correctly;
histrangelist = [(numpy.nanmin(A),numpy.nanmax(A)),(numpy.nanmin(B),numpy.nanmax(B)),
(numpy.nanmin(C),numpy.nanmax(C)),(numpy.nanmin(D),numpy.nanmax(D))]
data = numpy.array([A,B,C,D])
labels = ['A','3','C','D']
fig = GridPlot(data,labels=labels, no_tick_labels=True, color='black',
hist=True, histbins=3, histloc='tl', histrangelist=histrangelist, fig=None)
# Data of note to plot in different color
A2 = numpy.array([1,2,3,4])
B2 = numpy.array([8,7,3,6])
C2 = numpy.array([6,3,4,7])
D2 = numpy.array([5,2,4,5])
data2 = numpy.array([A2,B2,C2,D2])
fig = GridPlot(data2,labels=labels, no_tick_labels=True, color='red',
hist=True, histbins=3, histloc='tr', histrangelist=histrangelist, fig=fig)
return fig
def plot_result(self, result):
"""
It plots the resulting Q and q when atype is set to 'tsl' or 'asl'
:param result:
Event Sync result from compute()
:type result: dict
:returns: plt.figure
-- figure plot
"""
' Raise error if parameters are not in the correct type '
if not(isinstance(result, dict)) : raise TypeError("Requires result to be a dictionary")
' Raise error if not the good dictionary '
if not 'Q' in result : raise ValueError("Requires dictionary to be the output of compute() method")
if not 'q' in result : raise ValueError("Requires dictionary to be the output of compute() method")
x=np.arange(0, result['Q'].size, 1)
figure, axarr = plt.subplots(2, sharex=True)
axarr[0].set_title('Synchrony and time delay pattern')
axarr[0].set_xlabel('Samples')
axarr[1].set_xlabel('Samples')
axarr[0].set_ylim(0,np.nanmax(result['Q']))
axarr[0].plot(x, result['Q'], label="Synchrony (Qn)")
axarr[1].set_ylim(np.nanmin(result['q']),np.nanmax(result['q']))
axarr[1].plot(x, result['q'], label="Time delay pattern (qn)")
axarr[0].legend(loc='best')
axarr[1].legend(loc='best')
return figure
def plot_nontarget_betas_n_back(t_vols_n_back_beta_1, b_vols_smooth_n_back, in_brain_mask, brain_structure, nice_cmap, n_back):
beta_index = 1
b_vols_smooth_n_back[~in_brain_mask] = np.nan
t_vols_n_back_beta_1[~in_brain_mask] = np.nan
min_val = np.nanmin(b_vols_smooth_n_back[...,(40,50,60),beta_index])
max_val = np.nanmax(b_vols_smooth_n_back[...,(40,50,60),beta_index])
plt.figure()
for map_index, depth in (((3,2,1), 40),((3,2,3), 50),((3,2,5), 60)):
plt.subplot(*map_index)
plt.title("z=%d,%s" % (depth, n_back + "-back nontarget,beta values"))
plt.imshow(brain_structure[...,depth], alpha=0.5)
plt.imshow(b_vols_smooth_n_back[...,depth,beta_index], cmap=nice_cmap, alpha=0.5, vmin=min_val, vmax=max_val)
plt.colorbar()
plt.tight_layout()
t_min_val = np.nanmin(t_vols_n_back_beta_1[...,(40,50,60)])
t_max_val = np.nanmax(t_vols_n_back_beta_1[...,(40,50,60)])
for map_index, depth in (((3,2,2), 40),((3,2,4), 50),((3,2,6), 60)):
plt.subplot(*map_index)
plt.title("z=%d,%s" % (depth, n_back + "-back nontarget,t values"))
plt.imshow(brain_structure[...,depth], alpha=0.5)
plt.imshow(t_vols_n_back_beta_1[...,depth], cmap=nice_cmap, alpha=0.5, vmin=t_min_val, vmax=t_max_val)
plt.colorbar()
plt.tight_layout()
plt.savefig(os.path.join(output_filename, "sub011_nontarget_betas_%s_back.png" % (n_back)), format='png', dpi=500)
def plot_all_time_series(config_list, output_dir):
"""Plot column charts of the raw total time/energy spent in each profiler category.
Keyword arguments:
config_list -- [(config, result of process_config_dir(...))]
output_dir -- where to write plots to
"""
time_series_out_dir = path.join(output_dir, 'time_series')
os.makedirs(time_series_out_dir)
max_end_times = []
max_power_values = []
for (c, cd) in config_list:
for (t, td) in cd:
trial_max_end_times = map(np.nanmax, filter(lambda x: len(x) > 0, [te for (p, ts, te, es, ee) in td]))
max_end_times.append(np.nanmax(trial_max_end_times))
for (p, ts, te, es, ee) in td:
# We only care about the energy profiler (others aren't reliable for instant power anyway)
if p == ENERGY_PROFILER_NAME and len(te) > 0:
max_power_values.append(np.nanmax(hb_energy_times_to_power(es, ee, ts, te)))
max_time = np.nanmax(max_end_times)
max_power = np.nanmax(np.array(max_power_values)) * 1.2 # leave a little space at the top
for (config, config_data) in config_list:
[plot_trial_time_series(config, trial, trial_data, max_time, max_power, time_series_out_dir)
for (trial, trial_data) in config_data]
def test_threshold_filter_nan(self):
src = self.make_src(nan=True)
self.e.add_source(src)
threshold = Threshold()
self.e.add_filter(threshold)
self.assertEqual(np.nanmin(src.scalar_data), np.nanmin(threshold.outputs[0].point_data.scalars.to_array()))
self.assertEqual(np.nanmax(src.scalar_data), np.nanmax(threshold.outputs[0].point_data.scalars.to_array()))
def plot_noise_regressor_betas(b_vols_smooth, t_vols_beta_6_to_9, brain_structure, in_brain_mask, nice_cmap):
plt.figure()
min_val = np.nanmin(b_vols_smooth[...,40,(6,7,9)])
max_val = np.nanmax(b_vols_smooth[...,40,(6,7,9)])
plt.subplot(3,2,1)
plt.title("z=%d,%s" % (40, "linear drift,betas"))
b_vols_smooth[~in_brain_mask] = np.nan
plt.imshow(brain_structure[...,40], alpha=0.5)
plt.imshow(b_vols_smooth[...,40,6], cmap=nice_cmap, alpha=0.5, vmin=min_val, vmax=max_val)
plt.colorbar()
plt.tight_layout()
plt.subplot(3,2,3)
plt.title("z=%d,%s" % (40, "quadratic drift,betas"))
b_vols_smooth[~in_brain_mask] = np.nan
plt.imshow(brain_structure[...,40], alpha=0.5)
plt.imshow(b_vols_smooth[...,40,7], cmap=nice_cmap, alpha=0.5, vmin=min_val, vmax=max_val)
plt.colorbar()
plt.tight_layout()
plt.subplot(3,2,5)
plt.title("z=%d,%s" % (40, "second PC,betas"))
b_vols_smooth[~in_brain_mask] = np.nan
plt.imshow(brain_structure[...,40], alpha=0.5)
plt.imshow(b_vols_smooth[...,40,9], cmap=nice_cmap, alpha=0.5, vmin=min_val, vmax=max_val)
plt.colorbar()
plt.tight_layout()
t_vols_beta_6_to_9[0][~in_brain_mask] = np.nan
t_vols_beta_6_to_9[1][~in_brain_mask] = np.nan
t_vols_beta_6_to_9[3][~in_brain_mask] = np.nan
t_min_val = np.nanmin([t_vols_beta_6_to_9[0][...,40], t_vols_beta_6_to_9[1][...,40], t_vols_beta_6_to_9[3][...,40]])
t_max_val = np.nanmax([t_vols_beta_6_to_9[0][...,40], t_vols_beta_6_to_9[1][...,40], t_vols_beta_6_to_9[3][...,40]])
plt.subplot(3,2,2)
plt.title("z=%d,%s" % (40, "linear drift,t values"))
plt.imshow(brain_structure[...,40], alpha=0.5)
plt.imshow(t_vols_beta_6_to_9[0][...,40], cmap=nice_cmap, alpha=0.5, vmin=t_min_val, vmax=t_max_val)
plt.colorbar()
plt.tight_layout()
plt.subplot(3,2,4)
plt.title("z=%d,%s" % (40, "quadratic drift,t values"))
plt.imshow(brain_structure[...,40], alpha=0.5)
plt.imshow(t_vols_beta_6_to_9[1][...,40], cmap=nice_cmap, alpha=0.5, vmin=t_min_val, vmax=t_max_val)
plt.colorbar()
plt.tight_layout()
plt.subplot(3,2,6)
plt.title("z=%d,%s" % (40, "second PC,t values"))
plt.imshow(brain_structure[...,40], alpha=0.5)
plt.imshow(t_vols_beta_6_to_9[3][...,40], cmap=nice_cmap, alpha=0.5, vmin=t_min_val, vmax=t_max_val)
plt.colorbar()
plt.tight_layout()
plt.savefig(os.path.join(output_filename, "sub001_noise_regressors_betas_map.png"), format='png', dpi=500)
def preprocess_crowdlabels(self, crowdlabels):
# Initialise all objects relating to the crowd labels.
C = {}
crowdlabels[np.isnan(crowdlabels)] = -1
if self.discretedecisions:
crowdlabels = np.round(crowdlabels).astype(int)
if self.table_format_flag:# crowd labels as a full KxN table? If false, use diags sparse 3-column list, where 1st
logging.error("Can't use table format with preference pairs at the moment.")
return
if self.K < int(np.nanmax(crowdlabels[:,0]))+1:
self.K = int(np.nanmax(crowdlabels[:,0]))+1 # add one because indexes start from 0
for l in range(self.nscores):
lIdxs = np.argwhere(crowdlabels[:, 3] == l)[:,0]
data = np.ones((len(lIdxs), 1)).reshape(-1)
rows = np.array(crowdlabels[lIdxs, 1]).reshape(-1) * self.N + crowdlabels[lIdxs, 2]
cols = np.array(crowdlabels[lIdxs, 0]).reshape(-1)
Cl = csr_matrix(coo_matrix((data,(rows,cols)), shape=(self.N**2, self.K)))
C[l] = Cl
# Set and reset object properties for the new dataset
self.C = C
self.lnpCT = np.zeros((self.N, self.nclasses))
self.conf_mat_ind = []
# pre-compute the indices into the pi arrays
# repeat for test labels only
self.Ctest = {}
for l in range(self.nscores):
self.Ctest[l] = C[l][self.testidxs, :]
# Reset the pre-calculated data for the training set in case goldlabels has changed
self.alpha_tr = []
txt_sim_list.append(txt_sim)
# calculate img similarity
if index in img_similarity:
img_sim = img_similarity[index]
else:
try:
img_sim = img_model.similarity(str(item), str(related_item))
except:
img_sim = np.nan
img_similarity[index] = img_sim
img_sim_list.append(img_sim)
txt_feature.append([eachrow[0], item,
np.nanmax(txt_sim_list),
np.nanmean(txt_sim_list),
np.nanstd(txt_sim_list),
np.nansum(txt_sim_list),
np.sum(np.isnan(txt_sim_list))])
img_feature.append([eachrow[0], item,
np.nanmax(img_sim_list),
np.nanmean(img_sim_list),
np.nanstd(img_sim_list),
np.nansum(img_sim_list),
np.sum(np.isnan(img_sim_list))])
gc.collect()
# In[ ]:
def normalize(self, indices):
if self.features.size:
nanmin = np.nanmin(self.features[:, indices], 0)
nanmax = np.nanmax(self.features[:, indices], 0)
self.features[:, indices] = (self.features[:, indices] -
nanmin) / (nanmax - nanmin)
def write_depth_seg_image(dpth_img, msk, fname, overlay=[0,0,255], transp=0.15): img = dpth_img * (255.0/np.nanmax(dpth_img)) img = np.stack((img,)*3, axis=-1) for i, hue in enumerate(overlay): img[msk, i] = hue*transp + img[msk, i] * (1 - transp) cv2.imwrite(fname, img)
rain[rain < 0.05] = 0 # Filtering the error values
meteo_secs = mdate.epoch2num(meteo_time)
meteo.close()
# %%
# =========================
# # Get maxima and minima:
# =========================
print(Time[TMT.nanargmax(Z_dbz)[0]])
print(time.strftime('%H:%M:%S', time.gmtime((Time[TMT.nanargmax(Z_dbz)[0]])))[0:5])
MaxEcho = 'Highest Echo = ' + str(np.nanmax(Z_dbz))[0:5] + ' dBZ (' + str(
time.strftime('%H:%M:%S', time.gmtime((Time[TMT.nanargmax(Z_dbz)[0]])))[0:5]) + ' UTC)'
MinTemp = 'Min Temp = ' + str(np.nanmin(temp))[0:6] + ' °C (' + str(
time.strftime('%H:%M:%S', time.gmtime((meteo_time[TMT.nanargmin(temp)[0]])))[0:5]) + ' UTC)'
MaxTemp = 'Max Temp = ' + str(np.nanmax(temp))[0:6] + ' °C (' + str(
time.strftime('%H:%M:%S', time.gmtime((meteo_time[TMT.nanargmax(temp)[0]])))[0:5]) + ' UTC)'
DeltaTemp = '$\Delta$ Temp = ' + str(abs(abs(np.nanmax(temp)) - abs(np.nanmin(temp))))[0:6] + ' °C'
temp_str = MinTemp + '\n' + MaxTemp + '\n' + DeltaTemp
MaxRain = 'Max Precip = ' + str(np.nanmax(rain))[0:6] + ' mm/h (' + str(
time.strftime('%H:%M:%S', time.gmtime((meteo_time[TMT.nanargmax(rain)[0]])))[0:5]) + ' UTC)'
MinWind = 'Min Wind = ' + str(np.nanmin(wind_speed))[0:6] + ' m/s (' + str(
time.strftime('%H:%M:%S', time.gmtime((meteo_time[TMT.nanargmin(wind_speed)[0]])))[0:5]) + ' UTC)'
MaxWind = 'Max Wind = ' + str(np.nanmax(wind_speed))[0:6] + ' m/s (' + str(
nv = len(v); nvm = nv/nbins * nbins; v_mean = v[:nvm]; v_mean = v_mean.reshape([nvm/nbins, nbins]); v_mean = np.nanmean(v_mean, axis = 1); v_mean = np.hstack([v_mean, [np.nanmean(v[nvm:])]]); ts = int(exp_times[wid] / time_bin); v_24hr[wid, ts:ts+len(r_mean)] = v_mean.copy(); plt.figure(12); plt.clf(); #plt.plot(v_mean) #plt.ylim(0,20) np.nanmax(roam_24hr) rrange = np.nanpercentile(roam_24hr, [5, 95]); plt.imshow(roam_24hr, aspect = 'auto', vmax = rrange[1], interpolation = 'none') plt.tight_layout() # hr lines nhr = 60 * 60 / time_bin; nday = 24 * nhr; for d in range(ntimes/nhr): plt.plot([d*nhr, d*nhr], [-0.5, len(roam_24hr)-0.5], 'k') for d in range(ntimes/nday): plt.plot([d*nday, d*nday], [-0.5, len(roam_24hr)-0.5], 'r', linewidth = 2) plt.xlim(0, roam_24hr.shape[1]);
def process(self, sort_by_time=False,
aggregate: str or None = None,
progress: TextIO or None = sys.stderr,
info: TextIO or None = None):
"""Processes all added files/URLs and yields the results"""
if aggregate:
aggregates = ('min', 'max', 'median', 'mean')
if aggregate not in aggregates:
raise ValueError(f'"aggregate" not in {str(aggregates)}')
count, total = 0, sum(_[-1] for _ in self._data)
data = {}
messages = None if info is None else []
# load model now: it takes ~=1 second and is usually lazy loaded,
# but this means that the progressbar would show misleading results
# at the beginning
load_default_trained_model()
with ProgressBar(progress) as pbar:
# self._data has generally only one element in the current
# implementation (see module function `process`), however, it
# already supports multiple call of its `add_*` methods above
for key, streamiterator, metadata_path, length in self._data:
try:
inv = read_inventory(metadata_path)
except Exception as exc:
if messages is not None:
messages.append(f'Metadata error, {str(exc)}. {key}')
streamiterator = [] # hack to skip iteration below
feats = []
ids = []
kount = 0
try:
for fpath, stream in streamiterator:
kount += 1
for trace in stream:
(id_, st_, et_), feat = trace_idfeatures(trace, inv)
feats.append(feat)
ids.append((fpath, id_, st_, et_))
except Exception as exc:
if messages is not None:
messages.append(f'{str(exc)}. {key}')
feats = [] # hack to stop after updating the pbar
count += 1
pbar.update(count / total)
if kount < length:
count += length - kount
pbar.update(count / total)
if not feats:
continue
scores = aa_scores(np.asarray(feats))
iter_ = zip(ids, scores)
if aggregate:
data = defaultdict(lambda: [None, []])
for (fpath, id_, stime, etime), score in iter_:
timeranges, scores_ = data[id_]
if not timeranges:
data[id_][0] = [stime, etime]
else:
timeranges[0] = min(stime, timeranges[0])
timeranges[1] = max(etime, timeranges[1])
scores_.append(score)
ids = []
scores = []
for id_, (timeranges, scores_) in data.items():
ids.append(('', id_, *timeranges))
scores_ = np.asarray(scores_)
if np.isnan(scores_).all():
scores.append(np.nan)
elif aggregate == 'mean':
scores.append(np.nanmean(scores_))
elif aggregate == 'min':
scores.append(np.nanmin(scores_))
elif aggregate == 'max':
scores.append(np.nanmax(scores_))
else:
scores.append(np.nanmedian(scores_))
# now sort (if needed) and print them at once:
iter_ = zip(ids, scores)
if sort_by_time:
yield sorted(iter_, key=lambda _: _[0][1])
yield iter_
for msg in (messages or []):
print(msg, file=info)
def train_model(train_dir=None, validation_dir=None, debug=True):
config = GEDIconfig()
if train_dir is None: # Use globals
train_data = os.path.join(config.tfrecord_dir,
config.tf_record_names['train'])
meta_data = np.load(
os.path.join(config.tfrecord_dir,
'%s_%s' % (config.tvt_flags[0], config.max_file)))
else:
meta_data = np.load(
os.path.join(train_dir,
'%s_%s' % (config.tvt_flags[0], config.max_file)))
# Prepare image normalization values
if config.max_gedi is None:
max_value = np.nanmax(meta_data['max_array']).astype(np.float32)
if max_value == 0:
max_value = None
print 'Derived max value is 0'
else:
print 'Normalizing with empirical max.'
if 'min_array' in meta_data.keys():
# min_value = np.min(meta_data['min_array']).astype(np.float32)
print 'Normalizing with empirical min.'
else:
# min_value = None
print 'Not normalizing with a min.'
else:
max_value = config.max_gedi
# min_value = config.min_gedi
ratio = meta_data['ratio']
print 'Ratio is: %s' % ratio
if validation_dir is None: # Use globals
validation_data = os.path.join(config.tfrecord_dir,
config.tf_record_names['val'])
elif validation_dir is False:
pass # Do not use validation data during training
# Make output directories if they do not exist
dt_stamp = re.split(
'\.', str(datetime.now()))[0].\
replace(' ', '_').replace(':', '_').replace('-', '_')
dt_dataset = config.which_dataset + '_' + dt_stamp + '/'
config.train_checkpoint = os.path.join(config.train_checkpoint,
dt_dataset) # timestamp this run
out_dir = os.path.join(config.results, dt_dataset)
dir_list = [
config.train_checkpoint, config.train_summaries, config.results,
out_dir
]
[make_dir(d) for d in dir_list]
# im_shape = get_image_size(config)
im_shape = config.gedi_image_size
print '-' * 60
print('Training model:' + dt_dataset)
print '-' * 60
# Prepare data on CPU
assert os.path.exists(train_data)
assert os.path.exists(validation_data)
assert os.path.exists(config.vgg16_weight_path)
with tf.device('/cpu:0'):
train_images, train_labels, train_times = inputs(
train_data,
config.train_batch,
im_shape,
config.model_image_size,
# max_value=max_value,
# min_value=min_value,
train=config.data_augmentations,
num_epochs=config.epochs,
normalize=config.normalize,
return_filename=True)
val_images, val_labels, val_times = inputs(
validation_data,
config.validation_batch,
im_shape,
config.model_image_size,
# max_value=max_value,
# min_value=min_value,
num_epochs=config.epochs,
normalize=config.normalize,
return_filename=True)
train_image_list, val_image_list = [], []
for idx in range(int(train_images.get_shape()[1])):
train_image_list += [tf.gather(train_images, idx, axis=1)]
val_image_list += [tf.gather(val_images, idx, axis=1)]
tf.summary.image('train_image_frame_%s' % idx,
train_image_list[idx])
tf.summary.image('validation_image_frame_%s' % idx,
val_image_list[idx])
# Prepare model on GPU
with tf.device('/gpu:0'):
with tf.variable_scope('match'):
# Build matching model for frame 0
model_0 = matching_gedi.model_struct()
frame_activity = []
frame_activity += [
tf.nn.l2_normalize(model_0.build(train_image_list[0]), 0,
1e-12)
]
with tf.variable_scope('match', reuse=tf.AUTO_REUSE):
# Build matching model for other frames
for idx in range(1, len(train_image_list)):
frame_activity += [
tf.nn.l2_normalize(model_0.build(train_image_list[idx]), 0,
1e-12)
]
pos = l2_dist(frame_activity[0], frame_activity[1], axis=1)
neg = l2_dist(frame_activity[0], frame_activity[2], axis=1)
# loss = tf.reduce_mean(tf.pow(pos - neg + 0.2, 2))
loss = tf.reduce_mean(tf.maximum(pos - neg + 0.01, 0.))
# loss = tf.reduce_mean(tf.nn.relu(1 - (neg / (pos + 0.2)))) * 100
tf.summary.scalar('Triplet_loss', loss)
# Weight decay
if config.wd_layers is not None:
_, l2_wd_layers = fine_tune_prepare_layers(
tf.trainable_variables(), config.wd_layers)
l2_wd_layers = [x for x in l2_wd_layers if 'biases' not in x.name]
if len(l2_wd_layers) > 0:
loss += (config.wd_penalty *
tf.add_n([tf.nn.l2_loss(x) for x in l2_wd_layers]))
# Optimize
train_op = tf.train.AdamOptimizer(config.new_lr).minimize(loss)
train_accuracy = tf.reduce_mean(
tf.cast(
tf.equal(
tf.nn.relu(tf.sign(neg - pos)), # 1 if pos < neg
tf.cast(tf.ones_like(train_labels), tf.float32)),
tf.float32))
tf.summary.scalar('training_accuracy', train_accuracy)
# Setup validation op
if validation_data is not False:
with tf.variable_scope('match', tf.AUTO_REUSE) as match:
# Build matching model for frame 0
match.reuse_variables()
val_model_0 = matching_gedi.model_struct()
val_frame_activity = []
val_frame_activity += [
tf.nn.l2_normalize(val_model_0.build(val_image_list[0]), 1,
1e-12)
]
# Build matching model for other frames
for idx in range(1, len(train_image_list)):
val_frame_activity += [
tf.nn.l2_normalize(
val_model_0.build(val_image_list[idx]), 1, 1e-12)
]
val_pos = l2_dist(val_frame_activity[0],
val_frame_activity[1],
axis=1)
val_neg = l2_dist(val_frame_activity[0],
val_frame_activity[2],
axis=1)
# val_loss = tf.reduce_mean(tf.pow(val_pos - val_neg + 0.2, 2))
val_loss = tf.reduce_mean(tf.maximum(val_pos - val_neg + 0.01, 0.))
# val_loss = tf.reduce_mean(tf.nn.relu(1 - (val_neg / (val_pos + 0.2)))) * 100
tf.summary.scalar('Validation_triplet_loss', val_loss)
# Calculate validation accuracy
val_accuracy = tf.reduce_mean(
tf.cast(
tf.equal(
tf.nn.relu(tf.sign(val_neg - val_pos)), # 1 if pos < neg
tf.cast(tf.ones_like(val_labels), tf.float32)),
tf.float32))
tf.summary.scalar('val_accuracy', val_accuracy)
# Set up summaries and saver
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=config.keep_checkpoints)
summary_op = tf.summary.merge_all()
# Initialize the graph
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
# Need to initialize both of these if supplying num_epochs to inputs
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
summary_dir = os.path.join(config.train_summaries,
config.which_dataset + '_' + dt_stamp)
summary_writer = tf.summary.FileWriter(summary_dir, sess.graph)
# Set up exemplar threading
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Train operations
train_dict = {
'train_op': train_op,
'loss': loss,
'pos': pos,
'neg': neg,
'train_accuracy': train_accuracy,
'val_accuracy': val_accuracy,
}
val_dict = {'val_accuracy': val_accuracy}
if debug:
for idx in range(len(train_image_list)):
train_dict['train_im_%s' % idx] = train_image_list[idx]
for idx in range(len(val_image_list)):
val_dict['val_im_%s' % idx] = val_image_list[idx]
# Start training loop
np.save(out_dir + 'meta_info', config)
step, losses = 0, [] # val_max = 0
try:
# print response
while not coord.should_stop():
start_time = time.time()
train_values = sess.run(train_dict.values())
it_train_dict = {
k: v
for k, v in zip(train_dict.keys(), train_values)
}
losses += [it_train_dict['loss']]
duration = time.time() - start_time
if np.isnan(it_train_dict['loss']).sum():
assert not np.isnan(it_train_dict['loss']),\
'Model loss = NaN'
if step % config.validation_steps == 0:
if validation_data is not False:
val_values = sess.run(val_dict.values())
it_val_dict = {
k: v
for k, v in zip(val_dict.keys(), val_values)
}
val_acc = it_val_dict['val_accuracy']
else:
val_acc -= 1 # Store every checkpoint
# Summaries
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, step)
# Training status and validation accuracy
format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; '
'%.3f sec/batch) | Training accuracy = %s | '
'Validation accuracy = %s | '
'logdir = %s')
print(format_str %
(datetime.now(), step,
it_train_dict['loss'], config.train_batch / duration,
float(duration), it_train_dict['train_accuracy'],
it_train_dict['val_accuracy'], summary_dir))
# Save the model checkpoint if it's the best yet
if 1: # val_acc >= val_max:
saver.save(sess,
os.path.join(config.train_checkpoint,
'model_' + str(step) + '.ckpt'),
global_step=step)
# Store the new max validation accuracy
# val_max = val_acc
else:
# Training status
format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; '
'%.3f sec/batch) | Training accuracy = %s | ')
print(format_str %
(datetime.now(), step,
it_train_dict['loss'], config.train_batch / duration,
float(duration), it_train_dict['train_accuracy']))
# End iteration
step += 1
except tf.errors.OutOfRangeError:
print('Done training for %d epochs, %d steps.' % (config.epochs, step))
finally:
coord.request_stop()
np.save(os.path.join(config.tfrecord_dir, 'training_loss'), losses)
coord.join(threads)
sess.close()
# Keeping track of performance on the simple validation task by getting a precision recall curve.
model_name = "bert_small"
vectorize = lambda x: vectorize_with_bert(" ".join(preprocess_string(x)),
model, tokenizer, "sum", 1)
get_similarity = lambda s1, s2: 1 - cosine(vectorize(s1), vectorize(s2))
df[model_name] = df.apply(
lambda x: get_similarity(x["concept_1"], x["concept_2"]), axis=1)
y_true = list(df["class"].values)
y_prob = list(df[model_name].values)
precision, recall, thresholds = precision_recall_curve(y_true, y_prob)
f_beta = lambda pr, re, beta: [((1 + beta**2) * p * r) /
((((beta**2) * p) + r))
for p, r in zip(pr, re)]
f_1_scores = f_beta(precision, recall, beta=1)
f_1_max = np.nanmax(f_1_scores)
rows.append((model_name, epoch_i, loss.item(), f_1_max))
# Writing results of the validation to those files.
df.to_csv(output_path_for_results, index=False)
header = ["model", "epoch", "training_loss", "f1_max"]
pd.DataFrame(rows, columns=header).to_csv(output_path_for_results_summary,
index=False)
# In[44]:
output_dir = "../models/bert_small/model_save_{}/".format(
datetime.datetime.now().strftime('%m_%d_%Y_h%Hm%Ms%S'))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
model_to_save = model.module if hasattr(model, 'module') else model
def test_IEEE_case_9_with_bad_data():
# 1. Create network
# test grid: IEEE case 9 (HV)
# overall: 9 buses, 8 lines
net = pp.networks.case9()
pp.create_measurement(net, "v", "bus", 1.0, 0.01, bus=1) # V at bus 1
pp.create_measurement(net, "v", "bus", 1.02, 0.01, bus=2) # V at bus 2
pp.create_measurement(net, "v", "bus", 0.9822, 0.01, bus=3) # V at bus 3
pp.create_measurement(net, "v", "bus", 0.979, 0.01, bus=4) # V at bus 4
pp.create_measurement(net, "v", "bus", 1.005, 0.01, bus=5) # V at bus 5
pp.create_measurement(net, "v", "bus", 0.997, 0.01, bus=7) # V at bus 7
pp.create_measurement(net, "v", "bus", 0.953, 0.01, bus=8) # V at bus 8
pp.create_measurement(net, "p", "bus", 72000, 100., bus=0)
pp.create_measurement(net, "p", "bus", 162780, 100., bus=1)
pp.create_measurement(net, "p", "bus", 84958, 70., bus=2)
pp.create_measurement(net, "p", "bus", 0., 1., bus=3)
pp.create_measurement(net, "p", "bus", -89967., 20., bus=4)
pp.create_measurement(net, "p", "bus", 0., 10., bus=5)
pp.create_measurement(net, "p", "bus", -100059., 30., bus=6)
pp.create_measurement(net, "p", "bus", 0., 10., bus=7)
pp.create_measurement(net, "p", "bus", -125100, 50., bus=8)
pp.create_measurement(net, "q", "bus", 24000, 100., bus=0)
pp.create_measurement(net, "q", "bus", 14500, 100., bus=1)
pp.create_measurement(net, "q", "bus", 3644, 70., bus=2)
pp.create_measurement(net, "q", "bus", 0., 1., bus=3)
pp.create_measurement(net, "q", "bus", -30041., 20., bus=4)
pp.create_measurement(net, "q", "bus", 0., 10., bus=5)
pp.create_measurement(net, "q", "bus", -35087, 30., bus=6)
pp.create_measurement(net, "q", "bus", 0., 10., bus=7)
pp.create_measurement(net, "q", "bus", -49900, 50., bus=8)
# 2. Do state estimation
success_SE = estimate(net, init='flat', ref_power=1e6)
v_est_SE = net.res_bus_est.vm_pu.values
delta_SE = net.res_bus_est.va_degree.values
# 3. Create false measurement (very close to useful values)
pp.create_measurement(net, "v", "bus", 0.2, 0.01, bus=0) # V at bus 0
# 4. Do chi2-test
bad_data_detected = chi2_analysis(net, init='flat', ref_power=1e6)
# 5. Perform rn_max_test
success_rn_max = remove_bad_data(net,
init='flat',
ref_power=1e6,
rn_max_threshold=7.0)
v_est_rn_max = net.res_bus_est.vm_pu.values
delta_est_rn_max = net.res_bus_est.va_degree.values
diff_v = v_est_SE - v_est_rn_max
diff_delta = delta_SE - delta_est_rn_max
assert success_SE
assert bad_data_detected
assert success_rn_max
assert (np.nanmax(abs(diff_v)) < 1e-5)
assert (np.nanmax(abs(diff_delta)) < 1e-5)
def q_learning(self):
ns = len(self.state2ind.keys())
na = len(self.actions.keys())
discount = self.gamma
lr = self.alpha
# initialization
self.allowed_movements()
# ADD YOUR CODE SNIPPET BETWEEN EX. 2.1
# Initialize a numpy array with ns state rows and na state columns with float values from 0.0 to 1.0.
Q = np.random.rand(ns, na)
# ADD YOUR CODE SNIPPET BETWEEN EX. 2.1
for s in range(ns):
list_pos = self.allowed_moves[s]
for i in range(4):
if i not in list_pos:
Q[s, i] = np.nan
Q_old = Q.copy()
diff = np.infty
end_episode = False
init_pos_tuple = self.settings.init_pos_diver
init_pos = self.ind2state[(init_pos_tuple[0], init_pos_tuple[1])]
episode = 0
R_total = 0
current_total_steps = 0
steps = 0
# ADD YOUR CODE SNIPPET BETWEEN EX. 2.3
# Change the while loop to incorporate a threshold limit, to stop training when the mean difference
# in the Q table is lower than the threshold
#while episode <= self.episode_max or diff > self.threshold:
while diff > self.threshold:
if episode > self.episode_max:
break
# ADD YOUR CODE SNIPPET BETWEENEX. 2.3
s_current = init_pos
R_total = 0
steps = 0
while not end_episode:
# selection of action
list_pos = self.allowed_moves[s_current]
# ADD YOUR CODE SNIPPET BETWEEN EX 2.1 and 2.2
# Chose an action from all possible actions
# 2.2 below, choosing an action based on highest q-value at the given state:
action = np.nanargmax(Q[s_current])
#print("chosen action", action, "\n")
# ADD YOUR CODE SNIPPET BETWEEN EX 2.1 and 2.2
# ADD YOUR CODE SNIPPET BETWEEN EX 5
# Use the epsilon greedy algorithm to retrieve an action
# ADD YOUR CODE SNIPPET BETWEEN EX 5
# compute reward
action_str = self.action_list[action]
msg = {"action": action_str, "exploration": True}
self.sender(msg)
# wait response from game
msg = self.receiver()
R = msg["reward"]
R_total += R
s_next_tuple = msg["state"]
end_episode = msg["end_episode"]
s_next = self.ind2state[s_next_tuple]
# ADD YOUR CODE SNIPPET BETWEEN EX. 2.2
# Implement the Bellman Update equation to update Q
Q[s_current, action] = (
1 - self.alpha) * Q[s_current, action] + self.alpha * (
R + self.gamma * np.nanmax(Q[s_next]))
# ADD YOUR CODE SNIPPET BETWEEN EX. 2.2
s_current = s_next
current_total_steps += 1
steps += 1
# ADD YOUR CODE SNIPPET BETWEEN EX. 2.3
# Compute the absolute value of the mean between the Q and Q-old
diff = abs(np.nanmean(Q) - np.nanmean(Q_old))
# ADD YOUR CODE SNIPPET BETWEEN EX. 2.3
Q_old[:] = Q
print(
"Episode: {}, Steps {}, Diff: {:6e}, Total Reward: {}, Total Steps {}"
.format(episode, steps, diff, R_total, current_total_steps))
episode += 1
end_episode = False
return Q
def test_3bus_with_2_slacks():
# load the net which already contains 3 buses
net = load_3bus_network()
# add the same net with different slack (no galvanic connection)
# skip bus index 4 as further stability test
pp.create_bus(net, name="bus5", vn_kv=1., index=5)
pp.create_bus(net, name="bus6", vn_kv=1., index=6)
pp.create_bus(net, name="bus7", vn_kv=1., index=7)
pp.create_ext_grid(net, 5)
pp.create_line_from_parameters(net,
5,
6,
1,
r_ohm_per_km=.01,
x_ohm_per_km=.03,
c_nf_per_km=0.,
max_i_ka=1)
pp.create_line_from_parameters(net,
5,
7,
1,
r_ohm_per_km=.02,
x_ohm_per_km=.05,
c_nf_per_km=0.,
max_i_ka=1)
pp.create_line_from_parameters(net,
6,
7,
1,
r_ohm_per_km=.03,
x_ohm_per_km=.08,
c_nf_per_km=0.,
max_i_ka=1)
pp.create_measurement(net, "v", "bus", 1.006, .004, bus=5) # V at bus 5
pp.create_measurement(net, "v", "bus", .968, .004, bus=6) # V at bus 6
pp.create_measurement(net, "p", "bus", -501, 10, 6) # P at bus 6
pp.create_measurement(net, "q", "bus", -286, 10, 6) # Q at bus 6
pp.create_measurement(net, "p", "line", 888, 8, 5,
3) # Pline (bus 1 -> bus 2) at bus 5
pp.create_measurement(net, "p", "line", 1173, 8, 5,
4) # Pline (bus 1 -> bus 3) at bus 5
pp.create_measurement(net, "q", "line", 568, 8, 5,
3) # Qline (bus 1 -> bus 2) at bus 5
pp.create_measurement(net, "q", "line", 663, 8, 5,
4) # Qline (bus 1 -> bus 3) at bus 5
# 2. Do state estimation
success = estimate(net, init='flat', maximum_iterations=10)
v_result = net.res_bus_est.vm_pu.values
delta_result = net.res_bus_est.va_degree.values
target_v = np.array(
[0.9996, 0.9741, 0.9438, np.nan, 0.9996, 0.9741, 0.9438])
diff_v = target_v - v_result
target_delta = np.array([
0.0, -1.2475469989322963, -2.7457167371166862, np.nan, 0.0,
-1.2475469989322963, -2.7457167371166862
])
diff_delta = target_delta - delta_result
assert success
assert (np.nanmax(abs(diff_v)) < 1e-4)
assert (np.nanmax(abs(diff_delta)) < 1e-4)
def test_init_slack_with_multiple_transformers(angles=True):
np.random.seed(123)
net = pp.create_empty_network()
pp.create_bus(net, 220, index=0)
pp.create_bus(net, 110, index=1)
pp.create_bus(net, 110, index=2)
pp.create_bus(net, 110, index=3)
pp.create_bus(net, 10, index=4)
pp.create_bus(net, 10, index=5)
pp.create_bus(net, 10, index=6)
pp.create_bus(net, 10, index=7, in_service=False)
pp.create_transformer(net,
3,
7,
std_type="63 MVA 110/10 kV",
in_service=False)
pp.create_transformer(net, 3, 4, std_type="63 MVA 110/10 kV")
pp.create_transformer(net, 0, 1, std_type="100 MVA 220/110 kV")
pp.create_line(net, 1, 2, 2.0, std_type="N2XS(FL)2Y 1x120 RM/35 64/110 kV")
pp.create_line(net, 1, 3, 2.0, std_type="N2XS(FL)2Y 1x120 RM/35 64/110 kV")
pp.create_line(net, 4, 5, 2.0, std_type="NA2XS2Y 1x95 RM/25 12/20 kV")
pp.create_line(net, 5, 6, 2.0, std_type="NA2XS2Y 1x95 RM/25 12/20 kV")
pp.create_load(net, 2, 5000, 3300)
pp.create_load(net, 5, 900, 500)
pp.create_load(net, 6, 700, 300)
pp.create_ext_grid(net,
bus=0,
vm_pu=1.04,
va_degree=10.,
name="Slack 220 kV")
pp.runpp(net, calculate_voltage_angles=angles)
for bus, row in net.res_bus[net.bus.in_service == True].iterrows():
pp.create_measurement(net, "v", "bus", row.vm_pu * r(0.01), 0.01, bus)
if row.p_kw != 0.:
continue
pp.create_measurement(net, "p", "bus", -row.p_kw * r(),
max(1.0, abs(0.03 * row.p_kw)), bus)
pp.create_measurement(net, "q", "bus", -row.q_kvar * r(),
max(1.0, abs(0.03 * row.q_kvar)), bus)
pp.create_measurement(net,
"p",
"line",
net.res_line.p_from_kw[0],
10.,
bus=1,
element=0)
pp.create_measurement(net,
"q",
"line",
net.res_line.q_from_kvar[0],
10.,
bus=1,
element=0)
pp.create_measurement(net,
"p",
"line",
net.res_line.p_from_kw[2],
10.,
bus=4,
element=2)
pp.create_measurement(net,
"q",
"line",
net.res_line.q_from_kvar[2],
10.,
bus=4,
element=2)
pp.create_measurement(net,
"p",
"line",
net.res_line.p_from_kw[3],
10.,
bus=5,
element=3)
pp.create_measurement(net,
"q",
"line",
net.res_line.q_from_kvar[3],
10.,
bus=5,
element=3)
success = estimate(net, init='slack', calculate_voltage_angles=angles)
# pretty high error for vm_pu (half percent!)
assert success
assert (np.nanmax(
np.abs(net.res_bus.vm_pu.values - net.res_bus_est.vm_pu.values)) <
0.006)
assert (np.nanmax(
np.abs(net.res_bus.va_degree.values -
net.res_bus_est.va_degree.values)) < 0.006)
def test_3bus_with_out_of_service_bus():
# Test case from book "Power System State Estimation", A. Abur, A. G. Exposito, p. 20ff.
# S_ref = 1 MVA (PP standard)
# V_ref = 1 kV
# Z_ref = 1 Ohm
# The example only had per unit values, but pandapower expects kV, MVA, kW, kVar
# Measurements should be in kW/kVar/A - Voltage in p.u.
# 1. Create network
net = pp.create_empty_network()
pp.create_bus(net, name="bus1", vn_kv=1.)
pp.create_bus(net, name="bus2", vn_kv=1.)
pp.create_bus(net, name="bus3", vn_kv=1.)
pp.create_bus(net, name="bus4", vn_kv=1.,
in_service=0) # out-of-service bus test
pp.create_ext_grid(net, 0)
pp.create_line_from_parameters(net,
0,
1,
1,
r_ohm_per_km=.01,
x_ohm_per_km=.03,
c_nf_per_km=0.,
max_i_ka=1)
pp.create_line_from_parameters(net,
0,
2,
1,
r_ohm_per_km=.02,
x_ohm_per_km=.05,
c_nf_per_km=0.,
max_i_ka=1)
pp.create_line_from_parameters(net,
1,
2,
1,
r_ohm_per_km=.03,
x_ohm_per_km=.08,
c_nf_per_km=0.,
max_i_ka=1)
pp.create_measurement(net, "v", "bus", 1.006, .004, bus=0) # V at bus 1
pp.create_measurement(net, "v", "bus", .968, .004, bus=1) # V at bus 2
pp.create_measurement(net, "p", "bus", -501, 10, 1) # P at bus 2
pp.create_measurement(net, "q", "bus", -286, 10, 1) # Q at bus 2
pp.create_measurement(net, "p", "line", 888, 8, 0,
0) # Pline (bus 1 -> bus 2) at bus 1
pp.create_measurement(net, "p", "line", 1173, 8, 0,
1) # Pline (bus 1 -> bus 3) at bus 1
pp.create_measurement(net, "q", "line", 568, 8, 0,
0) # Qline (bus 1 -> bus 2) at bus 1
pp.create_measurement(net, "q", "line", 663, 8, 0,
1) # Qline (bus 1 -> bus 3) at bus 1
# 2. Do state estimation
success = estimate(net, init='flat')
v_result = net.res_bus_est.vm_pu.values
delta_result = net.res_bus_est.va_degree.values
target_v = np.array([[0.9996, 0.9741, 0.9438, np.nan]])
diff_v = target_v - v_result
target_delta = np.array([[0., -1.2475, -2.7457, np.nan]])
diff_delta = target_delta - delta_result
assert success
assert (np.nanmax(abs(diff_v)) < 1e-4)
assert (np.nanmax(abs(diff_delta)) < 1e-4)
def test_3bus():
# 1. Create network
net = pp.create_empty_network()
pp.create_bus(net, name="bus1", vn_kv=1.)
pp.create_bus(net, name="bus2", vn_kv=1.)
pp.create_bus(net, name="bus3", vn_kv=1.)
pp.create_ext_grid(net, 0)
pp.create_line_from_parameters(net,
0,
1,
1,
r_ohm_per_km=0.7,
x_ohm_per_km=0.2,
c_nf_per_km=0,
max_i_ka=1)
pp.create_line_from_parameters(net,
0,
2,
1,
r_ohm_per_km=0.8,
x_ohm_per_km=0.8,
c_nf_per_km=0,
max_i_ka=1)
pp.create_line_from_parameters(net,
1,
2,
1,
r_ohm_per_km=1,
x_ohm_per_km=0.6,
c_nf_per_km=0,
max_i_ka=1)
pp.create_measurement(net,
"p",
"line",
-0.0011e3,
0.01e3,
bus=0,
element=0) # p12
pp.create_measurement(net, "q", "line", 0.024e3, 0.01e3, bus=0,
element=0) # q12
pp.create_measurement(net, "p", "bus", 0.018e3, 0.01e3, bus=2) # p3
pp.create_measurement(net, "q", "bus", -0.1e3, 0.01e3, bus=2) # q3
pp.create_measurement(net, "v", "bus", 1.08, 0.05, 0) # u1
pp.create_measurement(net, "v", "bus", 1.015, 0.05, 2) # u3
# 2. Do state estimation
success = estimate(net, init='flat')
v_result = net.res_bus_est.vm_pu.values
delta_result = net.res_bus_est.va_degree.values
target_v = np.array([1.0627, 1.0589, 1.0317])
diff_v = target_v - v_result
target_delta = np.array([0., 0.8677, 3.1381])
diff_delta = target_delta - delta_result
assert success
assert (np.nanmax(abs(diff_v)) < 1e-4)
assert (np.nanmax(abs(diff_delta)) < 1e-4)
def plot_good_bad_stns(pws_in_coords_df, ids_pws_stns_gd, ids_pws_stns_bad,
zvalues, prim_netw_ppt, zvalues_bad, prim_netwx,
prim_netwy, event_date):
xstns_good = pws_in_coords_df.loc[ids_pws_stns_gd, 'X'].values.ravel()
ystns_good = pws_in_coords_df.loc[ids_pws_stns_gd, 'Y'].values.ravel()
xstns_bad = pws_in_coords_df.loc[ids_pws_stns_bad, 'X'].values.ravel()
ystns_bad = pws_in_coords_df.loc[ids_pws_stns_bad, 'Y'].values.ravel()
max_ppt = max(np.nanmax(zvalues), np.nanmax(prim_netw_ppt))
interval_ppt = np.linspace(0.0, 0.99)
colors_ppt = plt.get_cmap('Blues')(interval_ppt)
cmap_ppt = LinearSegmentedColormap.from_list('name', colors_ppt)
# cmap_ppt = plt.get_cmap('jet_r')
cmap_ppt.set_over('navy')
interval_ppt_bad = np.linspace(0.02, 0.95)
colors_ppt_bad = plt.get_cmap('autumn')(interval_ppt_bad)
cmap_ppt_bad = LinearSegmentedColormap.from_list(
'name', colors_ppt_bad)
plt.ioff()
plt.figure(figsize=(12, 8), dpi=100)
plt.scatter(prim_netwx,
prim_netwy,
c=prim_netw_ppt,
cmap=cmap_ppt,
marker=',',
s=10,
alpha=0.75,
vmin=0,
vmax=max_ppt,
label='prim_netw %d' % prim_netwx.size)
sc = plt.scatter(xstns_good,
ystns_good,
c=zvalues,
cmap=cmap_ppt,
marker='.',
s=10,
alpha=0.75,
vmin=0,
vmax=max_ppt,
label='PWS Good %d' % xstns_good.size)
plt.scatter(xstns_bad,
ystns_bad,
alpha=0.75,
c=zvalues_bad,
cmap=cmap_ppt_bad,
marker='X',
s=20,
vmin=0,
vmax=max_ppt,
label='PWS Bad %d' % xstns_bad.size)
# plt.show()
plt.xlabel('X [m]')
plt.ylabel('Y [m]')
plt.axis('equal')
plt.legend(loc=0)
cbar = plt.colorbar(sc, extend='max')
cbar.ax.get_yaxis().labelpad = 15
cbar.ax.set_ylabel('[mm/hr]')
plt.title('Event date %s ' % (event_date))
plt.grid(alpha=0.25)
plt.savefig(
os.path.join(
out_save_dir, 'event_date_%s.png' %
(str(event_date).replace('-', '_').replace(':', '_'))))
plt.close()
def test_3bus_with_transformer():
np.random.seed(12)
# 1. Create network
net = pp.create_empty_network()
pp.create_bus(net, name="bus1", vn_kv=10.)
pp.create_bus(net, name="bus2", vn_kv=10.)
pp.create_bus(net, name="bus3", vn_kv=10.)
pp.create_bus(net, name="bus4", vn_kv=110.)
pp.create_ext_grid(net, bus=3, vm_pu=1.01)
pp.create_line_from_parameters(net,
0,
1,
1,
r_ohm_per_km=.01,
x_ohm_per_km=.03,
c_nf_per_km=0.,
max_i_ka=1)
pp.create_line_from_parameters(net,
0,
2,
1,
r_ohm_per_km=.02,
x_ohm_per_km=.05,
c_nf_per_km=0.,
max_i_ka=1)
pp.create_line_from_parameters(net,
1,
2,
1,
r_ohm_per_km=.03,
x_ohm_per_km=.08,
c_nf_per_km=0.,
max_i_ka=1)
pp.create_transformer(net, 3, 0, std_type="25 MVA 110/10 kV")
pp.create_load(net, 1, 450, 300)
pp.create_load(net, 2, 350, 200)
pp.runpp(net, calculate_voltage_angles=True)
pp.create_measurement(net,
"v",
"bus",
r2(net.res_bus.vm_pu.iloc[0], .004),
.004,
bus=0)
pp.create_measurement(net,
"v",
"bus",
r2(net.res_bus.vm_pu.iloc[1], .004),
.004,
bus=1)
pp.create_measurement(net,
"v",
"bus",
r2(net.res_bus.vm_pu.iloc[3], .004),
.004,
bus=3)
pp.create_measurement(net,
"p",
"bus",
-r2(net.res_bus.p_kw.iloc[1], 10),
10,
bus=1)
pp.create_measurement(net,
"q",
"bus",
-r2(net.res_bus.q_kvar.iloc[1], 10),
10,
bus=1)
pp.create_measurement(net,
"p",
"bus",
-r2(net.res_bus.p_kw.iloc[2], 10),
10,
bus=2)
pp.create_measurement(net,
"q",
"bus",
-r2(net.res_bus.q_kvar.iloc[2], 10),
10,
bus=2)
pp.create_measurement(net, "p", "bus", 0., 1.0, bus=0)
pp.create_measurement(net, "q", "bus", 0., 1.0, bus=0)
pp.create_measurement(net, "p", "line",
r2(net.res_line.p_from_kw.iloc[0], 8), 8, 0, 0)
pp.create_measurement(net, "p", "line",
r2(net.res_line.p_from_kw.iloc[1], 8), 8, 0, 1)
pp.create_measurement(net,
"p",
"transformer",
r2(net.res_trafo.p_hv_kw.iloc[0], 10),
10,
bus=3,
element=0) # transformer meas.
pp.create_measurement(net,
"q",
"transformer",
r2(net.res_trafo.q_hv_kvar.iloc[0], 10),
10,
bus=3,
element=0) # at hv side
# 2. Do state estimation
success = estimate(net,
init='slack',
tolerance=5e-5,
maximum_iterations=10)
v_result = net.res_bus_est.vm_pu.values
delta_result = net.res_bus_est.va_degree.values
diff_v = net.res_bus.vm_pu.values - v_result
diff_delta = net.res_bus.va_degree.values - delta_result
assert success
assert (np.nanmax(abs(diff_v)) < 6e-4)
assert (np.nanmax(abs(diff_delta)) < 1.4e-4)
def main():
#%% First setup some parameters for data and motion correction
# dataset dependent parameters
fr = 15.49 # imaging rate in frames per second
decay_time = 0.9 # length of a typical transient in seconds
dxy = (1.4, 1.4) # spatial resolution in x and y in (um per pixel)
# note the lower than usual spatial resolution here
max_shift_um = (12., 12.) # maximum shift in um
patch_motion_um = (100., 100.) # patch size for non-rigid correction in um
# motion correction parameters
pw_rigid = True # flag to select rigid vs pw_rigid motion correction
# maximum allowed rigid shift in pixels
max_shifts = [int(a/b) for a, b in zip(max_shift_um, dxy)]
# start a new patch for pw-rigid motion correction every x pixels
strides = tuple([int(a/b) for a, b in zip(patch_motion_um, dxy)])
# overlap between pathes (size of patch in pixels: strides+overlaps)
overlaps = (24, 24)
# maximum deviation allowed for patch with respect to rigid shifts
max_deviation_rigid = 3
mc_dict = {
'fnames': fnames,
'fr': fr,
'decay_time': decay_time,
'dxy': dxy,
'pw_rigid': pw_rigid,
'max_shifts': max_shifts,
'strides': strides,
'overlaps': overlaps,
'max_deviation_rigid': max_deviation_rigid,
'border_nan': 'copy'
}
opts = params.CNMFParams(params_dict=mc_dict)
# %% start a cluster for parallel processing
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %%% MOTION CORRECTION
# first we create a motion correction object with the specified parameters
mc = MotionCorrect(fnames, dview=dview, **opts.get_group('motion'))
# note that the file is not loaded in memory
# %% Run (piecewise-rigid motion) correction using NoRMCorre
mc.motion_correct(save_movie=True)
# %% MEMORY MAPPING
border_to_0 = 0 if mc.border_nan is 'copy' else mc.border_to_0
# you can include the boundaries of the FOV if you used the 'copy' option
# during motion correction, although be careful about the components near
# the boundaries
# memory map the file in order 'C'
fname_new = cm.save_memmap(mc.mmap_file, base_name='_memmap_', order='C',
border_to_0=border_to_0) # exclude borders
# now load the file
Yr, dims, T = cm.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
# load frames in python format (T x X x Y)
# %% restart cluster to clean up memory
cm.stop_server(dview=dview)
c, dview, n_processes = cm.cluster.setup_cluster(
backend='local', n_processes=None, single_thread=False)
# %% parameters for source extraction and deconvolution
p = 1 # order of the autoregressive system
gnb = 3 # number of global background components
merge_thr = 0.85 # merging threshold, max correlation allowed
rf = 20 # half-size of the patches in pixels. e.g., if rf=25, patches are 50x50
stride_cnmf = 6 # amount of overlap between the patches in pixels
K = 3 # number of components per patch
gSig = [7, 7] # expected half size of neurons in pixels
method_init = 'greedy_roi' # initialization method (if analyzing dendritic data using 'sparse_nmf')
ssub = 2 # spatial subsampling during initialization
tsub = 2 # temporal subsampling during intialization
s_min = -20 # minimum signal amplitude needed in order for a transient to be considered as activity
# parameters for component evaluation
opts_dict = {'fnames': fnames,
'fr': fr,
'nb': gnb,
'rf': rf,
'K': K,
'gSig': gSig,
'stride': stride_cnmf,
'method_init': method_init,
'rolling_sum': True,
'merge_thr': merge_thr,
'n_processes': n_processes,
'only_init': True,
'ssub': ssub,
'tsub': tsub,
's_min': s_min}
opts.change_params(params_dict=opts_dict)
# %% RUN CNMF ON PATCHES
# First extract spatial and temporal components on patches and combine them
# for this step deconvolution is turned off (p=0)
opts.change_params({'p': 0})
cnm = cnmf.CNMF(n_processes, params=opts, dview=dview)
cnm = cnm.fit(images)
# %% ALTERNATE WAY TO RUN THE PIPELINE AT ONCE
# you can also perform the motion correction plus cnmf fitting steps
# simultaneously after defining your parameters object using
# cnm1 = cnmf.CNMF(n_processes, params=opts, dview=dview)
# cnm1.fit_file(motion_correct=True)
# %% RE-RUN seeded CNMF on accepted patches to refine and perform deconvolution
cnm.params.change_params({'p': p})
cnm2 = cnm.refit(images, dview=dview)
# %% COMPONENT EVALUATION
# the components are evaluated in three ways:
# a) the shape of each component must be correlated with the data
# b) a minimum peak SNR is required over the length of a transient
# c) each shape passes a CNN based classifier
min_SNR = 2 # Overall minimum signal to noise ratio for accepting a component
rval_thr = 0.85 # space correlation threshold for accepting a component
cnn_thr = 0.99 # threshold for CNN based classifier
cnn_lowest = 0.15 # neurons with cnn probability lower than this value are rejected
min_size_neuro = 0.1*gSig[0]*np.pi**2
max_size_neuro = 2.5*gSig[0]*np.pi**2
cnm2.params.set('quality', {'decay_time': decay_time,
'min_SNR': min_SNR,
'rval_thr': rval_thr,
'use_cnn': True,
'min_cnn_thr': cnn_thr,
'cnn_lowest': cnn_lowest,
'min_size_neuro': min_size_neuro,
'max_size_neuro': max_size_neuro,})
cnm2.estimates.evaluate_components(images, cnm2.params, dview=dview)
#%% update object with selected components
cnm2.estimates.select_components(use_object=True)
#%% Extract DF/F values
cnm2.estimates.detrend_df_f(quantileMin=8, frames_window=250)
#cnm2.estimates.threshold_spatial_components(maxthr=0.85)
#cnm2.estimates.remove_small_large_neurons(min_size_neuro=min_size_neuro, max_size_neuro=max_size_neuro)
# %% Generate heat image of video and save data
Cn = cm.local_correlations(images, swap_dim=False)
Cn[np.isnan(Cn)] = 0
cnm2.estimates.Cn = Cn
cnm2.save(cnm2.mmap_file[:-4] + 'hdf5')
#%% STOP CLUSTER and clean up log files
cm.stop_server(dview=dview)
#Create denoised video as matrix
denoised = cm.movie(cnm2.estimates.A.dot(cnm2.estimates.C) + \
cnm2.estimates.b.dot(cnm2.estimates.f)).reshape(dims + (-1,), order='F').transpose([2, 0, 1])
#Normalizing denoised matrix to the [0,255] range prior to type conversion
max_pixel = np.nanmax(denoised)
norm_denoised = (denoised/max_pixel)*255
#Type conversion so imageio doesn't get mad
norm_denoised = np.uint8(norm_denoised)
#Saving a denoised avi of the analyzed video
sname = Path(args.in_file).stem + '.avi'
imageio.mimwrite(sname, norm_denoised, fps=15.49)
def test_3bus_with_bad_data():
net = pp.create_empty_network()
pp.create_bus(net, name="bus1", vn_kv=1.)
pp.create_bus(net, name="bus2", vn_kv=1.)
pp.create_bus(net, name="bus3", vn_kv=1.)
pp.create_ext_grid(net, 0)
pp.create_line_from_parameters(net,
0,
1,
1,
r_ohm_per_km=0.7,
x_ohm_per_km=0.2,
c_nf_per_km=0,
max_i_ka=1)
pp.create_line_from_parameters(net,
0,
2,
1,
r_ohm_per_km=0.8,
x_ohm_per_km=0.8,
c_nf_per_km=0,
max_i_ka=1)
pp.create_line_from_parameters(net,
1,
2,
1,
r_ohm_per_km=1,
x_ohm_per_km=0.6,
c_nf_per_km=0,
max_i_ka=1)
pp.create_measurement(net,
"p",
"line",
-0.0011e3,
0.01e3,
bus=0,
element=0) # p12
pp.create_measurement(net, "q", "line", 0.024e3, 0.01e3, bus=0,
element=0) # q12
pp.create_measurement(net, "p", "bus", 0.018e3, 0.01e3, bus=2) # p3
pp.create_measurement(net, "q", "bus", -0.1e3, 0.01e3, bus=2) # q3
pp.create_measurement(net, "v", "bus", 1.08, 0.05, 0) # u1
pp.create_measurement(net, "v", "bus", 1.015, 0.05, 2) # u3
# 1. Create false voltage measurement for testing bad data detection (-> should be removed)
pp.create_measurement(net, "v", "bus", 1.3, 0.01, bus=1) # V at bus 2
# 2. Do chi2-test
bad_data_detected = chi2_analysis(net, init='flat')
# 3. Perform rn_max_test
success_rn_max = remove_bad_data(net, init='flat')
v_est_rn_max = net.res_bus_est.vm_pu.values
delta_est_rn_max = net.res_bus_est.va_degree.values
target_v = np.array([1.0627, 1.0589, 1.0317])
diff_v = target_v - v_est_rn_max
target_delta = np.array([0., 0.8677, 3.1381])
diff_delta = target_delta - delta_est_rn_max
assert bad_data_detected
assert success_rn_max
assert (np.nanmax(abs(diff_v)) < 1e-4)
assert (np.nanmax(abs(diff_delta)) < 1e-4)
def image_as_uint(im, bitdepth=None):
""" Convert the given image to uint (default: uint8)
If the dtype already matches the desired format, it is returned
as-is. If the image is float, and all values are between 0 and 1,
the values are multiplied by np.power(2.0, bitdepth). In all other
situations, the values are scaled such that the minimum value
becomes 0 and the maximum value becomes np.power(2.0, bitdepth)-1
(255 for 8-bit and 65535 for 16-bit).
"""
if not bitdepth:
bitdepth = 8
if not isinstance(im, np.ndarray):
raise ValueError('Image must be a numpy array')
if bitdepth == 8:
out_type = np.uint8
elif bitdepth == 16:
out_type = np.uint16
else:
raise ValueError('Bitdepth must be either 8 or 16')
dtype_str1 = str(im.dtype)
dtype_str2 = out_type.__name__
if ((im.dtype == np.uint8 and bitdepth == 8)
or (im.dtype == np.uint16 and bitdepth == 16)):
# Already the correct format? Return as-is
return im
if (dtype_str1.startswith('float') and np.nanmin(im) >= 0
and np.nanmax(im) <= 1):
_precision_warn(dtype_str1, dtype_str2, 'Range [0, 1].')
im = im.astype(
np.float64) * (np.power(2.0, bitdepth) - 1) + 0.499999999
elif im.dtype == np.uint16 and bitdepth == 8:
_precision_warn(dtype_str1, dtype_str2, 'Losing 8 bits of resolution.')
im = np.right_shift(im, 8)
elif im.dtype == np.uint32:
_precision_warn(dtype_str1, dtype_str2,
'Losing {} bits of resolution.'.format(32 - bitdepth))
im = np.right_shift(im, 32 - bitdepth)
elif im.dtype == np.uint64:
_precision_warn(
dtype_str1, dtype_str2,
'Losing {} bits of resolution.'.format(64 - bitdepth, ))
im = np.right_shift(im, 64 - bitdepth)
else:
mi = np.nanmin(im)
ma = np.nanmax(im)
if not np.isfinite(mi):
raise ValueError('Minimum image value is not finite')
if not np.isfinite(ma):
raise ValueError('Maximum image value is not finite')
if ma == mi:
raise ValueError('Max value == min value, ambiguous given dtype')
_precision_warn(dtype_str1, dtype_str2,
'Range [{}, {}].'.format(mi, ma))
# Now make float copy before we scale
im = im.astype('float64')
# Scale the values between 0 and 1 then multiply by the max value
im = (im - mi) / (ma - mi) * (np.power(2.0, bitdepth) -
1) + 0.499999999
assert np.nanmin(im) >= 0
assert np.nanmax(im) < np.power(2.0, bitdepth)
return im.astype(out_type)
def on_evt_filter_pws(args):
(path_to_prim_netw_ppt_hdf5, prim_netw_in_coords_df,
path_to_neatmo_ppt_hdf5, pws_in_coords_df, ids_pws_to_use, time_list,
df_save_results) = args
HDF5_pws_ppt = HDF5(infile=path_to_neatmo_ppt_hdf5)
# all_pws_ids = HDF5_pws_ppt.get_all_names()
all_pws_ids_to_use = ids_pws_to_use
HDF5_prim_netw_ppt = HDF5(infile=path_to_prim_netw_ppt_hdf5)
all_prim_netw_ids = HDF5_prim_netw_ppt.get_all_names()
def scale_vg_based_on_prim_netw_ppt(ppt_dwd_vals, vg_sill_b4_scale):
# sacle variogram based on dwd ppt
# vg_sill = float(vg_model_to_scale.split(" ")[0])
dwd_vals_var = np.var(ppt_dwd_vals)
vg_scaling_ratio = dwd_vals_var / vg_sill_b4_scale
if vg_scaling_ratio == 0:
vg_scaling_ratio = vg_sill_b4_scale
# rescale variogram
vgs_model_dwd_ppt = str(np.round(
vg_scaling_ratio, 4)) + ' ' + vg_model_to_scale.split(" ")[1]
# vgs_model_dwd_ppt
return vgs_model_dwd_ppt # vg_scaling_ratio
def plot_good_bad_stns(pws_in_coords_df, ids_pws_stns_gd, ids_pws_stns_bad,
zvalues, prim_netw_ppt, zvalues_bad, prim_netwx,
prim_netwy, event_date):
xstns_good = pws_in_coords_df.loc[ids_pws_stns_gd, 'X'].values.ravel()
ystns_good = pws_in_coords_df.loc[ids_pws_stns_gd, 'Y'].values.ravel()
xstns_bad = pws_in_coords_df.loc[ids_pws_stns_bad, 'X'].values.ravel()
ystns_bad = pws_in_coords_df.loc[ids_pws_stns_bad, 'Y'].values.ravel()
max_ppt = max(np.nanmax(zvalues), np.nanmax(prim_netw_ppt))
interval_ppt = np.linspace(0.0, 0.99)
colors_ppt = plt.get_cmap('Blues')(interval_ppt)
cmap_ppt = LinearSegmentedColormap.from_list('name', colors_ppt)
# cmap_ppt = plt.get_cmap('jet_r')
cmap_ppt.set_over('navy')
interval_ppt_bad = np.linspace(0.02, 0.95)
colors_ppt_bad = plt.get_cmap('autumn')(interval_ppt_bad)
cmap_ppt_bad = LinearSegmentedColormap.from_list(
'name', colors_ppt_bad)
plt.ioff()
plt.figure(figsize=(12, 8), dpi=100)
plt.scatter(prim_netwx,
prim_netwy,
c=prim_netw_ppt,
cmap=cmap_ppt,
marker=',',
s=10,
alpha=0.75,
vmin=0,
vmax=max_ppt,
label='prim_netw %d' % prim_netwx.size)
sc = plt.scatter(xstns_good,
ystns_good,
c=zvalues,
cmap=cmap_ppt,
marker='.',
s=10,
alpha=0.75,
vmin=0,
vmax=max_ppt,
label='PWS Good %d' % xstns_good.size)
plt.scatter(xstns_bad,
ystns_bad,
alpha=0.75,
c=zvalues_bad,
cmap=cmap_ppt_bad,
marker='X',
s=20,
vmin=0,
vmax=max_ppt,
label='PWS Bad %d' % xstns_bad.size)
# plt.show()
plt.xlabel('X [m]')
plt.ylabel('Y [m]')
plt.axis('equal')
plt.legend(loc=0)
cbar = plt.colorbar(sc, extend='max')
cbar.ax.get_yaxis().labelpad = 15
cbar.ax.set_ylabel('[mm/hr]')
plt.title('Event date %s ' % (event_date))
plt.grid(alpha=0.25)
plt.savefig(
os.path.join(
out_save_dir, 'event_date_%s.png' %
(str(event_date).replace('-', '_').replace(':', '_'))))
plt.close()
#==========================================================================
#
#==========================================================================
for ix, date_to_correct in enumerate(time_list):
print(ix, '/', len(time_list), '--', date_to_correct)
# pws_data = pd.read_feather(path_netatamo_edf_fk, columns=pws_ids_str)
pws_data_evt = HDF5_pws_ppt.get_pandas_dataframe_for_date(
ids=all_pws_ids_to_use,
event_date=date_to_correct).dropna(how='all', axis=1)
if len(pws_data_evt.columns) > 0:
pws_stns_evt = pws_data_evt.columns.to_list()
cmn_pws_event = pws_in_coords_df.index.intersection(pws_stns_evt)
# coords of stns to correct
xstns_interp = pws_in_coords_df.loc[cmn_pws_event,
'X'].values.ravel()
ystns_interp = pws_in_coords_df.loc[cmn_pws_event,
'Y'].values.ravel()
cmn_pws_data_evt = pws_data_evt.loc[:, cmn_pws_event]
# prim_netw data
ppt_prim_netw_vals_evt = (
HDF5_prim_netw_ppt.get_pandas_dataframe_for_date(
ids=all_prim_netw_ids, event_date=date_to_correct))
prim_netw_stns_evt = ppt_prim_netw_vals_evt.columns.to_list()
cmn_prim_netw_stns_evt = prim_netw_in_coords_df.index.intersection(
prim_netw_stns_evt)
prim_netw_xcoords = prim_netw_in_coords_df.loc[
cmn_prim_netw_stns_evt, 'X'].values.ravel()
prim_netw_ycoords = prim_netw_in_coords_df.loc[
cmn_prim_netw_stns_evt, 'Y'].values.ravel()
# primary network values for event
cmn_ppt_prim_netw_vals_evt = ppt_prim_netw_vals_evt.loc[:,
cmn_prim_netw_stns_evt]
# scale variogram
vgs_model_dwd_ppt = scale_vg_based_on_prim_netw_ppt(
ppt_prim_netw_vals_evt.values, vg_sill_b4_scale)
# start kriging pws location
# OK_prim_netw_pws_crt = OKpy(xi=prim_netw_xcoords,
# yi=prim_netw_ycoords,
# zi=cmn_ppt_prim_netw_vals_evt.values.ravel(),
# xk=xstns_interp,
# yk=ystns_interp,
# model=vgs_model_dwd_ppt)
# using PYkrige
dwd_vals_var = np.var(cmn_ppt_prim_netw_vals_evt.values)
vg_scaling_ratio = dwd_vals_var / vg_sill_b4_scale
#
if vg_scaling_ratio == 0:
vg_scaling_ratio = vg_sill_b4_scale
OK_prim_netw_pws_crt = OKpy(
prim_netw_xcoords,
prim_netw_ycoords,
cmn_ppt_prim_netw_vals_evt.values.ravel(),
variogram_model=vg_model_str,
variogram_parameters={
'sill': vg_scaling_ratio,
'range': vg_range,
'nugget': 0
})
try:
# OK_prim_netw_pws_crt.krige()
# zvalues = OK_prim_netw_pws_crt.zk.copy()
#
# # calcualte standard deviation of estimated values
# std_est_vals = np.sqrt(OK_prim_netw_pws_crt.est_vars)
zvalues, est_var = OK_prim_netw_pws_crt.execute(
'points', np.array([xstns_interp]),
np.array([ystns_interp]))
std_est_vals = np.sqrt(est_var).data
except Exception as msg:
print('ror', msg)
# if neg assign 0
zvalues[zvalues < 0] = 0
# calcualte standard deviation of estimated values
# calculate difference observed and estimated
# values
diff_obsv_interp = np.abs(cmn_pws_data_evt.values - zvalues)
idx_good_stns = np.where(diff_obsv_interp <= 3 * std_est_vals)[1]
idx_bad_stns = np.where(diff_obsv_interp > 3 * std_est_vals)[1]
if len(idx_bad_stns) > 0:
# use additional filter
try:
ids_pws_stns_gd = np.take(cmn_pws_event,
idx_good_stns).ravel()
ids_pws_stns_bad = np.take(cmn_pws_event,
idx_bad_stns).ravel()
except Exception as msg:
print(msg)
# ids of bad stns
xstns_bad = pws_in_coords_df.loc[ids_pws_stns_bad,
'X'].values.ravel()
ystns_bad = pws_in_coords_df.loc[ids_pws_stns_bad,
'Y'].values.ravel()
# # check if bad are truly bad
xstns_good = pws_in_coords_df.loc[ids_pws_stns_gd,
'X'].values.ravel()
ystns_good = pws_in_coords_df.loc[ids_pws_stns_gd,
'Y'].values.ravel()
# coords of neighbors good
neighbors_coords = np.array([
(x, y) for x, y in zip(xstns_good, ystns_good)
])
# create a tree from coordinates
points_tree = spatial.cKDTree(neighbors_coords)
neighbors_coords_prim_netw = np.array([
(x, y)
for x, y in zip(prim_netw_xcoords, prim_netw_ycoords)
])
points_tree_prim_netw = spatial.cKDTree(
neighbors_coords_prim_netw)
# plt.ioff()
# plt.scatter(xstns_good, ystns_good, c='b')
# plt.scatter(prim_netw_xcoords, prim_netw_ycoords, c='g')
# plt.scatter(xstns_bad, ystns_bad, c='r')
# plt.show()
if len(idx_bad_stns) > 0 > 0:
for stn_ix, stn_bad in zip(idx_bad_stns, ids_pws_stns_bad):
ppt_bad = cmn_pws_data_evt.loc[:, stn_bad].values
# print('ppt_bad', ppt_bad)
if ppt_bad >= 0.:
xstn_bd = pws_in_coords_df.loc[stn_bad, 'X']
ystn_bd = pws_in_coords_df.loc[stn_bad, 'Y']
idxs_neighbours = points_tree.query_ball_point(
np.array((xstn_bd, ystn_bd)), 1e4)
ids_neighbours = ids_pws_stns_gd[idxs_neighbours]
ids_neighbours_evt = np.in1d(
cmn_pws_event, ids_neighbours)
idxs_neighbours_prim_netw = points_tree_prim_netw.query_ball_point(
np.array((xstn_bd, ystn_bd)), 1e4)
ids_neighbours_prim_netw_evt = np.array(
cmn_prim_netw_stns_evt
)[idxs_neighbours_prim_netw]
if len(ids_neighbours_evt) > 0:
ppt_pws_ngbrs = cmn_pws_data_evt.loc[:,
ids_neighbours_evt]
ppt_pws_data = ppt_pws_ngbrs.values
#---------------------------------------------
# xstn_ngbr = pws_in_coords_df.loc[
# ppt_pws_ngbrs.columns, 'X'].values.ravel()
# ystn_ngbr = pws_in_coords_df.loc[
# ppt_pws_ngbrs.columns, 'Y'].values.ravel()
#---------------------------------------------
if ppt_pws_data.size == 0:
ppt_pws_data = 1000
else:
ppt_pws_data = 1000
if len(ids_neighbours_prim_netw_evt) > 0:
ppt_prim_netw_ngbrs = cmn_ppt_prim_netw_vals_evt.loc[:,
ids_neighbours_prim_netw_evt]
ppt_prim_netw_data = ppt_prim_netw_ngbrs.values
#---------------------------------------------
# prim_netw_xstn_ngbr = prim_netw_in_coords_df.loc[
# ppt_prim_netw_ngbrs.columns, 'X'].values.ravel()
# prim_netw_ystn_ngbr = prim_netw_in_coords_df.loc[
# ppt_prim_netw_ngbrs.columns, 'Y'].values.ravel()
#---------------------------------------------
# plt.ioff()
# plt.scatter(xstn_bd, ystn_bd, c='r')
# plt.scatter(xstn_ngbr, ystn_ngbr, c='b')
# plt.scatter(prim_netw_xstn_ngbr,
# prim_netw_ystn_ngbr, c='g')
# plt.show()
if ppt_prim_netw_ngbrs.size == 0:
ppt_prim_netw_ngbrs = 1000
else:
ppt_prim_netw_ngbrs = 1000 # always wrong
try:
if (ppt_bad > np.nanmin(ppt_pws_data)
or ppt_bad >
np.nanmin(ppt_prim_netw_data)):
# print('added bad to good\n')
# print('ppt_bad', ppt_bad)
ids_pws_stns_gd_final = np.append(
ids_pws_stns_gd, stn_bad)
idx_good_stns_final = np.append(
idx_good_stns, stn_ix)
ids_pws_stns_bad_final = np.setdiff1d(
ids_pws_stns_bad, stn_bad)
ids_pws_stns_bad_final.size
idx_bad_stns_final = np.setdiff1d(
idx_bad_stns, stn_ix)
assert stn_bad in ids_pws_stns_gd_final
assert stn_bad not in ids_pws_stns_bad_final
else:
pass
# print('not added bad stn')
idx_good_stns_final = np.sort(
idx_good_stns_final)
idx_bad_stns_final = np.sort(
idx_bad_stns_final)
except Exception as msg:
print('error', msg)
continue
# raise Exception
else:
ids_pws_stns_gd_final = ids_pws_stns_gd
idx_good_stns_final = idx_good_stns
ids_pws_stns_bad_final = ids_pws_stns_bad
idx_bad_stns_final = idx_bad_stns
try:
print('Number of Stations with bad index \n',
len(idx_bad_stns_final))
print('Number of Stations with good index \n',
len(idx_good_stns_final))
# save results gd+1, bad -1
df_save_results.loc[date_to_correct,
ids_pws_stns_gd_final] = 1
df_save_results.loc[date_to_correct,
ids_pws_stns_bad_final] = -1
except Exception as msg3:
print(msg3)
continue
try:
zvalues_good = cmn_pws_data_evt.loc[
date_to_correct, ids_pws_stns_gd_final].values.ravel()
zvalues_bad = cmn_pws_data_evt.loc[
date_to_correct,
ids_pws_stns_bad_final].values.ravel()
# plot configuration
max_ppt = max(
np.nanmax(zvalues_good),
np.nanmax(cmn_ppt_prim_netw_vals_evt.values.ravel()))
if max_ppt >= 0:
print('plotting map')
plot_good_bad_stns(
pws_in_coords_df, ids_pws_stns_gd_final,
ids_pws_stns_bad_final, zvalues_good,
cmn_ppt_prim_netw_vals_evt.values.ravel(),
zvalues_bad, prim_netw_xcoords, prim_netw_ycoords,
date_to_correct)
plt.close()
except Exception as msg2:
print('error plotting ', msg2)
# break
df_save_results.dropna(how='all', inplace=True)
return df_save_results
def aabb(points):
points = np.asarray(points)
dim = points.shape[-1]
points = points.reshape(-1, dim)
p1, p2 = np.nanmin(points, 0), np.nanmax(points, 0)
return np.array([p1, p2])
from astropy.io import fits
hdu1 = fits.open('mom2_12co_pix_2_Tmb.fits')[0]
xcenter = 84
ycenter = -6
wid = 1.5
hei = 2.4
xpanels = 1
ypanels = 1
fig = plt.figure(figsize=(3 * xpanels * 1.1 * (wid / (wid + hei)) * 10.,
3 * ypanels / 1.1 * (hei / (wid + hei)) * 10.))
ff = aplpy.FITSFigure(hdu1, figure=fig)
ff.recenter(xcenter, ycenter, width=wid, height=hei)
ff.set_theme('publication')
#ff.set_system_latex(True)
maxcolor = np.nanmax(hdu1.data)
ff.show_colorscale(cmap='Purples', vmin=0, vmax=5, stretch='sqrt')
#ff.show_contour(mask_hdu, levels=1, colors='yellow', linewidths=0.1)
#ff.show_regions('olay.reg')
ff.show_regions('olay3.reg')
ff.add_colorbar()
ff.colorbar.set_font(size=12)
ff.colorbar.set_pad(0.5)
ff.set_tick_labels_font(size=12)
ff.colorbar.set_axis_label_text('km s$^{-1}$')
ff.colorbar.set_font(size=12)
ff.set_axis_labels_font(size=12)
ff.add_scalebar(0.286, corner='bottom right',
pad=10) # degree for 2pc at 400 pc
ff.scalebar.set_label('2 pc')
ff.scalebar.set_font_size(12)
c=color) #, label = label_name)#"tab:blue")
flux_axarr[0].plot(longitude_list,
rolling_mean_Osborn_flux,
ls=":",
lw=2.5,
zorder=3,
c=color) #, label = label_name)
#flux_axarr.fill_between(longitude_list,rolling_upper_percentile_flux,rolling_lower_percentile_flux, color = "tab:blue", zorder = 2, alpha = 0.7, label = str(flux_percentile)+"% percentile")
#flux_axarr.fill_between(longitude_list,rolling_upper_percentile_flux,rolling_second_upper_percentile_flux, color = "tab:blue", zorder = 2, alpha = 0.4, label = str(second_flux_percentile)+"% percentile")
#flux_axarr.fill_between(longitude_list,rolling_lower_percentile_flux,rolling_second_lower_percentile_flux, color = "tab:blue", zorder = 2, alpha = 0.4)
#bathymetrie_axes.plot(bathymetry_longitude_list,bathymetry_list)
if cruisename == "emb217":
flux_axarr[1].set_ylim(
(np.nanmin(bathymetry_list) - 5, np.nanmax(bathymetry_list)))
flux_axarr[1].invert_yaxis()
flux_axarr[1].set_ylabel("pressure [dbar]")
flux_axarr[1].fill_between(bathymetry_longitude_list,
bathymetry_list,
np.ones(len(bathymetry_list)) *
max(bathymetry_list),
color="lightgrey",
zorder=1,
alpha=0.8,
label="bathymetry")
#bathymetrie_axes.plot(bathymetry_longitude_list,interval_list[:,0])
#bathymetrie_axes.plot(bathymetry_longitude_list,interval_list[:,1])
flux_axarr[1].fill_between(bathymetry_longitude_list,
interval_list[:, 0],
def minimum_bounding_rectangle(points):
"""
Find the smallest bounding rectangle for a set of points.
Returns a set of points representing the corners of the bounding box.
:param points: an nx2 matrix of coordinates
:rval: an nx2 matrix of coordinates
"""
from scipy.ndimage.interpolation import rotate
pi2 = np.pi / 2.
# get the convex hull for the points
hull_points = points[ConvexHull(points).vertices]
# calculate edge angles
edges = np.zeros((len(hull_points) - 1, 2))
edges = hull_points[1:] - hull_points[:-1]
angles = np.zeros((len(edges)))
angles = np.arctan2(edges[:, 1], edges[:, 0])
angles = np.abs(np.mod(angles, pi2))
angles = np.unique(angles)
# find rotation matrices
# XXX both work
rotations = np.vstack([
np.cos(angles),
np.cos(angles - pi2),
np.cos(angles + pi2),
np.cos(angles)
]).T
# rotations = np.vstack([
# np.cos(angles),
# -np.sin(angles),
# np.sin(angles),
# np.cos(angles)]).T
rotations = rotations.reshape((-1, 2, 2))
# apply rotations to the hull
rot_points = np.dot(rotations, hull_points.T)
# find the bounding points
min_x = np.nanmin(rot_points[:, 0], axis=1)
max_x = np.nanmax(rot_points[:, 0], axis=1)
min_y = np.nanmin(rot_points[:, 1], axis=1)
max_y = np.nanmax(rot_points[:, 1], axis=1)
# find the box with the best area
areas = (max_x - min_x) * (max_y - min_y)
best_idx = np.argmin(areas)
# return the best box
x1 = max_x[best_idx]
x2 = min_x[best_idx]
y1 = max_y[best_idx]
y2 = min_y[best_idx]
r = rotations[best_idx]
rval = np.zeros((4, 2))
rval[0] = np.dot([x1, y2], r)
rval[1] = np.dot([x2, y2], r)
rval[2] = np.dot([x2, y1], r)
rval[3] = np.dot([x1, y1], r)
area = abs(y2 - y1) * abs(x2 - x1)
per = 2 * (abs(y2 - y1) + abs(x2 - x1))
return per, area
def interpolate2d(x, y, z, points, mode='linear', bounds_error=False):
"""Fundamental 2D interpolation routine
:param x: 1D array of x-coordinates of the mesh on which to interpolate
:type x: numpy.ndarray
:param y: 1D array of y-coordinates of the mesh on which to interpolate
:type y: numpy.ndarray
:param z: 2D array of values for each x, y pair
:type z: numpy.ndarry
:param points: Nx2 array of coordinates where interpolated values are
sought
:type points: numpy.narray
:param mode: Determines the interpolation order.
Options are:
* 'constant' - piecewise constant nearest neighbour interpolation
* 'linear' - bilinear interpolation using the four
nearest neighbours (default)
:type mode: str
:param bounds_error: If True (default) a BoundsError exception
will be raised when interpolated values are requested
outside the domain of the input data. If False, nan
is returned for those values
:type bounds_error: bool
:returns: 1D array with same length as points with interpolated values
:raises: Exception, BoundsError (see note about bounds_error)
..notes::
Input coordinates x and y are assumed to be monotonically increasing,
but need not be equidistantly spaced. No such assumption regarding
ordering of points is made.
z is assumed to have dimension M x N, where M = len(x) and N = len(y).
In other words it is assumed that the x values follow the first
(vertical) axis downwards and y values the second (horizontal) axis
from left to right.
If this routine is to be used for interpolation of raster grids where
data is typically organised with longitudes (x) going from left to
right and latitudes (y) from left to right then user
interpolate_raster in this module
"""
# Input checks
validate_mode(mode)
x, y, z, xi, eta = validate_inputs(x=x,
y=y,
z=z,
points=points,
bounds_error=bounds_error)
# Identify elements that are outside interpolation domain or NaN
outside = (xi < x[0]) + (eta < y[0]) + (xi > x[-1]) + (eta > y[-1])
outside += numpy.isnan(xi) + numpy.isnan(eta)
inside = -outside
xi = xi[inside]
eta = eta[inside]
# Find upper neighbours for each interpolation point
idx = numpy.searchsorted(x, xi, side='left')
idy = numpy.searchsorted(y, eta, side='left')
# Internal check (index == 0 is OK)
if len(idx) > 0 or len(idy) > 0:
if (max(idx) >= len(x)) or (max(idy) >= len(y)):
msg = ('Interpolation point outside domain. '
'This should never happen. '
'Please email [email protected]')
raise InaSAFEError(msg)
# Get the four neighbours for each interpolation point
x0 = x[idx - 1]
x1 = x[idx]
y0 = y[idy - 1]
y1 = y[idy]
z00 = z[idx - 1, idy - 1]
z01 = z[idx - 1, idy]
z10 = z[idx, idy - 1]
z11 = z[idx, idy]
# Coefficients for weighting between lower and upper bounds
old_set = numpy.seterr(invalid='ignore') # Suppress warnings
alpha = (xi - x0) / (x1 - x0)
beta = (eta - y0) / (y1 - y0)
numpy.seterr(**old_set) # Restore
if mode == 'linear':
# Bilinear interpolation formula
dx = z10 - z00
dy = z01 - z00
z = z00 + alpha * dx + beta * dy + alpha * beta * (z11 - dx - dy - z00)
else:
# Piecewise constant (as verified in input_check)
# Set up masks for the quadrants
left = alpha < 0.5
right = -left
lower = beta < 0.5
upper = -lower
lower_left = lower * left
lower_right = lower * right
upper_left = upper * left
# Initialise result array with all elements set to upper right
z = z11
# Then set the other quadrants
z[lower_left] = z00[lower_left]
z[lower_right] = z10[lower_right]
z[upper_left] = z01[upper_left]
# Self test
if len(z) > 0:
mz = numpy.nanmax(z)
mZ = numpy.nanmax(z)
# noinspection PyStringFormat
msg = ('Internal check failed. Max interpolated value %.15f '
'exceeds max grid value %.15f ' % (mz, mZ))
if not (numpy.isnan(mz) or numpy.isnan(mZ)):
if not mz <= mZ:
raise InaSAFEError(msg)
# Populate result with interpolated values for points inside domain
# and NaN for values outside
r = numpy.zeros(len(points))
r[inside] = z
r[outside] = numpy.nan
return r
def SDS(df, n=3):
"""Finds the `n` most dissimilar items. In the input matrix, the ith row (and
ith column) is an array belonging to item i. The matrix element (i,j) would then be the pairwise
dissimilarity metric between items i and j (for example, geometric RMSD between molecular conformers i and j).
Args:
df (pandas.DataFrame): Square matrix where each row (and by symmetry, column) is an array
corresponding to a specific item or object, and each element (i,j) the
floating point dissimilarity between items i and j. The element (i,i) must
be represented as np.nan (for log-summing).
If pairwise data between two
items is missing, this can also be represented as np.nan. What will
happen is the second item will automatically be set to np.nan in the
log-summation array once the first of one of the two items is chosen.
Thus, the second item will never be chosen. In this same manner,
entire missing items can be represented as arrays of np.nan,
as a trick to preserve externally related indexing.
n (int): Dissimilar set size to find.
1 < n < N, where N is the full population size.
Returns:
pandas.DataFrame containing indices of the items found in the dissimilar set of size `n`.
"""
# Check df matrix is square
N = len(df.index)
assert N == len(df.columns)
# Reduce n to maximum number of items if n is over, otherwise search will fail with an erorr.
M = len(df.dropna(how='all'))
if n > M:
n = M
print(f'Starting SDS search for most dissimilar set of size n = {n}...')
# First grab matrix indices of the two most dissimilar geometries
row_mx = []
for i in range(N):
row_mx.append(np.nanmax(df.loc[i]))
ind1 = np.nanargmax(row_mx)
ind2 = ind1 + 1 + np.nanargmax(row_mx[ind1+1:])
# Initialize the dissimilar matrix with the two most dissimilar
disarray = [np.array(df.loc[ind1]), np.array(df.loc[ind2])]
indices = [ind1, ind2]
# Find n-2 other most dissimilar
# Multiply the rows of the n-1 dissimilar set. Or,
# use log summing if N is large (e.g. 50000) to avoid
# exceeding floating point machine precision.
# This script uses log summing.
# The index of the largest value is the index of the nth
# item which makes the nth dissimilar set.
# Initialize array for log summing
logsum = [0 for x in range(N)]
logsum += np.log(disarray[0])
for i in range(n-2):
logsum += np.log(disarray[-1])
indn = np.nanargmax(logsum)
indices.append(indn)
disarray.append(np.array(df.loc[indn]))
return_df = pd.DataFrame([indices], index=['matrix index']).T
print('Finished')
return return_df
def extractSimData(sim, min_frames_visible=MIN_FRAMES_VISIBLE, check_only=False, param_class_name=None, \
postprocess_params=None):
""" Extract input parameters and model outputs from the simulation container and normalize them.
Arguments:
sim: [ErosionSimContainer object] Container with the simulation.
Keyword arguments:
min_frames_visible: [int] Minimum number of frames above the limiting magnitude
check_only: [bool] Only check if the simulation satisfies filters, don' compute eveything.
Speed up the evaluation. False by default.
param_class_name: [str] Override the simulation parameters object with an instance of the given
class. An exact name of the class needs to be given.
postprocess_params: [list] A list of limiting magnitude for wide and narrow fields, and the delay in
length measurements. None by default, in which case they will be generated herein.
Return:
- None if the simulation does not satisfy filter conditions.
- postprocess_params if check_only=True and the simulation satisfies the conditions.
- params, input_data_normed, simulated_data_normed if check_only=False and the simulation satisfies
the conditions.
"""
# Create a frash instance of the system parameters if the same parameters are used as in the simulation
if param_class_name is None:
#params_obj = getattr(GenerateSimulations, sim.params.__class__.__name__)
params = globals()[sim.params.__class__.__name__]()
# Override the system parameters using the given class
else:
params = globals()[param_class_name]()
### DRAW LIMITING MAGNITUDE AND LENGTH DELAY ###
# If the drawn values have already been given, use them
if postprocess_params is not None:
lim_mag, lim_mag_len, len_delay = postprocess_params
else:
# Draw limiting magnitude and length end magnitude
lim_mag = np.random.uniform(params.lim_mag_brightest, params.lim_mag_faintest)
lim_mag_len = np.random.uniform(params.lim_mag_len_end_brightest, params.lim_mag_len_end_faintest)
# Draw the length delay
len_delay = np.random.uniform(params.len_delay_min, params.len_delay_max)
postprocess_params = [lim_mag, lim_mag_len, len_delay]
lim_mag_faintest = np.max([lim_mag, lim_mag_len])
lim_mag_brightest = np.min([lim_mag, lim_mag_len])
### ###
# Fix NaN values in the simulated magnitude
sim.simulation_results.abs_magnitude[np.isnan(sim.simulation_results.abs_magnitude)] \
= np.nanmax(sim.simulation_results.abs_magnitude)
# Get indices that are above the faintest limiting magnitude
indices_visible = sim.simulation_results.abs_magnitude <= lim_mag_faintest
# If no points were visible, skip this solution
if not np.any(indices_visible):
return None
### CHECK METEOR VISIBILITY WITH THE BRIGTHER (DETECTION) LIMITING MAGNITUDE ###
### (in the CAMO widefield camera) ###
# Get indices of magnitudes above the brighter limiting magnitude
indices_visible_brighter = sim.simulation_results.abs_magnitude <= lim_mag_brightest
# If no points were visible, skip this solution
if not np.any(indices_visible_brighter):
return None
# Compute the minimum time the meteor needs to be visible
min_time_visible = min_frames_visible/params.fps + len_delay
time_lim_mag_bright = sim.simulation_results.time_arr[indices_visible_brighter]
time_lim_mag_bright -= time_lim_mag_bright[0]
# Check if the minimum time is satisfied
if np.max(time_lim_mag_bright) < min_time_visible:
return None
### ###
# Get the first index after the magnitude reaches visibility in the wide field
index_first_visibility = np.argwhere(indices_visible_brighter)[0][0]
# Set all visibility indices before the first one visible in the wide field to False
indices_visible[:index_first_visibility] = False
# Select time, magnitude, height, and length above the visibility limit
time_visible = sim.simulation_results.time_arr[indices_visible]
mag_visible = sim.simulation_results.abs_magnitude[indices_visible]
ht_visible = sim.simulation_results.brightest_height_arr[indices_visible]
len_visible = sim.simulation_results.brightest_length_arr[indices_visible]
# Resample the time to the system FPS
mag_interpol = scipy.interpolate.CubicSpline(time_visible, mag_visible)
ht_interpol = scipy.interpolate.CubicSpline(time_visible, ht_visible)
len_interpol = scipy.interpolate.CubicSpline(time_visible, len_visible)
# Create a new time array according to the FPS
time_sampled = np.arange(np.min(time_visible), np.max(time_visible), 1.0/params.fps)
# Create new mag, height and length arrays at FPS frequency
mag_sampled = mag_interpol(time_sampled)
ht_sampled = ht_interpol(time_sampled)
len_sampled = len_interpol(time_sampled)
# Normalize time to zero
time_sampled -= time_sampled[0]
### SIMULATE CAMO tracking delay for length measurements ###
# Zero out all length measurements before the length delay (to simulate the delay of CAMO
# tracking)
len_sampled[time_sampled < len_delay] = 0
###
# Set all magnitudes below the brightest limiting magnitude to the faintest magnitude
mag_sampled[mag_sampled > lim_mag] = params.lim_mag_len_end_faintest
# Normalize the first length to zero
first_length_index = np.argwhere(time_sampled >= len_delay)[0][0]
len_sampled[first_length_index:] -= len_sampled[first_length_index]
# ### Plot simulated data
# fig, (ax1, ax2, ax3) = plt.subplots(nrows=3)
# ax1.plot(time_sampled, mag_sampled)
# ax1.invert_yaxis()
# ax1.set_ylabel("Magnitude")
# ax2.plot(time_sampled, len_sampled/1000)
# ax2.set_ylabel("Length (km)")
# ax3.plot(time_sampled, ht_sampled/1000)
# ax3.set_xlabel("Time (s)")
# ax3.set_ylabel("Height (km)")
# plt.subplots_adjust(hspace=0)
# plt.show()
# ### ###
# Check that there are any length measurements
if not np.any(len_sampled > 0):
return None
# If the simulation should only be checked that it's good, return the postprocess parameters used to
# generate the data
if check_only:
return postprocess_params
### ADD NOISE ###
# Add noise to magnitude data
mag_sampled[mag_sampled <= lim_mag] += np.random.normal(loc=0.0, scale=params.mag_noise, \
size=len(mag_sampled[mag_sampled <= lim_mag]))
# Add noise to length data
len_sampled[first_length_index:] += np.random.normal(loc=0.0, scale=params.len_noise, \
size=len(len_sampled[first_length_index:]))
### ###
# Construct input data vector with normalized values
input_data_normed = sim.getNormalizedInputs()
# Normalize simulated data
ht_normed, len_normed, mag_normed = sim.normalizeSimulations(params, ht_sampled, len_sampled, mag_sampled)
# Generate vector with simulated data
simulated_data_normed = np.vstack([padOrTruncate(ht_normed, params.data_length), \
padOrTruncate(len_normed, params.data_length), \
padOrTruncate(mag_normed, params.data_length)])
# Return input data and results
return params, input_data_normed, simulated_data_normed
def mod_u_matrix(prototype_density, avg_preictal_times):
raw_dens = np.sum(prototype_density, axis=1)
norm_times = avg_preictal_times / np.nanmax(avg_preictal_times)
u_matrix = SOMPY.visualization.umatrix.UMatrixView(
*SOM.codebook.mapsize, title='SOM Visualization',
text_size=10).build_u_matrix(SOM)
prep_H = np.argmax(prototype_density, axis=1)
prep_H = label_hue_arr[prep_H]
prep_H[prep_H == label_hue_arr[label_indices['Preictal']]] -= \
.15 * norm_times[prep_H == label_hue_arr[label_indices['Preictal']]]
H = np.reshape(prep_H, SOM.codebook.mapsize)
S_prep = 0.75 * np.ones_like(u_matrix.flatten())
S_prep[raw_dens == 0] = 0
S = np.reshape(S_prep, SOM.codebook.mapsize)
V = 0.85 * (1 - u_matrix / np.max(u_matrix))
image_mod_mat = hsv2rgb(np.stack((H, S, V), axis=2))
plt.subplot(1, 2, 1)
plt.imshow(image_mod_mat)
plt.title('Prototype classes and Neighborhood Distances')
proj = SOM.project_data(SOM.data_raw)
coord = SOM.bmu_ind_to_xy(proj)
mn = np.min(u_matrix.flatten())
# mx = np.max(u_matrix.flatten())
std = np.std(u_matrix.flatten())
md = np.median(u_matrix.flatten())
mx = md + 0.5 * std
plt.contour(u_matrix,
np.linspace(mn, mx, 15),
linewidths=0.7,
cmap=plt.cm.get_cmap('Blues'))
plt.scatter(coord[:, 1],
coord[:, 0],
s=2,
alpha=1.,
c='Gray',
marker='o',
cmap='jet',
linewidths=3,
edgecolor='Gray')
plt.axis('off')
preictal_hues = np.linspace(label_hues['Preictal'],
label_hues['Preictal'] - .15, 100)
prelim_colormap = np.squeeze(
hsv2rgb(
np.dstack((preictal_hues, np.full_like(preictal_hues, .75),
np.full_like(preictal_hues, .85)))))
cmap = mpl.colors.ListedColormap(prelim_colormap)
ax, _ = mpl.colorbar.make_axes(plt.gca(), shrink=0.75)
cbar = mpl.colorbar.ColorbarBase(
ax,
cmap=cmap,
norm=mpl.colors.Normalize(vmin=np.nanmin(avg_preictal_times),
vmax=np.nanmax(avg_preictal_times)))
plt.subplot(1, 2, 2)
plt.imshow(np.log(np.reshape(raw_dens, SOM.codebook.mapsize) + .05),
cmap='gray')
plt.scatter(coord[:, 1],
coord[:, 0],
s=2,
alpha=1.,
c='Gray',
marker='o',
cmap='jet',
linewidths=3,
edgecolor='Gray')
plt.title('Prototype Density Map')
plt.axis('off')
def __init__(self,
subcase_id,
header,
title,
location,
scalar,
mask_value=None,
nlabels=None,
labelsize=None,
ncolors=None,
colormap='jet',
data_format=None,
uname='GuiResult'):
"""
subcase_id : int
the flag that points to self.subcases for a message
header : str
the sidebar word
title : str
the legend title
location : str
node, centroid
scalar : (n,) ndarray
the data to make a contour plot with
data_format : str
the type of data result (e.g. '%i', '%.2f', '%.3f')
uname : str
some unique name for ...
"""
GuiResultCommon.__init__(self)
self.subcase_id = subcase_id
#assert self.subcase_id > 0, self.subcase_id
self.title = title
self.header = header
#self.scale = scale
self.location = location
assert location in ['node', 'centroid'], location
self.subcase_id = subcase_id
self.uname = uname
if scalar is None:
raise RuntimeError('title=%r scalar is None...' % self.title)
assert scalar.shape[0] == scalar.size, 'shape=%s size=%s' % (str(
scalar.shape), scalar.size)
self.scalar = scalar
#self.data_type = self.dxyz.dtype.str # '<c8', '<f4'
self.data_type = self.scalar.dtype.str # '<c8', '<f4'
self.is_real = True if self.data_type in REAL_TYPES else False
self.is_complex = not self.is_real
self.nlabels = nlabels
self.labelsize = labelsize
self.ncolors = ncolors
self.colormap = colormap
#print('title=%r data_type=%r' % (self.title, self.data_type))
if self.data_type in INT_TYPES:
self.data_format = '%i'
elif data_format is None:
self.data_format = '%.2f'
else:
self.data_format = data_format
self.title_default = self.title
self.header_default = self.header
self.data_format_default = self.data_format
self.min_default = np.nanmin(self.scalar)
self.max_default = np.nanmax(self.scalar)
if self.data_type in INT_TYPES:
# turns out you can't have a NaN/inf with an integer array
# we need to recast it
if mask_value is not None:
inan_short = np.where(self.scalar == mask_value)[0]
if len(inan_short):
# overly complicated way to allow us to use ~inan to invert the array
inan = np.in1d(np.arange(len(self.scalar)), inan_short)
inan_remaining = self.scalar[~inan]
self.scalar = np.asarray(self.scalar, 'f')
self.data_type = self.scalar.dtype.str
self.data_format = '%.0f'
self.scalar[inan] = np.nan
self.min_default = inan_remaining.min()
self.max_default = inan_remaining.max()
else:
# handling VTK NaN oddinty
# filtering the inf values and replacing them with NaN
# 1.#R = inf
# 1.#J = nan
ifinite = np.isfinite(self.scalar)
if not np.all(ifinite):
self.scalar[~ifinite] = np.nan
try:
self.min_default = self.scalar[ifinite].min()
except ValueError:
print(self.title)
print(self.scalar)
raise
self.max_default = self.scalar[ifinite].max()
self.min_value = self.min_default
self.max_value = self.max_default
# Print some summary stats:
print(f"Count of tracks: {len(df_with_crossings)}")
for gidx,name in enumerate(gates['name']):
if ('levee' in name) or (name=='point'): continue
t_first=df_with_crossings[name+"_first"]
t_last=df_with_crossings[name+"_last"]
n_crossed=t_first.count()
n_multiple=(t_first<t_last).sum()
n_uncross=(df_with_crossings[name+"_uncrossed"]>0).sum()
print(f" {name:12}: {n_crossed:3} tracks, {n_multiple:2} recross, {n_uncross} uncross")
##
track_subs=[]
for idx,row in df_with_crossings.iterrows():
t_min=np.nanmax( [row['top_of_array_last'],
row['track']['tnum'].min()] )
# Ah - sj_lower_first is negative??
t_max=np.nanmin( [row['sj_lower_first'],
row['hor_lower_first'],
row['track']['tnum'].max()])
sel=( (t_min<=row['track']['tnum'].values)
& (row['track']['tnum'].values<=t_max ) )
track_sub=row['track'].iloc[sel,:].copy()
track_subs.append(track_sub)
df_trim_crossings=df_with_crossings.copy()
df_trim_crossings['track']=track_subs
valid=np.array( [len(t)>0 for t in track_subs] )
df_trim_crossings=df_trim_crossings.iloc[valid,:].copy()
# 134 to 134 tracks
