def lowess(x, y, f=2. / 3., iter=3):
"""lowess(x, y, f=2./3., iter=3) -> yest
Lowess smoother: Robust locally weighted regression.
The lowess function fits a nonparametric regression curve to a scatterplot.
The arrays x and y contain an equal number of elements; each pair
(x[i], y[i]) defines a data point in the scatterplot. The function returns
the estimated (smooth) values of y.
The smoothing span is given by f. A larger value for f will result in a
smoother curve. The number of robustifying iterations is given by iter. The
function will run faster with a smaller number of iterations.
x and y should be numpy float arrays of equal length. The return value is
also a numpy float array of that length.
e.g.
>>> import numpy
>>> x = numpy.array([4, 4, 7, 7, 8, 9, 10, 10, 10, 11, 11, 12, 12, 12,
... 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 16, 16,
... 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 20, 20, 20, 20,
... 20, 22, 23, 24, 24, 24, 24, 25], numpy.float)
>>> y = numpy.array([2, 10, 4, 22, 16, 10, 18, 26, 34, 17, 28, 14, 20, 24,
... 28, 26, 34, 34, 46, 26, 36, 60, 80, 20, 26, 54, 32, 40,
... 32, 40, 50, 42, 56, 76, 84, 36, 46, 68, 32, 48, 52, 56,
... 64, 66, 54, 70, 92, 93, 120, 85], numpy.float)
>>> result = lowess(x, y)
>>> len(result)
50
>>> print "[%0.2f, ..., %0.2f]" % (result[0], result[-1])
[4.85, ..., 84.98]
"""
n = len(x)
r = int(numpy.ceil(f * n))
h = [numpy.sort(abs(x - x[i]))[r] for i in range(n)]
w = numpy.clip(abs(([x] - numpy.transpose([x])) / h), 0.0, 1.0)
w = 1 - w * w * w
w = w * w * w
yest = numpy.zeros(n)
delta = numpy.ones(n)
for iteration in range(iter):
for i in xrange(n):
weights = delta * w[:, i]
weights_mul_x = weights * x
b1 = numpy.dot(weights, y)
b2 = numpy.dot(weights_mul_x, y)
A11 = sum(weights)
A12 = sum(weights_mul_x)
A21 = A12
A22 = numpy.dot(weights_mul_x, x)
determinant = A11 * A22 - A12 * A21
beta1 = (A22 * b1 - A12 * b2) / determinant
beta2 = (A11 * b2 - A21 * b1) / determinant
yest[i] = beta1 + beta2 * x[i]
residuals = y - yest
s = median(abs(residuals))
delta[:] = numpy.clip(residuals / (6 * s), -1, 1)
delta[:] = 1 - delta * delta
delta[:] = delta * delta
return yest
def mouseMoveEvent(self, ev):
if self.lastMousePos is None:
self.lastMousePos = Point(ev.pos())
delta = Point(ev.pos() - self.lastMousePos)
self.lastMousePos = Point(ev.pos())
QtGui.QGraphicsView.mouseMoveEvent(self, ev)
if not self.mouseEnabled:
return
self.sigSceneMouseMoved.emit(self.mapToScene(ev.pos()))
if self.clickAccepted: ## Ignore event if an item in the scene has already claimed it.
return
if ev.buttons() == QtCore.Qt.RightButton:
delta = Point(np.clip(delta[0], -50, 50), np.clip(-delta[1], -50, 50))
scale = 1.01 ** delta
self.scale(scale[0], scale[1], center=self.mapToScene(self.mousePressPos))
self.sigRangeChanged.emit(self, self.range)
elif ev.buttons() in [QtCore.Qt.MidButton, QtCore.Qt.LeftButton]: ## Allow panning by left or mid button.
px = self.pixelSize()
tr = -delta * px
self.translate(tr[0], tr[1])
self.sigRangeChanged.emit(self, self.range)
def clamp(f, t, l):
if(f != 0 or t != 0):
# only when something is set
if(f != 0 and t == 0):
# from is set
f, _ = tuple(numpy.clip([f, t], 0, l))
elif(f == 0 and t != 0):
# to is set
_, t = tuple(numpy.clip([f, t], 0, l))
elif(f != 0 and t != 0):
# both are set
f, t = tuple(numpy.clip([f, t], 0, l))
else:
print("wtf?")
pass
if(f > t):
# swap
a = f
f = t
t = a
if(f == t and f != 0 and t != 0):
f = f - 1
return f, t
def scale(self, factor_x, factor_y=None):
"""
Expand or contract the bounding box or contour around its center by a given factor
:param factor_x: The multiplicative scale parameter in the x direction
:type factor_x: float
:param factor_y: The multiplicative scale parameter in the y direction
:type factor_y: float
.. note::
if factor_y parameter is omitted, then the factor_x is used in both directions
.. note::
The scaling is done with respect to the contour's centroid as computed by the get_centroid
methods.
:Example:
::
shape = (100, 100, 3)
image = np.zeros(shape, dtype=np.uint8)
d = bounding_region(shape, contour=np.array([[[10, 20]],
[[25, 15]],
[[80, 65]],
[[60, 70]],
[[20, 75]],
[[5, 50]]]))
d.draw_contour(image, color=(0, 255, 0))
# Scale to half the size
d.scale(0.5)
d.draw_contour(image, color=(255, 255, 0))
d.draw_box(image)
cv2.imshow("Two contours", image)
cv2.waitKey(0)
"""
if self._empty:
return
if factor_y is None:
factor_y = factor_x
if self.image_shape is None:
raise Exception("Image shape is nescessary to compute the relative coordinates")
if self.box_is_primary:
shift_x = self.box[2] * (1.-factor_x) * 0.5
shift_y = self.box[3] * (1.-factor_y) * 0.5
self.box = np.array([np.maximum(self.box[0]+shift_x, 0),
np.maximum(self.box[1]+shift_y, 0),
np.minimum(self.box[2]*factor_x, self.image_shape[1]),
np.minimum(self.box[3]*factor_y, self.image_shape[0])]).astype(np.int32)
self._contour_from_box()
self._update_internals()
else:
(cx, cy) = self.get_centroid_pixels()
new_contour = np.zeros_like(self.contour, dtype=np.int32)
for i in xrange(self.contour.shape[0]):
new_contour[i][0][0] = np.clip(int(cx + (self.contour[i][0][0]-cx)*factor_x), a_min=0, a_max=self.image_shape[1])
new_contour[i][0][1] = np.clip(int(cy + (self.contour[i][0][1]-cy)*factor_y), a_min=0, a_max=self.image_shape[0])
self.contour = new_contour
self._box_from_contour()
self._update_internals()
def _transposeAndClip(self, contours):
cnts = list()
for i, cnt in enumerate(contours):
x = np.clip(cnt[0], 0, self.gsize)
y = np.clip(cnt[1], 0, self.gsize)
cnts.append(zip(x, y))
return cnts
def rmsprop_one_step(self, param_name, index, grad_args, decay = 0.9, momentum = 0, learning_rate_adapt = 0.05,
learning_rate_min = 1e-6, learning_rate_max = 10):
# RMSPROP: Tieleman, T. and Hinton, G. (2012), Lecture 6.5 - rmsprop, COURSERA: Neural Networks for Machine Learning
# Implementation based on https://github.com/BRML/climin/blob/master/climin/rmsprop.py
# We use Nesterov momentum: first, we make a step according to the momentum and then we calculate the gradient.
step1 = self.param_updates[param_name] * momentum
self.wrt[param_name].set_value(self.wrt[param_name].get_value()+step1)
grad = self.get_grad(*grad_args)
self.moving_mean_squared[param_name] = (decay * self.moving_mean_squared[param_name] + (1 - decay) * grad ** 2)
step2 = self.learning_rates[param_name] * grad / (self.moving_mean_squared[param_name] + 1e-8)**0.5
# DEBUG
if param_name == 'lhyp' or 'ls':
step2 = np.clip(step2, -0.1, 0.1)
self.wrt[param_name].set_value(self.wrt[param_name].get_value()+step2)
#self.params[param_name] += step2
step = step1 + step2
# Step rate adaption. If the current step and the momentum agree, we slightly increase the step rate for that dimension.
if learning_rate_adapt:
# This code might look weird, but it makes it work with both numpy and gnumpy.
step_non_negative = step > 0
step_before_non_negative = self.param_updates[param_name] > 0
agree = (step_non_negative == step_before_non_negative) * 1.#０か１が出る
adapt = 1 + agree * learning_rate_adapt * 2 - learning_rate_adapt
self.learning_rates[param_name] *= adapt
self.learning_rates[param_name] = np.clip(self.learning_rates[param_name], learning_rate_min, learning_rate_max)
self.param_updates[param_name] = step
def _compute_normalized_data(self, data_array):
"""
Apply `data_func`, then linearly scale to the unit interval, and
then apply `unit_func`.
"""
# FIXME: Deal with nans?
if self._dirty:
self._recalculate()
if self.data_func is not None:
data_array = self.data_func(data_array)
low, high = self.transformed_bounds
else:
low, high = self.range.low, self.range.high
range_diff = high - low
# Linearly transform the values to the unit interval.
if range_diff == 0.0 or isinf(range_diff):
# Handle null range, or infinite range (which can happen during
# initialization before range is connected to a data source).
norm_data = 0.5*ones_like(data_array)
else:
norm_data = empty(data_array.shape, dtype='float32')
norm_data[:] = data_array
norm_data -= low
norm_data /= range_diff
clip(norm_data, 0.0, 1.0, norm_data)
if self.unit_func is not None:
norm_data = self.unit_func(norm_data)
return norm_data
def lowess(x, y, f=2./3., iter=3):
"""lowess(x, y, f=2./3., iter=3) -> yest
Lowess smoother: Robust locally weighted regression.
The lowess function fits a nonparametric regression curve to a scatterplot.
The arrays x and y contain an equal number of elements; each pair
(x[i], y[i]) defines a data point in the scatterplot. The function returns
the estimated (smooth) values of y.
The smoothing span is given by f. A larger value for f will result in a
smoother curve. The number of robustifying iterations is given by iter. The
function will run faster with a smaller number of iterations."""
n = len(x)
r = int(ceil(f*n))
h = [np.sort(np.abs(x - x[i]))[r] for i in range(n)]
w = np.clip(np.abs((x[:,None] - x[None,:]) / h), 0.0, 1.0)
w = (1 - w**3)**3
yest = np.zeros(n)
delta = np.ones(n)
for iteration in range(iter):
for i in range(n):
weights = delta * w[:,i]
b = np.array([np.sum(weights*y), np.sum(weights*y*x)])
A = np.array([[np.sum(weights), np.sum(weights*x)],
[np.sum(weights*x), np.sum(weights*x*x)]])
beta = linalg.solve(A, b)
yest[i] = beta[0] + beta[1]*x[i]
residuals = y - yest
s = np.median(np.abs(residuals))
delta = np.clip(residuals / (6.0 * s), -1, 1)
delta = (1 - delta**2)**2
return yest
def numpy_run(self):
"""Forward propagation from batch on CPU only.
"""
super(All2AllStrictRELU, self).numpy_run()
self.output.map_write()
mem = self.output.mem
numpy.clip(mem, 0.0, 1.0e30, mem)
def f(t, y, f_return):
y_com = y[0]
v_com = y[1]
theta = y[2]
omega = y[3]
y_tire_front = y[4]
v_tire_front = y[5]
y_tire_rear = y[6]
v_tire_rear = y[7]
delta_fs = (y_com + Lf*math.sin(theta)) - y_tire_front - fl_fs
delta_rs = (y_com - Lr*math.sin(theta))- y_tire_rear - fl_rs
# The clipping cuts off interaction between the road and the tire if the tire goes airborne.
delta_ft = np.clip(np.array([y_tire_front - y_road(v_0*t + L) - r_F]), a_min=None, a_max=0)[0]
delta_rt = np.clip(np.array([y_tire_rear - y_road(v_0*t) - r_R]), a_min=None, a_max=0)[0]
v_fs = Lf*omega + v_com - v_tire_front
v_rs = -Lr*omega + v_com - v_tire_rear
if delta_ft == 0:
v_ft = 0
else:
v_ft = v_tire_front - v_road(v_0*t+L)
if delta_rt == 0:
v_rt = 0
else:
v_rt = v_tire_rear - v_road(v_0*t)
# The clipping and if-cases basically make it so that if the car tire is airborn, the tire-spring and tire-damper produce no forces on the tire.
f_return[0] = v_com
f_return[1] = -g - (k_fs/m_c)*delta_fs - (b_fs/m_c)*v_fs - (k_rs/m_c)*delta_rs - (b_rs/m_c)*v_rs
f_return[2] = omega
f_return[3] = (-Lf*math.cos(theta)*(k_fs*delta_fs + b_fs*v_fs) + Lr*math.cos(theta)*(k_rs*delta_rs + b_rs*v_rs))/Ic
f_return[4] = v_tire_front
f_return[5] = -g - (k_ft/m_ft)*delta_ft - (b_ft/m_ft)*v_ft + (k_fs/m_ft)*delta_fs + (b_fs/m_ft)*v_fs
f_return[6] = v_tire_rear
f_return[7] = -g - (k_rt/m_rt)*delta_rt - (b_rt/m_rt)*v_rt + (k_rs/m_rt)*delta_rs + (b_rs/m_rt)*v_rs
def isotonic_regression(y, sample_weight=None, y_min=None, y_max=None,
increasing=True):
"""Solve the isotonic regression model::
min sum w[i] (y[i] - y_[i]) ** 2
subject to y_min = y_[1] <= y_[2] ... <= y_[n] = y_max
where:
- y[i] are inputs (real numbers)
- y_[i] are fitted
- w[i] are optional strictly positive weights (default to 1.0)
Read more in the :ref:`User Guide <isotonic>`.
Parameters
----------
y : iterable of floating-point values
The data.
sample_weight : iterable of floating-point values, optional, default: None
Weights on each point of the regression.
If None, weight is set to 1 (equal weights).
y_min : optional, default: None
If not None, set the lowest value of the fit to y_min.
y_max : optional, default: None
If not None, set the highest value of the fit to y_max.
increasing : boolean, optional, default: True
Whether to compute ``y_`` is increasing (if set to True) or decreasing
(if set to False)
Returns
-------
y_ : list of floating-point values
Isotonic fit of y.
References
----------
"Active set algorithms for isotonic regression; A unifying framework"
by Michael J. Best and Nilotpal Chakravarti, section 3.
"""
order = np.s_[:] if increasing else np.s_[::-1]
y = np.array(y[order], dtype=np.float64)
if sample_weight is None:
sample_weight = np.ones(len(y), dtype=np.float64)
else:
sample_weight = np.array(sample_weight[order], dtype=np.float64)
_inplace_contiguous_isotonic_regression(y, sample_weight)
if y_min is not None or y_max is not None:
# Older versions of np.clip don't accept None as a bound, so use np.inf
if y_min is None:
y_min = -np.inf
if y_max is None:
y_max = np.inf
np.clip(y, y_min, y_max, y)
return y[order]
def process(self, image, out=None):
# 0.25 is the default value used in Ng's paper
alpha = self.specs.get('alpha', 0.25)
# check if we would like to do two-side thresholding. Default yes.
if self.specs.get('twoside', True):
# concatenate, and make sure the output is C_CONTIGUOUS
# for the temporary product, we check if we can utilize the
# buffer to save allocation time
product = mathutil.dot_image(image, self.dictionary.T)
imshape = product.shape[:-1]
N = product.shape[-1]
product.resize((np.prod(imshape), N))
if out is None:
out = np.empty((np.prod(imshape), N*2))
else:
out.resize((np.prod(imshape), N*2))
out[:,:N] = product
out[:,N:] = -product
out.resize(imshape + (N*2,))
elif self.specs['twoside'] == 'abs':
out = mathutil.dot_image(image, self.dictionary.T, out=out)
np.abs(out, out=out)
else:
out = mathutil.dot_image(image, self.dictionary.T, out=out)
# do threshold
out -= alpha
np.clip(out, 0., np.inf, out=out)
return out
def lossFun(inputs, targets, hprev):
"""
inputs,targets are both list of integers.
hprev is Hx1 array of initial hidden state
returns the loss, gradients on model parameters, and last hidden state
"""
xs, hs, ys, ps = {}, {}, {}, {}
hs[-1] = np.copy(hprev)
loss = 0
# forward pass
for t in xrange(len(inputs)):
xs[t] = np.zeros((vocab_size,1)) # encode in 1-of-k representation
xs[t][inputs[t]] = 1
hs[t] = np.tanh(np.dot(Wxh, xs[t]) + np.dot(Whh, hs[t-1]) + bh) # hidden state
ys[t] = np.dot(Why, hs[t]) + by # unnormalized log probabilities for next chars
ps[t] = np.exp(ys[t]) / np.sum(np.exp(ys[t])) # probabilities for next chars
loss += -np.log(ps[t][targets[t],0]) # softmax (cross-entropy loss)
# backward pass: compute gradients going backwards
dWxh, dWhh, dWhy = np.zeros_like(Wxh), np.zeros_like(Whh), np.zeros_like(Why)
dbh, dby = np.zeros_like(bh), np.zeros_like(by)
dhnext = np.zeros_like(hs[0])
for t in reversed(xrange(len(inputs))):
dy = np.copy(ps[t])
dy[targets[t]] -= 1 # backprop into y
dWhy += np.dot(dy, hs[t].T)
dby += dy
dh = np.dot(Why.T, dy) + dhnext # backprop into h
dhraw = (1 - hs[t] * hs[t]) * dh # backprop through tanh nonlinearity
dbh += dhraw
dWxh += np.dot(dhraw, xs[t].T)
dWhh += np.dot(dhraw, hs[t-1].T)
dhnext = np.dot(Whh.T, dhraw)
for dparam in [dWxh, dWhh, dWhy, dbh, dby]:
np.clip(dparam, -5, 5, out=dparam) # clip to mitigate exploding gradients
return loss, dWxh, dWhh, dWhy, dbh, dby, hs[len(inputs)-1]
def switchShape(tractor, shape, var):
# This p0/p1/changed is a hack to know which elements of 'var' to change.
p0 = np.array(tractor.getParams())
softe = shape.softe
# Actually switch the parameter space
newshape = EllipseE.fromEllipseESoft(shape, maxe=0.99)
shape.setParams([-np.inf] * shape.numberOfParams())
p1 = np.array(tractor.getParams())
# ASSUME that changing to EllipseE parameterization actually
# changes the values.
# Could do something like: gal.shape.setParams([-np.inf] * 3) to be sure.
changed = np.flatnonzero(p0 != p1)
print 'shape param indices:', changed
assert(len(changed) == 3)
# ASSUME ordering re, e1, e2
# We changed from log(re) to re.
var[changed[0]] *= newshape.re**2
# We changed from soft-e to e.
# If soft-e is huge, var(soft-e) is huge; e is ~1 and var(e) gets hugely shrunk.
efac = np.exp(-2. * softe)
var[changed[1]] *= efac
var[changed[2]] *= efac
# Impose a minimum and maximum variance on e
minv, maxv = 1e-6, 1.
var[changed[1]] = np.clip(var[changed[1]], minv, maxv)
var[changed[2]] = np.clip(var[changed[2]], minv, maxv)
return newshape
def adjust_to_be_viewed_with(self, X, orig, per_example = False):
# if the scale is set based on the data, display X oring the scale determined
# by orig
# assumes no preprocessing. need to make preprocessors mark the new ranges
rval = X.copy()
#patch old pkl files
if not hasattr(self,'center'):
self.center = False
if not hasattr(self,'rescale'):
self.rescale = False
if not hasattr(self,'gcn'):
self.gcn = False
if self.gcn is not None:
rval = X.copy()
if per_example:
for i in xrange(rval.shape[0]):
rval[i,:] /= np.abs(orig[i,:]).max()
else:
rval /= np.abs(orig).max()
rval = np.clip(rval, -1., 1.)
return rval
if not self.center:
rval -= 127.5
if not self.rescale:
rval /= 127.5
rval = np.clip(rval,-1.,1.)
return rval
def _to_raw(self, data1, data2):
from matplotlib import pyplot as plt
from matplotlib.colors import Normalize
cmapdir = options.config.get("webgl", "colormaps")
cmap = plt.imread(os.path.join(cmapdir, "%s.png"%self.cmap))
norm1 = Normalize(self.vmin, self.vmax)
norm2 = Normalize(self.vmin2, self.vmax2)
d1 = np.clip(norm1(data1), 0, 1)
d2 = np.clip(1 - norm2(data2), 0, 1)
dim1 = np.round(d1 * (cmap.shape[1]-1))
# Nans in data seemed to cause weird interaction with conversion to uint32
dim1 = np.nan_to_num(dim1).astype(np.uint32)
dim2 = np.round(d2 * (cmap.shape[0]-1))
dim2 = np.nan_to_num(dim2).astype(np.uint32)
colored = cmap[dim2.ravel(), dim1.ravel()]
r, g, b, a = colored.T
r.shape = dim1.shape
g.shape = dim1.shape
b.shape = dim1.shape
a.shape = dim1.shape
# Preserve nan values as alpha = 0
aidx = np.logical_or(np.isnan(data1),np.isnan(data2))
a[aidx] = 0
# Code from master, to handle alpha input, prob better here but not tested.
# # Possibly move this above setting nans to alpha = 0;
# # Possibly multiply specified alpha by alpha in colormap??
# if 'alpha' in self.attrs:
# # Over-write alpha from colormap / nans with alpha arg if provided.
# # Question: Might it be important tokeep alpha as an attr?
# a = self.attrs.pop('alpha')
return r, g, b, a
def updateParticles(self):
# Update positions with velocity
self.particles[:, 0:2] += self.particles[:, 4:6]
#np.clip(self.particles[:,0], 0, self.bounds[0], self.particles[:,0])
#np.clip(self.particles[:,1], 0, self.bounds[1], self.particles[:,1])
# Add noise to w,h
if self.SIGMA_size > 0.0001:
self.particles[:, 2:4] += random.normal(0, self.SIGMA_size, (self.particles.shape[0], 2))
#np.clip(self.particles[:,2], 1, self.bounds[0], self.particles[:,2])
#np.clip(self.particles[:,3], 1, self.bounds[1], self.particles[:,3])
# Add noise to velocities and clip
self.particles[:, 4:6] += random.normal(
0, self.SIGMA_velocity, (self.particles.shape[0], 2))
#np.clip(self.particles[:,4:6], -MAX_velocity,MAX_velocity, self.particles[:,4:6])
lb = [0, 0, 1, 1, -MAX_velocity, -MAX_velocity, 0]
ub = [self.bounds[1],
self.bounds[0],
self.bounds[1],
self.bounds[0],
MAX_velocity,
MAX_velocity,
1]
np.clip(self.particles, lb, ub, self.particles)
if np.max(self.particles[:, 0]) > self.bounds[1]:
print "Not clipped"
self.iterations += 1
def hls_to_rgb_perceptual(arr, out=None):
# uncorrected/spectral RGB values don't produce a nice color space
# this function attempts to produce something that has:
# * even brightness across hues
# * a hue curve closer to human concepts of rainbows
# use lookup table based on hue
hues = arr['hue']
lookup_index = ((hues - hues.astype(np.int)) * lookup_entries).astype(np.int)
out = hue_lookup.color_cache[lookup_index]
outview = out.view(np.float64).reshape(out.shape + (-1,))
# adjust L and S ...
luminances = np.clip(arr['light'], 0, 1).reshape((3600,1))
shades = np.clip(luminances * 2, 0, 1)
outview *= shades
pastels = np.clip(luminances * 2 - 1, 0, 1)
outview += pastels
grays = np.tile(np.clip(arr['light'], 0, 1),(3,1)).T
saturations = np.clip(arr['sat'], 0, 1).reshape((3600,1))
final = saturations*outview + (1-saturations)*grays
return final.view(dtypes.rgb_color).reshape(arr.shape)
def shapeShift(arr,newShape,offset=None,fillValue=0):
'''Create a new array with a specified shape and element value
and paste another array into it with an optional offset.
In 2D image processing, this like changing the canvas size and
then moving the image in x and y.
In the simple case of expanding the shape of an array, this is
equivalent to the following standard procedure:
newArray = zeros(shape)
newArray[:arr.shape[0],:arr.shape[1],...] = arr
However, shapeShift is more flexible because it can safely
clip for any shape and any offset (but using it just for cropping
an array is more efficiently done with slicing).
A more accurate name for this might be "copyDataToArrayOfNewSize", but
"shapeShift" is much easier to remember (and cooler).
'''
oldArr = np.asanyarray(arr)
newArr = np.zeros(newShape,dtype=oldArr.dtype)+fillValue
oldShape,newShape = np.array(oldArr.shape), np.array(newArr.shape)
offset = ( 0*oldShape if offset==None else np.array(offset) )
assert len(oldShape)==len(newShape)==len(offset)
oldStartEnd = np.transpose([ np.clip(i-offset,0,oldShape) for i in [0,newShape] ])
newStartEnd = np.transpose([ np.clip(i+offset,0,newShape) for i in [0,oldShape] ])
oldSlice = [ slice(start,end) for start,end in oldStartEnd ]
newSlice = [ slice(start,end) for start,end in newStartEnd ]
newArr[newSlice] = oldArr[oldSlice]
return newArr
def test_ambient_densities_2():
r = np.linspace(0., 10., 10)
t = [0., np.pi]
p = [0., 2 * np.pi]
g = SphericalPolarGrid(r, t, p)
# Set up envelope
p1 = PowerLawEnvelope()
p1.power = -2
p1.r_0 = 1.
p1.rho_0 = 10.
p1.rmin = 0.1
p1.rmax = 10.
a = AmbientMedium()
a.rho = 2.
a.rmin = 0.1
a.rmax = 10.
a.subtract = [p1]
expected = 10. * g.r ** -2
assert_array_almost_equal_nulp(p1.density(g)[0, 0, :], expected, 10)
expected = np.clip(2 - 10. * g.r ** -2, 0., np.inf)
assert_array_almost_equal_nulp(a.density(g)[0, 0, :], expected, 10)
expected = np.clip(10. * g.r ** -2, 2., np.inf)
assert_array_almost_equal_nulp((a.density(g) + p1.density(g))[0, 0, :],
expected, 10)
def get_counts_by_location(self, bamfile, chromosome, start, end):
if self.convert_roman:
chromosome = ROMAN_MAP[chromosome]
cmd = get_samtools_view_command(bamfile, chromosome,
start, end)
status, output = commands.getstatusoutput(cmd)
if status != 0:
print('Error with samtools : %s' % output)
array = np.zeros(end - start)
for (s, e, flag) in samtools_reads_iter(output, start, end):
sign = 1
if self.pos_neg:
if (flag & 0x10):
sign = -1
else:
sign = 1
s -= start
e -= start
s = np.clip(s, 0, array.shape[0] - 1)
e = np.clip(e, 0, array.shape[0] - 1)
array[s:e] += sign
return array
def test_nmf_negative_beta_loss():
# Test that an error is raised if beta_loss < 0 and X contains zeros.
# Test that the output has not NaN values when the input contains zeros.
n_samples = 6
n_features = 5
n_components = 3
rng = np.random.mtrand.RandomState(42)
X = rng.randn(n_samples, n_features)
np.clip(X, 0, None, out=X)
X_csr = sp.csr_matrix(X)
def _assert_nmf_no_nan(X, beta_loss):
W, H, _ = non_negative_factorization(
X, init='random', n_components=n_components, solver='mu',
beta_loss=beta_loss, random_state=0, max_iter=1000)
assert not np.any(np.isnan(W))
assert not np.any(np.isnan(H))
msg = "When beta_loss <= 0 and X contains zeros, the solver may diverge."
for beta_loss in (-0.6, 0.):
assert_raise_message(ValueError, msg, _assert_nmf_no_nan, X, beta_loss)
_assert_nmf_no_nan(X + 1e-9, beta_loss)
for beta_loss in (0.2, 1., 1.2, 2., 2.5):
_assert_nmf_no_nan(X, beta_loss)
_assert_nmf_no_nan(X_csr, beta_loss)
def K(self, X, X2=None,alpha=None,variance=None):
"""
Computes the covariance matrix cov(X[i,:],X2[j,:]).
Args:
X: Matrix where each row is a point.
X2: Matrix where each row is a point.
alpha: It's the scaled alpha.
Variance: Sigma hyperparameter.
"""
if alpha is None:
alpha=self.alpha
if variance is None:
variance=self.variance
if X2 is None:
X=X*alpha/self.scaleAlpha
Xsq=np.sum(np.square(X), 1)
r=-2.*np.dot(X, X.T) + (Xsq[:, None] + Xsq[None, :])
r = np.clip(r, 0, np.inf)
return variance*np.exp(-0.5*r)
else:
X=X*alpha/self.scaleAlpha
X2=X2*alpha/self.scaleAlpha
r=-2.*np.dot(X, X2.T) + (np.sum(np.square(X), 1)[:, None] + np.sum(np.square(X2), 1)[None, :])
r = np.clip(r, 0, np.inf)
return variance*np.exp(-0.5*r)
def _clipToSafeRange(min_, max_, isLog):
# Clip range if needed
minLimit = FLOAT32_MINPOS if isLog else FLOAT32_SAFE_MIN
min_ = numpy.clip(min_, minLimit, FLOAT32_SAFE_MAX)
max_ = numpy.clip(max_, minLimit, FLOAT32_SAFE_MAX)
assert min_ < max_
return min_, max_
def combine_images(imgs, alphas):
""" Combine multiple rgb images in one rgb image """
image_f = numpy.zeros(imgs[0].shape, dtype='float')
for i in range(0, len(imgs)):
image_f += alphas[i] * imgs[i]
numpy.clip(image_f, 0., 255., image_f)
return numpy.array(image_f, dtype='uint8')
def interpgrid(a, xi, yi):
"""Fast 2D, linear interpolation on an integer grid"""
Ny, Nx = np.shape(a)
if isinstance(xi, np.ndarray):
x = xi.astype(np.int)
y = yi.astype(np.int)
# Check that xn, yn don't exceed max index
xn = np.clip(x + 1, 0, Nx - 1)
yn = np.clip(y + 1, 0, Ny - 1)
else:
x = np.int(xi)
y = np.int(yi)
# conditional is faster than clipping for integers
if x == (Nx - 2): xn = x
else: xn = x + 1
if y == (Ny - 2): yn = y
else: yn = y + 1
a00 = a[y, x]
a01 = a[y, xn]
a10 = a[yn, x]
a11 = a[yn, xn]
xt = xi - x
yt = yi - y
a0 = a00 * (1 - xt) + a01 * xt
a1 = a10 * (1 - xt) + a11 * xt
ai = a0 * (1 - yt) + a1 * yt
if not isinstance(xi, np.ndarray):
if np.ma.is_masked(ai):
raise TerminateTrajectory
return ai
def _compute_disk_overlap(d, r1, r2):
"""
Compute surface overlap between two disks of radii ``r1`` and ``r2``,
with centers separated by a distance ``d``.
Parameters
----------
d : float
Distance between centers.
r1 : float
Radius of the first disk.
r2 : float
Radius of the second disk.
Returns
-------
vol: float
Volume of the overlap between the two disks.
"""
ratio1 = (d ** 2 + r1 ** 2 - r2 ** 2) / (2 * d * r1)
ratio1 = np.clip(ratio1, -1, 1)
acos1 = math.acos(ratio1)
ratio2 = (d ** 2 + r2 ** 2 - r1 ** 2) / (2 * d * r2)
ratio2 = np.clip(ratio2, -1, 1)
acos2 = math.acos(ratio2)
a = -d + r2 + r1
b = d - r2 + r1
c = d + r2 - r1
d = d + r2 + r1
area = (r1 ** 2 * acos1 + r2 ** 2 * acos2 -
0.5 * sqrt(abs(a * b * c * d)))
return area / (math.pi * (min(r1, r2) ** 2))
def test_special_sparse_dot():
# Test the function that computes np.dot(W, H), only where X is non zero.
n_samples = 10
n_features = 5
n_components = 3
rng = np.random.mtrand.RandomState(42)
X = rng.randn(n_samples, n_features)
np.clip(X, 0, None, out=X)
X_csr = sp.csr_matrix(X)
W = np.abs(rng.randn(n_samples, n_components))
H = np.abs(rng.randn(n_components, n_features))
WH_safe = nmf._special_sparse_dot(W, H, X_csr)
WH = nmf._special_sparse_dot(W, H, X)
# test that both results have same values, in X_csr nonzero elements
ii, jj = X_csr.nonzero()
WH_safe_data = np.asarray(WH_safe[ii, jj]).ravel()
assert_array_almost_equal(WH_safe_data, WH[ii, jj], decimal=10)
# test that WH_safe and X_csr have the same sparse structure
assert_array_equal(WH_safe.indices, X_csr.indices)
assert_array_equal(WH_safe.indptr, X_csr.indptr)
assert_array_equal(WH_safe.shape, X_csr.shape)
def spectrum(data, attribute, roi, slc, zaxis):
xaxis = slc.index('x')
yaxis = slc.index('y')
ndim, nz = data.ndim, data.shape[zaxis]
l, r, b, t = roi.xmin, roi.xmax, roi.ymin, roi.ymax
shp = data.shape
# The 'or 0' is because Numpy in Python 3 cannot deal with 'None'
l, r = np.clip([l or 0, r or 0], 0, shp[xaxis])
b, t = np.clip([b or 0, t or 0], 0, shp[yaxis])
# extract sub-slice, without changing dimension
slc = [slice(s, s + 1)
if s not in ['x', 'y'] else slice(None)
for s in slc]
slc[xaxis] = slice(l, r)
slc[yaxis] = slice(b, t)
slc[zaxis] = slice(None)
x = Extractor.abcissa(data, zaxis)
data = data[attribute, tuple(slc)]
finite = np.isfinite(data)
assert data.ndim == ndim
for i in reversed(list(range(ndim))):
if i != zaxis:
data = np.nansum(data, axis=i)
finite = finite.sum(axis=i)
assert data.ndim == 1
assert data.size == nz
data = (1. * data / finite).ravel()
return x, data
def _raisePermanenceToThreshold(self, perm, mask):
"""
This method ensures that each column has enough connections to input bits
to allow it to become active. Since a column must have at least
'self._stimulusThreshold' overlaps in order to be considered during the
inhibition phase, columns without such minimal number of connections, even
if all the input bits they are connected to turn on, have no chance of
obtaining the minimum threshold. For such columns, the permanence values
are increased until the minimum number of connections are formed.
Parameters:
----------------------------
@param perm: An array of permanence values for a column. The array is
"dense", i.e. it contains an entry for each input bit, even
if the permanence value is 0.
@param mask: the indices of the columns whose permanences need to be
raised.
"""
if len(mask) < self._stimulusThreshold:
raise Exception("This is likely due to a " +
"value of stimulusThreshold that is too large relative " +
"to the input size. [len(mask) < self._stimulusThreshold]")
numpy.clip(perm, self._synPermMin, self._synPermMax, out=perm)
while True:
numConnected = numpy.nonzero(perm > self._synPermConnected)[0].size
if numConnected >= self._stimulusThreshold:
return
perm[mask] += self._synPermBelowStimulusInc
def inverse_normalize(X):
return np.clip((X + 1.0) * 0.5, 0., 1.)
layer_names.append(layer.name)
images_per_row = 16
for layer_name, layer_activation in zip(layer_names, activations):
# Number of features in the feature map
# 就是每一个卷积核的结果是一个 feature,所有 features 组合起来就是 feature map
n_features = layer_activation.shape[-1]
# The feature map has shape (1, size, size, n_features).
size = layer_activation.shape[1]
n_rows = n_features // images_per_row
# Tiles the activation channels in this matrix
display_grid = np.zeros((size * n_rows, images_per_row * size))
for row in range(n_rows):
for col in range(images_per_row):
channel_image = layer_activation[0, :, :,
row * images_per_row + col]
channel_image -= channel_image.mean()
channel_image /= channel_image.std()
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255).astype('uint8')
display_grid[row * size:(row + 1) * size,
col * size:(col + 1) * size] = channel_image
scale = 1. / size
plt.figure(figsize=(scale * display_grid.shape[1],
scale * display_grid.shape[0]))
plt.title(layer_name)
plt.grid(False)
plt.imshow(display_grid, aspect='auto', cmap='viridis')
plt.show()
def orbit(self, azim, elev):
"""Orbits the camera around the center position. *azim* and *elev* are given in degrees."""
self.opts['azimuth'] += azim
#self.opts['elevation'] += elev
self.opts['elevation'] = np.clip(self.opts['elevation'] + elev, -90, 90)
self.update()
def quantize_and_clip(self, val):
if np.all(val == 0):
return val.astype(self.dtype)
iinfo = np.iinfo(self.dtype)
return np.clip(np.round(np.power(2, self.q) * val/self.scale, 0), iinfo.min, iinfo.max).astype(self.dtype)
def hamming_Z(m, v, tau):
pd = hamming_distrib(m, v, tau)
popul = v**m
Z = np.sum(pd * popul * np.exp(-np.arange(m + 1) / tau))
return np.clip(Z, a_max=1e30, a_min=1)
def _clip_reward(reward):
return np.clip(reward, -1.0, 1.0)
def touint16(img):
coef = 2**16 - 1
return (np.clip(img, 0., 1.)*coef).astype(np.uint16)
def touint8(img):
coef = 2**8 - 1
return (np.clip(img, 0., 1.)*coef).astype(np.uint8)
def main(source_img_root='./data', target_img_root='./data', source_name='image_2', target_name='image_1',
source_keypoint_path='', target_keypoint_path='', output_root='./output', target_folder=''):
if not os.path.exists(output_root):
os.mkdir(output_root)
source_fn = os.path.join(source_img_root, source_name)
target_fn = os.path.join(target_img_root, target_name)
target_seg_fn = os.path.join('./segmentation/segmentation_model/gray_atr/' + target_folder,
'.'.join(target_name.split('.')[:-1]) + '.png')
print(target_seg_fn, target_folder, target_name)
# print(target_seg_fn)
# source_fn = './visualize_landmark/0.jpg'
# target_fn = './visualize_landmark/1.jpg'
source_img = cv2.imread(source_fn)
target_img = cv2.imread(target_fn)
target_seg = cv2.imread(target_seg_fn, 0)
target_seg = (target_seg == 4).astype(np.float64)
sh, sw, _ = source_img.shape
th, tw, _ = target_img.shape
w = max(sw, tw)
h = max(sh, th)
target_seg = np.pad(target_seg, ((0, h - th), (0, w - tw)), 'constant', constant_values=(0, 0))
target_seg = np.expand_dims(target_seg, axis=2)
source_img = np.pad(source_img, ((0, h - sh), (0, w - sw), (0, 0)), 'constant', constant_values=(255, 255))
target_img = np.pad(target_img, ((0, h - th), (0, w - tw), (0, 0)), 'constant', constant_values=(255, 255))
source_keypoint, target_keypoint, raw_source_keypoint, raw_target_keypoint = \
load_keypoints(w=w, h=h, source_name=source_name, target_name=target_name,
source_keypoint_path=source_keypoint_path, target_keypoint_path=target_keypoint_path)
raw_target_keypoint, target_keypoint = get_align_keypoint(raw_target_keypoint, is_source=False)
raw_source_keypoint, source_keypoint = get_align_keypoint(raw_source_keypoint, is_source=True)
visualize(target_keypoint, target_fn)
visualize(source_keypoint, source_fn)
target_keypoint = normalize(target_keypoint[:-2, :], w, h)
source_keypoint = normalize(source_keypoint[:-2, :], w, h)
left_down = raw_source_keypoint[13, :] / 5 + raw_source_keypoint[14, :] * 4 / 5
right_down = raw_source_keypoint[17, :] / 5 + raw_source_keypoint[16, :] * 4 / 5
raw_source_keypoint[14, :] = left_down
raw_source_keypoint[16, :] = right_down
convex_poly = raw_source_keypoint[[6, 11, 12, 13, 14, 16, 17, 18, 19, 24, 3], :].astype(int)
mask = np.zeros((h, w, 1)).astype(np.uint8)
cv2.fillPoly(mask, [convex_poly], 255)
mask = mask / 255
mask = morpho(mask, 0, False)
mask = mask[:, :, np.newaxis]
source_img = source_img * mask + 0 * (1 - mask)
_, grid = TPS(target_keypoint, source_keypoint, width=w, height=h,
calc_new_pos=True)
grid = torch.from_numpy(grid)
# 619 246
# tensor([0.2597, 0.6458], dtype=torch.float64)
source_img = torch.from_numpy(source_img.astype(np.float64)).unsqueeze(dim=0).permute(0, 3, 1, 2)
target_img = torch.from_numpy(target_img.astype(np.float64)).unsqueeze(dim=0).permute(0, 3, 1, 2)
# print(grid)
grid = grid.unsqueeze(dim=0) * 2 - 1.0
# print(grid.shape)
# print(grid)
warp_img = F.grid_sample(source_img, grid, mode='bilinear', padding_mode='border')
warp_img = warp_img.squeeze(dim=0).permute(1, 2, 0)
warp_img = warp_img.numpy().astype(np.uint8)
raw_target_keypoint = raw_target_keypoint.astype(int)
left_down = raw_target_keypoint[13, :] / 5 + raw_target_keypoint[14, :] * 4 / 5
right_down = raw_target_keypoint[17, :] / 5 + raw_target_keypoint[16, :] * 4 / 5
raw_target_keypoint[14, :] = left_down
raw_target_keypoint[16, :] = right_down
convex_poly = raw_target_keypoint[[6, 11, 12, 13, 14, 16, 17, 18, 19, 24, 3], :].astype(int)
mask = np.zeros((h, w, 1)).astype(np.uint8)
cv2.fillPoly(mask, [convex_poly], 255)
mask = mask / 255
mask = morpho(mask, 10, False)
mask = mask[:, :, np.newaxis]
warp_img = warp_img * mask + 0 * (1 - mask)
cv2.imwrite(os.path.join(output_root, source_name.split('.')[0] + '_' + target_name.split('.')[0] + '_warp.jpg'),
warp_img[:th, :tw, :])
warp_img = np.clip(warp_img * 0.85, 0, 255).astype(np.uint8) ## brigter
feather = cv2.blur(mask, (51, 51))
mask = np.clip(feather * 2 - 1, 0, 1)
mask = np.concatenate([mask[:, :, np.newaxis], mask[:, :, np.newaxis], mask[:, :, np.newaxis]], 2)
mask = np.square(mask)
mask = mask * target_seg
# print(mask.max(.))
# print(mask.min())
# cv2.imwrite(os.path.join(output_root, source_name + '_' + target_name + '_mask.jpg'), warp_img)
# cv2.waitKey(0)
target_img = target_img.squeeze(dim=0).permute(1, 2, 0).numpy().astype(np.uint8)
target_img = np.clip(target_img * 1.15, 0, 255).astype(np.uint8)
warp_sum = np.sum(warp_img, axis=2)
target_sum = np.sum(target_img, axis=2)
# warp_img=warp_img
warp_smaller = warp_img > target_img # np.expand_dims(warp_sum < target_sum, axis=2)
warp_img = (warp_img * mask + (1 - mask) * target_img) * warp_smaller + target_img * (1 - warp_smaller)
result = warp_img[:th, :tw, :] # * mask + target_img * (1 - mask)
cv2.imwrite(os.path.join(output_root, source_name.split('.')[0] + '_' + target_name.split('.')[0] + '.jpg'), result)
def make_image(trainer):
np.random.seed(seed)
n_images = rows * cols
xp = enc.xp
w_in_h = 512
w_in_w = 128
w_out_h = 512
w_out_w = 128
in_ch = 2
out_ch = 2
#in_chに+1して3次元にしてRGBにしている
in_all = np.zeros((n_images, in_ch + 1, w_in_h, w_in_w)).astype("f")
gt_all = np.zeros((n_images, out_ch + 1, w_out_h, w_out_w)).astype("f")
gen_all = np.zeros(
(n_images, out_ch + 1, w_out_h, w_out_w)).astype("f")
for it in range(n_images):
batch = updater.get_iterator('test').next()
batchsize = len(batch)
x_in = xp.zeros((batchsize, in_ch, w_in_h, w_in_w)).astype("f")
t_out = xp.zeros((batchsize, out_ch, w_out_h, w_out_w)).astype("f")
for i in range(batchsize):
x_in[i, :] = xp.asarray(batch[i][0])
t_out[i, :] = xp.asarray(batch[i][1])
x_in = Variable(x_in)
z = enc(x_in)
x_out = dec(z)
#3次元目は0のまま
in_all[it, :in_ch] = x_in.data.get()[0, :]
gt_all[it, :in_ch] = t_out.get()[0, :]
gen_all[it, :in_ch] = x_out.data.get()[0, :]
def save_image(x, name, mode=None):
_, C, H, W = x.shape
x = x.reshape((rows, cols, C, H, W))
x = x.transpose(0, 3, 1, 4, 2)
if C == 1:
x = x.reshape((rows * H, cols * W))
else:
x = x.reshape((rows * H, cols * W, C))
preview_dir = '{}/preview'.format(dst)
preview_path = preview_dir +\
'/image_{}_{:0>8}.png'.format(name, trainer.updater.iteration)
if not os.path.exists(preview_dir):
os.makedirs(preview_dir)
Image.fromarray(x, mode=mode).convert('RGB').save(preview_path)
x = np.asarray(np.clip(gen_all * 128 + 128, 0.0, 255.0),
dtype=np.uint8)
save_image(x, "gen")
x = np.asarray(np.clip(in_all * 128 + 128, 0.0, 255.0), dtype=np.uint8)
save_image(x, "in")
x = np.asarray(np.clip(gt_all * 128 + 128, 0.0, 255.0), dtype=np.uint8)
save_image(x, "gt")
def intersect(box_a, box_b):
max_xy = np.minimum(box_a[:, 2:], box_b[2:])
min_xy = np.maximum(box_a[:, :2], box_b[:2])
inter = np.clip((max_xy - min_xy), a_min=0, a_max=np.inf)
return inter[:, 0] * inter[:, 1]
def __call__(self, image, boxes=None, labels=None):
return np.clip(image/255.0, 0.0, 1.0), boxes, labels
def _pcca_connected(P, n, pi=None):
r"""PCCA+ spectral clustering method with optimized memberships [1]_
Clusters the first n_cluster eigenvectors of a transition matrix in order to cluster the states.
This function assumes that the transition matrix is fully connected.
Parameters
----------
P : ndarray (n,n)
Transition matrix.
n : int
Number of clusters to group to.
pi: ndarray(n,), optional, default=None
Stationary distribution if available.
Returns
-------
chi : ndarray (n x m)
A matrix containing the probability or membership of each state to be assigned to each cluster.
The rows sum to 1.
References
----------
[1] S. Roeblitz and M. Weber, Fuzzy spectral clustering by PCCA+:
application to Markov state models and data classification.
Adv Data Anal Classif 7, 147-179 (2013).
"""
# test connectivity
from deeptime.markov.tools.estimation import connected_sets
labels = connected_sets(P)
n_components = len(labels) # (n_components, labels) = connected_components(P, connection='strong')
if n_components > 1:
raise ValueError("Transition matrix is disconnected. Cannot use pcca_connected.")
if pi is None:
from deeptime.markov.tools.analysis import stationary_distribution
pi = stationary_distribution(P)
else:
if pi.shape[0] != P.shape[0]:
raise ValueError(f"Stationary distribution must span entire state space but got {pi.shape[0]} states "
f"instead of {P.shape[0]}.")
pi /= pi.sum() # make sure it is normalized
from deeptime.markov.tools.analysis import is_reversible
if not is_reversible(P, mu=pi):
raise ValueError("Transition matrix does not fulfill detailed balance. "
"Make sure to call pcca with a reversible transition matrix estimate")
# TODO: Susanna mentioned that she has a potential fix for nonreversible matrices by replacing each complex conjugate
# pair by the real and imaginary components of one of the two vectors. We could use this but would then need to
# orthonormalize all eigenvectors e.g. using Gram-Schmidt orthonormalization. Currently there is no theoretical
# foundation for this, so I'll skip it for now.
# right eigenvectors, ordered
from deeptime.markov.tools.analysis import eigenvectors
evecs = eigenvectors(P, n)
# orthonormalize
for i in range(n):
evecs[:, i] /= math.sqrt(np.dot(evecs[:, i] * pi, evecs[:, i]))
# make first eigenvector positive
evecs[:, 0] = np.abs(evecs[:, 0])
# Is there a significant complex component?
if not np.alltrue(np.isreal(evecs)):
warnings.warn(
"The given transition matrix has complex eigenvectors, so it doesn't exactly fulfill detailed balance. "
"Forcing eigenvectors to be real and continuing. Be aware that this is not theoretically solid.")
evecs = np.real(evecs)
# create initial solution using PCCA+. This could have negative memberships
chi, rot_matrix = _pcca_connected_isa(evecs, n)
# optimize the rotation matrix with PCCA++.
rot_matrix = _opt_soft(evecs, rot_matrix, n)
# These memberships should be nonnegative
memberships = np.dot(evecs[:, :], rot_matrix)
# We might still have numerical errors. Force memberships to be in [0,1]
memberships = np.clip(memberships, 0., 1.)
for i in range(0, np.shape(memberships)[0]):
memberships[i] /= np.sum(memberships[i])
return memberships
if __name__ == "__main__":
"""
Driver for testing region_of_interest
"""
# # Create object for parsing command-line options
parser = argparse.ArgumentParser(
description="Read .npy file and test for get_module_depth.\
To read a .npy file, type \"python get_module_depth.py --i (image name).npy)\""
)
# # Add argument which takes path to a bag file as an input
parser.add_argument("-i",
"--input",
type=str,
help="Path to the .npy file")
# # Parse the command line arguments to an object
args = parser.parse_args()
if args.input:
depthImage = np.load(args.input)
else:
raise FileNotFoundError(
"No input parameter has been given. For help type --help")
# gets rid of outliers, should be done before the arguments are calculated at all
depthImage = np.clip(depthImage, np.percentile(depthImage, 10),
np.percentile(depthImage, 90))
# test values, should be replaced with values found using other vision tools
region_of_interest(depthImage, depthImage[560][650], (650, 560))
def bound(arr):
return np.clip(arr, 0, 255)
def ellipeinc_alt(phi, m):
beta = np.arcsin(np.clip(np.sqrt(m) * np.sin(phi), 0, 1))
return np.sqrt(m) * ellipeinc(beta, 1 / m) + ((1 - m) / np.sqrt(m)) * ellipkinc(
beta, 1 / m
)
def step(self, action):
self.steps += 1
assert self.action_space.contains(
action), "%r (%s) invalid" % (action, type(action))
# robot 的加速度是有限制的,这里的选取一定要慎重,有两种方案,要么把变化数值都设成很小(不宜,难以快速响应环境)
# 要么将publish的频率设限,20hz(+0.1情况下)以下
if action == 0:
self.gazebo_robot_speed += 0.10
elif action == 1:
self.gazebo_robot_speed -= 0.10
# elif action == 2:
# self.gazebo_robot_speed = 0.0
elif action == 2:
self.gazebo_robot_angle += 0.05
elif action == 3:
self.gazebo_robot_angle -= 0.05
elif action == 4:
self.gazebo_robot_angle = 0.0
elif action == 5:
pass # 保持当前状态
else:
gym.logger.warn("Unknown situation !")
self.gazebo_robot_speed = np.clip(self.gazebo_robot_speed,
self.min_speed, self.max_speed)
self.gazebo_robot_angle = np.clip(self.gazebo_robot_angle,
self.min_angle, self.max_angle)
# 与许多其它os不同之处,p2os设置好cmd_vel后,robot会一直保持该速度,
# self.cmd_twist.linear.x = 0
# gazebo中robot就算什么指令都没有,还是会有非常小的cmd_twist.linear.x与cmd_twist.angular.z,如果直接publish这些数据,只会造成误差越来越大
if abs(self.gazebo_robot_speed) < 0.025:
self.gazebo_robot_speed = 0.0
if abs(self.gazebo_robot_angle) < 0.025:
self.gazebo_robot_angle = 0.0
self.cmd_twist.linear.x = self.gazebo_robot_speed
self.cmd_twist.angular.z = self.gazebo_robot_angle
# attention! testing!
# self.cmd_twist.linear.x = 0.5
# self.cmd_twist.angular.z = 0.0
# print("gazebo: speed" + str(self.gazebo_robot_speed) + ", angle: " + str(self.gazebo_robot_angle))
# 应该新建一个服务,将最新的动作信息同步给该服务,在该服务中负责发布cmd_twist,或者使用线程
time.sleep(0.08) # 0.075
self.act_pub.publish(self.cmd_twist)
# 记录部分上一轮数据
self.laser_data = self.thread1.laser_data
self.last_robot_pose_x = self.gazebo_robot_pose_x
self.last_robot_pose_y = self.gazebo_robot_pose_y
# 执行完动作,检查一下状态值(这里的严谨性有待考察,因为这里并不是严格的action一下,返回一个状态)
self.gazebo_robot_pose_x, self.gazebo_robot_pose_y, self.gazebo_robot_speed, self.gazebo_robot_angle = \
self.get_gazebo_state()
self.sample_fuc()
# self.get_rid_of_bad_data()
self.int_sample_fuc()
# print(self.sample_laser_data)
# print(self.rew_sample_laser_data)
self.state = np.array(self.int_sample_laser_data[:])
# self.state = np.array([])
self.state_speed_wrapper()
self.state = np.append(self.state, [
int(self.goal_x * 10),
int(self.goal_y * 10),
int(self.gazebo_robot_pose_x * 10),
int(self.gazebo_robot_pose_y * 10),
int(self.last_robot_pose_x * 10),
int(self.last_robot_pose_y * 10), self.state_speed,
self.state_angle
])
# print(self.sample_laser_data)
# 应该把最后的位姿信息也作为限制, min(distance) 选择0.25,有待考察,因为将测距仪放在车头前部
if self.distance_to_goal() < self.distance_to_goal_threshold \
or min(self.sample_laser_data) < self.collision_threshold: # and (self.gazebo_robot_speed < 0.1):
self.done = True
if self.steps <= 10: # for debug
print("robot: " + str(ac_num) + " : ")
print(self.sample_laser_data)
print("x: " + str(self.gazebo_robot_pose_x) + "y: " +
str(self.gazebo_robot_pose_y))
# print(self.int_sample_laser_data)
self.state = self.state.reshape((18, ))
# self.state = self.state.reshape((4, ))
reward_num = self.reward() # reward值必须在更新last_distance_to_goal之前计算
self.last_distance_to_goal = self.distance_to_goal(
) # 更新距离,为下次计算reward准备
# self.last_laser_data = self.sample_laser_data[:] # 注意这里,坑.... copy!!!
return self.state, reward_num, self.done, {}
im3 = np.zeros_like(im1)
#-- filter coeff
d = 1
k = np.array([[
0,
-1,
0,
], [-1, 4, -1], [0, -1, 0]])
#-- canvas loop
for y in range(d, height - d):
if y % 10 == 0: print(y)
for x in range(d, width - d):
#-- get window
w = im1[y - d:y + d + 1, x - d:x + d + 1]
#-- filter
# s = np.clip(w.dot(k) + 128, 0, 255)
s = np.clip(np.einsum('ij,ij', w, k) + 128, 0, 255)
#-- put
im3[y, x] = s
#-- save to png
cv2.imwrite('z410.png', im3)
if save_image_dir is not None:
fname = os.path.join(save_image_dir,fname)
raw_im, raw_coords = raw_images[orig_idx]
mean_im = mean_images[orig_idx]
absdiff_im = absdiff_images[orig_idx]
morphed_im = morphed_images[orig_idx]
raw_l, raw_b = raw_coords[:2]
imh, imw = raw_im.shape[:2]
n_ims = 5
if 1:
# increase contrast
contrast_scale = 2.0
av_im_show = np.clip(av_im*contrast_scale,0,255)
margin = 10
scale = 3
# calculate the orientation line
yintercept = y0-slope*x0
xplt=np.array([lowerleft[0]-5,
lowerleft[0]+av_im_show.shape[1]+5])
yplt=slope*xplt+yintercept
if 1:
# only send non-nan values to plot
plt_good = ~np.isnan(xplt) & ~np.isnan(yplt)
xplt = xplt[ plt_good ]
yplt = yplt[ plt_good ]
def normalize(X):
if not X.dtype == np.float32 and not X.dtype == np.float64:
X = X.astype(np.float32) / 255.
if X is None:
return None
return np.clip(X * 2.0 - 1.0, -1., 1.)
def core_test_convolution_double_backward(inshape, kernel, outmaps, pad, stride,
dilation, group, channel_last, with_bias, base_axis, seed, ctx,
func_name, non_accum_check=True,
atol_f=1e-4, atol_b=1e-3, atol_accum=8e-2, dstep=1e-3):
from nbla_test_utils import backward_function_tester, grad_function_forward_function_output
from nnabla.backward_function.convolution import ConvolutionDataGrad, ConvolutionFilterGrad
if func_name == 'ConvolutionCuda':
pytest.skip('CUDA Convolution N-D is only supported in CUDNN extension')
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip(
'channel_last=True is only supported in CUDNN backend so far.')
if channel_last and func_name.endswith('Cudnn') and (np.any(np.asarray(dilation) > 1) or group > 1):
import nnabla_ext.cuda as nc
major, minor, revision = map(int, nc.__cudnn_version__.split('.'))
version = major * 1000 + minor * 100
if version < 7200:
pytest.skip(
'channel_last dilated convolution not work in CUDNN {}.'.format(version))
# base_axis = len(inshape) - len(kernel) - 1
inmaps = inshape[base_axis]
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
rng = np.random.RandomState(seed)
i = np.clip(rng.randn(*inshape).astype(np.float32), -0.8, 0.8)
kshape = (outmaps,) + (inmaps // group,) + kernel
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(kshape), len(kernel))
kshape = tuple(kshape[i] for i in t.inv_axes)
k = np.clip(rng.randn(*kshape).astype(np.float32), -0.8, 0.8)
b = None
if with_bias:
b = np.clip(rng.randn(outmaps).astype(np.float32), -0.8, 0.8)
inputs = [i, k, b]
atol_half = 1.0 if inmaps > 64 else 1e-1
func_args = [base_axis, pad, stride, dilation, group, channel_last]
# Convolution
backward_function_tester(rng, F.convolution, inputs,
func_args=func_args,
atol_f=atol_f, atol_accum=atol_accum, dstep=dstep,
ctx=ctx)
# DataGrad
df, y = grad_function_forward_function_output(ConvolutionDataGrad,
F.convolution,
ctx, inputs, *func_args)
df.xshape = i.shape
ginputs = [rng.randn(*y.shape), k]
backward_function_tester(rng, df, ginputs,
func_args=[],
atol_f=atol_f, atol_b=atol_b, atol_accum=atol_accum, dstep=dstep,
ctx=ctx, non_accum_check=non_accum_check)
# FilterGrad
df, y = grad_function_forward_function_output(ConvolutionFilterGrad,
F.convolution,
ctx, inputs, *func_args)
df.wshape = k.shape
ginputs = [rng.randn(*y.shape), i]
backward_function_tester(rng, df, ginputs,
func_args=[],
atol_f=atol_f, atol_b=atol_b, atol_accum=atol_accum, dstep=dstep,
ctx=ctx, non_accum_check=non_accum_check)
def plot_image_subregion(raw_im, mean_im, absdiff_im,
roiradius, fname, user_coords,
scale=4.0, view='orig',
extras=None):
if extras is None:
extras = {}
output_ext = os.path.splitext(fname)[1].lower()
roisize = 2*roiradius
imtypes=['raw','absdiff','mean']
margin = 10
square_edge = roisize*scale
width=int(round(len(imtypes)*square_edge + (len(imtypes)+1)*margin))
height=int(round(square_edge+2*margin))
if output_ext == '.pdf':
output_surface = cairo.PDFSurface(fname,
width, height)
elif output_ext == '.svg':
output_surface = cairo.SVGSurface(fname,
width, height)
elif output_ext == '.png':
output_surface = cairo.ImageSurface(
cairo.FORMAT_ARGB32,width, height)
else:
raise ValueError('unknown output extension %s'%output_ext)
ctx = cairo.Context(output_surface)
# fill with white
ctx.set_source_rgb(1,1,1)
ctx.rectangle(0,0,width,height)
ctx.fill()
user_l, user_b, user_r, user_t = user_coords
# setup transform
# calculate image boundary (user coords)
for im_idx,im in enumerate(imtypes):
if im=='raw':
display_im = raw_im
elif im=='mean':
display_im = mean_im
elif im=='absdiff':
display_im = np.clip( 5*absdiff_im,0,255)
# set transform - make a patch of the cairo
# device be addressed with our image space
# coords
device_l = (im_idx+1)*margin + im_idx*square_edge
device_b = margin
ctx.identity_matrix() # reset
if view=='orig':
matrix = cairo.Matrix(xx=scale,
yx=0,
xy=0,
yy=scale,
x0=(device_l-scale*user_l),
y0=(device_b-scale*user_b),
)
elif view=='rot -90':
matrix = cairo.Matrix(xx=0,
yx=scale,
xy=scale,
yy=0,
x0=(device_l-scale*user_b),
y0=(device_b-scale*user_l),
)
elif view=='rot 180':
matrix = cairo.Matrix(xx=-scale,
yx=0,
xy=0,
yy=-scale,
x0=(device_l+scale*user_r),
y0=(device_b+scale*user_t),
)
else:
raise ValueError("unknown view '%s'"%view)
ctx.set_matrix(matrix)
## print 'device_l-user_l, device_b-user_b',device_l-user_l, device_b-user_b
## #ctx.translate(device_l-user_l, device_b-user_b)
## if scale!= 1.0:
## ctx.scale( scale, scale )
## #raise NotImplementedError('')
## ctx.translate(device_l-user_l, device_b-user_b)
## #print 'square_edge/roisize, square_edge/roisize',square_edge/roisize, square_edge/roisize
## #ctx.scale( roisize/square_edge, square_edge/roisize)
if 1:
in_surface = benu.numpy2cairo(display_im.astype(np.uint8))
ctx.rectangle(user_l,user_b,display_im.shape[1],display_im.shape[0])
if 1:
ctx.save()
ctx.set_source_surface(in_surface,user_l,user_b)
ctx.paint()
ctx.restore()
else:
ctx.set_source_rgb(0,.3,0)
ctx.fill()
if 0:
ctx.move_to(user_l,user_b)
ctx.line_to(user_r,user_b)
ctx.line_to(user_r,user_t)
ctx.line_to(user_l,user_t)
ctx.line_to(user_l,user_b)
ctx.close_path()
ctx.set_source_rgb(0,1,0)
ctx.fill()
ctx.move_to(user_l+5,user_b+5)
ctx.line_to(user_r-40,user_b+5)
ctx.line_to(user_r-40,user_t-40)
ctx.line_to(user_l+5,user_t-40)
ctx.line_to(user_l+5,user_b+5)
ctx.close_path()
ctx.set_source_rgb(0,0,1)
ctx.fill()
if output_ext == '.png':
output_surface.write_to_png(fname)
else:
ctx.show_page()
output_surface.finish()
def render(self, tmp_dir):
logger = logging.getLogger('terrarium')
bbox = self._mercator_bbox
mid_dir = os.path.join(tmp_dir, self.output_dir, str(self.z),
str(self.x))
mkdir_p(mid_dir)
tile = self.tile_name()
logger.debug("Generating tile %r..." % tile)
with self.get_datasource(logger) as dst_ds:
dst_srs = dst_ds.GetProjection()
dst_gt = dst_ds.GetGeoTransform()
dst_x_size = dst_ds.RasterXSize
dst_y_size = dst_ds.RasterYSize
# we want the output to be 3-channels R, G, B with:
# uheight = height + 32768.0
# R = int(height) / 256
# G = int(height) % 256
# B = int(frac(height) * 256)
# Looks like gdal doesn't handle "nodata" across multiple channels,
# so we'll use R=0, which corresponds to height < 32,513 which is
# lower than any depth on Earth, so we should be okay.
mem_drv = gdal.GetDriverByName("MEM")
mem_ds = mem_drv.Create('', dst_x_size, dst_y_size, 3,
gdal.GDT_Byte)
mem_ds.SetGeoTransform(dst_gt)
mem_ds.SetProjection(dst_srs)
mem_ds.GetRasterBand(1).SetNoDataValue(0)
pixels = dst_ds.GetRasterBand(1).ReadAsArray(
0, 0, dst_x_size, dst_y_size)
# transform to uheight, clamping the range
pixels += 32768.0
numpy.clip(pixels, 0.0, 65535.0, out=pixels)
r = (old_div(pixels, 256)).astype(numpy.uint8)
res = mem_ds.GetRasterBand(1).WriteArray(r)
assert res == gdal.CPLE_None
g = (pixels % 256).astype(numpy.uint8)
res = mem_ds.GetRasterBand(2).WriteArray(g)
assert res == gdal.CPLE_None
b = ((pixels * 256) % 256).astype(numpy.uint8)
res = mem_ds.GetRasterBand(3).WriteArray(b)
assert res == gdal.CPLE_None
png_file = os.path.join(tmp_dir, self.output_dir, tile + ".png")
png_drv = gdal.GetDriverByName("PNG")
png_ds = png_drv.CreateCopy(png_file, mem_ds)
# explicitly delete the datasources. the Python-GDAL docs suggest
# that this is a good idea not only to dispose of memory buffers
# but also to ensure that the backing file handles are closed.
del mem_ds
del png_ds
assert os.path.isfile(png_file)
source_names = [type(s).__name__ for s in self.sources]
logger.info("Done generating tile %r from %s" %
(tile, ", ".join(source_names)))
def actor_critic(agent_name,
multiple_agents=False,
load_agent=False,
n_episodes=300,
max_t=1000,
train_mode=True):
""" Batch processed the states in a single forward pass with a single neural network
Params
======
multiple_agents (boolean): boolean for multiple agents
PER (boolean):
n_episodes (int): maximum number of training episodes
max_t (int): maximum number of timesteps per episode
"""
start = time.time()
device = get_device()
env, env_info, states, state_size, action_size, brain_name, num_agents = initialize_env(
multiple_agents, train_mode)
states = torch.from_numpy(states).to(device).float()
NUM_PROCESSES = num_agents
# Scores is Episode Rewards
scores = np.zeros(num_agents)
scores_window = deque(maxlen=100)
scores_episode = []
actor_critic = ActorCritic(state_size, action_size, device).to(device)
agent = A2C_ACKTR(agent_name,
actor_critic,
value_loss_coef=CRITIC_DISCOUNT,
entropy_coef=ENTROPY_BETA,
lr=LEARNING_RATE,
eps=EPS,
alpha=ALPHA,
max_grad_norm=MAX_GRAD_NORM,
acktr=False,
load_agent=load_agent)
rollouts = SimpleRolloutStorage(NUM_STEPS, NUM_PROCESSES, state_size,
action_size)
rollouts.to(device)
num_updates = NUM_ENV_STEPS // NUM_STEPS // NUM_PROCESSES
# num_updates = NUM_ENV_STEPS // NUM_STEPS
print("\n## Loaded environment and agent in {} seconds ##\n".format(
round((time.time() - start), 2)))
update_start = time.time()
timesteps = 0
episode = 0
if load_agent != False:
episode = agent.episode
while True:
"""CAN INSERT LR DECAY HERE"""
# if episode == MAX_EPISODES:
# return scores_episode
# Adds noise to agents parameters to encourage exploration
# agent.add_noise(PARAMETER_NOISE)
for step in range(NUM_STEPS):
step_start = time.time()
# Sample actions
with torch.no_grad():
values, actions, action_log_probs, _ = agent.act(states)
clipped_actions = np.clip(actions.cpu().numpy(), *ACTION_BOUNDS)
env_info = env.step(actions.cpu().numpy())[
brain_name] # send the action to the environment
next_states = env_info.vector_observations # get the next state
rewards = env_info.rewards # get the reward
rewards_tensor = np.array(env_info.rewards)
rewards_tensor[rewards_tensor == 0] = NEGATIVE_REWARD
rewards_tensor = torch.from_numpy(rewards_tensor).to(
device).float().unsqueeze(1)
dones = env_info.local_done
masks = torch.from_numpy(1 - np.array(dones).astype(int)).to(
device).float().unsqueeze(1)
rollouts.insert(states, actions, action_log_probs, values,
rewards_tensor, masks, masks)
next_states = torch.from_numpy(next_states).to(device).float()
states = next_states
scores += rewards
# print(rewards)
if timesteps % 100:
print('\rTimestep {}\tScore: {:.2f}\tmin: {:.2f}\tmax: {:.2f}'.
format(timesteps, np.mean(scores), np.min(scores),
np.max(scores)),
end="")
if np.any(dones):
print(
'\rEpisode {}\tScore: {:.2f}\tAverage Score: {:.2f}\tMin Score: {:.2f}\tMax Score: {:.2f}'
.format(episode, score, np.mean(scores_window),
np.min(scores), np.max(scores)),
end="\n")
update_csv(agent_name, episode, np.mean(scores_window),
np.max(scores))
if episode % 20 == 0:
agent.save_agent(agent_name,
score,
episode,
save_history=True)
else:
agent.save_agent(agent_name, score, episode)
episode += 1
scores = np.zeros(num_agents)
break
timesteps += 1
with torch.no_grad():
next_values, _, _, _ = agent.act(next_states)
rollouts.compute_returns(next_values, USE_GAE, GAMMA, GAE_LAMBDA)
agent.update(rollouts)
score = np.mean(scores)
scores_window.append(score) # save most recent score
scores_episode.append(score)
return scores_episode
def reduce_hu_intensity_range(img, minv=100, maxv=1500):
img = np.clip(img, minv, maxv)
img = 255 * normalise_zero_one(img)
return img
def choose_action(self, s):
action = self.sess.run(self.a, {self.S: s[np.newaxis, :]})[0]
action = np.clip(np.random.normal(action, self.var), -self.a_bound, self.a_bound)
return action
# print(state)
# assert 0
var = 3
t_0 = time.time()
for i in range(MAX_EPISODE):
s = env.reset()
s = s[:, np.newaxis]
ep_reward = 0
for j in range(MAX_EP_STEP):
env.render()
a = brain.choose_actions(s)
a = np.clip(np.random.normal(a, var), 0, a_h)
s_, r, done, info = env.step(a)
# print(np.shape(s_))
# s_ = np.squeeze(s_, axis=0)
brain.store_transition(s, a, r, s_)
# print('\n', brain.pointer)
if brain.pointer > MEMORY_SIZE:
# print(brain.pointer) % MEMORY_SIZE
var *= 0.995
brain.learn()
s = s_
def step(self, action):
action = np.asarray(action)
action = action.reshape(2)
action = action * self.action_scale
reward = 0
# Penalize large actions
reward += self.control_penalty * np.sum(action**2)
probable_vel = self._pursuer.velocity + action
vel = np.clip(probable_vel, -self.max_velocity_pursuer,
self.max_velocity_pursuer)
self._pursuer.set_velocity(vel)
probable_position = self._pursuer.position + self._pursuer.velocity
# Bounce pursuer on hitting an obstacle
pursuerfromobst = ssd.euclidean(probable_position,
self.obstaclesx_No_2)
is_colliding_pursuer = pursuerfromobst <= self._pursuer._radius + self.obstacle_radius
if is_colliding_pursuer:
current_dist = ssd.euclidean(self._pursuer.position,
self.obstaclesx_No_2)
displacement = self._pursuer.velocity * (
(current_dist - self.obstacle_radius - self._pursuer._radius) /
current_dist)
self._pursuer.set_position(self._pursuer.position + displacement)
self._pursuer.set_velocity(-1 * self._pursuer.velocity)
else:
self._pursuer.set_position(probable_position)
# Pursuer stop on hitting a wall
clippedx_2 = np.clip(self._pursuer.position, 0, 1)
vel_2 = self._pursuer.velocity
vel_2[self._pursuer.position != clippedx_2] = 0
self._pursuer.set_velocity(vel_2)
self._pursuer.set_position(clippedx_2)
self._evader.set_velocity(self.get_evaders_velocity())
probable_position_ev = self._evader.position + self._evader.velocity
clippedx_2 = np.clip(probable_position_ev, 0, 1)
# print(clippedx_2)
vel_2 = self._evader.velocity
vel_2[self._evader.position != clippedx_2] = 0
self._evader.set_velocity(vel_2)
# self._evader.set_position(clippedx_2)
# Bounce evader on hitting an obstacle
evfromobst = ssd.euclidean(clippedx_2, self.obstaclesx_No_2)
is_colliding_evader = evfromobst <= self._evader._radius + self.obstacle_radius
if is_colliding_evader:
# print("Collision")
current_dist = ssd.euclidean(self._evader.position,
self.obstaclesx_No_2)
displacement = self._evader.velocity * (
(current_dist - self.obstacle_radius - self._evader._radius) /
current_dist)
# self._evader.set_position(self._evader.position + displacement)
self._evader.set_position(self._evader.position)
self._evader.set_velocity(-1 * self._evader.velocity)
else:
self._evader.set_position(clippedx_2)
# print(self._evader.velocity,self._evader.position, self._food.position)
# check if evader caught its food
evfromfood = ssd.euclidean(self._evader.position, self._food.position)
is_colliding_food = evfromfood <= self._evader._radius + self._food._radius
food_caught = False
if is_colliding_food:
food_caught = True
self._food.set_position(self.np_random.rand(2))
self._food.set_position(
self._respawn(self._food.position, self._food._radius))
# reward += -self.food_reward
# Find collisions
# Evaders
pursuerfromev = ssd.euclidean(self._pursuer.position,
self._evader.position)
is_colliding_ev_pursuer = pursuerfromev <= self._pursuer._radius + self._evader._radius
evcaught = False
if is_colliding_ev_pursuer:
# print("EVADER CAUGHT!")
evcaught = True
reward += self.food_reward
self._evader.set_position(self.np_random.rand(2))
self._evader.set_position(
self._respawn(self._evader.position, self._evader._radius))
# Update reward based on these collisions
if self.is_observability_full:
obslist = self.get_full_observation()
else:
obslist = self.get_partial_observation()
# print self._pursuer.observation_space.shape
self._timesteps += 1
done = self.is_terminal
info = dict(evcatches=int(evcaught), foodcatches=int(food_caught))
rlist = np.array([reward])
# print(reward)
# print(obslist, rlist, done, info)
# print(rlist,self.control_penalty)
# print(rlist)
return obslist, rlist, done, info
cutting(merge)
to_categorical(merge)
merge['name'] = merge['name'].apply(lambda x:normalize_text2(x))
#merge['item_description'] = merge['item_description'].apply(lambda x:normalize_text(x))
print('[{}] Convert categorical completed'.format(time.time() - start_time))
mindf = int(merge.shape[0] / 400000)
wb = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2, "hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2**28, "norm": None, "tf": 'binary',
"idf": None,
}), procs=8)
wb.dictionary_freeze= True
X_train_name = wb.fit_transform(merge['name'][:nrow_valid])
X_test_name = wb.transform(merge['name'][nrow_valid:])
X_name = vstack((X_train_name,X_test_name))
X_name = X_name[:, np.array(np.clip(X_name[:nrow_train,:].getnnz(axis=0) - 1, 0, 1), dtype=bool)]
del(X_train_name)
del(X_test_name)
del(wb)
print(X_name.shape)
#X_test_name = X_name[:, np.array(np.clip(X_test_name.getnnz(axis=0) - 1, 0, 1), dtype=bool)]
print('[{}] Vectorize `name` completed.'.format(time.time() - start_time))
wb = CountVectorizer()
X_category1 = wb.fit_transform(merge['general_cat'])
X_category2 = wb.fit_transform(merge['subcat_1'])
X_category3 = wb.fit_transform(merge['subcat_2'])
print('[{}] Count vectorize `categories` completed.'.format(time.time() - start_time))
gc.collect()
info = psutil.virtual_memory()
def join_bins(bin_edges, probabilities, min_prob=0.05):
"""Join bins until at least the minimum probability is contained.
By joining adjacent bins, find the configuration with the maximum
number of bins that each contain at least a certain probability.
Parameters
----------
bin_edges : iterable of 2-tuple
Upper and lower bounds associated with each bin.
probabilities : array-like
Probabilities in each bin. The contiguous ranges where bin joining
must be attempted will automatically be determined.
min_prob : float
The minimum probability (inclusive) per (final) bin.
Returns
-------
bin_edges : list of 2-tuple
List containing the new bin edges.
probabilities : array
Probabilities corresponding to the above bins.
Raises
------
ValueError : If the number of bin edges does not match the number
of probabilities.
ValueError : If the sum of all `probabilities` does not exceed `min_prob`.
"""
if len(bin_edges) != len(probabilities):
raise ValueError("Length of bin_edges and probabilities must match.")
probabilities = np.asarray(probabilities)
if np.sum(probabilities) <= min_prob:
raise ValueError("Sum of probabilities must exceed min_prob.")
max_i = probabilities.shape[0] - 1
def _join(start_i, end_i):
"""Return new bin edges and probabilities after a join.
Parameters
----------
start_i : int
Beginning of the join.
end_i : int
End of the join.
Returns
-------
bin_edges : list of 2-tuple
List containing the new bin edges.
probabilities : array
Probabilities corresponding to the above bins.
"""
new_bin_edges = []
new_probabilities = []
# Remove all but the first index within the join window.
for i in filterfalse(
lambda x: x in range(start_i + 1, end_i + 1), range(max_i + 1),
):
if i == start_i:
new_bin_edges.append((bin_edges[start_i][0], bin_edges[end_i][1],))
new_probabilities.append(np.sum(probabilities[start_i : end_i + 1]))
else:
new_bin_edges.append(bin_edges[i])
new_probabilities.append(probabilities[i])
return join_bins(
bin_edges=new_bin_edges, probabilities=new_probabilities, min_prob=min_prob,
)
# Identify regions with low probabilities.
join_mask = probabilities < min_prob
if not np.any(join_mask):
# Joining is complete.
return (bin_edges, probabilities)
# Find the contiguous clusters.
labelled, n_clusters = label(join_mask)
variations = []
# Carry out contiguous bin joining around all clusters.
for cluster_i in range(1, n_clusters + 1):
cluster_indices = np.where(labelled == cluster_i)[0]
cluster_bounds = (cluster_indices[0], cluster_indices[-1])
# Also consider the adjacent bins, since this may be needed
# in some cases.
join_bounds = tuple(
np.clip(np.array([cluster_bounds[0] - 1, cluster_bounds[1] + 1]), 0, max_i)
)
for start_i in range(*join_bounds):
# Optimisation: prevent 'orphan' bins on the left.
if join_bounds[0] == 0 and start_i != 0:
continue
for end_i in range(start_i + 1, join_bounds[1] + 1):
# Optimisation: prevent 'orphan' bins on the right.
if join_bounds[1] == max_i and end_i != max_i:
continue
# If the sum of probabilities between `start_i` and `end_i`
# exceeds the minimum threshold, join the bins.
if np.sum(probabilities[start_i : end_i + 1]) >= min_prob:
variations.append(_join(start_i, end_i))
# Return the 'best' variation - the set of bins with the lowest variability,
# measured using the standard deviation of the probabilities. Only sets with
# the largest number of bins will be considered here.
lengths = [len(variation[0]) for variation in variations]
max_length = max(lengths)
long_variations = [
variation for variation in variations if len(variation[0]) == max_length
]
variation_stds = [np.std(variation[1]) for variation in long_variations]
min_index = np.argsort(variation_stds)[0]
return long_variations[min_index]