def hinge_loss(X, y, theta, reg_beta=0.0):
"""Computes hinge loss and gradient.
See square_loss for arguments and return value.
"""
k, n = X.shape
# margin is (k, 1)
margin = y * X.dot(theta)
loss = (np.sum(np.maximum(np.zeros_like(margin), 1 - margin)) / k +
np.dot(theta.T, theta) * reg_beta / 2)
dtheta = np.zeros_like(theta)
# yx is (k, n) where the elementwise multiplication by y is broadcast across
# the whole X.
yx = y * X
# We're going to select columns of yx, and each column turns into a vector.
# Precompute the margin_selector vector which has for each j whether the
# margin for that j was < 1.
# Note: still keeping an explicit loop over n since I don't expect the
# number of features to be very large. It's possible to fully vectorize this
# but that would make the computation even more obscure. I'll do that if
# performance becomes an issue with this version.
margin_selector = (margin < 1).ravel()
for j in range(n):
# Sum up the contributions to the jth theta element gradient from all
# input samples.
dtheta[j, 0] = (np.sum(np.where(margin_selector, -yx[:, j], 0)) / k +
reg_beta * theta[j, 0])
return loss.flat[0], dtheta
def backward_pass(self, accum_grad):
_, timesteps, _ = accum_grad.shape
# Variables where we save the accumulated gradient w.r.t each parameter
grad_U = np.zeros_like(self.U)
grad_V = np.zeros_like(self.V)
grad_W = np.zeros_like(self.W)
# The gradient w.r.t the layer input.
# Will be passed on to the previous layer in the network
accum_grad_next = np.zeros_like(accum_grad)
# Back Propagation Through Time
for t in reversed(range(timesteps)):
# Update gradient w.r.t V at time step t
grad_V += accum_grad[:, t].T.dot(self.states[:, t])
# Calculate the gradient w.r.t the state input
grad_wrt_state = accum_grad[:, t].dot(self.V) * self.activation.gradient(self.state_input[:, t])
# Gradient w.r.t the layer input
accum_grad_next[:, t] = grad_wrt_state.dot(self.U)
# Update gradient w.r.t W and U by backprop. from time step t for at most
# self.bptt_trunc number of time steps
for t_ in reversed(np.arange(max(0, t - self.bptt_trunc), t+1)):
grad_U += grad_wrt_state.T.dot(self.layer_input[:, t_])
grad_W += grad_wrt_state.T.dot(self.states[:, t_-1])
# Calculate gradient w.r.t previous state
grad_wrt_state = grad_wrt_state.dot(self.W) * self.activation.gradient(self.state_input[:, t_-1])
# Update weights
self.U = self.U_opt.update(self.U, grad_U)
self.V = self.V_opt.update(self.V, grad_V)
self.W = self.W_opt.update(self.W, grad_W)
return accum_grad_next
def test_matrix_assemble(dim):
eps = 1000*DOLFIN_EPS
(u, uu), (v, vv), (U, UU), dPP, bc = _create_dp_problem(dim)
# Scalar assemble
mat = assemble(u*v*U*dPP)
# Create a numpy matrix based on the local size of the vector
# and populate it with values from local vector
loc_range = u.vector().local_range()
vec_mat = np.zeros_like(mat.array())
vec_mat[range(loc_range[1] - loc_range[0]),
range(loc_range[0], loc_range[1])] = u.vector().get_local()
assert np.sum(np.absolute(mat.array() - vec_mat)) < eps
# Vector assemble
mat = assemble((uu[0]*vv[0]*UU[0] + uu[1]*vv[1]*UU[1])*dPP)
# Create a numpy matrix based on the local size of the vector
# and populate it with values from local vector
loc_range = uu.vector().local_range()
vec_mat = np.zeros_like(mat.array())
vec_mat[range(loc_range[1] - loc_range[0]),
range(loc_range[0], loc_range[1])] = uu.vector().get_local()
assert np.sum(np.absolute(mat.array() - vec_mat)) < eps
def dateRange(first, last):
# Type check, float --> int
if isinstance(first[0], float):
temp = np.zeros_like(first, dtype='int')
for i in xrange(temp.size):
temp[i] = first[i]
first = tuple(temp)
if isinstance(last[0], float):
temp = np.zeros_like(last, dtype='int')
for i in xrange(temp.size):
temp[i] = last[i]
last = tuple(temp)
# Initialize date dictionary
dateList = {}
# Populate dictionary
first = dt.datetime(*first[:6])
last = dt.datetime(*last[:6])
n = (last + dt.timedelta(days=1) - first).days
dateList['year'] = np.array([(first + dt.timedelta(days=i)).year for i in xrange(n)])
dateList['month'] = np.array([(first + dt.timedelta(days=i)).month for i in xrange(n)])
dateList['day'] = np.array([(first + dt.timedelta(days=i)).day for i in xrange(n)])
return dateList
def reg(psf_model, parms):
"""
Regularization and derivative.
"""
eps = parms.eps
if (eps is None):
return np.zeros_like(psf_model)
psf_shape = psf_model.shape
d = np.zeros_like(psf_model)
r = np.zeros_like(psf_model)
for i in range(psf_shape[0]):
for j in range(psf_shape[1]):
if i > 0:
r[i, j] += (psf_model[i, j] - psf_model[i - 1, j]) ** 2.
d[i, j] += 2. * (psf_model[i, j] - psf_model[i - 1, j])
if j > 0:
r[i, j] += (psf_model[i, j] - psf_model[i, j - 1]) ** 2.
d[i, j] += 2. * (psf_model[i, j] - psf_model[i, j - 1])
if i < psf_shape[0] - 1:
r[i, j] += (psf_model[i, j] - psf_model[i + 1, j]) ** 2.
d[i, j] += 2. * (psf_model[i, j] - psf_model[i + 1, j])
if j < psf_shape[1] - 1:
r[i, j] += (psf_model[i, j] - psf_model[i, j + 1]) ** 2.
d[i, j] += 2. * (psf_model[i, j] - psf_model[i, j + 1])
r *= eps
d *= eps
return r, d
def qspline1d_eval(cj, newx, dx=1.0, x0=0):
"""Evaluate a quadratic spline at the new set of points.
`dx` is the old sample-spacing while `x0` was the old origin. In
other-words the old-sample points (knot-points) for which the `cj`
represent spline coefficients were at equally-spaced points of::
oldx = x0 + j*dx j=0...N-1, with N=len(cj)
Edges are handled using mirror-symmetric boundary conditions.
"""
newx = (asarray(newx) - x0) / dx
res = zeros_like(newx)
if res.size == 0:
return res
N = len(cj)
cond1 = newx < 0
cond2 = newx > (N - 1)
cond3 = ~(cond1 | cond2)
# handle general mirror-symmetry
res[cond1] = qspline1d_eval(cj, -newx[cond1])
res[cond2] = qspline1d_eval(cj, 2 * (N - 1) - newx[cond2])
newx = newx[cond3]
if newx.size == 0:
return res
result = zeros_like(newx)
jlower = floor(newx - 1.5).astype(int) + 1
for i in range(3):
thisj = jlower + i
indj = thisj.clip(0, N - 1) # handle edge cases
result += cj[indj] * quadratic(newx - thisj)
res[cond3] = result
return res
def viterbi_decode(score, transition_params):
"""Decode the highest scoring sequence of tags outside of TensorFlow.
This should only be used at test time.
Args:
score: A [seq_len, num_tags] matrix of unary potentials.
transition_params: A [num_tags, num_tags] matrix of binary potentials.
Returns:
viterbi: A [seq_len] list of integers containing the highest scoring tag
indicies.
viterbi_score: A float containing the score for the Viterbi sequence.
"""
trellis = np.zeros_like(score)
backpointers = np.zeros_like(score, dtype=np.int32)
trellis[0] = score[0]
for t in range(1, score.shape[0]):
v = np.expand_dims(trellis[t - 1], 1) + transition_params
trellis[t] = score[t] + np.max(v, 0)
backpointers[t] = np.argmax(v, 0)
viterbi = [np.argmax(trellis[-1])]
for bp in reversed(backpointers[1:]):
viterbi.append(bp[viterbi[-1]])
viterbi.reverse()
viterbi_score = np.max(trellis[-1])
return viterbi, viterbi_score
def __init__(self,n_hidden,n_input,n_out,fnc = 'sigmoid',loss_fnc= softmax,batchsize = 10,epochs = 1,learning_rate = 0.1,reg = 0.0,momentum = 0.0):
self.nn = {}
self.nn['batchsize'] = batchsize
self.nn['epochs'] = epochs
self.nn['learning_rate'] = learning_rate
self.nn['reg'] = reg
self.nn['momentum'] = momentum
self.nn['loss_fnc'] = loss_fnc
self.nn['w1'] = np.random.randn(n_hidden*n_input).reshape(n_input,n_hidden)/math.sqrt(n_hidden*n_input)
self.nn['b1'] = np.zeros(n_hidden).reshape(n_hidden)
self.nn['w2'] = np.random.random(n_hidden*n_out).reshape(n_hidden,n_out)/math.sqrt(n_hidden*n_out)
self.nn['b2'] = np.zeros(n_out).reshape(n_out)
self.nn['dw1'] = np.zeros_like(self.nn['w1'])
self.nn['db1'] = np.zeros_like(self.nn['b1'])
self.nn['dw2'] = np.zeros_like(self.nn['w2'])
self.nn['db2'] = np.zeros_like(self.nn['b2'])
self.nn['p_dw1'] = self.nn['dw1']
self.nn['p_db1'] = self.nn['db1']
self.nn['p_dw2'] = self.nn['dw2']
self.nn['p_db2'] = self.nn['db2']
def totalvalue(cash_ini,orderform,valueform):
trades = pd.read_csv(orderform,header=None,sep=',')
trades = trades.dropna(axis = 1, how='all')
trades.columns = ['Year','Month','Day','Symbol','Order','Share']
dateall = []
for i in np.arange(len(trades.Year)):
dateall.append(dt.datetime(trades['Year'][i],trades['Month'][i],trades['Day'][i],16))
dateall = pd.to_datetime(dateall)
trades=trades.drop(['Year','Month','Day'],axis=1)
trades['Date']=dateall
trades.set_index('Date',inplace=True)
ls_symbols = []
for symbol in trades.Symbol:
if symbol not in ls_symbols:
ls_symbols.append(symbol)
startdate = dateall[0]
enddate = dateall[-1]
dt_timeofday = dt.timedelta(hours=16)
ldt_timestamps = du.getNYSEdays(startdate,enddate+dt_timeofday,dt_timeofday)
ls_keys = 'close'
c_dataobj = da.DataAccess('Yahoo')
price = c_dataobj.get_data(ldt_timestamps, ls_symbols, ls_keys)
orders = price*np.NaN
orders = orders.fillna(0)
for i in np.arange(len(trades.index)):
ind = trades.index[i]
if trades.ix[i,'Order']=='Buy':
orders.loc[ind,trades.ix[i,'Symbol']]+=trades.ix[i,'Share']
else:
orders.loc[ind,trades.ix[i,'Symbol']]+=-trades.ix[i,'Share']
# keys = ['price','orders']
# trading_table = pd.concat([ldf_data,orders],keys=keys,axis=1)
cash = np.zeros(np.size(price[ls_symbols[0]]),dtype=np.float)
cash[0] = cash_ini
# updating the cash value
for i in np.arange(len(orders.index)):
if i == 0:
cash[i] = cash[i] - pd.Series.sum(price.ix[i,:]*orders.ix[i,:])
else:
cash[i] = cash[i-1] - pd.Series.sum(price.ix[i,:]*orders.ix[i,:])
# updating ownership
ownership = orders*np.NaN
for i in np.arange(len(orders.index)):
ownership.ix[i,:]=orders.ix[:i+1,:].sum(axis=0)
# updating total portofolio value
value = np.zeros_like(cash)
for i in np.arange(len(ownership.index)):
value[i] = pd.Series.sum(price.ix[i,:]*ownership.ix[i,:])
keys = ['price','orders','ownership']
trading_table = pd.concat([price,orders,ownership],keys = keys, axis=1)
trading_table[('value','CASH')]=cash
trading_table[('value','STOCK')]=value
total = np.zeros_like(cash)
total = cash + value
trading_table[('value','TOTAL')]=total
trading_table[('value','TOTAL')].to_csv(valueform)
def get_image_from_kspace(k_real, k_imag):
"""
Return image from real and imaginary k-space values
:param k_real:
:param k_imag:
:return:
"""
k_space = (np.zeros_like(k_real) + 0j).astype('complex64')
ret_img = np.zeros_like(k_real)
if len(k_real.shape) == 2:
# Create k-space from real and imaginary part
k_space.real = k_real
k_space.imag = k_imag
ret_img = np.abs(ifft2c(k_space)).astype(np.float32)
else:
for i in range(0, k_real.shape[0]):
# Create k-space from image, assuming this is the fully-sampled k-space.
k_space[i,:,:] = (np.zeros_like(k_real[i,:,:]) + 0j).astype('complex64')
k_space[i,:,:].real = k_real[i,:,:]
k_space[i,:,:].imag = k_imag[i,:,:]
# Reconstruct image
ret_img[i, :, :] = np.abs(ifft2c(k_space[i, :, :])).astype(np.float32)
return ret_img
def finalize(self):
"""Calculates the flux, inverse variance and resolution for this spectrum.
Uses the accumulated data from all += operations so far but does not prevent
further accumulation. This is the expensive step in coaddition so we make
it something that you have to call explicitly. If you forget to do this,
the flux,ivar,resolution attributes will be None.
If the coadded resolution matrix is not invertible, a warning message is
printed and the returned flux vector is zero (but ivar and resolution are
still valid).
"""
# Convert to a dense matrix if necessary.
if scipy.sparse.issparse(self.Cinv):
self.Cinv = self.Cinv.todense()
# What pixels are we using?
mask = (np.diag(self.Cinv) > 0)
keep = np.arange(len(self.Cinv_f))[mask]
keep_t = keep[:,np.newaxis]
# Initialize the results to zero.
self.flux = np.zeros_like(self.Cinv_f)
self.ivar = np.zeros_like(self.Cinv_f)
R = np.zeros_like(self.Cinv)
# Calculate the deconvolved flux,ivar and resolution for ivar > 0 pixels.
self.ivar[mask],R[keep_t,keep] = decorrelate(self.Cinv[keep_t,keep])
try:
R_it = scipy.linalg.inv(R[keep_t,keep].T)
self.flux[mask] = R_it.dot(self.Cinv_f[mask])/self.ivar[mask]
except np.linalg.linalg.LinAlgError:
self.log.warning('resolution matrix is singular so no coadded fluxes available.')
# Convert R from a dense matrix to a sparse one.
self.resolution = desispec.resolution.Resolution(R)
def num_sdss_rand_both_catalogs(hemi, grid):
d_rand = load_sdss_rand_both_catalogs(hemi)
n_noisy = grid.num_from_radecz(d_rand['ra'],d_rand['dec'], d_rand['z'])
# get the distance-from-observer 3d grid
d_obs = grid.distance_from_observer()
d_obs_max = d_obs[n_noisy>0].max()
d_obs_1d = np.linspace(0, d_obs_max+1., 100)
n_1d = np.zeros_like(d_obs_1d)
delta_d_obs = d_obs_1d[1]-d_obs_1d[0]
for i,this_d_obs in enumerate(d_obs_1d):
wh=np.where((np.abs(d_obs-this_d_obs)<(0.5*delta_d_obs)) & (n_noisy>0))
if len(wh[0])==0: continue
n_1d[i] = np.median(n_noisy[wh])
# now interpolate n_1d onto 3d grid
from scipy import interpolate
f = interpolate.interp1d(d_obs_1d, n_1d)
n_median = np.zeros_like(n_noisy)
wh_ok_interp = np.where((d_obs>np.min(d_obs_1d))&(d_obs<np.max(d_obs_1d)))
n_median[wh_ok_interp] = f(d_obs[wh_ok_interp])
weight = np.zeros_like(n_median)
weight[n_noisy>12]=1.
n_median *= weight
#pl.figure(1); pl.clf(); pl.imshow(n_noisy[:,:,128], vmin=0,vmax=n_median.max()); pl.colorbar()
#pl.figure(2); pl.clf(); pl.imshow(n_median[:,:,128], vmin=0,vmax=n_median.max()); pl.colorbar()
#ipdb.set_trace()
return n_median, weight
def dataVtk_3dMatrix(points,bounds,vectors):
"""
Function that turns a vtk output formated data to 3d field matrix data
from [(x1,y1,z1),...,(xn,yn,zn)]
to [[[[x1,y1,z1],[...],[x3,y1,z1]],[[x1,y2,z1],[...],[...]],[[x1,y3,z1],[...],[...]]]
,[[[x1,y1,z2],[...],[...]],[...],[...]] , [.........]]
-points => list of the coordinates of the poitns where the data is located.
-bounds => bounds of the data.(Xmin,Xmax,Ymin,Ymax,Zmin,Zmax)
-vectors => vector data of the field at the 'points'
"""
#asign variables
(xmin,xmax,ymin,ymax,zmin,zmax) = bounds
#generate the output arrays
grid3d = N.mgrid[zmin:zmax+1, ymin:ymax+1, xmin:xmax+1]
pnts3d = N.zeros_like(grid3d[0],dtype= N.ndarray)
vect3d = N.zeros_like(grid3d[0],dtype= N.ndarray)
#loop and rearange
for i in range(len(points)):
x_t = points[i][0]
y_t = points[i][1]
z_t = points[i][2]
pnts3d[z_t+zmax][y_t+ymax][x_t+xmax] = points[i]
vect3d[z_t+zmax][y_t+ymax][x_t+xmax] = vectors[i]
return {'points':pnts3d,'vectors':vect3d}
def _transform_dense(self, X):
non_zero = (X != 0.0)
X_nz = X[non_zero]
X_step = np.zeros_like(X)
X_step[non_zero] = np.sqrt(X_nz * self.sample_interval_)
X_new = [X_step]
log_step_nz = self.sample_interval_ * np.log(X_nz)
step_nz = 2 * X_nz * self.sample_interval_
for j in range(1, self.sample_steps):
factor_nz = np.sqrt(step_nz /
np.cosh(np.pi * j * self.sample_interval_))
X_step = np.zeros_like(X)
X_step[non_zero] = factor_nz * np.cos(j * log_step_nz)
X_new.append(X_step)
X_step = np.zeros_like(X)
X_step[non_zero] = factor_nz * np.sin(j * log_step_nz)
X_new.append(X_step)
return np.hstack(X_new)
def fit_deriv(cls, x, amplitude, x_0, width):
"""One dimensional Box model derivative with respect to parameters"""
d_amplitude = cls.evaluate(x, 1, x_0, width)
d_x_0 = np.zeros_like(x)
d_width = np.zeros_like(x)
return [d_amplitude, d_x_0, d_width]
def _compute_rotations(positions, pos_seed, rot_seed):
logger.info("compute_rotations: starting aux calculations")
# logger.info("positions array {}".format(positions))
# logger.debug("rot_seed = {}".format(rot_seed))
# logger.debug("pos_seed = {}".format(pos_seed))
delta = np.zeros_like(positions)
delta[1:] = positions[1:] - positions[:-1]
delta[0] = positions[0] - pos_seed
# logger.debug("delta array {}".format(delta))
z = np.zeros_like(delta) # zero array like delta
z[delta == -5] = 1 # where delta = -5, set increment to 1
z[delta == 5] = -1 # where delta is 5, set increment to -1
logger.info("compute_rotations: done with aux calculations")
logger.info("compute_rotations: starting primary calculations")
z[0] += rot_seed
r = np.cumsum(z)
logger.info("compute_rotations: done with primary calculations")
return r
def predictionToPosition(self,pi, dim = 64): pirescale = np.expand_dims(pi, axis=1) pirescale = np.append(pirescale, np.zeros_like(pirescale), axis=1) positions = np.zeros_like(pirescale) positions[:,0] = pirescale[:,0] // dim positions[:,1] = pirescale[:,0] % dim return positions
def gradientOfNormalizedRespectW(self, sequence):
seqLength = sequence.Length
(logPotential0, logPotentials) = self.LogPotentialTable(sequence)
#print(logPotentials)
forwardMessages = self.forward(logPotential0, logPotentials, seqLength)
backwardMessages = self.backward(logPotentials, seqLength)
b = forwardMessages[seqLength-1].max()
logNormalized = np.log(np.exp(forwardMessages[seqLength-1] - b).sum()) + b
WnodeGradient = np.zeros_like(self.mWnode)
WedgeGradient = np.zeros_like(self.mWedge)
for i in range(0, seqLength):
for j in range(0, self.mLabelStateSize):
#print(forwardMessages[i][j] + backwardMessages[i][j] - logNormalized)
WnodeGradient[sequence.GetFeature(i, j)] += np.exp(forwardMessages[i][j] + backwardMessages[i][j] - logNormalized).clip(0.0, 1.0)
for i in range(1, seqLength):
for j in range(0, self.mLabelStateSize):
for k in range(0, self.mLabelStateSize):
#print(forwardMessages[i-1][j] + logPotentials[i-1][j][k] + backwardMessages[i][k] - logNormalized)
WedgeGradient[sequence.GetFeature(i, j, k)] += np.exp(forwardMessages[i-1][j] + logPotentials[i-1][j][k] + backwardMessages[i][k] - logNormalized).clip(0.0, 1.0)
# print(WnodeGradient, WedgeGradient)
return (WnodeGradient, WedgeGradient)
def generate_sky_model_y_hybrid(baselines,del_bl,num_bl,fits_file):
"""
y is a vector of the visibilities at different baselines
"""
healmap = a.map.Map(fromfits=fits_file)
px_array = n.arange(healmap.npix()) # gets an array of healpix pixel indices
rx,ry,rz = n.array(healmap.px2crd(px_array,ncrd=3)) # finds the topocentric coords for each healpix pixel
phi,theta = n.array(healmap.px2crd(px_array,ncrd=2)) # phi,theta in math coords
print px_array.shape
true_sky = healmap.map.map
beamsig_largebm = 10/(2*n.pi*del_bl*(num_bl-1))
beamsig_smallbm = 10/(2*n.pi*del_bl)
amp_largebm = uf.gaussian(beamsig_largebm,n.zeros_like(theta),phi)
amp_smallbm = uf.gaussian(beamsig_smallbm,n.zeros_like(theta),phi)
#smallbm_inds = (int(n.floor(num_bl/2)),int(n.floor(num_bl/2)))
smallbm_ind = int(n.floor(num_bl/2))*(num_bl+1)+1
dOmega = 4*n.pi/px_array.shape[0]
visibilities = n.zeros(baselines.shape[0],dtype='complex')
print baselines.shape[0]
for kk in range(baselines.shape[0]):
#print kk
bx,by,bz = baselines[kk]
if kk==smallbm_ind: amp = amp_smallbm
else: amp = amp_largebm
Vis = amp*true_sky*n.exp(2j*n.pi*(bx*rx+by*ry+bz*rz))*dOmega
visibilities[kk] = n.sum(Vis)
return visibilities
def pop(self, index=-1):
if not isinstance(index, int):
msg = "{0} indices must be integers, not {1}"
raise TypeError(msg.format(self.__class__.__name__,
index.__class__.__name__))
if index < 0:
index += self._len
if index < 0:
raise IndexError
if self.storage == "numpy":
ret_data = datacopy(self._data[index])
ret_lengths = None
if self._arity > 1:
ret_lengths = _get_lenarray_empty(ret_data.shape)
ret = self._element(
_mode="from_numpy",
data_store=ret_data,
len_data_store=ret_lengths,
)
self._data[index:self._len-1] = self._data[index+1:self._len]
try:
self._data[self._len-1] = np.zeros_like(self._data[self._len-1])
except ValueError: # numpy bug
for field in self._data.dtype.fields:
self._data[self._len-1][field] = np.zeros_like(self._data[self._len-1][field])
self._del_child_lengths(index)
elif self._elementary:
ret = self._data[:self._len][index]
self._data.__delitem__(index)
else:
ret = self._children[:self._len][index].copy()
self._children.__delitem__(index)
self._data.__delitem__(index)
self._len -= 1
return ret
def lu_fact(A):
"""
Function for computing the LU factorization
Inputs:
A: Matrix of the system
Output:
L: Lower triangular matrix
U: Upper triangular matrix
"""
n = np.size(A, 1)
L = np.zeros_like(A, float)
U = np.zeros_like(A, float)
for k in xrange(0, n-1):
for i in xrange(k+1, n):
if A[i, k] != 0.0:
lam = A[i, k] / A[k, k]
A[i, k+1:n] = A[i, k+1:n] - lam*A[k, k+1:n]
A[i, k] = lam
L = np.tril(A, -1) + np.identity(n)
U = np.triu(A)
return L, U
def fdtd(input_grid, steps):
grid = input_grid.copy()
old_grid = np.zeros_like(input_grid)
previous_grid = np.zeros_like(input_grid)
l_x = grid.shape[0]
l_y = grid.shape[1]
for i in range(steps):
np.copyto(previous_grid, old_grid)
np.copyto(old_grid, grid)
for x in range(l_x):
for y in range(l_y):
grid[x,y] = 0.0
if 0 < x+1 < l_x:
grid[x,y] += old_grid[x+1,y]
if 0 < x-1 < l_x:
grid[x,y] += old_grid[x-1,y]
if 0 < y+1 < l_y:
grid[x,y] += old_grid[x,y+1]
if 0 < y-1 < l_y:
grid[x,y] += old_grid[x,y-1]
grid[x,y] /= 2.0
grid[x,y] -= previous_grid[x,y]
return grid
def __init__(self, network, **kwargs):
# due to the way that theano handles updates, we cannot update a
# parameter twice during the same function call. so, instead of handling
# everything in the updates for self.f_learn(...), we split the
# parameter updates into two function calls. the first "prepares" the
# parameters for the gradient computation by moving the entire model one
# step according to the current velocity. then the second computes the
# gradient at that new model position and performs the usual velocity
# and parameter updates.
self.params = network.params(**kwargs)
self.momentum = kwargs.get('momentum', 0.5)
# set up space for temporary variables used during learning.
self._steps = []
self._velocities = []
for param in self.params:
v = param.get_value()
n = param.name
self._steps.append(theano.shared(np.zeros_like(v), name=n + '_step'))
self._velocities.append(theano.shared(np.zeros_like(v), name=n + '_vel'))
# step 1. move to the position in parameter space where we want to
# compute our gradient.
prepare = []
for param, step, velocity in zip(self.params, self._steps, self._velocities):
prepare.append((step, self.momentum * velocity))
prepare.append((param, param + step))
logging.info('compiling NAG adjustment function')
self.f_prepare = theano.function([], [], updates=prepare)
super(NAG, self).__init__(network, **kwargs)
def sor(A, b):
sol = []
n = len(A)
D = np.zeros_like(A)
L = np.zeros_like(A)
for i in range(0,n):
D[i][i] = A[i][i];
for i in range(0,n):
for j in range(0,i):
L[i][j] = -A[i][j];
omega = omegafind(A,D)
Q = D/omega -L
Tj = np.linalg.inv(Q).dot(Q-A)
c = np.linalg.inv(Q).dot(b)
x = np.zeros_like(b)
for itr in range(ITERATION_LIMIT):
x=Tj.dot(x) + c;
sol = x
return list(sol)
def filter_frames(self, data):
data = data[0]
lp = gaussian_filter(data, 100)
hp = data - lp # poormans background subtraction
hp -= np.min(hp)
sh = hp.shape
print "here"
hp = hp.astype('uint32')
hp = flex.int(hp)
print "here now"
mask = flex.bool(np.ones_like(hp).astype('bool'))
print "here now"
result1 = flex.bool(np.zeros_like(hp).astype('bool'))
spots = np.zeros_like(hp).astype('bool')
print "here now"
for i in range(3, self.parameters['spotsize'], 5):
print "here now"
algorithm = DispersionThreshold(sh, (i, i), 1, 1, 0, -1)
print "here now"
print type(hp), type(mask), type(result1)
thing = algorithm(hp, mask, result1)
print "here now"
spots = spots + result1.as_numpy_array()
return [data, spots*data]
def compute_normals(im_pos, n_offset=3):
"""
Converts an XYZ image to a Normal Image
--Input--
im_pos : ndarray (NxMx3)
Image with x/y/z values for each pixel
n_offset : int
Smoothness factor for calculating the gradient
--Output--
normals : ndarray (NxMx3)
Image with normal vectors for each pixel
"""
gradients_x = np.zeros_like(im_pos)
gradients_y = np.zeros_like(im_pos)
for i in range(3):
gradients_x[:, :, i], gradients_y[:, :, i] = np.gradient(im_pos[:, :, i], n_offset)
gradients_x /= np.sqrt(np.sum(gradients_x**2, -1))[:, :, None]
gradients_y /= np.sqrt(np.sum(gradients_y**2, -1))[:, :, None]
normals = np.cross(gradients_x.reshape([-1, 3]),
gradients_y.reshape([-1, 3])).reshape(im_pos.shape)
normals /= np.sqrt(np.sum(normals**2, -1))[:, :, None]
normals = np.nan_to_num(normals)
return normals
def initialize_adam(parameters):
'''
初始化v和s,他们都是字典类型的向量,都包含了以下字段
-key:'dW1','db1',...'dWL','dbL'
-values:与对应的梯度/参数相同维度的值为零的numpy矩阵
:param parameters: -包含了以下参数的字典变量
parameters['W'+str(l)] = W1
parameters['b'+str(l)] = bl
:return:
v - 包含梯度的指数加权平均值,字段如下:
v['dW'+str(l)] = ...
v['db'+str(l)] = ...
s - 包含平方梯度的指数加权平均值,字段如下:
s['dW'+str(l)] = ...
s['db'+str(l)] = ...
'''
L = len(parameters)//2
v= {}
s = {}
for l in range(L):
v['dW'+str(l+1)] = np.zeros_like(parameters['W'+str(l+1)])
v['db'+str(l+1)] = np.zeros_like(parameters['b'+str(l+1)])
s['dW'+str(l+1)] = np.zeros_like(parameters['W'+str(l+1)])
s['db'+str(l+1)] = np.zeros_like(parameters['b'+str(l+1)])
return(v,s)
def get_jk_coulomb(mol, dm, hermi=1, coulomb_allow='SSSS',
opt_llll=None, opt_ssll=None, opt_ssss=None):
if coulomb_allow.upper() == 'LLLL':
logger.info(mol, 'Coulomb integral: (LL|LL)')
j1, k1 = _call_veff_llll(mol, dm, hermi, opt_llll)
n2c = j1.shape[1]
vj = numpy.zeros_like(dm)
vk = numpy.zeros_like(dm)
vj[...,:n2c,:n2c] = j1
vk[...,:n2c,:n2c] = k1
elif coulomb_allow.upper() == 'SSLL' \
or coulomb_allow.upper() == 'LLSS':
logger.info(mol, 'Coulomb integral: (LL|LL) + (SS|LL)')
vj, vk = _call_veff_ssll(mol, dm, hermi, opt_ssll)
j1, k1 = _call_veff_llll(mol, dm, hermi, opt_llll)
n2c = j1.shape[1]
vj[...,:n2c,:n2c] += j1
vk[...,:n2c,:n2c] += k1
else: # coulomb_allow == 'SSSS'
logger.info(mol, 'Coulomb integral: (LL|LL) + (SS|LL) + (SS|SS)')
vj, vk = _call_veff_ssll(mol, dm, hermi, opt_ssll)
j1, k1 = _call_veff_llll(mol, dm, hermi, opt_llll)
n2c = j1.shape[1]
vj[...,:n2c,:n2c] += j1
vk[...,:n2c,:n2c] += k1
j1, k1 = _call_veff_ssss(mol, dm, hermi, opt_ssss)
vj[...,n2c:,n2c:] += j1
vk[...,n2c:,n2c:] += k1
return vj, vk
def calculate_feed_tonnage_constrain(self,schedule,opening = None,closing = None):
if opening is None:
opening = np.zeros(self.ndp,dtype=np.int)
else:
assert(schedule.shape[0] == opening.shape[0])
if closing is None:
closing = np.zeros(self.ndp,dtype=np.int)
closing[:] = self.nperiods - 1
else:
assert(schedule.shape[0] == closing.shape[0])
production_period = self.calculate_feed_tonnage(schedule,opening,closing)
#calculate the deviation from feed targets
#logger.debug("minimum_feed_production=%f",self.minimum_feed_production)
#logger.debug("maximum_feed_production=%f",self.maximum_feed_production)
minp = np.zeros_like(production_period)
indices = np.where(production_period < self.minimum_feed_production)[0]
if len(indices) > 0:
minp[indices] = self.minimum_feed_production - production_period[indices]
maxp = np.zeros_like(production_period)
indices = np.where(production_period > self.maximum_feed_production)[0]
if len(indices) > 0:
maxp[indices] = production_period[indices] - self.maximum_feed_production
return tuple(maxp) + tuple(minp)
def curvesSimilar(t1, y1, t2, y2, tol):
"""
This function returns True if the two given curves are similar enough within tol. Otherwise returns False.
t1: time/domain of standard curve we assume to be correct
y1: values of standard curve, usually either temperature in (K) or log of a mol fraction
t2: time/domain of test curve
y2: values of test curve, usually either temperature in (K) or log of a mol fraction
The test curve is first synchronized to the standard curve using geatNearestTime function. We then calculate the value of
abs((y1-y2')/y1), giving us a normalized difference for every point. If the average value of these differences is less
than tol, we say the curves are similar.
We choose this criteria because it is compatible with step functions we expect to see in ignition systems.
"""
# Make synchornized version of t2,y2 called t2sync,y2sync.
t2sync=numpy.zeros_like(t1)
y2sync=numpy.zeros_like(t1)
for i, timepoint1 in enumerate(t1):
time_index = findNearest(t2, timepoint1)
t2sync[i]=t2[time_index]
y2sync[i]=y2[time_index]
# Get R^2 value equivalent:
normalizedError=abs((y1-y2sync)/y1)
normalizedError=sum(normalizedError)/len(y1)
if normalizedError > tol:
return False
else:
return True
PORTAL_NUM = 11475124
pb = progressbar.ProgressBar()
head = np.load('head.npy')
def find(x):
if head[head[x]] != head[x]:
head[x] = find(head[x])
return head[x]
num = len(head)
count = np.zeros_like(head)
grp = np.zeros_like(head)
total = 0
pb.start(PORTAL_NUM)
for i in range(num):
count[find(i)] += 1
pb.update(i + 1)
if head[i] == i:
grp[i] = total
total += 1
pb.finish()
np.savez("result.npz", head, count, grp)
def calc_grad(self):
"""Compute parameter gradient."""
grad = np.zeros_like(self.W)
W_recs = [self.get_weights(self.W, (l, l))
for l in range(self.n_layers)]
batch_size = self.inputs.shape[0]
sig_len = self.inputs.shape[1]
# temporary space to minimize memory allocations
tmp_act = [np.zeros((batch_size, l), dtype=self.dtype)
for l in self.shape]
tmp_grad = np.zeros_like(grad)
if self.truncation is None:
trunc_per = trunc_len = sig_len
else:
trunc_per, trunc_len = self.truncation
for n in range(trunc_per - 1, sig_len, trunc_per):
# every trunc_per timesteps we want to run backprop
deltas = [np.zeros((batch_size, l), dtype=self.dtype)
for l in self.shape]
state_deltas = [None if not l.stateful else
np.zeros((batch_size, self.shape[i]),
dtype=self.dtype)
for i, l in enumerate(self.layers)]
# backpropagate error
for s in range(n, np.maximum(n - trunc_len, -1), -1):
# execute trunc_len steps of backprop through time
error = self.loss.d_loss([a[:, s] for a in self.activations],
self.targets[:, s])
error = [np.zeros_like(self.activations[i][:, s]) if e is None
else e for i, e in enumerate(error)]
for l in range(self.n_layers - 1, -1, -1):
for post in self.conns[l]:
error[l] += np.dot(deltas[post],
self.get_weights(self.W,
(l, post))[0].T,
out=tmp_act[l])
# feedforward gradient
W_grad, b_grad = self.get_weights(grad, (l, post))
W_tmp_grad, b_tmp_grad = self.get_weights(tmp_grad,
(l, post))
W_grad += np.dot(self.activations[l][:, s].T,
deltas[post], out=W_tmp_grad)
b_grad += np.sum(deltas[post], axis=0, out=b_tmp_grad)
# add recurrent error
if l in self.rec_layers:
error[l] += np.dot(deltas[l], W_recs[l][0].T,
out=tmp_act[l])
# compute deltas
if not self.layers[l].stateful:
self.J_dot(self.d_activations[l][:, s], error[l],
transpose_J=True, out=deltas[l])
else:
d_input = self.d_activations[l][:, s, ..., 0]
d_state = self.d_activations[l][:, s, ..., 1]
d_output = self.d_activations[l][:, s, ..., 2]
state_deltas[l] += self.J_dot(d_output, error[l],
transpose_J=True,
out=tmp_act[l])
self.J_dot(d_input, state_deltas[l], transpose_J=True,
out=deltas[l])
self.J_dot(d_state, state_deltas[l], transpose_J=True,
out=state_deltas[l])
# gradient for recurrent weights
if l in self.rec_layers:
W_grad, b_grad = self.get_weights(grad, (l, l))
W_tmp_grad, b_tmp_grad = self.get_weights(tmp_grad,
(l, l))
if s > 0:
W_grad += np.dot(self.activations[l][:, s - 1].T,
deltas[l], out=W_tmp_grad)
else:
# put remaining gradient into initial bias
b_grad += np.sum(deltas[l], axis=0,
out=b_tmp_grad)
grad /= batch_size
return grad
def main(args):
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
# ========= Define VIBE model ========= #
model = VIBE_Demo(
seqlen=16,
device=device,
n_layers=2,
hidden_size=1024,
add_linear=True,
use_residual=True,
).to(device)
# ========= Load pretrained weights ========= #
pretrained_file = download_ckpt(use_3dpw=False)
ckpt = torch.load(pretrained_file, map_location=device)
print(f'Performance of pretrained model on 3DPW: {ckpt["performance"]}')
ckpt = ckpt['gen_state_dict']
model.load_state_dict(ckpt, strict=False)
model.eval()
print(f'Loaded pretrained weights from \"{pretrained_file}\"')
total_time = time.time()
# ========= Run VIBE on crops ========= #
print(f'Running VIBE on crops...')
vibe_time = time.time()
image_folder = args.input_folder
dataset = InferenceFromCrops(image_folder=image_folder)
orig_height = orig_width = 512
dataloader = DataLoader(dataset,
batch_size=args.vibe_batch_size,
num_workers=0)
with torch.no_grad():
pred_cam, pred_verts, pred_pose, pred_betas, pred_joints3d, norm_joints2d = [], [], [], [], [], []
for batch_num, batch in enumerate(dataloader):
print("BATCH:", batch_num)
batch = batch.unsqueeze(0)
batch = batch.to(device)
batch_size, seqlen = batch.shape[:2]
output = model(batch)[-1]
pred_cam.append(output['theta'][:, :, :3].reshape(
batch_size * seqlen, -1))
pred_verts.append(output['verts'].reshape(batch_size * seqlen, -1,
3))
pred_pose.append(output['theta'][:, :, 3:75].reshape(
batch_size * seqlen, -1))
pred_betas.append(output['theta'][:, :, 75:].reshape(
batch_size * seqlen, -1))
pred_joints3d.append(output['kp_3d'].reshape(
batch_size * seqlen, -1, 3))
pred_cam = torch.cat(pred_cam, dim=0)
pred_verts = torch.cat(pred_verts, dim=0)
pred_pose = torch.cat(pred_pose, dim=0)
pred_betas = torch.cat(pred_betas, dim=0)
pred_joints3d = torch.cat(pred_joints3d, dim=0)
del batch
# ========= [Optional] run Temporal SMPLify to refine the results ========= #
if args.run_smplify and args.tracking_method == 'pose':
norm_joints2d = np.concatenate(norm_joints2d, axis=0)
norm_joints2d = convert_kps(norm_joints2d, src='staf', dst='spin')
norm_joints2d = torch.from_numpy(norm_joints2d).float().to(device)
# Run Temporal SMPLify
update, new_opt_vertices, new_opt_cam, new_opt_pose, new_opt_betas, \
new_opt_joints3d, new_opt_joint_loss, opt_joint_loss = smplify_runner(
pred_rotmat=pred_pose,
pred_betas=pred_betas,
pred_cam=pred_cam,
j2d=norm_joints2d,
device=device,
batch_size=norm_joints2d.shape[0],
pose2aa=False,
)
# update the parameters after refinement
print(
f'Update ratio after Temporal SMPLify: {update.sum()} / {norm_joints2d.shape[0]}'
)
pred_verts = pred_verts.cpu()
pred_cam = pred_cam.cpu()
pred_pose = pred_pose.cpu()
pred_betas = pred_betas.cpu()
pred_joints3d = pred_joints3d.cpu()
pred_verts[update] = new_opt_vertices[update]
pred_cam[update] = new_opt_cam[update]
pred_pose[update] = new_opt_pose[update]
pred_betas[update] = new_opt_betas[update]
pred_joints3d[update] = new_opt_joints3d[update]
elif args.run_smplify and args.tracking_method == 'bbox':
print(
'[WARNING] You need to enable pose tracking to run Temporal SMPLify algorithm!'
)
print('[WARNING] Continuing without running Temporal SMPLify!..')
# ========= Save results to a pickle file ========= #
output_path = image_folder.replace('cropped_frames', 'vibe_results')
os.makedirs(output_path, exist_ok=True)
pred_cam = pred_cam.cpu().numpy()
pred_verts = pred_verts.cpu().numpy()
pred_pose = pred_pose.cpu().numpy()
pred_betas = pred_betas.cpu().numpy()
pred_joints3d = pred_joints3d.cpu().numpy()
vibe_results = {
'pred_cam': pred_cam,
'verts': pred_verts,
'pose': pred_pose,
'betas': pred_betas,
'joints3d': pred_joints3d,
}
del model
end = time.time()
fps = len(dataset) / (end - vibe_time)
print(f'VIBE FPS: {fps:.2f}')
total_time = time.time() - total_time
print(
f'Total time spent: {total_time:.2f} seconds (including model loading time).'
)
print(
f'Total FPS (including model loading time): {len(dataset) / total_time:.2f}.'
)
print(
f'Saving vibe results to \"{os.path.join(output_path, "vibe_results.pkl")}\".'
)
with open(os.path.join(output_path, "vibe_results.pkl"), 'wb') as f_save:
pickle.dump(vibe_results, f_save)
if not args.no_render:
# ========= Render results as a single video ========= #
renderer = Renderer(resolution=(orig_width, orig_height),
orig_img=True,
wireframe=args.wireframe)
output_img_folder = os.path.join(output_path, 'vibe_images')
os.makedirs(output_img_folder, exist_ok=True)
print(f'Rendering output video, writing frames to {output_img_folder}')
image_file_names = sorted([
os.path.join(image_folder, x) for x in os.listdir(image_folder)
if x.endswith('.png') or x.endswith('.jpg')
])
for frame_idx in tqdm(range(len(image_file_names))):
img_fname = image_file_names[frame_idx]
img = cv2.imread(img_fname)
frame_verts = vibe_results['verts'][frame_idx]
frame_cam = vibe_results['pred_cam'][frame_idx]
mesh_filename = None
if args.save_obj:
mesh_folder = os.path.join(output_path, 'vibe_meshes')
os.makedirs(mesh_folder, exist_ok=True)
mesh_filename = os.path.join(mesh_folder,
f'{frame_idx:06d}.obj')
rend_img = renderer.render(
img,
frame_verts,
cam=frame_cam,
mesh_filename=mesh_filename,
)
whole_img = rend_img
if args.sideview:
side_img_bg = np.zeros_like(img)
side_rend_img90 = renderer.render(
side_img_bg,
frame_verts,
cam=frame_cam,
angle=90,
axis=[0, 1, 0],
)
side_rend_img270 = renderer.render(
side_img_bg,
frame_verts,
cam=frame_cam,
angle=270,
axis=[0, 1, 0],
)
if args.reposed_render:
smpl = SMPL('data/vibe_data', batch_size=1)
zero_pose = torch.from_numpy(
np.zeros((1, pred_pose.shape[-1]))).float()
zero_pose[:, 0] = np.pi
pred_frame_betas = torch.from_numpy(
pred_betas[frame_idx][None, :]).float()
with torch.no_grad():
reposed_smpl_output = smpl(
betas=pred_frame_betas,
body_pose=zero_pose[:, 3:],
global_orient=zero_pose[:, :3])
reposed_verts = reposed_smpl_output.vertices
reposed_verts = reposed_verts.cpu().detach().numpy()
reposed_cam = np.array([0.9, 0, 0])
reposed_rend_img = renderer.render(side_img_bg,
reposed_verts[0],
cam=reposed_cam)
reposed_rend_img90 = renderer.render(side_img_bg,
reposed_verts[0],
cam=reposed_cam,
angle=90,
axis=[0, 1, 0])
top_row = np.concatenate(
[img, reposed_rend_img, reposed_rend_img90], axis=1)
bot_row = np.concatenate(
[rend_img, side_rend_img90, side_rend_img270], axis=1)
whole_img = np.concatenate([top_row, bot_row], axis=0)
else:
top_row = np.concatenate([img, side_img_bg, side_img_bg],
axis=1)
bot_row = np.concatenate(
[rend_img, side_rend_img90, side_rend_img270], axis=1)
whole_img = np.concatenate([top_row, bot_row], axis=0)
# cv2.imwrite(os.path.join(output_img_folder, f'{frame_idx:06d}.png'), whole_img)
cv2.imwrite(
os.path.join(output_img_folder, os.path.basename(img_fname)),
whole_img)
# ========= Save rendered video ========= #
save_vid_path = os.path.join(output_path, 'vibe_video.mp4')
print(f'Saving result video to {save_vid_path}')
images_to_video(img_folder=output_img_folder,
output_vid_file=save_vid_path)
print('================= END =================')
def BostonData_view():
# Load the dataset
db = datasets.load_boston()
print("数据特征项:", db.feature_names)
db_X = db.data
print("X行列数:", db_X.shape)
db_y = db.target
print("y行列数:", db_y.shape)
# 构造pandas的Frame格式，X特征值
df = pd.DataFrame(db_X)
# 特征值名称【列名】
df.columns = db.feature_names
# 定义结果名称【列名】为Price，并且将target的值赋值给此列
df['Price'] = db_y
# 显示前5前数据【包括表头】
print("boston数据预览:\n", df.head())
# 制作组图，不同特征属性的散点图
sns.set(style='whitegrid', context='notebook')
cols = ['LSTAT', 'INDUS', 'NOX', 'RM', 'Price']
sns.pairplot(df[cols], height=2)
plt.show()
# 指定特征值与目标值的相关系数矩阵
# 可视化相关系数矩阵，理论：皮尔逊相关系数
cm = np.corrcoef(df[cols].values.T)
sns.set(font_scale=1.3)
hm = sns.heatmap(cm,
cbar=True,
annot=True,
square=True,
fmt='.2f',
annot_kws={'size': 13},
yticklabels=cols,
xticklabels=cols)
plt.show()
# 显示矩阵热力图的一半
mask = np.zeros_like(cm)
print(mask)
mask[np.triu_indices_from(mask)] = True
with sns.axes_style("white"):
ax = sns.heatmap(cm,
mask=mask,
vmax=1,
annot=True,
square=True,
fmt='.2f',
annot_kws={'size': 13},
yticklabels=cols,
xticklabels=cols)
plt.show()
# 所有特征值的相关系数，取相关系数大于0.5的制作相关系数矩阵
# 不用修改直接运行
corrmat = df.corr().abs() # 计算连续型特征之间的相关系数
# 将于SalePrice的相关系数大于5的特征取出来，并按照SalePrice降序排列，然后取出对应的特征名，保存在列表中
top_corr = corrmat[corrmat["Price"] > 0.5].sort_values(
by=["Price"], ascending=False).index
cm = abs(np.corrcoef(
df[top_corr].values.T)) # 注意这里要转置，否则变成样本之间的相关系数，而我们要计算的是特征之间的相关系数
# f, ax = plt.subplots(figsize=(20, 9))
sns.set(font_scale=1.3)
hm = sns.heatmap(cm,
cbar=True,
annot=True,
cmap='YlGnBu',
square=True,
fmt='.2f',
annot_kws={'size': 13},
yticklabels=top_corr.values,
xticklabels=top_corr.values)
plt.show()
# full_cost = np.sum(2*(np.sqrt(1 + np.array(along_track_diffs)**2) - 1))
full_cost = np.sum(np.abs(np.array(full_residuals)))/len(full_residuals)
full_cost_list.append(full_cost)
print('3D cost:', full_cost)
#print('RMS cost:', np.sum(rms_cost))
### PLOT VELOCITIES
plt.scatter(velocities[1:], obs_time[1:])
x_times = np.linspace(np.min(obs_time), np.max(obs_time), 100)
#plt.plot(line(x_times, *popt), x_times)
plt.plot(np.zeros_like(x_times) + v_init, x_times)
plt.gca().invert_yaxis()
plt.show()
# # Load the numpy array with the results from a file
# solutions = np.load(results_file)
# # Find the best solution among the top N
# best_solution = findBestSolution(solutions)
# print('BEST SOLUTION:', best_solution)
frameClone = old_frame.copy()
for o in range(op.nPoints):
for m in range(len(detected_key_points[o])):
cv.circle(frameClone, detected_key_points[o][m][0:2], 5, op.colors[o],
-1, cv.LINE_AA)
p0 = []
for key_point in detected_key_points:
for person_kp in key_point:
p0.append(np.matrix(person_kp[0:2]).astype(np.float32))
p0 = np.array(p0)
print(type(cv.goodFeaturesToTrack(old_gray, mask=None, **feature_params)[0]))
print(type(p0[0]))
# Create a mask image for drawing purposes
mask = np.zeros_like(old_frame)
while 1:
ret, frame = cap.read()
frame = frame[9:710, 443:836]
frame_gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
# calculate optical flow
p1, st, err = cv.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None,
**lk_params)
# Select good points
good_new = p1[st == 1]
good_old = p0[st == 1]
# draw the tracks
vertical_spacing = 0.05 # in meters
max_depth_of_section = 5 # meters
fp = 'deltaRCM_Output/pyDeltaRCM_output.nc'
nc = netCDF4.Dataset(fp)
strata_sf = nc.variables['strata_sand_frac'][:]
strata_depth = nc.variables['strata_depth'][:]
# shortcuts for array sizes
dz, dx, dy = strata_depth.shape
nx, ny, nz = dx, dy, int(5 / vertical_spacing)
# preserves only the oldest surface when cross-cutting
strata = np.zeros_like(strata_depth)
strata[-1, :, :] = strata_depth[-1, :, :]
for i in range(1, dz):
strata[-i - 1, :, :] = np.minimum(strata_depth[-i - 1, :, :],
strata[-i, :, :])
# combines depths and sand fractions into stratigraphy
stratigraphy = np.zeros((nz, nx, ny))
for j in range(dx):
mask = np.ones((nz, ny)) * -1
for i in np.arange(dz - 1, -1, -1):
def reset_test_buffer(self):
self.test_buffer = np.zeros_like(self.test_buffer, dtype=np.float32)
self.test_buff_pos = 0
self.test_size = 0
def make_covmap(goodval, wave_phase, pixel_phase):
covmap = np.zeros_like(wave_phase)
for i, j in zip(goodval[0], goodval[1]):
wavevals = wave_phase[:, i, j]
pixvals = pixel_phase[:, i, j]
hit_w = np.zeros(100)
for m in range(0, wavevals.size):
wval = int(round(wavevals[m] * 100))
if (wval == 100):
wval = 0 # Wrap around
if (wval < 25):
hit_w[wval:wval + 25] = 1
hit_w[0:wval] = 1
hit_w[100 - (25 - wval):] = 1
if ((wval >= 25) and (wval < 75)):
hit_w[wval - 25:wval + 25] = 1
if (wval >= 75):
hit_w[wval - 25:wval] = 1
hit_w[wval:] = 1
hit_w[0:wval - 75] = 1
hit_p = np.zeros(100)
for m in range(0, pixvals.size):
pval = int(round(pixvals[m] * 100))
if (pval == 100):
pval = 0 # ; Wrap around
if (pval < 25):
hit_p[pval:pval + 25] = 1
hit_p[0:pval] = 1
hit_p[100 - (25 - pval):] = 1
if ((pval >= 25) and (pval < 75)):
hit_p[pval - 25:pval + 25] = 1
if (pval >= 75):
hit_p[pval - 25:pval] = 1
hit_p[pval:] = 1
hit_p[0:pval - 75] = 1
temp_w = idlwrap.where(hit_w == 1)
nhit_w = len(temp_w)
temp_p = idlwrap.where(hit_p == 1)
nhit_p = len(temp_p)
covmap[0, i, j] = nhit_w / 100.
covmap[1, i, j] = nhit_p / 100.
return covmap
def _serialize_event_exp(self, config):
"""serialize event expression and sent them to game engine"""
game = self.game
# collect agent symbol
symbol2int = {}
config.symbol_ct = 0
def collect_agent_symbol(node, config):
for item in node.inputs:
if isinstance(item, EventNode):
collect_agent_symbol(item, config)
elif isinstance(item, AgentSymbol):
if item not in symbol2int:
symbol2int[item] = config.symbol_ct
config.symbol_ct += 1
for rule in config.reward_rules:
on = rule[0]
receiver = rule[1]
for symbol in receiver:
if symbol not in symbol2int:
symbol2int[symbol] = config.symbol_ct
config.symbol_ct += 1
collect_agent_symbol(on, config)
# collect event node
event2int = {}
config.node_ct = 0
def collect_event_node(node, config):
if node not in event2int:
event2int[node] = config.node_ct
config.node_ct += 1
for item in node.inputs:
if isinstance(item, EventNode):
collect_event_node(item, config)
for rule in config.reward_rules:
collect_event_node(rule[0], config)
# send to C++ engine
for sym in symbol2int:
no = symbol2int[sym]
_LIB.gridworld_define_agent_symbol(game, no, sym.group, sym.index)
for event in event2int:
no = event2int[event]
inputs = np.zeros_like(event.inputs, dtype=np.int32)
for i, item in enumerate(event.inputs):
if isinstance(item, EventNode):
inputs[i] = event2int[item]
elif isinstance(item, AgentSymbol):
inputs[i] = symbol2int[item]
else:
inputs[i] = item
n_inputs = len(inputs)
_LIB.gridworld_define_event_node(game, no, event.op, as_int32_c_array(inputs), n_inputs)
for rule in config.reward_rules:
# rule = [on, receiver, value, terminal]
on = event2int[rule[0]]
receiver = np.zeros_like(rule[1], dtype=np.int32)
for i, item in enumerate(rule[1]):
receiver[i] = symbol2int[item]
if len(rule[2]) == 1 and rule[2][0] == 'auto':
value = np.zeros(receiver, dtype=np.float32)
else:
value = np.array(rule[2], dtype=np.float32)
n_receiver = len(receiver)
_LIB.gridworld_add_reward_rule(game, on, as_int32_c_array(receiver),
as_float_c_array(value), n_receiver, rule[3])
def setup_atm(filename_free,filename_close,filename_far,TI=0.11,N=24001):
print 'read data'
s = Time.time()
"""free FAST"""
# filename = '/Users/ningrsrch/Dropbox/Projects/waked-loads/BYU/BYU/C676_W8_T11.0_P0.0_m2D_L-1.0/Model.out'
lines = np.loadtxt(filename_free,skiprows=8)
time = lines[:,0]
# mx = lines[:,11]
my = lines[:,12]
a = lines[:,5]
# o_free = lines[:,6]
Omega_free = np.mean(lines[:,6])
time = time-time[0]
m_free = interp1d(time, my, kind='linear')
a_free = interp1d(time, a, kind='linear')
# o_free = interp1d(time, o_free, kind='cubic')
# m_free = my
"""waked FAST CLOSE"""
# filename = '/Users/ningrsrch/Dropbox/Projects/waked-loads/BYU/BYU/C653_W8_T11.0_P0.0_4D_L0/Model.out'
lines = np.loadtxt(filename_close,skiprows=8)
time = lines[:,0]
# mx = lines[:,11]
my = lines[:,12]
a = lines[:,5]
# o_waked = lines[:,6]
Omega_waked = np.mean(lines[:,6])
time = time-time[0]
m_close = interp1d(time, my, kind='linear')
a_close = interp1d(time, a, kind='linear')
# o_close = interp1d(time, o_waked, kind='cubic')
# m_close = my
"""waked FAST FAR"""
# filename = '/Users/ningrsrch/Dropbox/Projects/waked-loads/BYU/BYU/C671_W8_T11.0_P0.0_10D_L0/Model.out'
lines = np.loadtxt(filename_far,skiprows=8)
time = lines[:,0]
# mx = lines[:,11]
my = lines[:,12]
a = lines[:,5]
# o_waked = lines[:,6]
time = time-time[0]
m_far = interp1d(time, my, kind='linear')
a_far = interp1d(time, a, kind='linear')
# o_far = interp1d(time, o_waked, kind='cubic')
# m_far = my
print Time.time()-s
"""setup the CCBlade loads"""
angles = np.linspace(0.,360.,50)
ccblade_free = np.zeros_like(angles)
ccblade_close = np.zeros_like(angles)
ccblade_far = np.zeros_like(angles)
turbineX_close = np.array([0.,126.4])*4.
turbineX_far = np.array([0.,126.4])*10.
turbineY_free = np.array([0.,1264000.])
turbineY_waked = np.array([0.,0.])
wind_speed = 8.
Rhub,r,chord,theta,af,Rhub,Rtip,B,rho,mu,precone,hubHt,nSector,pitch,yaw_deg = setup_airfoil()
print 'make CCBlade functions'
s = Time.time()
for i in range(len(angles)):
az = angles[i]
#freestream
x_locs,y_locs,z_locs = findXYZ(turbineX_close[1],turbineY_free[1],90.,r,yaw_deg,az)
speeds, _ = get_speeds(turbineX_close, turbineY_free, x_locs, y_locs, z_locs, wind_speed,TI=TI)
ccblade_free[i], _ = calc_moment(speeds,Rhub,r,chord,theta,af,Rhub,Rtip,B,rho,mu,precone,hubHt,nSector,Omega_free,pitch,azimuth=az)
#waked close
x_locs,y_locs,z_locs = findXYZ(turbineX_close[1],turbineY_waked[1],90.,r,yaw_deg,az)
speeds, _ = get_speeds(turbineX_close, turbineY_waked, x_locs, y_locs, z_locs, wind_speed,TI=TI)
ccblade_close[i], _ = calc_moment(speeds,Rhub,r,chord,theta,af,Rhub,Rtip,B,rho,mu,precone,hubHt,nSector,Omega_free,pitch,azimuth=az)
#waked far
x_locs,y_locs,z_locs = findXYZ(turbineX_far[1],turbineY_waked[1],90.,r,yaw_deg,az)
speeds, _ = get_speeds(turbineX_far, turbineY_waked, x_locs, y_locs, z_locs, wind_speed,TI=TI)
ccblade_far[i], _ = calc_moment(speeds,Rhub,r,chord,theta,af,Rhub,Rtip,B,rho,mu,precone,hubHt,nSector,Omega_waked,pitch,azimuth=az)
if i == 0:
free_speed = get_eff_turbine_speeds(turbineX_close, turbineY_free, wind_speed,TI=TI)[1]
waked_speed = get_eff_turbine_speeds(turbineX_close, turbineY_waked, wind_speed,TI=TI)[1]
f_free = interp1d(angles, ccblade_free/1000., kind='linear')
f_close = interp1d(angles, ccblade_close/1000., kind='linear')
f_far = interp1d(angles, ccblade_far/1000., kind='linear')
print Time.time()-s
t = np.linspace(0.,600.,N)
# t = time
dt = t[1]-t[0]
CC_free = np.zeros_like(t)
CC_close = np.zeros_like(t)
CC_far = np.zeros_like(t)
FAST_free = np.zeros_like(t)
FAST_close = np.zeros_like(t)
FAST_far = np.zeros_like(t)
"""get atm loads"""
# pos_free = 0.
# pos_close = 0.
# pos_far = 0.
print 'call CCBlade functions'
s = Time.time()
for i in range(len(t)):
# M_free[i] = f_free(pos_free)
# pos_free = (pos_free+(o_free((t[i+1]+t[i])/2.)*dt/60.)*360.)%360.
CC_free[i] = f_free(a_free(t[i]))
# M_close[i] = f_close(pos_close)
# pos_close = (pos_close+(o_close((t[i+1]+t[i])/2.)*dt/60.)*360.)%360.
CC_close[i] = f_close(a_close(t[i]))
# M_far[i] = f_far(pos_far)
# pos_far = (pos_far+(o_far((t[i+1]+t[i])/2.)*dt/60.)*360.)%360.
CC_far[i] = f_far(a_far(t[i]))
FAST_free[i] = m_free(t[i])
FAST_close[i] = m_close(t[i])
FAST_far[i] = m_far(t[i])
print Time.time()-s
atm_free = FAST_free-CC_free
atm_close = FAST_close-CC_close
atm_far = FAST_far-CC_far
# atm_free = m_free-CC_free
# atm_close = m_close-CC_close
# atm_far = m_far-CC_far
f_atm_free = interp1d(t, atm_free, kind='linear')
f_atm_close = interp1d(t, atm_close, kind='linear')
f_atm_far = interp1d(t, atm_far, kind='linear')
return f_atm_free,f_atm_close,f_atm_far,Omega_free,Omega_waked,free_speed,waked_speed
def __init__(self, model, upsample_size=UPSAMPLE_SIZE):
mask_size = np.ceil(np.array((32, 32), dtype=float) /
upsample_size)
mask_size = mask_size.astype(int)
self.mask_size = mask_size
mask = np.zeros(self.mask_size)
pattern = np.zeros((32, 32, 3))
mask = np.expand_dims(mask, axis=2)
mask_tanh = np.zeros_like(mask)
pattern_tanh = np.zeros_like(pattern)
# prepare mask related tensors
self.mask_tanh_tensor = K.variable(mask_tanh)
mask_tensor_unrepeat = (K.tanh(self.mask_tanh_tensor) \
/ (2 - K.epsilon()) + 0.5)
mask_tensor_unexpand = K.repeat_elements(
mask_tensor_unrepeat,
rep=3,
axis=2)
self.mask_tensor = K.expand_dims(mask_tensor_unexpand, axis=0)
upsample_layer = UpSampling2D(
size=(upsample_size, upsample_size))
mask_upsample_tensor_uncrop = upsample_layer(self.mask_tensor)
uncrop_shape = K.int_shape(mask_upsample_tensor_uncrop)[1:]
cropping_layer = Cropping2D(
cropping=((0, uncrop_shape[0] - 32),
(0, uncrop_shape[1] - 32)))
self.mask_upsample_tensor = cropping_layer(
mask_upsample_tensor_uncrop)
# self.mask_upsample_tensor = K.round(self.mask_upsample_tensor)
reverse_mask_tensor = (K.ones_like(self.mask_upsample_tensor) -
self.mask_upsample_tensor)
# prepare pattern related tensors
self.pattern_tanh_tensor = K.variable(pattern_tanh)
self.pattern_raw_tensor = (
(K.tanh(self.pattern_tanh_tensor) / (2 - K.epsilon()) + 0.5) *
255.0)
# prepare input image related tensors
# ignore clip operation here
# assume input image is already clipped into valid color range
input_tensor = K.placeholder((None,32,32,3))
input_raw_tensor = input_tensor
# IMPORTANT: MASK OPERATION IN RAW DOMAIN
X_adv_raw_tensor = (
reverse_mask_tensor * input_raw_tensor +
self.mask_upsample_tensor * self.pattern_raw_tensor)
X_adv_tensor = X_adv_raw_tensor
output_tensor = model(X_adv_tensor)
y_target_tensor = K.placeholder((None,43))
y_true_tensor = K.placeholder((None,43))
self.loss_ce = categorical_crossentropy(output_tensor, y_target_tensor)
self.hyperparameters = K.reshape(K.constant(np.array([1e-2, 1e-5, 1e-7, 1e-8, 0, 1e-2])), shape=(6, 1))
self.loss_reg = self.build_tabor_regularization(input_raw_tensor,
model, y_target_tensor,
y_true_tensor)
self.loss_reg = K.dot(K.reshape(self.loss_reg, shape=(1, 6)), self.hyperparameters)
self.loss = K.mean(self.loss_ce) + self.loss_reg
self.opt = Adam(lr=1e-3, beta_1=0.5, beta_2=0.9)
self.updates = self.opt.get_updates(
params=[self.pattern_tanh_tensor, self.mask_tanh_tensor],
loss=self.loss)
self.train = K.function(
[input_tensor, y_true_tensor, y_target_tensor],
[self.loss_ce, self.loss_reg, self.loss],
updates=self.updates)
def corner_detectTrack(camA, camB):
# Generate some random colors later used to display movement tracks
color3 = np.ones((100,1))*255; color2 = np.zeros((100,1)); color1 = np.zeros((100,1))
color = np.hstack((color1,color2,color3))
index = 0
#get ROIs from sv_getROIs module
[masked_grays1, boundboxes1] = get_ROIs(camA[0])
[masked_grays2, boundboxes2] = get_ROIs(camB[0])
# Take first frame
old_frame = cv2.imread(camA[0])
old_gray = cv2.imread(camA[0],0)
slidesA = [[None] for x in range(len(first_cam)-1)] #images from camA used for plotting
slidesB = [[None] for x in range(len(second_cam)-1)] #images from camB used for plotting
disps_roi = {'ROI # ' + str(u): 0 for u in range(num_roi)}
for q in range(0,num_roi):
imgA_before = []; imgA_after = []; H_matA = []
imgB_before = []; imgB_after = []; H_matB = []
#params for ShiTomasi corner detection
feature_params = dict(maxCorners = 100,
qualityLevel = 0.3,
minDistance = 7,
blockSize = 7,
gradientSize = 7,
mask = masked_grays2[q]) #ROI from trials
# goodFeaturesToTrack determines strong corners on an image
# can be used to initialize any point-based tracker such as the calcOpticalFlowPyrLK
# first search for the features in the rois from the first camera;
# search for those same initialized features in all the images in the second camera
p0 = cv2.goodFeaturesToTrack(old_gray, **feature_params) #search for features from first camera pic
prev_coord = p0
print('number of corners found: '+ str(len(p0)))
# Create a mask image for drawing purposes
mask = np.zeros_like(old_frame)
################################################################## for cam A ###########################################################
while index < (len(camA)-1):
print('image' + str(index))
frame = cv2.imread(camA[index+1])
frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Parameters for lucas kanade optical flow
lk_params = dict( winSize = (15,15),
maxLevel = 10,
criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10,0.03))
# calculate optical flow
# for a sparse feature set using the iterative LK method with pyramids
p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params)
#p1 = p1[st==1]
#p1 = p1[st==1] #shouldn't p1 = p1[st==1]??; i.e. only found points?
#print('p0 = ' + str(len(p0))); print('p1 = ' + str(len(p1))); print('prev_coord = ' + str(len(prev_coord)))
if (len(p1) != len(prev_coord)):
print('didnt find same corners')
#can't find corners --> iterate over new lk parameters
for i in range(20,100,2): #change criteria for max number of iterations first
lk_params['maxLevel'] = i
print('# Levels = ' + str(i))
p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params) #recalculate p1
#p1 = p1[st==1] #shouldn't p1 = p1[st==1]??
#print ('new p1 length = ' + str(len(p1)))
if (len(p1) != len(prev_coord)):
print('still bad. continue changing parameters')
continue
else:
print('found good parameters')
break
else:
print('found same corners')
#print('p0 = ' + str(len(p0))); print('p1 = ' + str(len(p1))); print('prev_coord = ' + str(len(prev_coord)))
#previous block of code should ensure that len(p1) == len(prev_coord)
H, Hmask = cv2.findHomography(p0, p1, cv2.RANSAC,5.0) #H is 3x3 homography matrix
#not really needed
#print(prev_coord[0][0])
imgA_before.append(prev_coord)
imgA_after.append(p1)
H_matA.append(H)
prev_coord = p1
#print(prev_coord[0][0])
# Select good points #does p0 need to be reshaped to this good_new at the end? shouldn't p1 = p1[st==1]??
good_new = p1
good_old = p0
# for i in range(len(p1)):
# frame = cv2.putText(frame, str(i), (int(p1[i][0]),int(p1[i][1]+100)), cv2.FONT_HERSHEY_TRIPLEX, 3, (0, 0, 255), 5)
# # draw the tracks
# for i,(new,old) in enumerate(zip(good_new,good_old)):
# a,b = new.ravel()
# c,d = old.ravel()
# mask = cv2.line(mask, (a,b),(c,d), color[i].tolist(), 10)
# frame = cv2.circle(frame,(a,b),7,color[i].tolist(),-1, lineType = 8)
#
#
# img = cv2.add(frame,mask)
# img = cv2.add(img,boundboxes1[q]) #plot ROIs and OG image; boundbox from trials
#
# #slides[index][q] = img #problems updating (only first collumn is updating)
# #slides = np.vstack((slides,img))
# #add images from consective images to slides_frames
# slidesA[index] = img
# Now update the previous frame and previous points
old_frame = frame.copy()
index = index + 1
#p0 = good_new.reshape(-1,1,2)
index = 0; #reset index for camB
print('finished tracking images from camA')
################################################################## for camB ###########################################################
old_frame = cv2.imread(camB[0])
old_gray = cv2.imread(camB[0],0)
while index < (len(camB)-1):
print('image' + str(index))
frame = cv2.imread(camB[index+1])
frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Parameters for lucas kanade optical flow
lk_params = dict( winSize = (15,15),
maxLevel = 10,
criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10,0.03))
# calculate optical flow
# for a sparse feature set using the iterative LK method with pyramids
p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params)
#p1 = p1[st==1]
#p1 = p1[st==1] #shouldn't p1 = p1[st==1]??; i.e. only found points?
#print('p0 = ' + str(len(p0))); print('p1 = ' + str(len(p1))); print('prev_coord = ' + str(len(prev_coord)))
if (len(p1) != len(prev_coord)):
print('didnt find same corners')
#can't find corners --> iterate over new lk parameters
for i in range(20,100,2): #change criteria for max number of iterations first
lk_params['maxLevel'] = i
print('# Levels = ' + str(i))
p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params) #recalculate p1
#p1 = p1[st==1] #shouldn't p1 = p1[st==1]??
#print ('new p1 length = ' + str(len(p1)))
if (len(p1) != len(prev_coord)):
print('still bad. continue changing parameters')
continue
else:
print('found good criteria')
break
else:
print('found same corners')
#print('p0 = ' + str(len(p0))); print('p1 = ' + str(len(p1))); print('prev_coord = ' + str(len(prev_coord)))
#previous block of code should ensure that len(p1) == len(prev_coord)
H, Hmask = cv2.findHomography(p0, p1, cv2.RANSAC,5.0) #H is 3x3 homography matrix
#not really needed
#print(prev_coord[0][0])
imgB_before.append(prev_coord)
imgB_after.append(p1)
H_matB.append(H)
prev_coord = p1
#print(prev_coord[0][0])
# Select good points #does p0 need to be reshaped to this good_new at the end? shouldn't p1 = p1[st==1]??
good_new = p1
good_old = p0
# for i in range(len(p1)):
# frame = cv2.putText(frame, str(i), (int(p1[i][0]),int(p1[i][1]+100)), cv2.FONT_HERSHEY_TRIPLEX, 3, (0, 0, 255), 5)
# draw the tracks
for i,(new,old) in enumerate(zip(good_new,good_old)):
a,b = new.ravel()
c,d = old.ravel()
mask = cv2.line(mask, (a,b),(c,d), color[i].tolist(), 12)
frame = cv2.circle(frame,(a,b),7,color[i].tolist(),-1, lineType = 10)
img = cv2.add(frame,mask)
img = cv2.add(img,boundboxes2[q]) #plot ROIs and OG image; boundbox from trials
#slides[index][q] = img #problems updating (only first collumn is updating)
#slides = np.vstack((slides,img))
#add images from consective images to slides_frames
slidesB[index] = img
# Now update the previous frame and previous points
old_frame = frame.copy()
index = index + 1
#p0 = good_new.reshape(-1,1,2)
index = 0 #reset index for new ROI
print('finished tracking images from camB')
tot_disps = get_3Ddisps(imgA_before, imgA_after, imgB_before, imgB_after)
disps_roi['ROI # ' + str(q)] = tot_disps
#iterate over each consecutive image set, stack ROIs for a respective set together, display, move to next set
#f, axarr = plt.subplots(2,2)
for s in range(0,len(slidesB)): #for now, just displays on camB images, and two images for the two ROIs
#plot_image = np.concatenate((slidesA[s], slidesB[s]), axis=1) #slide[image set#][roi #]
plot_image = slidesB[s]
plot_image = cv2.resize(plot_image, (1440, 810 )) #do we need to resize
cv2.imshow('all ROIs', plot_image)
# #axarr[1,0].imshow(slides[2][s])
# #axarr[1,1].imshow(slides[3][s])
#
k = cv2.waitKey(0) & 0xff
if k == 27:
break
cv2.destroyAllWindows()
return [tot_disps, disps_roi]
def get_loads_history(turbineX,turbineY,turb_index,Omega_free,Omega_waked,free_speed,waked_speed,f_atm_free,f_atm_close,f_atm_far,N=24001,TI=0.11,wind_speed=8.,rotor_diameter=126.4):
# def get_loads_history(turbineX,turbineY,turb_index,Omega_free,Omega_waked,free_speed,waked_speed,atm_free,atm_close,atm_far,time,wind_speed=8.,rotor_diameter=126.4,TI=0.11):
# print 'get loads history'
# npts = len(time)
nTurbines = len(turbineX)
Rhub,r,chord,theta,af,Rhub,Rtip,B,rho,mu,precone,hubHt,nSector,pitch,yaw_deg = setup_airfoil()
angles = np.linspace(0.,360.,100)
ccblade_moments = np.zeros_like(angles)
_, wake_radius = get_speeds(turbineX, turbineY, np.array([0.]), np.array([0.]), np.array([0.]), wind_speed, TI=TI)
# print 'getting CCBlade moments'
s = Time.time()
for i in range(len(ccblade_moments)):
az = angles[i]
x_locs,y_locs,z_locs = findXYZ(turbineX[turb_index],turbineY[turb_index],90.,r,yaw_deg,az)
speeds, _ = get_speeds(turbineX, turbineY, x_locs, y_locs, z_locs, wind_speed,TI=TI)
if i == 0:
actual_speed = get_eff_turbine_speeds(turbineX, turbineY, wind_speed,TI=TI)[1]
Omega = (Omega_waked + (Omega_free-Omega_waked)/(free_speed-waked_speed) * (actual_speed-waked_speed))
ccblade_moments[i], _ = calc_moment(speeds,Rhub,r,chord,theta,af,Rhub,Rtip,B,rho,mu,precone,hubHt,nSector,Omega,pitch,azimuth=az)
f_ccblade = interp1d(angles, ccblade_moments/1000., kind='linear')
# print Time.time()-s
t = np.linspace(0.,600.,N)
# t = time
dt = t[1]-t[0]
# moments = np.zeros(N)
moments = np.zeros(N)
m = np.zeros(N)
position = 0.
# s = Time.time()
# for i in range(N):
for i in range(N):
# print '1'
# s = Time.time()
amnt_waked = np.zeros(nTurbines)
dx_dist = np.zeros(nTurbines)
for waking in range(nTurbines):
dx = turbineX[turb_index]-turbineX[waking]
dy = turbineY[turb_index]-turbineY[waking]
dx_dist[waking] = dx
if dx < 0.:
amnt_waked[waking] = 0.
else:
amnt_waked[waking] = amount_waked(dy,wake_radius[turb_index][waking]*1.75,rotor_diameter,position)
waked_array = np.zeros(nTurbines)
dx_array = np.zeros(nTurbines)
# print Time.time()-s
# print '2'
# s = Time.time()
num = 0
indices = np.argsort(dx_dist)
for waking in range(nTurbines):
if dx_dist[indices[waking]] > 0.:
# if num == 0:
# if amnt_waked[indices[waking]] > 0.:
# waked_array[num] = amnt_waked[indices[waking]]
# dx_array[num] = dx_dist[indices[waking]]
# num += 1
# else:
if amnt_waked[indices[waking]] > np.sum(waked_array[0:num]):
waked_array[num] = amnt_waked[indices[waking]]-np.sum(waked_array[0:num])
dx_array[num] = dx_dist[indices[waking]]
num += 1
# print Time.time()-s
# print '3'
# s = Time.time()
down = dx_array/rotor_diameter
moments[i] = f_ccblade(position)
m[i] = moments[i]
unwaked = 1.-np.sum(waked_array)
# print 'unwaked', unwaked
for k in range(np.count_nonzero(waked_array)):
if down[k] < 4.:
moments[i] += f_atm_close(t[i])*waked_array[k]
# moments[i] += atm_close[i]*waked_array[k]
elif down[k] > 10.:
moments[i] += f_atm_far(t[i])*waked_array[k]
else:
moments[i] += (f_atm_close(t[i])*(10.-down[k])/6.+f_atm_far(t[i])*(down[k]-4.)/6.)*waked_array[k]
moments[i] += f_atm_free(t[i])*unwaked
position = (position+(Omega*(dt)/60.)*360.)%360.
# print Time.time()-s
# plt.plot(t,f_atm_close(t))
# plt.plot(t,m)
if turb_index == 1.:
plt.plot(t,moments)
# plt.show()
return moments
def bright_wise_histogram_equalization(self, img_arr, level=256, **args):
### split the image to three level according brightness, equalize histogram dividely
### @params img_arr : numpy.array uint8 type, 2-dim
### @params level : gray scale
### @return arr : the equalized image array
def special_histogram(img_arr, min_v, max_v):
### calculate a special histogram with max, min value
### @params img_arr : 1-dim numpy.array
### @params min_v : min gray scale
### @params max_v : max gray scale
### @return hists : list type, length = max_v - min_v + 1
hists = [0 for _ in range(max_v - min_v + 1)]
for v in img_arr:
hists[v - min_v] += 1
return hists
def special_histogram_cdf(hists, min_v, max_v):
### calculate a special histogram cdf with max, min value
### @params hists : list type
### @params min_v : min gray scale
### @params max_v : max gray scale
### @return hists_cdf : numpy.array
hists_cumsum = np.cumsum(np.array(hists))
hists_cdf = (max_v - min_v) / hists_cumsum[-1] * hists_cumsum + min_v
hists_cdf = hists_cdf.astype("uint8")
return hists_cdf
def pseudo_variance(arr):
### calculate a type of variance
### @params arr : 1-dim numpy.array
arr_abs = np.abs(arr - np.mean(arr))
return np.mean(arr_abs)
# search two grayscale level, which can split the image into
# three parts having approximately same number of pixels
(m, n) = img_arr.shape
hists = self.calc_histogram_(img_arr)
hists_arr = np.cumsum(np.array(hists))
hists_ratio = hists_arr / hists_arr[-1]
scale1 = None
scale2 = None
for i in range(len(hists_ratio)):
if hists_ratio[i] >= 0.333 and scale1 is None:
scale1 = i
if hists_ratio[i] >= 0.667 and scale2 is None:
scale2 = i
break
# split images
dark_index = (img_arr <= scale1)
mid_index = (img_arr > scale1) & (img_arr <= scale2)
bright_index = (img_arr > scale2)
# variance
dark_variance = pseudo_variance(img_arr[dark_index])
mid_variance = pseudo_variance(img_arr[mid_index])
bright_variance = pseudo_variance(img_arr[bright_index])
# build three level images
dark_img_arr = np.zeros_like(img_arr)
mid_img_arr = np.zeros_like(img_arr)
bright_img_arr = np.zeros_like(img_arr)
# histogram equalization individually
dark_hists = special_histogram(img_arr[dark_index], 0, scale1)
dark_cdf = special_histogram_cdf(dark_hists, 0, scale1)
mid_hists = special_histogram(img_arr[mid_index], scale1, scale2)
mid_cdf = special_histogram_cdf(mid_hists, scale1, scale2)
bright_hists = special_histogram(img_arr[bright_index], scale2, level - 1)
bright_cdf = special_histogram_cdf(bright_hists, scale2, level - 1)
def plot_hists(arr):
hists = [0 for i in range(256)]
for a in arr:
hists[a] += 1
self.draw_histogram_(hists)
# mapping
dark_img_arr[dark_index] = dark_cdf[img_arr[dark_index]]
mid_img_arr[mid_index] = mid_cdf[img_arr[mid_index] - scale1]
bright_img_arr[bright_index] = bright_cdf[img_arr[bright_index] - scale2]
# weighted sum
# fractor = dark_variance + mid_variance + bright_variance
# arr = (dark_variance * dark_img_arr + mid_variance * mid_img_arr + bright_variance * bright_img_arr)/fractor
arr = dark_img_arr + mid_img_arr + bright_img_arr
arr = arr.astype("uint8")
return arr
inh5file = h5py.File(fullfile, 'r')
x = inh5file['default']['turns'].value
# intv = int(np.ceil(len(x)/points))
for key in inh5file['default'].keys():
if (key in not_plot):
continue
val = inh5file['default'][key].value
if (key == 'profile'):
val = val[-1]
val = val.reshape(len(val))
if (key not in plot_dir):
plot_dir[key] = {}
if (workers not in plot_dir[key]):
plot_dir[key][workers] = {'num': 0}
if ('sum' not in plot_dir[key][workers]):
plot_dir[key][workers]['sum'] = np.zeros_like(val)
plot_dir[key][workers]['min'] = val
plot_dir[key][workers]['max'] = val
plot_dir[key][workers]['num'] += 1
plot_dir[key][workers]['sum'] += val
plot_dir[key][workers]['min'] = np.minimum(
plot_dir[key][workers]['min'], val)
plot_dir[key][workers]['max'] = np.maximum(
plot_dir[key][workers]['max'], val)
plot_dir[key][workers]['turns'] = x
inh5file.close()
# continue here, I need to iterate over the errors, create a figure for each
# iterate over the reduce values, add an error plot line for each acording to the intv etc
for ts in tss:
def get_item_based_topk(self, items, top_k=10, sort_top_k=True):
"""Get top K similar items to provided seed items based on similarity metric defined.
This method will take a set of items and use them to recommend the most similar items to that set
based on the similarity matrix fit during training.
This allows recommendations for cold-users (unseen during training), note - the model is not updated.
The following options are possible based on information provided in the items input:
1. Single user or seed of items: only item column (ratings are assumed to be 1)
2. Single user or seed of items w/ ratings: item column and rating column
3. Separate users or seeds of items: item and user column (user ids are only used to separate item sets)
4. Separate users or seeds of items with ratings: item, user and rating columns provided
Args:predict
items (pd.DataFrame): DataFrame with item, user (optional), and rating (optional) columns
top_k (int): number of top items to recommend
sort_top_k (bool): flag to sort top k results
Returns:
pd.DataFrame: sorted top k recommendation items
"""
# convert item ids to indices
item_ids = np.asarray(
list(
map(
lambda item: self.item2index.get(item, np.NaN),
items[self.col_item].values,
)))
# if no ratings were provided assume they are all 1
if self.col_rating in items.columns:
ratings = items[self.col_rating]
else:
ratings = pd.Series(np.ones_like(item_ids))
# create local map of user ids
if self.col_user in items.columns:
test_users = items[self.col_user]
user2index = {
x[1]: x[0]
for x in enumerate(items[self.col_user].unique())
}
user_ids = test_users.map(user2index)
else:
# if no user column exists assume all entries are for a single user
test_users = pd.Series(np.zeros_like(item_ids))
user_ids = test_users
n_users = user_ids.drop_duplicates().shape[0]
# generate pseudo user affinity using seed items
pseudo_affinity = sparse.coo_matrix(
(ratings, (user_ids, item_ids)),
shape=(n_users, self.n_items)).tocsr()
# calculate raw scores with a matrix multiplication
test_scores = pseudo_affinity.dot(self.item_similarity)
# remove items in the seed set so recommended items are novel
test_scores[user_ids, item_ids] = -np.inf
top_items, top_scores = get_top_k_scored_items(scores=test_scores,
top_k=top_k,
sort_top_k=sort_top_k)
df = pd.DataFrame({
self.col_user:
np.repeat(test_users.drop_duplicates().values, top_items.shape[1]),
self.col_item:
[self.index2item[item] for item in top_items.flatten()],
self.col_prediction:
top_scores.flatten(),
})
# drop invalid items
return df.replace(-np.inf, np.nan).dropna()
def sample_labelswitch(self):
"""
Tries to switch two random labels
"""
if npr.rand() < self.p_switch:
if self.K < 2:
return np.array([-1, -1])
labels = npr.choice(self.K, 2, replace=False)
if np.sum(self.active_komp[labels]) == 0:
return np.array([-1, -1])
lik_old, R_S_mu0, log_det_Q0, R_S0 = self.likelihood_prior(
self.mu[labels[0]],
self.sigma[labels[0]],
labels[0],
switchprior=True)
lik_oldt, R_S_mu1, log_det_Q1, R_S1 = self.likelihood_prior(
self.mu[labels[1]],
self.sigma[labels[1]],
labels[1],
switchprior=True)
#updated added alpha contribution
alpha_ = np.zeros_like(self.logisticNormal.alpha)
alpha_[:] = self.logisticNormal.alpha[:]
llik_alpha, _, __ = self.logisticNormal.get_lprior_grad_hess(
alpha_)
self.p[labels[0]], self.p[labels[1]] = self.p[labels[1]], self.p[
labels[0]]
self.logisticNormal.set_alpha_p(self.p)
lliks_alpha, _, __ = self.logisticNormal.get_lprior_grad_hess()
lik_old += lik_oldt + llik_alpha
lik_star = self.likelihood_prior(self.mu[labels[1]],
self.sigma[labels[1]],
labels[0],
R_S_mu0,
log_det_Q0,
R_S1,
switchprior=True)[0]
lik_star += self.likelihood_prior(self.mu[labels[0]],
self.sigma[labels[0]],
labels[1],
R_S_mu1,
log_det_Q1,
R_S0,
switchprior=True)[0]
lik_star += lliks_alpha
if np.log(npr.rand()) < lik_star - lik_old:
self.active_komp[labels[0]], self.active_komp[
labels[1]] = self.active_komp[labels[1]], self.active_komp[
labels[0]]
self.mu[labels[0]], self.mu[labels[1]] = self.mu[
labels[1]], self.mu[labels[0]]
self.sigma[labels[0]], self.sigma[labels[1]] = self.sigma[
labels[1]], self.sigma[labels[0]]
self.p[labels[0]], self.p[labels[1]] = self.p[
labels[1]], self.p[labels[0]]
self.updata_mudata()
return labels
self.logisticNormal.set_alpha(alpha_)
self.p = self.logisticNormal.get_p()
return np.array([-1, -1])
def window_mask(width, height, img_ref, center, level):
output = np.zeros_like(img_ref)
output[int(img_ref.shape[0] - (level + 1) * height):int(img_ref.shape[0] - level * height),
max(0, int(center - width / 2)):min(int(center + width / 2), img_ref.shape[1])] = 1
return output
w_static = w_static.assign_coords({'times': list(mask.keys())})
##############################################################################
# Compute peaks for each roi
##############################################################################
unique_rois = np.unique(w_static.roi.values)
freqs = w_static.freqs.values
peaks = np.zeros((len(unique_rois), w_static.sizes['freqs']))
peaks = xr.DataArray(peaks, dims=('roi', 'freqs'),
coords={'roi': unique_rois,
'freqs': freqs})
for i, roi in enumerate(unique_rois):
counts = np.zeros_like(freqs)
idx = w_static.roi.values == roi
w_sel = w_static.isel(roi=idx).stack(T=("trials", "roi")).sel(times="baseline")
p_idx = []
for t in range(w_sel.sizes["T"]):
temp, _ = find_peaks(w_sel[:, t], threshold=1e-8)
p_idx += [temp]
p_idx = list(itertools.chain(*p_idx))
p_idx, c = np.unique(p_idx, return_counts=True)
counts[p_idx.astype(int)] = c
peaks[i, :] = counts
# Path to results folder
_RESULTS = os.path.join(_ROOT,
"Results/lucy/peaks",
f"{sidx}.nc")
def cross_entropy_neighbors_in_rep(adata, use_rep, n_points=3):
"""Compare neighborhood graph of representation based on cross entropy.
`n_points` denotes the number of points to add as highlight annotation.
Returns
-------
The cross entropy and the geodesic-distance-weighted cross entropy as
``entropy, geo_entropy_d, geo_entropy_o``.
Adds the most overlapping or disconnected points as annotation to `adata`.
"""
# see below why we need this
if 'X_diffmap' not in adata.obsm.keys():
raise ValueError('Run `tl.diffmap` on `adata`, first.')
adata_ref = adata # simple renaming, don't need copy here
adata_cmp = adata.copy()
n_neighbors = adata_ref.uns['neighbors']['params']['n_neighbors']
from .neighbors import neighbors
neighbors(adata_cmp, n_neighbors=n_neighbors, use_rep=use_rep)
from .tools.diffmap import diffmap
diffmap(adata_cmp)
graph_ref = adata_ref.uns['neighbors']['connectivities']
graph_cmp = adata_cmp.uns['neighbors']['connectivities']
graph_ref = graph_ref.tocoo() # makes a copy
graph_cmp = graph_cmp.tocoo()
edgeset_ref = {e for e in zip(graph_ref.row, graph_ref.col)}
edgeset_cmp = {e for e in zip(graph_cmp.row, graph_cmp.col)}
edgeset_union = list(edgeset_ref.union(edgeset_cmp))
edgeset_union_indices = tuple(zip(*edgeset_union))
edgeset_union_indices = (np.array(edgeset_union_indices[0]), np.array(edgeset_union_indices[1]))
n_edges_ref = len(graph_ref.nonzero()[0])
n_edges_cmp = len(graph_cmp.nonzero()[0])
n_edges_union = len(edgeset_union)
logg.msg(
'... n_edges_ref', n_edges_ref,
'n_edges_cmp', n_edges_cmp,
'n_edges_union', n_edges_union)
graph_ref = graph_ref.tocsr() # need a copy of the csr graph anyways
graph_cmp = graph_cmp.tocsr()
p_ref = graph_ref[edgeset_union_indices].A1
p_cmp = graph_cmp[edgeset_union_indices].A1
# the following is how one compares it to log_loss form sklearn
# p_ref[p_ref.nonzero()] = 1
# from sklearn.metrics import log_loss
# print(log_loss(p_ref, p_cmp))
p_cmp = np.clip(p_cmp, EPS, 1-EPS)
ratio = np.clip(p_ref / p_cmp, EPS, None)
ratio_1m = np.clip((1 - p_ref) / (1 - p_cmp), EPS, None)
entropy = np.sum(p_ref * np.log(ratio) + (1-p_ref) * np.log(ratio_1m))
n_edges_fully_connected = (graph_ref.shape[0]**2 - graph_ref.shape[0])
entropy /= n_edges_fully_connected
fraction_edges = n_edges_ref / n_edges_fully_connected
naive_entropy = (fraction_edges * np.log(1./fraction_edges)
+ (1-fraction_edges) * np.log(1./(1-fraction_edges)))
logg.msg('cross entropy of naive sparse prediction {:.3e}'.format(naive_entropy))
logg.msg('cross entropy of random prediction {:.3e}'.format(-np.log(0.5)))
logg.info('cross entropy {:.3e}'.format(entropy))
# for manifold analysis, restrict to largest connected component in
# reference
# now that we clip at a quite high value below, this might not even be
# necessary
n_components, labels = scipy.sparse.csgraph.connected_components(graph_ref)
largest_component = np.arange(graph_ref.shape[0], dtype=int)
if n_components > 1:
component_sizes = np.bincount(labels)
logg.msg('largest component has size', component_sizes.max())
largest_component = np.where(
component_sizes == component_sizes.max())[0][0]
graph_ref_red = graph_ref.tocsr()[labels == largest_component, :]
graph_ref_red = graph_ref_red.tocsc()[:, labels == largest_component]
graph_ref_red = graph_ref_red.tocoo()
graph_cmp_red = graph_cmp.tocsr()[labels == largest_component, :]
graph_cmp_red = graph_cmp_red.tocsc()[:, labels == largest_component]
graph_cmp_red = graph_cmp_red.tocoo()
edgeset_ref_red = {e for e in zip(graph_ref_red.row, graph_ref_red.col)}
edgeset_cmp_red = {e for e in zip(graph_cmp_red.row, graph_cmp_red.col)}
edgeset_union_red = edgeset_ref_red.union(edgeset_cmp_red)
map_indices = np.where(labels == largest_component)[0]
edgeset_union_red = {
(map_indices[i], map_indices[j]) for (i, j) in edgeset_union_red}
from .neighbors import Neighbors
neigh_ref = Neighbors(adata_ref)
dist_ref = neigh_ref.distances_dpt # we expect 'X_diffmap' to be already present
neigh_cmp = Neighbors(adata_cmp)
dist_cmp = neigh_cmp.distances_dpt
d_cmp = np.zeros_like(p_ref)
d_ref = np.zeros_like(p_ref)
for i, e in enumerate(edgeset_union):
# skip contributions that are not in the largest component
if n_components > 1 and e not in edgeset_union_red:
continue
d_cmp[i] = dist_cmp[e]
d_ref[i] = dist_ref[e]
MAX_DIST = 1000
d_cmp = np.clip(d_cmp, 0.1, MAX_DIST) # we don't want to measure collapsing clusters
d_ref = np.clip(d_ref, 0.1, MAX_DIST)
weights = np.array(d_cmp / d_ref) # disconnected regions
weights_overlap = np.array(d_ref / d_cmp) # overlapping regions
# the following is just for annotation of figures
if 'highlights' not in adata_ref.uns:
adata_ref.uns['highlights'] = {}
else:
# remove old disconnected and overlapping points
new_highlights = {}
for k, v in adata_ref.uns['highlights'].items():
if v != 'O' and v not in {'D0', 'D1', 'D2', 'D3', 'D4'}:
new_highlights[k] = v
adata_ref.uns['highlights'] = new_highlights
# points that are maximally disconnected
max_weights = np.argpartition(weights, kth=-n_points)[-n_points:]
points = list(edgeset_union_indices[0][max_weights])
points2 = list(edgeset_union_indices[1][max_weights])
found_disconnected_points = False
for ip, p in enumerate(points):
if d_cmp[max_weights][ip] == MAX_DIST:
adata_ref.uns['highlights'][p] = 'D' + str(ip)
adata_ref.uns['highlights'][points2[ip]] = 'D' + str(ip)
found_disconnected_points = True
if found_disconnected_points:
logg.msg('most disconnected points', points)
logg.msg(' with weights', weights[max_weights].round(1))
max_weights = np.argpartition(
weights_overlap, kth=-n_points)[-n_points:]
points = list(edgeset_union_indices[0][max_weights])
for p in points:
adata_ref.uns['highlights'][p] = 'O'
logg.msg('most overlapping points', points)
logg.msg(' with weights', weights_overlap[max_weights].round(1))
logg.msg(' with d_rep', d_cmp[max_weights].round(1))
logg.msg(' with d_ref', d_ref[max_weights].round(1))
geo_entropy_d = np.sum(weights * p_ref * np.log(ratio))
geo_entropy_o = np.sum(weights_overlap * (1-p_ref) * np.log(ratio_1m))
geo_entropy_d /= n_edges_fully_connected
geo_entropy_o /= n_edges_fully_connected
logg.info('geodesic cross entropy {:.3e}'.format(geo_entropy_d + geo_entropy_o))
return entropy, geo_entropy_d, geo_entropy_o
plt.figure(figsize=(12, 288), facecolor='white')
df_ints = t_data.select_dtypes([float])
for c in df_ints.columns:
ax = plt.subplot(200, 4, plot_number)
sns.distplot(train[c])
plt.xticks(rotation=45)
plot_number = plot_number + 1
plt.tight_layout()
plt.show()
## looking at a correlation matrix
t_corr = t_data.corr()
# Set up the matplotlib figure
fig, ax = plt.subplots(figsize=(11, 9))
mask = np.zeros_like(t_corr)
mask[np.triu_indices_from(mask)] = True
with sns.axes_style("white"):
ax = sns.heatmap(t_corr,
mask=mask,
vmax=.3,
square=True,
center=0,
cmap=sns.color_palette("BrBG", 7))
plt.show()
print(t_corr)
from statsmodels.stats.outliers_influence import variance_inflation_factor
from statsmodels.tools.tools import add_constant
n_input = (r * 2 + 1)**2
n_hidden = 15
n_output = 5
w1 = np.random.randn(n_hidden, n_input) / np.sqrt(np.prod(n_input))
w2 = np.random.randn(5, n_hidden) / np.sqrt(n_hidden)
#ws = np.zeros((1, 5, 5))
#ws[0, 2, 2] = 0.1
#w1 = np.zeros((1, 5, 5))
#w1[0, 3, 2] = 0.1
#w2 = [1.0]
#agent.set_weights(ws, w1, w2)
rmsprop_dw1 = np.zeros_like(w1)
rmsprop_dw2 = np.zeros_like(w2)
mean_reward = []
all_w1 = []
all_w2 = []
upda = 0
noup = 0
nextupdate = datetime.datetime.now() + datetime.timedelta(seconds=10)
while True:
aifile = "net.txt"
with open(aifile, 'w') as f:
print('n_input:', w1.shape[1], file=f)
print('n_hidden:', w1.shape[0], file=f)
print('n_output:', w2.shape[0], file=f)
# result in an error if LaTeX is not installed on your system. In that case, # you can set usetex to False. from astroML.plotting import setup_text_plots setup_text_plots(fontsize=8, usetex=True) #------------------------------------------------------------ # Generate random x, y with a given covariance length np.random.seed(1) x = np.linspace(0, 1, 500) h = 0.01 C = np.exp(-0.5 * (x - x[:, None]) ** 2 / h ** 2) y = 0.8 + 0.3 * np.random.multivariate_normal(np.zeros(len(x)), C) #------------------------------------------------------------ # Define a normalized top-hat window function w = np.zeros_like(x) w[(x > 0.12) & (x < 0.28)] = 1 #------------------------------------------------------------ # Perform the convolution y_norm = np.convolve(np.ones_like(y), w, mode='full') valid_indices = (y_norm != 0) y_norm = y_norm[valid_indices] y_w = np.convolve(y, w, mode='full')[valid_indices] / y_norm # trick: convolve with x-coordinate to find the center of the window at # each point. x_w = np.convolve(x, w, mode='full')[valid_indices] / y_norm #------------------------------------------------------------
if __name__ == '__main__':
import pycuda.driver as drv
import numpy
import pycuda.autoinit
mod = drv.SourceModule("""
__global__ void multiply_them(float *dest, float *a, float *b)
{
const int i = threadIdx.x;
dest[i] = a[i] * b[i];
}
""")
multiply_them = mod.get_function("multiply_them")
a = numpy.random.randn(400).astype(numpy.float32)
b = numpy.random.randn(400).astype(numpy.float32)
dest = numpy.zeros_like(a)
multiply_them(drv.Out(dest), drv.In(a), drv.In(b), block=(400, 1, 1))
print dest - a * b
qf = {}
fibre_marc = {}
fibre_marc_e = {}
for yi,by in enumerate(xyz_pf):
# brass_y2[:,:,yi] = by
HxY[yi] = np.cross(symHKL[0],by)
HxY[yi] = HxY[yi] / np.linalg.norm(HxY[yi],axis=1)[:,None]
ome[yi] = np.arccos(np.dot(symHKL[0],by))
q0[yi] = {}
q[yi] = {}
qf[yi] = {}
fibre_marc[yi] = np.zeros_like(brassFibre[yi])
for hi,h in enumerate(HxY[yi]):
q0[yi][hi] = quat.normalize(np.hstack( [ np.cos(ome[yi][hi]/2), np.sin(ome[yi][hi]/2) * h ] ))
q[yi][hi] = quat.normalize(np.hstack( [ cphi[:, np.newaxis], np.tile( by, (len(cphi),1) ) * sphi[:, np.newaxis] ] ))
qf[yi][hi] = quat.multiply(q[yi][hi], q0[yi][hi])
for qi in range(qf[yi][hi].shape[0]):
fibre_marc[yi][qi,hi,:] = qf[yi][hi][qi,:]
phi1, Phi, phi2 = quat2eu(fibre_marc[yi])
phi1 = np.where(phi1 < 0, phi1 + 2*np.pi, phi1) #brnng back to 0 - 2pi
def wind_plot():
"""Create southerly wind test data."""
v = np.full((5, 5), 10, dtype=np.float64)
u = np.zeros_like(v)
x, y = np.meshgrid(np.linspace(-120, -60, 5), np.linspace(25, 50, 5))
return u, v, x, y
def getPreprocessedImage(self, evtNumber, image_property):
disp_medianCorrection = 19
disp_radialCorrection = 18
disp_gainMask = 17
disp_coordy = 16
disp_coordx = 15
disp_col = 14
disp_row = 13
disp_seg = 12
disp_quad = 11
disp_gain = 10
disp_commonMode = 9
disp_rms = 8
disp_status = 7
disp_pedestal = 6
disp_photons = 5
disp_raw = 4
disp_pedestalCorrected = 3
disp_commonModeCorrected = 2
disp_adu = 1
if image_property == disp_medianCorrection: # median subtraction
print "Sorry, this feature isn't available yet"
elif image_property == disp_radialCorrection: # radial subtraction + polarization corrected
self.getEvent(evtNumber)
calib = self.getCalib(evtNumber)
if calib:
self.pf.shape = self.parent.calib.shape
calib = self.rb.subtract_bkgd(calib * self.pf)
elif image_property == disp_adu: # gain and hybrid gain corrected
calib = self.getCalib(evtNumber)
elif image_property == disp_commonModeCorrected: # common mode corrected
calib = self.getCommonModeCorrected(evtNumber)
elif image_property == disp_pedestalCorrected: # pedestal corrected
calib = self.det.raw(self.evt).astype('float32')
if calib: calib -= self.det.pedestals(self.evt)
elif image_property == disp_raw: # raw
calib = self.det.raw(self.evt)
elif image_property == disp_photons: # photon counts
calib = self.det.photons(
self.evt,
mask=self.parent.mk.userMask,
adu_per_photon=self.parent.exp.aduPerPhoton)
if calib is None:
calib = np.zeros_like(self.parent.exp.detGuaranteed,
dtype='int32')
elif image_property == disp_pedestal: # pedestal
calib = self.parent.det.pedestals(self.parent.evt)
elif image_property == disp_status: # status
calib = self.parent.det.status(self.parent.evt)
elif image_property == disp_rms: # rms
calib = self.parent.det.rms(self.parent.evt)
elif image_property == disp_commonMode: # common mode
calib = self.getCommonMode(evtNumber)
elif image_property == disp_gain: # gain
calib = self.parent.det.gain(self.parent.evt)
elif image_property == disp_gainMask: # gain_mask
calib = self.parent.det.gain_mask(self.parent.evt)
elif image_property == disp_coordx: # coords_x
calib = self.parent.det.coords_x(self.parent.evt)
elif image_property == disp_coordy: # coords_y
calib = self.parent.det.coords_y(self.parent.evt)
shape = self.parent.det.shape(self.parent.evt)
if len(shape) == 3:
if image_property == disp_quad: # quad ind
calib = np.zeros(shape)
for i in range(shape[0]):
# FIXME: handle detectors properly
if shape[0] == 32: # cspad
calib[i, :, :] = int(i) % 8
elif shape[0] == 2: # cspad2x2
calib[i, :, :] = int(i) % 2
elif shape[0] == 4: # pnccd
calib[i, :, :] = int(i) % 4
elif image_property == disp_seg: # seg ind
calib = np.zeros(shape)
if shape[0] == 32: # cspad
for i in range(32):
calib[i, :, :] = int(i) / 8
elif shape[0] == 2: # cspad2x2
for i in range(2):
calib[i, :, :] = int(i)
elif shape[0] == 4: # pnccd
for i in range(4):
calib[i, :, :] = int(i)
elif image_property == disp_row: # row ind
calib = np.zeros(shape)
if shape[0] == 32: # cspad
for i in range(185):
calib[:, i, :] = i
elif shape[0] == 2: # cspad2x2
for i in range(185):
calib[:, i, :] = i
elif shape[0] == 4: # pnccd
for i in range(512):
calib[:, i, :] = i
elif image_property == disp_col: # col ind
calib = np.zeros(shape)
if shape[0] == 32: # cspad
for i in range(388):
calib[:, :, i] = i
elif shape[0] == 2: # cspad2x2
for i in range(388):
calib[:, :, i] = i
elif shape[0] == 4: # pnccd
for i in range(512):
calib[:, :, i] = i
def rewinder_likelihood(dt,
nsteps,
potential,
prog_xv,
star_xv,
m0,
mdot,
alpha,
betas,
theta,
selfgravity=False):
# full array of initial conditions for progenitor and stars
w0 = np.vstack((prog_xv, star_xv))
# integrate orbits
t, w = potential.integrate_orbit(w0, dt=dt, nsteps=nsteps)
t = t[:-1]
w = w[:-1]
# satellite mass
sat_mass = -mdot * t + m0
GMprog = potential.parameters['G'] * sat_mass
# compute approximations of tidal radius and velocity dispersion from mass enclosed
menc = potential.mass_enclosed(w[:, 0, :3]) # progenitor position orbit
E_scale = (sat_mass / menc)**(1 / 3.)
# compute naive tidal radius and velocity dispersion
rtide = E_scale * np.linalg.norm(w[:, 0, :3],
axis=-1) # progenitor orbital radius
vdisp = E_scale * np.linalg.norm(w[:, 0, 3:],
axis=-1) # progenitor orbital velocity
# get the instantaneous orbital plane basis vectors (x1,x2,x3)
basis = get_basis(w[:, 0], theta)
# star orbits relative to progenitor
dw = w[:, 1:] - w[:, 0:1]
# project orbits into new basis
w123 = np.zeros_like(dw)
for i in range(3):
w123[..., i] = np.sum(dw[..., :3] * basis[..., i][:, np.newaxis],
axis=-1)
w123[..., i + 3] = np.sum(dw[..., 3:] * basis[..., i][:, np.newaxis],
axis=-1)
w123[..., i] += alpha * betas[np.newaxis] * rtide[:, np.newaxis]
# write like this to allow for more general dispersions...probably want a covariance matrix
sigmas = np.zeros((nsteps, 1, 6))
sigmas[:, 0, 0] = rtide
sigmas[:, 0, 1] = rtide
sigmas[:, 0, 2] = rtide
sigmas[:, 0, 3] = vdisp
sigmas[:, 0, 4] = vdisp
sigmas[:, 0, 5] = vdisp
g = -0.5 * np.log(2 * np.pi) - np.log(sigmas) - 0.5 * (w123 / sigmas)**2
# compute an estimate of the jacobian
Rsun = 8.
R2 = (w[:, 1:, 0] + Rsun)**2 + w[:, 1:, 1]**2 + w[:, 1:, 2]**2
x2 = w[:, 1:, 2]**2 / R2
log_jac = np.log(R2 * R2 * np.sqrt(1. - x2))
return g.sum(axis=-1) + log_jac, w
# 'V5AxleArticHGV', 'V6orMoreAxleArticHGV', 'AllHGVs', 'AllMotorVehicles',
# 'Lat', 'Lon'])
#
#accidents_train = pd.merge(accidents_train, traffic_data, on = ['Location_Easting_OSGR','Location_Northing_OSGR'])
accidents_train['B_Latitude'] = np.array(pd.qcut(accidents_train['Latitude'], q=15, precision=1).astype(str))
accidents_train['B_Longitude'] = np.array(pd.qcut(accidents_train['Longitude'], q=10, precision=1).astype(str))
accidents_train['M_LAT_LON'] = accidents_train[['B_Latitude', 'B_Longitude']].apply(lambda x: '-'.join(x), axis=1)
accidents_train.drop(['B_Latitude', 'B_Longitude'], axis=1, inplace=True)
corr = accidents_train.corr()
# Generate a mask for the upper triangle
mask = np.zeros_like(corr, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
# Set up the matplotlib figure
f, ax = plt.subplots(figsize=(11,9))
# Generate a custom diverging colormap
cmap = sns.diverging_palette(220,10,as_cmap=True)
# Draw the heatmap with he mask and correct aspect ratio
sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0, square=True, linewidth=.5, cbar_kws={"shrink":.5})
# Drop null columns
accidents_train = accidents_train.drop(['Junction_Detail'], axis=1)
# Drop string variables with a lot of different values
accidents_train.drop(['LSOA_of_Accident_Location', 'Local_Authority_(District)', 'Local_Authority_(Highway)', '1st_Road_Number', '2nd_Road_Number'], axis=1, inplace=True)