def analyze(odd_ecc,correct):
def func(pars):
a,b,c = pars
return a*(x**2) + b*x + c
def errfunc(pars):
return y-func(pars)
#Analysis:
#Bin the eccentricities:
a = np.floor(odd_ecc)
eccs_used = np.unique(a)
#Initialize counters:
b = np.zeros(len(eccs_used))
c = np.zeros(len(eccs_used))
#Loop over the trials and add the correct to one counter and the number of
#trials to the other:
for i in xrange(len(correct)):
idx = np.where(eccs_used==np.floor(odd_ecc[i]))
b[idx]+=correct[i]
c[idx]+=1
p_correct = b/c
for i,p in enumerate(p_correct):
plt.plot(eccs_used[i],p,'o',color='b',markersize=c[i])
x = []
y = []
for i,this_ecc in enumerate(eccs_used):
x = np.hstack([x,c[i]*[this_ecc]])
y = np.hstack([y,c[i]*[p_correct[i]]])
def test_dofs(vector_array):
v = vector_array
np.random.seed(len(v) + 24 + v.dim)
for ind in valid_inds(v):
c = v.copy()
dofs = c[ind].dofs(np.array([], dtype=np.int))
assert isinstance(dofs, np.ndarray)
assert dofs.shape == (v.len_ind(ind), 0)
c = v.copy()
dofs = c[ind].dofs([])
assert isinstance(dofs, np.ndarray)
assert dofs.shape == (v.len_ind(ind), 0)
if v.dim > 0:
for count in (1, 5, 10):
c_ind = np.random.randint(0, v.dim, count)
c = v.copy()
dofs = c[ind].dofs(c_ind)
assert dofs.shape == (v.len_ind(ind), count)
c = v.copy()
dofs2 = c[ind].dofs(list(c_ind))
assert np.all(dofs == dofs2)
c = v.copy()
c.scal(3.)
dofs2 = c[ind].dofs(c_ind)
assert np.allclose(dofs * 3, dofs2)
c = v.copy()
dofs2 = c[ind].dofs(np.hstack((c_ind, c_ind)))
assert np.all(dofs2 == np.hstack((dofs, dofs)))
if hasattr(v, 'data'):
assert np.all(dofs == indexed(v.data, ind)[:, c_ind])
def moving_something(_fun, tseries, period, day_rs=16, is_days=True):
"""
Applies a function to a moving window of the time-series:
ft_ = function([ f(t-N), f(t). f(t+N)])
"""
# if the time-series is at a day-time step, update the window to a step-size of 16 days
if is_days:
p0 = period*day_rs
else:
p0 = period
# find upper and lower bounds of the moving window
half = p0//2
tlen = len(tseries)
twin = [0]*tlen
for im in range(tlen):
# find the something for the window that satisfy the edge conditions
if im < half:
# fold back onto the end of the time-series
twin[im] = _fun(np.hstack([tseries[tlen-(half-im):tlen],\
tseries[0:im+half]]))
elif im > tlen-half:
# fold back into the beginning of the time-series
twin[im] = _fun(np.hstack([tseries[im-half:tlen],\
tseries[0:half-(tlen-im)]]))
else:
twin[im] = _fun(tseries[im-half:im+half])
return twin
def get_roc_score(edges_pos, edges_neg, score_matrix, apply_sigmoid=False):
# Edge case
if len(edges_pos) == 0 or len(edges_neg) == 0:
return (None, None, None)
# Store positive edge predictions, actual values
preds_pos = []
pos = []
for edge in edges_pos:
if apply_sigmoid == True:
preds_pos.append(sigmoid(score_matrix[edge[0], edge[1]]))
else:
preds_pos.append(score_matrix[edge[0], edge[1]])
pos.append(1) # actual value (1 for positive)
# Store negative edge predictions, actual values
preds_neg = []
neg = []
for edge in edges_neg:
if apply_sigmoid == True:
preds_neg.append(sigmoid(score_matrix[edge[0], edge[1]]))
else:
preds_neg.append(score_matrix[edge[0], edge[1]])
neg.append(0) # actual value (0 for negative)
# Calculate scores
preds_all = np.hstack([preds_pos, preds_neg])
labels_all = np.hstack([np.ones(len(preds_pos)), np.zeros(len(preds_neg))])
roc_score = roc_auc_score(labels_all, preds_all)
# roc_curve_tuple = roc_curve(labels_all, preds_all)
ap_score = average_precision_score(labels_all, preds_all)
# return roc_score, roc_curve_tuple, ap_score
return roc_score, ap_score
def torgerson(distances, n_components=2):
"""
Perform classical mds (Torgerson scaling).
..note ::
If the distances are euclidean then this is equivalent to projecting
the original data points to the first `n` principal components.
"""
distances = np.asarray(distances)
assert distances.shape[0] == distances.shape[1]
N = distances.shape[0]
# O ^ 2
D_sq = distances ** 2
# double center the D_sq
rsum = np.sum(D_sq, axis=1, keepdims=True)
csum = np.sum(D_sq, axis=0, keepdims=True)
total = np.sum(csum)
D_sq -= rsum / N
D_sq -= csum / N
D_sq += total / (N ** 2)
B = np.multiply(D_sq, -0.5, out=D_sq)
U, L, _ = np.linalg.svd(B)
if n_components > N:
U = np.hstack((U, np.zeros((N, n_components - N))))
L = np.hstack((L, np.zeros((n_components - N))))
U = U[:, :n_components]
L = L[:n_components]
D = np.diag(np.sqrt(L))
return np.dot(U, D)
def __generateInitialSpline(self, splineOrder, timeOffsetPadding, numberOfKnots = None, framerate = None):
poseSpline = bsplines.BSplinePose(splineOrder, sm.RotationVector())
# Get the observation times.
times = np.array([observation.time().toSec() for observation in self.__observations ])
# get the pose values of the initial transformations at observation time
curve = np.matrix([ poseSpline.transformationToCurveValue( observation.T_t_c().T() ) for observation in self.__observations]).T
# make sure all values are well defined
if np.isnan(curve).any():
raise RuntimeError("Nans in curve values")
sys.exit(0)
# Add 2 seconds on either end to allow the spline to slide during optimization
times = np.hstack((times[0] - (timeOffsetPadding * 2.0), times, times[-1] + (timeOffsetPadding * 2.0)))
curve = np.hstack((curve[:,0], curve, curve[:,-1]))
self.__ensureContinuousRotationVectors(curve)
seconds = times[-1] - times[0]
# fixed number of knots
if (numberOfKnots is not None):
knots = numberOfKnots
# otherwise with framerate estimate
else:
knots = int(round(seconds * framerate/3))
print
print "Initializing a pose spline with %d knots (%f knots per second over %f seconds)" % ( knots, 100, seconds)
poseSpline.initPoseSplineSparse(times, curve, knots, 1e-4)
return poseSpline
def gen_coastline(lon, lat, bathy, depth=0):
"""
Given lon, lat, and bathymetry, generate vectors of line segments
of the coastline. This can be exported to matlab (via savemat) to be
used with the 'editmask' routine for creating grid masks.
Input
-----
lon : array,
longitudes of bathymetry locations
lat : array,
latitudes of bathymetry locations
bathy : array,
bathymetry (negative for ocean, positive for land) values
depth : float,
depth to use as the definition of the coast
Returns
-------
lon : ndarray,
vector of coastlines, separated by nan (matlab-style)
lat : ndarray,
vector of coastlines, separated by nan (matlab-style)
"""
CS = plt.contour(lon, lat, bathy, [depth - 0.25, depth + 0.25])
lon = list()
lat = list()
for col in CS.collections:
for path in col.get_paths():
lon.append(path.vertices[:, 0])
lon.append(np.nan)
lat.append(path.vertices[:, 1])
lat.append(np.nan)
return (np.hstack(lon), np.hstack(lat))
def shift(data,shiftdirection = "center"):
numshifts = {"right":1,"left":1,"up":1,"down":1}
shiftfuncs = {
"right": lambda arr:np.hstack((zerocol(outarr),outarr)),
"left": lambda arr: np.hstack((outarr,zerocol(outarr))),
"up": lambda arr: np.vstack((outarr,zerorow(outarr))),
"down": lambda arr: np.vstack((zerorow(outarr),outarr))
}
directionpairs = \
{"right":"left","left":"right","up":"down","down":"up"}
if shiftdirection != "center":
numshifts[shiftdirection] += 1
numshifts[directionpairs[shiftdirection]] -= 1
outarr = cp.deepcopy(data)
zerocol = lambda arr: np.zeros((arr.shape[0],1))
zerorow = lambda arr: np.zeros((1,arr.shape[1]))
for direction in numshifts.keys():
for numtimes in xrange(numshifts[direction]):
outarr = shiftfuncs[direction](outarr)
return outarr
def __mul__(self, df):
"""
extract and stack poles and zeros
TODO : handling simplification
"""
b1 = self.b
a1 = self.a
b2 = df.b
a2 = df.a
pb1 = np.poly1d(b1)
pa1 = np.poly1d(a1)
pb2 = np.poly1d(b2)
pa2 = np.poly1d(a2)
rpb1 = pb1.r
rpb2 = pb2.r
rpa1 = pa1.r
rpa2 = pa2.r
F = DF()
F.p = np.hstack((rpa1, rpa2))
F.z = np.hstack((rpb1, rpb2))
F.simplify()
return F
def test_07_01_make_ijv_outlines(self):
np.random.seed(70)
x = cpo.Objects()
ii, jj = np.mgrid[0:10, 0:20]
masks = [(ii - ic) ** 2 + (jj - jc) ** 2 < r ** 2
for ic, jc, r in ((4, 5, 5), (4, 12, 5), (6, 8, 5))]
i = np.hstack([ii[mask] for mask in masks])
j = np.hstack([jj[mask] for mask in masks])
v = np.hstack([[k + 1] * np.sum(mask) for k, mask in enumerate(masks)])
x.set_ijv(np.column_stack((i, j, v)), ii.shape)
x.parent_image = cpi.Image(np.zeros((10, 20)))
colors = np.random.uniform(size=(3, 3)).astype(np.float32)
image = x.make_ijv_outlines(colors)
i1 = [i for i, color in enumerate(colors) if np.all(color == image[0, 5, :])]
self.assertEqual(len(i1), 1)
i2 = [i for i, color in enumerate(colors) if np.all(color == image[0, 12, :])]
self.assertEqual(len(i2), 1)
i3 = [i for i, color in enumerate(colors) if np.all(color == image[-1, 8, :])]
self.assertEqual(len(i3), 1)
self.assertNotEqual(i1[0], i2[0])
self.assertNotEqual(i2[0], i3[0])
colors = colors[np.array([i1[0], i2[0], i3[0]])]
outlines = np.zeros((10, 20, 3), np.float32)
alpha = np.zeros((10, 20))
for i, (color, mask) in enumerate(zip(colors, masks)):
my_outline = outline(mask)
outlines[my_outline] += color
alpha[my_outline] += 1
alpha[alpha == 0] = 1
outlines /= alpha[:, :, np.newaxis]
np.testing.assert_almost_equal(outlines, image)
def display_layer(X, filename="../images/layer.png"):
"""
Produces an image, composed of the given N images, patches or neural network weights,
stored in the array X. Saves it with the given filename.
:param X: numpy array of size (NxD) — N images, patches or neural network weights
:param filename: a string, the name of the produced file
:return: None
"""
if not isinstance(X, np.ndarray):
raise TypeError("'X' must be a numpy array")
N, D = X.shape
d = get_reshaped_image_size(D)
if N == 1:
return X.reshape(d, d, 3)
divizors = [n for n in range(1, N) if N % n == 0]
im_sizes = divizors[int(len(divizors) / 2)], int(N / divizors[int(len(divizors) / 2)])
for i in range(im_sizes[0]):
# img_row = np.hstack((img_row, np.zeros((d, 1, 3))))
img_row = np.hstack((np.zeros((d, 1, 3)), np.array(X[i * im_sizes[0], :].reshape(d, d, 3))))
img_row = np.hstack((img_row, np.zeros((d, 1, 3))))
for j in range(1, im_sizes[1]):
img_row = np.hstack((img_row, X[i * im_sizes[1] + j, :].reshape(d, d, 3)))
img_row = np.hstack((img_row, np.zeros((d, 1, 3))))
if i == 0:
img = img_row
else:
img = np.vstack((img, img_row))
img = np.vstack((img, np.zeros((1, img.shape[1], 3))))
img = np.vstack((np.zeros((1, img.shape[1], 3)), img))
imsave(filename, img)
return img
def evaluate_transformation_matrix(self, dynamic_values, constant_values):
"""Returns the numerical transformation matrices for each time step.
Parameters
----------
dynamic_values : array_like, shape(m,) or shape(n, m)
The m state values for each n time step.
constant_values : array_like, shape(p,)
The p constant parameter values of the system.
Returns
-------
transform_matrix : numpy.array, shape(n, 4, 4)
A 4 x 4 transformation matrix for each time step.
"""
#If states is instance of numpy array, well and good.
#else convert it to one:
states = np.array(dynamic_values)
if len(states.shape) > 1:
n = states.shape[0]
new = np.zeros((n, 4, 4))
for i, time_instance in enumerate(states):
args = np.hstack((time_instance, constant_values))
new[i, :, :] = self._numeric_transform(*args)
else:
n = 1
args = np.hstack((states, constant_values))
new = self._numeric_transform(*args)
self._visualization_matrix = new.reshape(n, 16)
return self._visualization_matrix
def getScalars(self, inputData):
"""
Returns a numpy array containing the sub-field scalar value(s) for
each sub-field of the inputData. To get the associated field names for each of
the scalar values, call getScalarNames().
For a simple scalar encoder, the scalar value is simply the input unmodified.
For category encoders, it is the scalar representing the category string
that is passed in. For the datetime encoder, the scalar value is the
the number of seconds since epoch.
The intent of the scalar representation of a sub-field is to provide a
baseline for measuring error differences. You can compare the scalar value
of the inputData with the scalar value returned from topDownCompute() on a
top-down representation to evaluate prediction accuracy, for example.
@param inputData The data from the source. This is typically a object with
members
@returns array of scalar values
"""
retVals = numpy.array([])
if self.encoders is not None:
for (name, encoder, offset) in self.encoders:
values = encoder.getScalars(self._getInputValue(inputData, name))
retVals = numpy.hstack((retVals, values))
else:
retVals = numpy.hstack((retVals, inputData))
return retVals
def _train(self, samples):
"""Perform network training.
Parameter
---------
samples : array-like
Used for unsupervised training of the SOM.
"""
# XXX initialize with clever default, e.g. plain of first two PCA
# components
self._K = np.random.standard_normal(tuple(self.kshape) + (samples.shape[1],))
# units weight vector deltas for batch training
# (height x width x #features)
unit_deltas = np.zeros(self._K.shape, dtype='float')
# precompute distance kernel between elements in the Kohonen layer
# that will remain constant throughout the training
# (just compute one quadrant, as the distances are symmetric)
# XXX maybe do other than squared Euclidean?
dqd = np.fromfunction(lambda x, y: (x**2 + y**2)**0.5,
self.kshape, dtype='float')
# for all iterations
for it in xrange(1, self.niter + 1):
# compute the neighborhood impact kernel for this iteration
# has to be recomputed since kernel shrinks over time
k = self._compute_influence_kernel(it, dqd)
# for all training vectors
for s in samples:
# determine closest unit (as element coordinate)
b = self._get_bmu(s)
# train all units at once by unfolding the kernel (from the
# single quadrant that is precomputed), cutting it to the
# right shape and simply multiply it to the difference of target
# and all unit weights....
infl = np.vstack((
np.hstack((
# upper left
k[b[0]:0:-1, b[1]:0:-1],
# upper right
k[b[0]:0:-1, :self.kshape[1] - b[1]])),
np.hstack((
# lower left
k[:self.kshape[0] - b[0], b[1]:0:-1],
# lower right
k[:self.kshape[0] - b[0], :self.kshape[1] - b[1]]))
))
unit_deltas += infl[:,:,np.newaxis] * (s - self._K)
# apply cumulative unit deltas
self._K += unit_deltas
if __debug__:
debug("SOM", "Iteration %d/%d done: ||unit_deltas||=%g" %
(it, self.niter, np.sqrt(np.sum(unit_deltas **2))))
# reset unit deltas
unit_deltas.fill(0.)
def save(self,filename):
num_objs = len(self.objects)
data_size = 0
desc = []
data = []
for obj in self.objects:
if isinstance(obj,self.scalars):
desc.append(0)
data.append(obj)
data_size += 1
else:
assert(isinstance(obj,np.ndarray))
desc.append(len(obj.shape))
desc.append(obj.shape)
data.append(obj.flatten(order='F'))
data_size += np.prod(obj.shape)
desc = np.hstack(desc)
header_size = 3 + desc.size
output = np.hstack([num_objs,header_size,data_size]
+ [desc]
+ data).astype(np.double)
assert((header_size + data_size,) == output.shape)
output.tofile(filename)
def get_subgroup(self, objpath):
"""
Load a set of image frames from a given subgroup, sorted by the
ls_z_measured value.
Returns a pair of (imgdata, metadata).
"""
node0 = self.h5file.get_node('/', objpath + '/0')
rowlen = 8
imgrows = []
metadata = {'size_x': int(node0.shape[0]), 'size_y': int(node0.shape[1]), 'framedata': []}
j = 0
for (i, node) in sorted(self.h5file.get_node('/', objpath)._v_children.items(), key = lambda i: i[1].attrs['ls_z_measured']):
if len(imgrows) <= j/rowlen:
imgrows.append(node.read())
else:
imgrows[j/rowlen] = numpy.hstack((imgrows[j/rowlen], node.read()))
metadata['framedata'].append(
{'t': int(node.attrs['ls_time']),
'n': int(node.attrs['ls_n']),
'z_r': float(node.attrs['ls_z_request']),
'z': float(node.attrs['ls_z_measured'])})
j += 1
# Fully extend the last row
imgrows[-1] = numpy.hstack((imgrows[-1], numpy.zeros([metadata['size_y'], rowlen * metadata['size_x'] - imgrows[-1].shape[1]])))
imgdata = numpy.vstack(imgrows)
self.cache[objpath] = { "imgdata": imgdata, "metadata": metadata }
def _stimcorr_core(self, motionfile, intensityfile, designmatrix, cwd=None):
"""
Core routine for determining stimulus correlation
"""
if not cwd:
cwd = os.getcwd()
# read in motion parameters
mc_in = np.loadtxt(motionfile)
g_in = np.loadtxt(intensityfile)
g_in.shape = g_in.shape[0], 1
dcol = designmatrix.shape[1]
mccol = mc_in.shape[1]
concat_matrix = np.hstack((np.hstack((designmatrix, mc_in)), g_in))
cm = np.corrcoef(concat_matrix, rowvar=0)
corrfile = self._get_output_filenames(motionfile, cwd)
# write output to outputfile
file = open(corrfile, 'w')
file.write("Stats for:\n")
file.write("Stimulus correlated motion:\n%s\n" % motionfile)
for i in range(dcol):
file.write("SCM.%d:" % i)
for v in cm[i, dcol + np.arange(mccol)]:
file.write(" %.2f" % v)
file.write('\n')
file.write("Stimulus correlated intensity:\n%s\n" % intensityfile)
for i in range(dcol):
file.write("SCI.%d: %.2f\n" % (i, cm[i, -1]))
file.close()
def test_elastic_basis_pursuit():
"""
"""
xx2d = np.array(np.meshgrid(np.arange(0,50,1),
np.arange(0,50,1)))
bounds = [[0,50], [0,50], [0, None], [0, None]]
mean1 = [20, 20]
sigma1 = [10, 10]
params1 = np.hstack([mean1, sigma1])
kernel1 = ebp.gaussian_kernel(xx2d, params1)
mean2 = [30, 40]
sigma2 = [10, 50]
params2 = np.hstack([mean2, sigma2])
kernel2 = ebp.gaussian_kernel(xx2d, params2)
beta = [0.3, 0.7]
signal = (beta[0] * kernel1 + beta[1] * kernel2).ravel()
initial_theta = np.hstack([np.mean(xx2d, axis=(-1, -2)), 1, 1])
theta, err, r = ebp.elastic_basis_pursuit(xx2d.reshape(-1, signal.shape[0]),
signal,
ebp.leastsq_oracle,
ebp.gaussian_kernel,
initial_theta=initial_theta,
bounds=bounds,
max_iter=1000,
beta_tol=10e-6)
def test_kernel_error():
"""
"""
for xx2d in [np.array(np.meshgrid(np.arange(-100, 100, 5),
np.arange(-100, 100, 5))),
np.array(np.meshgrid(np.arange(-100, 100, 5),
np.arange(-100, 100, 5))).reshape(2, -1)]:
mean1 = [20, 20]
sigma1 = [10, 10]
params1 = np.hstack([mean1, sigma1])
mean2 = [30, 40]
sigma2 = [10, 50]
params2 = np.hstack([mean2, sigma2])
params = [params1, params2]
betas = [0.3, 0.7]
y = ebp.mixture_of_kernels(xx2d, betas, params,
ebp.gaussian_kernel)
err = ebp.kernel_err(y, xx2d, betas, params, ebp.gaussian_kernel)
npt.assert_almost_equal(err, np.zeros(err.shape))
def compute_sketch(self):
start_time = time.time()
if self.sketch is not None:
return self.sketch
mat_b = np.zeros([self.l + self.b_size, self.m])
# compute zero valued row list
zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b[:self.l, :], axis = 1)])[0]
zero_rows = np.hstack((zero_rows, np.arange(self.l, self.l + self.b_size))).tolist()
# repeat inserting each row of matrix A
for i in range(0, self.mat.shape[0]):
# insert a row into matrix B
mat_b[zero_rows[0], :] = self.mat[i, :]
# remove zero valued row from the list
zero_rows.remove(zero_rows[0])
# if there is no more zero valued row
if len(zero_rows) == 0:
# compute SVD of matrix B, we want to find the first l
self._sketch_func(mat_b)
# update the zero valued row list
zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b[:self.l, :], axis = 1)])[0]
zero_rows = np.hstack((zero_rows, np.arange(self.l, self.l + self.b_size))).tolist()
# why do we need this here?
# do we need to do a sketch one last time at the end?
self._sketch_func(mat_b)
# get rid of extra non-zero rows when we return
self.sketch = mat_b[:self.l, :]
self.sketching_time = time.time() - start_time
return self.sketch
def compute_sketch(self):
start_time = time.time()
if self.sketch is not None:
return self.sketch
# basically, want to init an empty csr matrix
if self.randomized and (self.m > 100 * self.l):
print "using sparse sketch"
# lets use the sparse version of randomized sketch here
return self.compute_sparse_sketch()
else:
self._sketch_func = self._fast_rand_sketch
# what do we do differently here? we need to iterate over the nzrow_inds,
mat_b = np.zeros([self.l + self.b_size, self.m])
# other way: np.where(~mat_b.any(axis=1))[0]
# zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b, axis = 1)])[0].tolist()
zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b[:self.l, :], axis = 1)])[0]
zero_rows = np.hstack((zero_rows, np.arange(self.l, self.l + self.b_size))).tolist()
# iterate through the nzrow_inds
for i in self.nzrow_inds:
mat_b[zero_rows[0], :] = self.mat.getrow(i).todense()
zero_rows.remove(zero_rows[0])
if len(zero_rows) == 0:
print "sketching ", i
self._sketch_func(mat_b)
zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b[:self.l, :], axis = 1)])[0]
zero_rows = np.hstack((zero_rows, np.arange(self.l, self.l + self.b_size))).tolist()
self._sketch_func(mat_b)
self.sketch = mat_b[:self.l, :]
self.sketching_time = time.time() - start_time
return self.sketch
def dobetterstuff(inpath):
data_files = [f for f in os.listdir(inpath) if f.endswith('.mel.npy')]
random.shuffle(data_files)
artists = set([f[:18] for f in data_files])
artist_string_to_id = dict([(s,i) for i, s in enumerate(artists)])
def get_split(datafiles___, splitpercent):
# gen = filtered_stratified_split(datafiles___,
# sklearn.cross_validation.StratifiedShuffleSplit,
# [1] * len(datafiles___), n_iterations=1, test_size=splitpercent)
gen = sklearn.cross_validation.ShuffleSplit(len(datafiles___), 1, splitpercent)
for i_trs, i_tes in gen:
return [datafiles___[i] for i in i_trs], [datafiles___[i] for i in i_tes]
training_files, test_files = get_split(data_files, .2)
training_files, validation_files = get_split(training_files, .2)
print training_files
print test_files
print validation_files
train_set_y = np.hstack([[artist_string_to_id[f[:18]]] * 129 for f in training_files])
train_set_x = np.vstack([np.load(os.path.join(inpath, f)) for f in training_files])
test_set_y = np.hstack([[artist_string_to_id[f[:18]]] * 129 for f in test_files])
test_set_x = np.vstack([np.load(os.path.join(inpath, f)) for f in test_files])
validation_set_y = np.hstack([[artist_string_to_id[f[:18]]] * 129 for f in validation_files])
validation_set_x = np.vstack([np.load(os.path.join(inpath, f)) for f in validation_files])
datasets = [(train_set_x, train_set_y), (validation_set_x, validation_set_y), (test_set_x, test_set_y)]
return datasets
def load_sdss_data_both_catalogs(hemi):
lowz = load_sdss_data('lowz', hemi)
cmass = load_sdss_data('cmass', hemi)
ra = np.hstack([lowz['ra'],cmass['ra']])
dec = np.hstack([lowz['dec'],cmass['dec']])
z = np.hstack([lowz['z'],cmass['z']])
return {'ra':ra, 'dec':dec, 'z':z}
def sample_trajectory(M, n_states):
# Samples trajectories from random nodes
# in our domain (M)
G, W = M.get_graph_inv()
N = G.shape[0]
if N >= n_states:
rand_ind = np.random.permutation(N)
else:
rand_ind = np.tile(np.random.permutation(N), (1, 10))
init_states = rand_ind[0:n_states].flatten()
goal_s = M.map_ind_to_state(M.targetx, M.targety)
states = []
states_xy = []
states_one_hot = []
# Get optimal path from graph
g_dense = W
g_masked = np.ma.masked_values(g_dense, 0)
g_sparse = csr_matrix(g_dense)
d, pred = dijkstra(g_sparse, indices=goal_s, return_predecessors=True)
for i in range(n_states):
path = trace_path(pred, goal_s, init_states[i])
path = np.flip(path, 0)
states.append(path)
for state in states:
L = len(state)
r, c = M.get_coords(state)
row_m = np.zeros((L, M.n_row))
col_m = np.zeros((L, M.n_col))
for i in range(L):
row_m[i, r[i]] = 1
col_m[i, c[i]] = 1
states_one_hot.append(np.hstack((row_m, col_m)))
states_xy.append(np.hstack((r, c)))
return states_xy, states_one_hot
def phase_step_spike_fq(self, spikes_time, full_step, nb_block, fs):
stance_spike_fq=[]
swing_spike_fq=[]
for step in full_step:
stance_block_duration = (step[1]-step[0])/nb_block
swing_block_duration = (step[2]-step[1])/nb_block
step_stance_count = []
step_swing_count = []
for i in range(nb_block):
step_stance_count.append(0)
step_swing_count.append(0)
for spike_time in spikes_time:
#if stance phase
if step[0] < spike_time/fs < step[1]:
list_block = np.arange(step[0], step[1], stance_block_duration)
list_block = np.hstack((list_block, step[1]))
for i in range(nb_block):
if list_block[i] < spike_time/fs < list_block[i+1]:
step_stance_count[i] += 1
#if swing phase
elif step[1] < spike_time/fs < step[2]:
list_block = np.arange(step[1], step[2], swing_block_duration)
list_block = np.hstack((list_block, step[2]))
for i in range(nb_block):
if list_block[i] < spike_time/fs < list_block[i+1]:
step_swing_count[i] += 1
# elif spike_time/fs > step[2]:
# break
stance_spike_fq.append(np.array(step_stance_count) / stance_block_duration)
swing_spike_fq.append(np.array(step_swing_count) / swing_block_duration)
return stance_spike_fq, swing_spike_fq
def run_classify(X_groups_train, y_train, X_groups_validate, y_validate):
"""
Although this function is given groups, it actually doesn't utilize the groups at all in the criterion
"""
method_label = "gridsearch_lasso"
X_validate = np.hstack(X_groups_validate)
max_power = np.log(50)
min_power = np.log(1e-4)
lambda_guesses = np.power(np.e, np.arange(min_power, max_power, (max_power - min_power - 1e-5) / (NUM_LAMBDAS - 1)))
print method_label, "lambda_guesses", lambda_guesses
X_train = np.hstack(X_groups_train)
problem_wrapper = LassoClassifyProblemWrapper(X_train, y_train, [])
best_cost = 1e5
best_betas = []
best_regularization = lambda_guesses[0]
for l1 in reversed(lambda_guesses):
betas = problem_wrapper.solve([l1])
current_cost, _ = testerror_logistic_grouped(X_validate, y_validate, betas)
if best_cost > current_cost:
best_cost = current_cost
best_betas = betas
best_regularization = l1
print method_label, "best_cost so far", best_cost, "best_regularization", best_regularization
sys.stdout.flush()
print method_label, "best_validation_error", best_cost
print method_label, "best lambdas:", best_regularization
return best_betas, best_cost
def test_fuzz(self):
# try a bunch of crazy inputs
rfuncs = (
np.random.uniform,
np.random.normal,
np.random.standard_cauchy,
np.random.exponential)
ntests = 100
for i in range(ntests):
rfunc = random.choice(rfuncs)
target_norm_1 = random.expovariate(1.0)
n = random.randrange(2, 16)
A_original = rfunc(size=(n,n))
E_original = rfunc(size=(n,n))
A_original_norm_1 = scipy.linalg.norm(A_original, 1)
scale = target_norm_1 / A_original_norm_1
A = scale * A_original
E = scale * E_original
M = np.vstack([
np.hstack([A, E]),
np.hstack([np.zeros_like(A), A])])
expected_expm = scipy.linalg.expm(A)
expected_frechet = scipy.linalg.expm(M)[:n, n:]
observed_expm, observed_frechet = expm_frechet(A, E)
assert_allclose(expected_expm, observed_expm)
assert_allclose(expected_frechet, observed_frechet)
def offsetPlane(plane, x, y):
"""
Takes a numpy 2D array and returns the same plane offset by x and y,
adding rows and columns of 0 values
"""
height, width = plane.shape
dataType = plane.dtype
# shift x by cropping, creating a new array of columns and stacking
# horizontally
if abs(x) > 0:
newCols = zeros((height, abs(x)), dataType)
x1 = max(0, 0 - x)
x2 = min(width, width - x)
crop = plane[0:height, x1:x2]
if x > 0:
plane = hstack((newCols, crop))
else:
plane = hstack((crop, newCols))
# shift y by cropping, creating a new array of rows and stacking
# vertically
if abs(y) > 0:
newRows = zeros((abs(y), width), dataType)
y1 = max(0, 0 - y)
y2 = min(height, height - y)
crop = plane[y1:y2, 0:width]
if y > 0:
plane = vstack((newRows, crop))
else:
plane = vstack((crop, newRows))
return plane
def _plot_traj(self, z, axes, units):
"""Plots spacecraft trajectory.
Args:
- z (``tuple``, ``list``, ``numpy.ndarray``): Decision chromosome.
- axes (``matplotlib.axes._subplots.Axes3DSubplot``): 3D axes to use for the plot
- units (``float``, ``int``): Length unit by which to normalise data.
Examples:
>>> prob.extract(pykep.trajopt.indirect_or2or).plot_traj(pop.champion_x)
"""
# times
t0 = pk.epoch(0)
tf = pk.epoch(z[0])
# Mean Anomalies
M0 = z[1] - self.elem0[1] * np.sin(z[1])
Mf = z[2] - self.elemf[1] * np.sin(z[2])
elem0 = np.hstack([self.elem0[:5], [M0]])
elemf = np.hstack([self.elemf[:5], [Mf]])
# Keplerian points
kep0 = pk.planet.keplerian(t0, elem0)
kepf = pk.planet.keplerian(tf, elemf)
# planets
pk.orbit_plots.plot_planet(
kep0, t0=t0, units=units, ax=axes, color=(0.8, 0.8, 0.8))
pk.orbit_plots.plot_planet(
kepf, t0=tf, units=units, ax=axes, color=(0.8, 0.8, 0.8))
def setUp(self):
db = pysal.open(pysal.examples.get_path("baltim.dbf"),'r')
self.ds_name = "baltim.dbf"
self.y_name = "PRICE"
self.y = np.array(db.by_col(self.y_name)).T
self.y.shape = (len(self.y),1)
self.x_names = ["NROOM","AGE","SQFT"]
self.x = np.array([db.by_col(var) for var in self.x_names]).T
ww = pysal.open(pysal.examples.get_path("baltim_q.gal"))
self.w = ww.read()
ww.close()
self.w_name = "baltim_q.gal"
self.w.transform = 'r'
self.regimes = db.by_col("CITCOU")
#Artficial:
n = 256
self.n2 = n/2
self.x_a1 = np.random.uniform(-10,10,(n,1))
self.x_a2 = np.random.uniform(1,5,(n,1))
self.q_a = self.x_a2 + np.random.normal(0,1,(n,1))
self.x_a = np.hstack((self.x_a1,self.x_a2))
self.y_a = np.dot(np.hstack((np.ones((n,1)),self.x_a)),np.array([[1],[0.5],[2]])) + np.random.normal(0,1,(n,1))
latt = int(np.sqrt(n))
self.w_a = pysal.lat2W(latt,latt)
self.w_a.transform='r'
self.regi_a = [0]*(n/2) + [1]*(n/2)
self.w_a1 = pysal.lat2W(latt/2,latt)
self.w_a1.transform='r'
['ApplicantIncome', 'CoapplicantIncome', 'LoanAmount', 'Loan_Amount_Term'])
ohe = OneHotEncoder(sparse=False)
mms = MinMaxScaler()
enc = {}
ans = 0
for col in X:
if any(continuous == col):
enc[col] = mms.fit_transform(
np.array(X_ohe[col]).reshape(-1, 1).astype(np.float64))
ans += enc[col].shape[1]
else:
tmp = ohe.fit_transform(X_ohe[col].factorize()[0].reshape(
-1, 1)).astype(np.int64)
enc[col] = tmp
ans += enc[col].shape[1]
X_fin = np.hstack([
enc['Gender'], enc['Married'], enc['Dependents'], enc['Education'],
enc['Self_Employed'], enc['ApplicantIncome'], enc['CoapplicantIncome'],
enc['LoanAmount'], enc['Loan_Amount_Term'], enc['Credit_History'],
enc['Property_Area']
])
# "#### 設問5：ローン審査の予測モデルとして、以下のパイプラインを構成して下さい。\n",
# "- 標準化とロジスティック回帰\n",
# "- 標準化とランダムフォレスト\n",
# "- 標準化と主成分分析とランダムフォレスト"
# num_pipeline =
def ocean_30_vs_90_with_colorbar():
dirs = "../result_h/mean_vector/ocean_currents_30_vs_90_with_colorbar/"
if not os.path.exists(dirs):
os.makedirs(dirs)
start_list.pop()
for i, start in enumerate(start_list):
print("****************** {}/{} *******************".format(
i + 1, M - 1))
helmert_30_30_fname = "../data/csv_Helmert_30/Helmert_30_" + str(
start)[:6] + ".csv"
data_30 = pd.read_csv(helmert_30_30_fname)
data_30_vec = [
np.array(data_30["ocean_u"]),
np.array(data_30["ocean_v"])
]
helmert_90_90_fname = "../data/csv_Helmert_both_90/Helmert_both_90_" + str(
start)[:6] + ".csv"
data_90 = pd.read_csv(helmert_90_90_fname)
data_90_vec = [
np.array(data_90["ocean_u_90"]),
np.array(data_90["ocean_v_90"])
]
vector_list = [data_30_vec, data_90_vec]
name_list = ["_ocean_30", "_ocean_90"]
for j in range(2):
m = Basemap(lon_0=180,
boundinglat=50,
resolution='l',
projection='npstere')
m.drawcoastlines(color='0.15')
m.fillcontinents(color='#555555')
lon = np.array(latlon_ex.Lon)
lat = np.array(latlon_ex.Lat)
x, y = m(lon, lat)
x1 = np.reshape(x, (145, 145), order='F')
y1 = np.reshape(y, (145, 145), order='F')
dx1 = (x1[1, 0] - x1[0, 0]) / 2
dy1 = (y1[0, 1] - y1[0, 0]) / 2
x2 = np.linspace(x1[0, 0], x1[144, 0], 145)
y2 = np.linspace(y1[0, 0], y1[0, 144], 145)
xx, yy = np.meshgrid(x2, y2)
xx, yy = xx.T, yy.T
vector_u = np.ma.masked_invalid(vector_list[j][0])
vector_v = np.ma.masked_invalid(vector_list[j][1])
vector_speed = np.sqrt(vector_u * vector_u + vector_v * vector_v)
data_non_wind = vector_speed
data_non_wind = np.ma.masked_invalid(data_non_wind)
data1 = np.reshape(data_non_wind, (145, 145), order='F')
xx = np.hstack([xx, xx[:, 0].reshape(145, 1)])
xx_ex = np.vstack([xx, (xx[144, :] + (xx[1, 0] - xx[0, 0]))])
yy = np.vstack([yy, yy[0, :]])
yy_ex = np.hstack([
(yy[:, 0].reshape(146, 1) + (yy[0, 0] - yy[0, 1])), yy
])
m.pcolormesh(xx_ex - dx1,
yy_ex + dy1,
data1,
cmap=plt.cm.jet,
vmax=0.2,
vmin=0)
m.colorbar(location='bottom')
m.quiver(x, y, vector_u, vector_v, color="k")
save_name = dirs + str(start)[:6] + name_list[j] + ".png"
print(save_name)
plt.savefig(save_name, dpi=450)
plt.close()
def boundary_conditions(x, t, z=None, nbsym=2):
"""
Extend the signal beyond it's bounds w.r.t mirror symmetry.
:param x: Signal to be mirrored.
:param t: Timestamps of the signal
:param z: Signal on whose extrema the interpolation is evaluated. (By \
default this is just ``x``)
:param nbsym: Number of points added to each end of the signal.
:type x: array-like
:type t: array-like
:type z: array-like
:type nbsym: int
:return: timestamps and values of extended extrema, ordered as (minima \
timestamps, maxima timestamps, minima values, maxima values.)
:rtype: tuple
"""
indmax = argrelmax(x)[0]
indmin = argrelmin(x)[0]
lx = x.shape[0] - 1
if indmin.shape[0] + indmax.shape[0] < 3:
raise ValueError("Not enough extrema.")
if indmax[0] < indmin[0]:
if x[0] > x[indmin[0]]:
lmax = indmax[1:np.min([indmax.shape[0], nbsym + 1])][::-1]
lmin = indmin[:np.min([indmin.shape[0], nbsym])][::-1]
lsym = indmax[0]
else:
lmax = indmax[1:np.min([indmax.shape[0], nbsym])][::-1]
lmin = indmin[:np.min([indmin.shape[0], nbsym - 1])][::-1]
lmin = np.hstack((lmin, [1]))
lsym = 1
else:
if x[0] < x[indmax[0]]:
lmax = indmax[:np.min([indmax.shape[0], nbsym])][::-1]
lmin = indmin[1:np.min([indmin.shape[0], nbsym + 1])][::-1]
lsym = indmin[0]
else:
lmax = indmax[:np.min([indmin.shape[0], nbsym - 1])][::-1]
lmax = np.hstack((lmax, [1]))
lmin = indmin[:np.min([indmax.shape[0], nbsym])][::-1]
lsym = 1
if indmax[-1] < indmin[-1]:
if x[-1] < x[indmax[-1]]:
rmax = indmax[(max([indmax.shape[0] - nbsym + 1, 1]) - 1):][::-1]
rmin = indmin[(max([indmin.shape[0] - nbsym, 1]) - 1):-1][::-1]
rsym = indmin[-1]
else:
rmax = indmax[max(indmax.shape[0] - nbsym + 1, 0):indmax.shape[0]][::-1]
rmax = np.hstack(([lx], rmax))
rmin = indmin[max(indmin.shape[0] - nbsym, 0):][::-1]
rsym = lx
else:
if x[-1] > x[indmin[-1]]:
rmax = indmax[max(indmax.shape[0] - nbsym - 1, 0):-1][::-1]
rmin = indmin[max(indmin.shape[0] - nbsym, 0):][::-1]
rsym = indmax[-1]
else:
rmax = indmax[max(indmax.shape[0] - nbsym, 0):][::-1]
rmin = indmin[max(indmin.shape[0] - nbsym + 1, 0):][::-1]
rmin = np.hstack(([lx], rmin))
rsym = lx
tlmin = 2 * t[lsym] - t[lmin]
tlmax = 2 * t[lsym] - t[lmax]
trmin = 2 * t[rsym] - t[rmin]
trmax = 2 * t[rsym] - t[rmax]
# In case symmetrized parts do not extend enough
if (tlmin[0] > t[0]) or (tlmax[0] > t[1]):
if lsym == indmax[0]:
lmax = indmax[:np.min((indmax.shape[0], nbsym))][::-1]
else:
lmin = indmin[:np.min((indmin.shape[0], nbsym))][::-1]
if lsym == 1:
raise Exception("Bug")
lsym = 1
tlmin = 2 * t[lsym] - t[lmin]
tlmax = 2 * t[lsym] - t[lmax]
if (trmin[-1] < t[lx]) or (trmax[-1] < t[lx]):
if rsym == indmax.shape[0]:
rmax = indmax[np.max([indmax.shape[0] - nbsym + 1,
1]):indmax.shape[0]][::-1]
else:
rmin = indmin[np.max([indmax.shape[0] - nbsym + 1,
1]):indmin.shape[0]][::-1]
if rsym == lx:
raise Exception("bug")
rsym = lx
trmin = 2 * t[rsym] - t[rmin]
trmax = 2 * t[rsym] - t[rmax]
if z is None:
z = x
zlmax = z[lmax]
zlmin = z[lmin]
zrmax = z[rmax]
zrmin = z[rmin]
tmin = map(np.array, [tlmin, t[indmin], trmin])
tmax = map(np.array, [tlmax, t[indmax], trmax])
zmin = map(np.array, [zlmin, z[indmin], zrmin])
zmax = map(np.array, [zlmax, z[indmax], zrmax])
tmin, tmax, zmin, zmax = map(np.hstack, [tmin, tmax, zmin, zmax])
return tmin, tmax, zmin, zmax
def run_env(budget_para):
# 训练
print('data loading')
test_data = pd.read_csv(data_type['data_path'] + data_type['campaign_id'] +
str(data_type['fraction_type']) + '/test_data.csv',
header=None).drop([0])
test_data.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2] \
= test_data.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2].astype(
int)
test_data.iloc[:, config['data_pctr_index']] \
= test_data.iloc[:, config['data_pctr_index']].astype(
float)
test_data.iloc[:, config['data_fraction_index']] \
= test_data.iloc[:, config['data_fraction_index']].astype(
int)
test_data = test_data.values
test_all_sample = pd.read_csv(
data_type['data_path'] + data_type['campaign_id'] +
str(data_type['fraction_type']) + '/test_all_sample.csv',
header=None).drop([0])
test_all_sample.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2] \
= test_all_sample.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2].astype(
int)
test_all_sample.iloc[:, config['data_pctr_index']] \
= test_all_sample.iloc[:, config['data_pctr_index']].astype(
float)
test_all_sample.iloc[:, config['data_fraction_index']] \
= test_all_sample.iloc[:, config['data_fraction_index']].astype(
int)
test_all_sample = test_all_sample.values
test_down_sample = pd.read_csv(
data_type['data_path'] + data_type['campaign_id'] +
str(data_type['fraction_type']) + '/test_down_sample.csv',
header=None).drop([0])
test_down_sample.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2] \
= test_down_sample.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2].astype(
int)
test_down_sample.iloc[:, config['data_pctr_index']] \
= test_down_sample.iloc[:, config['data_pctr_index']].astype(
float)
test_down_sample.iloc[:, config['data_fraction_index']] \
= test_down_sample.iloc[:, config['data_fraction_index']].astype(
int)
test_down_sample = test_down_sample.values
train_data = pd.read_csv(data_type['data_path'] +
data_type['campaign_id'] +
str(data_type['fraction_type']) + '/train_' +
data_type['type'] + '.csv')
train_data.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2] \
= train_data.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2].astype(
int)
train_data.iloc[:, config['data_pctr_index']] \
= train_data.iloc[:, config['data_pctr_index']].astype(
float)
train_data.iloc[:, config['data_fraction_index']] \
= train_data.iloc[:, config['data_fraction_index']].astype(
int)
train_data = train_data.values
# config['train_budget'] = np.sum(train_data[:, config['data_marketprice_index']])
config['train_budget'] = 32000000
budget = config['train_budget'] * budget_para
# config['test_budget'] = np.sum(test_data[:, config['data_marketprice_index']])
config['test_budget'] = 32000000
original_ctr = np.sum(
train_data[:, config['data_clk_index']]) / len(train_data)
total_clks = np.sum(train_data[:, config['data_clk_index']])
real_hour_clks = []
for i in range(data_type['fraction_type']):
real_hour_clks.append(
np.sum(train_data[train_data[:, config['data_fraction_index']] ==
i][:, config['data_clk_index']]))
td_error, action_loss = 0, 0
eCPC = choose_eCPC(data_type['campaign_id'], original_ctr)
e_results = []
e_actions = []
test_records = []
e_all_actions = []
test_all_records = []
e_down_actions = []
test_down_records = []
is_learn = False
fraction_type = data_type['fraction_type']
exploration_rate = config['exploration_rate']
for episode in range(config['train_episodes']):
e_clks = [0 for i in range(fraction_type)] # episode各个时段所获得的点击数，以下类推
e_profits = [0 for i in range(fraction_type)]
e_reward = [0 for i in range(fraction_type)]
e_cost = [0 for i in range(fraction_type)]
e_true_value = [0 for i in range(fraction_type)]
e_miss_true_value = [0 for i in range(fraction_type)]
e_win_imp_with_clk_value = [0 for i in range(fraction_type)]
e_win_imp_without_clk_cost = [0 for i in range(fraction_type)
] # 各个时段浪费在没有点击的曝光上的预算
e_lose_imp_with_clk_value = [0 for i in range(fraction_type)]
e_clk_aucs = [0 for i in range(fraction_type)]
e_clk_no_win_aucs = [0 for i in range(fraction_type)]
e_lose_imp_without_clk_cost = [0 for i in range(fraction_type)]
e_no_clk_aucs = [0 for i in range(fraction_type)]
e_no_clk_no_win_aucs = [0 for i in range(fraction_type)]
actions = [0 for i in range(fraction_type)]
init_action = 0
next_action = 0
state_ = np.array([])
break_time_slot = 0
real_clks = [0 for i in range(fraction_type)]
bid_nums = [0 for i in range(fraction_type)]
imps = [0 for i in range(fraction_type)]
ou_noise = OrnsteinUhlenbeckNoise(mu=np.zeros(1))
# 状态包括：当前CTR，
for t in range(fraction_type):
auc_datas = train_data[train_data[:, config['data_fraction_index']]
== t]
if t == 0:
state = np.array(
[1, 0, 0, 0]
) # current_time_slot, budget_left_ratio, cost_t_ratio, ctr_t, win_rate_t
action = RL.choose_action(state)
action = np.clip(np.random.normal(action, exploration_rate),
-0.99, 0.99)
init_action = action
bids = auc_datas[:, config['data_pctr_index']] * eCPC / (
1 + init_action)
bids = np.where(bids >= 300, 300, bids)
else:
state = state_
action = next_action
bids = auc_datas[:, config['data_pctr_index']] * eCPC / (
1 + action)
bids = np.where(bids >= 300, 300, bids)
actions[t] = action
win_auctions = auc_datas[
bids >= auc_datas[:, config['data_marketprice_index']]]
no_win_auctions = auc_datas[
bids <= auc_datas[:, config['data_marketprice_index']]]
e_cost[t] = np.sum(win_auctions[:,
config['data_marketprice_index']])
e_profits[t] = np.sum(
win_auctions[:, config['data_pctr_index']] * eCPC -
win_auctions[:, config['data_marketprice_index']])
e_true_value[t] = np.sum(
win_auctions[:, config['data_pctr_index']] * eCPC)
e_miss_true_value[t] = np.sum(
no_win_auctions[:, config['data_pctr_index']] * eCPC)
with_clk_win_auctions = win_auctions[
win_auctions[:, config['data_clk_index']] == 1]
e_win_imp_with_clk_value[t] = np.sum(
with_clk_win_auctions[:, config['data_pctr_index']] * eCPC)
e_win_imp_without_clk_cost[t] = np.sum(
win_auctions[win_auctions[:, config['data_clk_index']] ==
0][:, config['data_marketprice_index']])
with_clk_no_win_auctions = no_win_auctions[
no_win_auctions[:, config['data_clk_index']] == 1]
e_lose_imp_with_clk_value[t] = np.sum(
with_clk_no_win_auctions[:, config['data_pctr_index']] * eCPC)
e_clks[t] = np.sum(win_auctions[:, config['data_clk_index']],
dtype=int)
imps[t] = len(win_auctions)
real_clks[t] = np.sum(auc_datas[:, config['data_clk_index']],
dtype=int)
bid_nums[t] = len(auc_datas)
e_clk_aucs[t] = len(
auc_datas[auc_datas[:, config['data_clk_index']] == 1])
e_clk_no_win_aucs[t] = len(with_clk_no_win_auctions)
e_no_clk_aucs[t] = len(
auc_datas[auc_datas[:, config['data_clk_index']] == 0])
without_clk_no_win_auctions = no_win_auctions[
no_win_auctions[:, config['data_clk_index']] == 0]
e_lose_imp_without_clk_cost[t] = np.sum(
without_clk_no_win_auctions[:,
config['data_marketprice_index']])
e_no_clk_no_win_aucs[t] = len(without_clk_no_win_auctions)
market_prices_t = auc_datas[:, config['data_marketprice_index']]
bid_win_t = bids[
bids >= auc_datas[:, config['data_marketprice_index']]]
market_price_win_t = market_prices_t[
bids >= auc_datas[:, config['data_marketprice_index']]]
no_win_imps_market_prices_t = np.sum(
no_win_auctions[:, config['data_marketprice_index']])
if np.sum(e_cost) >= budget:
break_time_slot = t
temp_cost = 0
temp_lose_cost = 0
temp_win_auctions = 0
e_clks[t] = 0
e_profits[t] = 0
e_true_value[t] = 0
e_miss_true_value[t] = 0
e_win_imp_without_clk_cost[t] = 0
e_lose_imp_with_clk_value[t] = 0
real_clks[t] = 0
imps[t] = 0
bid_nums[t] = 0
e_win_imp_with_clk_value[t] = 0
e_clk_aucs[t] = 0
e_lose_imp_without_clk_cost[t] = 0
e_no_clk_aucs[t] = 0
e_no_clk_no_win_aucs[t] = 0
bids_t = []
market_prices_t = []
bid_win_t = []
market_price_win_t = []
no_win_imps_market_prices_t = 0
for i in range(len(auc_datas)):
if temp_cost >= (budget - np.sum(e_cost[:t])):
break
current_data = auc_datas[i, :]
temp_clk = int(current_data[config['data_clk_index']])
temp_market_price = current_data[
config['data_marketprice_index']]
if t == 0:
temp_action = init_action
else:
temp_action = next_action
bid = current_data[config['data_pctr_index']] * eCPC / (
1 + temp_action)
bid = bid if bid <= 300 else 300
real_clks[t] += temp_clk
bid_nums[t] += 1
if temp_clk == 1:
e_clk_aucs[t] += temp_clk
else:
e_no_clk_aucs[t] += 1
bids_t.append(bid)
market_prices_t.append(temp_market_price)
if bid >= temp_market_price:
if temp_clk == 0:
e_win_imp_without_clk_cost[t] += temp_market_price
else:
e_win_imp_with_clk_value[t] += current_data[
config['data_pctr_index']] * eCPC
bid_win_t.append(bid)
market_price_win_t.append(temp_market_price)
e_profits[t] += (
current_data[config['data_pctr_index']] * eCPC -
temp_market_price)
e_true_value[t] += current_data[
config['data_pctr_index']] * eCPC
e_clks[t] += temp_clk
imps[t] += 1
temp_cost += temp_market_price
temp_win_auctions += 1
else:
e_miss_true_value[t] += current_data[
config['data_pctr_index']] * eCPC
temp_lose_cost += temp_market_price
no_win_imps_market_prices_t += temp_market_price
if temp_clk == 1:
e_clk_no_win_aucs[t] += 1
e_lose_imp_with_clk_value[t] += current_data[
config['data_pctr_index']] * eCPC
else:
e_no_clk_no_win_aucs[t] += 1
e_cost[t] = temp_cost
ctr_t = e_clks[
t] / temp_win_auctions if temp_win_auctions > 0 else 0
win_rate_t = temp_win_auctions / bid_nums[t]
else:
ctr_t = np.sum(
win_auctions[:, config['data_clk_index']]) / len(
win_auctions) if len(win_auctions) > 0 else 0
win_rate_t = len(win_auctions) / len(auc_datas) if len(
auc_datas) > 0 else 0
budget_left_ratio = (budget - np.sum(e_cost[:t + 1])) / budget
budget_left_ratio = budget_left_ratio if budget_left_ratio >= 0 else 0
time_left_ratio = (fraction_type - 1 - t) / fraction_type
avg_time_spend = budget_left_ratio / time_left_ratio if time_left_ratio > 0 else 0
cost_t_ratio = e_cost[t] / budget
if t == 0:
state_ = np.array(
[avg_time_spend, cost_t_ratio, ctr_t, win_rate_t])
else:
state_ = np.array(
[avg_time_spend, cost_t_ratio, ctr_t, win_rate_t])
action_ = RL.choose_action(state_)
action_ = np.clip(np.random.normal(action_, exploration_rate),
-0.99, 0.99)
next_action = action_
reward_t = adjust_reward(
len(auc_datas), e_true_value, e_miss_true_value, bid_win_t,
market_price_win_t, e_win_imp_with_clk_value, e_cost,
e_win_imp_without_clk_cost, real_clks,
e_lose_imp_with_clk_value, e_clk_aucs, e_clk_no_win_aucs,
e_lose_imp_without_clk_cost, e_no_clk_aucs,
e_no_clk_no_win_aucs, no_win_imps_market_prices_t, budget,
total_clks, t)
reward = reward_t
e_reward[t] = reward
transition = np.hstack(
(state.tolist(), action, reward, state_.tolist()))
RL.store_transition(transition)
if np.sum(e_cost) >= budget:
break
if (episode > 0) and ((episode + 1) % config['observation_episode']
== 0):
e_result = [
np.sum(e_reward),
np.sum(e_profits), budget,
np.sum(e_cost),
int(np.sum(e_clks)),
int(np.sum(real_clks)),
np.sum(bid_nums),
np.sum(imps),
np.sum(e_cost) / np.sum(imps) if np.sum(imps) > 0 else 0,
break_time_slot, td_error, action_loss
]
e_results.append(e_result)
# 在原始论文中，每感知一次环境就要对模型进行一次训练
# 然而频繁地学习在未充分感知环境的情况下，会使模型陷入局部（当前）最优
# 因此可以每感知N次再对模型训练n次，这样会使得模型更稳定，并加快学习速度
is_learn = True
exploration_rate *= 0.999
if is_learn: # after observing config['observation_size'] times, for config['learn_iter'] learning time
for m in range(config['learn_iter']):
td_e, a_loss = RL.learn()
td_error, action_loss = td_e, a_loss
RL.soft_update(RL.Actor, RL.Actor_)
RL.soft_update(RL.Critic, RL.Critic_)
if m == config['learn_iter'] - 1:
is_learn = False
actions_df = pd.DataFrame(data=actions)
actions_df.to_csv(log_path + '/result_reward_1/' +
str(fraction_type) + '/train_actions_' +
str(budget_para) + '.csv')
hour_clks = {
'clks': e_clks,
'no_bid_clks': np.subtract(real_hour_clks, e_clks).tolist(),
'real_clks': real_hour_clks
}
hour_clks_df = pd.DataFrame(data=hour_clks)
hour_clks_df.to_csv(log_path + '/result_reward_1/' +
str(fraction_type) + '/train_hour_clks_' +
str(budget_para) + '.csv')
print(
'episode {}, reward={}, profits={}, budget={}, cost={}, clks={}, real_clks={}, bids={}, '
'imps={}, cpm={}, break_time_slot={}, td_error={}, action_loss={}\n'
.format(
episode + 1, np.sum(e_reward), np.sum(e_profits), budget,
np.sum(e_cost), int(np.sum(e_clks)),
int(np.sum(real_clks)), np.sum(bid_nums), np.sum(imps),
np.sum(e_cost) / np.sum(imps) if np.sum(imps) > 0 else 0,
break_time_slot, td_error, action_loss))
test_result, test_actions, test_hour_clks = test_env(
config['test_budget'] * budget_para, budget_para, test_data,
eCPC)
test_records.append(test_result)
e_actions.append(test_actions)
test_all_result, test_all_actions, test_hour_clks = test_env(
config['test_budget'] * budget_para, budget_para,
test_all_sample, eCPC)
test_all_records.append(test_all_result)
e_all_actions.append(test_all_actions)
test_down_result, test_down_actions, test_hour_clks = test_env(
config['test_budget'] * budget_para, budget_para,
test_down_sample, eCPC)
test_down_records.append(test_down_result)
e_down_actions.append(test_down_actions)
e_results_df = pd.DataFrame(data=e_results,
columns=[
'reward', 'profits', 'budget', 'cost',
'clks', 'real_clks', 'bids', 'imps', 'cpm',
'break_time_slot', 'td_error',
'action_loss'
])
e_results_df.to_csv(log_path + '/result_reward_1/' + str(fraction_type) +
'/train_episode_results_' + str(budget_para) + '.csv')
e_actions_df = pd.DataFrame(data=e_actions)
e_actions_df.to_csv(log_path + '/result_reward_1/' + str(fraction_type) +
'/test_episode_actions_' + str(budget_para) + '.csv')
## test_data
test_records_df = pd.DataFrame(data=test_records,
columns=[
'profits', 'budget', 'cost', 'clks',
'real_clks', 'bids', 'imps', 'cpm',
'break_time_slot'
])
test_records_df.to_csv(log_path + '/result_reward_1/' +
str(fraction_type) + '/test_episode_results_' +
str(budget_para) + '.csv')
e_actions_df = pd.DataFrame(data=e_actions)
e_actions_df.to_csv(log_path + '/result_reward_1/' + str(fraction_type) +
'/test_episode_actions_' + str(budget_para) + '.csv')
# test_all_sample
test_all_records_df = pd.DataFrame(data=test_all_records,
columns=[
'profits', 'budget', 'cost', 'clks',
'real_clks', 'bids', 'imps', 'cpm',
'break_time_slot'
])
test_all_records_df.to_csv(log_path + '/result_reward_1/' +
str(fraction_type) +
'/all_sample/test_episode_results_' +
str(budget_para) + '.csv')
e_all_actions_df = pd.DataFrame(data=e_all_actions)
e_all_actions_df.to_csv(log_path + '/result_reward_1/' +
str(fraction_type) +
'/all_sample/test_episode_actions_' +
str(budget_para) + '.csv')
# test_down_sample
test_down_records_df = pd.DataFrame(data=test_down_records,
columns=[
'profits', 'budget', 'cost',
'clks', 'real_clks', 'bids',
'imps', 'cpm', 'break_time_slot'
])
test_down_records_df.to_csv(log_path + '/result_reward_1/' +
str(fraction_type) +
'/down_sample/test_episode_results_' +
str(budget_para) + '.csv')
e_down_actions_df = pd.DataFrame(data=e_down_actions)
e_down_actions_df.to_csv(log_path + '/result_reward_1/' +
str(fraction_type) +
'/down_sample/test_episode_actions_' +
str(budget_para) + '.csv')
def execute(context):
df_persons = context.stage("synthesis.population.sociodemographics")[[
"person_id", "hts_person_id", "age", "sex", "residence_area_index"
]]
df_trips = pd.DataFrame(context.stage("data.hts.cleaned")[1], copy=True)
df_trips = df_trips[[
"person_id", "trip_id", "departure_time", "arrival_time", "mode",
"preceeding_purpose", "following_purpose"
]]
df_trips.to_csv("%s/HTS/trips.csv" % context.config("data_path"))
assert (len(df_trips) == len(df_trips.dropna()))
df_trips = df_trips.sort_values(by=["person_id", "trip_id"])
df_trip_counts = df_trips[[
"person_id"
]].groupby("person_id").size().reset_index(name="count")
df_trips["trip_id"] = np.hstack(
[np.arange(n) for n in df_trip_counts["count"].values])
df_trips.columns = [
"hts_person_id", "trip_id", "departure_time", "arrival_time", "mode",
"preceeding_purpose", "following_purpose"
]
# Merge trips to persons
df_trips = pd.merge(df_persons, df_trips)
df_trips.loc[df_trips["arrival_time"] < df_trips["departure_time"],
"arrival_time"] += 24.0 * 3600.0
df_trips.loc[:,
"travel_time"] = df_trips.loc[:,
"arrival_time"] - df_trips.loc[:,
"departure_time"]
df_trips = df_trips[[
"person_id", "trip_id", "departure_time", "arrival_time",
"travel_time", "mode", "preceeding_purpose", "following_purpose",
"age", "hts_person_id", "residence_area_index", "sex"
]]
df_trips = df_trips.sort_values(by=["person_id", "trip_id"])
# Diversify departure times
counts = df_trips[[
"person_id", "trip_id"
]].groupby("person_id").size().reset_index(name="count")["count"].values
interval = df_trips[[
"person_id", "departure_time"
]].groupby("person_id").min().reset_index()["departure_time"].values
interval = np.minimum(
1800.0, interval
) # If first departure time is just 5min after midnight, we only add a deviation of 5min
offset = np.random.random(size=(len(counts), )) * interval * 2.0 - interval
offset = np.repeat(offset, counts)
df_trips["departure_time"] += offset
df_trips["arrival_time"] += offset
df_trips["departure_time"] = np.round(df_trips["departure_time"])
df_trips["arrival_time"] = np.round(df_trips["arrival_time"])
return df_trips
import cv2
# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required = True, help = "Path to the image")
args = vars(ap.parse_args())
# load the image, clone it for output, and then convert it to grayscale
image = cv2.imread(args["image"])
output = image.copy()
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# detect circles in the image
circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.2, 100)
# ensure at least some circles were found
if circles is not None:
# convert the (x, y) coordinates and radius of the circles to integers
circles = np.round(circles[0, :]).astype("int")
# loop over the (x, y) coordinates and radius of the circles
for (x, y, r) in circles:
# draw the circle in the output image, then draw a rectangle
# corresponding to the center of the circle
cv2.circle(output, (x, y), r, (0, 255, 0), 4)
cv2.rectangle(output, (x - 5, y - 5), (x + 5, y + 5), (0, 128, 255), -1)
# show the output image
cv2.imshow("output", np.hstack([image, output]))
cv2.waitKey(0)
def _train_one_epoch(self, epoch):
# Train
try:
self.net.train()
_learning_rate_ = self._adjust_learning_rate(epoch)
Tools.print('Epoch: [{}] lr={} name={}'.format(
epoch, _learning_rate_, Config.name))
self.produce_class.reset()
avg_loss, avg_loss_1, avg_loss_2 = AverageMeter(), AverageMeter(
), AverageMeter()
for batch_idx, (inputs, _,
indexes) in enumerate(self.train_loader):
batch_size, _, c, w, h = inputs.shape
a = np.random.beta(1, 1, [batch_size, 1])
b = np.hstack([a, 1 - a])
ab = np.tile(b[..., None, None, None], [c, w, h])
x1_add_x2 = torch.mean(inputs * ab, dim=1,
keepdim=True).float()
now_inputs = torch.cat([inputs, x1_add_x2],
dim=1).view(-1, c, w, h)
now_inputs = Config.cuda(now_inputs)
self.optimizer.zero_grad()
out_logits, out_l2norm, out_softmax = self.net(now_inputs)
out_dim = out_softmax.shape[-1]
now_out_logits = out_logits.view(batch_size, -1, out_dim)
now_out_l2norm = out_l2norm.view(batch_size, -1, out_dim)
now_out_softmax = out_softmax.view(batch_size, -1, out_dim)
softmax_w = np.hstack([a, 1 - a, -np.ones_like(a)])
softmax_w = torch.tensor(
np.tile(softmax_w[..., None], [
out_dim,
])).float()
now_softmax_w = Config.cuda(softmax_w)
out_error = torch.sum(now_out_softmax * now_softmax_w, dim=1)
loss1 = torch.mean(torch.sum(torch.pow(out_error, 2), dim=1))
loss1 *= Config.mix_ratio
targets = self.produce_class.get_label(indexes[0])
targets2 = self.produce_class.get_label(indexes[1])
self.produce_class.cal_label(now_out_l2norm[:, 0, :],
indexes[0])
loss2 = self.criterion(now_out_logits[:, 0, :], targets)
if Config.has_ic_2:
loss2_2 = self.criterion(now_out_logits[:, 1, :], targets2)
loss2 = loss2 + loss2_2
pass
loss = (loss1 + loss2) if Config.has_mix else loss2
loss.backward()
self.optimizer.step()
avg_loss.update(loss.item(), inputs.size(0))
avg_loss_1.update(loss1.item(), inputs.size(0))
avg_loss_2.update(loss2.item(), inputs.size(0))
# if batch_idx % 100 == 0:
# Tools.print('Train: [{}] {}/{} Loss: {:.4f}/{:.4f} '
# 'Loss 1: {:.4f}/{:.4f} Loss 2: {:.4f}/{:.4f}'.format(
# epoch, batch_idx, len(self.train_loader), avg_loss.avg, avg_loss.val,
# avg_loss_1.avg, avg_loss_1.val, avg_loss_2.avg, avg_loss_2.val))
pass
Tools.print("Train: [{}] {}/{}".format(epoch,
self.produce_class.count,
self.produce_class.count_2))
Tools.print('Train: [{}] {} Loss: {:.4f}/{:.4f} '
'Loss 1: {:.4f}/{:.4f} Loss 2: {:.4f}/{:.4f}'.format(
epoch, len(self.train_loader), avg_loss.avg,
avg_loss.val, avg_loss_1.avg, avg_loss_1.val,
avg_loss_2.avg, avg_loss_2.val))
finally:
pass
# Test
try:
Tools.print("Test: [{}] .......".format(epoch))
_acc = self.test(epoch=epoch)
if _acc > self.best_acc:
Tools.print('Saving..')
state = {
'net': self.net.state_dict(),
'acc': _acc,
'epoch': epoch
}
torch.save(state, Config.checkpoint_path)
self.best_acc = _acc
pass
Tools.print('Test: [{}] best accuracy: {:.2f}'.format(
epoch, self.best_acc * 100))
finally:
pass
pass
ax.set_ylabel('age')
plt.savefig(r'', bbox_inches='tight')
df.groupby(['month', 'y']).size().unstack().plot(kind='bar', stacked=True)
plt.gca().set_ylabel('count')
plt.savefig(r'', bbox_inches='tight')
# df['pdays'].plot.hist(bins=50)
# plt.gca().set_ylabel('count')
# plt.savefig(r'', bbox_inches='tight')
df['pdays'].quantile(
q=[0, 0.005, 0.025, 0.1, 0.25, 0.5, 0.75, 0.9, 0.975, 0.995, 1])
df.boxplot(column='pdays')
###################### ANN Results ###################################
ax = pd.DataFrame(np.hstack(
(labels, tf.nn.sigmoid(model(df.values.astype(np.float32))).numpy())),
columns=['result', 'predicted']).boxplot(column='predicted',
by='result')
plt.gca().set_title('')
plt.gca().set_ylabel('predicted')
plt.savefig(r'D:\Personal\presentations\WF\pred_boxplot.png',
bbox_inches='tight')
sns.set(style="whitegrid")
ax = sns.boxplot(x="result",
y="predicted",
data=pd.DataFrame(np.hstack(
(labels, tf.nn.sigmoid(model(df.values.astype(
np.float32))).numpy())),
columns=['result', 'predicted']),
showfliers=False)
def runSingleFold(data_list, file_idx, fold_number):
print '| ---- ---- Fold #{} ---- ----'.format(fold_number)
number_of_views = len(data_list)
(training_blocks, tuning_blocks,
testing_blocks) = util.configure_blocks(fold_number)
data_pre_processor = DataPreProcessor(data_list, file_idx, training_blocks,
False)
training_data_per_view, training_labels_per_view = data_pre_processor.process(
)
data_pre_processor = DataPreProcessor(data_list, file_idx, tuning_blocks,
False)
tuning_data_per_view, tuning_labels_per_view = data_pre_processor.process()
data_pre_processor = DataPreProcessor(data_list, file_idx, testing_blocks,
False)
testing_data_per_view, testing_labels_per_view = data_pre_processor.process(
)
for i in range(number_of_views):
training_data = np.ndarray(shape=(0, np.shape(
training_data_per_view[i])[0]),
dtype=np.float)
training_labels = np.array([], dtype=np.int)
if use_full_phones:
training_data = training_data_per_view[i].transpose()
training_labels = training_labels_per_view[i]
else:
for j in range(len(training_labels_per_view[i])):
if (training_labels_per_view[i][j] in vowel_labels):
training_data = np.vstack(
(training_data,
training_data_per_view[i].transpose()[j, :]))
training_labels = np.hstack(
(training_labels, int(training_labels_per_view[i][j])))
param_msg = None
# Start tuning/testing
if classification_model == ClassificationModel.Kernel_SVM_RBF:
max_accuracy = 0.0
optimal_gamma = 0.0
for j in [3e-08, 3.5e-08, 4e-08, 4.5e-08]:
model = svm.SVC(decision_function_shape='ovo',
kernel='rbf',
gamma=j,
C=2)
model.fit(training_data, training_labels)
accuracy = getAccuracy(model, tuning_data_per_view[i],
tuning_labels_per_view[i])
if accuracy > max_accuracy:
max_accuracy = accuracy
optimal_gamma = j
param_msg = 'Optimal gamma value: {}'.format(optimal_gamma)
model = svm.SVC(decision_function_shape='ovo',
kernel='rbf',
gamma=optimal_gamma,
C=2)
else:
max_accuracy = 0.0
optimal_neighbors = 0
for j in [28, 32, 36, 40]:
model = neighbors.KNeighborsClassifier(j, weights='distance')
model.fit(training_data, training_labels)
accuracy = getAccuracy(model, tuning_data_per_view[i],
tuning_labels_per_view[i])
if accuracy > max_accuracy:
max_accuracy = accuracy
optimal_neighbors = j
param_msg = 'Optimal number of neighbors: {}'.format(
optimal_neighbors)
model = neighbors.KNeighborsClassifier(optimal_neighbors,
weights='distance')
model.fit(training_data, training_labels)
accuracy = getAccuracy(model, testing_data_per_view[i],
testing_labels_per_view[i])
print '| Accuracy for view {}: {:.3f}'.format(
i + 1, accuracy) + ', ' + param_msg
print '|'
# key bindings
if k == 27: # esc to exit
break
elif k == ord('0'): # BG drawing
print(" mark background regions with left mouse button \n")
value = DRAW_BG
elif k == ord('1'): # FG drawing
print(" mark foreground regions with left mouse button \n")
value = DRAW_FG
elif k == ord('2'): # PR_BG drawing
value = DRAW_PR_BG
elif k == ord('3'): # PR_FG drawing
value = DRAW_PR_FG
elif k == ord('s'): # save image
bar = np.zeros((img.shape[0],5,3),np.uint8)
res = np.hstack((img2,bar,img,bar,output))
cv2.imwrite('grabcut_output.png',res)
print(" Result saved as image \n")
elif k == ord('r'): # reset everything
print("resetting \n")
rect = (0,0,1,1)
drawing = False
rectangle = False
rect_or_mask = 100
rect_over = False
value = DRAW_FG
img = img2.copy()
mask = np.zeros(img.shape[:2],dtype = np.uint8) # mask initialized to PR_BG
output = np.zeros(img.shape,np.uint8) # output image to be shown
elif k == ord('n'): # segment the image
print(""" For finer touchups, mark foreground and background after pressing keys 0-3
dec['cdsvm'].append(dec1)
auc['cdsvm'].append(roc_auc_score(yt,dec1))
acc_all['cdsvm'].append(acc['cdsvm'])
auc_all['cdsvm'].append(auc['cdsvm'])
acc_std = {}
acc_mean = {}
auc_std = {}
auc_mean = {}
acc_all['cdsvm'] = np.array(acc_all['cdsvm'])
acc_mean['cdsvm'] = np.mean(acc_all['cdsvm'],axis=1).reshape((len(src_full),1))
acc_std['cdsvm'] = np.std(acc_all['cdsvm'],axis=1).reshape((len(src_full),1))
auc_all['cdsvm'] = np.array(auc_all['cdsvm'])
auc_mean['cdsvm'] = np.mean(auc_all['cdsvm'],axis=1).reshape((len(src_full),1))
auc_std['cdsvm'] = np.std(auc_all['cdsvm'],axis=1).reshape((len(src_full),1))
source_list = np.array(src_list).T
cdsvm_result = np.hstack((acc_all['cdsvm'], acc_mean['cdsvm'], acc_std['cdsvm'],
auc_all['cdsvm'], auc_mean['cdsvm'], auc_std['cdsvm']))
cdsvm_df = pd.DataFrame(cdsvm_result, index = src_str_list)
cdsvm_df.to_csv("%sfold_ica+cdsvm_result_%s_%s.csv"%(kfold, target[0], target[1]))
for i in dec:
dec[i].append(yt)
pd.DataFrame(np.array(dec[i])).to_csv('%sfold_dec_%s_%s_%s.csv'%(kfold,i,target[0], target[1]))
def evaluate_recall(self, candidate_boxes=None, thresholds=None,
area='all', limit=None):
"""Evaluate detection proposal recall metrics.
Returns:
results: dictionary of results with keys
'ar': average recall
'recalls': vector recalls at each IoU overlap threshold
'thresholds': vector of IoU overlap thresholds
'gt_overlaps': vector of all ground-truth overlaps
"""
# Record max overlap value for each gt box
# Return vector of overlap values
areas = {'all': 0, 'small': 1, 'medium': 2, 'large': 3,
'96-128': 4, '128-256': 5, '256-512': 6, '512-inf': 7}
area_ranges = [[0 ** 2, 1e5 ** 2], # all
[0 ** 2, 32 ** 2], # small
[32 ** 2, 96 ** 2], # medium
[96 ** 2, 1e5 ** 2], # large
[96 ** 2, 128 ** 2], # 96-128
[128 ** 2, 256 ** 2], # 128-256
[256 ** 2, 512 ** 2], # 256-512
[512 ** 2, 1e5 ** 2], # 512-inf
]
assert area in areas, 'unknown area range: {}'.format(area)
area_range = area_ranges[areas[area]]
gt_overlaps = np.zeros(0)
num_pos = 0
for i in range(self.num_images):
# Checking for max_overlaps == 1 avoids including crowd annotations
# (...pretty hacking :/)
max_gt_overlaps = self.roidb[i]['gt_overlaps'].toarray().max(axis=1)
gt_inds = np.where((self.roidb[i]['gt_classes'] > 0) &
(max_gt_overlaps == 1))[0]
gt_boxes = self.roidb[i]['boxes'][gt_inds, :]
gt_areas = self.roidb[i]['seg_areas'][gt_inds]
valid_gt_inds = np.where((gt_areas >= area_range[0]) &
(gt_areas <= area_range[1]))[0]
gt_boxes = gt_boxes[valid_gt_inds, :]
num_pos += len(valid_gt_inds)
if candidate_boxes is None:
# If candidate_boxes is not supplied, the default is to use the
# non-ground-truth boxes from this roidb
non_gt_inds = np.where(self.roidb[i]['gt_classes'] == 0)[0]
boxes = self.roidb[i]['boxes'][non_gt_inds, :]
else:
boxes = candidate_boxes[i]
if boxes.shape[0] == 0:
continue
if limit is not None and boxes.shape[0] > limit:
boxes = boxes[:limit, :]
overlaps = bbox_overlaps(boxes.astype(np.float),
gt_boxes.astype(np.float))
_gt_overlaps = np.zeros((gt_boxes.shape[0]))
for j in range(gt_boxes.shape[0]):
# find which proposal box maximally covers each gt box
argmax_overlaps = overlaps.argmax(axis=0)
# and get the iou amount of coverage for each gt box
max_overlaps = overlaps.max(axis=0)
# find which gt box is 'best' covered (i.e. 'best' = most iou)
gt_ind = max_overlaps.argmax()
gt_ovr = max_overlaps.max()
assert (gt_ovr >= 0)
# find the proposal box that covers the best covered gt box
box_ind = argmax_overlaps[gt_ind]
# record the iou coverage of this gt box
_gt_overlaps[j] = overlaps[box_ind, gt_ind]
assert (_gt_overlaps[j] == gt_ovr)
# mark the proposal box and the gt box as used
overlaps[box_ind, :] = -1
overlaps[:, gt_ind] = -1
# append recorded iou coverage level
gt_overlaps = np.hstack((gt_overlaps, _gt_overlaps))
gt_overlaps = np.sort(gt_overlaps)
if thresholds is None:
step = 0.05
thresholds = np.arange(0.5, 0.95 + 1e-5, step)
recalls = np.zeros_like(thresholds)
# compute recall for each iou threshold
for i, t in enumerate(thresholds):
recalls[i] = (gt_overlaps >= t).sum() / float(num_pos)
# ar = 2 * np.trapz(recalls, thresholds)
ar = recalls.mean()
return {'ar': ar, 'recalls': recalls, 'thresholds': thresholds,
'gt_overlaps': gt_overlaps}
S1 = np.eye(2)
S2 = np.array([[1, 0.95], [0.95, 1]])
m1 = np.array([0.75, 0]).reshape(-1, 1)
m2 = np.array([-0.75, 0])
xx = np.repeat(m1, n1).reshape(2, n1)
yy = np.repeat(m2, n2).reshape(2, n2)
x1 = np.linalg.cholesky(S1).T @ np.random.randn(2,n1) + xx
x2 = np.linalg.cholesky(S2).T @ np.random.randn(2,n2) + yy
x = np.concatenate([x1.T, x2.T])
y1 = -np.ones(n1).reshape(-1, 1)
y2 = np.ones(n2).reshape(-1, 1)
y = np.concatenate([y1, y2])
q = np.linspace(-4, 4, 81)
r = np.linspace(-4, 4, 81)
t1, t2 = np.meshgrid(q, r)
tgrid = np.hstack([t1.reshape(-1, 1), t2.reshape(-1, 1)])
class GPClassificationModel(AbstractVariationalGP):
def __init__(self, train_x):
variational_distribution = CholeskyVariationalDistribution(train_x.size(0))
variational_strategy = VariationalStrategy(self, train_x, variational_distribution)
super(GPClassificationModel, self).__init__(variational_strategy)
self.mean_module = gpytorch.means.ConstantMean()
self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel())
def forward(self, x):
mean_x = self.mean_module(x)
covar_x = self.covar_module(x)
latent_pred = gpytorch.distributions.MultivariateNormal(mean_x, covar_x)
return latent_pred
def train(model, init_lengthscale, init_sigmaf_2):
decoder = Decoder(input_size = 300)
print(" [*] Decoding result")
conf_pred = np.stack([one_hot(target, 9).numpy() for target in conf_target])
conf_pred = conf_pred.reshape(-1, 9)
output = decoder((torch.FloatTensor(conf_pred), loc_target))
all_boxes = [[[] for _ in range(1)] for _ in range(9)]
for j in range(1, 9):
dets = output[0, j, :]
mask = dets[:, 0].gt(0.).expand(5, dets.size(0)).t()
dets = torch.masked_select(dets, mask).view(-1, 5)
if dets.dim() == 0:
continue
boxes = dets[:, 1:]
scores = dets[:, 0].cpu().numpy()
cls_dets = np.hstack((boxes.cpu().numpy(), scores[:, np.newaxis])).astype(np.float32, copy = False)
all_boxes[j][0] = cls_dets
#print(" [*] label:")
#print(output[:, 0])
print(" [*] box:")
print(output)
def train():
# data load
voca, sentence_index = data_load()
voca_num = len(voca)
# write vocabrary lists
pickle.dump(voca, open("vocabrary_word2vec.bn", "wb"))
print("vocabrary num:", voca_num)
print("e.x.", voca[:5])
# model
model = Word2Vec(voca_num, dim=hidden_dim).to(device)
# minibatch index
mbi = 0
data_num = len(sentence_index)
train_ind = np.arange(data_num)
np.random.seed(0)
np.random.shuffle(train_ind)
# loss function
loss_fn = torch.nn.NLLLoss()
# each learning rate and iteration
for lr, ite in train_factors:
print("lr", lr, " ite", ite)
# optimizer
if opt == "SGD":
optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9)
elif opt == "Adam":
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
else:
raise Exception("invalid optimizer:", opt)
# each iteration
for ite in range(ite):
# get minibatch index
if mbi + mb > data_num:
mb_ind = copy(train_ind[mbi:])
np.random.shuffle(train_ind)
mb_ind = np.hstack((mb_ind, train_ind[:(mb-(data_num-mbi))]))
else:
mb_ind = train_ind[mbi: mbi+mb]
mbi += mb
# get minibatch
X_inds = [sentence_index[i] for i in mb_ind]
loss = 0
accuracy = 0.
total_len = 0
# each data of minibatch
for mb_index in range(mb):
# 1 data of minibatch
Xs = np.array(X_inds[mb_index]).reshape([-1, 1])
input_X = np.zeros([1, voca_num])
input_X[:, Xs[C]] = 1
input_X = torch.tensor(input_X, dtype=torch.float).to(device)
# reset graph
optimizer.zero_grad()
# data length
total_len += x_length
# forward network
ys = model(input_X)
# target label index
t_inds = [_i for _i in range(x_length) if _i != C]
# each target label
for i, y in zip(t_inds, ys):
# target label
t = torch.tensor(Xs[i], dtype=torch.long).to(device)
# get loss
loss += loss_fn(torch.log(y), t)
# count accuracy
if y.argmax() == t:
accuracy += 1
"""
# each target label
for i in range(x_length):
# forward network
y = model(input_X)
# target label
t = torch.tensor(Xs[i], dtype=torch.long).to(device)
#t = torch.tensor(Xs[i], dtype=torch.long).to(device).view(-1, voca_num)
# get loss
loss += loss_fn(torch.log(y), t)
# count accuracy
if y.argmax() == t:
accuracy += 1
"""
# loss backward
loss.backward()
# update weight
optimizer.step()
# get loss
loss = loss.item() / total_len
accuracy = accuracy / total_len
if (ite + 1) % 10 == 0:
print("iter :", ite+1, ",loss >>:", loss, "accuracy:", accuracy)
torch.save(model.state_dict(), 'word2vec.pt')
def match_prediction():
match, stat = load_data()
teams = calc_score()
teams_train = {}
for team in teams:
teams_train[team] = {'x': [], 'y': []}
for i in range(len(teams[team]) - TRAINSIZE):
teams_train[team]['x'].append(teams[team][i:i + TRAINSIZE])
teams_train[team]['y'].append(teams[team][i + TRAINSIZE])
teams_train[team]['x'] = np.array(teams_train[team]['x'])
teams_train[team]['y'] = np.array(teams_train[team]['y'])
x_train = []
y_train = []
for a in teams_train.values():
x_train.append(a['x'])
y_train.append(a['y'])
x_train = np.vstack(x_train)
y_train = np.hstack(y_train)
#
x_all = np.concatenate((x_train, y_train.reshape(-1, 1)), axis=1)
sc = StandardScaler()
x_all = sc.fit_transform(x_all)
x_train2 = x_all[:, :TRAINSIZE]
y_train2 = x_all[:, TRAINSIZE]
#x_train = teams_train['26']['x']
#y_train = teams_train['26']['y']
model = Sequential()
model.add(LSTM(32, input_shape=(TRAINSIZE, 1), return_sequences=True))
model.add(Dropout(0.1))
#model.add(LSTM(32, return_sequences=True))
#model.add(Dropout(0.1))
model.add(LSTM(32, return_sequences=False))
model.add(Dropout(0.1))
model.add(Dense(1, activation='linear'))
start = time.time()
adam = keras.optimizers.Adam(learning_rate=0.0005,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
nadam = keras.optimizers.Nadam(learning_rate=0.002,
beta_1=0.9,
beta_2=0.999)
sgd = keras.optimizers.SGD(learning_rate=0.001,
momentum=0.0,
nesterov=False)
model.compile(loss="mse", optimizer=adam)
print("Compilation Time : ", time.time() - start)
model.fit(x_train2.reshape(x_train2.shape + (1, )),
y_train2,
batch_size=16,
epochs=10,
validation_split=0.05)
plt.figure()
predict = model.predict(x_train2.reshape(x_train2.shape + (1, )))
plt.plot(y_train2)
plt.plot(predict)
class_data, class_label = make_class_data(model, match, teams)
s4teams = []
for a in list(match.values())[-380:]:
if a['home_team_id'] not in s4teams:
s4teams.append(a['home_team_id'])
s4_teams = {}
for a in teams:
if a in s4teams:
if len(teams[a]) == 38:
s4_teams[a] = [46, 46, 46, 46, 46] + teams[a]
else:
s4_teams[a] = teams[a][-43:]
teamsi = dict.fromkeys(s4_teams.keys(), 0)
test_data = []
test_label = []
for a in list(match.values())[-380:]:
home = a['home_team_id']
away = a['away_team_id']
if int(a['full_time_score'].split(':')[0]) > int(
a['full_time_score'].split(':')[1]):
y = [1, 0, 0]
elif int(a['full_time_score'].split(':')[0]) == int(
a['full_time_score'].split(':')[1]):
y = [0, 1, 0]
else:
y = [0, 0, 1]
ihome = teamsi[home]
iaway = teamsi[away]
h = np.array(s4_teams[home][ihome:ihome + TRAINSIZE] + [0]).reshape(
-1, 1)
a = np.array(s4_teams[away][iaway:iaway + TRAINSIZE] + [0]).reshape(
-1, 1)
h = sc.transform(h.reshape(1, -1))
a = sc.transform(a.reshape(1, -1))
h_score = model.predict(
h.reshape(-1, 1)[:TRAINSIZE].reshape(1, TRAINSIZE, 1))
a_score = model.predict(
a.reshape(-1, 1)[:TRAINSIZE].reshape(1, TRAINSIZE, 1))
test_data.append([h_score[0], a_score[0]])
test_label.append(y)
teamsi[home] += 1
teamsi[away] += 1
test_data = np.array(test_data)
test_label = np.array(test_label)
clf = RandomForestClassifier()
clf = clf.fit(class_data[:1300].reshape(1300, 2), class_label[:1300])
score = clf.score(test_data.reshape(380, 2), test_label)
print(score)
def exp_hog_parameters(args, var, to_log=True):
# define feature parameters
cell_per_block = var['cell_per_block'] # 2
color_space = var['color_space'] # can be RGB, HSV, LUV, HLS, YUV, YCrCb
hist_bins = var['hist_bins'] # 32 # number of histogram bins
hist_feat = var['hist_feat'] # histogram features on or off
hog_channel = var['hog_channel'] # 'ALL' # can be 0, 1, 2, or 'ALL'
hog_feat = var['hog_feat'] # HOG features on or off
orient = var['orient'] # 8
overlap = var['overlap'] # 0.5
pix_per_cell = var['pix_per_cell'] # 8
scale = var['scale'] # 1.0
spatial_feat = var['spatial_feat'] # True, spatial features on or off
spatial_size = var['spatial_size'] # (32,32) # spatial binning dimensions
x_start_stop = var['x_start_stop'] # [None, None]
y_start_stop = var['y_start_stop'] # [400, 656]
xy_window = var['xy_window'] # (128, 128)
# list_all_images
cars, notcars = list_all_images(args)
# choose random car/notcar indices
flag_random = False
if flag_random:
car_ind = np.random.randint(0, len(cars))
notcar_ind = np.random.randint(0, len(notcars))
else:
car_ind, notcar_ind = 2734, 7868
# read in car / notcar images
car_image = mpimg.imread(cars[car_ind])
notcar_image = mpimg.imread(notcars[notcar_ind])
num_img = 5
flag_random = False
if flag_random:
cars_image_to_plot = [[cars[index].split('\\')[-1][:-4], cars[index]] for index in
[random.randint(0, len(cars)) for i in range(num_img)]]
notcars_image_to_plot = [[notcars[index].split('\\')[-1][:-4], notcars[index]] for index in
[random.randint(0, len(notcars)) for i in range(num_img)]]
else:
cars_image_to_plot = [[index, cars[index]] for index in
[random.randint(0, len(cars)) for i in range(num_img)]]
notcars_image_to_plot = [[index, notcars[index]] for index in
[random.randint(0, len(notcars)) for i in range(num_img)]]
car_features, car_hog_image = single_img_features(car_image, color_space=color_space, spatial_size=spatial_size,
hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block, hog_channel=hog_channel,
spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat,
vis=True)
notcar_features, notcar_hog_image = single_img_features(notcar_image, color_space=color_space,
spatial_size=spatial_size,
hist_bins=hist_bins, orient=orient,
pix_per_cell=pix_per_cell,
cell_per_block=cell_per_block, hog_channel=hog_channel,
spatial_feat=spatial_feat, hist_feat=hist_feat,
hog_feat=hog_feat,
vis=True)
t = time.time()
n_samples = 1000
random_idxs = np.random.randint(0, len(cars), n_samples)
test_cars = np.array(cars)[random_idxs]
test_noncars = np.array(notcars)[random_idxs]
car_features = extract_features(test_cars, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat)
notcar_features = extract_features(test_noncars, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins,
orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat)
t_feature_computation = round(time.time() - t, 2)
print(t_feature_computation, 'Seconds to compute features...')
X = np.vstack((car_features, notcar_features)).astype(np.float64)
# fit a per_column scaler
X_scaler = StandardScaler().fit(X)
# apply the scaler to X
scaled_X = X_scaler.transform(X)
# define the labels vector
y = np.hstack(( np.ones(len(car_features)), np.zeros(len(notcar_features)) ))
# split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(scaled_X, y, test_size=0.1, random_state=rand_state)
print('Using:', orient, 'orientations,', pix_per_cell, 'pixels per cell,', cell_per_block,
'cells per block,', hist_bins, 'histogram bins, and', spatial_size, 'spatial sampling')
print('Feature vector length:', len(X_train[0]))
# use a linear SVC
svc = LinearSVC()
# check the training time for the SVC
t = time.time()
svc.fit(X_train, y_train) # https://stackoverflow.com/questions/40524790/valueerror-this-solver-needs-samples-of-at-least-2-classes-in-the-data-but-the
t_train = round(time.time()-t, 2)
print(t_train, 'Seconds to train SVC...')
# check the score of the SVC
accuracy = round(svc.score(X_test, y_test), 4)
print('Test Accuracy of SVC = ', accuracy)
log = [ cell_per_block, color_space, hist_bins,
hist_feat, hog_channel, orient,
pix_per_cell, spatial_feat, spatial_size,
accuracy, len(X_train[0]), t_feature_computation, t_train, t_feature_computation+t_train ]
# log = [ var['cell_per_block'], var['color_space'], var['hist_bins'],
# var['hist_feat'], var['hog_channel'], var['orient'],
# var['pix_per_cell'], var['spatial_feat'], var['spatial_size'],
# accuracy, len(X_train[0]), t_feature_computation, t_train, t_feature_computation+t_train ]
if to_log: log_write(args, log)
print(
len(overlap_idx), 'out of',
len(missed_idx[(len(missed_idx) - 1) - iter_idx]),
'misclassifications were overlapped between logistic regression and ResNet9\n\n'
)
missed_df = pd.DataFrame(X_test[overlap_idx], columns=df.columns)
missed_df.insert(loc=0, column='labels', value=y_test[overlap_idx])
# ## Stacking
# ### Create Training Set for Meta-Learner
X_train_meta = np.hstack(
(log_y_prob.reshape(len(log_y_prob),
1), rfc_y_prob.reshape(len(rfc_y_prob), 1),
knn_y_prob.reshape(len(knn_y_prob),
1), res_y_prob.reshape(len(res_y_prob), 1)))
y_train_meta = y_test
pos = []
neg = []
for i in range(len(y_train_meta)):
if y_train_meta[i] > 0:
pos.append(X_train_meta[i, :])
else:
neg.append(X_train_meta[i, :])
pos = np.asarray(pos)
neg = np.asarray(neg)
ret, frame = cap.read()
M = frame.shape[0]
N = frame.shape[1]
frame = cv2.resize(frame, (int(ratio*N),int(ratio*M)))
filterSz = 5
frameBl = cv2.blur(frame,(filterSz,filterSz))
labImg = cv2.cvtColor(frameBl, cv2.COLOR_BGR2LAB)
hsvImg = cv2.cvtColor(frameBl, cv2.COLOR_BGR2HSV)
[Blue,G,R] = cv2.split(frame)
[L,A,B] = cv2.split(labImg)
[H,S,V] = cv2.split(hsvImg)
rgbMerge = np.hstack((R,G,Blue))
labMerge = np.hstack((L,A,B))
hsvMerge = np.hstack((H,S,V))
finMerge = np.vstack((rgbMerge,labMerge, hsvMerge))
cv2.imshow('frame-Binary',finMerge)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
reg2 = LinearRegression().fit(W2, Y)
print("linear regression: {:.3f}".format((int_x[1] - int_x[0]) * reg1.coef_[0] * reg2.coef_[0][0]))
####################### instrumental variable ##################
X, Y, Z = data[:, 0][:], data[:, 1], data[:, 2]
array_YZ = np.vstack((Y, Z))
cov_YZ = np.cov(array_YZ)[0][1]
array_XZ = np.vstack((X, Z))
cov_XZ = np.cov(array_XZ)[0][1]
print("instrumental variable: {:.3f}".format((int_x[1] - int_x[0]) * cov_YZ / cov_XZ))
###################### propensity score ########################
X, Y, Z = data[:, 0][:, np.newaxis], data[:, 1][:, np.newaxis], data[:, 2][:, np.newaxis]
W1, W2 = data[:, 3][:, np.newaxis], data[:, 4][:, np.newaxis]
#### step 1 ####
fan = np.hstack((np.ones_like(X), Z, W1, W2))
reg1 = LinearRegression(fit_intercept=False).fit(fan, X)
Beta = reg1.coef_.transpose()
variance = np.mean((X - np.dot(fan, Beta))**2)
R = normal(X, X - np.dot(fan, Beta), variance)
#### step 2 ####
inpt = np.hstack((np.ones_like(X), X, R, X*R))
reg2 = LinearRegression(fit_intercept=False).fit(inpt, Y)
Alpha = reg2.coef_.transpose()
#### step3 ####
t0 = int_x[0] * np.ones_like(X)
r0 = normal(t0, np.dot(fan, Beta), variance)
inpt0 = np.hstack((np.ones_like(X), t0, r0, t0*r0))
y0 = np.mean(np.dot(inpt0, Alpha))
t1 = int_x[1] * np.ones_like(X)
def get_deflection(self, *args, label, index, indices, field_name):
"""Get the vector field, the field name and the corresponding pyvista mesh in format adapted to a glyph plot.
Parameters
----------
self : MeshSolution
a MeshSolution object
*args: list of strings
List of axes requested by the user, their units and values (optional)
label : str
a label
index : int
an index
indices : list
list of the points to extract (optional)
field_name : str
title of the field to display on plot
Returns
-------
vect_field : ndaray
vector field to plot
field_normal_amp : ndarray
scalar field which is the normal/radial component of vect_field (to adjust the colormap)
field_name : str
name of the field
mesh_pv : UnstructuredGrid
pyvista mesh to plot
"""
# Get mesh_pv and field
# Try to get normal field, else use radial
try:
mesh_pv, field_normal_amp, field_name = self.get_mesh_field_pv(
*args,
label=label,
index=index,
indices=indices,
field_name=field_name,
is_normal=True,
)
except:
mesh_pv, field_normal_amp, field_name = self.get_mesh_field_pv(
*args,
label=label,
index=index,
indices=indices,
field_name=field_name,
is_radial=True,
)
_, vect_field, _ = self.get_mesh_field_pv(
*args,
label=label,
index=index,
indices=indices,
field_name=field_name,
)
solution = self.get_solution(
label=label,
index=index,
)
axes_list = solution.get_axes_list(*args)
# Add third dimension if needed
ind = axes_list[0].index("component")
if axes_list[1][ind] == 2:
vect_field = hstack((vect_field, zeros((vect_field.shape[0], 1))))
if field_name is None:
if label is not None:
field_name = label
elif self.get_solution(index=index).label is not None:
field_name = self.get_solution(index=index).label
else:
field_name = "Field"
return vect_field, field_normal_amp, field_name, mesh_pv
def uniform_init(x_low, x_high, y_low, y_high, v_x, v_y, n):
return np.hstack(
(np.random.uniform(x_low, x_high,
(n, 1)), np.random.uniform(y_low, y_high, (n, 1)),
np.random.uniform(-v_x, v_x,
(n, 1)), np.random.uniform(-v_y, v_y, (n, 1))))
def loadBlender(fname_blender: str):
"""
Compile and return the fragments and collision shapes from ``fname``.
The ``fname`` file must specify a Blender file.
:param str fname: Blender file name
:return: (frags dict, cshapes dict)
"""
# Use Blender itself to load the Blender file. This will trigger a script
# inside Blender that creates a JSON file with custom scene data. The name
# of that file is returned in 'fname_az'.
fname_az = processBlenderFile(fname_blender)
# Load the collision shape data and geometry.
pp = pyassimp.postprocess
azFile = json.loads(open(fname_az, 'rb').read().decode('utf8'))
scene = pyassimp.load(fname_blender, pp.aiProcess_Triangulate)
del fname_az, pp
# Sanity check: both must contain information about the exact same objects.
assert set(azFile.keys()) == {_.name for _ in scene.meshes}
# Load all the materials.
materials = loadMaterials(scene, fname_blender)
# Compile fragments and collision shapes for Azrael.
frags, cshapes = {}, {}
for mesh in scene.meshes:
# Get the material for this mesh.
material = materials[mesh.materialindex]
# Iterate over the faces and rebuild the vertices.
azData = azFile[mesh.name]
vert = np.zeros(len(mesh.faces) * 9)
for idx, face in enumerate(mesh.faces):
tmp = [mesh.vertices[_] for _ in face]
start, stop = idx * 9, (idx + 1) * 9
vert[start:stop] = np.hstack(tmp)
del start, stop, tmp, idx, face
# Copy the UV coordinates, if there are any.
if len(mesh.texturecoords) > 0:
uv = np.zeros(len(mesh.faces) * 6)
for idx, face in enumerate(mesh.faces):
tmp = [mesh.texturecoords[0][_, :2] for _ in face]
start, stop = 6 * idx, 6 * (idx + 1)
uv[start:stop] = np.hstack(tmp)
del start, stop, tmp, idx, face
# Sanity check.
assert (len(uv) // 2 == len(vert) // 3)
else:
uv = 0.5 * np.ones(2 * (len(vert) // 3))
uv = []
# Azrael does not allow 'dots', yet Bullet uses it prominently to name
# objects. As a quick fix, simply replace the dots with something else.
# fixme: should be redundant once #149 (https://trello.com/c/wcHX3qGd)
# is implemented.
azname = mesh.name.replace('.', 'x')
# Unpack the position and orientation of the mesh in world coordinates.
pos, rot, dim = azData['pos'], azData['rot'], np.array(azData['dimensions'])
# Unpack the interior points and put them into any kind of byte string.
# This will be attached as yet another file to the fragment data.
interior_points = json.dumps(azData['interior_points']).encode('utf8')
# Create a RAW fragment.
scale = 1
rgb, width, height = material['RGB'], material['width'], material['height']
frag = demolib.getFragMetaRaw(vert, uv, rgb, scale, pos, rot)
frag.files['interior_points'] = interior_points
frags[azname] = frag
# Construct the BOX collision shape based on the Blender dimensions.
hlen_x, hlen_y, hlen_z = (dim / 2).tolist()
box = aztypes.CollShapeBox(hlen_x, hlen_y, hlen_z)
# Construct the CollshapeMeta data.
cshapes[azname] = aztypes.CollShapeMeta('box', pos, rot, box)
del azData, vert, dim, scale, rgb, width, height, hlen_x, hlen_y, hlen_z
return frags, cshapes
plt.xlabel("Step number")
plt.ylabel("MCMC chains of c")
plt.tight_layout()
plt.show()
# computing Medians for true values
medians = np.median(samples, axis=0)
a_true, b_true, c_true = np.average(medians,axis=0) # computing the the value of the parameters by averaging over the 50 chains
data = np.zeros([4000 * 50, 3])
for i in range(3):
data[:, i] = np.hstack(samples[:, :, i])
# We can plot the posterior PDF using the corner library.
fig = corner.corner(samples.reshape(40 * 5000, 3), labels=["a", "b", "c"], quantiles=[0.16, 0.5, 0.84],
truths=[a_true, b_true, c_true], show_titles=True)
plt.show()
X = np.linspace(0, 300, 1000)
for i in range(200):
r = np.random.randint(0, high=4000 * 50)
a = data[r, 0]
b = data[r, 1]
c = data[r, 2]
plt.plot(X, a * X ** 2 + b * X + c)
# 勾勒轮廓，为了接下来处理
cv2.drawContours(image_,contours,2,(0, 0, 255),2)
cv2.drawContours(image_,contours,4,(0, 0, 255),2)
cv2.drawContours(image_,contours,6,(0, 0, 255),2)
cv2.imshow("demo",image_)
cv2.waitKey(0)
# # 2 自定义用hough解决
contour = image_[:,:,2]
mask = cv2.inRange(contour,254,255)
cv2.imshow("demo",mask)
cv2.waitKey(0)
mask_canny = cv2.Canny(mask,100,30)
dst = np.hstack((mask_canny,mask))
cv2.namedWindow("circle")
cv2.createTrackbar('param1','circle',120,200,nothing)
cv2.createTrackbar('param2','circle',12,200,nothing)
cv2.createTrackbar('minRadius','circle',22,30,nothing)
cv2.createTrackbar('maxRadius','circle',43,100,nothing)
while (True):
image1 = np.copy(img)
# 137 33 24 44
param1 = cv2.getTrackbarPos('param1', 'circle')
param2 = cv2.getTrackbarPos('param2', 'circle')
minRadius = cv2.getTrackbarPos('minRadius', 'circle')
maxRadius = cv2.getTrackbarPos('maxRadius', 'circle')
circle = cv2.HoughCircles(mask,cv2.HOUGH_GRADIENT,1,5,param1=param1,param2=param2,minRadius=minRadius,maxRadius=maxRadius)
def get_current_visual(self, n):
real_A = tensor2img(self.real_A, n)
real_B = tensor2img(self.real_B, n)
fake_B = tensor2img(self.fake_B, n)
return np.hstack([real_A, real_B, fake_B])
def compute_rand_index_ijv(self, gt_ijv, test_ijv, shape):
'''Compute the Rand Index for an IJV matrix
This is in part based on the Omega Index:
Collins, "Omega: A General Formulation of the Rand Index of Cluster
Recovery Suitable for Non-disjoint Solutions", Multivariate Behavioral
Research, 1988, 23, 231-242
The basic idea of the paper is that a pair should be judged to
agree only if the number of clusters in which they appear together
is the same.
'''
#
# The idea here is to assign a label to every pixel position based
# on the set of labels given to that position by both the ground
# truth and the test set. We then assess each pair of labels
# as agreeing or disagreeing as to the number of matches.
#
# First, add the backgrounds to the IJV with a label of zero
#
gt_bkgd = numpy.ones(shape, bool)
gt_bkgd[gt_ijv[:, 0], gt_ijv[:, 1]] = False
test_bkgd = numpy.ones(shape, bool)
test_bkgd[test_ijv[:, 0], test_ijv[:, 1]] = False
gt_ijv = numpy.vstack([
gt_ijv,
numpy.column_stack([numpy.argwhere(gt_bkgd),
numpy.zeros(numpy.sum(gt_bkgd), gt_bkgd.dtype)])])
test_ijv = numpy.vstack([
test_ijv,
numpy.column_stack([numpy.argwhere(test_bkgd),
numpy.zeros(numpy.sum(test_bkgd), test_bkgd.dtype)])])
#
# Create a unified structure for the pixels where a fourth column
# tells you whether the pixels came from the ground-truth or test
#
u = numpy.vstack([
numpy.column_stack([gt_ijv, numpy.zeros(gt_ijv.shape[0], gt_ijv.dtype)]),
numpy.column_stack([test_ijv, numpy.ones(test_ijv.shape[0], test_ijv.dtype)])])
#
# Sort by coordinates, then by identity
#
order = numpy.lexsort([u[:, 2], u[:, 3], u[:, 0], u[:, 1]])
u = u[order, :]
# Get rid of any duplicate labelings (same point labeled twice with
# same label.
#
first = numpy.hstack([[True], numpy.any(u[:-1, :] != u[1:, :], 1)])
u = u[first, :]
#
# Create a 1-d indexer to point at each unique coordinate.
#
first_coord_idxs = numpy.hstack([
[0],
numpy.argwhere((u[:-1, 0] != u[1:, 0]) |
(u[:-1, 1] != u[1:, 1])).flatten() + 1,
[u.shape[0]]])
first_coord_counts = first_coord_idxs[1:] - first_coord_idxs[:-1]
indexes = centrosome.index.Indexes([first_coord_counts])
#
# Count the number of labels at each point for both gt and test
#
count_test = numpy.bincount(indexes.rev_idx, u[:, 3]).astype(numpy.int64)
count_gt = first_coord_counts - count_test
#
# For each # of labels, pull out the coordinates that have
# that many labels. Count the number of similarly labeled coordinates
# and record the count and labels for that group.
#
labels = []
for i in range(1, numpy.max(count_test) + 1):
for j in range(1, numpy.max(count_gt) + 1):
match = ((count_test[indexes.rev_idx] == i) &
(count_gt[indexes.rev_idx] == j))
if not numpy.any(match):
continue
#
# Arrange into an array where the rows are coordinates
# and the columns are the labels for that coordinate
#
lm = u[match, 2].reshape(numpy.sum(match) / (i + j), i + j)
#
# Sort by label.
#
order = numpy.lexsort(lm.transpose())
lm = lm[order, :]
#
# Find indices of unique and # of each
#
lm_first = numpy.hstack([
[0],
numpy.argwhere(numpy.any(lm[:-1, :] != lm[1:, :], 1)).flatten() + 1,
[lm.shape[0]]])
lm_count = lm_first[1:] - lm_first[:-1]
for idx, count in zip(lm_first[:-1], lm_count):
labels.append((count,
lm[idx, :j],
lm[idx, j:]))
#
# We now have our sets partitioned. Do each against each to get
# the number of true positive and negative pairs.
#
max_t_labels = reduce(max, [len(t) for c, t, g in labels], 0)
max_g_labels = reduce(max, [len(g) for c, t, g in labels], 0)
#
# tbl is the contingency table from Table 4 of the Collins paper
# It's a table of the number of pairs which fall into M sets
# in the ground truth case and N in the test case.
#
tbl = numpy.zeros(((max_t_labels + 1), (max_g_labels + 1)))
for i, (c1, tobject_numbers1, gobject_numbers1) in enumerate(labels):
for j, (c2, tobject_numbers2, gobject_numbers2) in \
enumerate(labels[i:]):
nhits_test = numpy.sum(
tobject_numbers1[:, numpy.newaxis] ==
tobject_numbers2[numpy.newaxis, :])
nhits_gt = numpy.sum(
gobject_numbers1[:, numpy.newaxis] ==
gobject_numbers2[numpy.newaxis, :])
if j == 0:
N = c1 * (c1 - 1) / 2
else:
N = c1 * c2
tbl[nhits_test, nhits_gt] += N
N = numpy.sum(tbl)
#
# Equation 13 from the paper
#
min_JK = min(max_t_labels, max_g_labels) + 1
rand_index = numpy.sum(tbl[:min_JK, :min_JK] * numpy.identity(min_JK)) / N
#
# Equation 15 from the paper, the expected index
#
e_omega = numpy.sum(numpy.sum(tbl[:min_JK, :min_JK], 0) *
numpy.sum(tbl[:min_JK, :min_JK], 1)) / N ** 2
#
# Equation 16 is the adjusted index
#
adjusted_rand_index = (rand_index - e_omega) / (1 - e_omega)
return rand_index, adjusted_rand_index
for i in eigen_pairs:
print(i[0])
##Get the k value for reduction
sum_eigVal = np.sum(eigen_val)
arry = []
for i in eigen_val:
if (i / sum_eigVal > 0.02):
arry.append(i)
type(arry)
dim = len(arry)
#TODO
matrix_res = []
for i in range(0, dim):
matrix_res.append(np.hstack(eigen_pairs[i][1].reshape(cols, 1)))
print('Matrix W:\n', matrix_res)
type(matrix_res)
matrix1 = np.asmatrix(matrix_res).T
pca_reduced = matrix1.T.dot(samples)
print(pca_reduced)
pca_reduced.shape
pca_reduced = pca_reduced.T
pca_reduced.shape
pca_reduced = pd.DataFrame(pca_reduced)
type(pca_reduced)
fig = plt.figure()
fig.add_subplot(1, 1, 1)
def measure_objects(self, workspace):
image_set = workspace.image_set
object_name_GT = self.object_name_GT.value
objects_GT = workspace.get_objects(object_name_GT)
iGT, jGT, lGT = objects_GT.ijv.transpose()
object_name_ID = self.object_name_ID.value
objects_ID = workspace.get_objects(object_name_ID)
iID, jID, lID = objects_ID.ijv.transpose()
ID_obj = 0 if len(lID) == 0 else max(lID)
GT_obj = 0 if len(lGT) == 0 else max(lGT)
xGT, yGT = objects_GT.shape
xID, yID = objects_ID.shape
GT_pixels = numpy.zeros((xGT, yGT))
ID_pixels = numpy.zeros((xID, yID))
total_pixels = xGT * yGT
GT_pixels[iGT, jGT] = 1
ID_pixels[iID, jID] = 1
GT_tot_area = len(iGT)
if len(iGT) == 0 and len(iID) == 0:
intersect_matrix = numpy.zeros((0, 0), int)
else:
#
# Build a matrix with rows of i, j, label and a GT/ID flag
#
all_ijv = numpy.column_stack(
(numpy.hstack((iGT, iID)),
numpy.hstack((jGT, jID)),
numpy.hstack((lGT, lID)),
numpy.hstack((numpy.zeros(len(iGT)), numpy.ones(len(iID))))))
#
# Order it so that runs of the same i, j are consecutive
#
order = numpy.lexsort((all_ijv[:, -1], all_ijv[:, 0], all_ijv[:, 1]))
all_ijv = all_ijv[order, :]
# Mark the first at each i, j != previous i, j
first = numpy.where(numpy.hstack(
([True],
~ numpy.all(all_ijv[:-1, :2] == all_ijv[1:, :2], 1),
[True])))[0]
# Count # at each i, j
count = first[1:] - first[:-1]
# First indexer - mapping from i,j to index in all_ijv
all_ijv_map = centrosome.index.Indexes([count])
# Bincount to get the # of ID pixels per i,j
id_count = numpy.bincount(all_ijv_map.rev_idx,
all_ijv[:, -1]).astype(int)
gt_count = count - id_count
# Now we can create an indexer that has NxM elements per i,j
# where N is the number of GT pixels at that i,j and M is
# the number of ID pixels. We can then use the indexer to pull
# out the label values for each to populate a sparse array.
#
cross_map = centrosome.index.Indexes([id_count, gt_count])
off_gt = all_ijv_map.fwd_idx[cross_map.rev_idx] + cross_map.idx[0]
off_id = all_ijv_map.fwd_idx[cross_map.rev_idx] + cross_map.idx[1] + \
id_count[cross_map.rev_idx]
intersect_matrix = scipy.sparse.coo_matrix(
(numpy.ones(len(off_gt)),
(all_ijv[off_id, 2], all_ijv[off_gt, 2])),
shape=(ID_obj + 1, GT_obj + 1)).toarray()[1:, 1:]
gt_areas = objects_GT.areas
id_areas = objects_ID.areas
FN_area = gt_areas[numpy.newaxis, :] - intersect_matrix
all_intersecting_area = numpy.sum(intersect_matrix)
dom_ID = []
for i in range(0, ID_obj):
indices_jj = numpy.nonzero(lID == i)
indices_jj = indices_jj[0]
id_i = iID[indices_jj]
id_j = jID[indices_jj]
ID_pixels[id_i, id_j] = 1
for i in intersect_matrix: # loop through the GT objects first
if len(i) == 0 or max(i) == 0:
id = -1 # we missed the object; arbitrarily assign -1 index
else:
id = numpy.where(i == max(i))[0][0] # what is the ID of the max pixels?
dom_ID += [id] # for ea GT object, which is the dominating ID?
dom_ID = numpy.array(dom_ID)
for i in range(0, len(intersect_matrix.T)):
if len(numpy.where(dom_ID == i)[0]) > 1:
final_id = numpy.where(intersect_matrix.T[i] == max(intersect_matrix.T[i]))
final_id = final_id[0][0]
all_id = numpy.where(dom_ID == i)[0]
nonfinal = [x for x in all_id if x != final_id]
for n in nonfinal: # these others cannot be candidates for the corr ID now
intersect_matrix.T[i][n] = 0
else:
continue
TP = 0
FN = 0
FP = 0
for i in range(0, len(dom_ID)):
d = dom_ID[i]
if d == -1:
tp = 0
fn = id_areas[i]
fp = 0
else:
fp = numpy.sum(intersect_matrix[i][0:d]) + numpy.sum(intersect_matrix[i][(d + 1)::])
tp = intersect_matrix[i][d]
fn = FN_area[i][d]
TP += tp
FN += fn
FP += fp
TN = max(0, total_pixels - TP - FN - FP)
def nan_divide(numerator, denominator):
if denominator == 0:
return numpy.nan
return float(numerator) / float(denominator)
accuracy = nan_divide(TP, all_intersecting_area)
recall = nan_divide(TP, GT_tot_area)
precision = nan_divide(TP, (TP + FP))
F_factor = nan_divide(2 * (precision * recall), (precision + recall))
true_positive_rate = nan_divide(TP, (FN + TP))
false_positive_rate = nan_divide(FP, (FP + TN))
false_negative_rate = nan_divide(FN, (FN + TP))
true_negative_rate = nan_divide(TN, (FP + TN))
shape = numpy.maximum(numpy.maximum(
numpy.array(objects_GT.shape), numpy.array(objects_ID.shape)),
numpy.ones(2, int))
rand_index, adjusted_rand_index = self.compute_rand_index_ijv(
objects_GT.ijv, objects_ID.ijv, shape)
m = workspace.measurements
m.add_image_measurement(self.measurement_name(FTR_F_FACTOR), F_factor)
m.add_image_measurement(self.measurement_name(FTR_PRECISION),
precision)
m.add_image_measurement(self.measurement_name(FTR_RECALL), recall)
m.add_image_measurement(self.measurement_name(FTR_TRUE_POS_RATE),
true_positive_rate)
m.add_image_measurement(self.measurement_name(FTR_FALSE_POS_RATE),
false_positive_rate)
m.add_image_measurement(self.measurement_name(FTR_TRUE_NEG_RATE),
true_negative_rate)
m.add_image_measurement(self.measurement_name(FTR_FALSE_NEG_RATE),
false_negative_rate)
m.add_image_measurement(self.measurement_name(FTR_RAND_INDEX),
rand_index)
m.add_image_measurement(self.measurement_name(FTR_ADJUSTED_RAND_INDEX),
adjusted_rand_index)
def subscripts(condition1, condition2):
x1, y1 = numpy.where(GT_pixels == condition1)
x2, y2 = numpy.where(ID_pixels == condition2)
mask = set(zip(x1, y1)) & set(zip(x2, y2))
return list(mask)
TP_mask = subscripts(1, 1)
FN_mask = subscripts(1, 0)
FP_mask = subscripts(0, 1)
TN_mask = subscripts(0, 0)
TP_pixels = numpy.zeros((xGT, yGT))
FN_pixels = numpy.zeros((xGT, yGT))
FP_pixels = numpy.zeros((xGT, yGT))
TN_pixels = numpy.zeros((xGT, yGT))
def maskimg(mask, img):
for ea in mask:
img[ea] = 1
return img
TP_pixels = maskimg(TP_mask, TP_pixels)
FN_pixels = maskimg(FN_mask, FN_pixels)
FP_pixels = maskimg(FP_mask, FP_pixels)
TN_pixels = maskimg(TN_mask, TN_pixels)
if self.wants_emd:
emd = self.compute_emd(objects_ID, objects_GT)
m.add_image_measurement(
self.measurement_name(FTR_EARTH_MOVERS_DISTANCE), emd)
if self.show_window:
workspace.display_data.true_positives = TP_pixels
workspace.display_data.true_negatives = TN_pixels
workspace.display_data.false_positives = FP_pixels
workspace.display_data.false_negatives = FN_pixels
workspace.display_data.statistics = [
(FTR_F_FACTOR, F_factor),
(FTR_PRECISION, precision),
(FTR_RECALL, recall),
(FTR_FALSE_POS_RATE, false_positive_rate),
(FTR_FALSE_NEG_RATE, false_negative_rate),
(FTR_RAND_INDEX, rand_index),
(FTR_ADJUSTED_RAND_INDEX, adjusted_rand_index)
]
if self.wants_emd:
workspace.display_data.statistics.append(
(FTR_EARTH_MOVERS_DISTANCE, emd))
#Performing meanshift on flatImg
ms1.fit(flat_image)
#(r,g,b) vectors corresponding to the different clusters after meanshift
labels1=ms1.labels_
#Remaining colors after meanshift
cluster_centers1 = ms1.cluster_centers_
#Finding and diplaying the number of clusters
labels_unique1 = np.unique(labels1)
n_clusters_1 = len(labels_unique1)
#print("number of estimated clusters : %d" % n_clusters_1)
# Displaying segmented image
seg1 = cluster_centers1[np.reshape(labels1, originShape[:2])]
segImage1 = seg1.astype(np.uint8)
imageR = cv2.resize(image, (250, 300))
segImage1_R = cv2.resize(segImage1, (250, 300))
print("Time takend to complete Segmentation is {0}seconds.".format(time.time()-start))
#Result = np.hstack((imageR,segImage1_R,segImage2_R,segImage3_R,segImage4_R))
Result = np.hstack((imageR,segImage1_R))
cv2.imshow('Image', Result)
cv2.waitKey(0)
cv2.destroyAllWindows()
