def _trimmed_mean_1d(arr, k):
"""Calculate trimmed mean on a 1d array.
Trim values largest than the k'th largest value or smaller than the k'th
smallest value
Parameters
----------
arr: ndarray, shape (n,)
The one-dimensional input array to perform trimmed mean on
k: int
The thresholding order for trimmed mean
Returns
-------
trimmed_mean: float
The trimmed mean calculated
"""
kth_smallest = np.partition(arr, k)[k-1]
kth_largest = -np.partition(-arr, k)[k-1]
cnt = 0
summation = 0.0
for elem in arr:
if elem >= kth_smallest and elem <= kth_largest:
cnt += 1
summation += elem
return summation / cnt
def arg_median(a):
if len(a) % 2 == 1:
return np.where( a == np.median(a) )[0][0]
else:
l,r = len(a)/2 -1, len(a)/2
left = np.partition(a, l)[l]
right = np.partition(a, r)[r]
return [np.where(a == left)[0][0], np.where(a==right)[0][0]]
def color_range(data): #Define the color range clean = data[data>0] min_element = clean.size/20 max_element = clean.size*9/10 vmin = np.partition(clean, min_element, axis=None)[min_element] #invece di subint[subint>0] possibile subint[:-(num_rows/down_fact)] vmax = np.partition(clean, max_element, axis=None)[max_element] return vmin,vmax
def _degree_feats(uts=None, G=None, name_ext="", exclude_id=None):
"""
Helper method for retrieve_feats().
Generate statistics on degree-related features in a Hypergraph (G), or a Hypergraph
constructed from provided utterances (uts)
:param uts: utterances to construct Hypergraph from
:param G: Hypergraph to calculate degree features statistics from
:param name_ext: Suffix to append to feature name
:param exclude_id: id of utterance to exclude from Hypergraph construction
:return: A dictionary from a thread root id to its stats dictionary,
which is a dictionary from feature names to feature values. For degree-related
features specifically.
"""
assert uts is None or G is None
if G is None:
G = HyperConvo._make_hypergraph(uts, exclude_id=exclude_id)
stat_funcs = {
"max": np.max,
"argmax": np.argmax,
"norm.max": lambda l: np.max(l) / np.sum(l),
"2nd-largest": lambda l: np.partition(l, -2)[-2] if len(l) > 1
else np.nan,
"2nd-argmax": lambda l: (-l).argsort()[1] if len(l) > 1 else np.nan,
"norm.2nd-largest": lambda l: np.partition(l, -2)[-2] / np.sum(l)
if len(l) > 1 else np.nan,
"mean": np.mean,
"mean-nonzero": lambda l: np.mean(l[l != 0]),
"prop-nonzero": lambda l: np.mean(l != 0),
"prop-multiple": lambda l: np.mean(l[l != 0] > 1),
"entropy": scipy.stats.entropy,
"2nd-largest / max": lambda l: np.partition(l, -2)[-2] / np.max(l)
if len(l) > 1 else np.nan
}
stats = {}
for from_hyper in [False, True]:
for to_hyper in [False, True]:
if not from_hyper and to_hyper: continue # skip c -> C
outdegrees = np.array(G.outdegrees(from_hyper, to_hyper))
indegrees = np.array(G.indegrees(from_hyper, to_hyper))
for stat, stat_func in stat_funcs.items():
stats["{}[outdegree over {}->{} {}responses]".format(stat,
HyperConvo._node_type_name(from_hyper),
HyperConvo._node_type_name(to_hyper),
name_ext)] = stat_func(outdegrees)
stats["{}[indegree over {}->{} {}responses]".format(stat,
HyperConvo._node_type_name(from_hyper),
HyperConvo._node_type_name(to_hyper),
name_ext)] = stat_func(indegrees)
return stats
def getMedianIndex(array): if len(array) % 2 == 1: print "Par" return np.where( array == np.median(array) )[0][0] else: try: print "Impar" l,r = len(array)/2 -1, len(array)/2 left = np.partition(array, l)[l] right = np.partition(array, r)[r] return [np.where(array == left)[0][0], np.where(array==right)[0][0]] except: print "Warning: excepction controlled in median : "+str(len(array)) return (len(array)/(np.e*2))
def avg_x_weak_weights(wmxO, x):
'''
Holds only the 4000-x strongest weight in every row and average the other ones
:param wmxO: original weight matrix (4000 * 4000 ndarray)
:return: wmxM: modified weight matrix (4000 * 4000 ndarray)
'''
nHolded = 4000 - x
wmxM = np.zeros((4000, 4000))
for i in range(0, 4000):
row = wmxO[i, :]
tmp = np.partition(-row, nHolded)
max = -tmp[:nHolded] # values of first 4000-x elements
rest = -tmp[nHolded:]
mu = np.mean(rest) # mean of the x weights
tmp = np.argpartition(-row, nHolded)
maxj = tmp[:nHolded] # indexes of first 4000-x elements
rowM = mu * np.ones((1, 4000))
for j, val in zip(maxj, max):
rowM[0, j] = val
wmxM[i, :] = rowM
return wmxM
def compute_csls_accuracy(x_src, x_tgt, lexicon, lexicon_size=-1, k=10, bsz=1024):
if lexicon_size < 0:
lexicon_size = len(lexicon)
idx_src = list(lexicon.keys())
x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8
x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8
sr = x_src[list(idx_src)]
sc = np.dot(sr, x_tgt.T)
similarities = 2 * sc
sc2 = np.zeros(x_tgt.shape[0])
for i in range(0, x_tgt.shape[0], bsz):
j = min(i + bsz, x_tgt.shape[0])
sc_batch = np.dot(x_tgt[i:j, :], x_src.T)
dotprod = np.partition(sc_batch, -k, axis=1)[:, -k:]
sc2[i:j] = np.mean(dotprod, axis=1)
similarities -= sc2[np.newaxis, :]
nn = np.argmax(similarities, axis=1).tolist()
correct = 0.0
for k in range(0, len(lexicon)):
if nn[k] in lexicon[idx_src[k]]:
correct += 1.0
return correct / lexicon_size
def make_query(self):
"""Return the index of the sample to be queried and labeled.
Returns
-------
ask_id: int
The entry_id of the sample this algorithm wants to query.
"""
dataset = self.dataset
self.model.train(dataset)
unlabeled_entry_ids, X_pool = zip(*dataset.get_unlabeled_entries())
if self.method == 'lc': # least confident
ask_id = np.argmin(
np.max(self.model.predict_real(X_pool), axis=1)
)
elif self.method == 'sm': # smallest margin
dvalue = self.model.predict_real(X_pool)
if np.shape(dvalue)[1] > 2:
# Find 2 largest decision values
dvalue = -(np.partition(-dvalue, 2, axis=1)[:, :2])
margin = np.abs(dvalue[:, 0] - dvalue[:, 1])
ask_id = np.argmin(margin)
return unlabeled_entry_ids[ask_id]
def _chunk_based_bmu_find(input_matrix, codebook, y2, nth=1):
"""
Finds the corresponding bmus to the input matrix.
:param input_matrix: a matrix of input data, representing input vector as
rows, and vectors features/dimention as cols
when parallelizing the search, the input_matrix can be
a sub matrix from the bigger matrix
:param codebook: matrix of weights to be used for the bmu search
:param y2: <not sure>
"""
dlen = input_matrix.shape[0]
nnodes = codebook.shape[0]
bmu = np.empty((dlen, 2))
# It seems that small batches for large dlen is really faster:
# that is because of ddata in loops and n_jobs. for large data it slows
# down due to memory needs in parallel
blen = min(50, dlen)
i0 = 0
while i0+1 <= dlen:
low = i0
high = min(dlen, i0+blen)
i0 = i0+blen
ddata = input_matrix[low:high+1]
d = np.dot(codebook, ddata.T)
d *= -2
d += y2.reshape(nnodes, 1)
bmu[low:high+1, 0] = np.argpartition(d, nth, axis=0)[nth-1]
bmu[low:high+1, 1] = np.partition(d, nth, axis=0)[nth-1]
del ddata
return bmu
def disp_results(fig, ax1, ax2, loss_iterations, losses, accuracy_iterations, accuracies, accuracies_iteration_checkpoints_ind, fileName, color_ind=0):
modula = len(plt.rcParams['axes.color_cycle'])
acrIterations =[]
top_acrs={}
if accuracies.size:
if accuracies.size>4:
top_n = 4
else:
top_n = accuracies.size -1
temp = np.argpartition(-accuracies, top_n)
result_indexces = temp[:top_n]
temp = np.partition(-accuracies, top_n)
result = -temp[:top_n]
for acr in result_indexces:
acrIterations.append(accuracy_iterations[acr])
top_acrs[str(accuracy_iterations[acr])]=str(accuracies[acr])
sorted_top4 = sorted(top_acrs.items(), key=operator.itemgetter(1))
maxAcc = np.amax(accuracies, axis=0)
iterIndx = np.argmax(accuracies)
maxAccIter = accuracy_iterations[iterIndx]
maxIter = accuracy_iterations[-1]
consoleInfo = format('\n[%s]:maximum accuracy [from 0 to %s ] = [Iteration %s]: %s ' %(fileName,maxIter,maxAccIter ,maxAcc))
plotTitle = format('max accuracy(%s) [Iteration %s]: %s ' % (fileName,maxAccIter, maxAcc))
print (consoleInfo)
#print (str(result))
#print(acrIterations)
# print 'Top 4 accuracies:'
print ('Top 4 accuracies:'+str(sorted_top4))
plt.title(plotTitle)
ax1.plot(loss_iterations, losses, color=plt.rcParams['axes.color_cycle'][(color_ind * 2 + 0) % modula])
ax2.plot(accuracy_iterations, accuracies, plt.rcParams['axes.color_cycle'][(color_ind * 2 + 1) % modula], label=str(fileName))
ax2.plot(accuracy_iterations[accuracies_iteration_checkpoints_ind], accuracies[accuracies_iteration_checkpoints_ind], 'o', color=plt.rcParams['axes.color_cycle'][(color_ind * 2 + 1) % modula])
plt.legend(loc='lower right')
def find_top_k(x, k):
#return an array where anything less than the top k values of an array is zero
if( np.count_nonzero(x) <k):
return x
else:
x[x < -1*np.partition(-1*x, k)[k]] = 0
return x
def distance_to_kth_neighbor(metric, k_neighbors):
"""Computes the distance to the kth neighbor for each point in a metric.
Args
----
metric : ndarray
A distance matrix in square or condensed form.
k_neighbors : int
The order of neighbor to which distance is computed, where the 0th
neighbor of any point is the point itself.
Returns
-------
distances : ndarray
Distance to the kth neighbor for each point in `metric`.
Note
----
It is an implementation detail that the input metric is coerced to a
squareform array.
"""
# coerce any condensed metric to square form
if metric.ndim == 1:
metric = _dist.squareform(metric)
if k_neighbors >= metric.shape[0]:
message = 'k_neighbors must be less than the number of points.'
raise ValueError(message)
if k_neighbors < 0:
raise ValueError('k_neighbors must be non-negative')
return _np.partition(metric, k_neighbors, axis=1)[:, k_neighbors]
def estimate_eps(dist_mat, n_closest=5):
"""
Estimates possible eps values (to be used with DBSCAN)
for a given distance matrix by looking at the largest distance "jumps"
amongst the `n_closest` distances for each item.
Tip: the value for `n_closest` is important - set it too large and you may only get
really large distances which are uninformative. Set it too small and you may get
premature cutoffs (i.e. select jumps which are really not that big).
TO DO this could be fancier by calculating support for particular eps values,
e.g. 80% are around 4.2 or w/e
"""
dist_mat = dist_mat.copy()
# To ignore i == j distances
dist_mat[np.where(dist_mat == 0)] = np.inf
estimates = []
for i in range(dist_mat.shape[0]):
# Indices of the n closest distances
row = dist_mat[i]
dists = sorted(np.partition(row, n_closest)[:n_closest])
difs = [(x, y,
(y - x)) for x, y in zip(dists, dists[1:])]
eps_candidate, _, jump = max(difs, key=lambda x: x[2])
estimates.append(eps_candidate)
return sorted(estimates)
def nsmall(arr, n, axis):
return np.partition(arr, n, axis)[n]
#def addToPlot(subplot, appendX, appendY):
# plt.figure(1)
# plt.subplot(subplot)
# plt.se
def test_uint64(self):
arr = np.arange(10, -1, -1, dtype='int64')
partition = np.partition(arr, self.pivot_index)
self._check_partition(partition, self.pivot_index)
self._check_content_along_axis(arr, partition, -1)
def make_query(self):
"""
Choices for method (default 'lc'):
'lc' (Least Confident), 'sm' (Smallest Margin)
"""
dataset = self.dataset
self.model.train(dataset)
unlabeled_entry_ids, X_pool = zip(*dataset.get_unlabeled_entries())
if self.method == 'lc': # least confident
ask_id = np.argmin(
np.max(self.model.predict_real(X_pool), axis=1)
)
elif self.method == 'sm': # smallest margin
dvalue = self.model.predict_real(X_pool)
if np.shape(dvalue)[1] > 2:
# Find 2 largest decision values
dvalue = -(np.partition(-dvalue, 2, axis=1)[:, :2])
margin = np.abs(dvalue[:, 0] - dvalue[:, 1])
ask_id = np.argmin(margin)
return unlabeled_entry_ids[ask_id]
def fast_abs_percentile(data, percentile=80):
""" A fast version of the percentile of the absolute value.
Parameters
==========
data: ndarray, possibly masked array
The input data
percentile: number between 0 and 100
The percentile that we are asking for
Returns
=======
value: number
The score at percentile
Notes
=====
This is a faster, and less accurate version of
scipy.stats.scoreatpercentile(np.abs(data), percentile)
"""
if hasattr(data, 'mask'):
# Catter for masked arrays
data = np.asarray(data[np.logical_not(data.mask)])
data = np.abs(data)
data = data.ravel()
index = int(data.size * .01 * percentile)
if partition is not None:
# Partial sort: faster than sort
return partition(data, index)[index + 1]
data.sort()
return data[index + 1]
def random_con_bitmask(prob, shape, mins=1):
"""Generate a random bitmask with constraints
The functions allows specifying the minimum number of True values along a
single dimension while counting over the other ones. `mins` can be a scalar
or a tuple for each dimension and must be less than the product of the size
of the other dimensions.
If you just want a random bitmask use np.random.random(shape) < prob"""
assert len(shape) > 1
vals = np.random.random(shape)
mask = vals < prob
total = vals.size
if isinstance(mins, abc.Sequence):
assert len(mins) == vals.ndim
assert all(0 < s <= total // m for s, m in zip(mins, vals.shape))
else:
assert mins > 0
mins = tuple(min(mins, total // m) for m in vals.shape)
for dim, num in enumerate(mins):
aligned = np.rollaxis(vals, dim).reshape(vals.shape[dim], -1)
thresh = np.partition(aligned, num - 1, 1)[:, num - 1]
thresh.shape += (1,) * (vals.ndim - dim - 1)
mask |= vals <= thresh
return mask
def sumvalues(self, q=0):
"""Sum of top q passowrd frequencies
"""
if q == 0:
return self._totalf
else:
return -np.partition(-self._freq_list, q)[:q].sum()
def test_partition_matrix_none():
# gh-4301
# 2018-04-29: moved here from core.tests.test_multiarray
a = np.matrix([[2, 1, 0]])
actual = np.partition(a, 1, axis=None)
expected = np.matrix([[0, 1, 2]])
assert_equal(actual, expected)
assert_(type(expected) is np.matrix)
def get_top_k_elements_per_row_sim_mat(similarity_matrix, k):
'''
Introduces another parameter, k, where we only keep the top k similarity
scores per row in the similarity matrix.
'''
for i, row in enumerate(similarity_matrix):
k = len(row) - k
kth_largest = np.partition(row, k)[k]
similarity_matrix[i] = [ele if ele >= kth_largest else 0 for ele in row]
return similarity_matrix
def _phase2(self): """ Execute phase 2 of the SP region. This phase is used to compute the active columns. Note - This should only be called after phase 1 has been called and after the inhibition radius and neighborhood have been updated. """ # Shift the outputs self.y[:, 1:] = self.y[:, :-1] self.y[:, 0] = 0 # Calculate k # - For a column to be active its overlap must be at least as large # as the overlap of the k-th largest column in its neighborhood. k = self._get_num_cols() if self.global_inhibition: # The neighborhood is all columns, thus the set of active columns # is simply columns that have an overlap >= the k-th largest in the # entire region # Compute the winning column indexes if self.learn: # Randomly break ties ix = np.argpartition(-self.overlap[:, 0] - self.prng.uniform(.1, .2, self.ncolumns), k - 1)[:k] else: # Choose the same set of columns each time ix = np.argpartition(-self.overlap[:, 0], k - 1)[:k] # Set the active columns self.y[ix, 0] = self.overlap[ix, 0] > 0 else: # The neighborhood is bounded by the inhibition radius, therefore # each column's neighborhood must be considered for i in xrange(self.ncolumns): # Get the neighbors ix = np.where(self.neighbors[i])[0] # Compute the minimum top overlap if ix.shape[0] <= k: # Desired number of candidates is at or below the desired # activity level, so find the overall min m = max(bn.nanmin(self.overlap[ix, 0]), 1) else: # Desired number of candidates is above the desired # activity level, so find the k-th largest m = max(-np.partition(-self.overlap[ix, 0], k - 1)[k - 1], 1) # Set the column activity if self.overlap[i, 0] >= m: self.y[i, 0] = True
def cluster_select_func(self):
num_selected = self.num_selected
if(hasattr(self,'alternate_clustering_input')):
inp = self.alternate_clustering_input
# print("alternate used...")
# print("alternate_clustering_input_shape: " + str(self.alternate_clustering_input.shape) + " net.input.shape: " +
# str(self.input.shape))
else:
inp = self.input
if(hasattr(self,'do_weighted_euclidean')):
self.distances = np.sum(self.centroids**2,1)[:,np.newaxis] \
- 2.0*np.dot(self.centroids*self.weights,inp) \
+ np.dot(self.weights**2,inp**2)
#temp_centroids = self.centroids/self.weights;
#temp_distances = np.sum(temp_centroids**2,1)[:,np.newaxis] - 2*np.dot(temp_centroids,self.input) + \
# np.sum(self.input**2,0)[np.newaxis,:]
#self.distances = temp_distances*(np.sum(self.weights**2,1)[:,np.newaxis])
#print("Distance error: " + str(np.sum(np.sum((temp_distances - self.distances)**2))))
print("Weighted Euclidean Distance")
elif(hasattr(self,'do_cosinedistance')):
self.distances = -np.dot(self.centroids,inp)/(np.sqrt(np.sum(self.centroids**2.,1)[:,np.newaxis]*np.sum(inp**2.,0)[np.newaxis,:]))
else:
#print("centroids shape: " + str(self.centroids.shape))
#print("inp shape: " + str(inp.shape))
#print("centroids dtype: " + str(self.centroids.dtype))
#print("inp dtype: " + str(inp.dtype))
self.distances = pairwise_distances(self.centroids,self.input.T)
#self.distances = np.sum(self.centroids**2,1)[:,np.newaxis] - 2*np.dot(self.centroids,inp) + \
# np.sum(inp**2,0)[np.newaxis,:]
distances_sorted = np.partition(self.distances,num_selected,axis=0)
#distances_sorted = np.sort(self.distances,axis=0)
#print("distances_sorted " + str(distances_sorted[0:10,0]))
self.selected_neurons = self.distances > distances_sorted[num_selected,:]
#keep track of this so we can count the number of times a centroid was selected
self.saved_selected_neurons = np.copy(self.selected_neurons)
#initialize selected count to 0
if(not hasattr(self,'selected_count')):
self.selected_count = np.zeros(self.saved_selected_neurons.shape[0])
if(not hasattr(self,'eligibility_count')):
self.eligibility_count = np.ones(self.saved_selected_neurons.shape[0])
self.centroids_prime = (np.dot(inp,(~self.selected_neurons).transpose())/ \
np.sum(~self.selected_neurons,1,dtype=np.float32)).transpose()
self.centroids_prime[np.isnan(self.centroids_prime)] = self.centroids[np.isnan(self.centroids_prime)]
if(hasattr(self,'do_weighted_euclidean')):
self.centroids_prime = self.centroids_prime*self.weights;
#save a copy of the full output -- it is useful to some tests
self.full_output = np.copy(self.output)
self.output[self.selected_neurons] = 0;
def min_payoffs(self):
# Computes the min payoff by finding the required facilities that have
# the lowest payoff when everyone uses them. This will fail if the
# highest order term isn't negative.
if self._min_payoffs is None:
self._min_payoffs = self.num_players.astype(float)
self._min_payoffs *= np.partition(
self.facility_coefs.dot(self.num_players ** np.arange(3)),
self.num_required - 1)[:self.num_required].sum()
self._min_payoffs.setflags(write=False)
return self._min_payoffs
def largestElement(x, n):
"""
Returns the n-th largest element of x
:param x:
:param n:
:return:
"""
# The n-th largest element is the (N+1-n)th smallest element
len = x.size
desc_pos = (len-n)
return np.partition(x, desc_pos)[desc_pos]
def fit(self, X, Y):
'''
X: (n, d) array-like of samples
Y: (n,) array-like of class labels
'''
X, Y, num_classes, n, d = self._process_inputs(X, Y)
tSb = np.zeros((d,d))
tSw = np.zeros((d,d))
for c in xrange(num_classes):
Xc = X[Y==c]
nc = Xc.shape[0]
# classwise affinity matrix
dist = pairwise_distances(Xc, metric='l2', squared=True)
# distances to k-th nearest neighbor
k = min(self.params['k'], nc-1)
sigma = np.sqrt(np.partition(dist, k, axis=0)[:,k])
local_scale = np.outer(sigma, sigma)
with np.errstate(divide='ignore', invalid='ignore'):
A = np.exp(-dist/local_scale)
A[local_scale==0] = 0
G = Xc.T.dot(A.sum(axis=0)[:,None] * Xc) - Xc.T.dot(A).dot(Xc)
tSb += G/n + (1-nc/n)*Xc.T.dot(Xc) + _sum_outer(Xc)/n
tSw += G/nc
tSb -= _sum_outer(X)/n - tSw
# symmetrize
tSb += tSb.T
tSb /= 2
tSw += tSw.T
tSw /= 2
if self.params['dim'] == d:
vals, vecs = scipy.linalg.eigh(tSb, tSw)
else:
vals, vecs = scipy.sparse.linalg.eigsh(tSb, k=self.params['dim'], M=tSw,
which='LA')
order = np.argsort(-vals)[:self.params['dim']]
vals = vals[order]
vecs = vecs[:,order]
if self.params['metric'] == 'weighted':
vecs *= np.sqrt(vals)
elif self.params['metric'] == 'orthonormalized':
vecs, _ = np.linalg.qr(vecs)
self._transformer = vecs.T
return self
def fast_abs_percentile(map, percentile=80):
""" A fast version of the percentile of the absolute value.
"""
if hasattr(map, 'mask'):
# Catter for masked arrays
map = np.asarray(map[np.logical_not(map.mask)])
map = np.abs(map)
map = map.ravel()
index = int(map.size * .01 * percentile)
if partition is not None:
# Partial sort: faster than sort
return partition(map, index)[index + 1]
return map.sort()[index]
def hard_sparsity(self, d, sparsity, save_metadata=True):
"""Alters the given linear decomposition, applying a median filter.
The filtering process is done in the time domain.
The argument provided is destroyed.
Args:
d: LinearDecomposition object to filter.
sparsity: maximum polyphony allowed.
save_metadata: flag indicating whether the metadata should be
computed. Default: True.
Returns:
Same decomposition given in arguments.
"""
if save_metadata:
metadata = md.ObjectMetadata(a)
else:
metadata = None
meta = md.Metadata(name="sparsity_level",
sparsity=sparsity,
activation_input=metadata,
original_method=None)
# Binarizes the data and adjusts the metadata
A = None
p = []
for k in d.data.right.keys():
if A is None:
A = d.data.right[k]
else:
A = numpy.vstack((A, d.data.right[k]))
p.append(k)
for i in xrange(A.shape[1]):
b = numpy.partition(A[:,i], sparsity-1)[sparsity-1]
A[:,i] = A[:,i] * (A[:,i] >= b)
for argk in xrange(len(p)):
d.data.right[p[argk]] = A[argk,:]
d.data.right[p[argk]].shape = (1, len(d.data.right[p[argk]]))
d.metadata.right[p[argk]] = md.Metadata(method="sparsity_level",
sparsity=sparsity,
activation_input=metadata,
original_method =
d.metadata.right[k])
return d
def topk(a, k, axis, keepdims):
""" Chunk and combine function of topk
Extract the k largest elements from a on the given axis.
If k is negative, extract the -k smallest elements instead.
Note that, unlike in the parent function, the returned elements
are not sorted internally.
"""
assert keepdims is True
axis = axis[0]
if abs(k) >= a.shape[axis]:
return a
a = np.partition(a, -k, axis=axis)
k_slice = slice(-k, None) if k > 0 else slice(-k)
return a[tuple(k_slice if i == axis else slice(None)
for i in range(a.ndim))]
def topk(a, k, axis, keepdims):
"""Kernel of topk and argtopk.
Extract the k largest elements from a on the given axis.
If k is negative, extract the -k smallest elements instead.
Note that, unlike in the parent function, the returned elements
are not sorted internally.
"""
axis = axis[0]
if abs(k) >= a.shape[axis]:
return a
a = np.partition(a, -k, axis=axis)
# return a[-k:] if k>0 else a[:-k], on arbitrary axis
return a[[
(slice(-k, None) if k > 0 else slice(None, -k))
if i == axis else slice(None)
for i in range(a.ndim)
]]
def schedule(self, simulation, changed):
clock = simulation.clock
# mark freed hosts
for task in changed:
if task.kind == csimdag.TaskKind.TASK_KIND_COMP_SEQ and task.state == csimdag.TaskState.TASK_STATE_DONE:
host = task.hosts[0]
if host != self.master_host:
task.hosts[0].data["is_free"] = True
# schedule unscheduled ready tasks
unscheduled = self.tasks[csimdag.TaskState.TASK_STATE_SCHEDULABLE]\
.by_func(lambda task: task.data["target_host"] is None)
num_unscheduled = len(unscheduled)
if num_unscheduled > 0:
# build ECT matrix
ECT = np.zeros((num_unscheduled, len(self.exec_hosts)))
for t, task in enumerate(unscheduled):
parents = [(p, p.parents[0].hosts[0]) for p in task.parents
if p.kind == csimdag.TaskKind.TASK_KIND_COMM_E2E]
for h, host in enumerate(self.exec_hosts):
comm_times = [
p_comm.get_ecomt(p_host, host)
for (p_comm, p_host) in parents
]
ECT[t][h] = max(host.data["est"], clock) + task.get_eet(
host) + (max(comm_times) if comm_times else 0.)
# build schedule
task_idx = np.arange(num_unscheduled)
for _ in range(0, num_unscheduled):
min_hosts = np.argmin(ECT, axis=1)
min_times = ECT[np.arange(ECT.shape[0]), min_hosts]
if ECT.shape[1] > 1:
min2_times = np.partition(ECT, 1)[:, 1]
# round sufferage values to eliminate the influence of floating point errors
sufferages = np.round(min2_times - min_times, decimals=2)
else:
# use min time for a single host case
sufferages = min_times
possible_schedules = []
for i in range(0, len(task_idx)):
best_host_idx = int(min_hosts[i])
best_ect = min_times[i]
sufferage = sufferages[i]
possible_schedules.append(
(i, best_host_idx, best_ect, sufferage))
t, h, ect = self.batch_heuristic(possible_schedules)
task = unscheduled[int(task_idx[t])]
host = self.exec_hosts[h]
# logging.info("%s -> %s" % (task.name, host.name))
task.data["target_host"] = host.name
host.data["tasks"].append(task)
task_time = ect - max(host.data["est"], clock)
host.data["est"] = ect
task_idx = np.delete(task_idx, t)
ECT = np.delete(ECT, t, 0)
ECT[:, h] += task_time
for host in self.exec_hosts:
if host.data["is_free"]:
tasks = host.data["tasks"]
if len(tasks) > 0:
task = tasks.pop(0)
task.schedule(host)
host.data["is_free"] = False
def test(db, split, testiter, debug=False, suffix=None):
result_dir = system_configs.result_dir
result_dir = os.path.join(result_dir, str(testiter), split)
class_name = []
for i in range(1, len(db._coco.cats)):
# if db._coco.cats[i] is None:
# continue
# else:
ind = db._cat_ids[i]
class_name.append(db._coco.cats[ind]['name'])
if suffix is not None:
result_dir = os.path.join(result_dir, suffix)
make_dirs([result_dir])
test_iter = system_configs.max_iter if testiter is None else testiter
print("loading parameters at iteration: {}".format(test_iter))
print("building neural network...")
nnet = NetworkFactory(db)
print("loading parameters...")
nnet.load_params(test_iter)
# test_file = "test.{}".format(db.data)
# testing = importlib.import_module(test_file).testing
nnet.cuda()
nnet.eval_mode()
debug_dir = os.path.join(result_dir, "debug")
if not os.path.exists(debug_dir):
os.makedirs(debug_dir)
if db.split != "trainval":
db_inds = db.db_inds[:100] if debug else db.db_inds
else:
db_inds = db.db_inds[:100] if debug else db.db_inds[:5000]
K = db.configs["top_k"]
ae_threshold = db.configs["ae_threshold"]
nms_kernel = db.configs["nms_kernel"]
scales = db.configs["test_scales"]
weight_exp = db.configs["weight_exp"]
merge_bbox = db.configs["merge_bbox"]
categories = db.configs["categories"]
nms_threshold = db.configs["nms_threshold"]
max_per_image = db.configs["max_per_image"]
nms_algorithm = {
"nms": 0,
"linear_soft_nms": 1,
"exp_soft_nms": 2
}[db.configs["nms_algorithm"]]
img_name = os.listdir(db._image_dir)
for i in range(0, len(img_name)):
top_bboxes = {}
# for ind in tqdm(range(0, num_images), ncols=80, desc="locating kps"):
db_ind = i + 1
# image_id = db.image_ids(db_ind)
image_id = img_name[i]
image_file = db._image_dir + '/' + img_name[i]
image = cv2.imread(image_file)
height, width = image.shape[0:2]
detections = []
for scale in scales:
new_height = int(height * scale)
new_width = int(width * scale)
new_center = np.array([new_height // 2, new_width // 2])
inp_height = new_height | 127
inp_width = new_width | 127
images = np.zeros((1, 3, inp_height, inp_width), dtype=np.float32)
ratios = np.zeros((1, 2), dtype=np.float32)
borders = np.zeros((1, 4), dtype=np.float32)
sizes = np.zeros((1, 2), dtype=np.float32)
out_height, out_width = (inp_height + 1) // 4, (inp_width + 1) // 4
height_ratio = out_height / inp_height
width_ratio = out_width / inp_width
resized_image = cv2.resize(image, (new_width, new_height))
resized_image, border, offset = crop_image(resized_image, new_center, [inp_height, inp_width])
resized_image = resized_image / 255.
normalize_(resized_image, db.mean, db.std)
images[0] = resized_image.transpose((2, 0, 1))
borders[0] = border
sizes[0] = [int(height * scale), int(width * scale)]
ratios[0] = [height_ratio, width_ratio]
images = np.concatenate((images, images[:, :, :, ::-1]), axis=0)
images = torch.from_numpy(images)
dets = kp_decode(nnet, images, K, ae_threshold=ae_threshold, kernel=nms_kernel)
dets = dets.reshape(2, -1, 8)
dets[1, :, [0, 2]] = out_width - dets[1, :, [2, 0]]
dets = dets.reshape(1, -1, 8)
_rescale_dets(dets, ratios, borders, sizes)
dets[:, :, 0:4] /= scale
detections.append(dets)
detections = np.concatenate(detections, axis=1)
classes = detections[..., -1]
classes = classes[0]
detections = detections[0]
# reject detections with negative scores
keep_inds = (detections[:, 4] > -1)
detections = detections[keep_inds]
classes = classes[keep_inds]
top_bboxes[image_id] = {}
for j in range(categories):
keep_inds = (classes == j)
top_bboxes[image_id][j + 1] = detections[keep_inds][:, 0:7].astype(np.float32)
if merge_bbox:
soft_nms_merge(top_bboxes[image_id][j + 1], Nt=nms_threshold, method=nms_algorithm,
weight_exp=weight_exp)
else:
soft_nms(top_bboxes[image_id][j + 1], Nt=nms_threshold, method=nms_algorithm)
top_bboxes[image_id][j + 1] = top_bboxes[image_id][j + 1][:, 0:5]
scores = np.hstack([
top_bboxes[image_id][j][:, -1]
for j in range(1, categories + 1)
])
if len(scores) > max_per_image:
kth = len(scores) - max_per_image
thresh = np.partition(scores, kth)[kth]
for j in range(1, categories + 1):
keep_inds = (top_bboxes[image_id][j][:, -1] >= thresh)
top_bboxes[image_id][j] = top_bboxes[image_id][j][keep_inds]
# result_json = os.path.join(result_dir, "results.json")
detections = db.convert_to_list(top_bboxes)
print('demo for {}'.format(image_id))
img = cv2.imread(image_file)
box = []
if detections is not None:
for i in range(len(detections)):
name = db._coco.cats[detections[i][1]]['name'] #db._coco.cats[ind]['name']
confi = detections[i][-1]
if confi <0.3:
continue
for j in range(0, 4):
box.append(detections[i][j + 2])
cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), (0, 255, 255), 1)
# cv2.putText(img, name[0] + ' ' + '{:.3f}'.format(confi), (int(box[0]), int(box[1] - 10)),
# cv2.FONT_ITALIC, 1, (0, 0, 255), 1)
while (box):
box.pop(-1)
cv2.imshow('Detecting image...', img)
# timer.total_time = 0
if cv2.waitKey(3000) & 0xFF == ord('q'):
break
print(detections)
def main():
logger = create_logger(save_dir=cfg.log_dir)
print = logger.info
print(cfg)
cfg.device = torch.device('cuda')
torch.backends.cudnn.benchmark = False
max_per_image = 100
Dataset_eval = COCO_eval if cfg.dataset == 'coco' else PascalVOC_eval
dataset = Dataset_eval(cfg.data_dir,
split='val',
img_size=cfg.img_size,
test_scales=cfg.test_scales,
test_flip=cfg.test_flip)
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=False,
collate_fn=dataset.collate_fn)
print('Creating model...')
if 'hourglass' in cfg.arch:
model = get_hourglass[cfg.arch]
elif 'resdcn' in cfg.arch:
model = get_pose_net(num_layers=int(cfg.arch.split('_')[-1]),
num_classes=dataset.num_classes)
else:
raise NotImplementedError
model = load_model(model, cfg.pretrain_dir)
model = model.to(cfg.device)
model.eval()
results = {}
with torch.no_grad():
for inputs in data_loader:
img_id, inputs = inputs[0]
detections = []
for scale in inputs:
inputs[scale]['image'] = inputs[scale]['image'].to(cfg.device)
output = model(inputs[scale]['image'])[-1]
dets = ctdet_decode(*output, K=cfg.test_topk)
dets = dets.detach().cpu().numpy().reshape(
1, -1, dets.shape[2])[0]
top_preds = {}
dets[:, :2] = transform_preds(
dets[:,
0:2], inputs[scale]['center'], inputs[scale]['scale'],
(inputs[scale]['fmap_w'], inputs[scale]['fmap_h']))
dets[:, 2:4] = transform_preds(
dets[:,
2:4], inputs[scale]['center'], inputs[scale]['scale'],
(inputs[scale]['fmap_w'], inputs[scale]['fmap_h']))
cls = dets[:, -1]
for j in range(dataset.num_classes):
inds = (cls == j)
top_preds[j + 1] = dets[inds, :5].astype(np.float32)
top_preds[j + 1][:, :4] /= scale
detections.append(top_preds)
bbox_and_scores = {}
for j in range(1, dataset.num_classes + 1):
bbox_and_scores[j] = np.concatenate([d[j] for d in detections],
axis=0)
if len(dataset.test_scales) > 1:
soft_nms(bbox_and_scores[j], Nt=0.5, method=2)
scores = np.hstack([
bbox_and_scores[j][:, 4]
for j in range(1, dataset.num_classes + 1)
])
if len(scores) > max_per_image:
kth = len(scores) - max_per_image
thresh = np.partition(scores, kth)[kth]
for j in range(1, dataset.num_classes + 1):
keep_inds = (bbox_and_scores[j][:, 4] >= thresh)
bbox_and_scores[j] = bbox_and_scores[j][keep_inds]
results[img_id] = bbox_and_scores
eval_results = dataset.run_eval(results, cfg.ckpt_dir)
print(eval_results)
import numpy as np
# numpy裡面的排序, default: np.sort為Quick Sort(Time: N logN)
x = np.array([2, 4, 3, 1, 5])
# print("np.sort(x): ", np.sort(x)) # 傳回排序後的陣列, 但不會改變原本的內容
# x.sort() # 會改變原本的陣列內容
# print("x: ", x)
# 另一種排序為np.argsort() 傳會被排序過後元素的索引值
i = np.argsort(x)
# print("index: ", i)
# 沿著行或列排序 增加axis參數即可
rand = np.random.RandomState(1)
array = rand.randint(0, 10, (4, 6))
print("Array: \n", array)
# print("row sorted: \n", np.sort(array, axis = 0)) # 行排序
# print("column sorted: \n", np.sort(array, axis = 1)) # 列排序
# 分區(Partitioning): 將陣列中最小的K個值放入左側 np.partition(array, K) #
x = np.array([7, 2, 3, 1, 6, 5, 4])
# print(np.partition(x, 2)) # 排列順序隨意 (BTW: 只排列K個數值, 則被分開的數值順序隨意)
print(np.partition(array, 2, axis=1)) # 每一列中的兩個最小值會被放在左側
def compute_2nd_bmu(self, inp):
#computes distances using euklid norm
dist = np.linalg.norm(self.weights-inp, axis=2)
return np.unravel_index(np.partition(dist,2)[2], dist.shape)
def main():
# Parse command line arguments
parser = argparse.ArgumentParser(description='Map the source embeddings into the target embedding space')
parser.add_argument('src_input', help='the input source embeddings')
parser.add_argument('trg_input', help='the input target embeddings')
parser.add_argument('--model_path', default=None, type=str, help='directory to save the model')
parser.add_argument('--geomm_embeddings_path', default=None, type=str, help='directory to save the output GeoMM latent space embeddings. The output embeddings are normalized.')
parser.add_argument('--encoding', default='utf-8', help='the character encoding for input/output (defaults to utf-8)')
parser.add_argument('--max_vocab', default=0,type=int, help='Maximum vocabulary to be loaded, 0 allows complete vocabulary')
parser.add_argument('--verbose', default=0,type=int, help='Verbose')
mapping_group = parser.add_argument_group('mapping arguments', 'Basic embedding mapping arguments')
mapping_group.add_argument('-dtrain', '--dictionary_train', default=sys.stdin.fileno(), help='the training dictionary file (defaults to stdin)')
mapping_group.add_argument('-dtest', '--dictionary_test', default=sys.stdin.fileno(), help='the test dictionary file (defaults to stdin)')
mapping_group.add_argument('--normalize', choices=['unit', 'center', 'unitdim', 'centeremb'], nargs='*', default=[], help='the normalization actions to perform in order')
geomm_group = parser.add_argument_group('GeoMM arguments', 'Arguments for GeoMM method')
geomm_group.add_argument('--l2_reg', type=float,default=1e2, help='Lambda for L2 Regularization')
geomm_group.add_argument('--max_opt_time', type=int,default=5000, help='Maximum time limit for optimization in seconds')
geomm_group.add_argument('--max_opt_iter', type=int,default=150, help='Maximum number of iterations for optimization')
eval_group = parser.add_argument_group('evaluation arguments', 'Arguments for evaluation')
eval_group.add_argument('--normalize_eval', action='store_true', help='Normalize the embeddings at test time')
eval_group.add_argument('--eval_batch_size', type=int,default=1000, help='Batch size for evaluation')
eval_group.add_argument('--csls_neighbourhood', type=int,default=10, help='Neighbourhood size for CSLS')
args = parser.parse_args()
BATCH_SIZE = args.eval_batch_size
## Logging
#method_name = os.path.join('logs','geomm')
#directory = os.path.join(os.path.join(os.getcwd(),method_name), datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
#if not os.path.exists(directory):
# os.makedirs(directory)
#log_file_name, file_extension = os.path.splitext(os.path.basename(args.dictionary_train))
#log_file_name = log_file_name + '.log'
#class Logger(object):
# def __init__(self):
# self.terminal = sys.stdout
# self.log = open(os.path.join(directory,log_file_name), "a")
# def write(self, message):
# self.terminal.write(message)
# self.log.write(message)
# def flush(self):
# #this flush method is needed for python 3 compatibility.
# #this handles the flush command by doing nothing.
# #you might want to specify some extra behavior here.
# pass
#sys.stdout = Logger()
if args.verbose:
print('Current arguments: {0}'.format(args))
dtype = 'float32'
if args.verbose:
print('Loading train data...')
# Read input embeddings
srcfile = open(args.src_input, encoding=args.encoding, errors='surrogateescape')
trgfile = open(args.trg_input, encoding=args.encoding, errors='surrogateescape')
src_words, x = embeddings.read(srcfile,max_voc=args.max_vocab, dtype=dtype)
trg_words, z = embeddings.read(trgfile,max_voc=args.max_vocab, dtype=dtype)
# Build word to index map
src_word2ind = {word: i for i, word in enumerate(src_words)}
trg_word2ind = {word: i for i, word in enumerate(trg_words)}
# Build training dictionary
noov=0
src_indices = []
trg_indices = []
f = open(args.dictionary_train, encoding=args.encoding, errors='surrogateescape')
for line in f:
src,trg = line.split()
if args.max_vocab:
src=src.lower()
trg=trg.lower()
try:
src_ind = src_word2ind[src]
trg_ind = trg_word2ind[trg]
src_indices.append(src_ind)
trg_indices.append(trg_ind)
except KeyError:
noov+=1
if args.verbose:
print('WARNING: OOV dictionary entry ({0} - {1})'.format(src, trg)) #, file=sys.stderr
f.close()
if args.verbose:
print('Number of training pairs having at least one OOV: {}'.format(noov))
src_indices = src_indices
trg_indices = trg_indices
if args.verbose:
print('Normalizing embeddings...')
# STEP 0: Normalization
for action in args.normalize:
if action == 'unit':
x = embeddings.length_normalize(x)
z = embeddings.length_normalize(z)
elif action == 'center':
x = embeddings.mean_center(x)
z = embeddings.mean_center(z)
elif action == 'unitdim':
x = embeddings.length_normalize_dimensionwise(x)
z = embeddings.length_normalize_dimensionwise(z)
elif action == 'centeremb':
x = embeddings.mean_center_embeddingwise(x)
z = embeddings.mean_center_embeddingwise(z)
# Step 1: Optimization
if args.verbose:
print('Beginning Optimization')
start_time = time.time()
x_count = len(set(src_indices))
z_count = len(set(trg_indices))
A = np.zeros((x_count,z_count))
# Creating dictionary matrix from training set
map_dict_src={}
map_dict_trg={}
I=0
uniq_src=[]
uniq_trg=[]
for i in range(len(src_indices)):
if src_indices[i] not in map_dict_src.keys():
map_dict_src[src_indices[i]]=I
I+=1
uniq_src.append(src_indices[i])
J=0
for j in range(len(trg_indices)):
if trg_indices[j] not in map_dict_trg.keys():
map_dict_trg[trg_indices[j]]=J
J+=1
uniq_trg.append(trg_indices[j])
for i in range(len(src_indices)):
A[map_dict_src[src_indices[i]],map_dict_trg[trg_indices[i]]]=1
np.random.seed(0)
Lambda=args.l2_reg
U1 = TT.matrix()
U2 = TT.matrix()
B = TT.matrix()
cost = TT.sum(((shared(x[uniq_src]).dot(U1.dot(B.dot(U2.T)))).dot(shared(z[uniq_trg]).T)-A)**2) + 0.5*Lambda*(TT.sum(B**2))
solver = ConjugateGradient(maxtime=args.max_opt_time,maxiter=args.max_opt_iter)
manifold =Product([Stiefel(x.shape[1], x.shape[1]),Stiefel(z.shape[1], x.shape[1]),PositiveDefinite(x.shape[1])])
problem = Problem(manifold=manifold, cost=cost, arg=[U1,U2,B], verbosity=3)
wopt = solver.solve(problem)
w= wopt
U1 = w[0]
U2 = w[1]
B = w[2]
### Save the models if requested
if args.model_path is not None:
os.makedirs(args.model_path,exist_ok=True)
np.savetxt('{}/U_src.csv'.format(args.model_path),U1)
np.savetxt('{}/U_tgt.csv'.format(args.model_path),U2)
np.savetxt('{}/B.csv'.format(args.model_path),B)
# Step 2: Transformation
xw = x.dot(U1).dot(scipy.linalg.sqrtm(B))
zw = z.dot(U2).dot(scipy.linalg.sqrtm(B))
end_time = time.time()
if args.verbose:
print('Completed training in {0:.2f} seconds'.format(end_time-start_time))
gc.collect()
### Save the GeoMM embeddings if requested
xw_n = embeddings.length_normalize(xw)
zw_n = embeddings.length_normalize(zw)
if args.geomm_embeddings_path is not None:
os.makedirs(args.geomm_embeddings_path,exist_ok=True)
out_emb_fname=os.path.join(args.geomm_embeddings_path,'src.vec')
with open(out_emb_fname,'w',encoding=args.encoding) as outfile:
embeddings.write(src_words,xw_n,outfile)
out_emb_fname=os.path.join(args.geomm_embeddings_path,'trg.vec')
with open(out_emb_fname,'w',encoding=args.encoding) as outfile:
embeddings.write(trg_words,zw_n,outfile)
# Step 3: Evaluation
if args.normalize_eval:
xw = xw_n
zw = zw_n
X = xw[src_indices]
Z = zw[trg_indices]
# Loading test dictionary
f = open(args.dictionary_test, encoding=args.encoding, errors='surrogateescape')
src2trg = collections.defaultdict(set)
trg2src = collections.defaultdict(set)
oov = set()
vocab = set()
for line in f:
src, trg = line.split()
if args.max_vocab:
src=src.lower()
trg=trg.lower()
try:
src_ind = src_word2ind[src]
trg_ind = trg_word2ind[trg]
src2trg[src_ind].add(trg_ind)
trg2src[trg_ind].add(src_ind)
vocab.add(src)
except KeyError:
oov.add(src)
src = list(src2trg.keys())
trgt = list(trg2src.keys())
oov -= vocab # If one of the translation options is in the vocabulary, then the entry is not an oov
coverage = len(src2trg) / (len(src2trg) + len(oov))
f.close()
translation = collections.defaultdict(int)
translation5 = collections.defaultdict(list)
translation10 = collections.defaultdict(list)
### compute nearest neigbours of x in z
t=time.time()
nbrhood_x=np.zeros(xw.shape[0])
for i in range(0, len(src), BATCH_SIZE):
j = min(i + BATCH_SIZE, len(src))
similarities = xw[src[i:j]].dot(zw.T)
similarities_x = -1*np.partition(-1*similarities,args.csls_neighbourhood-1 ,axis=1)
nbrhood_x[src[i:j]]=np.mean(similarities_x[:,:args.csls_neighbourhood],axis=1)
### compute nearest neigbours of z in x (GPU version)
nbrhood_z=np.zeros(zw.shape[0])
with cp.cuda.Device(1):
nbrhood_z2=cp.zeros(zw.shape[0])
batch_num=1
for i in range(0, zw.shape[0], BATCH_SIZE):
j = min(i + BATCH_SIZE, zw.shape[0])
similarities = -1*cp.partition(-1*cp.dot(cp.asarray(zw[i:j]),cp.transpose(cp.asarray(xw))),args.csls_neighbourhood-1 ,axis=1)[:,:args.csls_neighbourhood]
nbrhood_z2[i:j]=(cp.mean(similarities[:,:args.csls_neighbourhood],axis=1))
batch_num+=1
nbrhood_z=cp.asnumpy(nbrhood_z2)
#### compute nearest neigbours of z in x (CPU version)
#nbrhood_z=np.zeros(zw.shape[0])
#for i in range(0, len(zw.shape[0]), BATCH_SIZE):
# j = min(i + BATCH_SIZE, len(zw.shape[0]))
# similarities = zw[i:j].dot(xw.T)
# similarities_z = -1*np.partition(-1*similarities,args.csls_neighbourhood-1 ,axis=1)
# nbrhood_z[i:j]=np.mean(similarities_z[:,:args.csls_neighbourhood],axis=1)
#### find translation
#for i in range(0, len(src), BATCH_SIZE):
# j = min(i + BATCH_SIZE, len(src))
# similarities = xw[src[i:j]].dot(zw.T)
# similarities = np.transpose(np.transpose(2*similarities) - nbrhood_x[src[i:j]]) - nbrhood_z
# nn = similarities.argmax(axis=1).tolist()
# similarities = np.argsort((similarities),axis=1)
# nn5 = (similarities[:,-5:])
# nn10 = (similarities[:,-10:])
# for k in range(j-i):
# translation[src[i+k]] = nn[k]
# translation5[src[i+k]] = nn5[k]
# translation10[src[i+k]] = nn10[k]
#if args.geomm_embeddings_path is not None:
# delim=','
# os.makedirs(args.geomm_embeddings_path,exist_ok=True)
# translations_fname=os.path.join(args.geomm_embeddings_path,'translations.csv')
# with open(translations_fname,'w',encoding=args.encoding) as translations_file:
# for src_id in src:
# src_word = src_words[src_id]
# all_trg_words = [ trg_words[trg_id] for trg_id in src2trg[src_id] ]
# trgout_words = [ trg_words[j] for j in translation10[src_id] ]
# ss = list(nn10[src_id,:])
#
# p1 = ':'.join(all_trg_words)
# p2 = delim.join( [ '{}{}{}'.format(w,delim,s) for w,s in zip(trgout_words,ss) ] )
# translations_file.write( '{s}{delim}{p1}{delim}{p2}\n'.format(s=src_word, delim=delim, p1=p1, p2=p2) )
### find translation (and write to file if output requested)
delim=','
translations_file =None
if args.geomm_embeddings_path is not None:
os.makedirs(args.geomm_embeddings_path,exist_ok=True)
translations_fname=os.path.join(args.geomm_embeddings_path,'translations.csv')
translations_file = open(translations_fname,'w',encoding=args.encoding)
for i in range(0, len(src), BATCH_SIZE):
j = min(i + BATCH_SIZE, len(src))
similarities = xw[src[i:j]].dot(zw.T)
similarities = np.transpose(np.transpose(2*similarities) - nbrhood_x[src[i:j]]) - nbrhood_z
nn = similarities.argmax(axis=1).tolist()
similarities = np.argsort((similarities),axis=1)
nn5 = (similarities[:,-5:])
nn10 = (similarities[:,-10:])
for k in range(j-i):
translation[src[i+k]] = nn[k]
translation5[src[i+k]] = nn5[k]
translation10[src[i+k]] = nn10[k]
if args.geomm_embeddings_path is not None:
src_id=src[i+k]
src_word = src_words[src_id]
all_trg_words = [ trg_words[trg_id] for trg_id in src2trg[src_id] ]
trgout_words = [ trg_words[j] for j in translation10[src_id] ]
#ss = list(nn10[src_id,:])
p1 = ':'.join(all_trg_words)
p2 = ':'.join(trgout_words)
#p2 = delim.join( [ '{}{}{}'.format(w,delim,s) for w,s in zip(trgout_words,ss) ] )
translations_file.write( '{s}{delim}{p1}{delim}{p2}\n'.format(s=src_word, p1=p1, p2=p2, delim=delim) )
if args.geomm_embeddings_path is not None:
translations_file.close()
accuracy = np.mean([1 if translation[i] in src2trg[i] else 0 for i in src])
mean=0
for i in src:
for k in translation5[i]:
if k in src2trg[i]:
mean+=1
break
mean/=len(src)
accuracy5 = mean
mean=0
for i in src:
for k in translation10[i]:
if k in src2trg[i]:
mean+=1
break
mean/=len(src)
accuracy10 = mean
print('Coverage:{0:7.2%} Accuracy:{1:7.2%} Accuracy(Top 5):{2:7.2%} Accuracy(Top 10):{3:7.2%}'.format(coverage, accuracy, accuracy5, accuracy10))
def calculate_nearest_k(sim_mat, k):
sorted_mat = -np.partition(-sim_mat, k + 1, axis=1) # -np.sort(-sim_mat1)
nearest_k = sorted_mat[:, 0:k]
return np.mean(nearest_k, axis=1, keepdims=True)
def val_map(epoch):
print_log('\n Val@Epoch: %d' % epoch)
model.eval()
torch.cuda.empty_cache()
max_per_image = 100
results = {}
speed_list = []
with torch.no_grad():
for inputs in val_loader:
img_id, inputs = inputs[0]
start_image_time = time.time()
segmentations = []
for scale in inputs:
inputs[scale]['image'] = inputs[scale]['image'].to(
cfg.device)
# hmap, regs, w_h_, offsets, active_codes, inactive_codes, active_cls = model(inputs[scale]['image'])[-1]
hmap, regs, w_h_, offsets, _, _, active_codes, active_cls = model(
inputs[scale]['image'])[-1]
active_cls = torch.sigmoid(active_cls) > 0.5
active_mask = active_cls * 1
# inactive_mask = (~ active_cls) * 1
codes = active_codes * active_mask # + inactive_codes * inactive_mask
output = [hmap, regs, w_h_, codes, offsets]
segms = ctsegm_scale_decode(
*output,
torch.from_numpy(dictionary.astype(np.float32)).to(
cfg.device),
K=cfg.test_topk)
# segms = ctsegm_amodal_cmm_decode(*output,
# torch.from_numpy(dictionary.astype(np.float32)).to(cfg.device),
# K=cfg.test_topk)
segms = segms.detach().cpu().numpy().reshape(
1, -1, segms.shape[2])[0]
top_preds = {}
for j in range(cfg.n_vertices):
segms[:, 2 * j:2 * j + 2] = transform_preds(
segms[:, 2 * j:2 * j + 2], inputs[scale]['center'],
inputs[scale]['scale'],
(inputs[scale]['fmap_w'], inputs[scale]['fmap_h']))
segms[:, cfg.n_vertices * 2:cfg.n_vertices * 2 +
2] = transform_preds(
segms[:,
cfg.n_vertices * 2:cfg.n_vertices * 2 + 2],
inputs[scale]['center'], inputs[scale]['scale'],
(inputs[scale]['fmap_w'],
inputs[scale]['fmap_h']))
segms[:, cfg.n_vertices * 2 + 2:cfg.n_vertices * 2 +
4] = transform_preds(
segms[:, cfg.n_vertices * 2 +
2:cfg.n_vertices * 2 + 4],
inputs[scale]['center'], inputs[scale]['scale'],
(inputs[scale]['fmap_w'],
inputs[scale]['fmap_h']))
clses = segms[:, -1]
for j in range(val_dataset.num_classes):
inds = (clses == j)
top_preds[j + 1] = segms[inds, :cfg.n_vertices * 2 +
5].astype(np.float32)
top_preds[j + 1][:, :cfg.n_vertices * 2 + 4] /= scale
segmentations.append(top_preds)
end_image_time = time.time()
segms_and_scores = {
j: np.concatenate([d[j] for d in segmentations], axis=0)
for j in range(1, val_dataset.num_classes + 1)
}
scores = np.hstack([
segms_and_scores[j][:, cfg.n_vertices * 2 + 4]
for j in range(1, val_dataset.num_classes + 1)
])
if len(scores) > max_per_image:
kth = len(scores) - max_per_image
thresh = np.partition(scores, kth)[kth]
for j in range(1, val_dataset.num_classes + 1):
keep_inds = (
segms_and_scores[j][:, cfg.n_vertices * 2 + 4] >=
thresh)
segms_and_scores[j] = segms_and_scores[j][keep_inds]
results[img_id] = segms_and_scores
speed_list.append(end_image_time - start_image_time)
eval_results = val_dataset.run_eval(results, save_dir=cfg.ckpt_dir)
print_log(eval_results)
summary_writer.add_scalar('val_mAP/mAP', eval_results[0], epoch)
print_log('Average speed on val set:{:.2f}'.format(
1. / np.mean(speed_list)))
return eval_results[0]
profit = profit + CADCHF10080[5][tick+1] - CADCHF10080[5][tick]
if( (CADCHF10080[5][tick] < CADCHF10080[5][tick+1] ) and (Decision == 0) ):
profit = profit + (CADCHF10080[5][tick+1] - CADCHF10080[5][tick])*-1
if( (CADCHF10080[5][tick] > CADCHF10080[5][tick+1] ) and (Decision == 1) ):
profit = profit + (CADCHF10080[5][tick+1] - CADCHF10080[5][tick])
if( (CADCHF10080[5][tick] > CADCHF10080[5][tick+1] ) and (Decision == 0) ):
profit = profit + (CADCHF10080[5][tick+1] - CADCHF10080[5][tick])*-1
PopulationProfitArray[n] = profit
for B in range(1,TopSpecimenNumber+1):
BestSpecimens[B-1] = np.partition(PopulationProfitArray.flatten(), -B)[-B] #Encontrar el X más alto.
t, = np.where(PopulationProfitArray == BestSpecimens[B-1])
if(t.size > 1):
t = t[0]
BestSpecimensArray[B-1] = PopulationArray[t]
for B in range(0,TopSpecimenNumber):
if(TopSpecimenNumber < TopSpecimenNumber/2):
SN = random.randint(0,TopSpecimenNumber/2-1)
BestSpecimensSonsArray[B] = BestSpecimensArray[SN]
for O in range(0,5):
SN = random.randint(TopSpecimenNumber/2,TopSpecimenNumber-1)
Z = random.randint(0,NeuronAmmount-1)
BestSpecimensSonsArray[B][Z] = BestSpecimensArray[SN][Z]
t = time.time()
# load data
mat_contents = sio.loadmat(data_path + '/' + train_mat_names[id])
adj = mat_contents['adj']
yy = mat_contents['indset_label'].transpose()
nn, nr = yy.shape # number of nodes & results
# y_train = yy[:,np.random.randint(0,nr)]
# y_train = np.concatenate([1-np.expand_dims(y_train,axis=1), np.expand_dims(y_train,axis=1)],axis=1)
# sample an intermediate graph
yyr = yy[:, np.random.randint(0, nr)]
yyr_num = np.sum(yyr)
yyr_down_num = np.random.randint(0, yyr_num)
if yyr_down_num > 0:
yyr_down_prob = yyr * np.random.random_sample(yyr.shape)
yyr_down_flag = (yyr_down_prob >= np.partition(
yyr_down_prob, -yyr_down_num)[-yyr_down_num])
tmp = np.sum(adj[yyr_down_flag, :], axis=0) > 0
tmp = np.asarray(tmp).reshape(-1)
yyr_down_flag[tmp] = 1
adj_down = adj[yyr_down_flag == 0, :]
adj_down = adj_down[:, yyr_down_flag == 0]
yyr_down = yyr[yyr_down_flag == 0]
adj = adj_down
nn = yyr_down.shape[0]
yyr = yyr_down
y_train = np.concatenate(
[1 - np.expand_dims(yyr, axis=1),
np.expand_dims(yyr, axis=1)],
axis=1)
def SinglMatch(ps1, w, md, edges1C, gt, fx_C, fy_C, edges0C, gt_0, fx_C0,
fy_C0, mask_b_C0):
pbar = tqdm(total=1, position=0, desc="RECC ")
max_dist = int((md) / ps1)
contours, hierarchy = cv2.findContours((1 - mask_b_C0).astype(np.uint8),
cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE)
contour_sizes = [(cv2.contourArea(contour), contour)
for contour in contours]
biggest_contour = max(contour_sizes, key=lambda x: x[0])[1]
polygon = Polygon(np.array(biggest_contour[:, 0]))
x, y = polygon.exterior.xy
b1 = int(min(x))
b2 = int(max(x))
b3 = int(min(y))
b4 = int(max(y))
target = edges0C[b3:b4, b1:b2]
sum_target = np.sum(target)
xd = int(round((b4 - b3) / 2))
yd = int(round((b2 - b1) / 2))
buffer = 2 * max(xd, yd)
edges1Ca = np.zeros(
(edges1C.shape[0] + buffer * 2, edges1C.shape[1] + 2 * buffer))
edges1Ca[buffer:-buffer, buffer:-buffer] = edges1C
x_i_0 = b3 + xd
y_i_0 = b1 + yd
lat_0 = gt_0[3] + gt_0[5] * x_i_0 * fx_C0
lon_0 = gt_0[0] + gt_0[1] * y_i_0 * fy_C0
x_i_0_og = int(round((lat_0 - gt[3]) / (gt[5] * fx_C)))
y_i_0_og = int(round((lon_0 - gt[0]) / (gt[1] * fy_C)))
search_wide = np.zeros((2 * (max_dist + yd), 2 * (max_dist + xd)))
search_wide = edges1Ca[buffer + x_i_0_og - max_dist - xd:buffer +
x_i_0_og + max_dist + xd, buffer + y_i_0_og -
max_dist - yd:buffer + y_i_0_og + max_dist + yd]
RECC_total = np.zeros(edges1Ca.shape)
RECC_over = np.zeros(edges1Ca.shape)
RECC_over.fill(np.NaN)
circle2 = np.zeros((2 * max_dist, 2 * max_dist))
for x in range(circle2.shape[0]):
for y in range(circle2.shape[1]):
if (x - max_dist)**2 + (y - max_dist)**2 < max_dist**2:
circle2[x, y] = 1
circle2[circle2 == 0] = np.NaN
sum_patch = cv2.filter2D(search_wide, -1, np.ones(target.shape))
numerator = cv2.filter2D(search_wide, -1, target)
RECC_wide = numerator / (sum_patch + sum_target)
RECC_area = RECC_wide[xd:-xd, yd:-yd] * circle2
RECC_total.fill(np.NaN)
RECC_total[x_i_0_og - max_dist:x_i_0_og + max_dist,
y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area
max_one = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(), -1)[-1]
y_i = np.where(RECC_total >= max_one)[1][0]
x_i = np.where(RECC_total >= max_one)[0][0]
x_offset = (x_i - x_i_0_og) * ps1
y_offset = (y_i - y_i_0_og) * ps1
o_x = x_i
o_y = y_i
t_x = x_i_0
t_y = y_i_0
pbar.update(1)
return x_offset, y_offset, o_x, o_y, t_x, t_y
def constraints_s(self,
doThr,
K,
doRw,
nnegS,
Sfft=None,
Sfft_det=None,
S=None,
stds=None,
Swtrw=None,
oracle=False):
"""Apply the constraints on the sources (thresholding in the wavelet domain and possibly a projection on the
positive orthant. The input data are Sfft. The output data are S, as well as Sfft and Sfft_det.
Parameters
----------
doThr : bool
perform thresholding
K: float
L0 support of the sources
doRw: bool
do reweighting
nnegS: bool
apply non-negativity constraint on the sources
Sfft: np.ndarray
(n,p) complex array, estimated sources in Fourier domain (in-place update, default: self.Sfft)
Sfft_det: np.ndarray
(n,p) complex array, estimated sources with only the detail scales in Fourier domain (in-place update,
default: self.Sfft_det)
S: np.ndarray
(n,p) float array, estimated sources (in-place update, default: self.S)
stds: np.ndarray
(n,nscales) float array, std of the noise in the source space, per detail scale (default: mad or analytical
calculation)
Swtrw: np.ndarray
(n,p,nscales) float array, sources in the wavelet domain of previous iteration (default: self.Swtrw)
oracle: bool
perform an oracle update (using the ground-truth A and S)
Returns
-------
int
error code
"""
if Sfft is None:
Sfft = self.Sfft
if Sfft_det is None:
Sfft_det = self.Sfft_det
if S is None:
S = self.S
if Swtrw is None:
if not oracle:
Swtrw = self.Swtrw
else:
Swtrw = self.S0wt
if not doThr:
S[:] = fftt.ifft(Sfft)
if not nnegS: # nothing more to do
Sfft_det[:] = fftt.fftprod(Sfft, 1 - self.wt_filters[:, -1])
return 0
else:
if self.verb >= 3:
print("Maximal L0 norm of the sources: %.1f %%" % (K * 100))
Swt = fftt.wt_trans(Sfft, nscales=self.nscales, fft_in=True)
# Thresholding
for i in range(self.n):
for j in range(self.nscales):
Swtij = Swt[i, :, j]
Swtrwij = Swtrw[i, :, j]
if stds is not None:
std = stds[i, j]
elif self.useMad:
std = utils.mad(Swtij, M=self.M)
else:
std = self.nStd * np.sqrt(
np.sum(self.invOpSp[:, i] *
self.wt_filters[:, j]**2) / self.p)
thrd = self.k * std
# If oracle, threshold Swtrw
if oracle and self.L1 and doRw:
Swtrwij = (
Swtrwij - np.sign(Swtrwij) * (thrd - np.sqrt(
np.abs(
(Swtrwij - thrd * np.sign(Swtrwij)) *
(3 * thrd * np.sign(Swtrwij) + Swtrwij))))
) / 2 * (np.abs(Swtrwij) >= thrd)
# Support based threshold
if K != 1:
npix = np.sum(abs(Swtij) - thrd > 0)
Kval = np.maximum(np.int(K * npix), 5)
thrd = np.partition(abs(Swtij),
self.p - Kval)[self.p - Kval]
if self.verb == 4 and i == 0:
print("Threshold of source %i at scale %i: %.5e" %
(i + 1, j + 1, thrd))
elif self.verb == 5:
print("Threshold of source %i at scale %i: %.5e" %
(i + 1, j + 1, thrd))
# Adapt the threshold if reweighing demanded
if doRw and self.L1:
thrd = thrd / (np.abs(Swtrwij) /
np.maximum(1e-20, self.k * std) + 1)
else:
thrd = thrd * np.ones(self.p)
# Apply the threshold
Swtij[(abs(Swtij) < thrd)] = 0
if self.L1:
indNZ = np.where(abs(Swtij) > thrd)[0]
Swtij[indNZ] = Swtij[indNZ] - thrd[indNZ] * np.sign(
Swtij[indNZ])
Swt[i, :, j] = Swtij
# Reconstruct S
S[:] = fftt.wt_rec(Swt)
# Non-negativity constraint
if nnegS:
nneg = S > 0
S *= nneg
if oracle:
return 0
# Save the wavelet coefficients of S for next iteration
if doThr and doRw and self.L1 and not oracle:
if nnegS:
Swt *= nneg[:, :, np.newaxis]
self.Swtrw = Swt[:, :, :-1]
Sfft[:] = fftt.fft(S)
Sfft_det[:] = fftt.fftprod(Sfft, 1 - self.wt_filters[:, -1])
return 0
def GDP_PCA_plot(filename=None,
threshold=0.015,
lowerbound=2.0,
upperbound=1.0e4,
factor=0.08):
data = np.load('uploads/' + filename)
image1 = data.f.image #-np.median(data.f.image)
image2 = np.array(image1 * 255 / image1.max(), dtype='uint8')
image_avg = np.mean(np.partition(image2, int(len(image2) * 0.2)))
image2[image2 < image_avg] = int(image_avg)
#H1=cv2.GaussianBlur(image2,(3,3),1.0*np.std(image2))
H1 = gaussian_filter(image2, factor * np.std(image2), mode='nearest')
image2 = H1
blobs_log = blob_log(image2,
max_sigma=0.3 * np.std(image2),
min_sigma=0.02 * np.mean(image2),
num_sigma=20,
threshold=threshold,
overlap=0.6)
blobs_log[:, 2] = blobs_log[:, 2] * np.sqrt(2)
blobs = blobs_log[(blobs_log[:, 2] > lowerbound)
& (blobs_log[:, 2] < upperbound)]
xx = (data.f.X.min(), np.round(data.f.X.max(), -1))
yy = (data.f.Y.min(), np.round(data.f.Y.max(), -1))
x_step = (xx[1] - xx[0]) / np.shape(H1)[0]
y_step = (yy[1] - yy[0]) / np.shape(H1)[1]
#Number of NV
height = yy[1] - yy[0]
width = xx[1] - xx[0]
total = len(blobs)
per_nv = round(len(blobs) / float(height * width) * (20 * 20), 2)
########################################################
t = [
'Original Density Plot,'
' Filename=' + filename, 'Gaussian Filtered Density Plot',
'Total NVs =' + str(total) + ' , NVs per 20x20 pixel area = ' +
str(per_nv)
]
data_list = [image1, H1, H1]
color_list = [
Viridis256, cc.b_linear_bgy_10_95_c74, cc.b_linear_bgy_10_95_c74
]
return_list = []
return_list.append(head)
#hover tool hack to work for image function in bokeh.
##http://stackoverflow.com/questions/28176949/convert-list-of-tuples-to-structured-numpy-array
px = np.linspace(xx[0], xx[1], np.shape(H1)[0] / 2)
py = np.linspace(yy[0], yy[1], np.shape(H1)[1] / 2)
px = np.array(px, dtype='uint32')
py = np.array(py, dtype='uint32')
a = []
for i in px:
a.extend(zip(itertools.repeat(i), py))
dt = np.dtype('int,float')
X = np.array(a, dtype=dt)
x1 = X['f0']
y1 = X['f1']
##################################################################
for i in range(3):
color_mapper = LogColorMapper(palette=color_list[i], \
low=np.mean(data_list[i]), \
high=1.0*np.mean(data_list[i])+\
2.0*np.std(data_list[i]))
color_bar = ColorBar(color_mapper=color_mapper,\
label_standoff=12, \
border_line_color=None, \
location=(0,0))
p1 = figure(plot_width=600, plot_height=600,title=t[i],title_text_font_size='12pt',\
x_range=xx,y_range=xx,tools=TOOLS,toolbar_location="below",toolbar_sticky=False,responsive=True)
p1.square(x1, y1, alpha=1.0)
p1.image(image=[data_list[i]],
color_mapper=color_mapper,
dh=yy[1] - yy[0],
dw=xx[1] - xx[0],
x=xx[0],
y=xx[0])
p1.add_layout(color_bar, 'right')
if i == 2:
p1.circle(blobs[:, 1] * x_step + xx[0],
blobs[:, 0] * y_step + yy[0],
radius=blobs[:, 2] * 1.6,
radius_dimension='y',
line_color='red',
alpha=1.0,
line_width=3,
fill_color=None)
p1.title.text_font_size = "11pt"
p1.xaxis.axis_label_text_font_size = "13pt"
p1.yaxis.axis_label_text_font_size = "13pt"
#plots = {'Navy': p1, 'Blue': p2};
tuple_plot = components(p1)
#script2, div2 = components(p2);
return_list.append(list(tuple_plot))
p=Histogram(blobs[:,2],\
plot_width=600, plot_height=600,tools=TOOLS,\
toolbar_location="below",toolbar_sticky=False,\
responsive=True,\
title="Distribution of Radii of NV Centers")
return_list.append(list(components(p)))
return return_list
step_cpt += actor_steps
# save stuff
if step_cpt >= args.period:
# evaluate mean actor over several runs. Memory is not filled
# and steps are not counted
actor.set_params(es.mu)
f_mu, _ = evaluate(actor, env, memory=None, n_episodes=args.n_eval,
render=args.render)
prRed('Actor Mu Average Fitness:{}'.format(f_mu))
df.to_pickle(args.output + "/log.pkl")
res = {"total_steps": total_steps,
"average_score": np.mean(fitness),
"average_score_half": np.mean(np.partition(fitness, args.pop_size // 2 - 1)[args.pop_size // 2:]),
"average_score_rl": np.mean(fitness[:args.n_grad]),
"average_score_ea": np.mean(fitness[args.n_grad:]),
"best_score": np.max(fitness),
"mu_score": f_mu}
if args.save_all_models:
os.makedirs(args.output + "/{}_steps".format(total_steps),
exist_ok=True)
critic.save_model(
args.output + "/{}_steps".format(total_steps), "critic")
actor.set_params(es.mu)
actor.save_model(
args.output + "/{}_steps".format(total_steps), "actor_mu")
else:
critic.save_model(args.output, "critic")
len(x)) #stores distance to 2nd nearest neighbor
nn3 = np.zeros(len(x)) #stores CloudNum of 3rd nearest neighbor
mindist3 = np.zeros(
len(x)) #stores distance to 3rd nearest neighbor
k = 0
j = 0
while k < len(x):
for j in range(len(x)):
dist[k, j] = np.sqrt(
np.square(xs[k] - xs[j]) + np.square(y[k] - y[j]))
k += 1
dist[dist == 0] = np.nan
for k in range(len(x)):
ind = np.where(dist[k] == np.nanmin(
dist[k])) #index of nearest neighbor
ind2 = np.where(dist[k] == np.partition(
dist[k], 1)[1]) #index of 2nd nearest neighbor
ind3 = np.where(dist[k] == np.partition(
dist[k], 2)[2]) #index of 3rd nearest neighbor
mindist[k] = np.deg2rad(dist[k, int(ind[0])]) * distance[0]
nn[k] = int(ind[0]) + 1
if len(ind2[0]) == 1:
mindist2[k] = np.deg2rad(dist[k,
int(ind2[0])]) * distance[0]
nn2[k] = int(ind2[0]) + 1
else:
mindist2[k] = np.deg2rad(
dist[k, int(ind2[0][0])]) * distance[0]
nn2[k] = int(ind2[0][1]) + 1
if len(ind3[0]) == 1:
mindist3[k] = np.deg2rad(dist[k,
int(ind3[0])]) * distance[0]
# %% print(np.sort(a2, axis=0)) # %% print(np.sort(a2, axis=1)) # 부분 정렬 # - partition() : 배열에서 k개의 작은 값을 반환 # %% a1 = np.random.randint(1, 10, size=10) print(a1) # %% print(np.partition(a1, 3)) # %% a2 = np.random.randint(1, 10, size=(5, 5)) print(a2) # %% print(np.partition(a2, 3)) # %% print(np.partition(a2, 3, axis=0)) # %% print(np.partition(a2, 3, axis=1)) # 배열 입출력
def train(self, load_dir=None, save_dir=None):
######################
######## TODO ########
######################
if self.style == 'Real':
self.train_real(load_dir, save_dir)
else:
self.debug = True if self.num_chosen_wc < 1000 else False
if load_dir != None and os.path.exists(load_dir):
self.chosen_wcs = pickle.load(open(load_dir, 'rb'))
self.wc_errors = pickle.load(open("errors_" + load_dir, 'rb'))
self.chosen_wcs_predictions = pickle.load(
open("preds_" + load_dir, 'rb'))
else:
n = self.data.shape[0]
weights = np.array([1 / n for i in range(n)])
self.chosen_wcs = []
self.chosen_wcs_predictions = np.zeros((self.num_chosen_wc, n))
k = 1000 #if self.debug==False else 50
t = 5
self.wc_errors = {}
min_index = -1
for weak_classifier in self.weak_classifiers:
weak_classifier.sorted_indices = np.argsort(
weak_classifier.activations)
rem_classifiers = [
classifier for classifier in self.weak_classifiers
]
range_keys = [0, 9, 49,
99] #if self.debug == False else [0,4,9,14,19]
for i in tqdm(range(self.num_chosen_wc)):
my_result_array = Parallel(n_jobs=self.num_cores)(
delayed(wc.calc_error)(weights, self.labels)
for wc in rem_classifiers)
my_result_array = list(zip(*my_result_array))
error = np.asarray(my_result_array[0])
predict = np.asarray(my_result_array[1])
threshold = np.asarray(my_result_array[2])
pol = np.asarray(my_result_array[3])
min_wc_error = min(error)
min_index = np.argmin(error)
rem_classifiers[min_index].threshold = threshold[min_index]
rem_classifiers[min_index].polarity = pol[min_index]
predict_min = predict[min_index]
if i in range_keys:
top_k_errors = np.partition(error, k)[:k]
top_k_errors.sort()
self.wc_errors[i] = top_k_errors
predict_min = pol[min_index] * np.sign(
rem_classifiers[min_index].activations -
threshold[min_index])
print('Minimum error:', min_wc_error, 'Minimum index: ',
min_index, 'in iteration ', i, 'classifier id',
rem_classifiers[min_index])
x = (1 - min_wc_error) / min_wc_error
alpha_i = 0.5 * np.log(x)
if self.style == 'Real':
alpha_i = 1
#wc_error_min, predict_min = rem_classifiers[min_index].calc_error(weights, self.labels)
weights = weights * np.exp(
-alpha_i * self.labels * predict_min)
weights = weights / sum(weights)
#print("Weightsum",sum(weights))
p = rem_classifiers[min_index]
self.chosen_wc_ids.append(p.id)
print("id", id)
#self.wc_errors[i] = min_wc_error
self.chosen_wcs.append((alpha_i, p))
self.chosen_wcs_predictions[i] = alpha_i * predict_min
# print(i,":",self.chosen_wcs_predictions[i])
del rem_classifiers[min_index]
if save_dir is not None:
pickle.dump(self.wc_errors, open("errors_" + save_dir,
'wb'))
pickle.dump(self.chosen_wcs_predictions,
open("preds_" + save_dir, 'wb'))
pickle.dump(self.chosen_wcs, open(save_dir, 'wb'))
parser.add_argument('filename', help='submission', type=str)
parser.add_argument('--max_num', help='maximum number of predictions', type=int,
default=150 * 10**6)
parser.add_argument('-f', help='force overwrite', action='store_true')
args = parser.parse_args()
print(args)
if os.path.exists(args.result) and not args.f:
print(args.result, 'already exists, exiting')
sys.exit()
pool = multiprocessing.Pool()
print('reading predictions')
all_confs: List[float] = []
with open(args.filename) as f:
for confs in tqdm(pool.imap(read_confidences, f), total=100000):
all_confs.extend(confs)
print(f'sorting scores (total {len(all_confs)})')
assert len(all_confs) >= args.max_num
pos = len(all_confs) - args.max_num
threshold = np.partition(all_confs, pos)[pos]
print('applying threshold', threshold)
with open(args.filename) as f:
with open(args.result, 'w') as out:
for line in tqdm(pool.imap(partial(trim_line, threshold), f), total=100000):
out.write(line)
def MRAS_categ_elite(
args,
game_i,
N_0=400,
alpha=4,
kai_0=0,
rho_0=0.7,
epi_J=1e-6,
pbar=None,
verbose=False,
):
game_ll = args.games[game_i]
game = Game(game_ll)
best_reward, best_arm = np.max(game.game_ll), np.argmax(game.game_ll)
if verbose:
print(f"\n\nRunning game {game_i} with MRAS Categorical")
print(f"Arms distribution used {game_ll}")
# ---------------------------------------------------------------------------- #
# 1. Rewards collected and optimal arm pull (overall)
# ---------------------------------------------------------------------------- #
regret_ll = np.zeros(args.n)
var_regret_ll = np.zeros(args.n)
arm_ll = np.zeros((len(game), args.n))
for exp in range(args.repeat):
# ---------------------------------------------------------------------------- #
# 2. Rewards collected and optimal arm pull (for experiment)
# ---------------------------------------------------------------------------- #
regret_exp_ll = np.zeros(args.n)
sample_mean_ll = np.zeros(len(game))
theta = np.ones(len(game)) / len(game)
pulls_ll = np.zeros(len(game))
count = 0
# ---------------------------------------------------------------------------- #
# 3. Initialisation
##################################################
for k in range(10):
for j in range(len(game)):
arm_ll[j, k * len(game) + j] += 1
reward = game.get_reward(j)
regret_exp_ll[j] = best_reward - reward
# Update sample mean
pulls_ll[j] += 1
sample_mean_ll[j] = (reward - sample_mean_ll[j]) / pulls_ll[j]
count += 1
# Samples to take
N = N_0
kai = kai_0
kai_bar = kai_0
rho = rho_0
phase = 1
while count < args.n:
# pull sample
arms = np.random.choice(len(game), N, p=theta)
mask = np.zeros(len(game))
J_vec = np.zeros(N)
J_vec_arms = np.zeros(N, dtype=np.int)
for e, arm in enumerate(arms):
arm_ll[arm, count] += 1
# Update sample mean
reward = game.get_reward(arm)
regret_exp_ll[count] = best_reward - reward
pulls_ll[arm] += 1
sample_mean_ll[arm] += (reward -
sample_mean_ll[arm]) / pulls_ll[arm]
# J_vec[e] = sample_mean_ll[arm]
J_vec_arms[e] = arm
count += 1
if count >= args.n:
break
for e in range(len(J_vec_arms)):
J_vec[e] = sample_mean_ll[J_vec_arms[e]]
# Update elite set
k = int((1 - rho) * N) - 1
kai_bar = np.partition(J_vec, k)[k - 1]
# Check if threshold is acceptable
if kai_bar - kai > epi_J:
kai = kai_bar
else:
rho_bar = rho
while rho_bar > epi_J:
rho_bar = 0.9 * rho_bar
k = int((1 - rho_bar) * N) - 1
kai_bar = np.partition(J_vec, k)[k]
if kai_bar - kai > epi_J:
kai = kai_bar
rho = rho_bar
break
if rho > rho_bar:
N = int(alpha * N)
# Find elite mask
if len(J_vec_arms[np.where(J_vec >= kai)]) > 0:
for arm in J_vec_arms[np.where(J_vec >= kai)]:
mask[arm] += 1
# Update theta
theta[mask > 0] = (np.exp(phase * sample_mean_ll[mask > 0]) *
mask[mask > 0] / theta[mask > 0])
theta[mask == 0] = 0
theta = theta / np.sum(theta[mask > 0])
phase += 1
if verbose:
print(f"N value {N}")
print(f"\nTurn {count} of {args.n} Arm pulled {arms}")
print(f"Updated sample means {sample_mean_ll}")
print(f"Mask {mask}")
print(f"New theta: {theta}")
if pbar:
pbar.set_description(
f"Game:{game_i + 1} MRAS_Categ_Elite exp:{exp} arm: {arm}")
pbar.update(1)
regret_exp_ll = np.cumsum(regret_exp_ll)
regret_ll += regret_exp_ll
var_regret_ll += regret_exp_ll**2
regret_ll = regret_ll / args.repeat
var_regret_ll /= args.repeat
var_regret_ll -= regret_ll**2
var_regret_ll /= np.arange(1, args.n + 1)
var_regret_ll = np.sqrt(var_regret_ll)
arm_ll /= args.repeat
print(f"Final theta: {theta}")
return regret_ll, var_regret_ll, arm_ll
def process_back_data(file):
f_cam = 1000
n_tags = 23
no_subj = 1
pose_all = np.empty((no_subj), dtype=object)
mesh_all = np.empty((no_subj), dtype=object)
suct = 0
print('Calculating center of mass on file', file)
for i in np.array([1]): # files in os.listdir(path_loc):
file_path = file
f = open(file_path)
csv_f = csv.reader(f, delimiter='\t')
new_frame = np.empty((23, 7, 1))
new_frame[:] = np.NAN
full_array = new_frame.copy()
count = -1
for row in csv_f:
row = row[1:]
if (count == -1):
count = count+1
elif (row[1] == str(NULL_MARKER)):
full_array = np.append(full_array, new_frame, axis=2)
count = count+1
else:
if (int(row[0]) < 23):
read_array = np.asarray(row[2:]).astype(np.float)
quart = Quaternion(matrix=rf.eul2mat(read_array[4:]))
full_array[int(row[0]), :, count] = np.concatenate(
(np.array([-read_array[2], read_array[3], -read_array[1]]), quart.elements), axis=0)
full_array = full_array[:, :, :-1]
f.close()
pose_all[suct] = full_array
suct += 1
final_ori = np.zeros((full_array.shape[2], full_array.shape[1]))
cur_fullmesh = np.empty((n_tags*4, 3, full_array.shape[2]))
cur_fullmesh[:] = np.NaN
mesh_visible = np.empty((n_tags*4, 3, full_array.shape[2]))
mesh_visible[:] = np.NaN
vismesh_px = np.empty((n_tags*4, 2, full_array.shape[2]))
vismesh_px[:] = np.NaN
fullmesh_px = np.empty((n_tags*4, 2, full_array.shape[2]))
fullmesh_px[:] = np.NaN
err_dist = np.zeros((full_array.shape[2], 1))
avg_mesh = np.load("mesh_calib.npy")
for f_ct in range(0, full_array.shape[2]):
cur_mesh = np.empty((n_tags*4, 3))
cur_mesh[:] = np.NaN
for tag_ct in range(0, n_tags):
cur_mesh[tag_ct*4:(tag_ct+1)*4, :] = rf.orient_square(
full_array[tag_ct, 0:3, f_ct], Quaternion(full_array[tag_ct, 3:, f_ct]))
while True:
mesh1 = avg_mesh[(~np.isnan(cur_mesh[:, 1])), :]
mesh2 = cur_mesh[(~np.isnan(cur_mesh[:, 1])), :]
m1c = rmsd.centroid(mesh1)
m2c = rmsd.centroid(mesh2)
mesh1 -= m1c
mesh2 -= m2c
rot2_1 = rmsd.kabsch(mesh1, mesh2)
cur_fullmesh[:, :, f_ct] = np.dot(avg_mesh - m1c, rot2_1) + m2c
# if f_ct >= 2:
# cur_fullmesh[:,:,f_ct] = np.nansum((0.5*cur_fullmesh[:,:,f_ct],0.3*cur_fullmesh[:,:,f_ct-1],0.2*cur_fullmesh[:,:,f_ct-2]),axis = 0)
err = np.linalg.norm(
cur_fullmesh[:, :, f_ct] - cur_mesh, axis=1)
if mesh1.shape[0] == 0:
break
if np.partition(err, 3)[3] > 0.005:
break
elif (np.nanmean(err) < 0.005):
break
cur_mesh[np.nanargmax(err), :] = np.array(
[np.nan, np.nan, np.nan])
final_ori[f_ct, 0:3] = rmsd.centroid(cur_fullmesh[:, :, f_ct])
final_ori[f_ct, 3:] = Quaternion(matrix=rot2_1).elements
# mesh_visible[(~np.isnan(cur_mesh[:,1])),:] = cur_fullmesh[(~np.isnan(cur_mesh[:,1])),:]
vismesh_px[:, 0, f_ct] = 960 - \
(cur_mesh[:, 0]/cur_mesh[:, 2]*f_cam)
vismesh_px[:, 1, f_ct] = 540 + \
(cur_mesh[:, 1]/cur_mesh[:, 2]*f_cam)
fullmesh_px[:, 0, f_ct] = 960 - \
(cur_fullmesh[:, 0, f_ct]/cur_fullmesh[:, 2, f_ct]*f_cam)
fullmesh_px[:, 1, f_ct] = 540 + \
(cur_fullmesh[:, 1, f_ct]/cur_fullmesh[:, 2, f_ct]*f_cam)
err_dist[f_ct] = np.nanmean(np.linalg.norm(
cur_fullmesh[:, :, f_ct] - cur_mesh, axis=1))
if np.size(mesh2, axis=0) <= 12:
err_dist[f_ct] = 1
loc = rmsd.centroid(cur_fullmesh)
#pose_all[suct] = final_ori
#mesh_all[suct] = fullmesh_px
suct = suct+1
print(suct)
axis_px = np.empty((4, 2, full_array.shape[2]))
axis_px[:] = np.NaN
for ct in range(0, full_array.shape[2]):
rot_mat = Quaternion(np.array(
[final_ori[ct, 3], final_ori[ct, 4], final_ori[ct, 5], final_ori[ct, 6]])).rotation_matrix/25
ax = np.c_[rot_mat, final_ori[ct, 0:3]]
ax_ad = ax.copy()
for row in range(3):
ax_ad[:, row] = ax[:, row] + final_ori[ct, 0:3]
axis_px[:, 0, ct] = 960 - (ax_ad[0, :]/ax_ad[2, :]*f_cam)
axis_px[:, 1, ct] = 540 + (ax_ad[1, :]/ax_ad[2, :]*f_cam)
axis_px1 = np.empty((4, 2, full_array.shape[2]))
axis_px1[:] = np.NaN
for ct in range(0, full_array.shape[2]):
rot_mat = Quaternion(np.array(
[final_ori[ct, 6], final_ori[ct, 5], final_ori[ct, 4], final_ori[ct, 3]])).rotation_matrix/25
ax = np.c_[rot_mat, final_ori[ct, 0:3]]
ax_ad = ax.copy()
for row in range(3):
ax_ad[:, row] = ax[:, row] + final_ori[ct, 0:3]
axis_px1[:, 0, ct] = 960 - (ax_ad[0, :]/ax_ad[2, :]*f_cam)
axis_px1[:, 1, ct] = 540 + (ax_ad[1, :]/ax_ad[2, :]*f_cam)
# sio.savemat('meshsave_back_2.mat',{'vismesh_px':vismesh_px,'mesh_all':fullmesh_px,'pose_all':pose_all[1],'axis_px':axis_px,'axis_px1':axis_px1})
sio.savemat('meshsave_back_2.mat', {'vismesh_px': vismesh_px, 'mesh_all': fullmesh_px,
'pose_all': final_ori, 'axis_px': axis_px, 'axis_px1': axis_px1})
def partition(self, kth, axis=-1, kind='introselect', order=None):
return np.partition(self, kth, axis, kind, order)
def initialize(num_centers_ratio,
data_pipeline,
method=None,
metric='cosine',
max_iter=100,
tol=0.005,
scaling_method=1,
alpha=1.0,
p=3):
"""
This functions initializes representative prototypes (RBF centers) c and scaling factors sigma.
This function will generate one group of centers for all labels as a whole. Be cautious with initialize_per_label.
Args:
num_centers_ratio: The number of centers to be decided / total number of examples that belong to label l,
for l = 0, ..., num_classes - 1.
data_pipeline: A namedtuple consisting of the following elements.
reader, video-level features reader or frame-level features reader.
data_pattern, File Glob of data set.
batch_size, How many examples to handle per time.
num_readers, How many IO threads to prefetch examples.
method: The method to decide the centers. Possible choices are random selection, kmeans and online (kmeans).
Default is None, which represents randomly selecting a certain number of examples as centers.
metric: Distance metric, euclidean distance or cosine distance.
max_iter: The maximal number of iterations clustering to be done.
tol: The minimal reduction of objective function of clustering to be reached to stop iteration.
scaling_method: There are four choices. 1, all of them use the same sigma, the p smallest pairs of distances.
2, average of p nearest centers.
3, distance to the nearest center that has a different label (Not supported!).
4, mean distance between this center and all of its points.
alpha: The alpha parameter that should be set heuristically. It works like a learning rate. (mu in Zhang)
p: When scaling_method is 1 or 2, p is needed.
Returns:
centers (prototypes) and scaling factors (sigmas).
Raises:
ValueError if num_centers_ratio is not between 0.0 (open) and 1.0 (closed).
ValueError if metric is not euclidean or cosine.
ValueError if method is not one of None, kmeans or online.
NotImplementedError if scaling_method is 3 or 5.
ValueError if scaling method is not 1 - 5.
"""
logging.info('Generate a group of centers for all labels. See Schwenker.')
# Argument checking.
if (num_centers_ratio <= 0.0) or (num_centers_ratio > 1.0):
raise ValueError(
'num_centers_ratio must be larger than 0.0 and no greater than 1.0.'
)
logging.info('num_centers_ratio is {}.'.format(num_centers_ratio))
if ('euclidean' == metric) or ('cosine' == metric):
logging.info(
'Using {} distance. The larger, the less similar.'.format(metric))
else:
raise ValueError(
'Only euclidean and cosine distance are supported, {} passed.'.
format(metric))
# Sample features only.
_, centers, _, _ = random_sample(num_centers_ratio,
mask=(False, True, False, False),
data_pipeline=data_pipeline,
name_scope='sample_centers')
logging.info('Sampled {} centers totally.'.format(centers.shape[0]))
logging.debug('Randomly selected centers: {}'.format(centers))
# Used in scaling method 4. Average distance of each point with its cluster center.
per_clu_mean_dist = None
# Perform k-means or online k-means.
if method is None:
logging.info(
'Using randomly selected centers as model prototypes (centers).')
elif 'online' == method:
raise NotImplementedError(
'Only None (randomly select examples), online, kmeans are supported.'
)
elif 'kmeans' == method:
logging.info(
'Using k-means clustering result as model prototypes (centers).')
return_mean_clu_dist = (scaling_method == 4)
kmeans = KMeans(centers,
data_pipeline=data_pipeline,
metric=metric,
return_mean_clu_dist=return_mean_clu_dist)
kmeans.fit(max_iter=max_iter, tol=tol)
# Get current centers and update centers.
centers = kmeans.current_centers
per_clu_mean_dist = kmeans.per_clu_mean_dist
else:
raise ValueError(
'Only None (randomly select examples), online, kmeans are supported.'
)
# Compute scaling factors based on these centers.
num_centers = centers.shape[0]
sigmas = None
if scaling_method == 1:
# Equation 27.
pairwise_distances = sci_distance.pdist(centers, metric=metric)
p = min(p, len(pairwise_distances))
logging.info('Using {} minimal pairwise distances.'.format(p))
# np.partition second argument begins with 0.
sigmas = np.array(
[alpha * np.mean(np.partition(pairwise_distances, p - 1)[:p])] *
num_centers,
dtype=np.float32)
elif scaling_method == 2:
# Equation 28.
p = min(p, num_centers - 1)
logging.info('Using {} minimal distances per center.'.format(p))
if 'euclidean' == metric:
dis_fn = sci_distance.euclidean
else:
dis_fn = sci_distance.cosine
sigmas = []
for c in centers:
distances = [dis_fn(c, _c) for _c in centers]
# The distance between c and itself is zero and is in the left partition.
sigmas.append(alpha * np.sum(np.partition(distances, p)[:p + 1]) /
float(p))
sigmas = np.array(sigmas, dtype=np.float32)
elif scaling_method == 3:
# Equation 29.
raise NotImplementedError(
'Not supported when all labels use the same centers.')
elif scaling_method == 4:
# Equation 30.
if per_clu_mean_dist is None:
kmeans = KMeans(centers,
data_pipeline=data_pipeline,
metric=metric,
return_mean_clu_dist=True)
kmeans.fit(max_iter=1, tol=tol)
centers = kmeans.current_centers
per_clu_mean_dist = kmeans.per_clu_mean_dist
logging.info(
'Compute mean distance per cluster using kmeans or online kmeans.'
)
else:
logging.info(
'Reuse mean distance per cluster computed in kmeans or online kmeans.'
)
sigmas = alpha * per_clu_mean_dist
elif scaling_method == 5:
# Equation 31.
raise NotImplementedError(
'Only three methods are supported. Please read the documentation.')
else:
raise ValueError(
'Only three methods are supported. Please read the documentation.')
logging.debug('Scaling factor sigmas: {}'.format(sigmas))
return centers, sigmas
def partition_wrapper(arr, kth):
try:
return np.partition(arr, kth)
except AttributeError:
return np.sort(arr)
def qValues_planning_abstr(self,
state_abstr_val,
R,
gamma,
T,
Q,
d,
branching_factor=None):
""" Get the q values for pseudo-state(s) with a planning depth d.
This function is called recursively by decreasing the depth d at every step.
Arguments
---------
state_abstr_val : internal state(s).
R : float_model
Model that fits the reward
gamma : float_model
Model that fits the discount factor
T : transition_model
Model that fits the transition between abstract representation
Q : Q_model
Model that fits the optimal Q-value
d : int
planning depth
Returns
-------
The Q-values with planning depth d for the provided encoded state(s)
"""
#if(branching_factor==None or branching_factor>self._n_actions):
# branching_factor=self._n_actions
n = len(state_abstr_val)
identity_matrix = np.identity(self._n_actions)
this_branching_factor = branching_factor.pop(0)
if (n == 1):
# We require that the first branching factor is self._n_actions so that this function return values
# with the right dimension (=self._n_actions).
this_branching_factor = self._n_actions
if (d == 0):
if (this_branching_factor < self._n_actions):
return np.partition(
Q.predict([state_abstr_val]),
-this_branching_factor)[:, -this_branching_factor:]
else:
return Q.predict([state_abstr_val
]) # no change in the order of the actions
else:
if (this_branching_factor == self._n_actions):
# All actions are considered in the tree
# NB: For this case, we do not use argpartition because we want to keep the actions in the natural order
# That way, this function returns the Q-values for all actions with planning depth d in the right order
repeat_identity = np.repeat(identity_matrix,
len(state_abstr_val),
axis=0)
if (state_abstr_val.ndim == 2):
tile3_encoded_x = np.tile(state_abstr_val,
(self._n_actions, 1))
elif (state_abstr_val.ndim == 4):
tile3_encoded_x = np.tile(state_abstr_val,
(self._n_actions, 1, 1, 1))
else:
print("error")
else:
# A subset of the actions corresponding to the best estimated Q-values are considered et each branch
estim_Q_values = Q.predict([state_abstr_val])
ind = np.argpartition(
estim_Q_values,
-this_branching_factor)[:, -this_branching_factor:]
# Replacing ind if we want random branching
#ind = np.random.randint(0,self._n_actions,size=ind.shape)
repeat_identity = identity_matrix[ind].reshape(
n * this_branching_factor, self._n_actions)
tile3_encoded_x = np.repeat(state_abstr_val,
this_branching_factor,
axis=0)
r_vals_d0 = np.array(R.predict([tile3_encoded_x, repeat_identity]))
r_vals_d0 = r_vals_d0.flatten()
gamma_vals_d0 = np.array(
gamma.predict([tile3_encoded_x, repeat_identity]))
gamma_vals_d0 = gamma_vals_d0.flatten()
next_x_predicted = T.predict([tile3_encoded_x, repeat_identity])
return r_vals_d0 + gamma_vals_d0 * np.amax(
self.qValues_planning_abstr(
next_x_predicted,
R,
gamma,
T,
Q,
d=d - 1,
branching_factor=branching_factor).reshape(
len(state_abstr_val) * this_branching_factor,
branching_factor[0]),
axis=1).flatten()
def init_match(ps1, w, md, edges1C, gt, fx_C, fy_C, xb_C, yb_C, edges0C, gt_0,
fx_C0, fy_C0, xb_C0, yb_C0, mask_b_C0):
w = int(w / ps1)
buffer = 2 * w
edges1Ca = np.zeros(
(edges1C.shape[0] + buffer * 2, edges1C.shape[1] + 2 * buffer))
edges1Ca[buffer:-buffer, buffer:-buffer] = edges1C
max_dist = int((md) / ps1)
contours, hierarchy = cv2.findContours((1 - mask_b_C0).astype(np.uint8),
cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE)
contour_sizes = [(cv2.contourArea(contour), contour)
for contour in contours]
biggest_contour = max(contour_sizes, key=lambda x: x[0])[1]
polygon = Polygon(np.array(biggest_contour[:, 0]))
polygon = polygon.buffer(-w)
v = w
while polygon.is_empty or polygon.geom_type == 'MultiPolygon' or polygon.area / ps1 < 100:
v -= int(2 / ps1)
polygon = Polygon(np.array(biggest_contour[:, 0]))
polygon = polygon.buffer(-v)
if v != w:
print("WARNING : Polygon-buffer: " + str(v * ps1) + " < w...")
x, y = polygon.exterior.xy
distance = np.cumsum(
np.sqrt(np.ediff1d(x, to_begin=0)**2 + np.ediff1d(y, to_begin=0)**2))
distance = distance / distance[-1]
fx, fy = interp1d(distance, x), interp1d(distance, y)
alpha = np.linspace(0, 1, 200)
x_regular, y_regular = fx(alpha), fy(alpha)
grid = []
for i in range(len(x_regular)):
grid.append((int(round(x_regular[i])), int(round(y_regular[i]))))
x_offset = []
y_offset = []
o_x = np.zeros(len(grid))
o_y = np.zeros(len(grid))
t_x = np.zeros(len(grid))
t_y = np.zeros(len(grid))
cv = np.zeros(len(grid))
RECC_total = np.zeros(edges1Ca.shape)
RECC_over = np.zeros(edges1Ca.shape)
RECC_over.fill(np.NaN)
circle1 = np.zeros((2 * w, 2 * w))
for x in range(circle1.shape[0]):
for y in range(circle1.shape[1]):
if (x - w)**2 + (y - w)**2 < w**2:
circle1[x, y] = 1
circle2 = np.zeros((2 * max_dist, 2 * max_dist))
for x in range(circle2.shape[0]):
for y in range(circle2.shape[1]):
if (x - max_dist)**2 + (y - max_dist)**2 < max_dist**2:
circle2[x, y] = 1
circle2[circle2 == 0] = np.NaN
for i in tqdm(range(len(grid)),
position=0,
miniters=int(len(grid) / 10),
desc="RECC "):
x_i_0 = grid[i][1]
y_i_0 = grid[i][0]
target = edges0C[x_i_0 - w:x_i_0 + w, y_i_0 - w:y_i_0 + w] * circle1
if target.shape != (2 * w, 2 * w):
continue
sum_target = np.sum(target)
lat_0 = gt_0[3] + gt_0[5] * x_i_0 * fx_C0
lon_0 = gt_0[0] + gt_0[1] * y_i_0 * fy_C0
x_i_0_og = int(round((lat_0 - gt[3]) / (gt[5] * fx_C)))
y_i_0_og = int(round((lon_0 - gt[0]) / (gt[1] * fy_C)))
search_wide = np.zeros((2 * (max_dist + w), 2 * (max_dist + w)))
search_wide = edges1Ca[buffer + x_i_0_og - max_dist - w:buffer +
x_i_0_og + max_dist + w, buffer + y_i_0_og -
max_dist - w:buffer + y_i_0_og + max_dist + w]
if search_wide.shape != (2 * (max_dist + w), 2 * (max_dist + w)):
continue
sum_patch = cv2.filter2D(search_wide, -1, circle1)
numerator = cv2.filter2D(search_wide, -1, target)
RECC_wide = numerator / (sum_patch + sum_target)
RECC_area = RECC_wide[w:-w, w:-w] * circle2
RECC_total.fill(np.NaN)
RECC_total[x_i_0_og - max_dist:x_i_0_og + max_dist,
y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area
if np.nansum(RECC_over[x_i_0_og - max_dist:x_i_0_og + max_dist,
y_i_0_og - max_dist:y_i_0_og + max_dist]) == 0:
RECC_over[x_i_0_og - max_dist:x_i_0_og + max_dist,
y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area
max_one = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(),
-1)[-1]
max_n = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(),
-4 - 1)[-4 - 1]
y_i = np.where(RECC_total >= max_one)[1][0]
x_i = np.where(RECC_total >= max_one)[0][0]
y_n = np.where(RECC_total >= max_n)[1][0:-1]
x_n = np.where(RECC_total >= max_n)[0][0:-1]
cv[i] = sum(np.sqrt(np.square(x_i - x_n) + np.square(y_i - y_n))) / 4
x_offset.append(x_i - x_i_0_og)
y_offset.append(y_i - y_i_0_og)
o_x[i] = x_i
o_y[i] = y_i
t_x[i] = x_i_0
t_y[i] = y_i_0
x_offset = np.array(x_offset)
y_offset = np.array(y_offset)
a, b, c = np.histogram2d(x_offset[cv < 4],
y_offset[cv < 4],
bins=len(x_offset))
d, e = np.where(a == np.max(a))
if len(d) > 1:
i = [0, 0]
binnnn = len(x_offset) / 10
while len(i) > 1:
binnnn -= 1
f, g, h = np.histogram2d(x_offset[cv < 4],
y_offset[cv < 4],
bins=binnnn)
i, j = np.where(f == np.max(f))
diff = (b[d] - g[i])**2 + (c[e] - h[j])**2
ind = np.where(diff == np.min(diff))[0]
d = d[ind]
e = e[ind]
x_offset = (b[d] + b[d + 1]) / 2
y_offset = (c[e] + c[e + 1]) / 2
x_offset = x_offset[0] * ps1
y_offset = y_offset[0] * ps1
o_x = o_x[cv < 4]
o_y = o_y[cv < 4]
t_x = t_x[cv < 4]
t_y = t_y[cv < 4]
return x_offset, y_offset, o_x, o_y, t_x, t_y
def fft(self):
#Performs fft of each state channel seperately and returns (symmetry, [spacial peaks], [temporal peaks])
#//16 or //32 work best
filter_resolution=self.size//32
#'background' signal removal parameters
t0 = self.max_iters//2
q0 = self.size//2
# One hot decode real space trajectory and fft the indivual binary representations of each state
im_one_hot = one_hot_encode(self.image,self.states)
fd = (np.fft.fftshift(np.fft.fftn(im_one_hot,axes=(1,2,3))))
#sys.stdout.write("#")
#sys.stdout.flush()
#Data for visualising purposes
self.fft_data = np.abs(one_hot_decode(fd))
#Remove background signal (q=0,f=0)
ts,xs,ys=np.meshgrid(np.arange(self.max_iters),np.arange(self.size),np.arange(self.size),indexing="ij")
gs = gaus3d(ts,xs,ys,t0,q0,q0,4,4,4)
fd_filtered = np.zeros(fd.shape,dtype=fd.dtype)
for i in range(self.states):
#plt.matshow(fd[i,t0])
#plt.show()
fd_filtered[i] = (1-gs)*fd[i]
#fd_filtered[i] = fd[i]
#sys.stdout.write("#")
#sys.stdout.flush()
#Apply local max filter to enhance peaks and suppress noise
n_peaks = self.size*self.size//8
for i in range(self.states):
for t in range(self.max_iters):
slice = ndimage.maximum_filter(np.abs(fd_filtered[i,t]),filter_resolution,mode="wrap")
flat = np.abs((slice)).flatten()
k = np.partition(flat,-n_peaks)[-n_peaks]
slice[np.abs(slice)<k]=0
fd_filtered[i,t] = slice
#sys.stdout.write("#")
#sys.stdout.flush()
#Peak detection along time axis
#print(fd_filtered[0,:,q0,q0].shape)
max_t_peaks = np.zeros((4,self.states)).astype(int)
a,b = sp.signal.butter(3,0.3,analog=False)
for i in range(self.states):
tdata = np.pad(np.abs(np.sum(fd_filtered[i,t0:],axis=(1,2))),(0,filter_resolution),mode='reflect')
tdata = sp.signal.filtfilt(a,b,tdata)
#tdata = np.pad(np.abs(fd_filtered[i,t0:,q0,q0]),(0,filter_resolution),mode='reflect')
peaks,dicts = sp.signal.find_peaks(tdata,prominence=256)
if len(peaks)==0:
#No peaks found
ind_max=0
ind_2nd=0
max_t_peaks[0,i] = 0#peaks[ind_max]
max_t_peaks[1,i] = 0#peaks[ind_2nd]
elif len(peaks)==1:
#only one peak found
ind_max = np.argmax(dicts['prominences'])
ind_2nd=0
max_t_peaks[0,i] = peaks[ind_max]
max_t_peaks[1,i] = 0#peaks[ind_2nd]
else:
#at least 2 peaks found
ind_max = np.argmax(dicts['prominences'])
ind_2nd = np.argpartition(dicts['prominences'],-2)[-2]
max_t_peaks[0,i] = peaks[ind_max]
max_t_peaks[1,i] = peaks[ind_2nd]
max_t_peaks[2,i] = tdata[max_t_peaks[0,i]]
max_t_peaks[3,i] = tdata[max_t_peaks[1,i]]
#plt.plot(tdata)
#plt.scatter(peaks,tdata[peaks])
#plt.scatter(max_t_peaks[0,i],max_t_peaks[2,i],color="red")
#plt.scatter(max_t_peaks[1,i],max_t_peaks[3,i],color="green")
#plt.show()
max_t_peak_mean = (2/self.max_iters)*safe_average(max_t_peaks[0,max_t_peaks[0]!=0],max_t_peaks[2,max_t_peaks[0]!=0])
max_t_peak_var = (2/self.max_iters)*np.std(max_t_peaks[0,max_t_peaks[0]!=0])
harmonic_t_mean = (2/self.max_iters)*safe_average(max_t_peaks[1,max_t_peaks[1]!=0],max_t_peaks[3,max_t_peaks[1]!=0])
harmonic_t_var = (2/self.max_iters)*np.std(max_t_peaks[1,max_t_peaks[1]!=0])
temporal_peaks = np.array([[max_t_peak_mean,max_t_peak_var],[harmonic_t_mean,harmonic_t_var]])
#print(temporal_peaks)
#sys.stdout.write("#")
#sys.stdout.flush()
#Check for 4-fold symmetry
fd_filtered_rot = np.rot90(fd_filtered,axes=(2,3))
rot_coeff = np.einsum('ijkl,ijkl',fd_filtered_rot,fd_filtered)
norm = np.einsum('ijkl,ijkl',fd_filtered,fd_filtered)
symmetry_coeff = np.abs(rot_coeff/norm)
#sys.stdout.write("#")
#sys.stdout.flush()
#print(symmetry_coeff)
#Calculuate radial distribution functions of fft data
#print("Calculating rdf")
fd_rdf = np.zeros((self.states,self.max_iters,self.size//2)).astype(fd.dtype)
fd_rdf_filtered = np.zeros((self.states,self.max_iters,self.size//2)).astype(fd.dtype)
for x in range(self.size//2):
sq = square_ring(self.size,x+1)
fd_rdf_filtered[:,:,x] = np.einsum('ijkl,kl->ij',fd_filtered,sq)/np.sum(sq)
fd_rdf[:,:,x] = np.einsum('ijkl,kl->ij',fd,sq)/np.sum(sq)
#sys.stdout.write("#")
#sys.stdout.flush()
#peak detection on rdf
#print("Finding peaks")
max_peaks = np.zeros((4,self.states)).astype(int)
for i in range(self.states):
xdata = np.pad(np.abs(fd_rdf_filtered[i,t0]),(0,filter_resolution),mode='reflect')
#peaks,dicts = signal.find_peaks(xdata,height=256,prominence=None,width=filter_resolution)
peaks,dicts = sp.signal.find_peaks(xdata,prominence=None,width=filter_resolution)
if len(peaks)==0:
#No peaks found
ind_max=0
ind_2nd=0
max_peaks[0,i] = 0#peaks[ind_max]
max_peaks[1,i] = 0#peaks[ind_2nd]
elif len(peaks)==1:
#only one peak found
ind_max = np.argmax(dicts['prominences'])
ind_2nd=0
max_peaks[0,i] = peaks[ind_max]
max_peaks[1,i] = 0#peaks[ind_2nd]
else:
#at least 2 peaks found
ind_max = np.argmax(dicts['prominences'])
ind_2nd = np.argpartition(dicts['prominences'],-2)[-2]
max_peaks[0,i] = peaks[ind_max]
max_peaks[1,i] = peaks[ind_2nd]
#Magnitude at peaks
max_peaks[2,i] = xdata[max_peaks[0,i]]
max_peaks[3,i] = xdata[max_peaks[1,i]]
#plt.plot(xdata)
#plt.scatter(peaks,xdata[peaks])
#plt.show()
max_peak_mean = (2/self.size)*safe_average(max_peaks[0,max_peaks[0]!=0],max_peaks[2,max_peaks[0]!=0])
max_peak_var = (2/self.size)*np.std(max_peaks[0,max_peaks[0]!=0])
harmonic_mean = (2/self.size)*safe_average(max_peaks[1,max_peaks[1]!=0],max_peaks[3,max_peaks[1]!=0])
harmonic_var = (2/self.size)*np.std(max_peaks[1,max_peaks[1]!=0])
#print(max_peak_var)
spacial_peaks = np.array([[max_peak_mean,max_peak_var],[harmonic_mean,harmonic_var]])
#print(spacial_peaks)
#sys.stdout.write("#")
#sys.stdout.flush()
#waves[waves<thresh]=0
#stat_struct_filtered = np.zeros(stat_struct.shape,dtype='complex')
#stat_struct_filtered[np.abs(stat_struct_filtered)<thresh]=0
#waves_filtered = signal.convolve(waves,f_gs_3,mode='same')
#plt.matshow(gs)
#plt.show()
self.image = np.abs(np.fft.ifftn(np.fft.ifftshift(one_hot_decode(fd_filtered))))
"""
#Split up sections of 3d fft that correspond to certain features - disused
stat_struct = fd[:,t0]
stat_struct[:,q0,q0]=0
stat_struct_filtered = fd_filtered[:,t0]#ndimage.maximum_filter(np.abs(stat_struct),self.size//16,mode="wrap")
stat_struct_filtered_rot = fd_filtered_rot[:,t0]
blinkers = fd_filtered[:,:,q0,q0]
#blinkers[:,t0]=0
waves = np.copy(fd)
waves[:,t0] = 0
waves[:,:,q0,q0]=0
for i in range(self.states):
epsilon=1E-20
xdata = (np.abs(fd_rdf[i,t0]))
peaks,_ = signal.find_peaks(xdata,prominence=None,width=filter_resolution)
plt.plot(xdata)
plt.scatter(peaks,xdata[peaks])
plt.matshow((np.abs(stat_struct[i])))
plt.show()
#plt.plot(np.sum(np.abs(stat_struct[i]),axis=0))
#plt.plot(np.sum(np.abs(stat_struct[i]),axis=1))
#plt.show()
xdata = (np.abs(fd_rdf_filtered[i,t0]))
peaks,dicts = signal.find_peaks(xdata,prominence=None,width=filter_resolution)
#maximum peak height
#max_peaks[0,i] = peaks[np.argmax(xdata[peaks])]
#most prominant peak
max_peaks[0,i] = peaks[np.argmax(dicts['prominences'])]
max_peaks[1,i] = xdata[max_peaks[0,i]]
plt.plot(xdata)
plt.scatter(peaks,xdata[peaks])
plt.scatter(max_peaks[0,i],max_peaks[1,i],color="red")
plt.matshow((np.abs(stat_struct_filtered[i])))
#plt.matshow(laplace(np.abs(stat_struct[i])))
plt.show()
#plt.show()
#plt.matshow((np.abs(stat_struct_filtered_rot[i])))
#plt.show()
#plt.plot(np.sum(np.abs(stat_struct_filtered[i]),axis=0))
#plt.plot(np.sum(np.abs(stat_struct_filtered[i]),axis=1))
#plt.show()
for j in range(self.states):
#j=1
plt.plot(np.abs(blinkers[j]),label=str(j))
#plt.plot(np.abs(fd_ref[:,q0,q0]),label="reference")
#plt.plot(blinkers[j].real,alpha=0.2)
#plt.plot(blinkers[j].imag,alpha=0.2)
plt.legend()
plt.show()
for j in range(self.states):
plt.plot(np.abs(np.sum(fd_filtered[j],axis=(1,2))),label=str(j))
plt.legend()
plt.show()
for j in range(self.states):
plt.plot(np.sum(np.abs(fd_filtered[j]),axis=(1,2)),label=str(j))
plt.legend()
plt.show()
self.fft_data=(np.abs(one_hot_decode(fd_filtered)))
"""
#sys.stdout.write("|")
#sys.stdout.flush()
return (symmetry_coeff,spacial_peaks,temporal_peaks,fd[:,t0])
def PatchMatch(ps2, w, md, edges1F, gt, fx_F, fy_F, edges0F, gt_0, fx_F0,
fy_F0, contour_F0, x_offset, y_offset):
w = int(w / ps2)
buffer = 2 * w
edges1Fa = np.zeros(
(edges1F.shape[0] + buffer * 2, edges1F.shape[1] + 2 * buffer))
edges1Fa[buffer:-buffer, buffer:-buffer] = edges1F
max_dist = int((md) / (ps2 * 4))
contours, hierarchy = cv2.findContours((1 - contour_F0).astype(np.uint8),
cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE)
contour_sizes = [(cv2.contourArea(contour), contour)
for contour in contours]
biggest_contour = max(contour_sizes, key=lambda x: x[0])[1]
polygon = Polygon(np.array(biggest_contour[:, 0]))
polygon = polygon.buffer(-w)
v = w
while polygon.is_empty or polygon.geom_type == 'MultiPolygon' or polygon.area / ps2 < 100:
v -= int(2 / ps2)
polygon = Polygon(np.array(biggest_contour[:, 0]))
polygon = polygon.buffer(-v)
if v != w:
print("WARNING : Polygon-buffer: " + str(v) + " < w...")
x, y = polygon.exterior.xy
distance = np.cumsum(
np.sqrt(np.ediff1d(x, to_begin=0)**2 + np.ediff1d(y, to_begin=0)**2))
distance = distance / distance[-1]
fx, fy = interp1d(distance, x), interp1d(distance, y)
alpha = np.linspace(0, 1, 200)
x_regular, y_regular = fx(alpha), fy(alpha)
grid = []
for i in range(len(x_regular)):
grid.append((int(round(x_regular[i])), int(round(y_regular[i]))))
cv = np.zeros(len(grid))
dist_lon = np.zeros(len(grid))
dist_lat = np.zeros(len(grid))
origin_x = np.zeros(len(grid))
origin_y = np.zeros(len(grid))
target_lon = np.zeros(len(grid))
target_lat = np.zeros(len(grid))
o_x = np.zeros(len(grid))
o_y = np.zeros(len(grid))
t_x = np.zeros(len(grid))
t_y = np.zeros(len(grid))
dx = np.zeros(len(grid))
dy = np.zeros(len(grid))
RECC_total = np.zeros(edges1Fa.shape)
RECC_over = np.zeros(edges1Fa.shape)
RECC_over.fill(np.NaN)
circle1 = np.zeros((2 * w, 2 * w))
for x in range(circle1.shape[0]):
for y in range(circle1.shape[1]):
if (x - w)**2 + (y - w)**2 < w**2:
circle1[x, y] = 1
circle2 = np.zeros((2 * max_dist, 2 * max_dist))
for x in range(circle2.shape[0]):
for y in range(circle2.shape[1]):
if (x - max_dist)**2 + (y - max_dist)**2 < max_dist**2:
circle2[x, y] = 1
circle2[circle2 == 0] = np.NaN
for i in tqdm(range(len(grid)),
position=0,
miniters=int(len(grid) / 10),
desc="RECC "):
x_i_0 = grid[i][1]
y_i_0 = grid[i][0]
target = edges0F[x_i_0 - w:x_i_0 + w, y_i_0 - w:y_i_0 + w]
if target.shape != (2 * w, 2 * w):
continue
target = target * circle1
sum_target = np.sum(target)
lat_0 = gt_0[3] + gt_0[5] * x_i_0 * fx_F0
lon_0 = gt_0[0] + gt_0[1] * y_i_0 * fy_F0
x_i_0_og = int(
round((lat_0 - gt[3]) / (gt[5] * fx_F) + (x_offset / ps2)))
y_i_0_og = int(
round((lon_0 - gt[0]) / (gt[1] * fy_F) + (y_offset / ps2)))
search_wide = np.zeros((2 * (max_dist + w), 2 * (max_dist + w)))
search_wide = edges1Fa[buffer + x_i_0_og - max_dist - w:buffer +
x_i_0_og + max_dist + w, buffer + y_i_0_og -
max_dist - w:buffer + y_i_0_og + max_dist + w]
if search_wide.shape != (2 * (max_dist + w), 2 * (max_dist + w)):
continue
sum_patch = cv2.filter2D(search_wide, -1, circle1)
numerator = cv2.filter2D(search_wide, -1, target)
RECC_wide = numerator / (sum_patch + sum_target)
RECC_area = RECC_wide[w:-w, w:-w] * circle2
RECC_total.fill(np.NaN)
RECC_total[x_i_0_og - max_dist:x_i_0_og + max_dist,
y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area
if np.nansum(RECC_over[x_i_0_og - max_dist:x_i_0_og + max_dist,
y_i_0_og - max_dist:y_i_0_og + max_dist]) == 0:
RECC_over[x_i_0_og - max_dist:x_i_0_og + max_dist,
y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area
max_one = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(),
-1)[-1]
max_n = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(),
-4 - 1)[-4 - 1]
y_i = np.where(RECC_total >= max_one)[1][0]
x_i = np.where(RECC_total >= max_one)[0][0]
y_n = np.where(RECC_total >= max_n)[1][0:-1]
x_n = np.where(RECC_total >= max_n)[0][0:-1]
cv[i] = sum(np.sqrt(np.square(x_i - x_n) + np.square(y_i - y_n))) / 4
lon = gt[0] + gt[1] * y_i * fy_F + gt[2] * x_i * fx_F
lat = gt[3] + gt[4] * y_i * fy_F + gt[5] * x_i * fx_F
lon_0 = gt_0[0] + gt_0[1] * y_i_0 * fy_F0 + gt_0[2] * x_i_0 * fx_F0
lat_0 = gt_0[3] + gt_0[4] * y_i_0 * fy_F0 + gt_0[5] * x_i_0 * fx_F0
dist_lon[i] = lon_0 - lon
dist_lat[i] = lat_0 - lat
o_x[i] = x_i
o_y[i] = y_i
t_x[i] = x_i_0
t_y[i] = y_i_0
dx[i] = (x_i - x_i_0_og) * ps2
dy[i] = (y_i - y_i_0_og) * ps2
# For referencing:
origin_x[i] = x_i * fx_F
origin_y[i] = y_i * fy_F
target_lon[i] = lon_0
target_lat[i] = lat_0
return origin_x, origin_y, target_lon, target_lat, o_x, o_y, t_x, t_y, cv, dx, dy, RECC_over
fft_len = 2304 effect_fft = int(fft_len / 2) for k in range(3,4): data_zif90 = list_test[k][:-1] data_zif90 -= np.median(data_zif90) data_zif90[data_zif90<0] = 0 data_zif90 /= np.max(np.abs(data_zif90)) ########################替换peak为mean copy_no_peak = np.copy(data_zif90) h_the = np.partition(data_zif90, -num_large)[-num_large] # copy_no_peak[data_zif90 > h_the] = np.median(data_zif90) copy_no_peak[data_zif90 > h_the] = 0 ########################替换noise为0 copy_only_peak = np.copy(data_zif90) copy_only_peak[data_zif90 < h_the] = 0 # copy_only_peak[data_zif90 < h_the] = np.median(data_zif90) names= ['original', 'no_peak', 'only_peak'] data = np.zeros((len(names), fft_len)) data[0, 20:2251+20] = data_zif90 data[1, 20:2251+20] = copy_no_peak data[2, 20:2251+20] = copy_only_peak
def nsmall(a, n):
# find the smallest nth element of an np array
return np.partition(a, n)[n]
def _topk_py_impl(op, x, k, axis, idx_dtype):
ndim = x.ndim
assert -ndim <= axis < ndim
axis %= ndim
if k == 0:
raise ValueError("topk: kth cannot be zero")
elif k > x.shape[axis]:
raise ValueError(
f"topk: kth cannot be larger than the size of specified axis {int(axis)}"
)
if abs(k) == 1:
# negative k means min instead of max
fn_max = [None, np.max, np.min][k]
fn_argmax = [None, np.argmax, np.argmin][k]
if not op.return_indices:
return np.expand_dims(fn_max(x, axis=axis), axis)
elif op.return_values:
zi = np.expand_dims(fn_argmax(x, axis=axis), axis)
idx2 = tuple(
np.arange(s).reshape((s, ) + (1, ) *
(ndim - i - 1)) if i != axis else zi
for i, s in enumerate(x.shape))
zv = x[idx2]
return zv, zi.astype(idx_dtype)
else:
zi = np.expand_dims(fn_argmax(x, axis=axis), axis)
return zi.astype(idx_dtype)
if x.shape[axis] == abs(k):
if not op.return_indices:
return x.copy()
else:
l = axis
r = ndim - l
reps = list(x.shape)
reps[axis] = 1
zi = np.arange(abs(k), dtype=idx_dtype)
zi = zi.reshape((1, ) * l + (k, ) + (1, ) * (r - 1))
zi = np.tile(zi, reps)
if op.return_values:
return x.copy(), zi
else:
return zi
idx = [slice(None)] * ndim
idx[axis] = slice(-k, None) if k > 0 else slice(-k)
if not op.return_indices:
zv = np.partition(x, -k, axis=axis)[idx]
return zv
elif op.return_values:
zi = np.argpartition(x, -k, axis=axis)[idx]
idx2 = tuple(
np.arange(s).reshape((s, ) + (1, ) *
(ndim - i - 1)) if i != axis else zi
for i, s in enumerate(x.shape))
zv = x[idx2]
return zv, zi.astype(idx_dtype)
else:
zi = np.argpartition(x, -k, axis=axis)[idx]
return zi.astype(idx_dtype)
