def _trimmed_mean_1d(arr, k): """Calculate trimmed mean on a 1d array. Trim values largest than the k'th largest value or smaller than the k'th smallest value Parameters ---------- arr: ndarray, shape (n,) The one-dimensional input array to perform trimmed mean on k: int The thresholding order for trimmed mean Returns ------- trimmed_mean: float The trimmed mean calculated """ kth_smallest = np.partition(arr, k)[k-1] kth_largest = -np.partition(-arr, k)[k-1] cnt = 0 summation = 0.0 for elem in arr: if elem >= kth_smallest and elem <= kth_largest: cnt += 1 summation += elem return summation / cnt
def arg_median(a): if len(a) % 2 == 1: return np.where( a == np.median(a) )[0][0] else: l,r = len(a)/2 -1, len(a)/2 left = np.partition(a, l)[l] right = np.partition(a, r)[r] return [np.where(a == left)[0][0], np.where(a==right)[0][0]]
def color_range(data): #Define the color range clean = data[data>0] min_element = clean.size/20 max_element = clean.size*9/10 vmin = np.partition(clean, min_element, axis=None)[min_element] #invece di subint[subint>0] possibile subint[:-(num_rows/down_fact)] vmax = np.partition(clean, max_element, axis=None)[max_element] return vmin,vmax
def _degree_feats(uts=None, G=None, name_ext="", exclude_id=None): """ Helper method for retrieve_feats(). Generate statistics on degree-related features in a Hypergraph (G), or a Hypergraph constructed from provided utterances (uts) :param uts: utterances to construct Hypergraph from :param G: Hypergraph to calculate degree features statistics from :param name_ext: Suffix to append to feature name :param exclude_id: id of utterance to exclude from Hypergraph construction :return: A dictionary from a thread root id to its stats dictionary, which is a dictionary from feature names to feature values. For degree-related features specifically. """ assert uts is None or G is None if G is None: G = HyperConvo._make_hypergraph(uts, exclude_id=exclude_id) stat_funcs = { "max": np.max, "argmax": np.argmax, "norm.max": lambda l: np.max(l) / np.sum(l), "2nd-largest": lambda l: np.partition(l, -2)[-2] if len(l) > 1 else np.nan, "2nd-argmax": lambda l: (-l).argsort()[1] if len(l) > 1 else np.nan, "norm.2nd-largest": lambda l: np.partition(l, -2)[-2] / np.sum(l) if len(l) > 1 else np.nan, "mean": np.mean, "mean-nonzero": lambda l: np.mean(l[l != 0]), "prop-nonzero": lambda l: np.mean(l != 0), "prop-multiple": lambda l: np.mean(l[l != 0] > 1), "entropy": scipy.stats.entropy, "2nd-largest / max": lambda l: np.partition(l, -2)[-2] / np.max(l) if len(l) > 1 else np.nan } stats = {} for from_hyper in [False, True]: for to_hyper in [False, True]: if not from_hyper and to_hyper: continue # skip c -> C outdegrees = np.array(G.outdegrees(from_hyper, to_hyper)) indegrees = np.array(G.indegrees(from_hyper, to_hyper)) for stat, stat_func in stat_funcs.items(): stats["{}[outdegree over {}->{} {}responses]".format(stat, HyperConvo._node_type_name(from_hyper), HyperConvo._node_type_name(to_hyper), name_ext)] = stat_func(outdegrees) stats["{}[indegree over {}->{} {}responses]".format(stat, HyperConvo._node_type_name(from_hyper), HyperConvo._node_type_name(to_hyper), name_ext)] = stat_func(indegrees) return stats
def getMedianIndex(array): if len(array) % 2 == 1: print "Par" return np.where( array == np.median(array) )[0][0] else: try: print "Impar" l,r = len(array)/2 -1, len(array)/2 left = np.partition(array, l)[l] right = np.partition(array, r)[r] return [np.where(array == left)[0][0], np.where(array==right)[0][0]] except: print "Warning: excepction controlled in median : "+str(len(array)) return (len(array)/(np.e*2))
def avg_x_weak_weights(wmxO, x): ''' Holds only the 4000-x strongest weight in every row and average the other ones :param wmxO: original weight matrix (4000 * 4000 ndarray) :return: wmxM: modified weight matrix (4000 * 4000 ndarray) ''' nHolded = 4000 - x wmxM = np.zeros((4000, 4000)) for i in range(0, 4000): row = wmxO[i, :] tmp = np.partition(-row, nHolded) max = -tmp[:nHolded] # values of first 4000-x elements rest = -tmp[nHolded:] mu = np.mean(rest) # mean of the x weights tmp = np.argpartition(-row, nHolded) maxj = tmp[:nHolded] # indexes of first 4000-x elements rowM = mu * np.ones((1, 4000)) for j, val in zip(maxj, max): rowM[0, j] = val wmxM[i, :] = rowM return wmxM
def compute_csls_accuracy(x_src, x_tgt, lexicon, lexicon_size=-1, k=10, bsz=1024): if lexicon_size < 0: lexicon_size = len(lexicon) idx_src = list(lexicon.keys()) x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8 x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8 sr = x_src[list(idx_src)] sc = np.dot(sr, x_tgt.T) similarities = 2 * sc sc2 = np.zeros(x_tgt.shape[0]) for i in range(0, x_tgt.shape[0], bsz): j = min(i + bsz, x_tgt.shape[0]) sc_batch = np.dot(x_tgt[i:j, :], x_src.T) dotprod = np.partition(sc_batch, -k, axis=1)[:, -k:] sc2[i:j] = np.mean(dotprod, axis=1) similarities -= sc2[np.newaxis, :] nn = np.argmax(similarities, axis=1).tolist() correct = 0.0 for k in range(0, len(lexicon)): if nn[k] in lexicon[idx_src[k]]: correct += 1.0 return correct / lexicon_size
def make_query(self): """Return the index of the sample to be queried and labeled. Returns ------- ask_id: int The entry_id of the sample this algorithm wants to query. """ dataset = self.dataset self.model.train(dataset) unlabeled_entry_ids, X_pool = zip(*dataset.get_unlabeled_entries()) if self.method == 'lc': # least confident ask_id = np.argmin( np.max(self.model.predict_real(X_pool), axis=1) ) elif self.method == 'sm': # smallest margin dvalue = self.model.predict_real(X_pool) if np.shape(dvalue)[1] > 2: # Find 2 largest decision values dvalue = -(np.partition(-dvalue, 2, axis=1)[:, :2]) margin = np.abs(dvalue[:, 0] - dvalue[:, 1]) ask_id = np.argmin(margin) return unlabeled_entry_ids[ask_id]
def _chunk_based_bmu_find(input_matrix, codebook, y2, nth=1): """ Finds the corresponding bmus to the input matrix. :param input_matrix: a matrix of input data, representing input vector as rows, and vectors features/dimention as cols when parallelizing the search, the input_matrix can be a sub matrix from the bigger matrix :param codebook: matrix of weights to be used for the bmu search :param y2: <not sure> """ dlen = input_matrix.shape[0] nnodes = codebook.shape[0] bmu = np.empty((dlen, 2)) # It seems that small batches for large dlen is really faster: # that is because of ddata in loops and n_jobs. for large data it slows # down due to memory needs in parallel blen = min(50, dlen) i0 = 0 while i0+1 <= dlen: low = i0 high = min(dlen, i0+blen) i0 = i0+blen ddata = input_matrix[low:high+1] d = np.dot(codebook, ddata.T) d *= -2 d += y2.reshape(nnodes, 1) bmu[low:high+1, 0] = np.argpartition(d, nth, axis=0)[nth-1] bmu[low:high+1, 1] = np.partition(d, nth, axis=0)[nth-1] del ddata return bmu
def disp_results(fig, ax1, ax2, loss_iterations, losses, accuracy_iterations, accuracies, accuracies_iteration_checkpoints_ind, fileName, color_ind=0): modula = len(plt.rcParams['axes.color_cycle']) acrIterations =[] top_acrs={} if accuracies.size: if accuracies.size>4: top_n = 4 else: top_n = accuracies.size -1 temp = np.argpartition(-accuracies, top_n) result_indexces = temp[:top_n] temp = np.partition(-accuracies, top_n) result = -temp[:top_n] for acr in result_indexces: acrIterations.append(accuracy_iterations[acr]) top_acrs[str(accuracy_iterations[acr])]=str(accuracies[acr]) sorted_top4 = sorted(top_acrs.items(), key=operator.itemgetter(1)) maxAcc = np.amax(accuracies, axis=0) iterIndx = np.argmax(accuracies) maxAccIter = accuracy_iterations[iterIndx] maxIter = accuracy_iterations[-1] consoleInfo = format('\n[%s]:maximum accuracy [from 0 to %s ] = [Iteration %s]: %s ' %(fileName,maxIter,maxAccIter ,maxAcc)) plotTitle = format('max accuracy(%s) [Iteration %s]: %s ' % (fileName,maxAccIter, maxAcc)) print (consoleInfo) #print (str(result)) #print(acrIterations) # print 'Top 4 accuracies:' print ('Top 4 accuracies:'+str(sorted_top4)) plt.title(plotTitle) ax1.plot(loss_iterations, losses, color=plt.rcParams['axes.color_cycle'][(color_ind * 2 + 0) % modula]) ax2.plot(accuracy_iterations, accuracies, plt.rcParams['axes.color_cycle'][(color_ind * 2 + 1) % modula], label=str(fileName)) ax2.plot(accuracy_iterations[accuracies_iteration_checkpoints_ind], accuracies[accuracies_iteration_checkpoints_ind], 'o', color=plt.rcParams['axes.color_cycle'][(color_ind * 2 + 1) % modula]) plt.legend(loc='lower right')
def find_top_k(x, k): #return an array where anything less than the top k values of an array is zero if( np.count_nonzero(x) <k): return x else: x[x < -1*np.partition(-1*x, k)[k]] = 0 return x
def distance_to_kth_neighbor(metric, k_neighbors): """Computes the distance to the kth neighbor for each point in a metric. Args ---- metric : ndarray A distance matrix in square or condensed form. k_neighbors : int The order of neighbor to which distance is computed, where the 0th neighbor of any point is the point itself. Returns ------- distances : ndarray Distance to the kth neighbor for each point in `metric`. Note ---- It is an implementation detail that the input metric is coerced to a squareform array. """ # coerce any condensed metric to square form if metric.ndim == 1: metric = _dist.squareform(metric) if k_neighbors >= metric.shape[0]: message = 'k_neighbors must be less than the number of points.' raise ValueError(message) if k_neighbors < 0: raise ValueError('k_neighbors must be non-negative') return _np.partition(metric, k_neighbors, axis=1)[:, k_neighbors]
def estimate_eps(dist_mat, n_closest=5): """ Estimates possible eps values (to be used with DBSCAN) for a given distance matrix by looking at the largest distance "jumps" amongst the `n_closest` distances for each item. Tip: the value for `n_closest` is important - set it too large and you may only get really large distances which are uninformative. Set it too small and you may get premature cutoffs (i.e. select jumps which are really not that big). TO DO this could be fancier by calculating support for particular eps values, e.g. 80% are around 4.2 or w/e """ dist_mat = dist_mat.copy() # To ignore i == j distances dist_mat[np.where(dist_mat == 0)] = np.inf estimates = [] for i in range(dist_mat.shape[0]): # Indices of the n closest distances row = dist_mat[i] dists = sorted(np.partition(row, n_closest)[:n_closest]) difs = [(x, y, (y - x)) for x, y in zip(dists, dists[1:])] eps_candidate, _, jump = max(difs, key=lambda x: x[2]) estimates.append(eps_candidate) return sorted(estimates)
def nsmall(arr, n, axis): return np.partition(arr, n, axis)[n] #def addToPlot(subplot, appendX, appendY): # plt.figure(1) # plt.subplot(subplot) # plt.se
def test_uint64(self): arr = np.arange(10, -1, -1, dtype='int64') partition = np.partition(arr, self.pivot_index) self._check_partition(partition, self.pivot_index) self._check_content_along_axis(arr, partition, -1)
def make_query(self): """ Choices for method (default 'lc'): 'lc' (Least Confident), 'sm' (Smallest Margin) """ dataset = self.dataset self.model.train(dataset) unlabeled_entry_ids, X_pool = zip(*dataset.get_unlabeled_entries()) if self.method == 'lc': # least confident ask_id = np.argmin( np.max(self.model.predict_real(X_pool), axis=1) ) elif self.method == 'sm': # smallest margin dvalue = self.model.predict_real(X_pool) if np.shape(dvalue)[1] > 2: # Find 2 largest decision values dvalue = -(np.partition(-dvalue, 2, axis=1)[:, :2]) margin = np.abs(dvalue[:, 0] - dvalue[:, 1]) ask_id = np.argmin(margin) return unlabeled_entry_ids[ask_id]
def fast_abs_percentile(data, percentile=80): """ A fast version of the percentile of the absolute value. Parameters ========== data: ndarray, possibly masked array The input data percentile: number between 0 and 100 The percentile that we are asking for Returns ======= value: number The score at percentile Notes ===== This is a faster, and less accurate version of scipy.stats.scoreatpercentile(np.abs(data), percentile) """ if hasattr(data, 'mask'): # Catter for masked arrays data = np.asarray(data[np.logical_not(data.mask)]) data = np.abs(data) data = data.ravel() index = int(data.size * .01 * percentile) if partition is not None: # Partial sort: faster than sort return partition(data, index)[index + 1] data.sort() return data[index + 1]
def random_con_bitmask(prob, shape, mins=1): """Generate a random bitmask with constraints The functions allows specifying the minimum number of True values along a single dimension while counting over the other ones. `mins` can be a scalar or a tuple for each dimension and must be less than the product of the size of the other dimensions. If you just want a random bitmask use np.random.random(shape) < prob""" assert len(shape) > 1 vals = np.random.random(shape) mask = vals < prob total = vals.size if isinstance(mins, abc.Sequence): assert len(mins) == vals.ndim assert all(0 < s <= total // m for s, m in zip(mins, vals.shape)) else: assert mins > 0 mins = tuple(min(mins, total // m) for m in vals.shape) for dim, num in enumerate(mins): aligned = np.rollaxis(vals, dim).reshape(vals.shape[dim], -1) thresh = np.partition(aligned, num - 1, 1)[:, num - 1] thresh.shape += (1,) * (vals.ndim - dim - 1) mask |= vals <= thresh return mask
def sumvalues(self, q=0): """Sum of top q passowrd frequencies """ if q == 0: return self._totalf else: return -np.partition(-self._freq_list, q)[:q].sum()
def test_partition_matrix_none(): # gh-4301 # 2018-04-29: moved here from core.tests.test_multiarray a = np.matrix([[2, 1, 0]]) actual = np.partition(a, 1, axis=None) expected = np.matrix([[0, 1, 2]]) assert_equal(actual, expected) assert_(type(expected) is np.matrix)
def get_top_k_elements_per_row_sim_mat(similarity_matrix, k): ''' Introduces another parameter, k, where we only keep the top k similarity scores per row in the similarity matrix. ''' for i, row in enumerate(similarity_matrix): k = len(row) - k kth_largest = np.partition(row, k)[k] similarity_matrix[i] = [ele if ele >= kth_largest else 0 for ele in row] return similarity_matrix
def _phase2(self): """ Execute phase 2 of the SP region. This phase is used to compute the active columns. Note - This should only be called after phase 1 has been called and after the inhibition radius and neighborhood have been updated. """ # Shift the outputs self.y[:, 1:] = self.y[:, :-1] self.y[:, 0] = 0 # Calculate k # - For a column to be active its overlap must be at least as large # as the overlap of the k-th largest column in its neighborhood. k = self._get_num_cols() if self.global_inhibition: # The neighborhood is all columns, thus the set of active columns # is simply columns that have an overlap >= the k-th largest in the # entire region # Compute the winning column indexes if self.learn: # Randomly break ties ix = np.argpartition(-self.overlap[:, 0] - self.prng.uniform(.1, .2, self.ncolumns), k - 1)[:k] else: # Choose the same set of columns each time ix = np.argpartition(-self.overlap[:, 0], k - 1)[:k] # Set the active columns self.y[ix, 0] = self.overlap[ix, 0] > 0 else: # The neighborhood is bounded by the inhibition radius, therefore # each column's neighborhood must be considered for i in xrange(self.ncolumns): # Get the neighbors ix = np.where(self.neighbors[i])[0] # Compute the minimum top overlap if ix.shape[0] <= k: # Desired number of candidates is at or below the desired # activity level, so find the overall min m = max(bn.nanmin(self.overlap[ix, 0]), 1) else: # Desired number of candidates is above the desired # activity level, so find the k-th largest m = max(-np.partition(-self.overlap[ix, 0], k - 1)[k - 1], 1) # Set the column activity if self.overlap[i, 0] >= m: self.y[i, 0] = True
def cluster_select_func(self): num_selected = self.num_selected if(hasattr(self,'alternate_clustering_input')): inp = self.alternate_clustering_input # print("alternate used...") # print("alternate_clustering_input_shape: " + str(self.alternate_clustering_input.shape) + " net.input.shape: " + # str(self.input.shape)) else: inp = self.input if(hasattr(self,'do_weighted_euclidean')): self.distances = np.sum(self.centroids**2,1)[:,np.newaxis] \ - 2.0*np.dot(self.centroids*self.weights,inp) \ + np.dot(self.weights**2,inp**2) #temp_centroids = self.centroids/self.weights; #temp_distances = np.sum(temp_centroids**2,1)[:,np.newaxis] - 2*np.dot(temp_centroids,self.input) + \ # np.sum(self.input**2,0)[np.newaxis,:] #self.distances = temp_distances*(np.sum(self.weights**2,1)[:,np.newaxis]) #print("Distance error: " + str(np.sum(np.sum((temp_distances - self.distances)**2)))) print("Weighted Euclidean Distance") elif(hasattr(self,'do_cosinedistance')): self.distances = -np.dot(self.centroids,inp)/(np.sqrt(np.sum(self.centroids**2.,1)[:,np.newaxis]*np.sum(inp**2.,0)[np.newaxis,:])) else: #print("centroids shape: " + str(self.centroids.shape)) #print("inp shape: " + str(inp.shape)) #print("centroids dtype: " + str(self.centroids.dtype)) #print("inp dtype: " + str(inp.dtype)) self.distances = pairwise_distances(self.centroids,self.input.T) #self.distances = np.sum(self.centroids**2,1)[:,np.newaxis] - 2*np.dot(self.centroids,inp) + \ # np.sum(inp**2,0)[np.newaxis,:] distances_sorted = np.partition(self.distances,num_selected,axis=0) #distances_sorted = np.sort(self.distances,axis=0) #print("distances_sorted " + str(distances_sorted[0:10,0])) self.selected_neurons = self.distances > distances_sorted[num_selected,:] #keep track of this so we can count the number of times a centroid was selected self.saved_selected_neurons = np.copy(self.selected_neurons) #initialize selected count to 0 if(not hasattr(self,'selected_count')): self.selected_count = np.zeros(self.saved_selected_neurons.shape[0]) if(not hasattr(self,'eligibility_count')): self.eligibility_count = np.ones(self.saved_selected_neurons.shape[0]) self.centroids_prime = (np.dot(inp,(~self.selected_neurons).transpose())/ \ np.sum(~self.selected_neurons,1,dtype=np.float32)).transpose() self.centroids_prime[np.isnan(self.centroids_prime)] = self.centroids[np.isnan(self.centroids_prime)] if(hasattr(self,'do_weighted_euclidean')): self.centroids_prime = self.centroids_prime*self.weights; #save a copy of the full output -- it is useful to some tests self.full_output = np.copy(self.output) self.output[self.selected_neurons] = 0;
def min_payoffs(self): # Computes the min payoff by finding the required facilities that have # the lowest payoff when everyone uses them. This will fail if the # highest order term isn't negative. if self._min_payoffs is None: self._min_payoffs = self.num_players.astype(float) self._min_payoffs *= np.partition( self.facility_coefs.dot(self.num_players ** np.arange(3)), self.num_required - 1)[:self.num_required].sum() self._min_payoffs.setflags(write=False) return self._min_payoffs
def largestElement(x, n): """ Returns the n-th largest element of x :param x: :param n: :return: """ # The n-th largest element is the (N+1-n)th smallest element len = x.size desc_pos = (len-n) return np.partition(x, desc_pos)[desc_pos]
def fit(self, X, Y): ''' X: (n, d) array-like of samples Y: (n,) array-like of class labels ''' X, Y, num_classes, n, d = self._process_inputs(X, Y) tSb = np.zeros((d,d)) tSw = np.zeros((d,d)) for c in xrange(num_classes): Xc = X[Y==c] nc = Xc.shape[0] # classwise affinity matrix dist = pairwise_distances(Xc, metric='l2', squared=True) # distances to k-th nearest neighbor k = min(self.params['k'], nc-1) sigma = np.sqrt(np.partition(dist, k, axis=0)[:,k]) local_scale = np.outer(sigma, sigma) with np.errstate(divide='ignore', invalid='ignore'): A = np.exp(-dist/local_scale) A[local_scale==0] = 0 G = Xc.T.dot(A.sum(axis=0)[:,None] * Xc) - Xc.T.dot(A).dot(Xc) tSb += G/n + (1-nc/n)*Xc.T.dot(Xc) + _sum_outer(Xc)/n tSw += G/nc tSb -= _sum_outer(X)/n - tSw # symmetrize tSb += tSb.T tSb /= 2 tSw += tSw.T tSw /= 2 if self.params['dim'] == d: vals, vecs = scipy.linalg.eigh(tSb, tSw) else: vals, vecs = scipy.sparse.linalg.eigsh(tSb, k=self.params['dim'], M=tSw, which='LA') order = np.argsort(-vals)[:self.params['dim']] vals = vals[order] vecs = vecs[:,order] if self.params['metric'] == 'weighted': vecs *= np.sqrt(vals) elif self.params['metric'] == 'orthonormalized': vecs, _ = np.linalg.qr(vecs) self._transformer = vecs.T return self
def fast_abs_percentile(map, percentile=80): """ A fast version of the percentile of the absolute value. """ if hasattr(map, 'mask'): # Catter for masked arrays map = np.asarray(map[np.logical_not(map.mask)]) map = np.abs(map) map = map.ravel() index = int(map.size * .01 * percentile) if partition is not None: # Partial sort: faster than sort return partition(map, index)[index + 1] return map.sort()[index]
def hard_sparsity(self, d, sparsity, save_metadata=True): """Alters the given linear decomposition, applying a median filter. The filtering process is done in the time domain. The argument provided is destroyed. Args: d: LinearDecomposition object to filter. sparsity: maximum polyphony allowed. save_metadata: flag indicating whether the metadata should be computed. Default: True. Returns: Same decomposition given in arguments. """ if save_metadata: metadata = md.ObjectMetadata(a) else: metadata = None meta = md.Metadata(name="sparsity_level", sparsity=sparsity, activation_input=metadata, original_method=None) # Binarizes the data and adjusts the metadata A = None p = [] for k in d.data.right.keys(): if A is None: A = d.data.right[k] else: A = numpy.vstack((A, d.data.right[k])) p.append(k) for i in xrange(A.shape[1]): b = numpy.partition(A[:,i], sparsity-1)[sparsity-1] A[:,i] = A[:,i] * (A[:,i] >= b) for argk in xrange(len(p)): d.data.right[p[argk]] = A[argk,:] d.data.right[p[argk]].shape = (1, len(d.data.right[p[argk]])) d.metadata.right[p[argk]] = md.Metadata(method="sparsity_level", sparsity=sparsity, activation_input=metadata, original_method = d.metadata.right[k]) return d
def topk(a, k, axis, keepdims): """ Chunk and combine function of topk Extract the k largest elements from a on the given axis. If k is negative, extract the -k smallest elements instead. Note that, unlike in the parent function, the returned elements are not sorted internally. """ assert keepdims is True axis = axis[0] if abs(k) >= a.shape[axis]: return a a = np.partition(a, -k, axis=axis) k_slice = slice(-k, None) if k > 0 else slice(-k) return a[tuple(k_slice if i == axis else slice(None) for i in range(a.ndim))]
def topk(a, k, axis, keepdims): """Kernel of topk and argtopk. Extract the k largest elements from a on the given axis. If k is negative, extract the -k smallest elements instead. Note that, unlike in the parent function, the returned elements are not sorted internally. """ axis = axis[0] if abs(k) >= a.shape[axis]: return a a = np.partition(a, -k, axis=axis) # return a[-k:] if k>0 else a[:-k], on arbitrary axis return a[[ (slice(-k, None) if k > 0 else slice(None, -k)) if i == axis else slice(None) for i in range(a.ndim) ]]
def schedule(self, simulation, changed): clock = simulation.clock # mark freed hosts for task in changed: if task.kind == csimdag.TaskKind.TASK_KIND_COMP_SEQ and task.state == csimdag.TaskState.TASK_STATE_DONE: host = task.hosts[0] if host != self.master_host: task.hosts[0].data["is_free"] = True # schedule unscheduled ready tasks unscheduled = self.tasks[csimdag.TaskState.TASK_STATE_SCHEDULABLE]\ .by_func(lambda task: task.data["target_host"] is None) num_unscheduled = len(unscheduled) if num_unscheduled > 0: # build ECT matrix ECT = np.zeros((num_unscheduled, len(self.exec_hosts))) for t, task in enumerate(unscheduled): parents = [(p, p.parents[0].hosts[0]) for p in task.parents if p.kind == csimdag.TaskKind.TASK_KIND_COMM_E2E] for h, host in enumerate(self.exec_hosts): comm_times = [ p_comm.get_ecomt(p_host, host) for (p_comm, p_host) in parents ] ECT[t][h] = max(host.data["est"], clock) + task.get_eet( host) + (max(comm_times) if comm_times else 0.) # build schedule task_idx = np.arange(num_unscheduled) for _ in range(0, num_unscheduled): min_hosts = np.argmin(ECT, axis=1) min_times = ECT[np.arange(ECT.shape[0]), min_hosts] if ECT.shape[1] > 1: min2_times = np.partition(ECT, 1)[:, 1] # round sufferage values to eliminate the influence of floating point errors sufferages = np.round(min2_times - min_times, decimals=2) else: # use min time for a single host case sufferages = min_times possible_schedules = [] for i in range(0, len(task_idx)): best_host_idx = int(min_hosts[i]) best_ect = min_times[i] sufferage = sufferages[i] possible_schedules.append( (i, best_host_idx, best_ect, sufferage)) t, h, ect = self.batch_heuristic(possible_schedules) task = unscheduled[int(task_idx[t])] host = self.exec_hosts[h] # logging.info("%s -> %s" % (task.name, host.name)) task.data["target_host"] = host.name host.data["tasks"].append(task) task_time = ect - max(host.data["est"], clock) host.data["est"] = ect task_idx = np.delete(task_idx, t) ECT = np.delete(ECT, t, 0) ECT[:, h] += task_time for host in self.exec_hosts: if host.data["is_free"]: tasks = host.data["tasks"] if len(tasks) > 0: task = tasks.pop(0) task.schedule(host) host.data["is_free"] = False
def test(db, split, testiter, debug=False, suffix=None): result_dir = system_configs.result_dir result_dir = os.path.join(result_dir, str(testiter), split) class_name = [] for i in range(1, len(db._coco.cats)): # if db._coco.cats[i] is None: # continue # else: ind = db._cat_ids[i] class_name.append(db._coco.cats[ind]['name']) if suffix is not None: result_dir = os.path.join(result_dir, suffix) make_dirs([result_dir]) test_iter = system_configs.max_iter if testiter is None else testiter print("loading parameters at iteration: {}".format(test_iter)) print("building neural network...") nnet = NetworkFactory(db) print("loading parameters...") nnet.load_params(test_iter) # test_file = "test.{}".format(db.data) # testing = importlib.import_module(test_file).testing nnet.cuda() nnet.eval_mode() debug_dir = os.path.join(result_dir, "debug") if not os.path.exists(debug_dir): os.makedirs(debug_dir) if db.split != "trainval": db_inds = db.db_inds[:100] if debug else db.db_inds else: db_inds = db.db_inds[:100] if debug else db.db_inds[:5000] K = db.configs["top_k"] ae_threshold = db.configs["ae_threshold"] nms_kernel = db.configs["nms_kernel"] scales = db.configs["test_scales"] weight_exp = db.configs["weight_exp"] merge_bbox = db.configs["merge_bbox"] categories = db.configs["categories"] nms_threshold = db.configs["nms_threshold"] max_per_image = db.configs["max_per_image"] nms_algorithm = { "nms": 0, "linear_soft_nms": 1, "exp_soft_nms": 2 }[db.configs["nms_algorithm"]] img_name = os.listdir(db._image_dir) for i in range(0, len(img_name)): top_bboxes = {} # for ind in tqdm(range(0, num_images), ncols=80, desc="locating kps"): db_ind = i + 1 # image_id = db.image_ids(db_ind) image_id = img_name[i] image_file = db._image_dir + '/' + img_name[i] image = cv2.imread(image_file) height, width = image.shape[0:2] detections = [] for scale in scales: new_height = int(height * scale) new_width = int(width * scale) new_center = np.array([new_height // 2, new_width // 2]) inp_height = new_height | 127 inp_width = new_width | 127 images = np.zeros((1, 3, inp_height, inp_width), dtype=np.float32) ratios = np.zeros((1, 2), dtype=np.float32) borders = np.zeros((1, 4), dtype=np.float32) sizes = np.zeros((1, 2), dtype=np.float32) out_height, out_width = (inp_height + 1) // 4, (inp_width + 1) // 4 height_ratio = out_height / inp_height width_ratio = out_width / inp_width resized_image = cv2.resize(image, (new_width, new_height)) resized_image, border, offset = crop_image(resized_image, new_center, [inp_height, inp_width]) resized_image = resized_image / 255. normalize_(resized_image, db.mean, db.std) images[0] = resized_image.transpose((2, 0, 1)) borders[0] = border sizes[0] = [int(height * scale), int(width * scale)] ratios[0] = [height_ratio, width_ratio] images = np.concatenate((images, images[:, :, :, ::-1]), axis=0) images = torch.from_numpy(images) dets = kp_decode(nnet, images, K, ae_threshold=ae_threshold, kernel=nms_kernel) dets = dets.reshape(2, -1, 8) dets[1, :, [0, 2]] = out_width - dets[1, :, [2, 0]] dets = dets.reshape(1, -1, 8) _rescale_dets(dets, ratios, borders, sizes) dets[:, :, 0:4] /= scale detections.append(dets) detections = np.concatenate(detections, axis=1) classes = detections[..., -1] classes = classes[0] detections = detections[0] # reject detections with negative scores keep_inds = (detections[:, 4] > -1) detections = detections[keep_inds] classes = classes[keep_inds] top_bboxes[image_id] = {} for j in range(categories): keep_inds = (classes == j) top_bboxes[image_id][j + 1] = detections[keep_inds][:, 0:7].astype(np.float32) if merge_bbox: soft_nms_merge(top_bboxes[image_id][j + 1], Nt=nms_threshold, method=nms_algorithm, weight_exp=weight_exp) else: soft_nms(top_bboxes[image_id][j + 1], Nt=nms_threshold, method=nms_algorithm) top_bboxes[image_id][j + 1] = top_bboxes[image_id][j + 1][:, 0:5] scores = np.hstack([ top_bboxes[image_id][j][:, -1] for j in range(1, categories + 1) ]) if len(scores) > max_per_image: kth = len(scores) - max_per_image thresh = np.partition(scores, kth)[kth] for j in range(1, categories + 1): keep_inds = (top_bboxes[image_id][j][:, -1] >= thresh) top_bboxes[image_id][j] = top_bboxes[image_id][j][keep_inds] # result_json = os.path.join(result_dir, "results.json") detections = db.convert_to_list(top_bboxes) print('demo for {}'.format(image_id)) img = cv2.imread(image_file) box = [] if detections is not None: for i in range(len(detections)): name = db._coco.cats[detections[i][1]]['name'] #db._coco.cats[ind]['name'] confi = detections[i][-1] if confi <0.3: continue for j in range(0, 4): box.append(detections[i][j + 2]) cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), (0, 255, 255), 1) # cv2.putText(img, name[0] + ' ' + '{:.3f}'.format(confi), (int(box[0]), int(box[1] - 10)), # cv2.FONT_ITALIC, 1, (0, 0, 255), 1) while (box): box.pop(-1) cv2.imshow('Detecting image...', img) # timer.total_time = 0 if cv2.waitKey(3000) & 0xFF == ord('q'): break print(detections)
def main(): logger = create_logger(save_dir=cfg.log_dir) print = logger.info print(cfg) cfg.device = torch.device('cuda') torch.backends.cudnn.benchmark = False max_per_image = 100 Dataset_eval = COCO_eval if cfg.dataset == 'coco' else PascalVOC_eval dataset = Dataset_eval(cfg.data_dir, split='val', img_size=cfg.img_size, test_scales=cfg.test_scales, test_flip=cfg.test_flip) data_loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False, num_workers=1, pin_memory=False, collate_fn=dataset.collate_fn) print('Creating model...') if 'hourglass' in cfg.arch: model = get_hourglass[cfg.arch] elif 'resdcn' in cfg.arch: model = get_pose_net(num_layers=int(cfg.arch.split('_')[-1]), num_classes=dataset.num_classes) else: raise NotImplementedError model = load_model(model, cfg.pretrain_dir) model = model.to(cfg.device) model.eval() results = {} with torch.no_grad(): for inputs in data_loader: img_id, inputs = inputs[0] detections = [] for scale in inputs: inputs[scale]['image'] = inputs[scale]['image'].to(cfg.device) output = model(inputs[scale]['image'])[-1] dets = ctdet_decode(*output, K=cfg.test_topk) dets = dets.detach().cpu().numpy().reshape( 1, -1, dets.shape[2])[0] top_preds = {} dets[:, :2] = transform_preds( dets[:, 0:2], inputs[scale]['center'], inputs[scale]['scale'], (inputs[scale]['fmap_w'], inputs[scale]['fmap_h'])) dets[:, 2:4] = transform_preds( dets[:, 2:4], inputs[scale]['center'], inputs[scale]['scale'], (inputs[scale]['fmap_w'], inputs[scale]['fmap_h'])) cls = dets[:, -1] for j in range(dataset.num_classes): inds = (cls == j) top_preds[j + 1] = dets[inds, :5].astype(np.float32) top_preds[j + 1][:, :4] /= scale detections.append(top_preds) bbox_and_scores = {} for j in range(1, dataset.num_classes + 1): bbox_and_scores[j] = np.concatenate([d[j] for d in detections], axis=0) if len(dataset.test_scales) > 1: soft_nms(bbox_and_scores[j], Nt=0.5, method=2) scores = np.hstack([ bbox_and_scores[j][:, 4] for j in range(1, dataset.num_classes + 1) ]) if len(scores) > max_per_image: kth = len(scores) - max_per_image thresh = np.partition(scores, kth)[kth] for j in range(1, dataset.num_classes + 1): keep_inds = (bbox_and_scores[j][:, 4] >= thresh) bbox_and_scores[j] = bbox_and_scores[j][keep_inds] results[img_id] = bbox_and_scores eval_results = dataset.run_eval(results, cfg.ckpt_dir) print(eval_results)
import numpy as np # numpy裡面的排序, default: np.sort為Quick Sort(Time: N logN) x = np.array([2, 4, 3, 1, 5]) # print("np.sort(x): ", np.sort(x)) # 傳回排序後的陣列, 但不會改變原本的內容 # x.sort() # 會改變原本的陣列內容 # print("x: ", x) # 另一種排序為np.argsort() 傳會被排序過後元素的索引值 i = np.argsort(x) # print("index: ", i) # 沿著行或列排序 增加axis參數即可 rand = np.random.RandomState(1) array = rand.randint(0, 10, (4, 6)) print("Array: \n", array) # print("row sorted: \n", np.sort(array, axis = 0)) # 行排序 # print("column sorted: \n", np.sort(array, axis = 1)) # 列排序 # 分區(Partitioning): 將陣列中最小的K個值放入左側 np.partition(array, K) # x = np.array([7, 2, 3, 1, 6, 5, 4]) # print(np.partition(x, 2)) # 排列順序隨意 (BTW: 只排列K個數值, 則被分開的數值順序隨意) print(np.partition(array, 2, axis=1)) # 每一列中的兩個最小值會被放在左側
def compute_2nd_bmu(self, inp): #computes distances using euklid norm dist = np.linalg.norm(self.weights-inp, axis=2) return np.unravel_index(np.partition(dist,2)[2], dist.shape)
def main(): # Parse command line arguments parser = argparse.ArgumentParser(description='Map the source embeddings into the target embedding space') parser.add_argument('src_input', help='the input source embeddings') parser.add_argument('trg_input', help='the input target embeddings') parser.add_argument('--model_path', default=None, type=str, help='directory to save the model') parser.add_argument('--geomm_embeddings_path', default=None, type=str, help='directory to save the output GeoMM latent space embeddings. The output embeddings are normalized.') parser.add_argument('--encoding', default='utf-8', help='the character encoding for input/output (defaults to utf-8)') parser.add_argument('--max_vocab', default=0,type=int, help='Maximum vocabulary to be loaded, 0 allows complete vocabulary') parser.add_argument('--verbose', default=0,type=int, help='Verbose') mapping_group = parser.add_argument_group('mapping arguments', 'Basic embedding mapping arguments') mapping_group.add_argument('-dtrain', '--dictionary_train', default=sys.stdin.fileno(), help='the training dictionary file (defaults to stdin)') mapping_group.add_argument('-dtest', '--dictionary_test', default=sys.stdin.fileno(), help='the test dictionary file (defaults to stdin)') mapping_group.add_argument('--normalize', choices=['unit', 'center', 'unitdim', 'centeremb'], nargs='*', default=[], help='the normalization actions to perform in order') geomm_group = parser.add_argument_group('GeoMM arguments', 'Arguments for GeoMM method') geomm_group.add_argument('--l2_reg', type=float,default=1e2, help='Lambda for L2 Regularization') geomm_group.add_argument('--max_opt_time', type=int,default=5000, help='Maximum time limit for optimization in seconds') geomm_group.add_argument('--max_opt_iter', type=int,default=150, help='Maximum number of iterations for optimization') eval_group = parser.add_argument_group('evaluation arguments', 'Arguments for evaluation') eval_group.add_argument('--normalize_eval', action='store_true', help='Normalize the embeddings at test time') eval_group.add_argument('--eval_batch_size', type=int,default=1000, help='Batch size for evaluation') eval_group.add_argument('--csls_neighbourhood', type=int,default=10, help='Neighbourhood size for CSLS') args = parser.parse_args() BATCH_SIZE = args.eval_batch_size ## Logging #method_name = os.path.join('logs','geomm') #directory = os.path.join(os.path.join(os.getcwd(),method_name), datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) #if not os.path.exists(directory): # os.makedirs(directory) #log_file_name, file_extension = os.path.splitext(os.path.basename(args.dictionary_train)) #log_file_name = log_file_name + '.log' #class Logger(object): # def __init__(self): # self.terminal = sys.stdout # self.log = open(os.path.join(directory,log_file_name), "a") # def write(self, message): # self.terminal.write(message) # self.log.write(message) # def flush(self): # #this flush method is needed for python 3 compatibility. # #this handles the flush command by doing nothing. # #you might want to specify some extra behavior here. # pass #sys.stdout = Logger() if args.verbose: print('Current arguments: {0}'.format(args)) dtype = 'float32' if args.verbose: print('Loading train data...') # Read input embeddings srcfile = open(args.src_input, encoding=args.encoding, errors='surrogateescape') trgfile = open(args.trg_input, encoding=args.encoding, errors='surrogateescape') src_words, x = embeddings.read(srcfile,max_voc=args.max_vocab, dtype=dtype) trg_words, z = embeddings.read(trgfile,max_voc=args.max_vocab, dtype=dtype) # Build word to index map src_word2ind = {word: i for i, word in enumerate(src_words)} trg_word2ind = {word: i for i, word in enumerate(trg_words)} # Build training dictionary noov=0 src_indices = [] trg_indices = [] f = open(args.dictionary_train, encoding=args.encoding, errors='surrogateescape') for line in f: src,trg = line.split() if args.max_vocab: src=src.lower() trg=trg.lower() try: src_ind = src_word2ind[src] trg_ind = trg_word2ind[trg] src_indices.append(src_ind) trg_indices.append(trg_ind) except KeyError: noov+=1 if args.verbose: print('WARNING: OOV dictionary entry ({0} - {1})'.format(src, trg)) #, file=sys.stderr f.close() if args.verbose: print('Number of training pairs having at least one OOV: {}'.format(noov)) src_indices = src_indices trg_indices = trg_indices if args.verbose: print('Normalizing embeddings...') # STEP 0: Normalization for action in args.normalize: if action == 'unit': x = embeddings.length_normalize(x) z = embeddings.length_normalize(z) elif action == 'center': x = embeddings.mean_center(x) z = embeddings.mean_center(z) elif action == 'unitdim': x = embeddings.length_normalize_dimensionwise(x) z = embeddings.length_normalize_dimensionwise(z) elif action == 'centeremb': x = embeddings.mean_center_embeddingwise(x) z = embeddings.mean_center_embeddingwise(z) # Step 1: Optimization if args.verbose: print('Beginning Optimization') start_time = time.time() x_count = len(set(src_indices)) z_count = len(set(trg_indices)) A = np.zeros((x_count,z_count)) # Creating dictionary matrix from training set map_dict_src={} map_dict_trg={} I=0 uniq_src=[] uniq_trg=[] for i in range(len(src_indices)): if src_indices[i] not in map_dict_src.keys(): map_dict_src[src_indices[i]]=I I+=1 uniq_src.append(src_indices[i]) J=0 for j in range(len(trg_indices)): if trg_indices[j] not in map_dict_trg.keys(): map_dict_trg[trg_indices[j]]=J J+=1 uniq_trg.append(trg_indices[j]) for i in range(len(src_indices)): A[map_dict_src[src_indices[i]],map_dict_trg[trg_indices[i]]]=1 np.random.seed(0) Lambda=args.l2_reg U1 = TT.matrix() U2 = TT.matrix() B = TT.matrix() cost = TT.sum(((shared(x[uniq_src]).dot(U1.dot(B.dot(U2.T)))).dot(shared(z[uniq_trg]).T)-A)**2) + 0.5*Lambda*(TT.sum(B**2)) solver = ConjugateGradient(maxtime=args.max_opt_time,maxiter=args.max_opt_iter) manifold =Product([Stiefel(x.shape[1], x.shape[1]),Stiefel(z.shape[1], x.shape[1]),PositiveDefinite(x.shape[1])]) problem = Problem(manifold=manifold, cost=cost, arg=[U1,U2,B], verbosity=3) wopt = solver.solve(problem) w= wopt U1 = w[0] U2 = w[1] B = w[2] ### Save the models if requested if args.model_path is not None: os.makedirs(args.model_path,exist_ok=True) np.savetxt('{}/U_src.csv'.format(args.model_path),U1) np.savetxt('{}/U_tgt.csv'.format(args.model_path),U2) np.savetxt('{}/B.csv'.format(args.model_path),B) # Step 2: Transformation xw = x.dot(U1).dot(scipy.linalg.sqrtm(B)) zw = z.dot(U2).dot(scipy.linalg.sqrtm(B)) end_time = time.time() if args.verbose: print('Completed training in {0:.2f} seconds'.format(end_time-start_time)) gc.collect() ### Save the GeoMM embeddings if requested xw_n = embeddings.length_normalize(xw) zw_n = embeddings.length_normalize(zw) if args.geomm_embeddings_path is not None: os.makedirs(args.geomm_embeddings_path,exist_ok=True) out_emb_fname=os.path.join(args.geomm_embeddings_path,'src.vec') with open(out_emb_fname,'w',encoding=args.encoding) as outfile: embeddings.write(src_words,xw_n,outfile) out_emb_fname=os.path.join(args.geomm_embeddings_path,'trg.vec') with open(out_emb_fname,'w',encoding=args.encoding) as outfile: embeddings.write(trg_words,zw_n,outfile) # Step 3: Evaluation if args.normalize_eval: xw = xw_n zw = zw_n X = xw[src_indices] Z = zw[trg_indices] # Loading test dictionary f = open(args.dictionary_test, encoding=args.encoding, errors='surrogateescape') src2trg = collections.defaultdict(set) trg2src = collections.defaultdict(set) oov = set() vocab = set() for line in f: src, trg = line.split() if args.max_vocab: src=src.lower() trg=trg.lower() try: src_ind = src_word2ind[src] trg_ind = trg_word2ind[trg] src2trg[src_ind].add(trg_ind) trg2src[trg_ind].add(src_ind) vocab.add(src) except KeyError: oov.add(src) src = list(src2trg.keys()) trgt = list(trg2src.keys()) oov -= vocab # If one of the translation options is in the vocabulary, then the entry is not an oov coverage = len(src2trg) / (len(src2trg) + len(oov)) f.close() translation = collections.defaultdict(int) translation5 = collections.defaultdict(list) translation10 = collections.defaultdict(list) ### compute nearest neigbours of x in z t=time.time() nbrhood_x=np.zeros(xw.shape[0]) for i in range(0, len(src), BATCH_SIZE): j = min(i + BATCH_SIZE, len(src)) similarities = xw[src[i:j]].dot(zw.T) similarities_x = -1*np.partition(-1*similarities,args.csls_neighbourhood-1 ,axis=1) nbrhood_x[src[i:j]]=np.mean(similarities_x[:,:args.csls_neighbourhood],axis=1) ### compute nearest neigbours of z in x (GPU version) nbrhood_z=np.zeros(zw.shape[0]) with cp.cuda.Device(1): nbrhood_z2=cp.zeros(zw.shape[0]) batch_num=1 for i in range(0, zw.shape[0], BATCH_SIZE): j = min(i + BATCH_SIZE, zw.shape[0]) similarities = -1*cp.partition(-1*cp.dot(cp.asarray(zw[i:j]),cp.transpose(cp.asarray(xw))),args.csls_neighbourhood-1 ,axis=1)[:,:args.csls_neighbourhood] nbrhood_z2[i:j]=(cp.mean(similarities[:,:args.csls_neighbourhood],axis=1)) batch_num+=1 nbrhood_z=cp.asnumpy(nbrhood_z2) #### compute nearest neigbours of z in x (CPU version) #nbrhood_z=np.zeros(zw.shape[0]) #for i in range(0, len(zw.shape[0]), BATCH_SIZE): # j = min(i + BATCH_SIZE, len(zw.shape[0])) # similarities = zw[i:j].dot(xw.T) # similarities_z = -1*np.partition(-1*similarities,args.csls_neighbourhood-1 ,axis=1) # nbrhood_z[i:j]=np.mean(similarities_z[:,:args.csls_neighbourhood],axis=1) #### find translation #for i in range(0, len(src), BATCH_SIZE): # j = min(i + BATCH_SIZE, len(src)) # similarities = xw[src[i:j]].dot(zw.T) # similarities = np.transpose(np.transpose(2*similarities) - nbrhood_x[src[i:j]]) - nbrhood_z # nn = similarities.argmax(axis=1).tolist() # similarities = np.argsort((similarities),axis=1) # nn5 = (similarities[:,-5:]) # nn10 = (similarities[:,-10:]) # for k in range(j-i): # translation[src[i+k]] = nn[k] # translation5[src[i+k]] = nn5[k] # translation10[src[i+k]] = nn10[k] #if args.geomm_embeddings_path is not None: # delim=',' # os.makedirs(args.geomm_embeddings_path,exist_ok=True) # translations_fname=os.path.join(args.geomm_embeddings_path,'translations.csv') # with open(translations_fname,'w',encoding=args.encoding) as translations_file: # for src_id in src: # src_word = src_words[src_id] # all_trg_words = [ trg_words[trg_id] for trg_id in src2trg[src_id] ] # trgout_words = [ trg_words[j] for j in translation10[src_id] ] # ss = list(nn10[src_id,:]) # # p1 = ':'.join(all_trg_words) # p2 = delim.join( [ '{}{}{}'.format(w,delim,s) for w,s in zip(trgout_words,ss) ] ) # translations_file.write( '{s}{delim}{p1}{delim}{p2}\n'.format(s=src_word, delim=delim, p1=p1, p2=p2) ) ### find translation (and write to file if output requested) delim=',' translations_file =None if args.geomm_embeddings_path is not None: os.makedirs(args.geomm_embeddings_path,exist_ok=True) translations_fname=os.path.join(args.geomm_embeddings_path,'translations.csv') translations_file = open(translations_fname,'w',encoding=args.encoding) for i in range(0, len(src), BATCH_SIZE): j = min(i + BATCH_SIZE, len(src)) similarities = xw[src[i:j]].dot(zw.T) similarities = np.transpose(np.transpose(2*similarities) - nbrhood_x[src[i:j]]) - nbrhood_z nn = similarities.argmax(axis=1).tolist() similarities = np.argsort((similarities),axis=1) nn5 = (similarities[:,-5:]) nn10 = (similarities[:,-10:]) for k in range(j-i): translation[src[i+k]] = nn[k] translation5[src[i+k]] = nn5[k] translation10[src[i+k]] = nn10[k] if args.geomm_embeddings_path is not None: src_id=src[i+k] src_word = src_words[src_id] all_trg_words = [ trg_words[trg_id] for trg_id in src2trg[src_id] ] trgout_words = [ trg_words[j] for j in translation10[src_id] ] #ss = list(nn10[src_id,:]) p1 = ':'.join(all_trg_words) p2 = ':'.join(trgout_words) #p2 = delim.join( [ '{}{}{}'.format(w,delim,s) for w,s in zip(trgout_words,ss) ] ) translations_file.write( '{s}{delim}{p1}{delim}{p2}\n'.format(s=src_word, p1=p1, p2=p2, delim=delim) ) if args.geomm_embeddings_path is not None: translations_file.close() accuracy = np.mean([1 if translation[i] in src2trg[i] else 0 for i in src]) mean=0 for i in src: for k in translation5[i]: if k in src2trg[i]: mean+=1 break mean/=len(src) accuracy5 = mean mean=0 for i in src: for k in translation10[i]: if k in src2trg[i]: mean+=1 break mean/=len(src) accuracy10 = mean print('Coverage:{0:7.2%} Accuracy:{1:7.2%} Accuracy(Top 5):{2:7.2%} Accuracy(Top 10):{3:7.2%}'.format(coverage, accuracy, accuracy5, accuracy10))
def calculate_nearest_k(sim_mat, k): sorted_mat = -np.partition(-sim_mat, k + 1, axis=1) # -np.sort(-sim_mat1) nearest_k = sorted_mat[:, 0:k] return np.mean(nearest_k, axis=1, keepdims=True)
def val_map(epoch): print_log('\n Val@Epoch: %d' % epoch) model.eval() torch.cuda.empty_cache() max_per_image = 100 results = {} speed_list = [] with torch.no_grad(): for inputs in val_loader: img_id, inputs = inputs[0] start_image_time = time.time() segmentations = [] for scale in inputs: inputs[scale]['image'] = inputs[scale]['image'].to( cfg.device) # hmap, regs, w_h_, offsets, active_codes, inactive_codes, active_cls = model(inputs[scale]['image'])[-1] hmap, regs, w_h_, offsets, _, _, active_codes, active_cls = model( inputs[scale]['image'])[-1] active_cls = torch.sigmoid(active_cls) > 0.5 active_mask = active_cls * 1 # inactive_mask = (~ active_cls) * 1 codes = active_codes * active_mask # + inactive_codes * inactive_mask output = [hmap, regs, w_h_, codes, offsets] segms = ctsegm_scale_decode( *output, torch.from_numpy(dictionary.astype(np.float32)).to( cfg.device), K=cfg.test_topk) # segms = ctsegm_amodal_cmm_decode(*output, # torch.from_numpy(dictionary.astype(np.float32)).to(cfg.device), # K=cfg.test_topk) segms = segms.detach().cpu().numpy().reshape( 1, -1, segms.shape[2])[0] top_preds = {} for j in range(cfg.n_vertices): segms[:, 2 * j:2 * j + 2] = transform_preds( segms[:, 2 * j:2 * j + 2], inputs[scale]['center'], inputs[scale]['scale'], (inputs[scale]['fmap_w'], inputs[scale]['fmap_h'])) segms[:, cfg.n_vertices * 2:cfg.n_vertices * 2 + 2] = transform_preds( segms[:, cfg.n_vertices * 2:cfg.n_vertices * 2 + 2], inputs[scale]['center'], inputs[scale]['scale'], (inputs[scale]['fmap_w'], inputs[scale]['fmap_h'])) segms[:, cfg.n_vertices * 2 + 2:cfg.n_vertices * 2 + 4] = transform_preds( segms[:, cfg.n_vertices * 2 + 2:cfg.n_vertices * 2 + 4], inputs[scale]['center'], inputs[scale]['scale'], (inputs[scale]['fmap_w'], inputs[scale]['fmap_h'])) clses = segms[:, -1] for j in range(val_dataset.num_classes): inds = (clses == j) top_preds[j + 1] = segms[inds, :cfg.n_vertices * 2 + 5].astype(np.float32) top_preds[j + 1][:, :cfg.n_vertices * 2 + 4] /= scale segmentations.append(top_preds) end_image_time = time.time() segms_and_scores = { j: np.concatenate([d[j] for d in segmentations], axis=0) for j in range(1, val_dataset.num_classes + 1) } scores = np.hstack([ segms_and_scores[j][:, cfg.n_vertices * 2 + 4] for j in range(1, val_dataset.num_classes + 1) ]) if len(scores) > max_per_image: kth = len(scores) - max_per_image thresh = np.partition(scores, kth)[kth] for j in range(1, val_dataset.num_classes + 1): keep_inds = ( segms_and_scores[j][:, cfg.n_vertices * 2 + 4] >= thresh) segms_and_scores[j] = segms_and_scores[j][keep_inds] results[img_id] = segms_and_scores speed_list.append(end_image_time - start_image_time) eval_results = val_dataset.run_eval(results, save_dir=cfg.ckpt_dir) print_log(eval_results) summary_writer.add_scalar('val_mAP/mAP', eval_results[0], epoch) print_log('Average speed on val set:{:.2f}'.format( 1. / np.mean(speed_list))) return eval_results[0]
profit = profit + CADCHF10080[5][tick+1] - CADCHF10080[5][tick] if( (CADCHF10080[5][tick] < CADCHF10080[5][tick+1] ) and (Decision == 0) ): profit = profit + (CADCHF10080[5][tick+1] - CADCHF10080[5][tick])*-1 if( (CADCHF10080[5][tick] > CADCHF10080[5][tick+1] ) and (Decision == 1) ): profit = profit + (CADCHF10080[5][tick+1] - CADCHF10080[5][tick]) if( (CADCHF10080[5][tick] > CADCHF10080[5][tick+1] ) and (Decision == 0) ): profit = profit + (CADCHF10080[5][tick+1] - CADCHF10080[5][tick])*-1 PopulationProfitArray[n] = profit for B in range(1,TopSpecimenNumber+1): BestSpecimens[B-1] = np.partition(PopulationProfitArray.flatten(), -B)[-B] #Encontrar el X más alto. t, = np.where(PopulationProfitArray == BestSpecimens[B-1]) if(t.size > 1): t = t[0] BestSpecimensArray[B-1] = PopulationArray[t] for B in range(0,TopSpecimenNumber): if(TopSpecimenNumber < TopSpecimenNumber/2): SN = random.randint(0,TopSpecimenNumber/2-1) BestSpecimensSonsArray[B] = BestSpecimensArray[SN] for O in range(0,5): SN = random.randint(TopSpecimenNumber/2,TopSpecimenNumber-1) Z = random.randint(0,NeuronAmmount-1) BestSpecimensSonsArray[B][Z] = BestSpecimensArray[SN][Z]
t = time.time() # load data mat_contents = sio.loadmat(data_path + '/' + train_mat_names[id]) adj = mat_contents['adj'] yy = mat_contents['indset_label'].transpose() nn, nr = yy.shape # number of nodes & results # y_train = yy[:,np.random.randint(0,nr)] # y_train = np.concatenate([1-np.expand_dims(y_train,axis=1), np.expand_dims(y_train,axis=1)],axis=1) # sample an intermediate graph yyr = yy[:, np.random.randint(0, nr)] yyr_num = np.sum(yyr) yyr_down_num = np.random.randint(0, yyr_num) if yyr_down_num > 0: yyr_down_prob = yyr * np.random.random_sample(yyr.shape) yyr_down_flag = (yyr_down_prob >= np.partition( yyr_down_prob, -yyr_down_num)[-yyr_down_num]) tmp = np.sum(adj[yyr_down_flag, :], axis=0) > 0 tmp = np.asarray(tmp).reshape(-1) yyr_down_flag[tmp] = 1 adj_down = adj[yyr_down_flag == 0, :] adj_down = adj_down[:, yyr_down_flag == 0] yyr_down = yyr[yyr_down_flag == 0] adj = adj_down nn = yyr_down.shape[0] yyr = yyr_down y_train = np.concatenate( [1 - np.expand_dims(yyr, axis=1), np.expand_dims(yyr, axis=1)], axis=1)
def SinglMatch(ps1, w, md, edges1C, gt, fx_C, fy_C, edges0C, gt_0, fx_C0, fy_C0, mask_b_C0): pbar = tqdm(total=1, position=0, desc="RECC ") max_dist = int((md) / ps1) contours, hierarchy = cv2.findContours((1 - mask_b_C0).astype(np.uint8), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) contour_sizes = [(cv2.contourArea(contour), contour) for contour in contours] biggest_contour = max(contour_sizes, key=lambda x: x[0])[1] polygon = Polygon(np.array(biggest_contour[:, 0])) x, y = polygon.exterior.xy b1 = int(min(x)) b2 = int(max(x)) b3 = int(min(y)) b4 = int(max(y)) target = edges0C[b3:b4, b1:b2] sum_target = np.sum(target) xd = int(round((b4 - b3) / 2)) yd = int(round((b2 - b1) / 2)) buffer = 2 * max(xd, yd) edges1Ca = np.zeros( (edges1C.shape[0] + buffer * 2, edges1C.shape[1] + 2 * buffer)) edges1Ca[buffer:-buffer, buffer:-buffer] = edges1C x_i_0 = b3 + xd y_i_0 = b1 + yd lat_0 = gt_0[3] + gt_0[5] * x_i_0 * fx_C0 lon_0 = gt_0[0] + gt_0[1] * y_i_0 * fy_C0 x_i_0_og = int(round((lat_0 - gt[3]) / (gt[5] * fx_C))) y_i_0_og = int(round((lon_0 - gt[0]) / (gt[1] * fy_C))) search_wide = np.zeros((2 * (max_dist + yd), 2 * (max_dist + xd))) search_wide = edges1Ca[buffer + x_i_0_og - max_dist - xd:buffer + x_i_0_og + max_dist + xd, buffer + y_i_0_og - max_dist - yd:buffer + y_i_0_og + max_dist + yd] RECC_total = np.zeros(edges1Ca.shape) RECC_over = np.zeros(edges1Ca.shape) RECC_over.fill(np.NaN) circle2 = np.zeros((2 * max_dist, 2 * max_dist)) for x in range(circle2.shape[0]): for y in range(circle2.shape[1]): if (x - max_dist)**2 + (y - max_dist)**2 < max_dist**2: circle2[x, y] = 1 circle2[circle2 == 0] = np.NaN sum_patch = cv2.filter2D(search_wide, -1, np.ones(target.shape)) numerator = cv2.filter2D(search_wide, -1, target) RECC_wide = numerator / (sum_patch + sum_target) RECC_area = RECC_wide[xd:-xd, yd:-yd] * circle2 RECC_total.fill(np.NaN) RECC_total[x_i_0_og - max_dist:x_i_0_og + max_dist, y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area max_one = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(), -1)[-1] y_i = np.where(RECC_total >= max_one)[1][0] x_i = np.where(RECC_total >= max_one)[0][0] x_offset = (x_i - x_i_0_og) * ps1 y_offset = (y_i - y_i_0_og) * ps1 o_x = x_i o_y = y_i t_x = x_i_0 t_y = y_i_0 pbar.update(1) return x_offset, y_offset, o_x, o_y, t_x, t_y
def constraints_s(self, doThr, K, doRw, nnegS, Sfft=None, Sfft_det=None, S=None, stds=None, Swtrw=None, oracle=False): """Apply the constraints on the sources (thresholding in the wavelet domain and possibly a projection on the positive orthant. The input data are Sfft. The output data are S, as well as Sfft and Sfft_det. Parameters ---------- doThr : bool perform thresholding K: float L0 support of the sources doRw: bool do reweighting nnegS: bool apply non-negativity constraint on the sources Sfft: np.ndarray (n,p) complex array, estimated sources in Fourier domain (in-place update, default: self.Sfft) Sfft_det: np.ndarray (n,p) complex array, estimated sources with only the detail scales in Fourier domain (in-place update, default: self.Sfft_det) S: np.ndarray (n,p) float array, estimated sources (in-place update, default: self.S) stds: np.ndarray (n,nscales) float array, std of the noise in the source space, per detail scale (default: mad or analytical calculation) Swtrw: np.ndarray (n,p,nscales) float array, sources in the wavelet domain of previous iteration (default: self.Swtrw) oracle: bool perform an oracle update (using the ground-truth A and S) Returns ------- int error code """ if Sfft is None: Sfft = self.Sfft if Sfft_det is None: Sfft_det = self.Sfft_det if S is None: S = self.S if Swtrw is None: if not oracle: Swtrw = self.Swtrw else: Swtrw = self.S0wt if not doThr: S[:] = fftt.ifft(Sfft) if not nnegS: # nothing more to do Sfft_det[:] = fftt.fftprod(Sfft, 1 - self.wt_filters[:, -1]) return 0 else: if self.verb >= 3: print("Maximal L0 norm of the sources: %.1f %%" % (K * 100)) Swt = fftt.wt_trans(Sfft, nscales=self.nscales, fft_in=True) # Thresholding for i in range(self.n): for j in range(self.nscales): Swtij = Swt[i, :, j] Swtrwij = Swtrw[i, :, j] if stds is not None: std = stds[i, j] elif self.useMad: std = utils.mad(Swtij, M=self.M) else: std = self.nStd * np.sqrt( np.sum(self.invOpSp[:, i] * self.wt_filters[:, j]**2) / self.p) thrd = self.k * std # If oracle, threshold Swtrw if oracle and self.L1 and doRw: Swtrwij = ( Swtrwij - np.sign(Swtrwij) * (thrd - np.sqrt( np.abs( (Swtrwij - thrd * np.sign(Swtrwij)) * (3 * thrd * np.sign(Swtrwij) + Swtrwij)))) ) / 2 * (np.abs(Swtrwij) >= thrd) # Support based threshold if K != 1: npix = np.sum(abs(Swtij) - thrd > 0) Kval = np.maximum(np.int(K * npix), 5) thrd = np.partition(abs(Swtij), self.p - Kval)[self.p - Kval] if self.verb == 4 and i == 0: print("Threshold of source %i at scale %i: %.5e" % (i + 1, j + 1, thrd)) elif self.verb == 5: print("Threshold of source %i at scale %i: %.5e" % (i + 1, j + 1, thrd)) # Adapt the threshold if reweighing demanded if doRw and self.L1: thrd = thrd / (np.abs(Swtrwij) / np.maximum(1e-20, self.k * std) + 1) else: thrd = thrd * np.ones(self.p) # Apply the threshold Swtij[(abs(Swtij) < thrd)] = 0 if self.L1: indNZ = np.where(abs(Swtij) > thrd)[0] Swtij[indNZ] = Swtij[indNZ] - thrd[indNZ] * np.sign( Swtij[indNZ]) Swt[i, :, j] = Swtij # Reconstruct S S[:] = fftt.wt_rec(Swt) # Non-negativity constraint if nnegS: nneg = S > 0 S *= nneg if oracle: return 0 # Save the wavelet coefficients of S for next iteration if doThr and doRw and self.L1 and not oracle: if nnegS: Swt *= nneg[:, :, np.newaxis] self.Swtrw = Swt[:, :, :-1] Sfft[:] = fftt.fft(S) Sfft_det[:] = fftt.fftprod(Sfft, 1 - self.wt_filters[:, -1]) return 0
def GDP_PCA_plot(filename=None, threshold=0.015, lowerbound=2.0, upperbound=1.0e4, factor=0.08): data = np.load('uploads/' + filename) image1 = data.f.image #-np.median(data.f.image) image2 = np.array(image1 * 255 / image1.max(), dtype='uint8') image_avg = np.mean(np.partition(image2, int(len(image2) * 0.2))) image2[image2 < image_avg] = int(image_avg) #H1=cv2.GaussianBlur(image2,(3,3),1.0*np.std(image2)) H1 = gaussian_filter(image2, factor * np.std(image2), mode='nearest') image2 = H1 blobs_log = blob_log(image2, max_sigma=0.3 * np.std(image2), min_sigma=0.02 * np.mean(image2), num_sigma=20, threshold=threshold, overlap=0.6) blobs_log[:, 2] = blobs_log[:, 2] * np.sqrt(2) blobs = blobs_log[(blobs_log[:, 2] > lowerbound) & (blobs_log[:, 2] < upperbound)] xx = (data.f.X.min(), np.round(data.f.X.max(), -1)) yy = (data.f.Y.min(), np.round(data.f.Y.max(), -1)) x_step = (xx[1] - xx[0]) / np.shape(H1)[0] y_step = (yy[1] - yy[0]) / np.shape(H1)[1] #Number of NV height = yy[1] - yy[0] width = xx[1] - xx[0] total = len(blobs) per_nv = round(len(blobs) / float(height * width) * (20 * 20), 2) ######################################################## t = [ 'Original Density Plot,' ' Filename=' + filename, 'Gaussian Filtered Density Plot', 'Total NVs =' + str(total) + ' , NVs per 20x20 pixel area = ' + str(per_nv) ] data_list = [image1, H1, H1] color_list = [ Viridis256, cc.b_linear_bgy_10_95_c74, cc.b_linear_bgy_10_95_c74 ] return_list = [] return_list.append(head) #hover tool hack to work for image function in bokeh. ##http://stackoverflow.com/questions/28176949/convert-list-of-tuples-to-structured-numpy-array px = np.linspace(xx[0], xx[1], np.shape(H1)[0] / 2) py = np.linspace(yy[0], yy[1], np.shape(H1)[1] / 2) px = np.array(px, dtype='uint32') py = np.array(py, dtype='uint32') a = [] for i in px: a.extend(zip(itertools.repeat(i), py)) dt = np.dtype('int,float') X = np.array(a, dtype=dt) x1 = X['f0'] y1 = X['f1'] ################################################################## for i in range(3): color_mapper = LogColorMapper(palette=color_list[i], \ low=np.mean(data_list[i]), \ high=1.0*np.mean(data_list[i])+\ 2.0*np.std(data_list[i])) color_bar = ColorBar(color_mapper=color_mapper,\ label_standoff=12, \ border_line_color=None, \ location=(0,0)) p1 = figure(plot_width=600, plot_height=600,title=t[i],title_text_font_size='12pt',\ x_range=xx,y_range=xx,tools=TOOLS,toolbar_location="below",toolbar_sticky=False,responsive=True) p1.square(x1, y1, alpha=1.0) p1.image(image=[data_list[i]], color_mapper=color_mapper, dh=yy[1] - yy[0], dw=xx[1] - xx[0], x=xx[0], y=xx[0]) p1.add_layout(color_bar, 'right') if i == 2: p1.circle(blobs[:, 1] * x_step + xx[0], blobs[:, 0] * y_step + yy[0], radius=blobs[:, 2] * 1.6, radius_dimension='y', line_color='red', alpha=1.0, line_width=3, fill_color=None) p1.title.text_font_size = "11pt" p1.xaxis.axis_label_text_font_size = "13pt" p1.yaxis.axis_label_text_font_size = "13pt" #plots = {'Navy': p1, 'Blue': p2}; tuple_plot = components(p1) #script2, div2 = components(p2); return_list.append(list(tuple_plot)) p=Histogram(blobs[:,2],\ plot_width=600, plot_height=600,tools=TOOLS,\ toolbar_location="below",toolbar_sticky=False,\ responsive=True,\ title="Distribution of Radii of NV Centers") return_list.append(list(components(p))) return return_list
step_cpt += actor_steps # save stuff if step_cpt >= args.period: # evaluate mean actor over several runs. Memory is not filled # and steps are not counted actor.set_params(es.mu) f_mu, _ = evaluate(actor, env, memory=None, n_episodes=args.n_eval, render=args.render) prRed('Actor Mu Average Fitness:{}'.format(f_mu)) df.to_pickle(args.output + "/log.pkl") res = {"total_steps": total_steps, "average_score": np.mean(fitness), "average_score_half": np.mean(np.partition(fitness, args.pop_size // 2 - 1)[args.pop_size // 2:]), "average_score_rl": np.mean(fitness[:args.n_grad]), "average_score_ea": np.mean(fitness[args.n_grad:]), "best_score": np.max(fitness), "mu_score": f_mu} if args.save_all_models: os.makedirs(args.output + "/{}_steps".format(total_steps), exist_ok=True) critic.save_model( args.output + "/{}_steps".format(total_steps), "critic") actor.set_params(es.mu) actor.save_model( args.output + "/{}_steps".format(total_steps), "actor_mu") else: critic.save_model(args.output, "critic")
len(x)) #stores distance to 2nd nearest neighbor nn3 = np.zeros(len(x)) #stores CloudNum of 3rd nearest neighbor mindist3 = np.zeros( len(x)) #stores distance to 3rd nearest neighbor k = 0 j = 0 while k < len(x): for j in range(len(x)): dist[k, j] = np.sqrt( np.square(xs[k] - xs[j]) + np.square(y[k] - y[j])) k += 1 dist[dist == 0] = np.nan for k in range(len(x)): ind = np.where(dist[k] == np.nanmin( dist[k])) #index of nearest neighbor ind2 = np.where(dist[k] == np.partition( dist[k], 1)[1]) #index of 2nd nearest neighbor ind3 = np.where(dist[k] == np.partition( dist[k], 2)[2]) #index of 3rd nearest neighbor mindist[k] = np.deg2rad(dist[k, int(ind[0])]) * distance[0] nn[k] = int(ind[0]) + 1 if len(ind2[0]) == 1: mindist2[k] = np.deg2rad(dist[k, int(ind2[0])]) * distance[0] nn2[k] = int(ind2[0]) + 1 else: mindist2[k] = np.deg2rad( dist[k, int(ind2[0][0])]) * distance[0] nn2[k] = int(ind2[0][1]) + 1 if len(ind3[0]) == 1: mindist3[k] = np.deg2rad(dist[k, int(ind3[0])]) * distance[0]
# %% print(np.sort(a2, axis=0)) # %% print(np.sort(a2, axis=1)) # 부분 정렬 # - partition() : 배열에서 k개의 작은 값을 반환 # %% a1 = np.random.randint(1, 10, size=10) print(a1) # %% print(np.partition(a1, 3)) # %% a2 = np.random.randint(1, 10, size=(5, 5)) print(a2) # %% print(np.partition(a2, 3)) # %% print(np.partition(a2, 3, axis=0)) # %% print(np.partition(a2, 3, axis=1)) # 배열 입출력
def train(self, load_dir=None, save_dir=None): ###################### ######## TODO ######## ###################### if self.style == 'Real': self.train_real(load_dir, save_dir) else: self.debug = True if self.num_chosen_wc < 1000 else False if load_dir != None and os.path.exists(load_dir): self.chosen_wcs = pickle.load(open(load_dir, 'rb')) self.wc_errors = pickle.load(open("errors_" + load_dir, 'rb')) self.chosen_wcs_predictions = pickle.load( open("preds_" + load_dir, 'rb')) else: n = self.data.shape[0] weights = np.array([1 / n for i in range(n)]) self.chosen_wcs = [] self.chosen_wcs_predictions = np.zeros((self.num_chosen_wc, n)) k = 1000 #if self.debug==False else 50 t = 5 self.wc_errors = {} min_index = -1 for weak_classifier in self.weak_classifiers: weak_classifier.sorted_indices = np.argsort( weak_classifier.activations) rem_classifiers = [ classifier for classifier in self.weak_classifiers ] range_keys = [0, 9, 49, 99] #if self.debug == False else [0,4,9,14,19] for i in tqdm(range(self.num_chosen_wc)): my_result_array = Parallel(n_jobs=self.num_cores)( delayed(wc.calc_error)(weights, self.labels) for wc in rem_classifiers) my_result_array = list(zip(*my_result_array)) error = np.asarray(my_result_array[0]) predict = np.asarray(my_result_array[1]) threshold = np.asarray(my_result_array[2]) pol = np.asarray(my_result_array[3]) min_wc_error = min(error) min_index = np.argmin(error) rem_classifiers[min_index].threshold = threshold[min_index] rem_classifiers[min_index].polarity = pol[min_index] predict_min = predict[min_index] if i in range_keys: top_k_errors = np.partition(error, k)[:k] top_k_errors.sort() self.wc_errors[i] = top_k_errors predict_min = pol[min_index] * np.sign( rem_classifiers[min_index].activations - threshold[min_index]) print('Minimum error:', min_wc_error, 'Minimum index: ', min_index, 'in iteration ', i, 'classifier id', rem_classifiers[min_index]) x = (1 - min_wc_error) / min_wc_error alpha_i = 0.5 * np.log(x) if self.style == 'Real': alpha_i = 1 #wc_error_min, predict_min = rem_classifiers[min_index].calc_error(weights, self.labels) weights = weights * np.exp( -alpha_i * self.labels * predict_min) weights = weights / sum(weights) #print("Weightsum",sum(weights)) p = rem_classifiers[min_index] self.chosen_wc_ids.append(p.id) print("id", id) #self.wc_errors[i] = min_wc_error self.chosen_wcs.append((alpha_i, p)) self.chosen_wcs_predictions[i] = alpha_i * predict_min # print(i,":",self.chosen_wcs_predictions[i]) del rem_classifiers[min_index] if save_dir is not None: pickle.dump(self.wc_errors, open("errors_" + save_dir, 'wb')) pickle.dump(self.chosen_wcs_predictions, open("preds_" + save_dir, 'wb')) pickle.dump(self.chosen_wcs, open(save_dir, 'wb'))
parser.add_argument('filename', help='submission', type=str) parser.add_argument('--max_num', help='maximum number of predictions', type=int, default=150 * 10**6) parser.add_argument('-f', help='force overwrite', action='store_true') args = parser.parse_args() print(args) if os.path.exists(args.result) and not args.f: print(args.result, 'already exists, exiting') sys.exit() pool = multiprocessing.Pool() print('reading predictions') all_confs: List[float] = [] with open(args.filename) as f: for confs in tqdm(pool.imap(read_confidences, f), total=100000): all_confs.extend(confs) print(f'sorting scores (total {len(all_confs)})') assert len(all_confs) >= args.max_num pos = len(all_confs) - args.max_num threshold = np.partition(all_confs, pos)[pos] print('applying threshold', threshold) with open(args.filename) as f: with open(args.result, 'w') as out: for line in tqdm(pool.imap(partial(trim_line, threshold), f), total=100000): out.write(line)
def MRAS_categ_elite( args, game_i, N_0=400, alpha=4, kai_0=0, rho_0=0.7, epi_J=1e-6, pbar=None, verbose=False, ): game_ll = args.games[game_i] game = Game(game_ll) best_reward, best_arm = np.max(game.game_ll), np.argmax(game.game_ll) if verbose: print(f"\n\nRunning game {game_i} with MRAS Categorical") print(f"Arms distribution used {game_ll}") # ---------------------------------------------------------------------------- # # 1. Rewards collected and optimal arm pull (overall) # ---------------------------------------------------------------------------- # regret_ll = np.zeros(args.n) var_regret_ll = np.zeros(args.n) arm_ll = np.zeros((len(game), args.n)) for exp in range(args.repeat): # ---------------------------------------------------------------------------- # # 2. Rewards collected and optimal arm pull (for experiment) # ---------------------------------------------------------------------------- # regret_exp_ll = np.zeros(args.n) sample_mean_ll = np.zeros(len(game)) theta = np.ones(len(game)) / len(game) pulls_ll = np.zeros(len(game)) count = 0 # ---------------------------------------------------------------------------- # # 3. Initialisation ################################################## for k in range(10): for j in range(len(game)): arm_ll[j, k * len(game) + j] += 1 reward = game.get_reward(j) regret_exp_ll[j] = best_reward - reward # Update sample mean pulls_ll[j] += 1 sample_mean_ll[j] = (reward - sample_mean_ll[j]) / pulls_ll[j] count += 1 # Samples to take N = N_0 kai = kai_0 kai_bar = kai_0 rho = rho_0 phase = 1 while count < args.n: # pull sample arms = np.random.choice(len(game), N, p=theta) mask = np.zeros(len(game)) J_vec = np.zeros(N) J_vec_arms = np.zeros(N, dtype=np.int) for e, arm in enumerate(arms): arm_ll[arm, count] += 1 # Update sample mean reward = game.get_reward(arm) regret_exp_ll[count] = best_reward - reward pulls_ll[arm] += 1 sample_mean_ll[arm] += (reward - sample_mean_ll[arm]) / pulls_ll[arm] # J_vec[e] = sample_mean_ll[arm] J_vec_arms[e] = arm count += 1 if count >= args.n: break for e in range(len(J_vec_arms)): J_vec[e] = sample_mean_ll[J_vec_arms[e]] # Update elite set k = int((1 - rho) * N) - 1 kai_bar = np.partition(J_vec, k)[k - 1] # Check if threshold is acceptable if kai_bar - kai > epi_J: kai = kai_bar else: rho_bar = rho while rho_bar > epi_J: rho_bar = 0.9 * rho_bar k = int((1 - rho_bar) * N) - 1 kai_bar = np.partition(J_vec, k)[k] if kai_bar - kai > epi_J: kai = kai_bar rho = rho_bar break if rho > rho_bar: N = int(alpha * N) # Find elite mask if len(J_vec_arms[np.where(J_vec >= kai)]) > 0: for arm in J_vec_arms[np.where(J_vec >= kai)]: mask[arm] += 1 # Update theta theta[mask > 0] = (np.exp(phase * sample_mean_ll[mask > 0]) * mask[mask > 0] / theta[mask > 0]) theta[mask == 0] = 0 theta = theta / np.sum(theta[mask > 0]) phase += 1 if verbose: print(f"N value {N}") print(f"\nTurn {count} of {args.n} Arm pulled {arms}") print(f"Updated sample means {sample_mean_ll}") print(f"Mask {mask}") print(f"New theta: {theta}") if pbar: pbar.set_description( f"Game:{game_i + 1} MRAS_Categ_Elite exp:{exp} arm: {arm}") pbar.update(1) regret_exp_ll = np.cumsum(regret_exp_ll) regret_ll += regret_exp_ll var_regret_ll += regret_exp_ll**2 regret_ll = regret_ll / args.repeat var_regret_ll /= args.repeat var_regret_ll -= regret_ll**2 var_regret_ll /= np.arange(1, args.n + 1) var_regret_ll = np.sqrt(var_regret_ll) arm_ll /= args.repeat print(f"Final theta: {theta}") return regret_ll, var_regret_ll, arm_ll
def process_back_data(file): f_cam = 1000 n_tags = 23 no_subj = 1 pose_all = np.empty((no_subj), dtype=object) mesh_all = np.empty((no_subj), dtype=object) suct = 0 print('Calculating center of mass on file', file) for i in np.array([1]): # files in os.listdir(path_loc): file_path = file f = open(file_path) csv_f = csv.reader(f, delimiter='\t') new_frame = np.empty((23, 7, 1)) new_frame[:] = np.NAN full_array = new_frame.copy() count = -1 for row in csv_f: row = row[1:] if (count == -1): count = count+1 elif (row[1] == str(NULL_MARKER)): full_array = np.append(full_array, new_frame, axis=2) count = count+1 else: if (int(row[0]) < 23): read_array = np.asarray(row[2:]).astype(np.float) quart = Quaternion(matrix=rf.eul2mat(read_array[4:])) full_array[int(row[0]), :, count] = np.concatenate( (np.array([-read_array[2], read_array[3], -read_array[1]]), quart.elements), axis=0) full_array = full_array[:, :, :-1] f.close() pose_all[suct] = full_array suct += 1 final_ori = np.zeros((full_array.shape[2], full_array.shape[1])) cur_fullmesh = np.empty((n_tags*4, 3, full_array.shape[2])) cur_fullmesh[:] = np.NaN mesh_visible = np.empty((n_tags*4, 3, full_array.shape[2])) mesh_visible[:] = np.NaN vismesh_px = np.empty((n_tags*4, 2, full_array.shape[2])) vismesh_px[:] = np.NaN fullmesh_px = np.empty((n_tags*4, 2, full_array.shape[2])) fullmesh_px[:] = np.NaN err_dist = np.zeros((full_array.shape[2], 1)) avg_mesh = np.load("mesh_calib.npy") for f_ct in range(0, full_array.shape[2]): cur_mesh = np.empty((n_tags*4, 3)) cur_mesh[:] = np.NaN for tag_ct in range(0, n_tags): cur_mesh[tag_ct*4:(tag_ct+1)*4, :] = rf.orient_square( full_array[tag_ct, 0:3, f_ct], Quaternion(full_array[tag_ct, 3:, f_ct])) while True: mesh1 = avg_mesh[(~np.isnan(cur_mesh[:, 1])), :] mesh2 = cur_mesh[(~np.isnan(cur_mesh[:, 1])), :] m1c = rmsd.centroid(mesh1) m2c = rmsd.centroid(mesh2) mesh1 -= m1c mesh2 -= m2c rot2_1 = rmsd.kabsch(mesh1, mesh2) cur_fullmesh[:, :, f_ct] = np.dot(avg_mesh - m1c, rot2_1) + m2c # if f_ct >= 2: # cur_fullmesh[:,:,f_ct] = np.nansum((0.5*cur_fullmesh[:,:,f_ct],0.3*cur_fullmesh[:,:,f_ct-1],0.2*cur_fullmesh[:,:,f_ct-2]),axis = 0) err = np.linalg.norm( cur_fullmesh[:, :, f_ct] - cur_mesh, axis=1) if mesh1.shape[0] == 0: break if np.partition(err, 3)[3] > 0.005: break elif (np.nanmean(err) < 0.005): break cur_mesh[np.nanargmax(err), :] = np.array( [np.nan, np.nan, np.nan]) final_ori[f_ct, 0:3] = rmsd.centroid(cur_fullmesh[:, :, f_ct]) final_ori[f_ct, 3:] = Quaternion(matrix=rot2_1).elements # mesh_visible[(~np.isnan(cur_mesh[:,1])),:] = cur_fullmesh[(~np.isnan(cur_mesh[:,1])),:] vismesh_px[:, 0, f_ct] = 960 - \ (cur_mesh[:, 0]/cur_mesh[:, 2]*f_cam) vismesh_px[:, 1, f_ct] = 540 + \ (cur_mesh[:, 1]/cur_mesh[:, 2]*f_cam) fullmesh_px[:, 0, f_ct] = 960 - \ (cur_fullmesh[:, 0, f_ct]/cur_fullmesh[:, 2, f_ct]*f_cam) fullmesh_px[:, 1, f_ct] = 540 + \ (cur_fullmesh[:, 1, f_ct]/cur_fullmesh[:, 2, f_ct]*f_cam) err_dist[f_ct] = np.nanmean(np.linalg.norm( cur_fullmesh[:, :, f_ct] - cur_mesh, axis=1)) if np.size(mesh2, axis=0) <= 12: err_dist[f_ct] = 1 loc = rmsd.centroid(cur_fullmesh) #pose_all[suct] = final_ori #mesh_all[suct] = fullmesh_px suct = suct+1 print(suct) axis_px = np.empty((4, 2, full_array.shape[2])) axis_px[:] = np.NaN for ct in range(0, full_array.shape[2]): rot_mat = Quaternion(np.array( [final_ori[ct, 3], final_ori[ct, 4], final_ori[ct, 5], final_ori[ct, 6]])).rotation_matrix/25 ax = np.c_[rot_mat, final_ori[ct, 0:3]] ax_ad = ax.copy() for row in range(3): ax_ad[:, row] = ax[:, row] + final_ori[ct, 0:3] axis_px[:, 0, ct] = 960 - (ax_ad[0, :]/ax_ad[2, :]*f_cam) axis_px[:, 1, ct] = 540 + (ax_ad[1, :]/ax_ad[2, :]*f_cam) axis_px1 = np.empty((4, 2, full_array.shape[2])) axis_px1[:] = np.NaN for ct in range(0, full_array.shape[2]): rot_mat = Quaternion(np.array( [final_ori[ct, 6], final_ori[ct, 5], final_ori[ct, 4], final_ori[ct, 3]])).rotation_matrix/25 ax = np.c_[rot_mat, final_ori[ct, 0:3]] ax_ad = ax.copy() for row in range(3): ax_ad[:, row] = ax[:, row] + final_ori[ct, 0:3] axis_px1[:, 0, ct] = 960 - (ax_ad[0, :]/ax_ad[2, :]*f_cam) axis_px1[:, 1, ct] = 540 + (ax_ad[1, :]/ax_ad[2, :]*f_cam) # sio.savemat('meshsave_back_2.mat',{'vismesh_px':vismesh_px,'mesh_all':fullmesh_px,'pose_all':pose_all[1],'axis_px':axis_px,'axis_px1':axis_px1}) sio.savemat('meshsave_back_2.mat', {'vismesh_px': vismesh_px, 'mesh_all': fullmesh_px, 'pose_all': final_ori, 'axis_px': axis_px, 'axis_px1': axis_px1})
def partition(self, kth, axis=-1, kind='introselect', order=None): return np.partition(self, kth, axis, kind, order)
def initialize(num_centers_ratio, data_pipeline, method=None, metric='cosine', max_iter=100, tol=0.005, scaling_method=1, alpha=1.0, p=3): """ This functions initializes representative prototypes (RBF centers) c and scaling factors sigma. This function will generate one group of centers for all labels as a whole. Be cautious with initialize_per_label. Args: num_centers_ratio: The number of centers to be decided / total number of examples that belong to label l, for l = 0, ..., num_classes - 1. data_pipeline: A namedtuple consisting of the following elements. reader, video-level features reader or frame-level features reader. data_pattern, File Glob of data set. batch_size, How many examples to handle per time. num_readers, How many IO threads to prefetch examples. method: The method to decide the centers. Possible choices are random selection, kmeans and online (kmeans). Default is None, which represents randomly selecting a certain number of examples as centers. metric: Distance metric, euclidean distance or cosine distance. max_iter: The maximal number of iterations clustering to be done. tol: The minimal reduction of objective function of clustering to be reached to stop iteration. scaling_method: There are four choices. 1, all of them use the same sigma, the p smallest pairs of distances. 2, average of p nearest centers. 3, distance to the nearest center that has a different label (Not supported!). 4, mean distance between this center and all of its points. alpha: The alpha parameter that should be set heuristically. It works like a learning rate. (mu in Zhang) p: When scaling_method is 1 or 2, p is needed. Returns: centers (prototypes) and scaling factors (sigmas). Raises: ValueError if num_centers_ratio is not between 0.0 (open) and 1.0 (closed). ValueError if metric is not euclidean or cosine. ValueError if method is not one of None, kmeans or online. NotImplementedError if scaling_method is 3 or 5. ValueError if scaling method is not 1 - 5. """ logging.info('Generate a group of centers for all labels. See Schwenker.') # Argument checking. if (num_centers_ratio <= 0.0) or (num_centers_ratio > 1.0): raise ValueError( 'num_centers_ratio must be larger than 0.0 and no greater than 1.0.' ) logging.info('num_centers_ratio is {}.'.format(num_centers_ratio)) if ('euclidean' == metric) or ('cosine' == metric): logging.info( 'Using {} distance. The larger, the less similar.'.format(metric)) else: raise ValueError( 'Only euclidean and cosine distance are supported, {} passed.'. format(metric)) # Sample features only. _, centers, _, _ = random_sample(num_centers_ratio, mask=(False, True, False, False), data_pipeline=data_pipeline, name_scope='sample_centers') logging.info('Sampled {} centers totally.'.format(centers.shape[0])) logging.debug('Randomly selected centers: {}'.format(centers)) # Used in scaling method 4. Average distance of each point with its cluster center. per_clu_mean_dist = None # Perform k-means or online k-means. if method is None: logging.info( 'Using randomly selected centers as model prototypes (centers).') elif 'online' == method: raise NotImplementedError( 'Only None (randomly select examples), online, kmeans are supported.' ) elif 'kmeans' == method: logging.info( 'Using k-means clustering result as model prototypes (centers).') return_mean_clu_dist = (scaling_method == 4) kmeans = KMeans(centers, data_pipeline=data_pipeline, metric=metric, return_mean_clu_dist=return_mean_clu_dist) kmeans.fit(max_iter=max_iter, tol=tol) # Get current centers and update centers. centers = kmeans.current_centers per_clu_mean_dist = kmeans.per_clu_mean_dist else: raise ValueError( 'Only None (randomly select examples), online, kmeans are supported.' ) # Compute scaling factors based on these centers. num_centers = centers.shape[0] sigmas = None if scaling_method == 1: # Equation 27. pairwise_distances = sci_distance.pdist(centers, metric=metric) p = min(p, len(pairwise_distances)) logging.info('Using {} minimal pairwise distances.'.format(p)) # np.partition second argument begins with 0. sigmas = np.array( [alpha * np.mean(np.partition(pairwise_distances, p - 1)[:p])] * num_centers, dtype=np.float32) elif scaling_method == 2: # Equation 28. p = min(p, num_centers - 1) logging.info('Using {} minimal distances per center.'.format(p)) if 'euclidean' == metric: dis_fn = sci_distance.euclidean else: dis_fn = sci_distance.cosine sigmas = [] for c in centers: distances = [dis_fn(c, _c) for _c in centers] # The distance between c and itself is zero and is in the left partition. sigmas.append(alpha * np.sum(np.partition(distances, p)[:p + 1]) / float(p)) sigmas = np.array(sigmas, dtype=np.float32) elif scaling_method == 3: # Equation 29. raise NotImplementedError( 'Not supported when all labels use the same centers.') elif scaling_method == 4: # Equation 30. if per_clu_mean_dist is None: kmeans = KMeans(centers, data_pipeline=data_pipeline, metric=metric, return_mean_clu_dist=True) kmeans.fit(max_iter=1, tol=tol) centers = kmeans.current_centers per_clu_mean_dist = kmeans.per_clu_mean_dist logging.info( 'Compute mean distance per cluster using kmeans or online kmeans.' ) else: logging.info( 'Reuse mean distance per cluster computed in kmeans or online kmeans.' ) sigmas = alpha * per_clu_mean_dist elif scaling_method == 5: # Equation 31. raise NotImplementedError( 'Only three methods are supported. Please read the documentation.') else: raise ValueError( 'Only three methods are supported. Please read the documentation.') logging.debug('Scaling factor sigmas: {}'.format(sigmas)) return centers, sigmas
def partition_wrapper(arr, kth): try: return np.partition(arr, kth) except AttributeError: return np.sort(arr)
def qValues_planning_abstr(self, state_abstr_val, R, gamma, T, Q, d, branching_factor=None): """ Get the q values for pseudo-state(s) with a planning depth d. This function is called recursively by decreasing the depth d at every step. Arguments --------- state_abstr_val : internal state(s). R : float_model Model that fits the reward gamma : float_model Model that fits the discount factor T : transition_model Model that fits the transition between abstract representation Q : Q_model Model that fits the optimal Q-value d : int planning depth Returns ------- The Q-values with planning depth d for the provided encoded state(s) """ #if(branching_factor==None or branching_factor>self._n_actions): # branching_factor=self._n_actions n = len(state_abstr_val) identity_matrix = np.identity(self._n_actions) this_branching_factor = branching_factor.pop(0) if (n == 1): # We require that the first branching factor is self._n_actions so that this function return values # with the right dimension (=self._n_actions). this_branching_factor = self._n_actions if (d == 0): if (this_branching_factor < self._n_actions): return np.partition( Q.predict([state_abstr_val]), -this_branching_factor)[:, -this_branching_factor:] else: return Q.predict([state_abstr_val ]) # no change in the order of the actions else: if (this_branching_factor == self._n_actions): # All actions are considered in the tree # NB: For this case, we do not use argpartition because we want to keep the actions in the natural order # That way, this function returns the Q-values for all actions with planning depth d in the right order repeat_identity = np.repeat(identity_matrix, len(state_abstr_val), axis=0) if (state_abstr_val.ndim == 2): tile3_encoded_x = np.tile(state_abstr_val, (self._n_actions, 1)) elif (state_abstr_val.ndim == 4): tile3_encoded_x = np.tile(state_abstr_val, (self._n_actions, 1, 1, 1)) else: print("error") else: # A subset of the actions corresponding to the best estimated Q-values are considered et each branch estim_Q_values = Q.predict([state_abstr_val]) ind = np.argpartition( estim_Q_values, -this_branching_factor)[:, -this_branching_factor:] # Replacing ind if we want random branching #ind = np.random.randint(0,self._n_actions,size=ind.shape) repeat_identity = identity_matrix[ind].reshape( n * this_branching_factor, self._n_actions) tile3_encoded_x = np.repeat(state_abstr_val, this_branching_factor, axis=0) r_vals_d0 = np.array(R.predict([tile3_encoded_x, repeat_identity])) r_vals_d0 = r_vals_d0.flatten() gamma_vals_d0 = np.array( gamma.predict([tile3_encoded_x, repeat_identity])) gamma_vals_d0 = gamma_vals_d0.flatten() next_x_predicted = T.predict([tile3_encoded_x, repeat_identity]) return r_vals_d0 + gamma_vals_d0 * np.amax( self.qValues_planning_abstr( next_x_predicted, R, gamma, T, Q, d=d - 1, branching_factor=branching_factor).reshape( len(state_abstr_val) * this_branching_factor, branching_factor[0]), axis=1).flatten()
def init_match(ps1, w, md, edges1C, gt, fx_C, fy_C, xb_C, yb_C, edges0C, gt_0, fx_C0, fy_C0, xb_C0, yb_C0, mask_b_C0): w = int(w / ps1) buffer = 2 * w edges1Ca = np.zeros( (edges1C.shape[0] + buffer * 2, edges1C.shape[1] + 2 * buffer)) edges1Ca[buffer:-buffer, buffer:-buffer] = edges1C max_dist = int((md) / ps1) contours, hierarchy = cv2.findContours((1 - mask_b_C0).astype(np.uint8), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) contour_sizes = [(cv2.contourArea(contour), contour) for contour in contours] biggest_contour = max(contour_sizes, key=lambda x: x[0])[1] polygon = Polygon(np.array(biggest_contour[:, 0])) polygon = polygon.buffer(-w) v = w while polygon.is_empty or polygon.geom_type == 'MultiPolygon' or polygon.area / ps1 < 100: v -= int(2 / ps1) polygon = Polygon(np.array(biggest_contour[:, 0])) polygon = polygon.buffer(-v) if v != w: print("WARNING : Polygon-buffer: " + str(v * ps1) + " < w...") x, y = polygon.exterior.xy distance = np.cumsum( np.sqrt(np.ediff1d(x, to_begin=0)**2 + np.ediff1d(y, to_begin=0)**2)) distance = distance / distance[-1] fx, fy = interp1d(distance, x), interp1d(distance, y) alpha = np.linspace(0, 1, 200) x_regular, y_regular = fx(alpha), fy(alpha) grid = [] for i in range(len(x_regular)): grid.append((int(round(x_regular[i])), int(round(y_regular[i])))) x_offset = [] y_offset = [] o_x = np.zeros(len(grid)) o_y = np.zeros(len(grid)) t_x = np.zeros(len(grid)) t_y = np.zeros(len(grid)) cv = np.zeros(len(grid)) RECC_total = np.zeros(edges1Ca.shape) RECC_over = np.zeros(edges1Ca.shape) RECC_over.fill(np.NaN) circle1 = np.zeros((2 * w, 2 * w)) for x in range(circle1.shape[0]): for y in range(circle1.shape[1]): if (x - w)**2 + (y - w)**2 < w**2: circle1[x, y] = 1 circle2 = np.zeros((2 * max_dist, 2 * max_dist)) for x in range(circle2.shape[0]): for y in range(circle2.shape[1]): if (x - max_dist)**2 + (y - max_dist)**2 < max_dist**2: circle2[x, y] = 1 circle2[circle2 == 0] = np.NaN for i in tqdm(range(len(grid)), position=0, miniters=int(len(grid) / 10), desc="RECC "): x_i_0 = grid[i][1] y_i_0 = grid[i][0] target = edges0C[x_i_0 - w:x_i_0 + w, y_i_0 - w:y_i_0 + w] * circle1 if target.shape != (2 * w, 2 * w): continue sum_target = np.sum(target) lat_0 = gt_0[3] + gt_0[5] * x_i_0 * fx_C0 lon_0 = gt_0[0] + gt_0[1] * y_i_0 * fy_C0 x_i_0_og = int(round((lat_0 - gt[3]) / (gt[5] * fx_C))) y_i_0_og = int(round((lon_0 - gt[0]) / (gt[1] * fy_C))) search_wide = np.zeros((2 * (max_dist + w), 2 * (max_dist + w))) search_wide = edges1Ca[buffer + x_i_0_og - max_dist - w:buffer + x_i_0_og + max_dist + w, buffer + y_i_0_og - max_dist - w:buffer + y_i_0_og + max_dist + w] if search_wide.shape != (2 * (max_dist + w), 2 * (max_dist + w)): continue sum_patch = cv2.filter2D(search_wide, -1, circle1) numerator = cv2.filter2D(search_wide, -1, target) RECC_wide = numerator / (sum_patch + sum_target) RECC_area = RECC_wide[w:-w, w:-w] * circle2 RECC_total.fill(np.NaN) RECC_total[x_i_0_og - max_dist:x_i_0_og + max_dist, y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area if np.nansum(RECC_over[x_i_0_og - max_dist:x_i_0_og + max_dist, y_i_0_og - max_dist:y_i_0_og + max_dist]) == 0: RECC_over[x_i_0_og - max_dist:x_i_0_og + max_dist, y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area max_one = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(), -1)[-1] max_n = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(), -4 - 1)[-4 - 1] y_i = np.where(RECC_total >= max_one)[1][0] x_i = np.where(RECC_total >= max_one)[0][0] y_n = np.where(RECC_total >= max_n)[1][0:-1] x_n = np.where(RECC_total >= max_n)[0][0:-1] cv[i] = sum(np.sqrt(np.square(x_i - x_n) + np.square(y_i - y_n))) / 4 x_offset.append(x_i - x_i_0_og) y_offset.append(y_i - y_i_0_og) o_x[i] = x_i o_y[i] = y_i t_x[i] = x_i_0 t_y[i] = y_i_0 x_offset = np.array(x_offset) y_offset = np.array(y_offset) a, b, c = np.histogram2d(x_offset[cv < 4], y_offset[cv < 4], bins=len(x_offset)) d, e = np.where(a == np.max(a)) if len(d) > 1: i = [0, 0] binnnn = len(x_offset) / 10 while len(i) > 1: binnnn -= 1 f, g, h = np.histogram2d(x_offset[cv < 4], y_offset[cv < 4], bins=binnnn) i, j = np.where(f == np.max(f)) diff = (b[d] - g[i])**2 + (c[e] - h[j])**2 ind = np.where(diff == np.min(diff))[0] d = d[ind] e = e[ind] x_offset = (b[d] + b[d + 1]) / 2 y_offset = (c[e] + c[e + 1]) / 2 x_offset = x_offset[0] * ps1 y_offset = y_offset[0] * ps1 o_x = o_x[cv < 4] o_y = o_y[cv < 4] t_x = t_x[cv < 4] t_y = t_y[cv < 4] return x_offset, y_offset, o_x, o_y, t_x, t_y
def fft(self): #Performs fft of each state channel seperately and returns (symmetry, [spacial peaks], [temporal peaks]) #//16 or //32 work best filter_resolution=self.size//32 #'background' signal removal parameters t0 = self.max_iters//2 q0 = self.size//2 # One hot decode real space trajectory and fft the indivual binary representations of each state im_one_hot = one_hot_encode(self.image,self.states) fd = (np.fft.fftshift(np.fft.fftn(im_one_hot,axes=(1,2,3)))) #sys.stdout.write("#") #sys.stdout.flush() #Data for visualising purposes self.fft_data = np.abs(one_hot_decode(fd)) #Remove background signal (q=0,f=0) ts,xs,ys=np.meshgrid(np.arange(self.max_iters),np.arange(self.size),np.arange(self.size),indexing="ij") gs = gaus3d(ts,xs,ys,t0,q0,q0,4,4,4) fd_filtered = np.zeros(fd.shape,dtype=fd.dtype) for i in range(self.states): #plt.matshow(fd[i,t0]) #plt.show() fd_filtered[i] = (1-gs)*fd[i] #fd_filtered[i] = fd[i] #sys.stdout.write("#") #sys.stdout.flush() #Apply local max filter to enhance peaks and suppress noise n_peaks = self.size*self.size//8 for i in range(self.states): for t in range(self.max_iters): slice = ndimage.maximum_filter(np.abs(fd_filtered[i,t]),filter_resolution,mode="wrap") flat = np.abs((slice)).flatten() k = np.partition(flat,-n_peaks)[-n_peaks] slice[np.abs(slice)<k]=0 fd_filtered[i,t] = slice #sys.stdout.write("#") #sys.stdout.flush() #Peak detection along time axis #print(fd_filtered[0,:,q0,q0].shape) max_t_peaks = np.zeros((4,self.states)).astype(int) a,b = sp.signal.butter(3,0.3,analog=False) for i in range(self.states): tdata = np.pad(np.abs(np.sum(fd_filtered[i,t0:],axis=(1,2))),(0,filter_resolution),mode='reflect') tdata = sp.signal.filtfilt(a,b,tdata) #tdata = np.pad(np.abs(fd_filtered[i,t0:,q0,q0]),(0,filter_resolution),mode='reflect') peaks,dicts = sp.signal.find_peaks(tdata,prominence=256) if len(peaks)==0: #No peaks found ind_max=0 ind_2nd=0 max_t_peaks[0,i] = 0#peaks[ind_max] max_t_peaks[1,i] = 0#peaks[ind_2nd] elif len(peaks)==1: #only one peak found ind_max = np.argmax(dicts['prominences']) ind_2nd=0 max_t_peaks[0,i] = peaks[ind_max] max_t_peaks[1,i] = 0#peaks[ind_2nd] else: #at least 2 peaks found ind_max = np.argmax(dicts['prominences']) ind_2nd = np.argpartition(dicts['prominences'],-2)[-2] max_t_peaks[0,i] = peaks[ind_max] max_t_peaks[1,i] = peaks[ind_2nd] max_t_peaks[2,i] = tdata[max_t_peaks[0,i]] max_t_peaks[3,i] = tdata[max_t_peaks[1,i]] #plt.plot(tdata) #plt.scatter(peaks,tdata[peaks]) #plt.scatter(max_t_peaks[0,i],max_t_peaks[2,i],color="red") #plt.scatter(max_t_peaks[1,i],max_t_peaks[3,i],color="green") #plt.show() max_t_peak_mean = (2/self.max_iters)*safe_average(max_t_peaks[0,max_t_peaks[0]!=0],max_t_peaks[2,max_t_peaks[0]!=0]) max_t_peak_var = (2/self.max_iters)*np.std(max_t_peaks[0,max_t_peaks[0]!=0]) harmonic_t_mean = (2/self.max_iters)*safe_average(max_t_peaks[1,max_t_peaks[1]!=0],max_t_peaks[3,max_t_peaks[1]!=0]) harmonic_t_var = (2/self.max_iters)*np.std(max_t_peaks[1,max_t_peaks[1]!=0]) temporal_peaks = np.array([[max_t_peak_mean,max_t_peak_var],[harmonic_t_mean,harmonic_t_var]]) #print(temporal_peaks) #sys.stdout.write("#") #sys.stdout.flush() #Check for 4-fold symmetry fd_filtered_rot = np.rot90(fd_filtered,axes=(2,3)) rot_coeff = np.einsum('ijkl,ijkl',fd_filtered_rot,fd_filtered) norm = np.einsum('ijkl,ijkl',fd_filtered,fd_filtered) symmetry_coeff = np.abs(rot_coeff/norm) #sys.stdout.write("#") #sys.stdout.flush() #print(symmetry_coeff) #Calculuate radial distribution functions of fft data #print("Calculating rdf") fd_rdf = np.zeros((self.states,self.max_iters,self.size//2)).astype(fd.dtype) fd_rdf_filtered = np.zeros((self.states,self.max_iters,self.size//2)).astype(fd.dtype) for x in range(self.size//2): sq = square_ring(self.size,x+1) fd_rdf_filtered[:,:,x] = np.einsum('ijkl,kl->ij',fd_filtered,sq)/np.sum(sq) fd_rdf[:,:,x] = np.einsum('ijkl,kl->ij',fd,sq)/np.sum(sq) #sys.stdout.write("#") #sys.stdout.flush() #peak detection on rdf #print("Finding peaks") max_peaks = np.zeros((4,self.states)).astype(int) for i in range(self.states): xdata = np.pad(np.abs(fd_rdf_filtered[i,t0]),(0,filter_resolution),mode='reflect') #peaks,dicts = signal.find_peaks(xdata,height=256,prominence=None,width=filter_resolution) peaks,dicts = sp.signal.find_peaks(xdata,prominence=None,width=filter_resolution) if len(peaks)==0: #No peaks found ind_max=0 ind_2nd=0 max_peaks[0,i] = 0#peaks[ind_max] max_peaks[1,i] = 0#peaks[ind_2nd] elif len(peaks)==1: #only one peak found ind_max = np.argmax(dicts['prominences']) ind_2nd=0 max_peaks[0,i] = peaks[ind_max] max_peaks[1,i] = 0#peaks[ind_2nd] else: #at least 2 peaks found ind_max = np.argmax(dicts['prominences']) ind_2nd = np.argpartition(dicts['prominences'],-2)[-2] max_peaks[0,i] = peaks[ind_max] max_peaks[1,i] = peaks[ind_2nd] #Magnitude at peaks max_peaks[2,i] = xdata[max_peaks[0,i]] max_peaks[3,i] = xdata[max_peaks[1,i]] #plt.plot(xdata) #plt.scatter(peaks,xdata[peaks]) #plt.show() max_peak_mean = (2/self.size)*safe_average(max_peaks[0,max_peaks[0]!=0],max_peaks[2,max_peaks[0]!=0]) max_peak_var = (2/self.size)*np.std(max_peaks[0,max_peaks[0]!=0]) harmonic_mean = (2/self.size)*safe_average(max_peaks[1,max_peaks[1]!=0],max_peaks[3,max_peaks[1]!=0]) harmonic_var = (2/self.size)*np.std(max_peaks[1,max_peaks[1]!=0]) #print(max_peak_var) spacial_peaks = np.array([[max_peak_mean,max_peak_var],[harmonic_mean,harmonic_var]]) #print(spacial_peaks) #sys.stdout.write("#") #sys.stdout.flush() #waves[waves<thresh]=0 #stat_struct_filtered = np.zeros(stat_struct.shape,dtype='complex') #stat_struct_filtered[np.abs(stat_struct_filtered)<thresh]=0 #waves_filtered = signal.convolve(waves,f_gs_3,mode='same') #plt.matshow(gs) #plt.show() self.image = np.abs(np.fft.ifftn(np.fft.ifftshift(one_hot_decode(fd_filtered)))) """ #Split up sections of 3d fft that correspond to certain features - disused stat_struct = fd[:,t0] stat_struct[:,q0,q0]=0 stat_struct_filtered = fd_filtered[:,t0]#ndimage.maximum_filter(np.abs(stat_struct),self.size//16,mode="wrap") stat_struct_filtered_rot = fd_filtered_rot[:,t0] blinkers = fd_filtered[:,:,q0,q0] #blinkers[:,t0]=0 waves = np.copy(fd) waves[:,t0] = 0 waves[:,:,q0,q0]=0 for i in range(self.states): epsilon=1E-20 xdata = (np.abs(fd_rdf[i,t0])) peaks,_ = signal.find_peaks(xdata,prominence=None,width=filter_resolution) plt.plot(xdata) plt.scatter(peaks,xdata[peaks]) plt.matshow((np.abs(stat_struct[i]))) plt.show() #plt.plot(np.sum(np.abs(stat_struct[i]),axis=0)) #plt.plot(np.sum(np.abs(stat_struct[i]),axis=1)) #plt.show() xdata = (np.abs(fd_rdf_filtered[i,t0])) peaks,dicts = signal.find_peaks(xdata,prominence=None,width=filter_resolution) #maximum peak height #max_peaks[0,i] = peaks[np.argmax(xdata[peaks])] #most prominant peak max_peaks[0,i] = peaks[np.argmax(dicts['prominences'])] max_peaks[1,i] = xdata[max_peaks[0,i]] plt.plot(xdata) plt.scatter(peaks,xdata[peaks]) plt.scatter(max_peaks[0,i],max_peaks[1,i],color="red") plt.matshow((np.abs(stat_struct_filtered[i]))) #plt.matshow(laplace(np.abs(stat_struct[i]))) plt.show() #plt.show() #plt.matshow((np.abs(stat_struct_filtered_rot[i]))) #plt.show() #plt.plot(np.sum(np.abs(stat_struct_filtered[i]),axis=0)) #plt.plot(np.sum(np.abs(stat_struct_filtered[i]),axis=1)) #plt.show() for j in range(self.states): #j=1 plt.plot(np.abs(blinkers[j]),label=str(j)) #plt.plot(np.abs(fd_ref[:,q0,q0]),label="reference") #plt.plot(blinkers[j].real,alpha=0.2) #plt.plot(blinkers[j].imag,alpha=0.2) plt.legend() plt.show() for j in range(self.states): plt.plot(np.abs(np.sum(fd_filtered[j],axis=(1,2))),label=str(j)) plt.legend() plt.show() for j in range(self.states): plt.plot(np.sum(np.abs(fd_filtered[j]),axis=(1,2)),label=str(j)) plt.legend() plt.show() self.fft_data=(np.abs(one_hot_decode(fd_filtered))) """ #sys.stdout.write("|") #sys.stdout.flush() return (symmetry_coeff,spacial_peaks,temporal_peaks,fd[:,t0])
def PatchMatch(ps2, w, md, edges1F, gt, fx_F, fy_F, edges0F, gt_0, fx_F0, fy_F0, contour_F0, x_offset, y_offset): w = int(w / ps2) buffer = 2 * w edges1Fa = np.zeros( (edges1F.shape[0] + buffer * 2, edges1F.shape[1] + 2 * buffer)) edges1Fa[buffer:-buffer, buffer:-buffer] = edges1F max_dist = int((md) / (ps2 * 4)) contours, hierarchy = cv2.findContours((1 - contour_F0).astype(np.uint8), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) contour_sizes = [(cv2.contourArea(contour), contour) for contour in contours] biggest_contour = max(contour_sizes, key=lambda x: x[0])[1] polygon = Polygon(np.array(biggest_contour[:, 0])) polygon = polygon.buffer(-w) v = w while polygon.is_empty or polygon.geom_type == 'MultiPolygon' or polygon.area / ps2 < 100: v -= int(2 / ps2) polygon = Polygon(np.array(biggest_contour[:, 0])) polygon = polygon.buffer(-v) if v != w: print("WARNING : Polygon-buffer: " + str(v) + " < w...") x, y = polygon.exterior.xy distance = np.cumsum( np.sqrt(np.ediff1d(x, to_begin=0)**2 + np.ediff1d(y, to_begin=0)**2)) distance = distance / distance[-1] fx, fy = interp1d(distance, x), interp1d(distance, y) alpha = np.linspace(0, 1, 200) x_regular, y_regular = fx(alpha), fy(alpha) grid = [] for i in range(len(x_regular)): grid.append((int(round(x_regular[i])), int(round(y_regular[i])))) cv = np.zeros(len(grid)) dist_lon = np.zeros(len(grid)) dist_lat = np.zeros(len(grid)) origin_x = np.zeros(len(grid)) origin_y = np.zeros(len(grid)) target_lon = np.zeros(len(grid)) target_lat = np.zeros(len(grid)) o_x = np.zeros(len(grid)) o_y = np.zeros(len(grid)) t_x = np.zeros(len(grid)) t_y = np.zeros(len(grid)) dx = np.zeros(len(grid)) dy = np.zeros(len(grid)) RECC_total = np.zeros(edges1Fa.shape) RECC_over = np.zeros(edges1Fa.shape) RECC_over.fill(np.NaN) circle1 = np.zeros((2 * w, 2 * w)) for x in range(circle1.shape[0]): for y in range(circle1.shape[1]): if (x - w)**2 + (y - w)**2 < w**2: circle1[x, y] = 1 circle2 = np.zeros((2 * max_dist, 2 * max_dist)) for x in range(circle2.shape[0]): for y in range(circle2.shape[1]): if (x - max_dist)**2 + (y - max_dist)**2 < max_dist**2: circle2[x, y] = 1 circle2[circle2 == 0] = np.NaN for i in tqdm(range(len(grid)), position=0, miniters=int(len(grid) / 10), desc="RECC "): x_i_0 = grid[i][1] y_i_0 = grid[i][0] target = edges0F[x_i_0 - w:x_i_0 + w, y_i_0 - w:y_i_0 + w] if target.shape != (2 * w, 2 * w): continue target = target * circle1 sum_target = np.sum(target) lat_0 = gt_0[3] + gt_0[5] * x_i_0 * fx_F0 lon_0 = gt_0[0] + gt_0[1] * y_i_0 * fy_F0 x_i_0_og = int( round((lat_0 - gt[3]) / (gt[5] * fx_F) + (x_offset / ps2))) y_i_0_og = int( round((lon_0 - gt[0]) / (gt[1] * fy_F) + (y_offset / ps2))) search_wide = np.zeros((2 * (max_dist + w), 2 * (max_dist + w))) search_wide = edges1Fa[buffer + x_i_0_og - max_dist - w:buffer + x_i_0_og + max_dist + w, buffer + y_i_0_og - max_dist - w:buffer + y_i_0_og + max_dist + w] if search_wide.shape != (2 * (max_dist + w), 2 * (max_dist + w)): continue sum_patch = cv2.filter2D(search_wide, -1, circle1) numerator = cv2.filter2D(search_wide, -1, target) RECC_wide = numerator / (sum_patch + sum_target) RECC_area = RECC_wide[w:-w, w:-w] * circle2 RECC_total.fill(np.NaN) RECC_total[x_i_0_og - max_dist:x_i_0_og + max_dist, y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area if np.nansum(RECC_over[x_i_0_og - max_dist:x_i_0_og + max_dist, y_i_0_og - max_dist:y_i_0_og + max_dist]) == 0: RECC_over[x_i_0_og - max_dist:x_i_0_og + max_dist, y_i_0_og - max_dist:y_i_0_og + max_dist] = RECC_area max_one = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(), -1)[-1] max_n = np.partition(RECC_total[~np.isnan(RECC_total)].flatten(), -4 - 1)[-4 - 1] y_i = np.where(RECC_total >= max_one)[1][0] x_i = np.where(RECC_total >= max_one)[0][0] y_n = np.where(RECC_total >= max_n)[1][0:-1] x_n = np.where(RECC_total >= max_n)[0][0:-1] cv[i] = sum(np.sqrt(np.square(x_i - x_n) + np.square(y_i - y_n))) / 4 lon = gt[0] + gt[1] * y_i * fy_F + gt[2] * x_i * fx_F lat = gt[3] + gt[4] * y_i * fy_F + gt[5] * x_i * fx_F lon_0 = gt_0[0] + gt_0[1] * y_i_0 * fy_F0 + gt_0[2] * x_i_0 * fx_F0 lat_0 = gt_0[3] + gt_0[4] * y_i_0 * fy_F0 + gt_0[5] * x_i_0 * fx_F0 dist_lon[i] = lon_0 - lon dist_lat[i] = lat_0 - lat o_x[i] = x_i o_y[i] = y_i t_x[i] = x_i_0 t_y[i] = y_i_0 dx[i] = (x_i - x_i_0_og) * ps2 dy[i] = (y_i - y_i_0_og) * ps2 # For referencing: origin_x[i] = x_i * fx_F origin_y[i] = y_i * fy_F target_lon[i] = lon_0 target_lat[i] = lat_0 return origin_x, origin_y, target_lon, target_lat, o_x, o_y, t_x, t_y, cv, dx, dy, RECC_over
fft_len = 2304 effect_fft = int(fft_len / 2) for k in range(3,4): data_zif90 = list_test[k][:-1] data_zif90 -= np.median(data_zif90) data_zif90[data_zif90<0] = 0 data_zif90 /= np.max(np.abs(data_zif90)) ########################替换peak为mean copy_no_peak = np.copy(data_zif90) h_the = np.partition(data_zif90, -num_large)[-num_large] # copy_no_peak[data_zif90 > h_the] = np.median(data_zif90) copy_no_peak[data_zif90 > h_the] = 0 ########################替换noise为0 copy_only_peak = np.copy(data_zif90) copy_only_peak[data_zif90 < h_the] = 0 # copy_only_peak[data_zif90 < h_the] = np.median(data_zif90) names= ['original', 'no_peak', 'only_peak'] data = np.zeros((len(names), fft_len)) data[0, 20:2251+20] = data_zif90 data[1, 20:2251+20] = copy_no_peak data[2, 20:2251+20] = copy_only_peak
def nsmall(a, n): # find the smallest nth element of an np array return np.partition(a, n)[n]
def _topk_py_impl(op, x, k, axis, idx_dtype): ndim = x.ndim assert -ndim <= axis < ndim axis %= ndim if k == 0: raise ValueError("topk: kth cannot be zero") elif k > x.shape[axis]: raise ValueError( f"topk: kth cannot be larger than the size of specified axis {int(axis)}" ) if abs(k) == 1: # negative k means min instead of max fn_max = [None, np.max, np.min][k] fn_argmax = [None, np.argmax, np.argmin][k] if not op.return_indices: return np.expand_dims(fn_max(x, axis=axis), axis) elif op.return_values: zi = np.expand_dims(fn_argmax(x, axis=axis), axis) idx2 = tuple( np.arange(s).reshape((s, ) + (1, ) * (ndim - i - 1)) if i != axis else zi for i, s in enumerate(x.shape)) zv = x[idx2] return zv, zi.astype(idx_dtype) else: zi = np.expand_dims(fn_argmax(x, axis=axis), axis) return zi.astype(idx_dtype) if x.shape[axis] == abs(k): if not op.return_indices: return x.copy() else: l = axis r = ndim - l reps = list(x.shape) reps[axis] = 1 zi = np.arange(abs(k), dtype=idx_dtype) zi = zi.reshape((1, ) * l + (k, ) + (1, ) * (r - 1)) zi = np.tile(zi, reps) if op.return_values: return x.copy(), zi else: return zi idx = [slice(None)] * ndim idx[axis] = slice(-k, None) if k > 0 else slice(-k) if not op.return_indices: zv = np.partition(x, -k, axis=axis)[idx] return zv elif op.return_values: zi = np.argpartition(x, -k, axis=axis)[idx] idx2 = tuple( np.arange(s).reshape((s, ) + (1, ) * (ndim - i - 1)) if i != axis else zi for i, s in enumerate(x.shape)) zv = x[idx2] return zv, zi.astype(idx_dtype) else: zi = np.argpartition(x, -k, axis=axis)[idx] return zi.astype(idx_dtype)