def compute_distances(self, x1, x2): """ The method uses a function implemented in Cython. Data (`x1` and `x2`) is accompanied by two tables. One is a 2-d table in which elements of `x1` (`x2`) are replaced by 0's and 1's. The other is a vector indicating rows (or column) with nan values. The function in Cython uses a fast loop without any conditions to compute distances between rows without missing values, and a slower loop for those with missing values. """ nonzeros1 = np.not_equal(x1, 0).view(np.int8) if self.axis == 1: nans1 = _distance.any_nan_row(x1) if x2 is None: nonzeros2, nans2 = nonzeros1, nans1 else: nonzeros2 = np.not_equal(x2, 0).view(np.int8) nans2 = _distance.any_nan_row(x2) return _distance.jaccard_rows( nonzeros1, nonzeros2, x1, x1 if x2 is None else x2, nans1, nans2, self.ps, x2 is not None) else: nans1 = _distance.any_nan_row(x1.T) return _distance.jaccard_cols( nonzeros1, x1, nans1, self.ps)
def best_grid(wavelengths1, wavelengths2, key): """ Return the best wavelength grid to regrid to arrays Considering the two wavelength grids passed in parameters, this function compute the best new grid that will be used to regrid the two spectra before combining them. We do not use np.unique as it is much slowe than finding the unique elements by hand. Parameters ---------- wavelengths1, wavelengths2: array of floats The wavelength grids to be 'regridded'. key: tuple Key to key the results in cache. Returns ------- new_grid: array of floats Array containing all the wavelengths found in the input arrays. """ if key in best_grid_cache: return best_grid_cache[key] wl = np.concatenate((wavelengths1, wavelengths2)) wl.sort(kind='mergesort') flag = np.ones(len(wl), dtype=bool) np.not_equal(wl[1:], wl[:-1], out=flag[1:]) best_grid_cache[key] = wl[flag] return wl[flag]
def node_can_drain(self, the_node): """Check if a node has drainage away from the current lake/depression. Parameters ---------- the_node : int The node to test. nodes_this_depression : array_like of int Nodes that form a pit. Returns ------- boolean ``True`` if the node can drain. Otherwise, ``False``. """ nbrs = self._node_nbrs[the_node] not_bad = nbrs != LOCAL_BAD_INDEX_VALUE not_too_high = self._elev[nbrs] < self._elev[the_node] not_current_lake = np.not_equal(self.flood_status[nbrs], _CURRENT_LAKE) not_flooded = np.not_equal(self.flood_status[nbrs], _FLOODED) all_probs = np.logical_and( np.logical_and(not_bad, not_too_high), np.logical_and(not_current_lake, not_flooded)) if np.any(all_probs): return True else: return False
def parseArgs(data, targetClass, otherClass = None, **args) : '''parse arguments for a feature scoring function''' if 'feature' in args : feature = args['feature'] else : feature = None if 'Y' in args : Y = args['Y'] if otherClass is None : otherI = numpy.nonzero(numpy.not_equal(Y, targetClass))[0] else : otherI = numpy.nonzero(numpy.equal(Y, otherClass))[0] targetClassSize = numpy.sum(numpy.equal(Y, targetClass)) else : Y = None if otherClass is None : otherI = numpy.nonzero(numpy.not_equal(data.labels.Y, targetClass))[0] else : otherI = data.labels.classes[otherClass] targetClassSize = len(data.labels.classes[targetClass]) otherClassSize = len(otherI) return Y, targetClassSize, otherClassSize, otherI, feature
def test_prelu_param_updates(self): x_train, _, y_train, _ = simple_classification() prelu_layer1 = layers.PRelu(20, alpha=0.25) prelu_layer2 = layers.PRelu(1, alpha=0.25) gdnet = algorithms.GradientDescent( [ layers.Input(10), prelu_layer1, prelu_layer2, ] ) prelu1_alpha_before_training = prelu_layer1.alpha.get_value() prelu2_alpha_before_training = prelu_layer2.alpha.get_value() gdnet.train(x_train, y_train, epochs=10) prelu1_alpha_after_training = prelu_layer1.alpha.get_value() prelu2_alpha_after_training = prelu_layer2.alpha.get_value() self.assertTrue(all(np.not_equal( prelu1_alpha_before_training, prelu1_alpha_after_training, ))) self.assertTrue(all(np.not_equal( prelu2_alpha_before_training, prelu2_alpha_after_training, )))
def scoreDuplicates(records, data_model, pool, threshold=0): record, records = peek(records) id_type = idType(record) score_dtype = [('pairs', id_type, 2), ('score', 'f4', 1)] record_chunks = grouper(records, 100000) scoring_function = ScoringFunction(data_model, threshold, score_dtype) results = [pool.apply_async(scoring_function, (chunk,)) for chunk in record_chunks] for r in results : r.wait() scored_pairs = numpy.concatenate([r.get() for r in results]) scored_pairs.sort() flag = numpy.ones(len(scored_pairs), dtype=bool) numpy.not_equal(scored_pairs[1:], scored_pairs[:-1], out=flag[1:]) return scored_pairs[flag]
def average_without_padding(x, ids, padding_id, cuda=False, eps=1e-8): if cuda: mask = Variable(torch.from_numpy(np.not_equal(ids, padding_id).astype(int)[:,:,np.newaxis])).float().cuda().permute(1, 2, 0).expand_as(x) else: mask = Variable(torch.from_numpy(np.not_equal(ids, padding_id).astype(int)[:,:,np.newaxis])).float().permute(1, 2, 0).expand_as(x) s = torch.sum(x*mask, dim=2) / (torch.sum(mask, dim=2)+eps) return s
def get_calipso_phase_inner(features, qual_min=CALIPSO_QUAL_VALUES['medium'], max_layers=1, same_phase_in_top_three_lay=True): """ Returns Calipso cloud phase. Pixels with quality lower than *qual_min* are masked out. Screen out pixels with more than *max_layers* layers. """ if same_phase_in_top_three_lay: phase1 = get_bits(features[:,0], CALIPSO_PHASE_BITS, shift=True) phase2 = get_bits(features[:,1], CALIPSO_PHASE_BITS, shift=True) phase3 = get_bits(features[:,2], CALIPSO_PHASE_BITS, shift=True) two_layer_pixels = features[:, 2] >1 three_layer_pixels = features[:, 3] >1 lay1_lay2_differ = np.logical_and(two_layer_pixels, np.not_equal(phase1, phase2)) lay2_lay3_differ = np.logical_and(three_layer_pixels, np.not_equal(phase2, phase3)) varying_phases_in_top_3lay = np.logical_or(lay1_lay2_differ, lay2_lay3_differ) # Reduce to single layer, masking any multilayer pixels features = np.ma.array(features[:, 0], mask=(features[:, max_layers:] > 1).any(axis=-1)) if same_phase_in_top_three_lay: features = np.ma.array(features, mask = varying_phases_in_top_3lay) phase = get_bits(features, CALIPSO_PHASE_BITS, shift=True) qual = get_bits(features, CALIPSO_QUAL_BITS, shift=True) # Don't care about pixels with lower than *qual_min* quality return np.ma.array(phase, mask=qual < qual_min)
def _calc_errors(truth, prediction, class_number=1): tp = np.sum(np.equal(truth,class_number)*np.equal(prediction,class_number)) tn = np.sum(np.not_equal(truth,class_number)*np.not_equal(prediction,class_number)) fp = np.sum(np.not_equal(truth,class_number)*np.equal(prediction,class_number)) fn = np.sum(np.equal(truth,class_number)*np.not_equal(prediction,class_number)) return tp, tn, fp, fn
def merge(a, b): # http://stackoverflow.com/questions/12427146/combine-two-arrays-and-sort c = np.concatenate((a, b)) c.sort(kind='mergesort') flag = np.ones(len(c), dtype=bool) np.not_equal(c[1:], c[:-1], out=flag[1:]) return c[flag]
def oht_model( gw, oro, fsns, flns, shfl, lhfl ): """parameters; must be dimensioned as specified: gwi : gaussian weights (lat) oroi : orography data array (lat,lon) requires the lat and lon are attached coordinates of oro and that oro and the following variables are 2D arrays (lat,lon). fsnsi: net shortwave solar flux at surface (lat,lon) flnsi: net longwave solar flux at surface (lat,lon) shfli: sensible heat flux at surface (lat,lon) lhfli: latent heat flux at surface (lat,lon) """ re = 6.371e6 # radius of earth coef = re**2/1.e15 # scaled by PW heat_storage = 0.3 # W/m^2 adjustment for ocean heat storage nlat = oro.shape[0] nlon = oro.shape[1] dlon = 2.*pi/nlon # dlon in radians lat = latAxis(oro) i65n = numpy.where( lat[:]>=65 )[0][0] # assumes that lat[i+1]>lat[i] i65s = numpy.where( lat[:]<=-65 )[0][-1] # assumes that lat[i+1]>lat[i] # get the mask for the ocean basins basins_mask = ocean_mask(oro) # returns 2D array(lat,lon) # compute net surface energy flux netflux = fsns-flns-shfl-lhfl-heat_storage # compute the net flux for the basins netflux_basin = numpy.ma.empty( (3,nlat,nlon) ) netflux_basin[0,:,:] = netflux[:,:] netflux_basin[1,:,:] = netflux[:,:] netflux_basin[2,:,:] = netflux[:,:] netflux_basin[:,:,:] = numpy.ma.masked # to make sure the mask array gets created netflux_basin._mask[0,:,:] = numpy.not_equal(basins_mask,1) # False on Pacific netflux_basin._mask[1,:,:] = numpy.not_equal(basins_mask,2) # False on Atlantic netflux_basin._mask[2,:,:] = numpy.not_equal(basins_mask,3) # False on Indian # sum flux over the longitudes in each basin heatflux = numpy.ma.sum( netflux_basin, axis=2 ) # compute implied heat transport in each basin oft = cdms2.createVariable( numpy.ma.masked_all((4,nlat)) ) oft.setAxisList( [cdms2.createAxis([0,1,2,3],id='basin numer'),lat] ) # These ! signs assign a name to a dimension of oft: #oft!0 = "basin number" # 0:pacific, 1:atlantic, 2:indian, 3:total #oft!1 = "lat" for n in range(3): for j in range(i65n,i65s-1,-1): #start sum at most northern point # ...assumes that lat[i+1]>lat[i] oft[n,j] = -coef*dlon*numpy.ma.sum( heatflux[n,j:i65n+1]*gw[j:i65n+1] ) # compute total implied ocean heat transport at each latitude # as the sum over the basins at that latitude for j in range( i65n, i65s-1, -1 ): oft[3,j] = numpy.ma.sum( oft[0:3,j] ) return oft # 2D array(4,lat)
def shrink_hyperrect(x0, x1, L, R): """ """ L_or_R = (x1 >= x0) #Modifications to R R[L_or_R] = x1[L_or_R] np.not_equal(L_or_R, True, L_or_R) #Modifications to L L[L_or_R] = x1[L_or_R] return L, R
def _numpy(self, data, weights, shape): q = self.quantity(data) self._checkNPQuantity(q, shape) self._checkNPWeights(weights, shape) weights = self._makeNPWeights(weights, shape) newentries = weights.sum() import numpy selection = numpy.isnan(q) numpy.bitwise_not(selection, selection) subweights = weights.copy() subweights[selection] = 0.0 self.nanflow._numpy(data, subweights, shape) # avoid nan warning in calculations by flinging the nans elsewhere numpy.bitwise_not(selection, selection) q = numpy.array(q, dtype=numpy.float64) q[selection] = self.high weights = weights.copy() weights[selection] = 0.0 numpy.greater_equal(q, self.low, selection) subweights[:] = weights subweights[selection] = 0.0 self.underflow._numpy(data, subweights, shape) numpy.less(q, self.high, selection) subweights[:] = weights subweights[selection] = 0.0 self.overflow._numpy(data, subweights, shape) if all(isinstance(value, Count) and value.transform is identity for value in self.values) and numpy.all(numpy.isfinite(q)) and numpy.all(numpy.isfinite(weights)): # Numpy defines histograms as including the upper edge of the last bin only, so drop that weights[q == self.high] == 0.0 h, _ = numpy.histogram(q, self.num, (self.low, self.high), weights=weights) for hi, value in zip(h, self.values): value.fill(None, float(hi)) else: q = numpy.array(q, dtype=numpy.float64) numpy.subtract(q, self.low, q) numpy.multiply(q, self.num, q) numpy.divide(q, self.high - self.low, q) numpy.floor(q, q) q = numpy.array(q, dtype=int) for index, value in enumerate(self.values): numpy.not_equal(q, index, selection) subweights[:] = weights subweights[selection] = 0.0 value._numpy(data, subweights, shape) # no possibility of exception from here on out (for rollback) self.entries += float(newentries)
def _build_y(self, X, y, sample_weight, trim_duplicates=True): """Build the y_ IsotonicRegression.""" check_consistent_length(X, y, sample_weight) X, y = [check_array(x, ensure_2d=False) for x in [X, y]] y = as_float_array(y) self._check_fit_data(X, y, sample_weight) # Determine increasing if auto-determination requested if self.increasing == 'auto': self.increasing_ = check_increasing(X, y) else: self.increasing_ = self.increasing # If sample_weights is passed, removed zero-weight values and clean # order if sample_weight is not None: sample_weight = check_array(sample_weight, ensure_2d=False) mask = sample_weight > 0 X, y, sample_weight = X[mask], y[mask], sample_weight[mask] else: sample_weight = np.ones(len(y)) order = np.lexsort((y, X)) X, y, sample_weight = [astype(array[order], np.float64, copy=False) for array in [X, y, sample_weight]] unique_X, unique_y, unique_sample_weight = _make_unique( X, y, sample_weight) # Store _X_ and _y_ to maintain backward compat during the deprecation # period of X_ and y_ self._X_ = X = unique_X self._y_ = y = isotonic_regression(unique_y, unique_sample_weight, self.y_min, self.y_max, increasing=self.increasing_) # Handle the left and right bounds on X self.X_min_, self.X_max_ = np.min(X), np.max(X) if trim_duplicates: # Remove unnecessary points for faster prediction keep_data = np.ones((len(y),), dtype=bool) # Aside from the 1st and last point, remove points whose y values # are equal to both the point before and the point after it. keep_data[1:-1] = np.logical_or( np.not_equal(y[1:-1], y[:-2]), np.not_equal(y[1:-1], y[2:]) ) return X[keep_data], y[keep_data] else: # The ability to turn off trim_duplicates is only used to it make # easier to unit test that removing duplicates in y does not have # any impact the resulting interpolation function (besides # prediction speed). return X, y
def LabelPerimeter(L, Connectivity=4): """Converts a label or binary mask image to a binary perimeter image. Uses 4-neighbor or 8-neighbor shifts to detect pixels whose values do not agree with their neighbors. Parameters ---------- L : array_like A label or binary mask image. Connectivity : double or int Neighborhood connectivity to evaluate. Valid values are 4 or 8. Default value = 4. Returns ------- Mask : array_like A binary image where object perimeter pixels have value 1, and non-perimeter pixels have value 0. See Also -------- EmbedBounds """ # initialize temporary variable Mask = np.zeros(L.shape) Temp = np.zeros(L.shape) # check left-right neighbors Temp[:, 0:-2] = np.not_equal(L[:, 0:-2], L[:, 1:-1]) Temp[:, 1:-1] = np.logical_or(Temp[:, 1:-1], Temp[:, 0:-2]) Mask = np.logical_or(Mask, Temp) # check up-down neighbors Temp[0:-2, :] = np.not_equal(L[0:-2, :], L[1:-1, :]) Temp[1:-1, :] = np.logical_or(Temp[1:-1, :], Temp[0:-2, :]) Mask = np.logical_or(Mask, Temp) # additional calculations if Connectivity == 8 if(Connectivity == 8): # slope 1 diagonal shift Temp[1:-1, 0:-2] = np.not_equal(L[0:-2, 1:-2], L[1:-1, 0:-2]) Temp[0:-2, 1:-1] = np.logical_or(Temp[0:-2, 1:-1], Temp[1:-1, 0:-2]) Mask = np.logical_or(Mask, Temp) # slope -1 diagonal shift Temp[1:-1, 1:-1] = np.not_equal(L[0:-2, 0:-2], L[1:-1, 1:-1]) Temp[0:-2, 0:-2] = np.logical_or(Temp[0:-2, 0:-2], Temp[1:-1, 1:-1]) Mask = np.logical_or(Mask, Temp) # generate label-valued output return Mask.astype(np.uint32) * L
def fact(x): p = equal(x, 0) k = add(x, p) i = not_equal(k, 1).astype(int) z = 1 while i.any().astype(bool): z = multiply(z, k) subtract(k, i, k) i = not_equal(k, 1).astype(int) return z
def getModelData(model, air, selected_mogs, type1, type2=""): data = np.array([]) type2 = "" tt = np.array([]) et = np.array([]) in_vect = np.array([]) mogs = [] for i in selected_mogs: mogs.append(model.mogs[i]) if type1 == "tt": fac_dt = 1 mog = mogs[0] ind = np.not_equal(mog.tt, -1).T tt, t0 = mog.getCorrectedTravelTimes(air) tt = tt.T et = fac_dt * mog.f_et * mog.et.T in_vect = mog.in_vect.T no = np.arange(mog.data.ntrace).T if len(mogs) > 1: for n in range(1, len(model.mogs)): mog = mogs[n] ind = np.concatenate((ind, np.not_equal(mog.tt, -1).T), axis=0) tt = np.concatenate((tt, mog.getCorrectedTravelTimes(air)[0].T), axis=0) et = np.concatenate((et, fac_dt * mog.et * mog.f_et.T), axis=0) in_vect = np.concatenate((in_vect, mog.in_vect.T), axis=0) no = np.concatenate((no, np.arange(mog.ntrace + 1).T), axis=0) ind = np.equal((ind.astype(int) + in_vect.astype(int)), 2) data = np.array([tt[ind], et[ind], no[ind]]).T return data, ind if type2 == "depth": data, ind = getModelData(model, air, selected_mogs, type1) # @UndefinedVariable mog = mogs[0] tt = mog.Tx_z_orig.T et = mog.Rx_z_orig.T in_vect = mog.in_vect.T if len(mogs) > 1: for n in (1, len(mogs)): tt = np.concatenate((tt, mogs[n].Tx_z_orig.T), axis=0) et = np.concatenate((et, mogs[n].Rx_z_orig.T), axis=0) in_vect = np.concatenate((in_vect, mogs[n].in_vect.T), axis=0) ind = np.equal((ind.astype(int) + in_vect.astype(int)), 2) data = np.array([tt[ind], et[ind], no[ind]]).T return data, ind
def step_callback(*args, **kwargs): nonlocal model, optimiser, context, w, b, var, call_count context.optimiser_updated = False mon.update_optimiser(context, *args, **kwargs) w_new, b_new, var_new = model.enquire_session().run([model.w.unconstrained_tensor, model.b.unconstrained_tensor, model.var.unconstrained_tensor]) self.assertTrue(np.alltrue(np.not_equal(w, w_new))) self.assertTrue(np.alltrue(np.not_equal(b, b_new))) self.assertTrue(np.alltrue(np.not_equal(var, var_new))) self.assertTrue(context.optimiser_updated) call_count += 1 w, b, var = w_new, b_new, var_new
def svm_bench(): data_file = "./data/dataset.pkl" train_set, valid_set, test_set, word2id, pop2id, type2id = dataset.load_data(data_file) train_set_x, train_set_y = train_set train_set_pop_y, train_set_type_y, train_set_loc_y = train_set_y valid_set_x, valid_set_y = valid_set valid_set_pop_y, valid_set_type_y, valid_set_loc_y = valid_set_y test_set_x, test_set_y = test_set test_set_pop_y, test_set_type_y, test_set_loc_y = test_set_y id2word = {v:k for k,v in word2id.items()} word_train_set_x = [sen_dig2word(doc, id2word) for doc in train_set_x] word_valid_set_x = [sen_dig2word(doc, id2word) for doc in valid_set_x] word_test_set_x = [sen_dig2word(doc, id2word) for doc in test_set_x] # construct the word count matrix # construct the word count matrix count_vect = CountVectorizer() x_train_count = count_vect.fit_transform(word_train_set_x) x_valid_count = count_vect.transform(word_valid_set_x) x_test_count = count_vect.transform(word_test_set_x) tfidf_transformer = TfidfTransformer() x_train_tfidf = tfidf_transformer.fit_transform(x_train_count) x_valid_tfidf = tfidf_transformer.transform(x_valid_count) x_test_tfidf = tfidf_transformer.transform(x_test_count) # train the pop model pop_clf = svm.LinearSVC().fit(x_train_tfidf, train_set_pop_y) pop_pred = pop_clf.predict(x_valid_tfidf) pop_pred_test = pop_clf.predict(x_test_tfidf) # compute the performance pop_errors = np.mean(np.not_equal(pop_pred, valid_set_pop_y)) pop_errors_test = np.mean(np.not_equal(pop_pred_test, test_set_pop_y)) # train the event type model type_clf = svm.LinearSVC().fit(x_train_tfidf, train_set_type_y) type_pred = type_clf.predict(x_valid_tfidf) type_pred_test = type_clf.predict(x_test_tfidf) # compute the performance type_errors = np.mean(np.not_equal(type_pred, valid_set_type_y)) type_errors_test = np.mean(np.not_equal(type_pred_test, test_set_type_y)) print "SVM Valid--> Type error: %0.2f, Popuation error: %0.2f" % (type_errors, pop_errors) print "SVM Tes--> Type error: %0.2f, Popuation error: %0.2f" % (type_errors_test, pop_errors_test)
def test_simplify(self): origEdgeImages = dm.load_rasters_from_dir('../images/manual/edges') redr_edge_images = [] simp_edge_images = [] edgeSets = [] simp_edge_sets = [] for img in origEdgeImages: edges = ch.chain(img) edgeSets.append(edges) simplified_set = [] edge_image = np.zeros_like(img) simp_edge_image = np.zeros_like(img) for chain in edges: edge = et.Edge(chain) edge.draw(edge_image, 255) simp_chain = ch.simplify_chain(chain,1) simp_edge = et.Edge(simp_chain) simp_edge.draw(simp_edge_image, 255) simplified_set.append(simp_chain) redr_edge_images.append(edge_image) simp_edge_images.append(simp_edge_image) simp_edge_sets.append(simplified_set) redr_matches_orig = np.array_equal(img, edge_image) if not redr_matches_orig: bad_pixels = np.argwhere(np.not_equal(img,edge_image).astype(np.uint8)) copy = cv2.cvtColor(np.copy(img),cv2.COLOR_GRAY2BGR) for pixel in bad_pixels: copy[pixel[0],pixel[1]] = (0,0,255) for chain in edges: for pt in chain: copy[pt[0],pt[1]] = (255,128,2) cv2.imwrite('bad_pixels.png',copy) self.assertTrue(redr_matches_orig, "The redrawn edge image does not match the original image."+ " Percentage of unmatched pixels: %f" % (float(np.not_equal(img,edge_image).sum())/img.size)) simp_matches_orig = np.array_equal(img, simp_edge_image) if not simp_matches_orig: bad_pixels = np.argwhere(np.not_equal(img,simp_edge_image).astype(np.uint8)) copy = cv2.cvtColor(np.copy(img),cv2.COLOR_GRAY2BGR) for pixel in bad_pixels: if(simp_edge_image[pixel[0],pixel[1]] != 0): copy[pixel[0],pixel[1]] = (0,0,255) for simp_chain in simplified_set: for pt in simp_chain: copy[pt[0],pt[1]] = (255,128,2) cv2.imwrite('bad_pixels.png',copy) self.assertTrue(simp_matches_orig, "The simplified edge image does not match the original image."+ " Percentage of unmatched pixels: %f" % (float(np.not_equal(img,simp_edge_image).sum())/img.size))
def test_visiting_stepping(self): lu = list(zip(*self.ld_bounds)) lower = np.array(lu[0]) upper = np.array(lu[1]) dim = lower.size vd = VisitingDistribution(lower, upper, self.qv, self.rs) values = np.zeros(dim) x_step_low = vd.visiting(values, 0, self.high_temperature) # Make sure that only the first component is changed assert_equal(np.not_equal(x_step_low, 0), True) values = np.zeros(dim) x_step_high = vd.visiting(values, dim, self.high_temperature) # Make sure that component other than at dim has changed assert_equal(np.not_equal(x_step_high[0], 0), True)
def test_acceptRejectMove(self, blues_sim, state_keys, caplog): # Check positions are different from stepNCMC md_state = BLUESSimulation.getStateFromContext(blues_sim._md_sim.context, state_keys) ncmc_state = BLUESSimulation.getStateFromContext(blues_sim._ncmc_sim.context, state_keys) assert np.not_equal(md_state['positions'], ncmc_state['positions']).all() caplog.set_level(logging.INFO) blues_sim._acceptRejectMove() ncmc_state = BLUESSimulation.getStateFromContext(blues_sim._ncmc_sim.context, state_keys) md_state = BLUESSimulation.getStateFromContext(blues_sim._md_sim.context, state_keys) if 'NCMC MOVE ACCEPTED' in caplog.text: assert np.equal(md_state['positions'], ncmc_state['positions']).all() elif 'NCMC MOVE REJECTED' in caplog.text: assert np.not_equal(md_state['positions'], ncmc_state['positions']).all()
def compute_image(nf, colorizer, region, antialiased = 3): A, B = region.A, region.B out = np.zeros((A, B, 3), dtype = float) ll = region.lowerleft() dx = region.dx() # -1 will be used for missing the target target = np.zeros((A, B), dtype = int) steps = np.zeros((A, B), dtype = float) # First, compute every pixel for i, j in region.pixels(): # compute center of pixel z = ll + dx * (i + 1j * j) result = nf.converge(z) if result is None: target[i, j] = -1 steps[i, j] = 0 else: target[i, j] = result[0] steps[i, j] = result[1] out[i, j, :] = colorizer.color(result) if antialiased >= 2: aa = list(np.linspace(-1, 1, antialiased + 2)[1:-1]) # Now, antialias the boundaries boundary = np.zeros((A, B), dtype = bool) boundary[1:, :] |= np.not_equal(target[1:, :], target[:-1, :]) boundary[:-1, :] |= np.not_equal(target[1:, :], target[:-1, :]) boundary[:, 1:] |= np.not_equal(target[:, 1:], target[:, :-1]) boundary[:, :-1] |= np.not_equal(target[:, 1:], target[:, :-1]) for i, j in region.pixels(): if not boundary[i, j]: continue colors = [] for i_ in aa: for j_ in aa: z = ll + dx * (i + i_ + 1j * (j + j_)) result = nf.converge(z) colors.append(colorizer.color(result)) out[i, j, :] = colorizer.mean(colors) return colorizer.quantize(out)
def _numpy(self, data, weights, shape): q = self.quantity(data) self._checkNPQuantity(q, shape) self._checkNPWeights(weights, shape) weights = self._makeNPWeights(weights, shape) newentries = weights.sum() import numpy selection = numpy.isnan(q) numpy.bitwise_not(selection, selection) subweights = weights.copy() subweights[selection] = 0.0 self.nanflow._numpy(data, subweights, shape) # switch to float here like in bin.py else numpy throws # TypeError on trivial integer cases such as: # >>> q = numpy.array([1,2,3,4]) # >>> np.divide(q,1,q) # >>> np.floor(q,q) q = numpy.array(q, dtype=numpy.float64) neginfs = numpy.isneginf(q) posinfs = numpy.isposinf(q) numpy.subtract(q, self.origin, q) numpy.divide(q, self.binWidth, q) numpy.floor(q, q) q = numpy.array(q, dtype=numpy.int64) q[neginfs] = LONG_MINUSINF q[posinfs] = LONG_PLUSINF selected = q[weights > 0.0] selection = numpy.empty(q.shape, dtype=numpy.bool) for index in numpy.unique(selected): if index != LONG_NAN: bin = self.bins.get(index) if bin is None: bin = self.value.zero() self.bins[index] = bin numpy.not_equal(q, index, selection) subweights[:] = weights subweights[selection] = 0.0 bin._numpy(data, subweights, shape) # no possibility of exception from here on out (for rollback) self.entries += float(newentries)
def test_testUfuncs1 (self): "Test various functions such as sin, cos." (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d self.assertTrue (eq(numpy.cos(x), cos(xm))) self.assertTrue (eq(numpy.cosh(x), cosh(xm))) self.assertTrue (eq(numpy.sin(x), sin(xm))) self.assertTrue (eq(numpy.sinh(x), sinh(xm))) self.assertTrue (eq(numpy.tan(x), tan(xm))) self.assertTrue (eq(numpy.tanh(x), tanh(xm))) olderr = numpy.seterr(divide='ignore', invalid='ignore') try: self.assertTrue (eq(numpy.sqrt(abs(x)), sqrt(xm))) self.assertTrue (eq(numpy.log(abs(x)), log(xm))) self.assertTrue (eq(numpy.log10(abs(x)), log10(xm))) finally: numpy.seterr(**olderr) self.assertTrue (eq(numpy.exp(x), exp(xm))) self.assertTrue (eq(numpy.arcsin(z), arcsin(zm))) self.assertTrue (eq(numpy.arccos(z), arccos(zm))) self.assertTrue (eq(numpy.arctan(z), arctan(zm))) self.assertTrue (eq(numpy.arctan2(x, y), arctan2(xm, ym))) self.assertTrue (eq(numpy.absolute(x), absolute(xm))) self.assertTrue (eq(numpy.equal(x, y), equal(xm, ym))) self.assertTrue (eq(numpy.not_equal(x, y), not_equal(xm, ym))) self.assertTrue (eq(numpy.less(x, y), less(xm, ym))) self.assertTrue (eq(numpy.greater(x, y), greater(xm, ym))) self.assertTrue (eq(numpy.less_equal(x, y), less_equal(xm, ym))) self.assertTrue (eq(numpy.greater_equal(x, y), greater_equal(xm, ym))) self.assertTrue (eq(numpy.conjugate(x), conjugate(xm))) self.assertTrue (eq(numpy.concatenate((x, y)), concatenate((xm, ym)))) self.assertTrue (eq(numpy.concatenate((x, y)), concatenate((x, y)))) self.assertTrue (eq(numpy.concatenate((x, y)), concatenate((xm, y)))) self.assertTrue (eq(numpy.concatenate((x, y, x)), concatenate((x, ym, x))))
def test_testUfuncs1(self): # Test various functions such as sin, cos. (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d assert_(eq(np.cos(x), cos(xm))) assert_(eq(np.cosh(x), cosh(xm))) assert_(eq(np.sin(x), sin(xm))) assert_(eq(np.sinh(x), sinh(xm))) assert_(eq(np.tan(x), tan(xm))) assert_(eq(np.tanh(x), tanh(xm))) with np.errstate(divide='ignore', invalid='ignore'): assert_(eq(np.sqrt(abs(x)), sqrt(xm))) assert_(eq(np.log(abs(x)), log(xm))) assert_(eq(np.log10(abs(x)), log10(xm))) assert_(eq(np.exp(x), exp(xm))) assert_(eq(np.arcsin(z), arcsin(zm))) assert_(eq(np.arccos(z), arccos(zm))) assert_(eq(np.arctan(z), arctan(zm))) assert_(eq(np.arctan2(x, y), arctan2(xm, ym))) assert_(eq(np.absolute(x), absolute(xm))) assert_(eq(np.equal(x, y), equal(xm, ym))) assert_(eq(np.not_equal(x, y), not_equal(xm, ym))) assert_(eq(np.less(x, y), less(xm, ym))) assert_(eq(np.greater(x, y), greater(xm, ym))) assert_(eq(np.less_equal(x, y), less_equal(xm, ym))) assert_(eq(np.greater_equal(x, y), greater_equal(xm, ym))) assert_(eq(np.conjugate(x), conjugate(xm))) assert_(eq(np.concatenate((x, y)), concatenate((xm, ym)))) assert_(eq(np.concatenate((x, y)), concatenate((x, y)))) assert_(eq(np.concatenate((x, y)), concatenate((xm, y)))) assert_(eq(np.concatenate((x, y, x)), concatenate((x, ym, x))))
def __ne__(self, other): try: return np.not_equal(self, other) except Exception as exc: if isinstance(other, Quantity): raise exc return True
def test_get_speech_features_from_file_augmentation(self): augmentation = { 'time_stretch_ratio': 0.0, 'noise_level_min': -90, 'noise_level_max': -46, } filename = 'open_seq2seq/test_utils/toy_speech_data/wav_files/46gc040q.wav' num_features = 161 input_features_clean = get_speech_features_from_file( filename, num_features, augmentation=None, ) input_features_augm = get_speech_features_from_file( filename, num_features, augmentation=augmentation, ) # just checking that result is different with and without augmentation self.assertTrue(np.all(np.not_equal(input_features_clean, input_features_augm))) augmentation = { 'time_stretch_ratio': 0.2, 'noise_level_min': -90, 'noise_level_max': -46, } input_features_augm = get_speech_features_from_file( filename, num_features, augmentation=augmentation, ) self.assertNotEqual( input_features_clean.shape[0], input_features_augm.shape[0], ) self.assertEqual( input_features_clean.shape[1], input_features_augm.shape[1], )
def show_overlay(img3d, cc3d, ncc=10, s=85, xyz = 'xy',alpha=.8): """Shows the connected components overlayed over img3d Input ====== img3d -- 3d array cc3d -- 3d array ( preferably of same shape as img3d, use get_3d_cc(...) ) ncc -- where to cut off the color scale s -- slice to show xyz -- which projection to use in {'xy','xz','yz'} """ cc = get_slice(cc3d,s,xyz) img = get_slice(img3d,s,xyz) notcc = np.isnan(cc) incc = np.not_equal(notcc,True) img4 = plt.cm.gray(img/np.nanmax(img)) if ncc is not np.Inf: cc = plt.cm.jet(cc/float(ncc)) else: cc = plt.cm.jet(np.log(cc)/np.log(np.nanmax(cc))) cc[notcc,:]=img4[notcc,:] cc[incc,3] = 1-img[incc]/(2*np.nanmax(img)) plt.imshow(cc)
def db_spline(y): ''' Background subtraction by fitting spline to the baseline. Arguments: y -- input array, 1D np.array Returns: y_db -- debaselined array, 1D np.array ''' # Because of the discharge disturbance, the background does not exactly # match the signal. This creates a curved baseline after background # subtraction. Try to fit a B-spline to the baseline. x = np.arange(len(y)) / (len(y)-1) # Construct weight weight = weight_spline(y) w_nonzero = np.not_equal(weight, 0) # Interpolate spline spline = interpolate.UnivariateSpline(x, y, w=weight, k=5) # Remove spline y_db = y - spline(x) # Remove linear feature ppoly = np.polyfit(x[w_nonzero], y_db[w_nonzero], 1) y_db = y_db - np.polyval(ppoly, x) return y_db
def read_data(filename): """ Reads an Intan Technologies RHD2000 data file generated by the evaluation board GUI. Returns data in a dictionary in raw sample units Adapted from Intan sample code... Michael Gibson 17 July 2015 Modified Adrian Foy Sep 2018 """ tic = time.time() fid = open(filename, 'rb') filesize = os.path.getsize(filename) header = read_header(fid) print('{} amplifier channels'.format(header['num_amplifier_channels'])) print('{} auxiliary input channels'.format( header['num_aux_input_channels'])) print('{} supply voltage channels'.format( header['num_supply_voltage_channels'])) print('{} board ADC channels'.format(header['num_board_adc_channels'])) print('{} board digital input channels'.format( header['num_board_dig_in_channels'])) print('{} board digital output channels'.format( header['num_board_dig_out_channels'])) print('{} temperature sensors channels'.format( header['num_temp_sensor_channels'])) # Determine how many samples the data file contains. bytes_per_block = get_bytes_per_data_block(header) # How many data blocks remain in this file? data_present = False bytes_remaining = filesize - fid.tell() if bytes_remaining > 0: data_present = True if bytes_remaining % bytes_per_block != 0: raise Exception( 'File size wrong: should have a whole number of data blocks') num_data_blocks = int(bytes_remaining / bytes_per_block) num_amplifier_samples = header[ 'num_samples_per_data_block'] * num_data_blocks num_aux_input_samples = int( (header['num_samples_per_data_block'] / 4) * num_data_blocks) num_supply_voltage_samples = 1 * num_data_blocks num_board_adc_samples = header[ 'num_samples_per_data_block'] * num_data_blocks num_board_dig_in_samples = header[ 'num_samples_per_data_block'] * num_data_blocks num_board_dig_out_samples = header[ 'num_samples_per_data_block'] * num_data_blocks record_time = num_amplifier_samples / header['sample_rate'] if data_present: print( 'File contains {:0.3f} seconds of data. Amplifiers sampled at {:0.2f} kS/s.' .format(record_time, header['sample_rate'] / 1000)) else: print('Header contains no data. Amplifiers sampled at {:0.2f} kS/s.'. format(header['sample_rate'] / 1000)) if data_present: # Pre-allocate memory for data. print('') print('Allocating memory for data...') data = {} data['num_amplifier_samples'] = num_amplifier_samples if (header['version']['major'] == 1 and header['version']['minor'] >= 2 ) or (header['version']['major'] > 1): data['t_amplifier'] = np.zeros(num_amplifier_samples, dtype=np.int) else: data['t_amplifier'] = np.zeros(num_amplifier_samples, dtype=np.uint) # NOTE: Changed from uint to uint16 data['amplifier_data'] = np.zeros( [header['num_amplifier_channels'], num_amplifier_samples], dtype=np.uint16) data['aux_input_data'] = np.zeros( [header['num_aux_input_channels'], num_aux_input_samples], dtype=np.uint) data['supply_voltage_data'] = np.zeros([ header['num_supply_voltage_channels'], num_supply_voltage_samples ], dtype=np.uint) data['temp_sensor_data'] = np.zeros( [header['num_temp_sensor_channels'], num_supply_voltage_samples], dtype=np.uint) data['board_adc_data'] = np.zeros( [header['num_board_adc_channels'], num_board_adc_samples], dtype=np.uint) # by default, this script interprets digital events (digital inputs and outputs) as booleans # if unsigned int values are preferred(0 for False, 1 for True), # replace the 'dtype=np.bool' argument with 'dtype=np.uint' as shown # the commented line below illustrates this for digital input data; # the same can be done for digital out # data['board_dig_in_data'] = np.zeros([header['num_board_dig_in_channels'], # num_board_dig_in_samples], dtype=np.uint) data['board_dig_in_data'] = np.zeros( [header['num_board_dig_in_channels'], num_board_dig_in_samples], dtype=np.bool) data['board_dig_in_raw'] = np.zeros(num_board_dig_in_samples, dtype=np.uint) data['board_dig_out_data'] = np.zeros( [header['num_board_dig_out_channels'], num_board_dig_out_samples], dtype=np.bool) data['board_dig_out_raw'] = np.zeros(num_board_dig_out_samples, dtype=np.uint) # Read sampled data from file. print('Reading data from file...') # Initialize indices used in looping indices = {} indices['amplifier'] = 0 indices['aux_input'] = 0 indices['supply_voltage'] = 0 indices['board_adc'] = 0 indices['board_dig_in'] = 0 indices['board_dig_out'] = 0 print_increment = 10 percent_done = print_increment for i in range(num_data_blocks): read_one_data_block(data, header, indices, fid) # Increment indices indices['amplifier'] += header['num_samples_per_data_block'] indices['aux_input'] += int(header['num_samples_per_data_block'] / 4) indices['supply_voltage'] += 1 indices['board_adc'] += header['num_samples_per_data_block'] indices['board_dig_in'] += header['num_samples_per_data_block'] indices['board_dig_out'] += header['num_samples_per_data_block'] fraction_done = 100 * (1.0 * i / num_data_blocks) if fraction_done >= percent_done: print('.') # print('{}% done...'.format(percent_done)) percent_done = percent_done + print_increment # Make sure we have read exactly the right amount of data. bytes_remaining = filesize - fid.tell() if bytes_remaining != 0: raise Exception('Error: End of file not reached.') # Close data file. fid.close() if (data_present): print('Parsing data...') # Extract digital input channels to separate variables. for i in range(header['num_board_dig_in_channels']): data['board_dig_in_data'][i, :] = np.not_equal( np.bitwise_and( data['board_dig_in_raw'], (1 << header['board_dig_in_channels'][i]['native_order'])), 0) # Extract digital output channels to separate variables. for i in range(header['num_board_dig_out_channels']): data['board_dig_out_data'][i, :] = np.not_equal( np.bitwise_and(data['board_dig_out_raw'], ( 1 << header['board_dig_out_channels'][i]['native_order'])), 0) # Scale voltage levels appropriately. # NOTE: Commented out to reduce size of file by 4x by storing # 16 bit ints in the resulting .npy # data['amplifier_data'] = np.multiply( # 0.195, (data['amplifier_data'].astype(np.int32) - 32768)) # units = microvolts data['aux_input_data'] = np.multiply( 37.4e-6, data['aux_input_data']) # units = volts data['supply_voltage_data'] = np.multiply( 74.8e-6, data['supply_voltage_data']) # units = volts if header['eval_board_mode'] == 1: data['board_adc_data'] = np.multiply( 152.59e-6, (data['board_adc_data'].astype(np.int32) - 32768)) # units = volts elif header['eval_board_mode'] == 13: data['board_adc_data'] = np.multiply( 312.5e-6, (data['board_adc_data'].astype(np.int32) - 32768)) # units = volts else: data['board_adc_data'] = np.multiply( 50.354e-6, data['board_adc_data']) # units = volts data['temp_sensor_data'] = np.multiply( 0.01, data['temp_sensor_data']) # units = deg C # Check for gaps in timestamps. num_gaps = np.sum( np.not_equal(data['t_amplifier'][1:] - data['t_amplifier'][:-1], 1)) assert num_gaps == 0 # We don't handle missing samples in all our downstream analysis # Scale time steps (units = seconds). data['t_amplifier'] = data['t_amplifier'] / header['sample_rate'] data['t_aux_input'] = data['t_amplifier'][range( 0, len(data['t_amplifier']), 4)] data['t_supply_voltage'] = data['t_amplifier'][range( 0, len(data['t_amplifier']), header['num_samples_per_data_block'])] data['t_board_adc'] = data['t_amplifier'] data['t_dig'] = data['t_amplifier'] data['t_temp_sensor'] = data['t_supply_voltage'] # If the software notch filter was selected during the recording, apply the # same notch filter to amplifier data here. # assert header['notch_filter_frequency'] == 0 if header['notch_filter_frequency'] > 0: print('Applying notch filter...') # print_increment = 10 # percent_done = print_increment # for i in range(header['num_amplifier_channels']): # data['amplifier_data'][i, :] = notch_filter( # data['amplifier_data'][i, :], header['sample_rate'], # header['notch_filter_frequency'], 10) # # fraction_done = 100 * (i / header['num_amplifier_channels']) # if fraction_done >= percent_done: # print('{}% done...'.format(percent_done)) # percent_done += print_increment else: data = [] # Move variables to result struct. result = data_to_result(header, data, data_present) # Add back in num samples for use in the load_experiment function result['num_amplifier_samples'] = data['num_amplifier_samples'] print('num_amplifier_samples', result['num_amplifier_samples']) print('Done! Elapsed time: {0:0.1f} seconds'.format(time.time() - tic)) return result
def _ConstantValue(tensor, partial): # TODO(touts): Support Variables? if not isinstance(tensor, ops.Tensor): raise TypeError("tensor is not a Tensor") if tensor.op.type == "Const": return MakeNdarray(tensor.op.get_attr("value")) elif tensor.op.type == "Shape": input_shape = tensor.op.inputs[0].get_shape() if input_shape.is_fully_defined(): return np.array([dim.value for dim in input_shape.dims], dtype=tensor.dtype.as_numpy_dtype) else: return None elif tensor.op.type == "Size": input_shape = tensor.op.inputs[0].get_shape() if input_shape.is_fully_defined(): return np.prod([dim.value for dim in input_shape.dims], dtype=np.int32) else: return None elif tensor.op.type == "Rank": input_shape = tensor.op.inputs[0].get_shape() if input_shape.ndims is not None: return np.ndarray(shape=(), buffer=np.array([input_shape.ndims], dtype=np.int32), dtype=np.int32) else: return None elif tensor.op.type == "Range": start = constant_value(tensor.op.inputs[0]) if start is None: return None limit = constant_value(tensor.op.inputs[1]) if limit is None: return None delta = constant_value(tensor.op.inputs[2]) if delta is None: return None return np.arange(start, limit, delta, dtype=tensor.dtype.as_numpy_dtype) elif tensor.op.type == "Cast": pre_cast = constant_value(tensor.op.inputs[0]) if pre_cast is None: return None cast_dtype = dtypes.as_dtype(tensor.op.get_attr("DstT")) return pre_cast.astype(cast_dtype.as_numpy_dtype) elif tensor.op.type == "Concat": dim = constant_value(tensor.op.inputs[0]) if dim is None: return None values = [] for x in tensor.op.inputs[1:]: value = constant_value(x) if value is None: return None values.append(value) return np.concatenate(values, axis=dim) elif tensor.op.type == "ConcatV2": dim = constant_value(tensor.op.inputs[-1]) if dim is None: return None values = [] for x in tensor.op.inputs[:-1]: value = constant_value(x) if value is None: return None values.append(value) return np.concatenate(values, axis=dim) elif tensor.op.type == "Pack": values = [] # Some imported GraphDefs have Pack ops with zero inputs. Those are invalid # and shouldn't be produced, but to deal sensibly with them here we check # and return None. if not tensor.op.inputs: return None # We can't handle axis != 0 Packs at the moment. if tensor.op.get_attr("axis") != 0: return None for x in tensor.op.inputs: value = constant_value(x, partial) if value is None and not partial: return None values.append(value) return np.array(values) elif tensor.op.type == "Fill": fill_shape = tensor.shape fill_value = constant_value(tensor.op.inputs[1]) if fill_shape.is_fully_defined() and fill_value is not None: return np.full(fill_shape.as_list(), fill_value, dtype=fill_value.dtype) else: return None elif tensor.op.type == "Equal": value1 = constant_value(tensor.op.inputs[0]) if value1 is None: return None value2 = constant_value(tensor.op.inputs[1]) if value2 is None: return None return np.equal(value1, value2) elif tensor.op.type == "NotEqual": value1 = constant_value(tensor.op.inputs[0]) if value1 is None: return None value2 = constant_value(tensor.op.inputs[1]) if value2 is None: return None return np.not_equal(value1, value2) else: return None
def get_exist_rewards(batch, anses): A_in_C = np.equal(np.array(batch[0]), anses[:, None, None]) exist_rewards = np.any(A_in_C, axis=(1, 2)) exist_rewards = np.logical_and(exist_rewards, np.not_equal(anses, 0)) return exist_rewards
def compare_dataframes(base, compare, return_orphans=True, ignore_case=True, print_info=False, convert_np_timestamps=True): """ Compare all common index and common column DataFrame values and report if any value is not equal in a returned dataframe. Values are compared only by index and column label, not order. Therefore, the only values compared are within common index rows and common columns. The routine will report the common columns and any unique index rows when the print_info option is selected (True). Inputs are pandas dataframes and/or pandas series. This function works well when comparing initial data lists, such as those which may be received from opposing parties. If return_orphans, returns tuple (diffs, base_loners, compare_loners), else returns diffs. diffs is a differential dataframe. inputs base baseline dataframe or series compare dataframe or series to compare against the baseline (base) return_orphans separately calculate and return the rows which are unique to base and compare ignore_case convert the column labels and column data to be compared to lowercase - this will avoid differences detected based on string case print_info option to print out to console verbose statistical information and the dataframe(s) instead of returning dataframe(s) convert_np_timestamps numpy returns datetime64 objects when the source is a datetime date-only object. this option will convert back to a date-only object for comparison. """ try: assert ((isinstance(base, pd.DataFrame)) | (isinstance(base, pd.Series))) and \ ((isinstance(compare, pd.DataFrame)) | (isinstance(compare, pd.Series))) except AssertionError: print('Routine aborted. Inputs must be a pandas dataframe or series.') return if isinstance(base, pd.Series): base = pd.DataFrame(base) if isinstance(compare, pd.Series): compare = pd.DataFrame(compare) common_rows = list(base.index[base.index.isin(compare.index)]) if print_info: print('\nROW AND INDEX INFORMATION:\n') print('base length:', len(base)) print('comp length:', len(compare)) print('common index count:', len(common_rows), '\n') # orphans section--------------------------------------------------------- if return_orphans: base_orphans = list(base.index[~base.index.isin(compare.index)]) compare_orphans = list(compare.index[~compare.index.isin(base.index)]) base_col_name = 'base_orphans' compare_col_name = 'compare_orphans' base_loners = pd.DataFrame(base_orphans, columns=[base_col_name]) compare_loners = pd.DataFrame(compare_orphans, columns=[compare_col_name]) def find_label_locs(df, orphans): loc_list = [] for orphan in orphans: loc_list.append(df.index.get_loc(orphan)) return loc_list if base_orphans: base_loners['index_loc'] = find_label_locs(base, base_orphans) if print_info: print('BASE LONERS (rows, by index):') print(base_loners, '\n') else: if print_info: print('''There are no unique index rows in the base input vs. the compare input.\n''') if compare_orphans: compare_loners['index_loc'] = find_label_locs( compare, compare_orphans) if print_info: print('COMPARE LONERS (rows, by index):') print(compare_loners, '\n') else: if print_info: print('''There are no unique index rows in the compare input vs. the base input.\n''') # ----------------------------------------------------------------------- base = base.loc[common_rows].copy() compare = compare.loc[common_rows].copy() unequal_cols = [] equal_cols = [] if ignore_case: base.columns = map(str.lower, base.columns) compare.columns = map(str.lower, compare.columns) common_cols = list(base.columns[base.columns.isin(compare.columns)]) base_only_cols = list(base.columns[~base.columns.isin(compare.columns)]) comp_only_cols = list(compare.columns[~compare.columns.isin(base.columns)]) oddballs = base_only_cols.copy() oddballs.extend(comp_only_cols) all_columns = common_cols.copy() all_columns.extend(oddballs) if print_info: same_col_list = [] print('\nCOMMON COLUMN equivalency:\n') for col in common_cols: if ignore_case: try: base[col] = base[col].str.lower() compare[col] = compare[col].str.lower() except: pass same_col = base[col].sort_index().equals(compare[col].sort_index()) if print_info: same_col_list.append(same_col) if not same_col: unequal_cols.append(col) else: equal_cols.append(col) base = base[unequal_cols] compare = compare[unequal_cols] if print_info: same_col_df = pd.DataFrame(list(zip(common_cols, same_col_list)), columns=['common_col', 'equivalent?']) same_col_df.sort_values(['equivalent?', 'common_col'], inplace=True) same_col_df.reset_index(drop=True, inplace=True) print(same_col_df, '\n') print('\nCOLUMN INFORMATION:') print('\ncommon columns:\n', common_cols) print('\ncommon and equal columns:\n', equal_cols) print('\ncommon but unequal columns:\n', unequal_cols) print('\ncols only in base:\n', base_only_cols) print('\ncols only in compare:\n', comp_only_cols, '\n') col_df = pd.DataFrame(index=[all_columns]) column_names = [ 'equal_cols', 'unequal_cols', 'common_cols', 'base_only_cols', 'comp_only_cols', 'all_columns' ] for result_name in column_names: i = 0 col_arr = np.empty_like(all_columns) for name in all_columns: if name in eval(result_name): col_arr[i] = name i += 1 col_df[result_name] = col_arr col_df.sort_values(['unequal_cols', 'equal_cols'], inplace=True) col_df.reset_index(drop=True, inplace=True) col_df.rename(columns={ 'unequal_cols': 'not_equal', 'base_only_cols': 'base_only', 'comp_only_cols': 'comp_only' }, inplace=True) print('\nCATEGORIZED COLUMN DATAFRAME:\n') print(col_df, '\n') zipped = [] col_counts = [] with warnings.catch_warnings(): warnings.simplefilter('ignore', category=FutureWarning) for col in base: base_np = base[col].values compare_np = compare[col].values try: unequal = np.not_equal(base_np, compare_np) except: try: mask = base.duplicated(subset=col, keep=False) dups = list(base[mask][col]) print('error, duplicate values:') print(pd.DataFrame(dups, columns=['dups'])) except: pass row_ = np.where(unequal)[0] index_ = base.iloc[row_].index col_ = np.array([col] * row_.size) base_ = base_np[unequal] compare_ = compare_np[unequal] if (base[col]).dtype == 'datetime64[ns]' and convert_np_timestamps: try: base_ = base_.astype('M8[D]') compare_ = compare_.astype('M8[D]') except: pass zipped.extend(list(zip(row_, index_, col_, base_, compare_))) col_counts.append(row_.size) diffs = pd.DataFrame(zipped, columns=['row', 'index', 'column', 'base', 'compare']) diffs.sort_values('row', inplace=True) diffs.reset_index(drop=True, inplace=True) if print_info: print('\nDIFFERENTIAL DATAFRAME:\n') print(diffs) print('\nSUMMARY:\n') print('''{!r} total differences found in common rows and columns\n'''.format(len(zipped))) if len(zipped) == 0: print('''Comparison complete, dataframes are equivalent. \nIndex and Column order may be different\n''') else: print( 'Breakdown by column:\n', pd.DataFrame(list(zip(base.columns, col_counts)), columns=['column', 'diff_count']), '\n') else: if return_orphans: return diffs, base_loners, compare_loners else: return diffs
def apply_perturbation(self, perturbed_nodes, params, weights, parallel, kind='element'): """ Perturbation simulator, actually applying the perturbation to all the nodes affected by the perturbation. The optimizer is run if any switch is present, and edges connecting its predecessors are removed if the switch state is set to 'False'. :param list perturbed_nodes: nodes(s) involved in the perturbing event. :param dict params: values for the optimizer evolutionary algorithm. Dict of: {str: int, str: int, str: float, str: float, str: int}: - 'npop': number of individuals for each population (default to 300) - 'ngen': total number of generations (default to 100) - 'indpb': independent probability for attributes to be changed (default to 0.6) - 'tresh': threshold for applying crossover/mutation (default to 0.5) - 'nsel': number of individuals to select (default to 5) :param dict weights: weights for fitness evaluation on individuals. Dict of: {str: float, str: float, str: float}: - 'w1': weight multiplying number of actions (default to 1.0) - 'w2': weight multiplying total final service (default to 1.0) - 'w3': weight multiplying final graph size (default to 1.0) :param bool parallel: flag for parallel fitness evaluation of initial population :param str kind: type of simulation, used to label output files, default to 'element' .. note:: A perturbation, depending on the considered system, may spread in all directions starting from the damaged component(s) and may be affect nearby areas. """ if self.G.switches: res = np.array( self.optimizer(perturbed_nodes, self.G.init_status, params, weights, parallel)) best = dict( zip(self.G.init_status.keys(), res[np.argmin(res[:, 1]), 0])) initial_condition_sw = list(self.G.init_status.values()) final_condition_sw = list(best.values()) flips = dict( zip(self.G.init_status.keys(), np.not_equal(initial_condition_sw, final_condition_sw))) init_open_edges = {} for sw, closed in self.G.init_status.items(): if not closed: logging.debug(f'Opened switch {sw} in initial configuration') for pred in list(self.G.predecessors(sw)): # if final config is closed for this switch I memorize it if flips[sw]: init_open_edges[sw] = {pred: self.G[pred][sw]} self.G.remove_edge(pred, sw) self.check_paths_and_measures(prefix='original_') self.G.clear_data([ 'shortest_path', 'shortest_path_length', 'efficiency', 'nodal_efficiency', 'local_efficiency', 'computed_service', 'closeness_centrality', 'betweenness_centrality', 'indegree_centrality', 'outdegree_centrality', 'degree_centrality' ]) if self.G.switches: for sw, closed in best.items(): if flips[sw]: if not closed: logging.debug( f'Switch {sw} finally open, first closed') for pre in list(self.G.predecessors(sw)): self.G.remove_edge(pre, sw) else: logging.debug( f'Switch {sw} finally closed, first open') for pre, attrs in init_open_edges[sw].items(): self.G.add_edge(pre, sw, **attrs) logging.debug(f'BEST: {best}, with fitness: {np.min(res[:, 1])}') self.G.final_status = best for node in perturbed_nodes: if node in self.G.nodes(): self.delete_a_node(node) self.check_paths_and_measures(prefix='final_') self.paths_df.to_csv('service_paths_' + str(kind) + '_perturbation.csv', index=False) status_area_fields = ['final_status', 'mark_status', 'status_area'] self.update_output(status_area_fields) self.update_status_areas(self.damaged_areas) self.graph_characterization_to_file(str(kind) + '_perturbation.csv')
# coding: utf-8 #from toy_problems import Data import numpy as np from sklearn.semi_supervised import LabelPropagation, LabelSpreading from new_stability import * GAMMA = 3e-6 original = np.load("datasets/spines2.npz") shapes = np.sum(original["shapes_n"], axis=2) ##at least one classifier chose label for a spine both_label_vector = np.logical_and(np.not_equal(original["kl1"], None), np.not_equal(original["kl2"], None)) ##both classyfiers agreed on spine both_match_vector = np.logical_and(both_label_vector, original["kl1"] == original["kl2"]) ##both stubby, long & thin or mushroom longs = original["kl1"] == "Long & thin" mushrooms = original["kl1"] == "Mushroom" stubbies = original["kl1"] == "Stubby" ##roznice w populacjach poszczegolnych typow #print sum(longs), sum(mushrooms), sum(stubbies) both_match_long_stubby_mush = lsm = both_match_vector & (longs | mushrooms | stubbies) def getIndicesOfTrue(vector): return np.array([i for i, x in enumerate(vector) if x]) def getLearningAndThrowAwayInd(len_learn, len_throw_away, learnable_obs=lsm):
def read_snap_dataset_as_list(dir, name): list_dir = os.listdir(dir) if len(list_dir) == 1 and osp.isdir(osp.join(dir, list_dir[0])): dir = osp.join(dir, list_dir[0]) if 'ego-' in name: files = [ file for file in os.listdir(dir) if osp.isfile(osp.join(dir, file)) ] files.sort() data_list = [] for i in range(5, len(files), 5): circles_file = files[i] edges_file = files[i + 1] egofeat_file = files[i + 2] feat_file = files[i + 3] # featnames_file = files[i+4] x = torch.from_numpy(np.loadtxt(osp.join(dir, feat_file))).long() indices = x[:, 0] indices_assoc = to_assoc(indices) x = x[:, 1:] circles = [] f = open(osp.join(dir, circles_file), "r") c_line = f.readline() while not c_line == '': circles.append([ from_assoc(indices_assoc, int(i)) for i in c_line.split()[1:] ]) c_line = f.readline() f.close() edge_index = np.loadtxt(osp.join(dir, edges_file)) edge_index = torch.from_numpy(edge_index).transpose(0, 1).long() # TODO find more efficient way to do this for i in range(edge_index.shape[0]): for j in range(edge_index.shape[1]): edge_index[i][j] = from_assoc(indices_assoc, edge_index[i][j].item()) x_ego = np.loadtxt(osp.join(dir, egofeat_file)) x_ego = torch.from_numpy(x_ego).long() x = torch.cat((x, x_ego.unsqueeze(0))) # ego Node is connected so every other node edge_index_ego = torch.fill_(torch.zeros((2, x.shape[0] - 1)), x.shape[0] - 1) edge_index_ego[0] = torch.arange(x.shape[0] - 1) # nodes undirected in ego-Facebook if name == 'ego-Facebook': edge_index_ego2 = torch.fill_(torch.zeros((2, x.shape[0] - 1)), x.shape[0] - 1) edge_index_ego2[1] = torch.arange(x.shape[0] - 1) edge_index = torch.cat((edge_index, edge_index_ego.long(), edge_index_ego2.long()), dim=1) edge_index = torch.cat((edge_index, edge_index_ego.long()), dim=1) data = Data(x=x, edge_index=edge_index, circles=circles) data_list.append(data) return data_list elif 'soc-' in name: if name == 'soc-Pokec': # TODO? read out features from 'soc-pokec-profiles.txt' edge_index = np.loadtxt( osp.join(dir, 'soc-pokec-relationships.txt')) edge_index = torch.from_numpy(edge_index).transpose(0, 1).long() data = Data(edge_index=edge_index) return [data] else: list_dir = os.listdir(dir) if len(list_dir) == 1 and osp.isfile(osp.join(dir, list_dir[0])): edge_index = np.loadtxt(osp.join(dir, list_dir[0])) ids = np.unique(edge_index) for i, j in zip(ids, range(len(ids))): edge_index[edge_index == i] = j assert np.sum( np.not_equal(np.unique(edge_index), np.arange(len(ids)))) == 0 edge_index = torch.from_numpy(edge_index).transpose(0, 1)\ .long() data = Data(edge_index=edge_index) return [data] elif 'wiki-' in name: if name == 'wiki-Vote': list_dir = os.listdir(dir) if len(list_dir) == 1 and osp.isfile(osp.join(dir, list_dir[0])): edge_index = np.loadtxt(osp.join(dir, list_dir[0])) ids = np.unique(edge_index) for i, j in zip(ids, range(len(ids))): edge_index[edge_index == i] = j assert np.sum( np.not_equal(np.unique(edge_index), np.arange(len(ids)))) == 0 edge_index = torch.from_numpy(edge_index).transpose(0, 1)\ .long() data = Data(edge_index=edge_index) return [data] elif name == 'wiki-RfA': list_dir = os.listdir(dir) if len(list_dir) == 1 and osp.isfile(osp.join(dir, list_dir[0])): i = 0 with open(osp.join(dir, list_dir[0])) as f: line = f.readline() while not line == '': print(i, line) if i == 10: raise i += 1 line = f.readline()
KDTree_Y = np.matrix(Result).astype(int).transpose() KDTree_Y_class0 = np.matrix(Result[1] <= 5000).astype(int).transpose() KDTree_Y_class1 = np.matrix(Result[1] > 5000).astype(int).transpose() accuracy = (KDTree_Y == test_Y).mean() error = (KDTree_Y != test_Y).mean() print('Accuracy_KDT', accuracy) print('Error_KDT', error) Correct_class0_kdt = np.where(np.equal(test_Y, KDTree_Y_class0))[0].tolist() Elements_Correctclass0_kdt = training_X[Correct_class0_kdt] Elements_Correctclass0_kdt_x = Elements_Correctclass0[:, 0] Elements_Correctclass0_kdt_y = Elements_Correctclass0[:, 1] Correct_class1_kdt = np.where(np.equal(test_Y, KDTree_Y_class1))[0].tolist() Elements_Correctclass1_kdt = training_X[Correct_class1_kdt] Elements_Correctclass1_kdt_x = Elements_Correctclass1[:, 0] Elements_Correctclass1_kdt_y = Elements_Correctclass1[:, 1] Incorrect_class0_kdt = np.where(np.not_equal(test_Y, KDTree_Y_class0))[0].tolist() Elements_Incorrectclass0_kdt = training_X[Incorrect_class0_kdt] Elements_Incorrectclass0_kdt_x = Elements_Correctclass0[:, 0] Elements_Incorrectclass0_kdt_y = Elements_Correctclass0[:, 1] Incorrect_class1_kdt = np.where(np.not_equal(test_Y, KDTree_Y_class0))[0].tolist() Elements_Incorrectclass1_kdt = training_X[Incorrect_class1_kdt] Elements_Incorrectclass1_kdt_x = Elements_Correctclass1[:, 0] Elements_Incorrectclass1_kdt_y = Elements_Correctclass1[:, 1] mplot.plot(Training_class0_x, Training_class0_y, 'x', color='b') mplot.plot(Training_class1_x, Training_class1_y, 'x', color='m') mplot.plot(Elements_Correctclass0_kdt_x, Elements_Correctclass0_kdt_y, 'x', color='r') mplot.plot(Elements_Correctclass1_kdt_x,
def test_compress_video_call(self): test_input = np.arange(12).reshape(1, 3, 1, 2, 2) video_compression = VideoCompression(video_format="mp4", constant_rate_factor=50, channels_first=True) assert np.any(np.not_equal(video_compression(test_input)[0], test_input))
def divz(X,Y): return X/np.where(Y,Y,Y+1)*np.not_equal(Y,0)
def fit(self): """ Trains a model """ if self.optimizers is None: self.setup_optimizers() num_train = self.dataset.train_X.shape[0] loss_history = [] train_acc_history = [] val_acc_history = [] for epoch in range(self.num_epochs): shuffled_indices = np.arange(num_train) np.random.shuffle(shuffled_indices) sections = np.arange(self.batch_size, num_train, self.batch_size) batches_indices = np.array_split(shuffled_indices, sections) batch_losses = [] for batch_indices in batches_indices: # TODO Generate batches based on batch_indices and # use model to generate loss and gradients for all # the params # в качестве модели - self.model !!! #raise Exception("Not implemented!") loss = self.model.compute_loss_and_gradients( self.dataset.train_X[batch_indices], self.dataset. train_y[batch_indices]) # прошли вперед, назад, # посчитали градиент for param_name, param in self.model.params().items(): optimizer = self.optimizers[param_name] param.value = optimizer.update(param.value, param.grad, self.learning_rate) batch_losses.append(loss) if np.not_equal(self.learning_rate_decay, 1.0): # TODO: Implement learning rate decay #raise Exception("Not implemented!") self.learning_rate *= self.learning_rate_decay ave_loss = np.mean(batch_losses) train_accuracy = self.compute_accuracy(self.dataset.train_X, self.dataset.train_y) val_accuracy = self.compute_accuracy(self.dataset.val_X, self.dataset.val_y) print("Loss: %f, Train accuracy: %f, val accuracy: %f" % (batch_losses[-1], train_accuracy, val_accuracy)) loss_history.append(ave_loss) train_acc_history.append(train_accuracy) val_acc_history.append(val_accuracy) return loss_history, train_acc_history, val_acc_history
def test_broadcast(): a = sym.Variable("a") b = sym.Variable("b") shape = {'a': (3, 4, 5), 'b': (1, 5)} def _collapse(g): return g.reshape(-1, shape['b'][-1]).sum(0, keepdims=True) y = sym.broadcast_add(a, b) def _backward_add(head_grads, a, b): da = head_grads db = _collapse(head_grads) return da, db check_function(y, lambda a, b: a + b, _backward_add, shape=shape) y = sym.broadcast_sub(a, b) def _backward_sub(head_grads, a, b): da = head_grads db = -_collapse(head_grads) return da, db check_function(y, lambda a, b: a - b, _backward_sub, shape=shape) y = sym.broadcast_mul(a, b) def _backward_mul(head_grads, a, b): da = head_grads * b db = _collapse(head_grads * a) return da, db check_function(y, lambda a, b: a * b, _backward_mul, shape=shape) y = sym.broadcast_div(a, b) def _backward_div(head_grads, a, b): da = head_grads / b db = _collapse(- head_grads * a / b**2) return da, db # We avoid computing numerical derivatives too close to zero here check_function(y, lambda a, b: a / b, _backward_div, shape=shape, numerical_grads=False) check_function(y, lambda a, b: a / b, _backward_div, shape=shape, in_range={'b': (0.1, 20)}) y = sym.broadcast_mod(a, b) check_function(y, lambda a, b: np.mod(a, b), in_range={'a': (0.001, 100), 'b': (1, 100)}, dtype='int32', shape=shape) y = sym.broadcast_max(a, b) check_function(y, lambda a, b: np.maximum(a, b), shape=shape) y = sym.broadcast_min(a, b) check_function(y, lambda a, b: np.minimum(a, b), shape=shape) y = sym.broadcast_pow(a, b) check_function(y, lambda a, b: np.power(a, b), in_range={'a': (0.001, 100), 'b': (0.001, 2)}, shape=shape) y = sym.broadcast_left_shift(a, b) check_function(y, lambda a, b: a << b, dtype='int32', shape=shape) y = sym.broadcast_right_shift(a, b) check_function(y, lambda a, b: a >> b, dtype='int32', shape=shape) y = sym.broadcast_greater(a, b) check_function(y, lambda a, b: np.greater(a, b), shape=shape) y = sym.broadcast_less(a, b) check_function(y, lambda a, b: np.less(a, b), shape=shape) y = sym.broadcast_equal(a, b) check_function(y, lambda a, b: np.equal(a, b), in_range={'a': (-2, 2), 'b': (-2, 2)}, dtype='int32', shape=shape) y = sym.broadcast_not_equal(a, b) check_function(y, lambda a, b: np.not_equal(a, b), in_range={'a': (-2, 2), 'b': (-2, 2)}, dtype='int32', shape=shape) y = sym.broadcast_greater_equal(a, b) check_function(y, lambda a, b: np.greater_equal(a, b), in_range={'a': (-3, 3), 'b': (-3, 3)}, dtype='int32', shape=shape) y = sym.broadcast_less_equal(a, b) check_function(y, lambda a, b: np.less_equal(a, b), in_range={'a': (-3, 3), 'b': (-3, 3)}, dtype='int32', shape=shape)
def test_half_ufuncs(self): """Test the various ufuncs""" a = np.array([0, 1, 2, 4, 2], dtype=float16) b = np.array([-2, 5, 1, 4, 3], dtype=float16) c = np.array([0, -1, -np.inf, np.nan, 6], dtype=float16) assert_equal(np.add(a, b), [-2, 6, 3, 8, 5]) assert_equal(np.subtract(a, b), [2, -4, 1, 0, -1]) assert_equal(np.multiply(a, b), [0, 5, 2, 16, 6]) assert_equal(np.divide(a, b), [0, 0.199951171875, 2, 1, 0.66650390625]) assert_equal(np.equal(a, b), [False, False, False, True, False]) assert_equal(np.not_equal(a, b), [True, True, True, False, True]) assert_equal(np.less(a, b), [False, True, False, False, True]) assert_equal(np.less_equal(a, b), [False, True, False, True, True]) assert_equal(np.greater(a, b), [True, False, True, False, False]) assert_equal(np.greater_equal(a, b), [True, False, True, True, False]) assert_equal(np.logical_and(a, b), [False, True, True, True, True]) assert_equal(np.logical_or(a, b), [True, True, True, True, True]) assert_equal(np.logical_xor(a, b), [True, False, False, False, False]) assert_equal(np.logical_not(a), [True, False, False, False, False]) assert_equal(np.isnan(c), [False, False, False, True, False]) assert_equal(np.isinf(c), [False, False, True, False, False]) assert_equal(np.isfinite(c), [True, True, False, False, True]) assert_equal(np.signbit(b), [True, False, False, False, False]) assert_equal(np.copysign(b, a), [2, 5, 1, 4, 3]) assert_equal(np.maximum(a, b), [0, 5, 2, 4, 3]) x = np.maximum(b, c) assert_(np.isnan(x[3])) x[3] = 0 assert_equal(x, [0, 5, 1, 0, 6]) assert_equal(np.minimum(a, b), [-2, 1, 1, 4, 2]) x = np.minimum(b, c) assert_(np.isnan(x[3])) x[3] = 0 assert_equal(x, [-2, -1, -np.inf, 0, 3]) assert_equal(np.fmax(a, b), [0, 5, 2, 4, 3]) assert_equal(np.fmax(b, c), [0, 5, 1, 4, 6]) assert_equal(np.fmin(a, b), [-2, 1, 1, 4, 2]) assert_equal(np.fmin(b, c), [-2, -1, -np.inf, 4, 3]) assert_equal(np.floor_divide(a, b), [0, 0, 2, 1, 0]) assert_equal(np.remainder(a, b), [0, 1, 0, 0, 2]) assert_equal(np.divmod(a, b), ([0, 0, 2, 1, 0], [0, 1, 0, 0, 2])) assert_equal(np.square(b), [4, 25, 1, 16, 9]) assert_equal(np.reciprocal(b), [-0.5, 0.199951171875, 1, 0.25, 0.333251953125]) assert_equal(np.ones_like(b), [1, 1, 1, 1, 1]) assert_equal(np.conjugate(b), b) assert_equal(np.absolute(b), [2, 5, 1, 4, 3]) assert_equal(np.negative(b), [2, -5, -1, -4, -3]) assert_equal(np.positive(b), b) assert_equal(np.sign(b), [-1, 1, 1, 1, 1]) assert_equal(np.modf(b), ([0, 0, 0, 0, 0], b)) assert_equal(np.frexp(b), ([-0.5, 0.625, 0.5, 0.5, 0.75], [2, 3, 1, 3, 2])) assert_equal(np.ldexp(b, [0, 1, 2, 4, 2]), [-2, 10, 4, 64, 12])
verbose_ops_dict={ 'data': data_ph, 'labels': label_ph, 'mu': model.z_mu, 'sigma': model.z_sigma, 'codes': model.latent_posterior_sample }, feed_dict_fn=test_feed_dict_fn) data_vals = np.stack(eval_dict['data']) label_vals = np.argmax(np.stack(eval_dict['labels']), axis=1) mu_vals = np.stack(eval_dict['mu']) sigma_vals = np.stack(eval_dict['sigma']) code_vals = np.concatenate(eval_dict['codes'], axis=0) prediction_vals = model._decision_tree.predict(code_vals) mislabeled_mask = np.not_equal(label_vals, prediction_vals) mislabeled_idxs = np.arange(len(data_vals))[mislabeled_mask] mislabeled_data = data_vals[mislabeled_mask] mislabeled_label = label_vals[mislabeled_mask] mislabeled_mu = mu_vals[mislabeled_mask] mislabeled_sigma = sigma_vals[mislabeled_mask] mislabeled_code = code_vals[mislabeled_mask] mislabeled_prediction = prediction_vals[mislabeled_mask] c_means, c_sds = model.aggregate_posterior_parameters( session, label_ph, train_batches_per_epoch, train_feed_dict_fn) latent_code_ph = tf.placeholder(tf.float32) for id, instance_mu, instance_sigma, label, prediction in zip( mislabeled_idxs, mislabeled_mu, mislabeled_sigma, mislabeled_label, mislabeled_prediction):
def assign_instances_for_scan(scene_name, pred_info, gt_file): try: gt_ids = util_3d.load_ids(gt_file) except Exception as e: util.print_error('unable to load ' + gt_file + ': ' + str(e)) # get gt instances gt_instances = util_3d.get_instances(gt_ids, VALID_CLASS_IDS, CLASS_LABELS, ID_TO_LABEL) # associate gt2pred = gt_instances.copy() for label in gt2pred: for gt in gt2pred[label]: gt['matched_pred'] = [] pred2gt = {} for label in CLASS_LABELS: pred2gt[label] = [] num_pred_instances = 0 # mask of void labels in the groundtruth bool_void = np.logical_not(np.in1d(gt_ids // 1000, VALID_CLASS_IDS)) # go thru all prediction masks nMask = pred_info['label_id'].shape[0] for i in range(nMask): label_id = int(pred_info['label_id'][i]) conf = pred_info['conf'][i] if not label_id in ID_TO_LABEL: continue label_name = ID_TO_LABEL[label_id] # read the mask pred_mask = pred_info['mask'][i] # (N), long if len(pred_mask) != len(gt_ids): util.print_error('wrong number of lines in mask#%d: ' % (i) + '(%d) vs #mesh vertices (%d)' % (len(pred_mask), len(gt_ids))) # convert to binary pred_mask = np.not_equal(pred_mask, 0) num = np.count_nonzero(pred_mask) if num < MIN_REGION_SIZES[0]: continue # skip if empty pred_instance = {} pred_instance['filename'] = '{}_{:03d}'.format(scene_name, num_pred_instances) pred_instance['pred_id'] = num_pred_instances pred_instance['label_id'] = label_id pred_instance['vert_count'] = num pred_instance['confidence'] = conf pred_instance['void_intersection'] = np.count_nonzero( np.logical_and(bool_void, pred_mask)) # matched gt instances matched_gt = [] # go thru all gt instances with matching label for (gt_num, gt_inst) in enumerate(gt2pred[label_name]): intersection = np.count_nonzero( np.logical_and(gt_ids == gt_inst['instance_id'], pred_mask)) if intersection > 0: gt_copy = gt_inst.copy() pred_copy = pred_instance.copy() gt_copy['intersection'] = intersection pred_copy['intersection'] = intersection matched_gt.append(gt_copy) gt2pred[label_name][gt_num]['matched_pred'].append(pred_copy) pred_instance['matched_gt'] = matched_gt num_pred_instances += 1 pred2gt[label_name].append(pred_instance) return gt2pred, pred2gt
def erode(self, dt, flooded_depths=None, **kwds): """Erode and deposit on the channel bed for a duration of *dt*. Erosion occurs according to the sediment dependent rules specified during initialization. Parameters ---------- dt : float (years, only!) Timestep for which to run the component. flooded_depths : array or field name (m) Depths of flooding at each node, zero where no lake. Note that the component will dynamically update this array as it fills nodes with sediment (...but does NOT update any other related lake fields). """ grid = self.grid node_z = grid.at_node['topographic__elevation'] node_A = grid.at_node['drainage_area'] flow_receiver = grid.at_node['flow__receiver_node'] s_in = grid.at_node['flow__upstream_node_order'] node_S = grid.at_node['topographic__steepest_slope'] if type(flooded_depths) is str: flooded_depths = mg.at_node[flooded_depths] # also need a map of initial flooded conds: flooded_nodes = flooded_depths > 0. elif type(flooded_depths) is np.ndarray: assert flooded_depths.size == self.grid.number_of_nodes flooded_nodes = flooded_depths > 0. # need an *updateable* record of the pit depths else: # if None, handle in loop flooded_nodes = None steepest_link = 'flow__link_to_receiver_node' link_length = np.empty(grid.number_of_nodes, dtype=float) link_length.fill(np.nan) draining_nodes = np.not_equal(grid.at_node[steepest_link], BAD_INDEX_VALUE) core_draining_nodes = np.intersect1d(np.where(draining_nodes)[0], grid.core_nodes, assume_unique=True) link_length[core_draining_nodes] = grid.length_of_d8[ grid.at_node[steepest_link][core_draining_nodes]] if self.Qc == 'MPM': if self.Dchar_in is not None: self.Dchar = self.Dchar_in else: assert not self.set_threshold, ( "Something is seriously wrong with your model " + "initialization.") assert self.override_threshold, ( "You need to confirm to the module you intend it to " + "internally calculate a shear stress threshold, " + "with set_threshold_from_Dchar in the input file.") # we need to adjust the thresholds for the Shields number # & gs dynamically: variable_thresh = self.shields_crit * self.g * ( self.sed_density - self.fluid_density) * self.Dchar if self.lamb_flag: variable_shields_crit = 0.15 * node_S**0.25 try: variable_thresh = (variable_shields_crit * self.shields_prefactor_to_shear) except AttributeError: variable_thresh = ( variable_shields_crit * self.shields_prefactor_to_shear_noDchar * self.Dchar) node_Q = self.k_Q * self.runoff_rate * node_A**self._c shear_stress_prefactor_timesAparts = (self.shear_stress_prefactor * node_Q**self.point6onelessb) try: transport_capacities_thresh = ( self.thresh * self.Qs_thresh_prefactor * self.runoff_rate**(0.66667 * self._b) * node_A**self.Qs_power_onAthresh) except AttributeError: transport_capacities_thresh = ( variable_thresh * self.Qs_thresh_prefactor * self.runoff_rate**(0.66667 * self._b) * node_A**self.Qs_power_onAthresh) transport_capacity_prefactor_withA = ( self.Qs_prefactor * self.runoff_rate**(0.6 + self._b / 15.) * node_A**self.Qs_power_onA) internal_t = 0. break_flag = False dt_secs = dt * 31557600. counter = 0 rel_sed_flux = np.empty_like(node_Q) # excess_vol_overhead = 0. while 1: # ^use the break flag, to improve computational efficiency for # runs which are very stable # we assume the drainage structure is forbidden to change # during the whole dt # note slopes will be *negative* at pits # track how many loops we perform: counter += 1 downward_slopes = node_S.clip(0.) # this removes the tendency to transfer material against # gradient, including in any lake depressions # we DON'T immediately zero trp capacity in the lake. # positive_slopes = np.greater(downward_slopes, 0.) slopes_tothe07 = downward_slopes**0.7 transport_capacities_S = (transport_capacity_prefactor_withA * slopes_tothe07) trp_diff = (transport_capacities_S - transport_capacities_thresh).clip(0.) transport_capacities = np.sqrt(trp_diff * trp_diff * trp_diff) shear_stress = (shear_stress_prefactor_timesAparts * slopes_tothe07) shear_tothe_a = shear_stress**self._a dt_this_step = dt_secs - internal_t # ^timestep adjustment is made AFTER the dz calc node_vol_capacities = transport_capacities * dt_this_step sed_into_node = np.zeros(grid.number_of_nodes, dtype=float) dz = np.zeros(grid.number_of_nodes, dtype=float) len_s_in = s_in.size cell_areas = self.cell_areas try: raise NameError # ^tripped out deliberately for now; doesn't appear to # accelerate much weave.inline(self.routing_code, [ 'len_s_in', 'sed_into_node', 'transport_capacities', 'dz', 'cell_areas', 'dt_this_step', 'flow__receiver_node' ]) except NameError: for i in s_in[::-1]: # work downstream cell_area = cell_areas[i] if flooded_nodes is not None: flood_depth = flooded_depths[i] else: flood_depth = 0. sed_flux_into_this_node = sed_into_node[i] node_capacity = transport_capacities[i] # ^we work in volume flux, not volume per se here node_vol_capacity = node_vol_capacities[i] if flood_depth > 0.: node_vol_capacity = 0. # requires special case handling - as much sed as # possible is dumped here, then the remainder # passed on if sed_flux_into_this_node < node_vol_capacity: # ^note incision is forbidden at capacity # flooded nodes never enter this branch # #implementing the pseudoimplicit method: try: thresh = variable_thresh except: # it doesn't exist thresh = self.thresh dz_prefactor = self._K_unit_time * dt_this_step * ( shear_tothe_a[i] - thresh).clip(0.) vol_prefactor = dz_prefactor * cell_area (dz_here, sed_flux_out, rel_sed_flux_here, error_in_sed_flux) = \ self.get_sed_flux_function_pseudoimplicit( sed_flux_into_this_node, node_vol_capacity, vol_prefactor, dz_prefactor) # note now dz_here may never create more sed than # the out can transport... assert sed_flux_out <= node_vol_capacity, ( 'failed at node ' + str(s_in.size - i) + ' with rel sed flux ' + str(sed_flux_out / node_capacity)) rel_sed_flux[i] = rel_sed_flux_here vol_pass = sed_flux_out else: rel_sed_flux[i] = 1. vol_dropped = (sed_flux_into_this_node - node_vol_capacity) dz_here = -vol_dropped / cell_area # with the pits, we aim to inhibit incision, but # depo is OK. We have already zero'd any adverse # grads, so sed can make it to the bottom of the # pit but no further in a single step, which seems # raeasonable. Pit should fill. if flood_depth <= 0.: vol_pass = node_vol_capacity else: height_excess = -dz_here - flood_depth # ...above water level if height_excess <= 0.: vol_pass = 0. # dz_here is already correct flooded_depths[i] += dz_here else: dz_here = -flood_depth vol_pass = height_excess * cell_area # ^bit cheeky? flooded_depths[i] = 0. # note we must update flooded depths # transiently to conserve mass # do we need to retain a small downhill slope? # ...don't think so. Will resolve itself on next # timestep. dz[i] -= dz_here sed_into_node[flow_receiver[i]] += vol_pass break_flag = True node_z[grid.core_nodes] += dz[grid.core_nodes] if break_flag: break # do we need to reroute the flow/recalc the slopes here? # -> NO, slope is such a minor component of Diff we'll be OK # BUT could be important not for the stability, but for the # actual calc. So YES. node_S = np.zeros_like(node_S) node_S[core_draining_nodes] = ( node_z - node_z[flow_receiver] )[core_draining_nodes] / link_length[core_draining_nodes] internal_t += dt_this_step # still in seconds, remember elif self.Qc == 'power_law': transport_capacity_prefactor_withA = self._Kt * node_A**self._mt erosion_prefactor_withA = self._K_unit_time * node_A**self._m # ^doesn't include S**n*f(Qc/Qc) internal_t = 0. break_flag = False dt_secs = dt * 31557600. counter = 0 rel_sed_flux = np.empty_like(node_A) while 1: counter += 1 # print counter downward_slopes = node_S.clip(0.) # positive_slopes = np.greater(downward_slopes, 0.) slopes_tothen = downward_slopes**self._n slopes_tothent = downward_slopes**self._nt transport_capacities = (transport_capacity_prefactor_withA * slopes_tothent) erosion_prefactor_withS = (erosion_prefactor_withA * slopes_tothen) # no time, no fqs # shear_tothe_a = shear_stress**self._a dt_this_step = dt_secs - internal_t # ^timestep adjustment is made AFTER the dz calc node_vol_capacities = transport_capacities * dt_this_step sed_into_node = np.zeros(grid.number_of_nodes, dtype=float) dz = np.zeros(grid.number_of_nodes, dtype=float) cell_areas = self.cell_areas for i in s_in[::-1]: # work downstream cell_area = cell_areas[i] if flooded_nodes is not None: flood_depth = flooded_depths[i] else: flood_depth = 0. sed_flux_into_this_node = sed_into_node[i] node_capacity = transport_capacities[i] # ^we work in volume flux, not volume per se here node_vol_capacity = node_vol_capacities[i] if flood_depth > 0.: node_vol_capacity = 0. if sed_flux_into_this_node < node_vol_capacity: # ^note incision is forbidden at capacity dz_prefactor = dt_this_step * erosion_prefactor_withS[i] vol_prefactor = dz_prefactor * cell_area (dz_here, sed_flux_out, rel_sed_flux_here, error_in_sed_flux) = \ self.get_sed_flux_function_pseudoimplicit( sed_flux_into_this_node, node_vol_capacity, vol_prefactor, dz_prefactor) # note now dz_here may never create more sed than the # out can transport... assert sed_flux_out <= node_vol_capacity, ( 'failed at node ' + str(s_in.size - i) + ' with rel sed flux ' + str(sed_flux_out / node_capacity)) rel_sed_flux[i] = rel_sed_flux_here vol_pass = sed_flux_out else: rel_sed_flux[i] = 1. vol_dropped = (sed_flux_into_this_node - node_vol_capacity) dz_here = -vol_dropped / cell_area try: isflooded = flooded_nodes[i] except TypeError: # was None isflooded = False if flood_depth <= 0. and not isflooded: vol_pass = node_vol_capacity # we want flooded nodes which have already been # filled to enter the else statement else: height_excess = -dz_here - flood_depth # ...above water level if height_excess <= 0.: vol_pass = 0. # dz_here is already correct flooded_depths[i] += dz_here else: dz_here = -flood_depth vol_pass = height_excess * cell_area # ^bit cheeky? flooded_depths[i] = 0. dz[i] -= dz_here sed_into_node[flow_receiver[i]] += vol_pass break_flag = True node_z[grid.core_nodes] += dz[grid.core_nodes] if break_flag: break # do we need to reroute the flow/recalc the slopes here? # -> NO, slope is such a minor component of Diff we'll be OK # BUT could be important not for the stability, but for the # actual calc. So YES. node_S = np.zeros_like(node_S) # print link_length[core_draining_nodes] node_S[core_draining_nodes] = ( node_z - node_z[flow_receiver] )[core_draining_nodes] / link_length[core_draining_nodes] internal_t += dt_this_step # still in seconds, remember active_nodes = grid.core_nodes if self.return_ch_props: # add the channel property field entries, # 'channel__width', 'channel__depth', and 'channel__discharge' W = self.k_w * node_Q**self._b H = shear_stress / self.rho_g / node_S # ...sneaky! grid.at_node['channel__width'][:] = W grid.at_node['channel__depth'][:] = H grid.at_node['channel__discharge'][:] = node_Q grid.at_node['channel__bed_shear_stress'][:] = shear_stress grid.at_node[ 'channel_sediment__volumetric_transport_capacity'][:] = transport_capacities grid.at_node['channel_sediment__volumetric_flux'][:] = sed_into_node grid.at_node['channel_sediment__relative_flux'][:] = rel_sed_flux # elevs set automatically to the name used in the function call. self.iterations_in_dt = counter return grid, grid.at_node['topographic__elevation']
def acr_subs_get_data(idxPrc, # Process ID #noqa strSubId, # Data struc - Subject ID lstVtkDpth01, # Data struc - Pth vtk I varNumDpth, # Data struc - Num. depth levels strPrcdData, # Data struc - Str prcd VTK data varNumLne, # Data struc - Lns prcd data VTK lgcSlct01, # Criterion 1 - Yes or no? strCsvRoi, # Criterion 1 - CSV path varNumHdrRoi, # Criterion 1 - Header lines lgcSlct02, # Criterion 2 - Yes or no? strVtkSlct02, # Criterion 2 - VTK path varThrSlct02, # Criterion 2 - Threshold lgcSlct03, # Criterion 3 - Yes or no? strVtkSlct03, # Criterion 3 - VTK path varThrSlct03, # Criterion 3 - Threshold lgcSlct04, # Criterion 4 - Yes or no? strVtkSlct04, # Criterion 4 - VTK path tplThrSlct04, # Criterion 4 - Threshold lgcNormDiv, # Normalisation - Yes or no? varNormIdx, # Normalisation - Reference varDpi, # Plot - Dots per inch varYmin, # Plot - Minimum of Y axis varYmax, # Plot - Maximum of Y axis lstConLbl, # Plot - Condition labels strXlabel, # Plot - X axis label strYlabel, # Plot - Y axis label strTitle, # Plot - Title strPltOtPre, # Plot - Output file path prefix strPltOtSuf, # Plot - Output file path suffix strMetaCon, # Metacondition (stim/periphery) queOut): # Queue for output list """ Obtaining & plotting single subject data for across subject analysis. This function loads the data for each subject for a multi-subject analysis and passes the data to the parent function for visualisation. """ # Only print status messages if this is the first of several parallel # processes: if idxPrc == 0: print('------Loading single subject data: ' + strSubId) # ************************************************************************* # *** Import data # Import CSV file with ROI definition if lgcSlct01: if idxPrc == 0: print('---------Importing CSV file with ROI definition (first ' + 'criterion)') aryRoiVrtx = load_csv_roi(strCsvRoi, varNumHdrRoi) # Otherwise, create dummy vector (for function I/O) else: aryRoiVrtx = 0 # Import second criterion vtk file (all depth levels) if lgcSlct02: if idxPrc == 0: print('---------Importing second criterion vtk file (all depth ' + 'levels).') arySlct02 = load_vtk_multi(strVtkSlct02, strPrcdData, varNumLne, varNumDpth) # Otherwise, create dummy vector (for function I/O) else: arySlct02 = 0 # Import third criterion vtk file (all depth levels) if lgcSlct03: if idxPrc == 0: print('---------Importing third criterion vtk file (all depth ' + 'levels).') arySlct03 = load_vtk_multi(strVtkSlct03, strPrcdData, varNumLne, varNumDpth) # Otherwise, create dummy array (for function I/O) else: arySlct03 = 0 # Import fourth criterion vtk file (one depth level) if lgcSlct04: if idxPrc == 0: print('---------Importing fourth criterion vtk file (one depth ' + 'level).') arySlct04 = load_vtk_multi(strVtkSlct04, strPrcdData, varNumLne, varNumDpth) # Otherwise, create dummy array (for function I/O): else: arySlct04 = 0 # Import depth data vtk files if idxPrc == 0: print('---------Importing depth data vtk files.') # Number of input files (i.e. number of conditions): varNumCon = len(lstVtkDpth01) # List for input data: lstDpthData01 = [None] * varNumCon # Loop through input data files: for idxIn in range(0, varNumCon): # Import data from file: lstDpthData01[idxIn] = load_vtk_multi(lstVtkDpth01[idxIn], strPrcdData, varNumLne, varNumDpth) if idxPrc == 0: print('------------File ' + str(idxIn + 1) + ' out of ' + str(varNumCon)) # ************************************************************************* # ************************************************************************* # *** Convert cope to percent signal change # According to the FSL documentation # (https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/FEAT/UserGuide), the PEs can be # scaled to signal change with respect to the mean (over time within # voxel): "This is achieved by scaling the PE or COPE values by (100*) the # peak-peak height of the regressor (or effective regressor in the case of # COPEs) and then by dividing by mean_func (the mean over time of # filtered_func_data)." However, this PSC would be with respect to the # temporal mean, but we are interested in the PSC with respect to # pre-stimulus baseline. Thus, we extract the difference (a scaling # factor) between these two (i.e. temporal mean vs. pre-stimulus baseline) # from the respective FSL design matrix (`design.mat` in the FEAT # directory). The scaling factor is approximately 1.4 (slightly different # values for sustained and transient predictors, but close enough not to # matter). This scaling factor needs to be applied after the procedure # described in the FSL documentation. Thus, the final PSC is calculated as # follows: `(PE * (100 * peak-peak height) / tmean) * 1.4`. The pp-height # is obtained from `design.mat`. # Only perform scaling if the data is from an FSL cope file: if (('cope' in lstVtkDpth01[0]) or ('_pe' in lstVtkDpth01[0])): if idxPrc == 0: print('---------Convert cope to percent signal change.') # The peak-peak height depends on the predictor (i.e. condition). if 'sst' in lstVtkDpth01[0]: varPpheight = 1.268049 elif 'trn' in lstVtkDpth01[0]: varPpheight = 0.2269044 else: if idxPrc == 0: print(('------------WARNING: Cannot determine condition from ' + 'file name, peak-peak height of the regressor is set ' + 'to 1.')) varPpheight = 1.0 # Loop through input data files: for idxIn in range(0, varNumCon): # Get PEs: aryTmp = lstDpthData01[idxIn].astype(np.float64) # In order to avoid division by zero, avoid zero-voxels: lgcTmp = np.not_equal(arySlct03, 0.0) # Apply PSC scaling, as described above: aryTmp[lgcTmp] = np.multiply( np.divide( np.multiply(aryTmp[lgcTmp], (100.0 * varPpheight) ), arySlct03[lgcTmp]), 1.0 # 1.4 ) # Put scaled PEs back into list (now PSC with respect to # pre-stimulus baseline): lstDpthData01[idxIn] = aryTmp # ************************************************************************* # ************************************************************************* # *** Select vertices lstDpthData01, varNumInc, vecInc = \ slct_vrtcs(varNumCon, # Number of conditions lstDpthData01, # List with depth-sampled data I lgcSlct01, # Criterion 1 - Yes or no? aryRoiVrtx, # Criterion 1 - Data (ROI) lgcSlct02, # Criterion 2 - Yes or no? arySlct02, # Criterion 2 - Data varThrSlct02, # Criterion 2 - Threshold lgcSlct03, # Criterion 3 - Yes or no? arySlct03, # Criterion 3 - Data varThrSlct03, # Criterion 3 - Threshold lgcSlct04, # Criterion 4 - Yes or no? arySlct04, # Criterion 4 - Data tplThrSlct04, # Criterion 4 - Threshold idxPrc) # Process ID # ************************************************************************* # ************************************************************************* # *** Create VTK mesh mask if idxPrc == 0: print('---------Creating VTK mesh mask.') # We would like to be able to visualise the selected vertices on the # cortical surface, i.e. on a vtk mesh. vtk_msk(strSubId, # Data struc - Subject ID lstVtkDpth01[0], # Data struc - Path first data vtk file strPrcdData, # Data struc - Str. prcd. VTK data varNumLne, # Data struc - Lns. prcd. data VTK strCsvRoi, # Data struc - ROI CSV fle (outpt. naming) vecInc, # Vector with included vertices strMetaCon) # Metacondition (stimulus or periphery) # ************************************************************************* # ************************************************************************* # *** Calculate mean & conficende interval if idxPrc == 0: print('---------Plot results - mean over vertices.') # Prepare arrays for results (mean & confidence interval): aryDpthMean = np.zeros((varNumCon, varNumDpth)) aryDpthConf = np.zeros((varNumCon, varNumDpth)) # Fill array with data - loop through input files: for idxIn in range(0, varNumCon): # Loop through depth levels: for idxDpth in range(0, varNumDpth): # Avoid warning in case of empty array (i.e. no vertices included # in ROI for current ROI/subject/hemisphere): if np.greater(np.sum(vecInc), 0): # Retrieve all vertex data for current input file & current # depth level: aryTmp = lstDpthData01[idxIn][:, idxDpth] # Calculate mean over vertices: varTmp = np.mean(aryTmp) else: # No vertices in ROI: varTmp = 0.0 # Place mean in array: aryDpthMean[idxIn, idxDpth] = varTmp # Calculate 95% confidence interval for the mean, obtained by # multiplying the standard error of the mean (SEM) by 1.96. We # obtain the SEM by dividing the standard deviation by the # squareroot of the sample size n. We get n by taking 1/8 of the # number of vertices, which corresponds to the number of voxels in # native resolution. varTmp = np.multiply(np.divide(np.std(aryTmp), np.sqrt(aryTmp.size * 0.125)), 1.96) # Place confidence interval in array: aryDpthConf[idxIn, idxDpth] = varTmp # Calculate standard error of the mean. # varTmp = np.divide(np.std(aryTmp), # np.sqrt(aryTmp.size * 0.125)) # Place SEM in array: # aryDpthConf[idxIn, idxDpth] = varTmp # Calculate standard deviation over vertices: # varTmp = np.std(aryTmp) # Place standard deviation in array: # aryDpthConf[idxIn, idxDpth] = varTmp # Normalise by division: if lgcNormDiv: if idxPrc == 0: print('---------Normalisation by division.') # Vector for subtraction: # vecSub = np.array(aryDpthMean[varNormIdx, :], ndmin=2) # Divide all rows by reference row: # aryDpthMean = np.divide(aryDpthMean, vecSub) # Calculate 'grand mean', i.e. the mean PE across depth levels and # conditions: varGrndMean = np.mean(aryDpthMean) # varGrndMean = np.median(aryDpthMean) # Divide all values by the grand mean: aryDpthMean = np.divide(np.absolute(aryDpthMean), varGrndMean) aryDpthConf = np.divide(np.absolute(aryDpthConf), varGrndMean) # ************************************************************************* # ************************************************************************* # *** Create plot if False: # File name for figure: strPltOt = strPltOtPre + strSubId + strPltOtSuf # Title, including information about number of vertices: strTitleTmp = (strTitle + ', ' + str(varNumInc) + ' vertices') plt_dpth_prfl(aryDpthMean, # Data: aryData[Condition, Depth] aryDpthConf, # Error shading: aryError[Condition, Depth] varNumDpth, # Number of depth levels (on the x-axis) varNumCon, # Number of conditions (separate lines) varDpi, # Resolution of the output figure varYmin, # Minimum of Y axis varYmax, # Maximum of Y axis False, # Boolean: whether to convert y axis to % lstConLbl, # Labels for conditions (separate lines) strXlabel, # Label on x axis strYlabel, # Label on y axis strTitleTmp, # Figure title True, # Boolean: whether to plot a legend strPltOt) # ************************************************************************* # ************************************************************************* # *** Return # Output list: lstOut = [idxPrc, aryDpthMean, varNumInc] queOut.put(lstOut)
im = ns.add_noise(original, noise_type, noise, max_val=255 / scaling) misc.save_image(im * scaling, "{}.png".format(noisyImName)) im, scaling2 = misc.get_image("{}.png".format(noisyImName), n) im *= scaling2 / scaling shift = im.ravel().min() - 1e-10 if shift > 0: shift = 0 sparsities = [] method_outputs = initialize_method_outputs(W_SOFT, W_HARD, SOFT, HARD) # compute sparsity paths for lamb in lambdas: _, Z, obj = ot_sparse_projection. \ wasserstein_image_filtering_invertible_dictionary(im - shift, filter_handler, gamma, lamb * mask) sparsity_pattern = np.not_equal(0, Z) _, Z_wasserstein_hard, obj_hard = ot_sparse_projection. \ OtFilteringSpecificPattern(filter_handler, gamma, sparsity_pattern, ).projection(im - shift) add_values(method_outputs, W_SOFT, Z) add_values(method_outputs, W_HARD, Z_wasserstein_hard) sparsity = misc.get_sparsity(Z) sparsities.append(sparsity) Y_l2, Z_l2 = l2.sparse_projection(im, filter_handler, sparsity) Y_l2_hard, Z_l2_hard = l2.hard_thresholding(im, filter_handler, sparsity) add_values(method_outputs, HARD, Z_l2_hard) add_values(method_outputs, SOFT, Z_l2) # plot scores for key, sim in similarities.items(): table_scores[noise_type][noise][key] = {} plt.figure(figsize=[9., 3.])
def num_changed(old, new): return np.sum(np.not_equal(old, new))
def onp_not_equal(a, b): return onp.not_equal(a, b)
def test_batch_iter_with_shuffle(): data = np.arange(36).reshape(12, 3) bi = iterator.BatchIterator(4, True) data2 = np.vstack([items[0] for items in bi(data)]) assert_equal(np.any(np.not_equal(data, data2)), True) assert_array_equal(data, np.sort(data2, axis=0))
def get_batch(self, batch_size): in_data = [] slot_data = [] slot_weight = [] length = [] intents = [] batch_in = [] batch_slot = [] max_len = 0 #used to record word(not id) in_seq = [] slot_seq = [] intent_seq = [] for i in range(batch_size): inp = self.__fd_in.readline() if inp == '': self.end = 1 break slot = self.__fd_slot.readline() intent = self.__fd_intent.readline() inp = inp.rstrip() slot = slot.rstrip() intent = intent.rstrip() in_seq.append(inp) slot_seq.append(slot) intent_seq.append(intent) iii = inp sss = slot inp = sentenceToIds(inp, self.__in_vocab) slot = sentenceToIds(slot, self.__slot_vocab) intent = sentenceToIds(intent, self.__intent_vocab) batch_in.append(np.array(inp)) batch_slot.append(np.array(slot)) length.append(len(inp)) intents.append(intent[0]) if len(inp) != len(slot): print(iii, sss) print(inp, slot) exit(0) if len(inp) > max_len: max_len = len(inp) length = np.array(length) intents = np.array(intents) #print(max_len) #print('A'*20) for i, s in zip(batch_in, batch_slot): in_data.append(padSentence(list(i), max_len, self.__in_vocab)) slot_data.append(padSentence(list(s), max_len, self.__slot_vocab)) #print(s) in_data = np.array(in_data) slot_data = np.array(slot_data) #print(in_data) #print(slot_data) #print(type(slot_data)) for s in slot_data: weight = np.not_equal(s, np.zeros(s.shape)) weight = weight.astype(np.float32) slot_weight.append(weight) slot_weight = np.array(slot_weight) return in_data, slot_data, slot_weight, length, intents, in_seq, slot_seq, intent_seq
with open('/data/y_raw.pickle', 'rb') as f: output = pickle.load(f) print(len(output)) print("Data loaded\n") # Split data train_x, test_x = train_test_split(input, test_size=0.1, random_state=42) train_y, test_y = train_test_split(output, test_size=0.1, random_state=42) clf = Pipeline([('classif', RandomForestClassifier())]) clf.fit(train_x, train_y) predicted = clf.predict(test_x) print('Correct predictions: {:4.2f}'.format(np.mean(predicted == test_y))) print("Accuracy:", metrics.accuracy_score(test_y, predicted)) print("Precision", metrics.precision_score(test_y, predicted, average='weighted')) print("Recall:", metrics.recall_score(test_y, predicted, average='weighted')) print("F1:", metrics.f1_score(test_y, predicted, average='weighted')) # Writing the hamming loss formula incorrect = np.not_equal(predicted, test_y) misclass = np.count_nonzero(incorrect) hamm_loss = (np.sum(misclass / n_classes)) / len(test_x) print("Hamming loss:,", hamm_loss) print("Verify hamming loss value against sklearn library:", metrics.hamming_loss(test_y, predicted))
f = file.as_matrix() # The training dataset in array form X_train = f[:split, 1:] y_train = f[:split, 0] # The Cross validation set X_cv = f[split: , 1:] y_cv = f[split: , 0] # Declaring error arrays error_cv = [] import numpy as np for layer_size in [200, 225, 250, 275]: for reg in [10, 25, 50 , 75 , 100]: nn = MLPClassifier(solver='lbfgs' , alpha = reg , activation='logistic' , hidden_layer_sizes=(layer_size,)) nn.fit( X_train[ :m , ] , y_train[ :m , ] ) y_pred = nn.predict(X_cv) error_cv.append( sum( np.not_equal(y_pred ,y_cv ) )/len(y_pred) ) max_index = error_cv.index(np.amin(error_cv)) print("Layer_size: " ,min_index%4) print("alpha: " , min_index/4)
def read_data(filename): """Reads Intan Technologies RHD2000 data file generated by evaluation board GUI. Data are returned in a dictionary, for future extensibility. """ tic = time.time() fid = open(filename, 'rb') filesize = os.path.getsize(filename) header = read_header(fid) print('Found {} amplifier channel{}.'.format( header['num_amplifier_channels'], plural(header['num_amplifier_channels']))) print('Found {} auxiliary input channel{}.'.format( header['num_aux_input_channels'], plural(header['num_aux_input_channels']))) print('Found {} supply voltage channel{}.'.format( header['num_supply_voltage_channels'], plural(header['num_supply_voltage_channels']))) print('Found {} board ADC channel{}.'.format( header['num_board_adc_channels'], plural(header['num_board_adc_channels']))) print('Found {} board digital input channel{}.'.format( header['num_board_dig_in_channels'], plural(header['num_board_dig_in_channels']))) print('Found {} board digital output channel{}.'.format( header['num_board_dig_out_channels'], plural(header['num_board_dig_out_channels']))) print('Found {} temperature sensors channel{}.'.format( header['num_temp_sensor_channels'], plural(header['num_temp_sensor_channels']))) print('') # Determine how many samples the data file contains. bytes_per_block = get_bytes_per_data_block(header) # How many data blocks remain in this file? data_present = False bytes_remaining = filesize - fid.tell() if bytes_remaining > 0: data_present = True if bytes_remaining % bytes_per_block != 0: raise Exception( 'Something is wrong with file size : should have a whole number of data blocks' ) num_data_blocks = int(bytes_remaining / bytes_per_block) num_amplifier_samples = 60 * num_data_blocks num_aux_input_samples = 15 * num_data_blocks num_supply_voltage_samples = 1 * num_data_blocks num_board_adc_samples = 60 * num_data_blocks num_board_dig_in_samples = 60 * num_data_blocks num_board_dig_out_samples = 60 * num_data_blocks record_time = num_amplifier_samples / header['sample_rate'] if data_present: print( 'File contains {:0.3f} seconds of data. Amplifiers were sampled at {:0.2f} kS/s.' .format(record_time, header['sample_rate'] / 1000)) else: print( 'Header file contains no data. Amplifiers were sampled at {:0.2f} kS/s.' .format(header['sample_rate'] / 1000)) if data_present: # Pre-allocate memory for data. print('') print('Allocating memory for data...') data = {} if (header['version']['major'] == 1 and header['version']['minor'] >= 2 ) or (header['version']['major'] > 1): data['t_amplifier'] = np.zeros(num_amplifier_samples, dtype=np.int) else: data['t_amplifier'] = np.zeros(num_amplifier_samples, dtype=np.uint) data['amplifier_data'] = np.zeros( [header['num_amplifier_channels'], num_amplifier_samples], dtype=np.uint) data['aux_input_data'] = np.zeros( [header['num_aux_input_channels'], num_aux_input_samples], dtype=np.uint) data['supply_voltage_data'] = np.zeros([ header['num_supply_voltage_channels'], num_supply_voltage_samples ], dtype=np.uint) data['temp_sensor_data'] = np.zeros( [header['num_temp_sensor_channels'], num_supply_voltage_samples], dtype=np.uint) data['board_adc_data'] = np.zeros( [header['num_board_adc_channels'], num_board_adc_samples], dtype=np.uint) data['board_dig_in_data'] = np.zeros( [header['num_board_dig_in_channels'], num_board_dig_in_samples], dtype=np.uint) data['board_dig_in_raw'] = np.zeros(num_board_dig_in_samples, dtype=np.uint) data['board_dig_out_data'] = np.zeros( [header['num_board_dig_out_channels'], num_board_dig_out_samples], dtype=np.uint) data['board_dig_out_raw'] = np.zeros(num_board_dig_out_samples, dtype=np.uint) # Read sampled data from file. print('Reading data from file...') # Initialize indices used in looping indices = {} indices['amplifier'] = 0 indices['aux_input'] = 0 indices['supply_voltage'] = 0 indices['board_adc'] = 0 indices['board_dig_in'] = 0 indices['board_dig_out'] = 0 print_increment = 10 percent_done = print_increment for i in range(num_data_blocks): read_one_data_block(data, header, indices, fid) # Increment indices indices['amplifier'] += 60 indices['aux_input'] += 15 indices['supply_voltage'] += 1 indices['board_adc'] += 60 indices['board_dig_in'] += 60 indices['board_dig_out'] += 60 fraction_done = 100 * (1.0 * i / num_data_blocks) if fraction_done >= percent_done: print('{}% done...'.format(percent_done)) percent_done = percent_done + print_increment # Make sure we have read exactly the right amount of data. bytes_remaining = filesize - fid.tell() if bytes_remaining != 0: raise Exception('Error: End of file not reached.') # Close data file. fid.close() if (data_present): print('Parsing data...') # Extract digital input channels to separate variables. for i in range(header['num_board_dig_in_channels']): data['board_dig_in_data'][i, :] = np.not_equal( np.bitwise_and( data['board_dig_in_raw'], (1 << header['board_dig_in_channels'][i]['native_order'])), 0) # Extract digital output channels to separate variables. for i in range(header['num_board_dig_out_channels']): data['board_dig_out_data'][i, :] = np.not_equal( np.bitwise_and(data['board_dig_out_raw'], ( 1 << header['board_dig_out_channels'][i]['native_order'])), 0) # Scale voltage levels appropriately. data['amplifier_data'] = np.multiply( 0.195, (data['amplifier_data'].astype(np.int32) - 32768)) # units = microvolts data['aux_input_data'] = np.multiply( 37.4e-6, data['aux_input_data']) # units = volts data['supply_voltage_data'] = np.multiply( 74.8e-6, data['supply_voltage_data']) # units = volts if header['eval_board_mode'] == 1: data['board_adc_data'] = np.multiply( 152.59e-6, (data['board_adc_data'].astype(np.int32) - 32768)) # units = volts else: data['board_adc_data'] = np.multiply( 50.354e-6, data['board_adc_data']) # units = volts data['temp_sensor_data'] = np.multiply( 0.01, data['temp_sensor_data']) # units = deg C # Check for gaps in timestamps. num_gaps = np.sum( np.not_equal(data['t_amplifier'][1:] - data['t_amplifier'][:-1], 1)) if num_gaps == 0: print('No missing timestamps in data.') else: print( 'Warning: {0} gaps in timestamp data found. Time scale will not be uniform!' .format(num_gaps)) # Scale time steps (units = seconds). data['t_amplifier'] = data['t_amplifier'] / header['sample_rate'] data['t_aux_input'] = data['t_amplifier'][range( 0, len(data['t_amplifier']), 4)] data['t_supply_voltage'] = data['t_amplifier'][range( 0, len(data['t_amplifier']), 60)] data['t_board_adc'] = data['t_amplifier'] data['t_dig'] = data['t_amplifier'] data['t_temp_sensor'] = data['t_supply_voltage'] # If the software notch filter was selected during the recording, apply the # same notch filter to amplifier data here. if header['notch_filter_frequency'] > 0: print('Applying notch filter...') print_increment = 10 percent_done = print_increment for i in range(header['num_amplifier_channels']): data['amplifier_data'][i, :] = notch_filter( data['amplifier_data'][i, :], header['sample_rate'], header['notch_filter_frequency'], 10) fraction_done = 100 * (i / header['num_amplifier_channels']) if fraction_done >= percent_done: print('{}% done...'.format(percent_done)) percent_done += print_increment else: data = [] # Move variables to result struct. result = data_to_result(header, data, data_present) print('Done! Elapsed time: {0:0.1f} seconds'.format(time.time() - tic)) return result
def add_not_equal_to_test(self, var_name, limit_value, test_meaning=None, test_assessment='Bad', test_number=None, flag_value=False, limit_attr_name=None, prepend_text=None): """ Method to perform a not equal to test and add result to ancillary quality control variable. If ancillary quality control variable does not exist it will be created. Parameters ---------- var_name : str Data variable name. limit_value : int or float or None Limit value to use in test. The value will be written to the quality control variable as an attribute. If set to None will return without setttin test. test_meaning : str The optional text description to add to flag_meanings describing the test. Will add a default if not set. test_assessment : str Optional single word describing the assessment of the test. Will set a default if not set. test_number : int Optional test number to use. If not set will ues next available test number. flag_value : boolean Indicates that the tests are stored as integers not bit packed values in quality control variable. limit_attr_name : str Optional attribute name to store the limit_value under quality control ancillary variable. prepend_text : str Optional text to prepend to the test meaning. Example is indicate what institution added the test. Returns ------- test_info : tuple A tuple containing test information including var_name, qc variable name, test_number, test_meaning, test_assessment """ if limit_value is None: return if limit_attr_name is None: if test_assessment == 'Suspect' or test_assessment == 'Indeterminate': attr_name = 'warn_not_equal_to' else: attr_name = 'fail_not_equal_to' else: attr_name = limit_attr_name if test_meaning is None: test_meaning = 'Data value not equal to {}.'.format(attr_name) if prepend_text is not None: test_meaning = ': '.join((prepend_text, test_meaning)) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=RuntimeWarning) index = np.not_equal(self._obj[var_name].values, limit_value) result = self._obj.qcfilter.add_test( var_name, index=index, test_number=test_number, test_meaning=test_meaning, test_assessment=test_assessment, flag_value=flag_value) # Ensure limit_value attribute is matching data type limit_value = np.array(limit_value, dtype=self._obj[var_name].values.dtype.type) qc_var_name = result['qc_variable_name'] self._obj[qc_var_name].attrs[attr_name] = limit_value return result
def binary_op(self, op, rhs1, rhs2, where, args, stacklevel): if self.shadow: rhs1 = self.runtime.to_eager_array( rhs1, stacklevel=(stacklevel + 1) ) rhs2 = self.runtime.to_eager_array( rhs2, stacklevel=(stacklevel + 1) ) if where is not None and isinstance(where, NumPyThunk): where = self.runtime.to_eager_array( where, stacklevel=(stacklevel + 1) ) elif self.deferred is None: if where is not None and isinstance(where, NumPyThunk): self.check_eager_args((stacklevel + 1), rhs1, rhs2, where) else: self.check_eager_args((stacklevel + 1), rhs1, rhs2) if self.deferred is not None: self.deferred.binary_op( op, rhs1, rhs2, where, args, stacklevel=(stacklevel + 1) ) else: if op == NumPyOpCode.ADD: np.add( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.LOGICAL_AND: np.logical_and( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.DIVIDE: np.divide( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.EQUAL: np.equal( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.FLOOR_DIVIDE: np.floor_divide( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.GREATER_EQUAL: np.greater_equal( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.GREATER: np.greater( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) # elif op == NumPyOpCode.SHIFT_LEFT: # np.left_shift(rhs1.array, rhs2.array, out=self.array, # where=where if not isinstance(where, EagerArray) # else where.array) # elif op == NumPyOpCode.SHIFT_RIGHT: # np.right_shift(rhs1.array, rhs2.array, out=self.array, # where=where if not isinstance(where, EagerArray) # else where.array) elif op == NumPyOpCode.MOD: np.mod( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.MULTIPLY: np.multiply( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.LOGICAL_OR: np.logical_or( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.POWER: np.power( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.SUBTRACT: np.subtract( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.LOGICAL_XOR: np.logical_xor( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.LESS_EQUAL: np.less_equal( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.LESS: np.less( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.MAXIMUM: np.maximum( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.MINIMUM: np.minimum( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) elif op == NumPyOpCode.NOT_EQUAL: np.not_equal( rhs1.array, rhs2.array, out=self.array, where=where if not isinstance(where, EagerArray) else where.array, ) else: raise RuntimeError("unsupported binary op " + str(op)) self.runtime.profile_callsite(stacklevel + 1, False)
def fit(self): """ Trains a model """ if self.optimizers is None: self.setup_optimizers() num_train = self.dataset.train_X.shape[0] loss_history = [] train_acc_history = [] val_acc_history = [] first_layer_W = {} for param_name, _ in self.model.params().items(): first_layer_W[param_name]=[] for epoch in range(self.num_epochs): shuffled_indices = np.arange(num_train) np.random.shuffle(shuffled_indices) sections = np.arange(self.batch_size, num_train, self.batch_size) batches_indices = np.array_split(shuffled_indices, sections) batch_losses = [] for batch_indices in batches_indices: # TODO Generate batches based on batch_indices and # use model to generate loss and gradients for all # the params batch = self.dataset.train_X[batch_indices] label = self.dataset.train_y[batch_indices] loss = self.model.compute_loss_and_gradients(batch, label) batch_losses.append(loss) for param_name, param in self.model.params().items(): optimizer = self.optimizers[param_name] # print('PARAM PRE', param_name) first_layer_W[param_name].append(param.value.sum()) param.value = optimizer.update(param.value, param.grad, self.learning_rate) # print('PARAM POST', param_name) # print(param.value.sum(), '\n=========') if np.not_equal(self.learning_rate_decay, 1.0): # TODO: Implement learning rate decay self.learning_rate *= self.learning_rate_decay pass ave_loss = np.mean(batch_losses) train_accuracy = self.compute_accuracy(self.dataset.train_X, self.dataset.train_y) val_accuracy = self.compute_accuracy(self.dataset.val_X, self.dataset.val_y) print("Loss: %f, Train accuracy: %f, val accuracy: %f" % (batch_losses[-1], train_accuracy, val_accuracy)) loss_history.append(ave_loss) train_acc_history.append(train_accuracy) val_acc_history.append(val_accuracy) return loss_history, train_acc_history, val_acc_history, first_layer_W
# *** Normalisation if lgcNorm: # Get prestimulus baseline: aryBse = np.copy(aryTmpBlcks[:, :, :, :, int(varVolsPre + tplBase[0]): int(varVolsPre + tplBase[1])] ).astype(np.float32) # Mean for each voxel over time (i.e. over the pre-stimulus # baseline): aryBseMne = np.mean(aryBse, axis=4).astype(np.float32) # Get indicies of voxels that have a non-zero prestimulus baseline: aryNonZero = np.not_equal(aryBseMne, 0.0) # Divide all voxels that are non-zero in the pre-stimulus baseline by # the prestimulus baseline: aryTmpBlcks[aryNonZero] = np.divide(aryTmpBlcks[aryNonZero], aryBseMne[aryNonZero, None] ).astype(np.float32) # ------------------------------------------------------------------------- # *** Save segment if lgcSegs: # Loop through runs again in order to save segments: for index_03 in range(0, varTmpNumBlck):
def vis_image_pair_opencv(im_list, boxes_list, segms_list=None, keypoints_list=None, track=None, thresh=0.9, kp_thresh=2, track_thresh=0.7, show_box=False, dataset=None, show_class=False, show_track=False, show_track_ids=False, colors=None, color_inds_list=None): """Visualize object associations in an image pair.""" classes_list = [] keep_idx = [[], []] sorted_inds_list = [] ret = [] color_inds_list_new = [None, None] masks_list = [] for i, im in enumerate(im_list): boxes = boxes_list[i] segms = None if segms_list is None else segms_list[i] keypoints = None if keypoints_list is None else keypoints_list[i] if isinstance(boxes, list): boxes, segms, keypoints, classes = convert_from_cls_format( boxes, segms, keypoints) boxes_list[i] = boxes segms_list[i] = segms keypoints_list[i] = keypoints classes_list.append(classes) if boxes is None or boxes.shape[0] == 0 or max(boxes[:, 4]) < thresh: if im is not None: ret.append(im) continue if segms is not None and len(segms) > 0: masks_list.append(mask_util.decode(segms)) # Display in largest to smallest order to reduce occlusion areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) sorted_inds = np.argsort(-areas) sorted_inds_list.append(sorted_inds) for idx in sorted_inds: bbox = boxes[idx, :4] score = boxes[idx, -1] if score >= thresh: keep_idx[i].append(idx) n_rois = len(boxes_list[0]) m_rois = len(boxes_list[1]) assert (len(sorted_inds_list[0]) == n_rois) assert (len(sorted_inds_list[1]) == m_rois) assert (track.shape == (n_rois, m_rois)) if color_inds_list is None: color_inds_list = [None, None] for i in xrange(2): if color_inds_list[i] is not None: assert (len(color_inds_list[i]) == len(boxes_list[i])) else: color_inds_list[i] = [None] * len(boxes_list[i]) color_inds_list_new[i] = [None] * len(boxes_list[i]) assign_inds_one = [] track_prob_mat = np.zeros((n_rois, m_rois)) for i in keep_idx[0]: for j in keep_idx[1]: track_prob_mat[i, j] = track[m_rois * i + j] track_prob_mat = np.where(track_prob_mat > track_thresh, track_prob_mat, np.zeros((n_rois, m_rois))) track_prob_mat = np.where( np.array([[class_one == class_two for class_two in classes_list[1]] for class_one in classes_list[0]]), track_prob_mat, np.zeros((n_rois, m_rois))) assign_inds_list = linear_sum_assignment(-track_prob_mat) if colors is None: colors = distinct_colors(min(len(keep_idx[0]), len(keep_idx[1]))) for i, im in enumerate(im_list): boxes = boxes_list[i] classes = classes_list[i] assign_inds = assign_inds_list[i] if i == 1: color_inds_list[0] = color_inds_list_new[0] for idx in keep_idx[i]: bbox = boxes[idx, :4] score = boxes[idx, -1] if idx not in assign_inds: continue i_other = (0 if i == 1 else 1) assign_inds_other = assign_inds_list[i_other] i_assign = assign_inds.tolist().index(idx) idx_other = assign_inds_other[i_assign] if idx_other not in keep_idx[i_other]: continue if i == 0: assign_inds_one.append(i_assign) i_color = color_inds_list[i][idx] if i_color is None: if color_inds_list[i_other][idx_other] is None: color_inds = np.unique([ x for x in color_inds_list[i_other] + color_inds_list[i] if x is not None ]) if len(color_inds): i_small_idx = np.where( np.not_equal(color_inds, np.array(range(len(color_inds)))))[0] if len(i_small_idx): i_color = i_small_idx[0] else: i_color = max(color_inds) + 1 else: i_color = assign_inds_one.index(i_assign) assert (i_color not in color_inds) else: i_color = color_inds_list[i_other][idx_other] assert (i_color is not None) color_inds_list_new[i][idx] = i_color if im is not None: if i == 0: track_prob = track_prob_mat[idx, idx_other] else: track_prob = track_prob_mat[idx_other, idx] # show box (off by default) if track_prob < track_thresh: color = (127.5, 127.5, 127.5) thick = 1 else: color = colors[i_color] thick = 2 if show_box: im = vis_bbox(im, (bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1]), color=color, thick=thick) # show class (off by default) class_str = "" if show_class: class_str = get_class_string(classes[idx], score, dataset) if show_track: class_str += " | " if show_track: class_str += "{:.2f}".format(track_prob) if show_track_ids: class_str += ", {} --> {}".format(idx, idx_other) if show_class or show_track: im = vis_class(im, (bbox[0], bbox[1] - 2), class_str) # show mask if segms is not None and len(segms) > idx: color_mask = np.array(colors[i_color]) im = vis_mask(im, masks_list[i][..., idx], color_mask) # show keypoints if keypoints is not None and len(keypoints) > idx: im = vis_keypoints(im, keypoints_list[i][idx], 2) if im is not None: im = vis_class(im, (10, 20), str(i)) ret.append(im) ret.append(track_prob_mat) ret += color_inds_list_new return ret