def hit(self, r: RayList, t_min: float, t_max: Union[float, cp.ndarray]) \ -> HitRecordList: if isinstance(t_max, (int, float, cp.floating)): t_max_list = cp.full(len(r), t_max, cp.float32) else: t_max_list = t_max oc: Vec3List = r.origin() - self.center a: cp.ndarray = r.direction().length_squared() half_b: cp.ndarray = oc @ r.direction() c: cp.ndarray = oc.length_squared() - self.radius**2 discriminant_list: cp.ndarray = half_b**2 - a*c discriminant_condition = discriminant_list > 0 if not discriminant_condition.any(): return HitRecordList.new(len(r)).set_compress_info(None) # Calculate t positive_discriminant_list = ( discriminant_list * discriminant_condition ) root = cp.sqrt(positive_discriminant_list) non_zero_a = a - (a == 0) t_0 = (-half_b - root) / non_zero_a t_1 = (-half_b + root) / non_zero_a # Choose t t_0_condition = ( (t_min < t_0) & (t_0 < t_max_list) & discriminant_condition ) t_1_condition = ( (t_min < t_1) & (t_1 < t_max_list) & (~t_0_condition) & discriminant_condition ) t = cp.where(t_0_condition, t_0, 0) t = cp.where(t_1_condition, t_1, t) # Compression condition = t > 0 full_rate = condition.sum() / len(t) if full_rate > 0.5: idx = None else: idx = cp.where(condition)[0] t = t[idx] r = RayList( Vec3List(r.orig.get_ndarray(idx)), Vec3List(r.dir.get_ndarray(idx)) ) # Wrap up result point = r.at(t) outward_normal = (point - self.center) / self.radius result = HitRecordList( point, t, cp.full(len(r), self.material.idx, dtype=cp.int32) ).set_face_normal(r, outward_normal).set_compress_info(idx) return result
def fetch_optim(user, tabular_np, neighbors_indeces): optim_indeces = cupy.where(tabular_np[user] == 0)[0] for i in neighbors_indeces: optim_indeces = cupy.unique( cupy.concatenate((optim_indeces, cupy.where(tabular_np[i] > 0)[0]), 0)) optim_values = cupy.take(tabular_np, optim_indeces, axis=1) return optim_values, optim_indeces
def adex_spike(V, w, c): """ Check potential thresholds for new spikes """ # Spike projects 0mV or 1 mV (nothing or unit value) to the network # with the current parameters spike = V > c['Vth'] V = cp.where(spike, c['V_r'], V) w = cp.where(spike, w + c['b'], w) return V, w, spike
def estimate_size(self, masks): d = cp.sqrt( cp.sum(masks, axis=(1, 2, 3)) / (masks.shape[1] * masks.shape[2])) # big: 2, medium: 1, small: 0 size = cp.zeros(d.shape) cp.where(d >= self.size_degree[0], 2, size) cp.where(d >= self.size_degree[1], 1, size) print('estimate_size Done.') return size
def voltMeterwStep(ne, ex_mat, step_arr=None, parser=None): ''' Returns all measurements with this step_arr and ex_mat takes: ex_mat - array shape (n_source/sinks, 2) - excitation matrix with source and sink for each measurement step_arr - array shape (n_source/sinks) - step between measuring electrodes for each source/sink pair parser - string returns: pair_mat - array shape (n_measurements, 2) - matrix with all possible meas. electrode combinations ind_new - array shape (n_measurements) - helper array ''' if step_arr is None: step_arr = 1 + cp.arange((ex_mat.shape[0])) % (ne) elif type(step_arr) is int: step_arr = step_arr * cp.ones(ex_mat.shape[0]) % ne drv_a = ex_mat[:, 0] drv_b = ex_mat[:, 1] i0 = drv_a if parser == 'fmmu' else 0 A = cp.arange(ne) #M = cp.dot(cp.ones(ex_mat.shape[0])[:,None], A[None, :]) % self.ne #N = (M + step_arr[:, None]) % self.ne M = cp.arange(ex_mat.shape[0] * ne) % ne N = (M.reshape((ex_mat.shape[0], ne)) + step_arr[:, None]) % ne pair_mat = cp.stack((N.ravel(), M), axis=-1) #ind_new = cp.arange(pair_mat.shape[0]) % ex_mat.shape[0] ind_new = cp.arange(ex_mat.shape[0]) ind_new = cp.tile(ind_new, (ne, 1)).T.ravel() #print('before indtest', ind_new[20:70]) nz2 = cp.where(pair_mat == drv_a[ind_new, None]) nz3 = cp.where(pair_mat == drv_b[ind_new, None]) #print(ind_new) ind_ = cp.arange(pair_mat.shape[0]) ind_fin = cp.sum(ind_[:, None] == nz2[0][None], axis=1) ind_fin2 = cp.sum(ind_[:, None] == nz3[0][None], axis=1) ind_test = cp.less((ind_fin + ind_fin2), 0.5 * cp.ones(len(ind_fin))) pair_mat = pair_mat[ind_test, :] ind_new = ind_new[ind_test] sort_index = cp.argsort(ind_new) #print('after indtest', ind_new[20:70]) #meas = cp.concatenate((ex_mat[ind_new], pair_mat), axis=1) #print(meas[20:70]) return pair_mat, ex_mat[ind_new], ind_new
def _run_cupy_bin(data, bins, new_values): # replace inf by nan to avoid classify these values as we want to treat them as outliers data = cupy.where(data == cupy.inf, cupy.nan, data) data = cupy.where(data == -cupy.inf, cupy.nan, data) bins_cupy = cupy.asarray(bins) new_values_cupy = cupy.asarray(new_values) out = cupy.empty(data.shape, dtype='f4') out[:] = cupy.nan griddim, blockdim = cuda_args(data.shape) _run_gpu_bin[griddim, blockdim](data, bins_cupy, new_values_cupy, out) return out
def digitize(x, bins): # With right = Flase and bins in increasing order out = np.full(shape=x.shape, fill_value=0, dtype=np.int32) for i in range(1, len(bins)): bool_arr = np.logical_and(bins[i - 1] <= x, x < bins[i]) matched = np.where(bool_arr) out[matched] = i bool_arr = x >= bins[-1] matched = np.where(bool_arr) out[matched] = len(bins) return out
def __init__(self, low, high, res, basis, spectrum=False, fine=False, linspace=False): self.low = low self.high = high self.res = int(res) # somehow gets non-int... self.res_ghosts = int(res + 2) # resolution including ghosts self.order = basis.order # domain and element widths self.length = self.high - self.low self.dx = self.length / self.res # element Jacobian self.J = 2.0 / self.dx # The grid does not have a basis but does have quad weights self.quad_weights = cp.tensordot(cp.ones(self.res), cp.asarray(basis.weights), axes=0) # arrays self.arr = np.zeros((self.res_ghosts, self.order)) self.create_grid(basis.nodes) self.arr_cp = cp.asarray(self.arr) self.midpoints = np.array([(self.arr[i, -1] + self.arr[i, 0]) / 2.0 for i in range(1, self.res_ghosts - 1)]) self.arr_max = np.amax(abs(self.arr)) # velocity axis gets a positive/negative indexing slice self.one_negatives = cp.where(condition=self.arr_cp < 0, x=1, y=0) self.one_positives = cp.where(condition=self.arr_cp >= 0, x=1, y=0) # fine array if fine: fine_num = 25 # 200 for 1D poisson study self.arr_fine = np.array([np.linspace(self.arr[i, 0], self.arr[i, -1], num=fine_num) for i in range(self.res_ghosts)]) if linspace: lin_num = 400 self.arr_lin = np.linspace(self.low, self.high, num=lin_num) # spectral coefficients if spectrum: self.nyquist_number = 2.0 * self.res # # 2.5 * # mode number of nyquist frequency # print(self.nyquist_number) self.k1 = 2.0 * np.pi / self.length # fundamental mode self.wave_numbers = self.k1 * np.arange(1 - self.nyquist_number, self.nyquist_number) self.d_wave_numbers = cp.asarray(self.wave_numbers) self.grid_phases = cp.asarray(np.exp(1j * np.tensordot(self.wave_numbers, self.arr[1:-1, :], axes=0))) if linspace: self.lin_phases = cp.asarray(np.exp(1j * np.tensordot(self.wave_numbers, self.arr_lin, axes=0))) # Spectral matrices self.spectral_transform = basis.fourier_transform_array(self.midpoints, self.J, self.wave_numbers) self.inverse_transform = basis.inverse_transform_array(self.midpoints, self.J, self.wave_numbers)
def create_fwd(P): # convolution function fZ = cp.fft.fftshift(fzeta_loop_weights( P.Ntheta, P.Nrho, 2*P.beta, P.g-cp.log(P.am), 0, 4)) # (lp2C1,lp2C2), transformed log-polar to Cartesian coordinates tmp1 = cp.outer(cp.exp(cp.array(P.rhosp)), cp.cos(cp.array(P.thsp))).flatten() tmp2 = cp.outer(cp.exp(cp.array(P.rhosp)), cp.sin(cp.array(P.thsp))).flatten() lp2C1 = [None]*P.Nspan lp2C2 = [None]*P.Nspan for k in range(P.Nspan): lp2C1[k] = ((tmp1-(1-P.aR))*cp.cos(k*P.beta+P.beta/2) - tmp2*cp.sin(k*P.beta+P.beta/2))/P.aR lp2C2[k] = ((tmp1-(1-P.aR))*cp.sin(k*P.beta+P.beta/2) + tmp2*cp.cos(k*P.beta+P.beta/2))/P.aR lp2C2[k] *= (-1) # adjust for Tomopy cids = cp.where((lp2C1[k]**2+lp2C2[k]**2) <= 1)[0] lp2C1[k] = lp2C1[k][cids] lp2C2[k] = lp2C2[k][cids] # pids, index in polar grids after splitting by spans pids = [None]*P.Nspan [s0, th0] = cp.meshgrid(P.s, P.proj) th0 = th0.flatten() s0 = s0.flatten() for k in range(0, P.Nspan): pids[k] = cp.where((th0 >= k*P.beta-P.beta/2) & (th0 < k*P.beta+P.beta/2))[0] # (p2lp1,p2lp2), transformed polar to log-polar coordinates p2lp1 = [None]*P.Nspan p2lp2 = [None]*P.Nspan for k in range(P.Nspan): th00 = th0[pids[k]]-k*P.beta s00 = s0[pids[k]] p2lp1[k] = th00 p2lp2[k] = np.log(s00*P.aR+(1-P.aR)*np.cos(th00)) # adapt for gpu interp for k in range(0, P.Nspan): lp2C1[k] = (lp2C1[k]+1)/2*(P.N-1) lp2C2[k] = (lp2C2[k]+1)/2*(P.N-1) p2lp1[k] = (p2lp1[k]-P.thsp[0])/(P.thsp[-1]-P.thsp[0])*(P.Ntheta-1) p2lp2[k] = (p2lp2[k]-P.rhosp[0])/(P.rhosp[-1]-P.rhosp[0])*(P.Nrho-1) const = cp.sqrt(P.N*P.osangles/P.Nproj)*cp.pi/4 / \ P.aR/cp.sqrt(2) # adjust constant fZgpu = fZ[:, :P.Ntheta//2+1]*const if(P.interp_type == 'cubic'): fZgpu = fZgpu/(P.B3com[:, :P.Ntheta//2+1]) Pfwd0 = Pfwd(fZgpu, lp2C1, lp2C2, p2lp1, p2lp2, cids, pids) # array representation parsi, parsf = savePfwdpars(Pfwd0) return Pfwd0, parsi, parsf
def ts_max(x, window): if window > len(x): return cp.full(len(x), cp.nan) # 把空值填充为-inf x = cp.where(cp.isinf(x) | cp.isnan(x), -cp.inf, x) prefix = cp.full(window - 1, cp.nan) x_rolling_array = cp_rolling_window(x, window) result = cp.max(x_rolling_array, axis=1) # 结果中如果存在-inf,说明此window均为空,返回nan result = result.astype(float) result = cp.where(cp.isinf(result), cp.nan, result) return cp.concatenate((prefix, result))
def ts_argmin(x, window): if window > len(x): return cp.full(len(x), cp.nan) # 将nan及-inf填充为inf x = cp.where(cp.isinf(x) | cp.isnan(x), cp.inf, x) prefix = cp.full(window - 1, cp.nan) x_rolling_array = cp_rolling_window(x, window) result = cp.argmin(x_rolling_array, axis=1) # 找到window中全为-inf的情况,填充为nan result = result.astype(float) result = cp.where(cp.isinf(x_rolling_array).all(axis=1), cp.nan, result) result += 1 return cp.concatenate((prefix, result))
def julia_set_cp(zs, phase): ns = cp.zeros_like(Z, dtype=cp.float32) for i in range(n_iteration): # cupy doesn't support complex in where, we need to decompose it to real and img parts zs_real = cp.where( cp.abs(zs) < R, cp.real(zs**2 + 0.7885 * cp.exp(phase)), cp.real(zs)) zs_imag = cp.where( cp.abs(zs) < R, cp.imag(zs**2 + 0.7885 * cp.exp(phase)), cp.imag(zs)) zs = zs_real + 1j * zs_imag not_diverged = cp.abs(zs) < R ns = ns + not_diverged.astype(cp.float32) return ns, zs
def MWU_game_algorithm_experiment(payoff_mat, phi=1/2, steps_number=10000): payoff_mat = np.array(payoff_mat) rows_number = payoff_mat.shape[0] cols_number = payoff_mat.shape[1] p_0 = np.ones((1, rows_number)) p_0 = p_0/rows_number p_t = p_0 j_sumed = np.zeros((cols_number, 1)) smallest_column_payoff = 1 p_best = p_0 p_t_sum = np.zeros((1, rows_number)) start = time.time() row_row = [] col_col = [] row_col = [] times = [] curr_index = 125 for i in range (1, steps_number + 1): payoffs = np.matmul(p_t, payoff_mat) j_best_response = np.argmax(payoffs) if(payoffs[0, j_best_response] < smallest_column_payoff): smallest_column_payoff = payoffs[0, j_best_response] p_best = p_t j_sumed[j_best_response] += 1 m_t = payoff_mat[:,j_best_response] m_t_negative = (m_t < 0) p_t_significant = (p_t > SIGNIFICANCE_CONST) to_update = np.logical_or(m_t_negative, p_t_significant[0]) m_t_updating = np.where(to_update,m_t,0) p_t_updating = np.where(to_update,p_t,0) p_t = np.multiply((1 - phi * m_t_updating), p_t_updating) p_t = p_t/p_t.sum() p_t_sum = p_t_sum + p_t if(i == curr_index): j_distribution = j_sumed / j_sumed.sum() print(i) now = time.time() times.append(now - start) row_row.append(max(epsilon_value(p_best, np.transpose(p_best), payoff_mat))) col_col.append(max(epsilon_value(np.transpose(j_distribution), j_distribution, payoff_mat))) row_col.append(max(epsilon_value(p_best, j_distribution, payoff_mat))) start -= (time.time() - now) curr_index *= 2 # game_value = np.matmul(np.matmul(p_best, payoff_mat), j_distribution)[0][0] # print() return times, row_row, col_col, row_col
def ray_color(r: RayList, world: HittableList, depth: int) \ -> Tuple[Optional[RayList], Optional[Vec3List], Vec3List]: length = len(r) if not r.direction().e.any(): return None, None, Vec3List.new_zero(length) # Calculate object hits rec_list: HitRecordList = world.hit(r, 0.001, cp.inf) # Useful empty arrays empty_vec3list = Vec3List.new_zero(length) empty_array_float = cp.zeros(length, cp.float32) empty_array_bool = cp.zeros(length, cp.bool) empty_array_int = cp.zeros(length, cp.int32) # Background / Sky unit_direction = r.direction().unit_vector() sky_condition = Vec3List.from_array((unit_direction.length() > 0) & (rec_list.material == 0)) t = (unit_direction.y() + 1) * 0.5 blue_bg = (Vec3List.from_vec3(Color(1, 1, 1), length).mul_ndarray(1 - t) + Vec3List.from_vec3(Color(0.5, 0.7, 1), length).mul_ndarray(t)) result_bg = Vec3List(cp.where(sky_condition.e, blue_bg.e, empty_vec3list.e)) if depth <= 1: return None, None, result_bg # Material scatter calculations materials: Dict[int, Material] = world.get_materials() scattered_list = RayList.new_zero(length) attenuation_list = Vec3List.new_zero(length) for mat_idx in materials: mat_condition = (rec_list.material == mat_idx) mat_condition_3 = Vec3List.from_array(mat_condition) if not mat_condition.any(): continue ray = RayList( Vec3List(cp.where(mat_condition_3.e, r.orig.e, empty_vec3list.e)), Vec3List(cp.where(mat_condition_3.e, r.dir.e, empty_vec3list.e))) rec = HitRecordList( Vec3List( cp.where(mat_condition_3.e, rec_list.p.e, empty_vec3list.e)), cp.where(mat_condition, rec_list.t, empty_array_float), cp.where(mat_condition, rec_list.material, empty_array_int), Vec3List( cp.where(mat_condition_3.e, rec_list.normal.e, empty_vec3list.e)), cp.where(mat_condition, rec_list.front_face, empty_array_bool)) ray, rec, idx_list = compress(ray, rec) scattered, attenuation = materials[mat_idx].scatter(ray, rec) scattered, attenuation = decompress(scattered, attenuation, idx_list, length) scattered_list += scattered attenuation_list += attenuation return scattered_list, attenuation_list, result_bg
def leaky_relu(self, Z): '''Leaky ReLU function''' A = cp.where(Z >= 0, Z, Z * 0.01) assert (A.shape == Z.shape) return A
def get_text_predictions(df, max_features=25_000): model = TfidfVectorizer(stop_words='english', binary=True, max_features=max_features) text_embeddings = model.fit_transform(df_cu['title']).toarray() print('Finding similar titles...') CHUNK = 1024 * 4 CTS = len(df) // CHUNK if (len(df) % CHUNK) != 0: CTS += 1 preds = [] for j in range(CTS): a = j * CHUNK b = (j + 1) * CHUNK b = min(b, len(df)) print('chunk', a, 'to', b) # COSINE SIMILARITY DISTANCE cts = cupy.matmul(text_embeddings, text_embeddings[a:b].T).T for k in range(b - a): IDX = cupy.where(cts[k, ] > 0.7)[0] o = df.iloc[cupy.asnumpy(IDX)].posting_id.values preds.append(o) del model, text_embeddings gc.collect() return preds
def non_maximum_suppression(bbox, thresh, score=None, limit=None): """非极大值抑制""" bbox_y1 = bbox[:, 0] bbox_x1 = bbox[:, 1] bbox_y2 = bbox[:, 2] bbox_x2 = bbox[:, 3] area = (bbox_x2 - bbox_x1 + 1) * (bbox_y2 - bbox_y1 + 1) n_bbox = bbox.shape[0] if score is not None: order = score.argsort()[::-1].astype(np.int32) else: order = cp.arange(n_bbox, dtype=np.int32) keep = [] # 预测框之间进行两两比较,去除重叠面积iou大于thresh的框 while order.size > 0: i = order[0] keep.append(i) xx1 = cp.maximum(bbox_x1[i], bbox_x1[order[1:]]) yy1 = cp.maximum(bbox_y1[i], bbox_y1[order[1:]]) xx2 = cp.minimum(bbox_x2[i], bbox_x2[order[1:]]) yy2 = cp.minimum(bbox_y2[i], bbox_y2[order[1:]]) width = cp.maximum(0., (xx2 - xx1 + 1)) height = cp.maximum(0., (yy2 - yy1 + 1)) inter = width * height iou = inter / (area[i] + area[order[1:]] - inter) index = cp.where(iou <= thresh)[0] order = order[(index + 1).tolist()] if limit is not None: keep = keep[:limit] return cp.asnumpy(keep)
def test_lambdaop_misalign(cpu): size = 12 df0 = pd.DataFrame({ "a": np.arange(size), "b": np.random.choice(["apple", "banana", "orange"], size), "c": np.random.choice([0, 1], size), }) ddf0 = dd.from_pandas(df0, npartitions=4) cont_names = ColumnGroup(["a"]) cat_names = ColumnGroup(["b"]) label = ColumnGroup(["c"]) if cpu: label_feature = label >> (lambda col: np.where(col == 4, 0, 1)) else: label_feature = label >> (lambda col: cp.where(col == 4, 0, 1)) workflow = nvt.Workflow(cat_names + cont_names + label_feature) dataset = nvt.Dataset(ddf0, cpu=cpu) transformed = workflow.transform(dataset) assert_eq_dd( df0[["a", "b"]], transformed.to_ddf().compute()[["a", "b"]], check_index=False, )
def signedpower(x, a): #将inf和-inf处理为nan #x[cp.isinf(x)] = cp.nan # 经测试where替换更快一些 x = cp.where(cp.isinf(x), cp.nan, x) return cp.sign(x) * cp.abs(x)**a
def registration_flow_batch(self, psi, g, mmin, mmax, flow=None, pars=[0.5, 3, 20, 16, 5, 1.1, 4]): """Find optical flow for all projections in parallel on CPU""" if (flow is None): flow = np.zeros([*psi.shape, 2], dtype='float32') total = 0 for ids in chunk(range(self.ntheta), self.ptheta): flownew = flow[ids] with cf.ThreadPoolExecutor(16) as e: #update flow in place e.map(partial(self.registration_flow, psi[ids], g[ids], mmin[ids], mmax[ids], flownew, pars), range(len(ids))) # control Farneback's (may diverge for small window sizes) #err = np.linalg.norm( # g[ids]-self.apply_flow_gpu_batch(psi[ids], flownew), axis=(1, 2)) #err1 = np.linalg.norm( # g[ids]-self.apply_flow_gpu_batch(psi[ids], flow[ids]), axis=(1, 2)) #idsgood = np.where(err1 >= err)[0] g_gpu = cp.array(g[ids]) psi_gpu = cp.array(psi[ids]) flownew_gpu = cp.array(flownew) flow_gpu = cp.array(flow[ids]) err = cp.linalg.norm( g_gpu-self.apply_flow_gpu(psi_gpu, flownew_gpu,0), axis=(1, 2)) err1 = cp.linalg.norm( g_gpu-self.apply_flow_gpu(psi_gpu, flow_gpu, 0), axis=(1, 2)) idsgood = cp.where(err1 >= err)[0].get() total += len(idsgood) flow[ids[idsgood]] = flownew[idsgood] # print('bad alignment for:', self.ntheta-total) return flow
def _rank(x): x = cp.where(cp.isinf(x) | cp.isnan(x), cp.inf, x) # 此方式,只一次排序,对于长数组更有优势 temp = x.argsort() ranks = cp.empty_like(temp) ranks[temp] = cp.arange(len(x)) ranks = ranks.astype(float) # 将nan和inf的位置恢复为nan,不参与rank ranks = cp.where(cp.isinf(x), cp.nan, ranks) #ranks[cp.isinf(x)] = cp.nan # 返回x的最后一个元素的排名 return (ranks + 1)[-1]
def CheckIfEscaped(photon_state, tau_atm, escaped_mu): ###################################################################################### # This checks if a photon has escaped, calculates the momentum that photon takes with it # then removes that photon from the list. # # photon_state -- A CuPy array consisting of [x,y,z,phi,theta,lambda] for each photon. # tau_atm -- The depth of the atmosphere. ###################################################################################### # Looks for all the photons above the atmosphere. escaped_photons = cp.where(photon_state[:, 2] >= tau_atm)[0] momentum_transfer = 0 if len(escaped_photons) > 0: # Find the angles the particles escaped at escaped_mu += cp.cos(photon_state[escaped_photons, 4]).tolist() # Calculates the momentum transferred, this is the difference between the initial z momentum and the final z momentum momentum_transfer = float( cp.sum(h * (-cp.cos(photon_state[escaped_photons, 4])) / photon_state[escaped_photons, 5])) # Remove the escaped photons photon_state = RemovePhotons(photon_state, escaped_photons) return photon_state, momentum_transfer, escaped_mu
def preprocess(img, pos, half_size=15, device=0): with cp.cuda.Device(device): pos_x_left, pos_x_right = pos[:, 0] - half_size, pos[:, 0] + half_size pos_y_left, pos_y_right = pos[:, 1] - half_size, pos[:, 1] + half_size #TODO: current using filtering option, in the near future, change to padding option SELECT_MAP = (pos_x_left >= 0) * (pos_y_left >= 0) * ( pos_x_right < 2048) * (pos_y_right < 2048) SELECT_INDEX = cp.where(SELECT_MAP > 0)[0] pos_x_left, pos_x_right, pos_y_left, pos_y_right = pos_x_left[ SELECT_INDEX], pos_x_right[SELECT_INDEX], pos_y_left[ SELECT_INDEX], pos_y_right[SELECT_INDEX] pos = pos[SELECT_INDEX] # shape should be dx * dy * dz pos_x = pos_x_left pos_x = cp.expand_dims(pos_x, axis=0) adding = cp.expand_dims(cp.arange(2 * half_size), 1) pos_x = pos_x + adding pos_y = pos_y_left pos_y = cp.expand_dims(pos_y, axis=0) adding = cp.expand_dims(cp.arange(2 * half_size), 1) pos_y = pos_y + adding # x * y * N _x = img[pos_x[:, cp.newaxis, :], pos_y] return _x.get(), pos.get() #, groups_start, groups_end
def add_water(self, atom_histograms_gpu): vacs = cp.prod(cp.where(atom_histograms_gpu == 0, True, False), axis=0) # average number of water molecules in a voxel vox_wat_num = water_num_dens * self.d1 * self.d2 * self.dz box = (self.n_slice, self.n1, self.n2) oxygens = cp.where(vacs, cp.random.poisson(vox_wat_num, box), 0).astype(cp.int) hydrogens = cp.where(vacs, cp.random.poisson(vox_wat_num * 2, box), 0).astype(cp.int) unique_elements_list = list(self.unique_elements) for z, hist in [(1, hydrogens), (8, oxygens)]: idx = unique_elements_list.index(z) atom_histograms_gpu[idx] += hist return atom_histograms_gpu
def find_similar_titles(): preds = [] CHUNK = 1024 * 4 print('Finding similar titles...') CTS = len(test) // CHUNK if len(test) % CHUNK != 0: CTS += 1 for j in range(CTS): a = j * CHUNK b = (j + 1) * CHUNK b = min(b, len(test)) print('chunk', a, 'to', b) # COSINE SIMILARITY DISTANCE cts = cupy.matmul(text_embeddings, text_embeddings[a:b].T).T for k in range(b - a): IDX = cupy.where(cts[k,] > 0.7)[0] o = test.iloc[cupy.asnumpy(IDX)].posting_id.values preds.append(o) del model, text_embeddings _ = gc.collect() test['preds'] = preds test.head()
def remove_outliers(self, data): """Remove outliers""" if (int(self.dezinger) > 0): r = int(self.dezinger) fdata = ndimage.median_filter(data, [1, r, r]) ids = cp.where(cp.abs(fdata - data) > 0.5 * cp.abs(fdata)) data[ids] = fdata[ids]
def filter_data_by_class_ovo(x, y, classes, prevent_relabel=False): x = SVM._select_classes(x, y, classes) y = SVM._select_classes(y, y, classes) if prevent_relabel == False: y = xp.where(y == classes[0], 1, -1) return x, y
def _validate_n_bins(self, n_features): """Returns n_bins_, the number of bins per feature. """ orig_bins = self.n_bins if isinstance(orig_bins, numbers.Number): if not isinstance(orig_bins, numbers.Integral): raise ValueError("{} received an invalid n_bins type. " "Received {}, expected int." .format(KBinsDiscretizer.__name__, type(orig_bins).__name__)) if orig_bins < 2: raise ValueError("{} received an invalid number " "of bins. Received {}, expected at least 2." .format(KBinsDiscretizer.__name__, orig_bins)) return np.full(n_features, orig_bins, dtype=int) n_bins = check_array(orig_bins, dtype=np.int, copy=True, ensure_2d=False) if n_bins.ndim > 1 or n_bins.shape[0] != n_features: raise ValueError("n_bins must be a scalar or array " "of shape (n_features,).") bad_nbins_value = (n_bins < 2) | (n_bins != orig_bins) violating_indices = np.where(bad_nbins_value)[0] if violating_indices.shape[0] > 0: indices = ", ".join(str(i) for i in violating_indices) raise ValueError("{} received an invalid number " "of bins at indices {}. Number of bins " "must be at least 2, and must be an int." .format(KBinsDiscretizer.__name__, indices)) return n_bins
def calc_neighbourhood_stats(self, distances, radius): neighbours = distances < radius cp.fill_diagonal(neighbours, False) neighbours_states_sum = (self.state[cp.newaxis, :, :] * neighbours[:, :, cp.newaxis]).sum(axis=1) neighbours_num = neighbours.sum(axis=1) has_neighbours = neighbours_num > 0 neighbours_num = neighbours_num[cp.where(has_neighbours)[0], cp.newaxis] neighbours_states_sum = neighbours_states_sum[ cp.where(has_neighbours)[0], :] return has_neighbours, neighbours_num, neighbours_states_sum
def _fast_gradient_descent(self): eta_init = self._optimal_eta_init() alpha = xp.zeros(self._n) theta = xp.zeros(self._n) eta = eta_init grad_theta = self._compute_gradient(theta) objective_val_size = int( self._max_iter // 10) + (1 if self._max_iter % 10 == 0 else 0) objective_vals = xp.ones(objective_val_size) misclassification_error_per_iter = xp.ones(objective_val_size) iter = 0 while iter < self._max_iter: eta = self._backtracking_line_search(theta, eta=eta) alpha_new = theta - eta * grad_theta theta = alpha_new + iter / (iter + 3) * (alpha_new - alpha) grad_theta = self._compute_gradient(theta) alpha = alpha_new iter += 1 if self._display_plots and iter % 10 == 0: objective_vals[int(iter / 10)] = self._objective(alpha) self._coef_matrix = alpha misclassification_error_per_iter[int( iter / 10)] = self.compute_misclassification_error( self._x, xp.where(self._y > 0, self._primary_class, self._secondary_class)) return alpha, objective_vals, misclassification_error_per_iter
def interval(self, mx, size): """Generate multiple integers independently sampled uniformly from ``[0, mx]``. Args: mx (int): Upper bound of the interval size (None or int or tuple): Shape of the array or the scalar returned. Returns: int or cupy.ndarray: If ``None``, an :class:`cupy.ndarray` with shape ``()`` is returned. If ``int``, 1-D array of length size is returned. If ``tuple``, multi-dimensional array with shape ``size`` is returned. Currently, each element of the array is ``numpy.int32``. """ dtype = numpy.int32 if size is None: return self.interval(mx, 1).reshape(()) elif isinstance(size, int): size = (size, ) if mx == 0: return cupy.zeros(size, dtype=dtype) mask = (1 << mx.bit_length()) - 1 mask = cupy.array(mask, dtype=dtype) ret = cupy.zeros(size, dtype=dtype) sample = cupy.zeros(size, dtype=dtype) done = cupy.zeros(size, dtype=numpy.bool_) while True: curand.generate( self._generator, sample.data.ptr, sample.size) sample &= mask success = sample <= mx ret = cupy.where(success, sample, ret) done |= success if done.all(): return ret
def toi(x): return cupy.where(x, 1, 0)
def testInhibition(self): """ Test if the firing number of coincidences after inhibition equals spatial pooler numActiveColumnsPerInhArea. """ # Miscellaneous variables: # n, w: n, w of encoders # inputLen: Length of binary input # synPermConnected: Spatial pooler synPermConnected # synPermActiveInc: Spatial pooler synPermActiveInc # connectPct: Initial connect percentage of permanences # columnDimensions: Number of spatial pooler coincidences # numActiveColumnsPerInhArea: Spatial pooler numActiveColumnsPerInhArea # stimulusThreshold: Spatial pooler stimulusThreshold # spSeed: Spatial pooler for initial permanences # stimulusThresholdInh: Parameter for inhibition, default value 0.00001 # kDutyCycleFactor: kDutyCycleFactor for dutyCycleTieBreaker in # Inhibition # spVerbosity: Verbosity to print other sp initial parameters # testIter: Testing iterations n = 100 w = 15 inputLen = 300 columnDimensions = 2048 numActiveColumnsPerInhArea = 40 stimulusThreshold = 0 spSeed = 1956 stimulusThresholdInh = 0.00001 kDutyCycleFactor = 0.01 spVerbosity = 0 testIter = 100 spTest = SpatialPooler( columnDimensions=(columnDimensions, 1), inputDimensions=(1, inputLen), potentialRadius=inputLen / 2, numActiveColumnsPerInhArea=numActiveColumnsPerInhArea, spVerbosity=spVerbosity, stimulusThreshold=stimulusThreshold, seed=spSeed ) initialPermanence = spTest._initialPermanence() spTest._masterPotentialM, spTest._masterPermanenceM = ( spTest._makeMasterCoincidences(spTest.numCloneMasters, spTest._coincRFShape, spTest.potentialPct, initialPermanence, spTest.random)) spTest._updateInhibitionObj() boostFactors = cupy.ones(columnDimensions) for i in range(testIter): spTest._iterNum = i # random binary input input_ = cupy.zeros((1, inputLen)) nonzero = numpy.random.random(inputLen) input_[0][cupy.where (nonzero < float(w)/float(n))] = 1 # overlap step spTest._computeOverlapsFP(input_, stimulusThreshold=spTest.stimulusThreshold) spTest._overlaps *= boostFactors onCellIndices = cupy.where(spTest._overlaps > 0) spTest._onCells.fill(0) spTest._onCells[onCellIndices] = 1 denseOn = spTest._onCells # update _dutyCycleBeforeInh spTest.dutyCyclePeriod = min(i + 1, 1000) spTest._dutyCycleBeforeInh = ( (spTest.dutyCyclePeriod - 1) * spTest._dutyCycleBeforeInh +denseOn) / spTest.dutyCyclePeriod dutyCycleTieBreaker = spTest._dutyCycleAfterInh.copy() dutyCycleTieBreaker *= kDutyCycleFactor # inhibition step numOn = spTest._inhibitionObj.compute( spTest._overlaps + dutyCycleTieBreaker, spTest._onCellIndices, stimulusThresholdInh, # stimulusThresholdInh max(spTest._overlaps)/1000, # addToWinners ) # update _dutyCycleAfterInh spTest._onCells.fill(0) onCellIndices = spTest._onCellIndices[0:numOn] spTest._onCells[onCellIndices] = 1 denseOn = spTest._onCells spTest._dutyCycleAfterInh = (((spTest.dutyCyclePeriod-1) * spTest._dutyCycleAfterInh + denseOn) / spTest.dutyCyclePeriod) # learning step spTest._adaptSynapses(onCellIndices, [], input_) # update boostFactor spTest._updateBoostFactors() boostFactors = spTest._firingBoostFactors # update dutyCycle and boost if ((spTest._iterNum+1) % 50) == 0: spTest._updateInhibitionObj() spTest._updateMinDutyCycles( spTest._dutyCycleBeforeInh, spTest.minPctDutyCycleBeforeInh, spTest._minDutyCycleBeforeInh) spTest._updateMinDutyCycles( spTest._dutyCycleAfterInh, spTest.minPctDutyCycleAfterInh, spTest._minDutyCycleAfterInh) # test numOn and spTest.numActiveColumnsPerInhArea self.assertEqual(numOn, spTest.numActiveColumnsPerInhArea, "Error at input %s, actual numOn are: %i, " "numActivePerInhAre is: %s" % ( i, numOn, numActiveColumnsPerInhArea))