def toGPU(batch, gpu_num): class Data(): pass data = Data() if gpu_num == 0: setattr(data, "image_0", batch[0]) setattr(data, "text_0", batch[1]) setattr(data, "wrong_image_0", batch[2]) setattr(data, "wrong_text_0", batch[3]) else: split_size = len(batch[0]) // 2 image = xp.array_split(xp.array(batch[0], dtype="float32"), gpu_num) text = [batch[1][i*split_size:(i+1)*split_size] for i in range(gpu_num)] wrong_image = xp.array_split(xp.array(batch[2], dtype="float32"), gpu_num) wrong_text = [batch[3][i*split_size:(i+1)*split_size] for i in range(gpu_num)] for i in range(gpu_num): setattr(data, f"image_{i}", cuda.to_gpu(image[i], i)) setattr(data, f"text_{i}", cuda.to_gpu(text[i], i)) setattr(data, f"wrong_image_{i}", cuda.to_gpu(wrong_image[i], i)) setattr(data, f"wrong_text_{i}", cuda.to_gpu(wrong_text[i], i)) return data
def batched_silhouette_scores(embeddings, clusters, batch_size=BATCH_SIZE): """Calculate silhouette score in batches on the CPU. Compatible with data on GPU or CPU Args: embeddings (cudf.DataFrame or cupy.ndarray): input features to clustering clusters (cudf.DataFrame or cupy.ndarray): cluster values for each data point batch_size (int, optional): Size for batching. Returns: float: mean silhouette score from batches """ # Function to calculate batched results def _silhouette_scores(input_data): embeddings, clusters = input_data return silhouette_score(cupy.asnumpy(embeddings), cupy.asnumpy(clusters)) if hasattr(embeddings, 'values'): embeddings = embeddings.values embeddings = cupy.asarray(embeddings) if hasattr(clusters, 'values'): clusters = clusters.values clusters = cupy.asarray(clusters) n_data = len(embeddings) msg = 'Calculating silhouette score on {} molecules'.format(n_data) if batch_size < n_data: msg += ' with batch size of {}'.format(batch_size) logger.info(msg + ' ...') n_chunks = int(math.ceil(n_data / batch_size)) embeddings_chunked = cupy.array_split(embeddings, n_chunks) clusters_chunked = cupy.array_split(clusters, n_chunks) # Calculate scores on batches and return the average scores = list( map(_silhouette_scores, zip(embeddings_chunked, clusters_chunked))) return numpy.nanmean(numpy.array(scores))
def toGPU(batch, gpu_num): class Data(): pass data = Data() if gpu_num == 0: setattr(data, "image_0", batch[0]) setattr(data, "depth_0", batch[1]) setattr(data, "text_0", batch[2]) setattr(data, "wrong_image_0", batch[3]) setattr(data, "wrong_depth_0", batch[4]) setattr(data, "wrong_text_0", batch[5]) else: split_size = len(batch[1]) // 2 image = [batch[0][i*split_size:(i+1)*split_size] for i in range(gpu_num)] depth = [batch[1][i*split_size:(i+1)*split_size] for i in range(gpu_num)] image = [{key:cuda.to_gpu(np.array([data[key] for data in image[i]], dtype="float32"), i) for key in image_keys} for i in range(gpu_num)] depth = [{key:cuda.to_gpu(np.array([data[key] for data in depth[i]], dtype="float32"), i) for key in depth_keys} for i in range(gpu_num)] text = xp.array_split(xp.array(batch[2], dtype="float32"), gpu_num) wrong_image = [batch[3][i*split_size:(i+1)*split_size] for i in range(gpu_num)] wrong_depth = [batch[4][i*split_size:(i+1)*split_size] for i in range(gpu_num)] wrong_image = [{key:cuda.to_gpu(np.array([data[key] for data in wrong_image[i]], dtype="float32"), i) for key in image_keys} for i in range(gpu_num)] wrong_depth = [{key:cuda.to_gpu(np.array([data[key] for data in wrong_depth[i]], dtype="float32"), i) for key in depth_keys} for i in range(gpu_num)] wrong_text = xp.array_split(xp.array(batch[5], dtype="float32"), gpu_num) for i in range(gpu_num): setattr(data, f"image_{i}", image[i]) setattr(data, f"depth_{i}", depth[i]) setattr(data, f"text_{i}", cuda.to_gpu(text[i], i)) setattr(data, f"wrong_image_{i}", wrong_image[i]) setattr(data, f"wrong_depth_{i}", wrong_depth[i]) setattr(data, f"wrong_text_{i}", cuda.to_gpu(wrong_text[i], i)) return data
def gen_disctance_list_ds(self, w, h, height, downsampling_xy, pts_cp): ### Generate Distance-List ### with DownSampling # print("Distance") px_list = [] for i in range(w): for j in range(h): px = [j * downsampling_xy, i * downsampling_xy, height] px_list.append([px]) ### pos-numpy array (from Image) pos_cp = cp.array(px_list) # print(pos_cp) # print("pos.shape :", pos_cp.shape) ### Separate Process ### https://qiita.com/kazuki_hayakawa/items/557edd922f9f1fafafe0 SPLIT = 250 pos_cp_split = cp.array_split(pos_cp, SPLIT) # print(len(pos_cp_split)) dist_tmp = [] for i in range(SPLIT): tmp_p = pos_cp_split[i] # print("pts.shape :", tmp_p.shape) ### pts-numpy array (from STL) # print("pts.shape :", pts_cp.shape) ### d = self.clac_all_distance(tmp_p, pts_cp) dist_tmp.append(d) dist_list = cp.concatenate(dist_tmp, 0) # print(len(dist_list)) return dist_list
def forward(self, x, y): y = chainermn.functions.bcast(self.comm, y, 0) partions = cp.array_split(x, self.comm.size, -2) # This part needs fixing. Probably all conditions are not checked if self.comm.rank == 0: x = partions[0] elif self.comm.rank == 1: x = partions[1] elif self.comm.rank == 2: x = partions[2] elif self.comm.rank == 3: x = partions[3] else: print("Rank does not exist") h = FX.halo_exchange_3d(self.comm, x, k_size=3, index=1, pad=0) h = F.leaky_relu(self.Conv1(h)) h = F.average_pooling_3d(h, ksize=2, stride=2) h = FX.halo_exchange_3d(self.comm, h, k_size=3, index=2, pad=0) h = F.leaky_relu(self.Conv2(h)) h = F.average_pooling_3d(h, ksize=2, stride=2) hs = chainermnx.functions.spatialallgather(self.comm, h) h = F.concat(hs, -2) h = F.leaky_relu(self.Conv3(h)) h = F.average_pooling_3d(h, ksize=2, stride=2) h = F.leaky_relu(self.Conv4(h)) h = F.average_pooling_3d(h, ksize=2, stride=2) h = F.leaky_relu(self.Conv5(h)) h = F.average_pooling_3d(h, ksize=2, stride=2) h = F.leaky_relu(self.Conv6(h)) h = F.average_pooling_3d(h, ksize=2, stride=2) h = F.leaky_relu(self.Conv7(h)) h = F.leaky_relu(self.FC1(h)) h = F.leaky_relu(self.FC2(h)) h = self.Output(h) loss = F.mean_squared_error(h, y) chainer.report({'loss': loss}, self) # print("Rank ", self.comm.rank, "Completed forward and y is ", y) return loss
def _stratify_split(X, y, n_train, n_test, x_numba, y_numba, random_state): """ Function to perform a stratified split based on y lables. Based on scikit-learn stratified split implementation. Parameters ---------- X, y: Shuffled input data and labels n_train: Number of samples in train set n_test: number of samples in test set x_numba: Determines whether the data should be converted to numba y_numba: Determines whether the labales should be converted to numba Returns ------- X_train, X_test: Data X divided into train and test sets y_train, y_test: Labels divided into train and test sets """ x_cudf = False y_cudf = False if isinstance(X, cudf.DataFrame): x_cudf = True elif hasattr(X, "__cuda_array_interface__"): X = cp.asarray(X) x_order = _strides_to_order(X.__cuda_array_interface__['strides'], cp.dtype(X.dtype)) if isinstance(y, cudf.Series): y_cudf = True elif hasattr(y, "__cuda_array_interface__"): y = cp.asarray(y) y_order = _strides_to_order(y.__cuda_array_interface__['strides'], cp.dtype(y.dtype)) elif isinstance(y, cudf.DataFrame): y_cudf = True # ensuring it has just one column if y.shape[1] != 1: raise ValueError('Expected one label, but found y' 'with shape = %d' % (y.shape)) classes, y_indices = cp.unique(y.values if y_cudf else y, return_inverse=True) n_classes = classes.shape[0] class_counts = cp.bincount(y_indices) if n_train < n_classes: raise ValueError('The train_size = %d should be greater or ' 'equal to the number of classes = %d' % (n_train, n_classes)) if n_test < n_classes: raise ValueError('The test_size = %d should be greater or ' 'equal to the number of classes = %d' % (n_test, n_classes)) class_indices = cp.array_split(cp.argsort(y_indices), n_classes) X_train = None # random_state won't be None or int, that's handled earlier if isinstance(random_state, np.random.RandomState): random_state = cp.random.RandomState(seed=random_state.get_state()[1]) # Break ties n_i = _approximate_mode(class_counts, n_train, random_state) class_counts_remaining = class_counts - n_i t_i = _approximate_mode(class_counts_remaining, n_test, random_state) for i in range(n_classes): permutation = random_state.permutation(class_counts[i].item()) perm_indices_class_i = class_indices[i].take(permutation) if hasattr(X, "__cuda_array_interface__") or \ isinstance(X, cupyx.scipy.sparse.csr_matrix): X_train_i = cp.array(X[perm_indices_class_i[:n_i[i]]], order=x_order) X_test_i = cp.array(X[perm_indices_class_i[n_i[i]:n_i[i] + t_i[i]]], order=x_order) y_train_i = cp.array(y[perm_indices_class_i[:n_i[i]]], order=y_order) y_test_i = cp.array(y[perm_indices_class_i[n_i[i]:n_i[i] + t_i[i]]], order=y_order) if X_train is None: X_train = cp.array(X_train_i, order=x_order) y_train = cp.array(y_train_i, order=y_order) X_test = cp.array(X_test_i, order=x_order) y_test = cp.array(y_test_i, order=y_order) else: X_train = cp.concatenate([X_train, X_train_i], axis=0) X_test = cp.concatenate([X_test, X_test_i], axis=0) y_train = cp.concatenate([y_train, y_train_i], axis=0) y_test = cp.concatenate([y_test, y_test_i], axis=0) elif x_cudf: X_train_i = X.iloc[perm_indices_class_i[:n_i[i]]] X_test_i = X.iloc[perm_indices_class_i[n_i[i]:n_i[i] + t_i[i]]] y_train_i = y.iloc[perm_indices_class_i[:n_i[i]]] y_test_i = y.iloc[perm_indices_class_i[n_i[i]:n_i[i] + t_i[i]]] if X_train is None: X_train = X_train_i y_train = y_train_i X_test = X_test_i y_test = y_test_i else: X_train = cudf.concat([X_train, X_train_i], ignore_index=False) X_test = cudf.concat([X_test, X_test_i], ignore_index=False) y_train = cudf.concat([y_train, y_train_i], ignore_index=False) y_test = cudf.concat([y_test, y_test_i], ignore_index=False) if x_numba: X_train = cuda.as_cuda_array(X_train) X_test = cuda.as_cuda_array(X_test) elif x_cudf: X_train = cudf.DataFrame(X_train) X_test = cudf.DataFrame(X_test) if y_numba: y_train = cuda.as_cuda_array(y_train) y_test = cuda.as_cuda_array(y_test) elif y_cudf: y_train = cudf.DataFrame(y_train) y_test = cudf.DataFrame(y_test) return X_train, X_test, y_train, y_test
locals()[item+'_1951_2014'] = cp.load(prism_2_hist[i]) # combine two-stage dataset into a historical data set, time range: 1895-2014 locals()[item+'_h'] = cp.concatenate((locals() [item+'_1895_1950'], locals()[item+'_1951_2014'])) # load future data, time range: 2006-2099 locals()[item+"_f_a_"+mdl] = np.load(ppt_future[mdl_c]) locals()[item+"_f_"+mdl] = np.delete(locals()[item+"_f_a_"+mdl], [it for it in range(0, (2015-2006)*12)], 0) locals()[item+"_f_"+mdl] = cp.array(locals()[item+"_f_"+mdl]) # stack historical and future data locals()[item+"_all_"+mdl] = cp.concatenate((locals() [item+'_h'], locals()[item+"_f_"+mdl]), axis=0) # calculate seasonal data, time ra locals()[item+'_mon_fmt_'+mdl] = locals()[item+"_all_"+mdl].reshape( (int(locals()[item+"_all_"+mdl].shape[0]/12), 12, len(dist_id_sel))) locals()[item+'_seasonal_all_'+mdl] = cp.array_split(locals() [item+'_mon_fmt_'+mdl], 4, axis=1) locals()[item+'_sp_'+mdl] = locals()[item + '_seasonal_all_'+mdl][0].mean(axis=1) locals()[item+'_sm_'+mdl] = locals()[item + '_seasonal_all_'+mdl][1].mean(axis=1) locals()[item+'_fl_'+mdl] = locals()[item + '_seasonal_all_'+mdl][2].mean(axis=1) locals()[item+'_wt_'+mdl] = locals()[item + '_seasonal_all_'+mdl][3].mean(axis=1) # construct A matrix (without npp): 1980-2014 locals()[item+'_sp_'+mdl+"_A"] = locals()[item+'_sp_' + mdl][yr_total.index("1980"):yr_total.index("2015"), ].mean(axis=0) locals()[item+'_sm_'+mdl+"_A"] = locals()[item+'_sm_' + mdl][yr_total.index("1980"):yr_total.index("2015"), ].mean(axis=0) locals()[item+'_fl_'+mdl+"_A"] = locals()[item+'_fl_' + mdl][yr_total.index("1980"):yr_total.index("2015"), ].mean(axis=0)
def FilterData(matrizOnibusCpu, matrizLinhasCpu, busIdList, lineIdList, CONFIGS, logging): distanceTolerance = float( CONFIGS['default_correction_method']['distanceTolerance']) detectionPercentage = float( CONFIGS['default_correction_method']['detectionPercentage']) busStepSize = int(CONFIGS['default_correction_method']['busStepSize']) lineStepSize = int(CONFIGS['default_correction_method']['lineStepSize']) matrizOnibus = cp.asarray(matrizOnibusCpu) matrizLinhas = cp.asarray(matrizLinhasCpu) busesList = cp.array_split( matrizOnibus, int(matrizOnibus.shape[0] / busStepSize if matrizOnibus.shape[0] > busStepSize else 1)) for index in range(1, len(busesList)): if len(busesList[index].shape) == 2: busesList[index][None, :] busesList[index] = cp.hsplit(busesList[index], [ int(cp.argwhere(cp.isnan(busesList[index][0, :, 1]))[0]), busesList[index].shape[1] ])[0] linesList = cp.array_split( matrizLinhas, int(matrizLinhas.shape[0] / lineStepSize if matrizLinhas.shape[0] > lineStepSize else 1)) for index in range(1, len(linesList)): if len(linesList[index].shape) == 2: linesList[index][None, :] linesList[index] = cp.hsplit(linesList[index], [ int(cp.argwhere(cp.isnan(linesList[index][0, :, 1]))[0]), linesList[index].shape[1] ])[0] #busDataset = tr.utils.data.DataLoader(busesList,drop_last=True) #lineDataset = tr.utils.data.DataLoader(linesList,drop_last=True) #algorithm = Algorithm() #algorithm.cuda() fullResults = None for nowBus, busTensor in enumerate(busesList): allLinesResult = None for nowLine, lineTensor in enumerate(linesList): # Algorithm segment #algRes = algorithm.FullAlgorithm(busTensor,lineTensor,distanceTolerance,detectionPercentage) #algRes = FullAlgorithm(busTensor,lineTensor,distanceTolerance,detectionPercentage) algRes = cp.asnumpy( Algorithm(busTensor, lineTensor, TOLERANCE=distanceTolerance, detectionPercentage=detectionPercentage)) #segmentResults = algRes.numpy() # Concatenation to full results matrix if allLinesResult is None: allLinesResult = np.copy(algRes) else: allLinesResult = np.concatenate([allLinesResult, algRes], axis=1) if fullResults is None: fullResults = np.copy(allLinesResult) else: fullResults = np.concatenate([fullResults, allLinesResult], axis=0) #fullResults = fullResults > detectionPercentage lineLabel = [(i[0], str(i[1])) for i in lineIdList] busLabel = [i[0] for i in busIdList] results = pd.DataFrame(fullResults.T, index=pd.MultiIndex.from_tuples(lineLabel), columns=busLabel) return results