def back_prop(self, dloss_dy): """ Do backpropagation. Parameters ---------- dloss_dy : cp.array of floats, shape (nr_examples,) + self.output_size Derivative of the loss with respect to output values. Returns ------- dloss_dx : cp.array of floats, shape (nr_examples,) + self.input_size Outputs. """ nr_examples = dloss_dy.shape[0] # Calculate derivatives. dloss_dz = self.ac_func_d(self.z_cache, dloss_dy) self.dloss_dw = (1 / nr_examples) * cp.tensordot( self.x_cache, dloss_dz, axes=((0, 3, 5), (0, 1, 2))) d_y_times_filters = cp.tensordot(self.d_y, self.weights, axes=((1, ), (2, ))) dz_dx = cp.tensordot(self.d_x, d_y_times_filters, axes=((1, ), (3, ))) dloss_dx = cp.tensordot(dloss_dz, dz_dx, axes=((1, 2, 3), (1, 3, 5))) dloss_dx = dloss_dx[:, self.padding[1][0]:self.padded_size[0] - self.padding[1][1], self.padding[2][0]:self.padded_size[1] - self.padding[2][1], :] self.dloss_db = cp.average(self.dloss_dw, axis=(0, 1, 2)) # Return derivative of loss with respect to inputs x return dloss_dx
def estimate_position_density(place_bin_centers, positions, position_std, block_size=100, sample_weights=None): ''' Parameters ---------- place_bin_centers : ndarray, shape (n_position_bins, n_position_dims) positions : ndarray, shape (n_time, n_position_dims) position_std : float Returns ------- position_density : ndarray, shape (n_position_bins,) ''' n_position_bins = place_bin_centers.shape[0] if block_size is None: block_size = n_position_bins position_density = cp.empty((n_position_bins, )) for start_ind in range(0, n_position_bins, block_size): block_inds = slice(start_ind, start_ind + block_size) position_density[block_inds] = cp.average( estimate_position_distance(place_bin_centers[block_inds], positions, position_std), axis=0, weights=sample_weights) return position_density
def get_amce(self, k, lag_mode='absmax', exclude_k=True, exclude_self_effects=True): """Returns the average mediated causal effect. This is the Average Mediated Causal Effect (AMCE) through a variable k With lag_mode='absmax' this is based on the lag of maximum CE for each pair. Parameters ---------- k : int Index of variable. lag_mode : {'absmax', 'all_lags'} Lag mode. Either average across all lags between each pair or only at the lag of maximum absolute causal effect. exclude_k : bool, optional (default: True) Whether to exclude causal effects through the variable itself at previous lags. exclude_self_effects : bool, optional (default: True) Whether to exclude causal self effects of variables on themselves. Returns ------- amce : float Average Mediated Causal Effect. """ all_but_k = np.ones(self.N, dtype='bool') if exclude_k: all_but_k[k] = False N_new = self.N - 1 else: N_new = self.N if exclude_self_effects: weights = np.identity(N_new) == False else: weights = np.ones((N_new, N_new), dtype='bool') if self.tau_max < 2: raise ValueError("Mediation only nonzero for tau_max >= 2") all_mce = self.psi[2:, :, :] - self.all_psi_k[k, 2:, :, :] # all_mce[:, range(self.N), range(self.N)] = 0. if lag_mode == 'absmax': return np.average(np.abs( all_mce[:, all_but_k, :][:, :, all_but_k]).max(axis=0), weights=weights) elif lag_mode == 'all_lags': return np.abs(all_mce[:, all_but_k, :][:, :, all_but_k]).mean() else: raise ValueError("lag_mode = %s not implemented" % lag_mode)
repeats += 1 # -------------------------------------------------------------------------- x = np.insert(x, 0, count, axis=1) pop_d = cp.array(x) # -------------------------------------------------------------------------- # Picking best solution old_cost = minimum_cost # best_sol = pop_d[0,:] best_sol = pop_d[pop_d[:,-1].argmin()] minimum_cost = best_sol[-1] worst_sol = pop_d[pop_d[:,-1].argmax()] worst_cost = worst_sol[-1] delta = worst_cost-minimum_cost average = cp.average(pop_d[:,-1]) if minimum_cost == old_cost: # To calculate for how long the quality did not improve count_index += 1 else: count_index = 0 # Shuffle the population after a certain number of generations without improvement assign_child_1 = False # if count_index >= 5000 and count%last_shuffle == 0: # last_shuffle *= 2.5 # count_index = 0 # r = 1 # #r = random.randint(1, 2)
# open mrc file with mrcfile.open(density_stack, permissive=True) as mrc: cp_density_stack = cp.asarray(mrc.data, dtype="float32") mrc.close sta_dens = cp_density_stack.shape[0] row = cp_density_stack.shape[1] col = cp_density_stack.shape[2] print("sta_dens = " + str(sta_dens)) print("row = " + str(row)) print("col = " + str(col)) print("") average_dens = cp.average(cp_density_stack, axis=(1, 2)) #print(average_dens) #print(average_dens.shape) temp_dens = cp.zeros(cp_density_stack.shape) if (temp_dens_flag == 1): temp_dens_2D = Image.open(temp_dens_name) temp_dens_2D = cp.asarray(temp_dens_2D, dtype="float32") temp_dens_2D = cp.flipud(temp_dens_2D) temp_dens_2D_ave = cp.average(temp_dens_2D) temp_dens_2D[:, :] = temp_dens_2D[:, :] - temp_dens_2D_ave temp_dens[:, :, :] = temp_dens_2D[:, :] else: temp_dens[:, :, :] = cp_density_stack[0, :, :] #average_temp_dens=cp.average(temp_dens,axis=(1,2))
def online_updating_user(self, new_user_with_data=None, coselection=True, convergence_ratio_threshold=0.0001, max_iteration=1000, topK=5, input_P_as_reference=True, input_V_as_reference=False): cleaned_new_user_data = new_user_with_data.copy() cleaned_new_user_data.drop_duplicates(inplace=True) assert len(set(cleaned_new_user_data.UserID) ) == 1, "New User Data should only include one user." cleaned_new_user_data.drop_duplicates(inplace=True) # check items from new data is aligned with self.item_original_id_list assert set(cleaned_new_user_data.ItemID).issubset( set(self.item_original_id_list)), "New Data Contains Wrong Item ID" # check Actions type are in ['V', 'P'] assert set(cleaned_new_user_data.Action).issubset( {'V', 'P'}), "New Data Contains Wrong Action Type" # check u in user_original_id_list or not (u exists or not) u_original_id = set(cleaned_new_user_data.UserID).pop() if u_original_id not in self.user_original_id_list: u_already_exist = False self.user_original_id_list.append(u_original_id) # convert u's string id to integer index id u = self.user_original_id_list.index(u_original_id) self.user_list.append(u) self.num_u += 1 else: u_already_exist = True u = self.user_original_id_list.index(u_original_id) # convert itemid into index id cleaned_new_user_data.ItemID = cleaned_new_user_data.ItemID.apply( lambda x: self.item_original_id_list.index(x)) # build I_u_t, I_u_a for user u assert (len(self.S) != 0) & (len( self.U_item) != 0), "Item-set Coselection Not Initialized." self.I_u_t_train[u] = self.I_u_t_train[u].union( set(cleaned_new_user_data[cleaned_new_user_data.Action == 'P']. ItemID)) if u_already_exist else set(cleaned_new_user_data[ cleaned_new_user_data.Action == 'P'].ItemID) self.I_u_a_train[u] = self.I_u_a_train[u].union( set(cleaned_new_user_data[cleaned_new_user_data.Action == 'V']. ItemID)) if u_already_exist else set(cleaned_new_user_data[ cleaned_new_user_data.Action == 'V'].ItemID) # calculate auxiliary-target correlation C self.alpha_u[u] = dict() I_t_u = self.I_u_t_train[u] # Notice we only have View action so we do not need filtered item set for x in ['V']: I_a_u = self.I_u_a_train[u] # Equation Reference to page 86 section 3.3 # NOTE # if I_t_u is 0, then we set C_u_at to be 0 # if I_a_u is 0, then we set C_u_ta to be 0 C_u_at = len(I_t_u.intersection(I_a_u)) / len(I_t_u) if len( I_t_u) != 0 else 0 C_u_ta = len(I_t_u.intersection(I_a_u)) / len(I_a_u) if len( I_a_u) != 0 else 0 # if C_u_ta + C_u_at == 0, then we set alpha_u of user u to be 1 # hence, C_u_X here is 1 / omega because alpha_u = omega * C_u_X # in this case, user u is not in train, perhaps in test C_u_X = 2 * C_u_at * C_u_ta / ( C_u_ta + C_u_at) if C_u_ta + C_u_at != 0 else (1 / self.omega) # set final alpha_u to 1 if C_u_ta + C_u_at == 0 self.alpha_u[u][x] = C_u_X # We have only one auxiliary action 'V' self.alpha_u[u]['alpha'] = self.omega * self.rho * C_u_X # generate item-set based on co-selection # build U for i in self.I_u_t_train[u]: self.U_item[i].add(u) # build S for i in self.item_list: for j in self.I_u_t_train[u]: # If more than 2 users have target action on i and j, then j is added in S[i] if len(self.U_item[i].intersection(self.U_item[j])) >= 2: self.S[i].add(j) self.S[j].add(i) # If u not exists, we initialize her user vector if not u_already_exist: # set random seed cupy.random.seed(self.random_state) u_vector = cupy.random.normal(loc=0.0, scale=0.1, size=(1, self.dim + 1)) u_vector[:, -1] = 1.0 self.U = cupy.concatenate((self.U, u_vector)) assert (self.U[u, :] == u_vector ).all(), "New User Int ID assigned wrongly" # initialize estimation self.estimation = cupy.dot(self.U, self.V) del u_vector # start training loss_u = 0.0 convergence_hit = 0 all_item = set(self.item_list) with trange(max_iteration) as t: for index in t: # build I # uniformly sample a item i from I_u_t I_u_t = self.I_u_t_train[u] if len(I_u_t) != 0: i = choice(sorted(I_u_t)) # build I = I_u_t cap S_i if coselection: I = I_u_t.intersection(self.S[i]) else: # if no coselection, we set I as the set of purchased items by user u # no uniform sampling, like COFISET I = I_u_t else: # if no item in I_u_t, then set I to empty set i = None I = set() # build J, since we only have one auxiliary action, we follow the uniform sampling I_u_oa = self.I_u_a_train[u] - I_u_t if len(I_u_oa) != 0: j = choice(sorted(I_u_oa)) if coselection: # NOTE: typo in paper? J = I_u_oa.intersection(self.S[j]) else: # if no coselection, we set J as the set of only-auxiliary items by user u # no uniform sampling, like COFISET J = I_u_oa else: # if no item in I_u_oa, then set J to empty set j = None J = set() # build K I_u_n = all_item - I_u_t - I_u_oa if len(I_u_n) != 0: k = choice(sorted(I_u_n)) # build K if coselection: # NOTE: typo in paper? K = I_u_n.intersection(self.S[k]) else: # if no coselection, we set K as the set of no-action items by user u # no uniform sampling, like COFISET K = I_u_n else: # if no item in I_u_n, then set K to empty set k = None K = set() # calculate intermediate variables # get specific alpha_u spec_alpha_u = self.alpha_u[u]['alpha'] U_u = self.U[u, :-1].copy() sorted_I = sorted(I) sorted_J = sorted(J) sorted_K = sorted(K) # get r_hat_uIJ, r_hat_uJK, r_hat_uIK r_hat_uI = cupy.average( self.estimation[u, sorted_I]) if len(I) != 0 else cupy.array( [0]) r_hat_uJ = cupy.average( self.estimation[u, sorted_J]) if len(J) != 0 else cupy.array( [0]) r_hat_uK = cupy.average( self.estimation[u, sorted_K]) if len(K) != 0 else cupy.array( [0]) r_hat_uIJ = r_hat_uI - r_hat_uJ r_hat_uJK = r_hat_uJ - r_hat_uK r_hat_uIK = r_hat_uI - r_hat_uK # get V_bar_I, V_bar_J, V_bar_K V_bar_I = cupy.average(self.V[:-1, sorted_I], axis=1) if len(I) != 0 else cupy.zeros( shape=(self.dim, )) V_bar_J = cupy.average(self.V[:-1, sorted_J], axis=1) if len(J) != 0 else cupy.zeros( shape=(self.dim, )) V_bar_K = cupy.average(self.V[:-1, sorted_K], axis=1) if len(K) != 0 else cupy.zeros( shape=(self.dim, )) # get b_I, b_J, b_K b_I = cupy.average( self.V[-1, sorted_I]) if len(I) != 0 else cupy.array([0]) b_J = cupy.average( self.V[-1, sorted_J]) if len(J) != 0 else cupy.array([0]) b_K = cupy.average( self.V[-1, sorted_K]) if len(K) != 0 else cupy.array([0]) # here we want to examine the condition of empty sets indicator_I = indicator(len(I) == 0) indicator_J = indicator(len(J) == 0) indicator_K = indicator(len(K) == 0) indicator_sum = indicator_I + indicator_J + indicator_K if 0 <= indicator_sum <= 1: # these are the cases when only one set are empty or no set is empty # when all three are not empty, or I is empty, or K is empty, it is # easy to rewrite the obj by multiplying the indicator # when J is empty, we have to rewrite the obj if indicator_J == 1: # when J is empty # NABLA U_u df_dUu = sigmoid(-r_hat_uIK) * (V_bar_I - V_bar_K) dR_dUu = 2 * self.lambda_u * U_u # update U_u = U_u + gamma * (df_dUu - dR_dUu) self.U[u, :-1] += self.gamma * (df_dUu - dR_dUu) # calculate loss f_Theta = cupy.log(sigmoid(r_hat_uIK)) regular = self.lambda_u * cupy.linalg.norm(U_u, ord=2) + \ self.lambda_v * ( cupy.linalg.norm(V_bar_I, ord=2) + cupy.linalg.norm(V_bar_K, ord=2)) + \ self.lambda_b * (b_I ** 2 + b_K ** 2) new_loss_u = f_Theta - regular else: # when J is not empty # NABLA U_u df_dUu = (1 - indicator_I) * sigmoid(- r_hat_uIJ / spec_alpha_u) / spec_alpha_u * ( V_bar_I - V_bar_J) + \ (1 - indicator_K) * sigmoid(- r_hat_uJK) * (V_bar_J - V_bar_K) dR_dUu = 2 * self.lambda_u * U_u # update U_u = U_u + gamma * (df_dUu - dR_dUu) self.U[u, :-1] += self.gamma * (df_dUu - dR_dUu) # calculate loss f_Theta = (1 - indicator_I) * cupy.log(sigmoid(r_hat_uIJ / spec_alpha_u)) + \ (1 - indicator_K) * cupy.log(sigmoid(r_hat_uJK)) regula = self.lambda_u * cupy.linalg.norm(U_u, ord=2) + \ self.lambda_v * ((cupy.linalg.norm(V_bar_I, ord=2) if len(I) != 0 else 0) + (cupy.linalg.norm(V_bar_J, ord=2) if len(J) != 0 else 0) + (cupy.linalg.norm(V_bar_K, ord=2) if len(K) != 0 else 0)) + \ self.lambda_b * ((b_I if len(I) != 0 else 0) ** 2 + (b_J if len(J) != 0 else 0) ** 2 + (b_K if len(K) != 0 else 0) ** 2) new_loss_u = f_Theta - regula else: # these are the cases when at least two sets are empty # at these cases, we ignore this user and continue the loop continue ratio_delta_loss = abs(new_loss_u - loss_u) / abs( loss_u) if loss_u != 0.0 else 1.0 loss_u = new_loss_u # Postfix will be displayed on the right, # formatted automatically based on argument's datatype t.set_postfix(ratio_delta_loss=ratio_delta_loss, len_I=len(I), len_J=len(J), len_K=len(K)) if abs(ratio_delta_loss) < convergence_ratio_threshold: convergence_hit += 1 if convergence_hit == 49: break else: convergence_hit = 0 del U_u, r_hat_uI, r_hat_uJ, r_hat_uK, r_hat_uIJ, r_hat_uJK, r_hat_uIK, V_bar_I, V_bar_J, V_bar_K, b_I, b_J, b_K, df_dUu, dR_dUu # recalculate estimation self.estimation[u, :] = cupy.dot(self.U[u, :], self.V) # after convergence, we recommend top-k items for user u est_pref_of_u = self.estimation[u, :].copy() # Next is the case when user u is in train data # get the ranking for user u's pref of item user_rec_dict = dict() est_pref_sort_index = est_pref_of_u.argsort()[::-1].get() rec_item_cnt = 0 # case of recommending on test data # if input_P_as_reference: # for item_id in est_pref_sort_index: # if rec_item_cnt == topK: # break # # we only consider the item that is not in train data for user u # if item_id not in self.I_u_t_train[u]: # user_rec_dict[self.item_original_id_list[item_id]] = est_pref_of_u[item_id] # rec_item_cnt += 1 ## case of recommending on train data # else: # for item_id in set(est_pref_sort_index[:topK]): # user_rec_dict[self.item_original_id_list[item_id]] = est_pref_of_u[item_id] for item_id in est_pref_sort_index: if rec_item_cnt == topK: break # we only consider the item that is not in train data for user u item_in_P = item_id in I_t_u item_in_A = item_id in I_a_u if item_in_P & input_P_as_reference: continue if item_in_A & input_V_as_reference: continue user_rec_dict[ self.item_original_id_list[item_id]] = est_pref_of_u[item_id] rec_item_cnt += 1 del est_pref_of_u, est_pref_sort_index # free all unused memory in GPU mempool = cupy.get_default_memory_pool() mempool.free_all_blocks() # return rec dict return user_rec_dict
def fit(self, X, eval_X, y=None, model_saved_path='bprh_model.pkl', iter_to_save=5000, coselection_saved_path='data/item-set-coselection.pkl', iter_to_log=100, correlation=True, coselection=False, plot_metric=False, log_metric=False): # Here we do not load model -> train a new model if self.existed_model_path is None: # To make sure train and test works with inconsistent user and item list, # we transform user and item's string ID to int ID so that their ID is their index in U and V print("Registering Model Parameters") # rename user and item self.user_original_id_list = sorted( set(X.UserID).union(set(eval_X.UserID))) self.item_original_id_list = sorted( set(X.ItemID).union(set(eval_X.ItemID))) self.train_data = X.copy() self.test_data = eval_X.copy() self.train_data.UserID = self.train_data.UserID.apply( lambda x: self.user_original_id_list.index(x)) self.train_data.ItemID = self.train_data.ItemID.apply( lambda x: self.item_original_id_list.index(x)) self.test_data.UserID = self.test_data.UserID.apply( lambda x: self.user_original_id_list.index(x)) self.test_data.ItemID = self.test_data.ItemID.apply( lambda x: self.item_original_id_list.index(x)) self.item_list = [ idx[0] for idx in enumerate(self.item_original_id_list) ] self.user_list = [ idx[0] for idx in enumerate(self.user_original_id_list) ] self.num_u = len(self.user_list) self.num_i = len(self.item_list) # build I_u_t, I_u_a (pre-computing for acceleration) self.build_itemset_for_user() # Calculate auxiliary-target correlation C for every user and each types of auxiliary action if correlation: self.alpha_u = self.auxiliary_target_correlation( X=self.train_data) else: print( "No auxiliary-target correlation - all alpha_u equal to one" ) alpha_u_all_ones = dict() user_set_bar = tqdm(self.user_list) for u in user_set_bar: alpha_u_all_ones[u] = dict() alpha_u_all_ones[u]['alpha'] = 1.0 self.alpha_u = alpha_u_all_ones.copy() # Generate item-set based on co-selection if coselection: self.S, self.U_item = self.itemset_coselection( X=self.train_data) # Initialization of User and Item Matrices if self.random_state is not None: cupy.random.seed(self.random_state) else: cupy.random.seed(0) print("Initializing User and Item Matrices") # NOTE: Initialization is influenced by mean and std self.U = cupy.random.normal(size=(self.num_u, self.dim + 1), loc=0.0, scale=0.1) self.V = cupy.random.normal(size=(self.dim + 1, self.num_i), loc=0.0, scale=0.1) # self.U = cupy.zeros(shape=(self.num_u, self.dim + 1)) # self.V = cupy.zeros(shape=(self.dim + 1, self.num_i)) self.U[:, -1] = 1.0 # estimation is U dot V self.estimation = cupy.dot(self.U, self.V) # Configure loss plots layout if plot_metric: groups = { 'Precision@K': ['Precision@5', 'Precision@10'], 'Recall@K': ['Recall@5', 'Recall@10'], 'AUC': ['AUC'] } plot_losses = PlotLosses(groups=groups) # Start Iteration all_item = set(self.item_list) user_in_train = sorted(set(self.train_data.UserID)) print("Start Training") with trange(self.num_iter) as t: for index in t: # Description will be displayed on the left # t.set_description('ITER %i' % index) # Build u, I, J, K # uniformly sample a user from U u = choice(user_in_train) # build I # uniformly sample a item i from I_u_t I_u_t = self.I_u_t_train[u] if len(I_u_t) != 0: i = choice(sorted(I_u_t)) # build I = I_u_t cap S_i if coselection: I = I_u_t.intersection(self.S[i]) else: # if no coselection, we set I as the set of purchased items by user u # no uniform sampling, like COFISET I = I_u_t else: # if no item in I_u_t, then set I to empty set i = None I = set() # build J, since we only have one auxiliary action, we follow the uniform sampling I_u_oa = self.I_u_a_train[u] - I_u_t if len(I_u_oa) != 0: j = choice(sorted(I_u_oa)) if coselection: # NOTE: typo in paper? J = I_u_oa.intersection(self.S[j]) else: # if no coselection, we set J as the set of only-auxiliary items by user u # no uniform sampling, like COFISET J = I_u_oa else: # if no item in I_u_oa, then set J to empty set j = None J = set() # build K I_u_n = all_item - I_u_t - I_u_oa if len(I_u_n) != 0: k = choice(sorted(I_u_n)) # build K if coselection: # NOTE: typo in paper? K = I_u_n.intersection(self.S[k]) else: # if no coselection, we set K as the set of no-action items by user u # no uniform sampling, like COFISET K = I_u_n else: # if no item in I_u_n, then set K to empty set k = None K = set() # calculate intermediate variables # get specific alpha_u spec_alpha_u = self.alpha_u[u]['alpha'] U_u = self.U[u, :-1].copy() sorted_I = sorted(I) sorted_J = sorted(J) sorted_K = sorted(K) # get r_hat_uIJ, r_hat_uJK, r_hat_uIK r_hat_uI = cupy.average( self.estimation[u, sorted_I]) if len(I) != 0 else cupy.array( [0]) r_hat_uJ = cupy.average( self.estimation[u, sorted_J]) if len(J) != 0 else cupy.array( [0]) r_hat_uK = cupy.average( self.estimation[u, sorted_K]) if len(K) != 0 else cupy.array( [0]) r_hat_uIJ = r_hat_uI - r_hat_uJ r_hat_uJK = r_hat_uJ - r_hat_uK r_hat_uIK = r_hat_uI - r_hat_uK # get V_bar_I, V_bar_J, V_bar_K V_bar_I = cupy.average(self.V[:-1, sorted_I], axis=1) if len(I) != 0 else cupy.zeros( shape=(self.dim, )) V_bar_J = cupy.average(self.V[:-1, sorted_J], axis=1) if len(J) != 0 else cupy.zeros( shape=(self.dim, )) V_bar_K = cupy.average(self.V[:-1, sorted_K], axis=1) if len(K) != 0 else cupy.zeros( shape=(self.dim, )) # get b_I, b_J, b_K b_I = cupy.average( self.V[-1, sorted_I]) if len(I) != 0 else cupy.array([0]) b_J = cupy.average( self.V[-1, sorted_J]) if len(J) != 0 else cupy.array([0]) b_K = cupy.average( self.V[-1, sorted_K]) if len(K) != 0 else cupy.array([0]) # here we want to examine the condition of empty sets indicator_I = indicator(len(I) == 0) indicator_J = indicator(len(J) == 0) indicator_K = indicator(len(K) == 0) indicator_sum = indicator_I + indicator_J + indicator_K if 0 <= indicator_sum <= 1: # these are the cases when only one set are empty or no set is empty # when all three are not empty, or I is empty, or K is empty, it is # easy to rewrite the obj by multiplying the indicator # when J is empty, we have to rewrite the obj if indicator_J == 1: # when J is empty # NABLA U_u df_dUu = sigmoid(-r_hat_uIK) * (V_bar_I - V_bar_K) dR_dUu = 2 * self.lambda_u * U_u # update U_u = U_u + gamma * (df_dUu - dR_dUu) self.U[u, :-1] += self.gamma * (df_dUu - dR_dUu) # NABLA V_i df_dbi = (1 - indicator_I ) * sigmoid(-r_hat_uIK) / indicator_len(I) dR_dbi = ( 1 - indicator_I ) * 2 * self.lambda_b * b_I / indicator_len(I) df_dVi = df_dbi * U_u dR_dVi = 2 * self.lambda_v * V_bar_I / indicator_len(I) # update V_i = V_i + gamma * (df_dVi - dR_dVi) self.V[:-1, sorted_I] += self.gamma * ( df_dVi - dR_dVi)[:, None] # trick: transpose here # update b_i = b_i + gamma * (df_dbi - dR_dbi) self.V[-1, sorted_I] += self.gamma * (df_dbi - dR_dbi) # No change on J # NABLA V_k df_dbk = (1 - indicator_K ) * -sigmoid(-r_hat_uIK) / indicator_len(K) dR_dbk = ( 1 - indicator_K ) * 2 * self.lambda_b * b_K / indicator_len(K) df_dVk = df_dbk * U_u dR_dVk = 2 * self.lambda_v * V_bar_K / indicator_len(K) # update V_k = V_k + gamma * (df_dVk - dR_dVk) self.V[:-1, sorted_K] += self.gamma * ( df_dVk - dR_dVk)[:, None] # trick: transpose here # update b_k = b_k + gamma * (df_dbk - dR_dbk) self.V[-1, sorted_K] += self.gamma * (df_dbk - dR_dbk) else: # when J is not empty # NABLA U_u df_dUu = (1 - indicator_I) * sigmoid(- r_hat_uIJ / spec_alpha_u) / spec_alpha_u * ( V_bar_I - V_bar_J) + \ (1 - indicator_K) * sigmoid(- r_hat_uJK) * (V_bar_J - V_bar_K) dR_dUu = 2 * self.lambda_u * U_u # update U_u = U_u + gamma * (df_dUu - dR_dUu) self.U[u, :-1] += self.gamma * (df_dUu - dR_dUu) # NABLA V_i df_dbi = (1 - indicator_I) * sigmoid( -r_hat_uIJ / spec_alpha_u) / (indicator_len(I) * spec_alpha_u) dR_dbi = ( 1 - indicator_I ) * 2 * self.lambda_b * b_I / indicator_len(I) df_dVi = df_dbi * U_u dR_dVi = 2 * self.lambda_v * V_bar_I / indicator_len(I) # update V_i = V_i + gamma * (df_dVi - dR_dVi) self.V[:-1, sorted_I] += self.gamma * ( df_dVi - dR_dVi)[:, None] # trick: transpose here # update b_i = b_i + gamma * (df_dbi - dR_dbi) self.V[-1, sorted_I] += self.gamma * (df_dbi - dR_dbi) # NABLA V_j df_dbj = (1 - indicator_I) * ( -sigmoid(-r_hat_uIJ / spec_alpha_u) / spec_alpha_u + (1 - indicator_K) * sigmoid(-r_hat_uJK)) / indicator_len(J) dR_dbj = 2 * self.lambda_b * b_J / indicator_len(J) df_dVj = df_dbj * U_u dR_dVj = 2 * self.lambda_v * V_bar_J / indicator_len(J) # update V_j = V_j + gamma * (df_dVj - dR_dVj) self.V[:-1, sorted_J] += self.gamma * ( df_dVj - dR_dVj)[:, None] # trick: transpose here # update b_j = b_j + gamma * (df_dbj - dR_dbj) self.V[-1, sorted_J] += self.gamma * (df_dbj - dR_dbj) # NABLA V_k df_dbk = (1 - indicator_K ) * -sigmoid(-r_hat_uJK) / indicator_len(K) dR_dbk = ( 1 - indicator_K ) * 2 * self.lambda_b * b_K / indicator_len(K) df_dVk = df_dbk * U_u dR_dVk = 2 * self.lambda_v * V_bar_K / indicator_len(K) # update V_k = V_k + gamma * (df_dVk - dR_dVk) self.V[:-1, sorted_K] += self.gamma * ( df_dVk - dR_dVk)[:, None] # trick: transpose here # update b_k = b_k + gamma * (df_dbk - dR_dbk) self.V[-1, sorted_K] += self.gamma * (df_dbk - dR_dbk) else: # these are the cases when at least two sets are empty # at these cases, we ignore this user and continue the loop continue # calculate loss # f_Theta = cupy.log(sigmoid(r_hat_uIJ / spec_alpha_u)) + cupy.log(sigmoid(r_hat_uJK)) # regula = self.lambda_u * cupy.linalg.norm(U_u, ord=2) + self.lambda_v * ( # (cupy.linalg.norm(V_bar_I, ord=2) if len(I) != 0 else 0) + ( # cupy.linalg.norm(V_bar_J, ord=2) if len(J) != 0 else 0) + ( # cupy.linalg.norm(V_bar_K, ord=2)) if len(K) != 0 else 0) + self.lambda_b * ( # (b_I if len(I) != 0 else 0) ** 2 + (b_J if len(J) != 0 else 0) ** 2 + ( # b_K if len(K) != 0 else 0) ** 2) # bprh_loss = f_Theta - regula # update estimation old_estimation = self.estimation.copy() # self.estimation = cupy.dot(self.U, self.V) all_sampled_item = sorted(set.union(I, J, K)) # for sampled_item in all_sampled_item: # self.estimation[:, sampled_item] = cupy.dot(self.U, self.V[:, sampled_item]) self.estimation[:, all_sampled_item] = cupy.dot( self.U, self.V[:, all_sampled_item]) self.estimation[u, :] = cupy.dot(self.U[u, :], self.V) # estimation changed est_changed = cupy.linalg.norm(self.estimation - old_estimation) # we only save model to file when the num of iter % iter_to_save == 0 if (index + 1) % iter_to_save == 0: self.save(model_path=model_saved_path + "_" + str(index)) # we only calculate metric when the num of iter % iter_to_log == 0 if (index + 1) % iter_to_log == 0: if log_metric | plot_metric: # calculate metrics on test data user_to_eval = sorted(set(self.test_data.UserID)) scoring_list_5, precision_5, recall_5, avg_auc = self.scoring( user_to_eval=user_to_eval, ground_truth=self.test_data, K=5, train_data_as_reference_flag=True) scoring_list_10, precision_10, recall_10, _ = self.scoring( user_to_eval=user_to_eval, ground_truth=self.test_data, K=10, train_data_as_reference_flag=True) if log_metric: self.eval_hist.append([ index, precision_5, precision_10, recall_5, recall_10, avg_auc ]) if plot_metric: plot_losses.update({ 'Precision@5': precision_5, 'Precision@10': precision_10, 'Recall@5': recall_5, 'Recall@10': recall_10, 'AUC': avg_auc }) plot_losses.send() # Postfix will be displayed on the right, # formatted automatically based on argument's datatype t.set_postfix(est_changed=est_changed, len_I=len(I), len_J=len(J), len_K=len(K)) del U_u, r_hat_uI, r_hat_uJ, r_hat_uK, r_hat_uIJ, r_hat_uJK, r_hat_uIK, V_bar_I, V_bar_J, V_bar_K, b_I, b_J, b_K, df_dUu, dR_dUu mempool = cupy.get_default_memory_pool() mempool.free_all_blocks() print("Training Finished")
cp_autocorrelation = cp.fft.fft2(cp_diff, norm="ortho") cp_autocorrelation = cp.fft.fftshift(cp_autocorrelation) cp_autocorrelation_abs=cp.absolute(cp_autocorrelation) del cp_autocorrelation np_autocorrelation_abs=cp.asnumpy(cp_autocorrelation_abs) foname=header + "_autocorrelation.tif" tifffile.imsave(foname ,np_autocorrelation_abs) #TH_mode="OTSU" TH_mode="keiken" if(TH_mode=="keiken"): ave_autocorrelation=cp.average(cp_autocorrelation_abs) std_autocorrelation=cp.std(cp_autocorrelation_abs) th = ave_autocorrelation + std_autocorrelation print("threshold = " + str(th)) elif(TH_mode=="OTSU"): max_autocorrelation=np.max(np_autocorrelation_abs*10) np_autocorrelation_abs_uint8=np.asarray(np_autocorrelation_abs*10, dtype=np.uint8) ret2, np_autocorrelation_abs_otsu = cv2.threshold(np_autocorrelation_abs_uint8, 0, int(max_autocorrelation), cv2.THRESH_OTSU) th=float(format(ret2))/10.0 print("threshold = " + str(th)) cp_th=cp.where(cp_autocorrelation_abs>=th,float(1),float(0)) cp_th=cp_th.astype(cp.float32) np_th=cp.asnumpy(cp_th)
def barycenter_unbalanced_sinkhorn2D_gpu(A, Cx, Cy, reg, reg_m, weights=None, numItermax=1000, stopThr=1e-6, verbose=False, log=False, logspace=False, reg_K=1e-16): """ ---------- A : np.ndarray (dimx,dimy, n_hists) `n_hists` training distributions a_i of dimension dimxdim Cx : np.ndarray (dimx, dimx) x part of separable cost matrix for OT. Cy : np.ndarray (dimy, dimy) y part of separable cost matrix for OT. reg : float Entropy regularization term > 0 reg_m: float Marginal relaxation term > 0 weights : np.ndarray (n_hists,) optional Weight of each distribution (barycentric coodinates) If None, uniform weights are used. numItermax : int, optional Max number of iterations stopThr : float, optional Stop threshol on error (> 0) verbose : bool, optional Print information along iterations log : bool, optional record log if True logspace : bool, optional compuation done in logspace if True Returns ------- q : (dim,) ndarray Unbalanced Wasserstein barycenter log : dict log dictionary return only if log==True in parameters References ---------- .. [3] Benamou, J. D., Carlier, G., Cuturi, M., Nenna, L., & Peyré, G. (2015). Iterative Bregman projections for regularized transportation problems. SIAM Journal on Scientific Computing, 37(2), A1111-A1138. .. [10] Chizat, L., Peyré, G., Schmitzer, B., & Vialard, F. X. (2016). Scaling algorithms for unbalanced transport problems. arXiv preprin arXiv:1607.05816. """ dimx, dimy, n_hists = A.shape if weights is None: weights = cp.ones(n_hists) / n_hists else: assert (len(weights) == A.shape[2]) weights = cp.asarray(weights / weights.sum()) if log: log = {'err': []} fi = reg_m / (reg_m + reg) if logspace: v = cp.zeros((dimx, dimy, 1)) * cp.log(1 / dimx / dimy) u = cp.zeros((dimx, dimy)) * cp.log(1 / dimx / dimy) q = cp.zeros((dimx, dimy)) * cp.log(1 / dimx / dimy) err = 1. for i in range(numItermax): uprev = u.copy() vprev = v.copy() qprev = q.copy() lKv = prod_separable_logspace(Cx, Cy, reg, v) u = fi * (cp.log(A) - cp.maximum(lKv, cp.log(reg_K))) lKtu = prod_separable_logspace(Cx.T, Cy.T, reg, u) mlktu = (1 - fi) * cp.max(lKtu, axis=2) q = (1 / (1 - fi)) * ((cp.log( cp.average(cp.exp((1 - fi) * lKtu - mlktu[:, :, None]), axis=2, weights=weights))) + mlktu) Q = q[:, :, None] v = fi * (Q - cp.maximum(lKtu, cp.log(reg_K))) if (cp.any(lKtu == -cp.inf) or cp.any(cp.isnan(u)) or cp.any(cp.isnan(v))): # we have reached the machine precision # come back to previous solution and quit loop print('Numerical errors at iteration %s' % i) u = uprev v = vprev # q = qprev break # compute change in barycenter if (i % 10 == 0) or i == 0: err = abs(cp.exp(qprev) * (1 - cp.exp(q - qprev))).max() err /= max(abs(cp.exp(q)).max(), abs(cp.exp(qprev)).max(), 1.) if log: log['err'].append(err) # if barycenter did not change + at least 10 iterations - stop if err < stopThr and i > 10: break if verbose: if i % 100 == 0: print('{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) print('{:5d}|{:8e}|'.format(i, err.get())) if log: log['niter'] = i log['logu'] = ((u + 1e-300)) log['logv'] = ((v + 1e-300)) return cp.exp(q), log else: return cp.exp(q) else: v = cp.ones((dimx, dimy, 1)) / dimx / dimy u = cp.ones((dimx, dimy)) / dimx / dimy q = cp.ones((dimx, dimy)) / dimx / dimy err = 1. Kx = cp.exp(-Cx / reg) Ky = cp.exp(-Cy / reg) for i in range(numItermax): uprev = u.copy() vprev = v.copy() qprev = q.copy() Kv = cp.tensordot(cp.tensordot(Kx, v, axes=([1], [0])), Ky, axes=([1], [0])).swapaxes(1, 2) u = cp.power(cp.divide(A, (Kv + reg_K)), fi) Ktu = cp.tensordot(cp.tensordot(cp.transpose(Kx), u, axes=([1], [0])), cp.transpose(Ky), axes=([1], [0])).swapaxes(1, 2) q = cp.power(cp.dot(cp.power(Ktu, (1 - fi)), weights), (1 / (1 - fi))) v = cp.power(cp.divide(q[:, :, None], (Ktu + reg_K)), fi) if (cp.any(Ktu == 0) or cp.any(cp.isnan(u)) or cp.any(cp.isnan(v)) or cp.any(cp.isinf(u)) or cp.any(cp.isinf(v))): # we have reached the machine precision # come back to previous solution and quit loop print('Numerical errors at iteration %s' % i) u = uprev v = vprev q = qprev break # compute change in barycenter if (i % 10 == 0) or i == 0: err = cp.abs(q - qprev).max() err /= max(cp.abs(q).max(), cp.abs(qprev).max(), 1.) if log: log['err'].append(err) # if barycenter did not change + at least 10 iterations - stop if err < stopThr and i > 10: break if verbose: if i % 100 == 0: print('{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) print('{:5d}|{:8e}|'.format(i, err.get())) if log: log['niter'] = i log['logu'] = (cp.log(u + 1e-300)) log['logv'] = (cp.log(v + 1e-300)) return q, log else: return q
def forward(self, x): h = F.relu(self.conv1(x)) h = F.relu(self.conv2(h)) h = F.max_pooling_2d(h, (1, 2), stride=(1, 2)) h = F.reshape(h, (h.data.shape[0], h.data.shape[2], h.data.shape[1], h.data.shape[3])) h = F.relu(self.conv3(h)) h = F.relu(self.conv4(h)) h = F.max_pooling_2d(h, (5, 3)) h = F.relu(self.conv5(h)) h = F.relu(self.conv6(h)) h = F.max_pooling_2d(h, (1, 2)) h = F.relu(self.conv7(h)) h = F.relu(self.conv8(h)) h = F.max_pooling_2d(h, (1, 2)) h = F.relu(self.conv9(h)) h = F.relu(self.conv10(h)) h = F.max_pooling_2d(h, (1, 2)) #Get power spectrum by FFT. data_shape = (6,) # frequency by 1.25Hz # sampling_rate=1000Hz # freq_names = {'delta','theta','lalpha','halpha','beta','lgamma'}; # freq_bands = [1 4; 4 8; 8 10; 10 13; 13 30; 30 50]; # delta:1.25-3.75: # theta: 5-7.5: # lalpha:8.75-10: # halpha:11.25-12.5: # beta:13.75-30: # lgamma:31.25-50: tmp = cupy.abs(cupy.fft.fft(x)) delta = cupy.average(tmp[:, :, :, 1:4], axis=3) theta = cupy.average(tmp[:, :, :, 4:7], axis=3) lalpha = cupy.average(tmp[:, :, :, 7:9], axis=3) halpha = cupy.average(tmp[:, :, :, 9:11], axis=3) beta = cupy.average(tmp[:, :, :, 11:25], axis=3) lgamma = cupy.average(tmp[:, :, :, 25:41], axis=3) Sum = delta + theta + lalpha + halpha + beta + lgamma power_spectral = cupy.zeros((x.shape[0], x.shape[1], x.shape[2], 6)) power_spectral[:, :, :, 0] = cupy.divide(delta, Sum) power_spectral[:, :, :, 1] = cupy.divide(theta, Sum) power_spectral[:, :, :, 2] = cupy.divide(lalpha, Sum) power_spectral[:, :, :, 3] = cupy.divide(halpha, Sum) power_spectral[:, :, :, 4] = cupy.divide(beta, Sum) power_spectral[:, :, :, 5] = cupy.divide(lgamma, Sum) power_spectral = chainer.Variable(power_spectral) power_spectral = F.cast(power_spectral, cupy.float32) h = F.reshape(h, (h.shape[0], h.shape[1] * h.shape[2] * h.shape[3])) power_spectral = F.reshape( power_spectral, (power_spectral.shape[0], power_spectral.shape[1] * power_spectral.shape[2] * power_spectral.shape[3])) h = F.relu(self.norm1(self.fc11(h))) h = F.dropout(h) h = F.relu(self.norm2(self.fc12(h))) h = F.dropout(h) #Concatenate the features extracted by deep neural network and relative power spectrum h = F.concat((h, power_spectral), axis=1) h = self.fc13(h) if chainer.config.train: return h return F.softmax(h)
def fit(self, x, y_true, epochs=1, learning_rate=0.001, batch_size=100, optimizer='same', verbose=False): """ Fit neural net to training set x, y_true Parameters ---------- x : cp.array of floats, shape (number of examples, number of features) Input features. y_true : cp.array of floats, shape (number of examples, number of classes) One-hot encoded labels. epochs : int, optional Number of epochs. learning_rate : float, optional Learning rate of the network. Default: 0.001. batch_size : int, optional Batch size. Default: 100. optimizer : str or None, optional Optimizer identifier. If you want to keep the existing optimizer, set to 'same'. Default: 'same'. verbose : bool, optional Print update for each batch. Default: False. """ # Set optimizer. if optimizer != 'same': self.optimizer = edonet.optimizers.choose(self, optimizer) # Calculate number of batches. nr_examples = x.shape[0] nr_batches = int(-(-nr_examples // batch_size)) # Iterate over epochs. for epoch in range(epochs): print("Epoch: ", epoch) avg_loss = cp.zeros(nr_batches) avg_acc = cp.zeros(nr_batches) # Iterate over batches. for i in range(nr_batches): # Forward propagation. x_batch = x[i * batch_size:min(nr_examples, (i + 1) * batch_size):, :] y_batch = y_true[i * batch_size:min(nr_examples, (i + 1) * batch_size):, :] y_pred = self._predict(x_batch, remove_dropout=False) # Calculate average loss. avg_loss[i] = cp.average(self.loss(y_pred, y_batch)) avg_acc[i] = edonet.metrics.accuracy(y_batch, y_pred) # Print status. if verbose: print("- Batch: %i/%i, loss: %.3f, acc: %.3f" % (i, nr_batches, avg_loss[i], avg_acc[i])) # Backpropagation and gradient descent. self.grad_desc(y_pred, y_batch, learning_rate) print("- Average loss: ", cp.average(avg_loss))
def cov(a, y=None, rowvar=True, bias=False, ddof=None, fweights=None, aweights=None, *, dtype=None): """Returns the covariance matrix of an array. This function currently does not support ``fweights`` and ``aweights`` options. Args: a (cupy.ndarray): Array to compute covariance matrix. y (cupy.ndarray): An additional set of variables and observations. rowvar (bool): If ``True``, then each row represents a variable, with observations in the columns. Otherwise, the relationship is transposed. bias (bool): If ``False``, normalization is by ``(N - 1)``, where N is the number of observations given (unbiased estimate). If ``True``, then normalization is by ``N``. ddof (int): If not ``None`` the default value implied by bias is overridden. Note that ``ddof=1`` will return the unbiased estimate and ``ddof=0`` will return the simple average. fweights (cupy.ndarray, int): 1-D array of integer frequency weights. the number of times each observation vector should be repeated. It is required that fweights >= 0. However, the function will not error when fweights < 0 for performance reasons. aweights (cupy.ndarray): 1-D array of observation vector weights. These relative weights are typically large for observations considered "important" and smaller for observations considered less "important". If ``ddof=0`` the array of weights can be used to assign probabilities to observation vectors. It is required that aweights >= 0. However, the function will not error when aweights < 0 for performance reasons. dtype: Data type specifier. By default, the return data-type will have at least `numpy.float64` precision. Returns: cupy.ndarray: The covariance matrix of the input array. .. seealso:: :func:`numpy.cov` """ if ddof is not None and ddof != int(ddof): raise ValueError('ddof must be integer') if a.ndim > 2: raise ValueError('Input must be <= 2-d') if dtype is None: if y is None: dtype = numpy.promote_types(a.dtype, numpy.float64) else: if y.ndim > 2: raise ValueError('y must be <= 2-d') dtype = functools.reduce(numpy.promote_types, (a.dtype, y.dtype, numpy.float64)) X = cupy.array(a, ndmin=2, dtype=dtype) if not rowvar and X.shape[0] != 1: X = X.T if X.shape[0] == 0: return cupy.array([]).reshape(0, 0) if y is not None: y = cupy.array(y, copy=False, ndmin=2, dtype=dtype) if not rowvar and y.shape[0] != 1: y = y.T X = _core.concatenate_method((X, y), axis=0) if ddof is None: ddof = 0 if bias else 1 w = None if fweights is not None: if not isinstance(fweights, cupy.ndarray): raise TypeError("fweights must be a cupy.ndarray") if fweights.dtype.char not in 'bBhHiIlLqQ': raise TypeError("fweights must be integer") fweights = fweights.astype(dtype=float) if fweights.ndim > 1: raise RuntimeError("cannot handle multidimensional fweights") if fweights.shape[0] != X.shape[1]: raise RuntimeError("incompatible numbers of samples and fweights") w = fweights if aweights is not None: if not isinstance(fweights, cupy.ndarray): raise TypeError("aweights must be a cupy.ndarray") aweights = aweights.astype(dtype=float) if aweights.ndim > 1: raise RuntimeError("cannot handle multidimensional aweights") if aweights.shape[0] != X.shape[1]: raise RuntimeError("incompatible numbers of samples and aweights") if w is None: w = aweights else: w *= aweights avg, w_sum = cupy.average(X, axis=1, weights=w, returned=True) w_sum = w_sum[0] # Determine the normalization if w is None: fact = X.shape[1] - ddof elif ddof == 0: fact = w_sum elif aweights is None: fact = w_sum - ddof else: fact = w_sum - ddof * sum(w * aweights) / w_sum if fact <= 0: warnings.warn('Degrees of freedom <= 0 for slice', RuntimeWarning, stacklevel=2) fact = 0.0 X -= X.mean(axis=1)[:, None] if w is None: X_T = X.T else: X_T = (X * w).T out = X.dot(X_T.conj()) * (1 / cupy.float64(fact)) return out.squeeze()
(x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = np.array(x_train).reshape((60000, 784)) x_test = np.array(x_test).reshape((10000, 784)) y_train = np.array(to_categorical(y_train)) y_test = np.array(to_categorical(y_test)) TEST_BATCH_SIZE = 1000 TRAIN_BATCH_SIZE = 1000 lr = 1e-3 for epoch in range(100000): print("=" * 10, "EPOCH", epoch, "=" * 10) test_indices = np.random.randint(x_test.shape[0], size=TEST_BATCH_SIZE) test_pred, test_real = d(x_test[test_indices]), y_test[test_indices] print("Loss:", np.average(-np.log(test_pred[:, np.argmax(test_real, -1)]))) print( "Accuracy:", np.sum(np.argmax(test_pred, -1) == np.argmax(test_real, -1)) / (test_pred.shape[0]) * 100, "%") for i in range(100): train_indices = np.random.randint(x_train.shape[0], size=TRAIN_BATCH_SIZE) d.train_batch(x_train[train_indices], y_train[train_indices], lr=lr, cross_entropy=True) lr *= 0.999