def analyze_network_cc(neighbor_indices, cells_response_curve_parts, kpss_and_adf_filter, max_tpd): result_df = pd.DataFrame(columns=[ 'source', 'destination', 'part', 'topological_distance', 'cross_correlation' ]) simple_network = build_network(neighbor_indices) tp_indices_by_td_all = get_all_indices_with_toplogical_distance_all( simple_network, max_tpd, neighbor_indices) number_of_parts = len(cells_response_curve_parts) pool = Pool() results = [ pool.apply_async(analyze_network_cc_by_part_private, (create_analyze_network_cc_by_td_context( tp_indices_by_td_all, neighbor_indices, cells_response_curve_parts[part], kpss_and_adf_filter, max_tpd, part), )) for part in range(0, number_of_parts) ] for p_result in results: result_df = result_df.append(p_result.get()) pool.close() return result_df
def analyze_network(neighbor_indices, cells_response_curve_parts, kpss_and_adf_filter, max_tpd): from tqdm.notebook import tqdm result_df = pd.DataFrame(columns=['source', 'destination', 'part', 'topological_distance', 'optimal_lag', 'granger_causality_mag', 'granger_causality_pvalue']) simple_network = build_network(neighbor_indices) tp_indices_by_td_all = get_all_indices_with_toplogical_distance_all(simple_network, max_tpd, neighbor_indices) number_of_parts = len(cells_response_curve_parts) pool = Pool() results = [pool.apply_async(analyze_network_by_part_private, (create_analyze_network_by_td_context(tp_indices_by_td_all, neighbor_indices, cells_response_curve_parts[part], kpss_and_adf_filter, max_tpd, part),)) for part in range(0, number_of_parts)] for p_result in tqdm(results, total=number_of_parts): result_df = result_df.append(p_result.get()) pool.close() return result_df
def analyze_network_by_td(neighbor_indices, cells_response_curve, kpss_and_adf_filter, max_tpd, random_mode, max_neighbors): ''' Analyze the network by topological distance. Parameters ---------- neighbor_indices : dict The key is the node idice and the value are list of neighbor indices cells_response_curve : array[#cells, #frames] The cell response of each cell max_tpd : int Max topological distance to estimate random_mode : string 'g' - global mode. 'c' - cell mode. max_neighbors : int Max neighbors for each topological distance. In global mode 'g' the max_neighbors is counted globaly. In cell mode 'c' the max_neighbors is counted per cell. If there is more than the max_neighbors then @max_neighbors are chosen uniformly. Returns ------- tp_indices_by_td : dict The key is the distance and the value is dict which the key are the node idice and the value are list of neighbor indices by topological distance. tp_indices_random_by_td : dict Same as @tp_indices_by_td but after random choice. optimal_lag_by_td: dict The key is the distance and the value is the optimal lag for each pairs. ''' #key - topological distance #value - dict(vertice index, neighbors indexes) neighbors of each vertices tp_indices_by_td = {} tp_indices_random_by_td = {} optimal_lag_by_td = {} simple_network = build_network(neighbor_indices) tp_indices_by_td_all = get_all_indices_with_toplogical_distance_all( simple_network, max_tpd, neighbor_indices) pool = Pool(processes=4) results = [ pool.apply_async(analyze_network_by_td_private, (create_analyze_network_by_td_context( tp_indices_by_td_all, neighbor_indices, random_mode, max_neighbors, cells_response_curve, kpss_and_adf_filter, tp_distance), )) for tp_distance in range(1, max_tpd + 1) ] for td in range(1, max_tpd + 1): tp_indices_by_td[td], tp_indices_random_by_td[td], optimal_lag_by_td[ td] = results[td - 1].get() pool.close() ''' for td in range(1, max_tpd + 1): #print 'analyze_network_by_td td: ' + str(td) + ' range: 1-' + str(max_tpd) tp_indices_by_td[td] = get_all_indices_with_toplogical_distance_specific(tp_indices_by_td_all, td, neighbor_indices) #choice random neighbors if random_mode == 'g': tp_indices_random_by_td[td] = choice_random_neighbors_global_level(tp_indices_by_td[td], count_neighbors(tp_indices_by_td[td]), max_neighbors) elif random_mode == 'c': tp_indices_random_by_td[td] = choice_random_neighbors_cell_level(tp_indices_by_td[td], count_neighbors(tp_indices_by_td[td]), max_neighbors) else: raise Exception('random_mode is unknown. please choose global or cell level.') #Var Analysis Find The Optimal Lag by Information Criterion optimal_lag_by_td[td] = get_collective_optimal_lag(neighbor_indices, cells_response_curve, tp_indices_random_by_td[td], kpss_and_adf_filter) ''' return tp_indices_by_td, tp_indices_random_by_td, optimal_lag_by_td
def calc_hub_indv_perm_scores(analyze_cell_stats_flat, neighbor_indices, pipe_norm_df, kpss_and_adf_filter, centroids, cells_response_curve_parts, get_role, base_path): # get original hub/individual cells indices state_matrix = analyze_cell_stats_flat.unstack()['Manual_' + str(0.5)].transpose() cells_state_matrix_loc_indexes = np.where(state_matrix == 0)[1] cells_state_matrix_loc_indexes = np.concatenate( [cells_state_matrix_loc_indexes, np.where(state_matrix == 1)[1]]) cells_indices = state_matrix.iloc[:, cells_state_matrix_loc_indexes].columns cells_indices = cells_indices.astype(int) orginal_hub_ind_count_ratio = get_hub_ind_count_ratio( state_matrix, cells_indices) hub_indv_perm_scores = load_object(base_path + 'hub_indv_perm_scores_df') print(hub_indv_perm_scores) if hub_indv_perm_scores is None: from module.networkhelper import build_network from module.networkhelper import get_all_indices_with_toplogical_distance_all, get_all_indices_with_toplogical_distance_specific from module.stathelper import check_raw_gc, get_collective_optimal_lag from modulev2.analyzetools import analyze_network #calculate neighbor ##build network simple_network = build_network(neighbor_indices) tp_indices_by_td_all = get_all_indices_with_toplogical_distance_all( simple_network, 2, neighbor_indices) tp_indices_1 = get_all_indices_with_toplogical_distance_specific( tp_indices_by_td_all, 1, neighbor_indices) tp_indices_2 = get_all_indices_with_toplogical_distance_specific( tp_indices_by_td_all, 2, neighbor_indices) from multiprocessing import Pool from tqdm.notebook import tqdm import time import multiprocessing class NoDaemonProcess(multiprocessing.Process): # make 'daemon' attribute always return False def _get_daemon(self): return False def _set_daemon(self, value): pass daemon = property(_get_daemon, _set_daemon) # We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool # because the latter is only a wrapper function, not a proper class. class MyPool(multiprocessing.pool.Pool): Process = NoDaemonProcess def mute(): sys.stdout = open(os.devnull, 'w') if not sys.warnoptions: import warnings warnings.simplefilter("ignore") scores = [] from concurrent.futures import ProcessPoolExecutor #pool = ThreadPool(processes=10, maxtasksperchild=1) pool_size = 100 ''' context_array = np.zeros((pool_size), dtype=object) for i in range(0, pool_size): context_array[i] = get_hub_ind_count_ratio_perm_function_context(cells_indices, tp_indices_1, tp_indices_2, kpss_and_adf_filter, pipe_norm_df, centroids, cells_response_curve_parts, get_role) ''' ''' with ProcessPoolExecutor(max_workers = 50) as executor: results = list(tqdm(executor.map(get_hub_ind_count_ratio_perm_function, context_array), total=len(context_array))) for result in results: scores.append(result[0]) ''' #for score in tqdm(pool.imap_unordered(get_hub_ind_count_ratio_perm_function, context_array), total=1): # scores.append(score[0]) # pass for i in tqdm(range(pool_size)): scores.append( get_hub_ind_count_ratio_perm(cells_indices, tp_indices_1, tp_indices_2, kpss_and_adf_filter, pipe_norm_df, centroids, cells_response_curve_parts, get_role)[0]) hub_indv_perm_scores = scores save_object(hub_indv_perm_scores, base_path + 'hub_indv_perm_scores_df') return orginal_hub_ind_count_ratio, hub_indv_perm_scores