def create(self, numOfEpoch, *args): self.args = args self.data = dict() for item in self.args: self.data[item] = np.frombuffer(sharedmem.full(numOfEpoch, [0] * numOfEpoch, dtype=np.float64), dtype=np.float64) self.data['tried_times'] = np.frombuffer(sharedmem.full(1, [0], dtype=np.int8), dtype=np.int8)
def coarse_graph(graph, similarity, max_levels, method_option, reduction_factor): #### Coarsening #### while not graph['level'] == max_levels: # The common neighbors measure is used to contract the network. graph['similarity'] = getattr(Similarity(graph, graph['adjlist']), similarity) matching = sharedmem.full(graph.vcount(), range(graph.vcount()), dtype='int') processes = [] # Sets the method that will be used for creating the matching. method = None if(method_option == 0): method = graph.greed_two_hops elif(method_option == 1): method = graph.greed_rand_two_hops for layer in options.layers: start = sum(graph['vertices'][0:layer]) end = sum(graph['vertices'][0:layer + 1]) processes.append(Process(target = method, args = (range(start, end), reduction_factor[layer], matching))) for p in processes: p.start() for p in processes: p.join() # Coarsening the graph. coarser = graph.coarsening(matching) graph = coarser return graph
def adjoint_transform(self, coeffs, do_norm=True, total_cores=None): if total_cores is None: total_cores = self.total_cores # result = sm.full((self.height, self.width), 0, dtype='complex128') result = sm.full((self.height, self.width), 0, dtype=DTYPES[1]) numexpr_flag = NUM_FFTW_THREADS != 1 with sharedmem_pool(total_cores, numexpr=numexpr_flag) as pool: def work(i): # local_fft = make_fft(self.width, self.height) self.add_adjoint_part(coeffs[i], i, result, do_norm, pool) # if do_norm: # # temp = ((spects[i] / self._space_norms[i]) * # # my_fft_shift(self._fft(coeffs[i], local_fft))) # temp = ((self._spectrograms[i] / self._space_norms[i]) * # my_fft_shift(self._fft(coeffs[i]))) # else: # # temp = (self._spectrograms[i] * # # my_fft_shift(self._fft(coeffs[i], local_fft))) # temp = (self._spectrograms[i] * # my_fft_shift(self._fft(coeffs[i]))) # with pool.critical: # result[:] += temp pool.map(work, list(range(len(self._spectrograms)))) # return ifft2(my_ifft_shift(result)) return self._ifft(my_ifft_shift(result))
def tf_trainAndVal(idxData, prlog, train_flag, resultDir, trainDataShared, spcl): # creat shared memory variable # data_shared, dataLabel_shared, data_shared_stepInfo, data_shared_stateInfo = sharedDataCreat(batch_size, batchBlockNum, global_step_local=train_flag[0]) read_timer_batch_org = sharedmem.full(idxData.maxStep * 2, [-1] * idxData.maxStep * 2, dtype=np.float64) read_timer_batch = np.frombuffer(read_timer_batch_org, dtype=np.float64).reshape( idxData.maxStep, 2) # creat a processor to manage reading data simutanously Process(target=readData, args=(trainDataShared, idxData, spcl), kwargs={ 'mode': 'train', 'read_timer_batch': read_timer_batch }).start() # a process to clean cache simutanously, for performance testing if cleanCacheFlag == 1: Process(target=cleanMem, args=(trainDataShared.step, idxData.maxStep, spcl)).start() # f*****g train now tfTrain(trainDataShared, idxData, prlog, train_flag, resultDir, spcl) # terminate reading processors, # stop all children process spcl.passiveFinishAll()
def generate_roadmap_parallel(samples, env, max_dist, leafsize, knn): """Parallelized roadmap generator """ n_sample = len(samples) leafsize = knn if len(samples) < leafsize: leafsize = len(samples) - 1 import sharedmem sample_ids = np.arange(n_sample, dtype='i') roadmap = sharedmem.full((n_sample, knn), 0) # Start multi processing over samples with sharedmem.MapReduce() as pool: if n_sample % sharedmem.cpu_count() == 0: chunksize = n_sample / sharedmem.cpu_count() else: chunksize = n_sample / sharedmem.cpu_count() + 1 def work(i): skdtree = KDTree(samples, leafsize=leafsize) sub_sample_ids = sample_ids[slice(i, i + chunksize)] for j, sub_sample_id in enumerate(sub_sample_ids): x = samples[sub_sample_id] try: inds, dists = skdtree.search(x, k=leafsize) except: print "skdtree search failed" sys.exit() edge_id = [] append = edge_id.append for ii, (ind, dist) in enumerate(zip(inds, dists)): if dist > max_dist: break # undirected if len(edge_id) >= knn: break # directed? append(ind) # to complement fewer number of edges for vectorized valueiteration if len(edge_id) < knn: for ii in range(0, len(inds)): #for ind in edge_id: # edge_id.append(ind) # if len(edge_id) >= knn: break append(inds[0]) if len(edge_id) >= knn: break assert len( edge_id ) <= leafsize, "fewer leaves than edges {} (dists={})".format( len(edge_id), dists[:len(edge_id)]) for k in range(len(edge_id)): roadmap[sub_sample_id][k] = edge_id[k] pool.map(work, range(0, n_sample, chunksize)) #, reduce=reduce) # convert sharedmem array to list roadmap = np.array(roadmap).astype(int) skdtree = None #KDTree(samples, leafsize=leafsize) return roadmap.tolist(), skdtree
def test_create(): a = sharedmem.empty(100, dtype='f8') l = list(a) b = sharedmem.empty_like(a) assert b.shape == a.shape b = sharedmem.empty_like(l) assert b.shape == a.shape c = sharedmem.full(100, 1.0, dtype='f8') assert c.shape == a.shape c = sharedmem.full_like(a, 1.0) assert c.shape == a.shape
def create(self, sampleShape_local, labelShape_local, batch_size_local, batchPoolSize_local, batchBlockNum_local, latestStep): datashape = [batchPoolSize_local, batch_size_local] + sampleShape_local labelshape = [batchPoolSize_local, batch_size_local] + labelShape_local self.data = np.frombuffer(sharedmem.empty(datashape, dtype=np.float32), dtype=np.float32).reshape(datashape) self.label = np.frombuffer(sharedmem.empty(labelshape, dtype=np.int64), dtype=np.int64).reshape(labelshape) self.step = np.frombuffer(sharedmem.full( batchPoolSize_local, [latestStep] * batchPoolSize_local, dtype=np.int64), dtype=np.int64) # 0: ready to use, 1: used, 2:full of updating, 3 updating conti. self.state = np.frombuffer(sharedmem.full( batchPoolSize_local * (batchBlockNum_local + 1), [1] * batchPoolSize_local * (batchBlockNum_local + 1), dtype=np.int8), dtype=np.int8).reshape( batchBlockNum_local + 1, batchPoolSize_local)
def inverse_transform(self, coeffs, real=False, do_norm=True, total_cores=None): if total_cores is None: total_cores = self.total_cores # result = sm.full((self.height, self.width), 0, dtype='complex128') result = sm.full((self.height, self.width), 0, dtype=DTYPES[1]) with sharedmem_pool(total_cores) as pool: def work(i): # local_fft = make_fft(self.width, self.height) # coeff_f = my_fft_shift(self._fft(coeffs[i], local_fft)) self.add_inverse_part(coeffs[i], i, result, do_norm, pool) # coeff_fourier = my_fft_shift(self._fft(coeffs[i])) # assert coeff_fourier is not None # silence pyflakes # spec = self._spectrograms[i] # dual_w = self.dual_frame_weight # temp = np.zeros((self.height, self.width), # dtype=DTYPES[1]) # if do_norm: # norm = self._space_norms[i] # ne.evaluate('spec * norm * coeff_fourier / dual_w', # out=temp) # else: # ne.evaluate('spec * coeff_fourier / dual_w', out=temp) # with pool.critical: # result[:] += temp pool.map(work, list(range(len(self._spectrograms)))) result = np.array(result) if real: # return np.real(ifft2(my_ifft_shift(result))) return np.real(self._ifft(my_ifft_shift(result))) else: # return ifft2(my_ifft_shift(result)) return self._ifft(my_ifft_shift(result))
def readData(dataShared, idxData, spcl, **kwargs): lock = Lock() mode = kwargs['mode'] spcl.start(mode, -1) # the read manager will be record in the final row print '%s read manager - %d - mother start' % (mode, os.getpid()) if mode == 'train': readNum = readProcessorNum elif mode == 'val' or mode == 'test': readNum = eval_readProcessorNum # a flag which tell the readData manager that the fill slaver is activated. token = sharedmem.full(readNum, [0] * readNum, dtype=np.int8) manager = [[]] * readNum for num in range(readNum): manager[num] = Process(target=fillData, args=(lock, dataShared, idxData, spcl, num, readNum, token), kwargs=kwargs) manager[num].start() for num in range(readNum): manager[num].join() while True: if sum(token) == readNum and spcl.evaluateChild( os.getpid(), mode, readNum) == 'dead': break time.sleep(1) spcl.activeFinish(mode, -1) print '%s read manager - %d - mother dead' % (mode, os.getpid()) print np.concatenate( (spcl.table[0, :, :], spcl.table[1, :, :], spcl.table[2, :, :]), 1)
def get_features_from_states(env, states, feature_fn): import sharedmem n_states = len(states) feat_len = len(feature_fn(env, states[0])) state_ids = np.arange(n_states, dtype='i') features = sharedmem.full((n_states, feat_len), 0.) # Start multi processing over support states with sharedmem.MapReduce() as pool: if n_states % sharedmem.cpu_count() == 0: chunksize = n_states / sharedmem.cpu_count() else: chunksize = n_states / sharedmem.cpu_count() + 1 def work(i): s_ids = state_ids[slice(i, i + chunksize)] for j, s_id in enumerate(s_ids): s = states[s_id] # state id in states features[s_id] = feature_fn(env, s) pool.map(work, range(0, n_states, chunksize)) #, reduce=reduce) return np.array(features)
def __init__(self, size=1, lock=None): super().__init__(lock) self.size = size self.index = sharedmem.full(1, 0, int)
def generate_graph(PB, diff_l, diff_h): print('generating graph...') #getcontext().prec = 6 #ttl_path=Decimal(0.0) ttl_path = Decimal((2**(timestamp - 2) - 1)) #eps_edges=np.float64(0.0) ''' for idx in range(2,timestamp+1): #if is_in_bound(1,idx,idx-1,diff_l,diff_h): ttl_path+= Decimal(find_total_path_with_edge(1, 1, idx, 1, timestamp)) ''' lock1 = mp.Lock() lock2 = mp.Lock() lock3 = mp.Lock() #for i in range(num_timeseries): # lock3.append(mp.Lock()) manager = mp.Manager() #ttl_Prest = np.array([Decimal(0)]*num_timeseries,dtype=np.dtype(Decimal)) ttl_Prest = np.zeros(num_timeseries, dtype=np.float128) #print('total prest type') #print(type(ttl_Prest[0])) #print(ttl_Prest.shape) #''' ttl_nodes = timestamp * timestamp num_cores = mp.cpu_count() if num_cores > ttl_nodes: n_jobs = ttl_nodes else: n_jobs = num_cores batch_size = int(ttl_nodes / n_jobs) node_num = 1 start = time.time() for batch in range(batch_size): job = [] num_path = mp.Value('i', 0) Prest = shm.full(num_timeseries, 0, dtype=np.float128) #Prest = shm.full(num_timeseries,Decimal(0),dtype=np.dtype(Decimal)) #print('prest type when created:') #print(type(Prest[0])) for idx in range(node_num, min(node_num + n_jobs, ttl_nodes)): node_start = int(idx / timestamp + 1) node_end = int(idx % timestamp + 1) if node_start >= node_end: #or not is_in_bound(node_start,node_end,node_end-node_start,diff_l,diff_h): continue #firstend=node_start+diff_l #print(node_start,node_end,firstend,diff_l) #if (node_end-firstend)<diff_l: # continue key = str(node_start) + '-' + str(node_end) print(key) p = mp.Process(target=create_graph_path, args=(key, node_start, node_end, PB, Prest, idx, timestamp, lock1, lock3, ttl_path)) job.append(p) p.start() node_num += n_jobs _ = [p.terminate() for p in job] _ = [p.join() for p in job] #ttl_path+=num_path.value ttl_Prest = np.add(ttl_Prest, Prest) #print('process Prest:') #print(Prest) #''' stop = time.time() comp_time = stop - start print('time: %.4f' % comp_time) #print('Prest:') #print(ttl_Prest) #print('nodes') #print(graph_nodes.keys()) return ttl_Prest, ttl_path, comp_time
def create(self, readProcessorNum, eval_readProcessorNum): lenth = max(readProcessorNum, eval_readProcessorNum) + 1 self.table = np.frombuffer(sharedmem.full(lenth * 3 * 4, [-1] * lenth * 3 * 4, dtype=np.int64), dtype=np.int64).reshape(3, lenth, 4)
# CNN-training parametes # os.environ["CUDA_VISIBLE_DEVICES"]="1" num_epochs = 20 SEED = 66478 num_label_classes = 2 max_keep = 5 loss_frequency = 200 # how much steps to print the loss infomation def data_type(): return tf.float32 # early stop parametes train_flag = sharedmem.full(1, [-1], dtype=np.int64) initialLR = 0.01 reTryNum = 5 decentRate = 0.5 stopLR = initialLR * (decentRate**reTryNum) def main(): for vp in range(10): for cross in range(10): # setup result output path if not os.path.isdir(os.path.join(projectPath, 'View{}'.format(vp))): os.makedirs(os.path.join(projectPath, 'View{}'.format(vp)))
def computeQ(mdp, support_states, error=1e-10, support_features=None, support_feature_state_dict=None, cstr_fn=None, add_no_cstr=True, max_cnt=100, **kwargs): """Compute Q using multi-process """ # initialization of variables n_support_states = len(support_states) n_actions, n_states = mdp.n_actions, mdp.n_states eps = np.finfo(float).eps roadmap = mdp.roadmap states = mdp.states gamma = mdp.gamma T = mdp.T rewards = mdp.get_rewards() #from IPython import embed; embed(); sys.exit() if rewards is None: rewards=np.zeros(len(mdp.states)) else: rewards = np.array(rewards) rewards[np.where(rewards>0)]=0. support_state_ids = np.arange(n_support_states, dtype='i') if support_features is not None: support_feature_ids, support_feature_values = support_features computed_f_id = sharedmem.full(len(support_feature_values), False, dtype='b') else: return NotImplementedError if cstr_fn is None: support_q_mat = sharedmem.full((n_support_states, mdp.n_states, mdp.n_actions), 0.) support_values = sharedmem.full((n_support_states, mdp.n_states), 0.) support_validity = sharedmem.full((n_support_states), True) else: if add_no_cstr: n_cstr_fn = len(cstr_fn) + 1 else: n_cstr_fn = len(cstr_fn) support_q_mat = sharedmem.full((n_support_states, n_cstr_fn, mdp.n_states, mdp.n_actions), 0.) support_values = sharedmem.full((n_support_states, n_cstr_fn, mdp.n_states), 0.) support_validity = sharedmem.full((n_support_states, n_cstr_fn), True) if len(cstr_fn)>0: feat_map = kwargs['feat_map'] roadmap = kwargs['roadmap'] states = mdp.states cstr_T = [] for i in range(len(cstr_fn)): validity_map = cstr_fn[i](None, f=feat_map)[roadmap] validity_map[:,0] = True Tc = mdp.T*validity_map[:,np.newaxis,:] Tc[:,:,0] = eps sum_T = np.sum(Tc, axis=-1) Tc /= sum_T[:,:,np.newaxis] cstr_T.append(Tc) # Start multi processing over support states with sharedmem.MapReduce() as pool: if n_support_states % sharedmem.cpu_count() == 0: chunksize = n_support_states / sharedmem.cpu_count() else: chunksize = n_support_states / sharedmem.cpu_count() + 1 def work(i): state_ids = support_state_ids[slice (i, i + chunksize)] new_rewards = copy.copy(rewards) values = np.zeros(n_states) for j, state_id in enumerate(state_ids): s = support_states[state_id] # state id in states # vi agent mdp = vi.valueIterAgent(n_actions, n_states, roadmap, None, states, gamma=gamma, T=T) mdp.set_goal(s) if support_feature_ids is None: if new_rewards[s] >= 0.: new_rewards[s] = 1. else: # find all states that gives f_g in states f_id = support_feature_ids[state_id] goal_state_ids = support_feature_state_dict[f_id] if computed_f_id[f_id]: continue else: computed_f_id[f_id] = True new_rewards[goal_state_ids] = 1. mdp.set_rewards(new_rewards) # Store q_mat and validity mat per state if cstr_fn is not None: for k in range(len(cstr_fn)): # check if the goal is isolated if np.sum(cstr_T[k][goal_state_ids])>0.: values, param_dict = mdp.solve_mdp(error, init_values=values, T=cstr_T[k], max_cnt=max_cnt, goal=s, return_params=True) support_q_mat[state_id][k] = param_dict['q'] support_validity[state_id][k] = cstr_fn[k](s) support_values[state_id][k] = values if add_no_cstr: values, param_dict = mdp.solve_mdp(error, init_values=values, T=T, max_cnt=max_cnt, ## goal=s, return_params=True) support_q_mat[state_id][-1] = param_dict['q'] support_validity[state_id][-1] = True support_values[state_id][-1] = values else: values, param_dict = mdp.solve_mdp(error, init_values=values, max_cnt=max_cnt, return_params=True) support_q_mat[state_id] = param_dict['q'] support_values[state_id] = values # find all states that gives f_g in states for gs in goal_state_ids: k = support_states.index(gs) if k!=state_id: support_q_mat[k] = support_q_mat[state_id] # reset ## new_rewards = copy.copy(rewards) if support_feature_ids is None: new_rewards[s] = 0. else: new_rewards[goal_state_ids] = 0. pool.map(work, range(0, n_support_states, chunksize))#, reduce=reduce) # convert sharedmem array to dict support_q_mat_dict = {} support_values_dict = {} support_validity_dict = {} for i, s in enumerate(support_states): support_q_mat_dict[s] = np.array(support_q_mat[i]) support_values_dict[s] = np.array(support_values[i]) if cstr_fn is not None: support_validity_dict[s] = np.array(support_validity[i]) if cstr_fn is not None: return support_q_mat_dict, support_values_dict, support_validity_dict else: return support_q_mat_dict, support_values_dict
def main(): """ Main entry point for the application when run from the command line. """ # Timing instanciation timing = Timing(['Snippet', 'Time [m]', 'Time [s]']) with timing.timeit_context_add('Pre-processing'): # Setup parse options command line current_path = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) parser = args.setup_parser(current_path + '/args/coarsening.json') options = parser.parse_args() args.update_json(options) args.check_output(options) # Log instanciation log = helper.initialize_logger(dir='log', output='log') if options.input and options.vertices is None: log.warning('Vertices are required when input is given.') sys.exit(1) # Create default values for optional parameters if options.reduction_factor is None: options.reduction_factor = [0.5] * len(options.vertices) if options.max_levels is None: options.max_levels = [3] * len(options.vertices) if options.matching is None: options.matching = ['rgmb'] * len(options.vertices) if options.similarity is None: options.similarity = ['weighted_common_neighbors'] * len( options.vertices) if options.itr is None: options.itr = [10] * len(options.vertices) if options.upper_bound is None: options.upper_bound = [2.0] * len(options.vertices) if options.global_min_vertices is None: options.global_min_vertices = [None] * len(options.vertices) if options.tolerance is None: options.tolerance = [0.01] * len(options.vertices) # Validation of list values if len(options.reduction_factor) == 1: options.reduction_factor = [options.reduction_factor[0]] * len( options.vertices) if len(options.max_levels) == 1: options.max_levels = [options.max_levels[0]] * len( options.vertices) if len(options.matching) == 1: options.matching = [options.matching[0]] * len(options.vertices) if len(options.similarity) == 1: options.similarity = [options.similarity[0]] * len( options.vertices) if len(options.itr) == 1: options.itr = [options.itr[0]] * len(options.vertices) if len(options.upper_bound) == 1: options.upper_bound = [options.upper_bound[0]] * len( options.vertices) if len(options.global_min_vertices) == 1: options.global_min_vertices = [options.global_min_vertices[0] ] * len(options.vertices) if len(options.tolerance) == 1: options.tolerance = [options.tolerance[0]] * len(options.vertices) # Verification of the dimension of the parameters if len(options.vertices) != len(options.reduction_factor): log.warning('Sizes of input arguments -v and -r do not match.') sys.exit(1) if len(options.vertices) != len(options.max_levels): log.warning('Sizes of input arguments -v and -m do not match.') sys.exit(1) if len(options.vertices) != len(options.matching): log.warning('Sizes of input arguments -v and -c do not match.') sys.exit(1) if len(options.vertices) != len(options.similarity): log.warning('Sizes of input arguments -v and -s do not match.') sys.exit(1) if len(options.vertices) != len(options.itr): log.warning('Size of input arguments -v and -i do not match.') sys.exit(1) if len(options.vertices) != len(options.upper_bound): log.warning('Size of input arguments -v and -ub do not match.') sys.exit(1) if len(options.vertices) != len(options.global_min_vertices): log.warning('Size of input arguments -v and -gmv do not match.') sys.exit(1) if len(options.vertices) != len(options.tolerance): log.warning('Size of input arguments -v and -t do not match.') sys.exit(1) # Validation of matching method valid_matching = ['rgmb', 'gmb', 'mlpb', 'hem', 'lem', 'rm'] for index, matching in enumerate(options.matching): matching = matching.lower() if matching not in valid_matching: log.warning('Matching ' + matching + ' method is unvalid.') sys.exit(1) options.matching[index] = matching # Validation of sedd priority valid_seed_priority = ['strength', 'degree', 'random'] for index, seed_priority in enumerate(options.seed_priority): seed_priority = seed_priority.lower() if seed_priority not in valid_seed_priority: log.warning('Seed priotiry ' + seed_priority + ' is unvalid.') sys.exit(1) options.seed_priority[index] = seed_priority # Validation reverse for index, reverse in enumerate(options.reverse): if reverse.lower() in ('yes', 'true', 't', 'y', '1'): options.reverse[index] = True elif reverse.lower() in ('no', 'false', 'f', 'n', '0'): options.reverse[index] = False else: log.warning('Boolean value expected in -rv.') sys.exit(1) # isinstance(data[i][k], bool) # Validation of similarity measure valid_similarity = [ 'common_neighbors', 'weighted_common_neighbors', 'salton', 'preferential_attachment', 'jaccard', 'weighted_jaccard', 'adamic_adar', 'resource_allocation', 'sorensen', 'hub_promoted', 'hub_depressed', 'leicht_holme_newman' ] for index, similarity in enumerate(options.similarity): similarity = similarity.lower() if similarity not in valid_similarity: log.warning('Similarity ' + similarity + ' misure is unvalid.') sys.exit(1) options.similarity[index] = similarity for layer in range(len(options.vertices)): if options.matching[layer] in ['rgmb', 'gmb', 'hem', 'lem', 'rm']: if options.global_min_vertices[layer] is not None: options.global_min_vertices[layer] = None text = 'Matching method ' + options.matching[layer] text += ' (setted in layer ' text += str(layer) + ') does not accept -gmv parameter.' log.warning(text) if options.reduction_factor[layer] > 0.5: options.reduction_factor[layer] = 0.5 text = 'Matching method ' + options.matching[layer] text += ' (setted in layer ' text += str(layer) + ') does not accept -rf > 0.5.' log.warning(text) # Load bipartite graph with timing.timeit_context_add('Load'): graph = helpermgraph.load(options.input, options.vertices) graph['level'] = [0] * graph['layers'] source_ecount = graph.ecount() # Coarsening with timing.timeit_context_add('Coarsening'): hierarchy_graphs = [] hierarchy_levels = [] running = True while running: running = False membership = sharedmem.full(graph.vcount(), range(graph.vcount()), dtype='int') levels = graph['level'] contract = False processes = [] for layer in range(len(graph['vertices'])): matching_layer = True if (options.global_min_vertices[layer] is None): if levels[layer] >= options.max_levels[layer]: matching_layer = False elif (graph['vertices'][layer] <= options.global_min_vertices[layer]): matching_layer = False if matching_layer: contract = True running = True levels[layer] += 1 graph['similarity'] = getattr( Similarity(graph, graph['adjlist']), options.similarity[layer]) start = sum(graph['vertices'][0:layer]) end = sum(graph['vertices'][0:layer + 1]) vertices = range(start, end) param = dict( reduction_factor=options.reduction_factor[layer]) if options.matching[layer] in ['mlpb', 'gmb', 'rgmb']: param['vertices'] = vertices param['reverse'] = options.reverse[layer] if options.matching[layer] in ['mlpb', 'rgmb']: param['seed_priority'] = options.seed_priority[layer] if options.matching[layer] in ['mlpb']: param['upper_bound'] = options.upper_bound[layer] param['n'] = options.vertices[layer] param[ 'global_min_vertices'] = options.global_min_vertices[ layer] param['tolerance'] = options.tolerance[layer] param['itr'] = options.itr[layer] if options.matching[layer] in ['hem', 'lem', 'rm']: one_mode_graph = graph.weighted_one_mode_projection( vertices) matching_method = getattr(one_mode_graph, options.matching[layer]) else: matching_method = getattr(graph, options.matching[layer]) processes.append( Process(target=matching_method, args=[membership], kwargs=param)) for p in processes: p.start() for p in processes: p.join() if contract: coarse = graph.contract(membership) coarse['level'] = levels if coarse.vcount() == graph.vcount(): break graph = coarse if options.save_hierarchy or not running: hierarchy_graphs.append(graph) hierarchy_levels.append(levels[:]) if not hierarchy_graphs: hierarchy_graphs.append(graph) hierarchy_levels.append(levels[:]) # Save with timing.timeit_context_add('Save'): output = options.output for index, obj in enumerate(zip(hierarchy_levels, hierarchy_graphs)): levels, graph = obj index += 1 if options.save_conf or options.show_conf: d = {} d['source_input'] = options.input d['source_vertices'] = [ options.vertices[0], options.vertices[1] ] d['source_vcount'] = options.vertices[0] + options.vertices[1] d['source_ecount'] = source_ecount d['ecount'] = graph.ecount() d['vcount'] = graph.vcount() d['vertices'] = graph['vertices'] d['reduction_factor'] = options.reduction_factor d['max_levels'] = options.max_levels d['achieved_levels'] = graph['level'] d['similarity'] = options.similarity d['matching'] = options.matching d['level'] = levels d['upper_bound'] = options.upper_bound d['global_min_vertices'] = options.global_min_vertices d['itr'] = options.itr if options.save_conf: with open(output + '-' + str(index) + '.conf', 'w+') as f: json.dump(d, f, indent=4) if options.show_conf: print(json.dumps(d, indent=4)) if options.save_ncol: graph.write(output + '-' + str(index) + '.ncol', format='ncol') if options.save_source: with open(output + '-' + str(index) + '.source', 'w+') as f: for v in graph.vs(): f.write(' '.join(map(str, v['source'])) + '\n') if options.save_membership: membership = [0] * (options.vertices[0] + options.vertices[1]) for v in graph.vs(): for source in v['source']: membership[source] = v.index numpy.savetxt(output + '-' + str(index) + '.membership', membership, fmt='%d') if options.save_predecessor: with open(output + '-' + str(index) + '.predecessor', 'w+') as f: for v in graph.vs(): f.write(' '.join(map(str, v['predecessor'])) + '\n') if options.save_successor: numpy.savetxt(output + '-' + str(index) + '.successor', graph.vs['successor'], fmt='%d') if options.save_weight: numpy.savetxt(output + '-' + str(index) + '.weight', graph.vs['weight'], fmt='%d') if options.save_gml: del graph['adjlist'] del graph['similarity'] graph['layers'] = str(graph['layers']) graph['vertices'] = ','.join(map(str, graph['vertices'])) graph['level'] = ','.join(map(str, graph['level'])) graph.vs['name'] = map(str, range(0, graph.vcount())) graph.vs['type'] = map(str, graph.vs['type']) graph.vs['weight'] = map(str, graph.vs['weight']) graph.vs['successor'] = map(str, graph.vs['successor']) for v in graph.vs(): v['source'] = ','.join(map(str, v['source'])) v['predecessor'] = ','.join(map(str, v['predecessor'])) graph.write(output + '-' + str(index) + '.gml', format='gml') if not options.save_hierarchy: break if options.show_timing: timing.print_tabular() if options.save_timing_csv: timing.save_csv(output + '-timing.csv') if options.save_timing_json: timing.save_json(output + '-timing.json')
import time import numpy as np import sharedmem from multiprocessing import Process, Manager def function(counts, lock): with lock: if counts[0]['foo'] == 0: time.sleep(1) counts[0]['foo'] += 1 processes = [] manager = Manager() counts = sharedmem.full(1, (0, ), dtype=[('foo', 'i1')]) lock = manager.Lock() for i in range(2): processes.append(Process(target=function, args=(counts, lock))) processes[-1].start() for process in processes: process.join() print(counts[0]['foo'])
def __init__(self, path: str, log1p: Optional[bool] = False, nproc: Optional[int] = 1, selection: Optional[list] = None, silent: Optional[bool] = False) -> None: """ PyTorch Dataset wrapper to handle the GSE93421 hdf5 dataset :param path: path to hdf5 file for e18 Mouse Data :param nproc: number of processes to use in contructing vectors :param ratio: ratio of the dataset to actually load :param selection: list of cells to load data for :param silent: whether print statements should print """ hdf5 = h5py.File(path, 'r', driver='core') self.dims = len(hdf5['mm10']['genes']) # allow a customizable selection of cells if selection is not None: self._len = len(selection) else: self._len = len(hdf5['mm10']['indptr']) selection = range(0, self._len) self.selection = selection # get a list that can be shared between processes selected_cells = sm.empty(self._len, dtype=np.int) for i in range(0, self._len): selected_cells[i] = self.selection[i] #self.cells = sm.full((self._len,self.dims),0,dtype=np.int16) # Load all of the important information into memory ############# # Data ############# if not silent: print("Reading data ...") start = time() ds = hdf5['mm10']['data'] data = sm.empty(len(ds), dtype=ds.dtype) ds.read_direct(data) tmp = ds.dtype end = time() if not silent: print("\t" + str(end - start) + "s ...") ############# # Indices ############# if not silent: print("Reading indices ...") start = time() ds = hdf5['mm10']['indices'] indx = sm.empty(len(ds), dtype=ds.dtype) ds.read_direct(indx) end = time() if not silent: print("\t" + str(end - start) + "s ...") ############# # Indptr ############# if not silent: print("Reading indptr ...") start = time() ds = hdf5['mm10']['indptr'] iptr = sm.empty(len(ds), dtype=ds.dtype) ds.read_direct(iptr) end = time() if not silent: print("\t" + str(end - start) + "s ...") hdf5.close() del ds ########################### # Create empty cell vectors ########################### # build the vector foreach cell if not silent: print("Creating 0 vectors ...") start = time() self.data = sm.full((self._len, self.dims), 0, dtype=tmp) #self.cells = sm.full((self._len,self.dims),0,dtype=float) end = time() if not silent: print("\t" + str(end - start) + "s ...") ########################### # Multi-core loading ... ########################### if not silent: print("Building Tensor List ...") start = time() with sm.MapReduce(np=nproc) as pool: pool.map( _build_tensor, list( zip([self.data] * nproc, [iptr] * nproc, [indx] * nproc, [data] * nproc, range(0, nproc), [nproc] * nproc, [selected_cells] * nproc, [log1p] * nproc))) end = time() if not silent: print("\t" + str(end - start) + "s ...") # Some explicit cleanup to conserve memory # Not sure if necessary, but I don't trust Python del iptr del indx del data del selected_cells
def computePolicies(mdp, goal_states, error=1e-10): """Compute Q using multi-process """ # initialization of variables n_goal_states = len(goal_states) n_actions, n_states = mdp.n_actions, mdp.n_states roadmap = mdp.roadmap states = mdp.states gamma = mdp.gamma T = mdp.T rewards = mdp.get_rewards() #from IPython import embed; embed(); sys.exit() if rewards is None: rewards=np.zeros(len(mdp.states)) else: rewards = np.array(rewards) rewards[np.where(rewards>0)]=0. goal_state_ids = np.arange(n_goal_states, dtype='i') goal_policy_mat = sharedmem.full((n_goal_states, mdp.n_states, mdp.n_actions), 0.) ## goal_values = sharedmem.full((n_goal_states, mdp.n_states), 0.) ## goal_validity = sharedmem.full((n_goal_states), True) # Start multi processing over goal states with sharedmem.MapReduce() as pool: if n_goal_states % sharedmem.cpu_count() == 0: chunksize = n_goal_states / sharedmem.cpu_count() else: chunksize = n_goal_states / sharedmem.cpu_count() + 1 def work(i): state_ids = goal_state_ids[slice (i, i + chunksize)] #0,1,2,3 new_rewards = copy.copy(rewards) values = np.zeros(n_states) for j, goal_state_id in enumerate(state_ids): s = goal_states[goal_state_id] # state id in states ## s_idx = states.index(s) # vi agent mdp = vi.valueIterAgent(n_actions, n_states, roadmap, None, states, gamma=gamma, T=T) mdp.set_goal(s) if new_rewards[s] >= 0.: new_rewards[s] = 1. mdp.set_rewards(new_rewards) # Store q_mat and validity mat per state policy, _ = mdp.find_policy(error) goal_policy_mat[goal_state_id] = policy # find all states that gives f_g in states for k, gs in enumerate(goal_states): if s == gs: goal_policy_mat[k] = policy # reset new_rewards[s] = 0. pool.map(work, range(0, n_goal_states, chunksize))#, reduce=reduce) # convert sharedmem array to list policies = [] for i in range(n_goal_states): policies.append( np.array(goal_policy_mat[i]) ) return policies