def evaluate(self, batch_size=100, cpu=-1, filtering=True) -> Dict[str, float]: """Evaluates a model by retrieving scores from the (implemented) score_batch function. :param batch_size: Size of a test batch :param cpu: Number of processors to use, -1 means all processors are used. :return: Dictionary containing the evaluation results (keys: 'hits@1', 'hits@3', 'hits@10', 'mrr') """ self.filtering = filtering start = time.time() n_batches, batches = self.dl.get_test_batches(batch_size) if cpu == 1 or cpu == 0: result = [] for batch in tqdm(batches, total=n_batches): result.append(self.evaluate_batch(batch)) elif cpu == -1: pool = mp.Pool(mp.cpu_count()) result = pool.map(self.evaluate_batch, batches) else: pool = mp.Pool(cpu) result = pool.map(self.evaluate_batch, batches) print('Evaluation took {:.3f} seconds'.format(time.time() - start)) return self.get_result(result)
def average_several_run(run_func, args, n_times=4, n_paral=2, **run_params): """Get average result after several running. Args: run_func [func]: running function like 'Classify.train_test' args [dict]: all model arguments n_times [int]: run several times for average n_paral [int]: number of parallel processes run_params [dict]: parameters for runing function Returns: max_socres [dict]: dict of the maximum scores after n_times running """ dnnnlp.verbose.config(0) assert not n_times % n_paral, ValueError( "'n_times' should be an integral multiple of 'n_paral'.") pool = mp.Pool(processes=1) check = pool.apply_async(_device_count) device_count = check.get() scores, processes = [], [] pool = mp.Pool(processes=n_paral) if args.n_gpu > 0: assert n_paral * args.n_gpu <= device_count, "Not enough GPU devices." for t in range(n_times): if args.n_gpu > 0: run_params['device_id'] = (t % n_paral) * args.n_gpu else: run_params['device_id'] = -1 processes.append(pool.apply_async(run_func, kwds=run_params.copy())) if (t + 1) % n_paral == 0: for i, p in enumerate(processes): result = p.get() scores.append(result) print() ptable = utils.display_prfacc(args.eval_metric, verbose=0) ptable.row(dict(result, **{"iter": t + 2 - len(processes) + i})) processes.clear() avg_scores = utils.average_prfacc(*scores) print() ptable = utils.display_prfacc(args.eval_metric, verbose=0) ptable.row(dict(avg_scores, **{"iter": 'AVG'})) dnnnlp.verbose.config(2) return avg_scores
def evaluate_mp(model, dataloader, Ks, num_processes, device): test_batch_size = dataloader.test_batch_size test_user_dict = dataloader.test_user_dict model.eval() model.to("cpu") user_ids = list(test_user_dict.keys()) user_ids_batches = [ user_ids[i:i + test_batch_size] for i in range(0, len(user_ids), test_batch_size) ] pool = mp.Pool(num_processes) res = pool.starmap(evaluate_batch, [(model, dataloader, batch_user, Ks) for batch_user in user_ids_batches]) pool.close() score_matrix = np.concatenate([r[0] for r in res], axis=0) metrics_dict = {k: {} for k in Ks} for k in Ks: for m in ['precision', 'recall', 'ndcg']: metrics_dict[k][m] = np.concatenate([r[1][k][m] for r in res]).mean() torch.cuda.empty_cache() model.to(device) return score_matrix, metrics_dict
def save_cache(self, cache_folder, epochs): """ Generates batch/episode indices and saves them into a torch file Args: cache_folder: string. folder where to save the cached indices epochs: int. number of epochs to generate Returns: list. Cached indices. """ logging.info("Saving cache to %s" % cache_folder) queue = multiprocessing.Queue() queue.put(1) nworkers = 32 last = nworkers + 1 seed = FLAGS.random_seed if FLAGS.random_seed is not None else get_random_initializer() with multiprocessing.Pool(nworkers, initializer=init_fn, initargs=(seed, queue, self, last)) as pool: _cache = pool.map_async(build_episode_indices, range(epochs)) for _ in tqdm(range(epochs)): queue.get(block=True) cache = _cache.get() del queue torch.save(cache, os.path.join(cache_folder, "cache.pt")) with open(os.path.join(cache_folder, 'ready'), 'w') as outfile: outfile.write('\n') self.cache = cache
def cost_fn_nav2d(theta_vals, params): global FVAL_CALLS FVAL_CALLS = FVAL_CALLS + 1 print("[cost_fn_nav2d] FVAL_CALLS: {0}".format(FVAL_CALLS)) theta = theta_vals_to_obj(theta_vals, params) n_data = params.leo.n_data_train # parallelized optimizer run pool = mp.Pool(processes=params.leo.pool_processes) optimizer_soln_fn = partial(optimizer_soln, theta, params) data_idxs = np.arange(0, n_data) result_opt = pool.map(optimizer_soln_fn, data_idxs) pool.close() pool.join() loss = 0.0 for data_idx in range(0, n_data): # x_opt, data = optimizer_soln(theta, params, data_idx) # serial run x_opt, data = result_opt[data_idx][0], result_opt[data_idx][1] x_gt = groundtruth_poses(data, params) loss = loss + loss_nav2d(x_opt, x_gt) loss = loss / n_data return loss
def get_pred_large(pan_2ch_all, vid_num, nframes_per_video=6): vid_num = len(pan_2ch_all) // nframes_per_video # 10 cpu_num = multiprocessing.cpu_count() // 2 # 32 --> 16 nprocs = min(vid_num, cpu_num) # 10 max_nframes = cpu_num * nframes_per_video nsplits = (len(pan_2ch_all) - 1) // max_nframes + 1 annotations, pan_all = [], [] for i in range(0, len(pan_2ch_all), max_nframes): print('==> Read and convert VPS output - split %d/%d' % ((i // max_nframes) + 1, nsplits)) pan_2ch_part = pan_2ch_all[i:min(i + max_nframes, len(pan_2ch_all ))] pan_2ch_split = np.array_split(pan_2ch_part, nprocs) workers = multiprocessing.Pool(processes=nprocs) processes = [] for proc_id, pan_2ch_set in enumerate(pan_2ch_split): p = workers.apply_async( self.converter_2ch_track_core, (proc_id, pan_2ch_set, color_generator)) processes.append(p) workers.close() workers.join() for p in processes: p = p.get() annotations.extend(p[0]) pan_all.extend(p[1]) pan_json = {'annotations': annotations} return pan_all, pan_json
def load_data(self, data_dir: str, num_workers: int = 1): data = list() if num_workers == -1: num_workers = mp.cpu_count() else: num_workers = num_workers chunk_paths = [ os.path.join(data_dir, chunk_file) for chunk_file in os.listdir(data_dir) if chunk_file.startswith('chunk') and chunk_file.endswith('pkl') ] if num_workers > 1: with mp.Pool(num_workers) as pool: result = pool.map(self.load_file, chunk_paths) for batches in result: data.extend(batches) else: for file_path in chunk_paths: data.extend(self.load_file(file_path)) return data
def validate(self, verbose=True): """Runs inference over the validation set periodically during training. Prints the validation loss and accuracy to their respective files. """ print('[INFO] Beginning validation.') with torch.no_grad(): n_val = len(self.X_val) pool = mp.Pool(self.params['n_processes']) inputs = [] for i in range(n_val): x = self.X_val[i, :, :].unsqueeze(0) y = self.y_val[i, :, :].unsqueeze(0) inputs.append((x, y)) results = pool.starmap(self.train_step, inputs) val_losses, val_accuracies = zip(*results) val_losses = torch.tensor(val_losses) val_accuracies = torch.tensor(val_accuracies) val_loss = torch.mean(val_losses, dim=0) val_acc = torch.mean(val_accuracies) val_loss = tuple(val_loss) print('[INFO] Validation complete.') with open(VAL_LOSS_FILE, 'a') as f: f.write('%d %f %f %f %f\n' % ((self.iter_count.item(), ) + val_loss)) f.close() with open(VAL_ACC_FILE, 'a') as f: f.write('%d %f\n' % (self.iter_count.item(), val_acc)) f.close()
def run_multiple_times(args, run_fct): cpu_count = mp.cpu_count() gpu_count = torch.cuda.device_count() # Clone arguments into list & Distribute workload across GPUs args_across_workers = [copy.deepcopy(args) for r in range(args.RUN_TIMES)] if gpu_count > 0: gpu_counter = 0 for r in range(args.RUN_TIMES): args_across_workers[r].device_id = gpu_counter gpu_counter += 1 if gpu_counter > gpu_count - 1: gpu_counter = 0 # Execute different runs/random seeds in parallel pool = mp.Pool(cpu_count - 1) df_across_runs = pool.map(run_fct, args_across_workers) pool.close() # Post process results df_concat = pd.concat(df_across_runs) by_row_index = df_concat.groupby(df_concat.index) df_means, df_stds = by_row_index.mean(), by_row_index.std() if args.SAVE: df_means.to_csv("logs/" + args.SAVE_FNAME + ".csv") return df_means, df_stds
def main(model, modeltype, gamename, ncpu): if modeltype == '2015': from model_15 import build_model elif modeltype == '2013': from model_13 import build_model config = pd.read_csv("config.csv") CONFIG = dict() CONFIG['game'] = gamename + '-v0' CONFIG['ep_max_step'] = 1500 CONFIG['eval_threshold'] = config[config['gamename']==gamename].iloc[0,1] CONFIG['l2coeff'] = 0.005 pool = mp.Pool(processes=ncpu) env = gym.make(CONFIG['game']) CONFIG['n_action'] = env.action_space.n test_times = 100 test_model =build_model(CONFIG) test_model.load_state_dict(torch.load(os.path.join(model_storage_path, model))) test_model.switch_to_vbn() test_rewards, _ = test(test_model, pool, env, test_times, CONFIG) print("test results:", np.array(test_rewards).mean()) print(str(test_rewards))
def __init__(self, args): self.args = args self.pool = multiprocessing.Pool(args.worker) env = gym.make(args.env_name) o_dim = env.observation_space.shape[0] s_dim = 128 a_dim = env.action_space.n self.population_status = [] for _ in range(args.population): individual_status = {} name = ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(8)) individual_status['name'] = name env_name = self.args.env_name individual_status['env_name'] = env_name policy_net = PolicyNet(o_dim, s_dim, a_dim) policy_net.share_memory() individual_status['policy_net'] = policy_net evaluate_net = EvaluateNet(o_dim, s_dim, a_dim) evaluate_net.share_memory() individual_status['evolution_net'] = evaluate_net steps = self.args.step_per_generation individual_status['steps'] = steps self.population_status.append(individual_status)
def pool_map(fn, data, num_workers=0, dlen=None, title=None): ''' Multithreaded map function that displays a progress bar Args: fn (function): Function to be applied to the elements in `data` data (iterable): Iterable on which the function `fn` is applied. num_workers (int): Number of threads to do the computation dlen (int): A way to supply the length of `data` separately (to display in progress bar) title (str): Title to be displayed next to the progress bar Returns: A list of results [fn(data[0]), .... fn(data[-1])] ''' result = [] if num_workers > 0: n = len(data) if hasattr(data, '__len__') else dlen desc = title if title is not None else fn.__name__ if hasattr( fn, '__name__') else None with mp.Pool(num_workers) as p: with tqdm.tqdm(total=n, desc=desc) as bar: for r in p.imap(fn, data): result.append(r) bar.update() else: result = [fn(d) for d in tqdm.tqdm(data)] return result
def _get_dataset(self, filename): dicts = self.processor._file_to_dicts(filename) #shuffle list of dicts here if we later want to have a random dev set splitted from train set if filename == self.processor.train_filename: if not self.processor.dev_filename: if self.processor.dev_split > 0.0: dicts = random.shuffle(dicts) dict_batches_to_process = int( len(dicts) / self.multiprocessing_chunk_size) num_cpus = min(mp.cpu_count(), self.max_processes, dict_batches_to_process) or 1 with ExitStack() as stack: p = stack.enter_context(mp.Pool(processes=num_cpus)) logger.info( f"Got ya {num_cpus} parallel workers to convert dict chunks to datasets (chunksize = {self.multiprocessing_chunk_size})..." ) log_ascii_workers(num_cpus, logger) results = p.imap( partial(self._multiproc, processor=self.processor), grouper(dicts, self.multiprocessing_chunk_size), chunksize=1, ) datasets = [] for dataset, tensor_names in tqdm(results, total=len(dicts) / self.multiprocessing_chunk_size): datasets.append(dataset) concat_datasets = ConcatDataset(datasets) return concat_datasets, tensor_names
def __init__(self, env, n_processes=0): if n_processes <= 0: n_processes = mp.cpu_count() init_seeds = mp.Queue() for seed in np.random.randint(Config.SEED_RANGE, size=n_processes): init_seeds.put(int(seed)) # initseeds self.pool = mp.Pool(n_processes, mp_sampler_init, (env, init_seeds)) self.rollout_f = stochastic_policy_rollout
def sample_parallel(self, env_fn, policy, min_steps, max_traj_len, deterministic=False): import torch.multiprocessing as mp from functools import partial, reduce worker = partial(self._sample, env_fn, policy, min_steps, max_traj_len, deterministic) with mp.Pool(processes=self.n_proc) as pool: # Call pool of workers, don't apply any arguments # TODO: this is a weird use of starmap, maybe Process is more suited? result = pool.starmap(worker, [() for _ in range(self.n_proc)]) def merge(buf1, buf2): buf2.states += buf1.states buf2.actions += buf1.actions buf2.rewards += buf1.rewards buf2.values += buf1.values buf2.returns += buf1.returns buf2.ep_returns += buf1.ep_returns buf2.ep_lens += buf1.ep_lens return buf2 memory = reduce(merge, result) return memory
def main(args): if args.load is False and os.path.isfile('./model/walker.pt'): while True: load = input('Are you sure you want to erase the previous training? (y/n) ') if load.lower() in ('y', 'yes', '1'): break elif load.lower() in ('n', 'no', '0'): import sys sys.exit() # create shared variables between all the processes manager = mp.Manager() # used to send the results of the net common_dict = manager.dict() # a queue of batches to be fed to the training net mem_queue = manager.Queue(1500 * mp.cpu_count()) # a queue of operations pending process_queue = manager.Queue(mp.cpu_count()-1) workers = mp.cpu_count() if args.train else 2 with mp.Pool(workers) as pool: try: print(f"Running pool with {workers} workers") pool.apply_async(gpu_thread, (args.load, mem_queue, process_queue, common_dict, 0)) if args.render: pool.apply_async(cpu_thread, (2 if not args.train else 1, mem_queue, process_queue, common_dict, 1)) for i in range(1+args.render, workers): pool.apply_async(cpu_thread, (0, mem_queue, process_queue, common_dict, i)) # Wait for children to finish pool.close() pool.join() except KeyboardInterrupt: pool.join()
def update_holdout_chromosomes(self, holdout_set): images = [] time_zero = time.time() block_size = HDF_MULTI_BLOCK_SIZE loc_chunks = list(range(0, self.__len__(), block_size)) num_processes = multiprocessing.cpu_count() print('Process with %d processors' % num_processes) with multiprocessing.Pool(num_processes) as pool: f = functools.partial(process_location, block_size=block_size, path=self.h5_path, holdout_set=holdout_set) for map_image in tqdm.tqdm(pool.imap_unordered(f, loc_chunks), total=len(loc_chunks)): images.extend(map_image) print('Saving num output locations:') print(len(images)) # Not necessary -- just for debug consistency -- remove to save time if big images.sort() print(images[:10]) print('Took %.2fs to process %d loc with %d processes' % (time.time() - time_zero, self.__len__(), num_processes)) for idx in tqdm.tqdm(images, total=len(images)): self.chromosome_holdout[idx] = True
def collect_data(top_player, players, times=100, repeat=1, log_dict=log_dict, value_policy=value_policy, p=None, queue=None, cuda=False, cpus=None): assert isinstance(players, list) if p is not None: assert isinstance(p, list) and len(p) == len(players) sum_p = sum(p) p = [i / sum_p for i in p] log_dict = os.path.join(log_dict, top_player) if not (os.path.exists(log_dict) and os.path.isdir(log_dict)): os.mkdir(log_dict) play_list = np.random.choice(len(players), times, replace=True, p=p) play_list = [(players[play_list[i]], i) for i in range(len(play_list))] with multiprocessing.Pool(cpus) as pool: pool.map( functools.partial(__collect_a_game, top_player=top_player, repeat=repeat, log_dict=log_dict, value_policy=value_policy, queue=queue, cuda=cuda), play_list)
def _get_dataset(self, filename): dicts = self.processor.file_to_dicts(filename) #shuffle list of dicts here if we later want to have a random dev set splitted from train set if self.processor.train_filename in filename: if not self.processor.dev_filename: if self.processor.dev_split > 0.0: random.shuffle(dicts) num_dicts = len(dicts) multiprocessing_chunk_size, num_cpus_used = calc_chunksize(num_dicts) with ExitStack() as stack: p = stack.enter_context(mp.Pool(processes=num_cpus_used)) logger.info( f"Got ya {num_cpus_used} parallel workers to convert {num_dicts} dictionaries " f"to pytorch datasets (chunksize = {multiprocessing_chunk_size})..." ) log_ascii_workers(num_cpus_used, logger) results = p.imap( partial(self._multiproc, processor=self.processor), grouper(dicts, multiprocessing_chunk_size), chunksize=1, ) datasets = [] with tqdm(total=len(dicts), unit=' Dicts') as pbar: for dataset, tensor_names in results: datasets.append(dataset) pbar.update(multiprocessing_chunk_size) concat_datasets = ConcatDataset(datasets) return concat_datasets, tensor_names
def _setup_multiprocess_pool(self, initargs=()): if self.number_of_processes > 1: logger.info('Starting multiprocess using ' + str(self.number_of_processes) + ' processes') assert len( mp.active_children() ) == 0, 'This should not happen. Has the cleanup() method been called ?' start = time.perf_counter() process_id = mp.Value('i', 0, lock=True) # shared between processes initargs = (process_id, initargs) self.pool = mp.Pool(processes=self.number_of_processes, maxtasksperchild=None, initializer=_initializer, initargs=initargs) logger.info('Multiprocess pool started using start method "' + mp.get_sharing_strategy() + '"' + ' in: ' + str(time.perf_counter() - start) + ' seconds') if torch.cuda.is_available( ) and self.number_of_processes > torch.cuda.device_count(): logger.warning( "You are trying to run more processes than there are available GPUs, " "it is advised to run `nvidia-cuda-mps-control` to leverage concurrent cuda executions. " "If run in background mode, don't forget to stop the daemon when done." )
def __call__(self, func, *func_args): with mp.Pool(self.n_threads) as pool: func_pickle = _FilterPickleHelper(func, *func_args) for keep, c in pool.imap_unordered(func_pickle, self.data, self.chunk_size): if keep: yield c
def get_gt(pan_gt_json_file=None, pan_gt_folder=None): if pan_gt_json_file is None: pan_gt_json_file = self.panoptic_json_file if pan_gt_folder is None: pan_gt_folder = self.panoptic_gt_folder with open(pan_gt_json_file, 'r') as f: pan_gt_json = json.load(f) files = [item['file_name'] for item in pan_gt_json['images']] if 'viper' in pan_gt_folder: files = [ _.split('/')[-1].replace('.jpg', '.png') for _ in files ] cpu_num = multiprocessing.cpu_count() files_split = np.array_split(files, cpu_num) workers = multiprocessing.Pool(processes=cpu_num) processes = [] for proc_id, files_set in enumerate(files_split): p = workers.apply_async(BaseDataset._load_image_single_core, (proc_id, files_set, pan_gt_folder)) processes.append(p) workers.close() workers.join() pan_gt_all = [] for p in processes: pan_gt_all.extend(p.get()) categories = pan_gt_json['categories'] categories = {el['id']: el for el in categories} color_gererator = IdGenerator(categories) return pan_gt_all, pan_gt_json, categories, color_gererator
def eval(self, data, output_trace_flag, max_eval_size=None): data_size = len(data) if max_eval_size is not None: data_size = min(data_size, max_eval_size) eval_data = data[:data_size] if self.processes == 1: cum_loss, cum_reward = self.batch_eval(eval_data, output_trace_flag, 0) else: cum_loss = 0 cum_reward = 0 try: mp.set_start_method('spawn') except RuntimeError: pass pool = mp.Pool(processes=self.processes) res = [] batch_per_process = data_size // self.processes if data_size % batch_per_process > 0: batch_per_process += 1 for st in range(0, data_size, batch_per_process): res += [ pool.apply_async(self.batch_eval, (eval_data[st:st + batch_per_process], output_trace_flag, st)) ] for i in range(len(res)): cur_cum_loss, cur_cum_reward = res[i].get() cum_loss += cur_cum_loss cum_reward += cur_cum_reward avg_loss = cum_loss / data_size avg_reward = cum_reward / data_size print('average pred reward: %.4f' % avg_reward) return avg_loss, avg_reward
def uniform_valid_perplexity(seqs, mask_function, num_workers=0): if num_workers is None or num_workers > 0: pool = multiprocessing.Pool(num_workers) map_fn = functools.partial(pool.imap, chunksize=8) else: map_fn = map valid_choices_fn = functools.partial(total_valid_choices, mask_function=mask_function) average_choices = np.empty(len(seqs)) average_entropy = np.empty(len(seqs)) seq_length = np.empty(len(seqs), dtype=np.int) for i, (choices, entropy, length) in enumerate( tqdm.tqdm(map_fn(valid_choices_fn, seqs), total=len(seqs))): average_choices[i] = choices average_entropy[i] = entropy seq_length[i] = length return { 'choices': average_choices, 'entropy': average_entropy, 'sequence_length': seq_length }
def start_pool(self, n_games, game_fn, device, *args): parent_conns = [] child_conns = [] parent_conns2 = [] child_conns2 = [] for i in range(n_games): parent_conn, child_conn = multiprocessing.Pipe() parent_conns.append(parent_conn) child_conns.append(child_conn) if self.net2: for i in range(n_games): parent_conn2, child_conn2 = multiprocessing.Pipe() parent_conns2.append(parent_conn2) child_conns2.append(child_conn2) pool = multiprocessing.Pool(processes=self.n_processes, initializer=np.random.seed) gpu_handler = multiprocessing.Process(target=handle_gpu, args=(copy.deepcopy(self.net), parent_conns, device)) gpu_handler2 = None if self.net2: gpu_handler2 = multiprocessing.Process(target=handle_gpu, args=(copy.deepcopy(self.net2), parent_conns2, device)) gpu_handler2.start() gpu_handler.start() if self.net2: examples = pool.map_async(self.single_game_fn, [(conn, self.game_name, self.kwargs, game_fn, child_conns2[idx], args) for idx, conn in enumerate(child_conns)]) else: examples = pool.map_async(self.single_game_fn, [(conn, self.game_name, self.kwargs, game_fn, None, args) for conn in child_conns]) return [gpu_handler, pool, examples, child_conns, parent_conns, gpu_handler2, child_conns2, parent_conns2]
def multi_process(self): print("Video processing using {} processes...".format(self.num_processes)) p = mp.Pool(self.num_processes) p.map(self.process_video_multiprocessing, range(self.num_processes)) self.combine_output_files(self.num_processes)
def main(): mp.set_start_method('spawn', force=True) with open(val_fpath, 'r') as fin: vid_list = [osp.join(vid_dir, el.strip()) for el in fin.readlines()] # vid_list = vid_list[:10] # DEBUG!!!! pool = mp.Pool(processes=8) all_preds = list(tqdm(pool.imap(compute_preds, vid_list), desc='Evaluating', total=len(vid_list))) pool.close() pool.join() preds, preds_gt, lbls, vid_fpaths = zip(*all_preds) nclasses = max(lbls) + 1 # lbls are 0 indexed preds_1hot = one_hot(preds, nclasses) preds_gt_1hot = one_hot(preds_gt, nclasses) lbls = np.array(lbls) # print(preds, preds_gt, lbls) corr = (np.array(preds) == np.array(lbls)) all_acc = all_localization_accuracies(lbls, preds_1hot) assert np.isclose(all_acc['top_1'], np.mean(corr)) print('Tracking baseline accuracies: {}'.format(all_acc)) store_preds(preds) store_failure_cases(vid_fpaths, preds, lbls) gt_acc = np.mean(np.array(np.array(preds_gt) == np.array(lbls))) all_acc_gt = all_localization_accuracies(lbls, preds_gt_1hot) assert all_acc_gt['top_1'] > 0.99 assert np.isclose(all_acc_gt['top_1'], gt_acc) print('Tracking baseline accuracy (GT end loc, this should be 1.0): {}'.format(all_acc_gt))
def run_multiple_times(args, run_fct): cpu_count = mp.cpu_count() gpu_count = torch.cuda.device_count() # Clone arguments into list & Distribute workload across GPUs args_across_workers = [copy.deepcopy(args) for r in range(args.RUN_TIMES)] if gpu_count > 0: gpu_counter = 0 for r in range(args.RUN_TIMES): args_across_workers[r].seed = r args_across_workers[r].device_id = gpu_counter gpu_counter += 1 if gpu_counter > gpu_count-1: gpu_counter = 0 # Execute different runs/random seeds in parallel pool = mp.Pool(cpu_count-1) df_across_runs = pool.map(run_fct, args_across_workers) pool.close() # Post process results df_concat = pd.concat(df_across_runs) by_row_index = df_concat.groupby(df_concat.index) df_means, df_stds = by_row_index.mean(), by_row_index.std() if args.ENV_ID == "dense-v0": sfname = "results/GRIDWORLD/" + str(args.RUN_TIMES) + "_RUNS_" + str(args.AGENT) + "_" + args.SAVE_FNAME else: sfname = "results/ATARI/" + str(args.RUN_TIMES) + "_RUNS_" + str(args.AGENT) + "_" + args.SAVE_FNAME print("Saved agents to {}".format(sfname)) df_means.to_csv(sfname) return df_means, df_stds
def __init__(self, fitness_fn: Callable, init_mean: np.ndarray=None, init_std: np.ndarray or float=0.5, pop_size: int=None, std_step: float=None, lr: float=0.3, log_time_interval: float=None, num_processes: int=None): if num_processes is None: num_processes = os.cpu_count() self.fitness_fn = fitness_fn self.cur_solution = init_mean self.lr = lr self.std = init_std self.log_time_interval = log_time_interval self.num_processes = num_processes self.dim = init_mean.size self.std_step = (3 + np.log(self.dim)) / (20 * np.sqrt(self.dim)) if std_step is None else std_step self.pop_size = int(round(4 + 3 * np.log(self.dim))) if pop_size is None else pop_size self.pop_size += self.pop_size % 2 self.R_rank = self.get_ranks(self.pop_size) self.grad_mean = np.zeros_like(self.cur_solution) self.max_R = None self.best_solution = self.cur_solution self._log_as_best = True self._last_log_time = time.time() self._pool = mp.Pool(initializer=self.process_fitness_init, initargs=(self.fitness_fn,)) \ if num_processes > 1 else None
def build_dataset(paths, num_workers, word_vocab, min_count, window, num_total_words, archive): func = partial(file_to_features, word_vocab=word_vocab, window=window, min_count=min_count, total_w=num_total_words) p = multiprocessing.Pool(num_workers, init_worker) files = [] file_counter = 0 filename = archive.format(file_counter) files.append(filename) archive_f = open(archive.format(file_counter), 'w', encoding='utf-8') archive_f.write('Source\tTarget\n') counter = 0 for x in p.imap(func, paths): if counter % 100 == 0: print(counter) counter += 1 for word, target in x: archive_f.write('{}\t{}\n'.format(word, target)) if archive_f.tell() > 5e+8: print('Changing file. Counter at {}'.format(counter)) file_counter += 1 filename = archive.format(file_counter) files.append(filename) archive_f = open(archive.format(file_counter), 'w', encoding='utf-8') archive_f.close() return files