def _report_rouge(self, predictions, references): a_lst = [] predictions = list(predictions) references = list(references) for i, p in enumerate(predictions): a_lst.append((p, references[i])) pool = Pool(24) rouge_scores = {"r1": [], "r2": [], "rl": []} for d in tqdm(pool.imap(_multi_rg, a_lst), total=len(a_lst)): if d is not None: rouge_scores["r1"].append(d[0]) rouge_scores["r2"].append(d[1]) rouge_scores["rl"].append(d[2]) pool.close() pool.join() r1 = np.mean(rouge_scores["r1"]) r2 = np.mean(rouge_scores["r2"]) rl = np.mean(rouge_scores["rl"]) if len(self.args.log_folds) > 0: with open(self.args.log_folds, mode='a') as f: f.write("{:.4f}\t{:.4f}\t{:.4f}".format(r1 / 100, r2 / 100, rl / 100)) f.write('\n') logger.info("Metric\tScore\t95% CI") logger.info("ROUGE-1\t{:.2f}\t({:.2f},{:.2f})".format(r1 * 100, 0, 0)) logger.info("ROUGE-2\t{:.2f}\t({:.2f},{:.2f})".format(r2 * 100, 0, 0)) logger.info("ROUGE-L\t{:.2f}\t({:.2f},{:.2f})".format(rl * 100, 0, 0)) logger.info("Data path: %s" % self.args.bert_data_path) logger.info("Model path: %s" % self.args.model_path) return r1, r2, rl
def async_build_examples( self, data_type: str, dials: List[Tuple[str, dict]]) -> Tuple[list, list]: """Use multiprocessing to process raw dialogue data. Args: data_type: train, dev or test dials: raw dialogues data Returns: new examples by all processes """ neg_examples = Manager().list() pos_examples = Manager().list() dials4single_process = (len(dials) - 1) // self.config['num_processes'] + 1 print(f'Single process have {dials4single_process} dials ...') pool = Pool(self.config['num_processes']) for i in range(self.config['num_processes']): pool.apply_async(func=self.iter_dials, args=(dials[dials4single_process * i:dials4single_process * (i + 1)], data_type, pos_examples, neg_examples, i)) pool.close() pool.join() pos_examples = list(pos_examples) neg_examples = list(neg_examples) return neg_examples, pos_examples
def step(self, items_seq, items_ori, items_batch=None, boxes_batch=None): if (items_batch != None) & (boxes_batch != None): self.items_batch = items_batch self.boxes_batch = boxes_batch self.batch_indx = list(range(self.BATCH_SIZE)) self.expected_items_n = [self.ITEMS_SEQ_LN] * BATCH_SIZE self.all_outs = {i: [] for i in range(self.BATCH_SIZE)} self.current_level = 0 self.items_batch_alligned = None self.boxes_batch_alligned = None items_seq_ = torch.LongTensor(items_seq).transpose(1, 0).expand( self.INPUT_SIZE, self.ITEMS_SEQ_LN, self.BATCH_SIZE).transpose(2, 0) items_ori_ = items_ori[torch.arange(self.BATCH_SIZE).expand( self.ITEMS_SEQ_LN, self.BATCH_SIZE).transpose(1, 0), torch.LongTensor(items_seq). expand(self.BATCH_SIZE, self.ITEMS_SEQ_LN)] self.items_batch_alligned = self.items_batch[[ self.base_indx_items, items_seq_, items_ori_ ]] lookup_sm = self.boxes_batch.expand( self.ITEMS_SEQ_LN, self.BATCH_SIZE, self.ITEMS_SEQ_LN, self.INPUT_SIZE).transpose( 1, 0) - self.items_batch_alligned.unsqueeze(2) validities = (lookup_sm >= 0).all(3).any(2).tolist() new_seq = [] for i, j in zip(items_seq, validities): new_seq.append([i[k] for k in range(len(i)) if j[k] == True]) self.batch_indx = [i for i in self.batch_indx if len(new_seq[i]) > 0] items_seq = [i for i in new_seq if len(i) > 0] zp = list( zip(self.batch_indx, self.items_batch[self.batch_indx], self.boxes_batch[self.batch_indx], items_seq, items_ori[self.batch_indx])) p = Pool(10) out = p.map(self.target_func, zp) p.close() p.join() out = [pickle.loads(i) for i in out] out_series = pd.Series(out) _ = out_series.apply(lambda x: self.dict_update(x)) # out = [i for i in out if i[1] < i[2]] self.batch_indx = [i[0] for i in out] self.current_level += 1 items_seq = [i[5] for i in out] all_rewards = [i[-1] * i[-2] for i in out] # filled_items_indx = {i:[i[2] for i in j] for i,j in self.all_outs.items() if len(j) > 0} # filled_items_HUs = {i:[i[7] for i in j if len(i[7]) > 0] for i,j in self.all_outs.items()} # all_rewards = [self.calc_reward(self.items_batch[i],i,filled_items_indx,filled_items_HUs) for i in range(self.BATCH_SIZE)] return all_rewards
def run_main(): deck_lists = list(map(int,args.decklists.split(","))) if args.decklists is not None else None if deck_lists is None: deck_lists = list(deck_id_2_name.keys()) else: assert all(key in deck_id_2_name for key in deck_lists) if deck_lists == [0, 1, 4, 5, 10, 12]: deck_lists = [0, 1, 4, 12, 5, 10] mylogger.info("deck_lists:{}".format(deck_lists)) D = [Deck() for i in range(len(deck_lists))] deck_index = 0 # sorted_keys = sorted(list(deck_id_2_name.keys())) # for i in sorted_keys: # if i not in deck_lists: # continue for i in deck_lists: mylogger.info("{}(deck_id:{}):{}".format(deck_index, i, key_2_tsv_name[i])) D[deck_index] = tsv_to_deck(key_2_tsv_name[i][0]) D[deck_index].set_leader_class(key_2_tsv_name[i][1]) deck_index += 1 Results = {} list_range = range(len(deck_lists)) #print(list(itertools.product(list_range,list_range))) Player1 = Player(9, True, policy=New_Dual_NN_Non_Rollout_OM_ISMCTSPolicy(model_name=args.model_name), mulligan=Min_cost_mulligan_policy()) if args.opponent is not None: if args.model_name is not None: if args.opponent == "Greedy": Player2 = Player(9, True, policy=NN_GreedyPolicy(model_name=args.model_name), mulligan=Min_cost_mulligan_policy()) elif args.opponent == "MCTS": Player2 = Player(9, True, policy=New_Dual_NN_Non_Rollout_OM_ISMCTSPolicy(model_name=args.model_name), mulligan=Min_cost_mulligan_policy()) else: Player2 = Player(9, True, policy=Opponent_Modeling_MCTSPolicy(), mulligan=Min_cost_mulligan_policy()) else: Player2 = Player(9, True, policy=AggroPolicy(), mulligan=Min_cost_mulligan_policy()) Player1.name = "Alice" Player2.name = "Bob" iteration = int(args.iteration) if args.iteration is not None else 10 deck_list_len = len(deck_lists) iter_data = [(deck_list_len*i+j,Player1, Player2,(i,j),(deck_lists[i],deck_lists[j]),iteration) for i,j in itertools.product(list_range,list_range)] pool = Pool(3) # 最大プロセス数:8 # memory = pool.map(preparation, iter_data) result = pool.map(multi_battle, iter_data) #result = list(tqdm(result, total=len(list_range)**2)) pool.close() # add this. pool.terminate() # add this. for data in result: Results[data[0]] = data[1] print(Results)
def final_experiments(): for i in range(2, 5): x_train, train, x_valid, valid, x_test, test, name = get_dataset(i) print("Running for", name, "dataset") x_train, e_train, t_train, x_valid, e_valid, t_valid, x_test, e_test, t_test = scale_data_to_torch( x_train, train, x_valid, valid, x_test, test) print("Dataset loaded and scaled") risk_set, risk_set_valid, risk_set_test = compute_risk_set( t_train, t_valid, t_test) print("Risk set computed") data_dict = { "x_train": x_train, "e_train": e_train, "t_train": t_train, "x_valid": x_valid, "e_valid": e_valid, "t_valid": t_valid, "x_test": x_test, "e_test": e_test, "t_test": t_test, "risk_set": risk_set, "risk_set_valid": risk_set_valid, "risk_set_test": risk_set_test } n_in = x_train.shape[1] linear_models = [2, 5, 10, 12] learning_rates = [1e-4, 1e-3] layer_sizes = [[n_in], [n_in, n_in], [n_in, n_in, n_in], [n_in, 20, 15]] data = [data_dict] hyperparams = [(linear_model, learning_rate, layer_size, seed, d) for layer_size in layer_sizes for learning_rate in learning_rates for linear_model in linear_models for seed in range(3) for d in data] print("Hyperparams initialized") p = Pool(50) print("Pool created") output = p.map(run_experiment, hyperparams) p.close() p.join() print("Models trained. Writing to file") filename = name + "_results.pkl" f = open(filename, "wb") pkl.dump(output, f) f.flush() f.close() print(name, "done") print("")
def mul_infer(args): setuplogging() set_start_method('spawn', force=True) root_data_dir = os.path.join(args.root_data_dir, 'testdata') checkpoint = torch.load(os.path.join(args.model_dir, args.load_ckpt_name), map_location=torch.device('cpu')) subcategory_dict = checkpoint['subcategory_dict'] category_dict = checkpoint['category_dict'] logging.info('load ckpt: {}'.format(args.load_ckpt_name)) check_preprocess_result(args, root_data_dir, mode='test', category=category_dict, subcategory=subcategory_dict) logging.info('finish the preprocess of docfeatures') docid_features, category_dict, subcategory_dict = read_news( args, root_data_dir) news_index = {} news_feature = [] cnt = 0 for k, v in docid_features.items(): news_index[k] = cnt news_feature.append(v) cnt += 1 news_num = len(news_feature) logging.info('news_num:{}'.format(news_num)) pool = Pool(processes=args.world_size) results = [] sigle_size = news_num // args.world_size for rank in range(args.world_size): start = sigle_size * rank end = sigle_size * (rank + 1) if rank == args.world_size - 1: end = news_num local_features = news_feature[start:end] result = pool.apply_async(sigle_process_infer, args=(rank, local_features, checkpoint, args)) results.append(result) pool.close() pool.join() results = [x.get() for x in results] news_vecs = np.concatenate(results, 0) return news_index, news_vecs
def runInParallel(args): cur_args = [] for i in range(args.r): cur_args.append({'seed': base_seed + i}) print('total to run', cur_args, 'nProc', args.nproc) if args.nproc > 1: pool = Pool(processes=int(args.nproc)) pool.map(single_run_algorithm, cur_args) pool.close() pool.join() else: for cur_arg in cur_args: single_run_algorithm(cur_arg)
def forward( self, images, ): parameters = [] num_workers = self.num_workers pool = Pool(num_workers) for bx in range(len(images)): bx_params = [bx, images[bx], True] parameters.append(bx_params, ) predictions = pool.map(worker_distance_transform, parameters) predictions = torch.stack(predictions) pool.close() pool.join() return predictions
def propagate(nnf, feat_A, feat_AP, feat_B, feat_BP, patch_size, iters=2, rand_search_radius=200): print("\tpatch_size:{}; num_iters:{}; rand_search_radius:{}".format(patch_size, iters, rand_search_radius)) nnd = np.zeros(nnf.shape[:2]) A_size = feat_A.shape[:2] B_size = feat_B.shape[:2] for ay in range(A_size[0]): for ax in range(A_size[1]): by, bx = nnf[ay, ax] nnd[ay, ax] = cal_dist(ay, ax, by, bx, feat_A, feat_AP, feat_B, feat_BP, A_size, B_size, patch_size) manager = mp.Manager() q = manager.Queue(A_size[1] * A_size[0]) cpus = min(mp.cpu_count(), A_size[0] // 20 + 1) for i in range(iters): p = Pool(cpus) ay_start = 0 while ay_start < A_size[0]: ax_start = 0 while ax_start < A_size[1]: p.apply_async(pixelmatch, args=(q, ax_start, ay_start, cpus, nnf, nnd, A_size, B_size, feat_A, feat_AP, feat_B, feat_BP, patch_size, rand_search_radius,)) ax_start += A_size[1] // cpus + 1 ay_start += A_size[0] // cpus + 1 p.close() p.join() while not q.empty(): ax, ay, xbest, ybest, dbest = q.get() nnf[ay, ax] = np.array([ybest, xbest]) nnd[ay, ax] = dbest return nnf, nnd
def create_episodes( self, n_episodes: int, n_processes: int, mcts_samples: int, mcts_temp: float, mcts_cpuct: int, mcts_observation_weight: float, model: Model, ) -> List[Tuple[List[ObservationType], List[np.ndarray], int, Summary]]: pool = Pool(n_processes) res = pool.starmap( self._generator.perform_episode, [[mcts_samples, mcts_temp, mcts_cpuct, mcts_observation_weight, model]] * n_episodes, ) pool.close() pool.terminate() pool.join() return res
def save(self): try: mp.set_start_method('spawn') except RuntimeError: pass pool = Pool(processes=4) self.labels = [] item = 0 for label_name in label_map.keys(): images = self._listdir(os.path.join(self.path, label_name)) for i in range(len(images)): rows, cols = [], [] files = glob.glob( os.path.join(self.path, label_name, images[i], str(self.magnify), '*.jpeg')) for file in files: filename = os.path.basename(file) nums = filename.split('_') row, col = int(nums[0]), int(nums[1]) rows.append(row) cols.append(col) num_row = max(rows) - min(rows) + 1 num_col = max(cols) - min(cols) + 1 patches = np.chararray((num_row, num_col), itemsize=1024) for file in files: filename = os.path.basename(file) nums = filename.split('_') row, col = int(nums[0]), int(nums[1]) patches[row - min(rows), col - min(cols)] = file self.labels.append(label_map[label_name]) # Save feature vector pool.apply_async(self.doit, args=(item, patches, num_row, num_col), error_callback=self.print_error) item += 1 # Save labels torch.save(self.labels, self._get_label_file()) pool.close() pool.join() print('done')
def fit(self, train_data, train_label): a = time() pool = Pool(16) results = [] for i in range(self.round): print(i) bag_index = np.random.choice(np.arange(train_label.shape[0]), train_label.shape[0]) data = train_data[bag_index] label = train_label[bag_index] results.append( pool.apply_async(self.parallel_fit, args=(self.clf[i], data, label))) pool.close() pool.join() for i, result in enumerate(results): self.clf[i] = result.get() print('Class %d cost %.1f seconds' % (i, time() - a))
class ParallelSimulator(Simulator): def __init__(self, simulator, workers=2): super(ParallelSimulator, self).__init__() self.pool = Pool(processes=workers) self.simulator = simulator self.workers = workers def _prepare_arguments(self, inputs): arguments = [] chunks = inputs.shape[0] // self.workers if chunks == 0: chunks = 1 chunks = inputs.split(chunks, dim=0) for chunk in chunks: a = (self.simulator, chunk) arguments.append(a) return arguments def forward(self, inputs): arguments = self._prepare_arguments(inputs) outputs = self.pool.map(self._simulate, arguments) outputs = torch.cat(outputs, dim=0) return outputs def terminate(self): self.pool.close() del self.pool self.pool = None self.simulator.terminate() @staticmethod def _simulate(arguments): simulator, inputs = arguments return simulator(inputs)
class ParallelApproximateBayesianComputation: def __init__(self, abc, workers=2): super(ParallelApproximateBayesianComputation, self).__init__() self.abc = abc self.pool = Pool(processes=workers) self.workers = workers def _prepare_arguments(self, observation, num_samples): arguments = [] inputs = torch.arange(num_samples) num_chunks = num_samples // self.workers if num_chunks == 0: num_chunks = 1 chunks = inputs.split(num_chunks, dim=0) for chunk in chunks: a = (self.abc, observation, len(chunk)) arguments.append(a) return arguments def sample(self, observation, num_samples=1): arguments = self._prepare_arguments(observation, num_samples) outputs = self.pool.map(self._sample, arguments) outputs = torch.cat(outputs, dim=0) return outputs def __del__(self): self.pool.close() del self.pool self.pool = None @staticmethod def _sample(arguments): abc, observation, n = arguments return abc.sample(observation, num_samples=n)
def start_multiprocessing(self, embeddings, naming_list, naming_dict, dataset): n_threads = 4 chunked = chunkIt(naming_list, n_threads) # multiprocess gridsearch and have a seperate thread for the progress bar. pool1 = Pool(processes=n_threads) m = Manager() q = m.Queue() p = Process(target=progressBar, args=(len(naming_list), q,)) p.start() results = pool1.starmap(self.determine_triplets, zip(n_threads * [q], chunked, n_threads * [naming_list], n_threads * [embeddings], n_threads * [naming_dict], n_threads * [dataset])) final_results = [] for r in results: final_results += r p.join() pool1.close() return final_results
class Environment(ParamTree): """Environment for executing training and post training evaluations. Takes care of process pool, reporting, and experiment parameters. Parameters ---------- params : dict or str Dictionary containing the parameters or a path to a parameters spec file. """ from .util import (default_params, setup_params, open_data, store_gen, store_gen_metrics, load_gen_metrics, stored_populations, stored_indiv_measurements, store_hof, load_hof, load_pop, load_indiv_measurements, env_path) def __init__(self, params): """Initialize an environment for training or post training analysis.""" super().__init__() self.setup_params(params) self.metrics = list() self.pool = None self.data_file = None self.hall_of_fame = list() # choose task self.task = select_task(self['task', 'name']) # choose adequate type of individuals if self['config', 'backend'].lower() == 'torch': import wann_genetic.individual.torch as backend else: import wann_genetic.individual.numpy as backend if self.task.is_recurrent: self.ind_class = backend.RecurrentIndividual else: self.ind_class = backend.Individual # only use enabled activations functions available_funcs = self.ind_class.Phenotype.available_act_functions enabled_acts = self['population', 'enabled_activation_funcs'] if self['population', 'enabled_activation_funcs'] != 'all': self.ind_class.Phenotype.enabled_act_functions = [ available_funcs[i] for i in enabled_acts ] def seed(self, seed): """Set seed to `seed` or from parameters. Parameters ---------- seed : int Seed to use. """ np.random.seed(seed) @property def elite_size(self): """Size of the elite (:math:`population\\ size * elite\\ ratio`).""" return int( np.floor(self['selection', 'elite_ratio'] * self['population', 'size'])) def sample_weights(self, n=None): if n is None: n = self['sampling']['num_weights_per_iteration'] dist = self['sampling', 'distribution'].lower() if dist == 'one': w = 1 elif dist == 'uniform': lower = self['sampling', 'lower_bound'] upper = self['sampling', 'upper_bound'] assert lower is not None and upper is not None w = np.random.uniform(lower, upper, size=n) elif dist == 'linspace': lower = self['sampling', 'lower_bound'] upper = self['sampling', 'upper_bound'] assert lower is not None and upper is not None w = np.linspace(lower, upper, num=n) elif dist == 'lognormal': mu = self['sampling', 'mean'] sigma = self['sampling', 'sigma'] assert mu is not None and sigma is not None w = np.random.lognormal(mu, sigma, size=n) elif dist == 'normal': mu = self['sampling', 'mean'] sigma = self['sampling', 'sigma'] assert mu is not None and sigma is not None w = np.random.normal(mu, sigma, size=n) else: raise RuntimeError(f'Distribution {dist} not implemented.') self['sampling', 'current_weight'] = w return w def setup_pool(self, n=None): """Setup process pool.""" if n is None: n = self['config', 'num_workers'] if n == 1: self.pool = None else: if self['config', 'backend'].lower() == 'torch': logging.info('Using torch multiprocessing') from torch.multiprocessing import Pool self.pool = Pool(n) else: logging.info('Using usual multiprocessing') from multiprocessing import Pool self.pool = Pool(n) def pool_map(self, func, iter): if self.pool is None: return map(func, iter) else: return self.pool.imap(func, iter) def setup_optimization(self): """Setup everything that is required for training (eg. loading test samples). """ log_path = self.env_path(self['storage', 'log_filename']) logging.info(f"Check log ('{log_path}') for details.") logger = logging.getLogger() fh = logging.FileHandler(log_path) fh.setFormatter( logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s')) logger.addHandler(fh) if self['config', 'debug']: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) logging.info(f"Package version {get_version()}") p = os.path.abspath(self['experiment_path']) logging.info(f'Saving data at {p}.') logging.debug('Loading training dataset') self.task.load_training(env=self) # log used parameters params_toml = toml.dumps(self.params) logging.debug(f"Running experiments with the following parameters:\n" f"{params_toml}") with open(self.env_path('params.toml'), 'w') as f: params = dict(self.params) # mark stored params as part of a report params['is_report'] = True toml.dump(params, f) def run(self): """Run optization and post optimization (if enabled).""" # set up logging, write params self.setup_optimization() # set up pool of workers self.setup_pool() with self.open_data('w'): # run optimization self.optimize() if self['postopt', 'run_postopt']: with self.open_data('r'): # evaluate individuals in hall of fame self.post_optimization() if self.pool is not None: logging.info('Closing pool') self.pool.close() def optimize(self): logging.info("Starting evolutionary algorithm") ts = TimeStore() alg = GeneticAlgorithm(self) first_generation = True self.seed(self['sampling', 'seed']) ts.start() for gen in np.arange(self['population', 'num_generations']) + 1: ts.start() pop = alg.ask() seed = self['sampling', 'post_init_seed'] if (first_generation and not isinstance(seed, bool)): self.seed(seed) # evaluate indivs weights = self.sample_weights() logging.debug(f'Sampled weight {weights}') make_measurements(self, pop, weights=weights) obj_values = np.array( [get_objective_values(ind, self.objectives) for ind in pop]) alg.tell(obj_values) logging.debug('Updating hall of fame') self.hall_of_fame = update_hall_of_fame(self, pop) ts.stop() avg = (ts.total / gen) expected_time = (self['population', 'num_generations'] - gen) * avg logging.info( f'Completed generation {gen}; {ts.dt:.02}s elapsed, {avg:.02}s avg, {ts.total:.02}s total. ' f'Expected time remaining: {expected_time:.02}s') self.store_data(gen, pop, dt=ts.dt) self.last_population = pop self.store_hof() def post_optimization(self): r = Report(self).run_evaluations( # run evaluations on test data num_weights=self['postopt', 'num_weights'], num_samples=self['postopt', 'num_samples'] # all ) if self['postopt', 'compile_report']: r.compile() # plot metrics, derive stats else: r.compile_stats() # at least derive and store stats def store_data(self, gen, pop, dt=-1): gen_metrics, indiv_metrics = self.population_metrics( gen=gen, population=pop, return_indiv_measurements=True, dt=dt) metric, metric_sign = self.hof_metric p = ("MAX" if metric_sign > 0 else "MIN") metric_value = gen_metrics[f"{p}:{metric}"] logging.info(f"#{gen} {p}:{metric}: {metric_value:.2}") self.metrics.append(gen_metrics) commit_freq = self['storage', 'commit_elite_freq'] if (commit_freq > 0 and gen % commit_freq == 0): self.store_gen(gen, population=pop[:self.elite_size], indiv_metrics=indiv_metrics) commit_freq = self['storage', 'commit_metrics_freq'] if (commit_freq > 0 and gen % commit_freq == 0): self.store_gen_metrics(pd.DataFrame(data=self.metrics)) def population_metrics(self, population, gen=None, dt=-1, return_indiv_measurements=False): """Get available measurements for all individuals in the population and calculate statistical key metrics. The statistical key metrics include: `Q_{0, 4}` The quartiles :math:`\\{1, 2, 3\\}` as well as the minimum and maximum :math:`(0,4)`. `MEAN`, `STD` Mean and standard deviation. `MIN`, `MEDIAN`, `MAX` Equal to `Q_0`, `Q_2`, `Q_3` Parameters ---------- population : [wann_genetic.Individual] List of individuals that constitute the population. gen : int Current generation index (required for caluclating the individuals age). return_indiv_measurements : bool, optional Whether to return the individual measurements as well. Returns ------- dict Dictionary of the produced measurements (cross product of key metrics and a list of individual measurements). """ if gen is None: gen = self['population', 'num_generations'] rows = list() for ind in population: data = ind.metadata(current_gen=gen) data.update(ind.measurements) rows.append(data) indiv_measurements = pd.DataFrame(data=rows) metrics = dict( num_unique_individuals=len(set(population)), num_individuals=len(population), delta_time=dt, # number of inds without edges num_no_edge_inds=np.sum( indiv_measurements['n_enabled_edges'] == 0), # number of inds without hidden nodes num_no_hidden_inds=np.sum(indiv_measurements['n_hidden'] == 0), # individual with the most occurences biggest_ind=max([population.count(i) for i in set(population)]), ) for name, values in indiv_measurements.items(): metrics[f'Q_0:{name}'] = metrics[f'MIN:{name}'] = values.min() metrics[f'Q_1:{name}'] = np.quantile(values, .25) metrics[f'Q_2:{name}'] = metrics[f'MEDIAN:{name}'] = values.median( ) metrics[f'Q_3:{name}'] = np.quantile(values, .75) metrics[f'Q_4:{name}'] = metrics[f'MAX:{name}'] = values.max() metrics[f'MEAN:{name}'] = values.mean() metrics[f'STD:{name}'] = values.std() if return_indiv_measurements: return metrics, indiv_measurements else: return metrics
class MetricTester: """Class used for efficiently run alot of parametrized tests in ddp mode. Makes sure that ddp is only setup once and that pool of processes are used for all tests. All tests should subclass from this and implement a new method called `test_metric_name` where the method `self.run_metric_test` is called inside. """ atol = 1e-8 def setup_class(self): """Setup the metric class. This will spawn the pool of workers that are used for metric testing and setup_ddp """ self.poolSize = NUM_PROCESSES self.pool = Pool(processes=self.poolSize) self.pool.starmap(setup_ddp, [(rank, self.poolSize) for rank in range(self.poolSize)]) def teardown_class(self): """ Close pool of workers """ self.pool.close() self.pool.join() def run_functional_metric_test( self, preds: Tensor, target: Tensor, metric_functional: Callable, sk_metric: Callable, metric_args: dict = None, fragment_kwargs: bool = False, **kwargs_update, ): """Main method that should be used for testing functions. Call this inside testing method Args: preds: torch tensor with predictions target: torch tensor with targets metric_functional: lightning metric class that should be tested sk_metric: callable function that is used for comparison metric_args: dict with additional arguments used for class initialization fragment_kwargs: whether tensors in kwargs should be divided as `preds` and `target` among processes kwargs_update: Additional keyword arguments that will be passed with preds and target when running update on the metric. """ device = 'cuda' if (torch.cuda.is_available() and torch.cuda.device_count() > 0) else 'cpu' _functional_test( preds=preds, target=target, metric_functional=metric_functional, sk_metric=sk_metric, metric_args=metric_args, atol=self.atol, device=device, fragment_kwargs=fragment_kwargs, **kwargs_update, ) def run_class_metric_test( self, ddp: bool, preds: Tensor, target: Tensor, metric_class: Metric, sk_metric: Callable, dist_sync_on_step: bool, metric_args: dict = None, check_dist_sync_on_step: bool = True, check_batch: bool = True, fragment_kwargs: bool = False, check_scriptable: bool = True, **kwargs_update, ): """Main method that should be used for testing class. Call this inside testing methods. Args: ddp: bool, if running in ddp mode or not preds: torch tensor with predictions target: torch tensor with targets metric_class: lightning metric class that should be tested sk_metric: callable function that is used for comparison dist_sync_on_step: bool, if true will synchronize metric state across processes at each ``forward()`` metric_args: dict with additional arguments used for class initialization check_dist_sync_on_step: bool, if true will check if the metric is also correctly calculated per batch per device (and not just at the end) check_batch: bool, if true will check if the metric is also correctly calculated across devices for each batch (and not just at the end) fragment_kwargs: whether tensors in kwargs should be divided as `preds` and `target` among processes kwargs_update: Additional keyword arguments that will be passed with preds and target when running update on the metric. """ if not metric_args: metric_args = {} if ddp: if sys.platform == "win32": pytest.skip("DDP not supported on windows") self.pool.starmap( partial( _class_test, preds=preds, target=target, metric_class=metric_class, sk_metric=sk_metric, dist_sync_on_step=dist_sync_on_step, metric_args=metric_args, check_dist_sync_on_step=check_dist_sync_on_step, check_batch=check_batch, atol=self.atol, fragment_kwargs=fragment_kwargs, check_scriptable=check_scriptable, **kwargs_update, ), [(rank, self.poolSize) for rank in range(self.poolSize)], ) else: device = 'cuda' if (torch.cuda.is_available() and torch.cuda.device_count() > 0) else 'cpu' _class_test( rank=0, worldsize=1, preds=preds, target=target, metric_class=metric_class, sk_metric=sk_metric, dist_sync_on_step=dist_sync_on_step, metric_args=metric_args, check_dist_sync_on_step=check_dist_sync_on_step, check_batch=check_batch, atol=self.atol, device=device, fragment_kwargs=fragment_kwargs, check_scriptable=check_scriptable, **kwargs_update, ) def run_precision_test_cpu( self, preds: Tensor, target: Tensor, metric_module: Metric, metric_functional: Callable, metric_args: dict = None, **kwargs_update, ): """Test if a metric can be used with half precision tensors on cpu Args: preds: torch tensor with predictions target: torch tensor with targets metric_module: the metric module to test metric_functional: the metric functional to test metric_args: dict with additional arguments used for class initialization kwargs_update: Additional keyword arguments that will be passed with preds and target when running update on the metric. """ metric_args = metric_args or {} _assert_half_support(metric_module(**metric_args), metric_functional, preds, target, device="cpu", **kwargs_update) def run_precision_test_gpu( self, preds: Tensor, target: Tensor, metric_module: Metric, metric_functional: Callable, metric_args: dict = None, **kwargs_update, ): """Test if a metric can be used with half precision tensors on gpu Args: preds: torch tensor with predictions target: torch tensor with targets metric_module: the metric module to test metric_functional: the metric functional to test metric_args: dict with additional arguments used for class initialization kwargs_update: Additional keyword arguments that will be passed with preds and target when running update on the metric. """ metric_args = metric_args or {} _assert_half_support(metric_module(**metric_args), metric_functional, preds, target, device="cuda", **kwargs_update) def run_differentiability_test( self, preds: Tensor, target: Tensor, metric_module: Metric, metric_functional: Callable, metric_args: dict = None, ): """Test if a metric is differentiable or not Args: preds: torch tensor with predictions target: torch tensor with targets metric_module: the metric module to test metric_args: dict with additional arguments used for class initialization """ metric_args = metric_args or {} # only floating point tensors can require grad metric = metric_module(**metric_args) if preds.is_floating_point(): preds.requires_grad = True out = metric(preds[0], target[0]) # metrics can return list of values if isinstance(out, list): assert all(metric.is_differentiable == o.requires_grad for o in out) else: assert metric.is_differentiable == out.requires_grad if metric.is_differentiable: # check for numerical correctness assert torch.autograd.gradcheck( partial(metric_functional, **metric_args), (preds[0].double(), target[0])) # reset as else it will carry over to other tests preds.requires_grad = False
def main(): opt = parse_args() device = 'cuda' if torch.cuda.is_available() else 'cpu' random.seed(opt.seed) numpy.random.seed(opt.seed) torch.manual_seed(opt.seed) data_path = 'traffic-data/state-action-cost/data_i80_v0' dataloader = DataLoader(None, opt, 'i80') ( forward_model, value_function, policy_network_il, policy_network_mper, data_stats ) = load_models(opt, data_path, device) splits = torch.load(path.join(data_path, 'splits.pth')) if opt.u_reg > 0.0: forward_model.train() forward_model.opt.u_hinge = opt.u_hinge if hasattr(forward_model, 'value_function'): forward_model.value_function.train() planning.estimate_uncertainty_stats( forward_model, dataloader, n_batches=50, npred=opt.npred) gym.envs.registration.register( id='I-80-v1', entry_point='map_i80_ctrl:ControlledI80', kwargs=dict( fps=10, nb_states=opt.ncond, display=False, delta_t=0.1, store_simulator_video=opt.save_sim_video, show_frame_count=False, ) ) print('Building the environment (loading data, if any)') env_names = { 'i80': 'I-80-v1', } env = gym.make(env_names[opt.map]) plan_file = build_plan_file_name(opt) print(f'[saving to {path.join(opt.save_dir, plan_file)}]') # different performance metrics time_travelled, distance_travelled, road_completed = [], [], [] collided, offscreen = [], [] # values saved for later inspection action_sequences, state_sequences, cost_sequences = [], [], [] image_sequences = [] writer = utils.create_tensorboard_writer(opt) n_test = len(splits['test_indx']) set_start_method('spawn') pool = Pool(opt.num_processes) async_results = [] time_started = time.time() total_images = 0 for j in range(n_test): # print(type(splits), len(splits['test_indx']), splits['test_indx'].shape, list(dataloader.car_sizes.keys())[0:5], list(dataloader.car_sizes[list(dataloader.car_sizes.keys())[0]].keys())[0:5],dataloader.car_sizes[list(dataloader.car_sizes.keys())[0]][list(dataloader.car_sizes[list(dataloader.car_sizes.keys())[0]].keys())[0]]) car_path = dataloader.ids[splits['test_indx'][j]] timeslot, car_id = utils.parse_car_path(car_path) car_sizes = torch.tensor( dataloader.car_sizes[sorted(list(dataloader.car_sizes.keys()))[ timeslot]][car_id] )[None, :] async_results.append( pool.apply_async( process_one_episode, ( opt, env, car_path, forward_model, policy_network_il, data_stats, plan_file, j, car_sizes ) ) ) for j in range(n_test): simulation_result = async_results[j].get() time_travelled.append(simulation_result.time_travelled) distance_travelled.append(simulation_result.distance_travelled) road_completed.append(simulation_result.road_completed) action_sequences.append(torch.from_numpy( simulation_result.action_sequence)) state_sequences.append(torch.from_numpy( simulation_result.state_sequence)) # image_sequences.append(torch.from_numpy( # simulation_result.image_sequence)) cost_sequences.append(simulation_result.cost_sequence) total_images += time_travelled[-1] collided.append(simulation_result.has_collided) offscreen.append(simulation_result.off_screen) log_string = ' | '.join(( f'ep: {j + 1:3d}/{n_test}', f'time: {time_travelled[-1]}', f'distance: {distance_travelled[-1]:.0f}', f'success: {road_completed[-1]:d}', f'mean time: {torch.Tensor(time_travelled).mean():.0f}', f'mean distance: {torch.Tensor(distance_travelled).mean():.0f}', f'mean success: {torch.Tensor(road_completed).mean():.3f}', )) print(log_string) utils.log(path.join(opt.save_dir, f'{plan_file}.log'), log_string) if writer is not None: # writer.add_video( # f'Video/success={simulation_result.road_completed:d}_{j}', # simulation_result.images.unsqueeze(0), # j # ) writer.add_scalar('ByEpisode/Success', simulation_result.road_completed, j) writer.add_scalar('ByEpisode/Collision', simulation_result.has_collided, j) writer.add_scalar('ByEpisode/OffScreen', simulation_result.off_screen, j) writer.add_scalar('ByEpisode/Distance', simulation_result.distance_travelled, j) pool.close() pool.join() diff_time = time.time() - time_started print('avg time travelled per second is', total_images / diff_time) torch.save({"road_completed" : road_completed, "collided": collided, "offscreen": offscreen}, path.join(opt.save_dir, f'{plan_file}.others')) torch.save(action_sequences, path.join( opt.save_dir, f'{plan_file}.actions')) torch.save(state_sequences, path.join(opt.save_dir, f'{plan_file}.states')) # torch.save(image_sequences, path.join(opt.save_dir, f'{plan_file}.images')) torch.save(cost_sequences, path.join(opt.save_dir, f'{plan_file}.costs')) if writer is not None: writer.close()
def eval_(self, X, *args): """ Evaluate a number of DARTS architecture in parallel. X should be a list of Genotypes defined by DARTS API. """ from math import ceil n_parallel = min(len(X), self.n_gpu) res = [] diag_stats = [] if n_parallel == 0: raise ValueError("No GPUs available!") elif n_parallel == 1: for i, genotype in enumerate(X): t = DARTSTrainer(self.data_path, self.save_path, genotype, self.dataset, cutout=self.cutout, auxiliary_tower=self.auxiliary, epochs=self.epochs, eval_policy=self.query_policy) print('Start training: ', i + 1, "/ ", len(X)) try: t.train() # bottleneck result = t.retrieve() res.append(1. - result[0] / 100.) # Turn into error diag_stats.append(result[1]) except Exception as e: logging.error( "An error occured in the current architecture. Assigning nan to the arch. The error is:" ) logging.error(e) res.append(np.nan) diag_stats.append(None) else: gpu_ids = range(n_parallel) num_reps = ceil(len(X) / float(n_parallel)) for i in range(num_reps): x = X[i * n_parallel:min((i + 1) * n_parallel, len( X))] # select the number of parallel archs to evaluate selected_gpus = gpu_ids[:len(x)] other_arg = [ self.data_path, self.save_path, self.dataset, self.cutout, self.epochs, self.query_policy ] args = list(map( list, zip( x, selected_gpus, ), )) args = [a + other_arg for a in args] pool = Pool(processes=len(x)) current_res = pool.starmap(parallel_eval, args) pool.close() pool.join() res.extend([i for i in current_res if i >= 0 ]) # Filter out the negative results due to errors res = np.array(res).flatten() if self.log_scale: res = np.log(res) if self.negative: res = -res return res, diag_stats
class MetricTester: """Class used for efficiently run alot of parametrized tests in ddp mode. Makes sure that ddp is only setup once and that pool of processes are used for all tests. All tests should subclass from this and implement a new method called `test_metric_name` where the method `self.run_metric_test` is called inside. """ atol = 1e-8 def setup_class(self): """Setup the metric class. This will spawn the pool of workers that are used for metric testing and setup_ddp """ self.poolSize = NUM_PROCESSES self.pool = Pool(processes=self.poolSize) self.pool.starmap(setup_ddp, [(rank, self.poolSize) for rank in range(self.poolSize)]) def teardown_class(self): """ Close pool of workers """ self.pool.close() self.pool.join() def run_functional_metric_test( self, preds: Tensor, target: Tensor, metric_functional: Callable, sk_metric: Callable, metric_args: dict = None, **kwargs_update, ): """Main method that should be used for testing functions. Call this inside testing method Args: preds: torch tensor with predictions target: torch tensor with targets metric_functional: lightning metric class that should be tested sk_metric: callable function that is used for comparison metric_args: dict with additional arguments used for class initialization kwargs_update: Additional keyword arguments that will be passed with preds and target when running update on the metric. """ _functional_test( preds=preds, target=target, metric_functional=metric_functional, sk_metric=sk_metric, metric_args=metric_args, atol=self.atol, **kwargs_update, ) def run_class_metric_test( self, ddp: bool, preds: Tensor, target: Tensor, metric_class: Metric, sk_metric: Callable, dist_sync_on_step: bool, metric_args: dict = None, check_dist_sync_on_step: bool = True, check_batch: bool = True, **kwargs_update, ): """Main method that should be used for testing class. Call this inside testing methods. Args: ddp: bool, if running in ddp mode or not preds: torch tensor with predictions target: torch tensor with targets metric_class: lightning metric class that should be tested sk_metric: callable function that is used for comparison dist_sync_on_step: bool, if true will synchronize metric state across processes at each ``forward()`` metric_args: dict with additional arguments used for class initialization check_dist_sync_on_step: bool, if true will check if the metric is also correctly calculated per batch per device (and not just at the end) check_batch: bool, if true will check if the metric is also correctly calculated across devices for each batch (and not just at the end) kwargs_update: Additional keyword arguments that will be passed with preds and target when running update on the metric. """ if not metric_args: metric_args = {} if ddp: if sys.platform == "win32": pytest.skip("DDP not supported on windows") self.pool.starmap( partial( _class_test, preds=preds, target=target, metric_class=metric_class, sk_metric=sk_metric, dist_sync_on_step=dist_sync_on_step, metric_args=metric_args, check_dist_sync_on_step=check_dist_sync_on_step, check_batch=check_batch, atol=self.atol, **kwargs_update, ), [(rank, self.poolSize) for rank in range(self.poolSize)], ) else: _class_test( 0, 1, preds=preds, target=target, metric_class=metric_class, sk_metric=sk_metric, dist_sync_on_step=dist_sync_on_step, metric_args=metric_args, check_dist_sync_on_step=check_dist_sync_on_step, check_batch=check_batch, atol=self.atol, **kwargs_update, ) def run_precision_test_cpu( self, preds: torch.Tensor, target: torch.Tensor, metric_module: Metric, metric_functional: Callable, metric_args: dict = {}, ): """Test if an metric can be used with half precision tensors on cpu Args: preds: torch tensor with predictions target: torch tensor with targets metric_module: the metric module to test metric_functional: the metric functional to test metric_args: dict with additional arguments used for class initialization """ _assert_half_support(metric_module(**metric_args), partial(metric_functional, **metric_args), preds, target, device="cpu") def run_precision_test_gpu( self, preds: torch.Tensor, target: torch.Tensor, metric_module: Metric, metric_functional: Callable, metric_args: dict = {}, ): """Test if an metric can be used with half precision tensors on gpu Args: preds: torch tensor with predictions target: torch tensor with targets metric_module: the metric module to test metric_functional: the metric functional to test metric_args: dict with additional arguments used for class initialization """ _assert_half_support(metric_module(**metric_args), partial(metric_functional, **metric_args), preds, target, device="cuda")
def train(self, n_process=None): assert n_process is not None pool = Pool(processes=n_process) outer_lr_scheduler = PiecewiseSchedule( [(0, self.outer_lr), (int(self.outer_n_epoch / 2), self.outer_lr * 0.1)], outside_value=self.outer_lr * 0.1) evoluted_loss = loss_net(self.inner_args['nclass']) evoluted_loss = torch.nn.DataParallel(evoluted_loss).cuda() evoluted_loss.train() for epoch in range(self.outer_n_epoch): phi_state_dict = evoluted_loss.state_dict() phi_noise = [] for i in range(self.outer_n_noise): noise = dict() for key in evoluted_loss.state_dict().keys(): noise[key] = self.outer_std * torch.randn_like( evoluted_loss.state_dict()[key]) + phi_state_dict[key] phi_noise.append(noise) start_time = time.time() assert self.outer_n_worker % self.outer_n_noise == 0 args = ((epoch, i, phi_noise[i], self.target_model, self.train_dataset, self.test_dataset, self.inner_args) for i in range(self.outer_n_noise)) results = pool.map_async(inner_train, args) results = results.get() result = [np.mean(r['accuracy']) for r in results] print( 'Epoch %d, mean accuracy: %f, max accuracy: %f(%d), min accuracy: %f(%d)' % (epoch, np.mean(result), np.max(result), result.index(np.max(result)), np.min(result), result.index(np.min(result)))) print('All inner loops completed, returns gathered ({:.2f} sec).'. format(time.time() - start_time)) print('\n') for key in evoluted_loss.state_dict(): grad = 0 for i in range(self.outer_n_noise): grad += result[i] * phi_noise[i][key].cpu( ) # - self.outer_l2 * evoluted_loss.state_dict()[key].cpu() lr = outer_lr_scheduler.value(epoch) adam = Adam(shape=grad.shape, stepsize=lr) evoluted_loss.state_dict()[key] -= adam.step( grad / self.outer_n_worker).cuda() if self.save_model: torch.save( { 'epoch': epoch + 1, 'state_dict': evoluted_loss.state_dict() }, self.log.path_model) self.log.log_tabular('epoch', epoch) for i in range(len(result)): self.log.log_tabular('reward %d' % i, result[i]) self.log.dump_tabular() pool.close()
class DataLoader(): def __init__(self, args): self.dir_bin = args.dir_bin line_load_list = self.dir_bin + 'line_load_list.t7' vocab_file = self.dir_bin + 'vocab.t7' assert os.path.isfile(self.dir_bin + 'specM.bin') assert os.path.isfile(self.dir_bin + 'specL.bin') assert os.path.isfile(self.dir_bin + 'text.bin') self.batch_size = args.batch_size self.trunc_size = args.trunc_size self.r_factor = args.r_factor self.dec_out_size = args.dec_out_size self.post_out_size = args.post_out_size self.shuffle_data = True if args.shuffle_data == 1 else False self.iter_per_epoch = None self.is_subbatch_end = True self.curr_split = None self.vocab_size = None self.process = None self.queue = Queue(maxsize=args.load_queue_size) self.n_workers = args.n_workers self.use_gpu = args.use_gpu self.num_gpu = len(args.gpu) if len(args.gpu) > 0 else 1 self.pinned_memory = True if args.pinned_memory == 1 and self.use_gpu else False self.vocab_size = self.get_num_vocab(vocab_file) text_limit = args.text_limit wave_limit = args.wave_limit # col1: idx / col2: wave_length / col3: text_length # col4: offset_M / col5: offset_L / col6: offset_T self.load_list = torch.load(line_load_list) spec_len_list = self.load_list[:, 1].clone() text_len_list = self.load_list[:, 2].clone() # exclude files whose wave length exceeds wave_limit sort_length, sort_idx = spec_len_list.sort() text_len_list = torch.gather(text_len_list, 0, sort_idx) sort_idx = sort_idx.view(-1, 1).expand_as(self.load_list) self.load_list = torch.gather(self.load_list, 0, sort_idx) end_idx = sort_length.le(wave_limit).sum() spec_len_list = sort_length[:end_idx] text_len_list = text_len_list[:end_idx] self.load_list = self.load_list[:end_idx] # exclude files whose text length exceeds text_limit sort_length, sort_idx = text_len_list.sort() spec_len_list = torch.gather(spec_len_list, 0, sort_idx) sort_idx = sort_idx.view(-1, 1).expand_as(self.load_list) self.load_list = torch.gather(self.load_list, 0, sort_idx) end_idx = sort_length.le(text_limit).sum() end_idx = end_idx - (end_idx % self.batch_size) # drop residual data text_len_list = sort_length[:end_idx] spec_len_list = spec_len_list[:end_idx] self.load_list = self.load_list[:end_idx] # sort by wave length _, sort_idx = spec_len_list.sort(0, descending=True) text_len_list = torch.gather(text_len_list, 0, sort_idx) sort_idx = sort_idx.view(-1, 1).expand_as(self.load_list) self.load_list = torch.gather(self.load_list, 0, sort_idx) # sort by text length in each batch (PackedSequence requires it) num_batches_per_epoch = self.load_list.size(0) // self.batch_size text_len_list = text_len_list.view(num_batches_per_epoch, -1) self.load_list = self.load_list.view(num_batches_per_epoch, -1, self.load_list.size(1)) sort_length, sort_idx = text_len_list.sort(1, descending=True) sort_idx = sort_idx.view(num_batches_per_epoch, -1, 1).expand_as(self.load_list) self.load_list = torch.gather(self.load_list, 1, sort_idx) # shuffle while preserving order in a batch if self.shuffle_data: _, sort_idx = torch.randn(num_batches_per_epoch).sort() sort_idx = sort_idx.view(-1, 1, 1).expand_as(self.load_list) self.load_list = torch.gather(self.load_list, 0, sort_idx) # nbpe x N x 6 self.load_list = self.load_list.long() # compute number of iterations needed spec_len_list = spec_len_list.view(num_batches_per_epoch, -1) spec_len_list, _ = spec_len_list.div(self.trunc_size).ceil().max(1) self.iter_per_epoch = int(spec_len_list.sum()) # set split cursor self.split_sizes = { 'train': self.load_list.size(0), 'val': -1, 'test': -1 } self.split_cursor = {'train': 0, 'val': 0, 'test': 0} def next_batch(self, split): T, idx = self.trunc_size, self.split_cursor[split] # seek and load data from raw files if self.is_subbatch_end: self.is_subbatch_end = False self.subbatch_cursor = 0 if self.curr_split != split: self.curr_split = split if self.process is not None: self.process.terminate() self.process = Process(target=self.start_async_loader, args=(split, self.split_cursor[split])) self.process.start() self.len_text, self.len_wave, self.curr_text, self.curr_specM, self.curr_specL = self.queue.get( ) self.split_cursor[split] = (idx + 1) % self.split_sizes[split] self.subbatch_max_len = self.len_wave.max() # Variables to return # +1 to length of y to consider shifting for target y subbatch_len_text = [x for x in self.len_text] subbatch_len_wave = [min(x, T) for x in self.len_wave] x_text = self.curr_text y_specM = self.curr_specM[:, self.subbatch_cursor:self.subbatch_cursor + max(subbatch_len_wave) + 1].contiguous() y_specL = self.curr_specL[:, self.subbatch_cursor:self.subbatch_cursor + max(subbatch_len_wave) + 1].contiguous() if self.use_gpu: if self.pinned_memory: x_text = x_text.pin_memory() y_specM = y_specM.pin_memory() y_specL = y_specL.pin_memory() x_text = x_text.cuda() y_specM = y_specM.cuda() y_specL = y_specL.cuda() # Advance split_cursor or Move on to the next batch if self.subbatch_cursor + T < self.subbatch_max_len: self.subbatch_cursor = self.subbatch_cursor + T self.len_wave.sub_(T).clamp_(min=0) else: self.is_subbatch_end = True # Don't compute for empty batch elements if subbatch_len_wave.count(0) > 0: self.len_wave_mask = [ idx for idx, l in enumerate(subbatch_len_wave) if l > 0 ] self.len_wave_mask = torch.LongTensor(self.len_wave_mask) if self.use_gpu: self.len_wave_mask = self.len_wave_mask.cuda() x_text = torch.index_select(x_text, 0, self.len_wave_mask) y_specM = torch.index_select(y_specM, 0, self.len_wave_mask) y_specL = torch.index_select(y_specL, 0, self.len_wave_mask) subbatch_len_text = [ subbatch_len_text[idx] for idx in self.len_wave_mask ] subbatch_len_wave = [ subbatch_len_wave[idx] for idx in self.len_wave_mask ] else: self.len_wave_mask = None return x_text, y_specM, y_specL, subbatch_len_wave, subbatch_len_text def start_async_loader(self, split, load_start_idx): # load batches to the queue asynchronously since it is a bottle-neck N, r = self.batch_size, self.r_factor load_curr_idx = load_start_idx while True: data_T, data_M, data_L, len_T, len_M = ([None for _ in range(N)] for _ in range(5)) # deploy workers to load data self.pool = Pool(self.n_workers) partial_func = partial(load_data_and_length, self.dir_bin, self.load_list[load_curr_idx]) results = self.pool.map_async(func=partial_func, iterable=range(N)) self.pool.close() self.pool.join() for result in results.get(): data_M[result[0]] = result[1] data_L[result[0]] = result[2] data_T[result[0]] = result[3] len_T[result[0]] = result[4] len_M[result[0]] = result[5] # TODO: output size is not accurate.. // len_text = torch.IntTensor(len_T) len_wave = torch.Tensor(len_M).div(r).ceil().mul( r).int() # consider r_factor curr_text = torch.LongTensor(N, len_text.max()).fill_( 0) # null-padding at tail curr_specM = torch.Tensor(N, len_wave.max() + 1, self.dec_out_size).fill_( 0) # null-padding at tail curr_specL = torch.Tensor(N, len_wave.max() + 1, self.post_out_size).fill_( 0) # null-padding at tail # fill the template tensors for j in range(N): curr_text[j, 0:data_T[j].size(0)].copy_(data_T[j]) curr_specM[j, 0:data_M[j].size(0)].copy_(data_M[j]) curr_specL[j, 0:data_L[j].size(0)].copy_(data_L[j]) self.queue.put( (len_text, len_wave, curr_text, curr_specM, curr_specL)) load_curr_idx = (load_curr_idx + 1) % self.split_sizes[split] def mask_prev_h(self, prev_h): if self.len_wave_mask is not None: if self.use_gpu: self.len_wave_mask = self.len_wave_mask.cuda() h_att, h_dec1, h_dec2 = prev_h h_att = torch.index_select(h_att.data, 1, self.len_wave_mask) # batch idx is h_dec1 = torch.index_select(h_dec1.data, 1, self.len_wave_mask) h_dec2 = torch.index_select(h_dec2.data, 1, self.len_wave_mask) prev_h = (Variable(h_att), Variable(h_dec1), Variable(h_dec2)) else: prev_h = prev_h return prev_h def get_num_vocab(self, vocab_file=None): if self.vocab_size: return self.vocab_size else: vocab_dict = torch.load(vocab_file) return len(vocab_dict) + 1 # +1 to consider null-padding
for i in range(0, table_prep_params['MAX_ROW_LEN'] - rows): table.append( [['<PAD>'] * table_prep_params['LENGTH_PER_CELL']] * table_prep_params['MAX_COL_LEN']) return table def table_words2index(tables): w2i = {w: i for i, w in enumerate(vocab)} for i, t in enumerate(tables): tables[i] = np.vectorize(lambda y: w2i[y])( np.array(t)).tolist() return tables p = Pool(processes=40) X = p.map(pad_table, X) p.close() p.join() X = table_words2index(X) X = np.array(X) print(X.shape) savepkl('./data/xp_2D_10-50_pad.pkl', X) else: device = torch.device( f"cuda:{1}" if torch.cuda.is_available() else 'cpu') dataset = T2VDataset(X, y, vocab, device, config) dataloader = DataLoader(dataset, batch_size=32, shuffle=True) X_, y_ = next(iter(dataloader)) print(X_.shape, y_.shape) print(time.time() - start)
from evostrat import compute_centered_ranks, NormalPopulation from evostrat.examples.lunar_lander import LunarLander if __name__ == '__main__': """ Lunar landers weights and biases are drawn from normal distributions with learned means and fixed standard deviation 0.1. This is the approach suggested in OpenAI ES [1] [1] - Salimans, Tim, et al. "Evolution strategies as a scalable alternative to reinforcement learning." arXiv preprint arXiv:1703.03864 (2017). """ param_shapes = {k: v.shape for k, v in LunarLander().get_params().items()} population = NormalPopulation(param_shapes, LunarLander.from_params, std=0.1) learning_rate = 0.1 iterations = 1000 pop_size = 200 optim = Adam(population.parameters(), lr=learning_rate) pbar = tqdm.tqdm(range(iterations)) pool = Pool() for _ in pbar: optim.zero_grad() raw_fit = population.fitness_grads(pop_size, pool, compute_centered_ranks) optim.step() pbar.set_description("fit avg: %0.3f, std: %0.3f" % (raw_fit.mean().item(), raw_fit.std().item())) if raw_fit.mean() > 200: print("Solved.") break pool.close()
class MetricTester: """ Class used for efficiently run alot of parametrized tests in ddp mode. Makes sure that ddp is only setup once and that pool of processes are used for all tests. All tests should subclass from this and implement a new method called `test_metric_name` where the method `self.run_metric_test` is called inside. """ def setup_class(self): """ Setup the metric class. This will spawn the pool of workers that are used for metric testing and setup_ddp """ try: set_start_method('spawn') except RuntimeError: pass self.poolSize = NUM_PROCESSES self.pool = Pool(processes=self.poolSize) self.pool.starmap(setup_ddp, [(rank, self.poolSize) for rank in range(self.poolSize)]) def teardown_class(self): """ Close pool of workers """ self.pool.close() self.pool.join() def run_metric_test( self, ddp: bool, preds: torch.Tensor, target: torch.Tensor, metric_class: Metric, sk_metric: Callable, dist_sync_on_step: bool, metric_args: dict = {}, check_dist_sync_on_step: bool = True, check_batch: bool = True, ): """ Main method that should be used for testing. Call this inside testing methods. Args: ddp: bool, if running in ddp mode or not preds: torch tensor with predictions target: torch tensor with targets metric_class: lightning metric class that should be tested sk_metric: callable function that is used for comparison dist_sync_on_step: bool, if true will synchronize metric state across processes at each ``forward()`` metric_args: dict with additional arguments used for class initialization check_dist_sync_on_step: bool, if true will check if the metric is also correctly calculated per batch per device (and not just at the end) check_batch: bool, if true will check if the metric is also correctly calculated across devices for each batch (and not just at the end) """ if ddp: if sys.platform == "win32": pytest.skip("DDP not supported on windows") self.pool.starmap( partial( _compute_batch, preds=preds, target=target, metric_class=metric_class, sk_metric=sk_metric, dist_sync_on_step=dist_sync_on_step, metric_args=metric_args, check_dist_sync_on_step=check_dist_sync_on_step, check_batch=check_batch, ), [(rank, self.poolSize) for rank in range(self.poolSize)], ) else: _compute_batch( 0, 1, preds=preds, target=target, metric_class=metric_class, sk_metric=sk_metric, dist_sync_on_step=dist_sync_on_step, metric_args=metric_args, check_dist_sync_on_step=check_dist_sync_on_step, check_batch=check_batch, )
def train(self, data, labels, val_data=None, val_labels=None, warm_start=False): """ :param data: :param labels: :param val_data: :param val_labels: :param warm_start: :return: """ # print('start train') # initialize variable data = torch.from_numpy(data).float() labels = torch.from_numpy(labels).char() if val_data is None: train_data, val_data = data, data train_labels, val_labels = labels, labels else: train_data = data train_labels = labels val_data = torch.from_numpy(val_data).float() val_labels = torch.from_numpy(val_labels).char() orig_cols = train_data.size(1) # counting class and get their index self.plus_row_index = [] self.minus_row_index = [] self.orig_minus = 0 self.orig_plus = 0 for idx, value in enumerate(train_labels): if value == 1: self.orig_plus += 1 self.plus_row_index.append(idx) else: self.orig_minus += 1 self.minus_row_index.append(idx) # balanced pick rows and cols plus = max(2, int(self.orig_plus * self.nrows)) minus = max(2, int(self.orig_minus * self.nrows)) num_cols = max(min(5, orig_cols), int(self.nfeatures * orig_cols)) # initialize up triangle matrix and reference index rows_sum = plus + minus if self.adv_train: rows_sum = rows_sum * 2 # plus = plus * 2 # minus = minus * 2 self.yp = torch.ones((self.width, rows_sum), dtype=torch.int8) self.ref_full_index = torch.repeat_interleave(torch.arange( self.updated_features * self.step.shape[0]).view((-1, 1)), rows_sum, dim=1) # multi-process c = self.round // self.num_gpus results = [] logs = {} # print('enter pool') for r in range(c + 1): pool = Pool(self.n_jobs) results = [] for t in range(min(self.n_jobs, self.round - r * self.num_gpus)): if warm_start and self.w_index != []: column_indices = self.w_index[r * self.num_gpus + t] w1 = self.w1[:, :, r * self.num_gpus + t] w2 = self.w2[:, r * self.num_gpus + t] else: column_indices = np.random.choice(np.arange(orig_cols), num_cols, replace=False) # column_indices = np.arange(orig_cols) self.w_index.append(column_indices) results.append( pool.apply_async(self.single_run, args=(train_data, train_labels, plus, minus, val_data, val_labels, column_indices, t % self.num_gpus))) pool.close() pool.join() for i, result in enumerate(results): temp_w, temp_b, temp_obj = result.get() # logs['vote%d_train' % i] = train_log # logs['vote%d_test' % i] = test_log if warm_start: self.w[:, i] = temp_w self.b[:, i] = temp_b self.obj[i] = temp_obj else: self.w.append(temp_w.view((-1, 1))) self.b.append(temp_b.view((1, 1))) self.obj.append(temp_obj) del pool, results if warm_start is False: self.w = torch.cat(self.w, dim=1) self.b = torch.cat(self.b, dim=1) self.obj = torch.Tensor(self.obj) best_index = self.obj.argmax() self.best_acc = self.obj[best_index] self.best_w = self.w[:, best_index] self.best_b = self.b[:, best_index] self.best_w_index = self.w_index[best_index] del self.yp, self.ref_full_index return
def train(self, data, labels, val_data=None, val_labels=None, warm_start=False): """ :param data: :param labels: :param val_data: :param val_labels: :param warm_start: :return: """ # initialize variable data = torch.from_numpy(data).float() labels = torch.from_numpy(labels).char() if val_data is None: # rows = labels.shape[0] # index = np.random.permutation(rows) # val_size = rows // 5 # val_data = data[index[:val_size]] # val_labels = labels[index[:val_size]] # train_data = data[index[val_size:]] # train_labels = labels[index[val_size:]] train_data, val_data = data, data train_labels, val_labels = labels, labels else: train_data = data train_labels = labels val_data = torch.from_numpy(val_data).float() val_labels = torch.from_numpy(val_labels).char() orig_cols = train_data.size(1) # counting class and get their index self.plus_row_index = [] self.minus_row_index = [] self.orig_minus = 0 self.orig_plus = 0 for idx, value in enumerate(train_labels): if value == 1: self.orig_plus += 1 self.plus_row_index.append(idx) else: self.orig_minus += 1 self.minus_row_index.append(idx) # balanced pick rows and cols plus = max(2, int(self.orig_plus * self.nrows)) minus = max(2, int(self.orig_minus * self.nrows)) num_cols = max(min(5, orig_cols), int(self.nfeatures * orig_cols)) # initialize up triangle matrix and reference index rows_sum = plus + minus if self.adv_train: rows_sum = rows_sum * 2 # plus = plus * 2 # minus = minus * 2 self.yp = torch.ones((rows_sum, rows_sum), dtype=torch.int8).triu_(0) self.ref_full_index = torch.repeat_interleave(torch.arange( self.updated_features * self.step.shape[0]).view((-1, 1)), rows_sum, dim=1) # multi-process pool = Pool(self.n_jobs) results = [] for r in range(self.round): if warm_start and self.w_index != []: column_indices = self.w_index[r] w = self.w[:, r] else: column_indices = np.random.choice(np.arange(orig_cols), num_cols, replace=False) self.w_index.append(column_indices) w = np.random.uniform(-1, 1, size=(num_cols, )).astype(np.float32) results.append( pool.apply_async( self.single_run_adv if self.adv_train else self.single_run, args=(train_data, train_labels, plus, minus, val_data, val_labels, w, column_indices, r % self.num_gpus))) pool.close() pool.join() for i, result in enumerate(results): temp_w, temp_b, temp_obj = result.get() if warm_start: self.w[:, i] = temp_w self.b[:, i] = temp_b self.obj[i] = temp_obj else: self.w.append(temp_w.view((-1, 1))) self.b.append(temp_b.view((1, 1))) self.obj.append(temp_obj) if warm_start is False: self.w = torch.cat(self.w, dim=1) self.b = torch.cat(self.b, dim=1) self.obj = torch.Tensor(self.obj) best_index = self.obj.argmax() self.best_acc = self.obj[best_index] self.best_w = self.w[:, best_index] self.best_b = self.b[:, best_index] self.best_w_index = self.w_index[best_index]
def train(self, data, labels, val_data=None, val_labels=None, warm_start=False): """ :param data: :param labels: :param val_data: :param val_labels: :param warm_start: :return: """ # initialize variable data = torch.from_numpy(data).float() labels = torch.from_numpy(labels).char() if val_data is None: train_data, val_data = data, data train_labels, val_labels = labels, labels else: train_data = data train_labels = labels val_data = torch.from_numpy(val_data).float() val_labels = torch.from_numpy(val_labels).char() orig_cols = train_data.size(1) # counting class and get their index self.plus_row_index = [] self.minus_row_index = [] self.orig_minus = 0 self.orig_plus = 0 for idx, value in enumerate(train_labels): if value == 1: self.orig_plus += 1 self.plus_row_index.append(idx) else: self.orig_minus += 1 self.minus_row_index.append(idx) # balanced pick rows and cols plus = max(2, int(self.orig_plus * self.nrows)) minus = max(2, int(self.orig_minus * self.nrows)) num_cols = max(min(5, orig_cols), int(self.nfeatures * orig_cols)) # initialize up triangle matrix and reference index rows_sum = plus + minus self.yp = torch.ones((self.width, rows_sum), dtype=torch.int8) self.ref_full_index1 = torch.repeat_interleave(torch.arange( self.updated_features * self.step.shape[0]).view((-1, 1)), rows_sum, dim=1) self.ref_full_index2 = torch.repeat_interleave(torch.arange( self.hidden_nodes * self.step.shape[0]).view((-1, 1)), rows_sum, dim=1) # multi-process c = self.round // self.num_gpus for r in range(c + 1): pool = Pool(self.n_jobs) results = [] for t in range(min(self.n_jobs, self.round - r * self.num_gpus)): if warm_start and self.w_index != []: column_indices = self.w_index[r * self.num_gpus + t] w1 = self.w1[:, :, r * self.num_gpus + t] w2 = self.w2[:, r * self.num_gpus + t] else: column_indices = np.random.choice(np.arange(orig_cols), num_cols, replace=False) # column_indices = np.arange(orig_cols) self.w_index.append(column_indices) results.append( pool.apply_async(self.single_run, args=(train_data, train_labels, plus, minus, val_data, val_labels, column_indices, t % self.num_gpus))) pool.close() pool.join() df = pd.DataFrame(columns=[]) for i, result in enumerate(results): temp_w1, temp_b1, temp_w2, temp_b2, temp_obj, uba, ba = result.get( ) df['vote %d imbalanced acc' % i] = uba df['vote %d balanced acc' % i] = ba # temp_w1, temp_b1, temp_w2, temp_b2, temp_obj = self.single_run(train_data, train_labels, plus, minus, val_data, val_labels, w1, w2, column_indices, r % self.num_gpus) if warm_start: self.w1[:, :, i] = temp_w1 self.w2[:, i] = temp_w2 self.b1[:, i] = temp_b1 self.b2[i] = temp_b2 self.obj[i] = temp_obj else: self.w1.append(temp_w1) self.w2.append(temp_w2) self.b1.append(temp_b1) self.b2.append(temp_b2) self.obj.append(temp_obj) del pool, results df.to_csv('v15.csv', index=False) if warm_start is False: self.w1 = torch.stack(self.w1, dim=2) self.w2 = torch.stack(self.w2, dim=1) self.b1 = torch.stack(self.b1, dim=1) self.b2 = torch.Tensor(self.b2) self.obj = torch.Tensor(self.obj) best_index = self.obj.argmax() self.best_acc = self.obj[best_index] self.best_w1 = self.w1[:, :, best_index] self.best_w2 = self.w2[:, best_index] self.best_b1 = self.b1[:, best_index] self.best_b2 = self.b2[best_index] self.best_w_index = self.w_index[best_index] return
def check_score(): print(args) p_size = cpu_num print("use cpu num:{}".format(p_size)) loss_history = [] check_deck_id = int( args.check_deck_id) if args.check_deck_id is not None else None cuda_flg = args.cuda == "True" #node_num = int(args.node_num) #net = New_Dual_Net(node_num) model_name = args.model_name existed_output_list = os.listdir(path="./Battle_Result") existed_output_list = [ f for f in existed_output_list if os.path.isfile(os.path.join("./Battle_Result", f)) ] result_name = "{}:{}".format(model_name.split(".")[0], args.deck_list) same_name_count = len( [1 for cell in existed_output_list if result_name in cell]) print("same_name_count:", same_name_count) result_name += "_{:0>3}".format(same_name_count + 1) PATH = 'model/' + model_name model_dict = torch.load(PATH) n_size = model_dict["final_layer.weight"].size()[1] net = New_Dual_Net(n_size, hidden_num=args.hidden_num[0]) net.load_state_dict(model_dict) opponent_net = None if args.opponent_model_name is not None: # opponent_net = New_Dual_Net(node_num) o_model_name = args.opponent_model_name PATH = 'model/' + o_model_name model_dict = torch.load(PATH) n_size = model_dict["final_layer.weight"].size()[1] opponent_net = New_Dual_Net(n_size, hidden_num=args.hidden_num[1]) opponent_net.load_state_dict(model_dict) if torch.cuda.is_available() and cuda_flg: net = net.cuda() opponent_net = opponent_net.cuda( ) if opponent_net is not None else None print("cuda is available.") #net.zero_grad() deck_sampling_type = False if args.deck is not None: deck_sampling_type = True G = Game() net.cpu() t3 = datetime.datetime.now() if args.greedy_mode is not None: p1 = Player(9, True, policy=Dual_NN_GreedyPolicy(origin_model=net)) else: p1 = Player(9, True, policy=New_Dual_NN_Non_Rollout_OM_ISMCTSPolicy( origin_model=net, cuda=cuda_flg, iteration=args.step_iter), mulligan=Min_cost_mulligan_policy()) #p1 = Player(9, True, policy=AggroPolicy()) p1.name = "Alice" if fixed_opponent is not None: if fixed_opponent == "Aggro": p2 = Player(9, False, policy=AggroPolicy(), mulligan=Min_cost_mulligan_policy()) elif fixed_opponent == "OM": p2 = Player(9, False, policy=Opponent_Modeling_ISMCTSPolicy()) elif fixed_opponent == "NR_OM": p2 = Player(9, False, policy=Non_Rollout_OM_ISMCTSPolicy(iteration=200), mulligan=Min_cost_mulligan_policy()) elif fixed_opponent == "ExItGreedy": tmp = opponent_net if opponent_net is not None else net p2 = Player(9, False, policy=Dual_NN_GreedyPolicy(origin_model=tmp)) elif fixed_opponent == "Greedy": p2 = Player(9, False, policy=New_GreedyPolicy(), mulligan=Simple_mulligan_policy()) elif fixed_opponent == "Random": p2 = Player(9, False, policy=RandomPolicy(), mulligan=Simple_mulligan_policy()) else: assert opponent_net is not None p2 = Player(9, False, policy=New_Dual_NN_Non_Rollout_OM_ISMCTSPolicy( origin_model=opponent_net, cuda=cuda_flg), mulligan=Min_cost_mulligan_policy()) # p2 = Player(9, False, policy=RandomPolicy(), mulligan=Min_cost_mulligan_policy()) p2.name = "Bob" Battle_Result = {} deck_list = tuple(map(int, args.deck_list.split(","))) print(deck_list) test_deck_list = deck_list # (0,1,4,10,13) test_deck_list = tuple(itertools.product(test_deck_list, test_deck_list)) test_episode_len = evaluate_num #100 episode_num = evaluate_num match_num = len(test_deck_list) manager = Manager() shared_array = manager.Array("i", [0 for _ in range(3 * len(test_deck_list))]) iter_data = [(p1, p2, shared_array, episode_num, p_id, test_deck_list) for p_id in range(p_size)] freeze_support() p1_name = p1.policy.name.replace("origin", args.model_name) if args.opponent_model_name is not None: p2_name = p2.policy.name.replace("origin", args.opponent_model_name) else: p2_name = p2.policy.name.replace("origin", args.model_name) print(p1_name) print(p2_name) pool = Pool(p_size, initializer=tqdm.set_lock, initargs=(RLock(), )) # 最大プロセス数:8 memory = pool.map(multi_battle, iter_data) pool.close() # add this. pool.terminate() # add this. print("\n" * (match_num + 1)) memory = list(memory) min_WR = 1.0 Battle_Result = {(deck_id[0], deck_id[1]): \ tuple(shared_array[3*index+1:3*index+3]) for index, deck_id in enumerate(test_deck_list)} print(shared_array) txt_dict = {} for key in sorted(list((Battle_Result.keys()))): cell = "{}:WR:{:.2%},first_WR:{:.2%}"\ .format(key,Battle_Result[key][0]/test_episode_len,2*Battle_Result[key][1]/test_episode_len) print(cell) txt_dict[key] = cell print(Battle_Result) # result_name = "{}:{}_{}".format(model_name.split(".")[0],args.deck_list,) # result_name = model_name.split(".")[0] + ":" + args.deck_list + "" deck_num = len(deck_list) # os.makedirs("Battle_Result", exist_ok=True) with open("Battle_Result/" + result_name, "w") as f: writer = csv.writer(f, delimiter='\t', lineterminator='\n') row = ["{} vs {}".format(p1_name, p2_name)] deck_names = [deck_id_2_name[deck_list[i]] for i in range(deck_num)] row = row + deck_names writer.writerow(row) for i in deck_list: row = [deck_id_2_name[i]] for j in deck_list: row.append(Battle_Result[(i, j)]) writer.writerow(row) for key in list(txt_dict.keys()): writer.writerow([txt_dict[key]])
import torch from torch.multiprocessing import Pool, Process, set_start_method from torch.autograd import Variable import numpy as np from scipy.ndimage import zoom def get_pred(args): img = args[0] scale = args[1] # feed input data input_img = Variable(torch.from_numpy(img), volatile=True).cuda() return input_img if __name__ == '__main__': try: set_start_method('spawn') except RuntimeError: pass img = np.float32(np.random.randint(0, 2, (300, 300, 3))) scales = [1,2,3,4,5] scale_list = [] for scale in scales: scale_list.append([img,scale]) multi_pool = Pool(processes=5) predictions = multi_pool.map(get_pred,scale_list) multi_pool.close() multi_pool.join()