def fit(self): self.dtree = DecisionTreeRegressor(self.criterion) hyperparameters = [ a for a in dir(self) if not a.startswith('__') and not a.startswith('_') ] params = {} properties = [ "data", "target", "fit", "predict", "get_metrics", "plot_tree", "best_score", "best_params", "criterion", "dtree", "grid_search" ] for hyperparameter in hyperparameters: if hyperparameter not in properties: params[hyperparameter] = getattr(self, hyperparameter, None) self.grid_search = GridSearchCV( self.dtree, param_grid=params, scoring="neg_mean_squared_error", n_jobs=1, cv=10, verbose=3, ) start_time = timer(None) self.grid_search.fit(self.data, self.target) timer(start_time) self._best_params = self.grid_search.best_params_ self._best_score = self.grid_search.best_score_
def evolve(self, logger): self.initial_fittest = self.get_fittest() self.networks_for_visualization = [] gen_image_path_format = '{}/snapshots/{}.png' image = gen_image_path_format.format(self.run_id, 'initial') self.networks_for_visualization.append((self.initial_fittest.network.as_json_dict(), 'Gen {}'.format('initial'), image)) for gen in range(1, self.generations): start_ga = datetime.datetime.now() logger.info("Generation: " + str(gen)) with timer("Generation {}".format(str(gen)), logger=logger): for phase in self.ga_steps: with timer(phase[0], logger=logger): phase[1]() gen_time = (datetime.datetime.now() - start_ga).total_seconds() self.statistics.gen_snapshot(gen=gen, time_spent=gen_time) image = gen_image_path_format.format(self.run_id, gen) network_visualization_info = (self.get_fittest().network.as_json_dict(), 'Gen {}'.format(gen), image) self.networks_for_visualization.append(network_visualization_info) logger.info('Adding relays') self.add_relays() logger.info('Recalculating fitness') self.calc_fitness() image = gen_image_path_format.format(self.run_id, self.generations) network_visualization_info = (self.get_fittest().network.as_json_dict(), 'Gen {}'.format(self.generations), image) self.networks_for_visualization.append(network_visualization_info) logger.info("Finished GA")
def train(self): self.scheduler.step() self.loss.step() epoch = self.scheduler.last_epoch + 1 lr = self.scheduler.get_lr()[0] self.ckp.visual("lr", lr, epoch) self.ckp.write_log('\n\n[Epoch {}]\tLearning rate: {:.2e}'.format( epoch, Decimal(lr))) self.loss.start_log() self.model.train() timer_data, timer_model = utils.timer(), utils.timer() for batch, (_input, _target, _, idx_scale) in enumerate(self.loader_train): _input, _target = self.prepare(_input, _target) timer_data.hold() timer_model.tic() self.optimizer.zero_grad() # output is the high-resolution image logits = self.model(_input, idx_scale) loss = self.loss(logits, _target) loss.backward() nn.utils.clip_grad_norm_(self.model.parameters(), self.args.grad_clip) self.optimizer.step() timer_model.hold() if (batch + 1) % self.args.print_every == 0: self.ckp.write_log('[{}/{}\t{}\t{:.1f}+{:.1f}s]'.format( (batch + 1) * self.args.batch_size, len(self.loader_train.dataset), self.loss.display_loss(batch), timer_model.release(), timer_data.release())) timer_data.tic() final_loss = self.loss.end_log(len(self.loader_train)) final_loss = final_loss.numpy()[-1] self.ckp.visual("train_loss", final_loss, epoch) self.error_last = self.loss.log[-1, -1] target = self.model torch.save( target.state_dict(), os.path.join(self.ckp.dir, 'model', 'model_{}.pt'.format(epoch)))
def __main__(): global parser parser = args_options() args = parser.parse_args() with timer(): exit(*main(args))
def __init__(self,params): self.params = params # Exception for not having self.serialport = serial.Serial("/dev/ttyO5",115200, timeout = 0.5) if 'period_gtfri' in params.keys(): self.timer_gtfri = timer(params['period_gtfri'],self.gtfri_method) if 'period_gtinf' in params.keys(): self.timer_gtinf = timer(params['period_gtinf'],self.gtinf_method) if 'period_gtudt' in params.keys(): self.timer_gtudt = timer(params['period_gtudt'],self.print_gtudt) # NEGLI'S WAY: self.kam_listener_thread =\ threading.Thread(target=self.kamaleon_listener, args=(1,))
def evolve(self, logger): for iteration in range(self.iterations): logger.info("Iteration %s", iteration) result = None for phase in self.phases: name = phase operation = self.phases[phase] with timer(op_name=name, logger=logger): result = operation(result, logger)
def make_relative_position(): with utils.timer("read data"): train_df = compe_data.read_train() test_df = compe_data.read_test() st_df = compe_data.read_structures() # merge structure to train and test with utils.timer("merge atom_st"): train_df = map_atom_info(train_df, st_df, 0) train_df = map_atom_info(train_df, st_df, 1) test_df = map_atom_info(test_df, st_df, 0) test_df = map_atom_info(test_df, st_df, 1) # make structure dict #with utils.timer("make st_dict"): # st_dict = make_st_dict(st_df) # del st_df # gc.collect() # train test each make relative position matrix train_df = each_make(train_df, st_df) test_df = each_make(test_df, st_df)
def valid(self): epoch = self.scheduler.last_epoch + 1 self.ckp.write_log('\n\nEvaluation during search process:') self.ckp.add_log(torch.zeros(1, len(self.scale))) self.model.eval() timer_valid = utils.timer() with torch.no_grad(): eval_psnr = 0 eval_ssim = 0 for batch, (_input, _target, _, idx_scale) in enumerate(self.loader_valid): _input, _target = self.prepare(_input.detach(), _target.detach()) timer_valid.tic() logits = self.model(_input) timer_valid.hold() logits = utils.quantize(logits, self.args.rgb_range) eval_psnr += utils.calc_psnr( logits, _target, self.scale[idx_scale], self.args.rgb_range, benchmark=False ) eval_ssim += utils.calc_batch_ssim( logits, _target, self.scale[idx_scale], benchmark=False ) self.ckp.log[-1, idx_scale] = eval_psnr / len(self.loader_valid) best = self.ckp.log.max(0) self.ckp.write_log( '[{} x{}]\tPSNR: {:.3f}\tSSIM: {:.4f}\t(best: {:.3f} @epoch {})'.format( self.args.data_valid, self.scale[idx_scale], self.ckp.log[-1, idx_scale], eval_ssim / len(self.loader_valid), best[0][idx_scale], best[1][idx_scale] + 1 ) ) self.ckp.visual("valid_PSNR", self.ckp.log[-1, idx_scale], epoch) self.ckp.visual("valid_SSIM", eval_ssim / len(self.loader_valid), epoch) self.ckp.write_log( 'Total time: {:.2f}s\n'.format(timer_valid.toc()), refresh=True )
def learn(): with utils.timer("load train and test data"): train_df = compe_data.read_train() test_df = compe_data.read_test() st_df = compe_data.read_structures() # set validation fold validation.set_fold( fold_type="GroupKFold", fold_num=settings.fold_num, random_state=settings.fold_seed, shuffle_flg=True, ) validation.make_splits(train_df, train_df[features.TARGET_COL], group_col=features.GROUP_COL) # make feature with utils.timer("feature make"): train_df, test_df, cat_cols = features.make(train_df, test_df, st_df) ## for debug train_df.head(30).to_csv(os.path.join(log_dir, "check_train_df.csv"), index=False) # predict only train feature # training flg_use_scc_meta = True #False if flg_use_scc_meta: with utils.timer("meta feature training"): flg_meta_learn = False #True#True feature_selection = True if flg_meta_learn: meta_train_df = pd.DataFrame() meta_test_df = pd.DataFrame() meta_features = compe_data.read_scalar_coupling_contributions() for meta_col in ["fc", "sd", "pso", "dso"]: meta_feat = meta_features[meta_col] logging.info("******************************") logging.info( "******** learning {} *********".format(meta_col)) logging.info("******************************") use_feats = [ x for x in train_df.columns if not x in features.EXCEPT_FEATURES ] train_y = meta_feat #train_df[features.TARGET_COL] # training per type types = train_df["type"].unique() oof = np.zeros(len(train_df)) test_preds = np.zeros(len(test_df)) types_clfs = {} types_scores = {} for type_name in types: logging.info("----- training type == {} -----".format( type_name)) type_idx = train_df.type == type_name test_type_idx = test_df.type == type_name if feature_selection: type_use_feats = feature_util.feature_select( use_feats, importance_df=pd.read_csv( f"./importance/permu_importance{type_name}.csv", names=["feature", "importance"]), threshold=-0.05, reverse=True, ) else: type_use_feats = use_feats.copy() select_feats = features.select_type_feats( use_feats, type_name) clfs, importances, val_score, oof_preds = \ training.train( train_df.loc[type_idx], train_y.loc[type_idx], use_feats = select_feats, train_type = type_name, permutation=False, cat_cols = cat_cols, log_dir = log_dir, target_mode = "meta", meta_col = meta_col ) types_clfs[type_name] = clfs types_scores[type_name] = val_score utils.save_importances( importances_=importances, save_dir=log_dir, prefix=f"{meta_col}_{type_name}") oof[type_idx] = oof_preds for clf in clfs: test_preds[test_type_idx] += \ clf.predict(test_df.loc[test_type_idx, use_feats], num_iteration=clf.best_iteration) / len(clfs) #total_cv = training.metric(train_df, oof) #print("TotalCV = {:.5f}".format(total_cv)) meta_train_df[meta_col] = oof meta_test_df[meta_col] = test_preds logging.info( "---------- meta {} types val score ----------".format( meta_col)) for type_name, score in types_scores.items(): logging.info("{0} : {1}".format(type_name, score)) # merge train and test_df meta_train_df.to_pickle("../pickle/meta_train.pkl") meta_test_df.to_pickle("../pickle/meta_test.pkl") else: meta_train_df = pd.read_pickle("../pickle/gnn_meta_train.pkl") meta_test_df = pd.read_pickle("../pickle/gnn_meta_test.pkl") train_df = utils.fast_concat(train_df, meta_train_df) test_df = utils.fast_concat(test_df, meta_test_df) #with utils.timer("reduce memory"): # train_df = utils.reduce_memory(train_df) # test_df = utils.reduce_memory(test_df) with utils.timer("training"): feature_selection = True use_feats = [ x for x in train_df.columns if not x in features.EXCEPT_FEATURES ] train_y = train_df[features.TARGET_COL] # training per type types = train_df["type"].unique() #types = train_df["new_type"].unique() #hinokkiタイプ oof = np.zeros(len(train_df)) types_clfs = {} types_scores = {} use_feats_dict = {} for type_name in types: logging.info("----- training type == {} -----".format(type_name)) type_idx = train_df.type == type_name if feature_selection: type_use_feats = feature_util.feature_select( use_feats, importance_df=pd.read_csv( f"./importance/permu_importance{type_name}.csv", names=["feature", "importance"]), #importance_path = f"./importance/{type_name}feature_importance_summary.csv", threshold=-0.05, reverse=True, ) else: type_use_feats = use_feats.copy() select_use_feats = features.select_type_feats( type_use_feats, type_name) use_feats_dict[type_name] = select_use_feats clfs, importances, val_score, oof_preds = \ training.train( train_df.loc[type_idx], train_y.loc[type_idx], use_feats = select_use_feats, train_type = type_name, #permutation=True, permutation=False, cat_cols = cat_cols, log_dir = log_dir, ) types_clfs[type_name] = clfs types_scores[type_name] = val_score utils.save_importances(importances_=importances, save_dir=log_dir, prefix=type_name) oof[type_idx] = oof_preds oof_df = pd.DataFrame({ "id": train_df.loc[type_idx, "id"], "oof_preds": oof_preds }) oof_df.to_csv(os.path.join(log_dir, "oof_{}.csv".format(type_name)), index=False) for type_name, score in types_scores.items(): logging.info("{0} : {1}".format(type_name, score)) total_cv = competition_metric(train_df, oof) logging.info("TotalCV = {:.5f}".format(total_cv)) del train_df gc.collect() # prediction with utils.timer("prediction"): prediction.predict(types_clfs, test_df, use_feats_dict=use_feats_dict, val_score=total_cv, log_dir=log_dir) return total_cv
def main(): parser = argparse.ArgumentParser( description='Create an optimized adhoc sensor network') parser.add_argument('--interest-areas', dest='interest_areas', required=True, help='a path to the interest areas json file') parser.add_argument( '--fitness-function', dest='fitness_function', required=True, type=int, help='1 (sum square cc size) or 3 (harmonic avg path length)') parser.add_argument( '--output-base-dir', dest='output_dir', required=True, help= 'The GA process output folder path. (process visualization and result)' ) parser.add_argument('--iterations', dest='iterations', required=False, type=int, default=300, help='The number of iterations for evolution.') parser.add_argument('--initial-population', dest='initial_population', required=False, type=int, default=10, help='The size of the initial population for the GA') parser.add_argument('--mutation-factor', dest='mutation_factor', required=False, type=float, default=1, help='The probability of mutation') parser.add_argument( '--visualize', dest='visualize', required=False, type=str2bool, default=False, help='How many processes to spawn for parallel calculation') parser.add_argument( '--parallel', dest='parallel', required=False, type=str2bool, default=False, help='How many processes to spawn for parallel calculation') parser.add_argument('--optimization-method', dest='optimization_method', required=False, default='ga') args = parser.parse_args() logger.info('validating and loading interest areas from %s', args.interest_areas) interest_areas = load_interest_areas(args.interest_areas) fitness_function = FitnessFunctions.get_fitness_function( args.fitness_function) run_id = '{}_{}'.format( args.fitness_function, hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:8]) if args.optimization_method == 'ga': logger.info('creating initial population of size %s', args.initial_population) with timer(op_name='evolution', logger=logger): if args.parallel: logger.info("starting GA process (%s) asynchronously", run_id) from multiprocessing.pool import Pool with Pool() as pool: ga = ParallelGA( interest_areas=interest_areas, initial_population_size=args.initial_population, generations=args.iterations, fitness_function=fitness_function, optimum=FitnessFunctions.get_fitness_function_optimum( args.fitness_function), pool=pool, mutation_factor=args.mutation_factor, run_id=run_id) ga.generate_initial_population() ga.evolve(logger=logger) else: logger.info("starting GA process (%s) synchronously", run_id) ga = GA(interest_areas=interest_areas, initial_population_size=args.initial_population, generations=args.iterations, fitness_function=fitness_function, optimum=FitnessFunctions.get_fitness_function_optimum( args.fitness_function), mutation_factor=args.mutation_factor, run_id=run_id) ga.generate_initial_population() ga.evolve(logger=logger) create_ga_process_files(process=ga, output_dir=args.output_dir, visualize_ga=args.visualize) elif args.optimization_method == 'sgd': sgd = SGD(run_id=run_id, interest_areas=interest_areas, fitness_function=fitness_function, optimum=FitnessFunctions.get_fitness_function_optimum( args.fitness_function), iterations=args.iterations) sgd.evolve(logger=logger) create_ga_process_files(process=sgd, output_dir=args.output_dir, visualize_ga=args.visualize) else: logger.error('Unknown optimization method %s. Please use GA or SGD', args.optimization_method) logger.info('Finished optimization process')
how="left") ##cosineを求める vec_0 = edge_df[["x_0", "y_0", "z_0"]].values vec_1 = edge_df[["x_1", "y_1", "z_1"]].values vec_2 = edge_df[["x_2", "y_2", "z_2"]].values edge_df["cos_012"] = np.einsum("ij,ij->i",(vec_0-vec_1),(vec_2-vec_1)) \ / (edge_df["dist_0"]*edge_df["dist_1"]) edge_df.drop( ["x_0", "x_1", "x_2", "y_0", "y_1", "y_2", "z_0", "z_1", "z_2"], axis=1, inplace=True) edge_df.to_pickle(f"../data/edge_angle/{molecule}.pkl") if __name__ == "__main__": with utils.timer("make_feature_per_molecule"): for mode in ["train", "test"]: meta_df = pd.read_pickle(f"../pickle/{mode}.pkl").set_index("id") molecules = meta_df["molecule_name"].unique().tolist() st_df = pd.read_pickle("../pickle/structures.pkl") ## train or validのstructureに絞る st_df = st_df[st_df.molecule_name.isin(molecules)]\ [["molecule_name","atom_index","atom","x","y","z"]] # 分子単位に処理 st_gr = st_df.groupby("molecule_name") st_dict = {} for molecule in tqdm(molecules): st_dict[molecule] = st_gr.get_group(molecule) all_file_num = len(molecules) with Pool(4) as p: res = p.map(make_per_molecule, molecules)
def learn(): with utils.timer("load train and test data"): train_pair = compe_data.read_train() test_pair = compe_data.read_test() st_df = compe_data.read_structures() with utils.timer("make atomic data frame"): train_df, test_df = make_atomic_data(train_pair, test_pair, st_df) # set validation fold validation.set_fold( fold_type = "GroupKFold", fold_num = settings.fold_num, random_state = settings.fold_seed, shuffle_flg = True, ) # validationのsplitは、本来のターゲットのindexに合わせる validation.make_splits( train_pair, train_pair[features.TARGET_COL], group_col=features.GROUP_COL ) validation.make_atom_splits(train_df, train_pair) # make feature with utils.timer("feature make"): train_df, test_df, cat_cols = atom_features.make( train_df, test_df, train_pair, test_pair ) pd.set_option("max_columns",100) ## for debug train_df.head(10).to_csv( os.path.join(log_dir,"check_atom_train_df.csv"),index=False) # predict only train feature # training with utils.timer("atomic_meta feature training"): meta_train_df = pd.DataFrame() meta_test_df = pd.DataFrame() meta_features = compe_data.read_mulliken_charges() #仮。変わるかも for meta_col in ["mulliken_charge"]: meta_feat = meta_features[meta_col] logging.info("-------- learning {} ---------".format(meta_col)) use_feats = [x for x in train_df.columns if not x in atom_features.EXCEPT_FEATURES] train_y = meta_feat # training per type test_preds = np.zeros(len(test_df)) clfs, importances, val_score, oof_preds = \ atom_training.train( train_df, train_y, use_feats = use_feats, permutation=False, cat_cols = cat_cols, log_dir = log_dir, target_mode = "meta" ) utils.save_importances(importances_=importances, save_dir=log_dir, prefix="meta_col_") for clf in clfs: test_preds += clf.predict(test_df.loc[:, use_feats], num_iteration=clf.best_iteration) / len(clfs) meta_train_df[meta_col] = oof_preds meta_test_df[meta_col] = test_preds meta_train_df["molecule_name"] = train_df["molecule_name"] meta_train_df["atom_index"] = train_df["atom_index"] meta_test_df["molecule_name"] = test_df["molecule_name"] meta_test_df["atom_index"] = test_df["atom_index"] # merge train and test_df meta_train_df.to_pickle("../pickle/atomic_meta_train.pkl") meta_test_df.to_pickle("../pickle/atomic_meta_test.pkl")
torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # ========== 1) Import and sort data ========== log.info("Loading data, create datasets") start = time.time() # TODO: Make this part into a module "make_dataset.py" dataset_name = "avatar8subs.npz" files_number = 24 # There should be 24 files (8 subjects x 3 trials) make_datasets(args, DATA_EEG, DATA_KIN, DIR_DATA_ALL, DIR_DATA_EACH, dataset_name, files_number) log.info("Finished loading and creating datasets") end = time.time() hours, mins, seconds = timer(start, end) log.info( f"Data loading took: {int(hours)} Hours {int(mins)} minutes {round(seconds, 3)} seconds") # Define some variables to keep track of the results num_eeg_train = len(list(DIR_DATA_EACH.glob('*.npz'))) key_eeg_train = [i.stem for i in DIR_DATA_EACH.glob('*.npz')] sep_fraction = 0.8 # If 0.8, 80% for train and 20% validation joints = ('hip', 'knee', 'ankle') dsets = ('train', 'valid', 'test') dtype = ('eeg_train', 'eeg_test', 'kin_train', 'kin_test') metrics = ('Hip R-val', 'Knee R-val', 'Ankle R-val', 'MSE', 'Hip R2', 'Knee R2', 'Ankle R2') # ========== 2) Training for each trial ========== log.info("Start training each trial") start = time.time()
def batch_loop(args,e,eval,dataset,loader,msg_pre='',exit_at_first_fwd=False): # Prepare if eval: model.eval() else: model.train() # Loop cum_losses=0 cum_num=0 msg_post='' for b,(x,info) in enumerate(loader): # Prepare data s=info[:,3].to(args.device) x=x.to(args.device) if not eval and args.augment>0: if args.augment>1: x=batch_data_augmentation.noiseg(x,0.001) x=batch_data_augmentation.emphasis(x,0.2) x=batch_data_augmentation.magnorm_flip(x,1) if args.augment>1: x=batch_data_augmentation.compress(x,0.1) # Forward z,log_det=model.forward(x,s) loss,losses=utils.loss_flow_nll(z,log_det) """ # Test reverse if e==0 and b==5: with torch.no_grad(): model.eval() z,_=model.forward(x,s) xhat=model.reverse(z,s) dif=(x-xhat).abs() print() print(z[0].view(-1).cpu()) print(xhat[0].view(-1).cpu()) print('AvgDif =',dif.mean().item(),' MaxDif =',dif.max().item()) print(dif.view(-1).cpu()) sys.exit() #""" # Exit? if exit_at_first_fwd: return loss,losses,msg_pre # Backward if not eval: optim.zero_grad() loss.backward() optim.step() # Report/print cum_losses+=losses*len(x) cum_num+=len(x) msg='\r| T = '+utils.timer(tstart,time.time())+' | ' msg+='Epoch = {:3d} ({:5.1f}%) | '.format(e+1,100*(b*args.sbatch+len(info))/len(dataset)) if eval: msg_post='Eval loss = ' else: msg_post='Train loss = ' for i in range(len(cum_losses)): msg_post+='{:7.2f} '.format(cum_losses[i]/cum_num) msg_post+='| ' print(msg+msg_pre+msg_post,end='') cum_losses/=cum_num return cum_losses[0],cum_losses,msg_pre+msg_post
def test(self): epoch = self.scheduler.last_epoch + 1 self.ckp.write_log('\nEvaluation: ') self.ckp.add_log(torch.zeros(1, len(self.loader_test), len(self.scale))) self.model.eval() timer_test = utils.timer() if self.args.save_results: self.ckp.begin_background() with torch.no_grad(): for idx_data, d in enumerate(self.loader_test): for idx_scale, scale in enumerate(self.scale): d.dataset.set_scale(idx_scale) eval_acc = 0 eval_acc_ssim = 0 for _input, _target, filename, _ in tqdm(d, ncols=80): filename = filename[0] _input, _target = self.prepare(_input, _target) timer_test.tic() logits = self.model(_input, idx_scale) timer_test.hold() logits = utils.quantize(logits, self.args.rgb_range) save_list = [logits] eval_acc += utils.calc_psnr( logits, _target, self.scale[idx_scale], self.args.rgb_range, benchmark=d.dataset.benchmark) eval_acc_ssim += utils.calc_ssim( logits, _target, self.scale[idx_scale], benchmark=d.dataset.benchmark) save_list.extend([_input, _target]) if self.args.save_results: self.ckp.save_results(filename, save_list, self.scale[idx_scale]) self.ckp.log[-1, idx_data, idx_scale] = eval_acc / len(d) best = self.ckp.log.max(0) self.ckp.write_log( '\n[{} x{}]\tPSNR: {:.3f}\tSSIM: {:.4f}(Best: {:.3f} @epoch {})' .format(d.dataset.name, self.scale[idx_scale], self.ckp.log[-1, idx_data, idx_scale], eval_acc_ssim / len(d), best[0][idx_data, idx_scale], best[1][idx_data, idx_scale] + 1)) if len(self.scale) == 1 and len(self.loader_test) == 1: self.ckp.visual("valid_PSNR", self.ckp.log[-1, idx_data, idx_scale], epoch) self.ckp.visual("valid_SSIM", eval_acc_ssim / len(d), epoch) self.ckp.write_log('Forward: {:.2f}s\n'.format(timer_test.toc())) self.ckp.write_log('Saving...') if self.args.save_results: self.ckp.end_background() if not self.args.test_only: self.ckp.save(self, epoch, is_best=(best[1][0, 0] + 1 == epoch)) self.ckp.write_log('Total: {:.2f}s\n'.format(timer_test.toc()), refresh=True)
# Seed np.random.seed(args.seed) torch.manual_seed(args.seed) if args.device == 'cuda': torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.cuda.manual_seed(args.seed) ######################################################################################################################## # Load model, pars, & check print('Load stuff') report, pars, model, _ = utils.load_stuff(args.base_fn_model) traintimes, lens, _, losses = report print('Training time = ' + utils.timer(traintimes[0], traintimes[1])) print('Training parameters =', pars) print('Training set lengths =', lens) try: losses_train, losses_valid, losses_test = np.vstack( losses['train']), np.vstack(losses['valid']), losses['test'] print('Best losses = ', np.min(losses_train, axis=0), np.min(losses_valid, axis=0), losses_test) except: print('[Could not load losses]') print('-' * 100) model = model.to(args.device) utils.print_model_report(model, verbose=1) print('-' * 100) # Check lchunk & stride
def search(self): self.scheduler.step() self.loss.step() epoch = self.scheduler.last_epoch + 1 lr = self.scheduler.get_lr()[0] self.ckp.visual("lr", lr, epoch) temperature = self.model.temperature self.ckp.visual("temp", temperature, epoch) genotype = self.model.genotype() self.ckp.write_log( '[Epoch {}]\tLearning rate: {:.2e}\tTemperature: {:.4e}\nGenotype: {}'.format( epoch, Decimal(lr), temperature, genotype ) ) if not self.args.use_sparsemax: print(F.softmax(self.model.alphas_normal, dim=-1)) self.loss.start_log() self.model.train() timer_data, timer_model = utils.timer(), utils.timer() for batch, (_input, _target, _, _) in enumerate(self.loader_search): _input, _target = self.prepare(_input.detach(), _target.detach()) timer_data.hold() timer_model.tic() input_search, target_search, _, _ = next(iter(self.loader_valid)) input_search, target_search = self.prepare( input_search.detach(), target_search.detach()) self.architect.step(_input, _target, input_search, target_search, lr, self.optimizer, unrolled=self.args.unrolled) self.optimizer.zero_grad() logits = self.model(_input) loss = self.loss(logits, _target) loss.backward(retain_graph=True) # TODO:check the diff between clip_grad_norm and clip_grad_value_(in EDSR) nn.utils.clip_grad_norm_(self.model.parameters(), self.args.grad_clip) self.optimizer.step() timer_model.hold() if (batch + 1) % self.args.print_every == 0: self.ckp.write_log('[{}/{}\t{}\t{:.1f}+{:.1f}s]'.format( (batch + 1) * self.args.batch_size, len(self.loader_search.dataset), self.loss.display_loss(batch), timer_model.release(), timer_data.release() )) timer_data.tic() final_loss = self.loss.end_log(len(self.loader_search)) final_loss = final_loss.numpy()[-1] self.ckp.visual("train_loss", final_loss, epoch) self.error_last = self.loss.log[-1, -1] if self.args.use_temp: self.model.temp_update(epoch)
print('Loop files') n = 0 for path_in in args.path_in: print(path_in) for dirpath, dirnames, filenames in os.walk(path_in): newpath = dirpath.replace(path_in, args.path_out) if not os.path.exists(newpath): os.system('mkdir ' + newpath) for fn in filenames: if not fn.endswith(args.extension): continue n += 1 fullfn_in = os.path.join(dirpath, fn) fullfn_out = fullfn_in.replace(path_in, args.path_out).replace( args.extension, '.pt') print( str(n) + '\t' + utils.timer(tstart, time.time()) + '\t' + fullfn_in + ' --> ' + fullfn_out) x, _ = librosa.load(fullfn_in, sr=args.sr) if not args.nonormalize: x -= np.mean(x) x *= args.maxmag / (np.max(np.abs(x)) + 1e-7) torch.save(torch.HalfTensor(x), fullfn_out) print('Done') print('[Do you need to run misc/rename_dataset.py? Check the readme]') ########################################################################################################################
def execute(): with utils.timer("load train and test data"): train_df = compe_data.read_train() test_df = compe_data.read_test() st_df = compe_data.read_structures() # set validation fold validation.set_fold( fold_type="GroupKFold", fold_num=2, #settings.fold_num, random_state=2222, shuffle_flg=True, ) validation.make_splits(train_df, train_df[features.TARGET_COL], group_col=features.GROUP_COL) # make feature with utils.timer("feature make"): train_df, test_df, cat_cols = features.make(train_df, test_df, st_df) del test_df gc.collect() # predict only train feature # training #with utils.timer("reduce memory"): # train_df = utils.reduce_memory(train_df) # #test_df = utils.reduce_memory(test_df) flg_use_scc_meta = True #False if flg_use_scc_meta: with utils.timer("meta feature training"): flg_meta_learn = True feature_selection = True if flg_meta_learn: meta_train_df = pd.DataFrame() meta_features = compe_data.read_scalar_coupling_contributions() for meta_col in ["fc", "sd", "pso", "dso"]: meta_feat = meta_features[meta_col] logging.info( "******************************".format(meta_col)) logging.info( "******** learning {} *********".format(meta_col)) logging.info( "******************************".format(meta_col)) use_feats = [ x for x in train_df.columns if not x in features.EXCEPT_FEATURES ] train_y = meta_feat #train_df[features.TARGET_COL] # training per type types = train_df["type"].unique() oof = np.zeros(len(train_df)) types_clfs = {} types_scores = {} for type_name in types: logging.info("----- training type == {} -----".format( type_name)) type_idx = train_df.type == type_name if feature_selection: type_use_feats = feature_util.feature_select( use_feats, importance_df=pd.read_csv( f"./importance/permu_importance{type_name}.csv", names=["feature", "importance"]), threshold=-0.05, reverse=True, ) else: type_use_feats = use_feats.copy() select_feats = features.select_type_feats( use_feats, type_name) param_tuning( train_df=train_df.loc[type_idx], y=train_y.loc[type_idx], use_feats=select_feats, objective_metric= mean_absolute_error, #competition_metric, train_type=type_name, type_name=meta_col + "_" + type_name, cat_cols=cat_cols, ) else: meta_train_df = pd.read_pickle("../pickle/meta_train.pkl") #train_df = utils.fast_concat(train_df, meta_train_df) #test_df = utils.fast_concat(test_df, meta_test_df) """
os.system('mkdir '+args.path_out) print('Loop files') n=0 for path_in in args.path_in: print(path_in) for dirpath,dirnames,filenames in os.walk(path_in): newpath=dirpath.replace(path_in,args.path_out) if not os.path.exists(newpath): os.system('mkdir '+newpath) for fn in filenames: if not fn.endswith(args.extension): continue n+=1 fullfn_in=os.path.join(dirpath,fn) fullfn_out=fullfn_in.replace(path_in,args.path_out).replace(args.extension,'.pt') print(str(n)+'\t'+utils.timer(tstart,time.time())+'\t'+fullfn_in+' --> '+fullfn_out) x,_=librosa.load(fullfn_in,sr=args.sr) if not args.nonormalize: x-=np.mean(x) x*=args.maxmag/(np.max(np.abs(x))+1e-7) torch.save(torch.HalfTensor(x),fullfn_out) print('Done') print('[Do you need to run misc/rename_dataset.py? Check the readme]') ########################################################################################################################
def main(cfg): ########################################################################### ######################### Some reminders to print ######################### ########################################################################### if cfg.to_log: print(f'log_path: {cfg.log_path}') print(f'model_checkpoint_path: {cfg.model_checkpoint_path}') ########################################################################### torch.manual_seed(0) np.random.seed(0) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.cuda.set_device(cfg.device_ids[0]) train_dataset = ActivityNetCaptionsIteratorDataset( cfg.start_token, cfg.end_token, cfg.pad_token, cfg.min_freq, cfg.train_batch_size, cfg.video_features_path, cfg.video_feature_name, cfg.filter_video_feats, cfg.average_video_feats, cfg.audio_features_path, cfg.audio_feature_name, cfg.filter_audio_feats, cfg.average_audio_feats, cfg.train_meta_path, cfg.val_1_meta_path, cfg.val_2_meta_path, torch.device(cfg.device), 'train', cfg.modality, cfg.use_categories, props_are_gt=True, get_full_feat=False ) val_1_dataset = ActivityNetCaptionsIteratorDataset( cfg.start_token, cfg.end_token, cfg.pad_token, cfg.min_freq, cfg.inference_batch_size, cfg.video_features_path, cfg.video_feature_name, cfg.filter_video_feats, cfg.average_video_feats, cfg.audio_features_path, cfg.audio_feature_name, cfg.filter_audio_feats, cfg.average_audio_feats, cfg.train_meta_path, cfg.val_1_meta_path, cfg.val_2_meta_path, torch.device(cfg.device), 'val_1', cfg.modality, cfg.use_categories, props_are_gt=True, get_full_feat=False ) val_2_dataset = ActivityNetCaptionsIteratorDataset( cfg.start_token, cfg.end_token, cfg.pad_token, cfg.min_freq, cfg.inference_batch_size, cfg.video_features_path, cfg.video_feature_name, cfg.filter_video_feats, cfg.average_video_feats, cfg.audio_features_path, cfg.audio_feature_name, cfg.filter_audio_feats, cfg.average_audio_feats, cfg.train_meta_path, cfg.val_1_meta_path, cfg.val_2_meta_path, torch.device(cfg.device), 'val_2', cfg.modality, cfg.use_categories, props_are_gt=True, get_full_feat=False ) # 'val_1' in phase doesn't really matter because props are for validation set # cfg.val_1_meta_path -> cfg.val_prop_meta val_pred_prop_dataset = ActivityNetCaptionsIteratorDataset( cfg.start_token, cfg.end_token, cfg.pad_token, cfg.min_freq, cfg.inference_batch_size, cfg.video_features_path, cfg.video_feature_name, cfg.filter_video_feats, cfg.average_video_feats, cfg.audio_features_path, cfg.audio_feature_name, cfg.filter_audio_feats, cfg.average_audio_feats, cfg.train_meta_path, cfg.val_prop_meta_path, cfg.val_2_meta_path, torch.device(cfg.device), 'val_1', cfg.modality, cfg.use_categories, props_are_gt=False, get_full_feat=False ) # make sure that DataLoader has batch_size = 1! train_loader = DataLoader(train_dataset, collate_fn=train_dataset.dont_collate) val_1_loader = DataLoader(val_1_dataset, collate_fn=val_1_dataset.dont_collate) val_2_loader = DataLoader(val_2_dataset, collate_fn=val_2_dataset.dont_collate) val_pred_prop_loader = DataLoader(val_pred_prop_dataset, collate_fn=val_2_dataset.dont_collate) model = SubsAudioVideoTransformer( train_dataset.trg_voc_size, train_dataset.subs_voc_size, cfg.d_aud, cfg.d_vid, cfg.d_model_audio, cfg.d_model_video, cfg.d_model_subs, cfg.d_ff_audio, cfg.d_ff_video, cfg.d_ff_subs, cfg.N, cfg.N, cfg.N, cfg.dout_p, cfg.H, cfg.use_linear_embedder ) criterion = LabelSmoothing(cfg.smoothing, train_dataset.pad_idx) # lr = 0 here have no impact on training (see lr scheduler) optimizer = torch.optim.Adam( model.parameters(), 0, (cfg.beta1, cfg.beta2), cfg.eps ) lr_scheduler = SimpleScheduler(optimizer, cfg.lr) loss_compute = SimpleLossCompute(criterion, lr_scheduler) model.to(torch.device(cfg.device)) # haven't tested for multi GPU for a while -- might not work. model = torch.nn.DataParallel(model, cfg.device_ids) param_num = sum(p.numel() for p in model.parameters() if p.requires_grad) print(f'Param Num: {param_num}') if cfg.to_log: os.makedirs(cfg.log_path) os.makedirs(cfg.model_checkpoint_path, exist_ok=True) # handles the case when model_checkpoint_path = log_path TBoard = tensorboard.SummaryWriter(log_dir=cfg.log_path) TBoard.add_text('config', cfg.get_params('md_table'), 0) TBoard.add_text('config/comment', cfg.comment, 0) TBoard.add_scalar('debug/param_number', param_num, 0) else: TBoard = None # keeping track of the best model best_metric = 0 # "early stopping" thing num_epoch_best_metric_unchanged = 0 for epoch in range(cfg.start_epoch, cfg.epoch_num): num_epoch_best_metric_unchanged += 1 if (num_epoch_best_metric_unchanged == cfg.early_stop_after) or (timer(cfg.curr_time) > 67): print(f'Early stop at {epoch}: unchanged for {num_epoch_best_metric_unchanged} epochs') print(f'Current timer: {timer(cfg.curr_time)}') break # train training_loop( model, train_loader, loss_compute, lr_scheduler, epoch, TBoard, cfg.modality, cfg.use_categories ) # validation (next word) val_1_loss = validation_next_word_loop( model, val_1_loader, greedy_decoder, loss_compute, lr_scheduler, epoch, cfg.max_len, cfg.videos_to_monitor, TBoard, cfg.modality, cfg.use_categories ) val_2_loss = validation_next_word_loop( model, val_2_loader, greedy_decoder, loss_compute, lr_scheduler, epoch, cfg.max_len, cfg.videos_to_monitor, TBoard, cfg.modality, cfg.use_categories ) val_loss_avg = (val_1_loss + val_2_loss) / 2 # validation (1-by-1 word) if epoch >= cfg.one_by_one_starts_at: # validation with g.t. proposals val_1_metrics = validation_1by1_loop( model, val_1_loader, greedy_decoder, loss_compute, lr_scheduler, epoch, cfg.max_len, cfg.log_path, cfg.verbose_evaluation, [cfg.reference_paths[0]], cfg.tIoUs, cfg.max_prop_per_vid, TBoard, cfg.modality, cfg.use_categories, ) val_2_metrics = validation_1by1_loop( model, val_2_loader, greedy_decoder, loss_compute, lr_scheduler, epoch, cfg.max_len, cfg.log_path, cfg.verbose_evaluation, [cfg.reference_paths[1]], cfg.tIoUs, cfg.max_prop_per_vid, TBoard, cfg.modality, cfg.use_categories, ) if cfg.to_log: # averaging metrics obtained from val_1 and val_2 metrics_avg = average_metrics_in_two_dicts(val_1_metrics, val_2_metrics) metrics_avg = metrics_avg['Average across tIoUs'] TBoard.add_scalar('metrics/val_loss_avg', val_loss_avg, epoch) TBoard.add_scalar('metrics/meteor', metrics_avg['METEOR'] * 100, epoch) TBoard.add_scalar('metrics/bleu4', metrics_avg['Bleu_4'] * 100, epoch) TBoard.add_scalar('val_avg/bleu3', metrics_avg['Bleu_3'] * 100, epoch) TBoard.add_scalar('val_avg/bleu2', metrics_avg['Bleu_2'] * 100, epoch) TBoard.add_scalar('val_avg/bleu1', metrics_avg['Bleu_1'] * 100, epoch) TBoard.add_scalar('val_avg/rouge_l', metrics_avg['ROUGE_L'] * 100, epoch) TBoard.add_scalar('val_avg/cider', metrics_avg['CIDEr'] * 100, epoch) TBoard.add_scalar('val_avg/precision', metrics_avg['Precision'] * 100, epoch) TBoard.add_scalar('val_avg/recall', metrics_avg['Recall'] * 100, epoch) # saving the model if it is better than the best so far if best_metric < metrics_avg['METEOR']: best_metric = metrics_avg['METEOR'] save_model( cfg, epoch, model, optimizer, val_1_loss, val_2_loss, val_1_metrics, val_2_metrics, train_dataset.trg_voc_size ) # reset the early stopping criterion num_epoch_best_metric_unchanged = 0 # put it after: so on zeroth epoch it is not zero TBoard.add_scalar('val_avg/best_metric_meteor', best_metric * 100, epoch) if cfg.to_log: # load the best model best_model_path = os.path.join(cfg.model_checkpoint_path, 'best_model.pt') checkpoint = torch.load(best_model_path) model.load_state_dict(checkpoint['model_state_dict']) val_metrics_pred_prop = validation_1by1_loop( model, val_pred_prop_loader, greedy_decoder, loss_compute, lr_scheduler, checkpoint['epoch'], cfg.max_len, cfg.log_path, cfg.verbose_evaluation, cfg.reference_paths, cfg.tIoUs, cfg.max_prop_per_vid, TBoard, cfg.modality, cfg.use_categories ) best_metric_pred_prop = val_metrics_pred_prop['Average across tIoUs']['METEOR'] print(f'best_metric: {best_metric}') print(f'best_metric_pred_prop: {best_metric_pred_prop}') TBoard.close()
import torch import models.losses as losses import models.models as models import dataloaders.dataloaders as dataloaders import utils.utils as utils from utils.fid_scores import fid_pytorch import config #--- read options ---# opt = config.read_arguments(train=True) #--- create utils ---# timer = utils.timer(opt) visualizer_losses = utils.losses_saver(opt) losses_computer = losses.losses_computer(opt) dataloader, dataloader_val = dataloaders.get_dataloaders(opt) im_saver = utils.image_saver(opt) fid_computer = fid_pytorch(opt, dataloader_val) #--- create models ---# model = models.OASIS_model(opt) model = models.put_on_multi_gpus(model, opt) #--- create optimizers ---# optimizerG = torch.optim.Adam(model.module.netG.parameters(), lr=opt.lr_g, betas=(opt.beta1, opt.beta2)) optimizerD = torch.optim.Adam(model.module.netD.parameters(), lr=opt.lr_d, betas=(opt.beta1, opt.beta2))
def batch_loop(args, e, eval, dataset, loader, msg_pre='', exit_at_first_fwd=False): # Prepare if eval: model.eval() else: model.train() # Loop cum_losses = 0 cum_num = 0 msg_post = '' for b, (x, info) in enumerate(loader): # Prepare data if not eval and args.augment > 0: if args.augment > 1: x = batch_data_augmentation.noiseg(x, 0.001) x = batch_data_augmentation.emphasis(x, 0.2) x = batch_data_augmentation.magnorm_flip(x, 1) if args.augment > 1: x = batch_data_augmentation.compress(x, 0.1) # get mel x = utils.get_mel(x) s = info[:, 3].to(args.device) x = x.to(args.device) # Forward z, log_det = model.forward(x, s) loss, losses = utils.loss_flow_nll(z, log_det) """ # Test reverse if e==0 and b==5: with torch.no_grad(): model.eval() z,_=model.forward(x,s) xhat=model.reverse(z,s) dif=(x-xhat).abs() print() print(z[0].view(-1).cpu()) print(xhat[0].view(-1).cpu()) print('AvgDif =',dif.mean().item(),' MaxDif =',dif.max().item()) print(dif.view(-1).cpu()) sys.exit() #""" # Exit? if exit_at_first_fwd: return loss, losses, msg_pre # Backward if not eval: optim.zero_grad() loss.backward() optim.step() # Report/print if eval: logger.add_scalar('eval_loss', loss, b + len(loader) * e) else: logger.add_scalar('train_loss', loss, b + len(loader) * e) cum_losses += losses * len(x) cum_num += len(x) msg = '\r| T = ' + utils.timer(tstart, time.time()) + ' | ' msg += 'Epoch = {:3d} ({:5.1f}%) | '.format(e + 1, 100 * ( b * args.sbatch + len(info)) / len(dataset)) if eval: msg_post = 'Eval loss = ' else: msg_post = 'Train loss = ' for i in range(len(cum_losses)): msg_post += '{:7.4f} '.format(cum_losses[i] / cum_num) msg_post += '| ' print(msg + msg_pre + msg_post, end='') if not eval: if (b + len(loader) * e) % 2000 == 0: fpath = os.path.join(args.base_fn_out, 'ckpt_{}'.format(b + len(loader) * e)) to_save = vars(args) try: model_state_dict = model.module.state_dict() except: model_state_dict = model.state_dict() to_save.update({ 'epoch': e, 'iter': b + len(loader) * e, 'model_state_dict': model_state_dict, 'optimizer_state_dict': optim.state_dict(), 'lr': lr, # shadowing the original lr 'loss_best': loss_best, 'losses': losses_track, }) torch.save(to_save, fpath + '.pt') torch.save(model.module, args.base_fn_out + '/ckpt_{}.model.pt'.format(b + len(loader) * e)) torch.save(optim, args.base_fn_out + '/ckpt_{}.optim.pt'.format(b + len(loader) * e)) make_audio_evals(dataset_test, loader_test, fpath, b + len(loader) * e, args.sw_path) cum_losses /= cum_num return cum_losses[0], cum_losses, msg_pre + msg_post