logger.info("Configuration file loaded from %s" % args.config) # read input csv file as pandas dataframe if args.input is not None: input = pd.read_csv(args.input, low_memory=False) logger.info('Input data loaded from %s', args.input) if args.step == 'acquire': output = acquire(**config['acquire']) elif args.step == 'featurize': output = featurize(input, **config['featurize']) elif args.step == 'clean': output = clean(input, **config['clean']) elif args.step == 'train': output, output1 = train(input, **config['train']) else: logger.warning('No such argument as %s', args.step) if args.output is not None: # if train, output model pkl file, else output csv file if args.step == 'train': try: pickle.dump(output, open(args.output, 'wb')) except: logger.error("Model object can't be dumped!") output1.to_csv(args.output1, index=False) logger.info("Output saved to %s" % args.output1) else:
def optimize_hp(opts): opts_list = get_opts_combs(opts) for opts in opts_list: train(opts)
import argparse # TODO: Refactoring needed from src.train import main_train as train from src.inference import main_inference as inference from src.constants import hyperparams, model_config if __name__ == "__main__": parser = argparse.ArgumentParser(description='Training start') parser.add_argument('env', metavar='ENV', type=str, default='localhost') parser.add_argument('mode', metavar='RUNNING', type=str, default='train') args = parser.parse_args() print('Environment: ', args.env) print('Mode: ', args.mode) if args.mode == 'train': train(args.env) else: if hyperparams['mixout_epochs'] == 0: ckpt_name = f'{model_config["model_name"]}_epoch{hyperparams["epochs"]}_{hyperparams["finetuning_epochs"]}' else: ckpt_name = f'{model_config["model_name"]}_epoch{hyperparams["epochs"]}_{hyperparams["finetuning_epochs"]}_{hyperparams["mixout_epochs"]}' inference( args.env, ckpt_name=ckpt_name, threshold_strategy="exact", ## "percentile" or "exact" threshold_percentile=100 - 0.59, ## for threshold_strategy == "percentile" threshold_exact=0.5, ## for threshold_strategy == "exact" if_use_log=False, logfile='011458.csv')
# about path tf.flags.DEFINE_string("data_name", "Fairchild_HDR", "data_name is Fairchild_HDR or Funt_HDR") tf.flags.DEFINE_string( "train_dir", "./dataset/training_samples", "Path to training image directory or an individual image") tf.flags.DEFINE_string( "valid_dir", "./dataset/valid_samples", "Path to valid image directory or an individual image") tf.flags.DEFINE_string( "test_dir", "./dataset/testing_samples", "Path to testing image directory or an individual image") tf.flags.DEFINE_string("out_dir", "results", "Path to output directory") tf.flags.DEFINE_string("summary_dir", "summary", "Path to output directory") tf.flags.DEFINE_string("dm", "./models_dm", "Path to trained CNN dm_weights") tf.flags.DEFINE_string("um", "./models_um", "Path to trained CNN um_weights") if FLAGS.model_type == 0: train(FLAGS) elif FLAGS.model_type == 1: test(FLAGS) else: predict(FLAGS) # Program over print('success! u are a smart boy!')
from env import OhmniInSpace from src import train if sys.argv[1] == '--test': if sys.argv[2] == 'py-env': ois = OhmniInSpace.PyEnv(gui=True) timestep = ois.reset() while not timestep.is_last(): timestep = ois.step(action=(0.4, 0.4)) (_, reward, discount, observation) = timestep print('Reward:', reward) ois.render() cv.imshow('Segmentation', observation) if cv.waitKey(10) & 0xFF == ord('q'): break if sys.argv[2] == 'tf-env': ois = OhmniInSpace.TfEnv() tf_env = ois.gen_env() print("TimeStep Specs:", tf_env.time_step_spec()) print("Action Specs:", tf_env.action_spec()) elif sys.argv[1] == '--ohmni': if sys.argv[2] == 'train': train.train() if sys.argv[2] == 'run': train.run() else: print("Error: Invalid option!")
def main(config, run_preprocessing, run_data_upload, log_dir): # Load experiment configuration with open(config) as f: config = yaml.load(f, Loader=yaml.FullLoader) # Get db connection conn = sql_utils.get_connection() # Get basic info of experiment exp_version = config['version'] exp_name = config["experiment_name"] exp_time = date_utils.get_current_time_string()[2:] username = getpass.getuser()[0] terminal_width = int(os.popen('stty size', 'r').read().split()[1]) print( f'Running Experiment: {username}_{exp_version}_{exp_name}_{exp_time}\n{"-" * terminal_width}\n' ) # Preprocessing preprocessing_prefix = config['preprocessing_config']['prefix'] if not run_preprocessing: print('Preprocessing skipped.') else: print('Preprocessing ...') run_preprocess(conn, config['preprocessing_config'], run_data_upload=run_data_upload) print('Preprocessing done.') # Get temporal configuration information train_dates_list, test_dates_list = parse_temporal_config( config['temporal_config']) # Training and evaluation test_results_over_time = [] experiment_loop = tqdm.tqdm(list(zip(train_dates_list, test_dates_list)), desc='Experiment Repeats') for train_dates, test_dates in experiment_loop: split_time_abbr = date_utils.date_to_string( test_dates['label_start_time']) split_time_abbr = split_time_abbr.replace('-', '')[2:] split_name = f'{split_time_abbr}' print(split_name) prefix = f'{username}_{exp_version}_{exp_name}_{exp_time}_{split_name}' experiment_table_prefix = f'experiments.{prefix}' train_save_dir = os.path.join(os.getcwd(), log_dir, prefix, 'train_' + exp_time) test_save_dir = os.path.join(os.getcwd(), log_dir, prefix, 'test_' + exp_time) # Prepare cohort as specified by our experiment configuration tqdm.tqdm.write('\nPreparing cohorts ...') train_feature_splits, train_label_splits = [], [] for i, train_dates_aod in enumerate(train_dates): train_feature_table, train_label_table = prepare_cohort( config, train_dates_aod, test_dates, preprocessing_prefix, experiment_table_prefix + f'_split{i}', include_test=False)[:2] train_feature_splits.append(train_feature_table) train_label_splits.append(train_label_table) test_feature_table, test_label_table = prepare_cohort( config, train_dates[-1], test_dates, preprocessing_prefix, experiment_table_prefix, include_train=False)[2:] train_feature_table = f'{experiment_table_prefix}_train_features' sql_utils.merge_tables(train_feature_splits, train_feature_table) train_label_table = f'{experiment_table_prefix}_train_labels' sql_utils.merge_tables(train_label_splits, train_label_table) # Delete intermediate cohort tables for i in range(len(train_dates)): cohort_table_name = f'{experiment_table_prefix}_split{i}_cohort' sql_utils.run_sql_from_string(conn, f'drop table {cohort_table_name};') # Train models as specified by our experiment configuration tqdm.tqdm.write('Training ...') model_summaries = train(config, train_feature_table, train_label_table, discard_columns=['split'], save_dir=train_save_dir) # Evaluate our models on the training data model_paths = glob.glob(f'{train_save_dir}/*.pkl') tqdm.tqdm.write('Evaluating on training data ...') train_results = evaluate(config, train_feature_table, train_label_table, model_paths, model_summaries, discard_columns=['split'], log_dir=train_save_dir) # Evaluate our models on the test data tqdm.tqdm.write('Evaluating on test data ...') test_results = evaluate(config, test_feature_table, test_label_table, model_paths, model_summaries, save_preds_to_db=True, save_prefix=f'{prefix}_test', log_dir=test_save_dir) test_results_over_time.append(test_results) # Save results to database train_results_name = f'{prefix}_train_results' test_results_name = f'{prefix}_test_results' train_results.to_sql(train_results_name, conn, schema='results') test_results.to_sql(test_results_name, conn, schema='results') # Plot test results over time test_results_tables_prefix = f'{username}_{exp_version}_{exp_name}_{exp_time}' plot_utils.plot_results_over_time(test_results_tables_prefix)
P_LABELS = Config.get("Default", "P_LABELS") P_PIXEL_MASK = Config.get("Default", "P_PIXEL_MASK") METHOD = Config.get("Default", "METHOD") ts = time.time() dt = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') result_path = os.path.join("results", "archive", METHOD, dt) os.makedirs(result_path) logging.basicConfig(filename=os.path.join(result_path, "info.log"), level=logging.INFO) model_path = os.path.join("models", METHOD, dt) os.makedirs(model_path) if METHOD == 'STAE': net = SpatialTemporalAutoencoder(tvol=TVOL, alpha=ALPHA, batch_size=BATCH_SIZE, lambd=LAMBDA) d = DataIteratorStae(P_TRAIN, P_TEST, P_LABELS, P_PIXEL_MASK, batch_size=BATCH_SIZE, tvol=TVOL, taug=TAUG) elif METHOD == 'CONVAE2D': net = ConvAE2d(tvol=TVOL, alpha=ALPHA, batch_size=BATCH_SIZE, lambd=LAMBDA) d = DataIteratorNormal(P_TRAIN, P_TEST, P_LABELS, P_PIXEL_MASK, batch_size=BATCH_SIZE, tvol=TVOL, taug=TAUG) elif METHOD == 'EXP': net = Experiment(tvol=TVOL, alpha=ALPHA, batch_size=BATCH_SIZE, lambd=LAMBDA) d = DataIteratorNormal(P_TRAIN, P_TEST, P_LABELS, P_PIXEL_MASK, batch_size=BATCH_SIZE, tvol=TVOL, taug=TAUG) else: raise ValueError('Incorrect method specification') frame_auc, frame_eer, pixel_auc, pixel_eer = train(data=d, model=net, num_iteration=NUM_ITER, result_path=result_path, model_path=model_path) logging.info("Best frame-level area under the roc curve: {0:g}".format(frame_auc)) logging.info("Frame-level equal error rate corresponding to this: {0:g}".format(frame_eer)) logging.info("Pixel-level area under the roc curve corresponding to this: {0:g}".format(pixel_auc)) logging.info("Pixel-level equal error rate corresponding to this: {0:g}".format(pixel_eer))
def test_one_epoch(): config = ['--no-visualize', '--num-examples=100', '--no-save'] metrics = train(['--epoch=1', '--name=TEST'] + config) assert round(metrics.get_primary_value(), 7) == 4.6288586
def trainFun(status_var, t): text = '样本训练完成' status_var.set(text) train(END, t)
def tune(project_parameters): project_parameters.mode = 'train' hyperparameter_space = _get_hyperparameter_space( project_parameters=project_parameters) tune_scheduler = ASHAScheduler(metric='accuracy_difference', mode='min') reporter = CLIReporter(metric_columns=[ 'train_loss', 'val_loss', 'test_loss', 'train_accuracy', 'val_accuracy', 'test_accuracy', 'accuracy_difference' ]) init(dashboard_host='0.0.0.0') tuning_result = ray.tune.run( run_or_experiment=partial(_tune_function, project_parameters=project_parameters), resources_per_trial={ 'cpu': project_parameters.tune_cpu, 'gpu': project_parameters.tune_gpu }, config=hyperparameter_space, num_samples=project_parameters.tune_iter, scheduler=tune_scheduler, local_dir=join(project_parameters.save_path, 'tuning_logs'), progress_reporter=reporter) best_trial = tuning_result.get_best_trial('accuracy_difference', 'min', 'last') if not project_parameters.tune_debug: project_parameters = _set_tune_project_parameters( hyperparameter=best_trial.config, project_parameters=project_parameters) result = train(project_parameters=project_parameters) result['tune'] = tuning_result else: result = {'tune': tuning_result} print('best trial name: {}'.format(best_trial)) print('best trial result: {}'.format( best_trial.last_result['accuracy_difference'])) print('best trial config: {}'.format(best_trial.config)) if 'cutoff_freq_low' in best_trial.config or 'cutoff_freq_high' in best_trial.config: best_trial.config['cutoff_freq'] = [] for key in ['cutoff_freq_low', 'cutoff_freq_high']: value = best_trial.config.pop(key, None) if value is not None: best_trial.config['cutoff_freq'].append(value) best_trial.config['cutoff_freq'] = ( '{},' * len(best_trial.config['cutoff_freq']))[:-1].format( *best_trial.config['cutoff_freq']) if 'parameters_config_path' in project_parameters: output = 'num_workers: {}'.format(project_parameters.num_workers) for k, v in best_trial.config.items(): if k == 'cutoff_freq': output += '\n{}:\n - {}\n - {}'.format( k, v.split(',')[0], v.split(',')[1]) else: output += '\n{}: {}'.format(k, v) print('best trial config command:\n{}'.format(output)) else: print('best trial config command: --num_workers {}{}'.format( project_parameters.num_workers, (' --{} {}' * len(best_trial.config)).format(*np.concatenate( list(zip(best_trial.config.keys(), best_trial.config.values())))))) shutdown() return result
from src.compare import compare if __name__ == "__main__": parser = argparse.ArgumentParser( prog='main.py', description= 'This program enables you to make and test image captioning model.', epilog='end', add_help=True, ) parser.add_argument('phase', help='to designate the phase (train, eval, infer)') phase = parser.parse_args().phase config = Config() if config.PREPARE_VOCAB: prepare_vocab(config.TRAIN_CAPTION_PATH, config.WORD_TO_ID_PATH, config.ID_TO_WORD_PATH) if phase == 'train': train() elif phase == 'eval': evalate() elif phase == 'infer': infer() elif phase == 'compare': compare() else: print('the argument should be train, eval, or infer')
def main(): if CONFIG['train'] == 'train': data.prepare(CONFIG) train.train(CONFIG, False) else: train.test(CONFIG, 'weighted_sampling', 100)
def run(cfg: DictConfig) -> None: logger.info('=' * 30) logger.info('::: pipeline start :::') logger.info('=' * 30) logger.info(f'git hash is: {hash_}') logger.info(f'all params\n{"="*80}\n{cfg.pretty()}\n{"="*80}') comment = cfg['globals']['comment'] assert comment != None, 'commentを入力してください。(globals.commet=hogehoge)' if cfg['globals']['debug']: logger.info('::: set debug mode :::') cfg = utils.get_debug_config(cfg) global_params = cfg["globals"] utils.set_seed(50) device = C.get_device(global_params["device"]) splitter = C.get_split(cfg) df, datadir = C.get_metadata(cfg) logger.info(f'meta_df: {df.shape}') output_dir = os.getcwd() output_dir_ignore = output_dir.replace('/data/', '/data_ignore/') if not os.path.exists(output_dir_ignore): os.makedirs(output_dir_ignore) for fold_i, (trn_idx, val_idx) in enumerate(splitter.split(df, y=df["ebird_code"])): if fold_i not in global_params["folds"]: continue logger.info("=" * 30) logger.info(f"Fold {fold_i}") logger.info("=" * 30) trn_df = df.loc[trn_idx, :].reset_index(drop=True) val_df = df.loc[val_idx, :].reset_index(drop=True) if global_params['remove_short']: logger.info(f'::: remove short duration :::') trn_df = utils.remove_short_duration(trn_df) if global_params['balanced']: logger.info(f'::: train class balanced :::') trn_df = utils.transform_balanced_dataset(trn_df) if global_params['mixup']: logger.info(f'::: perform mixup :::') if global_params['debug']: trn_df = utils.get_debug_df(trn_df) val_df = utils.get_debug_df(val_df) logger.info(f'trn_df: {trn_df.shape}') logger.info(f'val_df: {val_df.shape}') train_loader = C.get_loader(trn_df, datadir, cfg, 'train') valid_loader = C.get_loader(val_df, datadir, cfg, 'valid') model = models.get_model(cfg).to(device) criterion = C.get_criterion(cfg).to(device) optimizer = C.get_optimizer(model, cfg) scheduler = C.get_scheduler(optimizer, cfg) losses_train = [] losses_valid = [] epochs = [] best_f1 = 0 best_loss = 0 save_path = f'{output_dir_ignore}/{model.__class__.__name__}_fold{fold_i}.pth' early_stopping = EarlyStopping(patience=12, verbose=True, path=save_path) n_epoch = cfg['globals']['num_epochs'] for epoch in progress_bar(range(1, n_epoch + 1)): logger.info(f'::: epoch: {epoch}/{n_epoch} :::') loss_train = train(model, device, train_loader, optimizer, scheduler, criterion, global_params['mixup']) loss_valid, fscore_valid = get_epoch_loss_score( model, device, valid_loader, criterion) logger.info( f'loss_train: {loss_train:.6f}, loss_valid: {loss_valid:.6f}, f1(macro): {fscore_valid:.6f}' ) epochs.append(epoch) losses_train.append(loss_train) losses_valid.append(loss_valid) is_update = early_stopping(loss_valid, model, global_params['debug']) if is_update: best_loss = loss_valid best_f1 = fscore_valid if early_stopping.early_stop: logger.info("Early stopping") break # result handling rh.save_loss_figure(fold_i, epochs, losses_train, losses_valid, output_dir) rh.save_result_csv(fold_i, global_params['debug'], f'{model.__class__.__name__}', cfg['loss']['name'], best_loss, best_f1, comment, output_dir) logger.info( f'best_loss: {best_loss:.6f}, best_fscore(macro): {best_f1:.6f}') logger.info('::: success :::\n\n\n') # 開放 del train_loader del valid_loader del model del optimizer del scheduler gc.collect() torch.cuda.empty_cache()
for model_type in tqdm( ['logistic', 'rf', 'mlp2', 'svm'] ): #,'gb', 'logistic', 'dt', 'svm', 'gb', 'rf', 'mlp2', 'irf']): for num_feats in [ len(feat_names) ]: # [16, 25, len(feat_names)]: #[16, 25, len(feat_names)]: # number of total features to consider for balancing in ['ros']: # None, ros for balancing_ratio in [0.8, 1, 1.2]: # positive: negative hyperparams = [0] if model_type in [ 'logistic', 'rf', 'gb', 'dt', 'irf', 'qda' ] else [-1, 0, 1] for hyperparam in hyperparams: # 0 is default feats = feat_names[:num_feats] out_name = f'{model_type}_{num_feats}_{feature_selection}={feature_selection_num}_{balancing}={balancing_ratio}_h={hyperparam}_cal={calibrated}' train.train( df, feat_names=feats, cell_nums_feature_selection=dset[ 'feature_selection'], cell_nums_train=dset['train'], model_type=model_type, balancing=balancing, balancing_ratio=balancing_ratio, outcome_def=outcome_def, feature_selection=feature_selection, feature_selection_num= feature_selection_num, hyperparam=hyperparam, out_name=f'{out_dir}/{out_name}.pkl', calibrated=calibrated)
def loop(): device = torch.device("cuda") if not torch.cuda.is_available(): sys.exit("Error: CUDA requested but not available") # weighted values for loss functions # add a helper to return weights seamlessly try: weight = torch.Tensor([1.513212, 10.147043]) except KeyError: if model["opt"]["loss"] in ("CrossEntropy", "mIoU", "Focal"): sys.exit("Error: The loss function used, need dataset weights values") # loading Model net = UNet(num_classes) net = DataParallel(net) net = net.to(device) # define optimizer optimizer = Adam(net.parameters(), lr=lr) # resume training if model_path: chkpt = torch.load(model_path, map_location=device) net.load_state_dict(chkpt["state_dict"]) optimizer.load_state_dict(chkpt["optimizer"]) # select loss function, just set a default, or try to experiment if loss_func == "CrossEntropy": criterion = CrossEntropyLoss2d(weight=weight).to(device) elif loss_func == "mIoU": criterion = mIoULoss2d(weight=weight).to(device) elif loss_func == "Focal": criterion = FocalLoss2d(weight=weight).to(device) elif loss_func == "Lovasz": criterion = LovaszLoss2d().to(device) else: sys.exit("Error: Unknown Loss Function value !") #loading data train_loader, val_loader = get_dataset_loaders(target_size, batch_size, dataset_path) history = collections.defaultdict(list) # training loop for epoch in range(0, num_epochs): print("Epoch: " + str(epoch +1)) train_hist = train(train_loader, num_classes, device, net, optimizer, criterion) val_hist = validate(val_loader, num_classes, device, net, criterion) print("Train loss: {:.4f}, mIoU: {:.3f}, {} IoU: {:.3f}, MCC: {:.3f}".format( train_hist["loss"], train_hist["miou"], target_type, train_hist["fg_iou"], train_hist["mcc"])) print("Validation loss: {:.4f}, mIoU: {:.3f}, {} IoU: {:.3f}, MCC: {:.3f}".format( val_hist["loss"], val_hist["miou"], target_type, val_hist["fg_iou"], val_hist["mcc"])) for key, value in train_hist.items(): history["train " + key].append(value) for key, value in val_hist.items(): history["val " + key].append(value) if (epoch+1)%5 == 0: # plotter use history values, no need for log visual = "history-{:05d}-of-{:05d}.png".format(epoch + 1, num_epochs) plot(os.path.join(checkpoint_path, visual), history) if (epoch+1)%20 == 0: checkpoint = target_type + "-checkpoint-{:03d}-of-{:03d}.pth".format(epoch + 1, num_epochs) states = {"epoch": epoch + 1, "state_dict": net.state_dict(), "optimizer": optimizer.state_dict()} torch.save(states, os.path.join(checkpoint_path, checkpoint))
def _run_training(args): train(raw_data_file=args.raw_data_file, processed_data_folderpath=args.processed_data_folderpath, model_folderpath=args.model_folderpath, model_name=args.model_name)
parser.add_argument("--p_mcl", help="probability to execute MCL in Mixture MCL", default=0.95) args = parser.parse_args() if args.dataset == "simulation": cfg.DSOM_X = 129 cfg.DSOM_Y = 129 cfg.DSOM_batchsize = 32 args.batchsize = 32 else: cfg.DSOM_X = 320 cfg.DSOM_Y = 320 cfg.DSOM_batchsize = 16 args.batchsize = 16 if args.operation == "train": train_model = train(args.dataset,args.subdataset,int(args.batchsize), int(args.num_iters_per_epoch_train), args.num_iters_per_epoch_val, args.num_epochs, args.device, args.load_model, args.save_per_x_epoch, int(args.decay_per_x_step), args.decay_rate, args.val_per_x_iters) train_model.train() elif args.operation == "test_MCL": cfg.DSOM_batchsize = 1 args.batchsize = 1 cfg.DSOM_model_path = args.load_model if args.dataset == 'simulation': paras = {'Random_Produce': args.random_sampling, 'scan_res': cfg.simulation_Dataset_angle_step, 'scan_angle_st': cfg.simulation_Dataset_st_angle, 'max_scan_range': cfg.simulation_Dataset_max_range, 'var_rc_weighting_model': 5, 'laser_portions': cfg.simulation_Dataset_scan_steps, 'Random_sample_var_x': 0.1, 'Random_sample_var_y': 0.1, 'Random_sample_var_theta': 0.1, 'w_cut': float(args.w_cut), 'rc_lamda_short': float(args.rc_lamda_short), 'rc_w_hit': float(args.rc_w_hit), 'rc_w_short': float(args.rc_w_short),
from src.train import train from src.data import get_train_val_test, get_data import sys if __name__ == '__main__': batch_size = 64 print(sys.argv) model_name = sys.argv[1] dataset = sys.argv[2] epoch = int(sys.argv[3]) if dataset == 'fairface' or dataset == 'utkface': train_set, val_set, test_set = get_train_val_test( batch_size, 'rec/{}'.format(dataset)) else: train_path = sys.argv[4] val_path = sys.argv[5] train_set = get_data(batch_size, train_path, True) val_set = test_set = get_data(batch_size, val_path) train(model_name, dataset, train_set, val_set, test_set, batch_size, epoch=epoch)
def retrain(): _updateing = True train() _updateing = False return "success"
# Hyperparameters parser.add_argument('--learning-rate', '--lr', type=float, default=2.6e-3, help='learning rate') parser.add_argument('--momentum', '-m', type=float, default=0.9, help='momentum argument for SGD optimizer') parser.add_argument('--weight-decay', '--wd', type=float, default=0.0005, help='weight decay value') parser.add_argument('--warmup', type=int, default=None) parser.add_argument('--num-workers', type=int, default=4) parser.add_argument('--fp16', action='store_true') # Distributed parser.add_argument('--local_rank', default=0, type=int, help='Used for multi-process training. Can either be manually set ' + 'or automatically set by using \'python -m multiproc\'.') # Pipeline control parser.add_argument('--data_pipeline', type=str, default='dali', choices=['dali', 'no_dali'], help='data preprocessing pipline to use') return parser if __name__ == "__main__": parser = make_parser() args = parser.parse_args() torch.backends.cudnn.benchmark = True train(args)
def main(opt): start_epoch = 0 err_best = 1000 glob_step = 0 lr_now = opt.lr # save options log.save_options(opt, opt.out_dir) # create and initialise model # parents = [1, 2, 7, 7, 5, 7, 5, -1, 8, 7, 7, 10, 7] # assert len(parents) == 13 # adj = adj_mx_from_skeleton(13, parents) model = LinearModel( input_size=26, output_size=39, linear_size=opt.linear_size, num_stage=opt.num_stage, p_dropout=opt.dropout, ) # groups = [[2, 3], [5, 6], [1, 4], [0, 7], [8, 9], [14, 15], [11, 12], [10, 13]] # model = SemGCN(adj, 128, num_layers=4, p_dropout=0.0, nodes_group=None) # model = SemGCN() model = model.cuda() model.apply(weight_init) criterion = nn.MSELoss(size_average=True).cuda() optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) print(">>> total params: {:.2f}M".format( sum(p.numel() for p in model.parameters()) / 1000000.0)) # load pretrained ckpt if opt.load: print(">>> loading ckpt from '{}'".format(opt.load)) ckpt = torch.load(opt.load) start_epoch = ckpt["epoch"] err_best = ckpt["err"] glob_step = ckpt["step"] lr_now = ckpt["lr"] model.load_state_dict(ckpt["state_dict"]) optimizer.load_state_dict(ckpt["optimizer"]) print(">>> ckpt loaded (epoch: {} | err: {})".format( start_epoch, err_best)) if opt.test: log_file = "log_test.txt" else: log_file = "log_train.txt" if opt.resume: logger = log.Logger(os.path.join(opt.out_dir, log_file), resume=True) else: logger = log.Logger(os.path.join(opt.out_dir, log_file)) logger.set_names( ["epoch", "lr", "loss_train", "loss_test", "err_test"]) # data loading print("\n>>> loading data") stat_3d = torch.load(os.path.join(opt.data_dir, "stat_3d.pth.tar")) # test if opt.test: test_loader = DataLoader( dataset=data_loader(data_path=opt.data_dir, is_train=False), batch_size=opt.batch_size, shuffle=False, num_workers=opt.job, pin_memory=True, ) loss_test, err_test, joint_err, all_err, outputs, targets, inputs = test( test_loader, model, criterion, stat_3d) print(os.path.join(opt.out_dir, "test_results.pth.tar")) torch.save( { "loss": loss_test, "all_err": all_err, "test_err": err_test, "joint_err": joint_err, "output": outputs, "target": targets, "input": inputs, }, open(os.path.join(opt.out_dir, "test_results.pth.tar"), "wb"), ) # print("train {:.4f}".format(err_train), end="\t") print("test {:.4f}".format(err_test), end="\t") sys.exit() # load datasets for training test_loader = DataLoader( dataset=data_loader(data_path=opt.data_dir, is_train=False), batch_size=opt.batch_size, shuffle=False, num_workers=opt.job, pin_memory=True, ) train_loader = DataLoader( dataset=data_loader(data_path=opt.data_dir, is_train=True, noise=opt.noise), batch_size=opt.batch_size, shuffle=True, num_workers=opt.job, pin_memory=True, drop_last=False, ) # loop through epochs cudnn.benchmark = True for epoch in range(start_epoch, opt.epochs): print("==========================") print(">>> epoch: {} | lr: {:.10f}".format(epoch + 1, lr_now)) # train glob_step, lr_now, loss_train = train( train_loader, model, criterion, optimizer, lr_init=opt.lr, lr_now=lr_now, glob_step=glob_step, lr_decay=opt.lr_decay, gamma=opt.lr_gamma, max_norm=opt.max_norm, ) loss_test, err_test, _, _, _, _, _ = test(train_loader, model, criterion, stat_3d) # test loss_test, err_test, _, _, _, _, _ = test(test_loader, model, criterion, stat_3d) # update log file logger.append( [epoch + 1, lr_now, loss_train, loss_test, err_test], ["int", "float", "float", "float", "float"], ) # save ckpt is_best = err_test < err_best err_best = min(err_test, err_best) log.save_ckpt( { "epoch": epoch + 1, "lr": lr_now, "step": glob_step, "err": err_best, "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), }, ckpt_path=opt.out_dir, is_best=is_best, ) logger.close()
dropout_rate=args.dropout_rate, kernel_size=args.kernel_size, padding=args.padding) # Train the model. train_data = list(zip(src_train_sents, tgt_train_sents)) dev_data = list(zip(src_dev_sents, tgt_dev_sents)) dataset = {"train_data": train_data, "dev_data": dev_data} model_save_path = args.destination + "bin/model_%s_en.bin" % args.language tic = time.time() train(model, dataset, learning_rate=args.learning_rate, lr_decay=args.lr_decay, clip_grad=args.clip_grad, batch_size=args.batch_size, max_epochs=args.max_epochs, max_num_trial=args.max_num_trial, patience_limit=args.patience_limit, model_save_path=model_save_path, stdout=stdout) toc = time.time() print(f"Training took {((toc - tic) / 60):.3f} minutes", file=stdout) # Compute and print BLEU score. print("Computing corpuse level BLEU score..", file=stdout) test_data = [src_test_sents, tgt_test_sents] tic = time.time() bleu_score = compute_corpus_level_bleu_score(model=model, data=test_data) toc = time.time() print(f"Corpus BLEU: {bleu_score*100:.3f}. Computed in {toc-tic:.3f} seconds.",
cur_fold = i + 1 print("Current fold : {}".format(cur_fold)) train_df = train_csv.iloc[train_index, :] val_df = train_csv.iloc[val_index, :] print(train_df['diagnosis'].value_counts()) print(val_df['diagnosis'].value_counts()) train_df.reset_index(drop=True, inplace=True) val_df.reset_index(drop=True, inplace=True) train(train_df, val_df, path, weight_path, cur_fold, model_num=model_name[-1], n_epochs=n_epochs) elif pretraining == True: weight_path = None EXPERIMENT_NAME = "b4_384_Plateau_pretrain_only_resize_trans_v2" path = os.path.join( config.RES_PATH, f'{EXPERIMENT_NAME}_{model_name}-{nfold}fold-{now}') print(EXPERIMENT_NAME) if not os.path.exists(path): os.mkdir(path)
def main(cfg): SEED = cfg.values.seed BATCH_SIZE = cfg.values.train_args.batch_size IMAGE_SIZE = cfg.values.image_size USE_KFOLD = cfg.values.use_kfold NUM_FOLD = cfg.values.train_args.num_fold if USE_KFOLD else 0 seed_everything(SEED) print(f'Cuda is Available ? : {torch.cuda.is_available()}\n') data_df = pd.read_csv('E:/seti-breakthrough-listen/train_labels.csv') data_df['file_path'] = data_df['id'].apply(get_train_file_path) train_transform = albumentations.Compose([ albumentations.Resize(IMAGE_SIZE, IMAGE_SIZE), albumentations.HorizontalFlip(), albumentations.VerticalFlip(), # albumentations.Normalize(mean=(0.5, 0.5, 0.5), std=(0.25, 0.25, 0.25)), albumentations.pytorch.transforms.ToTensorV2() ]) val_transform = albumentations.Compose([ albumentations.Resize(IMAGE_SIZE, IMAGE_SIZE), # albumentations.Normalize(mean=(0.5, 0.5, 0.5), std=(0.25, 0.25, 0.25)), albumentations.pytorch.transforms.ToTensorV2() ]) if USE_KFOLD: kfold = StratifiedKFold(n_splits=NUM_FOLD, shuffle=True, random_state=SEED) for k, (train_index, val_index) in enumerate(kfold.split(data_df, data_df['target'])): print('\n') cpprint('=' * 15 + f'{k + 1}-Fold Cross Validation' + '=' * 15) train_df = data_df.iloc[train_index].reset_index(drop=True) val_df = data_df.iloc[val_index].reset_index(drop=True) train_loader = get_dataloader(df=train_df, transform=train_transform, batch_size=BATCH_SIZE, shuffle=True) val_loader = get_dataloader(df=val_df, transform=val_transform, batch_size=BATCH_SIZE, shuffle=False) val_labels = val_df['target'].values.tolist() train(cfg, train_loader, val_loader, val_labels, k + 1) else: print('\n') cpprint('=' * 15 + f'Start Training' + '=' * 15) train_df, val_df = train_test_split(data_df, test_size=0.2, shuffle=True, stratify=data_df['target'], random_state=SEED) train_loader = get_dataloader(df=train_df, transform=train_transform, batch_size=BATCH_SIZE, shuffle=True) val_loader = get_dataloader(df=val_df, transform=val_transform, batch_size=BATCH_SIZE, shuffle=False) val_labels = val_df['target'].values.tolist() train(cfg, train_loader, val_loader, val_labels, 0)
for batch in dataloader_train: # Get training data for this cycle input_batches, input_lengths = batch['input'], batch[ 'length'].numpy().tolist() input_batches, input_lengths = zip( *sorted(zip(input_batches, input_lengths), key=lambda x: x[1], reverse=True)) input_batches, input_lengths = torch.stack(input_batches), list( input_lengths) input_batches = input_batches[:, :max(input_lengths)] input_batches = input_batches.transpose(0, 1) # Run the train function loss = train(input_batches, input_lengths, input_batches, input_lengths, encoder, decoder, encoder_optimizer, decoder_optimizer, batch_size, clip, teacher_forcing_ratio) # Keep track of loss print_loss_total += loss batch_n += 1 if batch_n % print_every == 0: print_loss_avg = print_loss_total / print_every print_loss_total = 0 print_summary = 'Epoch:%d - Batch:%d - Train_loss:%.4f' % ( epoch, batch_n, print_loss_avg) logger.info(print_summary) if batch_n % save_every == 0: torch.save(encoder.state_dict(), args['save_path_encoder'])
from src.train import train train('SuperMarioBros-1-1-v2', 'results', False)
log_dir = os.path.join(args.base_dir, hp.log.log_dir, args.model) os.makedirs(log_dir, exist_ok=True) chkpt_path = args.checkpoint_path if args.checkpoint_path is not None else None logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler( os.path.join( log_dir, '%s-%d.log' % (args.model, time.time()))), logging.StreamHandler() ]) logger = logging.getLogger() logger.info(hp_str) for k, v in hp.data.items(): if v == '': logger.error("config {} is empty. Raising Error".format(k)) raise Exception("Please specify config for {}".format(k)) writer = MyWriter(hp, log_dir) trainloader = create_dataloader(hp, args, train=True) testloader = create_dataloader(hp, args, train=False) train(args, pt_dir, chkpt_path, trainloader, testloader, writer, logger, hp, hp_str)
0.7]), bias = 0.5 ) l2 = 0.5 model = RobustLogisticModel(1, l2_epsilon=l2, config=config) syndata = SyntheticDataGenerator(config) test_data = SyntheticDataGenerator(config) rob = 0.0 print(f'**** training on natural plus {rob} perturbed***') # custom train on natural data mixed with adversarially perturbed data train(model, syndata.data(), robust=rob, epochs=40) nat_nat = test(model, test_data.data()) nat_nat.update(train='nat', test='nat') print(f'Perf on perturbed data:') nat_per = test(model, test_data.data(), perturb=True) nat_per.update(train='nat', test='per') rob = 1.0 print(f'**** training on natural plus {rob} perturbed***') # custom train on natural data mixed with adversarially perturbed data train(model, syndata.data(), robust=rob, epochs=20) per_nat = test(model, test_data.data()) per_nat.update(train='per', test='nat') print(f'Perf on perturbed data:') per_per = test(model, test_data.data(), perturb=True) per_per.update(train='per', test='per')
# %% from src.utils import get_data_dir hidden_layers = 2 model = train({"model_name": "cnn", "data_path": os.path.join(get_data_dir(), "whitenoise_10_reflections"), "experiment_name": "3_to_4_conv2d_10_refs", "max_epochs": 15, "num_workers": 8, "lr": 0.001, "lr_sched_patience": 3, "lr_sched_thresh": 0.1, "hidden_layers": hidden_layers, "kernel_widths": [(10, 1)] * (hidden_layers + 1), "strides": [(1, 1)] * (hidden_layers + 1), "hidden_channels": [200]*hidden_layers, "sh_order_sig": 3, "sh_order_scm": 4, "gpus": -1, "residual_flag": True, "bias": True, "complex_conv": True, "fast_dev_run": False, "bandwidth": 2000, }) # # # %% # print(model.conv_layers[0].weight[0, :, 0]) # print(model.conv_layers[0].weight[1, :, 0]) # print(model.conv_layers[0].bias)
from src.train import train from src.utils import get_experiments_dir, get_data_dir opts = { # ---folders--- "data_path": os.path.join(get_data_dir(), 'SCM', 'image-method'), "logs_path": os.path.join(get_experiments_dir(), 'fc', 'image_method'), "experiment_name": 'fc_imagemethod_full', # ---network structure--- "model_name": 'fc', "input_sh_order": 3, "rank": None, # None -> output is full matrix, Int -> output is low rank matrix transformed into full matrix "hidden_layers": 1, "hidden_sizes": [3000], "residual_flag": True, "residual_only": False, # ---data--- "batch_size": 50, "num_workers": 15, # ---optimization--- "lr": 1e-3, "lr_sched_thresh": 0.01, "lr_sched_patience": 10, "max_epochs": 1000, "gpus": -1 } train(opts)