logger.info("Configuration file loaded from %s" % args.config)
# read input csv file as pandas dataframe
if args.input is not None:
input = pd.read_csv(args.input, low_memory=False)
logger.info('Input data loaded from %s', args.input)
if args.step == 'acquire':
output = acquire(**config['acquire'])
elif args.step == 'featurize':
output = featurize(input, **config['featurize'])
elif args.step == 'clean':
output = clean(input, **config['clean'])
elif args.step == 'train':
output, output1 = train(input, **config['train'])
else:
logger.warning('No such argument as %s', args.step)
if args.output is not None:
# if train, output model pkl file, else output csv file
if args.step == 'train':
try:
pickle.dump(output, open(args.output, 'wb'))
except:
logger.error("Model object can't be dumped!")
output1.to_csv(args.output1, index=False)
logger.info("Output saved to %s" % args.output1)
else:
def optimize_hp(opts):
opts_list = get_opts_combs(opts)
for opts in opts_list:
train(opts)
import argparse
# TODO: Refactoring needed
from src.train import main_train as train
from src.inference import main_inference as inference
from src.constants import hyperparams, model_config
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Training start')
parser.add_argument('env', metavar='ENV', type=str, default='localhost')
parser.add_argument('mode', metavar='RUNNING', type=str, default='train')
args = parser.parse_args()
print('Environment: ', args.env)
print('Mode: ', args.mode)
if args.mode == 'train':
train(args.env)
else:
if hyperparams['mixout_epochs'] == 0:
ckpt_name = f'{model_config["model_name"]}_epoch{hyperparams["epochs"]}_{hyperparams["finetuning_epochs"]}'
else:
ckpt_name = f'{model_config["model_name"]}_epoch{hyperparams["epochs"]}_{hyperparams["finetuning_epochs"]}_{hyperparams["mixout_epochs"]}'
inference(
args.env,
ckpt_name=ckpt_name,
threshold_strategy="exact", ## "percentile" or "exact"
threshold_percentile=100 -
0.59, ## for threshold_strategy == "percentile"
threshold_exact=0.5, ## for threshold_strategy == "exact"
if_use_log=False,
logfile='011458.csv')
# about path
tf.flags.DEFINE_string("data_name", "Fairchild_HDR",
"data_name is Fairchild_HDR or Funt_HDR")
tf.flags.DEFINE_string(
"train_dir", "./dataset/training_samples",
"Path to training image directory or an individual image")
tf.flags.DEFINE_string(
"valid_dir", "./dataset/valid_samples",
"Path to valid image directory or an individual image")
tf.flags.DEFINE_string(
"test_dir", "./dataset/testing_samples",
"Path to testing image directory or an individual image")
tf.flags.DEFINE_string("out_dir", "results", "Path to output directory")
tf.flags.DEFINE_string("summary_dir", "summary",
"Path to output directory")
tf.flags.DEFINE_string("dm", "./models_dm",
"Path to trained CNN dm_weights")
tf.flags.DEFINE_string("um", "./models_um",
"Path to trained CNN um_weights")
if FLAGS.model_type == 0:
train(FLAGS)
elif FLAGS.model_type == 1:
test(FLAGS)
else:
predict(FLAGS)
# Program over
print('success! u are a smart boy!')
from env import OhmniInSpace
from src import train
if sys.argv[1] == '--test':
if sys.argv[2] == 'py-env':
ois = OhmniInSpace.PyEnv(gui=True)
timestep = ois.reset()
while not timestep.is_last():
timestep = ois.step(action=(0.4, 0.4))
(_, reward, discount, observation) = timestep
print('Reward:', reward)
ois.render()
cv.imshow('Segmentation', observation)
if cv.waitKey(10) & 0xFF == ord('q'):
break
if sys.argv[2] == 'tf-env':
ois = OhmniInSpace.TfEnv()
tf_env = ois.gen_env()
print("TimeStep Specs:", tf_env.time_step_spec())
print("Action Specs:", tf_env.action_spec())
elif sys.argv[1] == '--ohmni':
if sys.argv[2] == 'train':
train.train()
if sys.argv[2] == 'run':
train.run()
else:
print("Error: Invalid option!")
def main(config, run_preprocessing, run_data_upload, log_dir):
# Load experiment configuration
with open(config) as f:
config = yaml.load(f, Loader=yaml.FullLoader)
# Get db connection
conn = sql_utils.get_connection()
# Get basic info of experiment
exp_version = config['version']
exp_name = config["experiment_name"]
exp_time = date_utils.get_current_time_string()[2:]
username = getpass.getuser()[0]
terminal_width = int(os.popen('stty size', 'r').read().split()[1])
print(
f'Running Experiment: {username}_{exp_version}_{exp_name}_{exp_time}\n{"-" * terminal_width}\n'
)
# Preprocessing
preprocessing_prefix = config['preprocessing_config']['prefix']
if not run_preprocessing:
print('Preprocessing skipped.')
else:
print('Preprocessing ...')
run_preprocess(conn,
config['preprocessing_config'],
run_data_upload=run_data_upload)
print('Preprocessing done.')
# Get temporal configuration information
train_dates_list, test_dates_list = parse_temporal_config(
config['temporal_config'])
# Training and evaluation
test_results_over_time = []
experiment_loop = tqdm.tqdm(list(zip(train_dates_list, test_dates_list)),
desc='Experiment Repeats')
for train_dates, test_dates in experiment_loop:
split_time_abbr = date_utils.date_to_string(
test_dates['label_start_time'])
split_time_abbr = split_time_abbr.replace('-', '')[2:]
split_name = f'{split_time_abbr}'
print(split_name)
prefix = f'{username}_{exp_version}_{exp_name}_{exp_time}_{split_name}'
experiment_table_prefix = f'experiments.{prefix}'
train_save_dir = os.path.join(os.getcwd(), log_dir, prefix,
'train_' + exp_time)
test_save_dir = os.path.join(os.getcwd(), log_dir, prefix,
'test_' + exp_time)
# Prepare cohort as specified by our experiment configuration
tqdm.tqdm.write('\nPreparing cohorts ...')
train_feature_splits, train_label_splits = [], []
for i, train_dates_aod in enumerate(train_dates):
train_feature_table, train_label_table = prepare_cohort(
config,
train_dates_aod,
test_dates,
preprocessing_prefix,
experiment_table_prefix + f'_split{i}',
include_test=False)[:2]
train_feature_splits.append(train_feature_table)
train_label_splits.append(train_label_table)
test_feature_table, test_label_table = prepare_cohort(
config,
train_dates[-1],
test_dates,
preprocessing_prefix,
experiment_table_prefix,
include_train=False)[2:]
train_feature_table = f'{experiment_table_prefix}_train_features'
sql_utils.merge_tables(train_feature_splits, train_feature_table)
train_label_table = f'{experiment_table_prefix}_train_labels'
sql_utils.merge_tables(train_label_splits, train_label_table)
# Delete intermediate cohort tables
for i in range(len(train_dates)):
cohort_table_name = f'{experiment_table_prefix}_split{i}_cohort'
sql_utils.run_sql_from_string(conn,
f'drop table {cohort_table_name};')
# Train models as specified by our experiment configuration
tqdm.tqdm.write('Training ...')
model_summaries = train(config,
train_feature_table,
train_label_table,
discard_columns=['split'],
save_dir=train_save_dir)
# Evaluate our models on the training data
model_paths = glob.glob(f'{train_save_dir}/*.pkl')
tqdm.tqdm.write('Evaluating on training data ...')
train_results = evaluate(config,
train_feature_table,
train_label_table,
model_paths,
model_summaries,
discard_columns=['split'],
log_dir=train_save_dir)
# Evaluate our models on the test data
tqdm.tqdm.write('Evaluating on test data ...')
test_results = evaluate(config,
test_feature_table,
test_label_table,
model_paths,
model_summaries,
save_preds_to_db=True,
save_prefix=f'{prefix}_test',
log_dir=test_save_dir)
test_results_over_time.append(test_results)
# Save results to database
train_results_name = f'{prefix}_train_results'
test_results_name = f'{prefix}_test_results'
train_results.to_sql(train_results_name, conn, schema='results')
test_results.to_sql(test_results_name, conn, schema='results')
# Plot test results over time
test_results_tables_prefix = f'{username}_{exp_version}_{exp_name}_{exp_time}'
plot_utils.plot_results_over_time(test_results_tables_prefix)
P_LABELS = Config.get("Default", "P_LABELS")
P_PIXEL_MASK = Config.get("Default", "P_PIXEL_MASK")
METHOD = Config.get("Default", "METHOD")
ts = time.time()
dt = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
result_path = os.path.join("results", "archive", METHOD, dt)
os.makedirs(result_path)
logging.basicConfig(filename=os.path.join(result_path, "info.log"), level=logging.INFO)
model_path = os.path.join("models", METHOD, dt)
os.makedirs(model_path)
if METHOD == 'STAE':
net = SpatialTemporalAutoencoder(tvol=TVOL, alpha=ALPHA, batch_size=BATCH_SIZE, lambd=LAMBDA)
d = DataIteratorStae(P_TRAIN, P_TEST, P_LABELS, P_PIXEL_MASK, batch_size=BATCH_SIZE, tvol=TVOL, taug=TAUG)
elif METHOD == 'CONVAE2D':
net = ConvAE2d(tvol=TVOL, alpha=ALPHA, batch_size=BATCH_SIZE, lambd=LAMBDA)
d = DataIteratorNormal(P_TRAIN, P_TEST, P_LABELS, P_PIXEL_MASK, batch_size=BATCH_SIZE, tvol=TVOL, taug=TAUG)
elif METHOD == 'EXP':
net = Experiment(tvol=TVOL, alpha=ALPHA, batch_size=BATCH_SIZE, lambd=LAMBDA)
d = DataIteratorNormal(P_TRAIN, P_TEST, P_LABELS, P_PIXEL_MASK, batch_size=BATCH_SIZE, tvol=TVOL, taug=TAUG)
else:
raise ValueError('Incorrect method specification')
frame_auc, frame_eer, pixel_auc, pixel_eer = train(data=d, model=net, num_iteration=NUM_ITER,
result_path=result_path, model_path=model_path)
logging.info("Best frame-level area under the roc curve: {0:g}".format(frame_auc))
logging.info("Frame-level equal error rate corresponding to this: {0:g}".format(frame_eer))
logging.info("Pixel-level area under the roc curve corresponding to this: {0:g}".format(pixel_auc))
logging.info("Pixel-level equal error rate corresponding to this: {0:g}".format(pixel_eer))
def test_one_epoch():
config = ['--no-visualize', '--num-examples=100', '--no-save']
metrics = train(['--epoch=1', '--name=TEST'] + config)
assert round(metrics.get_primary_value(), 7) == 4.6288586
def trainFun(status_var, t):
text = '样本训练完成'
status_var.set(text)
train(END, t)
def tune(project_parameters):
project_parameters.mode = 'train'
hyperparameter_space = _get_hyperparameter_space(
project_parameters=project_parameters)
tune_scheduler = ASHAScheduler(metric='accuracy_difference', mode='min')
reporter = CLIReporter(metric_columns=[
'train_loss', 'val_loss', 'test_loss', 'train_accuracy',
'val_accuracy', 'test_accuracy', 'accuracy_difference'
])
init(dashboard_host='0.0.0.0')
tuning_result = ray.tune.run(
run_or_experiment=partial(_tune_function,
project_parameters=project_parameters),
resources_per_trial={
'cpu': project_parameters.tune_cpu,
'gpu': project_parameters.tune_gpu
},
config=hyperparameter_space,
num_samples=project_parameters.tune_iter,
scheduler=tune_scheduler,
local_dir=join(project_parameters.save_path, 'tuning_logs'),
progress_reporter=reporter)
best_trial = tuning_result.get_best_trial('accuracy_difference', 'min',
'last')
if not project_parameters.tune_debug:
project_parameters = _set_tune_project_parameters(
hyperparameter=best_trial.config,
project_parameters=project_parameters)
result = train(project_parameters=project_parameters)
result['tune'] = tuning_result
else:
result = {'tune': tuning_result}
print('best trial name: {}'.format(best_trial))
print('best trial result: {}'.format(
best_trial.last_result['accuracy_difference']))
print('best trial config: {}'.format(best_trial.config))
if 'cutoff_freq_low' in best_trial.config or 'cutoff_freq_high' in best_trial.config:
best_trial.config['cutoff_freq'] = []
for key in ['cutoff_freq_low', 'cutoff_freq_high']:
value = best_trial.config.pop(key, None)
if value is not None:
best_trial.config['cutoff_freq'].append(value)
best_trial.config['cutoff_freq'] = (
'{},' * len(best_trial.config['cutoff_freq']))[:-1].format(
*best_trial.config['cutoff_freq'])
if 'parameters_config_path' in project_parameters:
output = 'num_workers: {}'.format(project_parameters.num_workers)
for k, v in best_trial.config.items():
if k == 'cutoff_freq':
output += '\n{}:\n - {}\n - {}'.format(
k,
v.split(',')[0],
v.split(',')[1])
else:
output += '\n{}: {}'.format(k, v)
print('best trial config command:\n{}'.format(output))
else:
print('best trial config command: --num_workers {}{}'.format(
project_parameters.num_workers,
(' --{} {}' * len(best_trial.config)).format(*np.concatenate(
list(zip(best_trial.config.keys(),
best_trial.config.values()))))))
shutdown()
return result
from src.compare import compare
if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog='main.py',
description=
'This program enables you to make and test image captioning model.',
epilog='end',
add_help=True,
)
parser.add_argument('phase',
help='to designate the phase (train, eval, infer)')
phase = parser.parse_args().phase
config = Config()
if config.PREPARE_VOCAB:
prepare_vocab(config.TRAIN_CAPTION_PATH, config.WORD_TO_ID_PATH,
config.ID_TO_WORD_PATH)
if phase == 'train':
train()
elif phase == 'eval':
evalate()
elif phase == 'infer':
infer()
elif phase == 'compare':
compare()
else:
print('the argument should be train, eval, or infer')
def main():
if CONFIG['train'] == 'train':
data.prepare(CONFIG)
train.train(CONFIG, False)
else:
train.test(CONFIG, 'weighted_sampling', 100)
def run(cfg: DictConfig) -> None:
logger.info('=' * 30)
logger.info('::: pipeline start :::')
logger.info('=' * 30)
logger.info(f'git hash is: {hash_}')
logger.info(f'all params\n{"="*80}\n{cfg.pretty()}\n{"="*80}')
comment = cfg['globals']['comment']
assert comment != None, 'commentを入力してください。(globals.commet=hogehoge)'
if cfg['globals']['debug']:
logger.info('::: set debug mode :::')
cfg = utils.get_debug_config(cfg)
global_params = cfg["globals"]
utils.set_seed(50)
device = C.get_device(global_params["device"])
splitter = C.get_split(cfg)
df, datadir = C.get_metadata(cfg)
logger.info(f'meta_df: {df.shape}')
output_dir = os.getcwd()
output_dir_ignore = output_dir.replace('/data/', '/data_ignore/')
if not os.path.exists(output_dir_ignore):
os.makedirs(output_dir_ignore)
for fold_i, (trn_idx,
val_idx) in enumerate(splitter.split(df, y=df["ebird_code"])):
if fold_i not in global_params["folds"]:
continue
logger.info("=" * 30)
logger.info(f"Fold {fold_i}")
logger.info("=" * 30)
trn_df = df.loc[trn_idx, :].reset_index(drop=True)
val_df = df.loc[val_idx, :].reset_index(drop=True)
if global_params['remove_short']:
logger.info(f'::: remove short duration :::')
trn_df = utils.remove_short_duration(trn_df)
if global_params['balanced']:
logger.info(f'::: train class balanced :::')
trn_df = utils.transform_balanced_dataset(trn_df)
if global_params['mixup']:
logger.info(f'::: perform mixup :::')
if global_params['debug']:
trn_df = utils.get_debug_df(trn_df)
val_df = utils.get_debug_df(val_df)
logger.info(f'trn_df: {trn_df.shape}')
logger.info(f'val_df: {val_df.shape}')
train_loader = C.get_loader(trn_df, datadir, cfg, 'train')
valid_loader = C.get_loader(val_df, datadir, cfg, 'valid')
model = models.get_model(cfg).to(device)
criterion = C.get_criterion(cfg).to(device)
optimizer = C.get_optimizer(model, cfg)
scheduler = C.get_scheduler(optimizer, cfg)
losses_train = []
losses_valid = []
epochs = []
best_f1 = 0
best_loss = 0
save_path = f'{output_dir_ignore}/{model.__class__.__name__}_fold{fold_i}.pth'
early_stopping = EarlyStopping(patience=12,
verbose=True,
path=save_path)
n_epoch = cfg['globals']['num_epochs']
for epoch in progress_bar(range(1, n_epoch + 1)):
logger.info(f'::: epoch: {epoch}/{n_epoch} :::')
loss_train = train(model, device, train_loader, optimizer,
scheduler, criterion, global_params['mixup'])
loss_valid, fscore_valid = get_epoch_loss_score(
model, device, valid_loader, criterion)
logger.info(
f'loss_train: {loss_train:.6f}, loss_valid: {loss_valid:.6f}, f1(macro): {fscore_valid:.6f}'
)
epochs.append(epoch)
losses_train.append(loss_train)
losses_valid.append(loss_valid)
is_update = early_stopping(loss_valid, model,
global_params['debug'])
if is_update:
best_loss = loss_valid
best_f1 = fscore_valid
if early_stopping.early_stop:
logger.info("Early stopping")
break
# result handling
rh.save_loss_figure(fold_i, epochs, losses_train, losses_valid,
output_dir)
rh.save_result_csv(fold_i, global_params['debug'],
f'{model.__class__.__name__}', cfg['loss']['name'],
best_loss, best_f1, comment, output_dir)
logger.info(
f'best_loss: {best_loss:.6f}, best_fscore(macro): {best_f1:.6f}')
logger.info('::: success :::\n\n\n')
# 開放
del train_loader
del valid_loader
del model
del optimizer
del scheduler
gc.collect()
torch.cuda.empty_cache()
for model_type in tqdm(
['logistic', 'rf', 'mlp2', 'svm']
): #,'gb', 'logistic', 'dt', 'svm', 'gb', 'rf', 'mlp2', 'irf']):
for num_feats in [
len(feat_names)
]: # [16, 25, len(feat_names)]: #[16, 25, len(feat_names)]: # number of total features to consider
for balancing in ['ros']: # None, ros
for balancing_ratio in [0.8, 1,
1.2]: # positive: negative
hyperparams = [0] if model_type in [
'logistic', 'rf', 'gb', 'dt', 'irf', 'qda'
] else [-1, 0, 1]
for hyperparam in hyperparams: # 0 is default
feats = feat_names[:num_feats]
out_name = f'{model_type}_{num_feats}_{feature_selection}={feature_selection_num}_{balancing}={balancing_ratio}_h={hyperparam}_cal={calibrated}'
train.train(
df,
feat_names=feats,
cell_nums_feature_selection=dset[
'feature_selection'],
cell_nums_train=dset['train'],
model_type=model_type,
balancing=balancing,
balancing_ratio=balancing_ratio,
outcome_def=outcome_def,
feature_selection=feature_selection,
feature_selection_num=
feature_selection_num,
hyperparam=hyperparam,
out_name=f'{out_dir}/{out_name}.pkl',
calibrated=calibrated)
def loop():
device = torch.device("cuda")
if not torch.cuda.is_available():
sys.exit("Error: CUDA requested but not available")
# weighted values for loss functions
# add a helper to return weights seamlessly
try:
weight = torch.Tensor([1.513212, 10.147043])
except KeyError:
if model["opt"]["loss"] in ("CrossEntropy", "mIoU", "Focal"):
sys.exit("Error: The loss function used, need dataset weights values")
# loading Model
net = UNet(num_classes)
net = DataParallel(net)
net = net.to(device)
# define optimizer
optimizer = Adam(net.parameters(), lr=lr)
# resume training
if model_path:
chkpt = torch.load(model_path, map_location=device)
net.load_state_dict(chkpt["state_dict"])
optimizer.load_state_dict(chkpt["optimizer"])
# select loss function, just set a default, or try to experiment
if loss_func == "CrossEntropy":
criterion = CrossEntropyLoss2d(weight=weight).to(device)
elif loss_func == "mIoU":
criterion = mIoULoss2d(weight=weight).to(device)
elif loss_func == "Focal":
criterion = FocalLoss2d(weight=weight).to(device)
elif loss_func == "Lovasz":
criterion = LovaszLoss2d().to(device)
else:
sys.exit("Error: Unknown Loss Function value !")
#loading data
train_loader, val_loader = get_dataset_loaders(target_size, batch_size, dataset_path)
history = collections.defaultdict(list)
# training loop
for epoch in range(0, num_epochs):
print("Epoch: " + str(epoch +1))
train_hist = train(train_loader, num_classes, device, net, optimizer, criterion)
val_hist = validate(val_loader, num_classes, device, net, criterion)
print("Train loss: {:.4f}, mIoU: {:.3f}, {} IoU: {:.3f}, MCC: {:.3f}".format(
train_hist["loss"], train_hist["miou"], target_type, train_hist["fg_iou"], train_hist["mcc"]))
print("Validation loss: {:.4f}, mIoU: {:.3f}, {} IoU: {:.3f}, MCC: {:.3f}".format(
val_hist["loss"], val_hist["miou"], target_type, val_hist["fg_iou"], val_hist["mcc"]))
for key, value in train_hist.items():
history["train " + key].append(value)
for key, value in val_hist.items():
history["val " + key].append(value)
if (epoch+1)%5 == 0:
# plotter use history values, no need for log
visual = "history-{:05d}-of-{:05d}.png".format(epoch + 1, num_epochs)
plot(os.path.join(checkpoint_path, visual), history)
if (epoch+1)%20 == 0:
checkpoint = target_type + "-checkpoint-{:03d}-of-{:03d}.pth".format(epoch + 1, num_epochs)
states = {"epoch": epoch + 1, "state_dict": net.state_dict(), "optimizer": optimizer.state_dict()}
torch.save(states, os.path.join(checkpoint_path, checkpoint))
def _run_training(args):
train(raw_data_file=args.raw_data_file, processed_data_folderpath=args.processed_data_folderpath,
model_folderpath=args.model_folderpath, model_name=args.model_name)
parser.add_argument("--p_mcl", help="probability to execute MCL in Mixture MCL", default=0.95)
args = parser.parse_args()
if args.dataset == "simulation":
cfg.DSOM_X = 129
cfg.DSOM_Y = 129
cfg.DSOM_batchsize = 32
args.batchsize = 32
else:
cfg.DSOM_X = 320
cfg.DSOM_Y = 320
cfg.DSOM_batchsize = 16
args.batchsize = 16
if args.operation == "train":
train_model = train(args.dataset,args.subdataset,int(args.batchsize), int(args.num_iters_per_epoch_train), args.num_iters_per_epoch_val, args.num_epochs, args.device, args.load_model, args.save_per_x_epoch, int(args.decay_per_x_step), args.decay_rate, args.val_per_x_iters)
train_model.train()
elif args.operation == "test_MCL":
cfg.DSOM_batchsize = 1
args.batchsize = 1
cfg.DSOM_model_path = args.load_model
if args.dataset == 'simulation':
paras = {'Random_Produce': args.random_sampling, 'scan_res': cfg.simulation_Dataset_angle_step,
'scan_angle_st': cfg.simulation_Dataset_st_angle,
'max_scan_range': cfg.simulation_Dataset_max_range,
'var_rc_weighting_model': 5,
'laser_portions': cfg.simulation_Dataset_scan_steps, 'Random_sample_var_x': 0.1, 'Random_sample_var_y': 0.1,
'Random_sample_var_theta': 0.1, 'w_cut': float(args.w_cut),
'rc_lamda_short': float(args.rc_lamda_short),
'rc_w_hit': float(args.rc_w_hit), 'rc_w_short': float(args.rc_w_short),
from src.train import train
from src.data import get_train_val_test, get_data
import sys
if __name__ == '__main__':
batch_size = 64
print(sys.argv)
model_name = sys.argv[1]
dataset = sys.argv[2]
epoch = int(sys.argv[3])
if dataset == 'fairface' or dataset == 'utkface':
train_set, val_set, test_set = get_train_val_test(
batch_size, 'rec/{}'.format(dataset))
else:
train_path = sys.argv[4]
val_path = sys.argv[5]
train_set = get_data(batch_size, train_path, True)
val_set = test_set = get_data(batch_size, val_path)
train(model_name,
dataset,
train_set,
val_set,
test_set,
batch_size,
epoch=epoch)
def retrain():
_updateing = True
train()
_updateing = False
return "success"
# Hyperparameters
parser.add_argument('--learning-rate', '--lr', type=float, default=2.6e-3,
help='learning rate')
parser.add_argument('--momentum', '-m', type=float, default=0.9,
help='momentum argument for SGD optimizer')
parser.add_argument('--weight-decay', '--wd', type=float, default=0.0005,
help='weight decay value')
parser.add_argument('--warmup', type=int, default=None)
parser.add_argument('--num-workers', type=int, default=4)
parser.add_argument('--fp16', action='store_true')
# Distributed
parser.add_argument('--local_rank', default=0, type=int,
help='Used for multi-process training. Can either be manually set ' +
'or automatically set by using \'python -m multiproc\'.')
# Pipeline control
parser.add_argument('--data_pipeline', type=str, default='dali',
choices=['dali', 'no_dali'],
help='data preprocessing pipline to use')
return parser
if __name__ == "__main__":
parser = make_parser()
args = parser.parse_args()
torch.backends.cudnn.benchmark = True
train(args)
def main(opt):
start_epoch = 0
err_best = 1000
glob_step = 0
lr_now = opt.lr
# save options
log.save_options(opt, opt.out_dir)
# create and initialise model
# parents = [1, 2, 7, 7, 5, 7, 5, -1, 8, 7, 7, 10, 7]
# assert len(parents) == 13
# adj = adj_mx_from_skeleton(13, parents)
model = LinearModel(
input_size=26,
output_size=39,
linear_size=opt.linear_size,
num_stage=opt.num_stage,
p_dropout=opt.dropout,
)
# groups = [[2, 3], [5, 6], [1, 4], [0, 7], [8, 9], [14, 15], [11, 12], [10, 13]]
# model = SemGCN(adj, 128, num_layers=4, p_dropout=0.0, nodes_group=None)
# model = SemGCN()
model = model.cuda()
model.apply(weight_init)
criterion = nn.MSELoss(size_average=True).cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr)
print(">>> total params: {:.2f}M".format(
sum(p.numel() for p in model.parameters()) / 1000000.0))
# load pretrained ckpt
if opt.load:
print(">>> loading ckpt from '{}'".format(opt.load))
ckpt = torch.load(opt.load)
start_epoch = ckpt["epoch"]
err_best = ckpt["err"]
glob_step = ckpt["step"]
lr_now = ckpt["lr"]
model.load_state_dict(ckpt["state_dict"])
optimizer.load_state_dict(ckpt["optimizer"])
print(">>> ckpt loaded (epoch: {} | err: {})".format(
start_epoch, err_best))
if opt.test:
log_file = "log_test.txt"
else:
log_file = "log_train.txt"
if opt.resume:
logger = log.Logger(os.path.join(opt.out_dir, log_file), resume=True)
else:
logger = log.Logger(os.path.join(opt.out_dir, log_file))
logger.set_names(
["epoch", "lr", "loss_train", "loss_test", "err_test"])
# data loading
print("\n>>> loading data")
stat_3d = torch.load(os.path.join(opt.data_dir, "stat_3d.pth.tar"))
# test
if opt.test:
test_loader = DataLoader(
dataset=data_loader(data_path=opt.data_dir, is_train=False),
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.job,
pin_memory=True,
)
loss_test, err_test, joint_err, all_err, outputs, targets, inputs = test(
test_loader, model, criterion, stat_3d)
print(os.path.join(opt.out_dir, "test_results.pth.tar"))
torch.save(
{
"loss": loss_test,
"all_err": all_err,
"test_err": err_test,
"joint_err": joint_err,
"output": outputs,
"target": targets,
"input": inputs,
},
open(os.path.join(opt.out_dir, "test_results.pth.tar"), "wb"),
)
# print("train {:.4f}".format(err_train), end="\t")
print("test {:.4f}".format(err_test), end="\t")
sys.exit()
# load datasets for training
test_loader = DataLoader(
dataset=data_loader(data_path=opt.data_dir, is_train=False),
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.job,
pin_memory=True,
)
train_loader = DataLoader(
dataset=data_loader(data_path=opt.data_dir,
is_train=True,
noise=opt.noise),
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.job,
pin_memory=True,
drop_last=False,
)
# loop through epochs
cudnn.benchmark = True
for epoch in range(start_epoch, opt.epochs):
print("==========================")
print(">>> epoch: {} | lr: {:.10f}".format(epoch + 1, lr_now))
# train
glob_step, lr_now, loss_train = train(
train_loader,
model,
criterion,
optimizer,
lr_init=opt.lr,
lr_now=lr_now,
glob_step=glob_step,
lr_decay=opt.lr_decay,
gamma=opt.lr_gamma,
max_norm=opt.max_norm,
)
loss_test, err_test, _, _, _, _, _ = test(train_loader, model,
criterion, stat_3d)
# test
loss_test, err_test, _, _, _, _, _ = test(test_loader, model,
criterion, stat_3d)
# update log file
logger.append(
[epoch + 1, lr_now, loss_train, loss_test, err_test],
["int", "float", "float", "float", "float"],
)
# save ckpt
is_best = err_test < err_best
err_best = min(err_test, err_best)
log.save_ckpt(
{
"epoch": epoch + 1,
"lr": lr_now,
"step": glob_step,
"err": err_best,
"state_dict": model.state_dict(),
"optimizer": optimizer.state_dict(),
},
ckpt_path=opt.out_dir,
is_best=is_best,
)
logger.close()
dropout_rate=args.dropout_rate,
kernel_size=args.kernel_size,
padding=args.padding)
# Train the model.
train_data = list(zip(src_train_sents, tgt_train_sents))
dev_data = list(zip(src_dev_sents, tgt_dev_sents))
dataset = {"train_data": train_data, "dev_data": dev_data}
model_save_path = args.destination + "bin/model_%s_en.bin" % args.language
tic = time.time()
train(model,
dataset,
learning_rate=args.learning_rate,
lr_decay=args.lr_decay,
clip_grad=args.clip_grad,
batch_size=args.batch_size,
max_epochs=args.max_epochs,
max_num_trial=args.max_num_trial,
patience_limit=args.patience_limit,
model_save_path=model_save_path,
stdout=stdout)
toc = time.time()
print(f"Training took {((toc - tic) / 60):.3f} minutes", file=stdout)
# Compute and print BLEU score.
print("Computing corpuse level BLEU score..", file=stdout)
test_data = [src_test_sents, tgt_test_sents]
tic = time.time()
bleu_score = compute_corpus_level_bleu_score(model=model, data=test_data)
toc = time.time()
print(f"Corpus BLEU: {bleu_score*100:.3f}. Computed in {toc-tic:.3f} seconds.",
cur_fold = i + 1
print("Current fold : {}".format(cur_fold))
train_df = train_csv.iloc[train_index, :]
val_df = train_csv.iloc[val_index, :]
print(train_df['diagnosis'].value_counts())
print(val_df['diagnosis'].value_counts())
train_df.reset_index(drop=True, inplace=True)
val_df.reset_index(drop=True, inplace=True)
train(train_df,
val_df,
path,
weight_path,
cur_fold,
model_num=model_name[-1],
n_epochs=n_epochs)
elif pretraining == True:
weight_path = None
EXPERIMENT_NAME = "b4_384_Plateau_pretrain_only_resize_trans_v2"
path = os.path.join(
config.RES_PATH,
f'{EXPERIMENT_NAME}_{model_name}-{nfold}fold-{now}')
print(EXPERIMENT_NAME)
if not os.path.exists(path):
os.mkdir(path)
def main(cfg):
SEED = cfg.values.seed
BATCH_SIZE = cfg.values.train_args.batch_size
IMAGE_SIZE = cfg.values.image_size
USE_KFOLD = cfg.values.use_kfold
NUM_FOLD = cfg.values.train_args.num_fold if USE_KFOLD else 0
seed_everything(SEED)
print(f'Cuda is Available ? : {torch.cuda.is_available()}\n')
data_df = pd.read_csv('E:/seti-breakthrough-listen/train_labels.csv')
data_df['file_path'] = data_df['id'].apply(get_train_file_path)
train_transform = albumentations.Compose([
albumentations.Resize(IMAGE_SIZE, IMAGE_SIZE),
albumentations.HorizontalFlip(),
albumentations.VerticalFlip(),
# albumentations.Normalize(mean=(0.5, 0.5, 0.5), std=(0.25, 0.25, 0.25)),
albumentations.pytorch.transforms.ToTensorV2()
])
val_transform = albumentations.Compose([
albumentations.Resize(IMAGE_SIZE, IMAGE_SIZE),
# albumentations.Normalize(mean=(0.5, 0.5, 0.5), std=(0.25, 0.25, 0.25)),
albumentations.pytorch.transforms.ToTensorV2()
])
if USE_KFOLD:
kfold = StratifiedKFold(n_splits=NUM_FOLD,
shuffle=True,
random_state=SEED)
for k, (train_index,
val_index) in enumerate(kfold.split(data_df,
data_df['target'])):
print('\n')
cpprint('=' * 15 + f'{k + 1}-Fold Cross Validation' + '=' * 15)
train_df = data_df.iloc[train_index].reset_index(drop=True)
val_df = data_df.iloc[val_index].reset_index(drop=True)
train_loader = get_dataloader(df=train_df,
transform=train_transform,
batch_size=BATCH_SIZE,
shuffle=True)
val_loader = get_dataloader(df=val_df,
transform=val_transform,
batch_size=BATCH_SIZE,
shuffle=False)
val_labels = val_df['target'].values.tolist()
train(cfg, train_loader, val_loader, val_labels, k + 1)
else:
print('\n')
cpprint('=' * 15 + f'Start Training' + '=' * 15)
train_df, val_df = train_test_split(data_df,
test_size=0.2,
shuffle=True,
stratify=data_df['target'],
random_state=SEED)
train_loader = get_dataloader(df=train_df,
transform=train_transform,
batch_size=BATCH_SIZE,
shuffle=True)
val_loader = get_dataloader(df=val_df,
transform=val_transform,
batch_size=BATCH_SIZE,
shuffle=False)
val_labels = val_df['target'].values.tolist()
train(cfg, train_loader, val_loader, val_labels, 0)
for batch in dataloader_train:
# Get training data for this cycle
input_batches, input_lengths = batch['input'], batch[
'length'].numpy().tolist()
input_batches, input_lengths = zip(
*sorted(zip(input_batches, input_lengths),
key=lambda x: x[1],
reverse=True))
input_batches, input_lengths = torch.stack(input_batches), list(
input_lengths)
input_batches = input_batches[:, :max(input_lengths)]
input_batches = input_batches.transpose(0, 1)
# Run the train function
loss = train(input_batches, input_lengths, input_batches,
input_lengths, encoder, decoder, encoder_optimizer,
decoder_optimizer, batch_size, clip,
teacher_forcing_ratio)
# Keep track of loss
print_loss_total += loss
batch_n += 1
if batch_n % print_every == 0:
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print_summary = 'Epoch:%d - Batch:%d - Train_loss:%.4f' % (
epoch, batch_n, print_loss_avg)
logger.info(print_summary)
if batch_n % save_every == 0:
torch.save(encoder.state_dict(), args['save_path_encoder'])
from src.train import train
train('SuperMarioBros-1-1-v2', 'results', False)
log_dir = os.path.join(args.base_dir, hp.log.log_dir, args.model)
os.makedirs(log_dir, exist_ok=True)
chkpt_path = args.checkpoint_path if args.checkpoint_path is not None else None
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(
os.path.join(
log_dir,
'%s-%d.log' % (args.model, time.time()))),
logging.StreamHandler()
])
logger = logging.getLogger()
logger.info(hp_str)
for k, v in hp.data.items():
if v == '':
logger.error("config {} is empty. Raising Error".format(k))
raise Exception("Please specify config for {}".format(k))
writer = MyWriter(hp, log_dir)
trainloader = create_dataloader(hp, args, train=True)
testloader = create_dataloader(hp, args, train=False)
train(args, pt_dir, chkpt_path, trainloader, testloader, writer, logger,
hp, hp_str)
0.7]),
bias = 0.5
)
l2 = 0.5
model = RobustLogisticModel(1, l2_epsilon=l2, config=config)
syndata = SyntheticDataGenerator(config)
test_data = SyntheticDataGenerator(config)
rob = 0.0
print(f'**** training on natural plus {rob} perturbed***')
# custom train on natural data mixed with adversarially perturbed data
train(model, syndata.data(), robust=rob, epochs=40)
nat_nat = test(model, test_data.data())
nat_nat.update(train='nat', test='nat')
print(f'Perf on perturbed data:')
nat_per = test(model, test_data.data(), perturb=True)
nat_per.update(train='nat', test='per')
rob = 1.0
print(f'**** training on natural plus {rob} perturbed***')
# custom train on natural data mixed with adversarially perturbed data
train(model, syndata.data(), robust=rob, epochs=20)
per_nat = test(model, test_data.data())
per_nat.update(train='per', test='nat')
print(f'Perf on perturbed data:')
per_per = test(model, test_data.data(), perturb=True)
per_per.update(train='per', test='per')
# %%
from src.utils import get_data_dir
hidden_layers = 2
model = train({"model_name": "cnn",
"data_path": os.path.join(get_data_dir(), "whitenoise_10_reflections"),
"experiment_name": "3_to_4_conv2d_10_refs",
"max_epochs": 15,
"num_workers": 8,
"lr": 0.001,
"lr_sched_patience": 3,
"lr_sched_thresh": 0.1,
"hidden_layers": hidden_layers,
"kernel_widths": [(10, 1)] * (hidden_layers + 1),
"strides": [(1, 1)] * (hidden_layers + 1),
"hidden_channels": [200]*hidden_layers,
"sh_order_sig": 3,
"sh_order_scm": 4,
"gpus": -1,
"residual_flag": True,
"bias": True,
"complex_conv": True,
"fast_dev_run": False,
"bandwidth": 2000,
})
#
# # %%
# print(model.conv_layers[0].weight[0, :, 0])
# print(model.conv_layers[0].weight[1, :, 0])
# print(model.conv_layers[0].bias)
from src.train import train
from src.utils import get_experiments_dir, get_data_dir
opts = {
# ---folders---
"data_path": os.path.join(get_data_dir(), 'SCM', 'image-method'),
"logs_path": os.path.join(get_experiments_dir(), 'fc', 'image_method'),
"experiment_name": 'fc_imagemethod_full',
# ---network structure---
"model_name": 'fc',
"input_sh_order": 3,
"rank":
None, # None -> output is full matrix, Int -> output is low rank matrix transformed into full matrix
"hidden_layers": 1,
"hidden_sizes": [3000],
"residual_flag": True,
"residual_only": False,
# ---data---
"batch_size": 50,
"num_workers": 15,
# ---optimization---
"lr": 1e-3,
"lr_sched_thresh": 0.01,
"lr_sched_patience": 10,
"max_epochs": 1000,
"gpus": -1
}
train(opts)
