def train(args):
import pytorch.training as training
from core.openwindow import OpenWindow
# Create training and validation sets
dataset = OpenWindow(args.dataset_dir)
if args.training_mask:
train_set = dataset['root'].subset('training', args.training_mask)
else:
train_set = dataset['training']
if args.validation_mask:
val_set = dataset['root'].subset('validation', args.validation_mask)
else:
val_set = dataset['validation']
# Ensure output directories exist
log_dir = args.log_dir / args.training_id
log_dir.mkdir(parents=True, exist_ok=True)
model_dir = args.model_dir / args.training_id
model_dir.mkdir(parents=True, exist_ok=True)
params = {
'seed': args.seed,
'batch_size': args.batch_size,
'n_epochs': args.n_epochs,
'lr': args.lr,
'lr_decay': args.lr_decay,
'lr_decay_rate': args.lr_decay_rate,
}
print(params, '\n')
print(f'log_dir: {log_dir}')
print(f'model_dir: {model_dir}')
training.train(train_set, val_set, log_dir, model_dir, **params)
def retraining(model, retrained_layers_schemes, config, dataset):
train_dataset, rotated_train_dataset = get_data_generator(
dataset.x_train, dataset.y_train, config.batch_size)
test_dataset, rotated_test_dataset = get_data_generator(
dataset.x_test, dataset.y_test, config.batch_size)
_, accuracy, _, _ = test(model, test_dataset, config.use_cuda,
config.loss_function)
_, rotated_accuracy, _, _ = test(model, rotated_test_dataset,
config.use_cuda, config.loss_function)
unrotated_accuracies = [accuracy]
rotated_accuracies = [rotated_accuracy]
models = {"None": model}
for retrained_layers in retrained_layers_schemes:
#TODO
retrained_model, retrained_model_optimizer = model_optimizer_generator(
previous_model=model, trainable_layers=retrained_layers)
# TODO
# freeze_layers_except(retrained_model.layers(),retrained_model.layer_names(),retrained_layers)
#for name, val in retrained_model.named_parameters():
# print(name, val.requires_grad)
retrained_layers_id = "_".join(retrained_layers)
print(f"Retraining {retrained_layers} with rotated dataset:")
history = train(retrained_model, config.retrain_epochs,
retrained_model_optimizer, config.use_cuda,
rotated_train_dataset, rotated_test_dataset,
config.loss_function)
models["retrained_" + retrained_layers_id] = retrained_model
_, accuracy, _, _ = test(retrained_model, test_dataset,
config.use_cuda, config.loss_function)
_, rotated_accuracy, _, _ = test(retrained_model, rotated_test_dataset,
config.use_cuda, config.loss_function)
unrotated_accuracies.append(accuracy)
rotated_accuracies.append(rotated_accuracy)
datasets = {
"test_dataset": test_dataset,
"rotated_test_dataset": rotated_test_dataset,
"train_dataset": train_dataset,
"rotated_train_dataset": rotated_train_dataset
}
print("Evaluating accuracy for all models/datasets:")
scores = eval_scores(models, datasets, config, config.loss_function)
return scores, models, unrotated_accuracies, rotated_accuracies
def do_run(model: nn.Module, dataset: datasets.ClassificationDataset,
t: tm.TransformationSet, o: Options, optimizer: Optimizer,
epochs: int, loss_function: torch.nn.Module,
epochs_callbacks: {int: Callable}):
train_dataset = get_data_generator(dataset.x_train, dataset.y_train, t,
o.batch_size, o.num_workers,
TransformationStrategy.random_sample)
test_dataset = get_data_generator(dataset.x_test, dataset.y_test, t,
o.batch_size, o.num_workers,
TransformationStrategy.random_sample)
history = train(model,
epochs,
optimizer,
o.use_cuda,
train_dataset,
test_dataset,
loss_function,
verbose=o.verbose_train,
epochs_callbacks=epochs_callbacks,
batch_verbose=o.verbose_batch)
return history
def run(config,
model,
rotated_model,
dataset,
plot_accuracy=False,
loss_function=torch.nn.NLLLoss(),
save_plots=False):
os.makedirs(experiment_plot_path(model.name, dataset.name), exist_ok=True)
train_dataset, rotated_train_dataset = get_data_generator(
dataset.x_train, dataset.y_train, config.batch_size)
test_dataset, rotated_test_dataset = get_data_generator(
dataset.x_test, dataset.y_test, config.batch_size)
# UNROTATED DATASET
if config.epochs == 0:
print(
f"### Skipping training model |{model.name}| with unrotated dataset |{dataset.name}|"
)
history = {}
else:
print(
f"### Training model |{model.name}| with unrotated dataset |{dataset.name}| for {config.epochs} epochs...",
flush=True)
history = train(model, config.epochs, config.optimizer,
config.use_cuda, train_dataset, test_dataset,
loss_function)
if plot_accuracy:
accuracy_plot_path = plot_history(history, "unrotated", model.name,
dataset.name, save_plots)
# ROTATED MODEL, UNROTATED DATASET
if config.pre_rotated_epochs == 0:
print(
f"### Skipping pretraining rotated model |{model.name}| with unrotated dataset |{dataset.name}|"
)
else:
print(
f"### Pretraining rotated model |{model.name}| with unrotated dataset |{dataset.name}|for {config.pre_rotated_epochs} epochs...",
flush=True)
pre_rotated_history = train(rotated_model, config.rotated_epochs,
config.rotated_optimizer, config.use_cuda,
train_dataset, test_dataset, loss_function)
if plot_accuracy:
plot_history(pre_rotated_history, "pre_rotated", model.name,
dataset.name, save_plots)
# ROTATED DATASET
if config.rotated_epochs == 0:
print(
f"### Skipping training of rotated model |{model.name}| with rotated dataset |{dataset.name}|"
)
else:
print(
f"### Training rotated model |{model.name}| with rotated dataset |{dataset.name}| for {config.rotated_epochs} epochs...",
flush=True)
rotated_history = train(rotated_model, config.rotated_epochs,
config.rotated_optimizer, config.use_cuda,
rotated_train_dataset, rotated_test_dataset,
loss_function)
if plot_accuracy:
rotated_accuracy_plot_path = plot_history(rotated_history,
"rotated",
rotated_model.name,
dataset.name, save_plots)
print("### Testing both models on both datasets...", flush=True)
models = {"rotated_model": rotated_model, "model": model}
datasets = {
"test_dataset": test_dataset,
"rotated_test_dataset": rotated_test_dataset,
"train_dataset": train_dataset,
"rotated_train_dataset": rotated_train_dataset
}
scores = eval_scores(models, datasets, config, loss_function)
train_test_path = train_test_accuracy_barchart2(scores, model.name,
dataset.name, save_plots)
experiment_plot = os.path.join(
"plots", f"{model.name}_{dataset.name}_train_rotated.png")
os.system(
f"convert {accuracy_plot_path} {rotated_accuracy_plot_path} {train_test_path} +append {experiment_plot}"
)
logging.info("training info saved to {experiment_plot}")
return scores
def train(dataset, args):
"""Train a model on a dataset.
Args:
dataset (Dataset): Information about the dataset.
args: Named tuple of configuration arguments.
"""
import pytorch.training as training
import relabel
import utils
# Load training data and metadata
x_train, df_train = _load_features(dataset, args.extraction_path,
args.block_size)
# Use subset of training set as validation set
mask = df_train.validation == 1
x_val = x_train[mask]
df_val = df_train[mask]
x_train = x_train[~mask]
df_train = df_train[~mask]
# Mask out training data based on user specification
if args.mask:
x_train, df_train = _mask_data(x_train, df_train, args.mask)
# Encode labels as one-hot vectors
y_train = pd.get_dummies(df_train.label)
y_val = pd.get_dummies(df_val.label)
# Relabel examples if relabeling is enabled
if args.relabel:
if not args.confidence_path:
args.confidence_path = args.pseudolabel_path
mask = df_train.manually_verified == 0
y_pred = pd.read_csv(args.pseudolabel_path, index_col=0)
y_conf = pd.read_csv(args.confidence_path, index_col=0).max(axis=1)
y_train[mask] = relabel.relabel(y_train[mask], y_pred, y_conf,
args.relabel_threshold,
args.relabel_weight)
# Ensure output directories exist
model_path = os.path.join(args.model_path, args.training_id)
log_path = os.path.join(args.log_path, args.training_id)
os.makedirs(model_path, exist_ok=True)
os.makedirs(log_path, exist_ok=True)
# Save hyperparameters to disk
params = {
k: v
for k, v in vars(args).items() if k in [
'model',
'mask',
'seed',
'batch_size',
'n_epochs',
'lr',
'lr_decay',
'lr_decay_rate',
'augment',
'overwrite',
'relabel',
'relabel_threshold',
'relabel_weight',
]
}
_log_parameters(os.path.join(model_path, 'parameters.json'), **params)
# Standardize data and save statistics to disk
mean, std = utils.statistics(x_train)
x_train, x_val = utils.standardize([x_train, x_val], mean, std)
pickle.dump((mean, std), open(os.path.join(model_path, 'scaler.p'), 'wb'))
training.train(x_train, y_train.values, x_val, y_val.values, df_val.index,
log_path, model_path, **params)