def train_model_opt(parameters):
print("Training model with hyper-parameters: {}\n\n\n".format(parameters))
args = parse_args()
# Load the config file
cfg = load_config(args)
# Make the results reproducible
fix_seed(cfg.SEED)
cfg.TRAIN.LR = parameters[0]
cfg.TRAIN.BATCH_SIZE = int(parameters[1])
# Preparing data
(train_loader,
valid_loader) = prepare_dataloaders(cfg)
# Define model architecture
vgg19 = VGG("VGG19", num_classes_length=7, num_classes_digits=10)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("Device used: ", device)
# We return negative accuracy since we have minimization function (gp_minimize)
return -train_model(vgg19, cfg=cfg, train_loader=train_loader, valid_loader=valid_loader, device=device)
torch.backends.cudnn.benchmark = False
if __name__ == '__main__':
# Load the config file.
load_config()
# Make the results reproductible.
fix_seed(cfg.SEED)
# Prepare data.
(train_loader, valid_loader) = prepare_dataloaders(
dataset_split=cfg.TRAIN.DATASET_SPLIT,
dataset_path=cfg.INPUT_DIR,
metadata_filename=cfg.METADATA_FILENAME,
batch_size=cfg.TRAIN.BATCH_SIZE,
sample_size=cfg.TRAIN.SAMPLE_SIZE,
valid_split=cfg.TRAIN.VALID_SPLIT)
# Define model architecture
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("Device used: ", device)
# Check output directory for most recent checkpoint. If it exists, load.
checkpoint = os.path.join(cfg.OUTPUT_DIR, 'checkpoint.pth.tar')
if os.path.isfile(checkpoint):
resume = True
state = torch.load(checkpoint)
base_iteration = state['iteration']
def eval_model(
dataset_dir,
metadata_filename,
model_filename,
model_cfg,
batch_size=32,
sample_size=-1,
):
"""
Validation loop.
Parameters
----------
dataset_dir : str
Directory with all the images.
metadata_filename : str
Absolute path to the metadata pickle file.
model_filename : str
path/filename where to save the model.
batch_size : int
Mini-batch size.
sample_size : int
Number of elements to use as sample size,
for debugging purposes only. If -1, use all samples.
Returns
-------
y_pred : ndarray
Prediction of the model.
"""
seed = 1234
print("pytorch/random seed: {}".format(seed))
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.backends.cudnn.deterministic = True
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
dataset_split = "test"
# dataset_split = 'train'
test_loader = prepare_dataloaders(
dataset_split=dataset_split,
dataset_path=dataset_dir,
metadata_filename=metadata_filename,
batch_size=batch_size,
sample_size=sample_size,
num_worker=0,
)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("Device used: ", device)
cfg_from_file(model_cfg)
# Load best model
model_dict = torch.load(model_filename, map_location=device)
current_hyper_params_dict = model_dict["hyper_params"]
model = ModularSVNHClassifier(
cfg.MODEL,
feature_transformation=ResNet34(
current_hyper_params_dict["FEATURES_OUTPUT_SIZE"]
),
length_classifier=LengthClassifier(
cfg.MODEL, current_hyper_params_dict["FEATURES_OUTPUT_SIZE"]
),
number_classifier=NumberClassifier,
hyper_params=current_hyper_params_dict,
)
model.load_state_dict(model_dict["model_state_dict"])
since = time.time()
model = model.to(device)
print("# Testing Model ... #")
stats = StatsRecorder()
performance_evaluator = PerformanceEvaluator(test_loader)
y_pred, y_true = performance_evaluator.evaluate(
model, device, stats, mode="test"
)
test_accuracy = stats.test_best_accuracy
print("===============================")
print("\n\nTest Set Accuracy: {}".format(test_accuracy))
time_elapsed = time.time() - since
print(
"\n\nTesting complete in {:.0f}m {:.0f}s".format(
time_elapsed // 60, time_elapsed % 60
)
)
y_true = np.asarray(y_true)
y_pred = np.asarray(y_pred)
return y_pred
def eval_model(dataset_dir,
metadata_filename,
model_filename,
batch_size=32,
sample_size=-1):
'''
Validation loop.
Parameters
----------
dataset_dir : str
Directory with all the images.
metadata_filename : str
Absolute path to the metadata pickle file.
model_filename : str
path/filename where to save the model.
batch_size : int
Mini-batch size.
sample_size : int
Number of elements to use as sample size,
for debugging purposes only. If -1, use all samples.
Returns
-------
y_pred : ndarray
Prediction of the model.
'''
seed = 1234
print('pytorch/random seed: {}'.format(seed))
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.backends.cudnn.deterministic = True
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
dataset_split = 'test'
test_loader = prepare_dataloaders(dataset_split=dataset_split,
dataset_path=dataset_dir,
metadata_filename=metadata_filename,
batch_size=batch_size,
sample_size=sample_size)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("Device used: ", device)
# Load best model
model = torch.load(model_filename, map_location=device)
since = time.time()
model = model.to(device)
model = model.eval()
print("# Testing Model ... #")
test_correct = 0
test_n_samples = 0
y_pred = []
for i, batch in enumerate(tqdm(test_loader)):
# get the inputs
inputs, targets = batch['image'], batch['target']
inputs = inputs.to(device)
target_ndigits = targets[:, 0].long()
target_ndigits = target_ndigits.to(device)
# Forward
output_len, output_seq = model(inputs)
predictions = gen_predictions(output_len, output_seq)
y_pred.extend(list(predictions))
time_elapsed = time.time() - since
print('\n\nTesting complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
y_pred = np.asarray(y_pred)
return y_pred
# Make the results reproductible
seed = cfg.SEED
model_dict = None
if args.model is not None:
model_dict = torch.load(args.model, map_location=device)
seed = model_dict["seed"]
fix_seed(seed)
# Prepare data
(train_loader, valid_loader) = prepare_dataloaders(
dataset_split=cfg.TRAIN.DATASET_SPLIT,
dataset_path=cfg.INPUT_DIR,
metadata_filename=cfg.METADATA_FILENAME,
batch_size=cfg.TRAIN.BATCH_SIZE,
sample_size=cfg.TRAIN.SAMPLE_SIZE,
valid_split=cfg.TRAIN.VALID_SPLIT,
test_split=cfg.TRAIN.TEST_SPLIT,
valid_metadata_filename=args.valid_metadata_filename,
valid_dataset_dir=args.valid_dataset_dir,
num_worker=cfg.TRAIN.NUM_WORKER,
)
print("Start training from ", cfg.INPUT_DIR)
hyper_param_search_state = None
if args.model is not None:
model_filename = Path(args.model)
print("\nLoading model from", model_filename.absolute())
model = torch.load(model_filename, map_location=device)
hyper_param_search_state = model_dict["hyper_param_search_state"]
def instantiate_model(hyper_params):
def eval_model(dataset_dir,
metadata_filename,
model_filename,
batch_size=32,
sample_size=-1):
'''
Validation loop.
Parameters
----------
dataset_dir : str
Directory with all the images.
metadata_filename : str
Absolute path to the metadata pickle file.
model_filename : str
path/filename where to save the model.
batch_size : int
Mini-batch size.
sample_size : int
Number of elements to use as sample size,
for debugging purposes only. If -1, use all samples.
Returns
-------
y_pred : ndarray
Prediction of the model.
'''
seed = 1234
print('pytorch/random seed: {}'.format(seed))
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.backends.cudnn.deterministic = True
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
dataset_split = 'test'
test_loader = prepare_dataloaders(dataset_split=dataset_split,
dataset_path=dataset_dir,
metadata_filename=metadata_filename,
batch_size=batch_size,
sample_size=sample_size)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("Device used: ", device)
# Load best model
model = torch.load(model_filename, map_location=device)
since = time.time()
model = model.to(device)
model = model.eval()
print("# Testing Model ... #")
test_correct = 0
test_n_samples = 0
y_true = []
y_pred = []
for i, batch in enumerate(tqdm(test_loader)):
# get the inputs
inputs, targets = batch['image'], batch['target']
inputs = inputs.to(device)
target_ndigits = targets[:, 0].long()
target_digits = targets[:, 1:].long()
target = torch.cat([
target_digits[:, digit_rank].unsqueeze(1) * 10**(4 - digit_rank)
for digit_rank in range(5)
],
dim=1)
target[target < 0] = 0
target = target.sum(1)
adjtargfor_length = torch.pow(torch.full_like(target_ndigits, 10),
5 - target_ndigits)
target = target / adjtargfor_length
# Forward
outputs_ndigits, outputs_digits = model(inputs)
outputs_ndigits = outputs_ndigits
outputs_digits = outputs_digits
# Statistics
predicted_ndigits = torch.max(outputs_ndigits, 1)[1]
predicted_digits = torch.cat([
torch.max(output_digit.data, dim=1)[1].unsqueeze(1) *
10**(4 - digit_rank)
for digit_rank, output_digit in enumerate(outputs_digits)
],
dim=1).sum(1)
adj_for_length = torch.pow(torch.full_like(predicted_ndigits, 10),
5 - predicted_ndigits)
predicted = predicted_digits / adj_for_length
predicted = predicted.cpu().numpy()
target = target.cpu().numpy()
y_pred.extend(list(predicted))
y_true.extend(list(target))
test_correct += (predicted == target).sum().item()
test_n_samples += target_ndigits.size(0)
test_accuracy = test_correct / test_n_samples
print('\n\nTest Set Accuracy: {:.4f}'.format(test_accuracy))
time_elapsed = time.time() - since
print('\n\nTesting complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
y_true = np.asarray(y_true)
y_pred = np.asarray(y_pred)
return y_pred
checkpoint = CheckpointSaver(args.checkpoint_dir)
# Load model from checkpoint
model, cfg = checkpoint.load(args.checkpoint_name)
# Make results reproducible
fix_seed(cfg.SEED)
else:
# Load the config file
cfg = load_config(args)
# Make results reproducible
fix_seed(cfg.SEED)
# Define model architecture
model = VGG('VGG19', num_classes_length=7, num_classes_digits=10)
# Prepare data
(train_loader, valid_loader) = prepare_dataloaders(cfg)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("Device used: ", device)
# Start model training
train_model(model,
cfg=cfg,
train_loader=train_loader,
valid_loader=valid_loader,
device=device)
