def evaluate_baseline(test_set,
baseline_model,
epochs=100,
shot=1,
way=5,
querysize=32,
verbose=0,
load_weights_dirname=None,
save_finetune_dirname=None):
test_set = loader.load_miniImgnet(test_path)
# create dataset for training, validation, testing
train_img, train_label, val_img, val_label, test_img, test_label, labels_pick = \
tester.create_finetuningset(dataset=test_set, way=way, shot=shot, querysize=querysize)
train_img = train_img / 255.
val_img = val_img / 255.
test_img = test_img / 255.
input_shape = train_img.shape[1:]
# baseline
# baseline = Baseline(input_shape=input_shape,
# classes=way,
# loadbb=True,
# load_weigths_path=os.path.join("../weights/{}/{}".format(load_weights_dirname, \
# 'baseline_training.h5')),
# freezebb=True)
# baseline.compile(loss='sparse_categorical_crossentropy', optimizer=Adam(lr=1e-3),
# metrics=['accuracy'])
baseline_model.load_weights(os.path.join("../weights/{}/{}".format(load_weights_dirname, \
'baseline_training.h5')), by_name=True, skip_mismatch=True)
print(K.eval(baseline_model.layers[-1].bias))
history_baseline = trainer.train_model(model=baseline_model,
x=train_img, y=train_label,
shuffle=True,
val_data=(val_img, val_label),
batch_size=querysize,
epochs=epochs ,
save_weights_path=os.path.join("../weights/{}/{}".format(save_finetune_dirname, \
'baseline_finetuning_{}shot.h5'.format(shot))),
monitor='val_acc',
verbose=verbose
)
# load best model and
baseline_model.load_weights(
'../weights/{}/baseline_finetuning_{}shot.h5'.format(
save_finetune_dirname, shot))
performance_baseline_finetune = baseline_model.evaluate(x=test_img,
y=test_label)
dirpath = "../results/{}".format(save_finetune_dirname)
if not os.path.exists(dirpath):
os.makedirs(dirpath)
with open(os.path.join(dirpath, "result_baseline.txt"), "a") as file:
np.savetxt(file,
np.array(performance_baseline_finetune).reshape(1, -1))
for param in model.features.parameters():
param.requires_grad = False
print("getting data")
data_loaders = get_data()
print("getting model")
model.to(device)
EPOCHS = 5
#weight = torch.tensor([0.327, 0.586, 2.0, 2.572, 26.452, 15.0]).to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr=0.001)
print("TRAINING CLASSIFIER")
model, val_acc_history = train_model(model=model,
dataloaders=data_loaders,
criterion=criterion,
optimizer=optimizer,
num_epochs=EPOCHS)
torch.save(model.state_dict(), 'code/models/saved_models/mvcnn_stage_1.pkl')
for param in model.parameters():
param.requires_grad = True
print("FINE TUNING")
EPOCHS = 3
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(),
lr=0.00005) # We use a smaller learning rate
# hyperparameter search
if args.search is not None:
traverser.traverser(*args.search)
# experiments
if args.experiment is not None:
# compare all regression and vectorization methods
if args.experiment == "compare_all":
experiments.benchmark_all(args.filename, False)
experiments.benchmark_all(args.filename, True)
# evaluate a regression method with a vectorization method
if args.experiment == "evaluate":
MSE, RMSE, MAE, r_square = evaluater.evaluate_baseline(
args.vectorizer, args.method, args.filename, args.extra_feat)
print("MSE:", MSE)
print("RMSE:", RMSE)
print("MAE:", MAE)
print("R square:", r_square)
# pretrained BERT + regression neural network
if args.experiment == "train_net":
if args.save_name is None:
args.save_name = args.filename
if args.pretask[0] is not None:
args.pretask[0] = int(args.pretask[0])
trainer.train_model(args.filename, 20, [10, 15, 18, 20], 128, 1e-3,
args.save_name, args.extra_feat,
args.conditional, args.pretask[0],
args.pretask[1], args.dropout, args.batchnorm,
args.no_freeze)
train_img = train_img / 255.
val_img = val_img / 255.
# baseline
baseline = Baseline(input_shape=input_shape, classes=classes)
baseline.compile(loss='sparse_categorical_crossentropy',
optimizer=Adam(lr=optimizer_lr),
metrics=['accuracy'])
history_baseline = trainer.train_model(model=baseline,
x=train_img, y=train_label,
shuffle=True,
val_data=(val_img, val_label),
batch_size=64,
epochs=train_epoch ,
# foldername=EXP_NAME,
# weights_file='baseline_training.h5', # save weights
save_weights_path=os.path.join("../weights/{}/{}".format(EXP_NAME, \
'baseline_training.h5')),
monitor='val_acc',
)
out_filepath = os.path.join("../output/{}/{}".format(
EXP_NAME, 'history_baseline_training.pickle'))
saver.save_output(history_baseline, out_filepath)
# baseline_plus
baseline_plus = Baseline_plus(input_shape=input_shape, classes=classes)
baseline_plus.compile(loss='sparse_categorical_crossentropy',
optimizer=Adam(lr=optimizer_lr),
metrics=['accuracy'])
history_baseline_plus = trainer.train_model(model=baseline_plus,
def main(configs):
from utils.generate_dataset_jsons import generate_dataset_jsons
generate_dataset_jsons(configs.dataset_folder)
with open('dataset_rgb_train.json') as json_file:
dataset_rgb_train_json = json.load(json_file)
with open('dataset_rgb_valid.json') as json_file:
dataset_rgb_valid_json = json.load(json_file)
with open('dataset_mmaps_train.json') as json_file:
dataset_mmaps_train_json = json.load(json_file)
with open('dataset_mmaps_valid.json') as json_file:
dataset_mmaps_valid_json = json.load(json_file)
with open('dataset_flow_train.json') as json_file:
dataset_flow_train_json = json.load(json_file)
with open('dataset_flow_valid.json') as json_file:
dataset_flow_valid_json = json.load(json_file)
torch.backends.cudnn.benchmark = True
if os.path.exists(configs.output_folder):
print('Warning: output folder {} already exists!'.format(
configs.output_folder))
try:
os.makedirs(configs.output_folder)
except FileExistsError:
pass
normalize = Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
normalize_5 = Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
randomHorizontalFlip = RandomHorizontalFlip()
randomMultiScaleCornerCrop = MultiScaleCornerCrop(
[1, 0.875, 0.75, 0.65625], 224)
train_transforms = Compose([
randomHorizontalFlip, randomMultiScaleCornerCrop,
ToTensor(), normalize
])
train_transforms.randomize_parameters()
valid_transforms = Compose([CenterCrop(224), ToTensor(), normalize])
train_transforms_mmaps = Compose([
randomHorizontalFlip, randomMultiScaleCornerCrop,
Scale(7),
ToTensor(),
Binarize()
])
valid_transforms_mmaps = Compose(
[CenterCrop(224), Scale(7),
ToTensor(), Binarize()])
train_transforms_5 = Compose([
randomHorizontalFlip, randomMultiScaleCornerCrop,
ToTensor(), normalize_5
])
valid_transforms_5 = Compose([CenterCrop(224), ToTensor(), normalize_5])
print('Loading dataset')
if configs.dataset == 'DatasetRGB':
dataset_rgb_train = DatasetRGB(configs.dataset_folder,
dataset_rgb_train_json,
spatial_transform=train_transforms,
seqLen=configs.dataset_rgb_n_frames,
minLen=5)
dataset_rgb_valid = DatasetRGB(configs.dataset_folder,
dataset_rgb_valid_json,
spatial_transform=valid_transforms,
seqLen=configs.dataset_rgb_n_frames,
minLen=5,
device='cuda')
dataset_train = dataset_rgb_train
dataset_valid = dataset_rgb_valid
forward_fn = forward_rgb
elif configs.dataset == 'DatasetRGBMMAPS':
dataset_rgb_train = DatasetRGB(configs.dataset_folder,
dataset_rgb_train_json,
spatial_transform=train_transforms,
seqLen=configs.dataset_rgb_n_frames,
minLen=5)
dataset_rgb_valid = DatasetRGB(configs.dataset_folder,
dataset_rgb_valid_json,
spatial_transform=valid_transforms,
seqLen=configs.dataset_rgb_n_frames,
minLen=5,
device='cuda')
dataset_mmaps_train = DatasetMMAPS(
configs.dataset_folder,
dataset_mmaps_train_json,
spatial_transform=train_transforms_mmaps,
seqLen=configs.dataset_rgb_n_frames,
minLen=1,
enable_randomize_transform=False)
dataset_mmaps_valid = DatasetMMAPS(
configs.dataset_folder,
dataset_mmaps_valid_json,
spatial_transform=valid_transforms_mmaps,
seqLen=configs.dataset_rgb_n_frames,
minLen=1,
enable_randomize_transform=False,
device='cuda')
dataset_rgbmmaps_train = DatasetRGBMMAPS(dataset_rgb_train,
dataset_mmaps_train)
dataset_rgbmmaps_valid = DatasetRGBMMAPS(dataset_rgb_valid,
dataset_mmaps_valid)
dataset_train = dataset_rgbmmaps_train
dataset_valid = dataset_rgbmmaps_valid
forward_fn = forward_rgbmmaps
elif configs.dataset == 'DatasetFlow':
dataset_flow_train = DatasetFlow(
configs.dataset_folder,
dataset_flow_train_json,
spatial_transform=train_transforms,
stack_size=configs.dataset_flow_stack_size,
sequence_mode='single_random')
dataset_flow_valid = DatasetFlow(
configs.dataset_folder,
dataset_flow_valid_json,
spatial_transform=valid_transforms,
stack_size=configs.dataset_flow_stack_size,
sequence_mode='single_midtime')
dataset_train = dataset_flow_train
dataset_valid = dataset_flow_valid
forward_fn = forward_flow
elif configs.dataset == 'DatasetFlowMultiple':
dataset_flow_train = DatasetFlow(
configs.dataset_folder,
dataset_flow_train_json,
spatial_transform=train_transforms_5,
stack_size=configs.dataset_flow_stack_size,
n_sequences=configs.dataset_flow_n_sequences,
sequence_mode='multiple_jittered')
dataset_flow_valid = DatasetFlow(
configs.dataset_folder,
dataset_flow_valid_json,
spatial_transform=valid_transforms_5,
stack_size=configs.dataset_flow_stack_size,
n_sequences=configs.dataset_flow_n_sequences,
sequence_mode='multiple')
dataset_train = dataset_flow_train
dataset_valid = dataset_flow_valid
forward_fn = forward_flowmultiple
elif configs.dataset == 'DatasetRGBFlow':
dataset_rgb_train = DatasetRGB(configs.dataset_folder,
dataset_rgb_train_json,
spatial_transform=train_transforms,
seqLen=configs.dataset_rgb_n_frames,
minLen=5)
dataset_rgb_valid = DatasetRGB(configs.dataset_folder,
dataset_rgb_valid_json,
spatial_transform=valid_transforms,
seqLen=configs.dataset_rgb_n_frames,
minLen=5)
dataset_flow_train = DatasetFlow(
configs.dataset_folder,
dataset_flow_train_json,
spatial_transform=train_transforms,
stack_size=configs.dataset_flow_stack_size,
sequence_mode='single_random',
enable_randomize_transform=False)
dataset_flow_valid = DatasetFlow(
configs.dataset_folder,
dataset_flow_valid_json,
spatial_transform=valid_transforms,
stack_size=configs.dataset_flow_stack_size,
sequence_mode='single_midtime',
enable_randomize_transform=False)
dataset_rgbflow_train = DatasetRGBFlow(dataset_rgb_train,
dataset_flow_train)
dataset_rgbflow_valid = DatasetRGBFlow(dataset_rgb_valid,
dataset_flow_valid)
dataset_train = dataset_rgbflow_train
dataset_valid = dataset_rgbflow_valid
forward_fn = forward_rgbflow
elif configs.dataset == 'DatasetRGBFlowMultiple':
dataset_rgb_train = DatasetRGB(configs.dataset_folder,
dataset_rgb_train_json,
spatial_transform=train_transforms,
seqLen=configs.dataset_rgb_n_frames,
minLen=5)
dataset_rgb_valid = DatasetRGB(configs.dataset_folder,
dataset_rgb_valid_json,
spatial_transform=valid_transforms,
seqLen=configs.dataset_rgb_n_frames,
minLen=5)
dataset_flow_train = DatasetFlow(
configs.dataset_folder,
dataset_flow_train_json,
spatial_transform=train_transforms_5,
stack_size=configs.dataset_flow_stack_size,
n_sequences=configs.dataset_rgb_n_frames,
sequence_mode='multiple',
enable_randomize_transform=False)
dataset_flow_valid = DatasetFlow(
configs.dataset_folder,
dataset_flow_valid_json,
spatial_transform=valid_transforms_5,
stack_size=configs.dataset_flow_stack_size,
n_sequences=configs.dataset_rgb_n_frames,
sequence_mode='multiple',
enable_randomize_transform=False)
dataset_rgbflow_train = DatasetRGBFlow(dataset_rgb_train,
dataset_flow_train)
dataset_rgbflow_valid = DatasetRGBFlow(dataset_rgb_valid,
dataset_flow_valid)
dataset_train = dataset_rgbflow_train
dataset_valid = dataset_rgbflow_valid
forward_fn = forward_rgbflowmultiple
else:
raise ValueError('Unknown dataset type: {}'.format(configs.dataset))
report_file = open(os.path.join(configs.output_folder, 'report.txt'), 'a')
for config in configs:
config_name = hashlib.md5(str(config).encode('utf-8')).hexdigest()
print('Running', config_name)
with open(
os.path.join(configs.output_folder,
config_name + '.params.txt'), 'w') as f:
f.write(pprint.pformat(config))
train_loader = torch.utils.data.DataLoader(
dataset_train, **config['TRAIN_DATA_LOADER'])
valid_loader = torch.utils.data.DataLoader(
dataset_valid, **config['VALID_DATA_LOADER'])
model_class = config['MODEL']['model']
model_params = {
k: v
for (k, v) in config['MODEL'].items() if k != 'model'
}
model = model_class(**model_params)
if config['TRAINING'].get('_model_state_dict', None) is not None:
model.load_weights(config['TRAINING']['_model_state_dict'])
model.train(config['TRAINING']['train_mode'])
model.cuda()
loss_class = config['LOSS']['loss']
loss_params = {
k: v
for (k, v) in config['LOSS'].items() if k != 'loss'
}
loss_fn = loss_class(**loss_params)
model_weights = []
for i in range(10):
group_name = '_group_' + str(i)
if group_name + '_params' in config['OPTIMIZER']:
model_weights_group = {
'params':
model.get_training_parameters(
name=config['OPTIMIZER'][group_name + '_params'])
}
for k in config['OPTIMIZER']:
if k.startswith(
group_name) and k != group_name + '_params':
model_weights_group[k[9:]] = config['OPTIMIZER'][k]
model_weights.append(model_weights_group)
if len(model_weights) == 0:
model_weights = model.get_training_parameters()
optimizer_class = config['OPTIMIZER']['optimizer']
optimizer_params = {
k: v
for (k, v) in config['OPTIMIZER'].items()
if k != 'optimizer' and not k.startswith('_')
}
optimizer = optimizer_class(model_weights, **optimizer_params)
scheduler_class = config['SCHEDULER']['scheduler']
scheduler_params = {
k: v
for (k, v) in config['SCHEDULER'].items()
if k != 'scheduler' and not k.startswith('_')
}
scheduler_lr = scheduler_class(optimizer, **scheduler_params)
model_state_dict_path = os.path.join(configs.output_folder,
config_name + '.model')
logfile = open(
os.path.join(configs.output_folder, config_name + '.log.txt'), 'w')
result = train_model(model=model,
train_loader=train_loader,
valid_loader=valid_loader,
forward_fn=forward_fn,
loss_fn=loss_fn,
optimizer=optimizer,
scheduler_lr=scheduler_lr,
model_state_dict_path=model_state_dict_path,
logfile=logfile,
**{
k: v
for (k, v) in config['TRAINING'].items()
if not k.startswith('_')
})
max_valid_accuracy_idx = np.argmax(result['accuracies_valid'])
print(
'{} | Train Loss {:04.2f} | Train Accuracy = {:05.2f}% | Valid Loss {:04.2f} | Valid Accuracy = {:05.2f}%'
.format(config_name,
result['losses_train'][max_valid_accuracy_idx],
result['accuracies_train'][max_valid_accuracy_idx] * 100,
result['losses_valid'][max_valid_accuracy_idx],
result['accuracies_valid'][max_valid_accuracy_idx] * 100),
file=report_file)