val_fold_index += 1
print("\n\n ---- Validation fold index: ", val_fold_index, "/", n_folds)
print(datetime.now(), len(train_id_type_list), len(val_id_type_list))
assert len(to_set(train_id_type_list)
& to_set(val_id_type_list)) == 0, "WTF"
cnn = params['network'](lr=params['lr_kwargs']['lr'],
**params,
**params['network_kwargs'])
params['save_prefix'] = params['save_prefix_template'].format(
cnn_name=cnn.name, fold_index=val_fold_index - 1)
print("\n {} - Loaded {} model ...".format(datetime.now(), cnn.name))
if 'pretrained_model' in params:
load_pretrained_model(cnn, **params)
print("\n {} - Start training ...".format(datetime.now()))
h = train(cnn, train_id_type_list, val_id_type_list, **params)
if h is None:
continue
hists.append(h)
# ### Validation all classes
n_runs = 2
n_folds = 5
run_counter = 0
cv_mean_scores = np.zeros((n_runs, n_folds))
val_fold_indices = [] # !!! CHECK BEFORE LOAD TO FLOYD
params['samples_per_epoch'] = 2 * len(train_id_type_list)
params['nb_val_samples'] = int(1.0 * len(val_id_type_list))
val_fold_index += 1
print("\n\n ---- Validation fold index: ", val_fold_index, "/", n_folds)
print(datetime.now(), len(train_id_type_list), len(val_id_type_list))
assert len(to_set(train_id_type_list) & to_set(val_id_type_list)) == 0, "WTF"
cnn = params['network'](lr=params['lr_kwargs']['lr'], **params, **params['network_kwargs'])
params['save_prefix'] = params['save_prefix_template'].format(cnn_name=cnn.name, fold_index=val_fold_index-1)
print("\n {} - Loaded {} model ...".format(datetime.now(), cnn.name))
if 'pretrained_model' in params:
load_pretrained_model(cnn, **params)
elif 'pretrained_model_template' in params:
params['pretrained_model'] = params['pretrained_model_template'].format(fold_index=(val_fold_index-1) % 3)
print((val_fold_index-1) % 3)
print(params['pretrained_model'])
load_pretrained_model(cnn, by_name=True, **params)
print("\n {} - Start training ...".format(datetime.now()))
h = train(cnn, train_id_type_list, val_id_type_list, **params)
if h is None:
continue
hists.append(h)
# ### Validation all classes