val_fold_index += 1 print("\n\n ---- Validation fold index: ", val_fold_index, "/", n_folds) print(datetime.now(), len(train_id_type_list), len(val_id_type_list)) assert len(to_set(train_id_type_list) & to_set(val_id_type_list)) == 0, "WTF" cnn = params['network'](lr=params['lr_kwargs']['lr'], **params, **params['network_kwargs']) params['save_prefix'] = params['save_prefix_template'].format( cnn_name=cnn.name, fold_index=val_fold_index - 1) print("\n {} - Loaded {} model ...".format(datetime.now(), cnn.name)) if 'pretrained_model' in params: load_pretrained_model(cnn, **params) print("\n {} - Start training ...".format(datetime.now())) h = train(cnn, train_id_type_list, val_id_type_list, **params) if h is None: continue hists.append(h) # ### Validation all classes n_runs = 2 n_folds = 5 run_counter = 0 cv_mean_scores = np.zeros((n_runs, n_folds)) val_fold_indices = [] # !!! CHECK BEFORE LOAD TO FLOYD
params['samples_per_epoch'] = 2 * len(train_id_type_list) params['nb_val_samples'] = int(1.0 * len(val_id_type_list)) val_fold_index += 1 print("\n\n ---- Validation fold index: ", val_fold_index, "/", n_folds) print(datetime.now(), len(train_id_type_list), len(val_id_type_list)) assert len(to_set(train_id_type_list) & to_set(val_id_type_list)) == 0, "WTF" cnn = params['network'](lr=params['lr_kwargs']['lr'], **params, **params['network_kwargs']) params['save_prefix'] = params['save_prefix_template'].format(cnn_name=cnn.name, fold_index=val_fold_index-1) print("\n {} - Loaded {} model ...".format(datetime.now(), cnn.name)) if 'pretrained_model' in params: load_pretrained_model(cnn, **params) elif 'pretrained_model_template' in params: params['pretrained_model'] = params['pretrained_model_template'].format(fold_index=(val_fold_index-1) % 3) print((val_fold_index-1) % 3) print(params['pretrained_model']) load_pretrained_model(cnn, by_name=True, **params) print("\n {} - Start training ...".format(datetime.now())) h = train(cnn, train_id_type_list, val_id_type_list, **params) if h is None: continue hists.append(h) # ### Validation all classes