def get_trainer(config, num_classes, start_epoch, perf_tester=None):
model_store = LocalModelStore(config['MODELLING']['architecture'],
config['GENERAL']['experiment_name'],
config['GENERAL']['root_dir'])
checkpoint_path_param_name = 'checkpoint_path'
checkpoint_path = None
if checkpoint_path_param_name in config['MODELLING']:
checkpoint_path = config['MODELLING'][checkpoint_path_param_name]
print(checkpoint_path)
if checkpoint_path == '':
checkpoint_path = None
else:
mlflow.log_param('checkpoint_path', checkpoint_path)
trainer_factory = TrainerFactory(
ArcFaceModelInitializer(json.loads(config['MODELLING']['feature_parallelized_architectures'])),
ArcFaceCriterionInitializer(),
GenericOptimizerInitializer(),
GenericLRSchedulerInitializer(),
model_store)
perf_threshold = None
num_epochs_to_test = None
num_batches_per_epoch_limit = int(config['MODELLING']['num_batches_per_epoch_limit'])
mlflow.log_param('num_batches_per_epoch_limit', num_batches_per_epoch_limit)
if perf_tester is not None:
perf_threshold = float(config['MODELLING']['perf_threshold'])
mlflow.log_param('perf_threshold', perf_threshold)
num_epochs_to_test = int(config['MODELLING']['num_epochs_to_test'])
mlflow.log_param('num_epochs_to_test', num_epochs_to_test)
perf_test_param_name = 'performance_test'
perf_test_name = 'None'
if perf_test_param_name in config['MODELLING']:
perf_test_name = config['MODELLING'][perf_test_param_name]
perf_logger = get_performance_logger(config)
logs_path = None
if 'logs_path' in config['MODELLING']:
logs_path = config['MODELLING']['logs_path']
trainer = trainer_factory.get_trainer(config['MODELLING']['architecture'],
config['OPTIMIZING']['optimizer'], json.loads(config['OPTIMIZING']['optimizer_params']),
config['MODELLING']['criterion_name'], json.loads(config['MODELLING']['criterion_params']),
config['OPTIMIZING']['lr_scheduler'], json.loads(config['OPTIMIZING']['lr_scheduler_params']),
num_classes,
config['MODELLING']['is_pretrained'] == 'True',
epoch=start_epoch,
checkpoint=checkpoint_path,
performance_tester=perf_tester,
performance_threshold=perf_threshold,
num_epochs_to_test=num_epochs_to_test,
num_batches_per_epoch_limit=num_batches_per_epoch_limit, test_type=perf_test_name,
perf_logger=perf_logger, logs_path=logs_path, finetuning=('FINETUNING' in config))
return trainer
def check_synapse_distances(architecture, experiment_name):
pretraining_dataset_path = "/home/administrator/experiments/familiarity/dataset/processed_pretraining_dataset/phase_perc_size/pretraining_fixed_{'train': 0.7, 'val': 0.2, 'test': 0.1}/train"
finetuning_dataset_path = "/home/administrator/experiments/familiarity/dataset/processed_finetuning_dataset/phase_perc_size/pretraining_fixed_A_{'train': 220, 'val': 50, 'test': 50}/train"
epoch = 119
root_dir = '/home/administrator/experiments/familiarity/'
datasets_paths = [pretraining_dataset_path]
idx_to_cls = get_full_dataset_idx_to_cls(datasets_paths)
model = ArcFaceModelInitializer(['vgg16'
]).get_model('vgg16',
False,
num_classes=len(idx_to_cls))
model_store = LocalModelStore(architecture, experiment_name, root_dir)
stuff = model_store.load_model_and_optimizer(model, epoch=epoch)
model = stuff[0]
fc8 = model.classifier[-1]
dist_mat = calculate_matrix_column_distances_gpu(fc8.weight, idx_to_cls)
dist_mat.to_csv(
os.path.join(root_dir, experiment_name, architecture, 'results',
'fc8_dist_mat.csv'))
def run_experiment(config_path):
# Get the configuration file
print(datetime.datetime.now())
start = time.perf_counter()
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
config.read(config_path)
print("Running experiment " + config['GENERAL']['experiment_name'])
num_iterations = int(config['GENERAL']['num_experiments'])
num_ids = json.loads(config['DATASET']['num_ids'])
num_pics = json.loads(config['DATASET']['num_pics'])
overall_index = 0
all_lfw_results = None
all_output = None
experiment_name = config['GENERAL']['experiment_name']
for id in num_ids:
for pic in num_pics:
for i in range(num_iterations):
print("________ ITER " + str(i) + " ________")
print("________ NUM IDS " + str(id) + " ________")
print("________ NUM PICS " + str(pic) + " ________")
config['GENERAL']['experiment_name'] = experiment_name + '_' + str(id) + '_' + str(pic) + '_' + str(i)
# Create image loader (by the configuration)
im_size = int(config['DATASET']['image_size'])
post_crop_im_size = int(config['DATASET']['post_crop_im_size'])
dataset_means = json.loads(config['DATASET']['dataset_means'])
dataset_stds = json.loads(config['DATASET']['dataset_stds'])
image_loader = RandomExperimentImageLoader(im_size, post_crop_im_size, dataset_means, dataset_stds)
print("training on dataset: ", config['DATASET']['raw_dataset_path'])
# Create dataloader for the training
dataloaders = dataloaders_setup(config, config['DATASET']['raw_dataset_path'], image_loader, id, pic)
# Get access to pre trained models
model_store = LocalModelStore(config['MODELLING']['architecture'],
config['GENERAL']['root_dir'],
config['GENERAL']['experiment_name'])
start_epoch = int(config['MODELLING']['start_epoch'])
end_epoch = int(config['MODELLING']['end_epoch'])
# creating the lfw tester
lfw_tester = get_lfw_test(config, image_loader)
# Creating the trainer and loading the pretrained model if specified in the configuration
trainer = get_trainer(config, id, start_epoch, lfw_tester)
# Will train the model from start_epoch to (end_epoch - 1) with the given dataloaders
trainer.train_model(start_epoch, end_epoch, dataloaders)
lfw_results = lfw_tester.test_performance(trainer.model)
lfw_results['index'] = overall_index
lfw_results['num_ids'] = id
lfw_results['num_pics'] = pic
if all_lfw_results is None:
all_lfw_results = lfw_results
else:
all_lfw_results = all_lfw_results.append(lfw_results)
print(lfw_results)
reps_behaviour_extractor = setup_pairs_reps_behaviour(config, image_loader)
if reps_behaviour_extractor != None:
output = reps_behaviour_extractor.compare_lists(trainer.model)
output['index'] = overall_index
output['num_ids'] = id
output['num_pics'] = pic
# saving results for the meanwhile
results_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'],
config['REP_BEHAVIOUR']['output_filename'] + '.csv')
lfw_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'], 'logs.csv')
os.makedirs(config['REP_BEHAVIOUR']['reps_results_path'], exist_ok=True)
output.to_csv(results_path)
lfw_results.to_csv(lfw_path)
if all_output is None:
all_output = output
else:
all_output = all_output.append(output)
overall_index += 1
print(all_output)
print(all_lfw_results)
config['GENERAL']['experiment_name'] = experiment_name
# lfw_path = os.path.join(root_dir, experiment_name + '_all', rep_dir, lfw_filename)
# results_path = os.path.join(root_dir, experiment_name + '_all', rep_dir, rep_filename)
results_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'],
config['REP_BEHAVIOUR']['output_filename'] + '.csv')
lfw_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'], config['LFW_TEST']['lfw_results_file'] + '.csv')
print('Saving results in ', results_path)
os.makedirs(config['REP_BEHAVIOUR']['reps_results_path'], exist_ok=True)
if all_output is not None:
all_output.to_csv(results_path)
all_lfw_results.to_csv(lfw_path)
end = time.perf_counter()
print(datetime.datetime.now())
print((end - start) / 3600)
print((time.process_time()) / 3600)
print('done')
post_crop_im_size = int(config['DATASET']['post_crop_im_size'])
dataset_means = json.loads(config['DATASET']['dataset_means'])
dataset_stds = json.loads(config['DATASET']['dataset_stds'])
image_loader = ImageLoader(im_size, post_crop_im_size, dataset_means,
dataset_stds)
filter = setup_dataset_filter(config)
processed_dataset, num_classes = filter.process_dataset(
config['DATASET']['raw_dataset_path'],
config['DATASET']['dataset_name'])
dataloaders = dataloaders_setup(config, processed_dataset, image_loader)
model_store = LocalModelStore(config['MODELLING']['architecture'],
config['GENERAL']['root_dir'],
config['GENERAL']['experiment_name'])
start_epoch = int(config['MODELLING']['start_epoch'])
end_epoch = int(config['MODELLING']['end_epoch'])
lfw_tester = get_lfw_test(config, image_loader)
trainer = get_trainer(config, num_classes, start_epoch, lfw_tester)
# trainer.train_model(start_epoch, end_epoch, dataloaders)
# print(lfw_tester.test_performance(trainer.model))
reps_behaviour_extractor = setup_pairs_reps_behaviour(config, image_loader)
def run_experiment(config_path):
# Get the configuration file
print(datetime.datetime.now())
start = time.perf_counter()
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
print(f'Running experiment {config_path}')
config.read(config_path)
print("Running experiment " + config['GENERAL']['experiment_name'])
if mlflow.get_experiment_by_name(config['GENERAL']['experiment_name']) is None:
mlflow.create_experiment(config['GENERAL']['experiment_name'], artifact_location=os.path.join(const.MLFLOW_ARTIFACT_STORE, config['GENERAL']['experiment_name']))
mlflow.set_experiment(config['GENERAL']['experiment_name'])
with mlflow.start_run():
print(mlflow.get_artifact_uri())
mlflow.log_artifact(config_path)
# Create image loader (by the configuration)
im_size = json.loads(config['DATASET']['image_size'])
mlflow.log_param('image_size', im_size)
if 'post_crop_im_size' in config['DATASET']:
post_crop_im_size = int(config['DATASET']['post_crop_im_size'])
elif 'net_input_size' in config['DATASET']:
post_crop_im_size = int(config['DATASET']['net_input_size'])
mlflow.log_param('post_crop_im_size', post_crop_im_size)
dataset_means = json.loads(config['DATASET']['dataset_means'])
mlflow.log_param('dataset_means', dataset_means)
dataset_stds = json.loads(config['DATASET']['dataset_stds'])
mlflow.log_param('dataset_stds', dataset_stds)
crop_scale = None
if 'crop_scale' in config['DATASET']:
crop_scale = json.loads(config['DATASET']['crop_scale'])
crop_scale = (crop_scale['max'], crop_scale['min'])
image_loader = ImageLoader(get_transforms(config['DATASET']))
# Create the dataset filters by config (if they are needed)
filter = setup_dataset_filter(config)
# Activate the filters
processed_dataset, num_classes = filter.process_dataset(
config['DATASET']['raw_dataset_path'],
config['DATASET']['dataset_name'])
print("training on dataset: ", processed_dataset)
mlflow.log_param('training_dataset', processed_dataset)
# Create dataloader for the training
dataloaders = dataloaders_setup(config, processed_dataset, image_loader)
# Get access to pre trained models
model_store = LocalModelStore(config['MODELLING']['architecture'],
config['GENERAL']['root_dir'],
config['GENERAL']['experiment_name'])
mlflow.log_param('architecture', config['MODELLING']['architecture'])
mlflow.log_param('experiment_name', config['GENERAL']['experiment_name'])
mlflow.log_param('root_dir', config['GENERAL']['root_dir'])
start_epoch = int(config['MODELLING']['start_epoch'])
end_epoch = int(config['MODELLING']['end_epoch'])
mlflow.log_param('start_epoch', start_epoch)
mlflow.log_param('end_epoch', end_epoch)
# creating the lfw tester
lfw_tester = get_lfw_test(config, image_loader)
num_classes = int(config['MODELLING']['num_classes'])
mlflow.log_param('num_classes', num_classes)
# Creating the trainer and loading the pretrained model if specified in the configuration
trainer = get_trainer(config, num_classes, start_epoch, lfw_tester)
if 'FINETUNING' in config:
model = trainer.model
print(config['FINETUNING']['classes_mode'])
int(config['FINETUNING']['num_classes'])
mlflow.log_param('finetuning', True)
if config['FINETUNING']['classes_mode'] == 'append':
model = modelling.finetuning.append_classes(trainer.model, int(config['FINETUNING']['num_classes']))
elif config['FINETUNING']['classes_mode'] == 'replace':
model = modelling.finetuning.replace_classes(trainer.model, int(config['FINETUNING']['num_classes']))
mlflow.log_param('finetuning_classes', int(config['FINETUNING']['num_classes']))
model = modelling.finetuning.freeze_layers(model, int(config['FINETUNING']['freeze_end']))
mlflow.log_param('freeze_depth', int(config['FINETUNING']['freeze_end']))
trainer.model = model
# Will train the model from start_epoch to (end_epoch - 1) with the given dataloaders
trainer.train_model(start_epoch, end_epoch, dataloaders)
flag = False
if flag:
if lfw_tester is not None:
lfw_results = lfw_tester.test_performance(trainer.model)
print(lfw_results)
lfw_path = os.path.join(config['LFW_TEST']['reps_results_path'], config['LFW_TEST']['output_filename'])
os.makedirs(config['LFW_TEST']['reps_results_path'], exist_ok=True)
lfw_results.to_csv(lfw_path)
reps_behaviour_extractor = setup_pairs_reps_behaviour(config, image_loader)
if reps_behaviour_extractor is not None:
output = reps_behaviour_extractor.compare_lists(trainer.model)
if output is not None:
if type(output) == pd.DataFrame:
results_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'],
config['REP_BEHAVIOUR']['output_filename'] + '.csv')
print('Saving results in ', results_path)
output.to_csv(results_path)
mlflow.log_artifact(results_path)
end = time.perf_counter()
print(datetime.datetime.now())
print((end - start) / 3600)
print((time.process_time()) / 3600)
print('done')
def run_experiment(config_path):
# Get the configuration file
print(datetime.datetime.now())
start = time.perf_counter()
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
config.read(config_path)
print("Running experiment " + config['GENERAL']['experiment_name'])
# Create image loader (by the configuration)
results_root = config['REP_BEHAVIOUR']['reps_results_path']
im_size = int(config['DATASET']['image_size'])
post_crop_im_size = int(config['DATASET']['post_crop_im_size'])
dataset_means = json.loads(config['DATASET']['dataset_means'])
dataset_stds = json.loads(config['DATASET']['dataset_stds'])
crop_scale=None
if 'crop_scale' in config['DATASET']:
crop_scale = json.loads(config['DATASET']['crop_scale'])
crop_scale = (crop_scale['max'], crop_scale['min'])
image_loader = ImageLoader(im_size, post_crop_im_size, dataset_means, dataset_stds, crop_scale=crop_scale)
# Get access to pre trained models
model_store = LocalModelStore(config['MODELLING']['architecture'],
config['GENERAL']['root_dir'],
config['GENERAL']['experiment_name'])
path_to_models = config['GENERAL']['weights_dir']
ids_dirs = glob.glob(os.path.join(path_to_models, '1000_ids'))
for ids_dir in ids_dirs:
ids_dirname = os.path.relpath(ids_dir, path_to_models)
num_classes = int(ids_dirname[:-4])
model = ModelInitializer(json.loads(config['MODELLING']['feature_parallelized_architectures'])).get_model(config['MODELLING']['architecture'], False, num_classes)
for experiment_dir in glob.glob(os.path.join(ids_dir, '*')):
experiment_dirname =os.path.relpath(experiment_dir, ids_dir)
num_images = int(experiment_dirname[5 + experiment_dirname.index('_'): -18])
loc = os.path.join(experiment_dir, 'vgg16', 'models', '119.pth')
if not os.path.exists(loc):
loc = os.path.join(experiment_dir, 'vgg16', 'models', '120.pth')
try:
model, _1, _2, _3 = model_store.load_model_and_optimizer_loc(model, model_location=loc)
os.makedirs(config['REP_BEHAVIOUR']['reps_results_path'], exist_ok=True)
config['REP_BEHAVIOUR']['reps_results_path'] = os.path.join(results_root,
str(num_classes) + '_' + str(num_images))
reps_behaviour_extractor = setup_pairs_reps_behaviour(config, image_loader)
if reps_behaviour_extractor != None:
output = reps_behaviour_extractor.compare_lists(model)
print('Saving results in ', config['REP_BEHAVIOUR']['reps_results_path'])
except:
print('Could not run loc')
end = time.perf_counter()
print(datetime.datetime.now())
print((end - start) / 3600)
print((time.process_time()) / 3600)
print('done')