def run_experiment(config_path):
# Get the configuration file
print(datetime.datetime.now())
start = time.perf_counter()
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
config.read(config_path)
print("Running experiment " + config['GENERAL']['experiment_name'])
num_iterations = int(config['GENERAL']['num_experiments'])
num_ids = json.loads(config['DATASET']['num_ids'])
num_pics = json.loads(config['DATASET']['num_pics'])
overall_index = 0
all_lfw_results = None
all_output = None
experiment_name = config['GENERAL']['experiment_name']
for id in num_ids:
for pic in num_pics:
for i in range(num_iterations):
print("________ ITER " + str(i) + " ________")
print("________ NUM IDS " + str(id) + " ________")
print("________ NUM PICS " + str(pic) + " ________")
config['GENERAL']['experiment_name'] = experiment_name + '_' + str(id) + '_' + str(pic) + '_' + str(i)
# Create image loader (by the configuration)
im_size = int(config['DATASET']['image_size'])
post_crop_im_size = int(config['DATASET']['post_crop_im_size'])
dataset_means = json.loads(config['DATASET']['dataset_means'])
dataset_stds = json.loads(config['DATASET']['dataset_stds'])
image_loader = RandomExperimentImageLoader(im_size, post_crop_im_size, dataset_means, dataset_stds)
print("training on dataset: ", config['DATASET']['raw_dataset_path'])
# Create dataloader for the training
dataloaders = dataloaders_setup(config, config['DATASET']['raw_dataset_path'], image_loader, id, pic)
# Get access to pre trained models
model_store = LocalModelStore(config['MODELLING']['architecture'],
config['GENERAL']['root_dir'],
config['GENERAL']['experiment_name'])
start_epoch = int(config['MODELLING']['start_epoch'])
end_epoch = int(config['MODELLING']['end_epoch'])
# creating the lfw tester
lfw_tester = get_lfw_test(config, image_loader)
# Creating the trainer and loading the pretrained model if specified in the configuration
trainer = get_trainer(config, id, start_epoch, lfw_tester)
# Will train the model from start_epoch to (end_epoch - 1) with the given dataloaders
trainer.train_model(start_epoch, end_epoch, dataloaders)
lfw_results = lfw_tester.test_performance(trainer.model)
lfw_results['index'] = overall_index
lfw_results['num_ids'] = id
lfw_results['num_pics'] = pic
if all_lfw_results is None:
all_lfw_results = lfw_results
else:
all_lfw_results = all_lfw_results.append(lfw_results)
print(lfw_results)
reps_behaviour_extractor = setup_pairs_reps_behaviour(config, image_loader)
if reps_behaviour_extractor != None:
output = reps_behaviour_extractor.compare_lists(trainer.model)
output['index'] = overall_index
output['num_ids'] = id
output['num_pics'] = pic
# saving results for the meanwhile
results_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'],
config['REP_BEHAVIOUR']['output_filename'] + '.csv')
lfw_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'], 'logs.csv')
os.makedirs(config['REP_BEHAVIOUR']['reps_results_path'], exist_ok=True)
output.to_csv(results_path)
lfw_results.to_csv(lfw_path)
if all_output is None:
all_output = output
else:
all_output = all_output.append(output)
overall_index += 1
print(all_output)
print(all_lfw_results)
config['GENERAL']['experiment_name'] = experiment_name
# lfw_path = os.path.join(root_dir, experiment_name + '_all', rep_dir, lfw_filename)
# results_path = os.path.join(root_dir, experiment_name + '_all', rep_dir, rep_filename)
results_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'],
config['REP_BEHAVIOUR']['output_filename'] + '.csv')
lfw_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'], config['LFW_TEST']['lfw_results_file'] + '.csv')
print('Saving results in ', results_path)
os.makedirs(config['REP_BEHAVIOUR']['reps_results_path'], exist_ok=True)
if all_output is not None:
all_output.to_csv(results_path)
all_lfw_results.to_csv(lfw_path)
end = time.perf_counter()
print(datetime.datetime.now())
print((end - start) / 3600)
print((time.process_time()) / 3600)
print('done')
def run_experiment(config_path):
# Get the configuration file
print(datetime.datetime.now())
start = time.perf_counter()
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
print(f'Running experiment {config_path}')
config.read(config_path)
print("Running experiment " + config['GENERAL']['experiment_name'])
if mlflow.get_experiment_by_name(config['GENERAL']['experiment_name']) is None:
mlflow.create_experiment(config['GENERAL']['experiment_name'], artifact_location=os.path.join(const.MLFLOW_ARTIFACT_STORE, config['GENERAL']['experiment_name']))
mlflow.set_experiment(config['GENERAL']['experiment_name'])
with mlflow.start_run():
print(mlflow.get_artifact_uri())
mlflow.log_artifact(config_path)
# Create image loader (by the configuration)
im_size = json.loads(config['DATASET']['image_size'])
mlflow.log_param('image_size', im_size)
if 'post_crop_im_size' in config['DATASET']:
post_crop_im_size = int(config['DATASET']['post_crop_im_size'])
elif 'net_input_size' in config['DATASET']:
post_crop_im_size = int(config['DATASET']['net_input_size'])
mlflow.log_param('post_crop_im_size', post_crop_im_size)
dataset_means = json.loads(config['DATASET']['dataset_means'])
mlflow.log_param('dataset_means', dataset_means)
dataset_stds = json.loads(config['DATASET']['dataset_stds'])
mlflow.log_param('dataset_stds', dataset_stds)
crop_scale = None
if 'crop_scale' in config['DATASET']:
crop_scale = json.loads(config['DATASET']['crop_scale'])
crop_scale = (crop_scale['max'], crop_scale['min'])
image_loader = ImageLoader(get_transforms(config['DATASET']))
# Create the dataset filters by config (if they are needed)
filter = setup_dataset_filter(config)
# Activate the filters
processed_dataset, num_classes = filter.process_dataset(
config['DATASET']['raw_dataset_path'],
config['DATASET']['dataset_name'])
print("training on dataset: ", processed_dataset)
mlflow.log_param('training_dataset', processed_dataset)
# Create dataloader for the training
dataloaders = dataloaders_setup(config, processed_dataset, image_loader)
# Get access to pre trained models
model_store = LocalModelStore(config['MODELLING']['architecture'],
config['GENERAL']['root_dir'],
config['GENERAL']['experiment_name'])
mlflow.log_param('architecture', config['MODELLING']['architecture'])
mlflow.log_param('experiment_name', config['GENERAL']['experiment_name'])
mlflow.log_param('root_dir', config['GENERAL']['root_dir'])
start_epoch = int(config['MODELLING']['start_epoch'])
end_epoch = int(config['MODELLING']['end_epoch'])
mlflow.log_param('start_epoch', start_epoch)
mlflow.log_param('end_epoch', end_epoch)
# creating the lfw tester
lfw_tester = get_lfw_test(config, image_loader)
num_classes = int(config['MODELLING']['num_classes'])
mlflow.log_param('num_classes', num_classes)
# Creating the trainer and loading the pretrained model if specified in the configuration
trainer = get_trainer(config, num_classes, start_epoch, lfw_tester)
if 'FINETUNING' in config:
model = trainer.model
print(config['FINETUNING']['classes_mode'])
int(config['FINETUNING']['num_classes'])
mlflow.log_param('finetuning', True)
if config['FINETUNING']['classes_mode'] == 'append':
model = modelling.finetuning.append_classes(trainer.model, int(config['FINETUNING']['num_classes']))
elif config['FINETUNING']['classes_mode'] == 'replace':
model = modelling.finetuning.replace_classes(trainer.model, int(config['FINETUNING']['num_classes']))
mlflow.log_param('finetuning_classes', int(config['FINETUNING']['num_classes']))
model = modelling.finetuning.freeze_layers(model, int(config['FINETUNING']['freeze_end']))
mlflow.log_param('freeze_depth', int(config['FINETUNING']['freeze_end']))
trainer.model = model
# Will train the model from start_epoch to (end_epoch - 1) with the given dataloaders
trainer.train_model(start_epoch, end_epoch, dataloaders)
flag = False
if flag:
if lfw_tester is not None:
lfw_results = lfw_tester.test_performance(trainer.model)
print(lfw_results)
lfw_path = os.path.join(config['LFW_TEST']['reps_results_path'], config['LFW_TEST']['output_filename'])
os.makedirs(config['LFW_TEST']['reps_results_path'], exist_ok=True)
lfw_results.to_csv(lfw_path)
reps_behaviour_extractor = setup_pairs_reps_behaviour(config, image_loader)
if reps_behaviour_extractor is not None:
output = reps_behaviour_extractor.compare_lists(trainer.model)
if output is not None:
if type(output) == pd.DataFrame:
results_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'],
config['REP_BEHAVIOUR']['output_filename'] + '.csv')
print('Saving results in ', results_path)
output.to_csv(results_path)
mlflow.log_artifact(results_path)
end = time.perf_counter()
print(datetime.datetime.now())
print((end - start) / 3600)
print((time.process_time()) / 3600)
print('done')
model_store = LocalModelStore(config['MODELLING']['architecture'],
config['GENERAL']['root_dir'],
config['GENERAL']['experiment_name'])
start_epoch = int(config['MODELLING']['start_epoch'])
end_epoch = int(config['MODELLING']['end_epoch'])
lfw_tester = get_lfw_test(config, image_loader)
trainer = get_trainer(config, num_classes, start_epoch, lfw_tester)
# trainer.train_model(start_epoch, end_epoch, dataloaders)
# print(lfw_tester.test_performance(trainer.model))
reps_behaviour_extractor = setup_pairs_reps_behaviour(config, image_loader)
output = reps_behaviour_extractor.test_behaviour(trainer.model)
results_path = os.path.join(config['REP_BEHAVIOUR']['reps_results_path'],
'comparisons_face_angles.pkl')
print('Saving results in ', results_path)
os.makedirs(config['REP_BEHAVIOUR']['reps_results_path'], exist_ok=True)
with open(results_path, 'wb') as f:
pickle.dump(output, f)
print('done')
# TODO: Add option to start from existing models.
# TODO: Divide the analysis from training and from the data prep - use dir tree as indicator to the model training
def run_experiment(config_path):
# Get the configuration file
print(datetime.datetime.now())
start = time.perf_counter()
config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
config.read(config_path)
print("Running experiment " + config['GENERAL']['experiment_name'])
# Create image loader (by the configuration)
results_root = config['REP_BEHAVIOUR']['reps_results_path']
im_size = int(config['DATASET']['image_size'])
post_crop_im_size = int(config['DATASET']['post_crop_im_size'])
dataset_means = json.loads(config['DATASET']['dataset_means'])
dataset_stds = json.loads(config['DATASET']['dataset_stds'])
crop_scale=None
if 'crop_scale' in config['DATASET']:
crop_scale = json.loads(config['DATASET']['crop_scale'])
crop_scale = (crop_scale['max'], crop_scale['min'])
image_loader = ImageLoader(im_size, post_crop_im_size, dataset_means, dataset_stds, crop_scale=crop_scale)
# Get access to pre trained models
model_store = LocalModelStore(config['MODELLING']['architecture'],
config['GENERAL']['root_dir'],
config['GENERAL']['experiment_name'])
path_to_models = config['GENERAL']['weights_dir']
ids_dirs = glob.glob(os.path.join(path_to_models, '1000_ids'))
for ids_dir in ids_dirs:
ids_dirname = os.path.relpath(ids_dir, path_to_models)
num_classes = int(ids_dirname[:-4])
model = ModelInitializer(json.loads(config['MODELLING']['feature_parallelized_architectures'])).get_model(config['MODELLING']['architecture'], False, num_classes)
for experiment_dir in glob.glob(os.path.join(ids_dir, '*')):
experiment_dirname =os.path.relpath(experiment_dir, ids_dir)
num_images = int(experiment_dirname[5 + experiment_dirname.index('_'): -18])
loc = os.path.join(experiment_dir, 'vgg16', 'models', '119.pth')
if not os.path.exists(loc):
loc = os.path.join(experiment_dir, 'vgg16', 'models', '120.pth')
try:
model, _1, _2, _3 = model_store.load_model_and_optimizer_loc(model, model_location=loc)
os.makedirs(config['REP_BEHAVIOUR']['reps_results_path'], exist_ok=True)
config['REP_BEHAVIOUR']['reps_results_path'] = os.path.join(results_root,
str(num_classes) + '_' + str(num_images))
reps_behaviour_extractor = setup_pairs_reps_behaviour(config, image_loader)
if reps_behaviour_extractor != None:
output = reps_behaviour_extractor.compare_lists(model)
print('Saving results in ', config['REP_BEHAVIOUR']['reps_results_path'])
except:
print('Could not run loc')
end = time.perf_counter()
print(datetime.datetime.now())
print((end - start) / 3600)
print((time.process_time()) / 3600)
print('done')