def _process_config(config_json, override_dotmap=None):
"""
Processes config file:
1) Converts it to a DotMap
2) Creates experiments path and required subdirs
3) Set up logging
"""
config = DotMap(config_json)
if override_dotmap is not None:
config.update(override_dotmap)
print("Loaded configuration: ")
pprint(config)
print()
print(" *************************************** ")
print(" Running experiment {}".format(config.exp_name))
print(" *************************************** ")
print()
# NOTE: current setup overwrite every time...
exp_base = config.exp_base
if 'atamkin' in exp_base or 'wumike' in exp_base:
# Excise existing user-specific prefix (like '/mnt/fs5/wumike/).
exp_base = '/'.join(exp_base.split('/')[4:])
if getpass.getuser() == 'atamkin':
prefix = '/mnt/fs2/atamkin/'
elif getpass.getuser() == 'wumike':
prefix = '/mnt/fs5/wumike/'
else:
raise RuntimeError(
f'Prefix not yet implemented for user {getpass.getuser()}.')
exp_base = os.path.join(prefix, exp_base)
# Uncomment me if you wish to not overwrite
# timestamp = strftime('%Y-%m-%d--%H_%M_%S', localtime())
exp_dir = os.path.join(exp_base, "experiments", config.exp_name)
config.exp_dir = exp_dir
# create some important directories to be used for the experiment.
config.checkpoint_dir = os.path.join(exp_dir, "checkpoints/")
config.out_dir = os.path.join(exp_dir, "out/")
config.log_dir = os.path.join(exp_dir, "logs/")
# will not create if already existing
makedirs([config.checkpoint_dir, config.out_dir, config.log_dir])
# save config to experiment dir
config_out = os.path.join(exp_dir, 'config.json')
save_json(config.toDict(), config_out)
# setup logging in the project
setup_logging(config.log_dir)
logging.getLogger().info(
"Configurations and directories successfully set up.")
return config
def _process_config(config_json, override_dotmap=None, exp_name_suffix=None):
"""
Processes config file:
1) Converts it to a DotMap
2) Creates experiments path and required subdirs
3) Set up logging
"""
config = DotMap(config_json)
if override_dotmap is not None:
config.update(override_dotmap)
if exp_name_suffix is not None:
config.exp_name = f'{config.exp_name}_{exp_name_suffix}'
print("Loaded configuration: ")
pprint(config)
print()
print(" *************************************** ")
print(" Running experiment {}".format(config.exp_name))
print(" *************************************** ")
print()
exp_base = config.exp_base or DEFAULT_EXP_BASE
# Uncomment me if you wish to not overwrite
# timestamp = strftime('%Y-%m-%d--%H_%M_%S', localtime())
exp_dir = os.path.join(exp_base, "experiments", config.exp_name)
config.exp_dir = exp_dir
# create some important directories to be used for the experiment.
config.checkpoint_dir = os.path.join(exp_dir, "checkpoints/")
config.out_dir = os.path.join(exp_dir, "out/")
config.log_dir = os.path.join(exp_dir, "logs/")
# will not create if already existing
makedirs([config.checkpoint_dir, config.out_dir, config.log_dir])
# save config to experiment dir
config_out = os.path.join(exp_dir, 'config.json')
save_json(config.toDict(), config_out)
# setup logging in the project
setup_logging(config.log_dir)
logging.getLogger().info(
"Configurations and directories successfully set up.")
return config
def main(base_model_name,
weights_file,
image_source,
predictions_file,
img_format='jpg'):
# load samples
if os.path.isfile(image_source):
image_dir, samples = image_file_to_json(image_source)
else:
image_dir = image_source
samples = image_dir_to_json(image_dir, img_type='jpg')
# build model and load weights
nima = Nima(base_model_name, weights=None)
nima.build()
nima.nima_model.load_weights(weights_file)
# initialize data generator
data_generator = TestDataGenerator(samples,
image_dir,
64,
10,
nima.preprocessing_function(),
img_format=img_format)
# get predictions
predictions = predict(nima.nima_model, data_generator)
# calc mean scores and add to samples
for i, sample in enumerate(samples):
sample['mean_score_prediction'] = calc_mean_score(predictions[i])
print(samples)
new_sample = sorted(samples,
key=lambda x: x["mean_score_prediction"],
reverse=True)
print("new_sample: ", new_sample)
# print(json.dumps(new_sample, indent=2))
with open("img_quality.csv", "w", encoding="utf-8") as f:
csv_writer = csv.writer(f)
csv_writer.writerow(['id', 'mean'])
for d in new_sample:
csv_writer.writerow([d["image_id"], d["mean_score_prediction"]])
if predictions_file is not None:
save_json(new_sample, predictions_file)
def _process_config(config_json, override_dotmap=None):
"""
Processes config file:
1) Converts it to a DotMap
2) Creates experiments path and required subdirs
3) Set up logging
"""
config = DotMap(config_json)
if override_dotmap is not None:
config.update(override_dotmap)
print("Loaded configuration: ")
pprint(config)
print()
print(" *************************************** ")
print(" Running experiment {}".format(config.exp_name))
print(" *************************************** ")
print()
exp_base = config.exp_base
timestamp = strftime('%Y-%m-%d--%H_%M_%S', localtime())
exp_dir = os.path.join(exp_base, "experiments", config.exp_name, timestamp)
# create some important directories to be used for the experiment.
config.summary_dir = os.path.join(exp_dir, "summaries/")
config.checkpoint_dir = os.path.join(exp_dir, "checkpoints/")
config.out_dir = os.path.join(exp_dir, "out/")
config.log_dir = os.path.join(exp_dir, "logs/")
makedirs([
config.summary_dir, config.checkpoint_dir, config.out_dir,
config.log_dir
])
# save config to experiment dir
config_out = os.path.join(exp_dir, 'config.json')
save_json(config.toDict(), config_out)
# setup logging in the project
setup_logging(config.log_dir)
logging.getLogger().info(
"Configurations and directories successfully set up.")
return config
def _process_config(config_json, override_dotmap=None):
"""
Processes config file:
1) Converts it to a DotMap
2) Creates experiments path and required subdirs
3) Set up logging
"""
config = DotMap(config_json)
if override_dotmap is not None:
config.update(override_dotmap)
print("Loaded configuration: ")
pprint(config)
print()
print(" *************************************** ")
print(" Running experiment {}".format(config.exp_name))
print(" *************************************** ")
print()
# if config.pretrained_exp_dir is not None:
# # don't make new dir more continuing training
# exp_dir = config.pretrained_exp_dir
# print("[INFO]: Continuing from previously finished training at %s." % exp_dir)
# else:
exp_base = config.exp_base
if config.debug:
exp_dir = os.path.join(exp_base, "experiments", config.exp_name,
'debug')
else:
if config.pretrained_exp_dir is not None and isinstance(
config.pretrained_exp_dir, str):
# don't make new dir more continuing training
exp_dir = config.pretrained_exp_dir
print('[INFO]: Backup previously trained model and config json')
os.system("cp %s/config.json %s/prev_config.json" %
(exp_dir, exp_dir))
os.system(
"cp %s/checkpoints/checkpoint.pth.tar %s/checkpoints/prev_checkpoint.pth.tar"
% (exp_dir, exp_dir))
os.system(
"cp %s/checkpoints/model_best.pth.tar %s/checkpoints/prev_model_best.pth.tar"
% (exp_dir, exp_dir))
elif config.continue_exp_dir is not None and isinstance(
config.continue_exp_dir, str):
exp_dir = config.continue_exp_dir
print('[INFO]: Backup previously trained model and config json')
os.system("cp %s/config.json %s/prev_config.json" %
(exp_dir, exp_dir))
os.system(
"cp %s/checkpoints/checkpoint.pth.tar %s/checkpoints/prev_checkpoint.pth.tar"
% (exp_dir, exp_dir))
os.system(
"cp %s/checkpoints/model_best.pth.tar %s/checkpoints/prev_model_best.pth.tar"
% (exp_dir, exp_dir))
else:
if config.exp_id is None:
config.exp_id = strftime('%Y-%m-%d--%H_%M_%S', localtime())
exp_dir = os.path.join(exp_base, "experiments", config.exp_name,
config.exp_id)
# create some important directories to be used for the experiment.
config.summary_dir = os.path.join(exp_dir, "summaries/")
config.checkpoint_dir = os.path.join(exp_dir, "checkpoints/")
config.out_dir = os.path.join(exp_dir, "out/")
config.log_dir = os.path.join(exp_dir, "logs/")
makedirs([
config.summary_dir, config.checkpoint_dir, config.out_dir,
config.log_dir
])
# save config to experiment dir
config_out = os.path.join(exp_dir, 'config.json')
save_json(config.toDict(), config_out)
# setup logging in the project
setup_logging(config.log_dir)
logging.getLogger().info("Experiment directory is located at %s" % exp_dir)
logging.getLogger().info(
"Configurations and directories successfully set up.")
return config
def main(target_file, source_file_mean):
df = get_dataframe(source_file_mean)
samples = parse_raw_data(df)
save_json(samples, target_file)
print('Done! Saved JSON at {}'.format(target_file))
def main(target_file, source_file_mean):
df = get_dataframe(source_file_mean)
samples = parse_raw_data(df)
save_json(samples, target_file)
print('Done! Saved JSON at {}'.format(target_file))