def delete_experiments(logdir, pattern):
"""delete experiments that match some pattern"""
with FileLockedTinyDB(logdir) as db:
query = db.query()
results = db.search(query.arg.test(lambda x: pattern in x))
print(logging_sep("-"))
print(f"Deleting {len(results)} / {len(db)} records")
db.remove(query.arg.test(lambda x: pattern in x))
def evaluate_minibatch_and_log(estimator, model, x, base_logger, desc, **parameters):
"""Evaluate the model using the estimator on a mini-batch of data and log to the Logger"""
print(logging_sep())
loss, diagnostics, output = estimator(model, x, backward=False, **parameters)
base_logger.info(
f"{desc} | L_{estimator.config['iw']} = {diagnostics['loss']['elbo'].mean().item():.6f}, "
f"KL = {diagnostics['loss']['kl'].mean().item():.6f}, "
f"NLL = {diagnostics['loss']['nll'].mean().item():.6f}, "
f"ESS = {diagnostics['loss']['ess'].mean().item():.6f}")
return diagnostics
def update_experiments(logdir, rules):
"""update the records in the database based on rules where rules follow the format `key:old_value,key:new_value`"""
with FileLockedTinyDB(logdir) as db:
query = db.query()
for rule in rules.split(','):
key, arg, new_arg = rule.split(':')
pattern = f"--{key} {arg}"
new_pattern = f"--{key} {new_arg}"
matching_exps = db.search(query.arg.test(lambda x: pattern in x))
print(logging_sep())
print(
f"Updating {len(matching_exps)} records with key = {key} : {arg} -> {new_arg}"
)
for exp in matching_exps:
exp['arg'] = exp['arg'].replace(pattern, new_pattern)
db.write_back(matching_exps)
print(
f"Classifier.forward: x.device = {x.device}, x.shape = {x.shape}")
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
x = x.view(-1, 320)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return x
# check GPUs
n_gpus = torch.cuda.device_count()
device_ids = list(range(n_gpus)) if n_gpus else None
print(logging_sep("="))
print(f"N gpus = {n_gpus}, Devices = {device_ids}")
if n_gpus == 1:
print("Use more than one GPU to test multi-GPUs capabilities.")
# init model and evaluator
model = Classifier()
evaluator = Classification(10)
model.to(available_device())
# fuse model + evaluator
pipeline = Pipeline(model, evaluator)
# wrap as DataParallel
parallel_pipeline = DataParallelPipeline(pipeline, device_ids=device_ids)
"After perturbation")
# define the gradients analysis arguments and store the meta-data
meta = {
'seed': opt['seed'],
'noise': epsilon,
'alpha': opt['alpha'],
'mc_samples': int(opt['mc_samples']),
**{k: v.mean().item()
for k, v in diagnostics['loss'].items()}
}
grad_args = {'mc_samples': int(opt['mc_samples']), **global_grad_args}
idx = None
for estimator_id in estimators:
print(logging_sep())
for iw in iws:
base_logger.info(f"{estimator_id} [K = {iw}]")
# create estimator
estimator = init_estimator(estimator_id,
iw,
alpha=opt['alpha'])
estimator.to(device)
parameters = {}
# evaluate the SNR and the Variance of the gradients
with ManualSeed(seed=opt['seed']):
analysis_data, grads_meta = get_gradients_statistics(
estimator,
model,
def read_experiments(opt, metrics, logger):
# get path to the experiment directory
path = os.path.join(opt.root, opt.exp)
experiments = [e for e in os.listdir(path) if '.' != e[0]]
logger.info(
f"{logging_sep('=')}\n# reading experiment at {path}\n{logging_sep('=')}"
)
# parse filters from opt
filters_inc, filters_exc = map(parse_comma_separated_list,
[opt.include, opt.exclude])
data = [] # store hyperparameters and configs
logs = [] # store tensorboard logs
pbar = tqdm(experiments)
for e in pbar:
pbar.set_description(f"{e}")
exp_path = os.path.join(path, e)
files = os.listdir(exp_path)
try:
if (not opt.non_completed) and (not Success.file in files):
logger.info(f" [Not completed] {e} ")
elif is_filtered(e, filters_inc, filters_exc):
logger.info(f" [Filtered] {e}")
else:
if not opt.non_completed and not is_successful(exp_path):
logger.info(f" [Failed] {e}")
else:
# reading configuration files with run parameter
with open(os.path.join(exp_path, 'config.json'),
'r') as fp:
args = json.load(fp)
# remove exceptions and parse `estimator` ids
args = drop_exceptions_from_args(args)
args.pop('hash', None)
if opt.parse_estimator_args and "-" in args['estimator']:
args = parse_estimator_args(args)
# read tensorboard logs
logs += [
r for r in read_tf_record(
exp_path, e, metrics['all_metrics_by_header'])
]
# gather config/hyperparameters
d = dict(args)
d['id'] = e
# append data and logs
data += [d]
# stop reading if > `max_records`
if opt.max_records > 0 and len(data) > opt.max_records:
break
except Exception as ex:
logger.info(
f" [Parsing Failed with Exception\n{logging_sep('=')}")
logger.info(logging_sep("="))
traceback.print_exception(type(ex), ex, ex.__traceback__)
logger.info(logging_sep("="))
# exit if not data
if len(data) == 0:
logger.info(
f"{logging_sep('=')}\nCouldn't read any record. Either they are errors or the experiments are not yet completed.\n{logging_sep('=')}"
)
exit()
# compile into a DataFrame
data = pd.DataFrame(data)
# replace NaN with False (important since nans are dropped afterwards)
# NaNs arise when using `parse_estimator_args`
data = data.fillna(False)
# compile logs into a DataFrame
logs = pd.DataFrame(logs)
# check data
found_keys = logs['_key'].unique()
for metric in metrics['all_metrics']:
if metric not in found_keys:
raise ValueError(
f"`{metric}` was not found in Tensorboard records. Found keys = `{found_keys}`"
)
return path, data, logs
def requeue_experiments(logdir, level=1, display_mode='status'):
"""
check queued==False records, find there corresponding experiment folder and
requeue (i.e. set record.queue==True) based on the `level`value. Levels:
* 0: do not requeue
* 1: requeue `aborted_by_user` (keyboard_interrupt or sigterm) and `not_found` (no match in experiments dir)
* 2: requeue `failed``
* 100: requeue `running` (without `success` file)
* 10000: all including `success`
"""
def requeue(queue, record, exp):
"""set record to `queued` flag to True, append record to the ´to_be_requeued´ queue and delete the ´success´ file"""
record['queued'] = True
queue += [record]
if exp is not None:
success_file = os.path.join(logdir, exp, Success.file)
if os.path.exists(success_file):
os.remove(success_file)
def remove_exp_from_queue(db_hashes_queue, exp_hash):
db_hashes_queue.pop(db_hashes_queue.index(exp_hash))
with FileLockedTinyDB(logdir) as db:
query = db.query()
parser = argparse.ArgumentParser()
add_base_args(parser, exp='sandbox')
add_run_args(parser)
add_model_architecture_args(parser)
add_active_units_args(parser)
add_gradient_analysis_args(parser)
get_hash = partial(get_hash_from_experiments, parser)
status = defaultdict(lambda: 0)
requeued_status = defaultdict(lambda: 0)
# count queued records
status['queued'] = db.count(query.queued == True)
# retrieve all non-queued records from the db and get the run_id hash
db_hashes = {
get_hash(record): record
for record in db.search(query.queued == False)
}
db_hashes_set = set(db_hashes.keys())
db_hashes_queue = list(db_hashes_set)
messages = []
# iterate through experiments files and store the experiments that have been interrupted
to_be_requeued = []
for exp in [e for e in os.listdir(logdir) if e[0] != '.']:
with open(os.path.join(logdir, exp, 'config.json'), 'r') as f:
exp_config = json.load(f)
exp_hash = exp_config['hash']
if exp_hash in db_hashes_set:
remove_exp_from_queue(db_hashes_queue, exp_hash)
if Success.file in os.listdir(os.path.join(logdir, exp)):
message = open(os.path.join(logdir, exp, Success.file),
'r').read()
if Success.aborted_by_user == message:
status_id = 'aborted_by_user'
status[status_id] += 1
if level >= 1:
requeued_status[status_id] += 1
requeue(to_be_requeued, db_hashes[exp_hash],
exp)
elif Success.success == message:
status_id = 'success'
if level >= 10000:
requeued_status[status_id] += 1
requeue(to_be_requeued, db_hashes[exp_hash],
exp)
status[status_id] += 1
elif Success.failure_base in message:
status_id = 'failed'
status[status_id] += 1
if level >= 2:
requeued_status[status_id] += 1
requeue(to_be_requeued, db_hashes[exp_hash],
exp)
else:
status_id = 'unknown'
status[status_id] += 1
if level >= 100:
requeued_status[status_id] += 1
requeue(to_be_requeued, db_hashes[exp_hash],
exp)
# store messages
messages += [{
'status': status_id,
'exp': exp,
'message': message
}]
else:
status['running'] += 1
if level >= 100:
requeued_status['running'] += 1
requeue(to_be_requeued, db_hashes[exp_hash], exp)
# process remaining exps in the queue
for exp_hash in db_hashes_queue:
status['not_found'] += 1
if level >= 1:
requeued_status['not_found'] += 1
requeue(to_be_requeued, db_hashes[exp_hash], None)
# requeue the stored experiments
db.write_back(to_be_requeued)
if display_mode == 'status': # print status
def requeue_message(n):
return f" --> requeued: {n}" if n > 0 else ""
status['queued'] += sum(
[v for k, v in requeued_status.items() if k != 'queued'])
assert status['queued'] == db.count(
query.queued == True) # safety check
for k, v in status.items():
print(
f" [{k}] {v - requeued_status[k]} experiments {requeue_message(requeued_status[k])}"
)
elif display_mode == 'messages': # print error messages
for data in messages:
if data['status'] in ['failed', 'unknown']:
print(
f"[{data['status']}] id = {data['exp']}\n{data['message']}"
)
print(logging_sep("-"))
else:
raise ValueError(f"Unknown display mode = `{display_mode}`.")
def run_manager():
"""
Run a set of experiments defined as a json file in `experiments/` using mutliprocessing.
This script is a quick & dirty implementation of a queue system using `filelock` and `tinydb`.
The manager creates a snapshot of the library to ensure consistency between runs. You may update the snapshot
manually using `--update_lib` or update the experiment file using `--update_exp`.
In that case starting a new manager using `--resume` will append potential new experiments to the database.
Use `--rf` to delete an existing experiment. Example:
```bash
python manager.py --exp gaussian-mixture-model --max_gpus 4 --processes 2
```
Check the experiment status using `dbutils.py` (see `python dbutils.py --help` for details about requeuing exps):
```bash
python dbutils.py --exp gaussian-mixture-model --check
```
Troubleshooting: If the `FileLockedTinyDB` exits without properly removing the `.lock` file, you may need to delete
it manually using `--purge_lock`. This must be used carefully as it may alter ongoing database transactions.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--script', default='run.py', help='script name')
parser.add_argument('--root', default='runs/', help='experiment directory')
parser.add_argument('--data_root', default='data/', help='data directory')
parser.add_argument(
'--exp',
default='gaussian-mixture-model',
type=str,
help='experiment id pointing to a .json file in experiments/')
parser.add_argument('--max_gpus',
default=8,
type=int,
help='maximum number of GPUs')
parser.add_argument('--max_load',
default=0.5,
type=float,
help='only use GPUs with load < `max_load`')
parser.add_argument('--max_memory',
default=0.01,
type=float,
help='only use GPUs with memory < `max_memory`')
parser.add_argument('--processes',
default=1,
type=int,
help='number of processes per GPU')
parser.add_argument('--rf',
action='store_true',
help='delete the previous experiment')
parser.add_argument('--resume',
action='store_true',
help='resume an experiment')
parser.add_argument('--update_exp',
action='store_true',
help='update snapshot experiment file')
parser.add_argument('--update_lib',
action='store_true',
help='update the entire snapshot')
parser.add_argument(
'--max_jobs',
default=-1,
type=int,
help='maximum jobs per thread (stop after `max_jobs` jobs)')
parser.add_argument(
'--requeue_level',
default=1,
type=int,
help=
'[db] Requeue level {0: nothing, 1: aborted_by_user/not_found, 2: failed, 100: not completed}'
)
parser.add_argument(
'--purge_lock',
action='store_true',
help=
'purge the `.lock` file [use only if the `.lock` file has not been properly removed].'
)
opt = parser.parse_args()
# get the list of devices
device_ids = GPUtil.getAvailable(order='memory',
limit=opt.max_gpus,
maxLoad=opt.max_load,
maxMemory=opt.max_memory,
includeNan=False,
excludeID=[],
excludeUUID=[])
if len(device_ids):
device_ids = [f"cuda:{d}" for d in device_ids]
else:
device_ids = ['cpu']
# total number of processes
processes = opt.processes * len(device_ids)
# get absolute path to logging directories
exps_root, exp_root, exp_data_root = get_abs_paths(opt.root, opt.exp,
opt.data_root)
if os.path.exists(exp_root):
warnings.warn(f"logging directory `{exp_root}` already exists.")
if not opt.resume:
if opt.rf:
warnings.warn(
f"Deleting existing logging directory `{exp_root}`.")
shutil.rmtree(exp_root)
else:
sys.exit()
# copy library to the `snapshot` directory
if not opt.resume:
shutil.copytree('./',
snapshot_dir(exp_root),
ignore=shutil.ignore_patterns('.*', '*.git', 'runs',
'reports', 'data',
'__pycache__'))
if opt.update_lib:
# move original lib
shutil.move(snapshot_dir(exp_root), f"{snapshot_dir(exp_root)}-saved")
# copy lib
shutil.copytree('./',
snapshot_dir(exp_root),
ignore=shutil.ignore_patterns('.*', '*.git', 'runs',
'reports', 'data',
'__pycache__', '*.psd'))
# udpate experiment file
if opt.update_exp:
_exp_file = f'experiments/{opt.exp}.json'
shutil.copyfile(_exp_file,
os.path.join(snapshot_dir(exp_root), _exp_file))
# move path to the snapshot directory to ensure consistency between runs (lib will be loaded from `./lib_snapshot/`)
os.chdir(snapshot_dir(exp_root))
# logging
logging.basicConfig(level=logging.INFO, handlers=[logging.StreamHandler()])
logger = logging.getLogger('exp-manager')
# log config
print(logging_sep("="))
logger.info(f"Available devices: {device_ids}")
logger.info(
f"Experiment id = {opt.exp}, running {opt.processes} processes/device, logdir = {exp_root}"
)
print(logging_sep())
# read the experiment file
experiment_args = read_experiment_json_file(opt.exp)
# replace the default `script` argument if specified
if "script" in experiment_args.keys():
opt.script = experiment_args.pop("script")
# define the arguments for each run
args = experiment_args['args'] # retrieve the list of arguments
args = [experiment_args['global'] + " " + a
for a in args] # append the `global` arguments
args = [
f"--exp {opt.exp} --root {exps_root} --data_root {exp_data_root} {a}"
for a in args
] # append specific parameters
if "parameters" in experiment_args.keys():
for _arg, values in experiment_args["parameters"].items():
_args = []
for v in values:
if isinstance(v, bool):
if v:
_args += [f"--{_arg} " + a for a in args]
else:
_args += args
else:
_args += [f"--{_arg} {v} " + a for a in args]
args = _args
# remove the lock file manually
if opt.purge_lock:
FileLockedTinyDB(exp_root).purge()
# write all experiments to `tinydb` database guarded by a `filelock`
with FileLockedTinyDB(exp_root) as db:
query = db.query()
# add missing exps (when using `--resume` + `--update_exp`)
n_added = 0
for i, a in enumerate(args):
if not db.contains(query.arg == a):
db.insert({'arg': a, 'queued': True, "job_id": "none"})
n_added += 1
# potentially check the database status requeue fail experiment
if opt.script == 'run.py': # only handled for `run.py` because matching exps relies on `get_run_parser`
with Header(f"Status"):
requeue_experiments(exp_root, opt.requeue_level)
# count queued exps and total exps
with FileLockedTinyDB(exp_root) as db:
n_queued_exps = db.count(query.queued == True)
n_exps = len(db)
# remaining queued experiments
logger.info(
f"Queued experiments : {n_queued_exps} / {n_exps}. Added exps. {n_added}"
)
# run processes in parallel (spawning `processes` processes)
pool = Pool(processes=processes)
job_args = [{
"opt": opt,
"exp_root": exp_root,
"devices": device_ids
} for _ in range(n_queued_exps)]
if opt.max_jobs > 0:
job_args = job_args[:opt.max_jobs * processes]
logger.info(
f"Max. number of jobs = {len(job_args)}, processes = {processes} [{len(device_ids)} devices]"
)
for _ in tqdm(pool.imap_unordered(retrieve_exp_and_run,
job_args,
chunksize=1),
total=n_queued_exps,
desc="Job Manager"):
pass
def __enter__(self):
if self.message is not None:
print(f"{logging_sep('=')}\n{self.message}\n{logging_sep('-')}")
else:
print(logging_sep('='))
model.to(device)
# define the evaluator
evaluator = VariationalInference(likelihood, iw_samples=1)
# define evaluation model with Exponential Moving Average
ema = EMA(model, opt.ema)
# data dependent init for weight normalization (automatically done during the first forward pass)
with torch.no_grad():
model.train()
x = next(iter(train_loader)).to(device)
model(x)
# print stages
print(logging_sep("=") + "\nGenerative model:\n" + logging_sep("-"))
for i, (convs, z) in reversed(list(enumerate(zip(stages, latents)))):
print("Stage #{0}".format(i + 1))
print("Stochastic layer:", z)
print("Deterministic block:", convs)
print(logging_sep("="))
# define freebits
n_latents = len(latents)
if opt.model_type == 'biva':
n_latents = 2 * n_latents - 1
freebits = [opt.freebits] * n_latents
# optimizer
optimizer = torch.optim.Adamax(model.parameters(),
lr=opt.lr,
model.to(device)
# define the evaluator
evaluator = VariationalInference(likelihood, iw_samples=1)
# define evaluation model with Exponential Moving Average
ema = EMA(model, opt.ema)
# data dependent init for weight normalization (automatically done during the first forward pass)
with torch.no_grad():
model.train()
x = next(iter(train_loader)).to(device)
model(x)
# print stages
print(logging_sep("=") + "\nGenerative model:\n" + logging_sep("-"))
for i, (convs, z) in reversed(list(enumerate(zip(stages, latents)))):
print(f"Stage #{i + 1}")
print("Stochastic layer:", z)
print("Deterministic block:", convs)
print(logging_sep("="))
# define freebits
n_latents = len(latents)
if opt.model_type == 'biva':
n_latents = 2 * n_latents - 1
freebits = [opt.freebits] * n_latents
# optimizer
optimizer = torch.optim.Adamax(model.parameters(),
lr=opt.lr,
def report():
"""
Read a entire `experiment` folder (i.e. containing multiple runs),
aggregate the data, dump .csv files and generate the plots.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--root', default='runs/', help='experiment directory')
parser.add_argument('--output',
default='reports/',
help='output directory')
parser.add_argument('--exp',
default='mini-vae',
type=str,
help='experiment id')
parser.add_argument('--include',
default='',
type=str,
help='filter run by id')
parser.add_argument('--exclude',
default='',
type=str,
help='filter run by id')
parser.add_argument(
'--pivot_metrics',
default='max:train:loss/L_k,min:train:loss/kl_q_p,mean:train:grads/snr',
type=str,
help='comma separated list of metrics to report in the table, '
'the prefix defines the aggregation function (min, mean, max)')
parser.add_argument(
'--metrics',
default=
'train:loss/L_k,train:loss/kl_q_p,train:loss/ess,train:grads/snr',
type=str,
help='comma separated list of keys to read from the tensorboard logs')
parser.add_argument(
'--ylims',
default='',
type=str,
help=
'comma separated list of limit values for the curve plot (syntax: key:min:max), '
'example: `train:loss/ess:1:3.5,train:loss/L_k:-89:-84,test:loss/L_k:-91:-87`'
)
parser.add_argument(
'--keys',
default='dataset, estimator, iw',
type=str,
help=
'comma separated list of keys to include in the report by decreasing order of importance, '
'more than 4 keys is not yet handled for plotting.')
parser.add_argument(
'--detailed_metrics',
default=
'train:loss/L_k,train:loss/kl_q_p,train:loss/ess,train:grads/snr',
type=str,
help='comma separated list of keys for the `detailed` plots. ')
parser.add_argument(
'--parse_estimator_args',
action='store_true',
help=
'parse estimator arguments such that `ovis-gamma1` -> `ovis` + gamma=1'
)
parser.add_argument('--latex',
action='store_true',
help='print as latex table')
parser.add_argument('--float_format', default=".2f", help='float format')
parser.add_argument(
'--downsample',
default=0,
type=int,
help='maximum number of point for the curves (downsampling)')
parser.add_argument('--ema',
default=0,
type=float,
help='exponential moving average')
parser.add_argument('--non_completed',
action='store_true',
help='allows reading non-completed runs')
parser.add_argument(
'--max_records',
default=-1,
type=int,
help='only read the first `max_records` data point (`-1` = no limit)')
parser.add_argument('--merge_args',
default='',
type=str,
help='comma separated list of args to be merged')
opt = parser.parse_args()
# define the run identifier
run_id = opt.exp
if len(opt.include):
run_id += f"-inc={opt.include}"
if len(opt.exclude):
run_id += f"-exc={opt.exclude}"
if opt.ema > 0:
run_id += f"-ema{opt.ema}"
# define, create output directory and get the logger
output_path = os.path.join(opt.output, run_id)
if os.path.exists(output_path):
rmtree(output_path)
os.makedirs(output_path)
logger, *_ = get_loggers(output_path,
keys=['report'],
format="%(message)s")
def print_df(df):
"""custom print function based on `opt.latex` and `opt.float_format`"""
if opt.latex:
logger.info(df.to_latex(float_format=f"%{opt.float_format}"))
else:
logger.info(df)
"""
read experiments, parse metrics and join attributes
"""
metrics = parse_keys_headers_metrics(opt)
path, data, logs = read_experiments(opt, metrics, logger)
data, logs, global_attributes = extract_global_attributes_and_join_into_logs(
opt, metrics, data, logs)
data = aggregate_metrics(data, logs, metrics)
# drop id (exp_id used for joins)
data.drop('id', 1, inplace=True)
logs.drop('id', 1, inplace=True)
# format estimator names
data['estimator'] = list(
map(format_estimator_name, data['estimator'].values))
logs['estimator'] = list(
map(format_estimator_name, logs['estimator'].values))
"""
print all results
"""
logger.info(f"{logging_sep('=')}\nGlobal Attributes\n{logging_sep('-')}")
for k, v in global_attributes.items():
logger.info(f"{k} : {v}")
logger.info(
f"{logging_sep('-')}\nExperiment path : {os.path.abspath(path)}\n{logging_sep('-')}"
)
logger.info(
f"Varying Parameters: {[k for k in data.keys() if k not in metrics['pivot_metrics']]}"
)
logger.info(
f"{logging_sep('-')}\nData (sorted by {metrics['pivot_metrics'][0]})\n{logging_sep('-')}"
)
print_df(data)
logger.info(logging_sep("="))
"""
build the pivot table and print
"""
pivot = build_pivot_table(opt, data, metrics)
logger.info("\n" + logging_sep('='))
logger.info("Pivot table\n" + logging_sep())
print_df(pivot)
logger.info(logging_sep('='))
# save to file
data.to_csv(os.path.join(output_path, "pivot.csv"))
"""
post processsing: smoothing + downsampling
"""
if opt.ema > 0:
logs = exponential_moving_average(logs, opt.ema)
if opt.downsample > 0:
logs = downsample(logs, opt.downsample, logger=logger)
"""drop nans + save to file"""
logs.dropna(inplace=True)
logs.to_csv(os.path.join(output_path, "curves.csv"))
"""plotting pivot plots, curve plots and detailed plots"""
# set style
set_matplotlib_style()
# plot for all auxiliary keys
ylims = parse_ylims(opt)
# ensure the first key to be `dataset`
keys = metrics['keys']
if logs['dataset'].nunique() > 1 and keys[0] != 'dataset':
keys = ['dataset'] + keys
# define keys used for styling
cat_key = keys[0]
main_key = keys[1] # color
aux_key = keys[2] if len(keys) > 2 else None # line style (in main plot)
third_key = keys[3] if len(
keys) > 3 else None # line style (in auxiliary plots)
meta = {
'log_rules': LOG_PLOT_RULES,
'metric_dict': METRIC_DISPLAY_NAME,
'agg_fns': metrics['pivot_metrics_agg_ids']
}
if logs[cat_key].nunique() > 1:
level = 1
# pivot plot
logger.info(f"|- Generating pivot plots for all keys {cat_key} ..")
_path = os.path.join(
output_path, f"{level}-pivot-plot-{cat_key}-hue={main_key}.png")
pivot_plot(data,
_path,
metrics['pivot_metrics'],
cat_key,
main_key,
aux_key,
style_key=third_key,
**meta)
# plot all data for each key
for cat in logs[cat_key].unique():
level = 2
print(logging_sep())
logger.info(f"[{cat_key} = {cat}]")
# slice data
cat_logs = logs[logs[cat_key] == cat]
cat_data = data[data[cat_key] == cat]
logger.info(f"|- Generating pivot plots with key = {cat_key} ..")
_path = os.path.join(
output_path,
f"{level}-{cat_key}={cat}-pivot-plot-hue={main_key}-style={third_key}.png"
)
pivot_plot(cat_data,
_path,
metrics['pivot_metrics'],
cat_key,
main_key,
aux_key,
style_key=third_key,
**meta)
logger.info(f"|- Generating simple plots for all aux_key = {aux_key}")
_path = os.path.join(
output_path,
f"{level}-{cat_key}={cat}-curves-hue={main_key}-style={aux_key}.png"
)
basic_curves_plot(cat_logs,
_path,
metrics['curves_metrics'],
main_key,
ylims=ylims,
style_key=aux_key,
**meta)
if aux_key is not None:
level = 3
# detailed plots
if len(metrics['detailed_metrics']):
logger.info(
f"|- Generating detailed plots for aux. key = {aux_key}")
_path = os.path.join(
output_path,
f"{level}-{cat_key}={cat}-detailed-plot-hue={main_key}-style={aux_key}.png"
)
detailed_curves_plot(cat_logs,
_path,
metrics['detailed_metrics'],
main_key,
aux_key,
style_key=third_key,
ylims=ylims,
**meta)
# on plot for each auxiliary key
# for each aux key..
aux_key_values = list(cat_logs[aux_key].unique())
for i, v in enumerate(sorted(aux_key_values)):
aux_cat_logs = cat_logs[cat_logs[aux_key] == v]
logger.info(
f"|--- Generating simple plots for {aux_key} = {v} [{i + 1} / {len(aux_key_values)}]"
)
# auxiliary plot
_path = os.path.join(
output_path,
f"{level}-{cat_key}={cat}-{aux_key}={v}-curves-hue={main_key}-style={third_key}.png"
)
basic_curves_plot(aux_cat_logs,
_path,
metrics['curves_metrics'],
main_key,
ylims=ylims,
style_key=third_key,
**meta)