def main():
"""See README for instructions on calling aggregate.
"""
description = "Aggregate study results across functions and optimizers"
args = parse_args(agg_parser(description))
logger.setLevel(logging.INFO) # Note this is the module-wide logger
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
# Get list of UUIDs
uuid_list = XRSerializer.get_uuids(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.EVAL)
uuid_list_ = XRSerializer.get_uuids(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.TIME)
assert uuid_list == uuid_list_, "UUID list does not match between time and eval results"
uuid_list_ = XRSerializer.get_uuids(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.SUGGEST_LOG)
assert uuid_list == uuid_list_, "UUID list does not match between suggest log and eval results"
# Get iterator of all experiment data dumps, load in and process, and concat
data_G = load_experiments(uuid_list, args[CmdArgs.db_root], args[CmdArgs.db])
all_perf, all_time, all_suggest, all_sigs = concat_experiments(data_G, ravel=args[CmdArgs.ravel])
# Check the concat signatures make are coherent
sig_errs, signatures_median = analyze_signatures(all_sigs)
logger.info("Signature errors:\n%s" % sig_errs.to_string())
print(json.dumps({"exp-agg sig errors": sig_errs.T.to_dict()}))
# Dump and save it all out
logger.info("saving")
meta = {"args": serializable_dict(args), "signature": signatures_median}
XRSerializer.save_derived(all_perf, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=EVAL_RESULTS)
XRSerializer.save_derived(all_time, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=TIME_RESULTS)
for test_case, ds in all_suggest.items():
XRSerializer.save_derived(ds, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=test_case)
logger.info("done")
def main(): # pragma: main
"""This is where experiments happen. Usually called by the experiment launcher."""
description = "Run a study with one benchmark function and an optimizer"
args = cmd.parse_args(cmd.experiment_parser(description))
opt_class = _get_opt_class(args[CmdArgs.optimizer])
experiment_main(opt_class, args=args)
def main():
"""See README for instructions on calling baseline.
"""
description = "Aggregate the baselines for later analysis in benchmark"
args = parse_args(general_parser(description))
logger.setLevel(logging.INFO) # Note this is the module-wide logger
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
do_baseline(args)
logger.info("done")
def main():
"""See README for instructions on calling launcher.
"""
description = "Launch series of studies across functions and optimizers"
args = cmd.parse_args(cmd.launcher_parser(description))
logger.setLevel(logging.INFO) # Note this is the module-wide logger
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
# Get optimizer settings, says which file to call for each optimizer
settings = cmd.load_optimizer_settings(args[CmdArgs.optimizer_root])
opt_file_lookup = {
optimizer: wrapper_file
for optimizer, (wrapper_file, _) in settings.items()
}
# Setup uuid
if args[CmdArgs.uuid] is None:
args[CmdArgs.uuid] = pyuuid.uuid4(
).hex # debatable if uuid1 or uuid4 is better here
else:
warnings.warn(
"User UUID supplied. This is only desired for debugging. Careless use could lead to study id conflicts.",
UserWarning,
)
run_uuid = pyuuid.UUID(hex=args[CmdArgs.uuid])
assert run_uuid.hex == args[CmdArgs.uuid]
logger.info("Supply --uuid %s to reproduce this run." % run_uuid.hex)
# Log all the options
print("Launcher options (JSON):\n")
print(json.dumps({"bayesmark-launch-args": cmd.serializable_dict(args)}))
print("\n")
# Set the master seed (derive from the uuid we just setup)
pyrandom.seed(run_uuid.int)
np.random.seed(random_seed(pyrandom))
# Now run it, either to dry run file or executes sub-processes
if args[CmdArgs.dry_run]:
with absopen(args[CmdArgs.jobs_file], "w") as fp:
dry_run(args, opt_file_lookup, run_uuid, fp)
else:
timeout = args[CmdArgs.timeout] if args[CmdArgs.timeout] > 0 else None
real_run(args, opt_file_lookup, run_uuid, timeout)
logger.info("done")
def main():
"""See README for instructions on calling analysis.
"""
description = "Analyze results from aggregated studies"
args = parse_args(general_parser(description))
logger.setLevel(logging.INFO) # Note this is the module-wide logger
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
# Load in the eval data and sanity check
perf_da, meta = XRSerializer.load_derived(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.EVAL_RESULTS)
perf_da = xru.only_dataarray(perf_da)
logger.info("Meta data from source file: %s" % str(meta["args"]))
# Check if there is baselines file, other make one
if cc.BASELINE not in XRSerializer.get_derived_keys(args[CmdArgs.db_root], db=args[CmdArgs.db]):
warnings.warn("Baselines not found. Need to construct baseline.")
do_baseline(args)
# Load in baseline scores data and sanity check (including compatibility with eval data)
baseline_ds, meta_ref = XRSerializer.load_derived(args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.BASELINE)
logger.info("baseline data from source ref file: %s" % str(meta_ref["args"]))
# Check test case signatures match between eval data and baseline data
sig_errs, signatures = analyze_signature_pair(meta["signature"], meta_ref["signature"])
logger.info("Signature errors:\n%s" % sig_errs.to_string())
# Do the actual computation
agg_result, summary = compute_aggregates(perf_da, baseline_ds)
final_score = summary[PERF_MED][{ITER: -1}]
logger.info("median score @ %d:\n%s" % (summary.sizes[ITER], xru.da_to_string(final_score)))
final_score = summary[PERF_MEAN][{ITER: -1}]
logger.info("mean score @ %d:\n%s" % (summary.sizes[ITER], xru.da_to_string(final_score)))
final_score = summary[NORMED_MEAN][{ITER: -1}]
logger.info("normed mean score @ %d:\n%s" % (summary.sizes[ITER], xru.da_to_string(final_score)))
# Now saving results
meta = {"args": serializable_dict(args), "signature": signatures}
XRSerializer.save_derived(agg_result, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.PERF_RESULTS)
XRSerializer.save_derived(summary, meta, args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.MEAN_SCORE)
logger.info("done")
def main():
"""See README for instructions on calling db_init.
"""
description = "Initialize the directories for running the experiments"
args = cmd.parse_args(cmd.general_parser(description))
assert not args[
CmdArgs.dry_run], "Dry run doesn't make any sense when building dirs"
logger.setLevel(logging.INFO) # Note this is the module-wide logger
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
XRSerializer.init_db(args[CmdArgs.db_root],
db=args[CmdArgs.db],
keys=EXP_VARS,
exist_ok=EXIST_OK)
logger.info("done")
def experiment_main(opt_class, args=None): # pragma: main
"""This is in effect the `main` routine for this experiment. However, it is called from the optimizer wrapper file
so the class can be passed in. The optimizers are assumed to be outside the package, so the optimizer class can't
be named from inside the main function without using hacky stuff like `eval`.
"""
if args is None:
description = "Run a study with one benchmark function and an optimizer"
args = cmd.parse_args(cmd.experiment_parser(description))
args[CmdArgs.opt_rev] = opt_class.get_version()
run_uuid = uuid.UUID(args[CmdArgs.uuid])
logging.captureWarnings(True)
# Setup logging to both a file and stdout (if verbose is set to True)
logger.setLevel(logging.INFO) # Note this is the module-wide logger
logfile = XRSerializer.logging_path(args[CmdArgs.db_root],
args[CmdArgs.db], run_uuid)
logger_file_handler = logging.FileHandler(logfile, mode="w")
logger.addHandler(logger_file_handler)
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
warnings_logger = logging.getLogger("py.warnings")
warnings_logger.addHandler(logger_file_handler)
if args[CmdArgs.verbose]:
warnings_logger.addHandler(logging.StreamHandler())
logger.info("running: %s" % str(cmd.serializable_dict(args)))
logger.info("cmd: %s" % cmd.cmd_str())
assert (args[CmdArgs.metric]
in METRICS_LOOKUP[get_problem_type(args[CmdArgs.data])]
), "reg/clf metrics can only be used on compatible dataset"
# Setup random streams for computing the signature, must use same seed
# across all runs to ensure signature is consistent. This seed is random:
_setup_seeds(
"7e9f2cabb0dd4f44bc10cf18e440b427") # pragma: allowlist secret
signature = get_objective_signature(args[CmdArgs.classifier],
args[CmdArgs.data],
args[CmdArgs.metric],
data_root=args[CmdArgs.data_root])
logger.info("computed signature: %s" % str(signature))
opt_kwargs = load_optimizer_kwargs(args[CmdArgs.optimizer],
args[CmdArgs.optimizer_root])
# Setup the call back for intermediate logging
if cc.BASELINE not in XRSerializer.get_derived_keys(args[CmdArgs.db_root],
db=args[CmdArgs.db]):
warnings.warn("Baselines not found. Will not log intermediate scores.")
callback = None
else:
test_case_str = SklearnModel.test_case_str(args[CmdArgs.classifier],
args[CmdArgs.data],
args[CmdArgs.metric])
optimizer_str = str_join_safe(
ARG_DELIM, (args[CmdArgs.optimizer], args[CmdArgs.opt_rev],
args[CmdArgs.rev]))
baseline_ds, baselines_meta = XRSerializer.load_derived(
args[CmdArgs.db_root], db=args[CmdArgs.db], key=cc.BASELINE)
# Check the objective function signatures match in the baseline file
sig_errs, _ = analyze_signature_pair({test_case_str: signature[1]},
baselines_meta["signature"])
logger.info("Signature errors:\n%s" % sig_errs.to_string())
print(json.dumps({"exp sig errors": sig_errs.T.to_dict()}))
def log_mean_score_json(evals, iters):
assert evals.shape == (len(OBJECTIVE_NAMES), )
assert not np.any(np.isnan(evals))
log_msg = {
cc.TEST_CASE: test_case_str,
cc.METHOD: optimizer_str,
cc.TRIAL: args[CmdArgs.uuid],
cc.ITER: iters,
}
for idx, obj in enumerate(OBJECTIVE_NAMES):
assert OBJECTIVE_NAMES[idx] == obj
# Extract relevant rescaling info
slice_ = {cc.TEST_CASE: test_case_str, OBJECTIVE: obj}
best_opt = baseline_ds[cc.PERF_BEST].sel(
slice_, drop=True).values.item()
base_clip_val = baseline_ds[cc.PERF_CLIP].sel(
slice_, drop=True).values.item()
# Perform the same rescaling as found in experiment_analysis.compute_aggregates()
score = linear_rescale(evals[idx],
best_opt,
base_clip_val,
0.0,
1.0,
enforce_bounds=False)
# Also, clip the score from below at -1 to limit max influence of single run on final average
score = np.clip(score, -1.0, 1.0)
score = score.item() # Make easiest for logging in JSON
assert isinstance(score, float)
# Note: This is not the raw score but the rescaled one!
log_msg[obj] = score
log_msg = json.dumps(log_msg)
print(log_msg, flush=True)
# One second safety delay to protect against subprocess stdout getting lost
sleep(1)
callback = log_mean_score_json
# Now set the seeds for the actual experiment
_setup_seeds(args[CmdArgs.uuid])
# Now do the experiment
logger.info("starting sklearn study %s %s %s %s %d %d" % (
args[CmdArgs.optimizer],
args[CmdArgs.classifier],
args[CmdArgs.data],
args[CmdArgs.metric],
args[CmdArgs.n_calls],
args[CmdArgs.n_suggest],
))
logger.info("with data root: %s" % args[CmdArgs.data_root])
function_evals, timing, suggest_log = run_sklearn_study(
opt_class,
opt_kwargs,
args[CmdArgs.classifier],
args[CmdArgs.data],
args[CmdArgs.metric],
args[CmdArgs.n_calls],
args[CmdArgs.n_suggest],
data_root=args[CmdArgs.data_root],
callback=callback,
)
# Curate results into clean dataframes
eval_ds = build_eval_ds(function_evals, OBJECTIVE_NAMES)
time_ds = build_timing_ds(*timing)
suggest_ds = build_suggest_ds(suggest_log)
# setup meta:
meta = {"args": cmd.serializable_dict(args), "signature": signature}
logger.info("saving meta data: %s" % str(meta))
# Now the final IO to export the results
logger.info("saving results")
XRSerializer.save(eval_ds,
meta,
args[CmdArgs.db_root],
db=args[CmdArgs.db],
key=cc.EVAL,
uuid_=run_uuid)
logger.info("saving timing")
XRSerializer.save(time_ds,
meta,
args[CmdArgs.db_root],
db=args[CmdArgs.db],
key=cc.TIME,
uuid_=run_uuid)
logger.info("saving suggest log")
XRSerializer.save(suggest_ds,
meta,
args[CmdArgs.db_root],
db=args[CmdArgs.db],
key=cc.SUGGEST_LOG,
uuid_=run_uuid)
logger.info("done")
def main(): # pragma: main
description = "Run a study with one benchmark function and an optimizer"
args = cmd.parse_args(cmd.experiment_parser(description))
opt_class = _get_opt_class(args[CmdArgs.optimizer])
experiment_main(opt_class, args=args)
def experiment_main(opt_class, args=None): # pragma: main
"""This is in effect the `main` routine for this experiment. However, it is called from the optimizer wrapper file
so the class can be passed in. The optimizers are assumed to be outside the package, so the optimizer classs can't
be named from inside the main function without using hacky stuff like `eval`.
"""
if args is None:
description = "Run a study with one benchmark function and an optimizer"
args = cmd.parse_args(cmd.experiment_parser(description))
args[CmdArgs.opt_rev] = opt_class.get_version()
run_uuid = uuid.UUID(args[CmdArgs.uuid])
logging.captureWarnings(True)
# Setup logging to both a file and stdout (if verbose is set to True)
logger.setLevel(logging.INFO) # Note this is the module-wide logger
logfile = XRSerializer.logging_path(args[CmdArgs.db_root],
args[CmdArgs.db], run_uuid)
logger_file_handler = logging.FileHandler(logfile, mode="w")
logger.addHandler(logger_file_handler)
if args[CmdArgs.verbose]:
logger.addHandler(logging.StreamHandler())
warnings_logger = logging.getLogger("py.warnings")
warnings_logger.addHandler(logger_file_handler)
if args[CmdArgs.verbose]:
warnings_logger.addHandler(logging.StreamHandler())
logger.info("running: %s" % str(cmd.serializable_dict(args)))
logger.info("cmd: %s" % cmd.cmd_str())
assert (args[CmdArgs.metric]
in METRICS_LOOKUP[get_problem_type(args[CmdArgs.data])]
), "reg/clf metrics can only be used on compatible dataset"
# Setup random streams for computing the signature, must use same seed
# across all runs to ensure signature is consistent. This seed is random:
_setup_seeds(
"7e9f2cabb0dd4f44bc10cf18e440b427") # pragma: allowlist secret
signature = get_objective_signature(args[CmdArgs.classifier],
args[CmdArgs.data],
args[CmdArgs.metric],
data_root=args[CmdArgs.data_root])
logger.info("computed signature: %s" % str(signature))
opt_kwargs = load_optimizer_kwargs(args[CmdArgs.optimizer],
args[CmdArgs.optimizer_root])
# Now set the seeds for the actual experiment
_setup_seeds(args[CmdArgs.uuid])
# Now do the experiment
logger.info("starting sklearn study %s %s %s %s %d %d" % (
args[CmdArgs.optimizer],
args[CmdArgs.classifier],
args[CmdArgs.data],
args[CmdArgs.metric],
args[CmdArgs.n_calls],
args[CmdArgs.n_suggest],
))
logger.info("with data root: %s" % args[CmdArgs.data_root])
function_evals, timing = run_sklearn_study(
opt_class,
opt_kwargs,
args[CmdArgs.classifier],
args[CmdArgs.data],
args[CmdArgs.metric],
args[CmdArgs.n_calls],
args[CmdArgs.n_suggest],
data_root=args[CmdArgs.data_root],
)
# Curate results into clean dataframes
eval_ds = build_eval_ds(function_evals)
time_ds = build_timing_ds(*timing)
# setup meta:
meta = {"args": cmd.serializable_dict(args), "signature": signature}
logger.info("saving meta data: %s" % str(meta))
# Now the final IO to export the results
logger.info("saving results")
XRSerializer.save(eval_ds,
meta,
args[CmdArgs.db_root],
db=args[CmdArgs.db],
key=cc.EVAL,
uuid_=run_uuid)
logger.info("saving timing")
XRSerializer.save(time_ds,
meta,
args[CmdArgs.db_root],
db=args[CmdArgs.db],
key=cc.TIME,
uuid_=run_uuid)
logger.info("done")