def make_short_standard_simulation(name, identity, data_id, learner, path):
simulation = autem.Simulation(
name,
[
loaders.OpenMLLoader(data_id),
scorers.LeagueScorer(scorers.accuracy_score, [ [ 1, 4, 5 ] ]),
workflows.StandardWorkflow(),
baselines.BaselineStats(identity['dataset']),
learner_builders[learner](),
reporters.Csv(path),
])
settings = autem.SimulationSettings(simulation)
settings.set_max_species(3)
return simulation
def run_balance_scale_mastery(seed):
baseline_name = "balance-scale"
experiment = "mastery_%s_s%d" % (baseline_name, seed)
study = "DEV"
version = benchmark.get_version()
simulation_name = "%s_%s_v%d" % (study, experiment, version)
configuration = baselines.get_baseline_configuration(baseline_name)
path = benchmark.get_simulations_path().joinpath(study).joinpath(
experiment)
utility.prepare_OpenML()
task_id = configuration["task_id"]
task = openml.tasks.get_task(task_id)
data_id = task.dataset_id
dataset = openml.datasets.get_dataset(data_id)
dataset_name = dataset.name
identity = {
'study': study,
'experiment': experiment,
'dataset': dataset_name,
'version': version
}
simulation = autem.Simulation(simulation_name, [
loaders.OpenMLLoader(data_id),
scorers.LeagueScorer(scorers.accuracy_score),
workflows.MasteryWorkflow(),
baselines.BaselineStats(baseline_name),
hyper_learners.ClassificationSVM(),
reporters.Csv(path),
])
settings = autem.SimulationSettings(simulation)
settings.set_identity(identity)
settings.set_n_jobs(4)
settings.set_seed(seed)
simulation.run()
def make_benchmark_simulation(study, baseline_name, configuration, learner):
experiment = baseline_name
baseline_configuration = baselines.get_baseline_configuration(baseline_name)
task_id = baseline_configuration["task_id"]
task = openml.tasks.get_task(task_id)
data_id = task.dataset_id
version = get_version()
configuration = baseline_configuration["Configuration"] if configuration is None else configuration
learner = baseline_configuration["Learner"] if learner is None else learner
configuration_valid = configuration in simulation_builders
if not configuration_valid:
print("Baseline %s configuration %s does not exist" % (baseline_name, configuration))
return None
name = "'%s_%s_%s v%d'" % (study, experiment, configuration, version)
identity = {
'study': study,
'experiment': experiment,
'dataset': baseline_name,
'version': version,
'configuration': configuration,
}
n_jobs = get_n_jobs()
seed = 1
path = get_simulations_path().joinpath(study).joinpath(experiment)
memory = str(path.joinpath("cache"))
utility.prepare_OpenML()
simulation_builder = simulation_builders[configuration]
simulation = simulation_builder(name, identity, data_id, learner, path)
settings = autem.SimulationSettings(simulation)
settings.set_identity(identity)
settings.set_n_jobs(4)
settings.set_seed(seed)
settings.set_memory(memory)
return simulation
def make_standard_simulation(study, baseline_name, hyperlearner):
prepare_OpenML()
hyper_configuration = configuration.get_hyper_configuration(baseline_name)
task_id = hyper_configuration["task_id"]
task = openml.tasks.get_task(task_id)
data_id = task.dataset_id
name = "%s %s" % (baseline_name, study)
path = configuration.get_hyper_simulations_path().joinpath(study).joinpath(
baseline_name)
n_jobs = 4
seed = 1
memory = str(path.joinpath("cache"))
identity = {
'study': study,
'dataset': baseline_name,
'scorer': 'League1x10',
'workflow': 'standard',
'learner': hyperlearner,
}
simulation = autem.Simulation(name, [
loaders.OpenMLLoader(data_id),
scorers.LeagueScorer(scorers.accuracy_score, [[1, 4, 5]]),
workflows.StandardWorkflow(),
learner_builders[hyperlearner](),
reporters.Csv(path),
])
settings = autem.SimulationSettings(simulation)
settings.set_identity(identity)
settings.set_n_jobs(4)
settings.set_seed(seed)
settings.set_memory(memory)
return simulation