def create_bench_jobs(context, batch, alltestcases):
'''
:param context:
:param batch:
:param alltestcases: list of testcases that are going to be autogenerated.
'''
config = get_dp_config()
algos = config.algos.expand_names(batch.algorithms)
# if not alltestcases:
# testcases = config.testcases.expand_names(batch.testcases)
# else:
# testcases = expand_string(batch.testcases, alltestcases)
# dict(id_algo, id_tc, id_discdds, plan_length) => PlanningResults
allplanning = StoreResults()
# dict(id_algo, id_tc, id_discdds, plan_length) => resultstats
allruns = StoreResults()
# dict(id_algo, id_tc) => DiffeoPlanningAlgorithm
algoinit = StoreResults()
config = get_dp_config()
# Let's instantiate all test cases and sort them by discdds
# so that we do only one initialization per algorithms
# id -> {dds -> [tc]}
id_discdds2testcases = defaultdict(lambda: {})
# alltc = {} # id -> Promise TestCase
id2t = batch.get_id_discdds2testcases(alltestcases)
for idd, idd_tcs in id2t.items():
for tcc in idd_tcs:
tc = context.comp(instantiate_testcase, tcc)
# alltc[tcc]
id_discdds2testcases[idd][tcc] = tc
# Load discdds before, they might be automatically generated
# as well so we want the generation to happen only once.
discdds = {} # id -> Promise DiffeoSystem
for id_discdds in id_discdds2testcases:
discdds[id_discdds] = context.comp(instantiate_discdds, id_discdds)
# for each algorithm
for id_algo in algos:
config.algos[id_algo] # check it is in the configuration
# for each dynamics
for id_discdds, dds in discdds.items():
job_id = 'init-%s-%s' % (id_algo, id_discdds)
# initialize the algorithm for that dynamics
algo = context.comp(init_algorithm,
id_algo,
id_discdds,
discdds[id_discdds],
job_id=job_id)
algoinit[dict(id_algo=id_algo, id_discdds=id_discdds)] = algo
# for each test case in that dynamics
for id_tc, tc in id_discdds2testcases[id_discdds].items():
# run the planning
job_id = 'plan-%s-%s' % (id_algo, id_tc)
result = context.comp(run_planning,
id_algo,
id_tc,
tc,
algo,
job_id=job_id)
# compute statistics
result_stats = context.comp(run_planning_stats,
result,
dds,
tc,
job_id=job_id + '-stats')
attrs = dict(id_algo=id_algo,
id_tc=id_tc,
id_discdds=id_discdds)
warnings.warn('this was an important attribute to have...')
# true_plan_length=len(tc.true_plan))
allruns[attrs] = result_stats
allplanning[attrs] = result
jobs_report_algo_init(context, algoinit)
jobs_report_tc(context, id_discdds2testcases)
jobs_report_dds(context, discdds)
allstats = StoreResults()
for key, run in allruns.items():
allstats[key] = context.comp(results2stats_dict,
run,
job_id=comp_stage_job_id(
run, 'statsdict'))
jobs_tables(context, allstats)
jobs_visualization(context, allruns)
def create_bench_jobs(context, batch, alltestcases):
'''
:param context:
:param batch:
:param alltestcases: list of testcases that are going to be autogenerated.
'''
config = get_dp_config()
algos = config.algos.expand_names(batch.algorithms)
# if not alltestcases:
# testcases = config.testcases.expand_names(batch.testcases)
# else:
# testcases = expand_string(batch.testcases, alltestcases)
# dict(id_algo, id_tc, id_discdds, plan_length) => PlanningResults
allplanning = StoreResults()
# dict(id_algo, id_tc, id_discdds, plan_length) => resultstats
allruns = StoreResults()
# dict(id_algo, id_tc) => DiffeoPlanningAlgorithm
algoinit = StoreResults()
config = get_dp_config()
# Let's instantiate all test cases and sort them by discdds
# so that we do only one initialization per algorithms
# id -> {dds -> [tc]}
id_discdds2testcases = defaultdict(lambda: {})
# alltc = {} # id -> Promise TestCase
id2t = batch.get_id_discdds2testcases(alltestcases)
for idd, idd_tcs in id2t.items():
for tcc in idd_tcs:
tc = context.comp(instantiate_testcase, tcc)
# alltc[tcc]
id_discdds2testcases[idd][tcc] = tc
# Load discdds before, they might be automatically generated
# as well so we want the generation to happen only once.
discdds = {} # id -> Promise DiffeoSystem
for id_discdds in id_discdds2testcases:
discdds[id_discdds] = context.comp(instantiate_discdds, id_discdds)
# for each algorithm
for id_algo in algos:
config.algos[id_algo] # check it is in the configuration
# for each dynamics
for id_discdds, dds in discdds.items():
job_id = 'init-%s-%s' % (id_algo, id_discdds)
# initialize the algorithm for that dynamics
algo = context.comp(init_algorithm, id_algo, id_discdds,
discdds[id_discdds], job_id=job_id)
algoinit[dict(id_algo=id_algo, id_discdds=id_discdds)] = algo
# for each test case in that dynamics
for id_tc, tc in id_discdds2testcases[id_discdds].items():
# run the planning
job_id = 'plan-%s-%s' % (id_algo, id_tc)
result = context.comp(run_planning, id_algo,
id_tc, tc, algo, job_id=job_id)
# compute statistics
result_stats = context.comp(run_planning_stats, result,
dds, tc,
job_id=job_id + '-stats')
attrs = dict(id_algo=id_algo, id_tc=id_tc,
id_discdds=id_discdds)
warnings.warn('this was an important attribute to have...')
# true_plan_length=len(tc.true_plan))
allruns[attrs] = result_stats
allplanning[attrs] = result
jobs_report_algo_init(context, algoinit)
jobs_report_tc(context, id_discdds2testcases)
jobs_report_dds(context, discdds)
allstats = StoreResults()
for key, run in allruns.items():
allstats[key] = context.comp(results2stats_dict, run,
job_id=comp_stage_job_id(run, 'statsdict'))
jobs_tables(context, allstats)
jobs_visualization(context, allruns)
def create_bench_jobs(config, algos, testcases, outdir):
# dict(id_algo, id_tc, id_discdds, plan_length) => PlanningResults
allplanning = StoreResults()
# dict(id_algo, id_tc, id_discdds, plan_length) => resultstats
allruns = StoreResults()
# dict(id_algo, id_tc) => DiffeoPlanningAlgorithm
algoinit = StoreResults()
rm = ReportManager(outdir)
comp_store(config, job_id='config')
# Let's instantiate all test cases and sort them by discdds
# so that we do only one initialization per algorithms
id_discdds2testcases = defaultdict(lambda: {})
alltc = {} # id -> Promise TestCase
for id_tc in testcases:
alltc[id_tc] = comp(instantiate_testcase, comp_store(config), id_tc)
# Do it once, now, to get its dds
tc1 = config.testcases.instance(id_tc)
id_discdds2testcases[tc1.id_discdds][id_tc] = tc1
# Load discdds before, they might be automatically generated
# as well so we want the generation to happen only once.
discdds = {} # id -> Promise DiffeoSystem
for id_discdds in id_discdds2testcases:
discdds[id_discdds] = comp(instantiate_discdds, comp_store(config), id_discdds)
# for each algorithm
for id_algo in algos:
config.algos[id_algo] # check it is in the configuration
# for each dynamics
for id_discdds, dds in discdds.items():
job_id = 'init-%s-%s' % (id_algo, id_discdds)
# initialize the algorithm for that dynamics
algo = comp(init_algorithm, comp_store(config),
id_algo, id_discdds, discdds[id_discdds], job_id=job_id)
algoinit[dict(id_algo=id_algo, id_discdds=id_discdds)] = algo
# for each test case in that dynamics
for id_tc, tc in id_discdds2testcases[id_discdds].items():
# run the planning
job_id = 'plan-%s-%s' % (id_algo, id_tc)
result = comp(run_planning, id_algo,
id_tc, alltc[id_tc], algo, job_id=job_id)
# compute statistics
result_stats = comp(run_planning_stats, result,
dds, alltc[id_tc],
job_id=job_id + '-stats')
attrs = dict(id_algo=id_algo, id_tc=id_tc,
id_discdds=tc.id_discdds,
true_plan_length=len(tc.true_plan))
allruns[attrs] = result_stats
allplanning[attrs] = result
jobs_report_algo_init(config, rm, algoinit)
jobs_report_tc(config, rm, testcases, alltc)
jobs_report_dds(config, rm, discdds)
allstats = StoreResults()
for key, run in allruns.items():
allstats[key] = comp(results2stats_dict, run,
job_id=comp_stage_job_id(run, 'statsdict'))
jobs_tables(allstats, rm)
jobs_visualization(config, allruns, rm)
rm.create_index_job()