Beispiel #1
0
def create_bench_jobs(context, batch, alltestcases):
    '''
    
    :param context: 
    :param batch: 
    :param alltestcases: list of testcases that are going to be autogenerated.
    '''
    config = get_dp_config()
    algos = config.algos.expand_names(batch.algorithms)
    #     if not alltestcases:
    #         testcases = config.testcases.expand_names(batch.testcases)
    #     else:
    #         testcases = expand_string(batch.testcases, alltestcases)

    # dict(id_algo, id_tc, id_discdds, plan_length) => PlanningResults
    allplanning = StoreResults()

    # dict(id_algo, id_tc, id_discdds, plan_length) => resultstats
    allruns = StoreResults()

    # dict(id_algo, id_tc) => DiffeoPlanningAlgorithm
    algoinit = StoreResults()

    config = get_dp_config()

    # Let's instantiate all test cases and sort them by discdds
    # so that we do only one initialization per algorithms

    # id -> {dds -> [tc]}
    id_discdds2testcases = defaultdict(lambda: {})
    #     alltc = {}  # id -> Promise TestCase

    id2t = batch.get_id_discdds2testcases(alltestcases)
    for idd, idd_tcs in id2t.items():
        for tcc in idd_tcs:
            tc = context.comp(instantiate_testcase, tcc)
            # alltc[tcc]
            id_discdds2testcases[idd][tcc] = tc

    # Load discdds before, they might be automatically generated
    # as well so we want the generation to happen only once.
    discdds = {}  # id -> Promise DiffeoSystem
    for id_discdds in id_discdds2testcases:
        discdds[id_discdds] = context.comp(instantiate_discdds, id_discdds)

    # for each algorithm
    for id_algo in algos:
        config.algos[id_algo]  # check it is in the configuration

        # for each dynamics
        for id_discdds, dds in discdds.items():
            job_id = 'init-%s-%s' % (id_algo, id_discdds)

            # initialize the algorithm for that dynamics
            algo = context.comp(init_algorithm,
                                id_algo,
                                id_discdds,
                                discdds[id_discdds],
                                job_id=job_id)
            algoinit[dict(id_algo=id_algo, id_discdds=id_discdds)] = algo

            # for each test case in that dynamics
            for id_tc, tc in id_discdds2testcases[id_discdds].items():
                # run the planning
                job_id = 'plan-%s-%s' % (id_algo, id_tc)
                result = context.comp(run_planning,
                                      id_algo,
                                      id_tc,
                                      tc,
                                      algo,
                                      job_id=job_id)

                # compute statistics
                result_stats = context.comp(run_planning_stats,
                                            result,
                                            dds,
                                            tc,
                                            job_id=job_id + '-stats')

                attrs = dict(id_algo=id_algo,
                             id_tc=id_tc,
                             id_discdds=id_discdds)
                warnings.warn('this was an important attribute to have...')
                # true_plan_length=len(tc.true_plan))
                allruns[attrs] = result_stats
                allplanning[attrs] = result

    jobs_report_algo_init(context, algoinit)
    jobs_report_tc(context, id_discdds2testcases)
    jobs_report_dds(context, discdds)

    allstats = StoreResults()
    for key, run in allruns.items():
        allstats[key] = context.comp(results2stats_dict,
                                     run,
                                     job_id=comp_stage_job_id(
                                         run, 'statsdict'))

    jobs_tables(context, allstats)
    jobs_visualization(context, allruns)
Beispiel #2
0
def create_bench_jobs(context, batch, alltestcases):
    '''
    
    :param context: 
    :param batch: 
    :param alltestcases: list of testcases that are going to be autogenerated.
    '''
    config = get_dp_config()
    algos = config.algos.expand_names(batch.algorithms)
#     if not alltestcases:
#         testcases = config.testcases.expand_names(batch.testcases)          
#     else:
#         testcases = expand_string(batch.testcases, alltestcases)

    # dict(id_algo, id_tc, id_discdds, plan_length) => PlanningResults
    allplanning = StoreResults()
    
    # dict(id_algo, id_tc, id_discdds, plan_length) => resultstats
    allruns = StoreResults() 
    
    # dict(id_algo, id_tc) => DiffeoPlanningAlgorithm
    algoinit = StoreResults()
    
    config = get_dp_config()
    
    # Let's instantiate all test cases and sort them by discdds
    # so that we do only one initialization per algorithms

    # id -> {dds -> [tc]}    
    id_discdds2testcases = defaultdict(lambda: {})
#     alltc = {}  # id -> Promise TestCase

    id2t = batch.get_id_discdds2testcases(alltestcases)
    for idd, idd_tcs in id2t.items():
        for tcc in idd_tcs:
            tc = context.comp(instantiate_testcase, tcc)
            # alltc[tcc]
            id_discdds2testcases[idd][tcc] = tc
            
    # Load discdds before, they might be automatically generated
    # as well so we want the generation to happen only once.
    discdds = {}  # id -> Promise DiffeoSystem
    for id_discdds in id_discdds2testcases: 
        discdds[id_discdds] = context.comp(instantiate_discdds, id_discdds)
        
    # for each algorithm
    for id_algo in algos:
        config.algos[id_algo]  # check it is in the configuration
            
        # for each dynamics
        for id_discdds, dds in discdds.items():
            job_id = 'init-%s-%s' % (id_algo, id_discdds)
            
            # initialize the algorithm for that dynamics
            algo = context.comp(init_algorithm, id_algo, id_discdds,
                                discdds[id_discdds], job_id=job_id)
            algoinit[dict(id_algo=id_algo, id_discdds=id_discdds)] = algo
            
            # for each test case in that dynamics
            for id_tc, tc in id_discdds2testcases[id_discdds].items():
                # run the planning
                job_id = 'plan-%s-%s' % (id_algo, id_tc)
                result = context.comp(run_planning, id_algo,
                              id_tc, tc, algo, job_id=job_id)
                
                # compute statistics
                result_stats = context.comp(run_planning_stats, result,
                                    dds, tc,
                                    job_id=job_id + '-stats') 
    
                attrs = dict(id_algo=id_algo, id_tc=id_tc,
                             id_discdds=id_discdds)
                warnings.warn('this was an important attribute to have...')
                # true_plan_length=len(tc.true_plan))
                allruns[attrs] = result_stats
                allplanning[attrs] = result
    
    jobs_report_algo_init(context, algoinit)
    jobs_report_tc(context, id_discdds2testcases)
    jobs_report_dds(context, discdds)

    allstats = StoreResults()
    for key, run in allruns.items():
        allstats[key] = context.comp(results2stats_dict, run,
                             job_id=comp_stage_job_id(run, 'statsdict'))

    jobs_tables(context, allstats)
    jobs_visualization(context, allruns)
Beispiel #3
0
def create_bench_jobs(config, algos, testcases, outdir):
    # dict(id_algo, id_tc, id_discdds, plan_length) => PlanningResults
    allplanning = StoreResults()
    # dict(id_algo, id_tc, id_discdds, plan_length) => resultstats
    allruns = StoreResults() 
    # dict(id_algo, id_tc) => DiffeoPlanningAlgorithm
    algoinit = StoreResults()
    
    rm = ReportManager(outdir)
    
    comp_store(config, job_id='config') 
    
    # Let's instantiate all test cases and sort them by discdds
    # so that we do only one initialization per algorithms
    id_discdds2testcases = defaultdict(lambda: {}) 
    alltc = {} # id -> Promise TestCase
    for id_tc in testcases:
        alltc[id_tc] = comp(instantiate_testcase, comp_store(config), id_tc)
        # Do it once, now, to get its dds
        tc1 = config.testcases.instance(id_tc) 
        id_discdds2testcases[tc1.id_discdds][id_tc] = tc1
    
    # Load discdds before, they might be automatically generated
    # as well so we want the generation to happen only once.
    discdds = {} # id -> Promise DiffeoSystem
    for id_discdds in id_discdds2testcases: 
        discdds[id_discdds] = comp(instantiate_discdds, comp_store(config), id_discdds)
        
    # for each algorithm
    for id_algo in algos:
        config.algos[id_algo] # check it is in the configuration
            
        # for each dynamics
        for id_discdds, dds in discdds.items():
            job_id = 'init-%s-%s' % (id_algo, id_discdds)
            
            # initialize the algorithm for that dynamics
            algo = comp(init_algorithm, comp_store(config),
                        id_algo, id_discdds, discdds[id_discdds], job_id=job_id)
            algoinit[dict(id_algo=id_algo, id_discdds=id_discdds)] = algo
            
            # for each test case in that dynamics
            for id_tc, tc in id_discdds2testcases[id_discdds].items():
                
                # run the planning
                job_id = 'plan-%s-%s' % (id_algo, id_tc)
                result = comp(run_planning, id_algo,
                              id_tc, alltc[id_tc], algo, job_id=job_id)
                
                # compute statistics
                result_stats = comp(run_planning_stats, result,
                                    dds, alltc[id_tc],
                                    job_id=job_id + '-stats') 
    
                attrs = dict(id_algo=id_algo, id_tc=id_tc,
                             id_discdds=tc.id_discdds,
                             true_plan_length=len(tc.true_plan))
                allruns[attrs] = result_stats
                allplanning[attrs] = result
    
    jobs_report_algo_init(config, rm, algoinit)
    jobs_report_tc(config, rm, testcases, alltc)
    jobs_report_dds(config, rm, discdds)

    allstats = StoreResults()
    for key, run in allruns.items():
        allstats[key] = comp(results2stats_dict, run,
                             job_id=comp_stage_job_id(run, 'statsdict'))

    jobs_tables(allstats, rm)
    jobs_visualization(config, allruns, rm)
    
    rm.create_index_job()