if not os.path.isdir(EXP_PATH): print("Please define a valid experiment path.") exit(1) suffix = "" if EXP_PATH.startswith("results/prost_"): suffix = "_{}".format(EXP_PATH[14:]) if suffix[-1] == "/": suffix = suffix[:-1] # Create a new experiment. exp = Experiment(path=EXP_PATH) # Add Prost parser. exp.add_parser("parser.py") exp.add_parse_again_step() # Add step that collects properties from run directories and # writes them to *-eval/properties. exp.add_fetcher(name="fetch") # Make a basic table report with IPC scores. ipc_scores = IPCScores() exp.add_report( ProstBaseReport(attributes=ATTRIBUTES, filter=[ipc_scores.store_rewards, ipc_scores.add_score]), outfile="report{}.html".format(suffix), )
"source /infai/blaas/virtualenvs/grounder/bin/activate\n"), export=["PATH", "DOWNWARD_BENCHMARKS", "POWER_LIFTED_DIR"]) else: SUITE = ['organic-synthesis-alkene:p2.pddl'] ENV = LocalEnvironment(processes=4) TIME_LIMIT = 1800 MEMORY_LIMIT = 16384 ATTRIBUTES = ['atoms', 'grounding_time', 'parsing_time', 'total_time'] # Create a new experiment. exp = Experiment(environment=ENV) # Add custom parser for Power Lifted. exp.add_parser('parser.py') NEW_GROUNDER_CONFIGS = [Configuration('new-grounder', [])] FD_CONFIGS = [Configuration('fd-grounder', [])] # Create one run for each instance and each configuration for config in NEW_GROUNDER_CONFIGS: for task in suites.build_suite(BENCHMARKS_DIR, SUITE): run = exp.add_run() run.add_resource('domain', task.domain_file, symlink=True) run.add_resource('problem', task.problem_file, symlink=True) run.add_command( 'run-search', [RUN_SCRIPT_DIR + '/ground.py', '-i', task.problem_file], time_limit=TIME_LIMIT, memory_limit=MEMORY_LIMIT)
ENV = BaselSlurmEnvironment(email="*****@*****.**") else: ENV = LocalEnvironment(processes=4) SUITE = [ 'grid', 'gripper:prob01.pddl', 'miconic:s1-0.pddl', 'mystery:prob07.pddl' ] ATTRIBUTES = [ 'coverage', 'error', 'evaluations', 'plan', 'times', 'trivially_unsolvable' ] TIME_LIMIT = 1800 MEMORY_LIMIT = 2048 # Create a new experiment. exp = Experiment(environment=ENV) # Add custom parser for FF. exp.add_parser('ff-parser.py') for task in suites.build_suite(BENCHMARKS_DIR, SUITE): run = exp.add_run() # Create symbolic links and aliases. This is optional. We # could also use absolute paths in add_command(). run.add_resource('domain', task.domain_file, symlink=True) run.add_resource('problem', task.problem_file, symlink=True) # 'ff' binary has to be on the PATH. # We could also use exp.add_resource(). run.add_command('run-planner', ['ff', '-o', '{domain}', '-f', '{problem}'], time_limit=TIME_LIMIT, memory_limit=MEMORY_LIMIT) # AbsoluteReport needs the following properties: # 'domain', 'problem', 'algorithm', 'coverage'. run.set_property('domain', task.domain)
TIME_LIMIT = 5 ATTRIBUTES = [ "cost", "coverage", "error", "g_values_over_time", "run_dir", "runtime", ] exp = Experiment(environment=ENVIRONMENT) exp.add_step("build", exp.build) exp.add_step("start", exp.start_runs) exp.add_fetcher(name="fetch") exp.add_parser(os.path.join(DIR, "singularity-parser.py")) def get_image(name): planner = name.replace("-", "_") image = os.path.join(IMAGES_DIR, name + ".img") assert os.path.exists(image), image return planner, image IMAGES = [get_image("fd1906-lama-first")] for planner, image in IMAGES: exp.add_resource(planner, image, symlink=True) singularity_script = os.path.join(DIR, "run-singularity.sh")
] ATTRIBUTES = [ "coverage", "error", "evaluations", "plan", "times", "trivially_unsolvable", ] TIME_LIMIT = 1800 MEMORY_LIMIT = 2048 # Create a new experiment. exp = Experiment(environment=ENV) # Add custom parser for FF. exp.add_parser("ff-parser.py") for task in suites.build_suite(BENCHMARKS_DIR, SUITE): run = exp.add_run() # Create symbolic links and aliases. This is optional. We # could also use absolute paths in add_command(). run.add_resource("domain", task.domain_file, symlink=True) run.add_resource("problem", task.problem_file, symlink=True) # 'ff' binary has to be on the PATH. # We could also use exp.add_resource(). run.add_command( "run-planner", ["ff", "-o", "{domain}", "-f", "{problem}"], time_limit=TIME_LIMIT, memory_limit=MEMORY_LIMIT, )
scale="linear"), Attribute("num_static_atoms", absolute=True, min_wins=False, scale="linear"), Attribute("evaluate_time", absolute=True, min_wins=True, scale="linear"), ] MEMORY_LIMIT = (16 * 3000) * 0.98 GENERATOR_TIME_LIMIT = 2 * 3600 GENERATOR_FEATURE_LIMIT = 1000000 # Create a new experiment. exp = Experiment(environment=ENV) # Add custom parser for FF. exp.add_parser("experiment_parser.py") for task in suites.build_suite(BENCHMARKS_DIR, SUITE): for num_threads in [1, 2, 4, 8, 12, 16]: for complexity in [10]: run = exp.add_run() # Create symbolic links and aliases. This is optional. We # could also use absolute paths in add_command(). run.add_resource("domain", task.domain_file, symlink=True) run.add_resource("problem", task.problem_file, symlink=True) run.add_resource("main", "main.py", symlink=True) # 'ff' binary has to be on the PATH. # We could also use exp.add_resource(). run.add_command( f"complexity-{complexity}-{num_threads}", [
args.TIME_LIMIT = config['TIME_LIMIT'] # seconds args.MEMORY_LIMIT = config['MEMORY_LIMIT'] # seconds #xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx# #---- SETUP EXPERIMENT -------------------------------------------------------# # Setup local/remote environment if REMOTE: ENV = None #To be use later for HPC infrastructure else: # NOTE: if "processes = NUM_PROCESSES" is omitted, then default is #CPUs ENV = LocalEnvironment(processes=args.NUM_PROC) exp = Experiment(path=join(args.EXP_DIR, "results"), environment=ENV) exp.add_parser(join(CWD, Path(LOG_PARSER))) if (len(args.steps)>0 and ('1' in args.steps or 'build' in args.steps)) or \ args.run_all_steps : # Don't over-write instead create backups index = 0 while( isdir(args.EXP_DIR) and index!=1000): index += 1 try : rename(args.EXP_DIR, args.EXP_DIR+'_'+str(index)) except : pass makedirs(args.EXP_DIR) # Add step to write experiment files to disk exp.add_step('build', exp.build)