def setup():
cwd = os.getcwd()
tools.makedirs(main_dir)
os.chdir(main_dir)
for opt in optimizations:
tools.makedirs(opt)
os.chdir(opt)
if not os.path.exists('planner.cc'):
cmd = ('svn co %s .' % PLANNER_URL).split()
ret = subprocess.call(cmd)
makefile_path = 'Makefile'
makefile = open(makefile_path).read()
planner_name = 'downward-' + opt
new_make = makefile.replace('-O3', '-' + opt)
new_make = new_make.replace('TARGET = downward',
'TARGET = ' + planner_name)
makefile_name = 'Makefile' # + '-' + opt
with open(makefile_name, 'w') as file:
file.write(new_make)
if not os.path.exists(planner_name):
# Needs compiling
subprocess.call([
'make',
]) # '-f', makefile_name])
os.chdir('../')
os.chdir(cwd)
def fetch(self):
total_dirs = len(self.run_dirs)
for index, run_dir in enumerate(self.run_dirs, 1):
prop_file = os.path.join(run_dir, 'properties')
props = tools.Properties(prop_file)
id = props.get('id')
dest_dir = os.path.join(self.eval_dir, *id)
tools.makedirs(dest_dir)
if self.copy_all:
tools.fast_updatetree(run_dir, dest_dir)
props['run_dir'] = run_dir
for filename, file_parser in self.file_parsers.items():
filename = os.path.join(run_dir, filename)
file_parser.load_file(filename)
props.update(file_parser.parse())
# Write new properties file
props.filename = os.path.join(dest_dir, 'properties')
props.write()
logging.info('Done Evaluating: %6d/%d' % (index, total_dirs))
def write_to_disk(self, content):
if self.dry or not content:
return
filename = self.get_filename()
tools.makedirs(os.path.dirname(filename))
with open(filename, 'w') as file:
file.write(content)
logging.info('Wrote file://%s' % filename)
def export_to_dir(self, outdir):
import shutil
sub_outdir = outdir if self.flat else os.path.join(outdir, self.dirname)
for track in self.tracks:
src = track.getFilePath()
if not os.path.exists(src):
logging.info('Skipping non-existent file %s.' % src)
continue
dest = os.path.join(sub_outdir, os.path.basename(src))
logging.info('Copying %s to %s' % (src, dest))
tools.makedirs(sub_outdir)
shutil.copy2(src, dest)
for subdir in self.subdirs:
subdir.export_to_dir(sub_outdir)
if self.flat:
logging.info('Export finished')
def export_to_dir(self, outdir):
import shutil
sub_outdir = outdir if self.flat else os.path.join(
outdir, self.dirname)
for track in self.tracks:
src = track.getFilePath()
if not os.path.exists(src):
logging.info('Skipping non-existent file %s.' % src)
continue
dest = os.path.join(sub_outdir, os.path.basename(src))
logging.info('Copying %s to %s' % (src, dest))
tools.makedirs(sub_outdir)
shutil.copy2(src, dest)
for subdir in self.subdirs:
subdir.export_to_dir(sub_outdir)
if self.flat:
logging.info('Export finished')
def fetch_dir(self, run_dir):
prop_file = os.path.join(run_dir, 'properties')
props = tools.Properties(prop_file)
id = props.get('id')
# Abort if an id cannot be read.
if not id:
logging.error('id is not set in %s.' % prop_file)
sys.exit(1)
dest_dir = os.path.join(self.eval_dir, *id)
if self.copy_all:
tools.makedirs(dest_dir)
tools.fast_updatetree(run_dir, dest_dir)
props['run_dir'] = os.path.relpath(run_dir, self.exp_dir)
for filename, file_parser in self.file_parsers.items():
# If filename is absolute it will not be changed here
path = os.path.join(run_dir, filename)
success = file_parser.load_file(path)
if success:
# Subclasses directly modify the properties during parsing
file_parser.parse(props)
else:
logging.error('File "%s" could not be read' % path)
if self.copy_all:
# Write new properties file
props.filename = os.path.join(dest_dir, 'properties')
props.write()
if self.check:
try:
self.check(props)
except AssertionError, e:
msg = 'Parsed properties not valid in %s: %s'
logging.error(msg % (prop_file, e))
print '*' * 60
props.write(sys.stdout)
print '*' * 60
def fetch(self):
total_dirs = self.exp_props.get('runs')
if not self.no_props_file:
combined_props_filename = os.path.join(self.eval_dir, 'properties')
combined_props = tools.Properties(combined_props_filename)
# Get all run_dirs
run_dirs = sorted(glob(os.path.join(self.exp_dir, 'runs-*-*', '*')))
for index, run_dir in enumerate(run_dirs, 1):
logging.info('Evaluating: %6d/%d' % (index, total_dirs))
id_string, props = self.fetch_dir(run_dir)
if not self.no_props_file:
props['id-string'] = id_string
combined_props[id_string] = props.dict()
tools.makedirs(self.eval_dir)
if not self.no_props_file:
self.apply_postprocess_functions(combined_props)
combined_props.write()
self.write_data_dump(combined_props)
def copyPdf(doclist,outdir,verbose=True):
'''Copy PDF to target location
<doclist>: list of meta data dicts
<outdir>: str, path to output folder
'''
if not os.path.isdir(outdir):
makedirs(outdir)
faillist=[]
num=len(doclist)
for ii,docii in enumerate(doclist):
pathii=docii['path']
if pathii is None:
continue
for jj,pjj in enumerate(pathii):
basedir,filename=os.path.split(pjj)
targetname=os.path.join(outdir,filename)
if not os.path.exists(pjj):
faillist.append(pjj)
continue
if verbose:
printNumHeader('Copying file:',ii+1,num,3)
printInd(filename,4)
try:
shutil.copy2(pjj,targetname)
except:
faillist.append(filename)
return faillist
def copyPdf(doclist,outdir,verbose=True):
'''Copy PDF to target location
<doclist>: list of meta data dicts
<outdir>: str, path to output folder
'''
if not os.path.isdir(outdir):
makedirs(outdir)
faillist=[]
num=len(doclist)
for ii,docii in enumerate(doclist):
pathii=docii['path']
if pathii is None:
continue
for jj,pjj in enumerate(pathii):
basedir,filename=os.path.split(pjj)
targetname=os.path.join(outdir,filename)
if not os.path.exists(pjj):
faillist.append(pjj)
continue
if verbose:
printNumHeader('Copying file:',ii+1,num,3)
printInd(filename,4)
try:
shutil.copy2(pjj,targetname)
except:
faillist.append(filename)
return faillist
def get_filename(self):
if self.outfile:
return os.path.abspath(self.outfile)
ext = self.extension or self.output_format.replace('xhtml', 'html')
tools.makedirs(tools.REPORTS_DIR)
return os.path.join(tools.REPORTS_DIR, '%s.%s' % (self.get_name(), ext))
"""
import os
import sys
import logging
import shlex
import experiments
import environments
import checkouts
import downward_suites
import downward_configs
import tools
PREPROCESSED_TASKS_DIR = os.path.join(tools.SCRIPTS_DIR, "preprocessed-tasks")
tools.makedirs(PREPROCESSED_TASKS_DIR)
LIMIT_TRANSLATE_TIME = 7200
LIMIT_TRANSLATE_MEMORY = 8192
LIMIT_PREPROCESS_TIME = 7200
LIMIT_PREPROCESS_MEMORY = 8192
LIMIT_SEARCH_TIME = 1800
LIMIT_SEARCH_MEMORY = 2048
# At least one of those must be found (First is taken if many are present)
PLANNER_BINARIES = ["downward", "downward-debug", "downward-profile", "release-search", "search"]
# The following are added only if they are present
PLANNER_HELPERS = [
"downward-1",
"downward-2",
import os
import sys
import logging
import re
import itertools
import tools
from tools import run_command, get_command_output
CHECKOUTS_DIR = os.path.join(tools.SCRIPTS_DIR, 'checkouts')
tools.makedirs(CHECKOUTS_DIR)
ABS_REV_CACHE = {}
class Checkout(object):
def __init__(self, part, repo, rev, checkout_dir):
# Directory name of the planner part (translate, preprocess, search)
self.part = part
self.repo = repo
self.rev = str(rev)
if not os.path.isabs(checkout_dir):
checkout_dir = os.path.join(CHECKOUTS_DIR, checkout_dir)
self.checkout_dir = os.path.abspath(checkout_dir)
def __lt__(self, other):
return self.name < other.name
def __eq__(self, other):
return self.name == other.name