def add_absolute_report_step(self, **kwargs):
"""Add step that makes an absolute report.
Absolute reports are useful for experiments that don't
compare revisions.
The report is written to the experiment evaluation directory.
All *kwargs* will be passed to the AbsoluteReport class. If
the keyword argument *attributes* is not specified, a
default list of attributes is used. ::
exp.add_absolute_report_step(attributes=["coverage"])
"""
kwargs.setdefault("attributes", self.DEFAULT_TABLE_ATTRIBUTES)
report = AbsoluteReport(**kwargs)
# oufile is of the form <rev1>-<rev2>-...-<revn>.<format>
outfile = ''
for rev in self.revisions:
outfile += rev
outfile += '-'
outfile = outfile[:len(outfile) - 1]
outfile += '.'
outfile += report.output_format
self.add_report(report, outfile=outfile)
self.add_step(
Step('publish-absolute-report', subprocess.call,
['publish', outfile]))
def add_step(self, name, function, *args, **kwargs):
"""Add a step to the list of experiment steps.
Use this method to add experiment steps like writing the
experiment file to disk, removing directories and publishing
results. To add fetch and report steps, use the convenience
methods :meth:`.add_fetcher` and :meth:`.add_report`.
*name* is a descriptive name for the step. When selecting steps
on the command line, you may either use step names or their
indices.
*function* must be a callable Python object, e.g., a function
or a class implementing `__call__`.
*args* and *kwargs* will be passed to *function* when the step
is executed.
>>> import shutil
>>> import subprocess
>>> from lab.experiment import Experiment
>>> exp = Experiment('/tmp/myexp')
>>> exp.add_step('build', exp.build)
>>> exp.add_step('start', exp.start_runs)
>>> exp.add_step('rm-eval-dir', shutil.rmtree, exp.eval_dir)
>>> exp.add_step('greet', subprocess.call, ['echo', 'Hello'])
"""
if not isinstance(name, tools.string_type):
logging.critical('Step name must be a string: {}'.format(name))
if not name:
logging.critical('Step name must not be empty')
if any(step.name == name for step in self.steps):
raise ValueError("Step names must be unique: {}".format(name))
self.steps.append(Step(name, function, *args, **kwargs))
def add_scatter_plot_step(self, attributes=None):
if attributes is None:
attributes = self.DEFAULT_SCATTER_PLOT_ATTRIBUTES
revisions = self._HACK_revisions
if revisions is None:
# TODO: See add_comparison_table_step.
raise NotImplementedError(
"only supported when specifying revisions in __init__")
if len(revisions) != 2:
# TODO: Should generalize this, too, by offering a general
# grouping function and then comparing any pair of
# settings in the same group.
raise NotImplementedError("need two revisions")
scatter_dir = os.path.join(self.eval_dir, "scatter")
def make_scatter_plots():
configs = [conf[0] for conf in self.configs]
for nick in configs:
config_before = "%s-%s" % (revisions[0], nick)
config_after = "%s-%s" % (revisions[1], nick)
for attribute in attributes:
name = "%s-%s-%s" % (self._report_prefix, attribute, nick)
report = ScatterPlotReport(
filter_config=[config_before, config_after],
attributes=[attribute],
get_category=lambda run1, run2: run1["domain"],
legend_location=(1.3, 0.5))
report(self.eval_dir, os.path.join(scatter_dir, name))
self.add_step(Step("make-scatter-plots", make_scatter_plots))
def add_comparison_table_step(self, **kwargs):
"""Add a step that makes pairwise revision comparisons.
Create comparative reports for all pairs of Fast Downward
revision triples. Each report pairs up the runs of the same
config and lists the two absolute attribute values and their
difference for all attributes in kwargs["attributes"].
All *kwargs* will be passed to the CompareRevisionsReport
class. If the keyword argument *attributes* is not
specified, a default list of attributes is used. ::
exp.add_comparison_table_step(attributes=["coverage"])
"""
kwargs.setdefault("attributes", self.DEFAULT_TABLE_ATTRIBUTES)
def make_comparison_tables():
for rev1, rev2 in itertools.combinations(self.revision_nicks, 2):
report = CompareRevisionsReport(rev1, rev2, **kwargs)
outfile = os.path.join(self.eval_dir,
"%s-%s-compare.html" % (rev1, rev2))
report(self.eval_dir, outfile)
self.add_step(Step("make-comparison-tables", make_comparison_tables))
def add_scatter_plot_step(self, attributes=None, relative=False):
"""Add a step that creates scatter plots for all revision pairs.
Create a scatter plot for each combination of attribute,
configuration and revision pair. If *attributes* is not
specified, a list of common scatter plot attributes is used.
For portfolios all attributes except "cost", "coverage" and
"plan_length" will be ignored. ::
exp.add_scatter_plot_step(attributes=["expansions"])
Use `relative=True` to create relative scatter plots. ::
exp.add_scatter_plot_step(relative=True)
"""
if attributes is None:
attributes = self.DEFAULT_SCATTER_PLOT_ATTRIBUTES
if relative:
scatter_plot_class = RelativeScatterPlotReport
scatter_dir = os.path.join(self.eval_dir, "relative-scatter")
else:
scatter_plot_class = ScatterPlotReport
scatter_dir = os.path.join(self.eval_dir, "scatter")
def is_portfolio(config_nick):
return "fdss" in config_nick
def make_scatter_plot(config_nick, rev1, rev2, attribute):
name = "-".join([self.name, rev1, rev2, attribute, config_nick])
print "Make scatter plot for", name
algo1 = "%s-%s" % (rev1, config_nick)
algo2 = "%s-%s" % (rev2, config_nick)
report = scatter_plot_class(
filter_config=[algo1, algo2],
attributes=[attribute],
get_category=lambda run1, run2: run1["domain"],
legend_location=(1.3, 0.5))
report(self.eval_dir,
os.path.join(scatter_dir, rev1 + "-" + rev2, name))
def make_scatter_plots():
for config_nick in self._config_nicks:
if is_portfolio(config_nick):
valid_attributes = [
attr for attr in attributes
if attr in self.PORTFOLIO_ATTRIBUTES
]
else:
valid_attributes = attributes
for rev1, rev2 in itertools.combinations(
self.revision_nicks, 2):
for attribute in valid_attributes:
make_scatter_plot(config_nick, rev1, rev2, attribute)
self.add_step(Step("make-scatter-plots", make_scatter_plots))
def main():
args = parse_custom_args()
if not args.revision:
rev = 'WORK'
name = 'current'
elif args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = checkouts.get_global_rev(REPO, args.revision)
name = rev
combo = [(Translator(REPO, rev=rev), Preprocessor(REPO, rev=rev), Planner(REPO, rev=rev))]
exp = DownwardExperiment(path=get_exp_dir(name, args.test), repo=REPO, combinations=combo)
exp.add_suite(SUITES[args.test])
for nick, config in CONFIGS[args.test]:
exp.add_config(nick, config)
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
dirty_paths = [path for path in [exp.preprocess_exp_path, exp.path, exp.eval_dir]
if os.path.exists(path)]
if dirty_paths:
logging.critical(
'The last run found a regression. Please inspect what '
'went wrong and then delete the following directories '
'manually: %s' % dirty_paths)
exp.add_step(Step('fetch-baseline-results', Fetcher(),
get_exp_dir('baseline', args.test) + '-eval',
exp.eval_dir))
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES), name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
# We abort if there is a regression and keep the directories.
exp.add_step(Step('rm-preprocess-dir', shutil.rmtree, exp.preprocess_exp_path))
exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
exp.add_step(Step('rm-preprocessed-tasks', shutil.rmtree, exp.preprocessed_tasks_dir))
exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir))
exp()
def add_comparison_table_step(self, **kwargs):
"""Add a step that makes pairwise revision comparisons.
Create comparative reports for all pairs of Fast Downward
revisions. Each report pairs up the runs of the same config and
lists the two absolute attribute values and their difference
for all attributes in kwargs["attributes"].
All *kwargs* will be passed to the CompareConfigsReport class.
If the keyword argument *attributes* is not specified, a
default list of attributes is used. ::
exp.add_comparison_table_step(attributes=["coverage"])
"""
kwargs.setdefault("attributes", self.DEFAULT_TABLE_ATTRIBUTES)
def make_comparison_tables():
for rev1, rev2 in itertools.combinations(self._revisions, 2):
compared_configs = []
for config in self._configs:
config_nick = config.nick
compared_configs.append(
("%s-%s" % (rev1, config_nick),
"%s-%s" % (rev2, config_nick),
"Diff (%s)" % config_nick))
report = CompareConfigsReport(compared_configs, **kwargs)
outfile = os.path.join(
self.eval_dir,
"%s-%s-%s-compare.%s" % (
self.name, rev1, rev2, report.output_format))
report(self.eval_dir, outfile)
def publish_comparison_tables():
for rev1, rev2 in itertools.combinations(self._revisions, 2):
outfile = os.path.join(
self.eval_dir,
"%s-%s-%s-compare.html" % (self.name, rev1, rev2))
subprocess.call(["publish", outfile])
self.add_step(Step("make-comparison-tables", make_comparison_tables))
self.add_step(Step(
"publish-comparison-tables", publish_comparison_tables))
def __init__(self, path, environment=None, cache_dir=None):
"""
Create a new experiment that will be built at *path* using the methods
provided by :ref:`Environment <environments>` *environment*. If
*environment* is None, ``LocalEnvironment`` is used (default).
Lab will use the *cache_dir* for storing temporary files.
In case you run :py:class:`Fast Downward experiments
<downward.experiments.DownwardExperiment>` this directory can become
very large (tens of GB) since it is used to cache revisions and
preprocessed tasks. By default *cache_dir* points to ``~/lab``.
An experiment consists of multiple steps. Every experiment will need at
least the following steps:
* Build the experiment.
* Run it.
* Fetch the results.
* Make a report.
In the "Run it" step all runs that have been added to the experiment
will be executed. Each run consists of one or multiple commands.
"""
_Buildable.__init__(self)
self.path = os.path.abspath(path)
if any(char in self.path for char in (':', ',')):
logging.critical('Path contains commas or colons: %s' % self.path)
self.environment = environment or LocalEnvironment()
self.environment.exp = self
self.cache_dir = cache_dir or tools.DEFAULT_USER_DIR
tools.makedirs(self.cache_dir)
self.shard_size = SHARD_SIZE
self.runs = []
self.set_property('experiment_file', self._script)
self.steps = Sequence()
self.add_step(Step('build', self.build))
self.add_step(Step('start', self.run))
self.add_fetcher(name='fetch')
def main():
args = parse_custom_args()
if args.revision.lower() == 'baseline':
rev = BASELINE
name = 'baseline'
else:
rev = args.revision
name = rev
exp = FastDownwardExperiment(path=get_exp_dir(name, args.test),
cache_dir=CACHE_DIR)
exp.add_suite(BENCHMARKS_DIR, SUITES[args.test])
for config_nick, config in CONFIGS[args.test]:
exp.add_algorithm(rev + "-" + config_nick, REPO, rev, config)
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='report')
# Only compare results if we are not running the baseline experiment.
if rev != BASELINE:
dirty_paths = [
path for path in [exp.path, exp.eval_dir] if os.path.exists(path)
]
if dirty_paths:
logging.critical(
'The last run found a regression. Please inspect what '
'went wrong and then delete the following directories '
'manually: %s' % dirty_paths)
exp.add_fetcher(src=get_exp_dir('baseline', args.test) + '-eval',
dest=exp.eval_dir,
name='fetch-baseline-results')
exp.add_report(AbsoluteReport(attributes=ABSOLUTE_ATTRIBUTES),
name='comparison')
exp.add_report(RegressionCheckReport(BASELINE, RELATIVE_CHECKS),
name='regression-check')
# We abort if there is a regression and keep the directories.
exp.add_step(Step('rm-exp-dir', shutil.rmtree, exp.path))
exp.add_step(Step('rm-eval-dir', shutil.rmtree, exp.eval_dir))
exp()
def add_fetcher(self, src=None, dest=None, name=None, **kwargs):
"""
Add a step that fetches results from experiment or
evaluation directories into a new or existing evaluation
directory. Use this method to combine results from multiple
experiments.
*src* can be an experiment or evaluation directory. It defaults to
``exp.path``.
*dest* must be a new or existing evaluation directory. It
defaults to ``exp.eval_dir``. If *dest* already contains
data, the old and new data will be merged, not replaced.
If no *name* is given, call this step "fetch-``basename(src)``".
Valid keyword args:
If *copy_all* is True (default: False), copy all files from the run
dirs to a new directory tree at *dest*. Without this option only
the combined properties file is written do disk.
If *write_combined_props* is True (default), write the combined
properties file.
You can include only specific domains or configurations by using
:py:class:`filters <.Report>`.
*parsers* can be a list of paths to parser scripts. If given, each
parser is called in each run directory and the results are added to
the properties file which is fetched afterwards. This option is
useful if you forgot to parse some attributes during the experiment.
Examples:
Merge the results from "other-exp" into this experiment's results::
exp.add_fetcher(src='/path/to/other-exp-eval')
Merge two evaluation directories at the location of the second one::
exp.add_fetcher(src=eval_dir1, dest=combined_eval_dir, name='merge')
Fetch only the runs for certain configuration from an older experiment::
exp.add_fetcher(src='/path/to/eval-dir',
filter_config_nick=['config_1', 'config_5'])
"""
src = src or self.path
dest = dest or self.eval_dir
name = name or 'fetch-%s' % os.path.basename(src)
self.add_step(Step(name, Fetcher(), src, dest, **kwargs))
def add_scatter_plot_step(self, relative=False, attributes=None):
"""Add step creating (relative) scatter plots for all revision pairs.
Create a scatter plot for each combination of attribute,
configuration and revisions pair. If *attributes* is not
specified, a list of common scatter plot attributes is used.
For portfolios all attributes except "cost", "coverage" and
"plan_length" will be ignored. ::
exp.add_scatter_plot_step(attributes=["expansions"])
"""
if relative:
report_class = RelativeScatterPlotReport
scatter_dir = os.path.join(self.eval_dir, "scatter-relative")
step_name = "make-relative-scatter-plots"
else:
report_class = ScatterPlotReport
scatter_dir = os.path.join(self.eval_dir, "scatter-absolute")
step_name = "make-absolute-scatter-plots"
if attributes is None:
attributes = self.DEFAULT_SCATTER_PLOT_ATTRIBUTES
def make_scatter_plot(config_nick, rev1, rev2, attribute):
name = "-".join([self.name, rev1, rev2, attribute, config_nick])
print "Make scatter plot for", name
algo1 = "{}-{}".format(rev1, config_nick)
algo2 = "{}-{}".format(rev2, config_nick)
report = report_class(
filter_config=[algo1, algo2],
attributes=[attribute],
get_category=lambda run1, run2: run1["domain"],
legend_location=(1.3, 0.5))
report(
self.eval_dir,
os.path.join(scatter_dir, rev1 + "-" + rev2, name))
def make_scatter_plots():
for config in self._configs:
for rev1, rev2 in itertools.combinations(self._revisions, 2):
for attribute in self.get_supported_attributes(
config.nick, attributes):
make_scatter_plot(config.nick, rev1, rev2, attribute)
self.add_step(Step(step_name, make_scatter_plots))
def add_report(self, report, name='', eval_dir='', outfile=''):
"""Add *report* to the list of experiment steps.
This method is a shortcut for
``add_step(Step(name, report, eval_dir, outfile))``.
If no *name* is given, use *outfile* or the *report*'s class name.
By default, use the experiment's standard *eval_dir*.
If *outfile* is omitted, compose a filename from *name* and the
*report*'s format. If *outfile* is a relative path, put it under
*eval_dir*.
"""
name = name or os.path.basename(
outfile) or report.__class__.__name__.lower()
eval_dir = eval_dir or self.eval_dir
outfile = outfile or '%s.%s' % (name, report.output_format)
if not os.path.isabs(outfile):
outfile = os.path.join(eval_dir, outfile)
self.add_step(Step(name, report, eval_dir, outfile))
def add_custom_comparison_table_step(self, name, **kwargs):
"""Add a step that compares the configurations given in
*compared_configs*.
*compared_configs* must be specified. See CompareConfigsReport class.
*name* is a custom name for the report.
All *kwargs* will be passed to the CompareConfigsReport class.
If the keyword argument *attributes* is not specified, a
default list of attributes is used. ::
exp.add_comparison_table_step(attributes=["coverage"])
"""
kwargs.setdefault("attributes", self.DEFAULT_TABLE_ATTRIBUTES)
report = CompareConfigsReport(**kwargs)
outfile = os.path.join(self.eval_dir,
name + "." + report.output_format)
self.add_report(report, outfile=outfile)
self.add_step(
Step('publish-custom-comparison-report', subprocess.call,
['publish', outfile]))
def add_absolute_report_step(self, **kwargs):
"""Add step that makes an absolute report.
Absolute reports are useful for experiments that don't compare
revisions.
The report is written to the experiment evaluation directory.
All *kwargs* will be passed to the AbsoluteReport class. If the
keyword argument *attributes* is not specified, a default list
of attributes is used. ::
exp.add_absolute_report_step(attributes=["coverage"])
"""
kwargs.setdefault("attributes", self.DEFAULT_TABLE_ATTRIBUTES)
report = AbsoluteReport(**kwargs)
outfile = os.path.join(
self.eval_dir,
get_experiment_name() + "." + report.output_format)
self.add_report(report, outfile=outfile)
self.add_step(Step(
'publish-absolute-report', subprocess.call, ['publish', outfile]))
ENV = LocalEnvironment(processes=6)
CONFIGS = [('lmcut', ['--search', 'astar(lmcut())'])]
ATTRIBUTES = [
'coverage', 'expansions', 'initial_h_value', 'cost', 'hstar_to_h'
]
#All with timeout
#exp = DownwardExperiment(path=EXPPATH, repo=REPO, environment=ENV, limits={'search_time': 100})
#exp.add_suite(suites.suite_all())
#Only lmcut domains without timeout
exp = DownwardExperiment(path=EXPPATH, repo=REPO, environment=ENV)
exp.add_suite(suites.suite_lmcut_domains())
for nick, config in CONFIGS:
exp.add_config(nick, config)
# Make a report containing absolute numbers with h*/h values.
report = os.path.join(exp.eval_dir, 'report.html')
exp.add_report(HstarToHRatio(attributes=ATTRIBUTES), outfile=report)
# "Publish" the results with "cat" for demonstration purposes.
exp.add_step(Step('publish-report', subprocess.call, ['cat', report]))
# Compress the experiment directory.
exp.add_step(Step.zip_exp_dir(exp))
# Parse the commandline and show or run experiment steps.
exp()
environment=ENV,
limits={'search_time': 300})
exp.add_suite({'airport', 'blocks', 'freecell'})
for nick, config in CONFIGS:
exp.add_config(nick, config)
# Make a report containing absolute numbers (this is the most common report).
file_name_for_report = 'report_' + nick + '.html'
report = os.path.join(exp.eval_dir, file_name_for_report)
file_name_for_preprocess = os.path.join(exp.eval_dir, 'preprocess')
exp.add_report(HstarToHRatioAndStatistics(nick,
file_name_for_preprocess,
attributes=ATTRIBUTES),
outfile=report)
# Plot
sub_dir = 'plots_' + nick
exp.add_step(
Step('report-plot-cat', ProblemPlotReport(), exp.eval_dir,
os.path.join(exp.eval_dir, sub_dir)))
# "Publish" the results with "cat" for demonstration purposes.
exp.add_step(Step('publish-report', subprocess.call, ['cat', report]))
# Compress the experiment directory.
exp.add_step(Step.zip_exp_dir(exp))
# Parse the commandline and show or run experiment steps.
exp()
def __init__(self, path, repo, environment=None, combinations=None,
compact=True, limits=None, cache_dir=None):
"""
The experiment will be built at *path*.
*repo* must be the path to a Fast Downward repository. Among other things
this repository is used to search for benchmark files.
*environment* must be an :ref:`Environment <environments>` instance.
By default the experiment is run locally.
If given, *combinations* must be a list of :ref:`Checkout <checkouts>`
tuples of the form (Translator, Preprocessor, Planner). If combinations
is None (default), perform an experiment with the working copy in *repo*.
The *compact* parameter is only relevant for the search
stage. If *compact* is ``False``, the preprocessed task and
the two PDDL files are **copied** into the respective run
directories for all configurations. This requires a lot of
space (tens of GB), so it is strongly recommended to use the
default (``compact=True``) which only references these
files. Use ``compact=False`` only if you really need a
portable experiment.
If *limits* is given, it must be a dictionary that maps a
subset of the keys below to seconds and MiB. It will be used
to overwrite the default limits::
default_limits = {
'translate_time': 7200,
'translate_memory': 8192,
'preprocess_time': 7200,
'preprocess_memory': 8192,
'search_time': 1800,
'search_memory': 2048,
}
*cache_dir* is used to cache Fast Downward clones and preprocessed
tasks. By default it points to ``~/lab``.
.. note::
The directory *cache_dir* can grow very large (tens of GB).
Example: ::
repo = '/path/to/downward-repo'
env = GkiGridEnvironment(queue='xeon_core.q', priority=-2)
combos = [(Translator(repo, rev=123),
Preprocessor(repo, rev='e2a018c865f7'),
Planner(repo, rev='tip')]
exp = DownwardExperiment('/tmp/path', repo, environment=env,
combinations=combos,
limits={'search_time': 30,
'search_memory': 1024})
"""
Experiment.__init__(self, path, environment=environment, cache_dir=cache_dir)
#if not repo or not os.path.isdir(repo):
# logging.critical('The path "%s" is not a local Fast Downward '
# 'repository.' % repo)
self.repo = repo
self.orig_path = self.path
self.search_exp_path = self.path
self.preprocess_exp_path = self.path + '-p'
self._path_to_python = None
Checkout.REV_CACHE_DIR = os.path.join(self.cache_dir, 'revision-cache')
self.preprocessed_tasks_dir = os.path.join(self.cache_dir, 'preprocessed-tasks')
tools.makedirs(self.preprocessed_tasks_dir)
self.combinations = (combinations or
[(Translator(repo), Preprocessor(repo), Planner(repo))])
self.compact = compact
self.suites = defaultdict(list)
self._algorithms = []
self._portfolios = []
limits = limits or {}
for key, value in limits.items():
if key not in LIMITS:
logging.critical('Unknown limit: %s' % key)
self.limits = LIMITS
self.limits.update(limits)
# Save if this is a compact experiment i.e. preprocessed tasks are referenced.
self.set_property('compact', compact)
# TODO: Integrate this into the API.
self.include_preprocess_results_in_search_runs = True
self.compilation_options = ['-j%d' % self._jobs]
self._search_parsers = []
self.add_search_parser(os.path.join(DOWNWARD_SCRIPTS_DIR, 'search_parser.py'))
# Remove the default experiment steps
self.steps = Sequence()
self.add_step(Step('build-preprocess-exp', self.build, stage='preprocess'))
self.add_step(Step('run-preprocess-exp', self.run, stage='preprocess'))
self.add_fetcher(src=self.preprocess_exp_path,
dest=self.preprocessed_tasks_dir,
name='fetch-preprocess-results',
copy_all=True,
write_combined_props=False)
self.add_step(Step('build-search-exp', self.build, stage='search'))
self.add_PAC_fetcher(src='/home/sternron/gal-dreiman/downward/lab/examples/PAC_Preprocess_Output-eval/preprocess',#TODO change to be parameter
dest=self.search_exp_path,
name='fetch-preprocess-results',
copy_all=True,
write_combined_props=False)#new featcher to copy preprocess for PAC results
self.add_step(Step('run-search-exp', self.run, stage='search'))
self.add_fetcher(src=self.search_exp_path, name='fetch-search-results')
def __init__(self,
path=None,
repo=None,
environment=None,
combinations=None,
limits=None,
attributes=None,
derived_properties=None,
priority=0,
queue=None,
processes=2,
email=None,
cache_dir=CACHE_DIR,
**kwargs):
if path is None:
path = os.path.splitext(os.path.basename(sys.argv[0]))[0]
expname = os.path.basename(path)
remote_exppath = os.path.join(REMOTE_EXPS, path)
local_exppath = os.path.join(LOCAL_EXPS, path)
if REMOTE:
exppath = remote_exppath
repo = repo or REMOTE_REPO
environment = environment or MaiaEnvironment(
priority=priority, queue=queue, email=email)
else:
exppath = local_exppath
repo = repo or LOCAL_REPO
environment = environment or LocalEnvironment(processes=processes)
DownwardExperiment.__init__(self,
path=exppath,
environment=environment,
repo=repo,
combinations=combinations,
limits=limits,
cache_dir=cache_dir,
**kwargs)
self.set_path_to_python(PYTHON)
if attributes is None:
attributes = ATTRIBUTES
# Add report steps
abs_report_file = os.path.join(self.eval_dir, '%s-abs.html' % expname)
self.add_report(AbsoluteReport(attributes=attributes,
colored=True,
derived_properties=derived_properties),
name='report-abs',
outfile=abs_report_file)
if REMOTE:
# Compress the experiment directory
self.add_step(Step.zip_exp_dir(self))
self.add_step(
Step('zip-eval-dir',
call, [
'tar', '-cjf', self.name + '-eval.tar.bz2',
self.name + '-eval'
],
cwd=os.path.dirname(self.path)))
self.add_step(Step.remove_exp_dir(self))
self.add_step(
Step('remove-eval-dir',
shutil.rmtree,
self.eval_dir,
ignore_errors=True))
if not REMOTE:
# Copy the results to local directory
self.add_step(
Step('scp-eval-dir', call, [
'scp', '-r',
'%s:%s-eval' % (SCP_LOGIN, remote_exppath),
'%s-eval' % local_exppath
]))
# Copy the results to local directory
self.add_step(
Step('scp-zipped-eval-dir', call, [
'scp', '-r',
'%s:%s-eval.tar.bz2' % (SCP_LOGIN, remote_exppath),
'%s-eval.tar.bz2' % local_exppath
]))
# Copy the zipped experiment directory to local directory
self.add_step(
Step('scp-exp-dir', call, [
'scp', '-r',
'%s:%s.tar.bz2' % (SCP_LOGIN, remote_exppath),
'%s.tar.bz2' % local_exppath
]))
# Unzip the experiment directory
self.add_step(Step.unzip_exp_dir(self))
self.add_step(
Step('unzip-eval-dir',
call, ['tar', '-xjf', self.name + '-eval.tar.bz2'],
cwd=os.path.dirname(self.path)))
# exp.add_suite({'blocks:probBLOCKS-8-0.pddl','blocks:probBLOCKS-8-1.pddl','blocks:probBLOCKS-4-0.pddl','blocks:probBLOCKS-5-1.pddl'})
#exp.add_suite({'gripper:prob02.pddl'})
# exp.add_suite({'zenotravel','trucks-strips','sokoban-sat11-strips','philosophers'})
for nick, config in CONFIGS:
exp.add_config(nick, config)
# Make a report containing absolute numbers (this is the most common report).
file_name_for_report = 'report_' + nick + '.html'
report = os.path.join(exp.eval_dir, file_name_for_report)
file_name_for_preprocess = os.path.join(exp.eval_dir, 'preprocess')
exp.add_report(HstarToHRatioAndStatistics(nick,
file_name_for_preprocess,
attributes=ATTRIBUTES),
outfile=report)
# Plot
sub_dir = 'plots_' + nick
exp.add_step(
Step('report-plot-cat', ProblemPlotReport(), exp.eval_dir,
os.path.join(exp.eval_dir, sub_dir)))
# "Publish" the results with "cat" for demonstration purposes.
# exp.add_step(Step('publish-report', subprocess.call, ['cat', report]))
# Compress the experiment directory.
# exp.add_step(Step.zip_exp_dir(exp))
# Parse the commandline and show or run experiment steps.
exp()
class PiReport(Report):
def get_text(self):
lines = []
for run_id, run in self.props.items():
lines.append('%s %s' % (run['time'], run['diff']))
return '\n'.join(lines)
exp = Experiment(EXPPATH)
exp.add_resource('PARSER', 'pi-parser-ext.py', 'pi-parser.py')
exp.add_resource('CALC', 'calculate.py', 'calculate.py')
for rounds in [1, 5, 10, 50, 100, 500, 1000, 5000, 10000]:
run = exp.add_run()
run.require_resource('PARSER')
run.require_resource('CALC')
run.add_command('calc-pi', ['CALC', rounds], time_limit=10, mem_limit=1024)
run.add_command('parse-pi', ['PARSER'])
run.set_property('id', ['calc-%d' % rounds])
def good(run):
return run['diff'] <= 0.01
exp.add_step(Step('report', Report(format='html', attributes=['pi', 'diff'],
filter=good), exp.eval_dir,
os.path.join(exp.eval_dir, 'report.html')))
exp.add_step(Step('plot', PiReport(),
exp.eval_dir, os.path.join(exp.eval_dir, 'plot.dat')))
exp()
exp.add_config('iter-hadd', [
'--heuristic', 'hadd=add()', '--search',
'iterated([lazy_greedy([hadd]),lazy_wastar([hadd])],repeat_last=true)'
])
exp.add_config('ipdb', ["--search", "astar(ipdb())"], timeout=10)
# Use original LAMA 2011 configuration
exp.add_config('lama11',
['ipc', 'seq-sat-lama-2011', '--plan-file', 'sas_plan'])
exp.add_config('fdss-1', ['ipc', 'seq-sat-fdss-1', '--plan-file', 'sas_plan'])
exp.add_portfolio(
os.path.join(REPO, 'src', 'search', 'downward-seq-opt-fdss-1.py'))
# Before we fetch the new results, delete the old ones
exp.steps.insert(
5,
Step('delete-old-results', shutil.rmtree, exp.eval_dir,
ignore_errors=True))
# Before we build the experiment, delete the old experiment directory
# and the preprocess directory
exp.steps.insert(
0, Step('delete-exp-dir', shutil.rmtree, exp.path, ignore_errors=True))
exp.steps.insert(
0,
Step('delete-preprocess-dir',
shutil.rmtree,
exp.preprocess_exp_path,
ignore_errors=True))
# Define some filters
exp.add_suite(['gripper:prob01.pddl'])
exp.add_suite('zenotravel:pfile2')
exp.add_config('ff', ['--search', 'lazy(single(ff()))'])
exp.add_config('add', ['--search', 'lazy(single(add()))'])
exp.add_portfolio(
os.path.join(REPO, 'src', 'search', 'downward-seq-sat-fdss-1.py'))
exp.add_report(AbsoluteReport('problem'),
name='make-report',
outfile='report-abs-p.html')
def solved(run):
return run['coverage'] == 1
exp.add_step(
Step('suite', SuiteReport(filter=solved), exp.eval_dir,
os.path.join(exp.eval_dir, 'suite.py')))
exp.add_step(
Step(
'scatter',
ScatterPlotReport(filter_config_nick=['ff', 'add'],
attributes='expansions',
format='png'), exp.eval_dir,
os.path.join(exp.eval_dir, 'scatter')))
exp()
def add_PAC_fetcher(self, src=None, dest=None, name=None, **kwargs):
src = src or self.path
dest = dest or self.eval_dir
name = name or 'fetch-%s' % os.path.basename(src)
self.add_step(Step(name, PACFetcher(), src, dest, **kwargs))
