def benchmark_app(app):
# Add customized primitives from a local source.
add_primitives_path(MLBLOCKS_PRIMITIVES)
# STEP-1: Generate tasks from a set of pipelines to solve a list of problems.
tasks = create_tasks(dataset_name=app.params.dataset,
beginning_stage=app.params.stage,
optimize=app.params.optimize)
# STEP-2: Evaluate the tasks and summarize the results of each run.
results = benchmark(tasks, output_path=app.params.path_to_save)
# STEP-3.1: Gain a pipeline-level summary by aggregation.
aggregate_results_by_pipeline(results,
'F1 Macro',
output_path=os.path.join(
app.params.path_to_save, 'pipeline.csv'))
# STEP-3.2: Gain a problem-level summary by aggregation.
aggregate_results_by_problem(results,
'F1 Macro',
record_time=False,
output_path=os.path.join(
app.params.path_to_save, 'problem.csv'))
def _add_primitives_paths(paths):
if paths:
for path in paths:
add_primitives_path(path)
# directory, add these directories to sys.path here. If the directory is
# relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
import os
import sys
import sphinx_rtd_theme # For read the docs theme
from recommonmark.parser import CommonMarkParser
# from recommonmark.transform import AutoStructify
sys.path.insert(0, os.path.abspath('..'))
import mlblocks
mlblocks.add_primitives_path('../mlblocks_primitives')
# -- General configuration ---------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
'sphinx.ext.napoleon',
'sphinx.ext.githubpages',
'sphinx.ext.graphviz',
'IPython.sphinxext.ipython_console_highlighting',
'IPython.sphinxext.ipython_directive',