def test_experiment_configuration(self):
# Activate tracking of pypads
from pypads.app.base import PyPads
tracker = PyPads()
tracker.start_track(experiment_name="ConfiguredExperiment")
from sklearn import datasets, metrics
from sklearn.tree import DecisionTreeClassifier
# load the iris datasets
dataset = datasets.load_iris()
# fit a model to the data
model = DecisionTreeClassifier()
model.fit(dataset.data, dataset.target)
# make predictions
expected = dataset.target
predicted = model.predict(dataset.data)
# summarize the fit of the model
print(metrics.classification_report(expected, predicted))
# print(metrics.confusion_matrix(expected, predicted))
# assert statements
# assert tracker._experiment.regex == "ConfiguredExperiment"
# TODO add asserts
tracker.api.end_run()
def test_pipeline(self):
"""
This example will track the experiment exection with the default configuration.
:return:
"""
# --------------------------- setup of the tracking ---------------------------
# Activate tracking of pypads
set_up_fns = {}
from pypads.app.base import PyPads
tracker = PyPads(setup_fns=set_up_fns, log_level="DEBUG")
tracker.start_track(experiment_name="1. Experiment")
import timeit
t = timeit.Timer(sklearn_pipeline_experiment)
print(t.timeit(1))
# --------------------------- asserts ---------------------------
run = mlflow.active_run()
assert tracker.api.active_run().info.run_id == run.info.run_id
# artifacts = [x for x in tracker.results.get_artifacts(run_id=run.info.run_id)]
# assert len(artifacts) > 0
tracker.api.end_run()
def test_punch_before_import(self):
from pypads.app.base import PyPads
from test_classes.dummy_mapping import _get_punch_dummy_mapping
tracker = PyPads(uri=TEST_FOLDER, mappings=_get_punch_dummy_mapping())
tracker.activate_tracking(reload_modules=True)
tracker.start_track()
from test_classes.dummy_classes import PunchDummy
from test_classes.dummy_classes import PunchDummy2
dummy2 = PunchDummy2(2)
assert hasattr(PunchDummy, "_pypads_mapping_PunchDummy")
assert hasattr(PunchDummy2, "_pypads_mapping_PunchDummy2")
assert hasattr(dummy2, "_pypads_mapping_PunchDummy2")
def test_punch_after_import_clear_imports(self):
from test_classes.dummy_classes import PunchDummy
from test_classes.dummy_classes import PunchDummy2
dummy2 = PunchDummy2(2)
from pypads.app.base import PyPads
from test_classes.dummy_mapping import _get_punch_dummy_mapping
# TODO Punching of globals?
tracker = PyPads(uri=TEST_FOLDER, mappings=_get_punch_dummy_mapping())
tracker.activate_tracking(clear_imports=True, reload_modules=False)
tracker.start_track()
from test_classes.dummy_classes import PunchDummy as c
from test_classes.dummy_classes import PunchDummy2 as d
assert hasattr(c, "_pypads_mapping_PunchDummy")
assert hasattr(d, "_pypads_mapping_PunchDummy2")
assert not hasattr(PunchDummy, "_pypads_mapping_PunchDummy")
assert not hasattr(PunchDummy2, "_pypads_mapping_PunchDummy2")
assert not hasattr(dummy2, "_pypads_mapping_PunchDummy2")
def test_punch_after_import(self):
from test_classes.dummy_classes import PunchDummy
from test_classes.dummy_classes import PunchDummy2
dummy2 = PunchDummy2(2)
from pypads.app.base import PyPads
from test_classes.dummy_mapping import _get_punch_dummy_mapping
# TODO PunchDummy2 has PunchDummy as reference
tracker = PyPads(uri=TEST_FOLDER, mappings=_get_punch_dummy_mapping())
tracker.activate_tracking(reload_modules=True)
tracker.start_track()
from test_classes.dummy_classes import PunchDummy as a
from test_classes.dummy_classes import PunchDummy2 as b
assert not hasattr(a, "_pypads_mapping_PunchDummy")
assert not hasattr(b, "_pypads_mapping_PunchDummy2")
assert not hasattr(PunchDummy, "_pypads_mapping_PunchDummy")
assert not hasattr(PunchDummy2, "_pypads_mapping_PunchDummy2")
assert not hasattr(dummy2, "_pypads_mapping_PunchDummy2")
def test_track_metric(self):
# --------------------------- setup of the tracking ---------------------------
# Activate tracking of pypads
from pypads.app.base import PyPads
name = "some_metric"
description = 'Some description'
value = 1
step = 0
keys = [
'experiment_id', 'run_id', 'category', 'storage_type',
'description', 'name', 'data'
]
tracker = PyPads(uri=TEST_FOLDER)
tracker.activate_tracking()
tracker.start_track(experiment_name='TEST CASE EXPERIMENT')
# meta = MetricMetaModel(url='https://some.metric.url', name='some_metric', description='some description',
# step=0)
tracker.api.log_metric(name,
value=value,
description=description,
step=step)
holder = tracker.api.get_programmatic_output()
meta = MetricMetaModel(name=name,
value_format='str',
data=str(value),
step=step,
description=description,
parent=holder,
produced_by=holder.produced_by,
part_of=get_reference(holder))
artifacts = [
x for x in tracker.results.get_metrics(
experiment_name='TEST CASE EXPERIMENT',
name=name,
step=step,
run_id=meta.run.uid)
]
# --------------------------- asserts ---------------------------
assert len(artifacts) == 1
for key in keys:
assert artifacts[0].dict().get(key) == meta.dict().get(key)
def test_default_tracking(self):
"""
This example will track the experiment exection with the default configuration.
:return:
"""
# --------------------------- setup of the tracking ---------------------------
# Activate tracking of pypads
from pypads.app.base import PyPads
tracker = PyPads(autostart=False, log_level="WARNING")
tracker.start_track(experiment_name="1. Experiment")
tracker.actuators.set_random_seed(seed=1)
import timeit
t = timeit.Timer(sklearn_simple_decision_tree_experiment)
from pypads import logger
logger.info(t.timeit(1))
# --------------------------- asserts ---------------------------
import mlflow
run = mlflow.active_run()
assert tracker.api.active_run().info.run_id == run.info.run_id
tracker.api.end_run()
def test_default_tracking(self):
"""
This example will track the experiment exection with the default configuration.
:return:
"""
# --------------------------- setup of the tracking ---------------------------
# Activate tracking of pypads
from pypads.app.base import PyPads
tracker = PyPads(uri="http://mlflow.padre-lab.eu")
tracker.activate_tracking()
tracker.start_track()
from pypads import logger
logger.info("Test logger")
import timeit
t = timeit.Timer(sklearn_simple_decision_tree_experiment)
from pypads import logger
logger.info(t.timeit(1))
# --------------------------- asserts ---------------------------
tracker.api.end_run()
from pypads.app.base import PyPads
from pypads.injections.setup.hardware import ICpuRSF
tracker = PyPads(setup_fns=[ICpuRSF()])
tracker.start_track(experiment_name="Pypads-padre")
import numpy as np
from sklearn import datasets
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import f1_score, make_scorer
# Define a pipeline to search for the best combination of PCA truncation
# and classifier regularization.
pca = PCA()
# set the tolerance to a large value to make the example faster
logistic = LogisticRegression(solver='liblinear',
max_iter=10000,
tol=0.1,
multi_class="ovr")
pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)])
X_digits, y_digits = datasets.load_digits(return_X_y=True)
# Parameters of pipelines can be set using ‘__’ separated parameter names:
param_grid = {
'pca__n_components': [5, 15, 30, 45, 64],
'logistic__C': np.logspace(-4, 4, 4),
}
search = GridSearchCV(pipe,
def wrapped_function(*args,
_pypads_cache=None,
_pypads_config=None,
_pypads_active_run_id=None,
_pypads_tracking_uri=None,
_pypads_affected_modules=None,
_pypads_triggering_process=None,
**kwargs):
from pypads.parallel.util import _pickle_tuple, _cloudpickle_tuple
from pypads import logger
# only if pads data was passed
if _pypads_active_run_id:
# noinspection PyUnresolvedReferences
from pypads.app import pypads
import mlflow
is_new_process = not pypads.current_pads
# If pads has to be reinitialized
if is_new_process:
import pypads
# reactivate this run in the foreign process
mlflow.set_tracking_uri(_pypads_tracking_uri)
mlflow.start_run(run_id=_pypads_active_run_id, nested=True)
start_time = time.time()
logger.debug("Init Pypads in:" +
str(time.time() - start_time))
# TODO update to new format
from pypads.app.base import PyPads
_pypads = PyPads(uri=_pypads_tracking_uri,
config=_pypads_config,
pre_initialized_cache=_pypads_cache)
_pypads.activate_tracking(
reload_warnings=False,
affected_modules=_pypads_affected_modules,
clear_imports=True,
reload_modules=True,
)
_pypads.start_track(disable_run_init=True)
def clear_mlflow():
"""
Don't close run. This function clears the run which was reactivated from the stack to stop a closing of it.
:return:
"""
if len(mlflow.tracking.fluent._active_run_stack) == 1:
mlflow.tracking.fluent._active_run_stack.pop()
import atexit
atexit.register(clear_mlflow)
# If pads already exists on process
else:
_pypads = pypads.current_pads
_pypads.cache.merge(_pypads_cache)
# Unpickle args
from pickle import loads
start_time = time.time()
a, b = loads(args[0])
logger.debug("Loading args from pickle in:" +
str(time.time() - start_time))
# Unpickle function
from cloudpickle import loads as c_loads
start_time = time.time()
wrapped_fn = c_loads(args[1])[0]
logger.debug("Loading punched function from pickle in:" +
str(time.time() - start_time))
args = a
kwargs = b
logger.debug("Started wrapped function on process: " +
str(os.getpid()))
out = wrapped_fn(*args, **kwargs)
return out, _pypads.cache
else:
return fn(*args, **kwargs)