def _store_results(self, output, logger_call):
if output:
logger_call.output = output.store()
logger_call.finish()
logger_call.store()
from pypads.app.pypads import get_current_pads
get_current_pads().cache.run_remove(env_cache(output))
def __new__(cls, *args, **kwargs):
try:
from pypads.app.pypads import get_current_pads
if get_current_pads() is not None:
logger.warning(
"Currently only one tracker can be activated at once."
"PyPads was already intialized. Reusing the old instance.")
return get_current_pads()
except Exception as e:
pass
return super().__new__(cls)
def make(uri) -> MLFlowBackend:
from pypads.app.pypads import get_current_pads, get_current_config
if uri.startswith("git://") or uri.startswith("/"):
if get_current_config()[mongo_db]:
return MongoSupportedLocalMlFlowBackend(
uri=uri, pypads=get_current_pads())
else:
return LocalMlFlowBackend(uri=uri, pypads=get_current_pads())
else:
if get_current_config()[mongo_db]:
return MongoSupportedRemoteMlFlowBackend(
uri=uri, pypads=get_current_pads())
else:
return RemoteMlFlowBackend(uri=uri, pypads=get_current_pads())
def _convert_context(cls, ctx):
"""
TODO is this still needed in this form?
:param ctx:
:return:
"""
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
if isinstance(ctx, List):
context = []
for c in ctx:
if isinstance(c, HttpUrl) or isinstance(
c, dict) or (isinstance(c, str) and os.path.isfile(c)):
context.append(c)
else:
splits = c.split(os.sep + "artifacts" + os.sep)
run_id = splits[0].split(os.sep)[-1]
context.append(
pads.backend.download_tmp_artifacts(
run_id, os.sep.join(splits[1:])))
ctx = context
else:
if not (isinstance(ctx, HttpUrl) or isinstance(ctx, dict) or
(isinstance(ctx, str) and os.path.isfile(ctx))):
splits = ctx.split(os.sep + "artifacts" + os.sep)
run_id = splits[0].split(os.sep)[-1]
ctx = pads.backend.download_tmp_artifacts(
run_id, os.sep.join(splits[1:]))
return ctx
def test_api_layered_tracking(self):
"""
This example will track the experiment execution with the default configuration.
:return:
"""
# --------------------------- setup of the tracking ---------------------------
# Activate tracking of pypads
from pypads.app.base import PyPads
tracker = PyPads(uri=TEST_FOLDER, config=config, hooks=hooks, events=events, autostart=True)
global experiment
experiment = tracker.api.track(experiment, anchors=["ran"])
global sub_experiment
sub_experiment = tracker.api.track(sub_experiment, anchors=["ran"])
global more_experiment
more_experiment = tracker.api.track(more_experiment, anchors=["ran"])
import timeit
t = timeit.Timer(experiment)
print(t.timeit(1))
# --------------------------- asserts ---------------------------
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
assert pads.cache.run_exists(id(logger))
assert pads.cache.run_get(id(logger)) == 16
# !-------------------------- asserts ---------------------------
def test_retry(self):
"""
This example will track a failure and only work on the second run.
:return:
"""
# --------------------------- setup of the tracking ---------------------------
# Activate tracking of pypads
from pypads.app.base import PyPads
tracker = PyPads(uri=TEST_FOLDER, config=config, hooks=hooks, events=events, setup_fns={}, autostart=True)
i = 0
def experiment():
print("I'm an function level experiment")
nonlocal i
if i == 0:
i = i + 1
raise Exception("Planed failure")
else:
return "I'm a retried return value."
experiment = tracker.api.track(experiment, anchors=["pypads_log"])
import timeit
t = timeit.Timer(experiment)
print(t.timeit(1))
# --------------------------- asserts ---------------------------
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
assert pads.cache.run_exists(id(logger))
self.assertEqual(i, 1)
def test_decorator_passed_event(self):
"""
This example will track the experiment exection with passed event.
:return:
"""
# --------------------------- setup of the tracking ---------------------------
# Activate tracking of pypads
from pypads.app.base import PyPads
tracker = PyPads(uri=TEST_FOLDER,
config=config,
hooks=hooks,
events=events,
autostart=True)
@tracker.decorators.track(event="pypads_log")
def experiment():
print("I'm an function level experiment")
return "I'm a return value."
import timeit
t = timeit.Timer(experiment)
print(t.timeit(1))
# --------------------------- asserts ---------------------------
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
assert pads.cache.run_exists(id(logger))
def _store_results(self, output, logger_call):
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
pads.cache.run_add(id(self), {
'id': id(self),
'logger_call': logger_call,
'output': output
})
def finalize(pads, *args, **kwargs):
data = pads.cache.run_get(id(self))
logger_call = data.get('logger_call')
output = data.get('output')
self.finalize_output(pads,
*args,
logger_call=logger_call,
output=output,
**kwargs)
logger_call.finish()
logger_call.store()
# Clean up saved env data for output
pads.cache.run_remove(env_cache(output))
pads.api.register_teardown_utility(
'{}_clean_up'.format(self.__class__.__name__),
finalize,
error_message="Couldn't finalize output of logger {},"
"because of exception: {} \nTrace:\n{}")
def __pre__(ctx, *args, _logger_call: InjectionLoggerEnv, **kwargs):
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
print("third")
if not pads.cache.run_exists(1):
raise ValueError("Not called as third")
pads.cache.run_add(2, True)
def test_order_lf(self):
"""
This example will track the experiment exection with the default configuration.
:return:
"""
# --------------------------- setup of the tracking ---------------------------
# Activate tracking of pypads
from pypads.app.base import PyPads
tracker = PyPads(uri=TEST_FOLDER,
config=config,
hooks=hooks,
events=events,
autostart=True)
tracker.api.track(experiment,
anchors=["order"],
ctx=sys.modules[__name__])
import timeit
t = timeit.Timer(experiment)
print(t.timeit(1))
# --------------------------- asserts ---------------------------
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
assert pads.cache.run_exists(0, 1, 2)
def _add_inherited_mapping(clazz, super_class):
from pypads.app.pypads import get_current_pads
found_mappings = set()
if clazz.__name__ not in get_current_pads(
).wrap_manager.class_wrapper.punched_class_names:
if hasattr(super_class, "_pypads_mapping_" + super_class.__name__):
for matched_mapping in getattr(
super_class, "_pypads_mapping_" + super_class.__name__):
"""
Build the package path matcher by looking at the superclass matched_mappings and deconstructing the
package_path of it. Taking from the matcher unmatched parts and adding them to the module and qualname.
"""
found_mapping = Mapping(PackagePathMatcher(".".join(
filter(lambda s: len(s) > 0, [
clazz.__module__, clazz.__qualname__, ".".join([
h.serialize() for h in matched_mapping.mapping.
matcher.matchers[len(matched_mapping.package_path.
segments):]
])
]))),
matched_mapping.mapping.in_collection,
{
h.anchor
for h in
matched_mapping.mapping.hooks
},
matched_mapping.mapping.values,
inherited=matched_mapping.mapping)
found_mappings.add(found_mapping)
return found_mappings
def check_determinism():
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
if is_package_available('tensorflow'):
import tensorflow
tf_version = tensorflow.version.VERSION
if tensorflow.match("(1\.(14|15)|2\.0)", tf_version):
if "TF_USE_CUDNN_AUTOTUNE" in os.environ:
logger.warning(
"When using TF auto-tuning of cuDNN convolution algorithms your experiment might"
" be non-deterministic.")
pads.api.set_tag("non-determinism", "CUDNN_AUTOTUNE")
if ("TF_CUDNN_DETERMINISTIC" not in os.environ
or (not os.environ["TF_CUDNN_DETERMINISTIC"]
and os.environ["TF_CUDNN_DETERMINISTIC"] is not 1)):
if not is_package_available("tfdeterminism"):
logger.warning(
"Your experiment might include a gpu-specific sources of non-determinism."
" See https://github.com/NVIDIA/tensorflow-determinism"
)
pads.api.set_tag(
"non-determinism",
"TF auto-tuning of cuDNN convolution algorithms (see multi-algo note)"
)
def __call_wrapped__(self, ctx, *args, _pypads_env: InjectionLoggerEnv,
_args, _kwargs, **_pypads_hook_params):
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
def tracer(frame, event, arg):
if event == 'return':
params = frame.f_locals.copy()
key = str(self)
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
pads.cache.run_add(key, params)
fn = _pypads_env.callback
if _pypads_env.call.call_id.is_wrapped():
fn = _pypads_env.callback.__wrapped__
try:
# tracer is activated on next call, return or exception
sys.setprofile(tracer)
fn(*_args, **_kwargs)
finally:
# deactivate tracer
sys.setprofile(None)
_return = super().__call_wrapped__(ctx,
_pypads_env=_pypads_env,
_args=_args,
_kwargs=_kwargs,
**_pypads_hook_params)
return _return
def tracer(frame, event, arg):
if event == 'return':
params = frame.f_locals.copy()
key = str(self)
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
pads.cache.run_add(key, params)
def tag_data(self):
if ResultType.tag not in self._results:
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
self._results[ResultType.tag] = [
pads.backend.get(**unwrap_typed_id(o)) for o in self.tags
]
return self._results[ResultType.tag]
def load(self):
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
if self.backend_uri is not pads.backend.uri:
# TODO init backend if possible?
logger.error("Can't load object due to unavailable backend.")
return None
return pads.backend.get(self.uid, self.storage_type)
def parameter_data(self):
if ResultType.parameter not in self._results:
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
self._results[ResultType.parameter] = [
pads.backend.get(**unwrap_typed_id(o)) for o in self.parameters
]
return self._results[ResultType.parameter]
def get_backend_uri():
try:
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
if pads:
return pads.uri
except (ImportError, UninitializedTrackerException):
pass # PyPads is not available here the backend uri is not set. Backend_uri has to be provided later on
return None
def _get_relevant_mappings(package: Package):
from pypads.app.pypads import get_current_pads
try:
return get_current_pads().mapping_registry.get_relevant_mappings(
package)
except Exception as e:
logger.debug('Getting on-import mappings failed due to : {}'.format(
str(e)))
return set()
def torch_cuda_seed(seed):
try:
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
pads.cache.run_add("torch.cuda.seed", seed)
return original_torch_cuda(seed)
except Exception as e:
Warning("Tracker failed to log the set seed because %s" %
str(e))
return original_torch_cuda(seed)
def numpy_seed(seed):
try:
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
pads.cache.run_add("numpy.random.seed", seed)
log_random_seed("numpy.random.seed")
return original_numpy(seed)
except Exception as e:
Warning("Tracker failed to log the set seed because %s" % str(e))
return original_numpy(seed)
def build_output(self, _pypads_env: InjectionLoggerEnv, _logger_call,
**kwargs):
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
if pads.cache.run_exists(id(self)):
logger_output = pads.cache.run_get(id(self)).get('output')
logger_output.add_call_env(_pypads_env)
return logger_output
else:
return super().build_output(_pypads_env, _logger_call)
def _path_to_id(path, run_id=None):
from pypads.app.pypads import get_current_pads
if run_id is None:
run = get_current_pads().api.active_run()
run_id = run.info.run_id
experiment_name = mlflow.get_experiment(
mlflow.active_run().info.experiment_id).name
else:
experiment_name = mlflow.get_experiment(
mlflow.active_run().info.experiment_id).name
return os.path.sep.join([experiment_name, run_id, path])
def finalize_output(pads, logger_call, output, *args, **kwargs):
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
log_to: LogTO = pads.cache.run_get("std_out_logger")
path = os.path.join(get_temp_folder(), "logfile.log")
if os.path.isfile(path):
log_to.path = pads.api.log_artifact(path, description="StdOut log of the current run",
artifact_path=log_to.path)
output.logs = log_to.store()
output.store()
def get_temp_folder(run=None):
"""
Get the base folder to log tmp files to. For now it can't be changed. Todo make configurable
:return:
"""
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
run = run if run else pads.api.active_run()
if run is None:
raise ValueError("No active run is defined.")
return os.path.join(pads.folder, "tmp", run.info.experiment_id,
run.info.run_id) + os.path.sep
def store_lib(self):
from pypads.app.pypads import get_current_pads
lib_repo = get_current_pads().library_repository
# TODO get hash uid for logger
lib_hash = persistent_hash(
(self._defined_in.name, self._defined_in.version))
if not lib_repo.has_object(uid=lib_hash):
lib_obj = lib_repo.get_object(uid=lib_hash)
lib_obj.log_json(self._defined_in)
else:
lib_obj = lib_repo.get_object(uid=lib_hash)
self.defined_in = lib_obj.get_reference()
def __pre__(self,
ctx,
*args,
_pypads_write_format=None,
_logger_call: LoggerCall,
_logger_output,
_args,
_kwargs,
**kwargs):
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
pads.cache.run_add("parameter_search", True)
def __call_wrapped__(self, ctx, *args, _pypads_env: InjectionLoggerEnv,
_logger_call, _logger_output, _args, _kwargs,
**kwargs):
"""
:param ctx:
:param args:
:param _pypads_result:
:param kwargs:
:return:
"""
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
_return, time = OriginalExecutor(fn=_pypads_env.callback)(*_args,
**_kwargs)
if isinstance(_return, GeneratorType):
items = list(_return)
def generator():
pads.cache.add("tracking_mode", "multiple")
logger.info("Detected splitting, Tracking splits started...")
if _logger_output.splits is None:
splits = SplitTO(parent=_logger_output)
else:
splits = _logger_output.splits
for r in items:
split_id = uuid.uuid4()
pads.cache.run_add("current_split", split_id)
train, test, val = splitter_output(r,
fn=_pypads_env.callback)
splits.add_split(split_id, train, test, val)
_logger_output.splits = splits
yield r
else:
def generator():
logger.info("Detected splitting, Tracking splits started...")
pads.cache.add("tracking_mode", "single")
if _logger_output.splits is None:
splits = SplitTO(parent=_logger_output)
else:
splits = _logger_output.splits
train, test, val = splitter_output(_return,
fn=_pypads_env.callback)
split_id = uuid.uuid4()
pads.cache.run_add("current_split", split_id)
splits.add_split(split_id, train, test, val)
_logger_output.splits = splits
return _return
return generator(), time
def log_random_seed(key):
from pypads.app.base import PyPads
from pypads.app.pypads import get_current_pads
pads: PyPads = get_current_pads()
# Get seed information from cache
if pads.cache.run_exists(key):
# TODO if tag already exists (set called multiple times) we need to handle that (save seed per function run?)
pads.api.set_tag("pypads." + key, pads.cache.run_get(key))
else:
warning(
"Can't log seed produced by seed generator. You have to enable ")
def __init__(self, parameter, experiment_id, run_id, data=None):
"""
:param parameter: Parameters given to the triggered logging env. Ex. defined on hooks.
:param experiment_id: The id of the experiment.
:param run_id: The id of the run.
:param data: Additional data for the logger env.
"""
self._data = data or {}
self._parameter = parameter
self._experiment_id = experiment_id
self._run_id = run_id
from pypads.app.pypads import get_current_pads
self._pypads = get_current_pads()
