def entry(_cls, *args, _pypads_context=context, pypads_mapped_by=mappings, **kwargs):
logger.debug("Call to tracked class method " + str(fn))
global error
if self._pypads.api.active_run():
error = False
with self._make_call(_cls, fn_reference) as call:
accessor = call.call_id
# add the function to the callback stack
callback = types.MethodType(fn, _cls)
# for every hook add
if self._is_skip_recursion(accessor):
logger.info("Skipping " + str(accessor.context.container.__name__) + "." + str(
accessor.wrappee.__name__))
out = callback(*args, **kwargs)
return out
hooks = context.get_hooks(fn)
for (h, config) in hooks:
c = self._add_hook(h, config, callback, call, context.get_wrap_metas(fn))
if c:
callback = types.MethodType(c, _cls)
# start executing the stack
out = callback(*args, **kwargs)
else:
if not error:
error = True
logger.error(
"No run was active to log your hooks. You may want to start a run with PyPads().start_track()")
callback = types.MethodType(fn, _cls)
out = callback(*args, **kwargs)
return out
def activate_tracking(self,
reload_modules=False,
reload_warnings=True,
clear_imports=False,
affected_modules=None):
"""
Function to duck punch all objects defined in the mapping files. This should at best be called before importing
any libraries.
:param affected_modules: Affected modules of the mapping files.
:param clear_imports: Clear imports after punching. CAREFUL THIS IS EXPERIMENTAL!
:param reload_warnings: Show warnings of affected modules which were already imported before the importlib was extended.
:param reload_modules: Force a reload of affected modules. CAREFUL THIS IS EXPERIMENTAL!
:return:
"""
if affected_modules is None:
# Modules are affected if they are mapped by a library or are already punched
affected_modules = self.wrap_manager.module_wrapper.punched_module_names | \
{l.name for l in self.mapping_registry.get_libraries()}
global tracking_active
if not tracking_active:
logger.info("Activating tracking by extending importlib...")
from pypads.app.pypads import set_current_pads
set_current_pads(self)
# Add our loader to the meta_path
extend_import_module()
import sys
import importlib
loaded_modules = [(name, module)
for name, module in sys.modules.items()]
for name, module in loaded_modules:
if self.is_affected_module(name, affected_modules):
if reload_warnings:
logger.warning(
name +
" was imported before PyPads. To enable tracking import PyPads before or use "
"reload_modules / clear_imports. Every already created instance is not tracked."
)
if clear_imports:
del sys.modules[name]
if reload_modules:
try:
spec = importlib.util.find_spec(module.__name__)
duck_punch_loader(spec)
loader = spec.loader
module = loader.load_module(module.__name__)
loader.exec_module(module)
importlib.reload(module)
except Exception as e:
logger.debug("Couldn't reload module " + str(e))
tracking_active = True
else:
# TODO check if a second tracker / tracker activation doesn't break the tracking
logger.warning("Tracking was already activated.")
return self
def __real_call__(self, *args, _pypads_env: LoggerEnv = None, **kwargs):
logger.debug("Called on Import function " + str(self))
_return = super().__real_call__(*args, _pypads_env=_pypads_env or LoggerEnv(parameter=dict(),
experiment_id=get_experiment_id(),
run_id=get_run_id(),
data={"category: ImportLogger"}),
**kwargs)
return _return
def _get_relevant_mappings(package: Package):
from pypads.app.pypads import get_current_pads
try:
return get_current_pads().mapping_registry.get_relevant_mappings(
package)
except Exception as e:
logger.debug('Getting on-import mappings failed due to : {}'.format(
str(e)))
return set()
def _add_hook(self, hook, config, callback, call: Call, mappings, data=None):
# For every hook we defined on the given function in out mapping file execute it before running the code
if not call.has_hook(hook):
return self._get_env_setter(
_pypads_env=InjectionLoggerEnv(mappings, hook, callback, call, config.parameters, get_experiment_id(),
get_run_id(), data=data))
else:
logger.debug(
f"{hook} defined hook with config {config} is tracked multiple times on {call}. Ignoring second hooking.")
return None
def store_original(self, wrappee):
try:
if not inspect.isfunction(wrappee):
holder = wrappee
else:
holder = self._c
setattr(holder, self.original_name(wrappee), copy(wrappee))
except TypeError as e:
logger.debug("Can't set attribute '" + wrappee.__name__ +
"' on '" + str(self._c) + "'.")
return self._c
def download_artifacts(self, run_id, relative_path, dst_path=None):
local_location = os.path.join(dst_path, relative_path)
if os.path.exists(local_location
): # TODO check file digest or something similar??
logger.debug(
f"Skipped downloading file because a file f{local_location} with the same name already exists."
)
return local_location
return artifact_utils.get_artifact_uri(run_id=run_id,
artifact_path=relative_path)
def defensive_exit(signum=None, frame=None):
global executed_exit_fns
try:
if fn not in executed_exit_fns:
logger.debug(f"Running exit fn {fn}.")
out = fn()
executed_exit_fns.add(fn)
return out
logger.debug(f"Already ran exit fn {fn}.")
return None
except (KeyboardInterrupt, Exception) as e:
logger.error("Couldn't run atexit function " + fn.__name__ +
" because of " + str(e))
def find_package_version(name: str):
try:
import sys
if name in sys.modules:
base_package = sys.modules[name]
if hasattr(base_package, "__version__"):
lib_version = getattr(base_package, "__version__")
return lib_version
else:
lib_version = pkg_resources.get_distribution(name).version
return lib_version
except Exception as e:
logger.debug("Couldn't get version of package {}".format(name))
return None
def mapping_applicable_filter(name):
if hasattr(ctx.container, name):
try:
return self.is_applicable(ctx,
getattr(ctx.container, name))
except RecursionError as rerr:
logger.error(
"Recursion error on '" + str(ctx) +
"'. This might be because __get_attr__ is being wrapped. "
+ str(rerr))
else:
logger.debug("Can't access attribute '" + str(name) +
"' on '" + str(ctx) + "'. Skipping.")
return False
def store_hook(self, hook, wrappee):
try:
if not inspect.isfunction(wrappee) or "<slot wrapper" in str(
wrappee):
holder = wrappee
else:
holder = self._c
# Set self reference
if not hasattr(holder, "_pypads_hooks_" + wrappee.__name__):
setattr(holder, "_pypads_hooks_" + wrappee.__name__, set())
getattr(holder, "_pypads_hooks_" + wrappee.__name__).add(hook)
except TypeError as e:
logger.debug("Can't set attribute '" + wrappee.__name__ +
"' on '" + str(self._c) + "'.")
raise e
def register_teardown(self,
name,
post_fn: Union[RunTeardown, SimpleRunFunction],
silent_duplicate=True):
"""
Register a new post run function.
:param name: Name of the registration
:param post_fn: Function to register
:param silent_duplicate: Ignore log output if post_run was already registered.
This makes sense if a logger running multiple times wants to register a single cleanup function.
:return:
"""
cache = self._get_teardown_cache()
if cache.exists(name):
if not silent_duplicate:
logger.debug("Post run fn with name '" + name +
"' already exists. Skipped.")
else:
cache.add(name, post_fn)
def wrap(self, clazz, context, matched_mappings: Set[MatchedMapping]):
"""
Wrap a class in given ctx with pypads functionality
:param clazz:
:param context:
:param matched_mappings:
:return:
"""
if clazz.__name__ not in self.punched_class_names:
for matched_mapping in matched_mappings:
try:
context.store_wrap_meta(matched_mapping, clazz)
except Exception:
return clazz
self.punched_class_names.add(clazz.__name__)
if not context.has_original(clazz):
context.store_original(clazz)
# Module was changed and should be added to the list of modules which have been changed
if hasattr(clazz, "__module__"):
self._pypads.wrap_manager.module_wrapper.add_punched_module_name(clazz.__module__)
attrs = {}
clazz_context = Context(clazz, ".".join([context.reference, clazz.__name__]))
for matched_mapping in matched_mappings:
# Try to wrap every attr of the class
for name in list(filter(
matched_mapping.mapping.applicable_filter(
clazz_context),
dir(clazz))):
if name not in attrs:
attrs[name] = set()
attrs[name].add(matched_mapping)
for name, mm in attrs.items():
self._pypads.wrap_manager.wrap(getattr(clazz, name), clazz_context, mm)
# Override class on module
context.overwrite(clazz.__name__, clazz)
else:
logger.debug("Class " + str(clazz) + "already duck-puched.")
return clazz
def __call__(self, ctx, *args, _pypads_env=None, **kwargs):
try:
return super().__call__(ctx,
*args,
_pypads_env=_pypads_env,
**kwargs)
except KeyboardInterrupt:
return self._handle_error(*args,
ctx=ctx,
_pypads_env=_pypads_env,
error=Exception("KeyboardInterrupt"),
**kwargs)
except Exception as e:
import traceback
logger.debug(traceback.format_exc())
return self._handle_error(*args,
ctx=ctx,
_pypads_env=_pypads_env,
error=e,
**kwargs)
def deactivate_tracking(self, run_atexits=False, reload_modules=True):
"""
Deacticate the current tracking and cleanup.
:param run_atexits: Run the registered atexit functions of pypads
:param reload_modules: Force a reload of affected modules
:return:
"""
# run atexit fns if needed
if run_atexits:
self.run_exit_fns()
# Remove atexit fns
for fn in self._atexit_fns:
atexit.unregister(fn)
import sys
import importlib
loaded_modules = [(name, module)
for name, module in sys.modules.items()]
for name, module in loaded_modules:
if self.is_affected_module(name):
del sys.modules[name]
if reload_modules:
# reload modules if they where affected
try:
spec = importlib.util.find_spec(module.__name__)
duck_punch_loader(spec)
loader = spec.loader
module = loader.load_module(module.__name__)
loader.exec_module(module)
importlib.reload(module)
except Exception as e:
logger.debug("Couldn't reload module " + str(e))
global tracking_active
tracking_active = False
# noinspection PyTypeChecker
from pypads.app.pypads import set_current_pads
set_current_pads(None)
def wrap(self, module, context, matched_mappings: Set[MatchedMapping]):
"""
Function to wrap modules with pypads functionality
:param module:
:param context:
:param matched_mappings:
:return:
"""
if module.__name__ not in self.punched_module_names:
self._pypads.wrap_manager.module_wrapper.add_punched_module_name(module.__name__)
for matched_mapping in matched_mappings:
context.store_wrap_meta(matched_mapping, module)
if not context.has_original(module):
context.store_original(module)
# Try to wrap every attr of the module
# Only get entries defined directly in this module
# https://stackoverflow.com/questions/22578509/python-get-only-classes-defined-in-imported-module-with-dir
attrs = {}
for matched_mapping in matched_mappings:
for name in list(filter(matched_mapping.mapping.applicable_filter(
Context(module, ".".join([context.reference, module.__name__]))),
[m for m, _ in inspect.getmembers(module, lambda x: hasattr(x,
"__module__") and x.__module__ == module.__name__)])):
attr = getattr(module, name)
if attr not in attrs:
attrs[attr] = set()
else:
attrs[attr].add(matched_mapping)
for attr, mm in attrs.items():
# Don't track imported modules
if not inspect.ismodule(attr):
self._pypads.wrap_manager.wrap(attr, module, mm)
else:
logger.debug("Module " + str(module) + " already duck-punched.")
return module
def _make_call(self, instance, fn_reference):
accessor = CallAccessor.from_function_reference(fn_reference, instance)
current_call = None
call = None
try:
current_call: Call = self._pypads.call_tracker.current_call()
try:
instance_str = str(instance)
except Exception as e:
if hasattr(instance, '__class__'):
if hasattr(instance.__class__, '__name__'):
instance_str = instance.__class__.__name__
else:
instance_str = str(instance.__class__)
else:
instance_str = ""
# Don't make a new call if the last call has the same identity as the current one
# Or if the instance method access yields a different method than the current original (inherited methods)
# And the instance as well as the function name where the same
if current_call and (accessor.is_call_identity(current_call.call_id) and
(instance and instance == current_call.call_id.instance
and not fn_reference.context.original(fn_reference.wrappee) == getattr(
instance,
fn_reference.fn_name,
fn_reference.wrappee))):
# if not fn_reference.context.original(
# fn_reference.wrappee) == fn_reference.wrappee and current_call is not None:
call = current_call
logger.debug(f"Reused existing call {call} in {fn_reference} of {instance_str}.")
else:
call = add_call(accessor)
logger.debug(f"Created new call to track {call} in {fn_reference} of {instance_str}.")
yield call
finally:
if call and not current_call == call:
finish_call(call)
def __real_call__(self, *args, **kwargs):
logger.debug("Called post run function " + str(self))
return super().__real_call__(*args, **kwargs)
def add_wrappings(self, module):
"""
Function that look for matched mappings and inject corresponding logging functionalities
:param self: context
:param module: module to be wrapped
"""
from pypads.app.pypads import current_pads
reference = module.__name__
# History to check if a class inherits a wrapping intra-module
mro_entry_history = {}
if current_pads:
# TODO we might want to make this configurable/improve performance.
# This looks at every imported class and every mapping.
# On execution of a module we search for relevant mappings
# For every var on module
try:
members = inspect.getmembers(
module, lambda x: hasattr(x, "__module__") and x.__module__ ==
module.__name__)
except Exception as e:
logger.debug(
"getmembers of inspect failed on module '" +
str(module.__name__) + "' with expection" + str(e) +
". Falling back to dir to get the members of the module.")
members = [(name, getattr(module, name)) for name in dir(module)]
for name, obj in members:
if obj is not None:
obj_ref = ".".join([reference, name])
package = Package(module, PackagePath(obj_ref))
# Skip modules if they are from another package for now
if inspect.ismodule(obj):
if not module.__name__.split(".")[0] == obj.__name__.split(
".")[0]:
continue
mappings = set()
if inspect.isclass(obj) and hasattr(obj, "mro"):
try:
# Look at the MRO and add classes to be punched which inherit from our punched classes
mro_ = obj.mro()[1:]
for entry in mro_:
if entry not in mro_entry_history.keys():
mro_entry_history[entry] = [obj]
else:
mro_entry_history[entry].append(obj)
if hasattr(entry,
"_pypads_mapping_" + entry.__name__):
found_mappings = _add_inherited_mapping(
obj, entry)
mappings = mappings.union(found_mappings)
except Exception as e:
logger.debug("Skipping some superclasses of " +
str(obj) + ". " + str(e))
mappings = mappings.union(_get_relevant_mappings(package))
if len(mappings) > 0:
if not has_delayed_wrapping():
current_pads.wrap_manager.wrap(
obj, Context(module, reference), {
MatchedMapping(mapping, package.path)
for mapping in mappings
})
else:
_first_in_queue = list(
_import_loggers_queues.keys())[0]
if not _first_in_queue in _wrapping_queues:
_wrapping_queues[_first_in_queue] = []
_wrapping_queues[_first_in_queue].append(
(obj, Context(module, reference), {
MatchedMapping(mapping, package.path)
for mapping in mappings
}))
if reference in _import_loggers_queues:
# execute import logger of this reference
while len(_import_loggers_queues[reference]) > 0:
(fn, config) = _import_loggers_queues[reference].pop()
fn(self)
del _import_loggers_queues[reference]
if reference in _wrapping_queues:
for (obj, ctx, mm) in _wrapping_queues[reference]:
current_pads.wrap_manager.wrap(obj, ctx, mm)
del _wrapping_queues[reference]
if reference in current_pads.wrap_manager.module_wrapper.punched_module_names:
logger.info(f"PyPads wrapped functions of module {reference}.")
def cleanup_cache(*args, run_id=self.run_id, **kwargs):
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
pads.cache.run_delete(run_id)
logger.debug("Cleared run cache after run " + run_id)
def __pre__(self, ctx, *args, _logger_call, **kwargs):
logger.debug("Entered " + str(_logger_call.original_call))
def __post__(self, ctx, *args, _logger_call, **kwargs):
logger.debug("Exited " + str(_logger_call.original_call))
def env_setter(_self, *args, _pypads_env=env, **kwargs):
logger.debug("Method hook " + str(cid.context) + str(cid.wrappee) + str(env.hook))
return self._wrapped_inner_function(_self, *args, _pypads_env=_pypads_env, **kwargs)
def wrapped_function(*args,
_pypads_cache=None,
_pypads_config=None,
_pypads_active_run_id=None,
_pypads_tracking_uri=None,
_pypads_affected_modules=None,
_pypads_triggering_process=None,
**kwargs):
from pypads.parallel.util import _pickle_tuple, _cloudpickle_tuple
from pypads import logger
# only if pads data was passed
if _pypads_active_run_id:
# noinspection PyUnresolvedReferences
from pypads.app import pypads
import mlflow
is_new_process = not pypads.current_pads
# If pads has to be reinitialized
if is_new_process:
import pypads
# reactivate this run in the foreign process
mlflow.set_tracking_uri(_pypads_tracking_uri)
mlflow.start_run(run_id=_pypads_active_run_id, nested=True)
start_time = time.time()
logger.debug("Init Pypads in:" +
str(time.time() - start_time))
# TODO update to new format
from pypads.app.base import PyPads
_pypads = PyPads(uri=_pypads_tracking_uri,
config=_pypads_config,
pre_initialized_cache=_pypads_cache)
_pypads.activate_tracking(
reload_warnings=False,
affected_modules=_pypads_affected_modules,
clear_imports=True,
reload_modules=True,
)
_pypads.start_track(disable_run_init=True)
def clear_mlflow():
"""
Don't close run. This function clears the run which was reactivated from the stack to stop a closing of it.
:return:
"""
if len(mlflow.tracking.fluent._active_run_stack) == 1:
mlflow.tracking.fluent._active_run_stack.pop()
import atexit
atexit.register(clear_mlflow)
# If pads already exists on process
else:
_pypads = pypads.current_pads
_pypads.cache.merge(_pypads_cache)
# Unpickle args
from pickle import loads
start_time = time.time()
a, b = loads(args[0])
logger.debug("Loading args from pickle in:" +
str(time.time() - start_time))
# Unpickle function
from cloudpickle import loads as c_loads
start_time = time.time()
wrapped_fn = c_loads(args[1])[0]
logger.debug("Loading punched function from pickle in:" +
str(time.time() - start_time))
args = a
kwargs = b
logger.debug("Started wrapped function on process: " +
str(os.getpid()))
out = wrapped_fn(*args, **kwargs)
return out, _pypads.cache
else:
return fn(*args, **kwargs)
