def entry(_cls, *args, _pypads_context=context, pypads_mapped_by=mappings, **kwargs): logger.debug("Call to tracked class method " + str(fn)) global error if self._pypads.api.active_run(): error = False with self._make_call(_cls, fn_reference) as call: accessor = call.call_id # add the function to the callback stack callback = types.MethodType(fn, _cls) # for every hook add if self._is_skip_recursion(accessor): logger.info("Skipping " + str(accessor.context.container.__name__) + "." + str( accessor.wrappee.__name__)) out = callback(*args, **kwargs) return out hooks = context.get_hooks(fn) for (h, config) in hooks: c = self._add_hook(h, config, callback, call, context.get_wrap_metas(fn)) if c: callback = types.MethodType(c, _cls) # start executing the stack out = callback(*args, **kwargs) else: if not error: error = True logger.error( "No run was active to log your hooks. You may want to start a run with PyPads().start_track()") callback = types.MethodType(fn, _cls) out = callback(*args, **kwargs) return out
def activate_tracking(self, reload_modules=False, reload_warnings=True, clear_imports=False, affected_modules=None): """ Function to duck punch all objects defined in the mapping files. This should at best be called before importing any libraries. :param affected_modules: Affected modules of the mapping files. :param clear_imports: Clear imports after punching. CAREFUL THIS IS EXPERIMENTAL! :param reload_warnings: Show warnings of affected modules which were already imported before the importlib was extended. :param reload_modules: Force a reload of affected modules. CAREFUL THIS IS EXPERIMENTAL! :return: """ if affected_modules is None: # Modules are affected if they are mapped by a library or are already punched affected_modules = self.wrap_manager.module_wrapper.punched_module_names | \ {l.name for l in self.mapping_registry.get_libraries()} global tracking_active if not tracking_active: logger.info("Activating tracking by extending importlib...") from pypads.app.pypads import set_current_pads set_current_pads(self) # Add our loader to the meta_path extend_import_module() import sys import importlib loaded_modules = [(name, module) for name, module in sys.modules.items()] for name, module in loaded_modules: if self.is_affected_module(name, affected_modules): if reload_warnings: logger.warning( name + " was imported before PyPads. To enable tracking import PyPads before or use " "reload_modules / clear_imports. Every already created instance is not tracked." ) if clear_imports: del sys.modules[name] if reload_modules: try: spec = importlib.util.find_spec(module.__name__) duck_punch_loader(spec) loader = spec.loader module = loader.load_module(module.__name__) loader.exec_module(module) importlib.reload(module) except Exception as e: logger.debug("Couldn't reload module " + str(e)) tracking_active = True else: # TODO check if a second tracker / tracker activation doesn't break the tracking logger.warning("Tracking was already activated.") return self
def __real_call__(self, *args, _pypads_env: LoggerEnv = None, **kwargs): logger.debug("Called on Import function " + str(self)) _return = super().__real_call__(*args, _pypads_env=_pypads_env or LoggerEnv(parameter=dict(), experiment_id=get_experiment_id(), run_id=get_run_id(), data={"category: ImportLogger"}), **kwargs) return _return
def _get_relevant_mappings(package: Package): from pypads.app.pypads import get_current_pads try: return get_current_pads().mapping_registry.get_relevant_mappings( package) except Exception as e: logger.debug('Getting on-import mappings failed due to : {}'.format( str(e))) return set()
def _add_hook(self, hook, config, callback, call: Call, mappings, data=None): # For every hook we defined on the given function in out mapping file execute it before running the code if not call.has_hook(hook): return self._get_env_setter( _pypads_env=InjectionLoggerEnv(mappings, hook, callback, call, config.parameters, get_experiment_id(), get_run_id(), data=data)) else: logger.debug( f"{hook} defined hook with config {config} is tracked multiple times on {call}. Ignoring second hooking.") return None
def store_original(self, wrappee): try: if not inspect.isfunction(wrappee): holder = wrappee else: holder = self._c setattr(holder, self.original_name(wrappee), copy(wrappee)) except TypeError as e: logger.debug("Can't set attribute '" + wrappee.__name__ + "' on '" + str(self._c) + "'.") return self._c
def download_artifacts(self, run_id, relative_path, dst_path=None): local_location = os.path.join(dst_path, relative_path) if os.path.exists(local_location ): # TODO check file digest or something similar?? logger.debug( f"Skipped downloading file because a file f{local_location} with the same name already exists." ) return local_location return artifact_utils.get_artifact_uri(run_id=run_id, artifact_path=relative_path)
def defensive_exit(signum=None, frame=None): global executed_exit_fns try: if fn not in executed_exit_fns: logger.debug(f"Running exit fn {fn}.") out = fn() executed_exit_fns.add(fn) return out logger.debug(f"Already ran exit fn {fn}.") return None except (KeyboardInterrupt, Exception) as e: logger.error("Couldn't run atexit function " + fn.__name__ + " because of " + str(e))
def find_package_version(name: str): try: import sys if name in sys.modules: base_package = sys.modules[name] if hasattr(base_package, "__version__"): lib_version = getattr(base_package, "__version__") return lib_version else: lib_version = pkg_resources.get_distribution(name).version return lib_version except Exception as e: logger.debug("Couldn't get version of package {}".format(name)) return None
def mapping_applicable_filter(name): if hasattr(ctx.container, name): try: return self.is_applicable(ctx, getattr(ctx.container, name)) except RecursionError as rerr: logger.error( "Recursion error on '" + str(ctx) + "'. This might be because __get_attr__ is being wrapped. " + str(rerr)) else: logger.debug("Can't access attribute '" + str(name) + "' on '" + str(ctx) + "'. Skipping.") return False
def store_hook(self, hook, wrappee): try: if not inspect.isfunction(wrappee) or "<slot wrapper" in str( wrappee): holder = wrappee else: holder = self._c # Set self reference if not hasattr(holder, "_pypads_hooks_" + wrappee.__name__): setattr(holder, "_pypads_hooks_" + wrappee.__name__, set()) getattr(holder, "_pypads_hooks_" + wrappee.__name__).add(hook) except TypeError as e: logger.debug("Can't set attribute '" + wrappee.__name__ + "' on '" + str(self._c) + "'.") raise e
def register_teardown(self, name, post_fn: Union[RunTeardown, SimpleRunFunction], silent_duplicate=True): """ Register a new post run function. :param name: Name of the registration :param post_fn: Function to register :param silent_duplicate: Ignore log output if post_run was already registered. This makes sense if a logger running multiple times wants to register a single cleanup function. :return: """ cache = self._get_teardown_cache() if cache.exists(name): if not silent_duplicate: logger.debug("Post run fn with name '" + name + "' already exists. Skipped.") else: cache.add(name, post_fn)
def wrap(self, clazz, context, matched_mappings: Set[MatchedMapping]): """ Wrap a class in given ctx with pypads functionality :param clazz: :param context: :param matched_mappings: :return: """ if clazz.__name__ not in self.punched_class_names: for matched_mapping in matched_mappings: try: context.store_wrap_meta(matched_mapping, clazz) except Exception: return clazz self.punched_class_names.add(clazz.__name__) if not context.has_original(clazz): context.store_original(clazz) # Module was changed and should be added to the list of modules which have been changed if hasattr(clazz, "__module__"): self._pypads.wrap_manager.module_wrapper.add_punched_module_name(clazz.__module__) attrs = {} clazz_context = Context(clazz, ".".join([context.reference, clazz.__name__])) for matched_mapping in matched_mappings: # Try to wrap every attr of the class for name in list(filter( matched_mapping.mapping.applicable_filter( clazz_context), dir(clazz))): if name not in attrs: attrs[name] = set() attrs[name].add(matched_mapping) for name, mm in attrs.items(): self._pypads.wrap_manager.wrap(getattr(clazz, name), clazz_context, mm) # Override class on module context.overwrite(clazz.__name__, clazz) else: logger.debug("Class " + str(clazz) + "already duck-puched.") return clazz
def __call__(self, ctx, *args, _pypads_env=None, **kwargs): try: return super().__call__(ctx, *args, _pypads_env=_pypads_env, **kwargs) except KeyboardInterrupt: return self._handle_error(*args, ctx=ctx, _pypads_env=_pypads_env, error=Exception("KeyboardInterrupt"), **kwargs) except Exception as e: import traceback logger.debug(traceback.format_exc()) return self._handle_error(*args, ctx=ctx, _pypads_env=_pypads_env, error=e, **kwargs)
def deactivate_tracking(self, run_atexits=False, reload_modules=True): """ Deacticate the current tracking and cleanup. :param run_atexits: Run the registered atexit functions of pypads :param reload_modules: Force a reload of affected modules :return: """ # run atexit fns if needed if run_atexits: self.run_exit_fns() # Remove atexit fns for fn in self._atexit_fns: atexit.unregister(fn) import sys import importlib loaded_modules = [(name, module) for name, module in sys.modules.items()] for name, module in loaded_modules: if self.is_affected_module(name): del sys.modules[name] if reload_modules: # reload modules if they where affected try: spec = importlib.util.find_spec(module.__name__) duck_punch_loader(spec) loader = spec.loader module = loader.load_module(module.__name__) loader.exec_module(module) importlib.reload(module) except Exception as e: logger.debug("Couldn't reload module " + str(e)) global tracking_active tracking_active = False # noinspection PyTypeChecker from pypads.app.pypads import set_current_pads set_current_pads(None)
def wrap(self, module, context, matched_mappings: Set[MatchedMapping]): """ Function to wrap modules with pypads functionality :param module: :param context: :param matched_mappings: :return: """ if module.__name__ not in self.punched_module_names: self._pypads.wrap_manager.module_wrapper.add_punched_module_name(module.__name__) for matched_mapping in matched_mappings: context.store_wrap_meta(matched_mapping, module) if not context.has_original(module): context.store_original(module) # Try to wrap every attr of the module # Only get entries defined directly in this module # https://stackoverflow.com/questions/22578509/python-get-only-classes-defined-in-imported-module-with-dir attrs = {} for matched_mapping in matched_mappings: for name in list(filter(matched_mapping.mapping.applicable_filter( Context(module, ".".join([context.reference, module.__name__]))), [m for m, _ in inspect.getmembers(module, lambda x: hasattr(x, "__module__") and x.__module__ == module.__name__)])): attr = getattr(module, name) if attr not in attrs: attrs[attr] = set() else: attrs[attr].add(matched_mapping) for attr, mm in attrs.items(): # Don't track imported modules if not inspect.ismodule(attr): self._pypads.wrap_manager.wrap(attr, module, mm) else: logger.debug("Module " + str(module) + " already duck-punched.") return module
def _make_call(self, instance, fn_reference): accessor = CallAccessor.from_function_reference(fn_reference, instance) current_call = None call = None try: current_call: Call = self._pypads.call_tracker.current_call() try: instance_str = str(instance) except Exception as e: if hasattr(instance, '__class__'): if hasattr(instance.__class__, '__name__'): instance_str = instance.__class__.__name__ else: instance_str = str(instance.__class__) else: instance_str = "" # Don't make a new call if the last call has the same identity as the current one # Or if the instance method access yields a different method than the current original (inherited methods) # And the instance as well as the function name where the same if current_call and (accessor.is_call_identity(current_call.call_id) and (instance and instance == current_call.call_id.instance and not fn_reference.context.original(fn_reference.wrappee) == getattr( instance, fn_reference.fn_name, fn_reference.wrappee))): # if not fn_reference.context.original( # fn_reference.wrappee) == fn_reference.wrappee and current_call is not None: call = current_call logger.debug(f"Reused existing call {call} in {fn_reference} of {instance_str}.") else: call = add_call(accessor) logger.debug(f"Created new call to track {call} in {fn_reference} of {instance_str}.") yield call finally: if call and not current_call == call: finish_call(call)
def __real_call__(self, *args, **kwargs): logger.debug("Called post run function " + str(self)) return super().__real_call__(*args, **kwargs)
def add_wrappings(self, module): """ Function that look for matched mappings and inject corresponding logging functionalities :param self: context :param module: module to be wrapped """ from pypads.app.pypads import current_pads reference = module.__name__ # History to check if a class inherits a wrapping intra-module mro_entry_history = {} if current_pads: # TODO we might want to make this configurable/improve performance. # This looks at every imported class and every mapping. # On execution of a module we search for relevant mappings # For every var on module try: members = inspect.getmembers( module, lambda x: hasattr(x, "__module__") and x.__module__ == module.__name__) except Exception as e: logger.debug( "getmembers of inspect failed on module '" + str(module.__name__) + "' with expection" + str(e) + ". Falling back to dir to get the members of the module.") members = [(name, getattr(module, name)) for name in dir(module)] for name, obj in members: if obj is not None: obj_ref = ".".join([reference, name]) package = Package(module, PackagePath(obj_ref)) # Skip modules if they are from another package for now if inspect.ismodule(obj): if not module.__name__.split(".")[0] == obj.__name__.split( ".")[0]: continue mappings = set() if inspect.isclass(obj) and hasattr(obj, "mro"): try: # Look at the MRO and add classes to be punched which inherit from our punched classes mro_ = obj.mro()[1:] for entry in mro_: if entry not in mro_entry_history.keys(): mro_entry_history[entry] = [obj] else: mro_entry_history[entry].append(obj) if hasattr(entry, "_pypads_mapping_" + entry.__name__): found_mappings = _add_inherited_mapping( obj, entry) mappings = mappings.union(found_mappings) except Exception as e: logger.debug("Skipping some superclasses of " + str(obj) + ". " + str(e)) mappings = mappings.union(_get_relevant_mappings(package)) if len(mappings) > 0: if not has_delayed_wrapping(): current_pads.wrap_manager.wrap( obj, Context(module, reference), { MatchedMapping(mapping, package.path) for mapping in mappings }) else: _first_in_queue = list( _import_loggers_queues.keys())[0] if not _first_in_queue in _wrapping_queues: _wrapping_queues[_first_in_queue] = [] _wrapping_queues[_first_in_queue].append( (obj, Context(module, reference), { MatchedMapping(mapping, package.path) for mapping in mappings })) if reference in _import_loggers_queues: # execute import logger of this reference while len(_import_loggers_queues[reference]) > 0: (fn, config) = _import_loggers_queues[reference].pop() fn(self) del _import_loggers_queues[reference] if reference in _wrapping_queues: for (obj, ctx, mm) in _wrapping_queues[reference]: current_pads.wrap_manager.wrap(obj, ctx, mm) del _wrapping_queues[reference] if reference in current_pads.wrap_manager.module_wrapper.punched_module_names: logger.info(f"PyPads wrapped functions of module {reference}.")
def cleanup_cache(*args, run_id=self.run_id, **kwargs): from pypads.app.pypads import get_current_pads pads = get_current_pads() pads.cache.run_delete(run_id) logger.debug("Cleared run cache after run " + run_id)
def __pre__(self, ctx, *args, _logger_call, **kwargs): logger.debug("Entered " + str(_logger_call.original_call))
def __post__(self, ctx, *args, _logger_call, **kwargs): logger.debug("Exited " + str(_logger_call.original_call))
def env_setter(_self, *args, _pypads_env=env, **kwargs): logger.debug("Method hook " + str(cid.context) + str(cid.wrappee) + str(env.hook)) return self._wrapped_inner_function(_self, *args, _pypads_env=_pypads_env, **kwargs)
def wrapped_function(*args, _pypads_cache=None, _pypads_config=None, _pypads_active_run_id=None, _pypads_tracking_uri=None, _pypads_affected_modules=None, _pypads_triggering_process=None, **kwargs): from pypads.parallel.util import _pickle_tuple, _cloudpickle_tuple from pypads import logger # only if pads data was passed if _pypads_active_run_id: # noinspection PyUnresolvedReferences from pypads.app import pypads import mlflow is_new_process = not pypads.current_pads # If pads has to be reinitialized if is_new_process: import pypads # reactivate this run in the foreign process mlflow.set_tracking_uri(_pypads_tracking_uri) mlflow.start_run(run_id=_pypads_active_run_id, nested=True) start_time = time.time() logger.debug("Init Pypads in:" + str(time.time() - start_time)) # TODO update to new format from pypads.app.base import PyPads _pypads = PyPads(uri=_pypads_tracking_uri, config=_pypads_config, pre_initialized_cache=_pypads_cache) _pypads.activate_tracking( reload_warnings=False, affected_modules=_pypads_affected_modules, clear_imports=True, reload_modules=True, ) _pypads.start_track(disable_run_init=True) def clear_mlflow(): """ Don't close run. This function clears the run which was reactivated from the stack to stop a closing of it. :return: """ if len(mlflow.tracking.fluent._active_run_stack) == 1: mlflow.tracking.fluent._active_run_stack.pop() import atexit atexit.register(clear_mlflow) # If pads already exists on process else: _pypads = pypads.current_pads _pypads.cache.merge(_pypads_cache) # Unpickle args from pickle import loads start_time = time.time() a, b = loads(args[0]) logger.debug("Loading args from pickle in:" + str(time.time() - start_time)) # Unpickle function from cloudpickle import loads as c_loads start_time = time.time() wrapped_fn = c_loads(args[1])[0] logger.debug("Loading punched function from pickle in:" + str(time.time() - start_time)) args = a kwargs = b logger.debug("Started wrapped function on process: " + str(os.getpid())) out = wrapped_fn(*args, **kwargs) return out, _pypads.cache else: return fn(*args, **kwargs)