def activate_tracking(self, reload_modules=False, reload_warnings=True, clear_imports=False, affected_modules=None): """ Function to duck punch all objects defined in the mapping files. This should at best be called before importing any libraries. :param affected_modules: Affected modules of the mapping files. :param clear_imports: Clear imports after punching. CAREFUL THIS IS EXPERIMENTAL! :param reload_warnings: Show warnings of affected modules which were already imported before the importlib was extended. :param reload_modules: Force a reload of affected modules. CAREFUL THIS IS EXPERIMENTAL! :return: """ if affected_modules is None: # Modules are affected if they are mapped by a library or are already punched affected_modules = self.wrap_manager.module_wrapper.punched_module_names | \ {l.name for l in self.mapping_registry.get_libraries()} global tracking_active if not tracking_active: logger.info("Activating tracking by extending importlib...") from pypads.app.pypads import set_current_pads set_current_pads(self) # Add our loader to the meta_path extend_import_module() import sys import importlib loaded_modules = [(name, module) for name, module in sys.modules.items()] for name, module in loaded_modules: if self.is_affected_module(name, affected_modules): if reload_warnings: logger.warning( name + " was imported before PyPads. To enable tracking import PyPads before or use " "reload_modules / clear_imports. Every already created instance is not tracked." ) if clear_imports: del sys.modules[name] if reload_modules: try: spec = importlib.util.find_spec(module.__name__) duck_punch_loader(spec) loader = spec.loader module = loader.load_module(module.__name__) loader.exec_module(module) importlib.reload(module) except Exception as e: logger.debug("Couldn't reload module " + str(e)) tracking_active = True else: # TODO check if a second tracker / tracker activation doesn't break the tracking logger.warning("Tracking was already activated.") return self
def tearDown(self): # TODO isn't run on unexpected errors from pypads.app.pypads import current_pads, set_current_pads if current_pads: current_pads.deactivate_tracking(run_atexits=True, reload_modules=False) # noinspection PyTypeChecker set_current_pads(None)
def deactivate_tracking(self, run_atexits=False, reload_modules=True): """ Deacticate the current tracking and cleanup. :param run_atexits: Run the registered atexit functions of pypads :param reload_modules: Force a reload of affected modules :return: """ # run atexit fns if needed if run_atexits: self.run_exit_fns() # Remove atexit fns for fn in self._atexit_fns: atexit.unregister(fn) import sys import importlib loaded_modules = [(name, module) for name, module in sys.modules.items()] for name, module in loaded_modules: if self.is_affected_module(name): del sys.modules[name] if reload_modules: # reload modules if they where affected try: spec = importlib.util.find_spec(module.__name__) duck_punch_loader(spec) loader = spec.loader module = loader.load_module(module.__name__) loader.exec_module(module) importlib.reload(module) except Exception as e: logger.debug("Couldn't reload module " + str(e)) global tracking_active tracking_active = False # noinspection PyTypeChecker from pypads.app.pypads import set_current_pads set_current_pads(None)
def __init__(self, uri=None, folder=None, mappings: List[MappingCollection] = None, hooks=None, events=None, setup_fns=None, config=None, pre_initialized_cache: PypadsCache = None, disable_plugins=None, autostart=None, log_level="WARNING", *args, **kwargs): from pypads.app.pypads import set_current_pads set_current_pads(self) from pypads.pads_loguru import logger_manager self._default_logger = logger_manager.add_default_logger( level=log_level) self._instance_modifiers = [] # Init variable to filled later in this constructor self._atexit_fns = [] # Init WrapManager self._wrap_manager = WrapManager(self) # Init API self._api = ApiPluginManager() # Init Decorators self._decorators = DecoratorPluginManager() # Init Actuators self._actuators = ActuatorPluginManager() # Init Validators self._validators = ValidatorPluginManager() # Init Results self._results = ResultPluginManager() # Init CallTracker self._call_tracker = CallTracker(self) # Init cache self._cache = pre_initialized_cache if pre_initialized_cache else PypadsCache( ) # Store folder into cache self._cache.add("folder", folder or PYPADS_FOLDER) # Store uri into cache self._cache.add("uri", uri or PYPADS_URI) # Store config into cache self.config = { **DEFAULT_CONFIG, **config } if config else DEFAULT_CONFIG # Enable git tracking of the current repository from pypads.app.misc.managed_git import ManagedGitFactory self._managed_git_factory = ManagedGitFactory(self) from pypads.app.backends.mlflow import MLFlowBackendFactory self._backend = MLFlowBackendFactory.make(self.uri) # Store config into cache self._cache.add("mappings", mappings) # Store hook registry into cache self._cache.add("hooks", hooks) # Store function registry into cache self._cache.add("events", events) self._library_repository = LibraryRepository() self._schema_repository = SchemaRepository() self._logger_repository = LoggerRepository() # Activate the discovered plugins if disable_plugins is None: # Temporarily disabling pypads_onto disable_plugins = ['pypads_onto'] for name, plugin in discovered_plugins.items(): if name not in disable_plugins: plugin.activate(self, *args, **kwargs) # Init mapping registry and repository self._mapping_repository = MappingRepository() self._mapping_registry = MappingRegistry.from_params(self, mappings) # Init hook registry self._hook_registry = HookRegistry.from_dict(self, hooks) # Init function registry self._function_registry = FunctionRegistry.from_dict(self, events) # Store config into cache self._cache.add("mappings", mappings) # Store hook registry into cache self._cache.add("hooks", hooks) # Store function registry into cache self._cache.add("events", events) # Initialize pre run functions before starting a run setup_fns = DEFAULT_SETUP_FNS if setup_fns is None else setup_fns for fn in setup_fns: self.api.register_setup(fn.__class__.__name__ + "_" + str(id(fn)), fn) # Execute instance modification functions given by a plugin for fn in self._instance_modifiers: fn(self) # Activate tracking by punching the import lib if autostart: self.activate_tracking() # Add cleanup functions def cleanup(): from pypads.app.pypads import get_current_pads pads: PyPads = get_current_pads() if pads.api.active_run(): pads.api.end_run() self.add_exit_fn(cleanup) # SIGKILL and SIGSTOP are not catchable signal.signal(signal.SIGTERM, self.run_exit_fns) signal.signal(signal.SIGINT, self.run_exit_fns) signal.signal(signal.SIGQUIT, self.run_exit_fns) if autostart: if isinstance(autostart, str): self.start_track(autostart) else: self.start_track()