def check_determinism(): from pypads.app.pypads import get_current_pads pads = get_current_pads() if is_package_available('tensorflow'): import tensorflow tf_version = tensorflow.version.VERSION if tensorflow.match("(1\.(14|15)|2\.0)", tf_version): if "TF_USE_CUDNN_AUTOTUNE" in os.environ: logger.warning( "When using TF auto-tuning of cuDNN convolution algorithms your experiment might" " be non-deterministic.") pads.api.set_tag("non-determinism", "CUDNN_AUTOTUNE") if ("TF_CUDNN_DETERMINISTIC" not in os.environ or (not os.environ["TF_CUDNN_DETERMINISTIC"] and os.environ["TF_CUDNN_DETERMINISTIC"] is not 1)): if not is_package_available("tfdeterminism"): logger.warning( "Your experiment might include a gpu-specific sources of non-determinism." " See https://github.com/NVIDIA/tensorflow-determinism" ) pads.api.set_tag( "non-determinism", "TF auto-tuning of cuDNN convolution algorithms (see multi-algo note)" )
def _wrapped_inner_function(_self, *args, _pypads_env: InjectionLoggerEnv, **kwargs): """ Wrapped function logic. This executes all hooks and the function itself. :param _self: Reference to :param args: :param kwargs: :return: """ env = _pypads_env call = env.call if not call.has_hook(env.hook): call.add_hook(env.hook) try: # check for name collision in parameters if set([k for k, v in kwargs.items()]) & set( [k for k, v in env.parameter.items()]): logger.warning("Hook parameter is overwriting a parameter in the standard " "model call. This most likely will produce side effects.") if env.hook: return env.hook(_self, _pypads_env=_pypads_env, *args, **kwargs) return env.callback(*args, **kwargs) finally: call.remove_hook(env.hook) else: return env.callback(*args, **kwargs)
def end_run(self): """ End the current run and run its tearDown functions. :return: """ run = self.active_run() # consolidated_dict = self.pypads.cache.get('consolidated_dict', None) # if consolidated_dict is not None: # # Dump data to disk # self.log_mem_artifact("consolidated_log", consolidated_dict, write_format=FileFormats.json) chached_fns = self._get_teardown_cache() fn_list = [v for i, v in chached_fns.items()] fn_list.sort(key=lambda t: t.order) for fn in fn_list: try: fn(self.pypads, _pypads_env=LoggerEnv(parameter=dict(), experiment_id=run.info.experiment_id, run_id=run.info.run_id), data={"category": "TearDownFn"}) except (KeyboardInterrupt, Exception) as e: logger.warning("Failed running post run function " + fn.__name__ + " because of exception: " + str(e)) mlflow.end_run() # --- Clean tmp files in disk cache after run --- folder = get_temp_folder(run) if os.path.exists(folder): import shutil shutil.rmtree(folder)
def data_path(data, *path, default=None, warning=None): """ Gets an data item of given dict at path :param data: :param path: :param default: :param warning: :return: """ cur = data for i, p in enumerate(path): if isinstance(cur, list): out = [] for list_element in cur: value = data_path(list_element, *path[i:]) if value is not None: if isinstance(value, list) and not len(path[i:]) == 0: out.extend(value) else: out.append(value) return out if len(out) > 0 else default elif p in cur: # If list recursively call itself # Multiple return values needed instead of one cur = cur[p] else: if warning is not None: logger.warning(warning) return default return cur
def commit(pads, *args, **kwargs): message = "Added results for run " + pads.api.active_run( ).info.run_id pads.managed_result_git.commit_changes(message=message) repo = pads.managed_result_git.repo remotes = repo.remotes if not remotes: logger.warning( "Your results don't have any remote repository set. Set a remote repository for" "to enable automatic pushing.") else: for remote in remotes: name, url = remote.name, list(remote.urls)[0] try: # check if remote repo is bare and if it is initialize it with a temporary local repo pads.managed_result_git.is_remote_empty(remote=name, remote_url=url, init=True) # stash current state repo.git.stash('push', '--include-untracked') # Force pull repo.git.pull(name, 'master', '--allow-unrelated-histories') # Push merged changes repo.git.push(name, 'master') logger.info("Pushed your results automatically to " + name + " @:" + url) # pop the stash repo.git.stash('pop') except Exception as e: logger.error( "pushing logs to remote failed due to this error '{}'" .format(str(e)))
def splitter_output(result, fn): # check if the output of the splitter is a tuple of indices try: if isinstance(result, Tuple): if "sklearn" in fn.__module__: return result[0].tolist(), result[1].tolist(), None elif "default_split" in fn.__name__: return result else: if len(result) < 4: result_ = [_tolist(r) for r in result] return tuple(result_ + [None] * (3 - len(result_))) else: return None, None, None else: if "torch" in fn.__module__: if hasattr(fn, "_dataset"): if hasattr(fn._dataset, "train"): if fn._dataset.train: return _tolist(result), None, None else: return None, _tolist(result), None return _tolist(result), None, None else: return None, None, None except Exception as e: logger.warning("Split tracking ommitted due to exception {}".format( str(e))) return None, None, None
def numpy_crawler(obj: Crawler, target_columns=None, **kwargs): logger.info("Detecting a dataset object of type 'numpy.ndarray'. Crawling any available metadata...") # , (obj.data[:, i].min(), obj.data[:, i].max()) if len(obj.data.shape) == 2: features = [(str(i), str(obj.data[:, i].dtype), False) for i in range(obj.data.shape[1])] else: # TODO for multidim datasets features = None metadata = {"type": str(obj.format), "shape": obj.data.shape, "features": features} metadata = {**metadata, **kwargs} targets = None try: if target_columns: targets = obj.data[:, target_columns] if isinstance(target_columns, Iterable): for c in target_columns: feature = metadata["features"][c] metadata["features"][c] = (feature[0], feature[1], True) else: feature = metadata["features"][target_columns] metadata["features"][target_columns] = (feature[0], feature[1], True) except Exception as e: logger.warning(str(e)) return obj.data, metadata, targets
def activate_tracking(self, reload_modules=False, reload_warnings=True, clear_imports=False, affected_modules=None): """ Function to duck punch all objects defined in the mapping files. This should at best be called before importing any libraries. :param affected_modules: Affected modules of the mapping files. :param clear_imports: Clear imports after punching. CAREFUL THIS IS EXPERIMENTAL! :param reload_warnings: Show warnings of affected modules which were already imported before the importlib was extended. :param reload_modules: Force a reload of affected modules. CAREFUL THIS IS EXPERIMENTAL! :return: """ if affected_modules is None: # Modules are affected if they are mapped by a library or are already punched affected_modules = self.wrap_manager.module_wrapper.punched_module_names | \ {l.name for l in self.mapping_registry.get_libraries()} global tracking_active if not tracking_active: logger.info("Activating tracking by extending importlib...") from pypads.app.pypads import set_current_pads set_current_pads(self) # Add our loader to the meta_path extend_import_module() import sys import importlib loaded_modules = [(name, module) for name, module in sys.modules.items()] for name, module in loaded_modules: if self.is_affected_module(name, affected_modules): if reload_warnings: logger.warning( name + " was imported before PyPads. To enable tracking import PyPads before or use " "reload_modules / clear_imports. Every already created instance is not tracked." ) if clear_imports: del sys.modules[name] if reload_modules: try: spec = importlib.util.find_spec(module.__name__) duck_punch_loader(spec) loader = spec.loader module = loader.load_module(module.__name__) loader.exec_module(module) importlib.reload(module) except Exception as e: logger.debug("Couldn't reload module " + str(e)) tracking_active = True else: # TODO check if a second tracker / tracker activation doesn't break the tracking logger.warning("Tracking was already activated.") return self
def read_pickle(p): try: with open(p, "rb") as fd: return pickle.load(fd) except FileNotFoundError: return None except Exception as e: logger.warning("Couldn't read pickle file. " + str(e))
def test_3d_mnist(self): # --------------------------- setup of the tracking --------------------------- try: import timeit t = timeit.Timer(torch_3d_mnist_example) print(t.timeit(1)) except RuntimeError as e: logger.warning("Torch bug on re-import: {}".format(str(e)))
def commit_changes(self, message=""): try: self.add_untracked_files() self._commit(message) except Exception as e: logger.warning("Failed to commit due to following exception: %s" % str(e)) pass
def _add_git_ignore(self): try: with open(self.repo.working_dir + "/.gitignore", "w") as file: file.write(GIT_IGNORE) self.repo.git.add(A=True) except Exception as e: logger.warning( "Could add .gitignore file to the repo due to this %s" % str(e))
def write_pickle(p, o): try: with open(p + ".pickle", "wb+") as fd: pickle.dump(o, fd) return fd.name except Exception as e: logger.warning( "Couldn't pickle output. Trying to save toString instead. " + str(e)) return write_text(p, o)
def _handle_error(self, *args, ctx, _pypads_env, error, **kwargs): if isinstance(error, StopIteration): logger.warning( "Ignoring recovery of this StopIteration error: {}".format( error)) original = _pypads_env.call.call_id.context.original( _pypads_env.callback) return original(ctx, *args, **kwargs) else: super()._handle_error(*args, ctx, _pypads_env, error, **kwargs)
def read_json(p): try: with open(p, "r") as fd: return json.load(fd) except FileNotFoundError: return None except Exception as e: logger.warning( "Couldn't read artifact as json. Trying to read it as text instead. " + str(e)) return read_text(p)
def __new__(cls, *args, **kwargs): try: from pypads.app.pypads import get_current_pads if get_current_pads() is not None: logger.warning( "Currently only one tracker can be activated at once." "PyPads was already intialized. Reusing the old instance.") return get_current_pads() except Exception as e: pass return super().__new__(cls)
def _add_json_ld(self, entry, json_ld, graph): for j in json_ld: if store_hash(graph.identifier, str(j)): j["@context"] = self._convert_context( dict_merge( entry.context, j["@context"])) if "@context" in j else entry.context try: graph.parse(data=dumps(j), format="json-ld") except Exception: logger.warning(f"Couldn't translate {j} to rdf.")
def read_artifact(path, read_format: FileFormats = None): if read_format is None: file_extension = path.split('.')[-1] read_format = get_by_value_in_enum(file_extension, FileFormats) if not read_format: logger.warning("Configured read format " + read_format + " not supported! ") return try: data = readers[read_format](path) except Exception as e: data = None return data
def __init__(self, name, description): """ Constructor for an anchor. :param name: Name of the anchor. :param description: String describing the anchor and its purpose. """ self._name = name self._description = description if self._name in anchors: logger.warning("Anchor with name {} already exists".format( self._name)) pass anchors[self._name] = self
def write_json(p, o): try: with open(p + ".json", "w+") as fd: if isinstance(o, str): fd.write(o) # TODO check if valid json? else: json.dump(o, fd) return fd.name except Exception as e: logger.warning( "Couldn't write meta as json. Trying to save it as text instead. " + str(e)) return write_text(p, o)
def dataframe_crawler(obj: Crawler, target_columns, **kwargs): logger.info("Detecting a dataset object of type 'pandas.DataFrame'. Crawling any available metadata...") data = obj.data features = [] for i, col in enumerate(data.columns): flag = col in target_columns if target_columns is not None else False features.append((col, str(data[col].dtype), flag)) metadata = {"type": str(obj.format), "shape": data.shape, "features": features} metadata = {**metadata, **kwargs} targets = None if target_columns is not None: targets = data[target_columns].values else: logger.warning("Target values might be innaccurate or not tracked.") return data, metadata, targets
def test_torch_sequential_class(self): # --------------------------- setup of the tracking --------------------------- # Activate tracking of pypads from pypads.app.base import PyPads tracker = PyPads(autostart="MNIST-Torch", setup_fns=[]) try: import timeit t = timeit.Timer(torch_simple_example) print(t.timeit(1)) except RuntimeError as e: logger.warning("Torch bug on re-import: {}".format(str(e))) # --------------------------- asserts --------------------------- # TODO Add asserts # !-------------------------- asserts --------------------------- tracker.api.end_run()
def _call(self, *args, _pypads_env: LoggerEnv, _logger_call, _logger_output, **kwargs): pads = _pypads_env.pypads if not pads.cache.run_exists("loguru_logger"): std_out_logger = LogTO(parent=_logger_output) pads.cache.run_add("loguru_logger", std_out_logger) from pypads.utils.logging_util import get_temp_folder folder = get_temp_folder() from pypads.pads_loguru import logger_manager lid = logger_manager.add(os.path.join(folder, "run_" + pads.api.active_run().info.run_id + ".log"), rotation="50 MB", enqueue=True) pads.cache.run_add("loguru_logger_lid", lid) else: logger.warning("LoguruRSF already registered")
def start_track(self, experiment_name=None, disable_run_init=False): """ Start a new run to track. :param experiment_name: The name of the mlflow experiment :param disable_run_init: Flag to indicate if the run_init functions are to be run on an already existing run. :return: """ if not tracking_active: self.activate_tracking() # check if there is already an active run run = mlflow.active_run() experiment = None if run is None: experiment_name = experiment_name or DEFAULT_EXPERIMENT_NAME # Create run if run doesn't already exist experiment = mlflow.get_experiment_by_name(experiment_name) experiment_id = experiment.experiment_id if experiment else mlflow.create_experiment( experiment_name) run = self.api.start_run(experiment_id=experiment_id) else: # if not disable_run_init: # self.api.run_setups(_pypads_env=LoggerEnv(parameter=dict(), experiment_id=experiment_id, run_id=run_id, # data={"category": "SetupFn"})) if experiment_name: # Check if we're still in the same experiment experiment = mlflow.get_experiment_by_name(experiment_name) experiment_id = experiment.experiment_id if experiment else mlflow.create_experiment( experiment_name) if run.info.experiment_id != experiment_id: experiment = mlflow.get_experiment_by_name(experiment_name) if experiment is None: experiment = self.backend.get_experiment(run.info.experiment_id) # override active run if used if experiment_name and run.info.experiment_id != experiment.experiment_id: logger.warning( "Active experiment_id of run doesn't match given input name " + experiment_name + ". Recreating new run.") try: self.api.start_run(experiment_id=experiment.experiment_id, nested=True) except Exception: mlflow.end_run() self.api.start_run(experiment_id=experiment.experiment_id) return self
def _to_node_label(wrappee, ref): if ref is not None: try: return str(wrappee) + str(id(ref)) except Exception as e: logger.warning( "Couldn't get the representation of wrappee. Fallback to id " + str(id(ref)) + ". " + str(e)) return str(id(wrappee)) + str(id(ref)) else: try: return str(wrappee) except Exception as e: logger.warning( "Couldn't get representation of the wrappee. Fallback to id " + str(id(wrappee)) + ". " + str(e)) return str(id(wrappee))
def _call(self, *args, _pypads_env: LoggerEnv, _logger_call, _logger_output, **kwargs): from pypads.app.pypads import get_current_pads pads = get_current_pads() if not pads.cache.run_exists("std_out_logger"): std_out_logger = LogTO(parent=_logger_output, path="logfile.log") pads.cache.run_add("std_out_logger", std_out_logger) else: logger.warning("StdOutRSF already registered") return import sys class Logger(object): def __init__(self): import re temp_folder = get_temp_folder() if not os.path.isdir(temp_folder): os.mkdir(temp_folder) # TODO close file? self.log = open(os.path.join(temp_folder, "logfile.log"), "a") self.re = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]') def write(self, message): self.log.write(self.re.sub('', message)) self.log.flush() def flush(self): # this flush method is needed for python 3 compatibility. # this handles the flush command by doing nothing. # you might want to specify some extra behavior here. pass stdout_logger = Logger() if hasattr(sys.stdout,'original_write'): original_function = getattr(sys.stdout, 'original_write') else: original_function = getattr(sys.stdout, 'write') setattr(sys.stdout, 'original_write', original_function) def modified_function(message): stdout_logger.write(message) sys.stdout.original_write(message) setattr(sys.stdout, 'write', modified_function)
def __pre__(self, ctx, *args, _logger_call, _logger_output, _args, _kwargs, **kwargs): """ :param ctx: :param args: :param kwargs: :return: """ from pypads.app.pypads import get_current_pads pads = get_current_pads() probabilities = None try: probabilities = ctx.predict(*_args, **_kwargs) except Exception as e: logger.warning("Couldn't compute probabilities because %s" % str(e)) pads.cache.run_add("probabilities", probabilities)
def add_result_remote(self, remote, uri): """ Add a remote to track the results. :param remote: Remote name to be added :param uri: Remote address to be added :return: """ if self.managed_result_git is None: logger.warning( "Can only add remotes to the result directory if it is managed by pypads git." ) else: try: self.managed_result_git.remote = remote self.managed_result_git.remote_uri = uri self.managed_result_git.repo.create_remote(remote, uri) except Exception as e: logger.warning("Failed to add remote due to exception: " + str(e))
def function_type(self): """ Get the function type of the accessor function. :return: """ # Return if already found if self._function_type: return self._function_type if self.context.is_module(): function_type = "staticmethod" else: # Get the function type (Method, unbound etc.) try: real_ctx = self.real_context() if real_ctx is None: raise ValueError("Couldn't find real context.") function_type = type( real_ctx.get_dict()[self.wrappee.__name__]) except Exception as e: logger.warning("Couldn't get function type of '" + str(self.wrappee.__name__) + "' on '" + str(self.real_context()) + ". Omit logging. " + str(e)) return None # TODO Can we find less error prone ways to get the type of the given fn? # Delegate decorator of sklearn obfuscates the real type. # if is_package_available("sklearn"): # from sklearn.utils.metaestimators import _IffHasAttrDescriptor # if function_type == _IffHasAttrDescriptor: if str( function_type ) == "<class 'sklearn.utils.metaestimators._IffHasAttrDescriptor'>": function_type = "wrapped" self.wrappee = self.real_context().get_dict()[ self.wrappee.__name__] # Set cached result self._function_type = function_type return function_type
def _call(self, *args, _pypads_env: LoggerEnv, **kwargs): pads = _pypads_env.pypads file = os.path.join(get_temp_folder(), str(os.getpid()) + "_trace.txt") proc = None if platform == "linux" or platform == "linux2": # https://stackoverflow.com/questions/4789837/how-to-terminate-a-python-subprocess-launched-with-shell-true proc = subprocess.Popen( ['sudo strace -p ' + str(os.getpid()) + ' &> ' + file], shell=True, preexec_fn=os.setsid) elif platform == "darwin": proc = subprocess.Popen( ['sudo dtruss -f -p ' + str(os.getpid()) + ' 2> ' + file], shell=True, preexec_fn=os.setsid) elif platform == "win32": logger.warning("No tracing supported on windows currently.") if proc: pads.api.register_teardown( "stop_dtrace_" + str(proc.pid), STraceStop(_pypads_proc=proc, _pypads_trace_file=file)) if proc.poll() == 1: logger.warning( "Can't dtruss/strace without sudo rights. To enable tracking allow user to execute dtruss/strace " "without sudo password with polkit or by modifiying visudo - /etc/sudoers:" "username ALL=NOPASSWD: /usr/bin/dtruss. To get the path to dtruss you can use 'which dtruss'. " "Be carefull about allowing permanent sudo rights to dtruss. This might introduce security risks." ) def safety_hook(): """ A None value indicates that the process hasn't terminated yet. """ if proc and proc.poll() is None: os.killpg(os.getpgid(proc.pid), signal.SIGTERM) proc.terminate() pads.add_exit_fn(safety_hook)