def end_run(self): """ End the current run and run its tearDown functions. :return: """ run = self.active_run() # consolidated_dict = self.pypads.cache.get('consolidated_dict', None) # if consolidated_dict is not None: # # Dump data to disk # self.log_mem_artifact("consolidated_log", consolidated_dict, write_format=FileFormats.json) chached_fns = self._get_teardown_cache() fn_list = [v for i, v in chached_fns.items()] fn_list.sort(key=lambda t: t.order) for fn in fn_list: try: fn(self.pypads, _pypads_env=LoggerEnv(parameter=dict(), experiment_id=run.info.experiment_id, run_id=run.info.run_id), data={"category": "TearDownFn"}) except (KeyboardInterrupt, Exception) as e: logger.warning("Failed running post run function " + fn.__name__ + " because of exception: " + str(e)) mlflow.end_run() # --- Clean tmp files in disk cache after run --- folder = get_temp_folder(run) if os.path.exists(folder): import shutil shutil.rmtree(folder)
def __init__(self): import re temp_folder = get_temp_folder() if not os.path.isdir(temp_folder): os.mkdir(temp_folder) # TODO close file? self.log = open(os.path.join(temp_folder, "logfile.log"), "a") self.re = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]')
def finalize_output(pads, logger_call, output, *args, **kwargs): from pypads.app.pypads import get_current_pads pads = get_current_pads() log_to: LogTO = pads.cache.run_get("std_out_logger") path = os.path.join(get_temp_folder(), "logfile.log") if os.path.isfile(path): log_to.path = pads.api.log_artifact(path, description="StdOut log of the current run", artifact_path=log_to.path) output.logs = log_to.store() output.store()
def download_tmp_artifacts(self, run_id, relative_path): """ Downloads the artifact at relative_path to a local temporary folder. :param run_id: :param relative_path: :return: """ local_path = get_temp_folder(self.get_run(run_id)) if not os.path.exists(os.path.dirname(local_path)): os.makedirs(os.path.dirname(local_path)) return self.download_artifacts(run_id=run_id, relative_path=relative_path, dst_path=local_path)
def finalize_output(pads, logger_call, output, *args, **kwargs): logs = pads.cache.run_get("loguru_logger") lid = pads.cache.run_get("loguru_logger_lid") folder = get_temp_folder() try: from pypads.pads_loguru import logger_manager logger_manager.remove(lid) except Exception: pass import glob for file in glob.glob(os.path.join(folder, "run_*.log")): pads.api.log_artifact(file, description="Logs of the current run", artifact_path=logs.path) output.logs = logs.store()
def _call(self, *args, _pypads_env: LoggerEnv, _logger_call, _logger_output, **kwargs): pads = _pypads_env.pypads if not pads.cache.run_exists("loguru_logger"): std_out_logger = LogTO(parent=_logger_output) pads.cache.run_add("loguru_logger", std_out_logger) from pypads.utils.logging_util import get_temp_folder folder = get_temp_folder() from pypads.pads_loguru import logger_manager lid = logger_manager.add(os.path.join(folder, "run_" + pads.api.active_run().info.run_id + ".log"), rotation="50 MB", enqueue=True) pads.cache.run_add("loguru_logger_lid", lid) else: logger.warning("LoguruRSF already registered")
def _call(self, *args, _pypads_env: LoggerEnv, **kwargs): pads = _pypads_env.pypads file = os.path.join(get_temp_folder(), str(os.getpid()) + "_trace.txt") proc = None if platform == "linux" or platform == "linux2": # https://stackoverflow.com/questions/4789837/how-to-terminate-a-python-subprocess-launched-with-shell-true proc = subprocess.Popen( ['sudo strace -p ' + str(os.getpid()) + ' &> ' + file], shell=True, preexec_fn=os.setsid) elif platform == "darwin": proc = subprocess.Popen( ['sudo dtruss -f -p ' + str(os.getpid()) + ' 2> ' + file], shell=True, preexec_fn=os.setsid) elif platform == "win32": logger.warning("No tracing supported on windows currently.") if proc: pads.api.register_teardown( "stop_dtrace_" + str(proc.pid), STraceStop(_pypads_proc=proc, _pypads_trace_file=file)) if proc.poll() == 1: logger.warning( "Can't dtruss/strace without sudo rights. To enable tracking allow user to execute dtruss/strace " "without sudo password with polkit or by modifiying visudo - /etc/sudoers:" "username ALL=NOPASSWD: /usr/bin/dtruss. To get the path to dtruss you can use 'which dtruss'. " "Be carefull about allowing permanent sudo rights to dtruss. This might introduce security risks." ) def safety_hook(): """ A None value indicates that the process hasn't terminated yet. """ if proc and proc.poll() is None: os.killpg(os.getpgid(proc.pid), signal.SIGTERM) proc.terminate() pads.add_exit_fn(safety_hook)
def finalize_output(pads, logger_call, output, *args, **kwargs): pipeline: PipelineTO = pads.cache.run_get("pipeline") from networkx import MultiDiGraph network: MultiDiGraph = pipeline.nx_network base_folder = get_temp_folder() path = os.path.join(base_folder, "pipeline_graph.png") if not os.path.exists(base_folder): pathlib.Path(base_folder).mkdir(parents=True, exist_ok=True) if is_package_available("agraph") and is_package_available( "graphviz") and is_package_available("pygraphviz"): from networkx.drawing.nx_agraph import to_agraph agraph = to_agraph(network) agraph.layout('dot') agraph.draw(path) pipeline.store_artifact( path, "pipeline_graph.png", description= "A depiction of the underlying pipeline of the experiment.") output.pipeline = pipeline.store()