def end_run(self):
"""
End the current run and run its tearDown functions.
:return:
"""
run = self.active_run()
# consolidated_dict = self.pypads.cache.get('consolidated_dict', None)
# if consolidated_dict is not None:
# # Dump data to disk
# self.log_mem_artifact("consolidated_log", consolidated_dict, write_format=FileFormats.json)
chached_fns = self._get_teardown_cache()
fn_list = [v for i, v in chached_fns.items()]
fn_list.sort(key=lambda t: t.order)
for fn in fn_list:
try:
fn(self.pypads,
_pypads_env=LoggerEnv(parameter=dict(),
experiment_id=run.info.experiment_id,
run_id=run.info.run_id),
data={"category": "TearDownFn"})
except (KeyboardInterrupt, Exception) as e:
logger.warning("Failed running post run function " +
fn.__name__ + " because of exception: " +
str(e))
mlflow.end_run()
# --- Clean tmp files in disk cache after run ---
folder = get_temp_folder(run)
if os.path.exists(folder):
import shutil
shutil.rmtree(folder)
def __init__(self):
import re
temp_folder = get_temp_folder()
if not os.path.isdir(temp_folder):
os.mkdir(temp_folder)
# TODO close file?
self.log = open(os.path.join(temp_folder, "logfile.log"), "a")
self.re = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]')
def finalize_output(pads, logger_call, output, *args, **kwargs):
from pypads.app.pypads import get_current_pads
pads = get_current_pads()
log_to: LogTO = pads.cache.run_get("std_out_logger")
path = os.path.join(get_temp_folder(), "logfile.log")
if os.path.isfile(path):
log_to.path = pads.api.log_artifact(path, description="StdOut log of the current run",
artifact_path=log_to.path)
output.logs = log_to.store()
output.store()
def download_tmp_artifacts(self, run_id, relative_path):
"""
Downloads the artifact at relative_path to a local temporary folder.
:param run_id:
:param relative_path:
:return:
"""
local_path = get_temp_folder(self.get_run(run_id))
if not os.path.exists(os.path.dirname(local_path)):
os.makedirs(os.path.dirname(local_path))
return self.download_artifacts(run_id=run_id,
relative_path=relative_path,
dst_path=local_path)
def finalize_output(pads, logger_call, output, *args, **kwargs):
logs = pads.cache.run_get("loguru_logger")
lid = pads.cache.run_get("loguru_logger_lid")
folder = get_temp_folder()
try:
from pypads.pads_loguru import logger_manager
logger_manager.remove(lid)
except Exception:
pass
import glob
for file in glob.glob(os.path.join(folder, "run_*.log")):
pads.api.log_artifact(file, description="Logs of the current run", artifact_path=logs.path)
output.logs = logs.store()
def _call(self, *args, _pypads_env: LoggerEnv, _logger_call, _logger_output, **kwargs):
pads = _pypads_env.pypads
if not pads.cache.run_exists("loguru_logger"):
std_out_logger = LogTO(parent=_logger_output)
pads.cache.run_add("loguru_logger", std_out_logger)
from pypads.utils.logging_util import get_temp_folder
folder = get_temp_folder()
from pypads.pads_loguru import logger_manager
lid = logger_manager.add(os.path.join(folder, "run_" + pads.api.active_run().info.run_id + ".log"),
rotation="50 MB",
enqueue=True)
pads.cache.run_add("loguru_logger_lid", lid)
else:
logger.warning("LoguruRSF already registered")
def _call(self, *args, _pypads_env: LoggerEnv, **kwargs):
pads = _pypads_env.pypads
file = os.path.join(get_temp_folder(), str(os.getpid()) + "_trace.txt")
proc = None
if platform == "linux" or platform == "linux2":
# https://stackoverflow.com/questions/4789837/how-to-terminate-a-python-subprocess-launched-with-shell-true
proc = subprocess.Popen(
['sudo strace -p ' + str(os.getpid()) + ' &> ' + file],
shell=True,
preexec_fn=os.setsid)
elif platform == "darwin":
proc = subprocess.Popen(
['sudo dtruss -f -p ' + str(os.getpid()) + ' 2> ' + file],
shell=True,
preexec_fn=os.setsid)
elif platform == "win32":
logger.warning("No tracing supported on windows currently.")
if proc:
pads.api.register_teardown(
"stop_dtrace_" + str(proc.pid),
STraceStop(_pypads_proc=proc, _pypads_trace_file=file))
if proc.poll() == 1:
logger.warning(
"Can't dtruss/strace without sudo rights. To enable tracking allow user to execute dtruss/strace "
"without sudo password with polkit or by modifiying visudo - /etc/sudoers:"
"username ALL=NOPASSWD: /usr/bin/dtruss. To get the path to dtruss you can use 'which dtruss'. "
"Be carefull about allowing permanent sudo rights to dtruss. This might introduce security risks."
)
def safety_hook():
"""
A None value indicates that the process hasn't terminated yet.
"""
if proc and proc.poll() is None:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
proc.terminate()
pads.add_exit_fn(safety_hook)
def finalize_output(pads, logger_call, output, *args, **kwargs):
pipeline: PipelineTO = pads.cache.run_get("pipeline")
from networkx import MultiDiGraph
network: MultiDiGraph = pipeline.nx_network
base_folder = get_temp_folder()
path = os.path.join(base_folder, "pipeline_graph.png")
if not os.path.exists(base_folder):
pathlib.Path(base_folder).mkdir(parents=True, exist_ok=True)
if is_package_available("agraph") and is_package_available(
"graphviz") and is_package_available("pygraphviz"):
from networkx.drawing.nx_agraph import to_agraph
agraph = to_agraph(network)
agraph.layout('dot')
agraph.draw(path)
pipeline.store_artifact(
path,
"pipeline_graph.png",
description=
"A depiction of the underlying pipeline of the experiment.")
output.pipeline = pipeline.store()