def get_logs(
self,
db_session: Session,
project: str,
uid: str,
size: int = -1,
offset: int = 0,
source: LogSources = LogSources.AUTO,
) -> typing.Tuple[str, bytes]:
"""
:return: Tuple with:
1. str of the run state (so watchers will know whether to continue polling for logs)
2. bytes of the logs themselves
"""
project = project or mlrun.mlconf.default_project
out = b""
log_file = log_path(project, uid)
data = get_db().read_run(db_session, uid, project)
if not data:
log_and_raise(HTTPStatus.NOT_FOUND.value, project=project, uid=uid)
run_state = data.get("status", {}).get("state", "")
if log_file.exists() and source in [LogSources.AUTO, LogSources.PERSISTENCY]:
with log_file.open("rb") as fp:
fp.seek(offset)
out = fp.read(size)
elif source in [LogSources.AUTO, LogSources.K8S]:
if get_k8s():
pods = get_k8s().get_logger_pods(project, uid)
if pods:
pod, pod_phase = list(pods.items())[0]
if pod_phase != PodPhases.pending:
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
return run_state, out
def store_log(
self, body: bytes, project: str, uid: str, append: bool = True,
):
project = project or mlrun.mlconf.default_project
log_file = log_path(project, uid)
log_file.parent.mkdir(parents=True, exist_ok=True)
mode = "ab" if append else "wb"
with log_file.open(mode) as fp:
fp.write(body)
def get_log(
db_session: Session,
project: str,
uid: str,
size: int = -1,
offset: int = 0,
source: LogSources = LogSources.AUTO,
):
out = b""
log_file = log_path(project, uid)
status = None
if log_file.exists() and source in [
LogSources.AUTO, LogSources.PERSISTENCY
]:
with log_file.open("rb") as fp:
fp.seek(offset)
out = fp.read(size)
status = ""
elif source in [LogSources.AUTO, LogSources.K8S]:
data = get_db().read_run(db_session, uid, project)
if not data:
log_and_raise(HTTPStatus.NOT_FOUND, project=project, uid=uid)
status = get_in(data, "status.state", "")
if get_k8s():
pods = get_k8s().get_logger_pods(uid)
if pods:
pod, new_status = list(pods.items())[0]
new_status = new_status.lower()
# TODO: handle in cron/tracking
if new_status != "pending":
resp = get_k8s().logs(pod)
if resp:
out = resp.encode()[offset:]
if status == "running":
now = now_date().isoformat()
update_in(data, "status.last_update", now)
if new_status == "failed":
update_in(data, "status.state", "error")
update_in(data, "status.error",
"error, check logs")
get_db().store_run(db_session, data, uid,
project)
if new_status == "succeeded":
update_in(data, "status.state", "completed")
get_db().store_run(db_session, data, uid,
project)
status = new_status
elif status == "running":
update_in(data, "status.state", "error")
update_in(data, "status.error",
"pod not found, maybe terminated")
get_db().store_run(db_session, data, uid, project)
status = "failed"
return out, status
def log_file_exists(self, project: str, uid: str) -> bool:
log_file = log_path(project, uid)
return log_file.exists()
def get_log_mtime(self, project: str, uid: str) -> int:
log_file = log_path(project, uid)
if not log_file.exists():
raise FileNotFoundError(f"Log file does not exist: {log_file}")
return log_file.stat().st_mtime
def store_log(body: bytes, project: str, uid: str, append: bool = True):
log_file = log_path(project, uid)
log_file.parent.mkdir(parents=True, exist_ok=True)
mode = "ab" if append else "wb"
with log_file.open(mode) as fp:
fp.write(body)