def _is_runtime_resource_run_in_transient_state(
self, db: DBInterface, db_session: Session,
runtime_resource: Dict) -> bool:
"""
A runtime can have different underlying resources (like pods or CRDs) - to generalize we call it runtime
resource. This function will verify whether the Run object related to this runtime resource is in transient
state. This is useful in order to determine whether an object can be removed. for example, a kubejob's pod
might be in completed state, but we would like to verify that the run is completed as well to verify the logs
were collected before we're removing the pod.
"""
project, uid = self._resolve_runtime_resource_run(runtime_resource)
# if no uid, assume in stable state
if not uid:
return False
run = db.read_run(db_session, uid, project)
if run.get('status',
{}).get('state') not in FunctionStates.stable_phases():
return True
# give some grace period
now = datetime.now(timezone.utc)
last_update_str = run.get('status', {}).get('last_update', now)
last_update = datetime.fromisoformat(last_update_str)
if last_update + timedelta(seconds=float(
config.runtime_resources_deletion_grace_period)) > now:
return True
return False
def _ensure_runtime_resource_run_logs_collected(self, db: DBInterface,
db_session: Session,
runtime_resource: Dict):
project, uid = self._resolve_runtime_resource_run(runtime_resource)
# if cannot resolve related run, assume collected
if not uid:
return
# import here to avoid circular imports
import mlrun.api.crud as crud
log_file_exists = crud.Logs.log_file_exists(project, uid)
store_log = False
if not log_file_exists:
store_log = True
else:
log_mtime = crud.Logs.get_log_mtime(project, uid)
log_mtime_datetime = datetime.fromtimestamp(
log_mtime, timezone.utc)
now = datetime.now(timezone.utc)
run = db.read_run(db_session, uid, project)
last_update_str = run.get('status', {}).get('last_update', now)
last_update = datetime.fromisoformat(last_update_str)
# this function is used to verify that logs collected from runtime resources before deleting them
# here we're using the knowledge that the function is called only after a it was verified that the runtime
# resource run is not in transient state, so we're assuming the run's last update is the last one, so if the
# log file was modified after it, we're considering it as all logs collected
if log_mtime_datetime < last_update:
store_log = True
if store_log:
logger.debug(f'Storing runtime resource log before deletion')
logs_from_k8s, _ = crud.Logs.get_log(db_session,
project,
uid,
source=LogSources.K8S)
crud.Logs.store_log(logs_from_k8s, project, uid, append=False)
