def _is_runtime_resource_run_in_transient_state( self, db: DBInterface, db_session: Session, runtime_resource: Dict) -> bool: """ A runtime can have different underlying resources (like pods or CRDs) - to generalize we call it runtime resource. This function will verify whether the Run object related to this runtime resource is in transient state. This is useful in order to determine whether an object can be removed. for example, a kubejob's pod might be in completed state, but we would like to verify that the run is completed as well to verify the logs were collected before we're removing the pod. """ project, uid = self._resolve_runtime_resource_run(runtime_resource) # if no uid, assume in stable state if not uid: return False run = db.read_run(db_session, uid, project) if run.get('status', {}).get('state') not in FunctionStates.stable_phases(): return True # give some grace period now = datetime.now(timezone.utc) last_update_str = run.get('status', {}).get('last_update', now) last_update = datetime.fromisoformat(last_update_str) if last_update + timedelta(seconds=float( config.runtime_resources_deletion_grace_period)) > now: return True return False
def _ensure_runtime_resource_run_logs_collected(self, db: DBInterface, db_session: Session, runtime_resource: Dict): project, uid = self._resolve_runtime_resource_run(runtime_resource) # if cannot resolve related run, assume collected if not uid: return # import here to avoid circular imports import mlrun.api.crud as crud log_file_exists = crud.Logs.log_file_exists(project, uid) store_log = False if not log_file_exists: store_log = True else: log_mtime = crud.Logs.get_log_mtime(project, uid) log_mtime_datetime = datetime.fromtimestamp( log_mtime, timezone.utc) now = datetime.now(timezone.utc) run = db.read_run(db_session, uid, project) last_update_str = run.get('status', {}).get('last_update', now) last_update = datetime.fromisoformat(last_update_str) # this function is used to verify that logs collected from runtime resources before deleting them # here we're using the knowledge that the function is called only after a it was verified that the runtime # resource run is not in transient state, so we're assuming the run's last update is the last one, so if the # log file was modified after it, we're considering it as all logs collected if log_mtime_datetime < last_update: store_log = True if store_log: logger.debug(f'Storing runtime resource log before deletion') logs_from_k8s, _ = crud.Logs.get_log(db_session, project, uid, source=LogSources.K8S) crud.Logs.store_log(logs_from_k8s, project, uid, append=False)