def resolve_mpijob_crd_version(api_context=False):
global cached_mpijob_crd_version
if not cached_mpijob_crd_version:
# config override everything
mpijob_crd_version = config.mpijob_crd_version
if not mpijob_crd_version:
in_k8s_cluster = get_k8s_helper(silent=True).is_running_inside_kubernetes_cluster()
if in_k8s_cluster:
k8s_helper = get_k8s_helper()
namespace = k8s_helper.resolve_namespace()
# try resolving according to mpi-operator that's running
res = k8s_helper.list_pods(namespace=namespace, selector='component=mpi-operator')
if len(res) > 0:
mpi_operator_pod = res[0]
mpijob_crd_version = mpi_operator_pod.metadata.labels.get('crd-version')
elif not in_k8s_cluster and not api_context:
# connect will populate the config from the server config
# TODO: something nicer
get_run_db().connect()
mpijob_crd_version = config.mpijob_crd_version
# If resolution failed simply use default
if not mpijob_crd_version:
mpijob_crd_version = MPIJobCRDVersions.default()
if mpijob_crd_version not in MPIJobCRDVersions.all():
raise ValueError(f'unsupported mpijob crd version: {mpijob_crd_version}. '
f'supported versions: {MPIJobCRDVersions.all()}')
cached_mpijob_crd_version = mpijob_crd_version
return cached_mpijob_crd_version
def list_pipelines(
project: str,
namespace: str = None,
sort_by: str = "",
page_token: str = "",
filter_: str = Query("", alias="filter"),
format_: mlrun.api.schemas.Format = Query(
mlrun.api.schemas.Format.metadata_only, alias="format"),
page_size: int = Query(None, gt=0, le=200),
):
total_size, next_page_token, runs = None, None, None
if get_k8s_helper(silent=True).is_running_inside_kubernetes_cluster():
total_size, next_page_token, runs = mlrun.api.crud.list_pipelines(
project,
namespace,
sort_by,
page_token,
filter_,
format_,
page_size,
)
return mlrun.api.schemas.PipelinesOutput(
runs=runs or [],
total_size=total_size or 0,
next_page_token=next_page_token or None,
)
async def move_api_to_online():
logger.info("Moving api to online")
initialize_project_member()
await initialize_scheduler()
# periodic cleanup is not needed if we're not inside kubernetes cluster
if get_k8s_helper(silent=True).is_running_inside_kubernetes_cluster():
_start_periodic_cleanup()
_start_periodic_runs_monitoring()
async def startup_event():
logger.info("configuration dump", dumped_config=config.dump_yaml())
await _initialize_singletons()
# periodic cleanup is not needed if we're not inside kubernetes cluster
if get_k8s_helper(silent=True).is_running_inside_kubernetes_cluster():
_start_periodic_cleanup()
_start_periodic_runs_monitoring()
def list_pipelines(
project: str,
namespace: str = config.namespace,
sort_by: str = "",
page_token: str = "",
filter_: str = Query("", alias="filter"),
format_: mlrun.api.schemas.PipelinesFormat = Query(
mlrun.api.schemas.PipelinesFormat.metadata_only, alias="format"),
page_size: int = Query(None, gt=0, le=200),
auth_verifier: mlrun.api.api.deps.AuthVerifierDep = Depends(
mlrun.api.api.deps.AuthVerifierDep),
db_session: Session = Depends(deps.get_db_session),
):
if project != "*":
mlrun.api.utils.clients.opa.Client().query_project_permissions(
project,
mlrun.api.schemas.AuthorizationAction.read,
auth_verifier.auth_info,
)
total_size, next_page_token, runs = None, None, []
if get_k8s_helper(silent=True).is_running_inside_kubernetes_cluster():
# we need to resolve the project from the returned run for the opa enforcement (project query param might be
# "*"), so we can't really get back only the names here
computed_format = (
mlrun.api.schemas.PipelinesFormat.metadata_only if format_
== mlrun.api.schemas.PipelinesFormat.name_only else format_)
total_size, next_page_token, runs = mlrun.api.crud.Pipelines(
).list_pipelines(
db_session,
project,
namespace,
sort_by,
page_token,
filter_,
computed_format,
page_size,
)
allowed_runs = mlrun.api.utils.clients.opa.Client(
).filter_project_resources_by_permissions(
mlrun.api.schemas.AuthorizationResourceTypes.pipeline,
runs,
lambda run: (
run["project"],
run["id"],
),
auth_verifier.auth_info,
)
if format_ == mlrun.api.schemas.PipelinesFormat.name_only:
allowed_runs = [run["name"] for run in allowed_runs]
return mlrun.api.schemas.PipelinesOutput(
runs=allowed_runs,
total_size=total_size or 0,
next_page_token=next_page_token or None,
)
async def startup_event():
logger.info("configuration dump", dumped_config=config.dump_yaml())
loop = asyncio.get_running_loop()
# Using python 3.8 default instead of 3.7 one - max(1, os.cpu_count()) * 5 cause it's causing to high memory
# consumption - https://bugs.python.org/issue35279
# TODO: remove when moving to python 3.8
max_workers = config.httpdb.max_workers or min(32, os.cpu_count() + 4)
loop.set_default_executor(
concurrent.futures.ThreadPoolExecutor(max_workers=max_workers))
await _initialize_singletons()
# periodic cleanup is not needed if we're not inside kubernetes cluster
if get_k8s_helper(silent=True).is_running_inside_kubernetes_cluster():
_start_periodic_cleanup()
_start_periodic_runs_monitoring()
def list_workflows(
experiment_id: str = None,
namespace: str = None,
sort_by: str = "",
page_token: str = "",
full: bool = False,
page_size: int = 10,
):
total_size, next_page_token, runs = None, None, None
if get_k8s_helper(silent=True).is_running_inside_kubernetes_cluster():
total_size, next_page_token, runs = list_pipelines(
full=full,
page_token=page_token,
page_size=page_size,
sort_by=sort_by,
experiment_id=experiment_id,
namespace=namespace,
)
return {
"runs": runs or [],
"total_size": total_size or 0,
"next_page_token": next_page_token or None,
}
def get_k8s() -> K8sHelper:
return get_k8s_helper(silent=True)
