def _create_run(uri, experiment_id, work_dir, entry_point): """ Create a ``Run`` against the current MLflow tracking server, logging metadata (e.g. the URI, entry point, and parameters of the project) about the run. Return an ``ActiveRun`` that can be used to report additional data about the run (metrics/params) to the tracking server. """ if _is_local_uri(uri): source_name = tracking.utils._get_git_url_if_present(_expand_uri(uri)) else: source_name = _expand_uri(uri) source_version = _get_git_commit(work_dir) existing_run = fluent.active_run() if existing_run: parent_run_id = existing_run.info.run_uuid else: parent_run_id = None tags = { MLFLOW_SOURCE_NAME: source_name, MLFLOW_SOURCE_TYPE: SourceType.to_string(SourceType.PROJECT), MLFLOW_PROJECT_ENTRY_POINT: entry_point } if source_version is not None: tags[MLFLOW_GIT_COMMIT] = source_version if parent_run_id is not None: tags[MLFLOW_PARENT_RUN_ID] = parent_run_id active_run = tracking.MlflowClient().create_run(experiment_id=experiment_id, tags=tags) return active_run
def _get_docker_tag_name(project_name, work_dir): """Returns an appropriate Docker tag for a project based on name and git hash.""" project_name = project_name if project_name else "docker-project" # Optionally include first 7 digits of git SHA in tag name, if available. git_commit = _get_git_commit(work_dir) version_string = "-" + git_commit[:7] if git_commit else "" return "mlflow-" + project_name + version_string
def _build_docker_image(work_dir, project, active_run): """ Build a docker image containing the project in `work_dir`, using the base image and tagging the built image with the project name specified by `project`. """ if not project.name: raise ExecutionException( "Project name in MLProject must be specified when using docker " "for image tagging.") tag_name = "mlflow-{name}-{version}".format( name=(project.name if project.name else "docker-project"), version=_get_git_commit(work_dir)[:7], ) dockerfile = ("FROM {imagename}\n" "LABEL Name={tag_name}\n" "COPY {build_context_path}/* /mlflow/projects/code/\n" "WORKDIR /mlflow/projects/code/\n").format( imagename=project.docker_env.get('image'), tag_name=tag_name, build_context_path=_PROJECT_TAR_ARCHIVE_NAME) build_ctx_path = _create_docker_build_ctx(work_dir, dockerfile) with open(build_ctx_path, 'rb') as docker_build_ctx: _logger.info("=== Building docker image %s ===", tag_name) client = docker.from_env() image = client.images.build(tag=tag_name, forcerm=True, dockerfile=posixpath.join( _PROJECT_TAR_ARCHIVE_NAME, _GENERATED_DOCKERFILE_NAME), fileobj=docker_build_ctx, custom_context=True, encoding="gzip") try: os.remove(build_ctx_path) except Exception: # pylint: disable=broad-except _logger.info("Temporary docker context file %s was not deleted.", build_ctx_path) tracking.MlflowClient().set_tag(active_run.info.run_id, MLFLOW_DOCKER_IMAGE_NAME, tag_name) tracking.MlflowClient().set_tag(active_run.info.run_id, MLFLOW_DOCKER_IMAGE_ID, image[0].id) return tag_name
def _create_run(uri, experiment_id, work_dir, entry_point): """ Create a ``Run`` against the current MLflow tracking server, logging metadata (e.g. the URI, entry point, and parameters of the project) about the run. Return an ``ActiveRun`` that can be used to report additional data about the run (metrics/params) to the tracking server. """ if _is_local_uri(uri): source_name = tracking.utils._get_git_url_if_present(_expand_uri(uri)) else: source_name = _expand_uri(uri) existing_run = fluent.active_run() if existing_run: parent_run_id = existing_run.info.run_uuid else: parent_run_id = None active_run = tracking.MlflowClient().create_run( experiment_id=experiment_id, source_name=source_name, source_version=_get_git_commit(work_dir), entry_point_name=entry_point, source_type=SourceType.PROJECT, parent_run_id=parent_run_id) return active_run