def get_context_id(v1_client: ApiV1Client, cluster_id: str, language: str): dbx_echo("Preparing execution context") lock_context_id = ContextLockFile.get_context() if _is_context_available(v1_client, cluster_id, lock_context_id): dbx_echo("Existing context is active, using it") return lock_context_id else: dbx_echo("Existing context is not active, creating a new one") context_id = create_context(v1_client, cluster_id, language) ContextLockFile.set_context(context_id) dbx_echo("New context prepared, ready to use it") return context_id
def launch(self): dbx_echo(f"Starting deployment to factory {self.factory_name}") prepared_activities = [] for job_spec in self._specs: job_name = job_spec.get("name") dbx_echo(f"Preparing job {job_name}") service_name = self._create_linked_service(job_spec) job_activity = self._generate_activity(job_spec, service_name) prepared_activities.append(job_activity) dbx_echo(f"Preparing job {job_name} - done") self._update_pipeline(prepared_activities) dbx_echo( f"Reflection to factory {self.factory_name} finished successfully")
def execute_command(v1_client: ApiV1Client, cluster_id: str, context_id: str, command: str, verbose=True): payload = { "language": "python", "clusterId": cluster_id, "contextId": context_id, "command": command, } command_execution_data = v1_client.execute_command(payload) command_id = command_execution_data["id"] execution_result = wait_for_command_execution(v1_client, cluster_id, context_id, command_id) if execution_result["status"] == "Cancelled": dbx_echo("Command cancelled") else: final_result = execution_result["results"]["resultType"] if final_result == "error": dbx_echo("Execution failed, please follow the given error") raise RuntimeError( f"Command execution failed. " f'Cluster error cause: {execution_result["results"]["cause"]}') if verbose: dbx_echo("Command successfully executed") print(execution_result["results"]["data"]) return execution_result["results"]["data"]
def _find_deployment_run(filter_string: str, tags: Dict[str, str], as_run_submit: bool, environment: str) -> Dict[str, Any]: runs = mlflow.search_runs(filter_string=filter_string, order_by=["start_time DESC"]) filter_conditions = [] if tags: dbx_echo("Filtering deployments with set of additional tags") for tag_name, tag_value in tags.items(): tag_column_name = f"tags.{tag_name}" if tag_column_name not in runs.columns: raise Exception( f"Tag {tag_name} not found in underlying MLflow experiment, please verify tag existence in the UI" ) tag_condition = runs[tag_column_name] == tag_value filter_conditions.append(tag_condition) full_filter = pd.DataFrame(filter_conditions).T.all(axis=1) # noqa _runs = runs[full_filter] else: dbx_echo("No additional tags provided") _runs = runs if as_run_submit: if "tags.dbx_deploy_type" not in _runs.columns: raise Exception("""" Run Submit API is available only when deployment was done with --files-only flag. Currently there is no deployments with such flag under given filters. Please re-deploy with --files-only flag and then re-run this launch command. """) _runs = _runs[_runs["tags.dbx_deploy_type"] == "files_only"] if _runs.empty: exception_string = """ No runs provided per given set of filters: {filter_string} """ if tags: exception_string = (exception_string + f""" With additional tags: {tags} """) if as_run_submit: exception_string = (exception_string + """ With file-based deployments (dbx_deployment_type='files_only'). """) experiment_location = InfoFile.get(environment).get("workspace_dir") exception_string = (exception_string + f""" To verify current status of deployments please check experiment UI in workspace dir: {experiment_location} """) raise Exception(exception_string) run_info = _runs.iloc[0].to_dict() dbx_echo("Successfully found deployment per given job name") return run_info
def _create_linked_service(self, job_spec: Dict[str, Any]) -> str: cluster_spec = job_spec.get("new_cluster") service_name = job_spec.get("name") + "-ls" dbx_echo(f"Preparing linked service {service_name}") if cluster_spec: service_spec = AzureDatabricksLinkedService( domain=self._config.host, access_token=SecureString(value=self._config.token), instance_pool_id=cluster_spec.get("instance_pool_id"), new_cluster_custom_tags=cluster_spec.get("custom_tags"), new_cluster_driver_node_type=cluster_spec.get( "driver_node_type_id"), new_cluster_enable_elastic_disk=cluster_spec.get( "enable_elastic_disk"), new_cluster_init_scripts=cluster_spec.get( "enable_elastic_disk"), new_cluster_log_destination=cluster_spec.get( "cluster_log_conf", {}).get("dbfs", {}).get("destination"), new_cluster_node_type=cluster_spec.get("node_type_id"), new_cluster_num_of_worker=cluster_spec.get("num_workers"), new_cluster_spark_conf=cluster_spec.get("spark_conf"), new_cluster_spark_env_vars=cluster_spec.get("spark_env_vars"), new_cluster_version=cluster_spec.get("spark_version"), ) else: service_spec = AzureDatabricksLinkedService( domain=self._config.host, access_token=SecureString(value=self._config.token), existing_cluster_id=cluster_spec.get("existing_cluster_id"), ) service_resource = LinkedServiceResource(properties=service_spec) self.adf_client.linked_services.create_or_update( self.resource_group, self.factory_name, service_name, service_resource) dbx_echo(f"Preparing linked service {service_name} - done") return service_name
def _adjust_job_definitions( jobs: List[Dict[str, Any]], artifact_base_uri: str, requirements_payload: List[Dict[str, str]], package_payload: List[Dict[str, str]], file_uploader: FileUploader, api_client: ApiClient, ): def adjustment_callback(p: Any): return _adjust_path(p, artifact_base_uri, file_uploader) for job in jobs: job["libraries"] = job.get("libraries", []) + package_payload _walk_content(adjustment_callback, job) job["libraries"] = job.get("libraries", []) + requirements_payload policy_name = job.get("new_cluster", {}).get("policy_name") if policy_name: dbx_echo( f"Processing policy name {policy_name} for job {job['name']}") policy_spec = _preprocess_policy_name(api_client, policy_name) policy = json.loads(policy_spec["definition"]) policy_props = PolicyParser(policy).parse() _deep_update(job["new_cluster"], policy_props, policy_name) job["new_cluster"]["policy_id"] = policy_spec["policy_id"]
def _update_pipeline(self, new_activities: List[DatabricksSparkPythonActivity]): dbx_echo(f"Updating pipeline {self.name}") current_pipeline = self.adf_client.pipelines.get( self.resource_group, self.factory_name, self.name) new_activities_dict = {a.name: a for a in new_activities} existing_activities_dict = { a.name: a for a in current_pipeline.activities } intersected = set(new_activities_dict.keys()).intersection( set(existing_activities_dict.keys())) final_activity_list = [] for name in intersected: _ex: Activity = existing_activities_dict.get(name) _new: Activity = new_activities_dict.get(name) _new.depends_on = _ex.depends_on final_activity_list.append(_new) for name, activity in existing_activities_dict.items(): if name not in intersected: final_activity_list.append(activity) for name, activity in new_activities_dict.items(): if name not in intersected: final_activity_list.append(activity) resource = PipelineResource(activities=final_activity_list) self.adf_client.pipelines.create_or_update(self.resource_group, self.factory_name, self.name, resource) dbx_echo(f"Updating pipeline {self.name} - done")
def _create_job(api_client: ApiClient, job: Dict[str, Any]) -> str: dbx_echo(f'Creating a new job with name {job["name"]}') try: jobs_api = JobsApi(api_client) job_id = jobs_api.create_job(job)["job_id"] except HTTPError as e: dbx_echo("Failed to create job with definition:") dbx_echo(json.dumps(job, indent=4)) raise e return job_id
def _update_job(jobs_service: JobsService, job_id: str, job: Dict[str, Any]) -> str: dbx_echo( f'Updating existing job with id: {job_id} and name: {job["name"]}') try: jobs_service.reset_job(job_id, job) except HTTPError as e: dbx_echo("Failed to update job with definition:") dbx_echo(json.dumps(job, indent=4)) raise e return job_id
def awake_cluster(cluster_service: ClusterService, cluster_id): cluster_info = cluster_service.get_cluster(cluster_id) if cluster_info["state"] in ["RUNNING", "RESIZING"]: dbx_echo("Cluster is ready") if cluster_info["state"] in ["TERMINATED", "TERMINATING"]: dbx_echo("Dev cluster is terminated, starting it") cluster_service.start_cluster(cluster_id) time.sleep(5) awake_cluster(cluster_service, cluster_id) elif cluster_info["state"] == "ERROR": raise RuntimeError("Cluster is mis-configured and cannot be started, please check cluster settings at first") elif cluster_info["state"] in ["PENDING", "RESTARTING"]: dbx_echo(f'Cluster is getting prepared, current state: {cluster_info["state"]}') time.sleep(5) awake_cluster(cluster_service, cluster_id)
def _wait_run(api_client: ApiClient, run_data: Dict[str, Any]) -> Dict[str, Any]: dbx_echo(f"Tracing run with id {run_data['run_id']}") while True: time.sleep( 5 ) # runs API is eventually consistent, it's better to have a short pause for status update status = _get_run_status(api_client, run_data) run_state = status["state"] result_state = run_state.get("result_state", None) life_cycle_state = run_state.get("life_cycle_state", None) state_message = run_state.get("state_message") dbx_echo(f"[Run Id: {run_data['run_id']}] Current run status info - " f"result state: {result_state}, " f"lifecycle state: {life_cycle_state}, " f"state message: {state_message}") if life_cycle_state in TERMINAL_RUN_LIFECYCLE_STATES: dbx_echo(f"Finished tracing run with id {run_data['run_id']}") return status
def execute( environment: str, cluster_id: str, cluster_name: str, job: str, deployment_file: str, requirements_file: str, no_package: bool, no_rebuild: bool, ): api_client = prepare_environment(environment) cluster_id = _preprocess_cluster_args(api_client, cluster_name, cluster_id) dbx_echo( f"Executing job: {job} in environment {environment} on cluster {cluster_name} (id: {cluster_id})" ) handle_package(no_rebuild) deployment = get_deployment_config(deployment_file).get_environment( environment) _verify_deployment(deployment, environment, deployment_file) found_jobs = [j for j in deployment["jobs"] if j["name"] == job] if not found_jobs: raise RuntimeError( f"Job {job} was not found in environment jobs, please check the deployment file" ) job_payload = found_jobs[0] entrypoint_file = job_payload.get("spark_python_task").get("python_file") if not entrypoint_file: raise FileNotFoundError( f"No entrypoint file provided in job {job}. " f"Please add one under spark_python_task.python_file section") cluster_service = ClusterService(api_client) dbx_echo("Preparing interactive cluster to accept jobs") awake_cluster(cluster_service, cluster_id) v1_client = ApiV1Client(api_client) context_id = get_context_id(v1_client, cluster_id, "python") file_uploader = FileUploader(api_client) with mlflow.start_run() as execution_run: artifact_base_uri = execution_run.info.artifact_uri localized_base_path = artifact_base_uri.replace("dbfs:/", "/dbfs/") requirements_fp = pathlib.Path(requirements_file) if requirements_fp.exists(): file_uploader.upload_file(requirements_fp) localized_requirements_path = f"{localized_base_path}/{str(requirements_fp)}" installation_command = f"%pip install -U -r {localized_requirements_path}" dbx_echo("Installing provided requirements") execute_command(v1_client, cluster_id, context_id, installation_command, verbose=False) dbx_echo("Provided requirements installed") else: dbx_echo( f"Requirements file {requirements_fp} is not provided" + ", following the execution without any additional packages") if not no_package: package_file = get_package_file() if not package_file: raise FileNotFoundError( "Project package was not found. Please check that /dist directory exists." ) file_uploader.upload_file(package_file) localized_package_path = f"{localized_base_path}/{str(package_file.as_posix())}" dbx_echo("Installing package") installation_command = f"%pip install --force-reinstall {localized_package_path}" execute_command(v1_client, cluster_id, context_id, installation_command, verbose=False) dbx_echo("Package installation finished") else: dbx_echo( "Package was disabled via --no-package, only the code from entrypoint will be used" ) tags = {"dbx_action_type": "execute", "dbx_environment": environment} mlflow.set_tags(tags) dbx_echo("Processing parameters") task_props: List[Any] = job_payload.get("spark_python_task").get( "parameters", []) if task_props: def adjustment_callback(p: Any): return _adjust_path(p, artifact_base_uri, file_uploader) _walk_content(adjustment_callback, task_props) task_props = ["python"] + task_props parameters_command = f""" import sys sys.argv = {task_props} """ execute_command(v1_client, cluster_id, context_id, parameters_command, verbose=False) dbx_echo("Processing parameters - done") dbx_echo("Starting entrypoint file execution") execute_command(v1_client, cluster_id, context_id, pathlib.Path(entrypoint_file).read_text()) dbx_echo("Command execution finished")
def deploy( deployment_file: str, jobs: str, requirements_file: str, tags: List[str], environment: str, no_rebuild: bool, no_package: bool, ): dbx_echo(f"Starting new deployment for environment {environment}") api_client = prepare_environment(environment) additional_tags = parse_multiple(tags) branch_name = get_current_branch_name() handle_package(no_rebuild) package_file = get_package_file() _verify_deployment_file(deployment_file) deployment_file_controller = DeploymentFile(deployment_file) deployment = deployment_file_controller.get_environment(environment) if not deployment: raise NameError(f""" Requested environment {environment} is non-existent in the deployment file {deployment_file}. Available environments are: {deployment_file_controller.get_all_environment_names()} """) if jobs: requested_jobs = jobs.split(",") else: requested_jobs = None requirements_payload = _preprocess_requirements(requirements_file) _preprocess_deployment(deployment, requested_jobs) _file_uploader = FileUploader(api_client) with mlflow.start_run() as deployment_run: artifact_base_uri = deployment_run.info.artifact_uri if no_package: dbx_echo( "No package definition will be added into job description") package_requirement = [] else: if package_file: package_requirement = [{"whl": str(package_file)}] else: dbx_echo( "Package file was not found! Please check your /dist/ folder" ) package_requirement = [] _adjust_job_definitions( deployment["jobs"], artifact_base_uri, requirements_payload, package_requirement, _file_uploader, ) deployment_data = _create_jobs(deployment["jobs"], api_client) _log_deployments(deployment_data) deployment_tags = { "dbx_action_type": "deploy", "dbx_environment": environment, "dbx_status": "SUCCESS", } deployment_tags.update(additional_tags) if branch_name: deployment_tags["dbx_branch_name"] = branch_name mlflow.set_tags(deployment_tags) dbx_echo( f"Deployment for environment {environment} finished successfully")
def launch( environment: str, job: str, trace: bool, kill_on_sigterm: bool, existing_runs: str, tags: List[str], parameters: List[str], ): dbx_echo(f"Launching job {job} on environment {environment}") api_client = prepare_environment(environment) additional_tags = parse_multiple(tags) override_parameters = parse_multiple(parameters) filter_string = generate_filter_string(environment, additional_tags) runs = mlflow.search_runs(filter_string=filter_string, max_results=1) if runs.empty: raise EnvironmentError( f""" No runs provided per given set of filters: {filter_string} Please check experiment UI to verify current status of deployments. """ ) run_info = runs.iloc[0].to_dict() dbx_echo("Successfully found deployment per given job name") deployment_run_id = run_info["run_id"] with mlflow.start_run(run_id=deployment_run_id) as deployment_run: with mlflow.start_run(nested=True): artifact_base_uri = deployment_run.info.artifact_uri deployments = _load_deployments(api_client, artifact_base_uri) job_id = deployments.get(job) if not job_id: raise Exception( f"Job with name {job} not found in the latest deployment" % job ) jobs_service = JobsService(api_client) active_runs = jobs_service.list_runs(job_id, active_only=True).get( "runs", [] ) for run in active_runs: if existing_runs == "pass": dbx_echo("Passing the existing runs status check") if existing_runs == "wait": dbx_echo( f'Waiting for job run with id {run["run_id"]} to be finished' ) _wait_run(api_client, run) if existing_runs == "cancel": dbx_echo(f'Cancelling run with id {run["run_id"]}') _cancel_run(api_client, run) if override_parameters: _prepared_parameters = sum( [[k, v] for k, v in override_parameters.items()], [] ) dbx_echo( f"Default launch parameters are overridden with the following: {_prepared_parameters}" ) run_data = jobs_service.run_now( job_id, python_params=_prepared_parameters ) else: run_data = jobs_service.run_now(job_id) if trace: dbx_echo("Tracing job run") if kill_on_sigterm: dbx_echo("Click Ctrl+C to stop the job run") try: dbx_status = _trace_run(api_client, run_data) except KeyboardInterrupt: dbx_status = "CANCELLED" dbx_echo("Cancelling the run gracefully") _cancel_run(api_client, run_data) dbx_echo("Run cancelled successfully") else: dbx_status = _trace_run(api_client, run_data) if dbx_status == "ERROR": raise Exception( "Tracked job failed during execution. " "Please check Databricks UI for job logs" ) dbx_echo("Launch command finished") else: dbx_status = "NOT_TRACKED" dbx_echo( "Job successfully launched in non-tracking mode. Please check Databricks UI for job status" ) deployment_tags = { "job_id": job_id, "run_id": run_data["run_id"], "dbx_action_type": "launch", "dbx_status": dbx_status, "dbx_environment": environment, } mlflow.set_tags(deployment_tags)
def execute( environment: str, cluster_id: str, cluster_name: str, job: str, deployment_file: str, requirements_file: str, no_rebuild: bool, ): api_client = prepare_environment(environment) cluster_id = _preprocess_cluster_args(api_client, cluster_name, cluster_id) dbx_echo( f"Executing job: {job} with environment: {environment} on cluster: {cluster_id}" ) handle_package(no_rebuild) deployment = DeploymentFile(deployment_file).get_environment(environment) if not deployment: raise NameError( f"Environment {environment} is not provided in deployment file {deployment_file}" + " please add this environment first" ) env_jobs = deployment.get("jobs") if not env_jobs: raise RuntimeError( f"No jobs section found in environment {environment}, please check the deployment file" ) found_jobs = [j for j in deployment["jobs"] if j["name"] == job] if not found_jobs: raise RuntimeError( f"Job {job} was not found in environment jobs, please check the deployment file" ) job_payload = found_jobs[0] entrypoint_file = job_payload.get("spark_python_task").get("python_file") if not entrypoint_file: raise FileNotFoundError( f"No entrypoint file provided in job {job}. " f"Please add one under spark_python_task.python_file section" ) cluster_service = ClusterService(api_client) dbx_echo("Preparing interactive cluster to accept jobs") awake_cluster(cluster_service, cluster_id) v1_client = ApiV1Client(api_client) context_id = get_context_id(v1_client, cluster_id, "python") file_uploader = FileUploader(api_client) with mlflow.start_run() as execution_run: artifact_base_uri = execution_run.info.artifact_uri localized_base_path = artifact_base_uri.replace("dbfs:/", "/dbfs/") requirements_fp = pathlib.Path(requirements_file) if requirements_fp.exists(): file_uploader.upload_file(requirements_fp) localized_requirements_path = ( f"{localized_base_path}/{str(requirements_fp)}" ) installation_command = f"%pip install -U -r {localized_requirements_path}" dbx_echo("Installing provided requirements") execute_command( v1_client, cluster_id, context_id, installation_command, verbose=False ) dbx_echo("Provided requirements installed") else: dbx_echo( f"Requirements file {requirements_fp} is not provided" + ", following the execution without any additional packages" ) project_package_path = list(pathlib.Path(".").rglob("dist/*.whl")) # Make sure we always take the newest package (i.e. sort by time) project_package_path.sort(key=os.path.getctime, reverse=True) project_package_path = project_package_path[0] file_uploader.upload_file(project_package_path) localized_package_path = ( f"{localized_base_path}/{str(project_package_path.as_posix())}" ) dbx_echo("Installing package") installation_command = f"%pip install {localized_package_path} --force-reinstall" execute_command( v1_client, cluster_id, context_id, installation_command, verbose=False ) dbx_echo("Package installation finished") tags = {"dbx_action_type": "execute", "dbx_environment": environment} mlflow.set_tags(tags) dbx_echo("Starting entrypoint file execution") execute_command( v1_client, cluster_id, context_id, pathlib.Path(entrypoint_file).read_text() ) dbx_echo("Command execution finished")
def launch( environment: str, job: str, trace: bool, kill_on_sigterm: bool, existing_runs: str, as_run_submit: bool, tags: List[str], parameters: List[str], parameters_raw: Optional[str], ): dbx_echo(f"Launching job {job} on environment {environment}") api_client = prepare_environment(environment) additional_tags = parse_multiple(tags) if parameters_raw: prepared_parameters = parameters_raw else: override_parameters = parse_multiple(parameters) prepared_parameters = sum([[k, v] for k, v in override_parameters.items()], []) filter_string = generate_filter_string(environment) run_info = _find_deployment_run(filter_string, additional_tags, as_run_submit, environment) deployment_run_id = run_info["run_id"] with mlflow.start_run(run_id=deployment_run_id) as deployment_run: with mlflow.start_run(nested=True): artifact_base_uri = deployment_run.info.artifact_uri if not as_run_submit: run_launcher = RunNowLauncher(job, api_client, artifact_base_uri, existing_runs, prepared_parameters) else: run_launcher = RunSubmitLauncher(job, api_client, artifact_base_uri, existing_runs, prepared_parameters, environment) run_data, job_id = run_launcher.launch() jobs_service = JobsService(api_client) run_info = jobs_service.get_run(run_data["run_id"]) run_url = run_info.get("run_page_url") dbx_echo(f"Run URL: {run_url}") if trace: if kill_on_sigterm: dbx_echo("Click Ctrl+C to stop the run") try: dbx_status = _trace_run(api_client, run_data) except KeyboardInterrupt: dbx_status = "CANCELLED" dbx_echo("Cancelling the run gracefully") _cancel_run(api_client, run_data) dbx_echo("Run cancelled successfully") else: dbx_status = _trace_run(api_client, run_data) if dbx_status == "ERROR": raise Exception( "Tracked run failed during execution. Please check Databricks UI for run logs" ) dbx_echo("Launch command finished") else: dbx_status = "NOT_TRACKED" dbx_echo( "Run successfully launched in non-tracking mode. Please check Databricks UI for job status" ) deployment_tags = { "job_id": job_id, "run_id": run_data.get("run_id"), "dbx_action_type": "launch", "dbx_status": dbx_status, "dbx_environment": environment, } mlflow.set_tags(deployment_tags)
def deploy( deployment_file: str, jobs: str, requirements_file: str, tags: List[str], environment: str, no_rebuild: bool, no_package: bool, files_only: bool, write_specs_to_file: Optional[str], branch_name: Optional[str], ): dbx_echo(f"Starting new deployment for environment {environment}") api_client = prepare_environment(environment) additional_tags = parse_multiple(tags) handle_package(no_rebuild) package_file = get_package_file() if not branch_name: branch_name = get_current_branch_name() _verify_deployment_file(deployment_file) deployment_file_config = get_deployment_config(deployment_file) deployment = deployment_file_config.get_environment(environment) if not deployment: raise NameError(f""" Requested environment {environment} is non-existent in the deployment file {deployment_file}. Available environments are: {deployment_file_config.get_all_environment_names()} """) if jobs: requested_jobs = jobs.split(",") else: requested_jobs = None requirements_payload = _preprocess_requirements(requirements_file) _preprocess_deployment(deployment, requested_jobs) _file_uploader = FileUploader(api_client) with mlflow.start_run() as deployment_run: artifact_base_uri = deployment_run.info.artifact_uri if no_package: dbx_echo( "No package definition will be added into job description") package_requirement = [] else: if package_file: package_requirement = [{"whl": str(package_file)}] else: dbx_echo( "Package file was not found! Please check your /dist/ folder" ) package_requirement = [] _adjust_job_definitions(deployment["jobs"], artifact_base_uri, requirements_payload, package_requirement, _file_uploader, api_client) if not files_only: dbx_echo("Updating job definitions") deployment_data = _create_jobs(deployment["jobs"], api_client) _log_dbx_file(deployment_data, "deployments.json") for job_spec in deployment.get("jobs"): permissions = job_spec.get("permissions") if permissions: job_name = job_spec.get("name") dbx_echo( f"Permission settings are provided for job {job_name}, setting it up" ) job_id = deployment_data.get(job_spec.get("name")) api_client.perform_query("PUT", f"/permissions/jobs/{job_id}", data=permissions) dbx_echo( f"Permission settings were successfully set for job {job_name}" ) dbx_echo("Updating job definitions - done") deployment_tags = { "dbx_action_type": "deploy", "dbx_environment": environment, "dbx_status": "SUCCESS", } deployment_spec = {environment: deployment} deployment_tags.update(additional_tags) if branch_name: deployment_tags["dbx_branch_name"] = branch_name if files_only: deployment_tags["dbx_deploy_type"] = "files_only" _log_dbx_file(deployment_spec, "deployment-result.json") mlflow.set_tags(deployment_tags) dbx_echo( f"Deployment for environment {environment} finished successfully") if write_specs_to_file: dbx_echo("Writing final job specifications into file") specs_file = pathlib.Path(write_specs_to_file) if specs_file.exists(): specs_file.unlink() specs_file.write_text(json.dumps(deployment_spec, indent=4))