def export_cli(tag, dry_run, notebook_path, delete, git_ssh_url, api_client: ApiClient, hcl, pattern_matches): if hcl: log.debug("this if debug") service = WorkspaceService(api_client) files = get_workspace_notebooks_recursive(service, notebook_path) with GitExportHandler(git_ssh_url, "notebooks", delete_not_found=delete, dry_run=dry_run, tag=tag) as gh: for file in files: identifier = normalize_identifier(f"databricks_notebook-{file.path}") content = get_content(service, file.path) if content is None: continue notebook_resource_data = { "@expr:content": f'filebase64("{identifier}")', "path": file.path, "overwrite": True, "mkdirs": True, "language": file.language, "format": "SOURCE", } name = "databricks_notebook" notebook_file_hcl = create_resource_from_dict(name, identifier, notebook_resource_data, False) processed_hcl_file = create_hcl_file(file.path, api_client.url, notebook_resource_data, notebook_file_hcl) gh.add_file(f"{identifier}.tf", processed_hcl_file) gh.add_file(f"files/{identifier}", content) hcl_errors = validate_hcl(notebook_file_hcl) if len(hcl_errors) > 0: log.error(f"Identified error in the following HCL Config: {notebook_file_hcl}") log.error(hcl_errors)
def plan(self): state_abs_path = os.path.join(self.artifact_dir, "terraform.tfstate") plan_abs_path = os.path.join(self.artifact_dir, "plan.out") return_code, stdout, stderr = self._terraform.plan(plan_abs_path, state_file_abs_path=state_abs_path) log.info(stdout) if return_code != 0: log.error(stderr)
def apply(self, custom_plan_path=None): state_abs_path = os.path.join(self.artifact_dir, "terraform.tfstate") plan_abs_path = os.path.join(self.artifact_dir, "plan.out") if custom_plan_path is not None: return_code, stdout, stderr = self._terraform.apply(custom_plan_path, state_file_abs_path=state_abs_path) else: return_code, stdout, stderr = self._terraform.apply(plan_abs_path, state_file_abs_path=state_abs_path) log.info(stdout) if return_code != 0: log.error(stderr)
def export_cli(tag, dry_run, dbfs_path, delete, git_ssh_url, api_client: ApiClient, hcl, pattern_matches): if hcl: log.debug("this if debug") service = DbfsService(api_client) files = get_dbfs_files_recursive(service, dbfs_path) log.info(files) with GitExportHandler(git_ssh_url, "dbfs", delete_not_found=delete, dry_run=dry_run, tag=tag) as gh: for file in files: assert "path" in file assert "is_dir" in file assert "file_size" in file if file["is_dir"]: continue base_name = file["path"] identifier = normalize_identifier( f"databricks_dbfs_file-{base_name}") dbfs_resource_data = { "@expr:source": f'pathexpand("{identifier}")', "@expr:content_b64_md5": f'md5(filebase64(pathexpand("{identifier}")))', "path": file["path"], "overwrite": True, "mkdirs": True, "validate_remote_file": True, } name = "databricks_dbfs_file" dbfs_file_hcl = create_resource_from_dict( name, identifier, dbfs_resource_data, False) processed_hcl_file = create_hcl_file(file['path'], api_client.url, dbfs_resource_data, dbfs_file_hcl) gh.add_file(f"{identifier}.tf", processed_hcl_file) gh.add_file(f"files/{identifier}", get_file_contents(service, file["path"])) hcl_errors = validate_hcl(dbfs_file_hcl) if len(hcl_errors) > 0: log.error( f"Identified error in the following HCL Config: {dbfs_file_hcl}" ) log.error(hcl_errors)
def export_cli(tag, dry_run, delete, git_ssh_url, api_client: ApiClient, hcl, pattern_matches): if hcl is True: service = PolicyService(api_client) created_policy_list = [] with GitExportHandler(git_ssh_url, "cluster_policies", delete_not_found=delete, dry_run=dry_run, tag=tag) as gh: for policy in service.list_policies()["policies"]: assert "definition" in policy assert "name" in policy assert "policy_id" in policy if not pattern_matches(policy["name"]): log.debug( f"{policy['name']} did not match pattern function {pattern_matches}" ) continue log.debug( f"{policy['name']} matched the pattern function {pattern_matches}" ) cluster_policy_tf_dict = { "@raw:definition": policy["definition"], "name": policy["name"] } name = "databricks_cluster_policy" identifier = normalize_identifier( f"databricks_cluster_policy-{policy['name']}-{policy['policy_id']}" ) created_policy_list.append(identifier) policy_hcl = create_resource_from_dict(name, identifier, cluster_policy_tf_dict, False) file_name_identifier = f"{identifier}.tf" processed_hcl_file = create_hcl_file(policy['policy_id'], api_client.url, cluster_policy_tf_dict, policy_hcl) gh.add_file(file_name_identifier, processed_hcl_file) hcl_errors = validate_hcl(policy_hcl) if len(hcl_errors) > 0: log.error( f"Identified error in the following HCL Config: {policy_hcl}" ) log.error(hcl_errors)
def get_content(service: WorkspaceService, path): data = service.export_workspace(path, format="SOURCE") if "content" not in data: log.error(f"Unable to find content for file {path}") return None return b64decode(data["content"].encode("utf-8")).decode("utf-8")
def _push_tags(self): if self._git_tag is not None: self.repo.git.push("origin", self._tag_value, "--porcelain", "--no-verify") else: log.error("Unable to find the tag")