def export_cli(tag, dry_run, notebook_path, delete, git_ssh_url, api_client: ApiClient, hcl, pattern_matches):
if hcl:
log.debug("this if debug")
service = WorkspaceService(api_client)
files = get_workspace_notebooks_recursive(service, notebook_path)
with GitExportHandler(git_ssh_url, "notebooks", delete_not_found=delete, dry_run=dry_run, tag=tag) as gh:
for file in files:
identifier = normalize_identifier(f"databricks_notebook-{file.path}")
content = get_content(service, file.path)
if content is None:
continue
notebook_resource_data = {
"@expr:content": f'filebase64("{identifier}")',
"path": file.path,
"overwrite": True,
"mkdirs": True,
"language": file.language,
"format": "SOURCE",
}
name = "databricks_notebook"
notebook_file_hcl = create_resource_from_dict(name, identifier, notebook_resource_data, False)
processed_hcl_file = create_hcl_file(file.path, api_client.url, notebook_resource_data,
notebook_file_hcl)
gh.add_file(f"{identifier}.tf", processed_hcl_file)
gh.add_file(f"files/{identifier}", content)
hcl_errors = validate_hcl(notebook_file_hcl)
if len(hcl_errors) > 0:
log.error(f"Identified error in the following HCL Config: {notebook_file_hcl}")
log.error(hcl_errors)
def plan(self):
state_abs_path = os.path.join(self.artifact_dir, "terraform.tfstate")
plan_abs_path = os.path.join(self.artifact_dir, "plan.out")
return_code, stdout, stderr = self._terraform.plan(plan_abs_path,
state_file_abs_path=state_abs_path)
log.info(stdout)
if return_code != 0:
log.error(stderr)
def apply(self, custom_plan_path=None):
state_abs_path = os.path.join(self.artifact_dir, "terraform.tfstate")
plan_abs_path = os.path.join(self.artifact_dir, "plan.out")
if custom_plan_path is not None:
return_code, stdout, stderr = self._terraform.apply(custom_plan_path,
state_file_abs_path=state_abs_path)
else:
return_code, stdout, stderr = self._terraform.apply(plan_abs_path,
state_file_abs_path=state_abs_path)
log.info(stdout)
if return_code != 0:
log.error(stderr)
def export_cli(tag, dry_run, dbfs_path, delete, git_ssh_url,
api_client: ApiClient, hcl, pattern_matches):
if hcl:
log.debug("this if debug")
service = DbfsService(api_client)
files = get_dbfs_files_recursive(service, dbfs_path)
log.info(files)
with GitExportHandler(git_ssh_url,
"dbfs",
delete_not_found=delete,
dry_run=dry_run,
tag=tag) as gh:
for file in files:
assert "path" in file
assert "is_dir" in file
assert "file_size" in file
if file["is_dir"]:
continue
base_name = file["path"]
identifier = normalize_identifier(
f"databricks_dbfs_file-{base_name}")
dbfs_resource_data = {
"@expr:source": f'pathexpand("{identifier}")',
"@expr:content_b64_md5":
f'md5(filebase64(pathexpand("{identifier}")))',
"path": file["path"],
"overwrite": True,
"mkdirs": True,
"validate_remote_file": True,
}
name = "databricks_dbfs_file"
dbfs_file_hcl = create_resource_from_dict(
name, identifier, dbfs_resource_data, False)
processed_hcl_file = create_hcl_file(file['path'],
api_client.url,
dbfs_resource_data,
dbfs_file_hcl)
gh.add_file(f"{identifier}.tf", processed_hcl_file)
gh.add_file(f"files/{identifier}",
get_file_contents(service, file["path"]))
hcl_errors = validate_hcl(dbfs_file_hcl)
if len(hcl_errors) > 0:
log.error(
f"Identified error in the following HCL Config: {dbfs_file_hcl}"
)
log.error(hcl_errors)
def export_cli(tag, dry_run, delete, git_ssh_url, api_client: ApiClient, hcl,
pattern_matches):
if hcl is True:
service = PolicyService(api_client)
created_policy_list = []
with GitExportHandler(git_ssh_url,
"cluster_policies",
delete_not_found=delete,
dry_run=dry_run,
tag=tag) as gh:
for policy in service.list_policies()["policies"]:
assert "definition" in policy
assert "name" in policy
assert "policy_id" in policy
if not pattern_matches(policy["name"]):
log.debug(
f"{policy['name']} did not match pattern function {pattern_matches}"
)
continue
log.debug(
f"{policy['name']} matched the pattern function {pattern_matches}"
)
cluster_policy_tf_dict = {
"@raw:definition": policy["definition"],
"name": policy["name"]
}
name = "databricks_cluster_policy"
identifier = normalize_identifier(
f"databricks_cluster_policy-{policy['name']}-{policy['policy_id']}"
)
created_policy_list.append(identifier)
policy_hcl = create_resource_from_dict(name, identifier,
cluster_policy_tf_dict,
False)
file_name_identifier = f"{identifier}.tf"
processed_hcl_file = create_hcl_file(policy['policy_id'],
api_client.url,
cluster_policy_tf_dict,
policy_hcl)
gh.add_file(file_name_identifier, processed_hcl_file)
hcl_errors = validate_hcl(policy_hcl)
if len(hcl_errors) > 0:
log.error(
f"Identified error in the following HCL Config: {policy_hcl}"
)
log.error(hcl_errors)
def get_content(service: WorkspaceService, path):
data = service.export_workspace(path, format="SOURCE")
if "content" not in data:
log.error(f"Unable to find content for file {path}")
return None
return b64decode(data["content"].encode("utf-8")).decode("utf-8")
def _push_tags(self):
if self._git_tag is not None:
self.repo.git.push("origin", self._tag_value, "--porcelain",
"--no-verify")
else:
log.error("Unable to find the tag")