def add_file(self, name, data):
write_path = os.path.join(self.resource_path, name)
os.makedirs(os.path.dirname(write_path), exist_ok=True)
log.info(f"Writing {self.directory} to path {write_path}")
with open(write_path, "w") as f:
f.write(data)
self.files_created.append(name)
def plan(self):
state_abs_path = os.path.join(self.artifact_dir, "terraform.tfstate")
plan_abs_path = os.path.join(self.artifact_dir, "plan.out")
return_code, stdout, stderr = self._terraform.plan(plan_abs_path,
state_file_abs_path=state_abs_path)
log.info(stdout)
if return_code != 0:
log.error(stderr)
def _remove_unmanaged_files(self):
deleted_file_paths_to_stage = []
files_to_delete = self._get_files_delete()
for abs_file_path in files_to_delete:
# Skip ignored files
file_name = ntpath.basename(abs_file_path)
if file_name not in self._ignore_remove_files:
log.info(f"Deleting {self.directory} in path {abs_file_path}")
os.remove(abs_file_path)
deleted_file_paths_to_stage.append(abs_file_path)
def apply(self, custom_plan_path=None):
state_abs_path = os.path.join(self.artifact_dir, "terraform.tfstate")
plan_abs_path = os.path.join(self.artifact_dir, "plan.out")
if custom_plan_path is not None:
return_code, stdout, stderr = self._terraform.apply(custom_plan_path,
state_file_abs_path=state_abs_path)
else:
return_code, stdout, stderr = self._terraform.apply(plan_abs_path,
state_file_abs_path=state_abs_path)
log.info(stdout)
if return_code != 0:
log.error(stderr)
def export_cli(tag, dry_run, dbfs_path, delete, git_ssh_url,
api_client: ApiClient, hcl, pattern_matches):
if hcl:
log.debug("this if debug")
service = DbfsService(api_client)
files = get_dbfs_files_recursive(service, dbfs_path)
log.info(files)
with GitExportHandler(git_ssh_url,
"dbfs",
delete_not_found=delete,
dry_run=dry_run,
tag=tag) as gh:
for file in files:
assert "path" in file
assert "is_dir" in file
assert "file_size" in file
if file["is_dir"]:
continue
base_name = file["path"]
identifier = normalize_identifier(
f"databricks_dbfs_file-{base_name}")
dbfs_resource_data = {
"@expr:source": f'pathexpand("{identifier}")',
"@expr:content_b64_md5":
f'md5(filebase64(pathexpand("{identifier}")))',
"path": file["path"],
"overwrite": True,
"mkdirs": True,
"validate_remote_file": True,
}
name = "databricks_dbfs_file"
dbfs_file_hcl = create_resource_from_dict(
name, identifier, dbfs_resource_data, False)
processed_hcl_file = create_hcl_file(file['path'],
api_client.url,
dbfs_resource_data,
dbfs_file_hcl)
gh.add_file(f"{identifier}.tf", processed_hcl_file)
gh.add_file(f"files/{identifier}",
get_file_contents(service, file["path"]))
hcl_errors = validate_hcl(dbfs_file_hcl)
if len(hcl_errors) > 0:
log.error(
f"Identified error in the following HCL Config: {dbfs_file_hcl}"
)
log.error(hcl_errors)
def export_cli(dry_run, tag, delete, git_ssh_url, api_client: ApiClient, hcl,
pattern_matches):
block_key_map = {}
ignore_attribute_key = {"last_updated_timestamp"}
required_attributes_key = {"key"}
if hcl:
secret_api = SecretApi(api_client)
scopes = secret_api.list_scopes()["scopes"]
log.info(scopes)
with GitExportHandler(git_ssh_url,
"secrets",
delete_not_found=delete,
dry_run=dry_run,
tag=tag) as gh:
for scope in scopes:
secrets = secret_api.list_secrets(scope["name"])["secrets"]
log.info(secrets)
for secret in secrets:
if not pattern_matches(secret["key"]):
log.debug(
f"{secret['key']} did not match pattern function {pattern_matches}"
)
continue
log.debug(
f"{secret['key']} matched the pattern function {pattern_matches}"
)
secret_resource_data = prep_json(block_key_map,
ignore_attribute_key,
secret,
required_attributes_key)
base_name = normalize_identifier(secret["key"])
name = "databricks_secret"
identifier = f"databricks_secret-{base_name}"
secret_resource_data["scope"] = scope["name"]
secret_hcl = create_resource_from_dict(
name, identifier, secret_resource_data, False)
file_name_identifier = f"{identifier}.tf"
gh.add_file(file_name_identifier, secret_hcl)
log.debug(secret_hcl)
def export_cli(dry_run, tag, delete, git_ssh_url, api_client: ApiClient, hcl,
pattern_matches):
block_key_map = {}
ignore_attribute_key = {}
required_attributes_key = {"instance_profile_arn"}
if hcl:
_data = {}
headers = None
profiles = api_client.perform_query(
'GET', '/instance-profiles/list', data=_data,
headers=headers)["instance_profiles"]
log.info(profiles)
with GitExportHandler(git_ssh_url,
"instance_profiles",
delete_not_found=delete,
dry_run=dry_run,
tag=tag) as gh:
for profile in profiles:
if not pattern_matches(profile["instance_profile_arn"]):
log.debug(
f"{profile['instance_profile_arn']} did not match pattern function {pattern_matches}"
)
continue
log.debug(
f"{profile['instance_profile_arn']} matched the pattern function {pattern_matches}"
)
profile_resource_data = prep_json(block_key_map,
ignore_attribute_key,
profile,
required_attributes_key)
base_name = normalize_identifier(
profile["instance_profile_arn"])
name = "databricks_instance_profile"
identifier = f"databricks_instance_profile-{base_name}"
#Force validation. If we import it, we might as well be able to use it
profile_resource_data["skip_validation"] = False
instance_profile_hcl = create_resource_from_dict(
name, identifier, profile_resource_data, False)
file_name_identifier = f"{identifier}.tf"
gh.add_file(file_name_identifier, instance_profile_hcl)
log.debug(instance_profile_hcl)
def export_cli(dry_run, tag, delete, git_ssh_url, api_client: ApiClient, hcl, pattern_matches):
block_key_map = {
"new_cluster": handle_block,
"notebook_task": handle_block,
"aws_attributes": handle_block,
"spark_env_vars": handle_block,
"autoscale": handle_block,
"spark_submit_task": handle_block,
"libraries": handle_libraries,
"email_notifications": handle_map,
"custom_tags": handle_map
}
ignore_attribute_key = {
"created_time", "creator_user_name", "job_id"
}
required_attributes_key = {
"max_concurrent_runs", "name"
}
if hcl:
job_api = JobsApi(api_client)
jobs = job_api.list_jobs()["jobs"]
log.info(jobs)
with GitExportHandler(git_ssh_url, "jobs", delete_not_found=delete, dry_run=dry_run, tag=tag) as gh:
for job in jobs:
if not pattern_matches(job["settings"]["name"]):
log.debug(f"{job['settings']['name']} did not match pattern function {pattern_matches}")
continue
log.debug(f"{job['settings']['name']} matched the pattern function {pattern_matches}")
job_resource_data = prep_json(block_key_map, ignore_attribute_key, job['settings'], required_attributes_key)
base_name = normalize_identifier(job['settings']['name'])
name = "databricks_job"
identifier = f"databricks_job-{base_name}"
#need to escape quotes in the name.
job_resource_data['name'] = job_resource_data['name'].replace('"','\\"')
instance_job_hcl = create_resource_from_dict(name, identifier, job_resource_data, False)
file_name_identifier = f"{identifier}.tf"
gh.add_file(file_name_identifier, instance_job_hcl)
log.debug(instance_job_hcl)
def _log_diff(self):
diff = self.repo.git.diff('HEAD', name_status=True)
if len(diff) is 0:
log.info("No files were changed and no diff was found.")
else:
for line in sorted(diff.split("\n")):
# get the relative path of the file relative to the handler directory
rel_path = line.split('\t')[1].replace(self.directory + "/",
"")
if line.startswith("D"):
self._git_removed.append(rel_path)
click.secho(f"File [A=added|M=modified|D=deleted]: {line}",
fg='red')
if line.startswith("A"):
self._git_added.append(rel_path)
click.secho(f"File [A=added|M=modified|D=deleted]: {line}",
fg='green')
if line.startswith("M"):
self._git_removed.append(rel_path)
click.secho(f"File [A=added|M=modified|D=deleted]: {line}",
fg='yellow')
def export_cli(dry_run, tag, delete, git_ssh_url, api_client: ApiClient, hcl,
pattern_matches):
block_key_map = {}
ignore_attribute_key = {}
required_attributes_key = {"principal", "permission"}
if hcl:
secret_api = SecretApi(api_client)
scopes = secret_api.list_scopes()["scopes"]
log.info(scopes)
with GitExportHandler(git_ssh_url,
"secret_acls",
delete_not_found=delete,
dry_run=dry_run,
tag=tag) as gh:
for scope in scopes:
acls = secret_api.list_acls(scope["name"])["items"]
log.info(acls)
for acl in acls:
acl_resource_data = prep_json(block_key_map,
ignore_attribute_key, acl,
required_attributes_key)
base_name = normalize_identifier(acl["principal"])
name = "databricks_secret_acl"
identifier = f"databricks_secret_acl-{base_name}"
acl_resource_data["scope"] = scope["name"]
acl_hcl = create_resource_from_dict(
name, identifier, acl_resource_data, False)
file_name_identifier = f"{identifier}.tf"
gh.add_file(file_name_identifier, acl_hcl)
log.debug(acl_hcl)
def export_cli(dry_run, tag, delete, git_ssh_url, api_client: ApiClient, hcl, pattern_matches):
block_key_map = {
"aws_attributes": handle_block,
"disk_spec": handle_block,
"custom_tags": handle_map
}
ignore_attribute_key = {
"stats", "state", "status", "default_tags", "instance_pool_id"
}
required_attributes_key = {
"instance_pool_name", "min_idle_instances", "idle_instance_autotermination_minutes", "node_type_id"
}
if hcl:
pool_api = InstancePoolsApi(api_client)
pools = pool_api.list_instance_pools()["instance_pools"]
log.info(pools)
with GitExportHandler(git_ssh_url, "instance_pools", delete_not_found=delete, dry_run=dry_run, tag=tag) as gh:
for pool in pools:
if not pattern_matches(pool["instance_pool_name"]):
log.debug(f"{pool['instance_pool_name']} did not match pattern function {pattern_matches}")
continue
log.debug(f"{pool['instance_pool_name']} matched the pattern function {pattern_matches}")
pool_resource_data = prep_json(block_key_map, ignore_attribute_key, pool, required_attributes_key)
base_name = normalize_identifier(pool["instance_pool_name"])
name = "databricks_instance_pool"
identifier = f"databricks_instance_pool-{base_name}"
instance_pool_hcl = create_resource_from_dict(name, identifier, pool_resource_data, False)
file_name_identifier = f"{identifier}.tf"
gh.add_file(file_name_identifier, instance_pool_hcl)
log.debug(instance_pool_hcl)
def __exit__(self, exc_type, exc_val, exc_tb):
# If not ignoring deleted remote state, delete all files not explicitly added
if self.delete_not_found is True:
self._remove_unmanaged_files()
log.info("===IDENTIFYING AND STAGING GIT CHANGES===")
# Stage Changes for logging diff
self._stage_changes()
# Log Changes
self._log_diff()
self._stage_changes()
# First differences need to be logged before applying change log
self._create_or_update_change_log()
# Stage stage the change log TODO: maybe this should be a decorator
self._stage_changes()
# Handle Dry Run
if self.dry_run is not None and self.dry_run is False:
# push all changes
self._push()
log.info("===FINISHED PUSHING CHANGES===")
# Tag and push previous changes
if self.tag is True:
# Create tag
self._create_tag()
# push the tag
self._push_tags()
log.info(f"===FINISHED PUSHING TAG {self._tag_value}===")
else:
log.info("===RUNNING IN DRY RUN MODE NOT PUSHING CHANGES===")
# clean temp folder
self.local_repo_path.cleanup()
def callback(ctx, param, value): # NOQA
if value is True:
log.info("===TAGGING IS ENABLED===")
return value
def callback(ctx, param, value): # NOQA
if value is True:
log.info("===RUNNING IN DRY RUN MODE===")
return value
def cmd(self, cmds, *args, **kwargs):
"""
run a terraform command, if success, will try to read state file
:param cmd: command and sub-command of terraform, seperated with space
refer to https://www.terraform.io/docs/commands/index.html
:param args: arguments of a command
:param kwargs: any option flag with key value without prefixed dash character
if there's a dash in the option name, use under line instead of dash,
ex. -no-color --> no_color
if it's a simple flag with no value, value should be IsFlagged
ex. cmd('taint', allow＿missing=IsFlagged)
if it's a boolean value flag, assign True or false
if it's a flag could be used multiple times, assign list to it's value
if it's a "var" variable flag, assign dictionary to it
if a value is None, will skip this option
if the option 'capture_output' is passed (with any value other than
True), terraform output will be printed to stdout/stderr and
"None" will be returned as out and err.
if the option 'raise_on_error' is passed (with any value that evaluates to True),
and the terraform command returns a nonzerop return code, then
a TerraformCommandError exception will be raised. The exception object will
have the following properties:
returncode: The command's return code
out: The captured stdout, or None if not captured
err: The captured stderr, or None if not captured
:return: ret_code, out, err
"""
capture_output = kwargs.pop('capture_output', True)
raise_on_error = kwargs.pop('raise_on_error', False)
if capture_output is True:
stderr = subprocess.PIPE
stdout = subprocess.PIPE
else:
stderr = sys.stderr
stdout = sys.stdout
# cmds = self.generate_cmd_string(cmd, *args, **kwargs)
log.info('command: {c}'.format(c=' '.join(cmds)))
working_folder = self.working_dir if self.working_dir else None
environ_vars = {}
if self.is_env_vars_included:
environ_vars = os.environ.copy()
p = subprocess.Popen(cmds, stdout=stdout, stderr=stderr,
cwd=working_folder, env=environ_vars)
synchronous = kwargs.pop('synchronous', True)
if not synchronous:
return p, None, None
out, err = p.communicate()
ret_code = p.returncode
log.debug('output: {o}'.format(o=out))
if capture_output is True:
out = out.decode('utf-8')
err = err.decode('utf-8')
else:
out = None
err = None
if ret_code != 0 and raise_on_error:
raise TerraformCommandError(
ret_code, ' '.join(cmds), out=out, err=err)
return ret_code, out, err
def __enter__(self):
# Get the terraform code
self._get_code()
# identify targets and changes
self._identify_files_to_stage()
# stage the terraform files
self.tf_stage_directory = tempfile.TemporaryDirectory()
self._add_provider_block()
log.info(f"staged provider")
self._add_output_tag_block()
log.info(f"added provider")
if self.backend_file is not None:
self._add_back_end_file()
log.info(f"added backend")
for file_path in self.targeted_files_abs_paths:
self._stage_file(file_path)
log.info("copied files")
contents = os.listdir(self.tf_stage_directory.name)
log.info(f"TF contents: {contents}")
self._terraform = Terraform(self.tf_stage_directory.name, True)
if self.init is True:
log.info("RUNNING TERRAFORM INIT")
self._init()
return self