def copy(self, **kwargs) -> None:
""" copy clones a remote git repo, and puts the requested files into the destination """
dest = self.get_destination(**kwargs)
branch = "master"
if "branch" in kwargs:
branch = kwargs["branch"]
if "sub_path" in kwargs:
sub_path = kwargs["sub_path"].strip("/")
else:
sub_path = ""
self.make_temp()
temp_path = f"{self._temp_dir}/{sub_path}"
pipe_exec(
f"git clone {self._source} --branch {branch} --single-branch ./",
cwd=self._temp_dir,
)
try:
self.check_conflicts(temp_path)
except FileExistsError as e:
self.clean_temp()
raise e
shutil.copytree(temp_path, dest, dirs_exist_ok=True)
self.clean_temp()
def test_pipe_exec(self, commands, exit_code, cwd, stdin, stdout, stderr):
(return_exit_code, return_stdout,
return_stderr) = pipe_exec(commands, cwd=cwd, stdin=stdin)
assert return_exit_code == exit_code
assert stdout.encode() in return_stdout.rstrip()
assert return_stderr.rstrip() in stderr.encode()
def get_state_item(working_dir, env, terraform_bin, state, item):
"""
get_state_item returns json encoded output from a terraform remote state
"""
base_dir, _ = os.path.split(working_dir)
try:
(exit_code, stdout, stderr) = pipe_exec(
f"{terraform_bin} output -json -no-color {item}",
cwd=f"{base_dir}/{state}",
env=env,
)
except FileNotFoundError:
# the remote state is not setup, likely do to use of --limit
# this is acceptable, and is the responsibility of the hook
# to ensure it has all values needed for safe execution
return None
if exit_code != 0:
raise HookError(
f"Error reading remote state item {state}.{item}, details: {stderr}"
)
if stdout is None:
raise HookError(
f"Remote state item {state}.{item} is empty; This is completely"
" unexpected, failing...")
json_output = json.loads(stdout)
return json.dumps(json_output, indent=None, separators=(",", ":"))
def copy(self, **kwargs) -> None:
""" copy clones a remote git repo, and puts the requested files into the destination """
dest = self.get_destination(**kwargs)
branch = "master"
git_cmd = "git"
git_args = ""
reset_repo = False
sub_path = ""
if "sub_path" in kwargs:
sub_path = kwargs["sub_path"].strip("/")
if "branch" in kwargs:
branch = kwargs["branch"]
if "git_cmd" in kwargs:
git_cmd = kwargs["git_cmd"]
if "git_args" in kwargs:
git_args = kwargs["git_args"]
if "reset_repo" in kwargs:
reset_repo = kwargs["reset_repo"]
self.make_temp()
temp_path = f"{self._temp_dir}/{sub_path}"
pipe_exec(
re.sub(
r"\s+",
" ",
f"{git_cmd} {git_args} clone {self._source} --branch {branch} --single-branch ./",
),
cwd=self._temp_dir,
)
try:
self.check_conflicts(temp_path)
except FileExistsError as e:
self.clean_temp()
raise e
if reset_repo:
self.repo_clean(f"{temp_path}")
shutil.copytree(temp_path, dest, dirs_exist_ok=True)
self.clean_temp()
def type_match(source: str, **kwargs) -> bool:
""" type matches uses git to see if the source is a valid git remote """
try:
(return_code, _, _) = pipe_exec(f"git ls-remote {source}")
except (PermissionError, FileNotFoundError):
return False
if return_code == 0:
return True
return False
def get_terraform_version(terraform_bin):
(return_code, stdout, stderr) = pipe_exec(f"{terraform_bin} version")
if return_code != 0:
click.secho(f"unable to get terraform version\n{stderr}", fg="red")
raise SystemExit(1)
version = stdout.decode("UTF-8").split("\n")[0]
version_search = re.search(r".* v\d+\.(\d+)\.(\d+)", version)
if version_search:
click.secho(
f"Terraform Version Result: {version}, using major:{version_search.group(1)}, minor:{version_search.group(2)}",
fg="yellow",
)
return (int(version_search.group(1)), int(version_search.group(2)))
else:
click.secho(f"unable to get terraform version\n{stderr}", fg="red")
raise SystemExit(1)
def type_match(source: str, **kwargs) -> bool:
""" type matches uses git to see if the source is a valid git remote """
git_cmd = "git"
git_args = ""
if "git_cmd" in kwargs:
git_cmd = kwargs["git_cmd"]
if "git_args" in kwargs:
git_args = kwargs["git_args"]
try:
(return_code, _,
_) = pipe_exec(f"{git_cmd} {git_args} ls-remote {source}")
except (PermissionError, FileNotFoundError):
return False
if return_code == 0:
return True
return False
def hook_exec(
phase,
command,
working_dir,
env,
terraform_path,
debug=False,
b64_encode=False,
):
"""
hook_exec executes a hook script.
Before execution it sets up the environment to make all terraform and remote
state variables available to the hook via environment vars
"""
key_replace_items = {
" ": "",
'"': "",
"-": "_",
".": "_",
}
val_replace_items = {
" ": "",
'"': "",
"\n": "",
}
local_env = env.copy()
hook_dir = f"{working_dir}/hooks"
hook_script = None
for f in os.listdir(hook_dir):
# this file format is specifically structured by the prep_def function
if os.path.splitext(f)[0] == f"{phase}_{command}":
hook_script = f"{hook_dir}/{f}"
# this should never have been called if the hook script didn't exist...
if hook_script is None:
raise HookError(f"hook script missing from {hook_dir}")
# populate environment with terraform remotes
if os.path.isfile(f"{working_dir}/worker-locals.tf"):
# I'm sorry. :-)
r = re.compile(
r"\s*(?P<item>\w+)\s*\=.+data\.terraform_remote_state\.(?P<state>\w+)\.outputs\.(?P<state_item>\w+)\s*"
)
with open(f"{working_dir}/worker-locals.tf") as f:
for line in f:
m = r.match(line)
if m:
item = m.group("item")
state = m.group("state")
state_item = m.group("state_item")
else:
continue
state_value = TerraformCommand.get_state_item(
working_dir, env, terraform_path, state, state_item)
if state_value is not None:
if b64_encode:
state_value = base64.b64encode(
state_value.encode("utf-8")).decode()
local_env[
f"TF_REMOTE_{state}_{item}".upper()] = state_value
# populate environment with terraform variables
if os.path.isfile(f"{working_dir}/worker.auto.tfvars"):
with open(f"{working_dir}/worker.auto.tfvars") as f:
for line in f:
tf_var = line.split("=")
# strip bad names out for env var settings
for k, v in key_replace_items.items():
tf_var[0] = tf_var[0].replace(k, v)
for k, v in val_replace_items.items():
tf_var[1] = tf_var[1].replace(k, v)
if b64_encode:
tf_var[1] = base64.b64encode(
tf_var[1].encode("utf-8")).decode()
local_env[f"TF_VAR_{tf_var[0].upper()}"] = tf_var[1]
else:
click.secho(
f"{working_dir}/worker.auto.tfvars not found!",
fg="red",
)
# execute the hook
(exit_code, stdout, stderr) = pipe_exec(
f"{hook_script} {phase} {command}",
cwd=hook_dir,
env=local_env,
)
# handle output from hook_script
if debug:
click.secho(f"exit code: {exit_code}", fg="blue")
for line in stdout.decode().splitlines():
click.secho(f"stdout: {line}", fg="blue")
for line in stderr.decode().splitlines():
click.secho(f"stderr: {line}", fg="red")
if exit_code != 0:
raise HookError("hook script {}")
def _run(self, definition, command, debug=False, plan_action="init"):
"""Run terraform."""
if self._tf_version_major >= 12:
params = {
"init":
f"-input=false -no-color -plugin-dir={self._temp_dir}/terraform-plugins",
"plan": "-input=false -detailed-exitcode -no-color",
"apply": "-input=false -no-color -auto-approve",
"destroy": "-input=false -no-color -force",
}
if self._tf_version_major >= 15:
params["destroy"] = "-input=false -no-color -auto-approve"
else:
params = {
"init": "-input=false -no-color",
"plan": "-input=false -detailed-exitcode -no-color",
"apply": "-input=false -no-color -auto-approve",
"destroy": "-input=false -no-color -force",
}
if plan_action == "destroy":
params["plan"] += " -destroy"
env = os.environ.copy()
for auth in self._authenticators:
env.update(auth.env())
env["TF_PLUGIN_CACHE_DIR"] = f"{self._temp_dir}/terraform-plugins"
working_dir = f"{self._temp_dir}/definitions/{definition.tag}"
command_params = params.get(command)
if not command_params:
raise ValueError(
f"invalid command passed to terraform, {command} has no defined params!"
)
# only execute hooks for plan/apply/destroy
try:
if TerraformCommand.check_hooks(
"pre", working_dir,
command) and command in ["apply", "destroy", "plan"]:
# pre exec hooks
# want to pass remotes
# want to pass tf_vars
click.secho(
f"found pre-{command} hook script for definition {definition.tag},"
" executing ",
fg="yellow",
)
TerraformCommand.hook_exec(
"pre",
command,
working_dir,
env,
self._terraform_bin,
debug=debug,
b64_encode=self._b64_encode,
)
except HookError as e:
click.secho(
f"hook execution error on definition {definition.tag}: {e}",
fg="red",
)
raise SystemExit(2)
click.secho(f"cmd: {self._terraform_bin} {command} {command_params}",
fg="yellow")
(exit_code, stdout, stderr) = pipe_exec(
f"{self._terraform_bin} {command} {command_params}",
cwd=working_dir,
env=env,
)
if debug:
click.secho(f"exit code: {exit_code}", fg="blue")
for line in stdout.decode().splitlines():
click.secho(f"stdout: {line}", fg="blue")
for line in stderr.decode().splitlines():
click.secho(f"stderr: {line}", fg="red")
# special handling of the exit codes for "plan" operations
if command == "plan":
if exit_code == 0:
return True
if exit_code == 1:
raise TerraformError
if exit_code == 2:
raise PlanChange
if exit_code:
raise TerraformError
# only execute hooks for plan/destroy
try:
if TerraformCommand.check_hooks(
"post", working_dir,
command) and command in ["apply", "destroy", "plan"]:
click.secho(
f"found post-{command} hook script for definition {definition.tag},"
" executing ",
fg="yellow",
)
TerraformCommand.hook_exec(
"post",
command,
working_dir,
env,
self._terraform_bin,
debug=debug,
b64_encode=self._b64_encode,
)
except HookError as e:
click.secho(
f"hook execution error on definition {definition.tag}: {e}",
fg="red")
raise SystemExit(2)
return True