def get_output_variable_with_retries(self, module, variable, retries):
"""Get output variable from terraform plan (with retries)"""
terraform_refresh = [
"refresh", "-input=false",
"-var", "gcp_credentials=" + self.gcp_credentials,
"-var", "gcp_project_name=" + self.gcp_project_name,
"-var", "azr_client_id=" + self.azr_client_id,
"-var", "azr_client_secret=" + self.azr_client_secret,
"-var", "azr_resource_group_name=" + self.azr_resource_group_name,
"-var", "azr_subscription_id=" + self.azr_subscription_id,
"-var", "azr_tennant_id=" + self.azr_tennant_id,
"-var", "image_tag=" + self.image_tag,
"-var", "image_version=" + self.image_family.replace(".", "-"),
"-var", "dns_managed_zone_domain=" + self.hosted_zone,
"-target=module." + module + ".azurerm_virtual_machine.vm"
]
# Check if there AWS keys or EC2 role should be used
if not self.aws_ec2_role:
terraform_refresh.append("-var")
terraform_refresh.append("aws_secret_key={}".format(self.s3_secret_key))
terraform_refresh.append("-var")
terraform_refresh.append("aws_access_key={}".format(self.s3_access_key_id))
log_info("Getting {0!r} output for {1!r} from terraform state...".format(variable, module))
var = self.get_output_variable(module, variable)
i = 1
cwd = os.getcwd()
os.chdir(self.terraform_dir)
while var is None and i <= retries:
log_warn('Retry to get {0!r} output from terraforn state. ({1} of {2})'.format(variable, i, retries))
log_info('Refresh terraform module...')
try:
if not self.config.get_debug():
sh.terraform(terraform_refresh)
else:
cmd = sh.terraform(terraform_refresh, _out=self.terraform_process_output, _bg=True)
cmd.wait()
except sh.ErrorReturnCode as err:
log_info(err.full_cmd)
log_info('Command output:' + err.stdout.decode('UTF-8').rstrip())
log_error(err.stderr.decode('UTF-8').rstrip(), nl=False)
log_error("Unexpected terraform error during refresh (status code {0!r})".format(err.exit_code))
var = self.get_output_variable(module, variable)
i = i + 1
os.chdir(cwd)
return var
def run(self, command):
self.command = command
tf_args = self.tf_args[:]
with sh.pushd(self.tf_module_path):
try:
if not self.overwrite_default_args:
log.info(
'Merging {} default and user provided args'.format(
command))
tf_args = TF_DEFAULT_ARGS[self.version].get(command,
[]) + tf_args
tf_args.insert(0, command)
if self.dry_run:
log.info('Dry run enabled.')
log.info('Would run: terraform {}'.format(
" ".join(tf_args)))
else:
terraform(tf_args)
except ErrorReturnCode as err:
log.error(err.stderr)
def iter_states(root=None):
root = root or os.getcwd()
curdir = os.getcwd()
for dpath, dnames, fnames in os.walk(root):
if '.terraform' in dnames:
try:
os.chdir(dpath)
output = sh.terraform("state", "pull").stdout.decode('utf-8')
start_index = output.find('{')
if start_index < 0:
start_index = 0
yield json.loads(output[start_index:])
finally:
os.chdir(curdir)
def iter_states(root=None):
root = root or os.getcwd()
curdir = os.getcwd()
for dpath, dnames, fnames in os.walk(root):
if '.terraform' in dnames:
try:
os.chdir(dpath)
output = sh.terraform("state", "pull").stdout.decode('utf-8')
start_index = output.find('{')
if start_index < 0:
start_index = 0
yield json.loads(output[start_index:])
finally:
os.chdir(curdir)
def get_output_variable(self, module, variable):
"""Get output variable from terraform plan"""
cwd = os.getcwd()
os.chdir(self.terraform_dir)
try:
cmd = sh.terraform("output", "-module=" + module, variable)
except sh.ErrorReturnCode as err:
log_info(err.full_cmd)
log_info('Command output:' + err.stdout.decode('UTF-8').rstrip())
log_error(err.stderr.decode('UTF-8').rstrip(), nl=False)
log_error("Unexpected terraform error during output (status code {0!r})".format(err.exit_code))
os.chdir(cwd)
return None
os.chdir(cwd)
if not bool(cmd.strip()):
return None
else:
return cmd.strip()
def get_or_install_tf_version(self, version=None):
if version:
return self._install_terraform(version)
installed_version = ""
latest_version = ""
try:
version_output = str(sh.terraform("--version"))
except:
version_output = ""
version_regex = "[0-9]+\.[0-9]+\.[0-9]+"
installed_search = re.search("Terraform v(%s)" % (version_regex), version_output)
if installed_search:
installed_version = installed_search.group(1)
if self.auto_increment or not installed_version:
latest_version = requests.get("https://api.github.com/repos/hashicorp/terraform/releases/latest").json()['tag_name'][1:]
if installed_version != latest_version and latest_version:
return self._install_terraform(latest_version)
return installed_version
def plan_execute(self):
"""Execute terraform plan"""
terraform_steps = {
'init': [
"init", "-input=false",
"-force-copy"
],
'apply': [
"apply", "-input=false",
"-var", "gcp_credentials=" + self.gcp_credentials,
"-var", "gcp_project_name=" + self.gcp_project_name,
"-var", "azr_client_id=" + self.azr_client_id,
"-var", "azr_client_secret=" + self.azr_client_secret,
"-var", "azr_resource_group_name=" + self.azr_resource_group_name,
"-var", "azr_subscription_id=" + self.azr_subscription_id,
"-var", "azr_tennant_id=" + self.azr_tennant_id,
"-var", "production=" + self.production,
"-var", "bastion_ip=" + self.config.get_attr('fabric_manager')['ip'],
"-var", "image_tag=" + self.image_tag,
"-var", "image_version=" + self.image_family.replace(".", "-"),
"-var", "dns_managed_zone_domain=" + self.hosted_zone,
"-auto-approve"
]
}
# Check if there AWS keys or EC2 role should be used
if not self.aws_ec2_role:
terraform_steps['init'].append("-backend-config=secret_key={}".format(self.s3_secret_key))
terraform_steps['init'].append("-backend-config=access_key={}".format(self.s3_access_key_id))
terraform_steps['apply'].append("-var")
terraform_steps['apply'].append("aws_secret_key={}".format(self.s3_secret_key))
terraform_steps['apply'].append("-var")
terraform_steps['apply'].append("aws_access_key={}".format(self.s3_access_key_id))
cwd = os.getcwd()
log_info("Running terraform init and apply...")
os.chdir(self.terraform_dir)
# Check if there is any terraform already running
proc_name = 'terraform'
proc_result = check_process_running(proc_name)
if proc_result[0]:
log_error("There is already {!r} process (PID {!r}) running for user {!r}. Please retry again "
"later...".format(proc_name, str(proc_result[1]), proc_result[2]))
return False, 1
def show_step(item):
"""Print current step"""
# We need to return next step as progressbar prints previously completed step
if item is not None:
t_keys = list(terraform_steps.keys())
idx = t_keys.index(item)
if idx == len(t_keys) - 1:
return '-> {0}'.format(item)
else:
return '-> {0}'.format(t_keys[idx + 1])
if not self.config.get_debug():
with click.progressbar(terraform_steps, item_show_func=show_step, show_eta=False) as bar:
for step in bar:
try:
sh.terraform(terraform_steps[step])
except sh.ErrorReturnCode as err:
log_info(err.full_cmd)
log_info('Command output:' + err.stdout.decode('UTF-8').rstrip())
log_error(err.stderr.decode('UTF-8').rstrip(), nl=False)
log_error("Unexpected terraform error during {0!s} (status code {1!r})".format(step,
err.exit_code))
os.chdir(cwd)
return False, err.exit_code
else:
for step in terraform_steps:
try:
cmd = sh.terraform(terraform_steps[step], _out=self.terraform_process_output, _bg=True)
cmd.wait()
except sh.ErrorReturnCode as err:
log_info(err.full_cmd)
log_info('Command output:' + err.stdout.decode('UTF-8').rstrip())
log_error(err.stderr.decode('UTF-8').rstrip(), nl=False)
log_error("Unexpected terraform error during {0!s} (status code {1!r})".format(step,
err.exit_code))
os.chdir(cwd)
return False, err.exit_code
os.chdir(cwd)
return True, 0
from jinja2 import Environment, FileSystemLoader
import os
import sys
import sh
token = os.getenv('TOKEN')
project = os.getenv('PROJECT')
def list_rules():
compute = googleapiclient.discovery.build('compute', 'v1')
result = compute.firewalls().list(project=project).execute()
return result['items']
if len(sys.argv) > 1:
if str(sys.argv[1]) == 'state':
for i in list_rules():
rule = "google_compute_firewall.{}".format(i['name'])
sh.terraform("import", rule, i['name'])
elif str(sys.argv[1]) == 'config':
env = Environment(loader=FileSystemLoader('templates'))
template = env.get_template('firewall.j2')
output_from_parsed_template = template.render(rules=list_rules())
print output_from_parsed_template
else:
raise ValueError(
'you need to specify to either generate config or state')
else:
raise ValueError('you need to specify to either generate config or state')