def show_step_inner_messaage(self, message, status, error_msg=None):
dot_len = self.column_length - len(message) - 30
print_message = "\t%s %s [%s]" % (
message, self._get_line_dots_in_color(dot_len),
self._get_status_in_color(status))
SysLog().write_debug_log(print_message)
print(print_message)
if error_msg:
print("\t\t%s" % self._get_error_msg_in_color(error_msg))
SysLog().write_error_log(error_msg)
def run(self, sys_args):
self.show_loading_messsage()
Settings.set('running_command', ' '.join(sys_args))
try:
SysLog().debug_started(Settings.running_command)
if self.do_pre_requisite_check():
command_class_instance = self.get_command_class_instance(
sys_args) # Get the command list and optional commands
self.execute(command_class_instance)
except Exception as e:
self.show_step_inner_error(
"Error occured, please check error log for more details")
SysLog().write_error_log(str(e))
def display_op_msg(self, display_op_list):
"""
Display output message at the end of process execution
Args:
display_op_list (list): List of key, value pairs to be displayed
"""
if display_op_list:
result_title = "OUTPUT"
column_length = self.column_length - 10
pre_star_count = math.ceil(
int(column_length - len(result_title) - 2) / 2)
post_star_count = math.floor(
int(column_length - len(result_title) - 2) / 2)
heading = "\n\t%s %s %s" % ("*" * pre_star_count, result_title,
"*" * post_star_count)
print(self._get_heading_message_in_color(heading, self.BCYAN_ANSI))
debug_log_msg = ""
for op in display_op_list:
for key, val in op.items():
key_value_msg = "\t%20s: %s" % (key, val)
debug_log_msg += key_value_msg + "\n"
print(self.GREEN_ANSI + key_value_msg + self.RESET_ANSI)
SysLog().write_debug_log(heading + "\n" + debug_log_msg)
end = "\t" + "*" * column_length
print(self._get_heading_message_in_color(end, self.BCYAN_ANSI))
print("\n")
def pre_terraform_apply(self):
status, msg = create_iam_service_linked_role(
"rds.amazonaws.com",
Settings.RESOURCE_DESCRIPTION,
Settings.AWS_AUTH_CRED)
SysLog().write_debug_log("RDS IAM Service Linked role creation: Status:%s, Message: %s" % (str(status), msg))
def show_step_inner_messaage(self, message, status, error_msg=None):
"""
Show an inner message
Args:
message (str): Message to be displayed on the line
"""
dot_len = self.column_length - len(message) - 30
print_message = "\t%s %s [%s]" % (
message, self._get_line_dots_in_color(dot_len),
self._get_status_in_color(status))
SysLog().write_debug_log(print_message)
print(print_message)
if error_msg:
print("\t\t%s" % self._get_error_msg_in_color(error_msg))
SysLog().write_error_log(error_msg)
def pre_terraform_apply(self):
status, msg = create_iam_service_linked_role(
Settings.AWS_ACCESS_KEY, Settings.AWS_SECRET_KEY,
"ecs.amazonaws.com", Settings.RESOURCE_DESCRIPTION)
SysLog().write_debug_log(
"ECS IAM Service Linked role creation: Status:%s, Message: %s" %
(str(status), msg))
def show_step_heading(self, heading, write_log=True):
if write_log:
SysLog().write_debug_log(heading)
step_count_num = Settings.get('step_count_num', 1)
print(
self._get_heading_message_in_color(
"\nStep %s: %s" % (str(step_count_num), heading),
self.BCYAN_ANSI))
step_count_num = Settings.set('step_count_num', step_count_num + 1)
def show_step_inner_warning(self, message):
"""
Show message as sep message i.e with a tab prefix to display warning
Args:
message (str): Message to be displayed on the line
"""
print_message = "\t%s" % message
SysLog().write_debug_log(print_message)
print(self.WARN_ANSI + print_message + self.RESET_ANSI)
def show_step_inner_error(self, message):
"""
Show message as sep message i.e with a tab prefix to display error
Args:
message (str): Message to be displayed on the line
"""
print_message = "\t%s" % self._get_error_msg_in_color(message)
SysLog().write_error_log(print_message)
print(print_message)
def generate_terraform(self):
if self.PROCESS: # Generate the resource terraform file only if the resource is to be processed
try:
terraform_args_dict = self.get_terraform_resource_args_dict()
self.create_terraform_resource_file(terraform_args_dict)
except Exception as e:
msg = 'Error occured in Terraform file generation. Resource: %s' % self.__class__.__name__
print(msg)
SysLog().write_error_log(str(e) + '\n' + msg)
sys.exit()
def post_terraform_apply(self):
archive_type = "zip"
s3_client = s3.get_s3_client(Settings.AWS_AUTH_CRED)
zip_file_name = Settings.RESOURCE_NAME_PREFIX + "-terraform-installer-backup"
zip_file_abs_path = os.path.join(Settings.BASE_APP_DIR, zip_file_name)
dir_to_archive = Settings.DATA_DIR
SysLog().write_debug_log("Started Archiving Terraform Directory")
shutil.make_archive(zip_file_abs_path, archive_type, dir_to_archive)
SysLog().write_debug_log("Completed Archiving")
bucket_name = BucketStorage.get_input_attr('bucket')
zip_file_name = zip_file_name + ".zip"
zip_file_abs_path = zip_file_abs_path + ".zip"
SysLog().write_debug_log("Started Uploading Archived Terraform(Zip File: %s) into S3 Bucket(Name: %s)" % (zip_file_abs_path, bucket_name))
s3_client.upload_file(
zip_file_abs_path,
bucket_name,
zip_file_name)
os.remove(zip_file_abs_path)
def exit_with_validation_errors(self, errors):
for key, error_list in errors.items():
msg = '\nValidation Error. Resource %s\n' % key
msg = msg + '\n'.join(error_list)
print(msg)
SysLog().write_error_log(msg, with_trace=False)
def show_step_inner_warning(self, message):
print_message = "\t%s" % message
SysLog().write_debug_log(print_message)
print(self.WARN_ANSI + print_message + self.RESET_ANSI)
class PyTerraform():
"""
This is the main class which bridges between the python_terraform class and framework system
Attributes:
log_obj (obj): SysLog object used to write logs
"""
log_obj = SysLog()
def terraform_init(self):
"""
Run terraform init and raise excpetion if there is any error or response of the command
Returns:
response (dict): Response after terraform init
"""
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
)
self.log_obj.write_debug_log(K.TERRAFORM_INIT_STARTED)
response = terraform.init()
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_INIT_ERROR)
raise Exception(response[2])
self.log_obj.write_terraform_init_log(response)
return response
def terraform_plan(self, resources=None):
"""
Run terraform plan and raise excpetion if there is any error or response of the command
Args:
resources (list): List of resources if there are targets else None
Returns:
response (dict): Response after terraform plan
"""
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
targets=self.get_target_resources(resources)
)
self.log_obj.write_debug_log(K.TERRAFORM_PLAN_STARTED)
response = terraform.plan()
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_PLAN_ERROR)
raise Exception(response[2])
self.log_obj.write_terraform_plan_log(response)
return response
def terraform_apply(self, resources=None):
"""
Run terraform apply and raise excpetion if there is any error or response of the command
Args:
resources (list): List of resources if there are targets else None
Returns:
response (dict): Response after terraform apply
"""
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
CMD = Settings.get('running_command', "Terraform Apply")
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
targets=self.get_target_resources(resources),
stdout_log_file=self.log_obj.get_terraform_install_log_file()
)
self.log_obj.write_terraform_apply_log_header()
# In order to -auto-approve we need to pass skip_plan=True for python3
response = terraform.apply(skip_plan=True)
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_APPLY_ERROR)
self.write_current_status(CMD, K.APPLY_STATUS_ERROR, response[2])
raise Exception(response[2])
self.write_current_status(CMD, K.APPLY_STATUS_COMPLETED, K.TERRAFORM_APPLY_COMPLETED)
return response
def terraform_destroy(self, resources=None):
"""
Run terraform destroy and raise excpetion if there is any error or response of the command
Args:
resources (list): List of resources if there are targets else None
Returns:
response (dict): Response after terraform destroy
"""
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
CMD = Settings.get('running_command', "Terraform Destroy")
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
targets=self.get_target_resources(resources),
stdout_log_file=self.log_obj.get_terraform_destroy_log_file()
)
self.log_obj.write_terraform_destroy_log_header()
kwargs = {"auto_approve": True}
response = terraform.destroy(**kwargs)
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_DESTROY_ERROR)
self.write_current_status(CMD, K.DESTROY_STATUS_ERROR, response[2])
raise Exception(response[2])
self.write_current_status(CMD, K.DESTROY_STATUS_COMPLETED, K.TERRAFORM_DESTROY_COMPLETED)
return response
def process_destroy_result(self, p):
"""
Store the destroy response and riase exception if there is any
Args:
p (process obj): process obj of the terraform destroy
"""
response = Terraform().return_process_result(p)
CMD = Settings.get('running_command', "Terraform Destroy")
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_DESTROY_ERROR)
self.write_current_status(CMD, K.DESTROY_STATUS_ERROR, response[2])
raise Exception(response[2])
self.write_current_status(CMD, K.DESTROY_STATUS_COMPLETED, K.TERRAFORM_DESTROY_COMPLETED)
def terraform_taint(self, resources):
"""
Run terraform taint on the mentioned resources
Args:
resources (list): List of resources if there are targets else None
Returns:
response (dict): Response after terraform taint
"""
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
)
taint_resources = self.get_taint_resources(resources)
self.log_obj.write_debug_log(K.TERRAFORM_TAINT_STARTED)
for resource_name in taint_resources:
response = terraform.cmd("taint", resource_name)
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_TAINT_ERROR)
raise Exception(response[2])
self.log_obj.write_debug_log(K.TERRAFORM_TAINT_COMPLETED)
return response
def get_target_resources(self, resources):
"""
Get list of terraform targets arguments to be supplied to terraform command
Args:
resources (list): List of resources if there are targets else None
Returns:
targets (list / none): list of resources to be added as targets if there is any else None
"""
if resources:
targets = []
for resource in resources:
# DO NOT process this resource as its definiiton asked to skip
if resource.PROCESS is False:
continue
if BaseTerraformVariable not in inspect.getmro(resource.__class__) and TerraformData not in inspect.getmro(resource.__class__):
targets.append(get_terraform_resource_path(resource))
return targets
return None
def get_taint_resources(self, resources):
"""
Get list of terraform resources to be tainted
Args:
resources (list): List of resources
Returns:
taint_resources (list): List of resources to be tainted
"""
taint_resources = []
for resource in resources:
if TerraformResource in inspect.getmro(resource.__class__):
taint_resources.append(get_terraform_resource_path(resource))
return taint_resources
@classmethod
def save_terraform_output(cls):
"""
Save terraform output to the output file
Returns:
output_dict (dict): Terraform output
"""
tf_output_file = get_terraform_latest_output_file()
output_dict = cls.load_terraform_output()
with open(tf_output_file, 'w') as jsonfile:
json.dump(output_dict, jsonfile, indent=4)
cls.log_obj.write_debug_log(K.TERRAFORM_OUTPUT_STORED)
return output_dict
@classmethod
def load_terraform_output(cls):
"""
Load terraform output form the output command
Returns:
output_dict (dict): Terraform output
"""
output_dict = {}
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
)
response = terraform.output()
if response:
for key, item in response.items():
key_splitted = key.split('-')
resource_key = '-'.join(key_splitted[0:-1])
if resource_key in output_dict:
output_dict[resource_key][key_splitted[-1]] = item['value']
else:
output_dict[resource_key] = {key_splitted[-1]: item['value']}
return output_dict
@classmethod
def load_terraform_output_from_json_file(cls):
"""
Load terraform output form the output file
Returns:
output_dict (dict): Terraform output
"""
tf_output_file = get_terraform_latest_output_file()
output_dict = {}
if os.path.exists(tf_output_file):
with open(tf_output_file) as jsonfile:
output_dict = json.load(jsonfile)
return output_dict
def write_current_status(self, command, status_code, description=""):
"""
Write current status for the executed comamnd to status file
Args:
command (str): Command name
status_code (str): Status of the current execution
description (str): Description of the current command
"""
current_status = self.get_current_status()
prev_status = None
if current_status:
prev_status = {
'status_code': current_status['status_code'],
'description': current_status['description'],
'last_exec_command': current_status['last_exec_command'],
'executed_time': current_status['executed_time']
}
current_status['status_code'] = status_code
current_status['description'] = description
current_status['last_exec_command'] = command
current_status['executed_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
if prev_status: # FIrst time previous status won't be available
current_status[prev_status['executed_time']] = prev_status
status_file = get_terraform_status_file()
with open(status_file, 'w') as jsonfile:
json.dump(current_status, jsonfile, indent=4)
@classmethod
def get_current_status(self):
"""
Write current status for the executed comamnd to status file
Returns:
status_dict (dict): Status dict to be written
"""
status_file = get_terraform_status_file()
status_dict = {}
if os.path.exists(status_file):
with open(status_file) as jsonfile:
status_dict = json.load(jsonfile)
return status_dict
def show_step_finish(self, end_heading, write_log=True, color=""):
if write_log:
SysLog().write_debug_log(end_heading)
end_heading = "\t%s" % end_heading
print(color + end_heading + self.RESET_ANSI)
class PyTerraform():
log_obj = SysLog()
def terraform_init(self):
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
)
self.log_obj.write_debug_log(K.TERRAFORM_INIT_STARTED)
response = terraform.init()
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_INIT_ERROR)
raise Exception(response[2])
self.log_obj.write_terraform_init_log(response)
return response
def terraform_plan(self, resources=None):
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
targets=self.get_target_resources(resources)
)
self.log_obj.write_debug_log(K.TERRAFORM_PLAN_STARTED)
response = terraform.plan()
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_PLAN_ERROR)
raise Exception(response[2])
self.log_obj.write_terraform_plan_log(response)
return response
def terraform_apply(self, resources=None):
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
CMD = Settings.get('running_command', "Terraform Apply")
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
targets=self.get_target_resources(resources),
stdout_log_file=self.log_obj.get_terraform_install_log_file()
)
self.log_obj.write_terraform_apply_log_header()
# In order to -auto-approve we need to pass skip_plan=True for python3
response = terraform.apply(skip_plan=True)
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_APPLY_ERROR)
self.write_current_status(CMD, K.APPLY_STATUS_ERROR, response[2])
raise Exception(response[2])
self.write_current_status(CMD, K.APPLY_STATUS_COMPLETED, K.TERRAFORM_APPLY_COMPLETED)
return response
def terraform_destroy(self, resources=None):
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
CMD = Settings.get('running_command', "Terraform Destroy")
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
targets=self.get_target_resources(resources),
stdout_log_file=self.log_obj.get_terraform_destroy_log_file()
)
self.log_obj.write_terraform_destroy_log_header()
kwargs = {"auto_approve": True}
response = terraform.destroy(**kwargs)
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_DESTROY_ERROR)
self.write_current_status(CMD, K.DESTROY_STATUS_ERROR, response[2])
raise Exception(response[2])
self.write_current_status(CMD, K.DESTROY_STATUS_COMPLETED, K.TERRAFORM_DESTROY_COMPLETED)
return response
def terraform_taint(self, resources):
if exists_teraform_lock():
raise Exception(K.ANOTHER_PROCESS_RUNNING)
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
)
taint_resources = self.get_taint_resources(resources)
self.log_obj.write_debug_log(K.TERRAFORM_TAINT_STARTED)
for resource_name in taint_resources:
response = terraform.cmd("taint", resource_name)
if response[0] == 1:
self.log_obj.write_debug_log(K.TERRAFORM_TAINT_ERROR)
raise Exception(response[2])
self.log_obj.write_debug_log(K.TERRAFORM_TAINT_COMPLETED)
return response
def get_target_resources(self, resources):
if resources:
targets = []
for resource in resources:
# DO NOT process this resource as its definiiton asked to skip
if resource.PROCESS is False:
continue
if BaseTerraformVariable not in inspect.getmro(resource.__class__) and TerraformData not in inspect.getmro(resource.__class__):
targets.append(get_terraform_resource_path(resource))
return targets
return None
def get_taint_resources(self, resources):
taint_resources = []
for resource in resources:
if TerraformResource in inspect.getmro(resource.__class__):
taint_resources.append(get_terraform_resource_path(resource))
return taint_resources
@classmethod
def save_terraform_output(cls):
tf_output_file = get_terraform_latest_output_file()
output_dict = cls.load_terraform_output()
with open(tf_output_file, 'w') as jsonfile:
json.dump(output_dict, jsonfile, indent=4)
cls.log_obj.write_debug_log(K.TERRAFORM_OUTPUT_STORED)
return output_dict
@classmethod
def load_terraform_output(cls):
output_dict = {}
terraform = Terraform(
working_dir=Settings.TERRAFORM_DIR,
)
response = terraform.output()
if response:
for key, item in response.items():
key_splitted = key.split('-')
resource_key = '-'.join(key_splitted[0:-1])
if resource_key in output_dict:
output_dict[resource_key][key_splitted[-1]] = item['value']
else:
output_dict[resource_key] = {key_splitted[-1]: item['value']}
return output_dict
@classmethod
def load_terraform_output_from_json_file(cls):
tf_output_file = get_terraform_latest_output_file()
output_dict = {}
if os.path.exists(tf_output_file):
with open(tf_output_file) as jsonfile:
output_dict = json.load(jsonfile)
return output_dict
def write_current_status(self, command, status_code, description=""):
current_status = self.get_current_status()
prev_status = None
if current_status:
prev_status = {
'status_code': current_status['status_code'],
'description': current_status['description'],
'last_exec_command': current_status['last_exec_command'],
'executed_time': current_status['executed_time']
}
current_status['status_code'] = status_code
current_status['description'] = description
current_status['last_exec_command'] = command
current_status['executed_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
if prev_status: # FIrst time previous status won't be available
current_status[prev_status['executed_time']] = prev_status
status_file = get_terraform_status_file()
with open(status_file, 'w') as jsonfile:
json.dump(current_status, jsonfile, indent=4)
@classmethod
def get_current_status(self):
status_file = get_terraform_status_file()
status_dict = {}
if os.path.exists(status_file):
with open(status_file) as jsonfile:
status_dict = json.load(jsonfile)
return status_dict
def show_step_inner_error(self, message):
print_message = "\t%s" % self._get_error_msg_in_color(message)
SysLog().write_error_log(print_message)
print(print_message)
