def __init__(self, working_directory=os.path.dirname(__file__)): self.logger = Log(__name__) self.APPLY_COMMAND = "apply" self.DESTROY_COMMAND = "destroy" self.PLAN_COMMAND = "plan" self.INIT_COMMAND = "init" self.working_directory = working_directory
def __init__(self, cluster_model, config_docs): self.cluster_model = cluster_model self.cluster_name = self.cluster_model.specification.name.lower() self.cluster_prefix = self.cluster_model.specification.prefix.lower() self.resource_group_name = resource_name(self.cluster_prefix, self.cluster_name, 'rg') self.config_docs = config_docs self.logger = Log(__name__)
def __init__(self, input_data): self.file = input_data.file self.skip_infrastructure = input_data.no_infra if hasattr( input_data, 'no_infra') else False self.logger = Log(__name__) self.cluster_model = None self.input_docs = [] self.configuration_docs = [] self.infrastructure_docs = []
class SpecCommand: def __init__(self): self.logger = Log(__name__) def __init__(self): self.logger = Log(__name__) def check_dependencies(self): required_gems = ['serverspec', 'rake', 'rspec_junit_formatter'] error_str = f'''Missing Ruby or one of the following Ruby gems: {', '.join(required_gems)} These need to be installed to run the cluster spec tests from epicli''' if shutil.which('ruby') == None or shutil.which('gem') == None: raise Exception(error_str) p = subprocess.Popen(['gem', 'query', '--local'], stdout=PIPE) out, err = p.communicate() if all(n in out.decode('utf-8') for n in required_gems) == False: raise Exception(error_str) def run(self, spec_output, inventory, user, key, group): self.check_dependencies() env = os.environ.copy() env['spec_output'] = spec_output env['inventory'] = inventory env['user'] = user env['keypath'] = key cmd = f'rake inventory={inventory} user={user} keypath={key} spec_output={spec_output} spec:{group}' self.logger.info(f'Running: "{cmd}"') logpipe = LogPipe(__name__) with Popen(cmd.split(' '), cwd=SPEC_TEST_PATH, env=env, stdout=logpipe, stderr=logpipe) as sp: logpipe.close() if sp.returncode != 0: raise Exception(f'Error running: "{cmd}"') else: self.logger.info(f'Done running: "{cmd}"') @staticmethod def get_spec_groups(): groups = os.listdir(SPEC_TEST_PATH + '/spec') groups.remove('spec_helper.rb') groups = ['all'] + groups sorted(groups, key=str.lower) return groups
class TerraformCommand: def __init__(self, working_directory=os.path.dirname(__file__)): self.logger = Log(__name__) self.APPLY_COMMAND = "apply" self.DESTROY_COMMAND = "destroy" self.PLAN_COMMAND = "plan" self.INIT_COMMAND = "init" self.working_directory = working_directory def apply(self, auto_approve=False, env=os.environ.copy()): self.run(self, self.APPLY_COMMAND, auto_approve=auto_approve, env=env) def destroy(self, auto_approve=False, env=os.environ.copy()): self.run(self, self.DESTROY_COMMAND, auto_approve=auto_approve, env=env) def plan(self, env=os.environ.copy()): self.run(self, self.PLAN_COMMAND, env=env) def init(self, env=os.environ.copy()): self.run(self, self.INIT_COMMAND, env=env) @staticmethod def run(self, command, env, auto_approve=False): cmd = ['terraform', command] if auto_approve: cmd.append('--auto-approve') if command == self.APPLY_COMMAND or command == self.DESTROY_COMMAND: cmd.append(f'-state={self.working_directory}/terraform.tfstate') cmd.append(self.working_directory) cmd = ' '.join(cmd) self.logger.info(f'Running: "{cmd}"') if Config().debug: env['TF_LOG'] = 'TRACE' logpipe = LogPipe(__name__) with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe, env=env, shell=True) as sp: logpipe.close() if sp.returncode != 0: raise Exception(f'Error running: "{cmd}"') else: self.logger.info(f'Done running "{cmd}"')
class Step(metaclass=ABCMeta): def __init__(self, step_name): self.logger = Log(step_name) def __enter__(self): self.start = time.time() self.logger.info('Starting run') return self def __exit__(self, exc_type, exc_value, traceback): passed = int(round((time.time() - self.start) * 1000)) self.logger.info('Run done in ' + str(passed) + 'ms')
class TerraformCommand: def __init__(self, working_directory=os.path.dirname(__file__)): self.logger = Log(__name__) self.APPLY_COMMAND = "apply" self.DESTROY_COMMAND = "destroy" self.PLAN_COMMAND = "plan" self.INIT_COMMAND = "init" self.working_directory = working_directory def apply(self, auto_approve=False): self.run(self, self.APPLY_COMMAND, auto_approve=auto_approve) def destroy(self, auto_approve=False): self.run(self, self.DESTROY_COMMAND, auto_approve=auto_approve) def plan(self): self.run(self, self.PLAN_COMMAND) def init(self): self.run(self, self.INIT_COMMAND) @staticmethod def run(self, command, auto_approve=False): cmd = ['terraform', command] if auto_approve: cmd.append('--auto-approve') if command == self.APPLY_COMMAND: cmd.append('-state=' + self.working_directory + '/terraform.tfstate') cmd.append(self.working_directory) self.logger.info('Running: "' + ' '.join(cmd) + '"') logpipe = LogPipe(__name__) with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe) as sp: logpipe.close() if sp.returncode != 0: raise Exception('Error running: "' + ' '.join(cmd) + '"') else: self.logger.info('Done running "' + ' '.join(cmd) + '"')
def __init__(self, input_data): self.file = input_data.file self.skip_infrastructure = getattr(input_data, 'no_infra', False) self.skip_config = getattr(input_data, 'skip_config', False) self.ansible_options = { 'profile_tasks': getattr(input_data, 'profile_ansible_tasks', False) } self.logger = Log(__name__) self.cluster_model = None self.input_docs = [] self.configuration_docs = [] self.infrastructure_docs = []
def __init__(self): self.logger = Log(__name__)
def __init__(self, working_directory=os.path.dirname(__file__)): self.logger = Log(__name__) self.working_directory = working_directory
def dump_config(config): logger = Log('config') for attr in config.__dict__: if attr.startswith('_'): logger.info('%s = %r' % (attr[1:], getattr(config, attr)))
def main(): config = Config() parser = argparse.ArgumentParser( description=__doc__, usage='''epicli <command> [<args>]''', formatter_class=argparse.RawDescriptionHelpFormatter) # setup some root arguments parser.add_argument('--version', action='version', help='Shows the CLI version', version=VERSION) parser.add_argument( '--licenses', action='version', help= 'Shows the third party packages and their licenses the CLI is using.', version=json.dumps(LICENSES, indent=4)) parser.add_argument( '-l', '--log-file', dest='log_name', type=str, help='The name of the log file written to the output directory') parser.add_argument('--log-format', dest='log_format', type=str, help='Format for the logging string.') parser.add_argument('--log-date-format', dest='log_date_format', type=str, help='Format for the logging date.') parser.add_argument( '--log-count', dest='log_count', type=str, help='Roleover count where each CLI run will generate a new log.') parser.add_argument('--log-type', choices=['plain', 'json'], default='plain', dest='log_type', action='store', help='Type of logs.') parser.add_argument( '--validate-certs', choices=['true', 'false'], default='true', action='store', dest='validate_certs', help= '''[Experimental]: Disables certificate checks for certain Ansible operations which might have issues behind proxies (https://github.com/ansible/ansible/issues/32750). Should NOT be used in production for security reasons.''' ) parser.add_argument( '--debug', dest='debug', action="store_true", help= 'Set this to output extensive debug information. Carries over to Ansible and Terraform.' ) parser.add_argument( '--auto-approve', dest='auto_approve', action="store_true", help='Auto approve any user input queries asked by Epicli') # some arguments we don't want available when running from the docker image. if not config.docker_cli: parser.add_argument( '-o', '--output', dest='output_dir', type=str, help='Directory where the CLI should write it`s output.') # setup subparsers subparsers = parser.add_subparsers() apply_parser(subparsers) validate_parser(subparsers) init_parser(subparsers) upgrade_parser(subparsers) backup_parser(subparsers) recovery_parser(subparsers) delete_parser(subparsers) prepare_parser(subparsers) # check if there were any variables and display full help if len(sys.argv) < 2: parser.print_help() sys.exit(1) arguments = sys.argv[1:] # add some arguments to the general config so we can easily use them throughout the CLI args = parser.parse_args(arguments) config.output_dir = args.output_dir if hasattr(args, 'output_dir') else None config.log_file = args.log_name config.log_format = args.log_format config.log_date_format = args.log_date_format config.log_type = args.log_type config.log_count = args.log_count config.validate_certs = True if args.validate_certs == 'true' else False if 'offline_requirements' in args and not args.offline_requirements is None: config.offline_requirements = args.offline_requirements if 'wait_for_pods' in args and not args.wait_for_pods is None: config.wait_for_pods = args.wait_for_pods config.debug = args.debug config.auto_approve = args.auto_approve try: return args.func(args) except Exception as e: logger = Log('epicli') logger.error(e, exc_info=config.debug) return 1
class AnsibleCommand: def __init__(self, working_directory=os.path.dirname(__file__)): self.logger = Log(__name__) self.working_directory = working_directory def __init__(self): self.logger = Log(__name__) def run_task(self, hosts, inventory, module, args): cmd = ['ansible'] cmd.extend(["-m", module]) if args is not None and len(args) > 0: cmd.extend(["-a", args]) if inventory is not None and len(inventory) > 0: cmd.extend(["-i", inventory]) cmd.append(hosts) self.logger.info('Running: "' + ' '.join(cmd) + '"') logpipe = LogPipe(__name__) with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe) as sp: logpipe.close() if sp.returncode != 0: raise Exception('Error running: "' + ' '.join(cmd) + '"') else: self.logger.info('Done running "' + ' '.join(cmd) + '"') def run_task_with_retries(self, inventory, module, args, hosts, retries, timeout=10): for i in range(retries): try: self.run_task(hosts=hosts, inventory=inventory, module=module, args=args) break except Exception as e: self.logger.error(e) self.logger.info("Retry running task: " + str(i + 1) + "/" + str(retries)) time.sleep(timeout) def run_playbook(self, inventory, playbook_path): cmd = ['ansible-playbook'] if inventory is not None and len(inventory) > 0: cmd.extend(["-i", inventory]) cmd.append(playbook_path) self.logger.info('Running: "' + ' '.join(cmd) + '"') logpipe = LogPipe(__name__) with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe) as sp: logpipe.close() if sp.returncode != 0: raise Exception('Error running: "' + ' '.join(cmd) + '"') else: self.logger.info('Done running "' + ' '.join(cmd) + '"') def run_playbook_with_retries(self, inventory, playbook_path, retries, timeout=10): for i in range(retries): try: self.run_playbook(inventory=inventory, playbook_path=playbook_path) return 0 except Exception as e: self.logger.error(e) self.logger.info("Retry running playbook: " + str(i + 1) + "/" + str(retries)) time.sleep(timeout) return 1
def main(): config = Config() parser = argparse.ArgumentParser( description=__doc__, usage='''epicli <command> [<args>]''', formatter_class=argparse.RawTextHelpFormatter) # setup some root arguments parser.add_argument('--version', action='version', help='Shows the CLI version', version=VERSION) parser.add_argument( '--licenses', action='version', help= 'Shows the third party packages and their licenses the CLI is using.', version=json.dumps(LICENSES, indent=4)) parser.add_argument( '-l', '--log-file', dest='log_name', type=str, help='The name of the log file written to the output directory') parser.add_argument('--log-format', dest='log_format', type=str, help='Format for the logging string.') parser.add_argument('--log-date-format', dest='log_date_format', type=str, help='Format for the logging date.') parser.add_argument( '--log-count', dest='log_count', type=str, help='Roleover count where each CLI run will generate a new log.') parser.add_argument('--log-type', choices=['plain', 'json'], default='plain', dest='log_type', action='store', help='Type of logs.') parser.add_argument( '--validate-certs', choices=['true', 'false'], default='true', action='store', dest='validate_certs', help= '''[Experimental]: Disables certificate checks for certain Ansible operations which might have issues behind proxies (https://github.com/ansible/ansible/issues/32750). Should NOT be used in production for security reasons.''') parser.add_argument( '--auto-approve', dest='auto_approve', action="store_true", help='Auto approve any user input queries asked by Epicli') # set debug verbosity level. def debug_level(x): x = int(x) if x < 0 or x > 4: raise argparse.ArgumentTypeError( "--debug value should be between 0 and 4") return x parser.add_argument( '--debug', dest='debug', type=debug_level, help='''Set this flag (0..4) to enable debug output where 0 is no debug output and 1..4 is debug output with different verbosity levels: Python : Anything heigher then 0 enables printing of Python stacktraces Ansible : 1..4 map to following Ansible verbosity levels: 1: -v 2: -vv 3: -vvv 4: -vvvv Terraform : 1..4 map to the following Terraform verbosity levels: 1: WARN 2: INFO 3: DEBUG 4: TRACE''') # some arguments we don't want available when running from the docker image. if not config.docker_cli: parser.add_argument( '-o', '--output', dest='output_dir', type=str, help='Directory where the CLI should write it`s output.') # setup subparsers subparsers = parser.add_subparsers() prepare_parser(subparsers) init_parser(subparsers) apply_parser(subparsers) upgrade_parser(subparsers) delete_parser(subparsers) test_parser(subparsers) ''' validate_parser(subparsers) ''' backup_parser(subparsers) recovery_parser(subparsers) # check if there were any variables and display full help if len(sys.argv) < 2: parser.print_help() sys.exit(1) arguments = sys.argv[1:] # add some arguments to the general config so we can easily use them throughout the CLI args = parser.parse_args(arguments) config.output_dir = getattr(args, 'output_dir', None) config.log_file = args.log_name config.log_format = args.log_format config.log_date_format = args.log_date_format config.log_type = args.log_type config.log_count = args.log_count config.validate_certs = True if args.validate_certs == 'true' else False if 'offline_requirements' in args and not args.offline_requirements is None: config.offline_requirements = args.offline_requirements if 'wait_for_pods' in args and not args.wait_for_pods is None: config.wait_for_pods = args.wait_for_pods config.debug = args.debug config.auto_approve = args.auto_approve try: return args.func(args) except Exception as e: logger = Log('epicli') logger.error(e, exc_info=(config.debug > 0)) dump_debug_info() return 1
class APIProxy: def __init__(self, cluster_model, config_docs): self.cluster_model = cluster_model self.cluster_name = self.cluster_model.specification.name.lower() self.cluster_prefix = self.cluster_model.specification.prefix.lower() self.resource_group_name = resource_name(self.cluster_prefix, self.cluster_name, 'rg') self.config_docs = config_docs self.logger = Log(__name__) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): pass def login_account(self): subscription_name = self.cluster_model.specification.cloud.subscription_name all_subscription = self.run(self, 'az login') subscription = select_first(all_subscription, lambda x: x['name'] == subscription_name) if subscription is None: raise Exception( f'User does not have access to subscription: "{subscription_name}"' ) return subscription def login_sp(self, sp_data): name = sp_data['name'] password = sp_data['password'] tenant = sp_data['tenant'] return self.run( self, f'az login --service-principal -u \'{name}\' -p "{password}" --tenant \'{tenant}\'', False) def set_active_subscribtion(self, subscription_id): self.run(self, f'az account set --subscription {subscription_id}') def get_active_subscribtion(self): subscription = self.run(self, f'az account show') return subscription def create_sp(self, app_name, subscription_id): #TODO: make role configurable? sp = self.run( self, f'az ad sp create-for-rbac -n \'{app_name}\' --role=\'Contributor\' --scopes=\'/subscriptions/{subscription_id}\'' ) # Sleep for a while. Sometimes the call returns before the rights of the SP are finished creating. self.wait(self, 60) return sp def get_ips_for_feature(self, component_key): look_for_public_ip = self.cluster_model.specification.cloud.use_public_ips cluster = cluster_tag(self.cluster_prefix, self.cluster_name) running_instances = self.run( self, f'az vm list-ip-addresses --ids $(az resource list --query "[?type==\'Microsoft.Compute/virtualMachines\' && tags.{component_key} == \'\' && tags.cluster == \'{cluster}\'].id" --output tsv)' ) result = [] for instance in running_instances: if isinstance(instance, list): instance = instance[0] name = instance['virtualMachine']['name'] if look_for_public_ip: ip = instance['virtualMachine']['network'][ 'publicIpAddresses'][0]['ipAddress'] else: ip = instance['virtualMachine']['network'][ 'privateIpAddresses'][0] result.append(AnsibleHostModel(name, ip)) return result def get_storage_account_primary_key(self, storage_account_name): keys = self.run( self, f'az storage account keys list -g \'{self.resource_group_name}\' -n \'{storage_account_name}\'' ) return keys[0]['value'] @staticmethod def wait(self, seconds): for x in range(0, seconds): self.logger.info(f'Waiting {seconds} seconds...{x}') time.sleep(1) @staticmethod def run(self, cmd, log_cmd=True): if log_cmd: self.logger.info('Running: "' + cmd + '"') logpipe = LogPipe(__name__) with Popen(cmd, stdout=PIPE, stderr=logpipe, shell=True) as sp: logpipe.close() try: data = sp.stdout.read().decode('utf-8') data = re.sub(r'\s+', '', data) data = re.sub(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]', '', data) output = json.loads(data) except: output = {} if sp.returncode != 0: if log_cmd: raise Exception(f'Error running: "{cmd}"') else: raise Exception(f'Error running Azure APIProxy cmd') else: if log_cmd: self.logger.info(f'Done running "{cmd}"') return output
def dump_debug_info(): def dump_external_debug_info(title, args): dump_file.write(f'\n\n*****{title}******\n') p = subprocess.Popen(args, stdout=subprocess.PIPE) out, err = p.communicate() lines = filter(lambda x: x.strip(), out.decode("utf-8").splitlines(keepends=True)) dump_file.writelines(lines) try: logger = Log('dump_debug_info') config = Config() timestr = time.strftime("%Y%m%d-%H%M%S") dump_path = os.getcwd() + f'/epicli_error_{timestr}.dump' dump_file = open(dump_path, 'w') dump_file.write('*****EPICLI VERSION******\n') dump_file.write(f'{VERSION}') dump_file.write('\n\n*****EPICLI ARGS******\n') dump_file.write(' '.join([*['epicli'], *sys.argv[1:]])) dump_file.write('\n\n*****EPICLI CONFIG******\n') for attr in config.__dict__: if attr.startswith('_'): dump_file.write('%s = %r\n' % (attr[1:], getattr(config, attr))) dump_file.write('\n\n*****SYSTEM******\n') system_data = { 'platform': platform.system(), 'release': platform.release(), 'type': platform.uname().system, 'arch': platform.uname().machine, 'cpus': json.dumps(os.cpu_count()), 'hostname': socket.gethostname() } dump_file.write(json.dumps(dict(system_data), indent=2)) dump_file.write('\n\n*****ENVIROMENT VARS******\n') dump_file.write(json.dumps(dict(os.environ), indent=2)) dump_file.write('\n\n*****PYTHON******\n') dump_file.write(f'python_version: {platform.python_version()}\n') dump_file.write(f'python_build: {platform.python_build()}\n') dump_file.write(f'python_revision: {platform.python_revision()}\n') dump_file.write(f'python_compiler: {platform.python_compiler()}\n') dump_file.write(f'python_branch: {platform.python_branch()}\n') dump_file.write( f'python_implementation: {platform.python_implementation()}\n') dump_external_debug_info('ANSIBLE VERSION', ['ansible', '--version']) dump_external_debug_info('ANSIBLE CONFIG', ['ansible-config', 'dump']) dump_external_debug_info('ANSIBLE-VAULT VERSION', ['ansible-vault', '--version']) dump_external_debug_info('TERRAFORM VERSION', ['terraform', '--version']) dump_external_debug_info('SKOPEO VERSION', ['skopeo', '--version']) dump_external_debug_info('RUBY VERSION', ['ruby', '--version']) dump_external_debug_info('RUBY GEM VERSION', ['gem', '--version']) dump_external_debug_info('RUBY INSTALLED GEMS', ['gem', 'query', '--local']) dump_file.write('\n\n*****LOG******\n') log_path = os.path.join(get_output_path(), config.log_file) dump_file.writelines([l for l in open(log_path).readlines()]) finally: dump_file.close() logger.info(f'Error dump has been written to: {dump_path}') logger.warning( 'This dump might contain sensitive information. Check before sharing.' )
class EpiphanyEngine: def __init__(self, input_data): self.file = input_data.file self.skip_infrastructure = input_data.no_infra if hasattr( input_data, 'no_infra') else False self.logger = Log(__name__) self.cluster_model = None self.input_docs = [] self.configuration_docs = [] self.infrastructure_docs = [] def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): pass def process_input_docs(self): # Load the user input YAML docs from the input file. if os.path.isabs(self.file): path_to_load = self.file else: path_to_load = os.path.join(os.getcwd(), self.file) user_file_stream = open(path_to_load, 'r') self.input_docs = safe_load_all(user_file_stream) # Merge the input docs with defaults with DefaultMerger(self.input_docs) as doc_merger: self.input_docs = doc_merger.run() # Get the cluster model. self.cluster_model = select_single( self.input_docs, lambda x: x.kind == 'epiphany-cluster') if self.cluster_model is None: raise Exception('No cluster model defined in input YAML file') # Validate input documents with SchemaValidator(self.cluster_model, self.input_docs) as schema_validator: schema_validator.run() def process_infrastructure_docs(self): # Build the infrastructure docs with provider_class_loader( self.cluster_model.provider, 'InfrastructureBuilder')( self.input_docs) as infrastructure_builder: self.infrastructure_docs = infrastructure_builder.run() # Validate infrastructure documents with SchemaValidator(self.cluster_model, self.infrastructure_docs) as schema_validator: schema_validator.run() def process_configuration_docs(self): # Append with components and configuration docs with ConfigurationAppender(self.input_docs) as config_appender: self.configuration_docs = config_appender.run() # Validate configuration documents with SchemaValidator(self.cluster_model, self.configuration_docs) as schema_validator: schema_validator.run() def collect_infrastructure_config(self): with provider_class_loader(self.cluster_model.provider, 'InfrastructureConfigCollector')([ *self.input_docs, *self.configuration_docs, *self.infrastructure_docs ]) as config_collector: config_collector.run() def verify(self): try: self.process_input_docs() self.process_configuration_docs() self.process_infrastructure_docs() save_manifest([ *self.input_docs, *self.configuration_docs, *self.infrastructure_docs ], self.cluster_model.specification.name) return 0 except Exception as e: self.logger.error( e, exc_info=True ) #TODO extensive debug output might not always be wanted. Make this configurable with input flag? return 1 def apply(self): try: self.process_input_docs() self.process_infrastructure_docs() if not self.skip_infrastructure: # Generate terraform templates with TerraformTemplateGenerator( self.cluster_model, self.infrastructure_docs) as template_generator: template_generator.run() # Run Terraform to create infrastructure with TerraformRunner( self.cluster_model.specification.name) as tf_runner: tf_runner.run() self.process_configuration_docs() self.collect_infrastructure_config() # Run Ansible to provision infrastructure docs = [ *self.input_docs, *self.configuration_docs, *self.infrastructure_docs ] with AnsibleRunner(self.cluster_model, docs) as ansible_runner: ansible_runner.run() # Save docs to manifest file save_manifest(docs, self.cluster_model.specification.name) return 0 except Exception as e: self.logger.error( e, exc_info=True ) # TODO extensive debug output might not always be wanted. Make this configurable with input flag? return 1 def dry_run(self): self.process_input_docs() self.process_configuration_docs() return [*self.input_docs, *self.configuration_docs]
class AnsibleCommand: def __init__(self, working_directory=os.path.dirname(__file__)): self.logger = Log(__name__) self.working_directory = working_directory def __init__(self): self.logger = Log(__name__) def run_task(self, hosts, inventory, module, args=None): cmd = ['ansible'] cmd.extend(["-m", module]) if args is not None and len(args) > 0: cmd.extend(["-a", args]) if inventory is not None and len(inventory) > 0: cmd.extend(["-i", inventory]) cmd.append(hosts) if Config().debug > 0: cmd.append(ansible_verbosity[Config().debug]) self.logger.info('Running: "' + ' '.join(module) + '"') logpipe = LogPipe(__name__) with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe) as sp: logpipe.close() if sp.returncode != 0: raise Exception('Error running: "' + ' '.join(cmd) + '"') else: self.logger.info('Done running "' + ' '.join(cmd) + '"') def run_task_with_retries(self, inventory, module, hosts, retries, timeout=10, args=None): for i in range(retries): try: self.run_task(hosts=hosts, inventory=inventory, module=module, args=args) break except Exception as e: self.logger.error(e) self.logger.info('Retry running task: ' + str(i + 1) + '/' + str(retries)) time.sleep(timeout) else: raise Exception( f'Failed running task after {str(retries)} retries') def run_playbook(self, inventory, playbook_path, vault_file=None): cmd = ['ansible-playbook'] if inventory is not None and len(inventory) > 0: cmd.extend(["-i", inventory]) if vault_file is not None: cmd.extend(["--vault-password-file", vault_file]) cmd.append(playbook_path) if Config().debug > 0: cmd.append(ansible_verbosity[Config().debug]) self.logger.info('Running: "' + ' '.join(playbook_path) + '"') logpipe = LogPipe(__name__) with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe) as sp: logpipe.close() if sp.returncode != 0: raise Exception('Error running: "' + ' '.join(cmd) + '"') else: self.logger.info('Done running "' + ' '.join(cmd) + '"') def run_playbook_with_retries(self, inventory, playbook_path, retries, timeout=10): for i in range(retries): try: self.run_playbook(inventory=inventory, playbook_path=playbook_path) break except Exception as e: self.logger.error(e) self.logger.info('Retry running playbook: ' + str(i + 1) + '/' + str(retries)) time.sleep(timeout) else: raise Exception( f'Failed running playbook after {str(retries)} retries')
class TerraformCommand: def __init__(self, working_directory=os.path.dirname(__file__)): self.logger = Log(__name__) self.APPLY_COMMAND = "apply" self.DESTROY_COMMAND = "destroy" self.PLAN_COMMAND = "plan" self.INIT_COMMAND = "init" self.working_directory = working_directory def apply(self, auto_approve=False, env=os.environ.copy()): self.run(self, self.APPLY_COMMAND, auto_approve=auto_approve, env=env, auto_retries=3) def destroy(self, auto_approve=False, env=os.environ.copy()): self.run(self, self.DESTROY_COMMAND, auto_approve=auto_approve, env=env) def plan(self, env=os.environ.copy()): self.run(self, self.PLAN_COMMAND, env=env) def init(self, env=os.environ.copy()): self.run(self, self.INIT_COMMAND, env=env) @staticmethod def run(self, command, env, auto_approve=False, auto_retries=1): cmd = ['terraform', command] if auto_approve: cmd.append('--auto-approve') if command == self.APPLY_COMMAND or command == self.DESTROY_COMMAND: cmd.append(f'-state={self.working_directory}/terraform.tfstate') cmd.append('-no-color') cmd.append(self.working_directory) cmd = ' '.join(cmd) self.logger.info(f'Running: "{cmd}"') if Config().debug > 0: env['TF_LOG'] = terraform_verbosity[Config().debug] retries = 1 do_retry = True while ((retries <= auto_retries) and do_retry): logpipe = LogPipe(__name__) with subprocess.Popen(cmd, stdout=logpipe, stderr=logpipe, env=env, shell=True) as sp: logpipe.close() retries = retries + 1 do_retry = next( (True for s in logpipe.stderrstrings if 'RetryableError' in s), False) if do_retry and retries <= auto_retries: self.logger.warning( f'Terraform failed with "RetryableError" error. Retry: ' + str(retries) + '/' + str(auto_retries)) if sp.returncode != 0: raise Exception(f'Error running: "{cmd}"') else: self.logger.info(f'Done running "{cmd}"')
def __init__(self, step_name): self.logger = Log(step_name)
def __init__(self, cluster_model, config_docs): self.cluster_model = cluster_model self.config_docs = config_docs self.logger = Log(__name__)