def run(self, context: ExecutionContext) -> bool: service_name = context.get_arg('name') strategy = str(context.get_arg('--strategy')) remove_previous_images = bool(context.get_arg('--remove-previous-images')) service = self.services(context).get_by_name(service_name) strategies = { 'rolling': lambda: self.deploy_rolling(service, context), 'compose': lambda: self.deploy_compose_like(service, context), 'recreate': lambda: self.deploy_recreate(service, context) } with self.hooks_executed(context, 'service-start-%s' % service_name): with self._old_images_clean_up(context, service, clear_images=remove_previous_images): if strategy in strategies: return strategies[strategy]() else: if strategy == 'auto': strategy = service.get_update_strategy(default='compose') if strategy in strategies: return strategies[strategy]() self.io().error_msg('Invalid strategy selected: %s' % strategy) return False
def run(self, context: ExecutionContext) -> bool: """Validate parameters and select action""" is_global = context.get_arg('--global') domain = context.get_arg('--domain') service = context.get_arg('--service') directory = self.get_data_path(context) + '/maintenance-mode' if not self._validate_switches(is_global, domain, service): self.io().error_msg( 'Cannot use together --global, --domain and --service switch. Pick one of them.' ) return False try: if is_global: return self.act([directory + '/on'], 'globally') elif service: return self.act_for_service(directory, service, context) elif domain: return self.act_for_domain(directory, domain, context) else: self.io().error_msg('Must specify --global or --domain switch') return False except PermissionError as e: self.io().error_msg( 'No write permissions. Set permissions or use sudo? %s' % str(e)) return False
def run(self, context: ExecutionContext) -> bool: services = self.get_matching_services(context) strategy = context.get_arg('--strategy') result = True with self.hooks_executed(context, 'start'): for service in services: self.io().h2( 'Starting "%s" (%i instances)...' % (service.get_name(), service.get_desired_replicas_count())) try: self.rkd( [ '--no-ui', ':harbor:service:up', service.get_name(), '--remove-previous-images' if context.get_arg('--remove-previous-images') else '', ('--strategy=%s' % strategy) if strategy else '' ], capture=not self.io().is_log_level_at_least('info')) self.io().success_msg('Service "%s" was started' % service.get_name()) except CalledProcessError as e: self.io().err(str(e)) self.io().error_msg('Cannot start service "%s"' % service.get_name()) result = False self.io().print_opt_line() return result
def build_image(ctx: ExecutionContext, this: TaskInterface) -> bool: """ Builds a docker image for given architecture, tags properly and push """ tag = ctx.get_arg('--git-tag') should_push = ctx.get_arg('--push') arch = ctx.get_arg('--arch') docker_tag = 'latest-dev-' + arch if tag: docker_tag = arch + '-' + tag image_name = '{repo}:{tag}'.format(tag=docker_tag, repo=REPO) this.sh( 'docker build . -f ./.infrastructure/{docker_arch}.Dockerfile -t {image}' .format(docker_arch=arch, image=image_name)) this.sh('docker tag {image} {image}-$(date "+%Y-%m-%d")'.format( image=image_name)) if should_push: this.sh('docker push {image}'.format(image=image_name)) this.sh( 'docker push {image}-$(date "+%Y-%m-%d")'.format(image=image_name)) return True
def run(self, context: ExecutionContext) -> bool: vault_opts = self._get_vault_opts(context) filename = context.get_arg('filename') mode = 'decrypt' if context.get_arg('--decrypt') else 'encrypt' self.sh('ansible-vault %s %s %s' % (mode, vault_opts, filename), capture=False) return True
def deploy_compose_like(self, service: ServiceDeclaration, ctx: ExecutionContext) -> bool: """Regular docker-compose up deployment (with downtime)""" self.io().info('Performing "compose" deployment for "%s"' % service.get_name()) self.containers(ctx).up(service, norecreate=bool(ctx.get_arg('--dont-recreate')), extra_args=ctx.get_arg('--extra-args')) return True
def run(self, ctx: ExecutionContext) -> bool: service_name = ctx.get_arg('name') timeout = int(ctx.get_arg('--timeout')) instance_num = int(ctx.get_arg('--instance')) if ctx.get_arg('--instance') else None service = self.services(ctx).get_by_name(service_name) try: container_name = self.containers(ctx).find_container_name(service, instance_num) container = self.containers(ctx).inspect_container(container_name) except ServiceNotCreatedException as e: self.io().error_msg(str(e)) return False started_at = time() self.io().info('Checking health of "%s" service - %s' % (service_name, container_name)) if container.has_health_check(): while True: if time() - started_at >= timeout: self.io().error_msg('Timeout of %is reached.' % timeout) return False health = container.get_health_status() # do not wait for result, do check manually in mean time if health == 'starting': self.io().warn('Docker reports "starting" - performing a manual check, we wont wait for docker') try: command = container.get_health_check_command().replace('"', '\\"') self.containers(ctx).exec_in_container( service_name=service_name, command='/bin/sh -c "%s"' % command, instance_num=instance_num ) self.io().success_msg('Service healthy after %is' % (time() - started_at)) return True except subprocess.CalledProcessError as e: self.io().debug(str(e.output)[0:128]) sleep(1) continue if health == 'healthy' or health == 'running': self.io().success_msg('Service healthy after %i' % (time() - started_at)) return True sleep(1) else: self.io().warn('Instance has no healthcheck defined!') return True
def prepare_tuple_for_single_container(self, ctx: ExecutionContext) -> tuple: service_name = ctx.get_arg('--name') service = self.services(ctx).get_by_name(service_name) instance_num = int(ctx.get_arg('--instance-num')) if ctx.get_arg('--instance-num') else None container_name = self.containers(ctx).find_container_name(service, instance_num) if not container_name: self.io().error_msg('Container not found') return None, None, None return container_name, service, instance_num
def run(self, ctx: ExecutionContext) -> bool: params = [] if ctx.get_arg('--quiet'): params.append('--quiet') if ctx.get_arg('--all'): params.append('--all') self.containers(ctx).ps(params) return True
def _preserve_vault_parameters_for_usage_in_inner_tasks( self, ctx: ExecutionContext): """Preserve original parameters related to Vault, so those parameters can be propagated to inner tasks""" try: vault_passwords = ctx.get_arg_or_env('--vault-passwords') except MissingInputException: vault_passwords = '' # keep the vault arguments for decryption of deployment.yml self.vault_args = ['--vault-passwords=' + vault_passwords] if ctx.get_arg('--ask-vault-pass'): self.vault_args.append('--ask-vault-pass')
def execute(self, context: ExecutionContext) -> bool: opts = '' if context.args['skip_existing']: opts += ' --skip-existing ' if context.get_arg('--url'): if context.get_arg('--test'): raise Exception('Cannot use --url and --test switch at once') opts += ' --repository-url=%s' % context.get_arg('--url') if context.get_arg('--test'): opts += ' --repository-url https://test.pypi.org/legacy/ ' self.sh(''' %s upload \\ --disable-progress-bar \\ --verbose \\ --username=%s \\ --password=%s \\ %s %s ''' % (context.get_env('TWINE_PATH'), context.get_arg('--username'), context.get_arg('--password'), opts, context.get_arg('--src'))) return True
def run(self, ctx: ExecutionContext) -> bool: service_name = ctx.get_arg('name') service = self.services(ctx).get_by_name(service_name) with_image = ctx.get_arg('--with-image') images = self.get_all_images_for_service(ctx, service) if with_image else [] self.containers(ctx).rm(service, extra_args=ctx.get_arg('--extra-args')) for image in images: try: self.containers(ctx).rm_image(image, capture=True) except: pass return True
def execute(self, ctx: ExecutionContext) -> bool: name = ctx.get_arg('name') category_name, pkg_name = self.extract_category_and_pkg_names(name) path = self.find_snippet_path(pkg_name, category_name) if not path: self.io().error( 'Snippet not found in any synchronized repository. ' + 'Did you forget to do :cooperative:sync?') return False self.io().info('Installing snippet from "%s"' % path) # mock rkd_path, so the snippet can override the tasks rkd_path = os.getenv('RKD_PATH', '') snippet_rkd_path = os.path.realpath('./' + path + '/.rkd') if snippet_rkd_path: os.putenv('RKD_PATH', (rkd_path + ':' + snippet_rkd_path).strip(':')) try: subprocess.check_call(['rkd', ':snippet:wizard', path]) subprocess.check_call(['rkd', ':snippet:install', path]) finally: if os.path.isfile('.rkd/tmp-wizard.json'): os.unlink('.rkd/tmp-wizard.json') os.putenv('RKD_PATH', rkd_path) return True
def _ask_and_set_var(self, ctx: ExecutionContext, arg_name: str, title: str, attribute: str, secret: bool): """Ask user an interactive question, then add answer to the deployment.yml loaded in memory The variable will be appended to any node, where the variable is empty. Example: We have 5 servers, 3 without a password. So the password will be applied to 3 servers. """ self.get_config() if not ctx.get_arg(arg_name): return wizard = Wizard(self).ask(title, attribute=attribute, secret=secret) for group_name, nodes in self._config['nodes'].items(): node_num = 0 for node in nodes: node_num += 1 if attribute in self._config['nodes'][group_name][node_num - 1]: continue self._config['nodes'][group_name][ node_num - 1][attribute] = wizard.answers[attribute]
def test_functional_hooks_are_executed_when_exists_and_files_with_extension_only_are_skipped( self): """Given we have an example hooks in pre-upgrade/whoami.sh and in post-upgrade/history.sh And we try to run those hooks using hooks_executed() Then we will see output produced by those scripts And .dotfiles will be ignored """ self._prepare_test_data() buffer = StringIO() hooks_capturing_io = IO() task = TestTask() task._io = BufferedSystemIO() ctx = ExecutionContext(TaskDeclaration(task), args={}, env={}) with hooks_capturing_io.capture_descriptors(stream=buffer, enable_standard_out=True): with task.hooks_executed(ctx, 'upgrade'): pass self.assertIn('>> This is a whoami.sh hook, test:', buffer.getvalue(), msg='Expected pre-upgrade hook to be ran') self.assertIn('25 June 1978 the rainbow flag was first flown', buffer.getvalue(), msg='Expected post-upgrade hook to be ran') self.assertIn('pre-upgrade/whoami.sh', task._io.get_value()) self.assertNotIn('.gitkeep', task._io.get_value())
def test_one_failed_step_is_preventing_next_steps_from_execution_and_result_is_marked_as_failure(self): """Check the correctness of error handling""" io = IO() str_io = StringIO() buffered = BufferedSystemIO() task_declaration = get_test_declaration() BasicTestingCase.satisfy_task_dependencies(task_declaration.get_task_to_execute(), io=buffered) ctx = ExecutionContext(task_declaration) executor = DeclarativeExecutor() executor.add_step('python', 'this.io().outln("Peter Kropotkin"); return True', task_name=':first', rkd_path='', envs={}) executor.add_step('bash', 'echo "Buenaventura Durruti"; exit 1', task_name=':second', rkd_path='', envs={}) executor.add_step('python', 'this.io().outln("This one will not show"); return True', task_name=':third', rkd_path='', envs={}) with io.capture_descriptors(target_files=[], stream=str_io, enable_standard_out=False): final_result = executor.execute_steps_one_by_one(ctx, task_declaration.get_task_to_execute()) output = str_io.getvalue() + buffered.get_value() self.assertIn('Peter Kropotkin', output) self.assertIn('Buenaventura Durruti', output) self.assertNotIn('This one will not show', output) self.assertEqual(False, final_result)
def mock_execution_context( task: TaskInterface, args: Dict[str, str] = None, env: Dict[str, str] = None, defined_args: Dict[str, dict] = None) -> ExecutionContext: """ Prepares a simplified rkd.api.contract.ExecutionContext instance :param task: :param args: :param env: :param defined_args: :return: """ if args is None: args = {} if env is None: env = {} if defined_args is None: defined_args = {} if args and not defined_args: for name, passed_value in args.items(): defined_args[name] = {'default': ''} return ExecutionContext(TaskDeclaration(task), parent=None, args=args, env=env, defined_args=defined_args)
def run(self, context: ExecutionContext) -> bool: path = self.get_app_repository_path(context.get_arg('name'), context) if not os.path.isfile(path): self.io().error_msg('Cannot pull a repository: Unknown application, "%s" file not found' % path) return False # load variables from shell into our current Python context project_vars = self._load_project_vars(path) project_root_path = self.get_apps_path(context) + '/www-data/' + self._get_var( project_vars, path, 'GIT_PROJECT_DIR') # 1) run pre_update hook self.contextual_sh(path, 'pre_update %s' % project_root_path) # 2a) clone fresh repository if not os.path.isdir(project_root_path): self.io().info('Cloning a fresh repository at "%s"' % project_root_path) self._clone_new_repository(path, project_vars) # 2b) update existing repository - pull changes else: self.io().info('Pulling in an existing repository at "%s"' % project_root_path) self._pull_changes_into_existing_repository(path, project_vars) # 3) run pre_update hook self.contextual_sh(path, 'post_update %s' % project_root_path) self.io().print_opt_line() self.io().success_msg('Application\'s repository updated.') return True
def run(self, context: ExecutionContext) -> bool: profile = context.get_arg('--profile') strategy = context.get_arg('--strategy') success = True with self.hooks_executed(context, 'upgrade'): self.rkd([ '--no-ui', ':harbor:templates:render', ':harbor:git:apps:update-all', ':harbor:pull', '--profile=%s' % profile, ':harbor:start', '--profile=%s' % profile, '--strategy=%s' % strategy, '--remove-previous-images' if context.get_arg('--remove-previous-images') else '', ':harbor:gateway:reload' ]) return success
def run(self, context: ExecutionContext) -> bool: vault_opts = self._get_vault_opts(context) filename = context.get_arg('filename') subprocess.check_call('ansible-vault edit %s %s' % (vault_opts, filename), shell=True) return True
def run(self, context: ExecutionContext) -> bool: cmd = context.get_arg('--cmd') try: subprocess.check_call('cd %s && vagrant %s' % (self.ansible_dir, cmd), shell=True) except subprocess.CalledProcessError: return False return True
def _get_vault_opts(self, ctx: ExecutionContext, chdir: str = '') -> str: """Creates options to pass in Ansible Vault commandline The output will be a temporary vault file with password entered inline or a --ask-vault-pass switch """ try: vault_passwords = ctx.get_arg_or_env('--vault-passwords').split( '||') except MissingInputException: vault_passwords = [] num = 0 opts = '' enforce_ask_pass = ctx.get_arg('--ask-vault-pass') for passwd in vault_passwords: num = num + 1 if not passwd: continue if passwd.startswith('./') or passwd.startswith('/'): if os.path.isfile(passwd): opts += ' --vault-password-file="%s" ' % (chdir + passwd) else: self.io().error( 'Vault password file "%s" does not exist, calling --ask-vault-pass' % passwd) enforce_ask_pass = True else: tmp_vault_file = self.temp.assign_temporary_file(mode=0o644) with open(tmp_vault_file, 'w') as f: f.write(passwd) opts += ' --vault-password-file="%s" ' % (chdir + tmp_vault_file) if enforce_ask_pass: opts += ' --ask-vault-pass ' return opts
def run(self, ctx: ExecutionContext) -> bool: command = ctx.get_arg('--command') shell = ctx.get_arg('--shell') tty = bool(ctx.get_arg('--no-tty')) interactive = bool(ctx.get_arg('--no-interactive')) container_name, service, instance_num = self.prepare_tuple_for_single_container(ctx) if not service: return False if shell != '/bin/sh' and command == '/bin/sh': command = shell self.containers(ctx).exec_in_container_passthrough( command, service, instance_num, shell=shell, tty=tty, interactive=interactive ) return True
def execute_mocked_task_and_get_output(self, task: TaskInterface, args=None, env=None) -> str: """ Run a single task, capturing it's output in a simplified way. There is no whole RKD bootstrapped in this operation. :param TaskInterface task: :param dict args: :param dict env: :return: """ if args is None: args = {} if env is None: env = {} ctx = ApplicationContext([], [], '') ctx.io = BufferedSystemIO() task.internal_inject_dependencies( io=ctx.io, ctx=ctx, executor=OneByOneTaskExecutor(ctx=ctx), temp_manager=TempManager()) merged_env = deepcopy(os.environ) merged_env.update(env) r_io = IO() str_io = StringIO() defined_args = {} for arg, arg_value in args.items(): defined_args[arg] = {'default': ''} with r_io.capture_descriptors(enable_standard_out=True, stream=str_io): try: # noinspection PyTypeChecker result = task.execute( ExecutionContext(TaskDeclaration(task), args=args, env=merged_env, defined_args=defined_args)) except Exception: self._restore_standard_out() print(ctx.io.get_value() + "\n" + str_io.getvalue()) raise return ctx.io.get_value() + "\n" + str_io.getvalue( ) + "\nTASK_EXIT_RESULT=" + str(result)
def _get_prepared_compose_driver(self, args: dict = {}, env: dict = {}) -> ComposeDriver: merged_env = deepcopy(os.environ) merged_env.update(env) task = self.satisfy_task_dependencies(TestTask(), BufferedSystemIO()) declaration = TaskDeclaration(task) ctx = ExecutionContext(declaration, args=args, env=merged_env) return ComposeDriver(task, ctx, TEST_PROJECT_NAME)
def execute(self, ctx: ExecutionContext) -> bool: wizard = Wizard(self) wizard.load_previously_stored_values() os.environ.update(wizard.answers) self.rkd([ ':j2:directory-to-directory', '--source="%s"' % ctx.get_arg('path') + '/files/', '--target="./"', '--pattern="(.*)"', '--copy-not-matching-files', '--template-filenames' ]) return True
def run(self, ctx: ExecutionContext) -> bool: services = self.get_matching_services(ctx) for service in services: self.io().info('Removing "%s"' % service.get_name()) self.rkd([ ':harbor:service:rm', service.get_name(), '--with-image' if ctx.get_arg('--with-image') else '' ]) return True
def test_non_existing_dir_is_skipped(self): """Assert that non-existing directory does not cause exception, but will be skipped""" task = TestTask() task._io = BufferedSystemIO() task._io.set_log_level('debug') ctx = ExecutionContext(TaskDeclaration(task), args={}, env={}) task.execute_hooks(ctx, 'non-existing-directory') self.assertIn( 'Hooks dir "./hooks.d//non-existing-directory/" not present, skipping', task._io.get_value())
def run(self, ctx: ExecutionContext) -> bool: self.io().h2('Validating NGINX configuration') self.containers(ctx).exec_in_container('gateway', 'nginx -t') self.io().h2('Reloading NGINX configuration') self.containers(ctx).exec_in_container('gateway', 'nginx -s reload') if ctx.get_env('DISABLE_SSL').lower() != 'true': self.io().h2('Reloading SSL configuration') self.make_sure_ssl_service_is_up(ctx) self.containers(ctx).exec_in_container('gateway_letsencrypt', '/app/signal_le_service') return True
def run(self, context: ExecutionContext) -> bool: self._preserve_vault_parameters_for_usage_in_inner_tasks(context) node_group = context.get_arg('--group') node_num = int(context.get_arg('--num')) - 1 should_print_password = context.get_arg('--print-password') try: config = self.get_config() node = self._get_node(node_group, node_num, config) if not node: return False if should_print_password: self._print_password(node) return self._ssh(node) except MissingDeploymentConfigurationError as e: self.io().error_msg(str(e)) return False