Exemple #1
0
def build_image(ctx: ExecutionContext, this: TaskInterface) -> bool:
    """ Builds a docker image for given architecture, tags properly and push """
    tag = ctx.get_arg('--git-tag')
    should_push = ctx.get_arg('--push')
    arch = ctx.get_arg('--arch')
    docker_tag = 'latest-dev-' + arch

    if tag:
        docker_tag = arch + '-' + tag

    image_name = '{repo}:{tag}'.format(tag=docker_tag, repo=REPO)

    this.sh(
        'docker build . -f ./.infrastructure/{docker_arch}.Dockerfile -t {image}'
        .format(docker_arch=arch, image=image_name))

    this.sh('docker tag {image} {image}-$(date "+%Y-%m-%d")'.format(
        image=image_name))

    if should_push:
        this.sh('docker push {image}'.format(image=image_name))
        this.sh(
            'docker push {image}-$(date "+%Y-%m-%d")'.format(image=image_name))

    return True
Exemple #2
0
    def run(self, context: ExecutionContext) -> bool:
        service_name = context.get_arg('name')
        strategy = str(context.get_arg('--strategy'))
        remove_previous_images = bool(context.get_arg('--remove-previous-images'))
        service = self.services(context).get_by_name(service_name)

        strategies = {
            'rolling': lambda: self.deploy_rolling(service, context),
            'compose': lambda: self.deploy_compose_like(service, context),
            'recreate': lambda: self.deploy_recreate(service, context)
        }

        with self.hooks_executed(context, 'service-start-%s' % service_name):
            with self._old_images_clean_up(context, service, clear_images=remove_previous_images):
                if strategy in strategies:
                    return strategies[strategy]()
                else:
                    if strategy == 'auto':
                        strategy = service.get_update_strategy(default='compose')

                        if strategy in strategies:
                            return strategies[strategy]()

                    self.io().error_msg('Invalid strategy selected: %s' % strategy)
                    return False
Exemple #3
0
    def run(self, context: ExecutionContext) -> bool:
        services = self.get_matching_services(context)
        strategy = context.get_arg('--strategy')
        result = True

        with self.hooks_executed(context, 'start'):
            for service in services:
                self.io().h2(
                    'Starting "%s" (%i instances)...' %
                    (service.get_name(), service.get_desired_replicas_count()))

                try:
                    self.rkd(
                        [
                            '--no-ui', ':harbor:service:up',
                            service.get_name(), '--remove-previous-images'
                            if context.get_arg('--remove-previous-images') else
                            '',
                            ('--strategy=%s' % strategy) if strategy else ''
                        ],
                        capture=not self.io().is_log_level_at_least('info'))

                    self.io().success_msg('Service "%s" was started' %
                                          service.get_name())

                except CalledProcessError as e:
                    self.io().err(str(e))
                    self.io().error_msg('Cannot start service "%s"' %
                                        service.get_name())
                    result = False

                self.io().print_opt_line()

        return result
    def run(self, context: ExecutionContext) -> bool:
        """Validate parameters and select action"""

        is_global = context.get_arg('--global')
        domain = context.get_arg('--domain')
        service = context.get_arg('--service')

        directory = self.get_data_path(context) + '/maintenance-mode'

        if not self._validate_switches(is_global, domain, service):
            self.io().error_msg(
                'Cannot use together --global, --domain and --service switch. Pick one of them.'
            )
            return False

        try:
            if is_global:
                return self.act([directory + '/on'], 'globally')

            elif service:
                return self.act_for_service(directory, service, context)

            elif domain:
                return self.act_for_domain(directory, domain, context)
            else:
                self.io().error_msg('Must specify --global or --domain switch')
                return False

        except PermissionError as e:
            self.io().error_msg(
                'No write permissions. Set permissions or use sudo? %s' %
                str(e))
            return False
Exemple #5
0
    def deploy_compose_like(self, service: ServiceDeclaration, ctx: ExecutionContext) -> bool:
        """Regular docker-compose up deployment (with downtime)"""

        self.io().info('Performing "compose" deployment for "%s"' % service.get_name())
        self.containers(ctx).up(service,
                                norecreate=bool(ctx.get_arg('--dont-recreate')),
                                extra_args=ctx.get_arg('--extra-args'))

        return True
Exemple #6
0
    def run(self, context: ExecutionContext) -> bool:
        vault_opts = self._get_vault_opts(context)
        filename = context.get_arg('filename')
        mode = 'decrypt' if context.get_arg('--decrypt') else 'encrypt'

        self.sh('ansible-vault %s %s %s' % (mode, vault_opts, filename),
                capture=False)

        return True
Exemple #7
0
    def run(self, ctx: ExecutionContext) -> bool:
        service_name = ctx.get_arg('name')
        timeout = int(ctx.get_arg('--timeout'))
        instance_num = int(ctx.get_arg('--instance')) if ctx.get_arg('--instance') else None

        service = self.services(ctx).get_by_name(service_name)

        try:
            container_name = self.containers(ctx).find_container_name(service, instance_num)
            container = self.containers(ctx).inspect_container(container_name)
        except ServiceNotCreatedException as e:
            self.io().error_msg(str(e))
            return False

        started_at = time()

        self.io().info('Checking health of "%s" service - %s' % (service_name, container_name))

        if container.has_health_check():
            while True:
                if time() - started_at >= timeout:
                    self.io().error_msg('Timeout of %is reached.' % timeout)
                    return False

                health = container.get_health_status()

                # do not wait for result, do check manually in mean time
                if health == 'starting':
                    self.io().warn('Docker reports "starting" - performing a manual check, we wont wait for docker')

                    try:
                        command = container.get_health_check_command().replace('"', '\\"')

                        self.containers(ctx).exec_in_container(
                            service_name=service_name,
                            command='/bin/sh -c "%s"' % command,
                            instance_num=instance_num
                        )

                        self.io().success_msg('Service healthy after %is' % (time() - started_at))
                        return True

                    except subprocess.CalledProcessError as e:
                        self.io().debug(str(e.output)[0:128])
                        sleep(1)
                        continue

                if health == 'healthy' or health == 'running':
                    self.io().success_msg('Service healthy after %i' % (time() - started_at))
                    return True

                sleep(1)
        else:
            self.io().warn('Instance has no healthcheck defined!')

        return True
Exemple #8
0
    def run(self, context: ExecutionContext) -> bool:
        playbook_name = context.get_arg_or_env('--playbook')
        inventory_name = context.get_arg_or_env('--inventory')
        git_private_key_path = context.get_arg_or_env('--git-key')
        branch = context.get_arg('--branch')
        profile = context.get_arg('--profile')
        debug = context.get_arg('--debug')

        # keep the vault arguments for decryption of deployment.yml
        self._preserve_vault_parameters_for_usage_in_inner_tasks(context)

        if not self.role_is_installed_and_configured():
            self.io().error_msg(
                'Deployment not configured. Use `harbor :deployment:files:update` first'
            )
            return False

        try:
            self.install_and_configure_role(context, force_update=False)

        except MissingDeploymentConfigurationError as e:
            self.io().error_msg(str(e))
            return False

        pwd_backup = os.getcwd()
        pid = None

        try:
            command = ''
            opts = ''

            if git_private_key_path:
                sock, pid = self.spawn_ssh_agent()
                command += 'export SSH_AUTH_SOCK=%s; export SSH_AGENT_PID=%i; ssh-add %s; sleep 5; ' % \
                           (sock, pid, git_private_key_path)

            if debug:
                opts += ' -vv '

            opts += ' -e git_branch="%s" ' % branch
            opts += ' -e harbor_deployment_profile="%s" ' % profile
            opts += self._get_vault_opts(context, '../../')

            os.chdir(self.ansible_dir)
            command += 'ansible-playbook ./%s -i %s %s' % (
                playbook_name, inventory_name, opts)

            self.spawn_ansible(command)
        finally:
            os.chdir(pwd_backup)

            if pid:
                self.kill_ssh_agent(pid)

        return True
Exemple #9
0
    def run(self, ctx: ExecutionContext) -> bool:
        params = []

        if ctx.get_arg('--quiet'):
            params.append('--quiet')

        if ctx.get_arg('--all'):
            params.append('--all')

        self.containers(ctx).ps(params)
        return True
Exemple #10
0
    def prepare_tuple_for_single_container(self, ctx: ExecutionContext) -> tuple:
        service_name = ctx.get_arg('--name')
        service = self.services(ctx).get_by_name(service_name)
        instance_num = int(ctx.get_arg('--instance-num')) if ctx.get_arg('--instance-num') else None
        container_name = self.containers(ctx).find_container_name(service, instance_num)

        if not container_name:
            self.io().error_msg('Container not found')
            return None, None, None

        return container_name, service, instance_num
Exemple #11
0
    def run(self, ctx: ExecutionContext) -> bool:
        service_name = ctx.get_arg('name')
        service = self.services(ctx).get_by_name(service_name)
        with_image = ctx.get_arg('--with-image')
        images = self.get_all_images_for_service(ctx, service) if with_image else []

        self.containers(ctx).rm(service, extra_args=ctx.get_arg('--extra-args'))

        for image in images:
            try:
                self.containers(ctx).rm_image(image, capture=True)
            except:
                pass

        return True
Exemple #12
0
    def _ask_and_set_var(self, ctx: ExecutionContext, arg_name: str,
                         title: str, attribute: str, secret: bool):
        """Ask user an interactive question, then add answer to the deployment.yml loaded in memory

        The variable will be appended to any node, where the variable is empty.
        Example: We have 5 servers, 3 without a password. So the password will be applied to 3 servers.
        """

        self.get_config()

        if not ctx.get_arg(arg_name):
            return

        wizard = Wizard(self).ask(title, attribute=attribute, secret=secret)

        for group_name, nodes in self._config['nodes'].items():
            node_num = 0

            for node in nodes:
                node_num += 1
                if attribute in self._config['nodes'][group_name][node_num -
                                                                  1]:
                    continue

                self._config['nodes'][group_name][
                    node_num - 1][attribute] = wizard.answers[attribute]
Exemple #13
0
    def execute(self, ctx: ExecutionContext) -> bool:
        name = ctx.get_arg('name')

        category_name, pkg_name = self.extract_category_and_pkg_names(name)
        path = self.find_snippet_path(pkg_name, category_name)

        if not path:
            self.io().error(
                'Snippet not found in any synchronized repository. ' +
                'Did you forget to do :cooperative:sync?')
            return False

        self.io().info('Installing snippet from "%s"' % path)

        # mock rkd_path, so the snippet can override the tasks
        rkd_path = os.getenv('RKD_PATH', '')
        snippet_rkd_path = os.path.realpath('./' + path + '/.rkd')

        if snippet_rkd_path:
            os.putenv('RKD_PATH',
                      (rkd_path + ':' + snippet_rkd_path).strip(':'))

        try:
            subprocess.check_call(['rkd', ':snippet:wizard', path])
            subprocess.check_call(['rkd', ':snippet:install', path])
        finally:
            if os.path.isfile('.rkd/tmp-wizard.json'):
                os.unlink('.rkd/tmp-wizard.json')

            os.putenv('RKD_PATH', rkd_path)

        return True
Exemple #14
0
    def run(self, context: ExecutionContext) -> bool:
        path = self.get_app_repository_path(context.get_arg('name'), context)

        if not os.path.isfile(path):
            self.io().error_msg('Cannot pull a repository: Unknown application, "%s" file not found' % path)
            return False

        # load variables from shell into our current Python context
        project_vars = self._load_project_vars(path)

        project_root_path = self.get_apps_path(context) + '/www-data/' + self._get_var(
            project_vars, path, 'GIT_PROJECT_DIR')

        # 1) run pre_update hook
        self.contextual_sh(path, 'pre_update %s' % project_root_path)

        # 2a) clone fresh repository
        if not os.path.isdir(project_root_path):
            self.io().info('Cloning a fresh repository at "%s"' % project_root_path)

            self._clone_new_repository(path, project_vars)

        # 2b) update existing repository - pull changes
        else:
            self.io().info('Pulling in an existing repository at "%s"' % project_root_path)
            self._pull_changes_into_existing_repository(path, project_vars)

        # 3) run pre_update hook
        self.contextual_sh(path, 'post_update %s' % project_root_path)

        self.io().print_opt_line()
        self.io().success_msg('Application\'s repository updated.')

        return True
Exemple #15
0
    def run(self, context: ExecutionContext) -> bool:
        profile = context.get_arg('--profile')
        strategy = context.get_arg('--strategy')
        success = True

        with self.hooks_executed(context, 'upgrade'):
            self.rkd([
                '--no-ui', ':harbor:templates:render',
                ':harbor:git:apps:update-all', ':harbor:pull',
                '--profile=%s' % profile, ':harbor:start',
                '--profile=%s' % profile,
                '--strategy=%s' % strategy, '--remove-previous-images'
                if context.get_arg('--remove-previous-images') else '',
                ':harbor:gateway:reload'
            ])

        return success
Exemple #16
0
    def execute(self, context: ExecutionContext) -> bool:
        opts = ''

        if context.args['skip_existing']:
            opts += ' --skip-existing '

        if context.get_arg('--url'):
            if context.get_arg('--test'):
                raise Exception('Cannot use --url and --test switch at once')

            opts += ' --repository-url=%s' % context.get_arg('--url')

        if context.get_arg('--test'):
            opts += ' --repository-url https://test.pypi.org/legacy/ '

        self.sh('''
            %s upload \\
                --disable-progress-bar \\
                --verbose \\
                --username=%s \\
                --password=%s \\
                %s %s
        ''' % (context.get_env('TWINE_PATH'), context.get_arg('--username'),
               context.get_arg('--password'), opts, context.get_arg('--src')))

        return True
Exemple #17
0
    def run(self, context: ExecutionContext) -> bool:
        vault_opts = self._get_vault_opts(context)
        filename = context.get_arg('filename')

        subprocess.check_call('ansible-vault edit %s %s' %
                              (vault_opts, filename),
                              shell=True)

        return True
Exemple #18
0
    def run(self, context: ExecutionContext) -> bool:
        cmd = context.get_arg('--cmd')

        try:
            subprocess.check_call('cd %s && vagrant %s' % (self.ansible_dir, cmd), shell=True)

        except subprocess.CalledProcessError:
            return False

        return True
Exemple #19
0
    def run(self, ctx: ExecutionContext) -> bool:
        command = ctx.get_arg('--command')
        shell = ctx.get_arg('--shell')
        tty = bool(ctx.get_arg('--no-tty'))
        interactive = bool(ctx.get_arg('--no-interactive'))

        container_name, service, instance_num = self.prepare_tuple_for_single_container(ctx)

        if not service:
            return False

        if shell != '/bin/sh' and command == '/bin/sh':
            command = shell

        self.containers(ctx).exec_in_container_passthrough(
            command, service, instance_num, shell=shell, tty=tty, interactive=interactive
        )

        return True
Exemple #20
0
    def execute(self, ctx: ExecutionContext) -> bool:
        wizard = Wizard(self)
        wizard.load_previously_stored_values()
        os.environ.update(wizard.answers)

        self.rkd([
            ':j2:directory-to-directory',
            '--source="%s"' % ctx.get_arg('path') + '/files/', '--target="./"',
            '--pattern="(.*)"', '--copy-not-matching-files',
            '--template-filenames'
        ])
        return True
Exemple #21
0
    def run(self, ctx: ExecutionContext) -> bool:
        services = self.get_matching_services(ctx)

        for service in services:
            self.io().info('Removing "%s"' % service.get_name())
            self.rkd([
                ':harbor:service:rm',
                service.get_name(),
                '--with-image' if ctx.get_arg('--with-image') else ''
            ])

        return True
Exemple #22
0
    def run(self, context: ExecutionContext) -> bool:
        self._preserve_vault_parameters_for_usage_in_inner_tasks(context)

        node_group = context.get_arg('--group')
        node_num = int(context.get_arg('--num')) - 1
        should_print_password = context.get_arg('--print-password')

        try:
            config = self.get_config()
            node = self._get_node(node_group, node_num, config)

            if not node:
                return False

            if should_print_password:
                self._print_password(node)

            return self._ssh(node)

        except MissingDeploymentConfigurationError as e:
            self.io().error_msg(str(e))
            return False
Exemple #23
0
    def _preserve_vault_parameters_for_usage_in_inner_tasks(
            self, ctx: ExecutionContext):
        """Preserve original parameters related to Vault, so those parameters can be propagated to inner tasks"""

        try:
            vault_passwords = ctx.get_arg_or_env('--vault-passwords')
        except MissingInputException:
            vault_passwords = ''

        # keep the vault arguments for decryption of deployment.yml
        self.vault_args = ['--vault-passwords=' + vault_passwords]
        if ctx.get_arg('--ask-vault-pass'):
            self.vault_args.append('--ask-vault-pass')
Exemple #24
0
    def run(self, ctx: ExecutionContext) -> bool:
        container_name, service, instance_num = self.prepare_tuple_for_single_container(ctx)
        buffered = bool(ctx.get_arg('--buffered'))

        if not service:
            self.io().error_msg('Service not found')
            return False

        self.io().outln(
            self.containers(ctx).get_logs(service, instance_num, raw=not buffered, follow=bool(ctx.get_arg('--follow')))
        )

        return True
Exemple #25
0
    def _get_vault_opts(self, ctx: ExecutionContext, chdir: str = '') -> str:
        """Creates options to pass in Ansible Vault commandline

        The output will be a temporary vault file with password entered inline or a --ask-vault-pass switch
        """

        try:
            vault_passwords = ctx.get_arg_or_env('--vault-passwords').split(
                '||')
        except MissingInputException:
            vault_passwords = []

        num = 0
        opts = ''
        enforce_ask_pass = ctx.get_arg('--ask-vault-pass')

        for passwd in vault_passwords:
            num = num + 1

            if not passwd:
                continue

            if passwd.startswith('./') or passwd.startswith('/'):
                if os.path.isfile(passwd):
                    opts += ' --vault-password-file="%s" ' % (chdir + passwd)
                else:
                    self.io().error(
                        'Vault password file "%s" does not exist, calling --ask-vault-pass'
                        % passwd)
                    enforce_ask_pass = True
            else:
                tmp_vault_file = self.temp.assign_temporary_file(mode=0o644)

                with open(tmp_vault_file, 'w') as f:
                    f.write(passwd)

                opts += ' --vault-password-file="%s" ' % (chdir +
                                                          tmp_vault_file)

        if enforce_ask_pass:
            opts += ' --ask-vault-pass '

        return opts
Exemple #26
0
    def run(self, ctx: ExecutionContext) -> bool:
        service_name = ctx.get_arg('name')
        service = self.services(ctx).get_by_name(service_name)

        if not service:
            self.io().error_msg('Service not found')
            return False

        try:
            containers = self.containers(ctx).find_all_container_names_for_service(service)
        except ServiceNotCreatedException:
            containers = []

        self.print_service_summary(ctx, service, len(containers))
        self.io().print_line()

        if containers:
            self.print_containers_summary(ctx, service, containers)

        return True
Exemple #27
0
    def execute(self, context: ExecutionContext) -> bool:
        cmd = 'export PYTHONUNBUFFERED=1; '

        if context.get_arg('--src-dir'):
            cmd += 'set +u; export PYTHONPATH="%s:$PYTHONPATH"; ' % context.get_arg(
                '--src-dir')

        cmd += '%s -m unittest discover -s %s ' % (
            context.get_arg('--python-bin'), context.get_arg('--tests-dir'))

        if context.get_arg('--pattern'):
            cmd += ' -p \'%s\' ' % context.get_arg('--pattern')

        if context.get_arg('--filter'):
            cmd += ' -k %s ' % context.get_arg('--filter')

        try:
            self.sh(cmd, verbose=True, strict=True)
        except CalledProcessError as e:
            return False

        return True
Exemple #28
0
    def run(self, ctx: ExecutionContext) -> bool:
        group_by = ctx.get_arg('--group-by')
        services = self.get_matching_services(ctx)

        # table
        table_headers = [
            'Priority', 'Name', 'Declared version', 'Replicas', 'URL', 'Ports',
            'Maintenance mode', 'Update strategy', 'Watchtower'
        ]
        table_body = []

        running = self.containers(ctx).get_created_containers(
            only_running=True)

        for service in services:
            domains = service.get_domains()

            # GROUP-BY: list per-domain
            if group_by == 'url':
                for domain in domains:
                    self._append_to_table(table_body, service, running, domain)

                # list also services that are not under a domain (internal services or exposed via ports)
                if not domains:
                    self._append_to_table(table_body,
                                          service,
                                          running,
                                          domain='-')

            # GROUP-BY: none
            if group_by == 'none':
                self._append_to_table(table_body,
                                      service,
                                      running,
                                      domain=self._format_domains(domains))

        self.io().outln(tabulate(table_body, headers=table_headers))

        return True
Exemple #29
0
    def run(self, context: ExecutionContext) -> bool:
        vault_opts = self._get_vault_opts(context)
        mode = 'decrypt' if context.get_arg('--decrypt') else 'encrypt'

        src = '.env'
        dst = '.env-prod'

        if mode == 'decrypt':
            src = '.env-prod'
            dst = '.env'

        self.sh('cp %s %s-tmp' % (src, dst))
        self.sh('ansible-vault %s %s %s-tmp' % (mode, vault_opts, dst),
                capture=False)
        self.sh('mv %s-tmp %s' % (dst, dst))

        if mode == 'encrypt':
            try:
                self.sh('git add %s' % dst)
            except:
                pass

        return True
Exemple #30
0
    def run(self, context: ExecutionContext) -> bool:
        path = self.get_app_repository_path(context.get_arg('name'), context)

        if not os.path.isfile(path):
            self.io().error_msg('Cannot pull a repository: Unknown application, "%s" file not found' % path)
            return False

        # load variables from shell into our current Python context
        project_vars = self._load_project_vars(path)

        writable_dirs = self._get_var(project_vars, path, 'WRITABLE_DIRS').replace('\\ ', '@SPACE@').split(' ')
        project_root_path = self.get_apps_path(context) + '/www-data/' + self._get_var(
            project_vars, path, 'GIT_PROJECT_DIR')
        container_user = self._get_var(project_vars, path, 'DEFAULT_CONTAINER_USER')

        for writable_dir in writable_dirs:
            writable_dir = writable_dir.replace('@SPACE@', ' ')

            command = 'sudo chown %s "%s/%s"' % (container_user, project_root_path, writable_dir)

            self.io().info(command)
            self.sh(command)

        return True