def build_image(ctx: ExecutionContext, this: TaskInterface) -> bool:
""" Builds a docker image for given architecture, tags properly and push """
tag = ctx.get_arg('--git-tag')
should_push = ctx.get_arg('--push')
arch = ctx.get_arg('--arch')
docker_tag = 'latest-dev-' + arch
if tag:
docker_tag = arch + '-' + tag
image_name = '{repo}:{tag}'.format(tag=docker_tag, repo=REPO)
this.sh(
'docker build . -f ./.infrastructure/{docker_arch}.Dockerfile -t {image}'
.format(docker_arch=arch, image=image_name))
this.sh('docker tag {image} {image}-$(date "+%Y-%m-%d")'.format(
image=image_name))
if should_push:
this.sh('docker push {image}'.format(image=image_name))
this.sh(
'docker push {image}-$(date "+%Y-%m-%d")'.format(image=image_name))
return True
def run(self, context: ExecutionContext) -> bool:
service_name = context.get_arg('name')
strategy = str(context.get_arg('--strategy'))
remove_previous_images = bool(context.get_arg('--remove-previous-images'))
service = self.services(context).get_by_name(service_name)
strategies = {
'rolling': lambda: self.deploy_rolling(service, context),
'compose': lambda: self.deploy_compose_like(service, context),
'recreate': lambda: self.deploy_recreate(service, context)
}
with self.hooks_executed(context, 'service-start-%s' % service_name):
with self._old_images_clean_up(context, service, clear_images=remove_previous_images):
if strategy in strategies:
return strategies[strategy]()
else:
if strategy == 'auto':
strategy = service.get_update_strategy(default='compose')
if strategy in strategies:
return strategies[strategy]()
self.io().error_msg('Invalid strategy selected: %s' % strategy)
return False
def run(self, context: ExecutionContext) -> bool:
services = self.get_matching_services(context)
strategy = context.get_arg('--strategy')
result = True
with self.hooks_executed(context, 'start'):
for service in services:
self.io().h2(
'Starting "%s" (%i instances)...' %
(service.get_name(), service.get_desired_replicas_count()))
try:
self.rkd(
[
'--no-ui', ':harbor:service:up',
service.get_name(), '--remove-previous-images'
if context.get_arg('--remove-previous-images') else
'',
('--strategy=%s' % strategy) if strategy else ''
],
capture=not self.io().is_log_level_at_least('info'))
self.io().success_msg('Service "%s" was started' %
service.get_name())
except CalledProcessError as e:
self.io().err(str(e))
self.io().error_msg('Cannot start service "%s"' %
service.get_name())
result = False
self.io().print_opt_line()
return result
def run(self, context: ExecutionContext) -> bool:
"""Validate parameters and select action"""
is_global = context.get_arg('--global')
domain = context.get_arg('--domain')
service = context.get_arg('--service')
directory = self.get_data_path(context) + '/maintenance-mode'
if not self._validate_switches(is_global, domain, service):
self.io().error_msg(
'Cannot use together --global, --domain and --service switch. Pick one of them.'
)
return False
try:
if is_global:
return self.act([directory + '/on'], 'globally')
elif service:
return self.act_for_service(directory, service, context)
elif domain:
return self.act_for_domain(directory, domain, context)
else:
self.io().error_msg('Must specify --global or --domain switch')
return False
except PermissionError as e:
self.io().error_msg(
'No write permissions. Set permissions or use sudo? %s' %
str(e))
return False
def deploy_compose_like(self, service: ServiceDeclaration, ctx: ExecutionContext) -> bool:
"""Regular docker-compose up deployment (with downtime)"""
self.io().info('Performing "compose" deployment for "%s"' % service.get_name())
self.containers(ctx).up(service,
norecreate=bool(ctx.get_arg('--dont-recreate')),
extra_args=ctx.get_arg('--extra-args'))
return True
def run(self, context: ExecutionContext) -> bool:
vault_opts = self._get_vault_opts(context)
filename = context.get_arg('filename')
mode = 'decrypt' if context.get_arg('--decrypt') else 'encrypt'
self.sh('ansible-vault %s %s %s' % (mode, vault_opts, filename),
capture=False)
return True
def run(self, ctx: ExecutionContext) -> bool:
service_name = ctx.get_arg('name')
timeout = int(ctx.get_arg('--timeout'))
instance_num = int(ctx.get_arg('--instance')) if ctx.get_arg('--instance') else None
service = self.services(ctx).get_by_name(service_name)
try:
container_name = self.containers(ctx).find_container_name(service, instance_num)
container = self.containers(ctx).inspect_container(container_name)
except ServiceNotCreatedException as e:
self.io().error_msg(str(e))
return False
started_at = time()
self.io().info('Checking health of "%s" service - %s' % (service_name, container_name))
if container.has_health_check():
while True:
if time() - started_at >= timeout:
self.io().error_msg('Timeout of %is reached.' % timeout)
return False
health = container.get_health_status()
# do not wait for result, do check manually in mean time
if health == 'starting':
self.io().warn('Docker reports "starting" - performing a manual check, we wont wait for docker')
try:
command = container.get_health_check_command().replace('"', '\\"')
self.containers(ctx).exec_in_container(
service_name=service_name,
command='/bin/sh -c "%s"' % command,
instance_num=instance_num
)
self.io().success_msg('Service healthy after %is' % (time() - started_at))
return True
except subprocess.CalledProcessError as e:
self.io().debug(str(e.output)[0:128])
sleep(1)
continue
if health == 'healthy' or health == 'running':
self.io().success_msg('Service healthy after %i' % (time() - started_at))
return True
sleep(1)
else:
self.io().warn('Instance has no healthcheck defined!')
return True
def run(self, context: ExecutionContext) -> bool:
playbook_name = context.get_arg_or_env('--playbook')
inventory_name = context.get_arg_or_env('--inventory')
git_private_key_path = context.get_arg_or_env('--git-key')
branch = context.get_arg('--branch')
profile = context.get_arg('--profile')
debug = context.get_arg('--debug')
# keep the vault arguments for decryption of deployment.yml
self._preserve_vault_parameters_for_usage_in_inner_tasks(context)
if not self.role_is_installed_and_configured():
self.io().error_msg(
'Deployment not configured. Use `harbor :deployment:files:update` first'
)
return False
try:
self.install_and_configure_role(context, force_update=False)
except MissingDeploymentConfigurationError as e:
self.io().error_msg(str(e))
return False
pwd_backup = os.getcwd()
pid = None
try:
command = ''
opts = ''
if git_private_key_path:
sock, pid = self.spawn_ssh_agent()
command += 'export SSH_AUTH_SOCK=%s; export SSH_AGENT_PID=%i; ssh-add %s; sleep 5; ' % \
(sock, pid, git_private_key_path)
if debug:
opts += ' -vv '
opts += ' -e git_branch="%s" ' % branch
opts += ' -e harbor_deployment_profile="%s" ' % profile
opts += self._get_vault_opts(context, '../../')
os.chdir(self.ansible_dir)
command += 'ansible-playbook ./%s -i %s %s' % (
playbook_name, inventory_name, opts)
self.spawn_ansible(command)
finally:
os.chdir(pwd_backup)
if pid:
self.kill_ssh_agent(pid)
return True
def run(self, ctx: ExecutionContext) -> bool:
params = []
if ctx.get_arg('--quiet'):
params.append('--quiet')
if ctx.get_arg('--all'):
params.append('--all')
self.containers(ctx).ps(params)
return True
def prepare_tuple_for_single_container(self, ctx: ExecutionContext) -> tuple:
service_name = ctx.get_arg('--name')
service = self.services(ctx).get_by_name(service_name)
instance_num = int(ctx.get_arg('--instance-num')) if ctx.get_arg('--instance-num') else None
container_name = self.containers(ctx).find_container_name(service, instance_num)
if not container_name:
self.io().error_msg('Container not found')
return None, None, None
return container_name, service, instance_num
def run(self, ctx: ExecutionContext) -> bool:
service_name = ctx.get_arg('name')
service = self.services(ctx).get_by_name(service_name)
with_image = ctx.get_arg('--with-image')
images = self.get_all_images_for_service(ctx, service) if with_image else []
self.containers(ctx).rm(service, extra_args=ctx.get_arg('--extra-args'))
for image in images:
try:
self.containers(ctx).rm_image(image, capture=True)
except:
pass
return True
def _ask_and_set_var(self, ctx: ExecutionContext, arg_name: str,
title: str, attribute: str, secret: bool):
"""Ask user an interactive question, then add answer to the deployment.yml loaded in memory
The variable will be appended to any node, where the variable is empty.
Example: We have 5 servers, 3 without a password. So the password will be applied to 3 servers.
"""
self.get_config()
if not ctx.get_arg(arg_name):
return
wizard = Wizard(self).ask(title, attribute=attribute, secret=secret)
for group_name, nodes in self._config['nodes'].items():
node_num = 0
for node in nodes:
node_num += 1
if attribute in self._config['nodes'][group_name][node_num -
1]:
continue
self._config['nodes'][group_name][
node_num - 1][attribute] = wizard.answers[attribute]
def execute(self, ctx: ExecutionContext) -> bool:
name = ctx.get_arg('name')
category_name, pkg_name = self.extract_category_and_pkg_names(name)
path = self.find_snippet_path(pkg_name, category_name)
if not path:
self.io().error(
'Snippet not found in any synchronized repository. ' +
'Did you forget to do :cooperative:sync?')
return False
self.io().info('Installing snippet from "%s"' % path)
# mock rkd_path, so the snippet can override the tasks
rkd_path = os.getenv('RKD_PATH', '')
snippet_rkd_path = os.path.realpath('./' + path + '/.rkd')
if snippet_rkd_path:
os.putenv('RKD_PATH',
(rkd_path + ':' + snippet_rkd_path).strip(':'))
try:
subprocess.check_call(['rkd', ':snippet:wizard', path])
subprocess.check_call(['rkd', ':snippet:install', path])
finally:
if os.path.isfile('.rkd/tmp-wizard.json'):
os.unlink('.rkd/tmp-wizard.json')
os.putenv('RKD_PATH', rkd_path)
return True
def run(self, context: ExecutionContext) -> bool:
path = self.get_app_repository_path(context.get_arg('name'), context)
if not os.path.isfile(path):
self.io().error_msg('Cannot pull a repository: Unknown application, "%s" file not found' % path)
return False
# load variables from shell into our current Python context
project_vars = self._load_project_vars(path)
project_root_path = self.get_apps_path(context) + '/www-data/' + self._get_var(
project_vars, path, 'GIT_PROJECT_DIR')
# 1) run pre_update hook
self.contextual_sh(path, 'pre_update %s' % project_root_path)
# 2a) clone fresh repository
if not os.path.isdir(project_root_path):
self.io().info('Cloning a fresh repository at "%s"' % project_root_path)
self._clone_new_repository(path, project_vars)
# 2b) update existing repository - pull changes
else:
self.io().info('Pulling in an existing repository at "%s"' % project_root_path)
self._pull_changes_into_existing_repository(path, project_vars)
# 3) run pre_update hook
self.contextual_sh(path, 'post_update %s' % project_root_path)
self.io().print_opt_line()
self.io().success_msg('Application\'s repository updated.')
return True
def run(self, context: ExecutionContext) -> bool:
profile = context.get_arg('--profile')
strategy = context.get_arg('--strategy')
success = True
with self.hooks_executed(context, 'upgrade'):
self.rkd([
'--no-ui', ':harbor:templates:render',
':harbor:git:apps:update-all', ':harbor:pull',
'--profile=%s' % profile, ':harbor:start',
'--profile=%s' % profile,
'--strategy=%s' % strategy, '--remove-previous-images'
if context.get_arg('--remove-previous-images') else '',
':harbor:gateway:reload'
])
return success
def execute(self, context: ExecutionContext) -> bool:
opts = ''
if context.args['skip_existing']:
opts += ' --skip-existing '
if context.get_arg('--url'):
if context.get_arg('--test'):
raise Exception('Cannot use --url and --test switch at once')
opts += ' --repository-url=%s' % context.get_arg('--url')
if context.get_arg('--test'):
opts += ' --repository-url https://test.pypi.org/legacy/ '
self.sh('''
%s upload \\
--disable-progress-bar \\
--verbose \\
--username=%s \\
--password=%s \\
%s %s
''' % (context.get_env('TWINE_PATH'), context.get_arg('--username'),
context.get_arg('--password'), opts, context.get_arg('--src')))
return True
def run(self, context: ExecutionContext) -> bool:
vault_opts = self._get_vault_opts(context)
filename = context.get_arg('filename')
subprocess.check_call('ansible-vault edit %s %s' %
(vault_opts, filename),
shell=True)
return True
def run(self, context: ExecutionContext) -> bool:
cmd = context.get_arg('--cmd')
try:
subprocess.check_call('cd %s && vagrant %s' % (self.ansible_dir, cmd), shell=True)
except subprocess.CalledProcessError:
return False
return True
def run(self, ctx: ExecutionContext) -> bool:
command = ctx.get_arg('--command')
shell = ctx.get_arg('--shell')
tty = bool(ctx.get_arg('--no-tty'))
interactive = bool(ctx.get_arg('--no-interactive'))
container_name, service, instance_num = self.prepare_tuple_for_single_container(ctx)
if not service:
return False
if shell != '/bin/sh' and command == '/bin/sh':
command = shell
self.containers(ctx).exec_in_container_passthrough(
command, service, instance_num, shell=shell, tty=tty, interactive=interactive
)
return True
def execute(self, ctx: ExecutionContext) -> bool:
wizard = Wizard(self)
wizard.load_previously_stored_values()
os.environ.update(wizard.answers)
self.rkd([
':j2:directory-to-directory',
'--source="%s"' % ctx.get_arg('path') + '/files/', '--target="./"',
'--pattern="(.*)"', '--copy-not-matching-files',
'--template-filenames'
])
return True
def run(self, ctx: ExecutionContext) -> bool:
services = self.get_matching_services(ctx)
for service in services:
self.io().info('Removing "%s"' % service.get_name())
self.rkd([
':harbor:service:rm',
service.get_name(),
'--with-image' if ctx.get_arg('--with-image') else ''
])
return True
def run(self, context: ExecutionContext) -> bool:
self._preserve_vault_parameters_for_usage_in_inner_tasks(context)
node_group = context.get_arg('--group')
node_num = int(context.get_arg('--num')) - 1
should_print_password = context.get_arg('--print-password')
try:
config = self.get_config()
node = self._get_node(node_group, node_num, config)
if not node:
return False
if should_print_password:
self._print_password(node)
return self._ssh(node)
except MissingDeploymentConfigurationError as e:
self.io().error_msg(str(e))
return False
def _preserve_vault_parameters_for_usage_in_inner_tasks(
self, ctx: ExecutionContext):
"""Preserve original parameters related to Vault, so those parameters can be propagated to inner tasks"""
try:
vault_passwords = ctx.get_arg_or_env('--vault-passwords')
except MissingInputException:
vault_passwords = ''
# keep the vault arguments for decryption of deployment.yml
self.vault_args = ['--vault-passwords=' + vault_passwords]
if ctx.get_arg('--ask-vault-pass'):
self.vault_args.append('--ask-vault-pass')
def run(self, ctx: ExecutionContext) -> bool:
container_name, service, instance_num = self.prepare_tuple_for_single_container(ctx)
buffered = bool(ctx.get_arg('--buffered'))
if not service:
self.io().error_msg('Service not found')
return False
self.io().outln(
self.containers(ctx).get_logs(service, instance_num, raw=not buffered, follow=bool(ctx.get_arg('--follow')))
)
return True
def _get_vault_opts(self, ctx: ExecutionContext, chdir: str = '') -> str:
"""Creates options to pass in Ansible Vault commandline
The output will be a temporary vault file with password entered inline or a --ask-vault-pass switch
"""
try:
vault_passwords = ctx.get_arg_or_env('--vault-passwords').split(
'||')
except MissingInputException:
vault_passwords = []
num = 0
opts = ''
enforce_ask_pass = ctx.get_arg('--ask-vault-pass')
for passwd in vault_passwords:
num = num + 1
if not passwd:
continue
if passwd.startswith('./') or passwd.startswith('/'):
if os.path.isfile(passwd):
opts += ' --vault-password-file="%s" ' % (chdir + passwd)
else:
self.io().error(
'Vault password file "%s" does not exist, calling --ask-vault-pass'
% passwd)
enforce_ask_pass = True
else:
tmp_vault_file = self.temp.assign_temporary_file(mode=0o644)
with open(tmp_vault_file, 'w') as f:
f.write(passwd)
opts += ' --vault-password-file="%s" ' % (chdir +
tmp_vault_file)
if enforce_ask_pass:
opts += ' --ask-vault-pass '
return opts
def run(self, ctx: ExecutionContext) -> bool:
service_name = ctx.get_arg('name')
service = self.services(ctx).get_by_name(service_name)
if not service:
self.io().error_msg('Service not found')
return False
try:
containers = self.containers(ctx).find_all_container_names_for_service(service)
except ServiceNotCreatedException:
containers = []
self.print_service_summary(ctx, service, len(containers))
self.io().print_line()
if containers:
self.print_containers_summary(ctx, service, containers)
return True
def execute(self, context: ExecutionContext) -> bool:
cmd = 'export PYTHONUNBUFFERED=1; '
if context.get_arg('--src-dir'):
cmd += 'set +u; export PYTHONPATH="%s:$PYTHONPATH"; ' % context.get_arg(
'--src-dir')
cmd += '%s -m unittest discover -s %s ' % (
context.get_arg('--python-bin'), context.get_arg('--tests-dir'))
if context.get_arg('--pattern'):
cmd += ' -p \'%s\' ' % context.get_arg('--pattern')
if context.get_arg('--filter'):
cmd += ' -k %s ' % context.get_arg('--filter')
try:
self.sh(cmd, verbose=True, strict=True)
except CalledProcessError as e:
return False
return True
def run(self, ctx: ExecutionContext) -> bool:
group_by = ctx.get_arg('--group-by')
services = self.get_matching_services(ctx)
# table
table_headers = [
'Priority', 'Name', 'Declared version', 'Replicas', 'URL', 'Ports',
'Maintenance mode', 'Update strategy', 'Watchtower'
]
table_body = []
running = self.containers(ctx).get_created_containers(
only_running=True)
for service in services:
domains = service.get_domains()
# GROUP-BY: list per-domain
if group_by == 'url':
for domain in domains:
self._append_to_table(table_body, service, running, domain)
# list also services that are not under a domain (internal services or exposed via ports)
if not domains:
self._append_to_table(table_body,
service,
running,
domain='-')
# GROUP-BY: none
if group_by == 'none':
self._append_to_table(table_body,
service,
running,
domain=self._format_domains(domains))
self.io().outln(tabulate(table_body, headers=table_headers))
return True
def run(self, context: ExecutionContext) -> bool:
vault_opts = self._get_vault_opts(context)
mode = 'decrypt' if context.get_arg('--decrypt') else 'encrypt'
src = '.env'
dst = '.env-prod'
if mode == 'decrypt':
src = '.env-prod'
dst = '.env'
self.sh('cp %s %s-tmp' % (src, dst))
self.sh('ansible-vault %s %s %s-tmp' % (mode, vault_opts, dst),
capture=False)
self.sh('mv %s-tmp %s' % (dst, dst))
if mode == 'encrypt':
try:
self.sh('git add %s' % dst)
except:
pass
return True
def run(self, context: ExecutionContext) -> bool:
path = self.get_app_repository_path(context.get_arg('name'), context)
if not os.path.isfile(path):
self.io().error_msg('Cannot pull a repository: Unknown application, "%s" file not found' % path)
return False
# load variables from shell into our current Python context
project_vars = self._load_project_vars(path)
writable_dirs = self._get_var(project_vars, path, 'WRITABLE_DIRS').replace('\\ ', '@SPACE@').split(' ')
project_root_path = self.get_apps_path(context) + '/www-data/' + self._get_var(
project_vars, path, 'GIT_PROJECT_DIR')
container_user = self._get_var(project_vars, path, 'DEFAULT_CONTAINER_USER')
for writable_dir in writable_dirs:
writable_dir = writable_dir.replace('@SPACE@', ' ')
command = 'sudo chown %s "%s/%s"' % (container_user, project_root_path, writable_dir)
self.io().info(command)
self.sh(command)
return True