def run(self, context: ExecutionContext) -> bool:
service_name = context.get_arg('name')
strategy = str(context.get_arg('--strategy'))
remove_previous_images = bool(context.get_arg('--remove-previous-images'))
service = self.services(context).get_by_name(service_name)
strategies = {
'rolling': lambda: self.deploy_rolling(service, context),
'compose': lambda: self.deploy_compose_like(service, context),
'recreate': lambda: self.deploy_recreate(service, context)
}
with self.hooks_executed(context, 'service-start-%s' % service_name):
with self._old_images_clean_up(context, service, clear_images=remove_previous_images):
if strategy in strategies:
return strategies[strategy]()
else:
if strategy == 'auto':
strategy = service.get_update_strategy(default='compose')
if strategy in strategies:
return strategies[strategy]()
self.io().error_msg('Invalid strategy selected: %s' % strategy)
return False
def run(self, context: ExecutionContext) -> bool:
"""Validate parameters and select action"""
is_global = context.get_arg('--global')
domain = context.get_arg('--domain')
service = context.get_arg('--service')
directory = self.get_data_path(context) + '/maintenance-mode'
if not self._validate_switches(is_global, domain, service):
self.io().error_msg(
'Cannot use together --global, --domain and --service switch. Pick one of them.'
)
return False
try:
if is_global:
return self.act([directory + '/on'], 'globally')
elif service:
return self.act_for_service(directory, service, context)
elif domain:
return self.act_for_domain(directory, domain, context)
else:
self.io().error_msg('Must specify --global or --domain switch')
return False
except PermissionError as e:
self.io().error_msg(
'No write permissions. Set permissions or use sudo? %s' %
str(e))
return False
def run(self, context: ExecutionContext) -> bool:
services = self.get_matching_services(context)
strategy = context.get_arg('--strategy')
result = True
with self.hooks_executed(context, 'start'):
for service in services:
self.io().h2(
'Starting "%s" (%i instances)...' %
(service.get_name(), service.get_desired_replicas_count()))
try:
self.rkd(
[
'--no-ui', ':harbor:service:up',
service.get_name(), '--remove-previous-images'
if context.get_arg('--remove-previous-images') else
'',
('--strategy=%s' % strategy) if strategy else ''
],
capture=not self.io().is_log_level_at_least('info'))
self.io().success_msg('Service "%s" was started' %
service.get_name())
except CalledProcessError as e:
self.io().err(str(e))
self.io().error_msg('Cannot start service "%s"' %
service.get_name())
result = False
self.io().print_opt_line()
return result
def build_image(ctx: ExecutionContext, this: TaskInterface) -> bool:
""" Builds a docker image for given architecture, tags properly and push """
tag = ctx.get_arg('--git-tag')
should_push = ctx.get_arg('--push')
arch = ctx.get_arg('--arch')
docker_tag = 'latest-dev-' + arch
if tag:
docker_tag = arch + '-' + tag
image_name = '{repo}:{tag}'.format(tag=docker_tag, repo=REPO)
this.sh(
'docker build . -f ./.infrastructure/{docker_arch}.Dockerfile -t {image}'
.format(docker_arch=arch, image=image_name))
this.sh('docker tag {image} {image}-$(date "+%Y-%m-%d")'.format(
image=image_name))
if should_push:
this.sh('docker push {image}'.format(image=image_name))
this.sh(
'docker push {image}-$(date "+%Y-%m-%d")'.format(image=image_name))
return True
def run(self, context: ExecutionContext) -> bool:
vault_opts = self._get_vault_opts(context)
filename = context.get_arg('filename')
mode = 'decrypt' if context.get_arg('--decrypt') else 'encrypt'
self.sh('ansible-vault %s %s %s' % (mode, vault_opts, filename),
capture=False)
return True
def deploy_compose_like(self, service: ServiceDeclaration, ctx: ExecutionContext) -> bool:
"""Regular docker-compose up deployment (with downtime)"""
self.io().info('Performing "compose" deployment for "%s"' % service.get_name())
self.containers(ctx).up(service,
norecreate=bool(ctx.get_arg('--dont-recreate')),
extra_args=ctx.get_arg('--extra-args'))
return True
def run(self, ctx: ExecutionContext) -> bool:
service_name = ctx.get_arg('name')
timeout = int(ctx.get_arg('--timeout'))
instance_num = int(ctx.get_arg('--instance')) if ctx.get_arg('--instance') else None
service = self.services(ctx).get_by_name(service_name)
try:
container_name = self.containers(ctx).find_container_name(service, instance_num)
container = self.containers(ctx).inspect_container(container_name)
except ServiceNotCreatedException as e:
self.io().error_msg(str(e))
return False
started_at = time()
self.io().info('Checking health of "%s" service - %s' % (service_name, container_name))
if container.has_health_check():
while True:
if time() - started_at >= timeout:
self.io().error_msg('Timeout of %is reached.' % timeout)
return False
health = container.get_health_status()
# do not wait for result, do check manually in mean time
if health == 'starting':
self.io().warn('Docker reports "starting" - performing a manual check, we wont wait for docker')
try:
command = container.get_health_check_command().replace('"', '\\"')
self.containers(ctx).exec_in_container(
service_name=service_name,
command='/bin/sh -c "%s"' % command,
instance_num=instance_num
)
self.io().success_msg('Service healthy after %is' % (time() - started_at))
return True
except subprocess.CalledProcessError as e:
self.io().debug(str(e.output)[0:128])
sleep(1)
continue
if health == 'healthy' or health == 'running':
self.io().success_msg('Service healthy after %i' % (time() - started_at))
return True
sleep(1)
else:
self.io().warn('Instance has no healthcheck defined!')
return True
def prepare_tuple_for_single_container(self, ctx: ExecutionContext) -> tuple:
service_name = ctx.get_arg('--name')
service = self.services(ctx).get_by_name(service_name)
instance_num = int(ctx.get_arg('--instance-num')) if ctx.get_arg('--instance-num') else None
container_name = self.containers(ctx).find_container_name(service, instance_num)
if not container_name:
self.io().error_msg('Container not found')
return None, None, None
return container_name, service, instance_num
def run(self, ctx: ExecutionContext) -> bool:
params = []
if ctx.get_arg('--quiet'):
params.append('--quiet')
if ctx.get_arg('--all'):
params.append('--all')
self.containers(ctx).ps(params)
return True
def _preserve_vault_parameters_for_usage_in_inner_tasks(
self, ctx: ExecutionContext):
"""Preserve original parameters related to Vault, so those parameters can be propagated to inner tasks"""
try:
vault_passwords = ctx.get_arg_or_env('--vault-passwords')
except MissingInputException:
vault_passwords = ''
# keep the vault arguments for decryption of deployment.yml
self.vault_args = ['--vault-passwords=' + vault_passwords]
if ctx.get_arg('--ask-vault-pass'):
self.vault_args.append('--ask-vault-pass')
def execute(self, context: ExecutionContext) -> bool:
opts = ''
if context.args['skip_existing']:
opts += ' --skip-existing '
if context.get_arg('--url'):
if context.get_arg('--test'):
raise Exception('Cannot use --url and --test switch at once')
opts += ' --repository-url=%s' % context.get_arg('--url')
if context.get_arg('--test'):
opts += ' --repository-url https://test.pypi.org/legacy/ '
self.sh('''
%s upload \\
--disable-progress-bar \\
--verbose \\
--username=%s \\
--password=%s \\
%s %s
''' % (context.get_env('TWINE_PATH'), context.get_arg('--username'),
context.get_arg('--password'), opts, context.get_arg('--src')))
return True
def run(self, ctx: ExecutionContext) -> bool:
service_name = ctx.get_arg('name')
service = self.services(ctx).get_by_name(service_name)
with_image = ctx.get_arg('--with-image')
images = self.get_all_images_for_service(ctx, service) if with_image else []
self.containers(ctx).rm(service, extra_args=ctx.get_arg('--extra-args'))
for image in images:
try:
self.containers(ctx).rm_image(image, capture=True)
except:
pass
return True
def execute(self, ctx: ExecutionContext) -> bool:
name = ctx.get_arg('name')
category_name, pkg_name = self.extract_category_and_pkg_names(name)
path = self.find_snippet_path(pkg_name, category_name)
if not path:
self.io().error(
'Snippet not found in any synchronized repository. ' +
'Did you forget to do :cooperative:sync?')
return False
self.io().info('Installing snippet from "%s"' % path)
# mock rkd_path, so the snippet can override the tasks
rkd_path = os.getenv('RKD_PATH', '')
snippet_rkd_path = os.path.realpath('./' + path + '/.rkd')
if snippet_rkd_path:
os.putenv('RKD_PATH',
(rkd_path + ':' + snippet_rkd_path).strip(':'))
try:
subprocess.check_call(['rkd', ':snippet:wizard', path])
subprocess.check_call(['rkd', ':snippet:install', path])
finally:
if os.path.isfile('.rkd/tmp-wizard.json'):
os.unlink('.rkd/tmp-wizard.json')
os.putenv('RKD_PATH', rkd_path)
return True
def _ask_and_set_var(self, ctx: ExecutionContext, arg_name: str,
title: str, attribute: str, secret: bool):
"""Ask user an interactive question, then add answer to the deployment.yml loaded in memory
The variable will be appended to any node, where the variable is empty.
Example: We have 5 servers, 3 without a password. So the password will be applied to 3 servers.
"""
self.get_config()
if not ctx.get_arg(arg_name):
return
wizard = Wizard(self).ask(title, attribute=attribute, secret=secret)
for group_name, nodes in self._config['nodes'].items():
node_num = 0
for node in nodes:
node_num += 1
if attribute in self._config['nodes'][group_name][node_num -
1]:
continue
self._config['nodes'][group_name][
node_num - 1][attribute] = wizard.answers[attribute]
def test_functional_hooks_are_executed_when_exists_and_files_with_extension_only_are_skipped(
self):
"""Given we have an example hooks in pre-upgrade/whoami.sh and in post-upgrade/history.sh
And we try to run those hooks using hooks_executed()
Then we will see output produced by those scripts
And .dotfiles will be ignored
"""
self._prepare_test_data()
buffer = StringIO()
hooks_capturing_io = IO()
task = TestTask()
task._io = BufferedSystemIO()
ctx = ExecutionContext(TaskDeclaration(task), args={}, env={})
with hooks_capturing_io.capture_descriptors(stream=buffer,
enable_standard_out=True):
with task.hooks_executed(ctx, 'upgrade'):
pass
self.assertIn('>> This is a whoami.sh hook, test:',
buffer.getvalue(),
msg='Expected pre-upgrade hook to be ran')
self.assertIn('25 June 1978 the rainbow flag was first flown',
buffer.getvalue(),
msg='Expected post-upgrade hook to be ran')
self.assertIn('pre-upgrade/whoami.sh', task._io.get_value())
self.assertNotIn('.gitkeep', task._io.get_value())
def test_one_failed_step_is_preventing_next_steps_from_execution_and_result_is_marked_as_failure(self):
"""Check the correctness of error handling"""
io = IO()
str_io = StringIO()
buffered = BufferedSystemIO()
task_declaration = get_test_declaration()
BasicTestingCase.satisfy_task_dependencies(task_declaration.get_task_to_execute(), io=buffered)
ctx = ExecutionContext(task_declaration)
executor = DeclarativeExecutor()
executor.add_step('python', 'this.io().outln("Peter Kropotkin"); return True', task_name=':first', rkd_path='', envs={})
executor.add_step('bash', 'echo "Buenaventura Durruti"; exit 1', task_name=':second', rkd_path='', envs={})
executor.add_step('python', 'this.io().outln("This one will not show"); return True', task_name=':third', rkd_path='', envs={})
with io.capture_descriptors(target_files=[], stream=str_io, enable_standard_out=False):
final_result = executor.execute_steps_one_by_one(ctx, task_declaration.get_task_to_execute())
output = str_io.getvalue() + buffered.get_value()
self.assertIn('Peter Kropotkin', output)
self.assertIn('Buenaventura Durruti', output)
self.assertNotIn('This one will not show', output)
self.assertEqual(False, final_result)
def mock_execution_context(
task: TaskInterface,
args: Dict[str, str] = None,
env: Dict[str, str] = None,
defined_args: Dict[str, dict] = None) -> ExecutionContext:
"""
Prepares a simplified rkd.api.contract.ExecutionContext instance
:param task:
:param args:
:param env:
:param defined_args:
:return:
"""
if args is None:
args = {}
if env is None:
env = {}
if defined_args is None:
defined_args = {}
if args and not defined_args:
for name, passed_value in args.items():
defined_args[name] = {'default': ''}
return ExecutionContext(TaskDeclaration(task),
parent=None,
args=args,
env=env,
defined_args=defined_args)
def run(self, context: ExecutionContext) -> bool:
path = self.get_app_repository_path(context.get_arg('name'), context)
if not os.path.isfile(path):
self.io().error_msg('Cannot pull a repository: Unknown application, "%s" file not found' % path)
return False
# load variables from shell into our current Python context
project_vars = self._load_project_vars(path)
project_root_path = self.get_apps_path(context) + '/www-data/' + self._get_var(
project_vars, path, 'GIT_PROJECT_DIR')
# 1) run pre_update hook
self.contextual_sh(path, 'pre_update %s' % project_root_path)
# 2a) clone fresh repository
if not os.path.isdir(project_root_path):
self.io().info('Cloning a fresh repository at "%s"' % project_root_path)
self._clone_new_repository(path, project_vars)
# 2b) update existing repository - pull changes
else:
self.io().info('Pulling in an existing repository at "%s"' % project_root_path)
self._pull_changes_into_existing_repository(path, project_vars)
# 3) run pre_update hook
self.contextual_sh(path, 'post_update %s' % project_root_path)
self.io().print_opt_line()
self.io().success_msg('Application\'s repository updated.')
return True
def run(self, context: ExecutionContext) -> bool:
profile = context.get_arg('--profile')
strategy = context.get_arg('--strategy')
success = True
with self.hooks_executed(context, 'upgrade'):
self.rkd([
'--no-ui', ':harbor:templates:render',
':harbor:git:apps:update-all', ':harbor:pull',
'--profile=%s' % profile, ':harbor:start',
'--profile=%s' % profile,
'--strategy=%s' % strategy, '--remove-previous-images'
if context.get_arg('--remove-previous-images') else '',
':harbor:gateway:reload'
])
return success
def run(self, context: ExecutionContext) -> bool:
vault_opts = self._get_vault_opts(context)
filename = context.get_arg('filename')
subprocess.check_call('ansible-vault edit %s %s' %
(vault_opts, filename),
shell=True)
return True
def run(self, context: ExecutionContext) -> bool:
cmd = context.get_arg('--cmd')
try:
subprocess.check_call('cd %s && vagrant %s' % (self.ansible_dir, cmd), shell=True)
except subprocess.CalledProcessError:
return False
return True
def _get_vault_opts(self, ctx: ExecutionContext, chdir: str = '') -> str:
"""Creates options to pass in Ansible Vault commandline
The output will be a temporary vault file with password entered inline or a --ask-vault-pass switch
"""
try:
vault_passwords = ctx.get_arg_or_env('--vault-passwords').split(
'||')
except MissingInputException:
vault_passwords = []
num = 0
opts = ''
enforce_ask_pass = ctx.get_arg('--ask-vault-pass')
for passwd in vault_passwords:
num = num + 1
if not passwd:
continue
if passwd.startswith('./') or passwd.startswith('/'):
if os.path.isfile(passwd):
opts += ' --vault-password-file="%s" ' % (chdir + passwd)
else:
self.io().error(
'Vault password file "%s" does not exist, calling --ask-vault-pass'
% passwd)
enforce_ask_pass = True
else:
tmp_vault_file = self.temp.assign_temporary_file(mode=0o644)
with open(tmp_vault_file, 'w') as f:
f.write(passwd)
opts += ' --vault-password-file="%s" ' % (chdir +
tmp_vault_file)
if enforce_ask_pass:
opts += ' --ask-vault-pass '
return opts
def run(self, ctx: ExecutionContext) -> bool:
command = ctx.get_arg('--command')
shell = ctx.get_arg('--shell')
tty = bool(ctx.get_arg('--no-tty'))
interactive = bool(ctx.get_arg('--no-interactive'))
container_name, service, instance_num = self.prepare_tuple_for_single_container(ctx)
if not service:
return False
if shell != '/bin/sh' and command == '/bin/sh':
command = shell
self.containers(ctx).exec_in_container_passthrough(
command, service, instance_num, shell=shell, tty=tty, interactive=interactive
)
return True
def execute_mocked_task_and_get_output(self,
task: TaskInterface,
args=None,
env=None) -> str:
"""
Run a single task, capturing it's output in a simplified way.
There is no whole RKD bootstrapped in this operation.
:param TaskInterface task:
:param dict args:
:param dict env:
:return:
"""
if args is None:
args = {}
if env is None:
env = {}
ctx = ApplicationContext([], [], '')
ctx.io = BufferedSystemIO()
task.internal_inject_dependencies(
io=ctx.io,
ctx=ctx,
executor=OneByOneTaskExecutor(ctx=ctx),
temp_manager=TempManager())
merged_env = deepcopy(os.environ)
merged_env.update(env)
r_io = IO()
str_io = StringIO()
defined_args = {}
for arg, arg_value in args.items():
defined_args[arg] = {'default': ''}
with r_io.capture_descriptors(enable_standard_out=True, stream=str_io):
try:
# noinspection PyTypeChecker
result = task.execute(
ExecutionContext(TaskDeclaration(task),
args=args,
env=merged_env,
defined_args=defined_args))
except Exception:
self._restore_standard_out()
print(ctx.io.get_value() + "\n" + str_io.getvalue())
raise
return ctx.io.get_value() + "\n" + str_io.getvalue(
) + "\nTASK_EXIT_RESULT=" + str(result)
def _get_prepared_compose_driver(self,
args: dict = {},
env: dict = {}) -> ComposeDriver:
merged_env = deepcopy(os.environ)
merged_env.update(env)
task = self.satisfy_task_dependencies(TestTask(), BufferedSystemIO())
declaration = TaskDeclaration(task)
ctx = ExecutionContext(declaration, args=args, env=merged_env)
return ComposeDriver(task, ctx, TEST_PROJECT_NAME)
def execute(self, ctx: ExecutionContext) -> bool:
wizard = Wizard(self)
wizard.load_previously_stored_values()
os.environ.update(wizard.answers)
self.rkd([
':j2:directory-to-directory',
'--source="%s"' % ctx.get_arg('path') + '/files/', '--target="./"',
'--pattern="(.*)"', '--copy-not-matching-files',
'--template-filenames'
])
return True
def run(self, ctx: ExecutionContext) -> bool:
services = self.get_matching_services(ctx)
for service in services:
self.io().info('Removing "%s"' % service.get_name())
self.rkd([
':harbor:service:rm',
service.get_name(),
'--with-image' if ctx.get_arg('--with-image') else ''
])
return True
def test_non_existing_dir_is_skipped(self):
"""Assert that non-existing directory does not cause exception, but will be skipped"""
task = TestTask()
task._io = BufferedSystemIO()
task._io.set_log_level('debug')
ctx = ExecutionContext(TaskDeclaration(task), args={}, env={})
task.execute_hooks(ctx, 'non-existing-directory')
self.assertIn(
'Hooks dir "./hooks.d//non-existing-directory/" not present, skipping',
task._io.get_value())
def run(self, ctx: ExecutionContext) -> bool:
self.io().h2('Validating NGINX configuration')
self.containers(ctx).exec_in_container('gateway', 'nginx -t')
self.io().h2('Reloading NGINX configuration')
self.containers(ctx).exec_in_container('gateway', 'nginx -s reload')
if ctx.get_env('DISABLE_SSL').lower() != 'true':
self.io().h2('Reloading SSL configuration')
self.make_sure_ssl_service_is_up(ctx)
self.containers(ctx).exec_in_container('gateway_letsencrypt', '/app/signal_le_service')
return True
def run(self, context: ExecutionContext) -> bool:
self._preserve_vault_parameters_for_usage_in_inner_tasks(context)
node_group = context.get_arg('--group')
node_num = int(context.get_arg('--num')) - 1
should_print_password = context.get_arg('--print-password')
try:
config = self.get_config()
node = self._get_node(node_group, node_num, config)
if not node:
return False
if should_print_password:
self._print_password(node)
return self._ssh(node)
except MissingDeploymentConfigurationError as e:
self.io().error_msg(str(e))
return False
