def test_generate_git_tgz(kctx: kitipy.Context, keep: bool):
"""(Re) Generate tests/git-archive.tgz.
This command has been implemented and used to generate the original
tests/git-archive.tgz file, which is used to test git-related helper
functions.
"""
tempdir = kctx.executor.mkdtemp()
commands = ['cd ' + tempdir, 'git init']
commits = (
('foo', 'v0.1'),
('bar', 'v0.2'),
('baz', 'v0.3'),
('pi', 'v0.4'),
('yolo', 'v0.5'),
)
for commit, tag in commits:
commands.append('touch ' + commit)
commands.append('git add ' + commit)
commands.append('git commit -m "%s"' % (commit))
commands.append('git tag %s HEAD' % (tag))
basedir = os.path.dirname(os.path.abspath(__file__))
tgz_path = os.path.join(basedir, 'tests', 'tasks', 'testdata',
'git-repo.tgz')
commands.append('tar zcf %s .' % (tgz_path))
try:
kctx.run(' && '.join(commands))
finally:
if not keep:
kctx.run('rm -rf %s' % (tempdir))
def deploy(kctx: kitipy.Context, version: str):
"""Deploy a given version to ECS."""
client = kitipy.libs.aws.ecs.new_client()
stack = kctx.config["stacks"][kctx.stack.name]
cluster_name = kctx.stage["ecs_cluster_name"]
service_name = versioned_service_name(stack)
service_def = stack["ecs_service_definition"](kctx)
task_def = stack["ecs_task_definition"](kctx)
task_def["containerDefinitions"] = stack["ecs_container_transformer"](
kctx, version)
task_def_tags = task_def.get("tags", [])
task_def_tags.append({'key': 'kitipy.image_tag', 'value': version})
task_def["tags"] = task_def_tags
try:
deployment_id = kitipy.libs.aws.ecs.upsert_service(
client, cluster_name, service_name, task_def, service_def)
except kitipy.libs.aws.ecs.ServiceDefinitionChangedError as err:
kctx.fail(
"Could not deploy the API: ECS service definition has " +
"changed - {0}. You have to increment the version ".format(err) +
"number in the ./tasks.py file before re-running this command.\n")
for event in kitipy.libs.aws.ecs.watch_deployment(client, cluster_name,
service_name,
deployment_id):
createdAt = event["createdAt"].isoformat()
message = event["message"]
kctx.info("[{createdAt}] {message}".format(createdAt=createdAt,
message=message))
def run_playbook(kctx: Context,
inventory: str,
playbook: str,
hosts: Optional[Tuple[str]] = None,
tags: Optional[Tuple[str]] = None,
ask_become_pass: bool = False):
"""Run a given Ansible playbook using ``ansible-playbook``.
Args:
kctx (kitipy.Context):
Context to use to run the playbook.
inventory (str):
Path to Ansible host inventory.
playbook (str):
Path to the Ansible playbook to run.
hosts (Optional[Tuple[str]]):
List of targeted hosts. Use None to target all hosts (default value).
tags (Optional[Tuple[str]]):
List of targeted tags. Use None to apply all the tags (default value).
ask_become_pass (bool):
Whether ``--ask-become-pass`` should be added to the ``ansible-playbook``
command.
"""
cmd = 'ansible-playbook -i %s' % (inventory)
if hosts is not None and len(hosts) > 0:
cmd += ' -l ' + ','.join(hosts)
if tags is not None and len(tags) > 0:
cmd += ' -t ' + ','.join(tags)
if ask_become_pass:
cmd += ' --ask-become-pass'
cmd += ' ' + playbook
kctx.local(cmd)
def test_unit(kctx: kitipy.Context, report: bool, coverage: bool):
# Be sure the SSH container used for tests purpose is up and running.
# @TODO: add a common way to kitipy to wait for a port to be open
kctx.invoke(kitipy.docker.tasks.up)
expected_services = len(kctx.stack.config['services'])
# @TODO: this won't work as is with Swarm, find how to generalize that sort of tests
tester = lambda kctx: expected_services == kctx.stack.count_running_services(
)
kitipy.wait_for(tester,
interval=1,
max_checks=5,
label="Waiting for services start up...")
# Host key might change if docker-compose down is used between two test run,
# thus we start by removing any existing host key.
kctx.local("ssh-keygen -R '[127.0.0.1]:2022' 1>/dev/null 2>&1")
kctx.local("ssh-keygen -R '[127.0.0.1]:2023' 1>/dev/null 2>&1")
kctx.local("ssh-keygen -R testhost 1>/dev/null 2>&1")
# Ensure the private key has the right chmod or the task might fail.
os.chmod("tests/.ssh/id_rsa", 0o0600)
# Ensure first that we're actually able to connect to SSH hosts, or
# tests will fail anyway.
kctx.local('ssh -F tests/.ssh/config testhost /bin/true 1>/dev/null 2>&1')
kctx.local('ssh -F tests/.ssh/config jumphost /bin/true 1>/dev/null 2>&1')
kctx.local(
'ssh -F tests/.ssh/config testhost-via-jumphost /bin/true 1>/dev/null 2>&1'
)
report_name = 'unit.xml' if report else None
pytest(kctx, report_name, coverage, 'tests/unit/ -vv')
def pytest(kctx: kitipy.Context, report_name: Optional[str], coverage: bool,
cmd: str, **args):
env = os.environ.copy()
env['PYTHONPATH'] = os.getcwd()
args.setdefault('env', env)
basecmd = 'pytest'
if report_name:
if not kctx.path_exists('.test-results'):
os.mkdir('.test-results')
basecmd += ' --junitxml=.test-results/%s' % (report_name)
if coverage:
basecmd += ' --cov=kitipy/'
kctx.local('%s %s' % (basecmd, cmd), **args)
def ensure_tag_is_recent(kctx: kitipy.Context, tag: str, last: int = 5):
"""Check if the given Git tag is recent enough (by default, one of the
last five).
Args:
kctx (kitipy.Context): Kitipy Context.
tag (str): Tag to look for.
"""
res = kctx.local(
"git for-each-ref --format='%%(refname:strip=2)' --sort=committerdate 'refs/tags/*' 2>/dev/null | tail -n%d | grep %s >/dev/null 2>&1"
% (last, tag),
check=False,
)
if res.returncode != 0:
kctx.fail(
'This tag seems too old: at least %d new tags have been released since %s.'
% (last, tag))
def format(kctx: kitipy.Context, show_diff, fix):
"""Run yapf to detect style divergences and fix them."""
if not show_diff and not fix:
kctx.fail(
"You can't use both --no-diff and --no-fix at the same time.")
confirm_msg = 'Do you want to reformat your code using yapf?'
dry_run = lambda: kctx.local('yapf --diff -r kitipy/ tests/ tasks*.py',
check=False)
apply = lambda: kctx.local('yapf -vv -p -i -r kitipy/ tests/ tasks*.py')
kitipy.confirm_and_apply(dry_run,
confirm_msg,
apply,
show_dry_run=show_diff,
ask_confirm=fix is None,
should_apply=fix if fix is not None else True)
def show(kctx: kitipy.Context, show_values: bool):
"""Show secrets stored by AWS Secrets Manager."""
# @TODO: kctx.stack should be the raw config dict
stack = kctx.config['stacks'][kctx.stack.name]
secrets = stack['secrets_resolver'](kctx)
kctx.echo(("NOTE: Secret values end with %s. This is here to help you " +
"see invisible characters (e.g. whitespace, line breaks, " +
"etc...).\n") % (secret_delimiter))
client = sm.new_client()
for secret_arn in secrets:
secret = sm.describe_secret_with_current_value(client, secret_arn)
kctx.echo("=================================")
kctx.echo("ID: %s" % (secret["ARN"]))
kctx.echo("Name: %s" % (secret["Name"]))
kctx.echo("Value: %s\n" %
(format_secret_value(secret["SecretString"], show_values)))
def run(kctx: kitipy.Context, container: str, command: List[str],
version: Optional[str]):
"""Run a given command in a oneoff task."""
client = kitipy.libs.aws.ecs.new_client()
stack = kctx.config["stacks"][kctx.stack.name]
cluster_name = kctx.stage["ecs_cluster_name"]
service_name = versioned_service_name(stack)
task_def = stack["ecs_task_definition"](kctx)
if version is None:
regular_task_def = kitipy.libs.aws.ecs.get_task_definition(
client, task_def["family"])
version = next((tag["value"] for tag in regular_task_def["tags"]
if tag["key"] == "kitipy.image_tag"), None)
if version is None:
kctx.fail(
"No --version flag was provided and no deployments have been found."
)
task_name = "{0}-{1}".format(service_name, "-".join(command))
containers = stack["ecs_container_transformer"](kctx, version)
containers = stack["ecs_oneoff_container_transformer"](kctx, containers,
container)
run_args = stack["ecs_service_definition"](kctx)
run_args = {
k: v
for k, v in run_args.items()
if k not in ["desiredCount", "loadBalancers"]
}
task_def["family"] = task_def["family"] + "-oneoff"
task_def["containerDefinitions"] = list(containers)
task_arn = kitipy.libs.aws.ecs.run_oneoff_task(client, cluster_name,
task_name, task_def,
container, command,
run_args)
task = kitipy.libs.aws.ecs.wait_until_task_stops(client, cluster_name,
task_arn)
show_task(kctx, task)
def galaxy_install(kctx: Context,
dest: str,
file: str = 'galaxy-requirements.yml',
force: bool = False):
"""Install Ansible dependencies using ``ansible-galaxy install``.
Args:
kctx (kitipy.Context):
Context to use to run ``ansible-galaxy``.
dest (str):
Directory path where dependencies should be installed.
file (str):
Requirements file (defaults to ``galaxy-requirements.yml``).
force (bool):
Whehter ``--force`` flag should be added to ``ansible-galaxy`` command.
"""
kctx.run('ansible-galaxy install -r %s -p %s %s' % (
file,
dest,
'--force' if force else '',
))
def ensure_tag_exists(kctx: kitipy.Context, tag: str):
"""Check if the given Git tag exists on both local copy and remote origin.
This is mostly useful to ensure no invalid tag is going to be deployed.
Args:
kctx (kitipy.Context): Kitipy context.
tag (str): Git tag to verify.
Raises:
ValueError: If the given Git tag does not exist either on local or
remote origin.
"""
res = kctx.local(
'git ls-remote --exit-code --tags origin refs/tags/%s >/dev/null 2>&1'
% (tag),
check=False)
if res.returncode != 0:
kctx.fail("The given tag is not available on Git remote origin.")
res = kctx.local(
'git ls-remote --exit-code --tags ./. refs/tags/%s >/dev/null 2>&1' %
(tag),
check=False)
if res.returncode != 0:
kctx.fail(
"The given tag is not available in your local Git repo. Please fetch remote tags before running this task again."
)
def validate_tag(kctx: kitipy.Context, image_ref: str):
"""Check if the given image reference exists on a remote Docker registry.
Args:
kctx (kitipy.Context): The current kitipy Context.
image_ref (str):
A full image reference composed of: the repository to check, the
base image name and the image tag.
Raises:
click.Exception: When the given image tag doesn't exist.
"""
if len(image_ref) == 0:
kctx.fail(
"No image tag provided. You can provide it through --tag flag or IMAGE_TAG env var."
)
images = (service['image'] for service in kctx.stack.config['services'])
for image in images:
result = actions.buildx_imagetools_inspect(image_ref, _check=False)
if result.returncode != 0:
kctx.fail('Image %s not found on remote registry.' % (image_ref))
def show_failed_containers(kctx: kitipy.Context,
task: mypy_boto3_ecs.type_defs.TaskTypeDef):
containers = list(filter(lambda c: c["exitCode"] > 0, task["containers"]))
if len(containers) == 0:
kctx.echo("Containers with nonzero exit code: (None)")
return
kctx.echo("Containers with nonzero exit code:")
for container in task["containers"]:
if container["exitCode"] == 0:
continue
reason = "exit code: {0}".format(container["exitCode"]) + (
" - " + container['reason'] if 'reason' in container else '')
kctx.echo(" * {name}: {reason}".format(name=container["name"],
reason=reason))
def docker_tasks(kctx: kitipy.Context):
if not isinstance(kctx.stack, stack.BaseStack):
kctx.fail("No valid Docker stack available in kipity Context.")
def show_task(kctx: kitipy.Context,
task: mypy_boto3_ecs.type_defs.TaskTypeDef,
image_tag: Optional[str] = None):
kctx.echo("=================================")
kctx.echo("Task ID: {0}".format(task_id_from_arn(task["taskArn"])))
kctx.echo("Task definition: {0}".format(
task_def_from_arn(task["taskDefinitionArn"])))
if image_tag:
kctx.echo("Image tag: {0}".format(image_tag))
kctx.echo("CPU / Memory: {0} / {1}".format(task["cpu"], task["memory"]))
kctx.echo("Last status / Desired status: {0} / {1}".format(
task["lastStatus"], task["desiredStatus"]))
if task["lastStatus"] == "RUNNING":
kctx.echo("Started at: {0}".format(task["startedAt"].isoformat()))
if task["lastStatus"] == "STOPPED":
kctx.echo("Stopped at: {0}".format(task["stoppedAt"].isoformat()))
kctx.echo("Reason: {0}".format(task["stoppedReason"]))
show_failed_containers(kctx, task)
kctx.echo("") # Put an empty line between each task
def lint(kctx: kitipy.Context):
"""Run mypy, a static type checker, to detect type errors."""
kctx.local('mypy -p kitipy')
def edit(kctx: kitipy.Context, secret_name):
"""Edit secrets stored by AWS Secrets Manager."""
stack = kctx.config['stacks'][kctx.stack.name]
secret_arn = stack['secret_arn_resolver'](kctx=kctx,
secret_name=secret_name)
client = sm.new_client()
secret = sm.describe_secret_with_current_value(client, secret_arn)
if secret == None:
kctx.fail("Secret \"%s\" not found." % (secret_name))
value = click.edit(text=secret['SecretString'])
if value == None:
kctx.info("Secret value was not changed. Aborting.")
raise click.exceptions.Abort()
trim_question = ("Your secret value ends with a new line. This is " +
"generally abnormal. Would you want to trim it " +
"automatically?")
if value.endswith("\n") and click.confirm(trim_question, default=True):
value = value.rstrip("\n")
kctx.echo(("NOTE: Secret values end with %s. This is here to help you " +
"see invisible characters (e.g. whitespace, line breaks, " +
"etc...).\n") % (secret_delimiter))
kctx.echo("ID: %s" % (secret["ARN"]))
kctx.echo("Name: %s" % (secret["Name"]))
kctx.echo("Previous value: %s" %
(format_secret_value(secret["SecretString"], True)))
kctx.echo("New value: %s" % (format_secret_value(value, True)))
click.confirm("\nDo you confirm this change?", abort=True)
sm.put_secret_value(client, secret["ARN"], value)
def test_all(kctx: kitipy.Context, report: bool, coverage: bool):
"""Execute all the tests suites."""
kctx.invoke(test_unit)
kctx.invoke(test_tasks)