def gitlog():
ecr = ECR()
staging_deployment = Deployment('staging')
staging_tag = staging_deployment.current_tag
if staging_tag is None:
raise HokusaiError("Could not find a tag for staging. Aborting.")
staging_tag = ecr.find_git_sha1_image_tag(staging_tag)
if staging_tag is None:
raise HokusaiError(
"Could not find a git SHA1 tag for tag %s. Aborting." %
staging_tag)
production_deployment = Deployment('production')
production_tag = production_deployment.current_tag
if production_tag is None:
raise HokusaiError("Could not find a tag for production. Aborting.")
production_tag = ecr.find_git_sha1_image_tag(production_tag)
if production_tag is None:
raise HokusaiError("Could not find a git SHA1 for tag %s. Aborting." %
production_tag)
print_green("Comparing %s to %s" % (production_tag, staging_tag))
shout("git log --right-only %s..%s" % (production_tag, staging_tag),
print_output=True)
def refresh(self):
deployment_timestamp = datetime.datetime.utcnow().strftime("%s%f")
for deployment in self.cache:
patch = {
"spec": {
"template": {
"metadata": {
"labels": {
"deploymentTimestamp": deployment_timestamp
}
}
}
}
}
print_green("Refreshing %s..." % deployment['metadata']['name'])
shout(
self.kctl.command(
"patch deployment %s -p '%s'" %
(deployment['metadata']['name'], json.dumps(patch))))
print_green("Waiting for refresh to complete...")
rollout_commands = [
self.kctl.command("rollout status deployment/%s" %
deployment['metadata']['name'])
for deployment in self.cache
]
return_code = shout_concurrent(rollout_commands)
if return_code:
raise HokusaiError("Refresh failed!", return_code=return_code)
def k8s_create(context, tag='latest', namespace=None, filename=None):
if filename is None:
kubernetes_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR,
"%s.yml" % context)
else:
kubernetes_yml = filename
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
ecr = ECR()
if not ecr.project_repo_exists():
raise HokusaiError(
"ECR repository %s does not exist... did you run `hokusai setup` for this project?"
% config.project_name)
if not ecr.tag_exists(tag):
raise HokusaiError(
"Image tag %s does not exist... did you run `hokusai registry push`?"
% tag)
if tag is 'latest' and not ecr.tag_exists(context):
ecr.retag(tag, context)
print_green("Updated tag 'latest' -> %s" % context)
if filename is None:
configmap = ConfigMap(context, namespace=namespace)
configmap.create()
print_green("Created configmap %s-environment" % config.project_name)
kctl = Kubectl(context, namespace=namespace)
shout(kctl.command("create --save-config -f %s" % kubernetes_yml),
print_output=True)
print_green("Created Kubernetes environment %s" % kubernetes_yml)
def gitdiff():
ecr = ECR()
staging_deployment = Deployment('staging')
staging_tag = staging_deployment.current_tag
if staging_tag is None:
raise HokusaiError("Could not find a tag for staging. Aborting.")
staging_tag = ecr.find_git_sha1_image_tag(staging_tag)
if staging_tag is None:
raise HokusaiError(
"Could not find a git SHA1 tag for tag %s. Aborting." %
staging_tag)
production_deployment = Deployment('production')
production_tag = production_deployment.current_tag
if production_tag is None:
raise HokusaiError("Could not find a tag for production. Aborting.")
production_tag = ecr.find_git_sha1_image_tag(production_tag)
if production_tag is None:
raise HokusaiError("Could not find a git SHA1 for tag %s. Aborting." %
production_tag)
print_green("Comparing %s to %s" % (production_tag, staging_tag))
for remote in shout('git remote').splitlines():
shout("git fetch %s" % remote)
shout("git diff %s %s" % (production_tag, staging_tag), print_output=True)
def k8s_create(context, tag='latest', namespace=None, filename=None, environment=()):
if filename is None:
yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, context))
else:
yaml_template = TemplateSelector().get(filename)
ecr = ECR()
if not ecr.project_repo_exists():
raise HokusaiError("ECR repository %s does not exist... did you run `hokusai setup` for this project?" % config.project_name)
if not ecr.tag_exists(tag):
raise HokusaiError("Image tag %s does not exist... did you run `hokusai registry push`?" % tag)
if tag == 'latest' and not ecr.tag_exists(context):
ecr.retag(tag, context)
print_green("Updated tag 'latest' -> %s" % context)
if filename is None:
configmap = ConfigMap(context, namespace=namespace)
for s in environment:
if '=' not in s:
raise HokusaiError("Error: environment variables must be of the form 'KEY=VALUE'")
split = s.split('=', 1)
configmap.update(split[0], split[1])
configmap.create()
print_green("Created configmap %s-environment" % config.project_name)
kctl = Kubectl(context, namespace=namespace)
yaml_spec = YamlSpec(yaml_template).to_file()
shout(kctl.command("create --save-config -f %s" % yaml_spec), print_output=True)
print_green("Created Kubernetes environment %s" % yaml_template)
def k8s_create(context, tag='latest', namespace=None, yaml_file_name=None):
if yaml_file_name is None: yaml_file_name = context
kubernetes_yml = os.path.join(os.getcwd(),
"hokusai/%s.yml" % yaml_file_name)
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
ecr = ECR()
if not ecr.project_repo_exists():
raise HokusaiError(
"ECR repository %s does not exist... did you run `hokusai setup` for this project?"
% config.project_name)
if not ecr.tag_exists(tag):
raise HokusaiError(
"Image tag %s does not exist... did you run `hokusai push`?" % tag)
if tag is 'latest' and not ecr.tag_exists(context):
ecr.retag(tag, context)
print_green("Updated tag 'latest' -> %s" % context)
kctl = Kubectl(context, namespace=namespace)
shout(kctl.command("create --save-config -f %s" % kubernetes_yml),
print_output=True)
print_green("Created Kubernetes environment %s" % kubernetes_yml)
def build(self, filename):
if filename is None:
docker_compose_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR,
BUILD_YAML_FILE)
legacy_docker_compose_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR,
LEGACY_BUILD_YAML_FILE)
if not os.path.isfile(docker_compose_yml) and not os.path.isfile(
legacy_docker_compose_yml):
raise HokusaiError(
"Yaml files %s / %s do not exist." %
(docker_compose_yml, legacy_docker_compose_yml))
if os.path.isfile(docker_compose_yml):
build_command = "docker-compose -f %s -p hokusai build" % docker_compose_yml
if os.path.isfile(legacy_docker_compose_yml):
build_command = "docker-compose -f %s -p hokusai build" % legacy_docker_compose_yml
else:
build_command = "docker-compose -f %s -p hokusai build" % filename
if config.pre_build:
build_command = "%s && %s" % (config.pre_build, build_command)
if config.post_build:
build_command = "%s && %s" % (build_command, config.post_build)
shout(build_command, print_output=True)
def dev_start(build, detach):
docker_compose_yml = os.path.join(os.getcwd(), 'hokusai/development.yml')
if not os.path.isfile(docker_compose_yml):
raise HokusaiError("Yaml file %s does not exist." % docker_compose_yml)
def cleanup(*args):
shout("docker-compose -f %s -p hokusai stop" % docker_compose_yml,
print_output=True)
for sig in EXIT_SIGNALS:
signal.signal(sig, cleanup)
opts = ''
if build:
opts += ' --build'
if detach:
opts += ' -d'
if not detach:
print_green(
"Starting development environment... Press Ctrl+C to stop.")
shout("docker-compose -f %s -p hokusai up%s" % (docker_compose_yml, opts),
print_output=True)
if detach:
print_green(
"Run `hokousai dev stop` to shut down, or `hokusai dev logs --follow` to tail output."
)
def gitlog():
ecr = ECR()
staging_deployment = Deployment('staging')
staging_tag = staging_deployment.current_tag
if staging_tag is None:
raise HokusaiError("Could not find a tag 'staging'. Aborting.")
if staging_tag == 'staging':
staging_tag = ecr.find_git_sha1_image_tag('staging')
if staging_tag is None:
print_red(
"Could not find a git SHA1 tag for 'staging'. Aborting.")
return -1
production_deployment = Deployment('production')
production_tag = production_deployment.current_tag
if production_tag is None:
raise HokusaiError("Could not find a tag 'production'. Aborting.")
if production_tag == 'production':
production_tag = ecr.find_git_sha1_image_tag('production')
if production_tag is None:
print_red(
"Could not find a git SHA1 tag for 'production'. Aborting.")
return -1
print_green("Comparing %s to %s" % (production_tag, staging_tag))
shout("git log --right-only %s..%s" % (production_tag, staging_tag),
print_output=True)
def dev_start(build, detach, filename):
if filename is None:
docker_compose_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR,
DEVELOPMENT_YML_FILE)
else:
docker_compose_yml = filename
if not os.path.isfile(docker_compose_yml):
raise HokusaiError("Yaml file %s does not exist." % docker_compose_yml)
def cleanup(*args):
shout("docker-compose -f %s -p hokusai stop" % docker_compose_yml,
print_output=True)
for sig in EXIT_SIGNALS:
signal.signal(sig, cleanup)
opts = ''
if build:
Docker().build(filename)
if detach:
opts += ' -d'
if not detach:
print_green(
"Starting development environment... Press Ctrl+C to stop.")
shout("docker-compose -f %s -p hokusai up%s" % (docker_compose_yml, opts),
print_output=True)
if detach:
print_green(
"Run `hokousai dev stop` to shut down, or `hokusai dev logs --follow` to tail output."
)
def dev_status():
docker_compose_yml = os.path.join(os.getcwd(), 'hokusai/development.yml')
if not os.path.isfile(docker_compose_yml):
raise HokusaiError("Yaml file %s does not exist." % docker_compose_yml)
shout("docker-compose -f %s -p hokusai ps" % docker_compose_yml,
print_output=True)
def test(build, cleanup):
docker_compose_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR, TEST_YML_FILE)
if not os.path.isfile(docker_compose_yml):
raise HokusaiError("Yaml file %s does not exist." % docker_compose_yml)
def on_cleanup(*args):
shout("docker-compose -f %s -p hokusai stop" % docker_compose_yml)
shout("docker-compose -f %s -p hokusai rm --force" % docker_compose_yml)
if cleanup:
for sig in EXIT_SIGNALS:
signal.signal(sig, on_cleanup)
opts = ' --abort-on-container-exit'
if build:
Docker().build()
print_green("Starting test environment... Press Ctrl+C to stop.", newline_after=True)
try:
shout("docker-compose -f %s -p hokusai up%s" % (docker_compose_yml, opts), print_output=True)
return_code = int(shout("docker wait hokusai_%s_1" % config.project_name))
except CalledProcessError:
if cleanup: on_cleanup()
raise HokusaiError('Tests Failed')
if return_code:
raise HokusaiError('Tests Failed - Exit Code: %s\n' % return_code, return_code=return_code)
else:
print_green("Tests Passed")
if cleanup: on_cleanup()
return return_code
def dev_start(build, detach, filename):
if filename is None:
yaml_template = TemplateSelector().get(
os.path.join(CWD, HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE))
else:
yaml_template = TemplateSelector().get(filename)
docker_compose_yml = YamlSpec(yaml_template).to_file()
follow_extends(docker_compose_yml)
def cleanup(*args):
shout("docker-compose -f %s -p hokusai stop" % docker_compose_yml,
print_output=True)
for sig in EXIT_SIGNALS:
signal.signal(sig, cleanup)
opts = ''
if build:
Docker().build(filename=yaml_template)
if detach:
opts += ' -d'
if not detach:
print_green(
"Starting development environment... Press Ctrl+C to stop.")
shout("docker-compose -f %s -p hokusai up%s" % (docker_compose_yml, opts),
print_output=True)
if detach:
print_green(
"Run `hokousai dev stop` to shut down, or `hokusai dev logs --follow` to tail output."
)
def run(self, image_tag, cmd, tty=False, env=(), constraint=()):
if os.environ.get('USER') is not None:
uuid = "%s-%s" % (os.environ.get('USER'), k8s_uuid())
else:
uuid = k8s_uuid()
name = "%s-hokusai-run-%s" % (config.project_name, uuid)
image_name = "%s:%s" % (config.aws_ecr_registry, image_tag)
container = {
"args":
cmd.split(' '),
"name":
name,
"image":
image_name,
"imagePullPolicy":
"Always",
'envFrom': [{
'configMapRef': {
'name': "%s-environment" % config.project_name
}
}]
}
if tty:
container.update({"stdin": True, "stdinOnce": True, "tty": True})
if env:
container['env'] = []
for s in env:
if '=' not in s:
raise HokusaiError(
"Error: environment variables must be of the form 'KEY=VALUE'"
)
split = s.split('=', 1)
container['env'].append({'name': split[0], 'value': split[1]})
spec = {"containers": [container]}
if constraint:
spec['nodeSelector'] = {}
for label in constraint:
if '=' not in label:
raise HokusaiError(
"Error: Node selectors must of the form 'key=value'")
split = label.split('=', 1)
spec['nodeSelector'][split[0]] = split[1]
overrides = {"apiVersion": "v1", "spec": spec}
if tty:
shout(self.kctl.command(
"run %s -t -i --image=%s --restart=Never --overrides='%s' --rm"
% (name, image_name, json.dumps(overrides))),
print_output=True)
else:
return returncode(
self.kctl.command(
"run %s --attach --image=%s --overrides='%s' --restart=Never --rm"
% (name, image_name, json.dumps(overrides))))
def dev_run(command):
docker_compose_yml = os.path.join(os.getcwd(), 'hokusai/development.yml')
if not os.path.isfile(docker_compose_yml):
raise HokusaiError("Yaml file %s does not exist." % docker_compose_yml)
shout("docker-compose -f %s -p hokusai exec %s %s" %
(docker_compose_yml, config.project_name, command),
print_output=True)
def environment_delete(context):
kubernetes_yml = os.path.join(os.getcwd(), "hokusai/%s.yml" % context)
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist for given context." %
kubernetes_yml)
kctl = Kubectl(context)
shout(kctl.command("delete -f %s" % kubernetes_yml), print_output=True)
print_green("Deleted remote environment %s" % context)
def k8s_delete(context, namespace=None, yaml_file_name=None):
if yaml_file_name is None: yaml_file_name = context
kubernetes_yml = os.path.join(CWD, "%s/%s.yml" % (HOKUSAI_CONFIG_DIR, yaml_file_name))
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
kctl = Kubectl(context, namespace=namespace)
shout(kctl.command("delete -f %s" % kubernetes_yml), print_output=True)
print_green("Deleted Kubernetes environment %s" % kubernetes_yml)
def k8s_update(context, namespace=None, yaml_file_name=None):
if yaml_file_name is None: yaml_file_name = context
kubernetes_yml = os.path.join(os.getcwd(), "hokusai/%s.yml" % yaml_file_name)
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
kctl = Kubectl(context, namespace=namespace)
shout(kctl.command("apply -f %s" % kubernetes_yml), print_output=True)
print_green("Updated Kubernetes environment %s" % kubernetes_yml)
def dev_run(command, service_name, stop):
docker_compose_yml = os.path.join(os.getcwd(), 'hokusai/development.yml')
if not os.path.isfile(docker_compose_yml):
raise HokusaiError("Yaml file %s does not exist." % docker_compose_yml)
shout("docker-compose -f %s -p hokusai run %s %s" %
(docker_compose_yml, service_name, command),
print_output=True)
if stop:
shout("docker-compose -f %s -p hokusai stop" % docker_compose_yml,
print_output=True)
def dev_status(filename):
if filename is None:
docker_compose_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR,
DEVELOPMENT_YML_FILE)
else:
docker_compose_yml = filename
if not os.path.isfile(docker_compose_yml):
raise HokusaiError("Yaml file %s does not exist." % docker_compose_yml)
shout("docker-compose -f %s -p hokusai ps" % docker_compose_yml,
print_output=True)
def dev_status(filename):
if filename is None:
yaml_template = TemplateSelector().get(
os.path.join(CWD, HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE))
else:
yaml_template = TemplateSelector().get(filename)
docker_compose_yml = YamlSpec(yaml_template).to_file()
follow_extends(docker_compose_yml)
shout("docker-compose -f %s -p hokusai ps" % docker_compose_yml,
print_output=True)
def dev_logs(follow, tail):
docker_compose_yml = os.path.join(os.getcwd(), 'hokusai/development.yml')
if not os.path.isfile(docker_compose_yml):
raise HokusaiError("Yaml file %s does not exist." % docker_compose_yml)
opts = ''
if follow:
opts += ' --follow'
if tail:
opts += " --tail=%i" % tail
shout("docker-compose -f %s -p hokusai logs%s" %
(docker_compose_yml, opts),
print_output=True)
def k8s_update(context, namespace=None, filename=None, check_branch="master",
check_remote=None, skip_checks=False, dry_run=False):
if filename is None:
yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, context))
else:
yaml_template = TemplateSelector().get(filename)
if not skip_checks:
current_branch = None
for branchname in shout('git branch').splitlines():
if '* ' in branchname:
current_branch = branchname.replace('* ', '')
break
if 'detached' in current_branch:
raise HokusaiError("Not on any branch! Aborting.")
if current_branch != check_branch:
raise HokusaiError("Not on %s branch! Aborting." % check_branch)
remotes = [check_remote] if check_remote else shout('git remote').splitlines()
for remote in remotes:
shout("git fetch %s" % remote)
if returncode("git diff --quiet %s/%s" % (remote, current_branch)):
raise HokusaiError("Local branch %s is divergent from %s/%s. Aborting." % (current_branch, remote, current_branch))
kctl = Kubectl(context, namespace=namespace)
yaml_spec = YamlSpec(yaml_template).to_file()
if dry_run:
shout(kctl.command("apply -f %s --dry-run" % yaml_spec), print_output=True)
print_green("Updated Kubernetes environment %s (dry run)" % yaml_template)
else:
shout(kctl.command("apply -f %s" % yaml_spec), print_output=True)
print_green("Updated Kubernetes environment %s" % yaml_template)
def build():
docker_compose_yml = os.path.join(os.getcwd(), 'hokusai/build.yml')
legacy_docker_compose_yml = os.path.join(os.getcwd(), 'hokusai/common.yml')
if not os.path.isfile(docker_compose_yml) and not os.path.isfile(
legacy_docker_compose_yml):
raise HokusaiError("Yaml files %s / %s do not exist." %
(docker_compose_yml, legacy_docker_compose_yml))
if os.path.isfile(docker_compose_yml):
shout("docker-compose -f %s -p hokusai build" % docker_compose_yml,
print_output=True)
if os.path.isfile(legacy_docker_compose_yml):
shout("docker-compose -f %s -p hokusai build" %
legacy_docker_compose_yml,
print_output=True)
def k8s_delete(context, namespace=None, filename=None):
if filename is None:
yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, context))
else:
yaml_template = TemplateSelector().get(filename)
if filename is None:
configmap = ConfigMap(context, namespace=namespace)
configmap.destroy()
print_green("Deleted configmap %s-environment" % config.project_name)
kctl = Kubectl(context, namespace=namespace)
yaml_spec = YamlSpec(yaml_template).to_file()
shout(kctl.command("delete -f %s" % yaml_spec), print_output=True)
print_green("Deleted Kubernetes environment %s" % yaml_template)
def test(build, cleanup, filename, service_name):
if filename is None:
yaml_template = TemplateSelector().get(
os.path.join(CWD, HOKUSAI_CONFIG_DIR, TEST_YML_FILE))
else:
yaml_template = TemplateSelector().get(filename)
docker_compose_yml = YamlSpec(yaml_template).to_file()
follow_extends(docker_compose_yml)
def on_cleanup(*args):
shout("docker-compose -f %s -p hokusai stop" % docker_compose_yml)
shout("docker-compose -f %s -p hokusai rm --force" %
docker_compose_yml)
if cleanup:
for sig in EXIT_SIGNALS:
signal.signal(sig, on_cleanup)
opts = ' --abort-on-container-exit'
if build:
Docker().build()
if service_name is None:
service_name = config.project_name
print_green("Starting test environment... Press Ctrl+C to stop.",
newline_after=True)
try:
shout("docker-compose -f %s -p hokusai up%s" %
(docker_compose_yml, opts),
print_output=True)
return_code = int(shout("docker wait hokusai_%s_1" % service_name))
except CalledProcessError:
if cleanup: on_cleanup()
raise HokusaiError('Tests Failed')
if return_code:
raise HokusaiError('Tests Failed - Exit Code: %s\n' % return_code,
return_code=return_code)
else:
print_green("Tests Passed")
if cleanup: on_cleanup()
return return_code
def build(self, filename=None):
if filename is None:
yaml_template = TemplateSelector().get(
os.path.join(CWD, HOKUSAI_CONFIG_DIR, BUILD_YAML_FILE))
else:
yaml_template = TemplateSelector().get(filename)
docker_compose_yml = YamlSpec(yaml_template).to_file()
build_command = "docker-compose -f %s -p hokusai build" % docker_compose_yml
if config.pre_build:
build_command = "%s && %s" % (config.pre_build, build_command)
if config.post_build:
build_command = "%s && %s" % (build_command, config.post_build)
shout(build_command, print_output=True)
def load(self):
payload = shout(
self.kctl.command("get configmap %s -o json" % self.name))
struct = json.loads(payload)
if 'data' in struct:
self.struct['data'] = struct['data']
else:
self.struct['data'] = {}
def test_shout(self, mocked_check_output):
with captured_output() as (out, err):
retval = shout('whoami')
self.assertEqual(retval, 'hokusai')
self.assertEqual(type(retval), str)
mocked_check_output.assert_called_once_with('whoami',
shell=True,
stderr=-2)
def gitlog():
ecr = ECR()
staging_tag = ecr.find_git_sha1_image_tag('staging')
if staging_tag is None:
raise HokusaiError("Could not find a tag for staging. Aborting.")
production_tag = ecr.find_git_sha1_image_tag('production')
if production_tag is None:
raise HokusaiError(
"Could not find a git SHA1 tag for production. Aborting.")
print_green("Comparing %s to %s" % (production_tag, staging_tag))
for remote in shout('git remote').splitlines():
shout("git fetch %s" % remote)
shout("git log --right-only %s..%s" % (production_tag, staging_tag),
print_output=True)