def k8s_create(context, tag='latest', namespace=None, filename=None, environment=()):
if filename is None:
yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, context))
else:
yaml_template = TemplateSelector().get(filename)
ecr = ECR()
if not ecr.project_repo_exists():
raise HokusaiError("ECR repository %s does not exist... did you run `hokusai setup` for this project?" % config.project_name)
if not ecr.tag_exists(tag):
raise HokusaiError("Image tag %s does not exist... did you run `hokusai registry push`?" % tag)
if tag == 'latest' and not ecr.tag_exists(context):
ecr.retag(tag, context)
print_green("Updated tag 'latest' -> %s" % context)
if filename is None:
configmap = ConfigMap(context, namespace=namespace)
for s in environment:
if '=' not in s:
raise HokusaiError("Error: environment variables must be of the form 'KEY=VALUE'")
split = s.split('=', 1)
configmap.update(split[0], split[1])
configmap.create()
print_green("Created configmap %s-environment" % config.project_name)
kctl = Kubectl(context, namespace=namespace)
yaml_spec = YamlSpec(yaml_template).to_file()
shout(kctl.command("create --save-config -f %s" % yaml_spec), print_output=True)
print_green("Created Kubernetes environment %s" % yaml_template)
def k8s_create(context, tag='latest', namespace=None, filename=None):
if filename is None:
kubernetes_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR,
"%s.yml" % context)
else:
kubernetes_yml = filename
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
ecr = ECR()
if not ecr.project_repo_exists():
raise HokusaiError(
"ECR repository %s does not exist... did you run `hokusai setup` for this project?"
% config.project_name)
if not ecr.tag_exists(tag):
raise HokusaiError(
"Image tag %s does not exist... did you run `hokusai registry push`?"
% tag)
if tag is 'latest' and not ecr.tag_exists(context):
ecr.retag(tag, context)
print_green("Updated tag 'latest' -> %s" % context)
if filename is None:
configmap = ConfigMap(context, namespace=namespace)
configmap.create()
print_green("Created configmap %s-environment" % config.project_name)
kctl = Kubectl(context, namespace=namespace)
shout(kctl.command("create --save-config -f %s" % kubernetes_yml),
print_output=True)
print_green("Created Kubernetes environment %s" % kubernetes_yml)
def k8s_status(context, resources, pods, describe, top, namespace=None, filename=None):
if filename is None:
yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, context))
else:
yaml_template = TemplateSelector().get(filename)
kctl = Kubectl(context, namespace=namespace)
yaml_spec = YamlSpec(yaml_template).to_file()
if describe:
kctl_cmd = "describe"
output = ""
else:
kctl_cmd = "get"
output = " -o wide"
if resources:
print_green("Resources", newline_before=True)
print_green("===========")
shout(kctl.command("%s -f %s%s" % (kctl_cmd, yaml_spec, output)), print_output=True)
if pods:
print_green("Pods", newline_before=True)
print_green("===========")
shout(kctl.command("%s pods --selector app=%s,layer=application%s" % (kctl_cmd, config.project_name, output)), print_output=True)
if top:
print_green("Top Pods", newline_before=True)
print_green("===========")
shout(kctl.command("top pods --selector app=%s,layer=application" % config.project_name), print_output=True)
def logs(context, timestamps, follow, tail):
kctl = Kubectl(context)
opts = ''
if timestamps:
opts += ' --timestamps'
if follow:
opts += ' --follow'
if tail:
opts += " --tail=%s" % tail
pods = kctl.get_object('pod',
selector="app=%s,layer=application" %
config.project_name)
pods = filter(lambda pod: pod['status']['phase'] == 'Running', pods)
containers = []
for pod in pods:
for container in pod['spec']['containers']:
containers.append({
'pod': pod['metadata']['name'],
'name': container['name']
})
commands = [
kctl.command("logs %s %s%s" %
(container['pod'], container['name'], opts))
for container in containers
]
return shout_concurrent(commands, print_output=True)
def setUpClass(cls):
kubernetes.ECR = MockECR
kubernetes.HOKUSAI_CONFIG_DIR = os.path.join(
CWD, 'test/fixtures/project/hokusai')
cls.kubernetes_yml = os.path.abspath(
os.path.join(kubernetes.HOKUSAI_CONFIG_DIR, 'minikube.yml'))
cls.kctl = Kubectl(TEST_KUBE_CONTEXT)
def k8s_create(context, tag='latest', namespace=None, yaml_file_name=None):
if yaml_file_name is None: yaml_file_name = context
kubernetes_yml = os.path.join(os.getcwd(),
"hokusai/%s.yml" % yaml_file_name)
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
ecr = ECR()
if not ecr.project_repo_exists():
raise HokusaiError(
"ECR repository %s does not exist... did you run `hokusai setup` for this project?"
% config.project_name)
if not ecr.tag_exists(tag):
raise HokusaiError(
"Image tag %s does not exist... did you run `hokusai push`?" % tag)
if tag is 'latest' and not ecr.tag_exists(context):
ecr.retag(tag, context)
print_green("Updated tag 'latest' -> %s" % context)
kctl = Kubectl(context, namespace=namespace)
shout(kctl.command("create --save-config -f %s" % kubernetes_yml),
print_output=True)
print_green("Created Kubernetes environment %s" % kubernetes_yml)
def k8s_status(context, resources, pods, describe, top, namespace=None, yaml_file_name=None):
if yaml_file_name is None: yaml_file_name = context
kubernetes_yml = os.path.join(CWD, "%s/%s.yml" % (HOKUSAI_CONFIG_DIR, yaml_file_name))
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
kctl = Kubectl(context, namespace=namespace)
if describe:
kctl_cmd = "describe"
output = ""
else:
kctl_cmd = "get"
output = " -o wide"
if resources:
print_green("Resources", newline_before=True)
print_green("===========")
shout(kctl.command("%s -f %s%s" % (kctl_cmd, kubernetes_yml, output)), print_output=True)
if pods:
print_green("Pods", newline_before=True)
print_green("===========")
shout(kctl.command("%s pods --selector app=%s,layer=application%s" % (kctl_cmd, config.project_name, output)), print_output=True)
if top:
print_green("Top Pods", newline_before=True)
print_green("===========")
shout(kctl.command("top pods --selector app=%s,layer=application" % config.project_name), print_output=True)
def logs(context, timestamps, follow, tail, previous, labels, namespace=None):
kctl = Kubectl(context, namespace=namespace)
opts = ''
if timestamps:
opts += ' --timestamps'
if previous:
opts += ' --previous'
if follow or config.follow_logs:
opts += ' --follow'
if tail or config.tail_logs:
num_tail = tail if tail else config.tail_logs
opts += " --tail=%s" % num_tail
selectors = ["app=%s" % config.project_name, "layer=application"]
for l in labels:
if '=' not in l:
raise HokusaiError("Error: label selectors of the form 'key=value'")
selectors.append(l)
pods = kctl.get_objects('pod', selector=(',').join(selectors))
pods = [pod for pod in pods if pod['status']['phase'] == 'Running']
containers = []
for pod in pods:
for container in pod['spec']['containers']:
containers.append({'pod': pod['metadata']['name'], 'name': container['name']})
commands = [kctl.command("logs %s %s%s" % (container['pod'], container['name'], opts)) for container in containers]
shout_concurrent(commands, print_output=True)
def k8s_update(context, namespace=None, filename=None, check_branch="master",
check_remote=None, skip_checks=False, dry_run=False):
if filename is None:
yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, context))
else:
yaml_template = TemplateSelector().get(filename)
if not skip_checks:
current_branch = None
for branchname in shout('git branch').splitlines():
if '* ' in branchname:
current_branch = branchname.replace('* ', '')
break
if 'detached' in current_branch:
raise HokusaiError("Not on any branch! Aborting.")
if current_branch != check_branch:
raise HokusaiError("Not on %s branch! Aborting." % check_branch)
remotes = [check_remote] if check_remote else shout('git remote').splitlines()
for remote in remotes:
shout("git fetch %s" % remote)
if returncode("git diff --quiet %s/%s" % (remote, current_branch)):
raise HokusaiError("Local branch %s is divergent from %s/%s. Aborting." % (current_branch, remote, current_branch))
kctl = Kubectl(context, namespace=namespace)
yaml_spec = YamlSpec(yaml_template).to_file()
if dry_run:
shout(kctl.command("apply -f %s --dry-run" % yaml_spec), print_output=True)
print_green("Updated Kubernetes environment %s (dry run)" % yaml_template)
else:
shout(kctl.command("apply -f %s" % yaml_spec), print_output=True)
print_green("Updated Kubernetes environment %s" % yaml_template)
def environment_delete(context):
kubernetes_yml = os.path.join(os.getcwd(), "hokusai/%s.yml" % context)
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist for given context." %
kubernetes_yml)
kctl = Kubectl(context)
shout(kctl.command("delete -f %s" % kubernetes_yml), print_output=True)
print_green("Deleted remote environment %s" % context)
def k8s_delete(context, namespace=None, yaml_file_name=None):
if yaml_file_name is None: yaml_file_name = context
kubernetes_yml = os.path.join(CWD, "%s/%s.yml" % (HOKUSAI_CONFIG_DIR, yaml_file_name))
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
kctl = Kubectl(context, namespace=namespace)
shout(kctl.command("delete -f %s" % kubernetes_yml), print_output=True)
print_green("Deleted Kubernetes environment %s" % kubernetes_yml)
def k8s_update(context, namespace=None, yaml_file_name=None):
if yaml_file_name is None: yaml_file_name = context
kubernetes_yml = os.path.join(os.getcwd(), "hokusai/%s.yml" % yaml_file_name)
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
kctl = Kubectl(context, namespace=namespace)
shout(kctl.command("apply -f %s" % kubernetes_yml), print_output=True)
print_green("Updated Kubernetes environment %s" % kubernetes_yml)
def __init__(self, context, namespace=None):
self.context = context
self.kctl = Kubectl(context, namespace=namespace)
self.struct = OrderedDict([('apiVersion', 'v1'), ('kind', 'ConfigMap'),
('metadata', {
'labels': {
'app': config.project_name
},
'name':
"%s-environment" % config.project_name
}), ('data', {})])
def __init__(self, context, namespace='default', name=None):
self.context = context
self.kctl = Kubectl(context, namespace=namespace)
self.name = name or "%s-environment" % config.project_name
self.metadata = {'name': self.name, 'namespace': namespace}
if name is None:
self.metadata['labels'] = {'app': config.project_name}
self.struct = {
'apiVersion': 'v1',
'kind': 'ConfigMap',
'metadata': self.metadata,
'data': {}
}
def __init__(self, context, deployment_name=None, namespace=None):
self.context = context
self.namespace = namespace
self.kctl = Kubectl(self.context, namespace=namespace)
self.ecr = ECR()
if deployment_name:
self.cache = [
self.kctl.get_object("deployment %s" % deployment_name)
]
else:
self.cache = self.kctl.get_objects(
'deployment',
selector="app=%s,layer=application" % config.project_name)
def k8s_delete(context, namespace=None, filename=None):
if filename is None:
yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, context))
else:
yaml_template = TemplateSelector().get(filename)
if filename is None:
configmap = ConfigMap(context, namespace=namespace)
configmap.destroy()
print_green("Deleted configmap %s-environment" % config.project_name)
kctl = Kubectl(context, namespace=namespace)
yaml_spec = YamlSpec(yaml_template).to_file()
shout(kctl.command("delete -f %s" % yaml_spec), print_output=True)
print_green("Deleted Kubernetes environment %s" % yaml_template)
def k8s_delete(context, namespace=None, filename=None):
if filename is None:
kubernetes_yml = os.path.join(CWD, HOKUSAI_CONFIG_DIR,
"%s.yml" % context)
else:
kubernetes_yml = filename
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist." % kubernetes_yml)
if filename is None:
configmap = ConfigMap(context, namespace=namespace)
configmap.destroy()
print_green("Deleted configmap %s-environment" % config.project_name)
kctl = Kubectl(context, namespace=namespace)
shout(kctl.command("delete -f %s" % kubernetes_yml), print_output=True)
print_green("Deleted Kubernetes environment %s" % kubernetes_yml)
def environment_status(context):
kctl = Kubectl(context)
print('')
print_green("Deployments")
print_green('-----------------------------------------------------------')
shout(kctl.command("get deployments --selector app=%s -o wide" %
config.project_name),
print_output=True)
print('')
print_green("Services")
print_green('-----------------------------------------------------------')
shout(kctl.command("get services --selector app=%s -o wide" %
config.project_name),
print_output=True)
print('')
print_green("Pods")
print_green('-----------------------------------------------------------')
shout(kctl.command("get pods --selector app=%s -o wide" %
config.project_name),
print_output=True)
def environment_create(context):
kubernetes_yml = os.path.join(os.getcwd(), "hokusai/%s.yml" % context)
if not os.path.isfile(kubernetes_yml):
raise HokusaiError("Yaml file %s does not exist for given context." %
kubernetes_yml)
ecr = ECR()
if not ecr.project_repository_exists():
raise HokusaiError(
"ECR repository %s does not exist... did you run `hokusai setup` for this project?"
% config.project_name)
if not ecr.tag_exists('latest'):
raise HokusaiError(
"Image tag 'latest' does not exist... did you run `hokusai push`?")
if not ecr.tag_exists(context):
ecr.retag('latest', context)
print_green("Updated tag 'latest' -> %s" % context)
kctl = Kubectl(context)
shout(kctl.command("create --save-config -f %s" % kubernetes_yml),
print_output=True)
print_green("Created remote environment %s" % context)
def __init__(self, context):
self.context = context
self.kctl = Kubectl(self.context)
def __init__(self, context):
self.context = context
self.kctl = Kubectl(self.context)
self.cache = self.kctl.get_object('deployment',
selector="app=%s,layer=application" %
config.project_name)
def check():
return_code = 0
def check_ok(check_item):
print_green('\u2714 ' + check_item + ' found')
def check_err(check_item):
print_red('\u2718 ' + check_item + ' not found')
try:
config.project_name
check_ok('Config project-name')
except HokusaiError:
check_err('Config project-name')
try:
shout('which docker')
check_ok('docker')
except CalledProcessError:
check_err('docker')
return_code += 1
try:
shout('which docker-compose')
check_ok('docker-compose')
except CalledProcessError:
check_err('docker-compose')
return_code += 1
try:
shout('which kubectl')
check_ok('kubectl')
except CalledProcessError:
check_err('kubectl')
return_code += 1
try:
shout('which git')
check_ok('git')
except CalledProcessError:
check_err('git')
return_code += 1
try:
boto3.client('sts', region_name=get_region_name()).get_caller_identity()
check_ok('Valid AWS credentials')
except (botoexceptions.ClientError, botoexceptions.NoCredentialsError):
check_err('Valid AWS credentials')
return_code += 1
ecr = ECR()
if ecr.project_repo_exists():
check_ok("ECR repository '%s'" % config.project_name)
else:
check_err("ECR repository '%s'" % config.project_name)
return_code += 1
try:
build_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, BUILD_YAML_FILE))
check_ok(HOKUSAI_CONFIG_DIR + '/' + os.path.split(build_template)[-1])
except HokusaiError:
check_err('hokusai/build.*')
return_code += 1
try:
development_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE))
check_ok(HOKUSAI_CONFIG_DIR + '/' + os.path.split(development_template)[-1])
except HokusaiError:
check_err('hokusai/development.*')
return_code += 1
try:
test_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, TEST_YML_FILE))
check_ok(HOKUSAI_CONFIG_DIR + '/' + os.path.split(test_template)[-1])
except HokusaiError:
check_err('hokusai/test.*')
return_code += 1
for context in ['staging', 'production']:
try:
if context in Kubectl('staging').contexts():
check_ok("kubectl context '%s'" % context)
else:
check_err("kubectl context '%s'" % context)
return_code += 1
except CalledProcessError:
check_err('%s context' % context)
return_code += 1
try:
context_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, context))
check_ok(HOKUSAI_CONFIG_DIR + '/' + os.path.split(context_template)[-1])
except HokusaiError:
check_err("hokusai/%s.*" % context)
return_code += 1
return return_code
def check():
return_code = 0
def check_ok(check_item):
print_green(u'\u2714 ' + check_item + ' found')
def check_err(check_item):
print_red(u'\u2718 ' + check_item + ' not found')
config.check()
try:
config.project_name
check_ok('Config project-name')
except HokusaiError:
check_err('Config project-name')
try:
shout('docker --version')
check_ok('docker')
except CalledProcessError:
check_err('docker')
return_code += 1
try:
shout('docker-compose --version')
check_ok('docker-compose')
except CalledProcessError:
check_err('docker-compose')
return_code += 1
try:
shout('kubectl version')
check_ok('kubectl')
except CalledProcessError:
check_err('kubectl')
return_code += 1
try:
shout('git version')
check_ok('git')
except CalledProcessError:
check_err('git')
return_code += 1
if os.environ.get('AWS_ACCESS_KEY_ID') is not None:
check_ok('$AWS_ACCESS_KEY_ID')
else:
check_err('$AWS_ACCESS_KEY_ID')
return_code += 1
if os.environ.get('AWS_SECRET_ACCESS_KEY') is not None:
check_ok('$AWS_SECRET_ACCESS_KEY')
else:
check_err('$AWS_SECRET_ACCESS_KEY')
return_code += 1
ecr = ECR()
if ecr.project_repo_exists():
check_ok("ECR repository '%s'" % config.project_name)
else:
check_err("ECR repository '%s'" % config.project_name)
return_code += 1
if not os.path.isfile(os.path.join(os.getcwd(), 'hokusai/build.yml')):
if os.path.isfile(os.path.join(os.getcwd(), 'hokusai/common.yml')):
check_ok('./hokusai/common.yml')
else:
check_err('./hokusai/build.yml')
else:
check_ok('./hokusai/build.yml')
return_code += 1
if os.path.isfile(os.path.join(os.getcwd(), 'hokusai/development.yml')):
check_ok('./hokusai/development.yml')
else:
check_err('./hokusai/development.yml')
return_code += 1
if os.path.isfile(os.path.join(os.getcwd(), 'hokusai/test.yml')):
check_ok('./hokusai/test.yml')
else:
check_err('./hokusai/test.yml')
return_code += 1
for context in ['staging', 'production']:
if context in Kubectl('staging').contexts():
check_ok("kubectl context '%s'" % context)
else:
check_err("kubectl context '%s'" % context)
return_code += 1
if os.path.isfile(os.path.join(os.getcwd(), "hokusai/%s.yml" % context)):
check_ok("./hokusai/%s.yml" % context)
else:
check_err("./hokusai/%s.yml" % context)
return_code += 1
return return_code
def __init__(self, context, namespace=None):
self.context = context
self.kctl = Kubectl(self.context, namespace=namespace)
self.ecr = ECR()
if os.path.isfile(os.path.join(os.getcwd(), 'hokusai/development.yml')):
check_ok('./hokusai/development.yml')
else:
check_err('./hokusai/development.yml')
return_code += 1
if os.path.isfile(os.path.join(os.getcwd(), 'hokusai/test.yml')):
check_ok('./hokusai/test.yml')
else:
check_err('./hokusai/test.yml')
return_code += 1
for context in ['staging', 'production']:
try:
if context in Kubectl('staging').contexts():
check_ok("kubectl context '%s'" % context)
else:
check_err("kubectl context '%s'" % context)
return_code += 1
if os.path.isfile(
os.path.join(os.getcwd(), "hokusai/%s.yml" % context)):
check_ok("./hokusai/%s.yml" % context)
else:
check_err("./hokusai/%s.yml" % context)
return_code += 1
except CalledProcessError:
check_err('%s context' % context)
return_code += 1