def test_api_object():
pod = Pod(None, {'metadata': {'name': 'myname'}})
assert repr(pod) == '<Pod myname>'
assert str(pod) == 'myname'
assert pod.metadata == {'name': 'myname'}
assert pod.labels == {}
assert pod.annotations == {}
def __produce_log_file(self, job_state):
pod_r = Pod.objects(self._pykube_api).filter(selector="app=" +
job_state.job_id)
log_string = ""
for pod_obj in pod_r.response['items']:
try:
pod = Pod(self._pykube_api, pod_obj)
log_string += "\n\n==== Pod " + pod.name + " log start ====\n\n"
log_string += pod.logs(timestamps=True)
log_string += "\n\n==== Pod " + pod.name + " log end ===="
except Exception as detail:
log.info(
"Could not write log file for pod %s due to HTTPError %s",
pod_obj['metadata']['name'], detail)
if isinstance(log_string, text_type):
log_string = log_string.encode('utf8')
logs_file_path = job_state.output_file
try:
with open(logs_file_path, mode="w") as logs_file:
logs_file.write(log_string)
except IOError as e:
log.error("Couldn't produce log files for %s", job_state.job_id)
log.exception(e)
return logs_file_path
def test_api_object():
pod = Pod(None, {"metadata": {"name": "myname"}})
assert repr(pod) == "<Pod myname>"
assert str(pod) == "myname"
assert pod.metadata == {"name": "myname"}
assert pod.labels == {}
assert pod.annotations == {}
def __get_pods(self):
self.__logger.info(f"checking __get_pods")
pod_objs = Pod.objects(self.__kube_api) \
.filter(namespace=self.job_namespace, selector="job-name=" + self.uu_name) \
.response['items']
return [Pod(self.__kube_api, p) for p in pod_objs]
def build_config_pod(self, os_version=""):
if len(os_version) == 0:
os_version = self.os_version
print("Creating config generation pod...")
return Pod(self.api,
{
"metadata":
{
"name": "generate-config",
"labels":
{
"purpose": "generate-config"
},
"namespace": "openshift-deploy"
},
"spec":
{
"containers":
[{
"name": "generate-config",
"image": "openshift/origin:" + os_version,
"imagePullPolicy": "Always",
"command": ["/bin/bash"],
"args": ["/etc/config_secret_script/create-config.sh"],
"ports": [],
"env":
[{
"name": "OPENSHIFT_INTERNAL_ADDRESS",
"value": "https://" + self.os_internal_ip
}, {
"name": "OPENSHIFT_EXTERNAL_ADDRESS",
"value": "https://" + self.os_external_ip
}, {
"name": "ETCD_ADDRESS",
"value": "http://etcd:4001"
}],
"volumeMounts":
[{
"mountPath": "/etc/config_secret_script",
"name": "config-secret-script",
"readOnly": True
}, {
"mountPath": "/etc/kube_config",
"name": "kube-config",
"readOnly": True
}]
}],
"volumes":
[{
"name": "config-secret-script",
"secret": {"secretName": "create-config-script"}
}, {
"name": "kube-config",
"secret": {"secretName": "kubeconfig"}
}],
"restartPolicy": "Never"
}
})
def get_mongo_pods():
return [
Pod(
None, {
'metadata': {
'labels': {
'hostname': 'fb-1.db.waverbase.com:%d' % p
}
},
'status': {
'podIP': '127.0.0.1:%d' % p
}
}) for p in range(base, base + num)
]
def __job_failed_due_to_low_memory(self, job_state):
"""
checks the state of the pod to see if it was killed
for being out of memory (pod status OOMKilled). If that is the case
marks the job for resubmission (resubmit logic is part of destinations).
"""
pods = Pod.objects(self._pykube_api).filter(selector="app=" + job_state.job_id)
pod = Pod(self._pykube_api, pods.response['items'][0])
if pod.obj['status']['phase'] == "Failed" and \
pod.obj['status']['containerStatuses'][0]['state']['terminated']['reason'] == "OOMKilled":
return True
return False
def test_update():
pod = Pod(
None, {
'metadata': {
'name': 'john',
'kind': 'test'
},
'annotations': 'a long string'
})
pod.obj = {'metadata': {'name': 'john'}}
pod.api = MagicMock()
pod.api.patch.return_value.json.return_value = obj_merge(
pod.obj, pod._original_obj, False)
pod.update(is_strategic=False)
assert pod.metadata == {'name': 'john'}
assert pod.annotations == {}
def test_map_empty_pod():
pod = Pod(None, {"metadata": {}, "spec": {"containers": []}, "status": {}})
assert map_pod(pod, 0, 0) == {
"application": "",
"component": "",
"container_images": [],
"team": "",
"requests": {
"cpu": 0,
"memory": 0
},
"usage": {
"cpu": 0,
"memory": 0
},
"cost": 0,
}
def run(self):
self._init_kubernetes()
self.log_event(
api.LogEntry(
self.run_id, api.TaskStatus.RUNNING,
"Starting task: {}.".format(self.task_family) + self.uu_name,
self.__repr__(), self.uu_name))
command = self.build_command()
full_command = []
full_command.extend(self.args)
if command:
full_command.append(command)
self.log_event(
api.LogEntry(
self.run_id, api.TaskStatus.RUNNING,
"{} task command: ".format(self.task_family) +
' '.join(full_command), self.__repr__(), self.uu_name))
# Render pod
pod_json = self.create_pod(full_command)
# Update user labels
pod_json['metadata']['labels'].update(self.labels)
if self.autoscaling_enabled:
pod_json['metadata']['labels'].update({"runid": self.run_id})
pod_json['spec']['nodeSelector'].update({"runid": self.run_id})
pod = Pod(self.__kube_api, pod_json)
try:
pod.create()
except HTTPError as error:
self.log_event(
api.LogEntry(
self.run_id, api.TaskStatus.RUNNING,
"Failed to create Kubernetes pod: " + self.uu_name +
"; error: " + error.message, self.__repr__(),
self.uu_name))
raise RuntimeError
# Track the Job (wait while active)
self.log_event(
api.LogEntry(self.run_id, api.TaskStatus.RUNNING,
"Start tracking Kubernetes pod: " + self.uu_name,
self.__repr__(), self.uu_name))
seen_events = set()
self.__track_pod(seen_events)
self.post_process()
def test_update():
pod = Pod(
None,
{
"metadata": {
"name": "john",
"kind": "test"
},
"annotations": "a long string"
},
)
pod.obj = {"metadata": {"name": "john"}}
pod.api = MagicMock()
pod.api.patch.return_value.json.return_value = obj_merge(
pod.obj, pod._original_obj, False)
pod.update(is_strategic=False)
assert pod.metadata == {"name": "john"}
assert pod.annotations == {}
def __get_pod_status(self):
# Look for the required pod
pods = Pod.objects(self.__kube_api).filter(selector="luigi_task_id=" +
self.job_uuid)
# Raise an exception if no such pod found
if len(pods.response["items"]) == 0:
self.log_event(
api.LogEntry(self.run_id, api.TaskStatus.FAILURE,
"Kubernetes pod failed to raise: " + self.uu_name,
self.__repr__(), self.uu_name))
raise RuntimeError("Kubernetes job " + self.uu_name + " not found")
# Figure out status and return it
pod = Pod(self.__kube_api, pods.response["items"][0])
if self.__SUCCESS_STATUS in pod.obj["status"]["phase"]:
return self.__SUCCESS_STATUS
if self.__FAILURE_STATUS in pod.obj["status"]["phase"]:
return self.__FAILURE_STATUS
return self.__RUNNING_STATUS
def test_map_pod_with_resources():
pod = Pod(
None,
{
"metadata": {
"labels": {
"app": "myapp",
"component": "mycomp",
"team": "myteam"
}
},
"spec": {
"containers": [{
"name": "main",
"image": "hjacobs/kube-downscaler:latest",
"resources": {
"requests": {
"cpu": "5m",
"memory": "200Mi"
}
},
}]
},
"status": {},
},
)
assert map_pod(pod, 0, 0) == {
"application": "myapp",
"component": "mycomp",
"team": "myteam",
"container_names": ["main"],
"container_images": ["hjacobs/kube-downscaler:latest"],
"requests": {
"cpu": 0.005,
"memory": 200 * 1024 * 1024
},
"usage": {
"cpu": 0,
"memory": 0
},
"cost": 0,
}
def __job_failed_due_to_low_memory(self, job_state):
"""
checks the state of the pod to see if it was killed
for being out of memory (pod status OOMKilled). If that is the case
marks the job for resubmission (resubmit logic is part of destinations).
"""
pods = Pod.objects(self._pykube_api).filter(selector="app=" +
job_state.job_id)
if len(pods.response['items']) == 0 or pods is None:
log.error("Cannot find API server or app={} doesn't find any pods".
format(str(job_state.job_id)))
return False
pod = Pod(self._pykube_api, pods.response['items'][0])
if pod.obj['status']['phase'] == "Failed" and \
pod.obj['status']['containerStatuses'][0]['state']['terminated']['reason'] == "OOMKilled":
return True
return False
def __produce_log_file(self, job_state):
pod_r = Pod.objects(self._pykube_api).filter(selector="app=" +
job_state.job_id)
logs = ""
for pod_obj in pod_r.response['items']:
try:
pod = Pod(self._pykube_api, pod_obj)
logs += "\n\n==== Pod " + pod.name + " log start ====\n\n"
logs += pod.logs(timestamps=True)
logs += "\n\n==== Pod " + pod.name + " log end ===="
except Exception as detail:
log.info("Could not write pod\'s " +
pod_obj['metadata']['name'] +
" log file due to HTTPError " + str(detail))
logs_file_path = job_state.output_file
logs_file = open(logs_file_path, mode="w")
if isinstance(logs, text_type):
logs = logs.encode('utf8')
logs_file.write(logs)
logs_file.close()
return logs_file_path
def build_execute_pod(self, command, admin_conf, os_version=""):
if len(os_version) == 0:
os_version = self.os_version
print("Creating command execution pod...")
name = "oskube-execute-" + random_string(4).lower()
command = "mkdir -p ~/.kube/ && echo \"$ADMIN_KUBECONFIG\" > ~/.kube/config && " + command
return Pod(self.api,
{
"metadata":
{
"name": name,
"labels":
{
"purpose": "exec-command"
},
"namespace": "openshift-origin"
},
"spec":
{
"containers":
[{
"name": "exec-command",
"image": "openshift/origin:" + os_version,
"imagePullPolicy": "Always",
"command": ["/bin/bash"],
"args": ["-c", command],
"ports": [],
"env":
[{
"name": "ADMIN_KUBECONFIG",
"value": admin_conf
}]
}],
"restartPolicy": "Never"
}
})
def editconfig(ctx):
"""Interactively edits master-config.yaml"""
ctx.temp_dir = tempfile.mkdtemp()
if ctx.auto_confirm:
print(
"Note: -y option is not supported for purely interactive commands."
)
ctx.auto_confirm = False
if not ctx.init_with_checks():
print("Failed cursory checks, exiting.")
exit(1)
if not ctx.consider_openshift_deployed:
print(
"I think OpenShift is not yet deployed. Use deploy first to create it."
)
exit(1)
old_secret = ctx.fetch_config_to_dir(ctx.temp_dir)
mc_path = ctx.temp_dir + "/master-config.yaml"
if not os.path.exists(mc_path):
print(
"Fetched config files but they don't contain master-config.yaml, something's wrong. Try getconfig."
)
shutil.rmtree(ctx.temp_dir)
exit(1)
last_mtime = os.path.getmtime(mc_path)
print("Config files are at: " + ctx.temp_dir)
print("Feel free to edit as you will...")
print("Launching editor...")
call([EDITOR, mc_path])
now_mtime = os.path.getmtime(mc_path)
if now_mtime == last_mtime:
print("No changes made, exiting.")
shutil.rmtree(ctx.temp_dir)
exit(0)
if not click.confirm("Do you want to upload the changed config files?"):
print("Okay, cancelling.")
shutil.rmtree(ctx.temp_dir)
exit(0)
print("Preparing to upload config files...")
# Serialize the config to a secret
openshift_config_kv = {}
for filen in os.listdir(ctx.temp_dir):
with open(ctx.temp_dir + "/" + filen, 'rb') as f:
openshift_config_kv[filen] = f.read()
openshift_config_secret = ctx.build_secret("openshift-config",
"openshift-origin",
openshift_config_kv)
openshift_config_secret._original_obj = old_secret.obj
print("Attempting to patch secret...")
openshift_config_secret.update()
print("Updates applied.")
if not click.confirm(
"Do you want to restart openshift to apply the changes?"):
print("Okay, I'm done. Have a nice day!")
shutil.rmtree(ctx.temp_dir)
exit(0)
print("Restarting openshift pod...")
try:
pods = Pod.objects(ctx.api).filter(namespace="openshift-origin",
selector={
"app": "openshift"
}).response["items"]
if len(pods) >= 1:
openshift_pod = Pod(ctx.api, pods[0])
print("Deleting pod " + openshift_pod.obj["metadata"]["name"] +
"...")
openshift_pod.delete()
except:
print(
"Something went wrong restarting openshift, do it yourself please!"
)
shutil.rmtree(ctx.temp_dir)
def __get_pods(self):
pod_objs = Pod.objects(self.__kube_api, namespace=self.kubernetes_namespace) \
.filter(selector="job-name=" + self.uu_name) \
.response['items']
return [Pod(self.__kube_api, p) for p in pod_objs]
def test_set_label():
pod = Pod(None, {'metadata': {'name': 'myname'}})
pod.labels['foo'] = 'bar'
assert pod.labels['foo'] == 'bar'
def test_set_annotation():
pod = Pod(None, {'metadata': {'name': 'myname'}})
pod.annotations['foo'] = 'bar'
assert pod.annotations['foo'] == 'bar'
def test_set_annotation():
pod = Pod(None, {"metadata": {"name": "myname"}})
pod.annotations["foo"] = "bar"
assert pod.annotations["foo"] == "bar"
def test_set_label():
pod = Pod(None, {"metadata": {"name": "myname"}})
pod.labels["foo"] = "bar"
assert pod.labels["foo"] == "bar"
def deploy(ctx, persistent_volume, load_balancer, public_hostname,
create_volume, master_config_override, server_key):
"""Deploy OpenShift to the cluster."""
if not load_balancer and public_hostname != None:
print(
"You must specify --load-balancer with --public-hostname, I can't map a public hostname without a load balancer."
)
exit(1)
if not ctx.init_with_checks():
print("Failed cursory checks, exiting.")
exit(1)
if ctx.consider_openshift_deployed:
print(
"I think OpenShift is already deployed. Use undeploy first to remove it before installing it again."
)
print(
"Consider if you really need a full redeploy. You can update without re-deploying!"
)
exit(1)
print()
if "openshift" in ctx.namespace_names or "openshift-origin" in ctx.namespace_names:
print(
"The namespaces 'openshift' and/or 'openshift-origin' exist, this indicates a potentially existing/broken install."
)
if ctx.auto_confirm:
print(
"Auto confirm (-y) option set, clearing existing installation."
)
else:
print("Really consider the decision you're about to make.")
if not click.confirm(
"Do you want to clear the existing installation?"):
print("Okay, cancelling.")
exit(1)
# Handle oddities with finalizers?
# todo: delete "openshift" and "openshift-infra" namespaces
ctx.delete_namespace_byname("openshift-origin")
time.sleep(1)
ctx.temp_dir = tempfile.mkdtemp()
print("Preparing to execute deploy...")
print("Deploy temp dir: " + ctx.temp_dir)
# Setup the deploy state namespace
ctx.cleanup_osdeploy_namespace()
ctx.create_osdeploy_namespace()
# Check the persistentvolume exists
if not create_volume and ctx.find_persistentvolume(
persistent_volume) == None:
print(" [!] persistentvolume with name " + persistent_volume +
" does not exist. Did you create it?")
exit(1)
# Create the namespaces
ctx.create_namespace("openshift-origin")
# Create the service
if load_balancer:
print("Will use load balancer type service.")
else:
print("Will use node port type service.")
os_service = ctx.create_os_service(load_balancer)
# Wait for it to be ready if it's a load balancer
if load_balancer:
print("Waiting for service load balancer IP to be allocated...")
ctx.wait_for_loadbalancer(os_service)
else:
os_service.reload()
internal_os_ip = os_service.obj["spec"]["clusterIP"]
ctx.os_internal_ip = internal_os_ip
if load_balancer:
tmp = os_service.obj["status"]["loadBalancer"]["ingress"][0]
external_os_ip = None
external_is_hostname = False
if "hostname" in tmp:
external_os_ip = tmp["hostname"]
external_is_hostname = True
else:
external_os_ip = tmp["ip"]
ctx.os_external_ip = external_os_ip
print("External OpenShift IP: " + external_os_ip)
else:
external_os_ip = internal_os_ip
print("External OpenShift IP: nodes (node port)")
print("Internal OpenShift IP: " + internal_os_ip)
if public_hostname != None:
print("You need to DNS map like this:")
if external_is_hostname:
print(public_hostname + ".\t300\tIN\tCNAME\t" + external_os_ip)
else:
print(public_hostname + ".\t300\tIN\tA\t" + external_os_ip)
ctx.os_external_ip = public_hostname
# Create a 'secret' containing the script to run to config.
create_config_script = (resource_string(ctx.scripts_resource,
'create-config.sh'))
# Build the secret
create_config_secret_kv = {"create-config.sh": create_config_script}
create_config_secret = ctx.build_secret("create-config-script",
"openshift-deploy",
create_config_secret_kv)
create_config_secret.create()
# Build the kubeconfig secret
kubeconfig_secret_kv = {
"kubeconfig": yaml.dump(ctx.config.doc).encode('ascii')
}
kubeconfig_secret = ctx.build_secret("kubeconfig", "openshift-deploy",
kubeconfig_secret_kv)
kubeconfig_secret.create()
# Generate the openshift config by running a temporary pod on the cluster
print("Generating openshift config via cluster...")
conf_pod = ctx.build_config_pod(ctx.os_version)
conf_pod.create()
with open(ctx.temp_dir + "/config_bundle.tar.gz", 'wb') as f:
conf_bundle = ctx.observe_config_pod(conf_pod)
conf_bundle_data = base64.b64decode(conf_bundle)
f.write(conf_bundle_data)
conf_pod.delete()
# Extract
tar = tarfile.open(ctx.temp_dir + "/config_bundle.tar.gz")
tar.extractall(ctx.temp_dir + "/config/")
tar.close()
# Move kubeconfig in
with open(ctx.temp_dir + "/config/external-master.kubeconfig", 'w') as f:
f.write(yaml.dump(ctx.config.doc))
# Delete tarfile
os.remove(ctx.temp_dir + "/config_bundle.tar.gz")
# Do some processing on the master-config yaml
conf = None
with open(ctx.temp_dir + '/config/master-config.yaml') as f:
conf = yaml.load(f)
conf = ctx.fix_master_config(conf)
# Write the serviceaccounts file again
with open(server_key, 'r') as fs:
with open(ctx.temp_dir + "/config/serviceaccounts.public.key",
'w') as fd:
fd.write(fs.read())
# Load patches if needed
master_config_override_kv = None
if master_config_override != None:
print("Loading " + master_config_override + "...")
with open(master_config_override, 'r') as f:
master_config_override_kv = yaml.load(f)
conf = deepupdate(conf, master_config_override_kv)
# Write the fixed master config
with open(ctx.temp_dir + "/config/master-config.yaml", 'w') as f:
f.write(yaml.dump(conf, default_flow_style=False))
# Allow the user to edit the openshift config last second
print("Generated updated master-config.yaml.")
if ctx.auto_confirm:
print(
"Auto confirm (-y) option set, skipping master-config.yaml edit opportunity."
)
else:
if click.confirm("Do you want to edit master-config.yaml?"):
call([EDITOR, ctx.temp_dir + "/config/master-config.yaml"])
# Cleanup a bit
kubeconfig_secret.delete()
create_config_secret.delete()
# Serialize the config to a secret
openshift_config_kv = {}
for filen in os.listdir(ctx.temp_dir + "/config"):
with open(ctx.temp_dir + "/config/" + filen, 'rb') as f:
openshift_config_kv[filen] = f.read()
openshift_config_secret = ctx.build_secret("openshift-config",
"openshift-origin",
openshift_config_kv)
# Save the secret
openshift_config_secret.create()
# Starting etcd setup... build PersistentVolumeClaim
etcd_pvc = ctx.build_pvc("openshift-etcd1", "openshift-origin", "2Gi",
create_volume)
etcd_pvc.create()
# Create the etcd controller
etcd_rc = ctx.build_etcd_rc("openshift-etcd1")
etcd_svc = ctx.build_etcd_service()
print("Creating etcd service...")
etcd_svc.create()
print("Creating etcd controller...")
etcd_rc.create()
print("Waiting for etcd pod to be created...")
etcd_pod = None
# Wait for the pod to exist
while etcd_pod == None:
etcd_pods = Pod.objects(ctx.api).filter(
selector={
"app": "etcd"
}, namespace="openshift-origin").response["items"]
if len(etcd_pods) < 1:
time.sleep(0.5)
continue
etcd_pod = Pod(ctx.api, etcd_pods[0])
# Wait for it to run
ctx.wait_for_pod_running(etcd_pod)
# Create the controller config
print("Creating openshift replication controller...")
openshift_rc = ctx.build_openshift_rc(ctx.os_version)
openshift_rc.create()
print("Waiting for openshift pod to be created...")
openshift_pod = None
# Wait for the pod to exist
while openshift_pod == None:
pods = Pod.objects(ctx.api).filter(namespace="openshift-origin",
selector={
"app": "openshift"
}).response["items"]
if len(pods) < 1:
time.sleep(0.5)
continue
openshift_pod = Pod(ctx.api, pods[0])
# Wait for it to run
ctx.wait_for_pod_running(openshift_pod)
print()
print(" == OpenShift Deployed ==")
print("External IP: " + ctx.os_external_ip)
ctx.fetch_namespaces()
ctx.cleanup_osdeploy_namespace()
shutil.rmtree(ctx.temp_dir)