def __init__(
self,
image_reference: str,
k8s_namespace_name: str,
env_vars: Optional[Dict] = None,
pod_name: Optional[str] = None,
working_dir: Optional[str] = None,
service_account_name: Optional[str] = None,
volume_mounts: Optional[List[VolumeSpec]] = None,
):
"""
:param image_reference: the pod will use this image
:param k8s_namespace_name: name of the namespace to deploy into
:param env_vars: additional environment variables to set in the pod
:param pod_name: name the pod like this, if not specified, generate something long and ugly
:param working_dir: path within the pod where we run commands by default
:param service_account_name: run the pod using this service account
:param volume_mounts: set these volume mounts in the sandbox
"""
self.image_reference: str = image_reference
self.service_account_name: Optional[str] = service_account_name
self.env_vars = env_vars
self.k8s_namespace_name = k8s_namespace_name
# regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?
self.cleaned_name = clean_string(image_reference)
if pod_name:
self.pod_name = pod_name
else:
self.pod_name = f"{self.cleaned_name}-{get_timestamp_now()}"
self.working_dir = working_dir
self.api: client.CoreV1Api = self.get_api_client()
self.volume_mounts: List[VolumeSpec] = volume_mounts or []
def create_pod_manifest(self, command: Optional[List] = None) -> dict:
env_image_vars = self.build_env_image_vars(self.env_vars)
# this is broken down for sake of mypy
container = {
"image": self.image_reference,
"name": self.pod_name,
"env": env_image_vars,
"imagePullPolicy": "IfNotPresent",
}
spec = {
"containers": [container],
"restartPolicy": "Never",
"automountServiceAccountToken": False,
}
pod_manifest = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": self.pod_name
},
"spec": spec,
}
if self.working_dir:
container["workingDir"] = self.working_dir
if command:
container["command"] = command
if self.service_account_name:
spec["serviceAccountName"] = self.service_account_name
if self.volume_mounts:
volume_mounts: List[Dict] = []
container["volumeMounts"] = volume_mounts
volumes: List[Dict] = []
spec["volumes"] = volumes
for vol in self.volume_mounts:
# local name b/w vol definition and container def
local_name = vol.name or clean_string(vol.pvc)
volume_mounts.append({
"mountPath": vol.path,
"name": local_name
})
if vol.pvc:
di = {
"name": local_name,
"persistentVolumeClaim": {
"claimName": vol.pvc
},
}
elif vol.name:
# will this work actually if the volume is created up front?
di = {"name": vol.name}
else:
raise RuntimeError(
"Please specified either volume.pvc or volume.name")
volumes.append(di)
return pod_manifest
def __init__(
self,
image_reference: str,
k8s_namespace_name: str,
env_vars: Optional[Dict] = None,
pod_name: Optional[str] = None,
working_dir: Optional[str] = None,
service_account_name: Optional[str] = None,
volume_mounts: Optional[List[VolumeSpec]] = None,
mapped_dir: Optional[MappedDir] = None,
):
"""
:param image_reference: the pod will use this image
:param k8s_namespace_name: name of the namespace to deploy into
:param env_vars: additional environment variables to set in the pod
:param pod_name: name the pod like this, if not specified, generate something long and ugly
:param working_dir: path within the pod where we run commands by default
:param service_account_name: run the pod using this service account
:param volume_mounts: set these volume mounts in the sandbox
:param mapped_dir, a mapping between a local dir which should be copied
to the sandbox, and then copied back once all the work is done
when this is set, working_dir args is being ignored and sandcastle invokes
all exec commands in the working dir of the mapped dir
"""
self.image_reference: str = image_reference
self.service_account_name: Optional[str] = service_account_name
self.env_vars = env_vars
self.k8s_namespace_name = k8s_namespace_name
# regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?
self.cleaned_name = clean_string(image_reference)
self.pod_name = pod_name or f"{self.cleaned_name}-{get_timestamp_now()}"
self.working_dir = working_dir
if working_dir and mapped_dir:
logger.warning("Ignoring working_dir because mapped_dir is set.")
self.working_dir = None
self.api: CoreV1Api = self.get_api_client()
self.volume_mounts: List[VolumeSpec] = volume_mounts or []
self.mapped_dir: MappedDir = mapped_dir
self.pvc: Optional[PVC] = None
self.pod_manifest: Dict = {}
def run_test_within_pod(test_path: str, with_pv_at: Optional[str] = None):
"""
run selected test from within an openshift pod
:param test_path: relative path to the test
:param with_pv_at: path to PV within the pod
"""
config.load_kube_config()
configuration = client.Configuration()
assert configuration.api_key
api = client.CoreV1Api(client.ApiClient(configuration))
container = {
"image":
TEST_IMAGE_NAME,
"name":
POD_NAME,
"tty":
True, # corols
"command": [
"bash",
"-c",
"ls -lha "
"&& id "
"&& pytest-3 --collect-only"
f"&& pytest-3 -vv -l -p no:cacheprovider {test_path}",
],
"imagePullPolicy":
"Never",
}
spec = {"containers": [container], "restartPolicy": "Never"}
pod_manifest = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": POD_NAME
},
"spec": spec,
}
if with_pv_at:
cleaned_test_name = clean_string(test_path)
ts = get_timestamp_now()
volume_name = f"{cleaned_test_name}-{ts}-vol"[-63:]
claim_name = f"{cleaned_test_name}-{ts}-pvc"[-63:]
container["env"] = [{"name": "SANDCASTLE_PVC", "value": claim_name}]
pvc_dict = {
"kind": "PersistentVolumeClaim",
"spec": {
"accessModes": ["ReadWriteMany"],
"resources": {
"requests": {
"storage": "1Gi"
}
},
},
"apiVersion": "v1",
"metadata": {
"name": claim_name
},
}
api.create_namespaced_persistent_volume_claim(NAMESPACE, pvc_dict)
container["volumeMounts"] = [{
"mountPath": with_pv_at,
"name": volume_name
}]
spec["volumes"] = [{
"name": volume_name,
"persistentVolumeClaim": {
"claimName": claim_name
}
}]
try:
api.delete_namespaced_pod(POD_NAME, NAMESPACE, body=V1DeleteOptions())
except ApiException as ex:
if ex.status != 404:
raise
try:
api.create_namespaced_pod(body=pod_manifest, namespace=NAMESPACE)
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not start on time.")
info = api.read_namespaced_pod(POD_NAME, NAMESPACE)
if info.status.phase == "Running":
break
time.sleep(2.0)
counter -= 1
print(
api.read_namespaced_pod_log(name=POD_NAME,
namespace=NAMESPACE,
follow=True))
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not finish on time.")
info = api.read_namespaced_pod(POD_NAME, NAMESPACE)
if info.status.phase == "Succeeded":
break
if info.status.phase == "Failed":
raise RuntimeError("Test failed")
time.sleep(2.0)
counter -= 1
finally:
print(
api.read_namespaced_pod_log(name=POD_NAME,
namespace=NAMESPACE,
follow=True))
api.delete_namespaced_pod(POD_NAME, NAMESPACE, body=V1DeleteOptions())
if with_pv_at:
api.delete_namespaced_persistent_volume_claim(
name=claim_name, namespace=NAMESPACE, body=V1DeleteOptions())
def run_test_within_pod(test_path: str,
with_pv_at: Optional[str] = None,
new_namespace: bool = False):
"""
run selected test from within an openshift pod
:param test_path: relative path to the test
:param with_pv_at: path to PV within the pod
:param new_namespace: create new namespace and pass it via env var
"""
config.load_kube_config()
configuration = client.Configuration()
assert configuration.api_key
api = client.CoreV1Api(client.ApiClient(configuration))
pod_name = f"test-orchestrator-{get_timestamp_now()}"
cont_cmd = [
"bash",
"-c",
"ls -lha "
f"&& pytest-3 -vv -l -p no:cacheprovider {test_path}",
]
container: Dict[str, Any] = {
"image": TEST_IMAGE_NAME,
"name": pod_name,
"tty": True, # corols
"command": cont_cmd,
"imagePullPolicy": "Never",
"env": [],
}
test_namespace = None
if new_namespace:
test_namespace = f"sandcastle-tests-{get_timestamp_now()}"
c = ["oc", "new-project", test_namespace]
run_command(c)
c = [
"oc",
"adm",
"-n",
test_namespace,
"policy",
"add-role-to-user",
"edit",
f"system:serviceaccount:{NAMESPACE}:default",
]
run_command(c)
container["env"] += [{
"name": "SANDCASTLE_TESTS_NAMESPACE",
"value": test_namespace
}]
spec = {"containers": [container], "restartPolicy": "Never"}
pod_manifest = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": pod_name
},
"spec": spec,
}
if with_pv_at:
cleaned_test_name = clean_string(test_path)
ts = get_timestamp_now()
volume_name = f"{cleaned_test_name}-{ts}-vol"[-63:]
claim_name = f"{cleaned_test_name}-{ts}-pvc"[-63:]
container["env"] = [{"name": "SANDCASTLE_PVC", "value": claim_name}]
pvc_dict = {
"kind": "PersistentVolumeClaim",
"spec": {
"accessModes": ["ReadWriteMany"],
"resources": {
"requests": {
"storage": "1Gi"
}
},
},
"apiVersion": "v1",
"metadata": {
"name": claim_name
},
}
api.create_namespaced_persistent_volume_claim(NAMESPACE, pvc_dict)
container["volumeMounts"] = [{
"mountPath": with_pv_at,
"name": volume_name
}]
spec["volumes"] = [{
"name": volume_name,
"persistentVolumeClaim": {
"claimName": claim_name
}
}]
try:
api.delete_namespaced_pod(pod_name, NAMESPACE, body=V1DeleteOptions())
except ApiException as ex:
if ex.status != 404:
raise
try:
api.create_namespaced_pod(body=pod_manifest, namespace=NAMESPACE)
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not start on time.")
info = api.read_namespaced_pod(pod_name, NAMESPACE)
if info.status.phase == "Running":
break
time.sleep(2.0)
counter -= 1
print(
api.read_namespaced_pod_log(name=pod_name,
namespace=NAMESPACE,
follow=True))
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not finish on time.")
info = api.read_namespaced_pod(pod_name, NAMESPACE)
if info.status.phase == "Succeeded":
break
if info.status.phase == "Failed":
raise RuntimeError("Test failed")
time.sleep(2.0)
counter -= 1
finally:
print(
api.read_namespaced_pod_log(name=pod_name,
namespace=NAMESPACE,
follow=True))
api.delete_namespaced_pod(pod_name, NAMESPACE, body=V1DeleteOptions())
if new_namespace:
run_command(["oc", "delete", "project", test_namespace])
if with_pv_at:
api.delete_namespaced_persistent_volume_claim(
name=claim_name, namespace=NAMESPACE, body=V1DeleteOptions())
def run_test_within_pod(test_path: str,
with_pv_at: Optional[str] = None,
new_namespace: bool = False):
"""
run selected test from within an openshift pod
:param test_path: relative path to the test
:param with_pv_at: path to PV within the pod
:param new_namespace: create new namespace and pass it via env var
"""
# this will connect to the cluster you have active right now - see `oc status`
current_context = check_output(["oc", "config",
"current-context"]).strip().decode()
api_client = new_client_from_config(context=current_context)
api = client.CoreV1Api(api_client)
# you need this for minishift; or do `eval $(minishift docker-env) && make build`
# # FIXME: get this from env or cli (`minishift openshift registry`)
# also it doesn't work for me right now, unable to reach minishift's docker registry
# registry = "172.30.1.1:5000"
# openshift_image_name = f"{registry}/myproject/{TEST_IMAGE_BASENAME}"
# # {'authorization': 'Bearer pRW5rGmqgBREnCeVweLcHbEhXvluvG1cImWfIrxWJ2A'}
# api_token = list(api_client.configuration.api_key.values())[0].split(" ")[1]
# check_call(["docker", "login", "-u", "developer", "-p", api_token, registry])
# check_call(["docker", "tag", TEST_IMAGE_NAME, openshift_image_name])
# check_call(["docker", "push", openshift_image_name])
pod_name = f"test-orchestrator-{get_timestamp_now()}"
cont_cmd = [
"bash",
"-c",
"~/setup_env_in_openshift.sh "
"&& ls -lha && pwd && id "
f"&& pytest-3 -vv -l -p no:cacheprovider {test_path}",
]
container: Dict[str, Any] = {
"image": TEST_IMAGE_NAME, # make sure this is correct
"name": pod_name,
"tty": True, # corols
"command": cont_cmd,
"imagePullPolicy": "Never",
"env": [],
}
user = f"system:serviceaccount:{NAMESPACE}:default"
enable_user_access_namespace(user, NAMESPACE)
test_namespace = None
if new_namespace:
test_namespace = f"sandcastle-tests-{get_timestamp_now()}"
c = ["oc", "new-project", test_namespace]
run_command(c)
enable_user_access_namespace(user, test_namespace)
container["env"] += [{
"name": "SANDCASTLE_TESTS_NAMESPACE",
"value": test_namespace
}]
spec = {"containers": [container], "restartPolicy": "Never"}
pod_manifest = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": pod_name
},
"spec": spec,
}
if with_pv_at:
cleaned_test_name = clean_string(test_path)
ts = get_timestamp_now()
volume_name = f"{cleaned_test_name}-{ts}-vol"[-63:]
claim_name = f"{cleaned_test_name}-{ts}-pvc"[-63:]
container["env"] = [{"name": "SANDCASTLE_PVC", "value": claim_name}]
pvc_dict = {
"kind": "PersistentVolumeClaim",
"spec": {
"accessModes": ["ReadWriteMany"],
"resources": {
"requests": {
"storage": "1Gi"
}
},
},
"apiVersion": "v1",
"metadata": {
"name": claim_name
},
}
api.create_namespaced_persistent_volume_claim(NAMESPACE, pvc_dict)
container["volumeMounts"] = [{
"mountPath": with_pv_at,
"name": volume_name
}]
spec["volumes"] = [{
"name": volume_name,
"persistentVolumeClaim": {
"claimName": claim_name
}
}]
try:
api.delete_namespaced_pod(pod_name, NAMESPACE, body=V1DeleteOptions())
except ApiException as ex:
if ex.status != 404:
raise
try:
api.create_namespaced_pod(body=pod_manifest, namespace=NAMESPACE)
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not start on time.")
info = api.read_namespaced_pod(pod_name, NAMESPACE)
if info.status.phase == "Running":
break
elif info.status.phase == "Failed":
print_pod_logs(api, pod_name, NAMESPACE)
raise RuntimeError("The pod failed to start.")
time.sleep(2.0)
counter -= 1
print_pod_logs(api, pod_name, NAMESPACE)
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not finish on time.")
info = api.read_namespaced_pod(pod_name, NAMESPACE)
if info.status.phase == "Succeeded":
break
if info.status.phase == "Failed":
raise RuntimeError("Test failed")
time.sleep(2.0)
counter -= 1
finally:
print_pod_logs(api, pod_name, NAMESPACE)
api.delete_namespaced_pod(pod_name, NAMESPACE, body=V1DeleteOptions())
if new_namespace:
run_command(["oc", "delete", "project", test_namespace])
if with_pv_at:
api.delete_namespaced_persistent_volume_claim(
name=claim_name, namespace=NAMESPACE, body=V1DeleteOptions())
def set_pod_manifest(self, command: Optional[List] = None):
env_image_vars = self.build_env_image_vars(self.env_vars)
# this is broken down for sake of mypy
container = {
"image": self.image_reference,
"name": self.pod_name,
"env": env_image_vars,
"imagePullPolicy": "IfNotPresent",
# we may be tempted to enable tty in the pod to get pretty terminal output
# but it's not as simple as that, for example
# npm loves to make beautiful terminal output full of colors and sunshine
# which would look hideous in ASCII logs (terminal escape sequences),
# therefore only allocate tty if you are debugging something locally by
# invoking real shell inside, tty should be set to false by default
# TODO: making this configurable would be the best
# "tty": True,
"resources": {
# 512/768 may feel like it's too much, but!
# git-clone needs a lot of memory
# nodejs is hungry, especially when npm compiles stuff
# https://developer.ibm.com/languages/node-js/articles/nodejs-memory-management-in-container-environments/#
"limits": {
"memory": "768Mi"
},
"requests": {
"memory": "512Mi"
},
},
}
spec = {
"containers": [container],
"restartPolicy": "Never",
"automountServiceAccountToken": False,
}
self.pod_manifest = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": self.pod_name
},
"spec": spec,
}
if self.working_dir:
container["workingDir"] = self.working_dir
if command:
container["command"] = command
if self.service_account_name:
spec["serviceAccountName"] = self.service_account_name
if self.mapped_dir and self.mapped_dir.with_interim_pvc:
self.pvc = PVC(path=self.mapped_dir.path)
self.api.create_namespaced_persistent_volume_claim(
namespace=self.k8s_namespace_name, body=self.pvc.to_dict())
self.volume_mounts.append(
VolumeSpec(
path=self.pvc.path,
volume_name=self.pvc.volume_name,
pvc=self.pvc.claim_name,
))
if self.volume_mounts:
volume_mounts: List[Dict] = []
container["volumeMounts"] = volume_mounts
volumes: List[Dict] = []
spec["volumes"] = volumes
for vol in self.volume_mounts:
# local name b/w vol definition and container def
local_name = vol.name or clean_string(vol.pvc)
volume_mounts.append({
"mountPath": str(vol.path),
"name": local_name,
"readOnly": vol.read_only,
})
if vol.pvc:
di = {
"name": local_name,
"persistentVolumeClaim": {
"claimName": vol.pvc
},
}
elif vol.name:
# if the volume is created up front, it's used then (surprise!)
# if not, an emptyDir PV is used
di = {"name": vol.name}
else:
raise RuntimeError(
"Please specified either volume.pvc or volume.name")
volumes.append(di)
