def _copy_path_from_pod(self, local_dir: Path, pod_dir: Path):
"""
copy content of a dir from pod to local dir
:param local_dir: path to the local dir
:param pod_dir: path within the pod
"""
try:
run_command([
"oc",
"rsync",
"--delete", # delete files in local_dir which are not in pod_dir
"--quiet=true", # avoid huge logs
f"--namespace={self.k8s_namespace_name}",
f"{self.pod_name}:{pod_dir}/", # trailing / to copy only content of dir
f"{local_dir}",
])
except Exception as ex:
# There is a race condition in k8s that it tells the pod is running even
# though it already killed an exec session and hence we couldn't copy
# anything from the pod
if not self.is_pod_running(
) or "not available in container" in str(ex):
logger.warning(
"The pod is not running while we tried to copy data out of it."
)
raise SandcastleException(
"Cannot copy data from the sandbox - the pod is not running."
)
raise
def test_md_e2e(tmp_path, git_url, branch):
# running in k8s
if "KUBERNETES_SERVICE_HOST" in os.environ:
t = Path(SANDCASTLE_MOUNTPOINT, f"clone-{get_timestamp_now()}")
else:
t = tmp_path
m_dir = MappedDir(t, SANDCASTLE_MOUNTPOINT, with_interim_pvc=True)
run_command(["git", "clone", "-b", branch, git_url, t])
o = Sandcastle(image_reference=SANDBOX_IMAGE,
k8s_namespace_name=NAMESPACE,
mapped_dir=m_dir)
o.run()
try:
o.exec(command=["packit", "--debug", "srpm"])
assert list(t.glob("*.src.rpm"))
o.exec(command=["packit", "--help"])
with pytest.raises(SandcastleCommandFailed) as ex:
o.exec(command=["bash", "-c", "echo 'I quit!'; exit 120"])
e = ex.value
assert "I quit!" in e.output
assert 120 == e.rc
assert "command terminated with non-zero exit code" in e.reason
finally:
o.delete_pod()
def build_now():
""" build a container image with sandcastle """
project_root = Path(__file__).parent.parent
run_command([
"docker",
"build",
"-t",
TEST_IMAGE_NAME,
"-f",
"Dockerfile.tests",
str(project_root),
])
def _copy_path_to_pod(self, local_path: Path, pod_dir: Path):
"""
copy local_path (dir or file) inside pod
:param local_path: path to a local file or a dir
:param pod_dir: Directory within the pod where the content of local_path is extracted
"""
with tempfile.TemporaryDirectory() as tmpdir:
tmp_tarball_dir = Path(tmpdir)
tmp_tarball_path = tmp_tarball_dir.joinpath("t.tar.gz")
cmd = ["tar", "--preserve-permissions", "-czf", tmp_tarball_path]
working_dir = local_path
if local_path.is_file():
items = [local_path]
working_dir = local_path.parent
else:
# this has to be list, because mypy:
# Incompatible types in assignment (expression has type
# "Generator[Path, None, None]", variable has type "List[Path]")
items = list(local_path.iterdir())
# tar: lost+found: Cannot utime: Operation not permitted
# list comprehension no likey mypy
for x in items:
if x.name == "lost+found":
continue
cmd.append(str(x.relative_to(working_dir)))
run_command(cmd, cwd=working_dir)
remote_tmp_dir = Path(self._do_exec(["mktemp", "-d"]).strip())
try:
remote_tar_path = remote_tmp_dir.joinpath("t.tar.gz")
# Copy /tmp/foo local file to /tmp/bar
# in a remote pod in namespace <some-namespace>:
# oc cp /tmp/foo <some-namespace>/<some-pod>:/tmp/bar
target = f"{self.k8s_namespace_name}/{self.pod_name}:{remote_tmp_dir}"
# if you're interested: the way openshift does this is that
# it creates a tarball locally
# and streams it via exec into the container to a pod process
run_command(["oc", "cp", tmp_tarball_path, target])
unpack_cmd = [
"tar",
"--preserve-permissions",
"-xzf",
str(remote_tar_path),
"-C",
str(pod_dir),
]
self._do_exec(unpack_cmd)
finally:
self._do_exec(["rm", "-rf", str(remote_tmp_dir)])
def _copy_path_from_pod(self, local_dir: Path, pod_dir: Path):
"""
copy content of a dir from pod to local dir
:param local_dir: path to the local dir
:param pod_dir: path within the pod
"""
run_command([
"oc",
"rsync",
"--delete", # delete files in local_dir which are not in pod_dir
"--quiet=true", # avoid huge logs
f"--namespace={self.k8s_namespace_name}",
f"{self.pod_name}:{pod_dir}/", # trailing / to copy only content of dir
f"{local_dir}",
])
def _copy_path_from_pod(self, local_dir: Path, pod_dir: Path):
"""
copy content of a dir from pod locally
:param local_dir: path to the local dir
:param pod_dir: path within the pod
"""
remote_tmp_dir = Path(self._do_exec(["mktemp", "-d"]).strip())
try:
remote_tar_path = remote_tmp_dir.joinpath("t.tar.gz")
grc = (
rf"cd {pod_dir} && ls -d -1 .* * | egrep -v '^\.$' | egrep -v '^\.\.$'"
)
tar_cmd = f"tar -czf {remote_tar_path} -C {pod_dir} $({grc})"
pack_cmd = ["bash", "-c", tar_cmd]
self._do_exec(pack_cmd)
# Copy /tmp/foo from a remote pod to /tmp/bar locally
# oc cp <some-namespace>/<some-pod>:/tmp/foo /tmp/bar
target = f"{self.k8s_namespace_name}/{self.pod_name}:{remote_tar_path}"
fd, tmp_tarball_path = tempfile.mkstemp()
try:
os.close(fd)
logger.info(f"copy {target} -> {tmp_tarball_path}")
run_command(["oc", "cp", target, tmp_tarball_path])
purge_dir_content(local_dir)
unpack_cmd = [
"tar",
"--preserve-permissions",
"-xzf",
tmp_tarball_path,
"-C",
str(local_dir),
]
run_command(unpack_cmd)
finally:
os.unlink(tmp_tarball_path)
finally:
self._do_exec(["rm", "-rf", str(remote_tmp_dir)])
def test_md_e2e(tmpdir, git_url, branch):
# running in k8s
if "KUBERNETES_SERVICE_HOST" in os.environ:
t = Path(SANDCASTLE_MOUNTPOINT).joinpath(f"clone-{get_timestamp_now()}")
else:
t = Path(tmpdir)
m_dir = MappedDir(str(t), SANDCASTLE_MOUNTPOINT, with_interim_pvc=True)
run_command(["git", "clone", "-b", branch, git_url, t])
o = Sandcastle(
image_reference=SANDBOX_IMAGE, k8s_namespace_name=NAMESPACE, mapped_dir=m_dir
)
o.run()
try:
o.exec(command=["packit", "--debug", "srpm"])
assert list(t.glob("*.src.rpm"))
o.exec(command=["packit", "--help"])
finally:
o.delete_pod()
def _copy_path_to_pod(self,
local_path: Path,
pod_dir: Path,
no_perms: bool = False):
"""
copy local_path (dir or file) inside pod
:param local_path: path to a local file or a dir
:param pod_dir: Directory within the pod where the content of local_path is extracted
:param no_perms: If true, do not transfer permissions
https://www.openshift.com/blog/transferring-files-in-and-out-of-containers-in-openshift-part-1-manually-copying-files
"""
if local_path.is_dir():
exclude = "--exclude=lost+found" # can't touch that
include = "--include=[]" # default
elif local_path.is_file():
exclude = "--exclude=*" # everything
include = f"--include={local_path.name}" # only the file
local_path = local_path.parent
else:
raise SandcastleException(
f"{local_path} is neither a dir nor a file")
cmd = [
"oc",
"rsync",
exclude,
include,
"--quiet=true", # avoid huge logs
f"--namespace={self.k8s_namespace_name}",
f"{local_path}/", # ??? rsync doesn't work without the trailing /
f"{self.pod_name}:{pod_dir}",
]
if no_perms:
cmd += ["--no-perms"]
run_command(cmd)
def test_md_new_namespace(tmpdir):
t = Path(tmpdir)
m_dir = MappedDir(str(t), SANDCASTLE_MOUNTPOINT, with_interim_pvc=True)
d = t.joinpath("dir")
d.mkdir()
d.joinpath("file").write_text("asd")
# running within openshift
namespace = os.getenv("SANDCASTLE_TESTS_NAMESPACE")
if not namespace:
# running on a host - you can't create new projects from inside a pod
namespace = f"sandcastle-tests-{get_timestamp_now()}"
c = ["oc", "new-project", namespace]
run_command(c)
try:
o = Sandcastle(
image_reference=SANDBOX_IMAGE,
k8s_namespace_name=namespace,
mapped_dir=m_dir,
)
o.run()
try:
o.exec(command=["ls", "-lha", f"./dir/file"])
assert d.joinpath("file").read_text() == "asd"
cmd = [
"bash",
"-c",
"curl -skL https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/metrics",
]
out = o.exec(command=cmd)
j = json.loads(out)
# a small proof we are safe
assert j["reason"] == "Forbidden"
finally:
o.delete_pod()
finally:
if not os.getenv("SANDCASTLE_TESTS_NAMESPACE"):
run_command(["oc", "delete", "project", namespace])
run_command(["oc", "project", NAMESPACE])
def run_test_within_pod(test_path: str,
with_pv_at: Optional[str] = None,
new_namespace: bool = False):
"""
run selected test from within an openshift pod
:param test_path: relative path to the test
:param with_pv_at: path to PV within the pod
:param new_namespace: create new namespace and pass it via env var
"""
config.load_kube_config()
configuration = client.Configuration()
assert configuration.api_key
api = client.CoreV1Api(client.ApiClient(configuration))
pod_name = f"test-orchestrator-{get_timestamp_now()}"
cont_cmd = [
"bash",
"-c",
"ls -lha "
f"&& pytest-3 -vv -l -p no:cacheprovider {test_path}",
]
container: Dict[str, Any] = {
"image": TEST_IMAGE_NAME,
"name": pod_name,
"tty": True, # corols
"command": cont_cmd,
"imagePullPolicy": "Never",
"env": [],
}
test_namespace = None
if new_namespace:
test_namespace = f"sandcastle-tests-{get_timestamp_now()}"
c = ["oc", "new-project", test_namespace]
run_command(c)
c = [
"oc",
"adm",
"-n",
test_namespace,
"policy",
"add-role-to-user",
"edit",
f"system:serviceaccount:{NAMESPACE}:default",
]
run_command(c)
container["env"] += [{
"name": "SANDCASTLE_TESTS_NAMESPACE",
"value": test_namespace
}]
spec = {"containers": [container], "restartPolicy": "Never"}
pod_manifest = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": pod_name
},
"spec": spec,
}
if with_pv_at:
cleaned_test_name = clean_string(test_path)
ts = get_timestamp_now()
volume_name = f"{cleaned_test_name}-{ts}-vol"[-63:]
claim_name = f"{cleaned_test_name}-{ts}-pvc"[-63:]
container["env"] = [{"name": "SANDCASTLE_PVC", "value": claim_name}]
pvc_dict = {
"kind": "PersistentVolumeClaim",
"spec": {
"accessModes": ["ReadWriteMany"],
"resources": {
"requests": {
"storage": "1Gi"
}
},
},
"apiVersion": "v1",
"metadata": {
"name": claim_name
},
}
api.create_namespaced_persistent_volume_claim(NAMESPACE, pvc_dict)
container["volumeMounts"] = [{
"mountPath": with_pv_at,
"name": volume_name
}]
spec["volumes"] = [{
"name": volume_name,
"persistentVolumeClaim": {
"claimName": claim_name
}
}]
try:
api.delete_namespaced_pod(pod_name, NAMESPACE, body=V1DeleteOptions())
except ApiException as ex:
if ex.status != 404:
raise
try:
api.create_namespaced_pod(body=pod_manifest, namespace=NAMESPACE)
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not start on time.")
info = api.read_namespaced_pod(pod_name, NAMESPACE)
if info.status.phase == "Running":
break
time.sleep(2.0)
counter -= 1
print(
api.read_namespaced_pod_log(name=pod_name,
namespace=NAMESPACE,
follow=True))
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not finish on time.")
info = api.read_namespaced_pod(pod_name, NAMESPACE)
if info.status.phase == "Succeeded":
break
if info.status.phase == "Failed":
raise RuntimeError("Test failed")
time.sleep(2.0)
counter -= 1
finally:
print(
api.read_namespaced_pod_log(name=pod_name,
namespace=NAMESPACE,
follow=True))
api.delete_namespaced_pod(pod_name, NAMESPACE, body=V1DeleteOptions())
if new_namespace:
run_command(["oc", "delete", "project", test_namespace])
if with_pv_at:
api.delete_namespaced_persistent_volume_claim(
name=claim_name, namespace=NAMESPACE, body=V1DeleteOptions())
def run_test_within_pod(test_path: str,
with_pv_at: Optional[str] = None,
new_namespace: bool = False):
"""
run selected test from within an openshift pod
:param test_path: relative path to the test
:param with_pv_at: path to PV within the pod
:param new_namespace: create new namespace and pass it via env var
"""
# this will connect to the cluster you have active right now - see `oc status`
current_context = check_output(["oc", "config",
"current-context"]).strip().decode()
api_client = new_client_from_config(context=current_context)
api = client.CoreV1Api(api_client)
# you need this for minishift; or do `eval $(minishift docker-env) && make build`
# # FIXME: get this from env or cli (`minishift openshift registry`)
# also it doesn't work for me right now, unable to reach minishift's docker registry
# registry = "172.30.1.1:5000"
# openshift_image_name = f"{registry}/myproject/{TEST_IMAGE_BASENAME}"
# # {'authorization': 'Bearer pRW5rGmqgBREnCeVweLcHbEhXvluvG1cImWfIrxWJ2A'}
# api_token = list(api_client.configuration.api_key.values())[0].split(" ")[1]
# check_call(["docker", "login", "-u", "developer", "-p", api_token, registry])
# check_call(["docker", "tag", TEST_IMAGE_NAME, openshift_image_name])
# check_call(["docker", "push", openshift_image_name])
pod_name = f"test-orchestrator-{get_timestamp_now()}"
cont_cmd = [
"bash",
"-c",
"~/setup_env_in_openshift.sh "
"&& ls -lha && pwd && id "
f"&& pytest-3 -vv -l -p no:cacheprovider {test_path}",
]
container: Dict[str, Any] = {
"image": TEST_IMAGE_NAME, # make sure this is correct
"name": pod_name,
"tty": True, # corols
"command": cont_cmd,
"imagePullPolicy": "Never",
"env": [],
}
user = f"system:serviceaccount:{NAMESPACE}:default"
enable_user_access_namespace(user, NAMESPACE)
test_namespace = None
if new_namespace:
test_namespace = f"sandcastle-tests-{get_timestamp_now()}"
c = ["oc", "new-project", test_namespace]
run_command(c)
enable_user_access_namespace(user, test_namespace)
container["env"] += [{
"name": "SANDCASTLE_TESTS_NAMESPACE",
"value": test_namespace
}]
spec = {"containers": [container], "restartPolicy": "Never"}
pod_manifest = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": pod_name
},
"spec": spec,
}
if with_pv_at:
cleaned_test_name = clean_string(test_path)
ts = get_timestamp_now()
volume_name = f"{cleaned_test_name}-{ts}-vol"[-63:]
claim_name = f"{cleaned_test_name}-{ts}-pvc"[-63:]
container["env"] = [{"name": "SANDCASTLE_PVC", "value": claim_name}]
pvc_dict = {
"kind": "PersistentVolumeClaim",
"spec": {
"accessModes": ["ReadWriteMany"],
"resources": {
"requests": {
"storage": "1Gi"
}
},
},
"apiVersion": "v1",
"metadata": {
"name": claim_name
},
}
api.create_namespaced_persistent_volume_claim(NAMESPACE, pvc_dict)
container["volumeMounts"] = [{
"mountPath": with_pv_at,
"name": volume_name
}]
spec["volumes"] = [{
"name": volume_name,
"persistentVolumeClaim": {
"claimName": claim_name
}
}]
try:
api.delete_namespaced_pod(pod_name, NAMESPACE, body=V1DeleteOptions())
except ApiException as ex:
if ex.status != 404:
raise
try:
api.create_namespaced_pod(body=pod_manifest, namespace=NAMESPACE)
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not start on time.")
info = api.read_namespaced_pod(pod_name, NAMESPACE)
if info.status.phase == "Running":
break
elif info.status.phase == "Failed":
print_pod_logs(api, pod_name, NAMESPACE)
raise RuntimeError("The pod failed to start.")
time.sleep(2.0)
counter -= 1
print_pod_logs(api, pod_name, NAMESPACE)
counter = 15
while True:
if counter < 0:
raise RuntimeError("Pod did not finish on time.")
info = api.read_namespaced_pod(pod_name, NAMESPACE)
if info.status.phase == "Succeeded":
break
if info.status.phase == "Failed":
raise RuntimeError("Test failed")
time.sleep(2.0)
counter -= 1
finally:
print_pod_logs(api, pod_name, NAMESPACE)
api.delete_namespaced_pod(pod_name, NAMESPACE, body=V1DeleteOptions())
if new_namespace:
run_command(["oc", "delete", "project", test_namespace])
if with_pv_at:
api.delete_namespaced_persistent_volume_claim(
name=claim_name, namespace=NAMESPACE, body=V1DeleteOptions())
def enable_user_access_namespace(user: str, namespace: str):
c = [
"oc", "adm", "-n", namespace, "policy", "add-role-to-user", "edit",
user
]
run_command(c)
