def _test_scp_put_files(host, user, password, temp_files):
"""
Test scp copy several files to remote:
$ scp local_file1 local_file2 local_file2 user@remote:/dest/path
"""
remote_path = temp_files['remote_dst_dir']
local_files = [
os.path.join(temp_files['local_src_dir'], f)
for f in temp_files['local_files']
]
remote_files = [
os.path.join(remote_path, os.path.basename(v)) for v in local_files
]
logger.debug('===== scp put multiple files =====')
cmd = 'scp {} {}@{}:{}'.format(' '.join(local_files), user, host,
remote_path)
_run_scp_command(cmd, user, host, password)
for local_path, remote_path in zip(local_files, remote_files):
retry(validate_transfer,
local_path=local_path,
remote_path=remote_path,
user=user,
host=host,
password=password,
tries=3,
interval=1)
def _test_sftp_get_dir(host, user, password, temp_files):
"""
Test sftp get directory
"""
remote_path = temp_files['remote_src_dir']
local_path = temp_files['local_dst_dir']
logger.debug('==== sftp get directory ====')
_sftp_communicate(remote_path,
local_path,
user,
host,
password,
command='get')
basename = os.path.basename(os.path.normpath(remote_path))
local_path = os.path.join(local_path, basename)
retry(validate_transfer,
local_path=local_path,
remote_path=remote_path,
user=user,
host=host,
password=password,
tries=3,
interval=1)
def test_cant_start_resized_pod_if_cpu_is_low(cluster):
cluster.set_system_setting("2", name="cpu_multiplier")
cluster.set_system_setting("20", name="memory_multiplier")
template = "wordpress.yaml"
log_debug("Starting first pod")
# TODO: remove sleep after fixing AC-5403
sleep(10)
pod = cluster.pods.create_pa(template, wait_for_status='running',
pod_name="wordpress1")
log_debug("Starting second pod")
cluster.pods.create_pa(template, wait_for_status='running',
pod_name="wordpress2")
log_debug("Starting third pod")
cluster.pods.create_pa(template, wait_for_status='running',
pod_name="wordpress3")
pod.change_kubes(kubes=9, container_name="wordpress")
log_debug("Make sure there is warning about lack of CPUs in k8s")
cmd = 'get events --namespace {} -o json'.format(pod.pod_id)
def _check_presence_of_cpu_warning():
_, out, _ = cluster.true_kubectl(cmd)
try:
next(e for e in json.loads(out)['items']
if e["reason"] == "FailedScheduling"
and K8S_CPU_LACK_ERROR in e["message"])
except StopIteration:
raise _NoResourseLackErrorInK8s("There aren't event with warning "
"about lack of CPU in k8s")
retry(_check_presence_of_cpu_warning, tries=10, interval=3)
def start(self):
self._rnd_sleep()
self._reserve_ips()
log_dict(self.env, "Cluster settings:")
LOG.debug("Running vagrant up...")
try:
with log_timing_ctx("vagrant up --no-provision"):
retry(self.vagrant.up,
tries=3,
interval=15,
provider="opennebula",
no_provision=True)
self.created_at = datetime.utcnow()
self._log_vm_ips()
except subprocess.CalledProcessError:
raise VmCreateError('Failed to create VMs in OpenNebula')
finally:
self._print_vagrant_log()
LOG.debug("Running vagrant provision...")
try:
with log_timing_ctx("vagrant provision"):
self.vagrant.provision()
except subprocess.CalledProcessError:
raise VmProvisionError('Failed Ansible provision')
finally:
self._print_vagrant_log()
self._save_reserved_ips()
def healthcheck(self):
if not (self.open_all_ports or self.ports):
raise Exception(
"Cannot perform nginx healthcheck without public IP")
self._generic_healthcheck()
# if shared IP is used, 404 is returned in a response to GET on
# pod's domain name for up to 40 seconds after pod is started
utils.retry(self.do_GET, tries=5, interval=10)
utils.assert_in("Welcome to nginx!", self.do_GET())
def wait_for_service_pods(self):
def _check_service_pods():
_, response, _ = self.kdctl('pods list --owner kuberdock-internal',
out_as_dict=True)
statuses = (pod['status'] for pod in response['data'])
if not all(s == 'running' for s in statuses):
raise exceptions.ServicePodsNotReady()
utils.retry(_check_service_pods, tries=40, interval=15)
def wait_ssh_conn(self, hosts, tries=20, interval=10):
if not isinstance(hosts, (list, tuple)):
hosts = [hosts]
for host in hosts:
utils.retry(self.ssh_exec,
tries,
interval,
node=host,
cmd="echo OK")
LOG.debug("SSH connection to host '{}' is OK.".format(host))
def _assert_ssh_cmd_output_expected(cmd,
expected=['root'],
using_bashc=False):
def _assertion():
out = _run_ssh_pexpect(cmd, password, using_bashc)
for t in expected:
assert t in out
retry(_assertion, tries=3, interval=1)
logger.debug(u'{}Validation: OK{}'.format(Fore.GREEN, Style.RESET_ALL))
def create(cls, cluster, image, name, kube_type, kubes, open_all_ports,
restart_policy, pvs, owner, password, ports_to_open, domain):
"""
Create new pod in kuberdock
:param open_all_ports: if true, open all ports of image (does not mean
these are Public IP ports, depends on a cluster setup)
:param ports_to_open: if open_all_ports is False, open only the ports
from this list
:return: object via which Kuberdock pod can be managed
"""
def _get_image_ports(img):
_, out, _ = cluster.kcli('image_info {}'.format(img),
out_as_dict=True,
user=owner)
return [
cls.Port(int(port['number']), port['protocol'])
for port in out['ports']
]
def _ports_to_dict(ports):
"""
:return: list of dictionaries with ports, necessary for
creation of general pod via kcli2
"""
ports_list = []
for port in ports:
ports_list.append(
dict(containerPort=port.port,
hostPort=port.port,
isPublic=(open_all_ports
or port.port in ports_to_open),
protocol=port.proto))
return ports_list
escaped_name = pipes.quote(name)
kube_types = {"Tiny": 0, "Standard": 1, "High memory": 2}
pod_spec = dict(kube_type=kube_types[kube_type],
restartPolicy=restart_policy,
name=escaped_name)
container = dict(kubes=kubes,
image=image,
name=utils.get_rnd_low_string(length=11))
ports = utils.retry(_get_image_ports, img=image)
container.update(ports=_ports_to_dict(ports))
if pvs is not None:
container.update(volumeMounts=[pv.volume_mount_dict for pv in pvs])
pod_spec.update(volumes=[pv.volume_dict for pv in pvs])
pod_spec.update(containers=[container])
if domain:
pod_spec["domain"] = domain
pod_spec = json.dumps(pod_spec, ensure_ascii=False)
_, out, _ = cluster.kcli2(u"pods create '{}'".format(pod_spec),
out_as_dict=True,
user=owner,
password=(password or owner))
this_pod_class = cls._get_pod_class(image)
return this_pod_class(cluster, image, name, kube_type, kubes,
open_all_ports, restart_policy, pvs, owner)
def preload_docker_image(self, image, node=None):
"""
Pulls given docker image in advance either for a specified node or
for all nodes in a cluster. Useful in cases when a node can be quite
overloaded. Then the image pull can take quite a long time and when you
create a test pod you can't be sure if the image pull takes too long
or something bad happened during pod creation. In these situations
use this function. With it you will be sure that image is in place.
:param image: name in a docker format (nginx, nginx:latest, etc)
:param node: a hostname. If not specified - replaced with all nodes in
cluster
"""
nodes = node if node is not None else self.node_names
# TODO parallel execution on all nodes
for node in nodes:
utils.retry(self.docker,
interval=5,
tries=3,
cmd='pull {}'.format(image),
node=node)
def _test_sftp_get_file(host, user, password, temp_files):
"""
Test sftp get file
"""
remote_path = temp_files['remote_src_file']
local_path = temp_files['local_dst_file']
logger.debug('==== sftp get file ====')
_sftp_communicate(remote_path,
local_path,
user,
host,
password,
command='get')
retry(validate_transfer,
local_path=local_path,
remote_path=remote_path,
user=user,
host=host,
password=password,
tries=3,
interval=1)
def _add_domain(cluster):
with open("tests_integration/assets/cpanel_credentials.json") as f:
creds = json.load(f)
for k, v in creds.items():
if k != "domain":
cluster.set_system_setting(escape_command_arg(v), name=k)
# Wait till DNS Pod is running
# It's impossible to import KUBERDOCK_DNS_POD_NAME from kubedock/kapi/nodes
# because nodes tries import from flask which isn't installed on the CI host
dns_pod = KDPod.get_internal_pod(cluster, "kuberdock-dns")
dns_pod.wait_for_status("running")
cluster.domains.add(name=creds["domain"], ignore_duplicates=True)
# Make sure ingress controller has been created
retry(KDPod.get_internal_pod,
tries=6,
interval=10,
cluster=cluster,
pod_name=KUBERDOCK_INGRESS_POD_NAME)
ingress_pod = KDPod.get_internal_pod(cluster, KUBERDOCK_INGRESS_POD_NAME)
ingress_pod.wait_for_status("running")
def _test_scp_put_dir(host, user, password, temp_files):
"""
Test scp copy a directory to remote:
$ scp -r local_directory user@host:/dst/path
"""
local_path = temp_files['local_src_dir']
remote_path = temp_files['remote_dst_dir']
logger.debug('===== scp put directory =====')
cmd = 'scp -r {} {}@{}:{}'.format(local_path, user, host, remote_path)
basename = os.path.basename(os.path.normpath(local_path))
remote_dir = os.path.join(remote_path, basename)
_run_scp_command(cmd, user, host, password)
retry(validate_transfer,
local_path=local_path,
remote_path=remote_dir,
user=user,
host=host,
password=password,
tries=3,
interval=1)
def _test_scp_get_file(host, user, password, temp_files):
"""
Test scp copy a single file from remote:
$ scp user@remote:/src/file/path dst/path
"""
remote_file = temp_files['remote_src_file']
local_dst_dir = temp_files['local_dst_dir']
basename = os.path.basename(os.path.normpath(remote_file))
local_file = os.path.join(local_dst_dir, basename)
logger.debug('==== scp get a single file ====')
cmd = 'scp {}@{}:{} {}'.format(user, host, remote_file, local_dst_dir)
_run_scp_command(cmd, user, host, password)
retry(validate_transfer,
local_path=local_file,
remote_path=remote_file,
user=user,
host=host,
password=password,
tries=3,
interval=1)
def _test_ssh_to_stopped_container(host, user, password):
"""
Try to ssh into stopped containers
"""
logger.debug('==== ssh to stopped containers ====')
cmd = 'ssh {}@{} ls -l /'.format(user, host)
def _assert_container_stopped(cmd, password, container_id):
out = _run_ssh_pexpect(cmd, password)
# Container is removed from docker after stopping. Sometimes this
# method is called before removing, sometimes - later. Therefore any
# of messages "Container ... is not running" or "No such
# container..." is expected
assert re.search(
r'(Container {0}.* is not running|'
r'No such container: {0})'.format(container_id), out)
retry(_assert_container_stopped,
cmd=cmd,
password=password,
container_id=user,
tries=3,
interval=1)
def _test_scp_put_file(host, user, password, temp_files):
"""
Test scp copy a single file on remote:
$ scp local_file user@remote:/dest/path
"""
local_path = temp_files['local_src_file']
remote_path = os.path.dirname(temp_files['remote_dst_file'])
logger.debug('===== scp put single file =====')
cmd = 'scp {} {}@{}:{}'.format(local_path, user, host, remote_path)
basename = os.path.basename(os.path.normpath(local_path))
remote_file = os.path.join(remote_path, basename)
_run_scp_command(cmd, user, host, password)
retry(validate_transfer,
local_path=local_path,
remote_path=remote_file,
user=user,
host=host,
password=password,
tries=3,
interval=1)
def _test_scp_get_dir(host, user, password, temp_files):
"""
Test scp copy directory from remote:
$ scp -r user@remote:path/to/src/dir dst/path
"""
remote_dir = temp_files['remote_src_dir']
local_dst_dir = temp_files['local_dst_dir']
basename = os.path.basename(os.path.normpath(remote_dir))
local_dir = os.path.join(local_dst_dir, basename)
logger.debug('==== scp get directory ====')
cmd = 'scp -r {}@{}:{} {}'.format(user, host, remote_dir, local_dst_dir)
_run_scp_command(cmd, user, host, password)
retry(validate_transfer,
local_path=local_dir,
remote_path=remote_dir,
user=user,
host=host,
password=password,
tries=3,
interval=1)
def wait_http_resp(self,
scheme="http",
path='/',
port=None,
code=200,
timeout=3,
tries=60,
internal=3):
port = port or self.HTTP_PORT
if port:
url = '{0}://{1}:{2}{3}'.format(scheme, self.host, port, path)
else:
url = '{0}://{1}{2}'.format(scheme, self.host, path)
LOG.debug('Expecting for response code {code} on url {url}, '
'total retries: {tries}'.format(code=code,
url=url,
tries=tries,
port=port))
def check(*args, **kwargs):
req = urllib2.urlopen(url, timeout=timeout)
assert req.code == code
utils.retry(check, tries=tries, internal=internal)
def _test_scp_get_files(host, user, password, temp_files, file_prefix='file'):
"""
Test scp copy several files from remote:
$ scp user@remote:path/to/files/{file1,file2,file3} dst/path
"""
logger.debug('==== scp get multiple files ====')
files = temp_files['remote_files']
remote_path = temp_files['remote_src_dir']
local_path = temp_files['local_dst_dir']
cmd = 'scp {}@{}:{}/{{{}}} {}'.format(user, host, remote_path,
','.join(files), local_path)
_run_scp_command(cmd, user, host, password)
remote_files = [
os.path.join(temp_files['remote_src_dir'], f)
for f in temp_files['remote_files']
]
local_files = [
os.path.join(local_path, '{}'.format(os.path.basename(f)))
for f in remote_files
]
remote_files = [os.path.join(remote_path, f) for f in files]
# validate each file separately
for local_path, remote_path in zip(local_files, remote_files):
retry(validate_transfer,
local_path=local_path,
remote_path=remote_path,
user=user,
host=host,
password=password,
tries=3,
interval=1)
def _compare_files(local_path, remote_path, ssh):
"""
Compare hashes of two files, one on local and another one on remote server
:param local_path: path to file on the local server
:param remote_path: path to file on the remote server
:param ssh: SSHClient instance to communicate with remote server
:returns: True/False on success/fail
:rtype: bool
"""
logger.debug(u'{}Comparing files. host: {} and container: {}{}'.format(
Style.DIM, local_path, remote_path, Style.RESET_ALL))
# Sometimes ssh_exec exits with a 0 code, but no stdout can be read,
# so we moved the file hash call inside a function.
def _remote_md5sum(ssh, path):
remote_cmd = 'md5sum {}'.format(path)
ret_code, out, err = ssh_exec(ssh=ssh, cmd=remote_cmd, get_pty=True)
_remote_file_hash = out.strip().split()[0].strip()
return _remote_file_hash
# Get hash of the remote file.
remote_file_hash = retry(_remote_md5sum,
ssh=ssh,
path=remote_path,
tries=3,
interval=1)
# Get hash of the local file
try:
with open(local_path) as f:
local_file_hash = hashlib.md5(f.read()).hexdigest()
except (OSError, IOError) as e:
raise FileTransferValidationFailed(str(e))
# Compare hashes
if local_file_hash != remote_file_hash:
message = 'Hashes not equal. Host: {} != container: {}'.format(
local_file_hash, remote_file_hash)
logger.debug(u'{}host: {} container: {}{}'.format(
Fore.RED, local_path, remote_path, Style.RESET_ALL))
raise FileTransferValidationFailed(message)
logger.debug(u'{}Validation: OK{}'.format(Fore.GREEN, Style.RESET_ALL))
return True
def _get_inventory(self):
with tempfile.NamedTemporaryFile(prefix="inv-", delete=False) as inv:
hosts = {}
for name in [
'master',
] + self.node_names:
hosts[name] = retry(self.get_host_ip,
tries=3,
interval=20,
hostname=name)
for host, host_ip in hosts.items():
inv.write('kd_{0} ansible_host={1} '
'ansible_ssh_user=centos '
'ansible_ssh_private_key_file={2}\n'.format(
host, host_ip, self.ssh_key))
inv.write('[master]\nkd_master\n')
hosts.pop('master')
inv.write('[node]\n{0}'.format('\n'.join("kd_{0}".format(node)
for node in hosts)))
return inv.name
def _compare_dirs(local_path, remote_path, ssh):
"""
Compare directories on the local and remote servers
:param local_path: path to the directory on a local server
:param remote_path: path to the directory on a remote server
:returns: True/False on success/fail
:rtype: bool
"""
# Sometimes ssh_exec exits with a 0 code, but no stdout can be read,
# so we moved the find call inside a function.
# Find directories on a remote server at a specified path
def _find_dirs(ssh, path):
remote_cmd = 'find {} -type d'.format(path)
ret_code, out, err = ssh_exec(ssh=ssh, cmd=remote_cmd, get_pty=True)
_remote_dirs = {d.strip() for d in out.strip().split('\n')}
assert any(_remote_dirs)
return _remote_dirs
# Get directories on the remote server
remote_dirs = retry(_find_dirs,
ssh=ssh,
path=remote_path,
tries=3,
interval=1)
# Get directories on the local server
local_dirs = {r for r, d, f in os.walk(local_path)}
# Error if number of directories differ
if local_dirs != remote_dirs:
loc_msg = '\n'.join(local_dirs)
rem_msg = '\n'.join(remote_dirs)
logger.debug(u'\n{}Host dirs:\n{}{}'.format(Fore.RED, loc_msg,
Style.RESET_ALL))
logger.debug(u'\n{}Container dirs:\n{}{}'.format(
Fore.RED, rem_msg, Style.RESET_ALL))
message = 'Number of directories differ'
raise FileTransferValidationFailed(message)
return True
def test_zfs_volumes_mount_properly(cluster):
"""
Automate TestRail case: Deploy with ZFS parameter
https://cloudlinux.testrail.net/index.php?/cases/view/81
"""
image = 'nginx'
pv_name = utils.get_rnd_low_string(prefix='zfs_pv_')
pv_mpath = '/usr/share/nginx/html'
pv = cluster.pvs.add('dummy', pv_name, pv_mpath)
pod = cluster.pods.create(image,
'nginx_zfs_volume_mounts',
pvs=[pv],
ports_to_open=(80, ),
start=True,
wait_for_status='running',
wait_ports=True)
pod_owner = cluster.users.get(name=pod.owner)
pv_mountpoint = os.path.join(ZFS_POOL_MOUNTPOINT, str(pod_owner.get('id')),
pv_name)
check_volume_mounts(cluster, pod, log_msg_prefix='BEFORE NODE REBOOT: ')
utils.log_debug("Write a file 'test.txt' to PV and get it via HTTP", LOG)
c_id = pod.get_container_id(container_image=image)
pod.docker_exec(c_id, 'echo -n TEST > {}/test.txt'.format(pv_mpath))
ret = pod.do_GET(path='/test.txt')
utils.assert_eq('TEST', ret)
# Reboot Node
cluster.nodes.get_node(pod.node).reboot()
utils.wait_for(lambda: c_id != pod.get_container_id(container_image=image))
pod.wait_for_ports()
check_volume_mounts(cluster, pod, log_msg_prefix='AFTER NODE REBOOT: ')
utils.log_debug(
"Make sure that we can get 'test.txt' via HTTP after node reboot", LOG)
ret = pod.do_GET(path='/test.txt')
utils.assert_eq('TEST', ret)
c_id = pod.get_container_id(container_image=image)
utils.log_debug('Restart Pod and check that volumes are mounted correctly')
pod.redeploy()
utils.wait_for(lambda: c_id != pod.get_container_id(container_image=image))
pod.wait_for_status('running')
check_volume_mounts(cluster, pod, log_msg_prefix='AFTER POD RESTART: ')
node = pod.node
pod.delete()
pv.delete()
utils.log_debug(
"Make sure that '{}' is not mounted after PV deletion".format(pv_name),
LOG)
with utils.assert_raises(NonZeroRetCodeException,
expected_ret_codes=GREP_EXIT_CODES):
utils.retry(assert_volume_mounts,
cluster=cluster,
mountpoint=pv_mountpoint,
node=node,
assertion=utils.assert_not_in,
tries=3,
interval=60)
utils.log_debug(
"Make sure that '{}' is not in mountpoints".format(pv_name), LOG)
pool_path = os.path.join(ZFS_POOL, str(pod_owner.get('id')), pv_name)
with utils.assert_raises(NonZeroRetCodeException,
'dataset does not exist'):
utils.retry(assert_zfs_mount_points,
cluster=cluster,
pool_path=pool_path,
volume_mp=pv_mountpoint,
node=node,
assertion=utils.assert_not_in,
tries=3,
interval=60)
def validate_transfer(local_path, remote_path, user, host, password):
"""
Validate that sftp transfer worked correctly. Check corresponding files.
:param local_path: path to file/directory on the local server
:param remote_path: path to file/directory on the remote server
:param user: user on the remote server
:param host: hostname/IP of the remote server
:param password: password for user on the remote server
:returns: None
"""
directory = not os.path.isfile(local_path)
if not directory:
with get_ssh(host, user, password) as conn:
assert _compare_files(local_path, remote_path, conn)
else:
# Sometimes ssh_exec exits with a 0 code, but no stdout can be read,
# so we moved the find call inside a function.
# Find files on a remote server at a specified path
def _find_files(ssh, path):
remote_cmd = 'find {} -type f'.format(path)
ret_code, out, err = ssh_exec(ssh=conn,
cmd=remote_cmd,
get_pty=True)
_remote_files = [f.strip() for f in out.strip().split('\n')]
assert any(_remote_files)
return _remote_files
# Get file-names on the remote server
with get_ssh(host, user, password) as conn:
remote_files = retry(_find_files,
ssh=conn,
path=remote_path,
tries=3,
interval=1)
# Get file names on the local server
local_files = [
os.path.join(d, p) for d, s, ps in os.walk(local_path) for p in ps
]
# Make sure the number of files is the same
if len(local_files) != len(remote_files):
loc_msg = '\n'.join(local_files)
rem_msg = '\n'.join(remote_files)
logger.debug(u'\n{}Local files:\n{}{}'.format(
Fore.RED, loc_msg, Style.RESET_ALL))
logger.debug(u'\n{}Remote files:\n{}{}'.format(
Fore.RED, rem_msg, Style.RESET_ALL))
message = 'Number of files differ'
raise FileTransferValidationFailed(message)
try:
with get_ssh(host, user, password) as conn:
# Validate directories
assert _compare_dirs(local_path=local_path,
remote_path=remote_path,
ssh=conn)
# Validate file hashes
local_files = sorted(local_files)
remote_files = sorted(remote_files)
for local_path, remote_path in zip(local_files, remote_files):
assert _compare_files(local_path, remote_path, conn)
except Exception as e:
raise e
def power_on(self, host):
vm_name = self.vm_names[host]
LOG.debug("VM Power On: '{}'".format(vm_name))
retry(self.vagrant.up, tries=3, vm_name=vm_name)
def ssh_credentials(self):
return utils.retry(self._get_creds, tries=10, interval=1)