def test_both_masters_stopped(start_multi_master):
'''
test when both masters are stopped on minion startup
'''
for host in ['master1', 'master2', 'minion1']:
salt_host = docker_client(host)
if 'minion' in host:
salt_host.exec_run('pkill salt-minion')
else:
salt_host.exec_run('pkill salt-master')
# make sure master is dead
ret = salt_host.exec_run('salt * test.ping')
assert b'Salt request timed out. The master is not responding' in ret.output
# start the minion and let it sit for 5 minutes
# to make sure it doesnt kill process
minion1 = docker_client('minion1')
minion1.exec_run('salt-minion -d')
time.sleep(300)
for master in ['master1', 'master2']:
salt_host = docker_client(master)
salt_host.exec_run('salt-master -d')
master1 = docker_client('master1')
ret = master1.exec_run('salt * test.ping')
assert ret.exit_code == 0
master2 = docker_client('master2')
ret = master2.exec_run('salt * test.ping')
assert ret.exit_code == 0
def test_refresh_pillar_masters(start_multi_master):
'''
test refreshing pillar when masters are up and
when only one is up
'''
# verify both masters are up and can run refresh
for master in ['master1', 'master2']:
salt_host = docker_client(master)
ret = salt_host.exec_run('salt * saltutil.refresh_pillar')
assert ret.exit_code == 0
master1 = docker_client('master1')
master1.exec_run('pkill salt-master')
# make sure master is dead
ret = master1.exec_run('salt * test.ping')
assert b'Salt request timed out. The master is not responding' in ret.output
master2 = docker_client('master2')
ret = master2.exec_run('salt * saltutil.refresh_pillar')
assert ret.exit_code == 0
master2.exec_run('pkill salt-master')
# make sure master is dead
ret = master2.exec_run('salt * test.ping')
assert b'Salt request timed out. The master is not responding' in ret.output
master1.exec_run('salt-master -d')
time.sleep(20)
ret = master1.exec_run('salt * saltutil.refresh_pillar')
assert ret.exit_code == 0
def test_one_master_up_on_startup(start_multi_master):
'''
test when one master is up when minion starts up
'''
for host in ['master2', 'minion1']:
salt_host = docker_client(host)
if 'minion' in host:
salt_host.exec_run('pkill salt-minion')
else:
salt_host.exec_run('pkill salt-master')
# make sure master is dead
ret = salt_host.exec_run('salt * test.ping')
assert b'Salt request timed out. The master is not responding' in ret.output
minion1 = docker_client('minion1')
minion1.exec_run('salt-minion -d')
time.sleep(20)
master1 = docker_client('master1')
ret = master1.exec_run('salt * test.ping')
assert ret.exit_code == 0
master2 = docker_client('master2')
salt_host.exec_run('salt-master -d')
time.sleep(20)
ret = master2.exec_run('salt * test.ping')
assert ret.exit_code == 0
def test_multi_master_failover_masters_stopped_on_start(start_multi_master):
'''
test multi-master failover when both masters are stopped
on start of minion. later on start master and make sure minion
re-connects
'''
# stop master1 and restart minion1
master1 = docker_client('master1')
master2 = docker_client('master2')
minion1 = docker_client('minion1')
for x, y in {
master1: 'master',
master2: 'master',
minion1: 'minion'
}.items():
x.exec_run('pkill salt-{0}'.format(y))
# start minion
minion1.exec_run('salt-minion -d')
time.sleep(200)
for master in [master1, master2]:
ret = master.exec_run('salt * test.ping')
assert b'The master is not responding' in ret.output
master2.exec_run('salt-master -d')
time.sleep(40)
ret = master2.exec_run('salt * test.ping')
assert ret.exit_code == 0
assert b'True' in ret.output
def test_multi_master_failover_restart_both_masters(start_multi_master):
'''
test multi-master failover when both masters are restarted
'''
master1 = docker_client('master1')
master2 = docker_client('master2')
ret = master1.exec_run('salt * test.ping')
assert ret.exit_code == 0
assert b'True' in ret.output
ret = master2.exec_run('salt * test.ping')
assert b'Minions returned with non-zero' in ret.output
# kill both masters
for master in [master1, master2]:
master.exec_run('pkill salt-master')
time.sleep(100)
for master in [master1, master2]:
master.exec_run('salt-master -d')
time.sleep(100)
ret = master1.exec_run('salt * test.ping')
assert ret.exit_code == 0
assert b'True' in ret.output
def test_multi_master_failover_stop_master(start_multi_master):
'''
test multi-master failover when both masters are running
then kill connected master and minion should migrate to
the other master
'''
master1 = docker_client('master1')
ret = master1.exec_run('salt * test.ping')
assert ret.exit_code == 0
assert b'True' in ret.output
master2 = docker_client('master2')
ret = master2.exec_run('salt * test.ping')
assert ret.exit_code == 1
assert b'Minions returned with non-zero' in ret.output
master1.exec_run('pkill salt-master')
time.sleep(40)
ret = master2.exec_run('salt * test.ping')
assert ret.exit_code == 0
assert b'True' in ret.output
ret = master1.exec_run('salt * test.ping')
assert b'The master is not responding' in ret.output
def test_multi_master_failover_stop_second_master(start_multi_master):
'''
test multi-master failover when first master is running
and second is not on startup of minion
'''
master2 = docker_client('master2')
master2.exec_run('pkill salt-master')
minion1 = docker_client('minion1')
minion1.exec_run('pkill salt-minion')
minion1.exec_run('salt-minion -d')
time.sleep(20)
ret = master2.exec_run('salt * test.ping')
assert b'The master is not responding' in ret.output
master1 = docker_client('master1')
ret = master1.exec_run('salt * test.ping')
assert ret.exit_code == 0
assert b'True' in ret.output
master2.exec_run('salt-master -d')
time.sleep(60)
master1.exec_run('pkill salt-master')
time.sleep(60)
ret = master2.exec_run('salt * test.ping')
assert ret.exit_code == 0
assert b'True' in ret.output
ret = master1.exec_run('salt * test.ping')
assert b'The master is not responding' in ret.output
def test_masters_down_minion_cmd(start_multi_master):
'''
test salt-call when both masters are down
'''
for master in ['master1', 'master2']:
salt_host = docker_client(master)
ret = salt_host.exec_run('pkill salt-master')
assert ret.exit_code == 0
minion1 = docker_client('minion1')
ret = minion1.exec_run('salt-call test.ping')
assert b'No master could be reached' in ret.output
def test_multi_master_failover(start_multi_master):
'''
test multi-master failover when both masters are running
only master1 should return miniong communication
'''
master1 = docker_client('master1')
ret = master1.exec_run('salt * test.ping')
assert ret.exit_code == 0
assert b'True' in ret.output
master2 = docker_client('master2')
ret = master2.exec_run('salt * test.ping')
assert ret.exit_code == 1
assert b'Minions returned with non-zero' in ret.output
def inspect_image(self, image, image_path=None):
with docker_utils.docker_client() as docker:
if image_path:
with open(image_path, 'r') as fd:
docker.load_image(fd.read())
image_dict = docker.inspect_image(image)
return image_dict
def delete_sandbox(self, context, sandbox_id):
with docker_utils.docker_client() as docker:
try:
docker.remove_container(sandbox_id, force=True)
except errors.APIError as api_error:
if '404' in str(api_error):
return
raise
def create_sandbox(self, context, container, image='kubernetes/pause'):
with docker_utils.docker_client() as docker:
name = self.get_sandbox_name(container)
response = docker.create_container(image, name=name,
hostname=name[:63])
sandbox_id = response['Id']
docker.start(sandbox_id)
return sandbox_id
def stop(self, container, timeout):
with docker_utils.docker_client() as docker:
if timeout:
docker.stop(container.container_id,
timeout=int(timeout))
else:
docker.stop(container.container_id)
container.status = fields.ContainerStatus.STOPPED
return container
def reboot(self, container, timeout):
with docker_utils.docker_client() as docker:
if timeout:
docker.restart(container.container_id,
timeout=int(timeout))
else:
docker.restart(container.container_id)
container.status = fields.ContainerStatus.RUNNING
return container
def _edit_salt_config(salt_host, conf, content, service):
'''
edit a salt config and restart service
'''
host = docker_client(salt_host)
host.exec_run('pkill {0}'.format(service))
host.exec_run('salt-call --local file.append {0} "{1}"'.format(
conf, content))
host.exec_run('{0} -d'.format(service))
time.sleep(100)
def delete(self, container, force):
with docker_utils.docker_client() as docker:
if container.container_id:
try:
docker.remove_container(container.container_id,
force=force)
except errors.APIError as api_error:
if '404' in str(api_error):
return
raise
def test_multi_second_master(start_multi_master):
'''
test second master stopped
'''
for master in ['master1', 'master2']:
salt_host = docker_client(master)
ret = salt_host.exec_run('salt * test.ping')
assert ret.exit_code == 0
master2 = docker_client('master2')
master2.exec_run('pkill salt-master')
# make sure master is dead
ret = master2.exec_run('salt * test.ping')
assert b'Salt request timed out. The master is not responding' in ret.output
master1 = docker_client('master1')
ret = master1.exec_run('salt * test.ping')
assert ret.exit_code == 0
def test_multi_first_master_down_startup(start_multi_master):
'''
test first master down when minion starts up
'''
# stop master1 and then start minion1
master1 = docker_client('master1')
master1.exec_run('pkill salt-master')
# make sure master is dead
ret = master1.exec_run('salt * test.ping')
assert b'Salt request timed out. The master is not responding' in ret.output
minion1 = docker_client('minion1')
minion1.exec_run('pkill salt-minion')
minion1.exec_run('salt-minion -d')
time.sleep(20)
master2 = docker_client('master2')
ret = master2.exec_run('salt * test.ping')
assert ret.exit_code == 0
def get_addresses(self, context, container):
sandbox_id = self.get_sandbox_id(container)
with docker_utils.docker_client() as docker:
response = docker.inspect_container(sandbox_id)
addr = response["NetworkSettings"]["IPAddress"]
addresses = {
'default': [
{
'addr': addr,
},
],
}
return addresses
def test_multi_master(request, start_multi_master):
'''
test multi-master when both masters are running
'''
transport = ' '.join([
x for x in request.keywords.keys() if x.startswith('test_')
]).split('[')[-1].strip(']')
for master in ['master1', 'master2']:
salt_host = docker_client(master)
ret = salt_host.exec_run('salt * test.ping')
assert ret.exit_code == 0
# verify we are using correct transport
ret = salt_host.exec_run('salt * config.get transport')
assert transport in str(ret.output)
def kill(self, container, signal=None):
with docker_utils.docker_client() as docker:
if signal is None or signal == 'None':
docker.kill(container.container_id)
else:
docker.kill(container.container_id, signal)
try:
response = docker.inspect_container(container.container_id)
except errors.APIError as api_error:
if '404' in str(api_error):
container.status = fields.ContainerStatus.ERROR
return container
raise
self._populate_container(container, response)
return container
def show(self, container):
with docker_utils.docker_client() as docker:
if container.container_id is None:
return container
response = None
try:
response = docker.inspect_container(container.container_id)
except errors.APIError as api_error:
if '404' in str(api_error):
container.status = fields.ContainerStatus.ERROR
return container
raise
self._populate_container(container, response)
return container
def update(self, container):
patch = container.obj_get_changes()
args = {}
memory = patch.get('memory')
if memory is not None:
args['mem_limit'] = memory
cpu = patch.get('cpu')
if cpu is not None:
args['cpu_quota'] = int(100000 * cpu)
args['cpu_period'] = 100000
with docker_utils.docker_client() as docker:
try:
resp = docker.update_container(container.container_id, **args)
return resp
except errors.APIError:
raise
def create(self, context, container, sandbox_id, image):
with docker_utils.docker_client() as docker:
name = container.name
if image['path']:
with open(image['path'], 'r') as fd:
docker.load_image(fd.read())
image = container.image
kwargs = {
'name': self.get_container_name(container),
'command': container.command,
'environment': container.environment,
'working_dir': container.workdir,
'labels': container.labels,
'tty': container.tty,
'stdin_open': container.stdin_open,
}
host_config = {}
host_config['network_mode'] = 'container:%s' % sandbox_id
# TODO(hongbin): Uncomment this after docker-py add support for
# container mode for pid namespace.
# host_config['pid_mode'] = 'container:%s' % sandbox_id
host_config['ipc_mode'] = 'container:%s' % sandbox_id
host_config['volumes_from'] = sandbox_id
if container.memory is not None:
host_config['mem_limit'] = container.memory
if container.cpu is not None:
host_config['cpu_quota'] = int(100000 * container.cpu)
host_config['cpu_period'] = 100000
if container.restart_policy is not None:
count = int(container.restart_policy['MaximumRetryCount'])
name = container.restart_policy['Name']
host_config['restart_policy'] = {'Name': name,
'MaximumRetryCount': count}
kwargs['host_config'] = docker.create_host_config(**host_config)
response = docker.create_container(image, **kwargs)
container.container_id = response['Id']
container.status = fields.ContainerStatus.STOPPED
container.save(context)
return container
def pause(self, container):
with docker_utils.docker_client() as docker:
docker.pause(container.container_id)
container.status = fields.ContainerStatus.PAUSED
return container
def unpause(self, container):
with docker_utils.docker_client() as docker:
docker.unpause(container.container_id)
container.status = fields.ContainerStatus.RUNNING
return container
def show_logs(self, container):
with docker_utils.docker_client() as docker:
return docker.get_container_logs(container.container_id)
def execute(self, container, command):
with docker_utils.docker_client() as docker:
create_res = docker.exec_create(
container.container_id, command, True, True, False)
exec_output = docker.exec_start(create_res, False, False, False)
return exec_output
def stop_sandbox(self, context, sandbox_id):
with docker_utils.docker_client() as docker:
docker.stop(sandbox_id)
def images(self, repo, quiet=False):
with docker_utils.docker_client() as docker:
response = docker.images(repo, quiet)
return response
def list(self):
with docker_utils.docker_client() as docker:
return docker.list_instances()