def test_metronome(dcos_api_session: DcosApiSession) -> None:
job = {
'description': 'Test Metronome API regressions',
'id': 'test.metronome',
'run': {
'cmd': 'ls',
'docker': {'image': 'busybox:latest'},
'cpus': 1,
'mem': 512,
'disk': 0,
'user': '******',
'restart': {'policy': 'ON_FAILURE'}
}
}
dcos_api_session.metronome_one_off(job)
def test_checks_cli(dcos_api_session: DcosApiSession) -> None:
base_cmd = [
'/opt/mesosphere/bin/dcos-shell',
'dcos-check-runner',
'check',
]
test_uuid = uuid.uuid4().hex
# Poststart node checks should pass.
dcos_api_session.metronome_one_off({
'id': 'test-checks-node-poststart-' + test_uuid,
'run': {
'cpus': .1,
'mem': 128,
'disk': 0,
'cmd': ' '.join(base_cmd + ['node-poststart']),
},
})
# Cluster checks should pass.
dcos_api_session.metronome_one_off({
'id': 'test-checks-cluster-' + test_uuid,
'run': {
'cpus': .1,
'mem': 128,
'disk': 0,
'cmd': ' '.join(base_cmd + ['cluster']),
},
})
# Check runner should only use the PATH and LD_LIBRARY_PATH from check config.
dcos_api_session.metronome_one_off({
'id': 'test-checks-env-' + test_uuid,
'run': {
'cpus':
.1,
'mem':
128,
'disk':
0,
'cmd':
' '.join([
'env',
'PATH=badvalue',
'LD_LIBRARY_PATH=badvalue',
'/opt/mesosphere/bin/dcos-check-runner',
'check',
'node-poststart',
]),
},
})
def test_if_marathon_app_can_be_deployed_with_nfs_csi_volume(
dcos_api_session: DcosApiSession) -> None:
"""Marathon app deployment integration test using an NFS CSI volume.
This test verifies that a Marathon app can be deployed which attaches to
an NFS volume provided by the NFS CSI plugin. In order to accomplish this,
we must first set up an NFS share on one agent.
"""
# We will run an NFS server on one agent and an app on another agent to
# verify CSI volume functionality.
if len(dcos_api_session.slaves) < 2:
pytest.skip("CSI Volume Tests require a minimum of two agents.")
expanded_config = test_helpers.get_expanded_config()
if expanded_config.get('security') == 'strict':
pytest.skip(
'Cannot setup NFS server as root user with EE strict mode enabled')
test_uuid = uuid.uuid4().hex
hosts = dcos_api_session.slaves[0], dcos_api_session.slaves[1]
# A helper to run a Metronome job as root to clean up the NFS share on an agent.
# We define this here so that it can be used during error handling.
def cleanup_nfs() -> None:
cleanup_command = """
sudo systemctl stop nfs-server && \
echo '' | sudo tee /etc/exports && \
sudo systemctl restart nfs-utils && \
sudo exportfs -arv && \
sudo rm -rf /var/lib/dcos-nfs-shares/test-volume-001
"""
cleanup_job = {
'description': 'Clean up NFS share',
'id': 'nfs-share-cleanup-{}'.format(test_uuid),
'run': {
'cmd': cleanup_command,
'cpus': 0.5,
'mem': 256,
'disk': 32,
'user': '******',
'restart': {
'policy': 'ON_FAILURE'
},
'placement': {
'constraints': [{
'attribute': '@hostname',
'operator': 'LIKE',
'value': hosts[0]
}]
}
}
}
dcos_api_session.metronome_one_off(cleanup_job)
# Run a Metronome job as root to set up the NFS share on an agent.
command = """sudo mkdir -p /var/lib/dcos-nfs-shares/test-volume-001 && \
sudo chown -R nobody: /var/lib/dcos-nfs-shares/test-volume-001 && \
sudo chmod 777 /var/lib/dcos-nfs-shares/test-volume-001 && \
echo '/var/lib/dcos-nfs-shares/test-volume-001 *(rw,sync)' | sudo tee /etc/exports && \
sudo systemctl restart nfs-utils && \
sudo exportfs -arv && \
sudo systemctl start nfs-server && \
sudo systemctl enable nfs-server
"""
setup_job = {
'description': 'Set up NFS share',
'id': 'nfs-share-setup-{}'.format(test_uuid),
'run': {
'cmd': command,
'cpus': 0.5,
'mem': 256,
'disk': 32,
'user': '******',
'restart': {
'policy': 'ON_FAILURE'
},
'placement': {
'constraints': [{
'attribute': '@hostname',
'operator': 'LIKE',
'value': hosts[0]
}]
}
}
}
dcos_api_session.metronome_one_off(setup_job)
# Create an app which writes to the NFS volume.
app = {
'id':
'csi-nfs-write-app-{}'.format(test_uuid),
'instances':
1,
'cpus':
0.5,
'mem':
256,
'cmd':
'echo some-stuff > test-volume-dir/output && sleep 999999',
'user':
'******',
'container': {
'type':
'MESOS',
'volumes': [{
'mode': 'rw',
'containerPath': 'test-volume-dir',
'external': {
'provider': 'csi',
'name': 'test-volume-001',
'options': {
'pluginName': 'nfs.csi.k8s.io',
'capability': {
'accessType': 'mount',
'accessMode': 'MULTI_NODE_MULTI_WRITER',
'fsType': 'nfs'
},
'volumeContext': {
'server': hosts[0],
'share': '/var/lib/dcos-nfs-shares/test-volume-001'
}
}
}
}]
},
'constraints': [['hostname', 'LIKE', hosts[1]]],
'healthChecks': [{
'protocol': 'COMMAND',
'command': {
'value': 'test `cat test-volume-dir/output` = some-stuff'
},
'gracePeriodSeconds': 5,
'intervalSeconds': 10,
'timeoutSeconds': 10,
'maxConsecutiveFailures': 3
}]
}
try:
with dcos_api_session.marathon.deploy_and_cleanup(app):
# Trivial app if it deploys, there is nothing else to check
pass
except Exception as error:
raise (error)
finally:
cleanup_nfs()
def test_move_external_volume_to_new_agent(
dcos_api_session: DcosApiSession) -> None:
"""Test that an external volume is successfully attached to a new agent.
If the dcos_api_session has only one agent, the volume will be detached and
reattached to the same agent.
"""
expanded_config = get_expanded_config()
if not (expanded_config['provider'] == 'aws'
or expanded_config['platform'] == 'aws'):
pytest.skip('Must be run in an AWS environment!')
if expanded_config.get('security') == 'strict':
pytest.skip('See: https://jira.mesosphere.com/browse/DCOS_OSS-4922')
hosts = dcos_api_session.slaves[0], dcos_api_session.slaves[-1]
test_uuid = uuid.uuid4().hex
test_label = 'integration-test-move-external-volume-{}'.format(test_uuid)
mesos_volume_path = 'volume'
docker_volume_path = '/volume'
base_app = {
'mem': 32,
'cpus': 0.1,
'instances': 1,
'container': {
'volumes': [{
'mode': 'RW',
'external': {
'name': test_label,
'provider': 'dvdi',
'options': {
'dvdi/driver': 'rexray'
}
}
}]
}
}
write_app = copy.deepcopy(base_app)
write_app.update({
'id':
'/{}/write'.format(test_label),
'cmd': (
# Check that the volume is empty.
'[ $(ls -A {volume_path}/ | grep -v --line-regexp "lost+found" | wc -l) -eq 0 ] && '
# Write the test UUID to a file.
'echo "{test_uuid}" >> {volume_path}/test && '
'while true; do sleep 1000; done').format(
test_uuid=test_uuid, volume_path=mesos_volume_path),
'constraints': [['hostname', 'LIKE', hosts[0]]],
})
write_app['container']['type'] = 'MESOS' # type: ignore
write_app['container']['volumes'][0][
'containerPath'] = mesos_volume_path # type: ignore
write_app['container']['volumes'][0]['external'][
'size'] = 1 # type: ignore
read_app = copy.deepcopy(base_app)
read_app.update({
'id':
'/{}/read'.format(test_label),
'cmd': (
# Diff the file and the UUID.
'echo "{test_uuid}" | diff - {volume_path}/test && '
'while true; do sleep 1000; done').format(
test_uuid=test_uuid, volume_path=docker_volume_path),
'constraints': [['hostname', 'LIKE', hosts[1]]],
})
read_app['container'].update({ # type: ignore
'type': 'DOCKER',
'docker': {
'image': 'busybox',
'network': 'HOST',
}
})
read_app['container']['volumes'][0][
'containerPath'] = docker_volume_path # type: ignore
# Volume operations can take several minutes.
timeout = 600
deploy_kwargs = {
'check_health': False,
# A volume might fail to attach because EC2. We can tolerate that and retry.
'ignore_failed_tasks': True,
'timeout': timeout
}
try:
with dcos_api_session.marathon.deploy_and_cleanup(
write_app, **deploy_kwargs):
logging.info('Successfully wrote to volume')
with dcos_api_session.marathon.deploy_and_cleanup(
read_app, **deploy_kwargs):
logging.info('Successfully read from volume')
finally:
logging.info('Deleting volume: ' + test_label)
delete_cmd = \
"/opt/mesosphere/bin/dcos-shell python " \
"/opt/mesosphere/active/dcos-integration-test/util/delete_ec2_volume.py {}".format(test_label)
delete_job = {
'id': 'delete-volume-' + test_uuid,
'run': {
'cpus': .1,
'mem': 128,
'disk': 0,
'cmd': delete_cmd
}
}
try:
# We use a metronome job to work around the `aws-deploy` integration tests where the master doesn't have
# volume permissions so all volume actions need to be performed from the agents.
dcos_api_session.metronome_one_off(delete_job, timeout=timeout)
except Exception as ex:
raise Exception('Failed to clean up volume {}: {}'.format(
test_label, ex)) from ex
