def ip_other_than_mom():
mom_ip = ip_of_mom()
agents = get_private_agents()
for agent in agents:
if agent != mom_ip:
return agent
return None
def ip_other_than_mom():
mom_ip = ip_of_mom()
agents = get_private_agents()
for agent in agents:
if agent != mom_ip:
return agent
return None
def test_external_volume():
volume_name = "marathon-si-test-vol-{}".format(uuid.uuid4().hex)
app_def = apps.external_volume_mesos_app()
app_def["container"]["volumes"][0]["external"]["name"] = volume_name
app_id = app_def['id']
# Tested with root marathon since MoM doesn't have
# --enable_features external_volumes option activated.
# First deployment should create the volume since it has a unique name
try:
print('INFO: Deploying {} with external volume {}'.format(app_id, volume_name))
client = marathon.create_client()
client.add_app(app_def)
deployment_wait(service_id=app_id)
# Create the app: the volume should be successfully created
common.assert_app_tasks_running(client, app_def)
common.assert_app_tasks_healthy(client, app_def)
# Scale down to 0
print('INFO: Scaling {} to 0 instances'.format(app_id))
client.stop_app(app_id)
deployment_wait(service_id=app_id)
# Scale up again: the volume should be successfully reused
print('INFO: Scaling {} back to 1 instance'.format(app_id))
client.scale_app(app_id, 1)
deployment_wait(service_id=app_id)
common.assert_app_tasks_running(client, app_def)
common.assert_app_tasks_healthy(client, app_def)
# Remove the app to be able to remove the volume
print('INFO: Finally removing {}'.format(app_id))
client.remove_app(app_id)
deployment_wait(service_id=app_id)
except Exception as e:
print('Fail to test external volumes: {}'.format(e))
raise e
finally:
# Clean up after the test: external volumes are not destroyed by marathon or dcos
# and have to be cleaned manually.
cmd = 'sudo /opt/mesosphere/bin/dvdcli remove --volumedriver=rexray --volumename={}'.format(volume_name)
removed = False
for agent in get_private_agents():
status, output = run_command_on_agent(agent, cmd) # NOQA
print('DEBUG: Failed to remove external volume with name={} on agent={}: {}'.format(
volume_name, agent, output))
if status:
removed = True
# Note: Removing the volume might fail sometimes because EC2 takes some time (~10min) to recognize that
# the volume is not in use anymore hence preventing it's removal. This is a known pitfall: we log the error
# and the volume should be cleaned up manually later.
if not removed:
print('WARNING: Failed to remove external volume with name={}'.format(volume_name))
else:
print('DEBUG: External volume with name={} successfully removed'.format(volume_name))
def test_external_volume():
volume_name = "marathon-si-test-vol-{}".format(uuid.uuid4().hex)
app_def = apps.external_volume_mesos_app()
app_def["container"]["volumes"][0]["external"]["name"] = volume_name
app_id = app_def['id']
# Tested with root marathon since MoM doesn't have
# --enable_features external_volumes option activated.
# First deployment should create the volume since it has a unique name
try:
print('INFO: Deploying {} with external volume {}'.format(app_id, volume_name))
client = marathon.create_client()
client.add_app(app_def)
deployment_wait(service_id=app_id)
# Create the app: the volume should be successfully created
common.assert_app_tasks_running(client, app_def)
common.assert_app_tasks_healthy(client, app_def)
# Scale down to 0
print('INFO: Scaling {} to 0 instances'.format(app_id))
client.stop_app(app_id)
deployment_wait(service_id=app_id)
# Scale up again: the volume should be successfully reused
print('INFO: Scaling {} back to 1 instance'.format(app_id))
client.scale_app(app_id, 1)
deployment_wait(service_id=app_id)
common.assert_app_tasks_running(client, app_def)
common.assert_app_tasks_healthy(client, app_def)
# Remove the app to be able to remove the volume
print('INFO: Finally removing {}'.format(app_id))
client.remove_app(app_id)
deployment_wait(service_id=app_id)
except Exception as e:
print('Fail to test external volumes: {}'.format(e))
raise e
finally:
# Clean up after the test: external volumes are not destroyed by marathon or dcos
# and have to be cleaned manually.
cmd = 'sudo /opt/mesosphere/bin/dvdcli remove --volumedriver=rexray --volumename={}'.format(volume_name)
removed = False
for agent in get_private_agents():
status, output = run_command_on_agent(agent, cmd) # NOQA
print('DEBUG: Failed to remove external volume with name={} on agent={}: {}'.format(
volume_name, agent, output))
if status:
removed = True
# Note: Removing the volume might fail sometimes because EC2 takes some time (~10min) to recognize that
# the volume is not in use anymore hence preventing it's removal. This is a known pitfall: we log the error
# and the volume should be cleaned up manually later.
if not removed:
print('WARNING: Failed to remove external volume with name={}'.format(volume_name))
else:
print('DEBUG: External volume with name={} successfully removed'.format(volume_name))
def archive_sandboxes():
# Nothing to setup
yield
logger.info('>>> Archiving Mesos sandboxes')
# We tarball the sandboxes from all the agents first and download them afterwards
for agent in get_private_agents():
file_name = 'sandbox_{}.tar.gz'.format(agent.replace(".", "_"))
cmd = 'sudo tar --exclude=provisioner -zcf {} /var/lib/mesos/slave'.format(file_name)
status, output = run_command_on_agent(agent, cmd) # NOQA
if status:
copy_file_from_agent(agent, file_name)
else:
logger.warning('Failed to tarball the sandbox from the agent={}, output={}'.format(agent, output))
def archive_sandboxes():
# Nothing to setup
yield
logger.info('>>> Archiving Mesos sandboxes')
# We tarball the sandboxes from all the agents first and download them afterwards
for agent in get_private_agents():
file_name = 'sandbox_{}.tar.gz'.format(agent.replace(".", "_"))
cmd = 'sudo tar --exclude=provisioner -zcf {} /var/lib/mesos/slave'.format(
file_name)
status, output = run_command_on_agent(agent, cmd) # NOQA
if status:
copy_file_from_agent(agent, file_name)
else:
logger.warning(
'Failed to tarball the sandbox from the agent={}, output={}'.
format(agent, output))
def cluster_info(mom_name='marathon-user'):
logger.info("DC/OS: %s, in %s mode", dcos_version(), ee_version())
agents = get_private_agents()
logger.info("Agents: %d", len(agents))
client = marathon.create_client()
about = client.get_about()
logger.info("Marathon version: %s", about.get("version"))
if service_available_predicate(mom_name):
with marathon_on_marathon(mom_name) as client:
try:
about = client.get_about()
logger.info("Marathon MoM version: {}".format(about.get("version")))
except Exception:
logger.info("Marathon MoM not present")
else:
logger.info("Marathon MoM not present")
def cluster_info(mom_name='marathon-user'):
logger.info("DC/OS: %s, in %s mode", dcos_version(), ee_version())
agents = get_private_agents()
logger.info("Agents: %d", len(agents))
client = marathon.create_client()
about = client.get_about()
logger.info("Marathon version: %s", about.get("version"))
if service_available_predicate(mom_name):
with marathon_on_marathon(mom_name) as client:
try:
about = client.get_about()
logger.info("Marathon MoM version: {}".format(
about.get("version")))
except Exception:
logger.info("Marathon MoM not present")
else:
logger.info("Marathon MoM not present")
def test_private_repository_docker_app():
username = os.environ['DOCKER_HUB_USERNAME']
password = os.environ['DOCKER_HUB_PASSWORD']
agents = get_private_agents()
common.create_docker_credentials_file(username, password)
common.copy_docker_credentials_file(agents)
app_def = apps.private_docker_app()
app_id = app_def["id"]
if ee_version() == 'strict':
app_def['user'] = '******'
common.add_dcos_marathon_user_acls()
client = marathon.create_client()
client.add_app(app_def)
deployment_wait(service_id=app_id)
common.assert_app_tasks_running(client, app_def)
def test_private_repository_docker_app():
username = os.environ['DOCKER_HUB_USERNAME']
password = os.environ['DOCKER_HUB_PASSWORD']
agents = get_private_agents()
common.create_docker_credentials_file(username, password)
common.copy_docker_credentials_file(agents)
app_def = apps.private_docker_app()
app_id = app_def["id"]
if ee_version() == 'strict':
app_def['user'] = '******'
common.add_dcos_marathon_user_acls()
client = marathon.create_client()
client.add_app(app_def)
deployment_wait(service_id=app_id)
common.assert_app_tasks_running(client, app_def)