def ajax_list_deployments(request):
mc = MarathonClient('http://{}:{}'.format(settings.MARATHON['host'],
settings.MARATHON['port']))
deployments = mc.list_deployments()
data = {}
data['deployments'] = deployments
return render(request, 'marathon_mgmt/ajax_list_deployments.html', data)
def ajax_deployments(request):
mc = MarathonClient('http://{}:{}'.format(settings.MARATHON['host'], settings.MARATHON['port']))
deployments = mc.list_deployments()
data = {}
for deployment in deployments:
deployment.complete = (deployment.current_step-1) * 100 / deployment.total_steps
data['deployments'] = deployments
data['total_depl'] = len(deployments)
return render(request, 'marathon_mgmt/ajax_deployments.html', data)
def ajax_deployments(request):
mc = MarathonClient('http://{}:{}'.format(settings.MARATHON['host'],
settings.MARATHON['port']))
deployments = mc.list_deployments()
data = {}
for deployment in deployments:
deployment.complete = (deployment.current_step -
1) * 100 / deployment.total_steps
data['deployments'] = deployments
data['total_depl'] = len(deployments)
return render(request, 'marathon_mgmt/ajax_deployments.html', data)
def test_get_deployments(m):
fake_response = '[ { "affectedApps": [ "/test" ], "id": "fakeid", "steps": [ [ { "action": "ScaleApplication", "app": "/test" } ] ], "currentActions": [ { "action": "ScaleApplication", "app": "/test" } ], "version": "fakeversion", "currentStep": 1, "totalSteps": 1 } ]'
m.get('http://fake_server/v2/deployments', text=fake_response)
mock_client = MarathonClient(servers='http://fake_server')
actual_deployments = mock_client.list_deployments()
expected_deployments = [ models.MarathonDeployment(
id=u"fakeid",
steps=[[models.MarathonDeploymentAction(action="ScaleApplication", app="/test")]],
current_actions=[models.MarathonDeploymentAction(action="ScaleApplication", app="/test")],
current_step=1,
total_steps=1,
affected_apps=[u"/test"],
version=u"fakeversion"
)]
assert expected_deployments == actual_deployments
def test_get_deployments_pre_1_0():
fake_response = """[
{
"affectedApps": [
"/test"
],
"id": "fakeid",
"steps": [
[
{
"action": "ScaleApplication",
"app": "/test"
}
]
],
"currentActions": [
{
"action": "ScaleApplication",
"app": "/test"
}
],
"version": "fakeversion",
"currentStep": 1,
"totalSteps": 1
}
]"""
with requests_mock.mock() as m:
m.get('http://fake_server/v2/deployments', text=fake_response)
mock_client = MarathonClient(servers='http://fake_server')
actual_deployments = mock_client.list_deployments()
expected_deployments = [
models.MarathonDeployment(
id=u"fakeid",
steps=[[
models.MarathonDeploymentAction(action="ScaleApplication",
app="/test")
]],
current_actions=[
models.MarathonDeploymentAction(action="ScaleApplication",
app="/test")
],
current_step=1,
total_steps=1,
affected_apps=[u"/test"],
version=u"fakeversion")
]
assert expected_deployments == actual_deployments
def poll_deployments_for_app(client: MarathonClient, app: MarathonApp) -> bool:
appid = '/' + app.id if not app.id.startswith('/') else app.id
try:
while True:
deployments = client.list_deployments()
if not deployments:
print('No deployments active. Assuming complete.')
return True
for deploy in deployments:
if appid not in deploy.affected_apps:
print("No deployment involving {}. Assuming complete.".
format(appid))
return True
time.sleep(0.1)
except KeyboardInterrupt:
if input(
'Really abort creation?\nType \'YES\' to continue: ') == 'YES':
print(
'Aborting. Rollback creation or delete application manually.')
return False
return poll_deployments_for_app(client, app)
class MarathonSpawner(Spawner):
app_image = Unicode("jupyterhub/singleuser", config=True)
app_prefix = Unicode(
"jupyter",
help=dedent(
"""
Prefix for app names. The full app name for a particular
user will be <prefix>/<username>.
"""
)
).tag(config=True)
marathon_host = Unicode(
u'',
help="Hostname of Marathon server").tag(config=True)
marathon_constraints = List(
[],
help='Constraints to be passed through to Marathon').tag(config=True)
ports = List(
[8888],
help='Ports to expose externally'
).tag(config=True)
volumes = List(
[],
help=dedent(
"""
A list in Marathon REST API format for mounting volumes into the docker container.
[
{
"containerPath": "/foo",
"hostPath": "/bar",
"mode": "RW"
}
]
Note that using the template variable {username} in containerPath,
hostPath or the name variable in case it's an external drive
it will be replaced with the current user's name.
"""
)
).tag(config=True)
network_mode = Unicode(
'BRIDGE',
help="Enum of BRIDGE or HOST"
).tag(config=True)
hub_ip_connect = Unicode(
"",
help="Public IP address of the hub"
).tag(config=True)
hub_port_connect = Integer(
-1,
help="Public PORT of the hub"
).tag(config=True)
format_volume_name = Any(
help="""Any callable that accepts a string template and a Spawner
instance as parameters in that order and returns a string.
"""
).tag(config=True)
@default('format_volume_name')
def _get_default_format_volume_name(self):
return default_format_volume_name
_executor = None
@property
def executor(self):
cls = self.__class__
if cls._executor is None:
cls._executor = ThreadPoolExecutor(1)
return cls._executor
def __init__(self, *args, **kwargs):
super(MarathonSpawner, self).__init__(*args, **kwargs)
self.marathon = MarathonClient(self.marathon_host)
@property
def container_name(self):
return '/%s/%s' % (self.app_prefix, self.user.name)
def get_state(self):
state = super(MarathonSpawner, self).get_state()
state['container_name'] = self.container_name
return state
def load_state(self, state):
if 'container_name' in state:
pass
def get_health_checks(self):
health_checks = []
health_checks.append(MarathonHealthCheck(
protocol='TCP',
port_index=0,
grace_period_seconds=300,
interval_seconds=60,
timeout_seconds=20,
max_consecutive_failures=0
))
return health_checks
def get_volumes(self):
volumes = []
for v in self.volumes:
mv = MarathonContainerVolume.from_json(v)
mv.container_path = self.format_volume_name(mv.container_path, self)
mv.host_path = self.format_volume_name(mv.host_path, self)
if mv.external and 'name' in mv.external:
mv.external['name'] = self.format_volume_name(mv.external['name'], self)
volumes.append(mv)
return volumes
def get_port_mappings(self):
port_mappings = []
for p in self.ports:
port_mappings.append(
MarathonContainerPortMapping(
container_port=p,
host_port=0,
protocol='tcp'
)
)
return port_mappings
def get_constraints(self):
constraints = []
for c in self.marathon_constraints:
constraints.append(MarathonConstraint.from_json(c))
@run_on_executor
def get_deployment(self, deployment_id):
deployments = self.marathon.list_deployments()
for d in deployments:
if d.id == deployment_id:
return d
return None
@run_on_executor
def get_deployment_for_app(self, app_name):
deployments = self.marathon.list_deployments()
for d in deployments:
if app_name in d.affected_apps:
return d
return None
def get_ip_and_port(self, app_info):
assert len(app_info.tasks) == 1
ip = socket.gethostbyname(app_info.tasks[0].host)
return (ip, app_info.tasks[0].ports[0])
@run_on_executor
def get_app_info(self, app_name):
try:
app = self.marathon.get_app(app_name, embed_tasks=True)
except NotFoundError:
self.log.info("The %s application has not been started yet", app_name)
return None
else:
return app
def _public_hub_api_url(self):
uri = urlparse(self.hub.api_url)
port = self.hub_port_connect if self.hub_port_connect > 0 else uri.port
ip = self.hub_ip_connect if self.hub_ip_connect else uri.hostname
return urlunparse((
uri.scheme,
'%s:%s' % (ip, port),
uri.path,
uri.params,
uri.query,
uri.fragment
))
def get_env(self):
env = super(MarathonSpawner, self).get_env()
env.update(dict(
# Jupyter Hub config
JPY_USER=self.user.name,
JPY_COOKIE_NAME=self.user.server.cookie_name,
JPY_BASE_URL=self.user.server.base_url,
JPY_HUB_PREFIX=self.hub.server.base_url,
))
if self.notebook_dir:
env['NOTEBOOK_DIR'] = self.notebook_dir
if self.hub_ip_connect or self.hub_port_connect > 0:
hub_api_url = self._public_hub_api_url()
else:
hub_api_url = self.hub.api_url
env['JPY_HUB_API_URL'] = hub_api_url
return env
@gen.coroutine
def start(self):
docker_container = MarathonDockerContainer(
image=self.app_image,
network=self.network_mode,
port_mappings=self.get_port_mappings())
app_container = MarathonContainer(
docker=docker_container,
type='DOCKER',
volumes=self.get_volumes())
# the memory request in marathon is in MiB
if hasattr(self, 'mem_limit') and self.mem_limit is not None:
mem_request = self.mem_limit / 1024.0 / 1024.0
else:
mem_request = 1024.0
app_request = MarathonApp(
id=self.container_name,
env=self.get_env(),
cpus=self.cpu_limit,
mem=mem_request,
container=app_container,
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
instances=1
)
app = self.marathon.create_app(self.container_name, app_request)
if app is False or app.deployments is None:
self.log.error("Failed to create application for %s", self.container_name)
return None
while True:
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
break
yield gen.sleep(1)
return (ip, port)
@gen.coroutine
def stop(self, now=False):
try:
status = self.marathon.delete_app(self.container_name)
except:
self.log.error("Could not delete application %s", self.container_name)
raise
else:
if not now:
while True:
deployment = yield self.get_deployment(status['deploymentId'])
if deployment is None:
break
yield gen.sleep(1)
@gen.coroutine
def poll(self):
deployment = yield self.get_deployment_for_app(self.container_name)
if deployment:
for current_action in deployment.current_actions:
if current_action.action == 'StopApplication':
self.log.error("Application %s is shutting down", self.container_name)
return 1
return None
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
return None
return 0
def test_get_deployments_post_1_0(m):
fake_response = """[
{
"id": "4d2ff4d8-fbe5-4239-a886-f0831ed68d20",
"version": "2016-04-20T18:00:20.084Z",
"affectedApps": [
"/test-trivial-app"
],
"steps": [
{
"actions": [
{
"type": "StartApplication",
"app": "/test-trivial-app"
}
]
},
{
"actions": [
{
"type": "ScaleApplication",
"app": "/test-trivial-app"
}
]
}
],
"currentActions": [
{
"action": "ScaleApplication",
"app": "/test-trivial-app",
"readinessCheckResults": []
}
],
"currentStep": 2,
"totalSteps": 2
}
]"""
m.get('http://fake_server/v2/deployments', text=fake_response)
mock_client = MarathonClient(servers='http://fake_server')
actual_deployments = mock_client.list_deployments()
expected_deployments = [
models.MarathonDeployment(
id=u"4d2ff4d8-fbe5-4239-a886-f0831ed68d20",
steps=[
models.MarathonDeploymentStep(actions=[
models.MarathonDeploymentAction(type="StartApplication",
app="/test-trivial-app")
], ),
models.MarathonDeploymentStep(actions=[
models.MarathonDeploymentAction(type="ScaleApplication",
app="/test-trivial-app")
], ),
],
current_actions=[
models.MarathonDeploymentAction(action="ScaleApplication",
app="/test-trivial-app",
readiness_check_results=[])
],
current_step=2,
total_steps=2,
affected_apps=[u"/test-trivial-app"],
version=u"2016-04-20T18:00:20.084Z")
]
# Helpful for tox to see the diff
assert expected_deployments[0].__dict__ == actual_deployments[0].__dict__
assert expected_deployments == actual_deployments
def wait_for_deployment(client: MarathonClient,
deployment: Union[MarathonDeployment, dict]) -> bool:
def show_affected_apps(target_deploy):
for aff in target_deploy.affected_apps:
print('- {}: {}ui/#/apps/{}/debug'.format(aff, srv,
aff.replace('/', '%2F')))
target = None
if isinstance(deployment, dict):
if not client.list_deployments():
print('No deployments in progress. Grace time of 3 seconds.')
time.sleep(3)
for current_deployment in client.list_deployments():
if deployment.get('deploymentId') == current_deployment.id:
target = current_deployment
else:
target = deployment
if target is None:
print('Deployment {} not found. Assuming complete.'.format(deployment))
return False
if target not in client.list_deployments():
print('Not found yet. Grace time of 3 seconds.')
time.sleep(3)
srv = client.servers if isinstance(client.servers,
str) else client.servers[0]
print('Watching deployment {}'.format(target.id))
print('Affected apps:')
show_affected_apps(target)
try:
i = 0
print('Waiting', end='')
sys.stdout.flush()
while target in client.list_deployments():
time.sleep(0.5)
i += 0.5
if i % 5 == 0:
print('.', end='')
sys.stdout.flush()
print()
except KeyboardInterrupt:
rollback = input('\nRoll back deployment? Type \'YES\' to confirm: ')
if rollback == 'YES':
prompt = 'Force it?\n' \
'Doing so does not create a new rollback deployment, but forcefully deletes the current one.\n' \
'WARNING: APPLICATION MAY BE STUCK IN AN INCONSISTENT STATE.\n' \
'NO FURTHER ROLLBACKS WILL BE PERFORMED (even if launched with `--fullrollback`)\n' \
'Type \'YES\' to confirm: '
if input(prompt) == 'YES':
client.delete_deployment(target.id, force=True)
print('Deployment deleted. Check status of applications:')
show_affected_apps(target)
sys.exit(2)
deployment = client.delete_deployment(target.id, force=False)
print('Rollback deployment launched: {}'.format(
deployment.get('deploymentId')))
# Do not wait for this one. It must not be cancelled.
# TODO: Review options?
return False
print('\nDeployment monitoring aborted. Check status in:')
print('- All deployments: {}/ui/#/deployments'.format(srv))
show_affected_apps(target)
sys.exit(1)
print('Deployment {} complete.'.format(target.id))
return True
class MarathonIF(object):
def __init__(self, marathon_addr, my_addr, mesos):
self.mcli = MarathonClient(marathon_addr)
self.myAddr = my_addr
self.mesos = mesos
def get_apps(self):
listapps = self.mcli.list_apps()
return listapps
def get_app(self, app_id):
try:
a = self.mcli.get_app(app_id)
except marathon.exceptions.NotFoundError as e: # NOQA
return None
return a
def delete_app(self, app_id, force=False):
return self.mcli.delete_app(app_id, force)
def delete_deployment(self, dep_id):
return self.mcli.delete_deployment(dep_id)
def get_deployments(self):
return self.mcli.list_deployments()
def delete_app_ifexisting(self, app_id, trys=4):
for idx in range(0, trys):
try:
a = self.get_app(app_id)
if a:
return self.delete_app(app_id)
return None
except:
e = sys.exc_info()[0]
pprint("<p>Error: %s</p>" % e)
time.sleep(10)
raise
@staticmethod
def is_valid_app_id(app_id):
# allowed: lowercase letters, digits, hyphens, slash, dot
if re.match("^[A-Za-z0-9-/.]*$", app_id):
return True
return False
def create_app(self, app_id, attr):
"""
Create and start an app.
:param app_id: (str) - Application ID
:param attr: marathon.models.app.MarathonApp application to create.
:return: the created app
"""
# Validate that app_id conforms to allowed naming scheme.
if not self.is_valid_app_id(app_id):
l.error("Error: Only lowercase letters, digits, hyphens are allowed in app_id. %s" % app_id)
raise Exception("Invalid app_id")
for idx in range(0, 10):
try:
a = self.mcli.create_app(app_id, attr)
return a
except marathon.exceptions.MarathonHttpError as e:
if str(e).find('App is locked by one or more deployments. Override with the option') >= 0:
time.sleep(1)
else:
raise
raise
def wait_app_removal(self, app):
cnt = 0
while True:
if not self.get_app(app):
break
time.sleep(0.2)
cnt += 1
if cnt > 0:
l.info("Stuck waiting for %s to be deleted CNT=%d" % (app, cnt))
return True
def wait_app_ready(self, app, running_count):
cnt = 0
while True:
a1 = self.get_app(app)
if a1.tasks_running == running_count:
return a1
cnt += 1
time.sleep(1)
if (cnt % 30) == 29:
l.info("[%d]Waiting for task to move to running stage, " % cnt +
"current stat staged=%d running=%d expected Running=%d" %
(a1.tasks_staged, a1.tasks_running, running_count))
def scale_app(self, app, scale):
return self.mcli.scale_app(app, scale)
def ping(self):
return self.mcli.ping()
class MarathonIF(object):
def __init__(self, marathon_addr, my_addr, mesos):
self.mcli = MarathonClient(marathon_addr)
self.myAddr = my_addr
self.mesos = mesos
def get_apps(self):
listapps = self.mcli.list_apps()
return listapps
def get_app(self, app_id):
try:
a = self.mcli.get_app(app_id)
except marathon.exceptions.NotFoundError as e: # NOQA
return None
return a
def delete_app(self, app_id, force=False):
return self.mcli.delete_app(app_id, force)
def delete_deployment(self, dep_id):
return self.mcli.delete_deployment(dep_id)
def get_deployments(self):
return self.mcli.list_deployments()
def delete_app_ifexisting(self, app_id, trys=4):
for idx in range(0, trys):
try:
a = self.get_app(app_id)
if a:
return self.delete_app(app_id)
return None
except:
e = sys.exc_info()[0]
pprint("<p>Error: %s</p>" % e)
time.sleep(10)
raise
def create_app(self, app_id, attr):
for idx in range(0, 10):
try:
a = self.mcli.create_app(app_id, attr)
return a
except marathon.exceptions.MarathonHttpError as e:
if str(e).find('App is locked by one or more deployments. Override with the option') >= 0:
time.sleep(1)
else:
raise
raise
def wait_app_removal(self, app):
cnt = 0
while True:
if not self.get_app(app):
break
time.sleep(0.2)
cnt += 1
if cnt > 0:
l.info("Stuck waiting for %s to be deleted CNT=%d" % (app, cnt))
return True
def wait_app_ready(self, app, running_count):
cnt = 0
while True:
a1 = self.get_app(app)
if a1.tasks_running == running_count:
return a1
cnt += 1
time.sleep(1)
if (cnt % 30) == 29:
l.info("[%d]Waiting for task to move to running stage, " % cnt +
"current stat staged=%d running=%d expected Running=%d" %
(a1.tasks_staged, a1.tasks_running, running_count))
def scale_app(self, app, scale):
return self.mcli.scale_app(app, scale)
def ping(self):
return self.mcli.ping()
class MarathonIF(object):
def __init__(self, marathon_addr, my_addr, mesos):
self.mcli = MarathonClient(marathon_addr)
self.myAddr = my_addr
self.mesos = mesos
def get_apps(self):
listapps = self.mcli.list_apps()
return listapps
def get_app(self, app_id, timeout=300):
st_time = time.time()
while (time.time() - st_time < timeout):
try:
try:
a = self.mcli.get_app(app_id)
except marathon.exceptions.NotFoundError as e: # NOQA
return None
return a
except:
l.info("mcli: get_app returned error")
l.info(traceback.format_exc())
l.info("Retrying after 10 secs timeout=%d", timeout)
time.sleep(10)
raise Exception(
"mcli get_app timed out, possible zookeper/marathon/mesos malfunction"
)
def delete_app(self, app_id, force=False, timeout=200):
st_time = time.time()
while (time.time() - st_time < timeout):
try:
self.mcli.delete_app(app_id, force)
return
except:
l.info("mcli: delete_app returned error")
l.info(traceback.format_exc())
l.info("Retrying after 10 secs timeout=%d", timeout)
time.sleep(10)
raise Exception(
"mcli delete_app timed out, possible zookeper/marathon/mesos malfunction"
)
def delete_deployment(self, dep_id):
return self.mcli.delete_deployment(dep_id)
def get_deployments(self):
return self.mcli.list_deployments()
def delete_app_ifexisting(self, app_id, trys=4):
for idx in range(0, trys):
try:
a = self.get_app(app_id)
if a:
return self.delete_app(app_id)
return None
except:
e = sys.exc_info()[0]
pprint("<p>Error: %s</p>" % e)
time.sleep(10)
raise
@staticmethod
def is_valid_app_id(app_id):
# allowed: lowercase letters, digits, hyphens, slash, dot
if re.match("^[A-Za-z0-9-/.]*$", app_id):
return True
return False
def create_app(self, app_id, attr):
"""
Create and start an app.
:param app_id: (str) - Application ID
:param attr: marathon.models.app.MarathonApp application to create.
:return: the created app
"""
# Validate that app_id conforms to allowed naming scheme.
if not self.is_valid_app_id(app_id):
l.error(
"Error: Only lowercase letters, digits, hyphens are allowed in app_id. %s"
% app_id)
raise Exception("Invalid app_id")
for idx in range(0, 10):
try:
a = self.mcli.create_app(app_id, attr)
return a
except marathon.exceptions.MarathonHttpError as e:
if str(
e
).find('App is locked by one or more deployments. Override with the option'
) >= 0:
time.sleep(1)
else:
raise
raise
def wait_app_removal(self, app):
cnt = 0
while True:
if not self.get_app(app):
break
time.sleep(0.2)
cnt += 1
if cnt > 0:
l.info("Stuck waiting for %s to be deleted CNT=%d" %
(app, cnt))
return True
def wait_app_ready(self, app, running_count, sleep_before_next_try=1):
cnt = 0
while True:
a1 = self.get_app(app)
# if tasks_running are greater (due to whatever reason, scale down accordingly)
if a1.tasks_running > running_count:
delta = a1.tasks_running - running_count
l.info("Found [%d] more apps, scaling down to [%d]", delta,
running_count)
self.scale_app(app, running_count)
# Allow for some time before next poll
time.sleep(1)
continue
if a1.tasks_running == running_count:
return a1
cnt += 1
time.sleep(sleep_before_next_try)
if (cnt % 30) == 29:
l.info(
"[%d]Waiting for task to move to running stage, " % cnt +
"current stat staged=%d running=%d expected Running=%d" %
(a1.tasks_staged, a1.tasks_running, running_count))
def scale_app(self, app, scale, timeout=300):
st_time = time.time()
while (time.time() - st_time < timeout):
try:
self.mcli.scale_app(app, scale)
return
except:
l.info("mcli: scale_app returned error")
l.info(traceback.format_exc())
l.info("Retrying after 10 secs timeout=%d", timeout)
time.sleep(10)
raise Exception(
"mcli scale_app timed out, possible zookeper/marathon/mesos malfunction"
)
def ping(self):
return self.mcli.ping()
def kill_task(self, app_id, task_id):
return self.mcli.kill_task(app_id, task_id)
class MarathonSpawner(Spawner):
app_image = Unicode("jupyterhub/singleuser:%s" % _jupyterhub_xy,
config=True)
app_prefix = Unicode("jupyter",
help=dedent("""
Prefix for app names. The full app name for a particular
user will be <prefix>/<username>.
""")).tag(config=True)
marathon_host = Unicode(
u'', help="Hostname of Marathon server").tag(config=True)
marathon_constraints = List(
[],
help='Constraints to be passed through to Marathon').tag(config=True)
ports = List([8888], help='Ports to expose externally').tag(config=True)
volumes = List([],
help=dedent("""
A list in Marathon REST API format for mounting volumes into the docker container.
[
{
"containerPath": "/foo",
"hostPath": "/bar",
"mode": "RW"
}
]
Note that using the template variable {username} in containerPath,
hostPath or the name variable in case it's an external drive
it will be replaced with the current user's name.
""")).tag(config=True)
network_mode = Unicode('BRIDGE',
help="Enum of BRIDGE or HOST").tag(config=True)
hub_ip_connect = Unicode(
"", help="Public IP address of the hub").tag(config=True)
@observe('hub_ip_connect')
def _ip_connect_changed(self, change):
if jupyterhub.version_info >= (0, 8):
warnings.warn(
"MarathonSpawner.hub_ip_connect is no longer needed with JupyterHub 0.8."
" Use JupyterHub.hub_connect_ip instead.",
DeprecationWarning,
)
hub_port_connect = Integer(-1,
help="Public PORT of the hub").tag(config=True)
@observe('hub_port_connect')
def _port_connect_changed(self, change):
if jupyterhub.version_info >= (0, 8):
warnings.warn(
"MarathonSpawner.hub_port_connect is no longer needed with JupyterHub 0.8."
" Use JupyterHub.hub_connect_port instead.",
DeprecationWarning,
)
format_volume_name = Any(
help="""Any callable that accepts a string template and a Spawner
instance as parameters in that order and returns a string.
""").tag(config=True)
@default('format_volume_name')
def _get_default_format_volume_name(self):
return default_format_volume_name
# fix default port to 8888, used in the container
@default('port')
def _port_default(self):
return 8888
# default to listening on all-interfaces in the container
@default('ip')
def _ip_default(self):
return '0.0.0.0'
_executor = None
@property
def executor(self):
cls = self.__class__
if cls._executor is None:
cls._executor = ThreadPoolExecutor(1)
return cls._executor
def __init__(self, *args, **kwargs):
super(MarathonSpawner, self).__init__(*args, **kwargs)
self.marathon = MarathonClient(self.marathon_host)
@property
def container_name(self):
return '/%s/%s' % (self.app_prefix, self.user.name)
def get_state(self):
state = super(MarathonSpawner, self).get_state()
state['container_name'] = self.container_name
return state
def load_state(self, state):
if 'container_name' in state:
pass
def get_health_checks(self):
health_checks = []
health_checks.append(
MarathonHealthCheck(protocol='TCP',
port_index=0,
grace_period_seconds=300,
interval_seconds=30,
timeout_seconds=20,
max_consecutive_failures=0))
return health_checks
def get_volumes(self):
volumes = []
for v in self.volumes:
mv = MarathonContainerVolume.from_json(v)
mv.container_path = self.format_volume_name(
mv.container_path, self)
mv.host_path = self.format_volume_name(mv.host_path, self)
if mv.external and 'name' in mv.external:
mv.external['name'] = self.format_volume_name(
mv.external['name'], self)
volumes.append(mv)
return volumes
def get_port_mappings(self):
port_mappings = []
for p in self.ports:
port_mappings.append(
MarathonContainerPortMapping(container_port=p,
host_port=0,
protocol='tcp'))
return port_mappings
def get_constraints(self):
constraints = []
for c in self.marathon_constraints:
constraints.append(MarathonConstraint.from_json(c))
return constraints
@run_on_executor
def get_deployment(self, deployment_id):
deployments = self.marathon.list_deployments()
for d in deployments:
if d.id == deployment_id:
return d
return None
@run_on_executor
def get_deployment_for_app(self, app_name):
deployments = self.marathon.list_deployments()
for d in deployments:
if app_name in d.affected_apps:
return d
return None
def get_ip_and_port(self, app_info):
assert len(app_info.tasks) == 1
ip = socket.gethostbyname(app_info.tasks[0].host)
return (ip, app_info.tasks[0].ports[0])
@run_on_executor
def get_app_info(self, app_name):
try:
app = self.marathon.get_app(app_name, embed_tasks=True)
except NotFoundError:
self.log.info("The %s application has not been started yet",
app_name)
return None
else:
return app
def _public_hub_api_url(self):
uri = urlparse(self.hub.api_url)
port = self.hub_port_connect if self.hub_port_connect > 0 else uri.port
ip = self.hub_ip_connect if self.hub_ip_connect else uri.hostname
return urlunparse((uri.scheme, '%s:%s' % (ip, port), uri.path,
uri.params, uri.query, uri.fragment))
def get_args(self):
args = super().get_args()
if self.hub_ip_connect:
# JupyterHub 0.7 specifies --hub-api-url
# on the command-line, which is hard to update
for idx, arg in enumerate(list(args)):
if arg.startswith('--hub-api-url='):
args.pop(idx)
break
args.append('--hub-api-url=%s' % self._public_hub_api_url())
return args
@gen.coroutine
def start(self):
docker_container = MarathonDockerContainer(
image=self.app_image,
network=self.network_mode,
port_mappings=self.get_port_mappings())
app_container = MarathonContainer(docker=docker_container,
type='DOCKER',
volumes=self.get_volumes())
# the memory request in marathon is in MiB
if hasattr(self, 'mem_limit') and self.mem_limit is not None:
mem_request = self.mem_limit / 1024.0 / 1024.0
else:
mem_request = 1024.0
cmd = self.cmd + self.get_args()
app_request = MarathonApp(id=self.container_name,
cmd=' '.join(cmd),
env=self.get_env(),
cpus=self.cpu_limit,
mem=mem_request,
container=app_container,
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
instances=1,
accepted_resource_roles=['*'])
self.log.info("Creating App: %s", app_request)
self.log.info("self.marathon: %s", self.marathon)
app = self.marathon.create_app(self.container_name, app_request)
if app is False or app.deployments is None:
self.log.error("Failed to create application for %s",
self.container_name)
self.log.error("app: %s", app)
return None
while True:
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
break
yield gen.sleep(1)
return (ip, port)
@gen.coroutine
def stop(self, now=False):
try:
status = self.marathon.delete_app(self.container_name)
except:
self.log.error("Could not delete application %s",
self.container_name)
raise
else:
if not now:
while True:
deployment = yield self.get_deployment(
status['deploymentId'])
if deployment is None:
break
yield gen.sleep(1)
@gen.coroutine
def poll(self):
deployment = yield self.get_deployment_for_app(self.container_name)
if deployment:
for current_action in deployment.current_actions:
if current_action.action == 'StopApplication':
self.log.error("Application %s is shutting down",
self.container_name)
return 1
return None
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
return None
return 0
def ajax_list_deployments(request):
mc = MarathonClient('http://{}:{}'.format(settings.MARATHON['host'], settings.MARATHON['port']))
deployments = mc.list_deployments()
data = {}
data['deployments'] = deployments
return render(request, 'marathon_mgmt/ajax_list_deployments.html', data)
def test_get_deployments_post_1_0():
fake_response = """[
{
"id": "4d2ff4d8-fbe5-4239-a886-f0831ed68d20",
"version": "2016-04-20T18:00:20.084Z",
"affectedApps": [
"/test-trivial-app"
],
"steps": [
{
"actions": [
{
"type": "StartApplication",
"app": "/test-trivial-app"
}
]
},
{
"actions": [
{
"type": "ScaleApplication",
"app": "/test-trivial-app"
}
]
}
],
"currentActions": [
{
"action": "ScaleApplication",
"app": "/test-trivial-app",
"readinessCheckResults": []
}
],
"currentStep": 2,
"totalSteps": 2
}
]"""
with requests_mock.mock() as m:
m.get('http://fake_server/v2/deployments', text=fake_response)
mock_client = MarathonClient(servers='http://fake_server')
actual_deployments = mock_client.list_deployments()
expected_deployments = [models.MarathonDeployment(
id=u"4d2ff4d8-fbe5-4239-a886-f0831ed68d20",
steps=[
models.MarathonDeploymentStep(
actions=[models.MarathonDeploymentAction(
type="StartApplication", app="/test-trivial-app")],
),
models.MarathonDeploymentStep(
actions=[models.MarathonDeploymentAction(
type="ScaleApplication", app="/test-trivial-app")],
),
],
current_actions=[models.MarathonDeploymentAction(
action="ScaleApplication", app="/test-trivial-app", readiness_check_results=[])
],
current_step=2,
total_steps=2,
affected_apps=[u"/test-trivial-app"],
version=u"2016-04-20T18:00:20.084Z"
)]
# Helpful for tox to see the diff
assert expected_deployments[0].__dict__ == actual_deployments[0].__dict__
assert expected_deployments == actual_deployments
class MarathonSpawner(Spawner):
# Load the app image
app_image = Unicode("jupyterhub/singleuser", config=True)
# The command to run
app_cmd = Unicode("jupyter notebook", config=True)
# This is the prefix in Marathon
app_prefix = Unicode(
"jupyter",
help=dedent(
"""
Prefix for app names. The full app name for a particular
user will be <prefix>/<username>.
"""
)
).tag(config=True)
user_web_port = Integer(0, help="Port that the Notebook is listening on").tag(config=True)
user_ssh_port = Integer(0, help="SSH Port that the container is listening on").tag(config=True)
user_ssh_host = Unicode('', help="Hostname of the ssh container").tag(config=True)
use_jupyterlab = Integer(0, help="Use Jupyterlab - Jupyterlab is 1 default is 0 or Jupyternotebook").tag(config=True)
user_ssh_hagroup = Unicode('', help="HAProxy group for ssh container port").tag(config=True)
# zeta_user_file are the users and their custom settings for installation in Zeta Architechure. If this is blank, defaults from Jupyter Hub are used for Mem, CPU, Ports, Image. If this is not blank, we will read from that file
zeta_user_file = Unicode(
"",
help="Path to json file that includes users and per user settings"
).tag(config=True)
no_user_file_fail = Bool(
True,
help="Is zeta_user_file is provided, but can't be opened fail. (Default). False loads defaults and tries to spawn"
).tag(config=True)
# Marathon Server
marathon_host = Unicode(
u'',
help="Hostname of Marathon server").tag(config=True)
marathon_user_name = Unicode(
u'',
help='Marathon user name'
).tag(config=True)
marathon_user_password = Unicode(
u'',
help='Marathon user password'
).tag(config=True)
fetch = List([], help='Optional files to fetch').tag(config=True)
custom_env = List(
[],
help='Additional ENVs to add to the default. Format is a list of 1 record dictionary. [{key:val}]'
).tag(config=True)
# Constraints in Marathon
marathon_constraints = List(
[],
help='Constraints to be passed through to Marathon').tag(config=True)
# Shared Notebook location
shared_notebook_dir = Unicode(
'', help="Shared Notebook location that users will get a link to in their notebook location - can be blank"
).tag(config=True)
ports = List(
[8888],
help='Ports to expose externally'
).tag(config=True)
volumes = List(
[],
help=dedent(
"""
A list in Marathon REST API format for mounting volumes into the docker container.
[
{
"containerPath": "/foo",
"hostPath": "/bar",
"mode": "RW"
}
]
Note that using the template variable {username} in containerPath,
hostPath or the name variable in case it's an external drive
it will be replaced with the current user's name.
"""
)
).tag(config=True)
network_mode = Unicode(
'BRIDGE',
help="Enum of BRIDGE or HOST"
).tag(config=True)
hub_ip_connect = Unicode(
"",
help="Public IP address of the hub"
).tag(config=True)
hub_port_connect = Integer(
-1,
help="Public PORT of the hub"
).tag(config=True)
format_volume_name = Any(
help="""Any callable that accepts a string template and a Spawner
instance as parameters in that order and returns a string.
"""
).tag(config=True)
@default('format_volume_name')
def _get_default_format_volume_name(self):
return default_format_volume_name
_executor = None
@property
def executor(self):
cls = self.__class__
if cls._executor is None:
cls._executor = ThreadPoolExecutor(1)
return cls._executor
def __init__(self, *args, **kwargs):
super(MarathonSpawner, self).__init__(*args, **kwargs)
self.marathon = MarathonClient(self.marathon_host,
self.marathon_user_name,
self.marathon_user_password)
@property
def container_name(self):
self.log.info("Container Name : %s / %s / %s",self.app_prefix, self.user.name, self.name )
try:
self.log.info("Debug %s", json.dumps(self.name))
except:
self.log.info("Could not log self")
return '/%s/%s%s' % (self.app_prefix, self.user.name, self.name)
def get_state(self):
state = super(MarathonSpawner, self).get_state()
state['container_name'] = self.container_name
return state
def load_state(self, state):
if 'container_name' in state:
pass
def get_health_checks(self):
health_checks = []
health_checks.append(MarathonHealthCheck(
protocol='TCP',
port_index=0,
grace_period_seconds=300,
interval_seconds=60,
timeout_seconds=20,
max_consecutive_failures=0
))
return health_checks
def get_volumes(self):
volumes = []
for v in self.volumes:
mv = MarathonContainerVolume.from_json(v)
mv.container_path = self.format_volume_name(mv.container_path, self)
mv.host_path = self.format_volume_name(mv.host_path, self)
if mv.external and 'name' in mv.external:
mv.external['name'] = self.format_volume_name(mv.external['name'], self)
volumes.append(mv)
out_vols = []
dups = {}
#Remove Duplicates there should be only one container path point for container
for x in volumes:
if x.container_path in dups:
pass
else:
out_vols.append(x)
dups[x.container_path] = 1
return out_vols
def get_app_cmd(self):
retval = self.app_cmd.replace("{username}", self.user.name)
retval = retval.replace("{userwebport}", str(self.user_web_port))
if self.use_jupyterlab == 1:
print("This is where I should do some thing if I want to run Jupyter lab")
if self.user_ssh_hagroup != "":
retval = retval.replace("{usersshport}", "$PORT0")
else:
retval = retval.replace("{usersshport}", str(self.user_ssh_port))
return retval
def get_port_mappings(self):
port_mappings = []
if self.network_mode == "BRIDGE":
for p in self.ports:
port_mappings.append(
MarathonContainerPortMapping(
container_port=p,
host_port=0,
protocol='tcp'
)
)
return port_mappings
def get_constraints(self):
constraints = []
for c in self.marathon_constraints:
constraints.append(MarathonConstraint.from_json(c))
@run_on_executor
def get_deployment(self, deployment_id):
deployments = self.marathon.list_deployments()
for d in deployments:
if d.id == deployment_id:
return d
return None
@run_on_executor
def get_deployment_for_app(self, app_name):
deployments = self.marathon.list_deployments()
for d in deployments:
if app_name in d.affected_apps:
return d
return None
def get_ip_and_port(self, app_info):
assert len(app_info.tasks) == 1
ip = socket.gethostbyname(app_info.tasks[0].host)
port = app_info.tasks[0].ports[0]
return (ip, port)
@run_on_executor
def get_app_info(self, app_name):
try:
app = self.marathon.get_app(app_name, embed_tasks=True)
except NotFoundError:
self.log.info("The %s application has not been started yet", app_name)
return None
else:
return app
def _public_hub_api_url(self):
uri = urlparse(self.hub.api_url)
port = self.hub_port_connect if self.hub_port_connect > 0 else uri.port
ip = self.hub_ip_connect if self.hub_ip_connect else uri.hostname
return urlunparse((
uri.scheme,
'%s:%s' % (ip, port),
uri.path,
uri.params,
uri.query,
uri.fragment
))
def get_env(self):
env = super(MarathonSpawner, self).get_env()
env.update(dict(
# Jupyter Hub config
JPY_USER=self.user.name,
# JPY_COOKIE_NAME=self.user.server.cookie_name,
# JPY_BASE_URL=self.user.server.base_url,
JPY_HUB_PREFIX=self.hub.server.base_url,
JPY_USER_WEB_PORT=str(self.user_web_port),
JPY_USER_SSH_PORT=str(self.user_ssh_port),
JPY_USER_SSH_HOST=str(self.user_ssh_host)
))
if self.notebook_dir:
env['NOTEBOOK_DIR'] = self.notebook_dir
if self.hub_ip_connect or self.hub_port_connect > 0:
hub_api_url = self._public_hub_api_url()
else:
hub_api_url = self.hub.api_url
env['JPY_HUB_API_URL'] = hub_api_url
for x in self.custom_env:
for k,v in x.items():
env[k] = str(v)
return env
def update_users(self):
# No changes if the zeta_user_file is blank
if self.zeta_user_file != "":
try:
j = open(self.zeta_user_file, "r")
user_file = j.read()
j.close()
user_ar = {}
for x in user_file.split("\n"):
if x.strip().find("#") != 0 and x.strip() != "":
y = json.loads(x)
if y['user'] == self.user.name:
user_ar = y
break
if len(user_ar) == 0:
self.log.error("Could not find current user %s in zeta_user_file %s - Not Spawning" % (self.user.name, self.zeta_user_file))
if self.no_user_file_fail == True:
raise Exception('no_user_file_fail is True, will not go on')
print("User List identified and loaded, setting values to %s" % user_ar)
self.cpu_limit = user_ar['cpu_limit']
self.mem_limit = user_ar['mem_limit']
self.user_ssh_port = user_ar['user_ssh_port']
self.user_web_port = user_ar['user_web_port']
self.user_ssh_host = user_ar['user_ssh_host']
try:
self.user_ssh_hagroup = user_ar['user_ssh_hagroup']
except:
self.user_ssh_hagroup = ""
try:
self.use_jupyterlab = int(user_ar['use_jupyterlab'])
except:
self.use_jupyterlab = 0
self.network_mode = user_ar['network_mode']
self.app_image = user_ar['app_image']
self.marathon_constraints = user_ar['marathon_constraints']
self.ports.append(self.user_web_port)
self.ports.append(self.user_ssh_port)
self.custom_env = self.custom_env + user_ar['custom_env']
self.volumes = self.volumes + user_ar['volumes']
print("User List Loaded!")
# { "user": "******", "cpu_limit": "1", "mem_limit": "2G", "user_ssh_port": 10500, "user_web_port:" 10400, "network_mode": "BRIDGE", "app_image": "$APP_IMG", "marathon_constraints": []}
except:
self.log.error("Could not find or open zeta_user_file: %s" % self.zeta_user_file)
if self.no_user_file_fail == True:
raise Exception("Could not open file and config says don't go on")
@gen.coroutine
def start(self):
# First make a quick call to determine if user info was updated
self.update_users()
# Go on to start the notebook
docker_container = MarathonDockerContainer(
image=self.app_image,
network=self.network_mode,
port_mappings=self.get_port_mappings())
app_container = MarathonContainer(
docker=docker_container,
type='DOCKER',
volumes=self.get_volumes())
# the memory request in marathon is in MiB
if hasattr(self, 'mem_limit') and self.mem_limit is not None:
mem_request = self.mem_limit / 1024.0 / 1024.0
else:
mem_request = 1024.0
if self.user_ssh_hagroup != "":
myports = [self.user_ssh_port]
labels = {"HAPROXY_GROUP": self.user_ssh_hagroup, "HA_EDGE_CONF": "1"}
else:
labels = {}
myports = []
app_request = MarathonApp(
id=self.container_name,
cmd=self.get_app_cmd(),
env=self.get_env(),
cpus=self.cpu_limit,
mem=mem_request,
container=app_container,
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
instances=1,
labels=labels,
ports=myports,
fetch=self.fetch,
)
app = self.marathon.create_app(self.container_name, app_request)
if app is False or app.deployments is None:
self.log.error("Failed to create application for %s", self.container_name)
return None
while True:
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
break
yield gen.sleep(1)
return (ip, port)
@gen.coroutine
def stop(self, now=False):
try:
status = self.marathon.delete_app(self.container_name)
except:
self.log.error("Could not delete application %s", self.container_name)
raise
else:
if not now:
while True:
deployment = yield self.get_deployment(status['deploymentId'])
if deployment is None:
break
yield gen.sleep(1)
@gen.coroutine
def poll(self):
deployment = yield self.get_deployment_for_app(self.container_name)
if deployment:
for current_action in deployment.current_actions:
if current_action.action == 'StopApplication':
self.log.error("Application %s is shutting down", self.container_name)
return 1
return None
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
return None
return 0
class MarathonIF(object):
def __init__(self, marathon_addr, my_addr, mesos):
self.mcli = MarathonClient(marathon_addr)
self.myAddr = my_addr
self.mesos = mesos
def get_apps(self):
listapps = self.mcli.list_apps()
return listapps
def get_app(self, app_id, timeout=300):
st_time = time.time()
while(time.time() - st_time < timeout):
try:
try:
a = self.mcli.get_app(app_id)
except marathon.exceptions.NotFoundError as e: # NOQA
return None
return a
except:
l.info("mcli: get_app returned error")
l.info(traceback.format_exc())
l.info("Retrying after 10 secs timeout=%d", timeout)
time.sleep(10)
raise Exception("mcli get_app timed out, possible zookeper/marathon/mesos malfunction")
def delete_app(self, app_id, force=False, timeout=200):
st_time = time.time()
while(time.time() - st_time < timeout):
try:
self.mcli.delete_app(app_id, force)
return
except:
l.info("mcli: delete_app returned error")
l.info(traceback.format_exc())
l.info("Retrying after 10 secs timeout=%d", timeout)
time.sleep(10)
raise Exception("mcli delete_app timed out, possible zookeper/marathon/mesos malfunction")
def delete_deployment(self, dep_id):
return self.mcli.delete_deployment(dep_id)
def get_deployments(self):
return self.mcli.list_deployments()
def delete_app_ifexisting(self, app_id, trys=4):
for idx in range(0, trys):
try:
a = self.get_app(app_id)
if a:
return self.delete_app(app_id)
return None
except:
e = sys.exc_info()[0]
pprint("<p>Error: %s</p>" % e)
time.sleep(10)
raise
@staticmethod
def is_valid_app_id(app_id):
# allowed: lowercase letters, digits, hyphens, slash, dot
if re.match("^[A-Za-z0-9-/.]*$", app_id):
return True
return False
def create_app(self, app_id, attr):
"""
Create and start an app.
:param app_id: (str) - Application ID
:param attr: marathon.models.app.MarathonApp application to create.
:return: the created app
"""
# Validate that app_id conforms to allowed naming scheme.
if not self.is_valid_app_id(app_id):
l.error("Error: Only lowercase letters, digits, hyphens are allowed in app_id. %s" % app_id)
raise Exception("Invalid app_id")
for idx in range(0, 10):
try:
a = self.mcli.create_app(app_id, attr)
return a
except marathon.exceptions.MarathonHttpError as e:
if str(e).find('App is locked by one or more deployments. Override with the option') >= 0:
time.sleep(1)
else:
raise
raise
def wait_app_removal(self, app):
cnt = 0
while True:
if not self.get_app(app):
break
time.sleep(0.2)
cnt += 1
if cnt > 0:
l.info("Stuck waiting for %s to be deleted CNT=%d" % (app, cnt))
return True
def wait_app_ready(self, app, running_count, sleep_before_next_try=1):
cnt = 0
while True:
a1 = self.get_app(app)
# if tasks_running are greater (due to whatever reason, scale down accordingly)
if a1.tasks_running > running_count:
delta = a1.tasks_running - running_count
l.info("Found [%d] more apps, scaling down to [%d]", delta, running_count)
self.scale_app(app, running_count)
# Allow for some time before next poll
time.sleep(1)
continue
if a1.tasks_running == running_count:
return a1
cnt += 1
time.sleep(sleep_before_next_try)
if (cnt % 30) == 29:
l.info("[%d]Waiting for task to move to running stage, " % cnt +
"current stat staged=%d running=%d expected Running=%d" %
(a1.tasks_staged, a1.tasks_running, running_count))
def scale_app(self, app, scale, timeout=300):
st_time = time.time()
while(time.time() - st_time < timeout):
try:
self.mcli.scale_app(app, scale)
return
except:
l.info("mcli: scale_app returned error")
l.info(traceback.format_exc())
l.info("Retrying after 10 secs timeout=%d", timeout)
time.sleep(10)
raise Exception("mcli scale_app timed out, possible zookeper/marathon/mesos malfunction")
def ping(self):
return self.mcli.ping()
def kill_task(self, app_id, task_id):
return self.mcli.kill_task(app_id, task_id)
