def _container_from_module(self):
if "container" not in self._module.params:
return MarathonContainer()
docker = None
type = "DOCKER"
volumes = None
if "docker" in self._module.params["container"]:
docker = self._module.params["container"]["docker"]
if "type" in self._module.params["container"]:
type = self._module.params["container"]["type"]
if "volumes" in self._module.params["container"]:
volumes = self._module.params["container"]["volumes"]
c = MarathonContainer(docker=docker, type=type, volumes=volumes)
# Set service ports to 0 for easier comparison
for pm in c.docker.port_mappings:
if pm.service_port is None:
pm.service_port = 0
return c
def get_app():
service_port = random.choice([x for x in SERVICE_PORT_RANGE])
container_args = {
'type': 'DOCKER',
'docker': {
'image': 'python:3.4.3',
'network': 'BRIDGE',
'portMappings': [
{
'containerPort': 8000,
'servicePort': service_port,
'protocol': 'tcp'
}
]
}
}
args = {
'id': './marathon-integration-test-{}-8000-{}'.format(str(uuid.uuid4()), service_port),
'instances': 1,
'cpus': 0.1,
'mem': 64.0,
'args': ['python', '-m', 'http.server', '8000'],
'container': MarathonContainer.from_json(container_args),
'health_checks': [MarathonHealthCheck.from_json(HEALTH_CHECK_ARGS)]
}
return MarathonApp(**args)
def create_app(self, id):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(
image=self.docker_image,
network="BRIDGE",
force_pull_image=True,
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(
protocol="HTTP",
path="/status",
grace_period_seconds=300,
interval_seconds=self.heath_check_interval,
timeout_seconds=20,
max_consecutive_failures=0
)
app_suffix = str(md5(str(random())).hexdigest())
app_name = self.app_base_name + app_suffix
new_app = MarathonApp(cpus=CPUS, mem=MEM, disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=self.instances_per_app,
max_launch_delay_seconds=5)
print("Creating {}".format(app_name))
self.marathon_cluster.create_app(app_id=app_name, app=new_app)
self.app_list_queue.put(app_name)
return None
def create_app(app_instances):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(image="nginx",
network="BRIDGE",
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(protocol="HTTP",
path="/",
grace_period_seconds=300,
interval_seconds=30,
timeout_seconds=20,
max_consecutive_failures=3)
app_name = str(hashlib.md5(str(random.random())).hexdigest())
logging.debug("Create cluster {}".format(app_name))
app_constraint = MarathonConstraint(field="hostname", operator="UNIQUE")
new_app = MarathonApp(cpus=CPUS,
mem=MEM,
disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=app_instances,
constraints=[app_constraint],
max_launch_delay_seconds=5)
print("Creating {}".format(app_name))
cluster.create_app(app_id=app_name, app=new_app)
return None
def get_docker_container(self):
docker = MarathonDockerContainer(
image=get_image_fullname(self.get_container_image()),
network=self.get_network(),
force_pull_image=self.get_force_pull_image(),
port_mappings=self.get_port_mappings(),
parameters=self.get_docker_parameters())
return MarathonContainer(docker=docker, volumes=self.get_volumes())
def start(self):
# First make a quick call to determine if user info was updated
self.update_users()
# Go on to start the notebook
docker_container = MarathonDockerContainer(
image=self.app_image,
network=self.network_mode,
port_mappings=self.get_port_mappings())
app_container = MarathonContainer(docker=docker_container,
type='DOCKER',
volumes=self.get_volumes())
# the memory request in marathon is in MiB
if hasattr(self, 'mem_limit') and self.mem_limit is not None:
mem_request = self.mem_limit / 1024.0 / 1024.0
else:
mem_request = 1024.0
if self.user_ssh_hagroup != "":
myports = [self.user_ssh_port]
labels = {
"HAPROXY_GROUP": self.user_ssh_hagroup,
"HA_EDGE_CONF": "1"
}
else:
labels = {}
myports = []
app_request = MarathonApp(
id=self.container_name,
cmd=self.get_app_cmd(),
env=self.get_env(),
cpus=self.cpu_limit,
mem=mem_request,
container=app_container,
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
instances=1,
labels=labels,
ports=myports,
fetch=self.fetch,
)
app = self.marathon.create_app(self.container_name, app_request)
if app is False or app.deployments is None:
self.log.error("Failed to create application for %s",
self.container_name)
return None
while True:
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
break
yield gen.sleep(1)
return (ip, port)
def create_app(client):
app = client.create_app(
'sleepy',
MarathonApp(cmd='while true; do sleep 33 ; done',
mem=32,
cpus=0.1,
container=MarathonContainer(
docker=MarathonDockerContainer(image='python:3'),
type='DOCKER')))
return app
def __init__(self,
scheduler,
executable='dask-worker',
docker_image='mrocklin/dask-distributed',
marathon_address='http://localhost:8080',
username=None,
password=None,
auth_token=None,
app_name=None,
**kwargs):
self.scheduler = scheduler
self.executor = ThreadPoolExecutor(1)
# Create Marathon App to run dask-worker
args = [
executable,
scheduler.address,
'--name',
'$MESOS_TASK_ID', # use Mesos task ID as worker name
'--worker-port',
'$PORT_WORKER',
'--nanny-port',
'$PORT_NANNY',
'--http-port',
'$PORT_HTTP'
]
ports = [{
'port': 0,
'protocol': 'tcp',
'name': port_name
} for port_name in ['worker', 'nanny', 'http']]
if 'mem' in kwargs:
args.extend(
['--memory-limit',
str(int(kwargs['mem'] * 0.6 * 1e6))])
kwargs['cmd'] = ' '.join(args)
container = MarathonContainer({'image': docker_image})
app = MarathonApp(instances=0,
container=container,
port_definitions=ports,
**kwargs)
# Connect and register app
self.client = MarathonClient(servers=marathon_address,
username=username,
password=password,
auth_token=auth_token)
self.app = self.client.create_app(app_name or 'dask-%s' % uuid.uuid4(),
app)
def create_app(app_instances,
delete=False,
timeout=1200,
need_statistics=True):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(image="nginx",
network="BRIDGE",
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(protocol="HTTP",
path="/",
grace_period_seconds=300,
interval_seconds=2,
timeout_seconds=20,
max_consecutive_failures=3)
app_name = str(hashlib.md5(str(random.random())).hexdigest())
logging.debug("Create cluster {}".format(app_name))
new_app = MarathonApp(cpus=CPUS,
mem=MEM,
disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=app_instances,
max_launch_delay_seconds=5)
start_time = time.time()
cluster.create_app(app_id=app_name, app=new_app)
logging.debug("Get tasks for cluster {}".format(app_name))
successful_instances, all_starting, app_full_time = \
check_operation_status(start_time, app_name, app_instances, timeout)
if delete:
logging.debug('Delete {}'.format(app_name))
delete_app(app_name, force=True)
if need_statistics:
return {
"app_name": app_name,
"app_full_time": round(app_full_time, 2),
"instances": app_instances,
"successful_instances": successful_instances,
"instances_mean": round(statistics.mean(all_starting), 2),
"instances_median": round(statistics.median(all_starting), 2),
"instances_min": round(min(all_starting), 2),
"instances_max": round(max(all_starting), 2),
"id_run": id_run
}
else:
return {"app_name": app_name}
def start(self):
docker_container = MarathonDockerContainer(
image=self.app_image,
network=self.network_mode,
port_mappings=self.get_port_mappings())
app_container = MarathonContainer(docker=docker_container,
type='DOCKER',
volumes=self.get_volumes())
# the memory request in marathon is in MiB
if hasattr(self, 'mem_limit') and self.mem_limit is not None:
mem_request = self.mem_limit / 1024.0 / 1024.0
else:
mem_request = 1024.0
cmd = self.cmd + self.get_args()
app_request = MarathonApp(id=self.container_name,
cmd=' '.join(cmd),
env=self.get_env(),
cpus=self.cpu_limit,
mem=mem_request,
container=app_container,
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
instances=1,
accepted_resource_roles=['*'])
self.log.info("Creating App: %s", app_request)
self.log.info("self.marathon: %s", self.marathon)
app = self.marathon.create_app(self.container_name, app_request)
if app is False or app.deployments is None:
self.log.error("Failed to create application for %s",
self.container_name)
self.log.error("app: %s", app)
return None
while True:
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
break
yield gen.sleep(1)
return (ip, port)
def start(self, nworkers=0):
# address = self.scheduler.address.replace('tcp://', '')
args = ['dask-worker', self.scheduler.address,
'--name', '$MESOS_TASK_ID', # use Mesos task ID as worker name
'--worker-port', '$PORT_WORKER',
'--bokeh-port', '$PORT_BOKEH',
'--nanny-port', '$PORT_NANNY',
'--nprocs', str(self.nprocs),
'--nthreads', str(self.nthreads)]
ports = [{'port': 0,
'protocol': 'tcp',
'name': name}
for name in ['worker', 'nanny', 'http', 'bokeh']]
# healths = [{'portIndex': i,
# 'protocol': 'TCP',
# 'gracePeriodSeconds': 300,
# 'intervalSeconds': 60,
# 'timeoutSeconds': 20,
# 'maxConsecutiveFailures': 3}
# for i, name in enumerate(['worker', 'nanny', 'http', 'bokeh'])]
healths = []
if 'mem' in self.options:
args.extend(['--memory-limit',
str(int(self.options['mem'] * 0.8 * 1e6))])
docker_parameters = [{"key": "volume", "value": v}
for v in self.volumes]
container = MarathonContainer({'image': self.docker,
'forcePullImage': True,
'parameters': docker_parameters})
command = ' '.join(args)
app = MarathonApp(instances=nworkers, container=container,
port_definitions=ports, cmd=command,
health_checks=healths,
**self.options)
self.client.update_app(self.name, app)
logger.info('Started marathon workers {}'.format(self.name))
def start(self):
docker_container = MarathonDockerContainer(
image=self.app_image, network=self.network_mode
) #,port_mappings=self.get_port_mappings())
app_container = MarathonContainer(docker=docker_container,
type='DOCKER',
volumes=self.get_volumes())
# the memory request in marathon is in MiB
if hasattr(self, 'mem_limit') and self.mem_limit is not None:
mem_request = self.mem_limit / 1024.0 / 1024.0
else:
mem_request = 1024.0
app_request = MarathonApp(id=self.container_name,
env=self.get_env(),
cpus=self.cpu_limit,
mem=mem_request,
container=app_container,
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
instances=1,
ports=[0])
app = self.marathon.create_app(self.container_name, app_request)
if app is False or app.deployments is None:
self.log.error("Failed to create application for %s",
self.container_name)
return None
while True:
app_info = yield self.get_app_info(self.container_name)
self.state = app_info.tasks[0].state
if app_info and app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
self.user.server.ip = ip
self.user.server.port = port
break
yield gen.sleep(1)
return (ip, port)
def create_app(client):
""" Spins up a service based on predefined parameters
Arguments: MarathonClient object
Returns: app creation JSON output
"""
# Creates app running a command within a docker container
app = client.create_app(
'sleepy',
MarathonApp(
cmd='while true; do sleep 33 ; done',
mem=32,
cpus=0.1,
container=MarathonContainer(
docker=MarathonDockerContainer(image='python:3'),
type='DOCKER'
)
)
)
return app
def deploy(marathon, name, docker, volume, scheduler_cpus, scheduler_mem,
adaptive, port, bokeh_port, constraint, maximum_over_capacity,
minimum_health_capacity, label, uri, jupyter, **kwargs):
name = name or 'daskathon-{}'.format(str(uuid.uuid4())[-4:])
kwargs['name'] = '{}-workers'.format(name)
kwargs['docker'] = docker
kwargs['port'] = port
kwargs['bokeh_port'] = bokeh_port
args = [('--{}'.format(k.replace('_', '-')), str(v))
for k, v in kwargs.items() if v not in (None, '')]
for c in constraint:
args.append(('--constraint', c))
for u in uri:
args.append(('--uri', u))
for v in volume:
args.append(('--volume', v))
if maximum_over_capacity:
args.append(('--maximum-over-capacity', str(maximum_over_capacity)))
if minimum_health_capacity:
args.append(
('--minimum-health-capacity', str(minimum_health_capacity)))
args = list(concat(args))
if adaptive:
args.append('--adaptive')
client = MarathonClient(marathon)
docker_parameters = [{"key": "volume", "value": v} for v in volume]
container = MarathonContainer({
'image': docker,
'forcePullImage': True,
'parameters': docker_parameters
})
args = ['daskathon', 'run'] + args + [marathon]
cmd = ' '.join(args)
healths = [{
'portIndex': i,
'protocol': 'TCP'
} for i, _ in enumerate(['scheduler', 'bokeh'])]
services = [('scheduler', port), ('bokeh', bokeh_port)]
ports = [{
'port': p,
'protocol': 'tcp',
'name': service
} for (service, p) in services]
constraints = [c.split(':')[:3] for c in constraint]
labels = dict([l.split(':') for l in label])
upgrade_strategy = {
'maximum_over_capacity': maximum_over_capacity,
'minimum_health_capacity': minimum_health_capacity
}
scheduler = MarathonApp(instances=1,
container=container,
cpus=scheduler_cpus,
mem=scheduler_mem,
task_kill_grace_period_seconds=20,
port_definitions=ports,
health_checks=healths,
constraints=constraints,
upgrade_strategy=upgrade_strategy,
labels=labels,
uris=uri,
require_ports=True,
cmd=cmd)
client.update_app('{}-scheduler'.format(name), scheduler)
if jupyter:
cmd = ('jupyter notebook --allow-root --no-browser '
'--NotebookApp.token=\'\' --ip 0.0.0.0 --port $PORT_NOTEBOOK')
ports = [{'port': 0, 'protocol': 'tcp', 'name': 'notebook'}]
jupyter = deepcopy(scheduler)
jupyter.cmd = cmd
jupyter.port_definitions = ports
client.update_app('{}-jupyter'.format(name), jupyter)
def start(self):
app_image = self.user_options.get('app_image', None) or self.app_image
force_pull_image = self.user_options.get('force_pull_image', False)
self.log.info("starting a Marathon app with image=%s" % app_image)
container_params = {
'image': app_image,
'force_pull_image': force_pull_image
}
docker_container = MarathonDockerContainer(**container_params)
app_container = MarathonContainer(docker=docker_container,
type='MESOS',
volumes=self.get_volumes())
cpu = self.user_options.get('cpu', None)
mem = self.user_options.get('mem', None)
disk = self.user_options.get('disk', None)
gpu = self.user_options.get('gpu', None)
self.log.info("resource: (cpu=%s, mem=%s, disk=%s, gpu=%s)" %
(cpu, mem, disk, gpu))
cmd = self.cmd + self.get_args()
env = self.get_env()
port_definitions = [PortDefinition(port=0, protocol='tcp')]
app_request = MarathonApp(
id=self.app_id,
cmd=' '.join(
cmd), # cmd does not use Docker image's default entrypoint
env=env,
cpus=cpu,
mem=mem,
disk=disk,
gpus=gpu,
user=self.mesos_user,
container=app_container,
port_definitions=port_definitions,
networks=[{
'mode': 'host'
}],
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
unreachable_strategy=self.unreachable_strategy,
instances=1)
app_info = self.get_app_info(self.app_id)
try:
if app_info:
self.marathon.update_app(self.app_id, app_request, force=True)
else:
self.marathon.create_app(self.app_id, app_request)
except Exception as e:
self.log.error("Failed to create application for %s: %s",
self.app_id, e)
raise e
while True:
app_info = yield self.get_app_info(self.app_id)
if app_info is None:
raise MarathonSpawnerException("Application %s is lost",
self.app_id)
elif app_info.instances == 0:
raise MarathonSpawnerException(
"No instance for application %s", self.app_id)
elif app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
break
yield gen.sleep(1)
return (ip, port)
