def create_app(self, id):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(
image=self.docker_image,
network="BRIDGE",
force_pull_image=True,
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(
protocol="HTTP",
path="/status",
grace_period_seconds=300,
interval_seconds=self.heath_check_interval,
timeout_seconds=20,
max_consecutive_failures=0
)
app_suffix = str(md5(str(random())).hexdigest())
app_name = self.app_base_name + app_suffix
new_app = MarathonApp(cpus=CPUS, mem=MEM, disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=self.instances_per_app,
max_launch_delay_seconds=5)
print("Creating {}".format(app_name))
self.marathon_cluster.create_app(app_id=app_name, app=new_app)
self.app_list_queue.put(app_name)
return None
def create_app(app_instances):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(image="nginx",
network="BRIDGE",
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(protocol="HTTP",
path="/",
grace_period_seconds=300,
interval_seconds=30,
timeout_seconds=20,
max_consecutive_failures=3)
app_name = str(hashlib.md5(str(random.random())).hexdigest())
logging.debug("Create cluster {}".format(app_name))
app_constraint = MarathonConstraint(field="hostname", operator="UNIQUE")
new_app = MarathonApp(cpus=CPUS,
mem=MEM,
disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=app_instances,
constraints=[app_constraint],
max_launch_delay_seconds=5)
print("Creating {}".format(app_name))
cluster.create_app(app_id=app_name, app=new_app)
return None
def get_docker_container(self):
docker = MarathonDockerContainer(
image=get_image_fullname(self.get_container_image()),
network=self.get_network(),
force_pull_image=self.get_force_pull_image(),
port_mappings=self.get_port_mappings(),
parameters=self.get_docker_parameters())
return MarathonContainer(docker=docker, volumes=self.get_volumes())
def start(self):
# First make a quick call to determine if user info was updated
self.update_users()
# Go on to start the notebook
docker_container = MarathonDockerContainer(
image=self.app_image,
network=self.network_mode,
port_mappings=self.get_port_mappings())
app_container = MarathonContainer(docker=docker_container,
type='DOCKER',
volumes=self.get_volumes())
# the memory request in marathon is in MiB
if hasattr(self, 'mem_limit') and self.mem_limit is not None:
mem_request = self.mem_limit / 1024.0 / 1024.0
else:
mem_request = 1024.0
if self.user_ssh_hagroup != "":
myports = [self.user_ssh_port]
labels = {
"HAPROXY_GROUP": self.user_ssh_hagroup,
"HA_EDGE_CONF": "1"
}
else:
labels = {}
myports = []
app_request = MarathonApp(
id=self.container_name,
cmd=self.get_app_cmd(),
env=self.get_env(),
cpus=self.cpu_limit,
mem=mem_request,
container=app_container,
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
instances=1,
labels=labels,
ports=myports,
fetch=self.fetch,
)
app = self.marathon.create_app(self.container_name, app_request)
if app is False or app.deployments is None:
self.log.error("Failed to create application for %s",
self.container_name)
return None
while True:
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
break
yield gen.sleep(1)
return (ip, port)
def create_app(client):
app = client.create_app(
'sleepy',
MarathonApp(cmd='while true; do sleep 33 ; done',
mem=32,
cpus=0.1,
container=MarathonContainer(
docker=MarathonDockerContainer(image='python:3'),
type='DOCKER')))
return app
def create_app(app_instances,
delete=False,
timeout=1200,
need_statistics=True):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(image="nginx",
network="BRIDGE",
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(protocol="HTTP",
path="/",
grace_period_seconds=300,
interval_seconds=2,
timeout_seconds=20,
max_consecutive_failures=3)
app_name = str(hashlib.md5(str(random.random())).hexdigest())
logging.debug("Create cluster {}".format(app_name))
new_app = MarathonApp(cpus=CPUS,
mem=MEM,
disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=app_instances,
max_launch_delay_seconds=5)
start_time = time.time()
cluster.create_app(app_id=app_name, app=new_app)
logging.debug("Get tasks for cluster {}".format(app_name))
successful_instances, all_starting, app_full_time = \
check_operation_status(start_time, app_name, app_instances, timeout)
if delete:
logging.debug('Delete {}'.format(app_name))
delete_app(app_name, force=True)
if need_statistics:
return {
"app_name": app_name,
"app_full_time": round(app_full_time, 2),
"instances": app_instances,
"successful_instances": successful_instances,
"instances_mean": round(statistics.mean(all_starting), 2),
"instances_median": round(statistics.median(all_starting), 2),
"instances_min": round(min(all_starting), 2),
"instances_max": round(max(all_starting), 2),
"id_run": id_run
}
else:
return {"app_name": app_name}
def start(self):
docker_container = MarathonDockerContainer(
image=self.app_image,
network=self.network_mode,
port_mappings=self.get_port_mappings())
app_container = MarathonContainer(docker=docker_container,
type='DOCKER',
volumes=self.get_volumes())
# the memory request in marathon is in MiB
if hasattr(self, 'mem_limit') and self.mem_limit is not None:
mem_request = self.mem_limit / 1024.0 / 1024.0
else:
mem_request = 1024.0
cmd = self.cmd + self.get_args()
app_request = MarathonApp(id=self.container_name,
cmd=' '.join(cmd),
env=self.get_env(),
cpus=self.cpu_limit,
mem=mem_request,
container=app_container,
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
instances=1,
accepted_resource_roles=['*'])
self.log.info("Creating App: %s", app_request)
self.log.info("self.marathon: %s", self.marathon)
app = self.marathon.create_app(self.container_name, app_request)
if app is False or app.deployments is None:
self.log.error("Failed to create application for %s",
self.container_name)
self.log.error("app: %s", app)
return None
while True:
app_info = yield self.get_app_info(self.container_name)
if app_info and app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
break
yield gen.sleep(1)
return (ip, port)
def start(self):
docker_container = MarathonDockerContainer(
image=self.app_image, network=self.network_mode
) #,port_mappings=self.get_port_mappings())
app_container = MarathonContainer(docker=docker_container,
type='DOCKER',
volumes=self.get_volumes())
# the memory request in marathon is in MiB
if hasattr(self, 'mem_limit') and self.mem_limit is not None:
mem_request = self.mem_limit / 1024.0 / 1024.0
else:
mem_request = 1024.0
app_request = MarathonApp(id=self.container_name,
env=self.get_env(),
cpus=self.cpu_limit,
mem=mem_request,
container=app_container,
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
instances=1,
ports=[0])
app = self.marathon.create_app(self.container_name, app_request)
if app is False or app.deployments is None:
self.log.error("Failed to create application for %s",
self.container_name)
return None
while True:
app_info = yield self.get_app_info(self.container_name)
self.state = app_info.tasks[0].state
if app_info and app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
self.user.server.ip = ip
self.user.server.port = port
break
yield gen.sleep(1)
return (ip, port)
def create_app(client):
""" Spins up a service based on predefined parameters
Arguments: MarathonClient object
Returns: app creation JSON output
"""
# Creates app running a command within a docker container
app = client.create_app(
'sleepy',
MarathonApp(
cmd='while true; do sleep 33 ; done',
mem=32,
cpus=0.1,
container=MarathonContainer(
docker=MarathonDockerContainer(image='python:3'),
type='DOCKER'
)
)
)
return app
def start(self):
app_image = self.user_options.get('app_image', None) or self.app_image
force_pull_image = self.user_options.get('force_pull_image', False)
self.log.info("starting a Marathon app with image=%s" % app_image)
container_params = {
'image': app_image,
'force_pull_image': force_pull_image
}
docker_container = MarathonDockerContainer(**container_params)
app_container = MarathonContainer(docker=docker_container,
type='MESOS',
volumes=self.get_volumes())
cpu = self.user_options.get('cpu', None)
mem = self.user_options.get('mem', None)
disk = self.user_options.get('disk', None)
gpu = self.user_options.get('gpu', None)
self.log.info("resource: (cpu=%s, mem=%s, disk=%s, gpu=%s)" %
(cpu, mem, disk, gpu))
cmd = self.cmd + self.get_args()
env = self.get_env()
port_definitions = [PortDefinition(port=0, protocol='tcp')]
app_request = MarathonApp(
id=self.app_id,
cmd=' '.join(
cmd), # cmd does not use Docker image's default entrypoint
env=env,
cpus=cpu,
mem=mem,
disk=disk,
gpus=gpu,
user=self.mesos_user,
container=app_container,
port_definitions=port_definitions,
networks=[{
'mode': 'host'
}],
constraints=self.get_constraints(),
health_checks=self.get_health_checks(),
unreachable_strategy=self.unreachable_strategy,
instances=1)
app_info = self.get_app_info(self.app_id)
try:
if app_info:
self.marathon.update_app(self.app_id, app_request, force=True)
else:
self.marathon.create_app(self.app_id, app_request)
except Exception as e:
self.log.error("Failed to create application for %s: %s",
self.app_id, e)
raise e
while True:
app_info = yield self.get_app_info(self.app_id)
if app_info is None:
raise MarathonSpawnerException("Application %s is lost",
self.app_id)
elif app_info.instances == 0:
raise MarathonSpawnerException(
"No instance for application %s", self.app_id)
elif app_info.tasks_healthy == 1:
ip, port = self.get_ip_and_port(app_info)
break
yield gen.sleep(1)
return (ip, port)