def zookeeper_health_check_get_app():
labels = {
'_client_conn_port': 31001,
'_follower_conn_port': 31002,
'_server_conn_port': 31003
}
app1 = MarathonApp(id='/new-zookeeper/zookeeper-1',
labels=labels,
tasks=[MarathonTask(host='127.0.0.1')],
tasks_unhealthy=0,
tasks_running=1,
mem=256,
cpus=0.5,
disk=256)
app2 = MarathonApp(id='/new-zookeeper/zookeeper-2',
labels=labels,
tasks=[MarathonTask(host='127.0.0.2')],
tasks_unhealthy=0,
tasks_running=1,
mem=256,
cpus=0.5,
disk=256)
app3 = MarathonApp(id='/new-zookeeper/zookeeper-3',
labels=labels,
tasks=[MarathonTask(host='127.0.0.3')],
tasks_unhealthy=0,
tasks_running=1,
mem=256,
cpus=0.5,
disk=256)
return [app1, app2, app3]
def tw_consumer_discover_list_apps():
return [
MarathonApp(id='/new-consumer/doctor-strange',
labels={'_tonomi_application': 'tw-consumer'}),
MarathonApp(id='/new-consumer2/trolls',
labels={'_tonomi_application': 'tw-consumer'})
]
def kafka_discover_list_apps():
return [
MarathonApp(id='/sandbox/kafka',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'kafka'
}),
MarathonApp(id='/new-kafka', labels={'_tonomi_application': 'kafka'})
]
def redis_discover_list_apps():
return [
MarathonApp(id='/sandbox/redis',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'redis'
}),
MarathonApp(id='/new-redis', labels={'_tonomi_application': 'redis'})
]
def spark_discover_list_apps():
return [
MarathonApp(id='/sandbox/spark',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'spark'
}),
MarathonApp(id='/new-spark', labels={'_tonomi_application': 'spark'})
]
def webui_discover_list_apps():
return [
MarathonApp(id='/sandbox/webui',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'webui'
}),
MarathonApp(id='/new-webui', labels={'_tonomi_application': 'webui'})
]
def environment_health_check_get_group():
zoo1 = MarathonApp(id='/sandbox/zookeeper/zookeeper-1',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'zookeeper'
})
zoo2 = MarathonApp(id='/sandbox/zookeeper/zookeeper-2',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'zookeeper'
})
zoo3 = MarathonApp(id='/sandbox/zookeeper/zookeeper-3',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'zookeeper'
})
redis1 = MarathonApp(id='/sandbox/redis/redis-master',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'redis'
})
redis2 = MarathonApp(id='/sandbox/redis/redis-slave',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'redis'
})
cass1 = MarathonApp(id='/sandbox/cassandra/cassandra-seed',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'cassandra'
})
cass2 = MarathonApp(id='/sandbox/cassandra/cassandra-node',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'cassandra'
})
kafka1 = MarathonApp(id='/sandbox/kafka/kafka-broker',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'kafka'
})
webui1 = MarathonApp(id='/sandbox/webui/webui-app',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'webui'
})
spark1 = MarathonApp(id='/sandbox/spark/spark-app',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'spark'
})
apps = [
zoo1, zoo2, zoo3, redis1, redis2, cass1, cass2, kafka1, webui1, spark1
]
return namedtuple('Struct', 'id apps')(id='/sandbox', apps=apps)
def zookeeper_discover_list_apps():
return [
MarathonApp(id='/new-zookeeper',
labels={'_tonomi_application': 'zookeeper'}),
MarathonApp(id='/sandbox/zookeeper',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'zookeeper'
})
]
def tw_consumer_health_check_get_app():
labels = {'_tonomi_application': 'tw-consumer'}
app1 = MarathonApp(id='/new-consumer/doctor-strange',
labels=labels,
cpus=0.1,
mem=256,
disk=0)
app1.tasks_unhealthy = 0
app1.tasks_running = 1
return [app1]
def marathon_discover_list_apps():
return [
MarathonApp(id='/sandbox/cassandra/cassandra-seed',
labels={
'_tonomi_environment': 'test',
'_tonomi_application': 'cassandra'
}),
MarathonApp(id='/sample2/cassandra-seed',
labels={'_tonomi_application': 'cassandra'}),
MarathonApp(id='/sample3', labels={})
]
def marathon_health_check_get_app():
docker = MarathonDockerContainer(image='image')
container = MarathonContainer(docker=docker)
app1 = MarathonApp(id='/sample',
labels={},
cpus=0.5,
mem=256,
disk=256,
instances=1,
container=container)
app1.tasks_unhealthy = 0
return [app1]
def webui_health_check_get_app():
port_mappings = [MarathonContainerPortMapping(service_port=1000)]
docker = MarathonDockerContainer(image='node', port_mappings=port_mappings)
container = MarathonContainer(docker=docker)
labels = {'_tonomi_environment': 'sandbox', '_tonomi_application': 'webui'}
app1 = MarathonApp(id='/sandbox/webui/webui-app',
labels=labels,
cpus=0.5,
mem=256,
disk=256,
instances=1,
container=container)
app1.tasks_unhealthy = 0
app1.tasks_running = 1
return [app1]
def environment_discover_list_apps():
return [
MarathonApp(id='/new-cassandra/cassandra-seed',
labels={'_tonomi_application': 'cassandra'}),
MarathonApp(id='/test/redis/redis-master',
labels={
'_tonomi_environment': 'test',
'_tonomi_application': 'redis'
}),
MarathonApp(id='/sandbox/spark/spark-app',
labels={
'_tonomi_environment': 'sandbox',
'_tonomi_application': 'spark'
})
]
def create_app(app_instances):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(image="nginx",
network="BRIDGE",
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(protocol="HTTP",
path="/",
grace_period_seconds=300,
interval_seconds=30,
timeout_seconds=20,
max_consecutive_failures=3)
app_name = str(hashlib.md5(str(random.random())).hexdigest())
logging.debug("Create cluster {}".format(app_name))
app_constraint = MarathonConstraint(field="hostname", operator="UNIQUE")
new_app = MarathonApp(cpus=CPUS,
mem=MEM,
disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=app_instances,
constraints=[app_constraint],
max_launch_delay_seconds=5)
print("Creating {}".format(app_name))
cluster.create_app(app_id=app_name, app=new_app)
return None
def update_and_wait_app(app_name, scale_param, scale, scale_type="*"):
timeout = 600
list_tasks = cluster.list_tasks(app_name)
app_instances = len(list_tasks)
cpus = CPUS
mem = MEM
disk = DISK
instances = app_instances
if scale_param == "cpu":
cpus = eval("{} {} {}".format(cpus, scale_type, scale))
if scale_param == "mem":
mem = eval("{} {} {}".format(mem, scale_type, scale))
if scale_param == "disk":
disk = eval("{} {} {}".format(disk, scale_type, scale))
if scale_param == "instances":
instances = eval("{} {} {}".format(instances, scale_type, scale))
updated_app = MarathonApp(cpus=cpus,
mem=mem,
disk=disk,
instances=instances)
start_time = time.time()
cluster.update_app(app_id=app_name, app=updated_app)
time.sleep(5)
successful_instances, all_starting, app_full_time = \
check_operation_status(start_time, app_name, instances,
timeout, list_tasks)
return calculate_results_per_operation([
app_name, app_instances, successful_instances, app_full_time,
all_starting
])
def launch_elsa(marathon, stats_file, scale_window):
logging.info('Start monitoring the inbound traffic on topics using %s' %
(stats_file))
# make sure the stats file is properly initialized:
if not os.path.exists(stats_file):
f = open(stats_file, 'w')
f.write('0')
f.close()
# launch the Elsa app via Marathon
c = MarathonClient(marathon)
c.create_app(
'elsa',
MarathonApp(cmd='/home/vagrant/elsa/launch-elsa.sh',
mem=200,
cpus=1,
user='******'))
# c.list_apps()
print(
'ElSA is deployed and running, waiting now 5 sec before starting auto-scale ...'
)
time.sleep(5) # allow time to deploy before autoscaling sets in
# kick off traffic monitoring and trigger autoscaling:
previous_topic_traffic = 0
try:
while True:
with open(stats_file, 'r') as elsa_file:
topic_traffic = int(elsa_file.read())
topic_traffic_diff = topic_traffic - previous_topic_traffic
print('Difference in traffic in the past %d seconds: %d' %
(scale_window, topic_traffic_diff))
previous_topic_traffic = topic_traffic
current_instance_num = c.get_app('elsa').instances
if topic_traffic_diff > TRAFFIC_INCREASE_THRESHOLD: # we see a surge of traffic above threshold ...
instance_multiplier = int(
topic_traffic_diff /
SCALE_FACTOR) # ... increase number of instances
c.scale_app('elsa',
current_instance_num * instance_multiplier)
print('Increasing number of instances to %d' %
(current_instance_num * instance_multiplier))
elif topic_traffic_diff < 0: # negative, back off exponentially
target_instance_num = int(current_instance_num / 2)
if target_instance_num > 1:
c.scale_app('elsa', target_instance_num)
print('Decreasing number of instances to %d' %
(target_instance_num))
else:
c.scale_app('elsa', 1)
print('Resetting number of instances to 1')
time.sleep(scale_window)
except KeyboardInterrupt:
print(
'ElSA has been stopped by user, halting app and rolling back deployment. Thanks and bye!'
)
c.delete_app('elsa', force=True)
def create_app(self, id):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(
image=self.docker_image,
network="BRIDGE",
force_pull_image=True,
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(
protocol="HTTP",
path="/status",
grace_period_seconds=300,
interval_seconds=self.heath_check_interval,
timeout_seconds=20,
max_consecutive_failures=0
)
app_suffix = str(md5(str(random())).hexdigest())
app_name = self.app_base_name + app_suffix
new_app = MarathonApp(cpus=CPUS, mem=MEM, disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=self.instances_per_app,
max_launch_delay_seconds=5)
print("Creating {}".format(app_name))
self.marathon_cluster.create_app(app_id=app_name, app=new_app)
self.app_list_queue.put(app_name)
return None
def create_hydra_app(self,
name,
app_path,
app_args,
cpus,
mem,
ports=None,
constraints=None):
""" Create an application that is a shell script.
"""
assert (name not in self.apps)
command = self.get_cmd(app_path, app_args)
app_uri = self.get_app_uri()
r = self.__mt.create_app(
name,
MarathonApp(cmd=command,
cpus=cpus,
mem=mem,
ports=ports,
constraints=constraints,
uris=[app_uri]))
self.apps[name] = {'app': r, 'type': 'script'}
self.wait_app_ready(name, 1)
self.refresh_app_info(name)
return r
def create(self, name, image, command='', **kwargs):
"""Create a new container"""
app_id = self._app_id(name)
l = locals().copy()
l.update(re.match(MATCH, name).groupdict())
image = self.registry + '/' + image
mems = kwargs.get('memory', {}).get(l['c_type'])
m = 0
if mems:
mems = mems.lower()
if mems[-2:-1].isalpha() and mems[-1].isalpha():
mems = mems[:-1]
m = int(mems[:-1])
c = 0.5
cpu = kwargs.get('cpu', {}).get(l['c_type'])
if cpu:
c = cpu
cmd = "docker run --name {name} -P {image} {command}".format(
**locals())
self.client.create_app(
app_id, MarathonApp(cmd=cmd, mem=m, cpus=c, instances=0))
for _ in xrange(POLL_ATTEMPTS):
if self.client.get_app(self._app_id(name)).tasks_running == 0:
return
time.sleep(1)
def kafka_health_check_get_app():
env = {'KAFKA_PORT': '31001'}
app1 = MarathonApp(id='/new-kafka/kafka-broker',
env=env,
tasks_unhealthy=0,
tasks_running=1,
mem=256,
cpus=0.5,
disk=256)
app2 = MarathonApp(id='/sandbox/kafka/kafka-broker',
env=env,
tasks_unhealthy=0,
tasks_running=1,
mem=256,
cpus=0.5,
disk=256)
return [app1, app2]
def create_marathon_app(app_id, config, client):
"""Create a new marathon application with a given
config and marathon client object.
:param config: The marathon configuration to be deployed
:param client: A MarathonClient object"""
client.create_app(app_id, MarathonApp(**config))
wait_for_create(app_id, client)
def post(self):
args = self.reqparse.parse_args()
docker_setup = {"docker": {"image": args['service']}, "type": "DOCKER", "volumes": []}
try:
return str(self.c.create_app(args['service'],
MarathonApp(container=docker_setup, mem=32.0, cpus=0.2, instances=1))), 201
except MarathonError as mhe:
return "Marathon API error: {}".format(mhe), 503
def __init__(self, name, image, volumes=[], network='BRIDGE', privileged=False, labels={}, cmd=None, constraints=[],
residency=None, env={}, health_checks=[], uris=[], cpus=0.5, mem=256, instances=1, disk=512,
port_mappings=[], dependencies=None):
self.name = name
docker = MarathonDockerContainer(image=image, network=network, privileged=privileged,
port_mappings=port_mappings)
container = MarathonContainer(docker=docker, volumes=volumes)
self.app = MarathonApp(id=name, cmd=cmd, cpus=cpus, mem=mem, instances=instances, dependencies=dependencies,
disk=disk, labels=labels, container=container, constraints=constraints,
residency=residency, env=env, health_checks=health_checks, uris=uris)
def redis_health_check_get_app():
app1 = MarathonApp(id='/sandbox/redis-master',
tasks=[MarathonTask(host='127.0.0.1')],
tasks_unhealthy=0,
tasks_running=1,
mem=256,
cpus=0.5,
disk=256)
app2 = MarathonApp(
id='/sandbox/redis-slave',
tasks=[MarathonTask(host='127.0.0.1')],
cmd=
'docker-entrypoint.sh redis-server --port $REDIS_PORT --slaveof 127.0.0.1 $REDIS_PORT',
tasks_running=1,
mem=256,
cpus=0.5,
disk=256)
return [app1, app2]
def create_app(app_instances, delete=False, timeout=20, need_statistics=True):
"""
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(
image="nginx",
network="BRIDGE",
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
"""
"""
http_health_check = MarathonHealthCheck(protocol="HTTP",
path="/",
grace_period_seconds=300,
interval_seconds=4,
timeout_seconds=10,
max_consecutive_failures=3)
"""
http_health_check = MarathonHealthCheck()
app_name = str(hashlib.md5(str(random.random())).hexdigest())
logging.debug("Create cluster {}".format(app_name))
new_app = MarathonApp(
cpus=CPUS,
mem=MEM,
disk=DISK,
#container=app_container,
cmd="while true ;do echo 'hello'; sleep 5; done",
#cmd="echo 'hello'",
#health_checks=[http_health_check],
health_checks=[],
instances=app_instances,
max_launch_delay_seconds=5)
start_time = time.time()
cluster.create_app(app_id=app_name, app=new_app)
logging.debug("Get tasks for cluster {}".format(app_name))
successful_instances, all_starting, app_full_time = \
check_operation_status(start_time, app_name, app_instances, timeout)
if delete:
logging.debug('Delete {}'.format(app_name))
delete_app(app_name, force=True)
if need_statistics:
return {
"app_name": app_name,
"app_full_time": round(app_full_time, 2),
"instances": app_instances,
"successful_instances": successful_instances,
"instances_mean": round(statistics.mean(all_starting), 2),
"instances_median": round(statistics.median(all_starting), 2),
"instances_min": round(min(all_starting), 2),
"instances_max": round(max(all_starting), 2),
"id_run": id_run
}
else:
return {"app_name": app_name}
def create_marathon_app(self, marathon_server, config_file_data):
"""
Creates the App name on the Marathon Server with minimal default values.
"""
self.logger.info("App wasn't found. Creating new marathon app")
self.logger.info(config_file_data["id"])
# Initialize app creation and name space
marathon_server.create_app(config_file_data["id"],
MarathonApp(cmd='test', mem=1, cpus=.01))
self.logger.info("Done creating marathon app.")
def create_app(client):
app = client.create_app(
'sleepy',
MarathonApp(cmd='while true; do sleep 33 ; done',
mem=32,
cpus=0.1,
container=MarathonContainer(
docker=MarathonDockerContainer(image='python:3'),
type='DOCKER')))
return app
def environment_health_check_get_app():
zoo1 = MarathonApp(id='/sandbox/zookeeper/zookeeper-1',
env={'ZOO_PORT': '10001'})
zoo2 = MarathonApp(id='/sandbox/zookeeper/zookeeper-2',
env={'ZOO_PORT': '10001'})
zoo3 = MarathonApp(id='/sandbox/zookeeper/zookeeper-3',
env={'ZOO_PORT': '10001'})
redis1 = MarathonApp(id='/sandbox/redis/redis-master',
env={'REDIS_PORT': '20001'})
redis2 = MarathonApp(id='/sandbox/redis/redis-slave',
env={'REDIS_PORT': '20001'})
cass_labels = {
'_jmx_port': '30001',
'_internode_communication_port': '30002',
'_tls_internode_communication_port': '30003',
'_thrift_client_port': '30004',
'_cql_native_port': '30005'
}
cass1 = MarathonApp(id='/sandbox/cassandra/cassandra-seed',
labels=cass_labels)
cass2 = MarathonApp(id='/sandbox/cassandra/cassandra-node',
labels=cass_labels)
kafka1 = MarathonApp(id='/sandbox/kafka/kafka-broker',
env={'KAFKA_PORT': '60001'})
port_mappings = [MarathonContainerPortMapping(service_port=90001)]
docker = MarathonDockerContainer(image='node', port_mappings=port_mappings)
container = MarathonContainer(docker=docker)
webui1 = MarathonApp(id='/sandbox/webui/webui-app', container=container)
port_mappings = [
MarathonContainerPortMapping(service_port=i)
for i in [80000, 80002, 80001]
]
docker = MarathonDockerContainer(image='spark',
port_mappings=port_mappings)
container = MarathonContainer(docker=docker)
spark1 = MarathonApp(id='/sandbox/spark/spark-app', container=container)
return [
zoo1, zoo2, zoo3, redis1, redis2, cass1, cass2, kafka1, webui1, spark1
]
def commit(self):
docker = MarathonDockerContainer(image=self.image, network='BRIDGE')
container = MarathonContainer(docker=docker)
self.cmd = '{} && wget --method=DELETE {}/v2/apps/{}'.format(
self.cmd, self.marathon_client.servers[0], self.app_name)
app = MarathonApp(id=self.app_name,
cmd=self.cmd,
cpus=self.cpus,
mem=self.mem,
instances=1,
container=container)
self.marathon_client.create_app(self.app_name, app)
def spark_health_check_get_app():
port_mappings = [
MarathonContainerPortMapping(service_port=10001),
MarathonContainerPortMapping(service_port=10002),
MarathonContainerPortMapping(service_port=10003),
MarathonContainerPortMapping(service_port=10004)
]
docker = MarathonDockerContainer(image='node', port_mappings=port_mappings)
container = MarathonContainer(docker=docker)
labels = {'_tonomi_environment': 'sandbox', '_tonomi_application': 'spark'}
app1 = MarathonApp(id='/sandbox/spark/spark-app',
labels=labels,
cpus=0.5,
mem=256,
disk=256,
instances=1,
container=container,
tasks=[MarathonTask(host='127.0.0.1')])
app1.tasks_unhealthy = 0
app1.tasks_running = 1
return [app1]
try:
client.create_app(marathon_app.id, marathon_app)
logging.info('Successfully created GeoServer app in Marathon.')
except:
logging.exception('Unable to create new Marathon App for GeoServer.')
sys.exit(1)
def block_for_healthy_app(client, app_name, target_healthy):
while client.get_app(app_name).tasks_healthy < target_healthy:
logging.info("Waiting for healthy app %s." % app_name)
time.sleep(5)
with open('configs/geoserver.json') as marathon_config:
marathon_app = MarathonApp.from_json(json.load(marathon_config))
# Shim in the appropriate config values from environment
marathon_app.id = GEOSERVER_APP
marathon_app.cpus = GEOSERVER_CPUS
marathon_app.mem = GEOSERVER_MEMORY
marathon_app.instances = 1
marathon_app.env['GOSU_USER'] = GOSU_USER
marathon_app.env['GEOSERVER_HOSTNAME'] = HAPROXY_VHOST
marathon_app.env['INSTANCE_MEMORY'] = str(GEOSERVER_MEMORY)
marathon_app.container.volumes[0].host_path = HOST_GEOSERVER_DATA_DIR
# If HOST_SUPPLEMENTAL_DATA_DIRS set, add read-only volume mounts as needed
if HOST_SUPPLEMENTAL_DATA_DIRS and len(HOST_SUPPLEMENTAL_DATA_DIRS.split(',')):
for host_dir in HOST_SUPPLEMENTAL_DATA_DIRS.split(','):
volume = MarathonContainerVolume(host_dir, host_dir, 'RO')
marathon_app.container.volumes.append(volume)
marathon_app.container.docker.image = GEOSERVER_IMAGE
