def __init__(self, name, ports, env_name=None):
self.name = name
self.ports = ports
self.env_name = env_name
volume_name = get_volume_name(name)
volumes = [
MarathonContainerVolume(container_path=volume_name,
host_path=None,
mode='RW',
persistent={'size': 512}),
MarathonContainerVolume(container_path='/var/lib/zookeeper',
host_path=volume_name,
mode='RW',
persistent=None)
]
constraints = [
MarathonConstraint(field='hostname', operator='LIKE', value='')
]
residency = Residency(task_lost_behavior='WAIT_FOREVER')
health_checks = [
MarathonHealthCheck(grace_period_seconds=300,
interval_seconds=20,
max_consecutive_failures=3,
protocol='TCP',
timeout_seconds=20,
ignore_http1xx=False,
port=ports[0])
]
cmd = 'export ZOO_SERVERS="{}" && /docker-entrypoint.sh zkServer.sh start-foreground'
labels = {
'_tonomi_application': 'zookeeper',
'_client_conn_port': str(ports[0]),
'_follower_conn_port': str(ports[1]),
'_server_conn_port': str(ports[2])
}
if self.env_name:
labels['_tonomi_environment'] = self.env_name
env = {'ZOO_MY_ID': '', 'ZOO_PORT': str(ports[0])}
super().__init__(name,
image='zookeeper',
volumes=volumes,
network='HOST',
labels=labels,
cmd=cmd,
constraints=constraints,
residency=residency,
env=env,
health_checks=health_checks,
cpus=0.5,
mem=400,
instances=1,
disk=400)
def create_app(self, id):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(
image=self.docker_image,
network="BRIDGE",
force_pull_image=True,
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(
protocol="HTTP",
path="/status",
grace_period_seconds=300,
interval_seconds=self.heath_check_interval,
timeout_seconds=20,
max_consecutive_failures=0
)
app_suffix = str(md5(str(random())).hexdigest())
app_name = self.app_base_name + app_suffix
new_app = MarathonApp(cpus=CPUS, mem=MEM, disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=self.instances_per_app,
max_launch_delay_seconds=5)
print("Creating {}".format(app_name))
self.marathon_cluster.create_app(app_id=app_name, app=new_app)
self.app_list_queue.put(app_name)
return None
def create_app(app_instances):
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(image="nginx",
network="BRIDGE",
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
http_health_check = MarathonHealthCheck(protocol="HTTP",
path="/",
grace_period_seconds=300,
interval_seconds=30,
timeout_seconds=20,
max_consecutive_failures=3)
app_name = str(hashlib.md5(str(random.random())).hexdigest())
logging.debug("Create cluster {}".format(app_name))
app_constraint = MarathonConstraint(field="hostname", operator="UNIQUE")
new_app = MarathonApp(cpus=CPUS,
mem=MEM,
disk=DISK,
container=app_container,
health_checks=[http_health_check],
instances=app_instances,
constraints=[app_constraint],
max_launch_delay_seconds=5)
print("Creating {}".format(app_name))
cluster.create_app(app_id=app_name, app=new_app)
return None
def create_app(app_instances, delete=False, timeout=20, need_statistics=True):
"""
port_mapping = MarathonContainerPortMapping(container_port=80,
protocol="tcp")
app_docker = MarathonDockerContainer(
image="nginx",
network="BRIDGE",
port_mappings=[port_mapping])
app_container = MarathonContainer(docker=app_docker)
"""
"""
http_health_check = MarathonHealthCheck(protocol="HTTP",
path="/",
grace_period_seconds=300,
interval_seconds=4,
timeout_seconds=10,
max_consecutive_failures=3)
"""
http_health_check = MarathonHealthCheck()
app_name = str(hashlib.md5(str(random.random())).hexdigest())
logging.debug("Create cluster {}".format(app_name))
new_app = MarathonApp(
cpus=CPUS,
mem=MEM,
disk=DISK,
#container=app_container,
cmd="while true ;do echo 'hello'; sleep 5; done",
#cmd="echo 'hello'",
#health_checks=[http_health_check],
health_checks=[],
instances=app_instances,
max_launch_delay_seconds=5)
start_time = time.time()
cluster.create_app(app_id=app_name, app=new_app)
logging.debug("Get tasks for cluster {}".format(app_name))
successful_instances, all_starting, app_full_time = \
check_operation_status(start_time, app_name, app_instances, timeout)
if delete:
logging.debug('Delete {}'.format(app_name))
delete_app(app_name, force=True)
if need_statistics:
return {
"app_name": app_name,
"app_full_time": round(app_full_time, 2),
"instances": app_instances,
"successful_instances": successful_instances,
"instances_mean": round(statistics.mean(all_starting), 2),
"instances_median": round(statistics.median(all_starting), 2),
"instances_min": round(min(all_starting), 2),
"instances_max": round(max(all_starting), 2),
"id_run": id_run
}
else:
return {"app_name": app_name}
def __init__(self,
name,
service_port=0,
cassandra_host=None,
cassandra_port=None,
env_name=None):
port_mappings = [
MarathonContainerPortMapping(container_port=3005,
host_port=0,
service_port=service_port,
protocol='tcp')
]
labels = {'_tonomi_application': 'webui'}
if env_name:
labels['_tonomi_environment'] = env_name
cmd = 'cd ${MESOS_SANDBOX}/webclient && npm install && NODE_ENV=production WEB_CLIENT_PORT=3005 npm start'
env = {
'CASSANDRA_HOST': cassandra_host,
'CASSANDRA_PORT': str(cassandra_port)
}
uris = [
'https://s3-us-west-1.amazonaws.com/streaming-artifacts/ui.tar.gz'
]
health_checks = [
MarathonHealthCheck(path='/',
protocol='HTTP',
port_index=0,
grace_period_seconds=300,
interval_seconds=60,
timeout_seconds=30,
max_consecutive_failures=3,
ignore_http1xx=True)
]
super().__init__(name,
image='node',
network='BRIDGE',
labels=labels,
cmd=cmd,
env=env,
health_checks=health_checks,
uris=uris,
cpus=0.5,
mem=300,
instances=2,
disk=256,
port_mappings=port_mappings)
def write(self, user, request_app, original_app):
hc_data = {
"command": None,
"gracePeriodSeconds": 30,
"intervalSeconds": 10,
"maxConsecutiveFailures": 3,
"path": "/marathon/healthcheck",
"portIndex": 0,
"protocol": "HTTP",
"timeoutSeconds": 5,
"ignoreHttp1xx": False
}
request_app.health_checks.append(MarathonHealthCheck.from_json(hc_data))
return request_app
def get_health_check(grace_period_seconds=300,
interval_seconds=20,
max_consecutive_failures=3,
timeout_seconds=20,
protocol='TCP',
ignore_http1xx=False,
port=None,
path=None,
port_index=None):
return MarathonHealthCheck(
grace_period_seconds=grace_period_seconds,
interval_seconds=interval_seconds,
max_consecutive_failures=max_consecutive_failures,
protocol=protocol,
timeout_seconds=timeout_seconds,
ignore_http1xx=ignore_http1xx,
port=port,
path=path,
port_index=port_index)
def get_health_checks(self):
from marathon.models import MarathonHealthCheck
checks = []
if self.project.health_check:
checks = [MarathonHealthCheck(port_index=0, path=self.project.health_check)]
return checks
cmd = 'bash ${{MESOS_SANDBOX}}/twitter-consumer-runner.sh --movie "{}" --search-since "{}"'.format(movie, search_since)
labels = {
'_tonomi_application': 'twitter-consumer'
}
env = {
'KAFKA_BROKER_LIST': kafka_broker,
'TWITTER_ACCESS_TOKEN': access_token,
'TWITTER_ACCESS_TOKEN_SECRET': access_token_secret,
'TWITTER_CONSUMER_KEY': consumer_key,
'TWITTER_CONSUMER_SECRET': consumer_secret
}
uris = ['https://s3-us-west-1.amazonaws.com/streaming-artifacts/twitter-consumer.tar.gz']
name = '{}/{}'.format(instance_name, ''.join([i for i in movie if i.isalpha() or i == ' ']).lower().replace(' ', '-'))
health_checks = [MarathonHealthCheck(grace_period_seconds=300, interval_seconds=60, max_consecutive_failures=3,
protocol='COMMAND', timeout_seconds=20, ignore_http1xx=False,
command={"value": 'test ! -z \"$(ps ax|egrep \"(twitter-consumer)*.(jar)\"|grep -v grep)\"'})]
consumer_app = Node(name=name, image='java:8', labels=labels, cmd=cmd, env=env, uris=uris, cpus=0.1, mem=256, disk=0,
health_checks=health_checks)
add_movie = CassandraAddMovie(marathon_client, cassandra_host, cassandra_port, movie)
add_movie.commit()
time.sleep(7)
try:
marathon_client.create_app(name, consumer_app.app)
except:
pass
def __init__(self,
name,
cassandra_host,
cassandra_port,
kafka_host,
kafka_port,
redis_host,
redis_port,
env_name=None):
self.name = name
self.cassandra_host = cassandra_host
self.cassandra_port = cassandra_port
self.redis_host = redis_host
self.redis_port = redis_port
self.kafka_host = kafka_host
self.kafka_port = kafka_port
self.env_name = env_name
port_mappings = [
MarathonContainerPortMapping(container_port=8088,
host_port=0,
service_port=0,
protocol='tcp'),
MarathonContainerPortMapping(container_port=8042,
host_port=0,
service_port=0,
protocol='tcp'),
MarathonContainerPortMapping(container_port=4040,
host_port=0,
service_port=0,
protocol='tcp'),
MarathonContainerPortMapping(container_port=2122,
host_port=0,
service_port=0,
protocol='tcp')
]
labels = {'_tonomi_application': 'spark'}
if self.env_name:
labels['_tonomi_environment'] = self.env_name
env = {
'CASSANDRA_HOST': self.cassandra_host,
'CASSANDRA_PORT': str(self.cassandra_port),
'KAFKA_BROKER_LIST': '{}:{}'.format(self.kafka_host,
self.kafka_port),
'REDIS_HOST': self.redis_host,
'REDIS_PORT': str(self.redis_port)
}
cmd = 'cd ${MESOS_SANDBOX} && bash ./streaming-runner.sh'
uris = [
'https://s3-us-west-1.amazonaws.com/streaming-artifacts/in-stream-tweets-analyzer.tar.gz',
'https://s3-us-west-1.amazonaws.com/streaming-artifacts/dictionary-populator.tar.gz'
]
health_checks = [
MarathonHealthCheck(grace_period_seconds=300,
interval_seconds=20,
max_consecutive_failures=3,
protocol='HTTP',
timeout_seconds=20,
ignore_http1xx=True,
port_index=2)
]
super().__init__(name,
image='sequenceiq/spark:1.6.0',
network='BRIDGE',
labels=labels,
cmd=cmd,
env=env,
health_checks=health_checks,
cpus=0.5,
mem=500,
instances=1,
disk=512,
port_mappings=port_mappings,
uris=uris)
def __init__(self, name, port, zoo_host, zoo_port, env_name=None):
self.name = name
self.port = port
self.zoo_host = zoo_host
self.zoo_port = zoo_port
self.env_name = env_name
volume_name = get_volume_name(name)
volumes = [
MarathonContainerVolume(container_path=volume_name,
host_path=None,
mode='RW',
persistent={'size': 512}),
MarathonContainerVolume(container_path='/var/lib/kafka',
host_path=volume_name,
mode='RW',
persistent=None)
]
port_mappings = [
MarathonContainerPortMapping(container_port=port,
host_port=port,
service_port=port,
protocol='tcp')
]
constraints = [MarathonConstraint(field='hostname', operator='UNIQUE')]
residency = Residency(task_lost_behavior='WAIT_FOREVER')
labels = {'_tonomi_application': 'kafka', '_cluster_port': str(port)}
if self.env_name:
labels['_tonomi_environment'] = self.env_name
cmd = 'export KAFKA_ADVERTISED_HOST_NAME=$HOST && start-kafka.sh'
env = {
'KAFKA_ADVERTISED_PORT': str(port),
'KAFKA_ZOOKEEPER_CONNECT': '{}:{}'.format(self.zoo_host,
self.zoo_port),
'KAFKA_PORT': str(port)
}
health_checks = [
MarathonHealthCheck(grace_period_seconds=300,
interval_seconds=20,
max_consecutive_failures=3,
protocol='TCP',
timeout_seconds=20,
ignore_http1xx=False,
port=port)
]
super().__init__(name,
image='wurstmeister/kafka',
network='BRIDGE',
labels=labels,
cmd=cmd,
env=env,
health_checks=health_checks,
cpus=0.4,
mem=300,
instances=3,
disk=256,
volumes=volumes,
port_mappings=port_mappings,
residency=residency,
constraints=constraints)
def __init__(self, name, ports, is_seed=True, env_name=None):
self.env_name = env_name
volume_name = get_volume_name(name)
volumes = [
MarathonContainerVolume(container_path=volume_name,
host_path=None,
mode='RW',
persistent={'size': 512}),
MarathonContainerVolume(container_path='/var/lib/cassandra',
host_path=volume_name,
mode='RW',
persistent=None)
]
constraints = [MarathonConstraint(field='hostname', operator='UNIQUE')]
residency = Residency(task_lost_behavior='WAIT_FOREVER')
health_checks = [
MarathonHealthCheck(grace_period_seconds=300,
interval_seconds=20,
max_consecutive_failures=3,
protocol='TCP',
timeout_seconds=20,
ignore_http1xx=False,
port=ports[9042])
]
ports_map = {
'p11': 9042,
'p12': ports[9042],
'p21': 9160,
'p22': ports[9160],
'p31': 7199,
'p32': ports[7199],
'p41': 7000,
'p42': ports[7000],
'p51': 7001,
'p52': ports[7001]
}
cmd = "chown -R cassandra /var/lib/cassandra && sed -i 's/{p11}/{p12}/' /etc/cassandra/default.conf/cqlshrc.sample && sed -i 's/{p31}/{p32}/' /etc/cassandra/default.conf/cassandra-env.sh && sed -i 's/{p41}/{p42}/;s/{p51}/{p52}/;s/{p11}/{p12}/;s/{p21}/{p22}/;s/{p31}/{p32}/' /etc/cassandra/default.conf/cassandra.yaml".format(
**ports_map)
if is_seed:
cmd += ' && cd ${MESOS_SANDBOX}/cassandra-schema && ./apply_schema.sh & start'
else:
cmd += ' && start'
labels = {
'_tonomi_application': 'cassandra',
'_jmx_port': str(ports[7199]),
'_internode_communication_port': str(ports[7000]),
'_tls_internode_communication_port': str(ports[7001]),
'_thrift_client_port': str(ports[9160]),
'_cql_native_port': str(ports[9042])
}
if self.env_name:
labels['_tonomi_environment'] = self.env_name
env = {'SEEDS': '', 'CASSANDRA_PORT': str(ports[9042])}
uris = [
'https://s3-us-west-1.amazonaws.com/streaming-artifacts/mk-cassandra-schema.tar.gz'
]
super().__init__(name,
image='poklet/cassandra',
volumes=volumes,
network='HOST',
labels=labels,
cmd=cmd,
constraints=constraints,
residency=residency,
env=env,
health_checks=health_checks,
uris=uris,
cpus=0.5,
mem=400,
instances=1,
disk=512)
def __init__(self, name, port, is_master=True, env_name=None):
self.name = name
self.env_name = env_name
volume_name = get_volume_name(name)
volumes = [
MarathonContainerVolume(container_path=volume_name,
host_path=None,
mode='RW',
persistent={'size': 512}),
MarathonContainerVolume(container_path='/var/lib/redis',
host_path=volume_name,
mode='RW',
persistent=None)
]
constraints = [MarathonConstraint(field='hostname', operator='UNIQUE')]
residency = Residency(task_lost_behavior='WAIT_FOREVER')
health_checks = [
MarathonHealthCheck(grace_period_seconds=300,
interval_seconds=20,
max_consecutive_failures=3,
protocol='TCP',
timeout_seconds=20,
ignore_http1xx=False,
port=port)
]
service_port = 0 if not is_master else port
port_mappings = [
MarathonContainerPortMapping(container_port=port,
host_port=port,
service_port=service_port,
protocol='tcp')
]
cmd = 'docker-entrypoint.sh redis-server --port $REDIS_PORT '
if not is_master:
cmd += '--slaveof {} $REDIS_PORT'
labels = {'_tonomi_application': 'redis', '_cluster_port': str(port)}
if self.env_name:
labels['_tonomi_environment'] = self.env_name
env = {'REDIS_PORT': str(port)}
super().__init__(name,
image='redis',
volumes=volumes,
network='BRIDGE',
labels=labels,
cmd=cmd,
constraints=constraints,
residency=residency,
env=env,
health_checks=health_checks,
cpus=0.5,
mem=300,
instances=1,
disk=512,
port_mappings=port_mappings)