def kill_marathon_app(full_appid, cluster, client, soa_dir):
service, instance, _, __ = (s.replace("--", "_")
for s in decompose_job_id(full_appid))
service_instance_config = marathon_tools.load_marathon_service_config(
service=service, instance=instance, cluster=cluster, soa_dir=soa_dir)
complete_config = service_instance_config.format_marathon_app_dict()
registrations = service_instance_config.get_registrations()
service_namespace_config = marathon_tools.load_service_namespace_config(
service=service, namespace=registrations[0])
drain_method = drain_lib.get_drain_method(
service_instance_config.get_drain_method(service_namespace_config),
service=service,
instance=instance,
registrations=registrations,
drain_method_params=service_instance_config.get_drain_method_params(
service_namespace_config),
)
bounce_func = bounce_lib.get_bounce_method_func("down")
while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
app_to_kill = client.get_app(full_appid)
(
old_app_live_happy_tasks,
old_app_live_unhappy_tasks,
old_app_draining_tasks,
old_app_at_risk_tasks,
) = get_tasks_by_state(
other_apps=[app_to_kill],
drain_method=drain_method,
service=service,
nerve_ns=registrations[0],
bounce_health_params=service_instance_config.
get_bounce_health_params(service_namespace_config),
)
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=complete_config,
new_app_running="",
happy_new_tasks=[],
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
old_app_draining_tasks=old_app_draining_tasks,
old_app_at_risk_tasks=old_app_at_risk_tasks,
serviceinstance=f"{service}.{instance}",
bounce_method="down",
service=service,
cluster=cluster,
instance=instance,
marathon_jobid=full_appid,
client=client,
soa_dir=soa_dir,
)
paasta_print("Sleeping for 10 seconds to give the tasks time to drain")
time.sleep(10)
paasta_print(f"Successfully killed {full_appid}")
def kill_marathon_app(full_appid, cluster, client, soa_dir):
service, instance, _, __ = (s.replace('--', '_') for s in decompose_job_id(full_appid))
service_instance_config = marathon_tools.load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
complete_config = service_instance_config.format_marathon_app_dict()
nerve_ns = service_instance_config.get_nerve_namespace()
service_namespace_config = marathon_tools.load_service_namespace_config(service=service, namespace=nerve_ns)
drain_method = drain_lib.get_drain_method(
service_instance_config.get_drain_method(service_namespace_config),
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=service_instance_config.get_drain_method_params(service_namespace_config),
)
bounce_func = bounce_lib.get_bounce_method_func('down')
while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
app_to_kill = client.get_app(full_appid)
(
old_app_live_happy_tasks,
old_app_live_unhappy_tasks,
old_app_draining_tasks,
old_app_at_risk_tasks,
) = get_tasks_by_state(
other_apps=[app_to_kill],
drain_method=drain_method,
service=service,
nerve_ns=nerve_ns,
bounce_health_params=service_instance_config.get_bounce_health_params(service_namespace_config),
)
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=complete_config,
new_app_running='',
happy_new_tasks=[],
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
old_app_draining_tasks=old_app_draining_tasks,
old_app_at_risk_tasks=old_app_at_risk_tasks,
serviceinstance="{}.{}".format(service, instance),
bounce_method='down',
service=service,
cluster=cluster,
instance=instance,
marathon_jobid=full_appid,
client=client,
soa_dir=soa_dir,
)
paasta_print("Sleeping for 10 seconds to give the tasks time to drain")
time.sleep(10)
paasta_print("Sucessfully killed {}".format(full_appid))
def test_register_drain_method():
with mock.patch('drain_lib._drain_methods'):
@drain_lib.register_drain_method('FAKEDRAINMETHOD')
class FakeDrainMethod(drain_lib.DrainMethod):
pass
assert type(drain_lib.get_drain_method('FAKEDRAINMETHOD', 'srv', 'inst', 'ns')) == FakeDrainMethod
def test_register_drain_method():
with mock.patch.dict(drain_lib._drain_methods):
@drain_lib.register_drain_method('FAKEDRAINMETHOD')
class FakeDrainMethod(drain_lib.DrainMethod):
pass
assert type(drain_lib.get_drain_method('FAKEDRAINMETHOD', 'srv', 'inst', 'ns')) == FakeDrainMethod
def recreate_drain_method(self):
"""Re-instantiate self.drain_method. Should be called after self.service_config changes."""
self.drain_method = drain_lib.get_drain_method(
name=self.service_config.get_drain_method(self.service_config.service_namespace_config),
service=self.service_name,
instance=self.instance_name,
nerve_ns=self.service_config.get_nerve_namespace(),
**self.service_config.get_drain_method_params(self.service_config.service_namespace_config)
)
def recreate_drain_method(self) -> None:
"""Re-instantiate self.drain_method. Should be called after self.service_config changes."""
self.drain_method = drain_lib.get_drain_method(
name=self.service_config.get_drain_method(self.service_config.service_namespace_config),
service=self.service_name,
instance=self.instance_name,
nerve_ns=self.service_config.get_nerve_namespace(),
**self.service_config.get_drain_method_params(self.service_config.service_namespace_config),
)
def test_register_drain_method():
with mock.patch.dict(drain_lib._drain_methods):
@drain_lib.register_drain_method("FAKEDRAINMETHOD")
class FakeDrainMethod(drain_lib.DrainMethod):
pass
assert (type(
drain_lib.get_drain_method("FAKEDRAINMETHOD", "srv", "inst",
"ns")) == FakeDrainMethod)
def deploy_service(
service,
instance,
marathon_jobid,
config,
client,
bounce_method,
drain_method_name,
drain_method_params,
nerve_ns,
bounce_health_params,
soa_dir,
):
"""Deploy the service to marathon, either directly or via a bounce if needed.
Called by setup_service when it's time to actually deploy.
:param service: The name of the service to deploy
:param instance: The instance of the service to deploy
:param marathon_jobid: Full id of the marathon job
:param config: The complete configuration dict to send to marathon
:param client: A MarathonClient object
:param bounce_method: The bounce method to use, if needed
:param drain_method_name: The name of the traffic draining method to use.
:param nerve_ns: The nerve namespace to look in.
:param bounce_health_params: A dictionary of options for bounce_lib.get_happy_tasks.
:returns: A tuple of (status, output) to be used with send_sensu_event"""
def log_deploy_error(errormsg, level='event'):
return _log(service=service,
line=errormsg,
component='deploy',
level='event',
cluster=cluster,
instance=instance)
short_id = marathon_tools.format_job_id(service, instance)
cluster = load_system_paasta_config().get_cluster()
existing_apps = marathon_tools.get_matching_apps(service,
instance,
client,
embed_failures=True)
new_app_list = [a for a in existing_apps if a.id == '/%s' % config['id']]
other_apps = [a for a in existing_apps if a.id != '/%s' % config['id']]
serviceinstance = "%s.%s" % (service, instance)
if new_app_list:
new_app = new_app_list[0]
if len(new_app_list) != 1:
raise ValueError("Only expected one app per ID; found %d" %
len(new_app_list))
new_app_running = True
happy_new_tasks = bounce_lib.get_happy_tasks(new_app, service,
nerve_ns,
**bounce_health_params)
else:
new_app_running = False
happy_new_tasks = []
try:
drain_method = drain_lib.get_drain_method(
drain_method_name,
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=drain_method_params,
)
except KeyError:
errormsg = 'ERROR: drain_method not recognized: %s. Must be one of (%s)' % \
(drain_method_name, ', '.join(drain_lib.list_drain_methods()))
log_deploy_error(errormsg)
return (1, errormsg)
old_app_live_happy_tasks, old_app_live_unhappy_tasks, old_app_draining_tasks = get_old_happy_unhappy_draining_tasks(
other_apps, drain_method, service, nerve_ns, bounce_health_params)
if new_app_running:
protected_draining_tasks = set()
if new_app.instances < config['instances']:
client.scale_app(app_id=new_app.id,
instances=config['instances'],
force=True)
elif new_app.instances > config['instances']:
num_tasks_to_scale = max(
min(len(new_app.tasks), new_app.instances) -
config['instances'], 0)
task_dict = get_old_happy_unhappy_draining_tasks_for_app(
new_app,
drain_method,
service,
nerve_ns,
bounce_health_params,
)
scaling_app_happy_tasks = list(task_dict['happy'])
scaling_app_unhappy_tasks = list(task_dict['unhappy'])
scaling_app_draining_tasks = list(task_dict['draining'])
tasks_to_move_draining = min(len(scaling_app_draining_tasks),
num_tasks_to_scale)
old_app_draining_tasks[new_app.id] = set(
scaling_app_draining_tasks[:tasks_to_move_draining])
protected_draining_tasks.update(
scaling_app_draining_tasks[:tasks_to_move_draining])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_draining
tasks_to_move_unhappy = min(len(scaling_app_unhappy_tasks),
num_tasks_to_scale)
old_app_live_unhappy_tasks[new_app.id] = set(
scaling_app_unhappy_tasks[:tasks_to_move_unhappy])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_unhappy
tasks_to_move_happy = min(len(scaling_app_happy_tasks),
num_tasks_to_scale)
old_app_live_happy_tasks[new_app.id] = set(
scaling_app_happy_tasks[:tasks_to_move_happy])
happy_new_tasks = scaling_app_happy_tasks[tasks_to_move_happy:]
# If any tasks on the new app happen to be draining (e.g. someone reverts to an older version with
# `paasta mark-for-deployment`), then we should undrain them.
for task in new_app.tasks:
if task not in protected_draining_tasks:
drain_method.stop_draining(task)
# Re-drain any already draining tasks on old apps
for tasks in old_app_draining_tasks.values():
for task in tasks:
drain_method.drain(task)
# log all uncaught exceptions and raise them again
try:
try:
bounce_func = bounce_lib.get_bounce_method_func(bounce_method)
except KeyError:
errormsg = 'ERROR: bounce_method not recognized: %s. Must be one of (%s)' % \
(bounce_method, ', '.join(bounce_lib.list_bounce_methods()))
log_deploy_error(errormsg)
return (1, errormsg)
try:
with bounce_lib.bounce_lock_zookeeper(short_id):
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=config,
new_app_running=new_app_running,
happy_new_tasks=happy_new_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
old_app_draining_tasks=old_app_draining_tasks,
service=service,
bounce_method=bounce_method,
serviceinstance=serviceinstance,
cluster=cluster,
instance=instance,
marathon_jobid=marathon_jobid,
client=client,
soa_dir=soa_dir,
)
except bounce_lib.LockHeldException:
log.error("Instance %s already being bounced. Exiting", short_id)
return (1, "Instance %s is already being bounced." % short_id)
except Exception:
loglines = ['Exception raised during deploy of service %s:' % service]
loglines.extend(traceback.format_exc().rstrip().split("\n"))
for logline in loglines:
log_deploy_error(logline, level='debug')
raise
return (0, 'Service deployed.')
def deploy_service(
service: str,
instance: str,
marathon_jobid: str,
config: marathon_tools.FormattedMarathonAppDict,
clients: marathon_tools.MarathonClients,
marathon_apps_with_clients: Collection[Tuple[MarathonApp, MarathonClient]],
bounce_method: str,
drain_method_name: str,
drain_method_params: Dict[str, Any],
nerve_ns: str,
bounce_health_params: Dict[str, Any],
soa_dir: str,
job_config: marathon_tools.MarathonServiceConfig,
bounce_margin_factor: float = 1.0,
) -> Tuple[int, str, Optional[float]]:
"""Deploy the service to marathon, either directly or via a bounce if needed.
Called by setup_service when it's time to actually deploy.
:param service: The name of the service to deploy
:param instance: The instance of the service to deploy
:param marathon_jobid: Full id of the marathon job
:param config: The complete configuration dict to send to marathon
:param clients: A MarathonClients object
:param bounce_method: The bounce method to use, if needed
:param drain_method_name: The name of the traffic draining method to use.
:param nerve_ns: The nerve namespace to look in.
:param bounce_health_params: A dictionary of options for bounce_lib.get_happy_tasks.
:param bounce_margin_factor: the multiplication factor used to calculate the number of instances to be drained
:returns: A tuple of (status, output, bounce_in_seconds) to be used with send_sensu_event"""
def log_deploy_error(errormsg: str, level: str = 'event') -> None:
return _log(
service=service,
line=errormsg,
component='deploy',
level='event',
cluster=cluster,
instance=instance,
)
system_paasta_config = load_system_paasta_config()
cluster = system_paasta_config.get_cluster()
existing_apps_with_clients = marathon_tools.get_matching_apps_with_clients(
service=service,
instance=instance,
marathon_apps_with_clients=marathon_apps_with_clients,
)
new_client = clients.get_current_client_for_service(job_config)
new_apps_with_clients_list: List[Tuple[MarathonApp, MarathonClient]] = []
other_apps_with_clients: List[Tuple[MarathonApp, MarathonClient]] = []
for a, c in existing_apps_with_clients:
if a.id == '/%s' % config['id'] and c == new_client:
new_apps_with_clients_list.append((a, c))
else:
other_apps_with_clients.append((a, c))
serviceinstance = "%s.%s" % (service, instance)
if new_apps_with_clients_list:
new_app, new_client = new_apps_with_clients_list[0]
if len(new_apps_with_clients_list) != 1:
raise ValueError(
"Only expected one app per ID per shard; found %d" %
len(new_apps_with_clients_list))
new_app_running = True
happy_new_tasks = bounce_lib.get_happy_tasks(
new_app,
service,
nerve_ns,
system_paasta_config,
**bounce_health_params,
)
else:
new_app_running = False
happy_new_tasks = []
try:
drain_method = drain_lib.get_drain_method(
drain_method_name,
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=drain_method_params,
)
except KeyError:
errormsg = 'ERROR: drain_method not recognized: %s. Must be one of (%s)' % \
(drain_method_name, ', '.join(drain_lib.list_drain_methods()))
log_deploy_error(errormsg)
return (1, errormsg, None)
try:
draining_hosts = get_draining_hosts()
except ReadTimeout as e:
errormsg = "ReadTimeout encountered trying to get draining hosts: %s" % e
return (1, errormsg, 60)
(
old_app_live_happy_tasks,
old_app_live_unhappy_tasks,
old_app_draining_tasks,
old_app_at_risk_tasks,
) = get_tasks_by_state(
other_apps_with_clients=other_apps_with_clients,
drain_method=drain_method,
service=service,
nerve_ns=nerve_ns,
bounce_health_params=bounce_health_params,
system_paasta_config=system_paasta_config,
log_deploy_error=log_deploy_error,
draining_hosts=draining_hosts,
)
# The first thing we need to do is take up the "slack" of old apps, to stop
# them from launching new things that we are going to have to end up draining
# and killing anyway.
for a, c in other_apps_with_clients:
marathon_tools.take_up_slack(app=a, client=c)
num_at_risk_tasks = 0
if new_app_running:
num_at_risk_tasks = get_num_at_risk_tasks(
new_app, draining_hosts=draining_hosts)
if new_app.instances < config['instances'] + num_at_risk_tasks:
log.info("Scaling %s up from %d to %d instances." %
(new_app.id, new_app.instances,
config['instances'] + num_at_risk_tasks))
new_client.scale_app(app_id=new_app.id,
instances=config['instances'] +
num_at_risk_tasks,
force=True)
# If we have more than the specified number of instances running, we will want to drain some of them.
# We will start by draining any tasks running on at-risk hosts.
elif new_app.instances > config['instances']:
num_tasks_to_scale = max(
min(len(new_app.tasks), new_app.instances) -
config['instances'], 0)
task_dict = get_tasks_by_state_for_app(
app=new_app,
drain_method=drain_method,
service=service,
nerve_ns=nerve_ns,
bounce_health_params=bounce_health_params,
system_paasta_config=system_paasta_config,
log_deploy_error=log_deploy_error,
draining_hosts=draining_hosts,
)
scaling_app_happy_tasks = list(task_dict['happy'])
scaling_app_unhappy_tasks = list(task_dict['unhappy'])
scaling_app_draining_tasks = list(task_dict['draining'])
scaling_app_at_risk_tasks = list(task_dict['at_risk'])
tasks_to_move_draining = min(len(scaling_app_draining_tasks),
num_tasks_to_scale)
old_app_draining_tasks[(new_app.id, new_client)] = set(
scaling_app_draining_tasks[:tasks_to_move_draining])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_draining
tasks_to_move_unhappy = min(len(scaling_app_unhappy_tasks),
num_tasks_to_scale)
old_app_live_unhappy_tasks[(new_app.id, new_client)] = set(
scaling_app_unhappy_tasks[:tasks_to_move_unhappy], )
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_unhappy
tasks_to_move_at_risk = min(len(scaling_app_at_risk_tasks),
num_tasks_to_scale)
old_app_at_risk_tasks[(new_app.id, new_client)] = set(
scaling_app_at_risk_tasks[:tasks_to_move_at_risk])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_at_risk
tasks_to_move_happy = min(len(scaling_app_happy_tasks),
num_tasks_to_scale)
old_app_live_happy_tasks[(new_app.id, new_client)] = set(
scaling_app_happy_tasks[:tasks_to_move_happy])
happy_new_tasks = scaling_app_happy_tasks[tasks_to_move_happy:]
# slack represents remaining the extra remaining instances that are configured
# in marathon that don't have a launched task yet. When scaling down we want to
# reduce this slack so marathon doesn't get a chance to launch a new task in
# that space that we will then have to drain and kill again.
marathon_tools.take_up_slack(client=new_client, app=new_app)
# TODO: don't take actions in deploy_service.
undrain_tasks(
to_undrain=new_app.tasks,
leave_draining=old_app_draining_tasks.get((new_app.id, new_client),
[]),
drain_method=drain_method,
log_deploy_error=log_deploy_error,
)
# log all uncaught exceptions and raise them again
try:
try:
bounce_func = bounce_lib.get_bounce_method_func(bounce_method)
except KeyError:
errormsg = 'ERROR: bounce_method not recognized: %s. Must be one of (%s)' % \
(bounce_method, ', '.join(bounce_lib.list_bounce_methods()))
log_deploy_error(errormsg)
return (1, errormsg, None)
bounce_again_in_seconds = do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=config,
new_app_running=new_app_running,
happy_new_tasks=happy_new_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
old_app_draining_tasks=old_app_draining_tasks,
old_app_at_risk_tasks=old_app_at_risk_tasks,
service=service,
bounce_method=bounce_method,
serviceinstance=serviceinstance,
cluster=cluster,
instance=instance,
marathon_jobid=marathon_jobid,
clients=clients,
soa_dir=soa_dir,
job_config=job_config,
bounce_margin_factor=bounce_margin_factor,
)
except bounce_lib.LockHeldException:
logline = 'Failed to get lock to create marathon app for %s.%s' % (
service, instance)
log_deploy_error(logline, level='debug')
return (0, "Couldn't get marathon lock, skipping until next time",
None)
except Exception:
logline = 'Exception raised during deploy of service %s:\n%s' % (
service, traceback.format_exc())
log_deploy_error(logline, level='debug')
raise
if num_at_risk_tasks:
bounce_again_in_seconds = 60
elif new_app_running:
if new_app.instances > config['instances']:
bounce_again_in_seconds = 60
return (0, 'Service deployed.', bounce_again_in_seconds)
def main():
args = parse_args()
full_appid = args.appname.lstrip('/')
soa_dir = args.soa_dir
marathon_config = marathon_tools.load_marathon_config()
client = marathon_tools.get_marathon_client(
url=marathon_config.get_url(),
user=marathon_config.get_username(),
passwd=marathon_config.get_password(),
)
if not marathon_tools.is_app_id_running(app_id=full_appid, client=client):
print("Couldn't find an app named {0}".format(full_appid))
sys.exit(1)
service, instance, _, __ = (s.replace('--', '_')
for s in decompose_job_id(full_appid))
complete_config = marathon_tools.create_complete_config(
service, instance, marathon_config)
cluster = load_system_paasta_config().get_cluster()
service_instance_config = marathon_tools.load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
nerve_ns = service_instance_config.get_nerve_namespace()
service_namespace_config = marathon_tools.load_service_namespace_config(
service=service, namespace=nerve_ns)
drain_method = drain_lib.get_drain_method(
service_instance_config.get_drain_method(service_namespace_config),
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=service_instance_config.get_drain_method_params(
service_namespace_config),
)
bounce_func = bounce_lib.get_bounce_method_func('down')
while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
app_to_kill = client.get_app(full_appid)
old_app_live_tasks, old_app_draining_tasks = get_old_live_draining_tasks(
[app_to_kill], drain_method)
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=complete_config,
new_app_running='',
happy_new_tasks=[],
old_app_live_tasks=old_app_live_tasks,
old_app_draining_tasks=old_app_draining_tasks,
serviceinstance="{0}.{1}".format(service, instance),
bounce_method='down',
service=service,
cluster=cluster,
instance=instance,
marathon_jobid=full_appid,
client=client,
soa_dir=soa_dir,
)
print "Sleeping for 10 seconds to give the tasks time to drain"
time.sleep(10)
print("Sucessfully killed {0}".format(full_appid))
def deploy_service(
service,
instance,
marathon_jobid,
config,
client,
bounce_method,
drain_method_name,
drain_method_params,
nerve_ns,
bounce_health_params,
soa_dir,
):
"""Deploy the service to marathon, either directly or via a bounce if needed.
Called by setup_service when it's time to actually deploy.
:param service: The name of the service to deploy
:param instance: The instance of the service to deploy
:param marathon_jobid: Full id of the marathon job
:param config: The complete configuration dict to send to marathon
:param client: A MarathonClient object
:param bounce_method: The bounce method to use, if needed
:param drain_method_name: The name of the traffic draining method to use.
:param nerve_ns: The nerve namespace to look in.
:param bounce_health_params: A dictionary of options for bounce_lib.get_happy_tasks.
:returns: A tuple of (status, output) to be used with send_sensu_event"""
def log_deploy_error(errormsg, level='event'):
return _log(
service=service,
line=errormsg,
component='deploy',
level='event',
cluster=cluster,
instance=instance
)
short_id = marathon_tools.format_job_id(service, instance)
cluster = load_system_paasta_config().get_cluster()
existing_apps = marathon_tools.get_matching_apps(service, instance, client, embed_failures=True)
new_app_list = [a for a in existing_apps if a.id == '/%s' % config['id']]
other_apps = [a for a in existing_apps if a.id != '/%s' % config['id']]
serviceinstance = "%s.%s" % (service, instance)
if new_app_list:
new_app = new_app_list[0]
if len(new_app_list) != 1:
raise ValueError("Only expected one app per ID; found %d" % len(new_app_list))
new_app_running = True
happy_new_tasks = bounce_lib.get_happy_tasks(new_app, service, nerve_ns, **bounce_health_params)
else:
new_app_running = False
happy_new_tasks = []
try:
drain_method = drain_lib.get_drain_method(
drain_method_name,
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=drain_method_params,
)
except KeyError:
errormsg = 'ERROR: drain_method not recognized: %s. Must be one of (%s)' % \
(drain_method_name, ', '.join(drain_lib.list_drain_methods()))
log_deploy_error(errormsg)
return (1, errormsg)
old_app_live_happy_tasks, old_app_live_unhappy_tasks, old_app_draining_tasks = get_old_happy_unhappy_draining_tasks(
other_apps,
drain_method,
service,
nerve_ns,
bounce_health_params
)
if new_app_running:
protected_draining_tasks = set()
if new_app.instances < config['instances']:
client.scale_app(app_id=new_app.id, instances=config['instances'], force=True)
elif new_app.instances > config['instances']:
num_tasks_to_scale = max(min(len(new_app.tasks), new_app.instances) - config['instances'], 0)
task_dict = get_old_happy_unhappy_draining_tasks_for_app(
new_app,
drain_method,
service,
nerve_ns,
bounce_health_params,
)
scaling_app_happy_tasks = list(task_dict['happy'])
scaling_app_unhappy_tasks = list(task_dict['unhappy'])
scaling_app_draining_tasks = list(task_dict['draining'])
tasks_to_move_draining = min(len(scaling_app_draining_tasks), num_tasks_to_scale)
old_app_draining_tasks[new_app.id] = set(scaling_app_draining_tasks[:tasks_to_move_draining])
protected_draining_tasks.update(scaling_app_draining_tasks[:tasks_to_move_draining])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_draining
tasks_to_move_unhappy = min(len(scaling_app_unhappy_tasks), num_tasks_to_scale)
old_app_live_unhappy_tasks[new_app.id] = set(scaling_app_unhappy_tasks[:tasks_to_move_unhappy])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_unhappy
tasks_to_move_happy = min(len(scaling_app_happy_tasks), num_tasks_to_scale)
old_app_live_happy_tasks[new_app.id] = set(scaling_app_happy_tasks[:tasks_to_move_happy])
happy_new_tasks = scaling_app_happy_tasks[tasks_to_move_happy:]
# If any tasks on the new app happen to be draining (e.g. someone reverts to an older version with
# `paasta mark-for-deployment`), then we should undrain them.
for task in new_app.tasks:
if task not in protected_draining_tasks:
drain_method.stop_draining(task)
# Re-drain any already draining tasks on old apps
for tasks in old_app_draining_tasks.values():
for task in tasks:
drain_method.drain(task)
# log all uncaught exceptions and raise them again
try:
try:
bounce_func = bounce_lib.get_bounce_method_func(bounce_method)
except KeyError:
errormsg = 'ERROR: bounce_method not recognized: %s. Must be one of (%s)' % \
(bounce_method, ', '.join(bounce_lib.list_bounce_methods()))
log_deploy_error(errormsg)
return (1, errormsg)
try:
with bounce_lib.bounce_lock_zookeeper(short_id):
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=config,
new_app_running=new_app_running,
happy_new_tasks=happy_new_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
old_app_draining_tasks=old_app_draining_tasks,
service=service,
bounce_method=bounce_method,
serviceinstance=serviceinstance,
cluster=cluster,
instance=instance,
marathon_jobid=marathon_jobid,
client=client,
soa_dir=soa_dir,
)
except bounce_lib.LockHeldException:
log.error("Instance %s already being bounced. Exiting", short_id)
return (1, "Instance %s is already being bounced." % short_id)
except Exception:
loglines = ['Exception raised during deploy of service %s:' % service]
loglines.extend(traceback.format_exc().rstrip().split("\n"))
for logline in loglines:
log_deploy_error(logline, level='debug')
raise
return (0, 'Service deployed.')
def main():
args = parse_args()
full_appid = args.appname.lstrip('/')
soa_dir = args.soa_dir
marathon_config = marathon_tools.load_marathon_config()
client = marathon_tools.get_marathon_client(
url=marathon_config.get_url(),
user=marathon_config.get_username(),
passwd=marathon_config.get_password(),
)
if not marathon_tools.is_app_id_running(app_id=full_appid, client=client):
print("Couldn't find an app named {0}".format(full_appid))
sys.exit(1)
service, instance, _, __ = (s.replace('--', '_') for s in decompose_job_id(full_appid))
complete_config = marathon_tools.create_complete_config(service, instance, marathon_config)
cluster = load_system_paasta_config().get_cluster()
service_instance_config = marathon_tools.load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
nerve_ns = service_instance_config.get_nerve_namespace()
service_namespace_config = marathon_tools.load_service_namespace_config(service=service, namespace=nerve_ns)
drain_method = drain_lib.get_drain_method(
service_instance_config.get_drain_method(service_namespace_config),
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=service_instance_config.get_drain_method_params(service_namespace_config),
)
bounce_func = bounce_lib.get_bounce_method_func('down')
while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
app_to_kill = client.get_app(full_appid)
old_app_live_tasks, old_app_draining_tasks = get_old_live_draining_tasks([app_to_kill], drain_method)
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=complete_config,
new_app_running='',
happy_new_tasks=[],
old_app_live_tasks=old_app_live_tasks,
old_app_draining_tasks=old_app_draining_tasks,
serviceinstance="{0}.{1}".format(service, instance),
bounce_method='down',
service=service,
cluster=cluster,
instance=instance,
marathon_jobid=full_appid,
client=client,
soa_dir=soa_dir,
)
print "Sleeping for 10 seconds to give the tasks time to drain"
time.sleep(10)
print("Sucessfully killed {0}".format(full_appid))
def deploy_service(
service,
instance,
marathon_jobid,
config,
client,
marathon_apps,
bounce_method,
drain_method_name,
drain_method_params,
nerve_ns,
bounce_health_params,
soa_dir,
bounce_margin_factor=1.0,
):
"""Deploy the service to marathon, either directly or via a bounce if needed.
Called by setup_service when it's time to actually deploy.
:param service: The name of the service to deploy
:param instance: The instance of the service to deploy
:param marathon_jobid: Full id of the marathon job
:param config: The complete configuration dict to send to marathon
:param client: A MarathonClient object
:param bounce_method: The bounce method to use, if needed
:param drain_method_name: The name of the traffic draining method to use.
:param nerve_ns: The nerve namespace to look in.
:param bounce_health_params: A dictionary of options for bounce_lib.get_happy_tasks.
:param bounce_margin_factor: the multiplication factor used to calculate the number of instances to be drained
:returns: A tuple of (status, output) to be used with send_sensu_event"""
def log_deploy_error(errormsg, level='event'):
return _log(service=service,
line=errormsg,
component='deploy',
level='event',
cluster=cluster,
instance=instance)
system_paasta_config = load_system_paasta_config()
cluster = system_paasta_config.get_cluster()
existing_apps = marathon_tools.get_matching_apps(service, instance,
marathon_apps)
new_app_list = [a for a in existing_apps if a.id == '/%s' % config['id']]
other_apps = [a for a in existing_apps if a.id != '/%s' % config['id']]
serviceinstance = "%s.%s" % (service, instance)
if new_app_list:
new_app = new_app_list[0]
if len(new_app_list) != 1:
raise ValueError("Only expected one app per ID; found %d" %
len(new_app_list))
new_app_running = True
happy_new_tasks = bounce_lib.get_happy_tasks(new_app, service,
nerve_ns,
system_paasta_config,
**bounce_health_params)
else:
new_app_running = False
happy_new_tasks = []
try:
drain_method = drain_lib.get_drain_method(
drain_method_name,
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=drain_method_params,
)
except KeyError:
errormsg = 'ERROR: drain_method not recognized: %s. Must be one of (%s)' % \
(drain_method_name, ', '.join(drain_lib.list_drain_methods()))
log_deploy_error(errormsg)
return (1, errormsg)
(
old_app_live_happy_tasks,
old_app_live_unhappy_tasks,
old_app_draining_tasks,
old_app_at_risk_tasks,
) = get_tasks_by_state(
other_apps,
drain_method,
service,
nerve_ns,
bounce_health_params,
system_paasta_config,
)
if new_app_running:
num_at_risk_tasks = get_num_at_risk_tasks(new_app)
if new_app.instances < config['instances'] + num_at_risk_tasks:
log.info("Scaling %s from %d to %d instances." %
(new_app.id, new_app.instances,
config['instances'] + num_at_risk_tasks))
client.scale_app(app_id=new_app.id,
instances=config['instances'] + num_at_risk_tasks,
force=True)
# If we have more than the specified number of instances running, we will want to drain some of them.
# We will start by draining any tasks running on at-risk hosts.
elif new_app.instances > config['instances']:
num_tasks_to_scale = max(
min(len(new_app.tasks), new_app.instances) -
config['instances'], 0)
task_dict = get_tasks_by_state_for_app(
new_app,
drain_method,
service,
nerve_ns,
bounce_health_params,
system_paasta_config,
)
scaling_app_happy_tasks = list(task_dict['happy'])
scaling_app_unhappy_tasks = list(task_dict['unhappy'])
scaling_app_draining_tasks = list(task_dict['draining'])
scaling_app_at_risk_tasks = list(task_dict['at_risk'])
tasks_to_move_draining = min(len(scaling_app_draining_tasks),
num_tasks_to_scale)
old_app_draining_tasks[new_app.id] = set(
scaling_app_draining_tasks[:tasks_to_move_draining])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_draining
tasks_to_move_unhappy = min(len(scaling_app_unhappy_tasks),
num_tasks_to_scale)
old_app_live_unhappy_tasks[new_app.id] = set(
scaling_app_unhappy_tasks[:tasks_to_move_unhappy])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_unhappy
tasks_to_move_at_risk = min(len(scaling_app_at_risk_tasks),
num_tasks_to_scale)
old_app_at_risk_tasks[new_app.id] = set(
scaling_app_at_risk_tasks[:tasks_to_move_at_risk])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_at_risk
tasks_to_move_happy = min(len(scaling_app_happy_tasks),
num_tasks_to_scale)
old_app_live_happy_tasks[new_app.id] = set(
scaling_app_happy_tasks[:tasks_to_move_happy])
happy_new_tasks = scaling_app_happy_tasks[tasks_to_move_happy:]
# TODO: don't take actions in deploy_service.
undrain_tasks(
to_undrain=new_app.tasks,
leave_draining=old_app_draining_tasks.get(new_app.id, []),
drain_method=drain_method,
log_deploy_error=log_deploy_error,
)
# log all uncaught exceptions and raise them again
try:
try:
bounce_func = bounce_lib.get_bounce_method_func(bounce_method)
except KeyError:
errormsg = 'ERROR: bounce_method not recognized: %s. Must be one of (%s)' % \
(bounce_method, ', '.join(bounce_lib.list_bounce_methods()))
log_deploy_error(errormsg)
return (1, errormsg)
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=config,
new_app_running=new_app_running,
happy_new_tasks=happy_new_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
old_app_draining_tasks=old_app_draining_tasks,
old_app_at_risk_tasks=old_app_at_risk_tasks,
service=service,
bounce_method=bounce_method,
serviceinstance=serviceinstance,
cluster=cluster,
instance=instance,
marathon_jobid=marathon_jobid,
client=client,
soa_dir=soa_dir,
bounce_margin_factor=bounce_margin_factor,
)
except bounce_lib.LockHeldException:
logline = 'Failed to get lock to create marathon app for %s.%s' % (
service, instance)
log_deploy_error(logline, level='debug')
return (0, "Couldn't get marathon lock, skipping until next time")
except Exception:
logline = 'Exception raised during deploy of service %s:\n%s' % (
service, traceback.format_exc())
log_deploy_error(logline, level='debug')
raise
return (0, 'Service deployed.')
def deploy_service(
service,
instance,
marathon_jobid,
config,
client,
marathon_apps,
bounce_method,
drain_method_name,
drain_method_params,
nerve_ns,
bounce_health_params,
soa_dir,
bounce_margin_factor=1.0,
):
"""Deploy the service to marathon, either directly or via a bounce if needed.
Called by setup_service when it's time to actually deploy.
:param service: The name of the service to deploy
:param instance: The instance of the service to deploy
:param marathon_jobid: Full id of the marathon job
:param config: The complete configuration dict to send to marathon
:param client: A MarathonClient object
:param bounce_method: The bounce method to use, if needed
:param drain_method_name: The name of the traffic draining method to use.
:param nerve_ns: The nerve namespace to look in.
:param bounce_health_params: A dictionary of options for bounce_lib.get_happy_tasks.
:param bounce_margin_factor: the multiplication factor used to calculate the number of instances to be drained
:returns: A tuple of (status, output) to be used with send_sensu_event"""
def log_deploy_error(errormsg, level='event'):
return _log(
service=service,
line=errormsg,
component='deploy',
level='event',
cluster=cluster,
instance=instance
)
short_id = marathon_tools.format_job_id(service, instance)
system_paasta_config = load_system_paasta_config()
cluster = system_paasta_config.get_cluster()
existing_apps = marathon_tools.get_matching_apps(service, instance, marathon_apps)
new_app_list = [a for a in existing_apps if a.id == '/%s' % config['id']]
other_apps = [a for a in existing_apps if a.id != '/%s' % config['id']]
serviceinstance = "%s.%s" % (service, instance)
if new_app_list:
new_app = new_app_list[0]
if len(new_app_list) != 1:
raise ValueError("Only expected one app per ID; found %d" % len(new_app_list))
new_app_running = True
happy_new_tasks = bounce_lib.get_happy_tasks(new_app, service, nerve_ns, system_paasta_config,
**bounce_health_params)
else:
new_app_running = False
happy_new_tasks = []
try:
drain_method = drain_lib.get_drain_method(
drain_method_name,
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=drain_method_params,
)
except KeyError:
errormsg = 'ERROR: drain_method not recognized: %s. Must be one of (%s)' % \
(drain_method_name, ', '.join(drain_lib.list_drain_methods()))
log_deploy_error(errormsg)
return (1, errormsg)
(old_app_live_happy_tasks,
old_app_live_unhappy_tasks,
old_app_draining_tasks,
old_app_at_risk_tasks,
) = get_tasks_by_state(
other_apps,
drain_method,
service,
nerve_ns,
bounce_health_params,
system_paasta_config,
)
if new_app_running:
num_at_risk_tasks = get_num_at_risk_tasks(new_app)
if new_app.instances < config['instances'] + num_at_risk_tasks:
log.info("Scaling %s from %d to %d instances." %
(new_app.id, new_app.instances, config['instances'] + num_at_risk_tasks))
client.scale_app(app_id=new_app.id, instances=config['instances'] + num_at_risk_tasks, force=True)
# If we have more than the specified number of instances running, we will want to drain some of them.
# We will start by draining any tasks running on at-risk hosts.
elif new_app.instances > config['instances']:
num_tasks_to_scale = max(min(len(new_app.tasks), new_app.instances) - config['instances'], 0)
task_dict = get_tasks_by_state_for_app(
new_app,
drain_method,
service,
nerve_ns,
bounce_health_params,
system_paasta_config,
)
scaling_app_happy_tasks = list(task_dict['happy'])
scaling_app_unhappy_tasks = list(task_dict['unhappy'])
scaling_app_draining_tasks = list(task_dict['draining'])
scaling_app_at_risk_tasks = list(task_dict['at_risk'])
tasks_to_move_draining = min(len(scaling_app_draining_tasks), num_tasks_to_scale)
old_app_draining_tasks[new_app.id] = set(scaling_app_draining_tasks[:tasks_to_move_draining])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_draining
tasks_to_move_unhappy = min(len(scaling_app_unhappy_tasks), num_tasks_to_scale)
old_app_live_unhappy_tasks[new_app.id] = set(scaling_app_unhappy_tasks[:tasks_to_move_unhappy])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_unhappy
tasks_to_move_at_risk = min(len(scaling_app_at_risk_tasks), num_tasks_to_scale)
old_app_at_risk_tasks[new_app.id] = set(scaling_app_at_risk_tasks[:tasks_to_move_at_risk])
num_tasks_to_scale = num_tasks_to_scale - tasks_to_move_at_risk
tasks_to_move_happy = min(len(scaling_app_happy_tasks), num_tasks_to_scale)
old_app_live_happy_tasks[new_app.id] = set(scaling_app_happy_tasks[:tasks_to_move_happy])
happy_new_tasks = scaling_app_happy_tasks[tasks_to_move_happy:]
# TODO: don't take actions in deploy_service.
undrain_tasks(
to_undrain=new_app.tasks,
leave_draining=old_app_draining_tasks.get(new_app.id, []),
drain_method=drain_method,
log_deploy_error=log_deploy_error,
)
# log all uncaught exceptions and raise them again
try:
try:
bounce_func = bounce_lib.get_bounce_method_func(bounce_method)
except KeyError:
errormsg = 'ERROR: bounce_method not recognized: %s. Must be one of (%s)' % \
(bounce_method, ', '.join(bounce_lib.list_bounce_methods()))
log_deploy_error(errormsg)
return (1, errormsg)
try:
with bounce_lib.bounce_lock_zookeeper(short_id):
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=config,
new_app_running=new_app_running,
happy_new_tasks=happy_new_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
old_app_draining_tasks=old_app_draining_tasks,
old_app_at_risk_tasks=old_app_at_risk_tasks,
service=service,
bounce_method=bounce_method,
serviceinstance=serviceinstance,
cluster=cluster,
instance=instance,
marathon_jobid=marathon_jobid,
client=client,
soa_dir=soa_dir,
bounce_margin_factor=bounce_margin_factor,
)
except bounce_lib.LockHeldException:
log.error("Instance %s already being bounced. Exiting", short_id)
return (1, "Instance %s is already being bounced." % short_id)
except Exception:
logline = 'Exception raised during deploy of service %s:\n%s' % (service, traceback.format_exc())
log_deploy_error(logline, level='debug')
raise
return (0, 'Service deployed.')
def deploy_service(
service,
instance,
marathon_jobid,
config,
client,
bounce_method,
drain_method_name,
drain_method_params,
nerve_ns,
bounce_health_params,
soa_dir,
):
"""Deploy the service to marathon, either directly or via a bounce if needed.
Called by setup_service when it's time to actually deploy.
:param service: The name of the service to deploy
:param instance: The instance of the service to deploy
:param marathon_jobid: Full id of the marathon job
:param config: The complete configuration dict to send to marathon
:param client: A MarathonClient object
:param bounce_method: The bounce method to use, if needed
:param drain_method_name: The name of the traffic draining method to use.
:param nerve_ns: The nerve namespace to look in.
:param bounce_health_params: A dictionary of options for bounce_lib.get_happy_tasks.
:returns: A tuple of (status, output) to be used with send_sensu_event"""
def log_deploy_error(errormsg, level="event"):
return _log(
service=service, line=errormsg, component="deploy", level="event", cluster=cluster, instance=instance
)
short_id = marathon_tools.format_job_id(service, instance)
cluster = load_system_paasta_config().get_cluster()
existing_apps = marathon_tools.get_matching_apps(service, instance, client, embed_failures=True)
new_app_list = [a for a in existing_apps if a.id == "/%s" % config["id"]]
other_apps = [a for a in existing_apps if a.id != "/%s" % config["id"]]
serviceinstance = "%s.%s" % (service, instance)
if new_app_list:
new_app = new_app_list[0]
if len(new_app_list) != 1:
raise ValueError("Only expected one app per ID; found %d" % len(new_app_list))
new_app_running = True
happy_new_tasks = bounce_lib.get_happy_tasks(new_app, service, nerve_ns, **bounce_health_params)
else:
new_app_running = False
happy_new_tasks = []
try:
drain_method = drain_lib.get_drain_method(
drain_method_name,
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=drain_method_params,
)
except KeyError:
errormsg = "ERROR: drain_method not recognized: %s. Must be one of (%s)" % (
drain_method_name,
", ".join(drain_lib.list_drain_methods()),
)
log_deploy_error(errormsg)
return (1, errormsg)
old_app_live_tasks, old_app_draining_tasks = get_old_live_draining_tasks(other_apps, drain_method)
# Re-drain any already draining tasks on old apps
for tasks in old_app_draining_tasks.values():
for task in tasks:
drain_method.drain(task)
# If any tasks on the new app happen to be draining (e.g. someone reverts to an older version with
# `paasta mark-for-deployment`), then we should undrain them.
if new_app_running:
for task in new_app.tasks:
drain_method.stop_draining(task)
# log all uncaught exceptions and raise them again
try:
try:
bounce_func = bounce_lib.get_bounce_method_func(bounce_method)
except KeyError:
errormsg = "ERROR: bounce_method not recognized: %s. Must be one of (%s)" % (
bounce_method,
", ".join(bounce_lib.list_bounce_methods()),
)
log_deploy_error(errormsg)
return (1, errormsg)
try:
with bounce_lib.bounce_lock_zookeeper(short_id):
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=config,
new_app_running=new_app_running,
happy_new_tasks=happy_new_tasks,
old_app_live_tasks=old_app_live_tasks,
old_app_draining_tasks=old_app_draining_tasks,
service=service,
bounce_method=bounce_method,
serviceinstance=serviceinstance,
cluster=cluster,
instance=instance,
marathon_jobid=marathon_jobid,
client=client,
soa_dir=soa_dir,
)
except bounce_lib.LockHeldException:
log.error("Instance %s already being bounced. Exiting", short_id)
return (1, "Instance %s is already being bounced." % short_id)
except Exception:
loglines = ["Exception raised during deploy of service %s:" % service]
loglines.extend(traceback.format_exc().rstrip().split("\n"))
for logline in loglines:
log_deploy_error(logline, level="debug")
raise
return (0, "Service deployed.")