def get_service_instances_needing_update(
marathon_clients: MarathonClients,
instances: Collection[Tuple[str, str]],
cluster: str,
) -> List[Tuple[str, str, MarathonServiceConfig, str]]:
marathon_apps = {}
for marathon_client in marathon_clients.get_all_clients():
marathon_apps.update(
{app.id: app
for app in get_all_marathon_apps(marathon_client)})
marathon_app_ids = marathon_apps.keys()
service_instances = []
for service, instance in instances:
try:
config = load_marathon_service_config_no_cache(
service=service,
instance=instance,
cluster=cluster,
soa_dir=DEFAULT_SOA_DIR,
)
config_app = config.format_marathon_app_dict()
app_id = "/{}".format(config_app["id"])
# Not ideal but we rely on a lot of user input to create the app dict
# and we really can't afford to bail if just one app definition is malformed
except Exception as e:
print("ERROR: Skipping {}.{} because: '{}'".format(
service, instance, str(e)))
continue
if (app_id not in marathon_app_ids
or marathon_apps[app_id].instances != config_app["instances"]):
service_instances.append((service, instance, config, app_id))
return service_instances
def get_service_instances_needing_update(marathon_client, instances, cluster):
marathon_apps = {
app.id: app
for app in get_all_marathon_apps(marathon_client)
}
marathon_app_ids = marathon_apps.keys()
service_instances = []
for service, instance in instances:
try:
config = load_marathon_service_config_no_cache(
service=service,
instance=instance,
cluster=cluster,
soa_dir=DEFAULT_SOA_DIR,
)
config_app = config.format_marathon_app_dict()
app_id = '/{}'.format(config_app['id'])
except (NoDockerImageError, InvalidJobNameError,
NoDeploymentsAvailable) as e:
print("DEBUG: Skipping %s.%s because: '%s'" %
(service, instance, str(e)))
continue
if app_id not in marathon_app_ids:
service_instances.append((service, instance))
elif marathon_apps[app_id].instances != config_app['instances']:
service_instances.append((service, instance))
return service_instances
def check_sha_changed(context, service_instance):
service, instance, _, _ = decompose_job_id(service_instance)
service_configuration_lib._yaml_cache = {}
context.marathon_config = load_marathon_service_config_no_cache(
service, instance, context.cluster)
assert context.app_id != context.marathon_config.format_marathon_app_dict(
)['id']
def get_service_instances_needing_update(
marathon_clients: MarathonClients,
instances: Collection[Tuple[str, str]],
cluster: str,
) -> List[Tuple[str, str]]:
marathon_apps = {}
for marathon_client in marathon_clients.get_all_clients():
marathon_apps.update(
{app.id: app
for app in get_all_marathon_apps(marathon_client)})
marathon_app_ids = marathon_apps.keys()
service_instances = []
for service, instance in instances:
try:
config = load_marathon_service_config_no_cache(
service=service,
instance=instance,
cluster=cluster,
soa_dir=DEFAULT_SOA_DIR,
)
config_app = config.format_marathon_app_dict()
app_id = '/{}'.format(config_app['id'])
except (NoDockerImageError, InvalidJobNameError,
NoDeploymentsAvailable, NoSlavesAvailableError) as e:
print("DEBUG: Skipping {}.{} because: '{}'".format(
service, instance, str(e)))
continue
if app_id not in marathon_app_ids:
service_instances.append((service, instance))
elif marathon_apps[app_id].instances != config_app['instances']:
service_instances.append((service, instance))
return service_instances
def deploy_marathon_service(service, instance, client, soa_dir,
marathon_config, marathon_apps):
"""deploy the service instance given and proccess return code
if there was an error we send a sensu alert.
:param service: The service name to setup
:param instance: The instance of the service to setup
:param client: A MarathonClient object
:param soa_dir: Path to yelpsoa configs
:param marathon_config: The service instance's configuration dict
:param marathon_apps: A list of all marathon app objects
:returns: A tuple of (status, bounce_in_seconds) to be used by paasta-deployd
bounce_in_seconds instructs how long until the deployd should try another bounce
None means that it is in a steady state and doesn't need to bounce again
"""
short_id = marathon_tools.format_job_id(service, instance)
try:
with bounce_lib.bounce_lock_zookeeper(short_id):
try:
service_instance_config = marathon_tools.load_marathon_service_config_no_cache(
service,
instance,
load_system_paasta_config().get_cluster(),
soa_dir=soa_dir,
)
except NoDeploymentsAvailable:
log.debug(
"No deployments found for %s.%s in cluster %s. Skipping." %
(service, instance,
load_system_paasta_config().get_cluster()))
return 0, None
except NoConfigurationForServiceError:
error_msg = "Could not read marathon configuration file for %s.%s in cluster %s" % \
(service, instance, load_system_paasta_config().get_cluster())
log.error(error_msg)
return 1, None
try:
status, output, bounce_again_in_seconds = setup_service(
service,
instance,
client,
service_instance_config,
marathon_apps,
soa_dir,
)
sensu_status = pysensu_yelp.Status.CRITICAL if status else pysensu_yelp.Status.OK
send_event(service, instance, soa_dir, sensu_status, output)
return 0, bounce_again_in_seconds
except (KeyError, TypeError, AttributeError, InvalidInstanceConfig,
NoSlavesAvailableError):
error_str = traceback.format_exc()
log.error(error_str)
send_event(service, instance, soa_dir,
pysensu_yelp.Status.CRITICAL, error_str)
return 1, None
except bounce_lib.LockHeldException:
log.error("Instance %s already being bounced. Exiting", short_id)
return 0, None
def update_context_marathon_config(context):
whitelist_keys = {
'id',
'backoff_factor',
'backoff_seconds',
'max_instances',
'mem',
'cpus',
'instances',
'marathon_shard',
'previous_marathon_shards',
}
with mock.patch.object(
MarathonServiceConfig,
'get_min_instances',
autospec=True,
return_value=1,
), mock.patch.object(
MarathonServiceConfig,
'get_max_instances',
autospec=True,
) as mock_get_max_instances:
mock_get_max_instances.return_value = context.max_instances if 'max_instances' in context else None
service_configuration_lib._yaml_cache = {}
context.job_config = marathon_tools.load_marathon_service_config_no_cache(
service=context.service,
instance=context.instance,
cluster=context.system_paasta_config.get_cluster(),
soa_dir=context.soa_dir,
)
context.current_client = context.marathon_clients.get_current_client_for_service(
context.job_config)
context.marathon_complete_config = {
key: value
for key, value in
context.job_config.format_marathon_app_dict().items()
if key in whitelist_keys
}
context.marathon_complete_config.update({
'cmd': '/bin/sleep 1m',
'constraints': None,
'container': {
'type': 'DOCKER',
'docker': {
'network': 'BRIDGE',
'image': 'busybox',
},
},
})
if 'max_instances' not in context:
context.marathon_complete_config['instances'] = context.instances
def update_context_marathon_config(context):
whitelist_keys = {
"id",
"backoff_factor",
"backoff_seconds",
"max_instances",
"mem",
"cpus",
"instances",
"marathon_shard",
"previous_marathon_shards",
}
with mock.patch.object(MarathonServiceConfig,
"get_min_instances",
autospec=True,
return_value=1), mock.patch.object(
MarathonServiceConfig,
"get_max_instances",
autospec=True) as mock_get_max_instances:
mock_get_max_instances.return_value = (
context.max_instances if "max_instances" in context else None)
service_configuration_lib._yaml_cache = {}
context.job_config = marathon_tools.load_marathon_service_config_no_cache(
service=context.service,
instance=context.instance,
cluster=context.system_paasta_config.get_cluster(),
soa_dir=context.soa_dir,
)
context.current_client = context.marathon_clients.get_current_client_for_service(
context.job_config)
context.marathon_complete_config = {
key: value
for key, value in
context.job_config.format_marathon_app_dict().items()
if key in whitelist_keys
}
context.marathon_complete_config.update({
"cmd": "/bin/sleep 1m",
"constraints": None,
"container": {
"type": "DOCKER",
"docker": {
"network": "BRIDGE",
"image": "busybox"
},
},
})
if "max_instances" not in context:
context.marathon_complete_config["instances"] = context.instances
def get_service_instances_with_changed_id(marathon_client, instances, cluster):
marathon_app_ids = list_all_marathon_app_ids(marathon_client)
service_instances = []
for service, instance in instances:
config = load_marathon_service_config_no_cache(service=service,
instance=instance,
cluster=cluster,
soa_dir=DEFAULT_SOA_DIR)
try:
config_app_id = config.format_marathon_app_dict()['id']
except NoDockerImageError:
config_app_id = None
if not config_app_id or (config_app_id not in marathon_app_ids):
service_instances.append((service, instance))
return service_instances
def check_app_running(context, service_instance, seconds):
service, instance, _, _ = decompose_job_id(service_instance)
service_configuration_lib._yaml_cache = {}
context.marathon_config = load_marathon_service_config_no_cache(
service, instance, context.cluster)
context.app_id = context.marathon_config.format_marathon_app_dict()['id']
step = 5
attempts = 0
while (attempts * step) < seconds:
if context.app_id in list_all_marathon_app_ids(
context.marathon_client):
break
time.sleep(step)
attempts += 1
assert context.app_id in list_all_marathon_app_ids(context.marathon_client)
context.old_app_id = context.app_id
def get_service_instances_needing_update(marathon_client, instances, cluster):
marathon_apps = {
app.id: app
for app in get_all_marathon_apps(marathon_client)
}
marathon_app_ids = marathon_apps.keys()
service_instances = []
for service, instance in instances:
config = load_marathon_service_config_no_cache(service=service,
instance=instance,
cluster=cluster,
soa_dir=DEFAULT_SOA_DIR)
try:
config_app = config.format_marathon_app_dict()
app_id = '/{}'.format(config_app['id'])
except NoDockerImageError:
config_app = None
if not config_app:
service_instances.append((service, instance))
elif app_id not in marathon_app_ids:
service_instances.append((service, instance))
elif marathon_apps[app_id].instances != config_app['instances']:
service_instances.append((service, instance))
return service_instances
def deploy_marathon_service(
service: str,
instance: str,
clients: marathon_tools.MarathonClients,
soa_dir: str,
marathon_apps_with_clients: Optional[Sequence[Tuple[MarathonApp,
MarathonClient]]],
system_paasta_config: Optional[SystemPaastaConfig] = None,
) -> Tuple[int, float]:
"""deploy the service instance given and process return code
if there was an error we send a sensu alert.
:param service: The service name to setup
:param instance: The instance of the service to setup
:param clients: A MarathonClients object
:param soa_dir: Path to yelpsoa configs
:param marathon_apps: A list of all marathon app objects
:returns: A tuple of (status, bounce_in_seconds) to be used by paasta-deployd
bounce_in_seconds instructs how long until the deployd should try another bounce
None means that it is in a steady state and doesn't need to bounce again
"""
if system_paasta_config is None:
system_paasta_config = load_system_paasta_config()
short_id = marathon_tools.format_job_id(service, instance)
try:
with bounce_lib.bounce_lock_zookeeper(
short_id, system_paasta_config=system_paasta_config):
try:
service_instance_config = marathon_tools.load_marathon_service_config_no_cache(
service,
instance,
system_paasta_config.get_cluster(),
soa_dir=soa_dir,
)
except NoDeploymentsAvailable:
log.debug(
"No deployments found for %s.%s in cluster %s. Skipping." %
(service, instance, system_paasta_config.get_cluster()))
return 0, None
except NoConfigurationForServiceError:
error_msg = (
"Could not read marathon configuration file for %s.%s in cluster %s"
% (service, instance, system_paasta_config.get_cluster()))
log.error(error_msg)
return 1, None
if marathon_apps_with_clients is None:
marathon_apps_with_clients = marathon_tools.get_marathon_apps_with_clients(
clients=clients.get_all_clients_for_service(
job_config=service_instance_config),
service_name=service,
instance_name=instance,
embed_tasks=True,
)
try:
with a_sync.idle_event_loop():
status, output, bounce_again_in_seconds = setup_service(
service=service,
instance=instance,
clients=clients,
job_config=service_instance_config,
marathon_apps_with_clients=marathon_apps_with_clients,
soa_dir=soa_dir,
system_paasta_config=system_paasta_config,
)
sensu_status = (pysensu_yelp.Status.CRITICAL
if status else pysensu_yelp.Status.OK)
send_event(
service,
instance,
soa_dir,
sensu_status,
output,
system_paasta_config,
service_instance_config,
)
return 0, bounce_again_in_seconds
except (
KeyError,
TypeError,
AttributeError,
InvalidInstanceConfig,
NoSlavesAvailableError,
):
error_str = traceback.format_exc()
log.error(error_str)
send_event(
service,
instance,
soa_dir,
pysensu_yelp.Status.CRITICAL,
error_str,
system_paasta_config,
service_instance_config,
)
return 1, None
except bounce_lib.LockHeldException:
log.error("Instance %s already being bounced. Exiting", short_id)
return 0, None
