def check_app_not_running(context, seconds):
step = 5
attempts = 0
while (attempts * step) < seconds:
if context.old_app_id not in list_all_marathon_app_ids(context.current_client):
return
time.sleep(step)
attempts += 1
assert context.old_app_id not in list_all_marathon_app_ids(context.current_client)
def not_see_it_in_list(context, shard_number=None):
full_list = []
if shard_number is None:
for client in context.marathon_clients.get_all_clients():
full_list.extend(marathon_tools.list_all_marathon_app_ids(client))
else:
full_list.extend(marathon_tools.list_all_marathon_app_ids(context.marathon_clients.current[shard_number]))
assert context.app_id not in full_list
def _clean_up_marathon_apps(context):
"""If a marathon client object exists in our context, delete any apps in Marathon and wait until they die."""
if hasattr(context, "marathon_clients"):
still_apps = True
while still_apps:
still_apps = False
for client in context.marathon_clients.get_all_clients():
apps = marathon_tools.list_all_marathon_app_ids(client)
if apps:
still_apps = True
else:
continue
paasta_print(
"after_scenario: Deleting %d apps to prep for the next scenario. %s"
% (len(apps), ",".join(apps)))
for app in apps:
if marathon_tools.is_app_id_running(app, client):
paasta_print(
"after_scenario: %s does look like it is running. Scaling down and killing it..."
% app)
client.scale_app(app, instances=0, force=True)
time.sleep(1)
client.delete_app(app, force=True)
else:
paasta_print(
"after_scenario: %s showed up in the app_list, but doesn't look like it is running?"
% app)
time.sleep(0.5)
for client in context.marathon_clients.get_all_clients():
while client.list_deployments():
paasta_print(
"after_scenario: There are still marathon deployments in progress. sleeping."
)
time.sleep(0.5)
def cleanup_apps(soa_dir):
"""Clean up old or invalid jobs/apps from marathon. Retrieves
both a list of apps currently in marathon and a list of valid
app ids in order to determine what to kill.
:param soa_dir: The SOA config directory to read from"""
log.info("Loading marathon configuration")
marathon_config = marathon_tools.load_marathon_config()
log.info("Connecting to marathon")
client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(),
marathon_config.get_password())
valid_services = get_services_for_cluster(instance_type='marathon', soa_dir=soa_dir)
running_app_ids = marathon_tools.list_all_marathon_app_ids(client)
for app_id in running_app_ids:
log.debug("Checking app id %s", app_id)
try:
service, instance, _, __ = marathon_tools.deformat_job_id(app_id)
except InvalidJobNameError:
log.warn("%s doesn't conform to paasta naming conventions? Skipping." % app_id)
continue
if (service, instance) not in valid_services:
delete_app(
app_id=app_id,
client=client,
soa_dir=soa_dir,
)
def check_app_running(context, service_instance, seconds):
service, instance, _, _ = decompose_job_id(service_instance)
service_configuration_lib._yaml_cache = {}
context.marathon_config = load_marathon_service_config(
service, instance, context.cluster)
context.app_id = context.marathon_config.format_marathon_app_dict()['id']
step = 5
attempts = 0
while (attempts * step) < seconds:
if context.app_id in list_all_marathon_app_ids(
context.marathon_client):
break
time.sleep(step)
attempts += 1
assert context.app_id in list_all_marathon_app_ids(context.marathon_client)
context.old_app_id = context.app_id
def cleanup_apps(soa_dir):
"""Clean up old or invalid jobs/apps from marathon. Retrieves
both a list of apps currently in marathon and a list of valid
app ids in order to determine what to kill.
:param soa_dir: The SOA config directory to read from"""
log.info("Loading marathon configuration")
marathon_config = marathon_tools.load_marathon_config()
log.info("Connecting to marathon")
client = marathon_tools.get_marathon_client(marathon_config.get_url(),
marathon_config.get_username(),
marathon_config.get_password())
valid_services = get_services_for_cluster(instance_type='marathon',
soa_dir=soa_dir)
running_app_ids = marathon_tools.list_all_marathon_app_ids(client)
for app_id in running_app_ids:
log.debug("Checking app id %s", app_id)
try:
service, instance, _, __ = marathon_tools.deformat_job_id(app_id)
except InvalidJobNameError:
log.warn(
"%s doesn't conform to paasta naming conventions? Skipping." %
app_id)
continue
if (service, instance) not in valid_services:
delete_app(
app_id=app_id,
client=client,
soa_dir=soa_dir,
)
def cleanup_apps(soa_dir, kill_threshold=0.5, force=False):
"""Clean up old or invalid jobs/apps from marathon. Retrieves
both a list of apps currently in marathon and a list of valid
app ids in order to determine what to kill.
:param soa_dir: The SOA config directory to read from
:param kill_threshold: The decimal fraction of apps we think is
sane to kill when this job runs.
:param force: Force the cleanup if we are above the kill_threshold"""
log.info("Loading marathon configuration")
marathon_config = marathon_tools.load_marathon_config()
log.info("Connecting to marathon")
client = marathon_tools.get_marathon_client(
marathon_config.get_url(),
marathon_config.get_username(),
marathon_config.get_password(),
)
valid_services = get_services_for_cluster(instance_type='marathon',
soa_dir=soa_dir)
running_app_ids = marathon_tools.list_all_marathon_app_ids(client)
running_apps = []
for app_id in running_app_ids:
try:
app_id = marathon_tools.deformat_job_id(app_id)
except InvalidJobNameError:
log.warn(
"%s doesn't conform to paasta naming conventions? Skipping." %
app_id)
continue
running_apps.append(app_id)
apps_to_kill = [(service, instance, git_sha, config_sha)
for service, instance, git_sha, config_sha in running_apps
if (service, instance) not in valid_services]
log.debug("Running apps: %s" % running_apps)
log.debug("Valid apps: %s" % valid_services)
log.debug("Terminating: %s" % apps_to_kill)
if running_apps:
above_kill_threshold = float(len(apps_to_kill)) / float(
len(running_apps)) > float(kill_threshold)
if above_kill_threshold and not force:
log.critical(
"Paasta was about to kill more than %s of the running services, this "
"is probably a BAD mistake!, run again with --force if you "
"really need to destroy everything" % kill_threshold, )
raise DontKillEverythingError
for running_app in apps_to_kill:
app_id = marathon_tools.format_job_id(*running_app)
delete_app(
app_id=app_id,
client=client,
soa_dir=soa_dir,
)
def get_service_instances_with_changed_id(marathon_client, instances, cluster):
marathon_app_ids = list_all_marathon_app_ids(marathon_client)
service_instances = []
for service, instance in instances:
config = load_marathon_service_config_no_cache(service=service,
instance=instance,
cluster=cluster,
soa_dir=DEFAULT_SOA_DIR)
try:
config_app_id = config.format_marathon_app_dict()['id']
except NoDockerImageError:
config_app_id = None
if not config_app_id or (config_app_id not in marathon_app_ids):
service_instances.append((service, instance))
return service_instances
def cleanup_apps(soa_dir, kill_threshold=0.5, force=False):
"""Clean up old or invalid jobs/apps from marathon. Retrieves
both a list of apps currently in marathon and a list of valid
app ids in order to determine what to kill.
:param soa_dir: The SOA config directory to read from
:param kill_threshold: The decimal fraction of apps we think is
sane to kill when this job runs.
:param force: Force the cleanup if we are above the kill_threshold"""
log.info("Loading marathon configuration")
marathon_config = marathon_tools.load_marathon_config()
log.info("Connecting to marathon")
client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(),
marathon_config.get_password())
valid_services = get_services_for_cluster(instance_type='marathon', soa_dir=soa_dir)
running_app_ids = marathon_tools.list_all_marathon_app_ids(client)
running_apps = []
for app_id in running_app_ids:
try:
app_id = marathon_tools.deformat_job_id(app_id)
except InvalidJobNameError:
log.warn("%s doesn't conform to paasta naming conventions? Skipping." % app_id)
continue
running_apps.append(app_id)
apps_to_kill = [(service, instance, git_sha, config_sha)
for service, instance, git_sha, config_sha in running_apps
if (service, instance) not in valid_services]
log.debug("Running apps: %s" % running_apps)
log.debug("Valid apps: %s" % valid_services)
log.debug("Terminating: %s" % apps_to_kill)
if running_apps:
above_kill_threshold = float(len(apps_to_kill)) / float(len(running_apps)) > float(kill_threshold)
if above_kill_threshold and not force:
log.critical("Paasta was about to kill more than %s of the running services, this "
"is probably a BAD mistake!, run again with --force if you "
"really need to destroy everything" % kill_threshold)
raise DontKillEverythingError
for running_app in apps_to_kill:
app_id = marathon_tools.format_job_id(*running_app)
delete_app(
app_id=app_id,
client=client,
soa_dir=soa_dir,
)
def _clean_up_marathon_apps(context):
"""If a marathon client object exists in our context, delete any apps in Marathon and wait until they die."""
if hasattr(context, 'marathon_client'):
while True:
apps = marathon_tools.list_all_marathon_app_ids(context.marathon_client)
if not apps:
break
print "after_scenario: Deleting %d apps to prep for the next scenario. %s" % (len(apps), ",".join(apps))
for app in apps:
if marathon_tools.is_app_id_running(app, context.marathon_client):
print "after_scenario: %s does look like it is running. Scaling down and killing it..." % app
context.marathon_client.scale_app(app, instances=0, force=True)
time.sleep(1)
context.marathon_client.delete_app(app, force=True)
else:
print "after_scenario: %s showed up in the app_list, but doesn't look like it is running?" % app
time.sleep(0.5)
while context.marathon_client.list_deployments():
print "after_scenario: There are still marathon deployments in progress. sleeping."
time.sleep(0.5)
def see_it_in_list(context):
assert context.app_id in marathon_tools.list_all_marathon_app_ids(context.marathon_client)
def not_see_it_in_list(context):
with requests_cache.disabled():
assert context.app_id not in marathon_tools.list_all_marathon_app_ids(context.marathon_client)
