def check_app_not_running(context, seconds): step = 5 attempts = 0 while (attempts * step) < seconds: if context.old_app_id not in list_all_marathon_app_ids(context.current_client): return time.sleep(step) attempts += 1 assert context.old_app_id not in list_all_marathon_app_ids(context.current_client)
def not_see_it_in_list(context, shard_number=None): full_list = [] if shard_number is None: for client in context.marathon_clients.get_all_clients(): full_list.extend(marathon_tools.list_all_marathon_app_ids(client)) else: full_list.extend(marathon_tools.list_all_marathon_app_ids(context.marathon_clients.current[shard_number])) assert context.app_id not in full_list
def _clean_up_marathon_apps(context): """If a marathon client object exists in our context, delete any apps in Marathon and wait until they die.""" if hasattr(context, "marathon_clients"): still_apps = True while still_apps: still_apps = False for client in context.marathon_clients.get_all_clients(): apps = marathon_tools.list_all_marathon_app_ids(client) if apps: still_apps = True else: continue paasta_print( "after_scenario: Deleting %d apps to prep for the next scenario. %s" % (len(apps), ",".join(apps))) for app in apps: if marathon_tools.is_app_id_running(app, client): paasta_print( "after_scenario: %s does look like it is running. Scaling down and killing it..." % app) client.scale_app(app, instances=0, force=True) time.sleep(1) client.delete_app(app, force=True) else: paasta_print( "after_scenario: %s showed up in the app_list, but doesn't look like it is running?" % app) time.sleep(0.5) for client in context.marathon_clients.get_all_clients(): while client.list_deployments(): paasta_print( "after_scenario: There are still marathon deployments in progress. sleeping." ) time.sleep(0.5)
def cleanup_apps(soa_dir): """Clean up old or invalid jobs/apps from marathon. Retrieves both a list of apps currently in marathon and a list of valid app ids in order to determine what to kill. :param soa_dir: The SOA config directory to read from""" log.info("Loading marathon configuration") marathon_config = marathon_tools.load_marathon_config() log.info("Connecting to marathon") client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(), marathon_config.get_password()) valid_services = get_services_for_cluster(instance_type='marathon', soa_dir=soa_dir) running_app_ids = marathon_tools.list_all_marathon_app_ids(client) for app_id in running_app_ids: log.debug("Checking app id %s", app_id) try: service, instance, _, __ = marathon_tools.deformat_job_id(app_id) except InvalidJobNameError: log.warn("%s doesn't conform to paasta naming conventions? Skipping." % app_id) continue if (service, instance) not in valid_services: delete_app( app_id=app_id, client=client, soa_dir=soa_dir, )
def check_app_running(context, service_instance, seconds): service, instance, _, _ = decompose_job_id(service_instance) service_configuration_lib._yaml_cache = {} context.marathon_config = load_marathon_service_config( service, instance, context.cluster) context.app_id = context.marathon_config.format_marathon_app_dict()['id'] step = 5 attempts = 0 while (attempts * step) < seconds: if context.app_id in list_all_marathon_app_ids( context.marathon_client): break time.sleep(step) attempts += 1 assert context.app_id in list_all_marathon_app_ids(context.marathon_client) context.old_app_id = context.app_id
def cleanup_apps(soa_dir): """Clean up old or invalid jobs/apps from marathon. Retrieves both a list of apps currently in marathon and a list of valid app ids in order to determine what to kill. :param soa_dir: The SOA config directory to read from""" log.info("Loading marathon configuration") marathon_config = marathon_tools.load_marathon_config() log.info("Connecting to marathon") client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(), marathon_config.get_password()) valid_services = get_services_for_cluster(instance_type='marathon', soa_dir=soa_dir) running_app_ids = marathon_tools.list_all_marathon_app_ids(client) for app_id in running_app_ids: log.debug("Checking app id %s", app_id) try: service, instance, _, __ = marathon_tools.deformat_job_id(app_id) except InvalidJobNameError: log.warn( "%s doesn't conform to paasta naming conventions? Skipping." % app_id) continue if (service, instance) not in valid_services: delete_app( app_id=app_id, client=client, soa_dir=soa_dir, )
def cleanup_apps(soa_dir, kill_threshold=0.5, force=False): """Clean up old or invalid jobs/apps from marathon. Retrieves both a list of apps currently in marathon and a list of valid app ids in order to determine what to kill. :param soa_dir: The SOA config directory to read from :param kill_threshold: The decimal fraction of apps we think is sane to kill when this job runs. :param force: Force the cleanup if we are above the kill_threshold""" log.info("Loading marathon configuration") marathon_config = marathon_tools.load_marathon_config() log.info("Connecting to marathon") client = marathon_tools.get_marathon_client( marathon_config.get_url(), marathon_config.get_username(), marathon_config.get_password(), ) valid_services = get_services_for_cluster(instance_type='marathon', soa_dir=soa_dir) running_app_ids = marathon_tools.list_all_marathon_app_ids(client) running_apps = [] for app_id in running_app_ids: try: app_id = marathon_tools.deformat_job_id(app_id) except InvalidJobNameError: log.warn( "%s doesn't conform to paasta naming conventions? Skipping." % app_id) continue running_apps.append(app_id) apps_to_kill = [(service, instance, git_sha, config_sha) for service, instance, git_sha, config_sha in running_apps if (service, instance) not in valid_services] log.debug("Running apps: %s" % running_apps) log.debug("Valid apps: %s" % valid_services) log.debug("Terminating: %s" % apps_to_kill) if running_apps: above_kill_threshold = float(len(apps_to_kill)) / float( len(running_apps)) > float(kill_threshold) if above_kill_threshold and not force: log.critical( "Paasta was about to kill more than %s of the running services, this " "is probably a BAD mistake!, run again with --force if you " "really need to destroy everything" % kill_threshold, ) raise DontKillEverythingError for running_app in apps_to_kill: app_id = marathon_tools.format_job_id(*running_app) delete_app( app_id=app_id, client=client, soa_dir=soa_dir, )
def get_service_instances_with_changed_id(marathon_client, instances, cluster): marathon_app_ids = list_all_marathon_app_ids(marathon_client) service_instances = [] for service, instance in instances: config = load_marathon_service_config_no_cache(service=service, instance=instance, cluster=cluster, soa_dir=DEFAULT_SOA_DIR) try: config_app_id = config.format_marathon_app_dict()['id'] except NoDockerImageError: config_app_id = None if not config_app_id or (config_app_id not in marathon_app_ids): service_instances.append((service, instance)) return service_instances
def cleanup_apps(soa_dir, kill_threshold=0.5, force=False): """Clean up old or invalid jobs/apps from marathon. Retrieves both a list of apps currently in marathon and a list of valid app ids in order to determine what to kill. :param soa_dir: The SOA config directory to read from :param kill_threshold: The decimal fraction of apps we think is sane to kill when this job runs. :param force: Force the cleanup if we are above the kill_threshold""" log.info("Loading marathon configuration") marathon_config = marathon_tools.load_marathon_config() log.info("Connecting to marathon") client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(), marathon_config.get_password()) valid_services = get_services_for_cluster(instance_type='marathon', soa_dir=soa_dir) running_app_ids = marathon_tools.list_all_marathon_app_ids(client) running_apps = [] for app_id in running_app_ids: try: app_id = marathon_tools.deformat_job_id(app_id) except InvalidJobNameError: log.warn("%s doesn't conform to paasta naming conventions? Skipping." % app_id) continue running_apps.append(app_id) apps_to_kill = [(service, instance, git_sha, config_sha) for service, instance, git_sha, config_sha in running_apps if (service, instance) not in valid_services] log.debug("Running apps: %s" % running_apps) log.debug("Valid apps: %s" % valid_services) log.debug("Terminating: %s" % apps_to_kill) if running_apps: above_kill_threshold = float(len(apps_to_kill)) / float(len(running_apps)) > float(kill_threshold) if above_kill_threshold and not force: log.critical("Paasta was about to kill more than %s of the running services, this " "is probably a BAD mistake!, run again with --force if you " "really need to destroy everything" % kill_threshold) raise DontKillEverythingError for running_app in apps_to_kill: app_id = marathon_tools.format_job_id(*running_app) delete_app( app_id=app_id, client=client, soa_dir=soa_dir, )
def _clean_up_marathon_apps(context): """If a marathon client object exists in our context, delete any apps in Marathon and wait until they die.""" if hasattr(context, 'marathon_client'): while True: apps = marathon_tools.list_all_marathon_app_ids(context.marathon_client) if not apps: break print "after_scenario: Deleting %d apps to prep for the next scenario. %s" % (len(apps), ",".join(apps)) for app in apps: if marathon_tools.is_app_id_running(app, context.marathon_client): print "after_scenario: %s does look like it is running. Scaling down and killing it..." % app context.marathon_client.scale_app(app, instances=0, force=True) time.sleep(1) context.marathon_client.delete_app(app, force=True) else: print "after_scenario: %s showed up in the app_list, but doesn't look like it is running?" % app time.sleep(0.5) while context.marathon_client.list_deployments(): print "after_scenario: There are still marathon deployments in progress. sleeping." time.sleep(0.5)
def see_it_in_list(context): assert context.app_id in marathon_tools.list_all_marathon_app_ids(context.marathon_client)
def not_see_it_in_list(context): with requests_cache.disabled(): assert context.app_id not in marathon_tools.list_all_marathon_app_ids(context.marathon_client)