def get_service_instances_needing_update( marathon_clients: MarathonClients, instances: Collection[Tuple[str, str]], cluster: str, ) -> List[Tuple[str, str, MarathonServiceConfig, str]]: marathon_apps = {} for marathon_client in marathon_clients.get_all_clients(): marathon_apps.update( {app.id: app for app in get_all_marathon_apps(marathon_client)}) marathon_app_ids = marathon_apps.keys() service_instances = [] for service, instance in instances: try: config = load_marathon_service_config_no_cache( service=service, instance=instance, cluster=cluster, soa_dir=DEFAULT_SOA_DIR, ) config_app = config.format_marathon_app_dict() app_id = "/{}".format(config_app["id"]) # Not ideal but we rely on a lot of user input to create the app dict # and we really can't afford to bail if just one app definition is malformed except Exception as e: print("ERROR: Skipping {}.{} because: '{}'".format( service, instance, str(e))) continue if (app_id not in marathon_app_ids or marathon_apps[app_id].instances != config_app["instances"]): service_instances.append((service, instance, config, app_id)) return service_instances
def get_service_instances_needing_update(marathon_client, instances, cluster): marathon_apps = { app.id: app for app in get_all_marathon_apps(marathon_client) } marathon_app_ids = marathon_apps.keys() service_instances = [] for service, instance in instances: try: config = load_marathon_service_config_no_cache( service=service, instance=instance, cluster=cluster, soa_dir=DEFAULT_SOA_DIR, ) config_app = config.format_marathon_app_dict() app_id = '/{}'.format(config_app['id']) except (NoDockerImageError, InvalidJobNameError, NoDeploymentsAvailable) as e: print("DEBUG: Skipping %s.%s because: '%s'" % (service, instance, str(e))) continue if app_id not in marathon_app_ids: service_instances.append((service, instance)) elif marathon_apps[app_id].instances != config_app['instances']: service_instances.append((service, instance)) return service_instances
def check_sha_changed(context, service_instance): service, instance, _, _ = decompose_job_id(service_instance) service_configuration_lib._yaml_cache = {} context.marathon_config = load_marathon_service_config_no_cache( service, instance, context.cluster) assert context.app_id != context.marathon_config.format_marathon_app_dict( )['id']
def get_service_instances_needing_update( marathon_clients: MarathonClients, instances: Collection[Tuple[str, str]], cluster: str, ) -> List[Tuple[str, str]]: marathon_apps = {} for marathon_client in marathon_clients.get_all_clients(): marathon_apps.update( {app.id: app for app in get_all_marathon_apps(marathon_client)}) marathon_app_ids = marathon_apps.keys() service_instances = [] for service, instance in instances: try: config = load_marathon_service_config_no_cache( service=service, instance=instance, cluster=cluster, soa_dir=DEFAULT_SOA_DIR, ) config_app = config.format_marathon_app_dict() app_id = '/{}'.format(config_app['id']) except (NoDockerImageError, InvalidJobNameError, NoDeploymentsAvailable, NoSlavesAvailableError) as e: print("DEBUG: Skipping {}.{} because: '{}'".format( service, instance, str(e))) continue if app_id not in marathon_app_ids: service_instances.append((service, instance)) elif marathon_apps[app_id].instances != config_app['instances']: service_instances.append((service, instance)) return service_instances
def deploy_marathon_service(service, instance, client, soa_dir, marathon_config, marathon_apps): """deploy the service instance given and proccess return code if there was an error we send a sensu alert. :param service: The service name to setup :param instance: The instance of the service to setup :param client: A MarathonClient object :param soa_dir: Path to yelpsoa configs :param marathon_config: The service instance's configuration dict :param marathon_apps: A list of all marathon app objects :returns: A tuple of (status, bounce_in_seconds) to be used by paasta-deployd bounce_in_seconds instructs how long until the deployd should try another bounce None means that it is in a steady state and doesn't need to bounce again """ short_id = marathon_tools.format_job_id(service, instance) try: with bounce_lib.bounce_lock_zookeeper(short_id): try: service_instance_config = marathon_tools.load_marathon_service_config_no_cache( service, instance, load_system_paasta_config().get_cluster(), soa_dir=soa_dir, ) except NoDeploymentsAvailable: log.debug( "No deployments found for %s.%s in cluster %s. Skipping." % (service, instance, load_system_paasta_config().get_cluster())) return 0, None except NoConfigurationForServiceError: error_msg = "Could not read marathon configuration file for %s.%s in cluster %s" % \ (service, instance, load_system_paasta_config().get_cluster()) log.error(error_msg) return 1, None try: status, output, bounce_again_in_seconds = setup_service( service, instance, client, service_instance_config, marathon_apps, soa_dir, ) sensu_status = pysensu_yelp.Status.CRITICAL if status else pysensu_yelp.Status.OK send_event(service, instance, soa_dir, sensu_status, output) return 0, bounce_again_in_seconds except (KeyError, TypeError, AttributeError, InvalidInstanceConfig, NoSlavesAvailableError): error_str = traceback.format_exc() log.error(error_str) send_event(service, instance, soa_dir, pysensu_yelp.Status.CRITICAL, error_str) return 1, None except bounce_lib.LockHeldException: log.error("Instance %s already being bounced. Exiting", short_id) return 0, None
def update_context_marathon_config(context): whitelist_keys = { 'id', 'backoff_factor', 'backoff_seconds', 'max_instances', 'mem', 'cpus', 'instances', 'marathon_shard', 'previous_marathon_shards', } with mock.patch.object( MarathonServiceConfig, 'get_min_instances', autospec=True, return_value=1, ), mock.patch.object( MarathonServiceConfig, 'get_max_instances', autospec=True, ) as mock_get_max_instances: mock_get_max_instances.return_value = context.max_instances if 'max_instances' in context else None service_configuration_lib._yaml_cache = {} context.job_config = marathon_tools.load_marathon_service_config_no_cache( service=context.service, instance=context.instance, cluster=context.system_paasta_config.get_cluster(), soa_dir=context.soa_dir, ) context.current_client = context.marathon_clients.get_current_client_for_service( context.job_config) context.marathon_complete_config = { key: value for key, value in context.job_config.format_marathon_app_dict().items() if key in whitelist_keys } context.marathon_complete_config.update({ 'cmd': '/bin/sleep 1m', 'constraints': None, 'container': { 'type': 'DOCKER', 'docker': { 'network': 'BRIDGE', 'image': 'busybox', }, }, }) if 'max_instances' not in context: context.marathon_complete_config['instances'] = context.instances
def update_context_marathon_config(context): whitelist_keys = { "id", "backoff_factor", "backoff_seconds", "max_instances", "mem", "cpus", "instances", "marathon_shard", "previous_marathon_shards", } with mock.patch.object(MarathonServiceConfig, "get_min_instances", autospec=True, return_value=1), mock.patch.object( MarathonServiceConfig, "get_max_instances", autospec=True) as mock_get_max_instances: mock_get_max_instances.return_value = ( context.max_instances if "max_instances" in context else None) service_configuration_lib._yaml_cache = {} context.job_config = marathon_tools.load_marathon_service_config_no_cache( service=context.service, instance=context.instance, cluster=context.system_paasta_config.get_cluster(), soa_dir=context.soa_dir, ) context.current_client = context.marathon_clients.get_current_client_for_service( context.job_config) context.marathon_complete_config = { key: value for key, value in context.job_config.format_marathon_app_dict().items() if key in whitelist_keys } context.marathon_complete_config.update({ "cmd": "/bin/sleep 1m", "constraints": None, "container": { "type": "DOCKER", "docker": { "network": "BRIDGE", "image": "busybox" }, }, }) if "max_instances" not in context: context.marathon_complete_config["instances"] = context.instances
def get_service_instances_with_changed_id(marathon_client, instances, cluster): marathon_app_ids = list_all_marathon_app_ids(marathon_client) service_instances = [] for service, instance in instances: config = load_marathon_service_config_no_cache(service=service, instance=instance, cluster=cluster, soa_dir=DEFAULT_SOA_DIR) try: config_app_id = config.format_marathon_app_dict()['id'] except NoDockerImageError: config_app_id = None if not config_app_id or (config_app_id not in marathon_app_ids): service_instances.append((service, instance)) return service_instances
def check_app_running(context, service_instance, seconds): service, instance, _, _ = decompose_job_id(service_instance) service_configuration_lib._yaml_cache = {} context.marathon_config = load_marathon_service_config_no_cache( service, instance, context.cluster) context.app_id = context.marathon_config.format_marathon_app_dict()['id'] step = 5 attempts = 0 while (attempts * step) < seconds: if context.app_id in list_all_marathon_app_ids( context.marathon_client): break time.sleep(step) attempts += 1 assert context.app_id in list_all_marathon_app_ids(context.marathon_client) context.old_app_id = context.app_id
def get_service_instances_needing_update(marathon_client, instances, cluster): marathon_apps = { app.id: app for app in get_all_marathon_apps(marathon_client) } marathon_app_ids = marathon_apps.keys() service_instances = [] for service, instance in instances: config = load_marathon_service_config_no_cache(service=service, instance=instance, cluster=cluster, soa_dir=DEFAULT_SOA_DIR) try: config_app = config.format_marathon_app_dict() app_id = '/{}'.format(config_app['id']) except NoDockerImageError: config_app = None if not config_app: service_instances.append((service, instance)) elif app_id not in marathon_app_ids: service_instances.append((service, instance)) elif marathon_apps[app_id].instances != config_app['instances']: service_instances.append((service, instance)) return service_instances
def deploy_marathon_service( service: str, instance: str, clients: marathon_tools.MarathonClients, soa_dir: str, marathon_apps_with_clients: Optional[Sequence[Tuple[MarathonApp, MarathonClient]]], system_paasta_config: Optional[SystemPaastaConfig] = None, ) -> Tuple[int, float]: """deploy the service instance given and process return code if there was an error we send a sensu alert. :param service: The service name to setup :param instance: The instance of the service to setup :param clients: A MarathonClients object :param soa_dir: Path to yelpsoa configs :param marathon_apps: A list of all marathon app objects :returns: A tuple of (status, bounce_in_seconds) to be used by paasta-deployd bounce_in_seconds instructs how long until the deployd should try another bounce None means that it is in a steady state and doesn't need to bounce again """ if system_paasta_config is None: system_paasta_config = load_system_paasta_config() short_id = marathon_tools.format_job_id(service, instance) try: with bounce_lib.bounce_lock_zookeeper( short_id, system_paasta_config=system_paasta_config): try: service_instance_config = marathon_tools.load_marathon_service_config_no_cache( service, instance, system_paasta_config.get_cluster(), soa_dir=soa_dir, ) except NoDeploymentsAvailable: log.debug( "No deployments found for %s.%s in cluster %s. Skipping." % (service, instance, system_paasta_config.get_cluster())) return 0, None except NoConfigurationForServiceError: error_msg = ( "Could not read marathon configuration file for %s.%s in cluster %s" % (service, instance, system_paasta_config.get_cluster())) log.error(error_msg) return 1, None if marathon_apps_with_clients is None: marathon_apps_with_clients = marathon_tools.get_marathon_apps_with_clients( clients=clients.get_all_clients_for_service( job_config=service_instance_config), service_name=service, instance_name=instance, embed_tasks=True, ) try: with a_sync.idle_event_loop(): status, output, bounce_again_in_seconds = setup_service( service=service, instance=instance, clients=clients, job_config=service_instance_config, marathon_apps_with_clients=marathon_apps_with_clients, soa_dir=soa_dir, system_paasta_config=system_paasta_config, ) sensu_status = (pysensu_yelp.Status.CRITICAL if status else pysensu_yelp.Status.OK) send_event( service, instance, soa_dir, sensu_status, output, system_paasta_config, service_instance_config, ) return 0, bounce_again_in_seconds except ( KeyError, TypeError, AttributeError, InvalidInstanceConfig, NoSlavesAvailableError, ): error_str = traceback.format_exc() log.error(error_str) send_event( service, instance, soa_dir, pysensu_yelp.Status.CRITICAL, error_str, system_paasta_config, service_instance_config, ) return 1, None except bounce_lib.LockHeldException: log.error("Instance %s already being bounced. Exiting", short_id) return 0, None