def kill_tasks_if_necessary(self, driver: MesosSchedulerDriver): base_task = self.service_config.base_task(self.system_paasta_config) all_tasks_with_params = self.task_store.get_all_tasks() new_tasks_with_params = self.get_new_tasks(base_task['name'], all_tasks_with_params) happy_new_tasks_with_params = self.get_happy_tasks(new_tasks_with_params) desired_instances = self.service_config.get_desired_instances() # this puts the most-desired tasks first. I would have left them in order of bad->good and used # new_tasks_by_desirability[:-desired_instances] instead, but list[:-0] is an empty list, rather than the full # list. new_task_ids_by_desirability = sorted( list(new_tasks_with_params.keys()), key=self.make_healthiness_sorter(base_task['name'], all_tasks_with_params), reverse=True, ) new_task_ids_to_kill = new_task_ids_by_desirability[desired_instances:] old_tasks_with_params = self.get_old_tasks(base_task['name'], all_tasks_with_params) old_draining_tasks_with_params = self.get_draining_tasks(old_tasks_with_params) old_non_draining_tasks = sorted( list( set(old_tasks_with_params.keys()) - set(old_draining_tasks_with_params), ), key=self.make_healthiness_sorter(base_task['name'], all_tasks_with_params), reverse=True, ) actions = bounce_lib.crossover_bounce( new_config={"instances": desired_instances}, new_app_running=True, happy_new_tasks=happy_new_tasks_with_params.keys(), old_non_draining_tasks=new_task_ids_to_kill + old_non_draining_tasks, ) with a_sync.idle_event_loop(): futures = [] for task in set(new_tasks_with_params.keys()) - set(actions['tasks_to_drain']): futures.append(asyncio.ensure_future(self.undrain_task(task))) for task in actions['tasks_to_drain']: futures.append(asyncio.ensure_future(self.drain_task(task))) if futures: a_sync.block(asyncio.wait, futures) async def kill_if_safe_to_kill(task_id: str): if await self.drain_method.is_safe_to_kill(self.make_drain_task(task_id)): self.kill_task(driver, task_id) futures = [] for task, parameters in all_tasks_with_params.items(): if parameters.is_draining and parameters.mesos_task_state in LIVE_TASK_STATES: futures.append(asyncio.ensure_future(kill_if_safe_to_kill(task))) if futures: a_sync.block(asyncio.wait, futures)
def deploy_marathon_service( service: str, instance: str, clients: marathon_tools.MarathonClients, soa_dir: str, marathon_apps_with_clients: Optional[Collection[Tuple[MarathonApp, MarathonClient]]], ) -> Tuple[int, float]: """deploy the service instance given and proccess return code if there was an error we send a sensu alert. :param service: The service name to setup :param instance: The instance of the service to setup :param clients: A MarathonClients object :param soa_dir: Path to yelpsoa configs :param marathon_apps: A list of all marathon app objects :returns: A tuple of (status, bounce_in_seconds) to be used by paasta-deployd bounce_in_seconds instructs how long until the deployd should try another bounce None means that it is in a steady state and doesn't need to bounce again """ short_id = marathon_tools.format_job_id(service, instance) try: with bounce_lib.bounce_lock_zookeeper(short_id): try: service_instance_config = marathon_tools.load_marathon_service_config_no_cache( service, instance, load_system_paasta_config().get_cluster(), soa_dir=soa_dir, ) except NoDeploymentsAvailable: log.debug( "No deployments found for %s.%s in cluster %s. Skipping." % (service, instance, load_system_paasta_config().get_cluster())) return 0, None except NoConfigurationForServiceError: error_msg = "Could not read marathon configuration file for %s.%s in cluster %s" % \ (service, instance, load_system_paasta_config().get_cluster()) log.error(error_msg) return 1, None if marathon_apps_with_clients is None: marathon_apps_with_clients = marathon_tools.get_marathon_apps_with_clients( clients=clients.get_all_clients_for_service( job_config=service_instance_config), embed_tasks=True, ) try: with a_sync.idle_event_loop(): status, output, bounce_again_in_seconds = setup_service( service=service, instance=instance, clients=clients, job_config=service_instance_config, marathon_apps_with_clients=marathon_apps_with_clients, soa_dir=soa_dir, ) sensu_status = pysensu_yelp.Status.CRITICAL if status else pysensu_yelp.Status.OK send_event(service, instance, soa_dir, sensu_status, output) return 0, bounce_again_in_seconds except (KeyError, TypeError, AttributeError, InvalidInstanceConfig, NoSlavesAvailableError): error_str = traceback.format_exc() log.error(error_str) send_event(service, instance, soa_dir, pysensu_yelp.Status.CRITICAL, error_str) return 1, None except bounce_lib.LockHeldException: log.error("Instance %s already being bounced. Exiting", short_id) return 0, None