def test_tasks_for_offer_chooses_port(self, system_paasta_config): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_configs = [] service_configs.append( NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 1, "cmd": "sleep 50", "drain_method": "test", }, branch_dict={ "docker_image": "busybox", "image_version": None, "desired_state": "start", "force_bounce": "0", }, soa_dir="/nail/etc/services", )) scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_configs[0], reconcile_start_time=0, staging_timeout=1, task_store_type=DictTaskStore, ) scheduler.registered(driver=mock.Mock(), frameworkId={"value": "foo"}, masterInfo=mock.Mock()) with mock.patch( "paasta_tools.utils.load_system_paasta_config", autospec=True, return_value=system_paasta_config, ): tasks, _ = scheduler.tasks_and_state_for_offer( mock.Mock(), make_fake_offer(port_begin=12345, port_end=12345), {}) assert { "name": "ports", "ranges": { "range": [{ "begin": 12345, "end": 12345 }] }, "type": "RANGES", } in tasks[0]["resources"]
def test_make_drain_task_works_with_hacheck_drain_method( self, system_paasta_config): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_config = NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 1, "cmd": 'sleep 50', "drain_method": "hacheck", "pool": "default", }, branch_dict={ 'docker_image': 'busybox', 'desired_state': 'start', 'force_bounce': '0', }, soa_dir='/nail/etc/services', ) scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_config, staging_timeout=1, task_store_type=DictTaskStore, ) fake_driver = mock.Mock() scheduler.registered( driver=fake_driver, frameworkId={'value': 'foo'}, masterInfo=mock.Mock(), ) # launch a task offer = make_fake_offer(port_begin=31337, port_end=31337) with mock.patch( 'paasta_tools.utils.load_system_paasta_config', autospec=True, return_value=system_paasta_config, ): scheduler.launch_tasks_for_offers( driver=fake_driver, offers=[offer], ) expected = "http://super_big_slave:6666/spool/service_name.instance_name/31337/status" actual = scheduler.drain_method.spool_url( scheduler.make_drain_task( list(scheduler.task_store.get_all_tasks().keys())[0]), ) assert actual == expected
def test_tasks_for_offer_chooses_port(self, system_paasta_config): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_configs = [] service_configs.append(NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 1, "cmd": 'sleep 50', "drain_method": "test", }, branch_dict={ 'docker_image': 'busybox', 'desired_state': 'start', 'force_bounce': '0', }, soa_dir='/nail/etc/services', )) scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_configs[0], reconcile_start_time=0, staging_timeout=1, task_store_type=DictTaskStore, ) scheduler.registered( driver=mock.Mock(), frameworkId={'value': 'foo'}, masterInfo=mock.Mock(), ) with mock.patch( 'paasta_tools.utils.load_system_paasta_config', autospec=True, return_value=system_paasta_config, ): tasks, _ = scheduler.tasks_and_state_for_offer( mock.Mock(), make_fake_offer(port_begin=12345, port_end=12345), {}, ) assert { 'name': 'ports', 'ranges': {'range': [{'begin': 12345, 'end': 12345}]}, 'type': 'RANGES', } in tasks[0]['resources']
def test_tasks_for_offer_chooses_port(self, system_paasta_config): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_configs = [] service_configs.append(NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 1, "cmd": 'sleep 50', "drain_method": "test" }, branch_dict={ 'docker_image': 'busybox', 'desired_state': 'start', 'force_bounce': '0', }, soa_dir='/nail/etc/services', )) scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_configs[0], reconcile_start_time=0, staging_timeout=1, ) with mock.patch('paasta_tools.utils.load_system_paasta_config', autospec=True, return_value=system_paasta_config ): tasks, _ = scheduler.tasks_and_state_for_offer( mock.Mock(), make_fake_offer(port_begin=12345, port_end=12345), {}) assert len(tasks) == 1 for task in tasks: for resource in task.resources: if resource.name == "ports": assert resource.ranges.range[0].begin == 12345 assert resource.ranges.range[0].end == 12345 break else: raise AssertionError("never saw a ports resource")
def test_offer_matches_pool(self): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_config = NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 1, "cmd": 'sleep 50', "drain_method": "test", "pool": "default", }, branch_dict={ 'docker_image': 'busybox', 'desired_state': 'start', 'force_bounce': '0', }, soa_dir='/nail/etc/services', ) scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_config, staging_timeout=1, task_store_type=DictTaskStore, ) scheduler.registered( driver=mock.Mock(), frameworkId={'value': 'foo'}, masterInfo=mock.Mock(), ) assert scheduler.offer_matches_pool( make_fake_offer(port_begin=12345, port_end=12345, pool="default")) assert not scheduler.offer_matches_pool( make_fake_offer( port_begin=12345, port_end=12345, pool="somethingelse")) assert not scheduler.offer_matches_pool( make_fake_offer(port_begin=12345, port_end=12345, pool=None))
def test_resource_offers_ignores_blacklisted_slaves( self, system_paasta_config): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_configs = [ NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 3, "cmd": 'sleep 50', "drain_method": "test", }, branch_dict={ 'docker_image': 'busybox', 'desired_state': 'start', }, soa_dir='/nail/etc/services', ), ] scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_configs[0], staging_timeout=1, task_store_type=DictTaskStore, ) fake_driver = mock.Mock() scheduler.registered( driver=fake_driver, frameworkId={'value': 'foo'}, masterInfo=mock.Mock(), ) scheduler.blacklist_slave('super big slave') assert len(scheduler.blacklisted_slaves) == 1 scheduler.resourceOffers(fake_driver, [make_fake_offer()]) assert len(scheduler.task_store.get_all_tasks()) == 0
def test_resource_offers_ignores_blacklisted_slaves( self, system_paasta_config): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_configs = [ NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 3, "cmd": 'sleep 50', "drain_method": "test" }, branch_dict={ 'docker_image': 'busybox', 'desired_state': 'start', }, ) ] scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_configs[0], staging_timeout=1, ) fake_driver = mock.Mock() scheduler.blacklist_slave('super big slave') assert len(scheduler.blacklisted_slaves) == 1 scheduler.resourceOffers(fake_driver, [make_fake_offer()]) assert len(scheduler.tasks_with_flags) == 0
def test_offer_matches_pool(self): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_config = NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 1, "cmd": 'sleep 50', "drain_method": "test", "pool": "default", }, branch_dict={ 'docker_image': 'busybox', 'desired_state': 'start', 'force_bounce': '0', }, ) scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_config, staging_timeout=1) assert scheduler.offer_matches_pool( make_fake_offer(port_begin=12345, port_end=12345, pool="default")) assert not scheduler.offer_matches_pool( make_fake_offer( port_begin=12345, port_end=12345, pool="somethingelse")) assert not scheduler.offer_matches_pool( make_fake_offer(port_begin=12345, port_end=12345, pool=None))
def test_start_upgrade_rollback_scaledown(self, system_paasta_config): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_configs = [] for force_bounce in range(2): service_configs.append(native_scheduler.NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 3, "cmd": 'sleep 50', "drain_method": "test" }, branch_dict={ 'docker_image': 'busybox', 'desired_state': 'start', 'force_bounce': str(force_bounce), }, )) scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_configs[0], staging_timeout=1, ) fake_driver = mock.Mock() # First, start up 3 old tasks old_tasks = scheduler.launch_tasks_for_offers(fake_driver, [make_fake_offer()]) assert len(scheduler.tasks_with_flags) == 3 # and mark the old tasks as up for task in old_tasks: scheduler.statusUpdate(fake_driver, mock.Mock(task_id=task.task_id, state=TASK_RUNNING)) assert len(scheduler.drain_method.downed_task_ids) == 0 # Now, change force_bounce scheduler.service_config = service_configs[1] # and start 3 more tasks new_tasks = scheduler.launch_tasks_for_offers(fake_driver, [make_fake_offer()]) assert len(scheduler.tasks_with_flags) == 6 # It should not drain anything yet, since the new tasks aren't up. scheduler.kill_tasks_if_necessary(fake_driver) assert len(scheduler.tasks_with_flags) == 6 assert len(scheduler.drain_method.downed_task_ids) == 0 # Now we mark the new tasks as up. for i, task in enumerate(new_tasks): scheduler.statusUpdate(fake_driver, mock.Mock(task_id=task.task_id, state=TASK_RUNNING)) # As each of these new tasks come up, we should drain an old one. assert len(scheduler.drain_method.downed_task_ids) == i + 1 # Now let's roll back and make sure it undrains the old ones and drains new. scheduler.service_config = service_configs[0] scheduler.kill_tasks_if_necessary(fake_driver) assert scheduler.drain_method.downed_task_ids == set() scheduler.kill_tasks_if_necessary(fake_driver) assert scheduler.drain_method.downed_task_ids == {t.task_id.value for t in new_tasks} # Once we drain the new tasks, it should kill them. assert fake_driver.killTask.call_count == 0 # we issue duplicate kills for tasks until we get notified about TASK_KILLED, so we keep track of # the unique IDs of tasks being killed. killed_tasks = set() def killTask_side_effect(task_id): killed_tasks.add(task_id.value) fake_driver.killTask.side_effect = killTask_side_effect scheduler.drain_method.mark_arbitrary_task_as_safe_to_kill() scheduler.kill_tasks_if_necessary(fake_driver) assert len(killed_tasks) == 1 scheduler.drain_method.mark_arbitrary_task_as_safe_to_kill() scheduler.kill_tasks_if_necessary(fake_driver) assert len(killed_tasks) == 2 scheduler.drain_method.mark_arbitrary_task_as_safe_to_kill() scheduler.kill_tasks_if_necessary(fake_driver) assert scheduler.drain_method.safe_to_kill_task_ids == {t.task_id.value for t in new_tasks} assert len(killed_tasks) == 3 for task in new_tasks: fake_driver.killTask.assert_any_call(task.task_id) # Now tell the scheduler those tasks have died. for task in new_tasks: scheduler.statusUpdate(fake_driver, mock.Mock(task_id=task.task_id, state=TASK_KILLED)) # Clean up the TestDrainMethod for the rest of this test. assert not list(scheduler.drain_method.downed_task_ids) # Now scale down old app scheduler.service_config.config_dict['instances'] = 2 scheduler.kill_tasks_if_necessary(fake_driver) assert len(scheduler.drain_method.downed_task_ids) == 1 # mark it as drained and let the scheduler kill it. scheduler.drain_method.mark_arbitrary_task_as_safe_to_kill() killed_tasks.clear() scheduler.kill_tasks_if_necessary(fake_driver) assert len(killed_tasks) == 1
def test_start_upgrade_rollback_scaledown(self, mock_log, system_paasta_config): service_name = "service_name" instance_name = "instance_name" cluster = "cluster" service_configs = [] for force_bounce in range(2): service_configs.append( NativeServiceConfig( service=service_name, instance=instance_name, cluster=cluster, config_dict={ "cpus": 0.1, "mem": 50, "instances": 3, "cmd": "sleep 50", "drain_method": "test", }, branch_dict={ "docker_image": "busybox", "desired_state": "start", "force_bounce": str(force_bounce), }, soa_dir="/nail/etc/services", ) ) scheduler = native_scheduler.NativeScheduler( service_name=service_name, instance_name=instance_name, cluster=cluster, system_paasta_config=system_paasta_config, service_config=service_configs[0], staging_timeout=1, task_store_type=DictTaskStore, ) fake_driver = mock.Mock() scheduler.registered( driver=fake_driver, frameworkId={"value": "foo"}, masterInfo=mock.Mock() ) with mock.patch( "paasta_tools.utils.load_system_paasta_config", autospec=True, return_value=system_paasta_config, ): # First, start up 3 old tasks old_tasks = scheduler.launch_tasks_for_offers( fake_driver, [make_fake_offer()] ) assert len(scheduler.task_store.get_all_tasks()) == 3 # and mark the old tasks as up for task in old_tasks: scheduler.statusUpdate( fake_driver, dict(task_id=task["task_id"], state=TASK_RUNNING) ) assert len(scheduler.drain_method.downed_task_ids) == 0 # Now, change force_bounce scheduler.service_config = service_configs[1] # and start 3 more tasks new_tasks = scheduler.launch_tasks_for_offers( fake_driver, [make_fake_offer()] ) assert len(scheduler.task_store.get_all_tasks()) == 6 # It should not drain anything yet, since the new tasks aren't up. scheduler.kill_tasks_if_necessary(fake_driver) assert len(scheduler.task_store.get_all_tasks()) == 6 assert len(scheduler.drain_method.downed_task_ids) == 0 # Now we mark the new tasks as up. for i, task in enumerate(new_tasks): scheduler.statusUpdate( fake_driver, dict(task_id=task["task_id"], state=TASK_RUNNING) ) # As each of these new tasks come up, we should drain an old one. assert len(scheduler.drain_method.downed_task_ids) == i + 1 # Now let's roll back and make sure it undrains the old ones and drains new. scheduler.service_config = service_configs[0] scheduler.kill_tasks_if_necessary(fake_driver) assert scheduler.drain_method.downed_task_ids == set() scheduler.kill_tasks_if_necessary(fake_driver) assert scheduler.drain_method.downed_task_ids == { t["task_id"]["value"] for t in new_tasks } # Once we drain the new tasks, it should kill them. assert fake_driver.killTask.call_count == 0 # we issue duplicate kills for tasks until we get notified about TASK_KILLED, so we keep track of # the unique IDs of tasks being killed. killed_tasks = set() def killTask_side_effect(task_id): killed_tasks.add(task_id["value"]) fake_driver.killTask.side_effect = killTask_side_effect scheduler.drain_method.mark_arbitrary_task_as_safe_to_kill() scheduler.kill_tasks_if_necessary(fake_driver) assert len(killed_tasks) == 1 scheduler.drain_method.mark_arbitrary_task_as_safe_to_kill() scheduler.kill_tasks_if_necessary(fake_driver) assert len(killed_tasks) == 2 scheduler.drain_method.mark_arbitrary_task_as_safe_to_kill() scheduler.kill_tasks_if_necessary(fake_driver) assert scheduler.drain_method.safe_to_kill_task_ids == { t["task_id"]["value"] for t in new_tasks } assert len(killed_tasks) == 3 for task in new_tasks: fake_driver.killTask.assert_any_call(task["task_id"]) # Now tell the scheduler those tasks have died. for task in new_tasks: scheduler.statusUpdate( fake_driver, dict(task_id=task["task_id"], state=TASK_KILLED) ) # Clean up the TestDrainMethod for the rest of this test. assert not list(scheduler.drain_method.downed_task_ids) # Now scale down old app scheduler.service_config.config_dict["instances"] = 2 scheduler.kill_tasks_if_necessary(fake_driver) assert len(scheduler.drain_method.downed_task_ids) == 1 # mark it as drained and let the scheduler kill it. scheduler.drain_method.mark_arbitrary_task_as_safe_to_kill() killed_tasks.clear() scheduler.kill_tasks_if_necessary(fake_driver) assert len(killed_tasks) == 1