def test_primes(self): """Test a bunch of different distributions.""" projects = [ data_types.OssFuzzProject(name='proj1', cpu_weight=2.0), data_types.OssFuzzProject(name='proj2', cpu_weight=3.0), data_types.OssFuzzProject(name='proj3', cpu_weight=5.0), data_types.OssFuzzProject(name='proj4', cpu_weight=7.0), data_types.OssFuzzProject(name='proj5', cpu_weight=11.0), ] result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 101) self.assertListEqual([7, 10, 18, 26, 40], result) self.assertEqual(101, sum(result)) result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 887) self.assertListEqual([63, 95, 158, 222, 349], result) self.assertEqual(887, sum(result)) result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 2741) self.assertListEqual([214, 313, 509, 705, 1000], result) self.assertEqual(2741, sum(result)) result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 3571) self.assertListEqual([356, 483, 738, 994, 1000], result) self.assertEqual(3571, sum(result))
def test_not_enough(self): """Tests allocation with not enough CPUs.""" projects = [ data_types.OssFuzzProject(name='proj1', cpu_weight=1.0), data_types.OssFuzzProject(name='proj2', cpu_weight=1.0), data_types.OssFuzzProject(name='proj3', cpu_weight=1.0), ] result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 1) self.assertListEqual([1, 0, 0], result) result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 2) self.assertListEqual([1, 1, 0], result)
def test_equal_uneven(self): """Tests for each project receiving equal share with an uneven division.""" projects = [ data_types.OssFuzzProject(name='proj1', cpu_weight=1.0), data_types.OssFuzzProject(name='proj2', cpu_weight=1.0), data_types.OssFuzzProject(name='proj3', cpu_weight=1.0), ] result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 31) self.assertListEqual([11, 10, 10], result) result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 32) self.assertListEqual([11, 11, 10], result)
def test_minimum(self): """Tests that projects are given a minimum share.""" projects = [ data_types.OssFuzzProject(name='proj1', cpu_weight=0.0), data_types.OssFuzzProject(name='proj2', cpu_weight=0.0), data_types.OssFuzzProject(name='proj3', cpu_weight=0.0), ] result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 3) self.assertListEqual([1, 1, 1], result) result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 10) self.assertListEqual([4, 3, 3], result)
def test_weight_preference(self): """Tests that remainders are given to projects with higher weights first. """ projects = [ data_types.OssFuzzProject(name='proj1', cpu_weight=1.0), data_types.OssFuzzProject(name='proj2', cpu_weight=1.01), data_types.OssFuzzProject(name='proj3', cpu_weight=1.1), ] result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 4) self.assertListEqual([1, 1, 2], result) result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 5) self.assertListEqual([1, 2, 2], result)
def test_maximum(self): """Tests that projects are capped at the maximum share.""" projects = [ data_types.OssFuzzProject(name='proj1', cpu_weight=1.0), data_types.OssFuzzProject(name='proj2', cpu_weight=1.0), data_types.OssFuzzProject(name='proj3', cpu_weight=1.0), ] result = manage_vms.OssFuzzClustersManager( 'clusterfuzz-external').distribute_cpus(projects, 10000) self.assertListEqual([1000, 1000, 1000], result)
def test_update_cpus(self): """Tests CPU distribution cron.""" self.maxDiff = None # pylint: disable=invalid-name manager = manage_vms.OssFuzzClustersManager('clusterfuzz-external') manager.update_clusters() proj1 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj1').get() self.assertIsNotNone(proj1) self.assertDictEqual({ 'name': 'proj1', 'clusters': [{ 'cluster': 'oss-fuzz-linux-zone2-pre', 'cpu_count': 100, 'gce_zone': 'us-east2-a', }, { 'cluster': 'oss-fuzz-linux-zone3-worker', 'cpu_count': 1, 'gce_zone': 'us-central1-d', }], }, proj1.to_dict()) proj2 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj2').get() self.assertIsNotNone(proj2) self.assertDictEqual({ 'name': 'proj2', 'clusters': [{ 'cluster': 'oss-fuzz-linux-zone2-pre', 'cpu_count': 200, 'gce_zone': 'us-east2-a', }, { 'cluster': 'oss-fuzz-linux-zone3-worker', 'cpu_count': 4, 'gce_zone': 'us-central1-d', }], }, proj2.to_dict()) proj3 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj3').get() self.assertIsNotNone(proj3) self.assertDictEqual({ 'name': 'proj3', 'clusters': [{ 'cluster': 'oss-fuzz-linux-zone2-pre', 'cpu_count': 499, 'gce_zone': 'us-east2-a', }, { 'cluster': 'oss-fuzz-linux-zone3-worker', 'cpu_count': 9, 'gce_zone': 'us-central1-d', }], }, proj3.to_dict()) proj4 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj4').get() self.assertIsNotNone(proj4) self.assertDictEqual({ 'name': 'proj4', 'clusters': [{ 'cluster': 'oss-fuzz-linux-zone2-pre', 'cpu_count': 99, 'gce_zone': 'us-east2-a', }, { 'cluster': 'oss-fuzz-linux-zone3-worker', 'cpu_count': 1, 'gce_zone': 'us-central1-d', }], }, proj4.to_dict()) proj5 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj5').get() self.assertIsNotNone(proj5) self.assertDictEqual({ 'name': 'proj5', 'clusters': [{ 'cluster': 'oss-fuzz-linux-zone2-pre', 'cpu_count': 99, 'gce_zone': 'us-east2-a', }, { 'cluster': 'oss-fuzz-linux-zone3-worker', 'cpu_count': 1, 'gce_zone': 'us-central1-d', }], }, proj5.to_dict()) proj6 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj6').get() self.assertIsNotNone(proj6) self.assertDictEqual({ 'name': 'proj6', 'clusters': [{ 'cluster': 'oss-fuzz-linux-zone3-worker-high-end', 'cpu_count': 2, 'gce_zone': 'us-central1-d', }], }, proj6.to_dict()) old_proj = ndb.Key(data_types.OssFuzzProjectInfo, 'old_proj').get() self.assertIsNone(old_proj) mock_bot_manager = self.mock.BotManager('clusterfuzz-external', 'us-east2-a') # proj1: new project. mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-proj1').create.assert_called_with( expected_instance_template('clusterfuzz-external', 'external-pre-zone2', 'proj1')) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj1').create.assert_called_with( 'oss-fuzz-linux-zone2-pre-proj1', 'oss-fuzz-linux-zone2-pre-proj1', size=100, wait_for_instances=False) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj1').resize.assert_not_called() # proj2: already exists. needs a resize. old cluster should be deleted. mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-proj2').create.assert_not_called() mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-proj2').delete.assert_not_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj2').create.assert_not_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj2').delete.assert_not_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj2').resize.assert_called_with( 200, wait_for_instances=False) mock_bot_manager.instance_template( 'old-cluster-proj2').delete.assert_called() mock_bot_manager.instance_group('old-cluster-proj2').delete.assert_called() # proj3: already exists. no changes needed. mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-proj3').delete.assert_not_called() mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-proj3').create.assert_not_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj3').create.assert_not_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj3').resize.assert_not_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj3').delete.assert_not_called() # proj4: needs a template update (version change). mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-proj4').delete.assert_called() mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-proj4').create.assert_called_with( expected_instance_template('clusterfuzz-external', 'external-pre-zone2', 'proj4')) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj4').delete.assert_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj4').create.assert_called_with( 'oss-fuzz-linux-zone2-pre-proj4', 'oss-fuzz-linux-zone2-pre-proj4', size=99, wait_for_instances=False) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj4').resize.assert_not_called() # proj5: needs a template update (disk size change). mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-proj5').delete.assert_called() mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-proj5').create.assert_called_with( expected_instance_template( 'clusterfuzz-external', 'external-pre-zone2', 'proj5', disk_size_gb=10)) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj5').delete.assert_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj5').create.assert_called_with( 'oss-fuzz-linux-zone2-pre-proj5', 'oss-fuzz-linux-zone2-pre-proj5', size=99, wait_for_instances=False) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-proj5').resize.assert_not_called() # proj6: high end project. for j in range(1, 6): mock_bot_manager.instance_group( 'oss-fuzz-linux-zone3-worker-high-end-proj' + str(j)).create.assert_not_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone3-worker-high-end-proj6').create.assert_called() # old_proj: deleted. mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-old-proj').create.assert_not_called() mock_bot_manager.instance_template( 'oss-fuzz-linux-zone2-pre-old-proj').delete.assert_called() mock_bot_manager.instance_group( 'oss-fuzz-linux-zone2-pre-old-proj').delete.assert_called() # host instances: created. mock_bot_manager.instance_template( 'oss-fuzz-linux-zone3-host').create.assert_called_with( expected_host_instance_template('clusterfuzz-external', 'host-zone3')) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone3-host').create.assert_called_with( 'oss-fuzz-linux-zone3-host', 'oss-fuzz-linux-zone3-host', size=2, wait_for_instances=False) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone3-host-high-end').create.assert_called_with( 'oss-fuzz-linux-zone3-host-high-end', 'oss-fuzz-linux-zone3-host-high-end', size=1, wait_for_instances=False) # Worker instances: created. mock_bot_manager.instance_template( 'oss-fuzz-linux-zone3-worker-proj1').create.assert_called_with( expected_instance_template( 'clusterfuzz-external', 'worker-zone3', 'proj1', service_account='*****@*****.**', tls_cert=True)) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone3-worker-proj1').create.assert_called_with( 'oss-fuzz-linux-zone3-worker-proj1', 'oss-fuzz-linux-zone3-worker-proj1', size=1, wait_for_instances=False) mock_bot_manager.instance_template( 'oss-fuzz-linux-zone3-worker-proj2').create.assert_called_with( expected_instance_template( 'clusterfuzz-external', 'worker-zone3', 'proj2', service_account='*****@*****.**', tls_cert=True)) mock_bot_manager.instance_group( 'oss-fuzz-linux-zone3-worker-proj2').create.assert_called_with( 'oss-fuzz-linux-zone3-worker-proj2', 'oss-fuzz-linux-zone3-worker-proj2', size=4, wait_for_instances=False) self.assertItemsEqual([{ 'instance_num': 0, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj1-0001', 'project_name': u'proj1', 'host_name': u'oss-fuzz-linux-zone3-host-abcd' }, { 'instance_num': 1, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj2-0001', 'project_name': u'proj2', 'host_name': u'oss-fuzz-linux-zone3-host-abcd' }, { 'instance_num': 2, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj2-0002', 'project_name': u'proj2', 'host_name': u'oss-fuzz-linux-zone3-host-abcd' }, { 'instance_num': 3, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj2-0003', 'project_name': u'proj2', 'host_name': u'oss-fuzz-linux-zone3-host-abcd' }, { 'instance_num': 4, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj2-0004', 'project_name': u'proj2', 'host_name': u'oss-fuzz-linux-zone3-host-abcd' }, { 'instance_num': 5, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0001', 'project_name': u'proj3', 'host_name': u'oss-fuzz-linux-zone3-host-abcd' }, { 'instance_num': 6, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0002', 'project_name': u'proj3', 'host_name': u'oss-fuzz-linux-zone3-host-abcd' }, { 'instance_num': 7, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0003', 'project_name': u'proj3', 'host_name': u'oss-fuzz-linux-zone3-host-abcd' }, { 'instance_num': 0, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0004', 'project_name': u'proj3', 'host_name': u'oss-fuzz-linux-zone3-host-efgh' }, { 'instance_num': 1, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0005', 'project_name': u'proj3', 'host_name': u'oss-fuzz-linux-zone3-host-efgh' }, { 'instance_num': 2, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0006', 'project_name': u'proj3', 'host_name': u'oss-fuzz-linux-zone3-host-efgh' }, { 'instance_num': 3, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0007', 'project_name': u'proj3', 'host_name': u'oss-fuzz-linux-zone3-host-efgh' }, { 'instance_num': 4, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0008', 'project_name': u'proj3', 'host_name': u'oss-fuzz-linux-zone3-host-efgh' }, { 'instance_num': 5, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0009', 'project_name': u'proj3', 'host_name': u'oss-fuzz-linux-zone3-host-efgh' }, { 'instance_num': 6, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj4-0001', 'project_name': u'proj4', 'host_name': u'oss-fuzz-linux-zone3-host-efgh' }, { 'instance_num': 7, 'worker_name': u'oss-fuzz-linux-zone3-worker-proj5-0001', 'project_name': u'proj5', 'host_name': u'oss-fuzz-linux-zone3-host-efgh' }, { 'instance_num': 0, 'worker_name': u'oss-fuzz-linux-zone3-worker-high-end-proj6-0001', 'project_name': u'proj6', 'host_name': u'oss-fuzz-linux-zone3-host-high-end-1' }, { 'instance_num': 1, 'worker_name': u'oss-fuzz-linux-zone3-worker-high-end-proj6-0002', 'project_name': u'proj6', 'host_name': u'oss-fuzz-linux-zone3-host-high-end-1' }], [ assignment.to_dict() for assignment in data_types.HostWorkerAssignment.query() ])
def test_assign_keep_existing(self): """Test that assignment keeps existing assignments.""" host_names = ['host'] worker_instances = [ manage_vms.WorkerInstance(name='worker-proj-0', project='proj'), manage_vms.WorkerInstance(name='worker-proj-1', project='proj'), manage_vms.WorkerInstance(name='worker-proj-2', project='proj'), manage_vms.WorkerInstance(name='worker-proj-3', project='proj'), manage_vms.WorkerInstance(name='worker-proj-4', project='proj'), manage_vms.WorkerInstance(name='worker-proj-5', project='proj'), manage_vms.WorkerInstance(name='worker-proj-6', project='proj'), manage_vms.WorkerInstance(name='worker-proj-7', project='proj'), ] data_types.HostWorkerAssignment( host_name='host', instance_num=2, worker_name='worker-proj-6', project_name='proj', id='host-2').put() data_types.HostWorkerAssignment( host_name='host', instance_num=3, worker_name='worker-proj-1', project_name='proj', id='host-3').put() data_types.HostWorkerAssignment( host_name='host', instance_num=0, worker_name='worker-nonexistent-1', project_name='nonexistent', id='host-0').put() manager = manage_vms.OssFuzzClustersManager('clusterfuzz-external') new_assignments = manager.do_assign_hosts_to_workers( host_names, worker_instances, 8) self.assertListEqual([ { 'host_name': u'host', 'instance_num': 0, 'project_name': 'proj', 'worker_name': 'worker-proj-0' }, { 'host_name': u'host', 'instance_num': 1, 'project_name': 'proj', 'worker_name': 'worker-proj-2' }, { 'host_name': u'host', 'instance_num': 4, 'project_name': 'proj', 'worker_name': 'worker-proj-3' }, { 'host_name': u'host', 'instance_num': 5, 'project_name': 'proj', 'worker_name': 'worker-proj-4' }, { 'host_name': u'host', 'instance_num': 6, 'project_name': 'proj', 'worker_name': 'worker-proj-5' }, { 'host_name': u'host', 'instance_num': 7, 'project_name': 'proj', 'worker_name': 'worker-proj-7' }, ], [assignment.to_dict() for assignment in new_assignments])