def test_primes(self):
"""Test a bunch of different distributions."""
projects = [
data_types.OssFuzzProject(name='proj1', cpu_weight=2.0),
data_types.OssFuzzProject(name='proj2', cpu_weight=3.0),
data_types.OssFuzzProject(name='proj3', cpu_weight=5.0),
data_types.OssFuzzProject(name='proj4', cpu_weight=7.0),
data_types.OssFuzzProject(name='proj5', cpu_weight=11.0),
]
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 101)
self.assertListEqual([7, 10, 18, 26, 40], result)
self.assertEqual(101, sum(result))
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 887)
self.assertListEqual([63, 95, 158, 222, 349], result)
self.assertEqual(887, sum(result))
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 2741)
self.assertListEqual([214, 313, 509, 705, 1000], result)
self.assertEqual(2741, sum(result))
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 3571)
self.assertListEqual([356, 483, 738, 994, 1000], result)
self.assertEqual(3571, sum(result))
def test_not_enough(self):
"""Tests allocation with not enough CPUs."""
projects = [
data_types.OssFuzzProject(name='proj1', cpu_weight=1.0),
data_types.OssFuzzProject(name='proj2', cpu_weight=1.0),
data_types.OssFuzzProject(name='proj3', cpu_weight=1.0),
]
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 1)
self.assertListEqual([1, 0, 0], result)
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 2)
self.assertListEqual([1, 1, 0], result)
def test_equal_uneven(self):
"""Tests for each project receiving equal share with an uneven division."""
projects = [
data_types.OssFuzzProject(name='proj1', cpu_weight=1.0),
data_types.OssFuzzProject(name='proj2', cpu_weight=1.0),
data_types.OssFuzzProject(name='proj3', cpu_weight=1.0),
]
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 31)
self.assertListEqual([11, 10, 10], result)
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 32)
self.assertListEqual([11, 11, 10], result)
def test_minimum(self):
"""Tests that projects are given a minimum share."""
projects = [
data_types.OssFuzzProject(name='proj1', cpu_weight=0.0),
data_types.OssFuzzProject(name='proj2', cpu_weight=0.0),
data_types.OssFuzzProject(name='proj3', cpu_weight=0.0),
]
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 3)
self.assertListEqual([1, 1, 1], result)
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 10)
self.assertListEqual([4, 3, 3], result)
def test_weight_preference(self):
"""Tests that remainders are given to projects with higher weights
first.
"""
projects = [
data_types.OssFuzzProject(name='proj1', cpu_weight=1.0),
data_types.OssFuzzProject(name='proj2', cpu_weight=1.01),
data_types.OssFuzzProject(name='proj3', cpu_weight=1.1),
]
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 4)
self.assertListEqual([1, 1, 2], result)
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 5)
self.assertListEqual([1, 2, 2], result)
def test_maximum(self):
"""Tests that projects are capped at the maximum share."""
projects = [
data_types.OssFuzzProject(name='proj1', cpu_weight=1.0),
data_types.OssFuzzProject(name='proj2', cpu_weight=1.0),
data_types.OssFuzzProject(name='proj3', cpu_weight=1.0),
]
result = manage_vms.OssFuzzClustersManager(
'clusterfuzz-external').distribute_cpus(projects, 10000)
self.assertListEqual([1000, 1000, 1000], result)
def test_update_cpus(self):
"""Tests CPU distribution cron."""
self.maxDiff = None # pylint: disable=invalid-name
manager = manage_vms.OssFuzzClustersManager('clusterfuzz-external')
manager.update_clusters()
proj1 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj1').get()
self.assertIsNotNone(proj1)
self.assertDictEqual({
'name':
'proj1',
'clusters': [{
'cluster': 'oss-fuzz-linux-zone2-pre',
'cpu_count': 100,
'gce_zone': 'us-east2-a',
}, {
'cluster': 'oss-fuzz-linux-zone3-worker',
'cpu_count': 1,
'gce_zone': 'us-central1-d',
}],
}, proj1.to_dict())
proj2 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj2').get()
self.assertIsNotNone(proj2)
self.assertDictEqual({
'name':
'proj2',
'clusters': [{
'cluster': 'oss-fuzz-linux-zone2-pre',
'cpu_count': 200,
'gce_zone': 'us-east2-a',
}, {
'cluster': 'oss-fuzz-linux-zone3-worker',
'cpu_count': 4,
'gce_zone': 'us-central1-d',
}],
}, proj2.to_dict())
proj3 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj3').get()
self.assertIsNotNone(proj3)
self.assertDictEqual({
'name':
'proj3',
'clusters': [{
'cluster': 'oss-fuzz-linux-zone2-pre',
'cpu_count': 499,
'gce_zone': 'us-east2-a',
}, {
'cluster': 'oss-fuzz-linux-zone3-worker',
'cpu_count': 9,
'gce_zone': 'us-central1-d',
}],
}, proj3.to_dict())
proj4 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj4').get()
self.assertIsNotNone(proj4)
self.assertDictEqual({
'name':
'proj4',
'clusters': [{
'cluster': 'oss-fuzz-linux-zone2-pre',
'cpu_count': 99,
'gce_zone': 'us-east2-a',
}, {
'cluster': 'oss-fuzz-linux-zone3-worker',
'cpu_count': 1,
'gce_zone': 'us-central1-d',
}],
}, proj4.to_dict())
proj5 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj5').get()
self.assertIsNotNone(proj5)
self.assertDictEqual({
'name':
'proj5',
'clusters': [{
'cluster': 'oss-fuzz-linux-zone2-pre',
'cpu_count': 99,
'gce_zone': 'us-east2-a',
}, {
'cluster': 'oss-fuzz-linux-zone3-worker',
'cpu_count': 1,
'gce_zone': 'us-central1-d',
}],
}, proj5.to_dict())
proj6 = ndb.Key(data_types.OssFuzzProjectInfo, 'proj6').get()
self.assertIsNotNone(proj6)
self.assertDictEqual({
'name':
'proj6',
'clusters': [{
'cluster': 'oss-fuzz-linux-zone3-worker-high-end',
'cpu_count': 2,
'gce_zone': 'us-central1-d',
}],
}, proj6.to_dict())
old_proj = ndb.Key(data_types.OssFuzzProjectInfo, 'old_proj').get()
self.assertIsNone(old_proj)
mock_bot_manager = self.mock.BotManager('clusterfuzz-external',
'us-east2-a')
# proj1: new project.
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-proj1').create.assert_called_with(
expected_instance_template('clusterfuzz-external',
'external-pre-zone2', 'proj1'))
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj1').create.assert_called_with(
'oss-fuzz-linux-zone2-pre-proj1',
'oss-fuzz-linux-zone2-pre-proj1',
size=100,
wait_for_instances=False)
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj1').resize.assert_not_called()
# proj2: already exists. needs a resize. old cluster should be deleted.
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-proj2').create.assert_not_called()
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-proj2').delete.assert_not_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj2').create.assert_not_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj2').delete.assert_not_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj2').resize.assert_called_with(
200, wait_for_instances=False)
mock_bot_manager.instance_template(
'old-cluster-proj2').delete.assert_called()
mock_bot_manager.instance_group('old-cluster-proj2').delete.assert_called()
# proj3: already exists. no changes needed.
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-proj3').delete.assert_not_called()
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-proj3').create.assert_not_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj3').create.assert_not_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj3').resize.assert_not_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj3').delete.assert_not_called()
# proj4: needs a template update (version change).
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-proj4').delete.assert_called()
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-proj4').create.assert_called_with(
expected_instance_template('clusterfuzz-external',
'external-pre-zone2', 'proj4'))
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj4').delete.assert_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj4').create.assert_called_with(
'oss-fuzz-linux-zone2-pre-proj4',
'oss-fuzz-linux-zone2-pre-proj4',
size=99,
wait_for_instances=False)
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj4').resize.assert_not_called()
# proj5: needs a template update (disk size change).
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-proj5').delete.assert_called()
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-proj5').create.assert_called_with(
expected_instance_template(
'clusterfuzz-external',
'external-pre-zone2',
'proj5',
disk_size_gb=10))
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj5').delete.assert_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj5').create.assert_called_with(
'oss-fuzz-linux-zone2-pre-proj5',
'oss-fuzz-linux-zone2-pre-proj5',
size=99,
wait_for_instances=False)
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-proj5').resize.assert_not_called()
# proj6: high end project.
for j in range(1, 6):
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone3-worker-high-end-proj' +
str(j)).create.assert_not_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone3-worker-high-end-proj6').create.assert_called()
# old_proj: deleted.
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-old-proj').create.assert_not_called()
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone2-pre-old-proj').delete.assert_called()
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone2-pre-old-proj').delete.assert_called()
# host instances: created.
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone3-host').create.assert_called_with(
expected_host_instance_template('clusterfuzz-external',
'host-zone3'))
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone3-host').create.assert_called_with(
'oss-fuzz-linux-zone3-host',
'oss-fuzz-linux-zone3-host',
size=2,
wait_for_instances=False)
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone3-host-high-end').create.assert_called_with(
'oss-fuzz-linux-zone3-host-high-end',
'oss-fuzz-linux-zone3-host-high-end',
size=1,
wait_for_instances=False)
# Worker instances: created.
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone3-worker-proj1').create.assert_called_with(
expected_instance_template(
'clusterfuzz-external',
'worker-zone3',
'proj1',
service_account='*****@*****.**',
tls_cert=True))
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone3-worker-proj1').create.assert_called_with(
'oss-fuzz-linux-zone3-worker-proj1',
'oss-fuzz-linux-zone3-worker-proj1',
size=1,
wait_for_instances=False)
mock_bot_manager.instance_template(
'oss-fuzz-linux-zone3-worker-proj2').create.assert_called_with(
expected_instance_template(
'clusterfuzz-external',
'worker-zone3',
'proj2',
service_account='*****@*****.**',
tls_cert=True))
mock_bot_manager.instance_group(
'oss-fuzz-linux-zone3-worker-proj2').create.assert_called_with(
'oss-fuzz-linux-zone3-worker-proj2',
'oss-fuzz-linux-zone3-worker-proj2',
size=4,
wait_for_instances=False)
self.assertItemsEqual([{
'instance_num': 0,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj1-0001',
'project_name': u'proj1',
'host_name': u'oss-fuzz-linux-zone3-host-abcd'
}, {
'instance_num': 1,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj2-0001',
'project_name': u'proj2',
'host_name': u'oss-fuzz-linux-zone3-host-abcd'
}, {
'instance_num': 2,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj2-0002',
'project_name': u'proj2',
'host_name': u'oss-fuzz-linux-zone3-host-abcd'
}, {
'instance_num': 3,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj2-0003',
'project_name': u'proj2',
'host_name': u'oss-fuzz-linux-zone3-host-abcd'
}, {
'instance_num': 4,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj2-0004',
'project_name': u'proj2',
'host_name': u'oss-fuzz-linux-zone3-host-abcd'
}, {
'instance_num': 5,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0001',
'project_name': u'proj3',
'host_name': u'oss-fuzz-linux-zone3-host-abcd'
}, {
'instance_num': 6,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0002',
'project_name': u'proj3',
'host_name': u'oss-fuzz-linux-zone3-host-abcd'
}, {
'instance_num': 7,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0003',
'project_name': u'proj3',
'host_name': u'oss-fuzz-linux-zone3-host-abcd'
}, {
'instance_num': 0,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0004',
'project_name': u'proj3',
'host_name': u'oss-fuzz-linux-zone3-host-efgh'
}, {
'instance_num': 1,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0005',
'project_name': u'proj3',
'host_name': u'oss-fuzz-linux-zone3-host-efgh'
}, {
'instance_num': 2,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0006',
'project_name': u'proj3',
'host_name': u'oss-fuzz-linux-zone3-host-efgh'
}, {
'instance_num': 3,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0007',
'project_name': u'proj3',
'host_name': u'oss-fuzz-linux-zone3-host-efgh'
}, {
'instance_num': 4,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0008',
'project_name': u'proj3',
'host_name': u'oss-fuzz-linux-zone3-host-efgh'
}, {
'instance_num': 5,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj3-0009',
'project_name': u'proj3',
'host_name': u'oss-fuzz-linux-zone3-host-efgh'
}, {
'instance_num': 6,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj4-0001',
'project_name': u'proj4',
'host_name': u'oss-fuzz-linux-zone3-host-efgh'
}, {
'instance_num': 7,
'worker_name': u'oss-fuzz-linux-zone3-worker-proj5-0001',
'project_name': u'proj5',
'host_name': u'oss-fuzz-linux-zone3-host-efgh'
}, {
'instance_num': 0,
'worker_name': u'oss-fuzz-linux-zone3-worker-high-end-proj6-0001',
'project_name': u'proj6',
'host_name': u'oss-fuzz-linux-zone3-host-high-end-1'
}, {
'instance_num': 1,
'worker_name': u'oss-fuzz-linux-zone3-worker-high-end-proj6-0002',
'project_name': u'proj6',
'host_name': u'oss-fuzz-linux-zone3-host-high-end-1'
}], [
assignment.to_dict()
for assignment in data_types.HostWorkerAssignment.query()
])
def test_assign_keep_existing(self):
"""Test that assignment keeps existing assignments."""
host_names = ['host']
worker_instances = [
manage_vms.WorkerInstance(name='worker-proj-0', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-1', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-2', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-3', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-4', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-5', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-6', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-7', project='proj'),
]
data_types.HostWorkerAssignment(
host_name='host',
instance_num=2,
worker_name='worker-proj-6',
project_name='proj',
id='host-2').put()
data_types.HostWorkerAssignment(
host_name='host',
instance_num=3,
worker_name='worker-proj-1',
project_name='proj',
id='host-3').put()
data_types.HostWorkerAssignment(
host_name='host',
instance_num=0,
worker_name='worker-nonexistent-1',
project_name='nonexistent',
id='host-0').put()
manager = manage_vms.OssFuzzClustersManager('clusterfuzz-external')
new_assignments = manager.do_assign_hosts_to_workers(
host_names, worker_instances, 8)
self.assertListEqual([
{
'host_name': u'host',
'instance_num': 0,
'project_name': 'proj',
'worker_name': 'worker-proj-0'
},
{
'host_name': u'host',
'instance_num': 1,
'project_name': 'proj',
'worker_name': 'worker-proj-2'
},
{
'host_name': u'host',
'instance_num': 4,
'project_name': 'proj',
'worker_name': 'worker-proj-3'
},
{
'host_name': u'host',
'instance_num': 5,
'project_name': 'proj',
'worker_name': 'worker-proj-4'
},
{
'host_name': u'host',
'instance_num': 6,
'project_name': 'proj',
'worker_name': 'worker-proj-5'
},
{
'host_name': u'host',
'instance_num': 7,
'project_name': 'proj',
'worker_name': 'worker-proj-7'
},
], [assignment.to_dict() for assignment in new_assignments])
