def get_or_create_host_worker_assignment(self, host_name, instance_num):
"""Get OSS-Fuzz host worker assignment (or create a new one if it doesn't
exist)."""
key_id = '%s-%d' % (host_name, instance_num)
key = ndb.Key(data_types.HostWorkerAssignment, key_id)
assignment = key.get()
if not assignment:
assignment = data_types.HostWorkerAssignment(
host_name=host_name, instance_num=instance_num, id=key_id)
assignment.put()
return assignment
def setUpClass(cls):
logs.configure_for_tests()
os.environ['HOST_INSTANCE_NAME'] = 'host'
os.environ['HOST_INSTANCE_NUM'] = '0'
os.environ['BOT_NAME'] = 'host-0'
os.environ['LOCAL_DEVELOPMENT'] = 'True'
os.environ['SOURCE_VERSION_OVERRIDE'] = 'VERSION'
os.environ['CONFIG_DIR_OVERRIDE'] = os.path.abspath(
os.path.join(os.environ['ROOT_DIR'], 'configs', 'test'))
cert_location = os.path.join(TEST_LIBS_DATA_DIR, 'untrusted_cert.pem')
key_location = os.path.join(TEST_LIBS_DATA_DIR, 'untrusted_key.pem')
os.environ['UNTRUSTED_TLS_CERT_FOR_TESTING'] = cert_location
os.environ['UNTRUSTED_TLS_KEY_FOR_TESTING'] = key_location
cls.bot_proc, bot_root_dir = _create_test_bot()
os.environ['TRUSTED_HOST'] = 'True'
os.environ['WORKER_ROOT_DIR'] = bot_root_dir
os.environ['WORKER_BOT_TMPDIR'] = os.path.join(bot_root_dir,
'bot_tmpdir')
# Explicitly patch datastore.ndb here, as otherwise we patch
# google.appengine.ext.ndb which is not what is imported everywhere.
ndb_patcher.patch_ndb(ndb)
environment.set_default_vars()
data_types.HostWorkerAssignment(host_name='host',
instance_num=0,
worker_name='localhost',
project_name='project',
id='host-0').put()
with open(cert_location) as f:
cert_contents = f.read()
with open(key_location) as f:
key_contents = f.read()
data_types.WorkerTlsCert(project_name='project',
cert_contents=cert_contents,
key_contents=key_contents,
id='project').put()
host.init()
def setUpClass(cls):
cls.saved_env = os.environ.copy()
os.environ['HOST_INSTANCE_NAME'] = 'host'
os.environ['HOST_INSTANCE_NUM'] = '0'
os.environ['BOT_NAME'] = 'host-0'
os.environ['LOCAL_DEVELOPMENT'] = 'True'
os.environ['SOURCE_VERSION_OVERRIDE'] = 'VERSION'
os.environ['CONFIG_DIR_OVERRIDE'] = os.path.abspath(
os.path.join(os.environ['ROOT_DIR'], 'configs', 'test'))
cert_location = os.path.join(TEST_LIBS_DATA_DIR, 'untrusted_cert.pem')
key_location = os.path.join(TEST_LIBS_DATA_DIR, 'untrusted_key.pem')
os.environ['UNTRUSTED_TLS_CERT_FOR_TESTING'] = cert_location
os.environ['UNTRUSTED_TLS_KEY_FOR_TESTING'] = key_location
cls.bot_proc, bot_root_dir = _create_test_bot()
os.environ['TRUSTED_HOST'] = 'True'
os.environ['WORKER_ROOT_DIR'] = bot_root_dir
os.environ['WORKER_BOT_TMPDIR'] = os.path.join(bot_root_dir,
'bot_tmpdir')
environment.set_default_vars()
data_types.HostWorkerAssignment(host_name='host',
instance_num=0,
worker_name='localhost',
project_name='project',
id='host-0').put()
with open(cert_location, 'rb') as f:
cert_contents = f.read()
with open(key_location, 'rb') as f:
key_contents = f.read()
data_types.WorkerTlsCert(project_name='project',
cert_contents=cert_contents,
key_contents=key_contents,
id='project').put()
host.init()
def setUp(self):
test_helpers.patch_environ(self)
test_helpers.patch(self, [
'base.utils.is_oss_fuzz',
'handlers.cron.helpers.bot_manager.BotManager',
'system.environment.is_running_on_app_engine',
'google.appengine.api.app_identity.get_application_id',
'google_cloud_utils.compute_engine_projects.load_project',
])
self.mock.is_oss_fuzz.return_value = True
self.mock.is_running_on_app_engine.return_value = True
self.mock.get_application_id.return_value = 'clusterfuzz-external'
self.mock.load_project.return_value = compute_engine_projects.Project(
project_id='clusterfuzz-external',
clusters=[
compute_engine_projects.Cluster(
name='oss-fuzz-linux-zone2-pre',
gce_zone='us-east2-a',
instance_count=997,
instance_template='external-pre-zone2',
distribute=True,
worker=False,
high_end=False),
compute_engine_projects.Cluster(
name='oss-fuzz-linux-zone3-host',
gce_zone='us-central1-d',
instance_count=2,
instance_template='host-zone3',
distribute=False,
worker=False,
high_end=False),
compute_engine_projects.Cluster(
name='oss-fuzz-linux-zone3-worker',
gce_zone='us-central1-d',
instance_count=16,
instance_template='worker-zone3',
distribute=True,
worker=True,
high_end=False),
compute_engine_projects.Cluster(
name='oss-fuzz-linux-zone3-host-high-end',
gce_zone='us-central1-d',
instance_count=1,
instance_template='host-high-end-zone3',
distribute=False,
worker=False,
high_end=True),
compute_engine_projects.Cluster(
name='oss-fuzz-linux-zone3-worker-high-end',
gce_zone='us-central1-d',
instance_count=2,
instance_template='worker-zone3',
distribute=True,
worker=True,
high_end=True),
],
instance_templates=[
{
'name': 'external-pre-zone2',
'description': '{"version": 1}',
'properties': {
'metadata': {
'items': [],
},
'disks': [{
'initializeParams': {
'diskSizeGb': 30,
},
}],
'serviceAccounts': [{
'email':
'email',
'scopes': [
'https://www.googleapis.com/auth/'
'devstorage.full_control',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/appengine.apis',
'https://www.googleapis.com/auth/prodxmon',
'https://www.googleapis.com/auth/bigquery',
]
}],
}
},
{
'name': 'host-zone3',
'description': '{"version": 1}',
'properties': {
'metadata': {
'items': [],
},
'disks': [{
'initializeParams': {
'diskSizeGb': 30,
},
}],
'serviceAccounts': [{
'email':
'email',
'scopes': [
'https://www.googleapis.com/auth/'
'devstorage.full_control',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/appengine.apis',
'https://www.googleapis.com/auth/prodxmon',
'https://www.googleapis.com/auth/bigquery',
]
}],
}
},
{
'name': 'worker-zone3',
'description': '{"version": 1}',
'properties': {
'metadata': {
'items': [],
},
'disks': [{
'initializeParams': {
'diskSizeGb': 30,
},
}],
'serviceAccounts': [{
'email':
'email',
'scopes': [
'https://www.googleapis.com/auth/'
'devstorage.full_control',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/prodxmon',
]
}],
}
},
{
'name': 'host-high-end-zone3',
'description': '{"version": 1}',
'properties': {
'metadata': {
'items': [],
},
'disks': [{
'initializeParams': {
'diskSizeGb': 100,
},
}],
'serviceAccounts': [{
'email':
'email',
'scopes': [
'https://www.googleapis.com/auth/'
'devstorage.full_control',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/prodxmon',
]
}],
}
},
],
host_worker_assignments=[
compute_engine_projects.HostWorkerAssignment(
host='oss-fuzz-linux-zone3-host',
worker='oss-fuzz-linux-zone3-worker',
workers_per_host=8),
compute_engine_projects.HostWorkerAssignment(
host='oss-fuzz-linux-zone3-host-high-end',
worker='oss-fuzz-linux-zone3-worker-high-end',
workers_per_host=2),
])
data_types.OssFuzzProject(
id='proj1',
name='proj1',
cpu_weight=1.0,
service_account='*****@*****.**').put()
data_types.OssFuzzProject(
id='proj2',
name='proj2',
cpu_weight=2.0,
service_account='*****@*****.**').put()
data_types.OssFuzzProject(
id='proj3',
name='proj3',
cpu_weight=5.0,
service_account='*****@*****.**').put()
data_types.OssFuzzProject(
id='proj4',
name='proj4',
cpu_weight=1.0,
service_account='*****@*****.**').put()
data_types.OssFuzzProject(
id='proj5',
name='proj5',
cpu_weight=1.0,
service_account='*****@*****.**',
disk_size_gb=10).put()
data_types.OssFuzzProject(
id='proj6',
name='proj6',
cpu_weight=1.0,
service_account='*****@*****.**',
high_end=True).put()
for j in range(1, 7):
project_name = 'proj%d' % j
data_types.WorkerTlsCert(
id=project_name,
project_name=project_name,
cert_contents=project_name + '_cert',
key_contents=project_name + '_key').put()
data_types.OssFuzzProjectInfo(id='old_proj', name='old_proj').put()
data_types.OssFuzzProjectInfo(
id='proj2',
name='proj2',
clusters=[
data_types.OssFuzzProjectInfo.ClusterInfo(
cluster='oss-fuzz-linux-zone2-pre',
gce_zone='us-east2-a',
cpu_count=1,
),
data_types.OssFuzzProjectInfo.ClusterInfo(
cluster='old-cluster',
gce_zone='us-east2-a',
cpu_count=1,
),
]).put()
data_types.OssFuzzProjectInfo(
id='proj3',
name='proj3',
clusters=[
data_types.OssFuzzProjectInfo.ClusterInfo(
cluster='oss-fuzz-linux-zone2-pre',
gce_zone='us-east2-a',
cpu_count=499,
)
]).put()
data_types.OssFuzzProjectInfo(
id='proj4',
name='proj4',
clusters=[
data_types.OssFuzzProjectInfo.ClusterInfo(
cluster='oss-fuzz-linux-zone2-pre',
gce_zone='us-east2-a',
cpu_count=99,
)
]).put()
data_types.OssFuzzProjectInfo(
id='proj5',
name='proj5',
clusters=[
data_types.OssFuzzProjectInfo.ClusterInfo(
cluster='oss-fuzz-linux-zone2-pre',
gce_zone='us-east2-a',
cpu_count=99,
)
]).put()
data_types.OssFuzzProjectInfo(
id='old_proj',
name='old_proj',
clusters=[
data_types.OssFuzzProjectInfo.ClusterInfo(
cluster='oss-fuzz-linux-zone2-pre',
gce_zone='us-east2-a',
cpu_count=5,
)
]).put()
data_types.HostWorkerAssignment(
id='old-host-0',
host_name='old-host',
worker_name='worker',
instance_num=0).put()
instance_groups = {}
instance_templates = {}
self.mock.BotManager.side_effect = functools.partial(
MockBotManager,
instance_groups=instance_groups,
instance_templates=instance_templates)
def test_assign_keep_existing(self):
"""Test that assignment keeps existing assignments."""
host_names = ['host']
worker_instances = [
manage_vms.WorkerInstance(name='worker-proj-0', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-1', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-2', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-3', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-4', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-5', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-6', project='proj'),
manage_vms.WorkerInstance(name='worker-proj-7', project='proj'),
]
data_types.HostWorkerAssignment(
host_name='host',
instance_num=2,
worker_name='worker-proj-6',
project_name='proj',
id='host-2').put()
data_types.HostWorkerAssignment(
host_name='host',
instance_num=3,
worker_name='worker-proj-1',
project_name='proj',
id='host-3').put()
data_types.HostWorkerAssignment(
host_name='host',
instance_num=0,
worker_name='worker-nonexistent-1',
project_name='nonexistent',
id='host-0').put()
manager = manage_vms.OssFuzzClustersManager('clusterfuzz-external')
new_assignments = manager.do_assign_hosts_to_workers(
host_names, worker_instances, 8)
self.assertListEqual([
{
'host_name': u'host',
'instance_num': 0,
'project_name': 'proj',
'worker_name': 'worker-proj-0'
},
{
'host_name': u'host',
'instance_num': 1,
'project_name': 'proj',
'worker_name': 'worker-proj-2'
},
{
'host_name': u'host',
'instance_num': 4,
'project_name': 'proj',
'worker_name': 'worker-proj-3'
},
{
'host_name': u'host',
'instance_num': 5,
'project_name': 'proj',
'worker_name': 'worker-proj-4'
},
{
'host_name': u'host',
'instance_num': 6,
'project_name': 'proj',
'worker_name': 'worker-proj-5'
},
{
'host_name': u'host',
'instance_num': 7,
'project_name': 'proj',
'worker_name': 'worker-proj-7'
},
], [assignment.to_dict() for assignment in new_assignments])
