def setUp(self): helpers.patch_environ(self) helpers.patch(self, [ 'base.persistent_cache.get_value', 'base.persistent_cache.set_value', 'base.utils.utcnow', 'time.sleep', ]) self.mock.get_value.return_value = None self.mock.sleep.return_value = None client = pubsub.PubSubClient() topic = pubsub.topic_name('test-clusterfuzz', 'jobs-linux') client.create_topic(topic) client.create_subscription( pubsub.subscription_name('test-clusterfuzz', 'jobs-linux'), topic) topic = pubsub.topic_name('test-clusterfuzz', 'high-end-jobs-linux') client.create_topic(topic) client.create_subscription( pubsub.subscription_name('test-clusterfuzz', 'high-end-jobs-linux'), topic) self.mock.utcnow.return_value = test_utils.CURRENT_TIME.replace( microsecond=0)
def create_pubsub_topic(client, project, name): """Create topic if it doesn't exist.""" full_name = pubsub.topic_name(project, name) if client.get_topic(full_name): return client.create_topic(full_name)
def add_task(command, argument, job_type, queue=None, wait_time=None): """Add a new task to the job queue.""" # Old testcases may pass in queue=None explicitly, # so we must check this here. if not queue: queue = default_queue() if wait_time is None: wait_time = random.randint(1, TASK_CREATION_WAIT_INTERVAL) if job_type != 'none': job = data_types.Job.query(data_types.Job.name == job_type).get() if not job: raise Error(f'Job {job_type} not found.') if job.is_external(): external_tasks.add_external_task(command, argument, job) return # Add the task. eta = utils.utcnow() + datetime.timedelta(seconds=wait_time) task = Task(command, argument, job_type, eta=eta) pubsub_client = pubsub.PubSubClient() pubsub_client.publish(pubsub.topic_name(utils.get_application_id(), queue), [task.to_pubsub_message()])
def create_pubsub_subscription(client, project, topic, name): """Create subscription if it doesn't exist.""" topic_name = pubsub.topic_name(project, topic) full_name = pubsub.subscription_name(project, name) if client.get_subscription(full_name): return client.create_subscription(full_name, topic_name)
def setUp(self): helpers.patch_environ(self) self.topic = pubsub.topic_name(PROJECT_NAME, 'test-topic') self.subscription = pubsub.subscription_name(PROJECT_NAME, 'subscription') self.client = pubsub.PubSubClient() self.client.create_topic(self.topic) self.client.create_subscription( self.subscription, self.topic, ack_deadline=ACK_DEADLINE)
def setUp(self): helpers.patch_environ(self) helpers.patch(self, [ 'google_cloud_utils.blobs.read_key', ]) self.mock.read_key.return_value = b'reproducer' self.client = pubsub.PubSubClient() self.topic = pubsub.topic_name('proj', 'repro') self.client.create_topic(self.topic) self.subscription = pubsub.subscription_name('proj', 'repro') self.client.create_subscription(self.subscription, self.topic) data_types.Job( name='libfuzzer_asan_blah_external', platform='LINUX', environment_string= ('RELEASE_BUILD_BUCKET_PATH = gs://bucket/a/b/release-([0-9]+).zip\n' 'PROJECT_NAME = proj\n'), external_reproduction_topic=self.topic, external_updates_subscription='projects/proj/subscriptions/updates' ).put() data_types.Job( name='libfuzzer_msan_blah_external', platform='LINUX', environment_string=( 'FUZZ_TARGET_BUILD_BUCKET_PATH = ' 'gs://bucket/a/b/%TARGET%/release-([0-9]+).zip\n' 'PROJECT_NAME = proj\n'), external_reproduction_topic=self.topic, external_updates_subscription='projects/proj/subscriptions/updates' ).put() data_types.FuzzTarget(id='libFuzzer_abc', engine='libFuzzer', binary='abc').put() self.testcase_0 = data_types.Testcase( fuzzer_name='libFuzzer', overridden_fuzzer_name='libFuzzer_abc', crash_revision=1336, minimized_keys='key') self.testcase_0.set_metadata('last_tested_revision', 1337) self.testcase_0.put() self.testcase_1 = data_types.Testcase( fuzzer_name='libFuzzer', overridden_fuzzer_name='libFuzzer_abc', crash_revision=1336, fuzzed_keys='key') self.testcase_1.put()
def create_pubsub_topics(project): """Create pubsub topics for tasks.""" for platform in PUBSUB_PLATFORMS: name = untrusted.queue_name(project, platform) client = pubsub.PubSubClient() application_id = app_identity.get_application_id() topic_name = pubsub.topic_name(application_id, name) if client.get_topic(topic_name) is None: client.create_topic(topic_name) subscription_name = pubsub.subscription_name(application_id, name) if client.get_subscription(subscription_name) is None: client.create_subscription(subscription_name, topic_name)
def add_task(command, argument, job_type, queue=None, wait_time=None): """Add a new task to the job queue.""" # Old testcases may pass in queue=None explicitly, # so we must check this here. if not queue: queue = default_queue() if wait_time is None: wait_time = random.randint(1, TASK_CREATION_WAIT_INTERVAL) # Add the task. eta = utils.utcnow() + datetime.timedelta(seconds=wait_time) task = Task(command, argument, job_type, eta=eta) pubsub_client = pubsub.PubSubClient() pubsub_client.publish(pubsub.topic_name(utils.get_application_id(), queue), [task.to_pubsub_message()])
def test_list_topics(self): """Test listing topics.""" expected = [] for i in xrange(5): topic = pubsub.topic_name(PROJECT_NAME, 'topic-{}'.format(i)) expected.append(topic) self.client.create_topic(topic) expected.append('projects/fake-project/topics/test-topic') topics = list(self.client.list_topics('projects/' + PROJECT_NAME)) self.assertItemsEqual(expected, topics) # Note: Page size appears to be ignored by the emulator. Even when creating # large amounts of topics to force paging, the nextPageToken returned is # buggy and results in infinite loops. topics = list( self.client.list_topics('projects/' + PROJECT_NAME, page_size=1)) self.assertItemsEqual(expected, topics)
def test_execute(self): """Tests executing of cron job.""" mock_storage = mock.MagicMock() mock_storage.buckets().insert().execute.return_value = 'timeCreated' self.mock.get_application_id_1.return_value = 'clusterfuzz-external' self.mock.get_application_id_2.return_value = 'clusterfuzz-external' self.mock.build.return_value = mock_storage pubsub_client = pubsub.PubSubClient() unmanaged_topic_name = pubsub.topic_name(app_identity.get_application_id(), 'jobs-linux') old_topic_name = pubsub.topic_name(app_identity.get_application_id(), 'jobs-shouldbedeleted') old_subscription_name = pubsub.subscription_name( app_identity.get_application_id(), 'jobs-shouldbedeleted') other_topic_name = pubsub.topic_name(app_identity.get_application_id(), 'other') pubsub_client.create_topic(unmanaged_topic_name) pubsub_client.create_topic(old_topic_name) pubsub_client.create_topic(other_topic_name) pubsub_client.create_subscription(old_subscription_name, old_topic_name) self.mock.get_projects.return_value = [ ('lib1', { 'homepage': 'http://example.com', 'primary_contact': '*****@*****.**', 'auto_ccs': [ '*****@*****.**', '*****@*****.**', ], }), ('lib2', { 'homepage': 'http://example2.com', 'disabled': True, 'fuzzing_engines': ['libfuzzer',], }), ('lib3', { 'homepage': 'http://example3.com', 'sanitizers': [ 'address', { 'memory': { 'experimental': True, }, }, 'undefined', ], 'auto_ccs': '*****@*****.**', 'disabled': False, 'fuzzing_engines': ['libfuzzer',], 'view_restrictions': 'none', }), ('lib4', { 'homepage': 'http://example4.com', 'sanitizers': ['address'], 'auto_ccs': '*****@*****.**', 'fuzzing_engines': ['none',], }), ('lib5', { 'homepage': 'http://example5.com', 'sanitizers': ['address'], 'fuzzing_engines': ['libfuzzer',], 'experimental': True, 'selective_unpack': True, }), ] mock_storage.buckets().get.side_effect = mock_bucket_get mock_storage.buckets().getIamPolicy.side_effect = mock_get_iam_policy mock_storage.buckets().setIamPolicy = CopyingMock() mock_storage.buckets().setIamPolicy.side_effect = mock_set_iam_policy self.app.get('/setup') job = data_types.Job.query( data_types.Job.name == 'libfuzzer_asan_lib1').get() self.assertIsNotNone(job) self.assertEqual(job.platform, 'LIB1_LINUX') self.assertItemsEqual(job.templates, ['asan', 'libfuzzer']) self.assertEqual( job.environment_string, 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-builds/lib1/lib1-address-([0-9]+).zip\n' 'FUZZ_LOGS_BUCKET = lib1-logs.clusterfuzz-external.appspot.com\n' 'CORPUS_BUCKET = lib1-corpus.clusterfuzz-external.appspot.com\n' 'QUARANTINE_BUCKET = lib1-quarantine.clusterfuzz-external.appspot.com\n' 'BACKUP_BUCKET = lib1-backup.clusterfuzz-external.appspot.com\n' 'AUTOMATIC_LABELS = Proj-lib1,Engine-libfuzzer\n' 'PROJECT_NAME = lib1\n' 'SUMMARY_PREFIX = lib1\n' 'REVISION_VARS_URL = https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib1/lib1-address-%s.srcmap.json\n' 'MANAGED = True\n') job = data_types.Job.query( data_types.Job.name == 'libfuzzer_asan_lib2').get() self.assertIsNotNone(job) self.assertEqual(job.platform, 'LIB2_LINUX') self.assertItemsEqual(job.templates, ['asan', 'libfuzzer']) self.assertEqual( job.environment_string, 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-builds/lib2/lib2-address-([0-9]+).zip\n' 'FUZZ_LOGS_BUCKET = lib2-logs.clusterfuzz-external.appspot.com\n' 'CORPUS_BUCKET = lib2-corpus.clusterfuzz-external.appspot.com\n' 'QUARANTINE_BUCKET = lib2-quarantine.clusterfuzz-external.appspot.com\n' 'BACKUP_BUCKET = lib2-backup.clusterfuzz-external.appspot.com\n' 'AUTOMATIC_LABELS = Proj-lib2,Engine-libfuzzer\n' 'PROJECT_NAME = lib2\n' 'SUMMARY_PREFIX = lib2\n' 'REVISION_VARS_URL = https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib2/lib2-address-%s.srcmap.json\n' 'MANAGED = True\n') job = data_types.Job.query( data_types.Job.name == 'libfuzzer_asan_lib3').get() self.assertIsNotNone(job) self.assertEqual(job.platform, 'LIB3_LINUX') self.assertItemsEqual(job.templates, ['asan', 'libfuzzer']) self.assertEqual( job.environment_string, 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-builds/lib3/lib3-address-([0-9]+).zip\n' 'FUZZ_LOGS_BUCKET = lib3-logs.clusterfuzz-external.appspot.com\n' 'CORPUS_BUCKET = lib3-corpus.clusterfuzz-external.appspot.com\n' 'QUARANTINE_BUCKET = lib3-quarantine.clusterfuzz-external.appspot.com\n' 'BACKUP_BUCKET = lib3-backup.clusterfuzz-external.appspot.com\n' 'AUTOMATIC_LABELS = Proj-lib3,Engine-libfuzzer\n' 'PROJECT_NAME = lib3\n' 'SUMMARY_PREFIX = lib3\n' 'REVISION_VARS_URL = https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib3/lib3-address-%s.srcmap.json\n' 'MANAGED = True\n' 'ISSUE_VIEW_RESTRICTIONS = none\n') job = data_types.Job.query( data_types.Job.name == 'libfuzzer_msan_lib3').get() self.assertIsNotNone(job) self.assertEqual(job.platform, 'LIB3_LINUX') self.assertItemsEqual(job.templates, ['msan', 'libfuzzer']) self.assertEqual( job.environment_string, 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-builds/lib3/lib3-memory-([0-9]+).zip\n' 'FUZZ_LOGS_BUCKET = lib3-logs.clusterfuzz-external.appspot.com\n' 'CORPUS_BUCKET = lib3-corpus.clusterfuzz-external.appspot.com\n' 'QUARANTINE_BUCKET = lib3-quarantine.clusterfuzz-external.appspot.com\n' 'BACKUP_BUCKET = lib3-backup.clusterfuzz-external.appspot.com\n' 'AUTOMATIC_LABELS = Proj-lib3,Engine-libfuzzer\n' 'PROJECT_NAME = lib3\n' 'SUMMARY_PREFIX = lib3\n' 'REVISION_VARS_URL = https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib3/lib3-memory-%s.srcmap.json\n' 'MANAGED = True\n' 'EXPERIMENTAL = True\n' 'ISSUE_VIEW_RESTRICTIONS = none\n') job = data_types.Job.query( data_types.Job.name == 'libfuzzer_ubsan_lib3').get() self.assertIsNotNone(job) self.assertEqual(job.platform, 'LIB3_LINUX') self.assertItemsEqual(job.templates, ['ubsan', 'libfuzzer']) self.assertEqual( job.environment_string, 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-builds/lib3/lib3-undefined-([0-9]+).zip\n' 'FUZZ_LOGS_BUCKET = lib3-logs.clusterfuzz-external.appspot.com\n' 'CORPUS_BUCKET = lib3-corpus.clusterfuzz-external.appspot.com\n' 'QUARANTINE_BUCKET = lib3-quarantine.clusterfuzz-external.appspot.com\n' 'BACKUP_BUCKET = lib3-backup.clusterfuzz-external.appspot.com\n' 'AUTOMATIC_LABELS = Proj-lib3,Engine-libfuzzer\n' 'PROJECT_NAME = lib3\n' 'SUMMARY_PREFIX = lib3\n' 'REVISION_VARS_URL = https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib3/lib3-undefined-%s.srcmap.json\n' 'MANAGED = True\n' 'ISSUE_VIEW_RESTRICTIONS = none\n') job = data_types.Job.query(data_types.Job.name == 'afl_asan_lib1').get() self.assertIsNotNone(job) self.assertEqual(job.platform, 'LIB1_LINUX') self.assertItemsEqual(job.templates, ['asan', 'afl']) self.assertEqual( job.environment_string, 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-builds-afl/lib1/lib1-address-([0-9]+).zip\n' 'FUZZ_LOGS_BUCKET = lib1-logs.clusterfuzz-external.appspot.com\n' 'CORPUS_BUCKET = lib1-corpus.clusterfuzz-external.appspot.com\n' 'QUARANTINE_BUCKET = lib1-quarantine.clusterfuzz-external.appspot.com\n' 'BACKUP_BUCKET = lib1-backup.clusterfuzz-external.appspot.com\n' 'AUTOMATIC_LABELS = Proj-lib1,Engine-afl\n' 'PROJECT_NAME = lib1\n' 'SUMMARY_PREFIX = lib1\n' 'REVISION_VARS_URL = https://commondatastorage.googleapis.com/' 'clusterfuzz-builds-afl/lib1/lib1-address-%s.srcmap.json\n' 'MANAGED = True\n' 'MINIMIZE_JOB_OVERRIDE = libfuzzer_asan_lib1\n') # Engine-less job. Manually managed. job = data_types.Job.query(data_types.Job.name == 'asan_lib4').get() self.assertIsNone(job) job = data_types.Job.query( data_types.Job.name == 'libfuzzer_asan_lib5').get() self.assertEqual(job.platform, 'LIB5_LINUX') self.assertEqual( job.environment_string, 'RELEASE_BUILD_BUCKET_PATH = ' 'gs://clusterfuzz-builds/lib5/lib5-address-([0-9]+).zip\n' 'FUZZ_LOGS_BUCKET = lib5-logs.clusterfuzz-external.appspot.com\n' 'CORPUS_BUCKET = lib5-corpus.clusterfuzz-external.appspot.com\n' 'QUARANTINE_BUCKET = lib5-quarantine.clusterfuzz-external.appspot.com\n' 'BACKUP_BUCKET = lib5-backup.clusterfuzz-external.appspot.com\n' 'AUTOMATIC_LABELS = Proj-lib5,Engine-libfuzzer\n' 'PROJECT_NAME = lib5\n' 'SUMMARY_PREFIX = lib5\n' 'REVISION_VARS_URL = https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib5/lib5-address-%s.srcmap.json\n' 'MANAGED = True\n' 'EXPERIMENTAL = True\n' 'UNPACK_ALL_FUZZ_TARGETS_AND_FILES = False\n') config = db_config.get() self.maxDiff = None # pylint: disable=invalid-name self.assertItemsEqual(config.revision_vars_url.splitlines(), [ u'libfuzzer_asan_lib2;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib2/lib2-address-%s.srcmap.json', u'libfuzzer_asan_lib3;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib3/lib3-address-%s.srcmap.json', u'libfuzzer_asan_lib1;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib1/lib1-address-%s.srcmap.json', u'libfuzzer_ubsan_lib1;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib1/lib1-undefined-%s.srcmap.json', u'libfuzzer_ubsan_lib2;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib2/lib2-undefined-%s.srcmap.json', u'afl_asan_lib1;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds-afl/lib1/lib1-address-%s.srcmap.json', u'blah;url2', u'libfuzzer_ubsan_lib3;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib3/lib3-undefined-%s.srcmap.json', u'libfuzzer_msan_lib3;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib3/lib3-memory-%s.srcmap.json', u'asan_lib4;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds-no-engine/lib4/lib4-address-%s.srcmap.json', u'libfuzzer_asan_lib5;https://commondatastorage.googleapis.com/' 'clusterfuzz-builds/lib5/lib5-address-%s.srcmap.json', ]) libfuzzer = data_types.Fuzzer.query( data_types.Fuzzer.name == 'libFuzzer').get() self.assertItemsEqual(libfuzzer.jobs, [ 'libfuzzer_asan_lib1', 'libfuzzer_asan_lib3', 'libfuzzer_asan_lib5', 'libfuzzer_msan_lib3', 'libfuzzer_ubsan_lib1', 'libfuzzer_ubsan_lib3', ]) afl = data_types.Fuzzer.query(data_types.Fuzzer.name == 'afl').get() self.assertItemsEqual(afl.jobs, [ 'afl_asan_lib1', ]) # Test that old unused jobs are deleted. self.assertIsNone( data_types.Job.query( data_types.Job.name == 'libfuzzer_asan_old_job').get()) self.assertIsNone( data_types.Job.query( data_types.Job.name == 'libfuzzer_msan_old_job').get()) # Unmanaged job should still exist. self.assertIsNotNone( data_types.Job.query(data_types.Job.name == 'unmanaged_job').get()) # Test that project settings are created. lib1_settings = ndb.Key(data_types.OssFuzzProject, 'lib1').get() self.assertIsNotNone(lib1_settings) self.assertDictEqual({ 'cpu_weight': 1.5, 'name': 'lib1', 'disk_size_gb': None, 'service_account': '*****@*****.**', 'high_end': False, 'ccs': [ '*****@*****.**', '*****@*****.**', '*****@*****.**' ], }, lib1_settings.to_dict()) lib2_settings = ndb.Key(data_types.OssFuzzProject, 'lib2').get() self.assertIsNone(lib2_settings) lib3_settings = ndb.Key(data_types.OssFuzzProject, 'lib3').get() self.assertIsNotNone(lib3_settings) self.assertDictEqual({ 'cpu_weight': 1.0, 'name': 'lib3', 'disk_size_gb': None, 'service_account': '*****@*****.**', 'high_end': False, 'ccs': ['*****@*****.**'], }, lib3_settings.to_dict()) lib4_settings = ndb.Key(data_types.OssFuzzProject, 'lib4').get() self.assertIsNotNone(lib4_settings) self.assertDictEqual({ 'cpu_weight': 1.0, 'name': 'lib4', 'disk_size_gb': None, 'service_account': '*****@*****.**', 'high_end': True, 'ccs': ['*****@*****.**'], }, lib4_settings.to_dict()) old_lib_settings = ndb.Key(data_types.OssFuzzProject, 'old_lib').get() self.assertIsNone(old_lib_settings) mock_storage.buckets().get.assert_has_calls([ mock.call(bucket='lib1-backup.clusterfuzz-external.appspot.com'), mock.call(bucket='lib1-corpus.clusterfuzz-external.appspot.com'), mock.call(bucket='lib1-quarantine.clusterfuzz-external.appspot.com'), mock.call(bucket='lib1-logs.clusterfuzz-external.appspot.com'), mock.call(bucket='lib2-backup.clusterfuzz-external.appspot.com'), mock.call(bucket='lib2-corpus.clusterfuzz-external.appspot.com'), mock.call(bucket='lib2-quarantine.clusterfuzz-external.appspot.com'), mock.call(bucket='lib2-logs.clusterfuzz-external.appspot.com'), mock.call(bucket='lib3-backup.clusterfuzz-external.appspot.com'), mock.call(bucket='lib3-corpus.clusterfuzz-external.appspot.com'), mock.call(bucket='lib3-quarantine.clusterfuzz-external.appspot.com'), mock.call(bucket='lib3-logs.clusterfuzz-external.appspot.com'), ]) mock_storage.buckets().insert.assert_has_calls([ mock.call( body={ 'name': 'lib1-backup.clusterfuzz-external.appspot.com', 'lifecycle': { 'rule': [{ 'action': { 'type': 'Delete' }, 'condition': { 'age': 100 } }] } }, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={'name': 'lib1-corpus.clusterfuzz-external.appspot.com'}, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={ 'name': 'lib1-quarantine.clusterfuzz-external.appspot.com', 'lifecycle': { 'rule': [{ 'action': { 'type': 'Delete' }, 'condition': { 'age': 90 } }] } }, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={ 'name': 'lib2-backup.clusterfuzz-external.appspot.com', 'lifecycle': { 'rule': [{ 'action': { 'type': 'Delete' }, 'condition': { 'age': 100 } }] } }, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={'name': 'lib2-corpus.clusterfuzz-external.appspot.com'}, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={ 'name': 'lib2-quarantine.clusterfuzz-external.appspot.com', 'lifecycle': { 'rule': [{ 'action': { 'type': 'Delete' }, 'condition': { 'age': 90 } }] } }, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={ 'name': 'lib2-logs.clusterfuzz-external.appspot.com', 'lifecycle': { 'rule': [{ 'action': { 'type': 'Delete' }, 'condition': { 'age': 14 } }] } }, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={ 'name': 'lib3-backup.clusterfuzz-external.appspot.com', 'lifecycle': { 'rule': [{ 'action': { 'type': 'Delete' }, 'condition': { 'age': 100 } }] } }, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={'name': 'lib3-corpus.clusterfuzz-external.appspot.com'}, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={ 'name': 'lib3-quarantine.clusterfuzz-external.appspot.com', 'lifecycle': { 'rule': [{ 'action': { 'type': 'Delete' }, 'condition': { 'age': 90 } }] } }, project='clusterfuzz-external'), mock.call().execute(), mock.call( body={ 'name': 'lib3-logs.clusterfuzz-external.appspot.com', 'lifecycle': { 'rule': [{ 'action': { 'type': 'Delete' }, 'condition': { 'age': 14 } }] } }, project='clusterfuzz-external'), mock.call().execute(), ]) mock_storage.buckets().setIamPolicy.assert_has_calls([ mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }] }, bucket='lib1-backup.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }] }, bucket='lib1-backup.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': [ 'user:[email protected]', 'user:[email protected]' ] }] }, bucket='lib1-backup.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': [ 'user:[email protected]', 'user:[email protected]' ] }, { 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib1-backup.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }] }, bucket='lib1-corpus.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }] }, bucket='lib1-corpus.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': [ 'user:[email protected]', 'user:[email protected]' ] }] }, bucket='lib1-corpus.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': [ 'user:[email protected]', 'user:[email protected]' ] }, { 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib1-corpus.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': [ 'user:[email protected]', 'user:[email protected]' ] }] }, bucket='lib1-logs.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': [ 'user:[email protected]', 'user:[email protected]' ] }] }, bucket='lib1-logs.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': [ 'user:[email protected]', 'user:[email protected]' ] }, { 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib1-logs.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }] }, bucket='lib1-quarantine.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }] }, bucket='lib1-quarantine.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': [ 'user:[email protected]', 'user:[email protected]' ] }] }, bucket='lib1-quarantine.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': [ 'user:[email protected]', 'user:[email protected]' ] }, { 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib1-quarantine.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket='clusterfuzz-external-deployment'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket='test-shared-corpus-bucket'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket='test-mutator-plugins-bucket'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket=u'global-corpus.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib2-backup.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib2-corpus.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib2-logs.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib2-quarantine.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket='clusterfuzz-external-deployment'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket='test-shared-corpus-bucket'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket='test-mutator-plugins-bucket'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket=u'global-corpus.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }] }, bucket='lib3-backup.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }, { 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib3-backup.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }] }, bucket='lib3-corpus.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }, { 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib3-corpus.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }, { 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib3-logs.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }] }, bucket='lib3-quarantine.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['user:[email protected]'] }, { 'role': 'roles/storage.objectAdmin', 'members': ['serviceAccount:[email protected]'] }] }, bucket='lib3-quarantine.clusterfuzz-external.appspot.com'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket='clusterfuzz-external-deployment'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket='test-shared-corpus-bucket'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket='test-mutator-plugins-bucket'), mock.call( body={ 'resourceId': 'fake', 'kind': 'storage#policy', 'etag': 'fake', 'bindings': [{ 'role': 'roles/storage.objectViewer', 'members': ['serviceAccount:[email protected]'] }] }, bucket=u'global-corpus.clusterfuzz-external.appspot.com') ]) mappings = data_types.FuzzerJob.query() tags_fuzzers_and_jobs = [(m.platform, m.fuzzer, m.job) for m in mappings] self.assertItemsEqual(tags_fuzzers_and_jobs, [ ('LIB1_LINUX', 'afl', 'afl_asan_lib1'), ('LIB1_LINUX', 'libFuzzer', 'libfuzzer_asan_lib1'), ('LIB3_LINUX', 'libFuzzer', 'libfuzzer_asan_lib3'), ('LIB3_LINUX', 'libFuzzer', 'libfuzzer_msan_lib3'), ('LIB1_LINUX', 'libFuzzer', 'libfuzzer_ubsan_lib1'), ('LIB3_LINUX', 'libFuzzer', 'libfuzzer_ubsan_lib3'), ('LIB5_LINUX', 'libFuzzer', 'libfuzzer_asan_lib5'), ]) all_permissions = [ entity.to_dict() for entity in data_types.ExternalUserPermission.query() ] self.assertItemsEqual(all_permissions, [{ 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'libfuzzer_asan_lib1', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'libfuzzer_ubsan_lib1', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'libfuzzer_ubsan_lib1', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'libfuzzer_ubsan_lib1', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'libfuzzer_asan_lib1', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'libfuzzer_asan_lib1', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'afl_asan_lib1', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'afl_asan_lib1', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'afl_asan_lib1', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'libfuzzer_msan_lib3', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'libfuzzer_ubsan_lib3', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'libfuzzer_asan_lib3', 'email': u'*****@*****.**' }, { 'entity_kind': 1L, 'is_prefix': False, 'auto_cc': 1L, 'entity_name': u'asan_lib4', 'email': u'*****@*****.**' }])