def test_get_config_all_partially_forbidden(self): self.mock(storage, 'get_config_sets_async', mock.Mock()) storage.get_config_sets_async.return_value = future([ storage.ConfigSet( id='services/x', location='https://x.googlesource.com/x', latest_revision='deadbeef', ), storage.ConfigSet( id='projects/y', location='https://y.googlesource.com/y', latest_revision='badcoffee', ), ]) self.mock(acl, 'can_read_config_sets', mock.Mock(return_value={ 'services/x': True, 'projects/y': False, })) resp = self.call_api('get_config_sets', {}).json_body self.assertEqual(resp, { 'config_sets': [ { 'config_set': 'services/x', 'location': 'https://x.googlesource.com/x', 'revision': { 'id': 'deadbeef', } }, ], })
def test_get_config_all(self): self.mock(storage, 'get_config_sets_async', mock.Mock()) storage.get_config_sets_async.return_value = future([ storage.ConfigSet( id='services/x', location='https://x.googlesource.com/x', latest_revision='deadbeef', latest_revision_url='https://x.googlesource.com/x/+/deadbeef', latest_revision_time=datetime.datetime(2016, 1, 1), latest_revision_committer_email='*****@*****.**', ), storage.ConfigSet( id='projects/y', location='https://y.googlesource.com/y', latest_revision='badcoffee', latest_revision_url='https://y.googlesource.com/y/+/badcoffee', latest_revision_time=datetime.datetime(2016, 1, 2), latest_revision_committer_email='*****@*****.**', ), ]) resp = self.call_api('get_config_sets', {}).json_body storage.get_config_sets_async.assert_called_once_with(config_set=None) self.assertEqual( resp, { 'config_sets': [ { 'config_set': 'services/x', 'location': 'https://x.googlesource.com/x', 'revision': { 'id': 'deadbeef', 'url': 'https://x.googlesource.com/x/+/deadbeef', 'timestamp': '1451606400000000', 'committer_email': '*****@*****.**', }, }, { 'config_set': 'projects/y', 'location': 'https://y.googlesource.com/y', 'revision': { 'id': 'badcoffee', 'url': 'https://y.googlesource.com/y/+/badcoffee', 'timestamp': '1451692800000000', 'committer_email': '*****@*****.**', }, }, ], })
def test_revision_revision_exists(self): self.mock(gitiles, 'get_archive', mock.Mock()) with open(TEST_ARCHIVE_PATH, 'r') as test_archive_file: gitiles.get_archive.return_value = test_archive_file.read() loc = gitiles.Location(hostname='localhost', project='project', treeish='master', path='/') cs = storage.ConfigSet( id='config_set', latest_revision=None, location=str(loc), ) rev = storage.Revision( parent=cs.key, id='deadbeef', ) ndb.put_multi([cs, rev]) gitiles_import._import_revision('config_set', loc, self.test_commit, False) cs_fresh = cs.key.get() self.assertEqual(cs_fresh.latest_revision, self.test_commit.sha)
def put_file(self, config_set, revision, path, content): confg_set_key = storage.ConfigSet(id=config_set, latest_revision=revision).put() rev_key = storage.Revision(id=revision, parent=confg_set_key).put() content_hash = storage.compute_hash(content) storage.File(id=path, parent=rev_key, content_hash=content_hash).put() storage.Blob(id=content_hash, content=content).put()
def test_get_config_with_include_files( self, mock_get_config_sets_async, mock_get_file_keys): mock_get_config_sets_async.return_value = future([ storage.ConfigSet( id='services/x', location='https://x.googlesource.com/x', latest_revision='deadbeef', latest_revision_url='https://x.googlesource.com/x/+/deadbeef', latest_revision_time=datetime.datetime(2016, 1, 1), latest_revision_committer_email='*****@*****.**', ), ]) class Key: def __init__(self, name): self.name = name def id(self): return self.name mock_get_file_keys.return_value = [ Key('README.md'), Key('rick.morty'), Key('pied.piper') ] req = { 'config_set': 'services/x', 'include_files': True, } resp = self.call_api('get_config_sets', req).json_body self.assertEqual(resp, { 'config_sets': [ { 'config_set': 'services/x', 'files': [ { 'path': 'README.md' }, { 'path': 'rick.morty' }, { 'path': 'pied.piper' } ], 'location': 'https://x.googlesource.com/x', 'revision': { 'committer_email': '*****@*****.**', 'id': 'deadbeef', 'timestamp': '1451606400000000', 'url': 'https://x.googlesource.com/x/+/deadbeef' } } ] })
def test_get_mapping_all_partially_forbidden(self): self.mock(storage, 'get_config_sets_async', mock.Mock()) storage.get_config_sets_async.return_value = future([ storage.ConfigSet(id='services/x', location='http://x'), storage.ConfigSet(id='services/y', location='http://y'), ]) self.mock(acl, 'can_read_config_set', mock.Mock(side_effect=[True, False])) resp = self.call_api('get_mapping', {}).json_body self.assertEqual(resp, { 'mappings': [ { 'config_set': 'services/x', 'location': 'http://x', }, ], })
def test_get_mapping_all(self): self.mock(storage, 'get_config_sets_async', mock.Mock()) storage.get_config_sets_async.return_value = future([ storage.ConfigSet(id='services/x', location='https://x'), storage.ConfigSet(id='services/y', location='https://y'), ]) resp = self.call_api('get_mapping', {}).json_body self.assertEqual(resp, { 'mappings': [ { 'config_set': 'services/x', 'location': 'https://x', }, { 'config_set': 'services/y', 'location': 'https://y', }, ], })
def test_deadline_exceeded(self): self.mock_get_log() self.mock(gitiles, 'get_archive', mock.Mock()) gitiles.get_archive.side_effect = urlfetch_errors.DeadlineExceededError storage.ConfigSet( location='https://localhost/project', latest_revision='deadbeef', version=0, id='config_set', ).put() with self.assertRaises(gitiles_import.Error): gitiles_import._import_config_set( 'config_set', gitiles.Location.parse('https://localhost/project')) self.assert_attempt(False, 'Could not import: deadline exceeded')
def put_file(self, config_set, revision, path, content): confg_set_key = storage.ConfigSet( id=config_set, location='https://x.com', latest_revision=revision, latest_revision_url='https://x.com/+/%s' % revision, latest_revision_time=datetime.datetime(2016, 1, 1), latest_revision_committer_email='*****@*****.**', ).put() rev_key = storage.Revision(id=revision, parent=confg_set_key).put() content_hash = storage.compute_hash(content) storage.File(id=path, parent=rev_key, content_hash=content_hash).put() storage.Blob(id=content_hash, content=content).put()
def test_import_config_set_without_force_update(self): self.mock_get_log() storage.ConfigSet( id='config_set', latest_revision='a1841f40264376d170269ee9473ce924b7c2c4e9', latest_revision_url='https://localhost/project/+/deadbeef/x', latest_revision_committer_email=self.john.email, latest_revision_time=self.john.time, location='https://localhost/project/+/master/x', version=2, ).put() self.mock(gitiles_import, '_import_revision', mock.Mock()) gitiles_import._import_config_set( 'config_set', gitiles.Location.parse('https://localhost/project/+/master/x')) self.assertFalse(gitiles_import._import_revision.called)
def test_import_project_ref_not_resolved(self): self.mock(projects, 'get_project', mock.Mock()) projects.get_project.return_value = service_config_pb2.Project( id='chromium', config_location=service_config_pb2.ConfigSetLocation( url='https://localhost/chromium/src/', storage_type=service_config_pb2.ConfigSetLocation.GITILES, ), ) self.mock(gitiles.Location, 'parse_resolve', mock.Mock(side_effect=gitiles.TreeishResolutionError())) storage.ConfigSet(id='projects/chromium', location='https://example.com').put() gitiles_import.import_project('chromium') self.assertIsNone(storage.ConfigSet.get_by_id('projects/chromium'))
def test_import_config_set(self): self.mock_get_log() self.mock_get_archive() storage.ConfigSet( location='https://localhost/project', latest_revision='deadbeef', version=0, id='config_set', ).put() gitiles_import._import_config_set( 'config_set', gitiles.Location.parse('https://localhost/project'), self.test_project_id) gitiles.get_log.assert_called_once_with( 'localhost', 'project', 'HEAD', '/', project_id=self.test_project_id, limit=1, deadline=15) saved_config_set = storage.ConfigSet.get_by_id('config_set') self.assertIsNotNone(saved_config_set) self.assertEqual(saved_config_set.latest_revision, 'a1841f40264376d170269ee9473ce924b7c2c4e9') self.assertTrue( storage.Revision.get_by_id( 'a1841f40264376d170269ee9473ce924b7c2c4e9', parent=saved_config_set.key)) self.assert_attempt(True, 'Imported') # Import second time, import_revision should not be called. self.mock(gitiles_import, '_import_revision', mock.Mock()) gitiles_import._import_config_set( 'config_set', gitiles.Location.parse('https://localhost/project'), self.test_project_id) self.assertFalse(gitiles_import._import_revision.called) self.assert_attempt(True, 'Up-to-date')
def test_get_mapping_one(self): self.mock(storage, 'get_config_sets_async', mock.Mock()) storage.get_config_sets_async.return_value = future([ storage.ConfigSet(id='services/x', location='https://x'), ]) req = { 'config_set': 'services/x', } resp = self.call_api('get_mapping', req).json_body storage.get_config_sets_async.assert_called_once_with( config_set='services/x') self.assertEqual(resp, { 'mappings': [ { 'config_set': 'services/x', 'location': 'https://x', }, ], })
def test_import_existing_config_set_with_log_failed(self): self.mock(gitiles_import, '_import_revision', mock.Mock()) self.mock(gitiles, 'get_log', mock.Mock(return_value=None)) cs = storage.ConfigSet( id='config_set', latest_revision='deadbeef', latest_revision_url='https://localhost/project/+/deadbeef/x', latest_revision_committer_email=self.john.email, latest_revision_time=self.john.time, location='https://localhost/project/+/master/x', ) cs.put() with self.assertRaises(gitiles_import.HistoryDisappeared): gitiles_import._import_config_set( 'config_set', gitiles.Location.parse('https://localhost/project')) self.assertIsNone(storage.last_import_attempt_key('config_set').get()) cs_fresh = cs.key.get() self.assertEqual(cs.latest_revision, cs_fresh.latest_revision)
def import_revision( config_set, base_location, revision, create_config_set=False): """Imports a referenced Gitiles revision into a config set. |base_location| will be used to set storage.ConfigSet.location. If |create_config_set| is True and Revision entity does not exist, then creates ConfigSet with latest_revision set to |location.treeish|. """ assert re.match('[0-9a-f]{40}', revision), ( '"%s" is not a valid sha' % revision ) logging.debug('Importing revision %s @ %s', config_set, revision) rev_key = ndb.Key( storage.ConfigSet, config_set, storage.Revision, revision) updated_config_set = storage.ConfigSet( id=config_set, latest_revision=revision, location=str(base_location)) if rev_key.get(): if create_config_set: updated_config_set.put() return # Fetch archive, extract files and save them to Blobs outside ConfigSet # transaction. location = base_location._replace(treeish=revision) archive = location.get_archive( deadline=get_gitiles_config().fetch_archive_deadline) if not archive: logging.error( 'Could not import %s: configuration does not exist', config_set) return logging.info('%s archive size: %d bytes' % (config_set, len(archive))) entites_to_put = [storage.Revision(key=rev_key)] if create_config_set: entites_to_put.append(updated_config_set) stream = StringIO.StringIO(archive) blob_futures = [] with tarfile.open(mode='r|gz', fileobj=stream) as tar: for item in tar: if not item.isreg(): # pragma: no cover continue with contextlib.closing(tar.extractfile(item)) as extracted: content = extracted.read() ctx = config.validation.Context.logging() validation.validate_config(config_set, item.name, content, ctx=ctx) if ctx.result().has_errors: logging.error('Invalid revision %s@%s', config_set, revision) return content_hash = storage.compute_hash(content) blob_futures.append(storage.import_blob_async( content=content, content_hash=content_hash)) entites_to_put.append( storage.File( id=item.name, parent=rev_key, content_hash=content_hash) ) # Wait for Blobs to be imported before proceeding. ndb.Future.wait_all(blob_futures) @ndb.transactional def do_import(): if not rev_key.get(): ndb.put_multi(entites_to_put) do_import() logging.info('Imported revision %s/%s', config_set, location.treeish)
def _import_revision(config_set, base_location, commit, force_update): """Imports a referenced Gitiles revision into a config set. |base_location| will be used to set storage.ConfigSet.location. Updates last ImportAttempt for the config set. Puts ConfigSet initialized from arguments. """ revision = commit.sha assert re.match('[0-9a-f]{40}', revision), ( '"%s" is not a valid sha' % revision ) rev_key = ndb.Key( storage.ConfigSet, config_set, storage.Revision, revision) location = base_location._replace(treeish=revision) attempt = storage.ImportAttempt( key=storage.last_import_attempt_key(config_set), revision=_commit_to_revision_info(commit, location)) cs_entity = storage.ConfigSet( id=config_set, latest_revision=revision, latest_revision_url=str(location), latest_revision_committer_email=commit.committer.email, latest_revision_time=commit.committer.time, location=str(base_location), version=storage.ConfigSet.CUR_VERSION, ) if not force_update and rev_key.get(): attempt.success = True attempt.message = 'Up-to-date' ndb.put_multi([cs_entity, attempt]) return rev_entities = [cs_entity, storage.Revision(key=rev_key)] # Fetch archive outside ConfigSet transaction. archive = location.get_archive( deadline=get_gitiles_config().fetch_archive_deadline) if not archive: logging.warning( 'Configuration %s does not exist. Probably it was deleted', config_set) attempt.success = True attempt.message = 'Config directory not found. Imported as empty' else: # Extract files and save them to Blobs outside ConfigSet transaction. files, validation_result = _read_and_validate_archive( config_set, rev_key, archive, location) if validation_result.has_errors: logging.warning('Invalid revision %s@%s', config_set, revision) notifications.notify_gitiles_rejection( config_set, location, validation_result) attempt.success = False attempt.message = 'Validation errors' attempt.validation_messages = [ storage.ImportAttempt.ValidationMessage( severity=config.Severity.lookup_by_number(m.severity), text=m.text, ) for m in validation_result.messages ] attempt.put() return rev_entities += files attempt.success = True attempt.message = 'Imported' @ndb.transactional def txn(): if force_update or not rev_key.get(): ndb.put_multi(rev_entities) attempt.put() txn() logging.info('Imported revision %s/%s', config_set, location.treeish)
def test_notify_gitiles_rejection(self): ctx = validation.Context() ctx.error('err') ctx.warning('warn') base = gitiles.Location.parse('https://example.com/x/+/infra/config') new_rev = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' new_loc = base._replace(treeish=new_rev) old_rev = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' old_loc = base._replace(treeish=old_rev) self.mock(notifications, '_send', mock.Mock()) john = gitiles.Contribution('John', '*****@*****.**', datetime.datetime(2015, 1, 1)) commit = gitiles.Commit(sha=new_rev, tree='badcoffee', parents=[], author=john, committer=john, message='New config', tree_diff=None) self.mock(gitiles, 'get_log_async', mock.Mock(return_value=ndb.Future())) gitiles.get_log_async.return_value.set_result( gitiles.Log(commits=[commit], next_cursor=None)) self.mock(template, 'render', mock.Mock()) self.mock(auth, 'list_group', mock.Mock()) auth.list_group.return_value = auth.GroupListing([ auth.Identity('user', '*****@*****.**'), auth.Identity('service', 'foo'), ], [], []) # Notify. notifications.notify_gitiles_rejection('projects/x', new_loc, ctx.result()) self.assertTrue(notifications._send.called) email = notifications._send.call_args[0][0] self.assertEqual( email.sender, 'sample-app.appspot.com <*****@*****.**>') self.assertEqual(email.subject, 'Config revision aaaaaaa is rejected') self.assertEqual(email.to, ['John <*****@*****.**>']) self.assertEqual(email.cc, {'*****@*****.**'}) template.render.assert_called_with( 'templates/validation_notification.html', { 'author': 'John', 'messages': [{ 'severity': 'ERROR', 'text': 'err' }, { 'severity': 'WARNING', 'text': 'warn' }], 'rev_link': new_loc, 'rev_hash': 'aaaaaaa', 'rev_repo': 'x', 'cur_rev_hash': None, 'cur_rev_link': None, }) # Do not send second time. notifications._send.reset_mock() notifications.notify_gitiles_rejection('projects/x', new_loc, ctx.result()) self.assertFalse(notifications._send.called) # Now with config set. ndb.Key(notifications.Notification, str(new_loc)).delete() storage.ConfigSet(id='projects/x', latest_revision=old_rev, latest_revision_url=str(old_loc), location=str(base)).put() template.render.reset_mock() notifications.notify_gitiles_rejection('projects/x', new_loc, ctx.result()) template.render.assert_called_with( 'templates/validation_notification.html', { 'author': 'John', 'messages': [{ 'severity': 'ERROR', 'text': 'err' }, { 'severity': 'WARNING', 'text': 'warn' }], 'rev_link': new_loc, 'rev_hash': 'aaaaaaa', 'rev_repo': 'x', 'cur_rev_hash': 'bbbbbbb', 'cur_rev_link': old_loc, })
def test_get_config_one_with_last_attempt(self): self.mock(storage, 'get_config_sets_async', mock.Mock()) storage.get_config_sets_async.return_value = future([ storage.ConfigSet( id='services/x', location='https://x.googlesource.com/x', latest_revision='deadbeef', latest_revision_url='https://x.googlesource.com/x/+/deadbeef', latest_revision_time=datetime.datetime(2016, 1, 1), latest_revision_committer_email='*****@*****.**', ), ]) storage.ImportAttempt( key=storage.last_import_attempt_key('services/x'), time=datetime.datetime(2016, 1, 2), revision=storage.RevisionInfo( id='badcoffee', url='https://x.googlesource.com/x/+/badcoffee', time=datetime.datetime(2016, 1, 1), committer_email='*****@*****.**', ), success=False, message='Validation errors', validation_messages=[ storage.ImportAttempt.ValidationMessage( severity=config.Severity.ERROR, text='error!', ), storage.ImportAttempt.ValidationMessage( severity=config.Severity.WARNING, text='warning!', ), ], ).put() req = { 'config_set': 'services/x', } resp = self.call_api('get_config_sets', req).json_body storage.get_config_sets_async.assert_called_once_with( config_set='services/x') expected_resp = { 'config_sets': [ { 'config_set': 'services/x', 'location': 'https://x.googlesource.com/x', 'revision': { 'id': 'deadbeef', 'url': 'https://x.googlesource.com/x/+/deadbeef', 'timestamp': '1451606400000000', 'committer_email': '*****@*****.**', }, }, ], } self.assertEqual(resp, expected_resp) req['include_last_import_attempt'] = True resp = self.call_api('get_config_sets', req).json_body expected_resp['config_sets'][0]['last_import_attempt'] = { 'timestamp': '1451692800000000', 'revision': { 'id': 'badcoffee', 'url': 'https://x.googlesource.com/x/+/badcoffee', 'timestamp': '1451606400000000', 'committer_email': '*****@*****.**', }, 'success': False, 'message': 'Validation errors', 'validation_messages': [ { 'severity': 'ERROR', 'text': 'error!', }, { 'severity': 'WARNING', 'text': 'warning!', }, ] } self.assertEqual(resp, expected_resp)