def get_config_sets(self, request): """Returns config sets.""" if request.config_set and not self.can_read_config_set(request.config_set): raise endpoints.ForbiddenException() config_sets = storage.get_config_sets_async( config_set=request.config_set).get_result() if request.include_last_import_attempt: attempts = ndb.get_multi([ storage.last_import_attempt_key(cs.key.id()) for cs in config_sets ]) else: attempts = [None] * len(config_sets) res = self.GetConfigSetsResponseMessage() for cs, attempt in zip(config_sets, attempts): if self.can_read_config_set(cs.key.id()): timestamp = None if cs.latest_revision_time: timestamp = utils.datetime_to_timestamp(cs.latest_revision_time) res.config_sets.append(ConfigSet( config_set=cs.key.id(), location=cs.location, revision=Revision( id=cs.latest_revision, url=cs.latest_revision_url, timestamp=timestamp, committer_email=cs.latest_revision_committer_email, ), last_import_attempt=attempt_to_msg(attempt), )) return res
def save_attempt(success, msg): storage.ImportAttempt( key=storage.last_import_attempt_key(config_set), revision=_commit_to_revision_info(commit, location), success=success, message=msg, ).put()
def save_attempt(success, msg): storage.ImportAttempt( key=storage.last_import_attempt_key(config_set), revision=_commit_to_revision_info(commit, location), success=success, message=msg, ).put()
def get_config_sets(self, request): """Returns config sets.""" if request.config_set and not can_read_config_set(request.config_set): raise endpoints.ForbiddenException() if request.include_files and not request.config_set: raise endpoints.BadRequestException( 'Must specify config_set to use include_files') config_sets = storage.get_config_sets_async( config_set=request.config_set).get_result() # The files property must always be a list of File objects (not None). files = [] if request.include_files: # There must be a single config set because request.config_set is # specified. cs = config_sets[0] if cs.latest_revision: file_keys = storage.get_file_keys(request.config_set, cs.latest_revision) files = [File(path=key.id()) for key in file_keys] if request.include_last_import_attempt: attempts = ndb.get_multi([ storage.last_import_attempt_key(cs.key.id()) for cs in config_sets ]) else: attempts = [None] * len(config_sets) res = self.GetConfigSetsResponseMessage() can_read = can_read_config_sets([cs.key.id() for cs in config_sets]) for cs, attempt in zip(config_sets, attempts): if not can_read[cs.key.id()]: continue if common.REF_CONFIG_SET_RGX.match(cs.key.id()): # Exclude ref configs from the listing for crbug.com/935667 # TODO(crbug.com/924803): remove ref configs altogether. continue cs_msg = ConfigSet( config_set=cs.key.id(), location=cs.location, files=files, last_import_attempt=attempt_to_msg(attempt), ) if cs.latest_revision: cs_msg.revision = Revision( id=cs.latest_revision, url=cs.latest_revision_url, committer_email=cs.latest_revision_committer_email, ) if cs.latest_revision_time: cs_msg.revision.timestamp = utils.datetime_to_timestamp( cs.latest_revision_time) res.config_sets.append(cs_msg) return res
def assert_attempt(self, success, msg, config_set=None, no_revision=False): config_set = config_set or 'config_set' attempt = storage.last_import_attempt_key(config_set).get() self.assertIsNotNone(attempt) if no_revision: self.assertIsNone(attempt.revision) else: self.assertEqual(attempt.revision.id, self.test_commit.sha) self.assertEqual(attempt.revision.time, self.test_commit.committer.time) self.assertEqual( attempt.revision.url, 'https://localhost/project/+/a1841f40264376d170269ee9473ce924b7c2c4e9' ) self.assertEqual(attempt.revision.committer_email, '*****@*****.**') self.assertEqual(attempt.success, success) self.assertEqual(attempt.message, msg) return attempt
def test_import_existing_config_set_with_log_failed(self): self.mock(gitiles_import, '_import_revision', mock.Mock()) self.mock(gitiles, 'get_log', mock.Mock(return_value=None)) cs = storage.ConfigSet( id='config_set', latest_revision='deadbeef', latest_revision_url='https://localhost/project/+/deadbeef/x', latest_revision_committer_email=self.john.email, latest_revision_time=self.john.time, location='https://localhost/project/+/master/x', ) cs.put() with self.assertRaises(gitiles_import.HistoryDisappeared): gitiles_import._import_config_set( 'config_set', gitiles.Location.parse('https://localhost/project')) self.assertIsNone(storage.last_import_attempt_key('config_set').get()) cs_fresh = cs.key.get() self.assertEqual(cs.latest_revision, cs_fresh.latest_revision)
def get_config_sets(self, request): """Returns config sets.""" if request.config_set and not self.can_read_config_set( request.config_set): raise endpoints.ForbiddenException() config_sets = storage.get_config_sets_async( config_set=request.config_set).get_result() if request.include_last_import_attempt: attempts = ndb.get_multi([ storage.last_import_attempt_key(cs.key.id()) for cs in config_sets ]) else: attempts = [None] * len(config_sets) res = self.GetConfigSetsResponseMessage() for cs, attempt in zip(config_sets, attempts): if self.can_read_config_set(cs.key.id()): timestamp = None if cs.latest_revision_time: timestamp = utils.datetime_to_timestamp( cs.latest_revision_time) res.config_sets.append( ConfigSet( config_set=cs.key.id(), location=cs.location, revision=Revision( id=cs.latest_revision, url=cs.latest_revision_url, timestamp=timestamp, committer_email=cs.latest_revision_committer_email, ), last_import_attempt=attempt_to_msg(attempt), )) return res
def _import_revision(config_set, base_location, commit, force_update): """Imports a referenced Gitiles revision into a config set. |base_location| will be used to set storage.ConfigSet.location. Updates last ImportAttempt for the config set. Puts ConfigSet initialized from arguments. """ revision = commit.sha assert re.match('[0-9a-f]{40}', revision), ( '"%s" is not a valid sha' % revision ) rev_key = ndb.Key( storage.ConfigSet, config_set, storage.Revision, revision) location = base_location._replace(treeish=revision) attempt = storage.ImportAttempt( key=storage.last_import_attempt_key(config_set), revision=_commit_to_revision_info(commit, location)) cs_entity = storage.ConfigSet( id=config_set, latest_revision=revision, latest_revision_url=str(location), latest_revision_committer_email=commit.committer.email, latest_revision_time=commit.committer.time, location=str(base_location), version=storage.ConfigSet.CUR_VERSION, ) if not force_update and rev_key.get(): attempt.success = True attempt.message = 'Up-to-date' ndb.put_multi([cs_entity, attempt]) return rev_entities = [cs_entity, storage.Revision(key=rev_key)] # Fetch archive outside ConfigSet transaction. archive = location.get_archive( deadline=get_gitiles_config().fetch_archive_deadline) if not archive: logging.warning( 'Configuration %s does not exist. Probably it was deleted', config_set) attempt.success = True attempt.message = 'Config directory not found. Imported as empty' else: # Extract files and save them to Blobs outside ConfigSet transaction. files, validation_result = _read_and_validate_archive( config_set, rev_key, archive, location) if validation_result.has_errors: logging.warning('Invalid revision %s@%s', config_set, revision) notifications.notify_gitiles_rejection( config_set, location, validation_result) attempt.success = False attempt.message = 'Validation errors' attempt.validation_messages = [ storage.ImportAttempt.ValidationMessage( severity=config.Severity.lookup_by_number(m.severity), text=m.text, ) for m in validation_result.messages ] attempt.put() return rev_entities += files attempt.success = True attempt.message = 'Imported' @ndb.transactional def txn(): if force_update or not rev_key.get(): ndb.put_multi(rev_entities) attempt.put() txn() logging.info('Imported revision %s/%s', config_set, location.treeish)
def test_get_config_one_with_last_attempt(self): self.mock(storage, 'get_config_sets_async', mock.Mock()) storage.get_config_sets_async.return_value = future([ storage.ConfigSet( id='services/x', location='https://x.googlesource.com/x', latest_revision='deadbeef', latest_revision_url='https://x.googlesource.com/x/+/deadbeef', latest_revision_time=datetime.datetime(2016, 1, 1), latest_revision_committer_email='*****@*****.**', ), ]) storage.ImportAttempt( key=storage.last_import_attempt_key('services/x'), time=datetime.datetime(2016, 1, 2), revision=storage.RevisionInfo( id='badcoffee', url='https://x.googlesource.com/x/+/badcoffee', time=datetime.datetime(2016, 1, 1), committer_email='*****@*****.**', ), success=False, message='Validation errors', validation_messages=[ storage.ImportAttempt.ValidationMessage( severity=config.Severity.ERROR, text='error!', ), storage.ImportAttempt.ValidationMessage( severity=config.Severity.WARNING, text='warning!', ), ], ).put() req = { 'config_set': 'services/x', } resp = self.call_api('get_config_sets', req).json_body storage.get_config_sets_async.assert_called_once_with( config_set='services/x') expected_resp = { 'config_sets': [ { 'config_set': 'services/x', 'location': 'https://x.googlesource.com/x', 'revision': { 'id': 'deadbeef', 'url': 'https://x.googlesource.com/x/+/deadbeef', 'timestamp': '1451606400000000', 'committer_email': '*****@*****.**', }, }, ], } self.assertEqual(resp, expected_resp) req['include_last_import_attempt'] = True resp = self.call_api('get_config_sets', req).json_body expected_resp['config_sets'][0]['last_import_attempt'] = { 'timestamp': '1451692800000000', 'revision': { 'id': 'badcoffee', 'url': 'https://x.googlesource.com/x/+/badcoffee', 'timestamp': '1451606400000000', 'committer_email': '*****@*****.**', }, 'success': False, 'message': 'Validation errors', 'validation_messages': [ { 'severity': 'ERROR', 'text': 'error!', }, { 'severity': 'WARNING', 'text': 'warning!', }, ] } self.assertEqual(resp, expected_resp)
def _import_revision(config_set, base_location, commit): """Imports a referenced Gitiles revision into a config set. |base_location| will be used to set storage.ConfigSet.location. Updates last ImportAttempt for the config set. If Revision entity does not exist, then creates ConfigSet initialized from arguments. """ revision = commit.sha assert re.match('[0-9a-f]{40}', revision), ( '"%s" is not a valid sha' % revision ) logging.debug('Importing revision %s @ %s', config_set, revision) rev_key = ndb.Key( storage.ConfigSet, config_set, storage.Revision, revision) location = base_location._replace(treeish=revision) attempt = storage.ImportAttempt( key=storage.last_import_attempt_key(config_set), revision=_commit_to_revision_info(commit, location)) if rev_key.get(): attempt.success = True attempt.message = 'Up-to-date' attempt.put() return rev_entities = [ storage.ConfigSet( id=config_set, latest_revision=revision, latest_revision_url=str(location), latest_revision_committer_email=commit.committer.email, latest_revision_time=commit.committer.time, location=str(base_location), ), storage.Revision(key=rev_key), ] # Fetch archive outside ConfigSet transaction. archive = location.get_archive( deadline=get_gitiles_config().fetch_archive_deadline) if not archive: logging.warning( 'Configuration %s does not exist. Probably it was deleted', config_set) attempt.success = True attempt.message = 'Config directory not found. Imported as empty' else: # Extract files and save them to Blobs outside ConfigSet transaction. files, validation_result = _read_and_validate_archive( config_set, rev_key, archive) if validation_result.has_errors: logging.warning('Invalid revision %s@%s', config_set, revision) notifications.notify_gitiles_rejection( config_set, location, validation_result) attempt.success = False attempt.message = 'Validation errors' attempt.validation_messages = [ storage.ImportAttempt.ValidationMessage( severity=config.Severity.lookup_by_number(m.severity), text=m.text, ) for m in validation_result.messages ] attempt.put() return rev_entities += files attempt.success = True attempt.message = 'Imported' @ndb.transactional def txn(): if not rev_key.get(): ndb.put_multi(rev_entities) attempt.put() txn() logging.info('Imported revision %s/%s', config_set, location.treeish)