def test_update_indexes(self): metadata = """{ "full_name": "full-name" }""" collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put() collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put() Content(id='bower', parent=collection_version_key, content="""{"dependencies": { "a": "org/element-1#1.0.0", "b": "org/element-2#1.0.0" }}""").put() VersionCache.update(collection_library_key) response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) # Triggers ingestions tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ensure_library_task('org', 'element-1'), util.ensure_library_task('org', 'element-2'), ], [task.url for task in tasks]) # Ensures collection references ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1")) self.assertIsNotNone(ref1) ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2")) self.assertIsNotNone(ref2) # Validate search index index = search.Index('repo') document = index.get('my/collection') self.assertIsNotNone(document) self.assertTrue(len(document.fields) > 0)
def handle_get(self, owner, repo, version): # FIXME: Make deletion transactional with check on library that tag is excluded. version_key = ndb.Key(Library, Library.id(owner, repo), Version, version) ndb.delete_multi(ndb.Query(ancestor=version_key).iter(keys_only=True)) if VersionCache.update(version_key.parent()): task_url = util.update_indexes_task(owner, repo) util.new_task(task_url, target='manage')
def test_analyzer_index_empty(self): metadata = """{ "full_name": "full-name" }""" library_key = Library(id='owner/repo', metadata=metadata).put() version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put() content = Content(id='analysis', parent=version_key, status=Status.pending) data = {"analyzerData": {}} content.json = data content.status = Status.ready content.put() VersionCache.update(library_key) response = self.app.get(util.update_indexes_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) index = search.Index('repo') document = index.get('owner/repo') self.assertIsNotNone(document) self.assertTrue(len(document.fields) > 0) elements = [field for field in document.fields if field.name == 'element'] self.assertEqual(len(elements), 0) behaviors = [field for field in document.fields if field.name == 'behavior'] self.assertEqual(len(behaviors), 0)
def test_update_indexes(self): metadata = """{ "full_name": "full-name" }""" collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put() collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put() Content(id='bower', parent=collection_version_key, content="""{"dependencies": { "a": "org/element-1#1.0.0", "b": "org/element-2#1.0.0" }}""").put() VersionCache.update(collection_library_key) response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) # Triggers ingestions tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ensure_library_task('org', 'element-1'), util.ensure_library_task('org', 'element-2'), ], [task.url for task in tasks]) # Ensures collection references ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1")) self.assertIsNotNone(ref1) ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2")) self.assertIsNotNone(ref2)
def handle_post(self): message_json = json.loads( urllib.unquote(self.request.body).rstrip('=')) message = message_json['message'] data = base64.b64decode(str(message['data'])) attributes = message['attributes'] owner = attributes['owner'] repo = attributes['repo'] version = attributes['version'] error = attributes.get('error', None) version_key = ndb.Key(Library, Library.id(owner, repo), Version, version) content = Content.get_by_id('analysis', parent=version_key) if content is None: return if data == '': content.set_json(None) else: content.set_json(json.loads(data)) if error is None: content.status = Status.ready content.error = None else: content.status = Status.error content.error = error content.put() if version_key.id() == Library.default_version_for_key_async( version_key.parent()).get_result(): task_url = util.update_indexes_task(owner, repo) util.new_task(task_url, target='manage')
def handle_get(self, scope, package, version): # FIXME: Make deletion transactional with check on library that tag is excluded. version_key = ndb.Key(Library, Library.id(scope, package), Version, version) ndb.delete_multi(ndb.Query(ancestor=version_key).iter(keys_only=True)) if VersionCache.update(version_key.parent()): task_url = util.update_indexes_task(scope, package) util.new_task(task_url, target='manage')
def handle_get(self): query = Library.query() cursor = None more = True task_count = 0 while more: keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor) for key in keys: task_count = task_count + 1 owner, repo = key.id().split('/', 1) task_url = util.update_indexes_task(owner, repo) util.new_task(task_url, target='manage') logging.info('triggered %d index updates', task_count)
def handle_get(self): query = Library.query() cursor = None more = True task_count = 0 while more: keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor) for key in keys: task_count = task_count + 1 owner, repo = key.id().split('/', 1) task_url = util.update_indexes_task(owner, repo) util.new_task(task_url, target='manage') logging.info('triggered %d index updates', task_count)
def handle_post(self): # Ignore payloads larger than 5 MB. if len(self.request.body) > 1048487 * 5: return message_json = json.loads(urllib.unquote(self.request.body).rstrip('=')) message = message_json['message'] data = base64.b64decode(str(message['data'])) attributes = message['attributes'] if len(attributes) == 0: logging.error(message) return owner = attributes['owner'] repo = attributes['repo'] version = attributes['version'] error = attributes.get('error', None) version_key = ndb.Key(Library, Library.id(owner, repo), Version, version) content = Content.get_by_id('analysis', parent=version_key) if content is None: return if data == '': content.set_json(None) else: content.set_json(json.loads(data)) if error is None: content.status = Status.ready content.error = None else: content.status = Status.error content.error = error content.put() if version_key.id() == Library.default_version_for_key_async(version_key.parent()).get_result(): task_url = util.update_indexes_task(owner, repo) util.new_task(task_url, target='manage')
def handle_post(self): message_json = json.loads(urllib.unquote(self.request.body).rstrip('=')) message = message_json['message'] data = base64.b64decode(str(message['data'])) attributes = message['attributes'] owner = attributes['owner'] repo = attributes['repo'] version = attributes['version'] error = attributes.get('error', None) version_key = ndb.Key(Library, Library.id(owner, repo), Version, version) content = Content.get_by_id('analysis', parent=version_key) if content is None: return if data == '': content.content = None elif len(data) > 500000: # Max entity size is only 1MB. logging.error('content was too large: %d %s %s', len(data), Library.id(owner, repo), version) error = 'content was too large: %d' % len(data) else: content.content = data if error is None: content.status = Status.ready content.error = None else: content.status = Status.error content.error = error content.put() if version_key.id() == Library.default_version_for_key_async(version_key.parent()).get_result(): task_url = util.update_indexes_task(owner, repo) util.new_task(task_url, target='manage')
def update_versions_and_index(self): if VersionCache.update(self.version_key.parent()): task_url = util.update_indexes_task(self.owner, self.repo) util.new_task(task_url, target='manage', transactional=True)
def update_versions_and_index(self): if VersionCache.update(self.version_key.parent()): task_url = util.update_indexes_task(self.owner, self.repo) util.new_task(task_url, target='manage', transactional=True)