def test_update_indexes(self):
metadata = """{
"full_name": "full-name"
}"""
collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put()
collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put()
Content(id='bower', parent=collection_version_key, content="""{"dependencies": {
"a": "org/element-1#1.0.0",
"b": "org/element-2#1.0.0"
}}""").put()
VersionCache.update(collection_library_key)
response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
# Triggers ingestions
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('org', 'element-1'),
util.ensure_library_task('org', 'element-2'),
], [task.url for task in tasks])
# Ensures collection references
ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1"))
self.assertIsNotNone(ref1)
ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2"))
self.assertIsNotNone(ref2)
# Validate search index
index = search.Index('repo')
document = index.get('my/collection')
self.assertIsNotNone(document)
self.assertTrue(len(document.fields) > 0)
def handle_get(self, owner, repo, version):
# FIXME: Make deletion transactional with check on library that tag is excluded.
version_key = ndb.Key(Library, Library.id(owner, repo), Version, version)
ndb.delete_multi(ndb.Query(ancestor=version_key).iter(keys_only=True))
if VersionCache.update(version_key.parent()):
task_url = util.update_indexes_task(owner, repo)
util.new_task(task_url, target='manage')
def test_analyzer_index_empty(self):
metadata = """{
"full_name": "full-name"
}"""
library_key = Library(id='owner/repo', metadata=metadata).put()
version_key = Version(id='v1.1.1', parent=library_key, sha='sha', status='ready').put()
content = Content(id='analysis', parent=version_key, status=Status.pending)
data = {"analyzerData": {}}
content.json = data
content.status = Status.ready
content.put()
VersionCache.update(library_key)
response = self.app.get(util.update_indexes_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
index = search.Index('repo')
document = index.get('owner/repo')
self.assertIsNotNone(document)
self.assertTrue(len(document.fields) > 0)
elements = [field for field in document.fields if field.name == 'element']
self.assertEqual(len(elements), 0)
behaviors = [field for field in document.fields if field.name == 'behavior']
self.assertEqual(len(behaviors), 0)
def test_update_indexes(self):
metadata = """{
"full_name": "full-name"
}"""
collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put()
collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put()
Content(id='bower', parent=collection_version_key, content="""{"dependencies": {
"a": "org/element-1#1.0.0",
"b": "org/element-2#1.0.0"
}}""").put()
VersionCache.update(collection_library_key)
response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
# Triggers ingestions
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('org', 'element-1'),
util.ensure_library_task('org', 'element-2'),
], [task.url for task in tasks])
# Ensures collection references
ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1"))
self.assertIsNotNone(ref1)
ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2"))
self.assertIsNotNone(ref2)
def handle_post(self):
message_json = json.loads(
urllib.unquote(self.request.body).rstrip('='))
message = message_json['message']
data = base64.b64decode(str(message['data']))
attributes = message['attributes']
owner = attributes['owner']
repo = attributes['repo']
version = attributes['version']
error = attributes.get('error', None)
version_key = ndb.Key(Library, Library.id(owner, repo), Version,
version)
content = Content.get_by_id('analysis', parent=version_key)
if content is None:
return
if data == '':
content.set_json(None)
else:
content.set_json(json.loads(data))
if error is None:
content.status = Status.ready
content.error = None
else:
content.status = Status.error
content.error = error
content.put()
if version_key.id() == Library.default_version_for_key_async(
version_key.parent()).get_result():
task_url = util.update_indexes_task(owner, repo)
util.new_task(task_url, target='manage')
def handle_get(self, scope, package, version):
# FIXME: Make deletion transactional with check on library that tag is excluded.
version_key = ndb.Key(Library, Library.id(scope, package), Version, version)
ndb.delete_multi(ndb.Query(ancestor=version_key).iter(keys_only=True))
if VersionCache.update(version_key.parent()):
task_url = util.update_indexes_task(scope, package)
util.new_task(task_url, target='manage')
def handle_get(self):
query = Library.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor)
for key in keys:
task_count = task_count + 1
owner, repo = key.id().split('/', 1)
task_url = util.update_indexes_task(owner, repo)
util.new_task(task_url, target='manage')
logging.info('triggered %d index updates', task_count)
def handle_get(self):
query = Library.query()
cursor = None
more = True
task_count = 0
while more:
keys, cursor, more = query.fetch_page(50, keys_only=True, start_cursor=cursor)
for key in keys:
task_count = task_count + 1
owner, repo = key.id().split('/', 1)
task_url = util.update_indexes_task(owner, repo)
util.new_task(task_url, target='manage')
logging.info('triggered %d index updates', task_count)
def handle_post(self):
# Ignore payloads larger than 5 MB.
if len(self.request.body) > 1048487 * 5:
return
message_json = json.loads(urllib.unquote(self.request.body).rstrip('='))
message = message_json['message']
data = base64.b64decode(str(message['data']))
attributes = message['attributes']
if len(attributes) == 0:
logging.error(message)
return
owner = attributes['owner']
repo = attributes['repo']
version = attributes['version']
error = attributes.get('error', None)
version_key = ndb.Key(Library, Library.id(owner, repo), Version, version)
content = Content.get_by_id('analysis', parent=version_key)
if content is None:
return
if data == '':
content.set_json(None)
else:
content.set_json(json.loads(data))
if error is None:
content.status = Status.ready
content.error = None
else:
content.status = Status.error
content.error = error
content.put()
if version_key.id() == Library.default_version_for_key_async(version_key.parent()).get_result():
task_url = util.update_indexes_task(owner, repo)
util.new_task(task_url, target='manage')
def handle_post(self):
message_json = json.loads(urllib.unquote(self.request.body).rstrip('='))
message = message_json['message']
data = base64.b64decode(str(message['data']))
attributes = message['attributes']
owner = attributes['owner']
repo = attributes['repo']
version = attributes['version']
error = attributes.get('error', None)
version_key = ndb.Key(Library, Library.id(owner, repo), Version, version)
content = Content.get_by_id('analysis', parent=version_key)
if content is None:
return
if data == '':
content.content = None
elif len(data) > 500000:
# Max entity size is only 1MB.
logging.error('content was too large: %d %s %s', len(data), Library.id(owner, repo), version)
error = 'content was too large: %d' % len(data)
else:
content.content = data
if error is None:
content.status = Status.ready
content.error = None
else:
content.status = Status.error
content.error = error
content.put()
if version_key.id() == Library.default_version_for_key_async(version_key.parent()).get_result():
task_url = util.update_indexes_task(owner, repo)
util.new_task(task_url, target='manage')
def update_versions_and_index(self):
if VersionCache.update(self.version_key.parent()):
task_url = util.update_indexes_task(self.owner, self.repo)
util.new_task(task_url, target='manage', transactional=True)
def update_versions_and_index(self):
if VersionCache.update(self.version_key.parent()):
task_url = util.update_indexes_task(self.owner, self.repo)
util.new_task(task_url, target='manage', transactional=True)
