def test_update_indexes(self):
metadata = """{
"full_name": "full-name"
}"""
collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put()
collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put()
Content(id='bower', parent=collection_version_key, content="""{"dependencies": {
"a": "org/element-1#1.0.0",
"b": "org/element-2#1.0.0"
}}""").put()
VersionCache.update(collection_library_key)
response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
# Triggers ingestions
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('org', 'element-1'),
util.ensure_library_task('org', 'element-2'),
], [task.url for task in tasks])
# Ensures collection references
ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1"))
self.assertIsNotNone(ref1)
ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2"))
self.assertIsNotNone(ref2)
# Validate search index
index = search.Index('repo')
document = index.get('my/collection')
self.assertIsNotNone(document)
self.assertTrue(len(document.fields) > 0)
def test_update_indexes(self):
metadata = """{
"full_name": "full-name"
}"""
collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put()
collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put()
Content(id='bower', parent=collection_version_key, content="""{"dependencies": {
"a": "org/element-1#1.0.0",
"b": "org/element-2#1.0.0"
}}""").put()
VersionCache.update(collection_library_key)
response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
# Triggers ingestions
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('org', 'element-1'),
util.ensure_library_task('org', 'element-2'),
], [task.url for task in tasks])
# Ensures collection references
ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1"))
self.assertIsNotNone(ref1)
ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2"))
self.assertIsNotNone(ref2)
def test_renamed_repo_is_renamed(self):
library = Library(id='org/repo',
metadata_etag='a',
contributors_etag='b',
tags_etag='c',
tag_map='{}',
spdx_identifier='MIT')
library.put()
self.respond_to_github(
'https://api.github.com/repos/org/repo',
json.dumps({
"name": "newname",
"owner": {
"login": "******"
},
}))
response = self.app.get('/task/update/org/repo',
headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = library.key.get()
self.assertIsNone(library)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('newowner', 'newname'),
], [task.url for task in tasks])
def test_ensure_when_present(self):
Library(id=Library.id('owner', 'repo')).put()
response = self.app.get(util.ensure_library_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([], [task.url for task in tasks])
def test_ensure_when_present(self):
Library(id=Library.id('owner', 'repo')).put()
response = self.app.get(util.ensure_library_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([], [task.url for task in tasks])
def update_metadata(self):
headers = {'Accept': 'application/vnd.github.drax-preview+json'}
response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers)
if response.status_code == 200:
try:
metadata = json.loads(response.content)
except ValueError:
return self.error("could not parse metadata")
repo = metadata.get('name', '').lower()
owner = metadata.get('owner', {}).get('login', '').lower()
if repo != '' and owner != '' and (repo != self.repo or owner != self.owner):
logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo))
delete_library(self.library.key)
task_url = util.ensure_library_task(owner, repo)
util.new_task(task_url, target='manage')
raise RequestAborted('repo has been renamed to %s', Library.id(owner, repo))
self.library.metadata = response.content
self.library.metadata_etag = response.headers.get('ETag', None)
self.library.metadata_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code == 404:
logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo))
delete_library(self.library.key)
raise RequestAborted('repo no longer exists')
elif response.status_code != 304:
return self.retry('could not update repo metadata (%d)' % response.status_code)
response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag)
if response.status_code == 200:
try:
json.loads(response.content)
except ValueError:
return self.error("could not parse contributors")
self.library.contributors = response.content
self.library.contributors_etag = response.headers.get('ETag', None)
self.library.contributors_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code != 304:
return self.retry('could not update contributors (%d)' % response.status_code)
response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag)
if response.status_code == 200:
try:
json.loads(response.content)
except ValueError:
return self.error("could not parse stats/participation")
self.library.participation = response.content
self.library.participation_etag = response.headers.get('ETag', None)
self.library.participation_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code == 202:
# GitHub is "computing" the data. We'll try again next update cycle.
# TODO: Alternatively we could retry this task
pass
elif response.status_code != 304:
return self.retry('could not update stats/participation (%d)' % response.status_code)
def test_ensure_when_missing(self):
response = self.app.get(util.ensure_library_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_library_task('owner', 'repo'),
], [task.url for task in tasks])
def test_ensure_when_missing(self):
response = self.app.get(util.ensure_library_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ingest_library_task('owner', 'repo'),
], [task.url for task in tasks])
def update_collection_dependencies(self, collection_version_key, bower):
dependencies = bower.get('dependencies', {})
for name in dependencies.keys():
dep = Dependency.from_string(dependencies[name])
if dep is None:
continue
library_key = ndb.Key(Library, Library.id(dep.owner, dep.repo))
CollectionReference.ensure(library_key, collection_version_key, semver=dep.version)
task_url = util.ensure_library_task(dep.owner.lower(), dep.repo.lower())
util.new_task(task_url, target='manage')
def update_collection_dependencies(self, collection_version_key, bower):
dependencies = bower.get('dependencies', {})
for name in dependencies.keys():
dep = Dependency.from_string(dependencies[name])
if dep is None:
continue
library_key = ndb.Key(Library, Library.id(dep.owner, dep.repo))
CollectionReference.ensure(library_key, collection_version_key, semver=dep.version)
task_url = util.ensure_library_task(dep.owner.lower(), dep.repo.lower())
util.new_task(task_url, target='manage')
def test_renamed_repo_is_renamed(self):
library = Library(id='org/repo', metadata_etag='a', contributors_etag='b', tags_etag='c', tag_map='{}', spdx_identifier='MIT')
library.put()
self.respond_to_github('https://api.github.com/repos/org/repo', json.dumps({
"name": "newname",
"owner": {"login": "******"},
}))
response = self.app.get('/task/update/org/repo', headers={'X-AppEngine-QueueName': 'default'})
self.assertEqual(response.status_int, 200)
library = library.key.get()
self.assertIsNone(library)
tasks = self.tasks.get_filtered_tasks()
self.assertEqual([
util.ensure_library_task('newowner', 'newname'),
], [task.url for task in tasks])
def update_metadata(self):
headers = {'Accept': 'application/vnd.github.drax-preview+json'}
response = util.github_get('repos',
self.owner,
self.repo,
etag=self.library.metadata_etag,
headers=headers)
if response.status_code == 200:
try:
metadata = json.loads(response.content)
except ValueError:
return self.error("could not parse metadata",
ErrorCodes.Library_parse_metadata)
repo = metadata.get('name', '').lower()
owner = metadata.get('owner', {}).get('login', '').lower()
if repo != '' and owner != '' and (repo != self.repo
or owner != self.owner):
logging.info('deleting renamed repo %s',
Library.id(self.owner, self.repo))
delete_library(self.library.key)
task_url = util.ensure_library_task(owner, repo)
util.new_task(task_url, target='manage')
raise RequestAborted('repo has been renamed to %s',
Library.id(owner, repo))
self.library.metadata = response.content
self.library.metadata_etag = response.headers.get('ETag', None)
self.library.metadata_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code == 404:
logging.info('deleting non-existing repo %s',
Library.id(self.owner, self.repo))
delete_library(self.library.key)
raise RequestAborted('repo no longer exists')
elif response.status_code != 304:
return self.retry('could not update repo metadata (%d)' %
response.status_code)
response = util.github_get('repos',
self.owner,
self.repo,
'contributors',
etag=self.library.contributors_etag)
if response.status_code == 200:
try:
json.loads(response.content)
except ValueError:
return self.error("could not parse contributors",
ErrorCodes.Library_parse_contributors)
self.library.contributors = response.content
self.library.contributors_etag = response.headers.get('ETag', None)
self.library.contributors_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code != 304:
return self.retry('could not update contributors (%d)' %
response.status_code)
response = util.github_get('repos',
self.owner,
self.repo,
'stats/participation ',
etag=self.library.participation_etag)
if response.status_code == 200:
try:
json.loads(response.content)
except ValueError:
return self.error("could not parse stats/participation",
ErrorCodes.Library_parse_stats)
self.library.participation = response.content
self.library.participation_etag = response.headers.get(
'ETag', None)
self.library.participation_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code == 202:
# GitHub is "computing" the data. We'll try again next update cycle.
# TODO: Alternatively we could retry this task
pass
elif response.status_code != 304:
return self.retry('could not update stats/participation (%d)' %
response.status_code)
def update_metadata(self):
# Query NPM registry API for packages
is_npm_package = self.scope.startswith('@')
if is_npm_package:
self.update_registry_info()
else:
self.owner = self.scope
self.repo = self.package
# Fetch GitHub metadata
headers = {'Accept': 'application/vnd.github.drax-preview+json'}
response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers)
if response.status_code == 200:
try:
metadata = json.loads(response.content)
except ValueError:
return self.error("could not parse metadata", ErrorCodes.Library_parse_metadata)
self.owner = metadata.get('owner', {}).get('login', '').lower()
self.repo = metadata.get('name', '').lower()
# Deleting is only necessary if Library entity is a GitHub repo
if (not is_npm_package) and self.repo != '' and self.owner != '' and (self.repo != self.package or self.owner != self.scope):
logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo))
delete_library(self.library.key)
task_url = util.ensure_library_task(self.owner, self.repo)
util.new_task(task_url, target='manage')
raise RequestAborted('repo has been renamed to %s', Library.id(self.owner, self.repo))
# If adding a NPM package that a Bower repo already points to, remove the bower one.
bower_library_id = Library.id(self.owner, self.repo)
if is_npm_package and bower_library_id is not None:
task_url = util.migrate_library_task(self.owner, self.repo, self.scope, self.package)
util.new_task(task_url, target='manage')
self.library.github_owner = self.owner
self.library.github_repo = self.repo
self.library.metadata = response.content
self.library.metadata_etag = response.headers.get('ETag', None)
self.library.metadata_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code == 404:
logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo))
delete_library(self.library.key)
raise RequestAborted('repo no longer exists')
elif response.status_code != 304:
return self.retry('could not update repo metadata (%d)' % response.status_code)
response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag)
if response.status_code == 200:
try:
json.loads(response.content)
except ValueError:
return self.error("could not parse contributors", ErrorCodes.Library_parse_contributors)
self.library.contributors = response.content
self.library.contributors_etag = response.headers.get('ETag', None)
self.library.contributors_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code != 304:
return self.retry('could not update contributors (%d)' % response.status_code)
response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag)
if response.status_code == 200:
try:
json.loads(response.content)
except ValueError:
return self.error("could not parse stats/participation", ErrorCodes.Library_parse_stats)
self.library.participation = response.content
self.library.participation_etag = response.headers.get('ETag', None)
self.library.participation_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code == 202:
# GitHub is "computing" the data. We'll try again next update cycle.
# TODO: Alternatively we could retry this task
pass
elif response.status_code != 304:
return self.retry('could not update stats/participation (%d)' % response.status_code)
def update_metadata(self):
# Query NPM registry API for packages
is_npm_package = self.scope.startswith('@')
if is_npm_package:
self.update_registry_info()
else:
self.owner = self.scope
self.repo = self.package
# Fetch GitHub metadata
headers = {'Accept': 'application/vnd.github.drax-preview+json'}
response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers)
if response.status_code == 200:
try:
metadata = json.loads(response.content)
except ValueError:
return self.error("could not parse metadata", ErrorCodes.Library_parse_metadata)
self.owner = metadata.get('owner', {}).get('login', '').lower()
self.repo = metadata.get('name', '').lower()
# Deleting is only necessary if Library entity is a GitHub repo
if (not is_npm_package) and self.repo != '' and self.owner != '' and (self.repo != self.package or self.owner != self.scope):
logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo))
delete_library(self.library.key)
task_url = util.ensure_library_task(self.owner, self.repo)
util.new_task(task_url, target='manage')
raise RequestAborted('repo has been renamed to %s', Library.id(self.owner, self.repo))
# If adding a NPM package that a Bower repo already points to, remove the bower one.
bower_library_id = Library.id(self.owner, self.repo)
if is_npm_package and bower_library_id is not None:
logging.info('removing bower repo %s', Library.id(self.owner, self.repo))
task_url = util.suppress_library_task(self.owner, self.repo)
util.new_task(task_url, target='manage')
self.library.github_owner = self.owner
self.library.github_repo = self.repo
self.library.metadata = response.content
self.library.metadata_etag = response.headers.get('ETag', None)
self.library.metadata_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code == 404:
logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo))
delete_library(self.library.key)
raise RequestAborted('repo no longer exists')
elif response.status_code != 304:
return self.retry('could not update repo metadata (%d)' % response.status_code)
response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag)
if response.status_code == 200:
try:
json.loads(response.content)
except ValueError:
return self.error("could not parse contributors", ErrorCodes.Library_parse_contributors)
self.library.contributors = response.content
self.library.contributors_etag = response.headers.get('ETag', None)
self.library.contributors_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code != 304:
return self.retry('could not update contributors (%d)' % response.status_code)
response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag)
if response.status_code == 200:
try:
json.loads(response.content)
except ValueError:
return self.error("could not parse stats/participation", ErrorCodes.Library_parse_stats)
self.library.participation = response.content
self.library.participation_etag = response.headers.get('ETag', None)
self.library.participation_updated = datetime.datetime.now()
self.library_dirty = True
elif response.status_code == 202:
# GitHub is "computing" the data. We'll try again next update cycle.
# TODO: Alternatively we could retry this task
pass
elif response.status_code != 304:
return self.retry('could not update stats/participation (%d)' % response.status_code)
