def test_update_indexes(self): metadata = """{ "full_name": "full-name" }""" collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put() collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put() Content(id='bower', parent=collection_version_key, content="""{"dependencies": { "a": "org/element-1#1.0.0", "b": "org/element-2#1.0.0" }}""").put() VersionCache.update(collection_library_key) response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) # Triggers ingestions tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ensure_library_task('org', 'element-1'), util.ensure_library_task('org', 'element-2'), ], [task.url for task in tasks]) # Ensures collection references ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1")) self.assertIsNotNone(ref1) ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2")) self.assertIsNotNone(ref2) # Validate search index index = search.Index('repo') document = index.get('my/collection') self.assertIsNotNone(document) self.assertTrue(len(document.fields) > 0)
def test_update_indexes(self): metadata = """{ "full_name": "full-name" }""" collection_library_key = Library(id='my/collection', status=Status.ready, kind='collection', metadata=metadata).put() collection_version_key = Version(id='v1.0.0', parent=collection_library_key, sha='sha', status=Status.ready).put() Content(id='bower', parent=collection_version_key, content="""{"dependencies": { "a": "org/element-1#1.0.0", "b": "org/element-2#1.0.0" }}""").put() VersionCache.update(collection_library_key) response = self.app.get(util.update_indexes_task('my', 'collection'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) # Triggers ingestions tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ensure_library_task('org', 'element-1'), util.ensure_library_task('org', 'element-2'), ], [task.url for task in tasks]) # Ensures collection references ref1 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-1")) self.assertIsNotNone(ref1) ref2 = CollectionReference.get_by_id(id="my/collection/v1.0.0", parent=ndb.Key(Library, "org/element-2")) self.assertIsNotNone(ref2)
def test_renamed_repo_is_renamed(self): library = Library(id='org/repo', metadata_etag='a', contributors_etag='b', tags_etag='c', tag_map='{}', spdx_identifier='MIT') library.put() self.respond_to_github( 'https://api.github.com/repos/org/repo', json.dumps({ "name": "newname", "owner": { "login": "******" }, })) response = self.app.get('/task/update/org/repo', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = library.key.get() self.assertIsNone(library) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ensure_library_task('newowner', 'newname'), ], [task.url for task in tasks])
def test_ensure_when_present(self): Library(id=Library.id('owner', 'repo')).put() response = self.app.get(util.ensure_library_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([], [task.url for task in tasks])
def update_metadata(self): headers = {'Accept': 'application/vnd.github.drax-preview+json'} response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers) if response.status_code == 200: try: metadata = json.loads(response.content) except ValueError: return self.error("could not parse metadata") repo = metadata.get('name', '').lower() owner = metadata.get('owner', {}).get('login', '').lower() if repo != '' and owner != '' and (repo != self.repo or owner != self.owner): logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo)) delete_library(self.library.key) task_url = util.ensure_library_task(owner, repo) util.new_task(task_url, target='manage') raise RequestAborted('repo has been renamed to %s', Library.id(owner, repo)) self.library.metadata = response.content self.library.metadata_etag = response.headers.get('ETag', None) self.library.metadata_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code == 404: logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo)) delete_library(self.library.key) raise RequestAborted('repo no longer exists') elif response.status_code != 304: return self.retry('could not update repo metadata (%d)' % response.status_code) response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag) if response.status_code == 200: try: json.loads(response.content) except ValueError: return self.error("could not parse contributors") self.library.contributors = response.content self.library.contributors_etag = response.headers.get('ETag', None) self.library.contributors_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code != 304: return self.retry('could not update contributors (%d)' % response.status_code) response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag) if response.status_code == 200: try: json.loads(response.content) except ValueError: return self.error("could not parse stats/participation") self.library.participation = response.content self.library.participation_etag = response.headers.get('ETag', None) self.library.participation_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code == 202: # GitHub is "computing" the data. We'll try again next update cycle. # TODO: Alternatively we could retry this task pass elif response.status_code != 304: return self.retry('could not update stats/participation (%d)' % response.status_code)
def test_ensure_when_missing(self): response = self.app.get(util.ensure_library_task('owner', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_library_task('owner', 'repo'), ], [task.url for task in tasks])
def update_collection_dependencies(self, collection_version_key, bower): dependencies = bower.get('dependencies', {}) for name in dependencies.keys(): dep = Dependency.from_string(dependencies[name]) if dep is None: continue library_key = ndb.Key(Library, Library.id(dep.owner, dep.repo)) CollectionReference.ensure(library_key, collection_version_key, semver=dep.version) task_url = util.ensure_library_task(dep.owner.lower(), dep.repo.lower()) util.new_task(task_url, target='manage')
def test_renamed_repo_is_renamed(self): library = Library(id='org/repo', metadata_etag='a', contributors_etag='b', tags_etag='c', tag_map='{}', spdx_identifier='MIT') library.put() self.respond_to_github('https://api.github.com/repos/org/repo', json.dumps({ "name": "newname", "owner": {"login": "******"}, })) response = self.app.get('/task/update/org/repo', headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = library.key.get() self.assertIsNone(library) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ensure_library_task('newowner', 'newname'), ], [task.url for task in tasks])
def update_metadata(self): headers = {'Accept': 'application/vnd.github.drax-preview+json'} response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers) if response.status_code == 200: try: metadata = json.loads(response.content) except ValueError: return self.error("could not parse metadata", ErrorCodes.Library_parse_metadata) repo = metadata.get('name', '').lower() owner = metadata.get('owner', {}).get('login', '').lower() if repo != '' and owner != '' and (repo != self.repo or owner != self.owner): logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo)) delete_library(self.library.key) task_url = util.ensure_library_task(owner, repo) util.new_task(task_url, target='manage') raise RequestAborted('repo has been renamed to %s', Library.id(owner, repo)) self.library.metadata = response.content self.library.metadata_etag = response.headers.get('ETag', None) self.library.metadata_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code == 404: logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo)) delete_library(self.library.key) raise RequestAborted('repo no longer exists') elif response.status_code != 304: return self.retry('could not update repo metadata (%d)' % response.status_code) response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag) if response.status_code == 200: try: json.loads(response.content) except ValueError: return self.error("could not parse contributors", ErrorCodes.Library_parse_contributors) self.library.contributors = response.content self.library.contributors_etag = response.headers.get('ETag', None) self.library.contributors_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code != 304: return self.retry('could not update contributors (%d)' % response.status_code) response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag) if response.status_code == 200: try: json.loads(response.content) except ValueError: return self.error("could not parse stats/participation", ErrorCodes.Library_parse_stats) self.library.participation = response.content self.library.participation_etag = response.headers.get( 'ETag', None) self.library.participation_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code == 202: # GitHub is "computing" the data. We'll try again next update cycle. # TODO: Alternatively we could retry this task pass elif response.status_code != 304: return self.retry('could not update stats/participation (%d)' % response.status_code)
def update_metadata(self): # Query NPM registry API for packages is_npm_package = self.scope.startswith('@') if is_npm_package: self.update_registry_info() else: self.owner = self.scope self.repo = self.package # Fetch GitHub metadata headers = {'Accept': 'application/vnd.github.drax-preview+json'} response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers) if response.status_code == 200: try: metadata = json.loads(response.content) except ValueError: return self.error("could not parse metadata", ErrorCodes.Library_parse_metadata) self.owner = metadata.get('owner', {}).get('login', '').lower() self.repo = metadata.get('name', '').lower() # Deleting is only necessary if Library entity is a GitHub repo if (not is_npm_package) and self.repo != '' and self.owner != '' and (self.repo != self.package or self.owner != self.scope): logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo)) delete_library(self.library.key) task_url = util.ensure_library_task(self.owner, self.repo) util.new_task(task_url, target='manage') raise RequestAborted('repo has been renamed to %s', Library.id(self.owner, self.repo)) # If adding a NPM package that a Bower repo already points to, remove the bower one. bower_library_id = Library.id(self.owner, self.repo) if is_npm_package and bower_library_id is not None: task_url = util.migrate_library_task(self.owner, self.repo, self.scope, self.package) util.new_task(task_url, target='manage') self.library.github_owner = self.owner self.library.github_repo = self.repo self.library.metadata = response.content self.library.metadata_etag = response.headers.get('ETag', None) self.library.metadata_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code == 404: logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo)) delete_library(self.library.key) raise RequestAborted('repo no longer exists') elif response.status_code != 304: return self.retry('could not update repo metadata (%d)' % response.status_code) response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag) if response.status_code == 200: try: json.loads(response.content) except ValueError: return self.error("could not parse contributors", ErrorCodes.Library_parse_contributors) self.library.contributors = response.content self.library.contributors_etag = response.headers.get('ETag', None) self.library.contributors_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code != 304: return self.retry('could not update contributors (%d)' % response.status_code) response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag) if response.status_code == 200: try: json.loads(response.content) except ValueError: return self.error("could not parse stats/participation", ErrorCodes.Library_parse_stats) self.library.participation = response.content self.library.participation_etag = response.headers.get('ETag', None) self.library.participation_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code == 202: # GitHub is "computing" the data. We'll try again next update cycle. # TODO: Alternatively we could retry this task pass elif response.status_code != 304: return self.retry('could not update stats/participation (%d)' % response.status_code)
def update_metadata(self): # Query NPM registry API for packages is_npm_package = self.scope.startswith('@') if is_npm_package: self.update_registry_info() else: self.owner = self.scope self.repo = self.package # Fetch GitHub metadata headers = {'Accept': 'application/vnd.github.drax-preview+json'} response = util.github_get('repos', self.owner, self.repo, etag=self.library.metadata_etag, headers=headers) if response.status_code == 200: try: metadata = json.loads(response.content) except ValueError: return self.error("could not parse metadata", ErrorCodes.Library_parse_metadata) self.owner = metadata.get('owner', {}).get('login', '').lower() self.repo = metadata.get('name', '').lower() # Deleting is only necessary if Library entity is a GitHub repo if (not is_npm_package) and self.repo != '' and self.owner != '' and (self.repo != self.package or self.owner != self.scope): logging.info('deleting renamed repo %s', Library.id(self.owner, self.repo)) delete_library(self.library.key) task_url = util.ensure_library_task(self.owner, self.repo) util.new_task(task_url, target='manage') raise RequestAborted('repo has been renamed to %s', Library.id(self.owner, self.repo)) # If adding a NPM package that a Bower repo already points to, remove the bower one. bower_library_id = Library.id(self.owner, self.repo) if is_npm_package and bower_library_id is not None: logging.info('removing bower repo %s', Library.id(self.owner, self.repo)) task_url = util.suppress_library_task(self.owner, self.repo) util.new_task(task_url, target='manage') self.library.github_owner = self.owner self.library.github_repo = self.repo self.library.metadata = response.content self.library.metadata_etag = response.headers.get('ETag', None) self.library.metadata_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code == 404: logging.info('deleting non-existing repo %s', Library.id(self.owner, self.repo)) delete_library(self.library.key) raise RequestAborted('repo no longer exists') elif response.status_code != 304: return self.retry('could not update repo metadata (%d)' % response.status_code) response = util.github_get('repos', self.owner, self.repo, 'contributors', etag=self.library.contributors_etag) if response.status_code == 200: try: json.loads(response.content) except ValueError: return self.error("could not parse contributors", ErrorCodes.Library_parse_contributors) self.library.contributors = response.content self.library.contributors_etag = response.headers.get('ETag', None) self.library.contributors_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code != 304: return self.retry('could not update contributors (%d)' % response.status_code) response = util.github_get('repos', self.owner, self.repo, 'stats/participation ', etag=self.library.participation_etag) if response.status_code == 200: try: json.loads(response.content) except ValueError: return self.error("could not parse stats/participation", ErrorCodes.Library_parse_stats) self.library.participation = response.content self.library.participation_etag = response.headers.get('ETag', None) self.library.participation_updated = datetime.datetime.now() self.library_dirty = True elif response.status_code == 202: # GitHub is "computing" the data. We'll try again next update cycle. # TODO: Alternatively we could retry this task pass elif response.status_code != 304: return self.retry('could not update stats/participation (%d)' % response.status_code)