def update_readme(self, is_npm_package): if is_npm_package: # Load registry metadata to fetch readme path. library = Library.get_by_id(Library.id(self.owner, self.repo)) registry_metadata = json.loads(library.registry_metadata) if library.registry_metadata else None readme_path = registry_metadata.get('readmeFilename', 'README.md') response = util.unpkg_get(self.owner, self.repo, self.version, readme_path) readme = response.content else: # Load readme from GitHub endpoint. response = util.github_get('repos', self.owner, self.repo, 'readme', params={"ref": self.sha}) if response.status_code == 200: readme = base64.b64decode(json.loads(response.content)['content']) elif response.status_code == 404: readme = None else: return self.retry('error fetching readme (%d)' % response.status_code) if readme is not None: # Store the raw readme markdown. try: Content(parent=self.version_key, id='readme', content=readme, status=Status.ready, etag=response.headers.get('ETag', None)).put() except db.BadValueError: return self.error("Could not store README.md as a utf-8 string", ErrorCodes.Version_utf) # Convert markdown to HTML and store the result. response = util.github_markdown(readme) if response.status_code == 200: Content(parent=self.version_key, id='readme.html', content=response.content, status=Status.ready, etag=response.headers.get('ETag', None)).put() else: return self.retry('error converting readme to markdown (%d)' % response.status_code)
def test_ingest_preview(self): self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') response = self.app.get(util.ingest_preview_task('org', 'repo'), params={'commit': 'commit-sha', 'url': 'url'}, headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertTrue(library.shallow_ingestion) version = Version.get_by_id('commit-sha', parent=library.key) self.assertEquals(version.status, Status.pending) self.assertEquals(version.sha, 'commit-sha') self.assertEquals(version.url, 'url') self.assertTrue(version.preview) tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 2) self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'commit-sha'), util.ingest_version_task('org', 'repo', 'commit-sha'), ], [task.url for task in tasks])
def test_ingest_collection(self): self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"keywords": ["element-collection"], "license": "MIT"}') self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/heads/master', '{"ref": "refs/heads/master", "object": {"sha": "master-sha"}}') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}') self.assertEqual(library.contributors, '["a"]') self.assertEqual(library.tags, ['v0.0.1']) version = ndb.Key(Library, 'org/repo', Version, 'v0.0.1').get() self.assertIsNone(version.error) self.assertEqual(version.status, Status.pending) self.assertEqual(version.sha, 'master-sha') tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'v0.0.1', 'master-sha'), util.ensure_author_task('org'), util.ingest_version_task('org', 'repo', 'v0.0.1'), ], [task.url for task in tasks])
def test_ingest_element_no_versions(self): self.respond_to_github( 'https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github( 'https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/tags', '''[]''') self.respond_to_github( 'https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNotNone(library.error) self.assertIsNotNone(json.loads(library.error).get('code', None)) self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}') self.assertEqual(library.contributors, '["a"]') self.assertEqual(library.tags, []) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ensure_author_task('org'), ], [task.url for task in tasks])
def test_ingest_element(self): self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/tags', '''[{"name": "v0.5.0", "commit": {"sha": "old"}},{"name": "v1.0.0", "commit": {"sha": "lol"}}]''') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}') self.assertEqual(library.contributors, '["a"]') self.assertEqual(library.tags, ['v0.5.0', 'v1.0.0']) version = ndb.Key(Library, 'org/repo', Version, 'v1.0.0').get() self.assertIsNotNone(version) self.assertIsNone(version.error) self.assertEqual(version.sha, 'lol') tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'v1.0.0'), util.ensure_author_task('org'), util.ingest_version_task('org', 'repo', 'v1.0.0'), ], [task.url for task in tasks])
def test_ingest_preview(self): self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github( 'https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github( 'https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github( 'https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') response = self.app.get(util.ingest_preview_task('org', 'repo'), params={ 'commit': 'commit-sha', 'url': 'url' }, headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertTrue(library.shallow_ingestion) version = Version.get_by_id('commit-sha', parent=library.key) self.assertEquals(version.status, Status.pending) self.assertEquals(version.sha, 'commit-sha') self.assertEquals(version.url, 'url') self.assertTrue(version.preview) tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 2) self.assertEqual([ util.ingest_analysis_task('org', 'repo', 'commit-sha'), util.ingest_version_task('org', 'repo', 'commit-sha'), ], [task.url for task in tasks])
def test_add_element(self): response = self.app.get('/manage/add/element/org/repo') self.assertEqual(response.status_int, 200) self.assertEqual(response.normal_body, 'OK') tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].url, util.ingest_library_task('org', 'repo', 'element')) self.respond_to_github('https://api.github.com/repos/org/repo', 'metadata bits') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/git/refs/tags', '[{"ref": "refs/tags/v1.0.0", "object": {"sha": "lol"}}]') response = self.app.get(util.ingest_library_task('org', 'repo', 'element')) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertEqual(library.kind, 'element') self.assertEqual(library.metadata, 'metadata bits') self.assertEqual(library.contributors, '["a"]') self.assertEqual(library.contributor_count, 1) version = ndb.Key(Library, 'org/repo', Version, 'v1.0.0').get() self.assertIsNone(version.error) self.assertEqual(version.sha, 'lol') tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 2) self.assertEqual(tasks[1].url, util.ingest_version_task('org', 'repo', 'v1.0.0') + '?latestVersion=True')
def handle_get(self, owner, repo, scope, package): library = Library.get_by_id(Library.id(owner, repo)) if library is None: return library.npm_package = scope + '/' + package library.put() # Remove from search indexes. index = search.Index('repo') index.delete(Library.id(owner, repo)) npm_library = Library.get_by_id(Library.id(scope, package)) if npm_library is not None: npm_library.migrated_from_bower = True npm_library.put()
def init_library(self, owner, repo, kind=None, create=True): self.owner = owner.lower() self.repo = repo.lower() if create: assert kind is not None self.library = Library.maybe_create_with_kind(self.owner, self.repo, kind) else: self.library = Library.get_by_id('%s/%s' % (owner, repo))
def init_library(self, owner, repo, create=True): self.owner = owner.lower() self.repo = repo.lower() if create: self.library = Library.get_or_insert(Library.id(owner, repo)) self.is_new = self.library.metadata is None and self.library.error is None else: self.library = Library.get_by_id(Library.id(owner, repo)) if self.library.status == Status.suppressed: raise RequestAborted('library is suppressed')
def init_library(self, scope, package, create=True): self.scope = scope.lower() self.package = package.lower() if create: self.library = Library.get_or_insert(Library.id(self.scope, self.package)) self.is_new = self.library.metadata is None and self.library.error is None else: self.library = Library.get_by_id(Library.id(self.scope, self.package)) if self.library.status == Status.suppressed: raise RequestAborted('library is suppressed')
def test_ingest_no_license(self): self.respond_to_github('https://api.github.com/repos/org/repo', '{"license": null, "owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{}') response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNotNone(library.error) self.assertEqual(library.status, Status.error)
def test_ingest_license_fallback(self): self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') self.respond_to_github('https://api.github.com/repos/org/repo/tags', '[{"name": "v1.0.0", "commit": {"sha": "lol"}}]') response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertEqual(library.status, Status.ready) self.assertEqual(library.spdx_identifier, 'MIT')
def test_ingest_commit(self): self.respond_to_github('https://api.github.com/repos/org/repo', 'metadata bits') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.app.get(util.ingest_commit_task('org', 'repo', 'element'), params={'commit': 'commit-sha', 'url': 'url'}) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNone(library.error) self.assertFalse(library.ingest_versions) version = Version.get_by_id(parent=library.key, id='commit-sha') self.assertEqual(version.sha, 'commit-sha') self.assertEqual(version.url, 'url') tasks = self.tasks.get_filtered_tasks() self.assertEqual(len(tasks), 1) self.assertEqual(tasks[0].url, util.ingest_version_task('org', 'repo', 'commit-sha'))
def test_ingest_element_no_versions(self): self.respond_to_github('https://raw.githubusercontent.com/org/repo/master/bower.json', '{"license": "MIT"}') self.respond_to_github('https://api.github.com/repos/org/repo', '{"owner":{"login":"******"},"name":"repo"}') self.respond_to_github('https://api.github.com/repos/org/repo/contributors', '["a"]') self.respond_to_github('https://api.github.com/repos/org/repo/tags', '''[]''') self.respond_to_github('https://api.github.com/repos/org/repo/stats/participation', '{}') response = self.app.get(util.ingest_library_task('org', 'repo'), headers={'X-AppEngine-QueueName': 'default'}) self.assertEqual(response.status_int, 200) library = Library.get_by_id('org/repo') self.assertIsNotNone(library) self.assertIsNotNone(library.error) self.assertEqual(library.metadata, '{"owner":{"login":"******"},"name":"repo"}') self.assertEqual(library.contributors, '["a"]') self.assertEqual(library.tags, []) tasks = self.tasks.get_filtered_tasks() self.assertEqual([ util.ensure_author_task('org'), ], [task.url for task in tasks])
def handle_get(self, owner, repo): library = Library.get_by_id(Library.id(owner, repo)) if library is None or library.shallow_ingestion: task_url = util.ingest_library_task(owner, repo) util.new_task(task_url, target='manage')
def handle_get(self, scope, package): library = Library.get_by_id(Library.id(scope, package)) if library is not None: library.status = Status.suppressed library.put()
def get(self, owner, repo, ver=None): owner = owner.lower() repo = repo.lower() library = Library.get_by_id('%s/%s' % (owner, repo), read_policy=ndb.EVENTUAL_CONSISTENCY) if library is None or library.error is not None: self.response.write(str(library)) self.response.set_status(404) return versions = library.versions() if ver is None: ver = versions[-1] version = Version.get_by_id(ver, parent=library.key, read_policy=ndb.EVENTUAL_CONSISTENCY) if version is None or version.error is not None: self.response.write(str(version)) self.response.set_status(404) return metadata = json.loads(library.metadata) dependencies = [] bower = Content.get_by_id('bower', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY) if bower is not None: try: bower_json = json.loads(bower.content) # TODO: Which exception is this for? # pylint: disable=bare-except except: bower_json = {} readme = Content.get_by_id('readme.html', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY) full_name_match = re.match(r'(.*)/(.*)', metadata['full_name']) result = { 'version': ver, 'versions': versions, 'readme': None if readme is None else readme.content, 'subscribers': metadata['subscribers_count'], 'stars': metadata['stargazers_count'], 'forks': metadata['forks'], 'contributors': library.contributor_count, 'open_issues': metadata['open_issues'], 'updated_at': metadata['updated_at'], 'owner': full_name_match.groups()[0], 'repo': full_name_match.groups()[1], 'bower': None if bower is None else { 'description': bower_json.get('description', ''), 'license': bower_json.get('license', ''), 'dependencies': bower_json.get('dependencies', []), 'keywords': bower_json.get('keywords', []), }, 'collections': [] } for collection in library.collections: if not versiontag.match(ver, collection.semver): continue collection_version = collection.version.id() collection_library = collection.version.parent().get() collection_metadata = json.loads(collection_library.metadata) collection_name_match = re.match(r'(.*)/(.*)', collection_metadata['full_name']) result['collections'].append({ 'owner': collection_name_match.groups()[0], 'repo': collection_name_match.groups()[1], 'version': collection_version }) if library.kind == 'collection': dependencies = [] version_futures = [] for dep in version.dependencies: parsed_dep = Dependency.fromString(dep) dep_key = ndb.Key(Library, "%s/%s" % (parsed_dep.owner.lower(), parsed_dep.repo.lower())) version_futures.append(Library.versions_for_key_async(dep_key)) for i, dep in enumerate(version.dependencies): parsed_dep = Dependency.fromString(dep) versions = version_futures[i].get_result() versions.reverse() while len(versions) > 0 and not versiontag.match(versions[0], parsed_dep.version): versions.pop() if len(versions) == 0: dependencies.append({ 'error': 'unsatisfyable dependency', 'owner': parsed_dep.owner, 'repo': parsed_dep.repo, 'versionSpec': parsed_dep.version }) else: dependencies.append(brief_metadata_from_datastore(parsed_dep.owner, parsed_dep.repo, versions[0])) result['dependencies'] = dependencies self.response.headers['Access-Control-Allow-Origin'] = '*' self.response.headers['Content-Type'] = 'application/json' self.response.write(json.dumps(result))