def test_from_string(self):
dependency = Dependency.from_string('owner/repo')
self.assertEqual(dependency.owner, 'owner')
self.assertEqual(dependency.repo, 'repo')
self.assertEqual(dependency.version, '*')
dependency = Dependency.from_string(
'https://github.com/owner/repo.git#master')
self.assertEqual(dependency.owner, 'owner')
self.assertEqual(dependency.repo, 'repo')
self.assertEqual(dependency.version, 'master')
def test_from_string(self):
dependency = Dependency.from_string('owner/repo')
self.assertEqual(dependency.owner, 'owner')
self.assertEqual(dependency.repo, 'repo')
self.assertEqual(dependency.version, '*')
dependency = Dependency.from_string('https://github.com/owner/repo.git#master')
self.assertEqual(dependency.owner, 'owner')
self.assertEqual(dependency.repo, 'repo')
self.assertEqual(dependency.version, 'master')
dependency = Dependency.from_string('https://github.com/owner/repo')
self.assertEqual(dependency.owner, 'owner')
self.assertEqual(dependency.repo, 'repo')
self.assertEqual(dependency.version, '*')
def get(self, owner, repo, version=None):
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Content-Type'] = 'application/json'
library_key = ndb.Key(Library, Library.id(owner, repo))
if version is None:
version = yield Library.default_version_for_key_async(library_key)
if version is None:
self.response.set_status(404)
return
version_key = ndb.Key(Library, library_key.id(), Version, version)
bower = yield Content.get_by_id_async('bower', parent=version_key)
if bower is None:
self.response.set_status(404)
return
bower_json = bower.get_json()
bower_dependencies = bower_json.get('dependencies', {})
dependencies = []
version_futures = []
for name in bower_dependencies.keys():
dependency = Dependency.from_string(bower_dependencies[name])
if dependency is None:
continue
dependencies.append(dependency)
dependency_library_key = ndb.Key(Library, Library.id(dependency.owner, dependency.repo))
version_futures.append(Library.versions_for_key_async(dependency_library_key))
dependency_futures = []
for i, dependency in enumerate(dependencies):
versions = yield version_futures[i]
def matches(version, spec):
try:
return versiontag.match(version, spec)
except ValueError:
# FIXME: What other cases do we need to support here?
return False
while len(versions) > 0 and not matches(versions[-1], dependency.version):
versions.pop()
if len(versions) > 0:
dependency_library_key = ndb.Key(Library, Library.id(dependency.owner.lower(), dependency.repo.lower()))
dependency_futures.append(LibraryMetadata.brief_async(dependency_library_key, versions[-1]))
results = []
for future in dependency_futures:
dependency_result = yield future
if dependency_result is not None:
results.append(dependency_result)
result = {
'results': results,
'count': len(results),
}
self.response.write(json.dumps(result))
def update_collection_dependencies(self, collection_version_key, bower):
dependencies = bower.get('dependencies', {})
for name in dependencies.keys():
dep = Dependency.from_string(dependencies[name])
if dep is None:
continue
library_key = ndb.Key(Library, Library.id(dep.owner, dep.repo))
CollectionReference.ensure(library_key, collection_version_key, semver=dep.version)
task_url = util.ensure_library_task(dep.owner.lower(), dep.repo.lower())
util.new_task(task_url, target='manage')
def update_collection_dependencies(self, collection_version_key, bower):
dependencies = bower.get('dependencies', {})
for name in dependencies.keys():
dep = Dependency.from_string(dependencies[name])
if dep is None:
continue
library_key = ndb.Key(Library, Library.id(dep.owner, dep.repo))
CollectionReference.ensure(library_key, collection_version_key, semver=dep.version)
task_url = util.ensure_library_task(dep.owner.lower(), dep.repo.lower())
util.new_task(task_url, target='manage')
def get(self, owner, repo, version):
logging.info('ingesting version %s/%s/%s', owner, repo, version)
key = ndb.Key(Library, '%s/%s' % (owner, repo), Version, version, Content, 'bower')
bower = json.loads(key.get().content)
ver = key.parent().get()
dependencies = bower.get('dependencies', {})
library_keys = []
dep_list = []
for name in dependencies.keys():
ver.dependencies.append(dependencies[name])
dep = Dependency.from_string(dependencies[name])
dep_list.append(dep)
library_keys.append(ndb.Key(Library, '%s/%s' % (dep.owner.lower(), dep.repo.lower())))
libraries = Library.get_or_create_list(library_keys)
for i, library in enumerate(libraries):
dep = dep_list[i]
library.collections.append(CollectionReference(version=key.parent(), semver=dep.version))
util.new_task('ingest/library', dep.owner.lower(), dep.repo.lower())
libraries.append(ver)
ndb.put_multi(libraries)
def get(self, owner, repo, ver=None):
owner = owner.lower()
repo = repo.lower()
library = Library.get_by_id('%s/%s' % (owner, repo), read_policy=ndb.EVENTUAL_CONSISTENCY)
if library is None or library.error is not None:
self.response.write(str(library))
self.response.set_status(404)
return
versions = library.versions()
if ver is None:
ver = versions[-1]
version = Version.get_by_id(ver, parent=library.key, read_policy=ndb.EVENTUAL_CONSISTENCY)
if version is None or version.error is not None:
self.response.write(str(version))
self.response.set_status(404)
return
metadata = json.loads(library.metadata)
dependencies = []
bower = Content.get_by_id('bower', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY)
if bower is not None:
try:
bower_json = json.loads(bower.content)
# TODO: Which exception is this for?
# pylint: disable=bare-except
except:
bower_json = {}
readme = Content.get_by_id('readme.html', parent=version.key, read_policy=ndb.EVENTUAL_CONSISTENCY)
full_name_match = re.match(r'(.*)/(.*)', metadata['full_name'])
result = {
'version': ver,
'versions': versions,
'readme': None if readme is None else readme.content,
'subscribers': metadata['subscribers_count'],
'stars': metadata['stargazers_count'],
'forks': metadata['forks'],
'contributors': library.contributor_count,
'open_issues': metadata['open_issues'],
'updated_at': metadata['updated_at'],
'owner': full_name_match.groups()[0],
'repo': full_name_match.groups()[1],
'bower': None if bower is None else {
'description': bower_json.get('description', ''),
'license': bower_json.get('license', ''),
'dependencies': bower_json.get('dependencies', []),
'keywords': bower_json.get('keywords', []),
},
'collections': []
}
for collection in library.collections:
if not versiontag.match(ver, collection.semver):
continue
collection_version = collection.version.id()
collection_library = collection.version.parent().get()
collection_metadata = json.loads(collection_library.metadata)
collection_name_match = re.match(r'(.*)/(.*)', collection_metadata['full_name'])
result['collections'].append({
'owner': collection_name_match.groups()[0],
'repo': collection_name_match.groups()[1],
'version': collection_version
})
if library.kind == 'collection':
dependencies = []
version_futures = []
for dep in version.dependencies:
parsed_dep = Dependency.fromString(dep)
dep_key = ndb.Key(Library, "%s/%s" % (parsed_dep.owner.lower(), parsed_dep.repo.lower()))
version_futures.append(Library.versions_for_key_async(dep_key))
for i, dep in enumerate(version.dependencies):
parsed_dep = Dependency.fromString(dep)
versions = version_futures[i].get_result()
versions.reverse()
while len(versions) > 0 and not versiontag.match(versions[0], parsed_dep.version):
versions.pop()
if len(versions) == 0:
dependencies.append({
'error': 'unsatisfyable dependency',
'owner': parsed_dep.owner,
'repo': parsed_dep.repo,
'versionSpec': parsed_dep.version
})
else:
dependencies.append(brief_metadata_from_datastore(parsed_dep.owner, parsed_dep.repo, versions[0]))
result['dependencies'] = dependencies
self.response.headers['Access-Control-Allow-Origin'] = '*'
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(result))