def _reload_version(self, key):
"""Reload a single package version from its tarball."""
version = PackageVersion.get(key)
with closing(cloud_storage.read(version.storage_path)) as f:
new_version = PackageVersion.from_archive(
f, uploader=version.uploader)
with models.transaction():
# Reload the old version in case anything (e.g. sort order) changed.
version = PackageVersion.get(key)
package = version.package
# We don't load new_version.package.latest_version here for two
# reasons. One is to avoid a needless data store lookup; the other
# is because it's possible that that version is being reloaded in
# another transaction and thus in a weird transitional state.
latest_version_key = Package.latest_version.get_value_for_datastore(
package)
if latest_version_key == key:
package.latest_version = new_version
new_version.created = version.created
new_version.downloads = version.downloads
new_version.sort_order = version.sort_order
version.delete()
new_version.put()
package.put()
memcache.incr('versions_reloaded')
def create(self, id, **kwargs):
"""Create a new package version.
This creates a single package version. It will also create all the
package metadata if that doesn't already exist. The package archive is
uploaded to cloud storage separately.
If the user doesn't own the package or isn't logged in with admin
privileges, this will return a 403. If the package already has a version
with this number, or if the version is invalid, this will return a 400.
Arguments:
id: The id of the package in cloud storage.
"""
try:
route = handlers.request().route
if 'id' in route: del route['id']
try:
with closing(cloud_storage.read('tmp/' + id)) as f:
version = PackageVersion.from_archive(
f, uploaderEmail=handlers.get_oauth_user().email())
except (KeyError, files.ExistenceError):
handlers.http_error(
403, "Package upload " + id + " does not exist.")
# If the package for this version already exists, make sure we're an
# uploader for it. If it doesn't, we're fine to create it anew.
if version.package.is_saved():
if not version.package.has_uploader_email(
handlers.get_oauth_user().email()):
handlers.http_error(
403, "You aren't an uploader for package '%s'." %
version.package.name)
elif version.package.has_version(version.version):
message = 'Package "%s" already has version "%s".' % \
(version.package.name, version.version)
handlers.http_error(400, message)
if self._should_update_latest_version(version):
version.package.latest_version = version
else:
version.package.latest_version = version
cloud_storage.modify_object(version.storage_path,
acl='public-read',
copy_source='tmp/' + id)
with models.transaction():
version.package.put()
version.put()
version.package.invalidate_cache()
deferred.defer(self._compute_version_order, version.package.name)
return handlers.json_success('%s %s uploaded successfully.' %
(version.package.name, version.version))
finally:
cloud_storage.delete_object('tmp/' + id)
def create(self, id, **kwargs):
"""Create a new package version.
This creates a single package version. It will also create all the
package metadata if that doesn't already exist. The package archive is
uploaded to cloud storage separately.
If the user doesn't own the package or isn't logged in with admin
privileges, this will return a 403. If the package already has a version
with this number, or if the version is invalid, this will return a 400.
Arguments:
id: The id of the package in cloud storage.
"""
try:
route = handlers.request().route
if 'id' in route: del route['id']
try:
with closing(cloud_storage.read('tmp/' + id)) as f:
version = PackageVersion.from_archive(
f, uploader=handlers.get_oauth_user())
except (KeyError, files.ExistenceError):
handlers.http_error(
403, "Package upload " + id + " does not exist.")
# If the package for this version already exists, make sure we're an
# uploader for it. If it doesn't, we're fine to create it anew.
if version.package.is_saved():
if not version.package.has_uploader(handlers.get_oauth_user()):
handlers.http_error(
403, "You aren't an uploader for package '%s'." %
version.package.name)
elif version.package.has_version(version.version):
message = 'Package "%s" already has version "%s".' % \
(version.package.name, version.version)
handlers.http_error(400, message)
if self._should_update_latest_version(version):
version.package.latest_version = version
else:
version.package.latest_version = version
cloud_storage.modify_object(version.storage_path,
acl='public-read',
copy_source='tmp/' + id)
with models.transaction():
version.package.put()
version.put()
deferred.defer(self._compute_version_order, version.package.name)
return handlers.json_success(
'%s %s uploaded successfully.' %
(version.package.name, version.version))
finally:
cloud_storage.delete_object('tmp/' + id)
def _count_download(self, key):
"""Increment the download count for a package version."""
with models.transaction():
version = PackageVersion.get(key)
version.downloads += 1
version.package.downloads += 1
version.put()
version.package.put()
def test_reload_preserves_sort_order(self):
self.be_admin_user()
self.post_package_version("1.2.3")
self.post_package_version("1.2.4")
self.post_package_version("1.2.4-pre")
self.testapp.post("/packages/versions/reload")
self.run_deferred_tasks()
version = PackageVersion.get_by_name_and_version("test-package", "1.2.3")
self.assertEqual(0, version.sort_order)
version = PackageVersion.get_by_name_and_version("test-package", "1.2.4-pre")
self.assertEqual(1, version.sort_order)
version = PackageVersion.get_by_name_and_version("test-package", "1.2.4")
self.assertEqual(2, version.sort_order)
def test_loads_readme_from_archive(self):
pubspec = {'name': 'test-package', 'version': '1.0.0'}
archive = self.tar_package(pubspec, {
'README': 'This is a README.',
})
version = PackageVersion.from_archive(StringIO(archive),
uploader=self.admin_user())
self.assertEqual('This is a README.', version.readme.text)
def _count_download(self, key):
"""Increment the download count for a package version."""
with models.transaction():
version = PackageVersion.get(key)
version.downloads += 1
version.package.downloads += 1
version.put()
version.package.put()
def reload(self, package_id):
"""Reload all package versions from their tarballs."""
if not users.is_current_user_admin(): handlers.http_error(403)
versions_to_reload = 0
for key in PackageVersion.all(keys_only=True).run():
versions_to_reload += 1
deferred.defer(self._reload_version, key)
memcache.set('versions_to_reload', versions_to_reload)
memcache.set('versions_reloaded', 0)
raise cherrypy.HTTPRedirect('/admin#tab-packages')
def reload(self):
"""Reload all package versions from their tarballs."""
if not handlers.is_current_user_dogfooder(): handlers.http_error(403)
query = PackageVersion.all(keys_only=True)
memcache.set('versions_to_reload', query.count())
memcache.set('versions_reloaded', 0)
for key in query.run():
name = 'reload-%s-%s' % (int(time.time()), key)
deferred.defer(self._reload_version, key, _name=name)
raise cherrypy.HTTPRedirect('/admin#tab-packages')
def test_reload_preserves_downloads(self):
self.be_admin_user()
self.post_package_version("1.2.3")
response = self.testapp.get("/packages/test-package/versions/1.2.3.tar.gz")
self.testapp.post("/packages/versions/reload")
self.run_deferred_tasks()
version = PackageVersion.get_by_name_and_version("test-package", "1.2.3")
self.assertEqual(1, version.downloads)
def test_reload_reloads_a_package_version(self):
self.be_admin_user()
self.post_package_version("1.2.3")
version = PackageVersion.get_by_name_and_version("test-package", "1.2.3")
version.libraries = ["wrong"]
version.put()
response = self.testapp.post("/packages/versions/reload", status=302)
self.assertEqual(response.headers["Location"], "http://localhost:80/admin#tab-packages")
self.run_deferred_tasks()
version = PackageVersion.get_by_name_and_version("test-package", "1.2.3")
self.assertEqual([], version.libraries)
# The latest_version of the parent package should also be updated
version = Package.get_by_key_name("test-package").latest_version
self.assertEqual([], version.libraries)
self.assert_package_updated_is_latest_version_created()
def reload(self, package_id):
"""Reload all package versions from their tarballs."""
if not handlers.is_current_user_dogfooder(): handlers.http_error(403)
query = PackageVersion.all(keys_only=True)
memcache.set('versions_to_reload', query.count())
memcache.set('versions_reloaded', 0)
for key in query.run():
name = 'reload-%s-%s' % (int(time.time()), key)
deferred.defer(self._reload_version, key, _name=name)
raise cherrypy.HTTPRedirect('/admin#tab-packages')
def _reload_version(self, key):
"""Reload a single package version from its tarball."""
version = PackageVersion.get(key)
logging.info('Reloading %s %s' % (version.package.name, version.version))
with closing(cloud_storage.read(version.storage_path)) as f:
new_version = PackageVersion.from_archive(
f, uploader=version.uploader)
with models.transaction():
# Reload the old version in case anything (e.g. sort order) changed.
version = PackageVersion.get(key)
package = version.package
# We don't load new_version.package.latest_version here for two
# reasons. One is to avoid a needless data store lookup; the other
# is because it's possible that that version is being reloaded in
# another transaction and thus in a weird transitional state.
latest_version_key = Package.latest_version.get_value_for_datastore(
package)
if latest_version_key == key:
package.latest_version = new_version
new_version.created = version.created
new_version.downloads = version.downloads
new_version.sort_order = version.sort_order
version.delete()
new_version.put()
# Only save the package if its latest version has been updated.
# Otherwise, its latest version may be being updated in parallel,
# causing icky bugs.
if latest_version_key == key:
package.put()
package.invalidate_cache()
count = memcache.incr('versions_reloaded')
logging.info('%s/%s versions reloaded' %
(count, memcache.get('versions_to_reload')))
def _reload_version(self, key):
"""Reload a single package version from its tarball."""
version = PackageVersion.get(key)
logging.info('Reloading %s %s' %
(version.package.name, version.version))
with closing(cloud_storage.read(version.storage_path)) as f:
new_version = PackageVersion.from_archive(
f, uploaderEmail=version.uploaderEmail)
with models.transaction():
# Reload the old version in case anything (e.g. sort order) changed.
version = PackageVersion.get(key)
package = version.package
# We don't load new_version.package.latest_version here for two
# reasons. One is to avoid a needless data store lookup; the other
# is because it's possible that that version is being reloaded in
# another transaction and thus in a weird transitional state.
latest_version_key = Package.latest_version.get_value_for_datastore(
package)
if latest_version_key == key:
package.latest_version = new_version
new_version.created = version.created
new_version.downloads = version.downloads
new_version.sort_order = version.sort_order
version.delete()
new_version.put()
# Only save the package if its latest version has been updated.
# Otherwise, its latest version may be being updated in parallel,
# causing icky bugs.
if latest_version_key == key:
package.put()
package.invalidate_cache()
count = memcache.incr('versions_reloaded')
logging.info('%s/%s versions reloaded' %
(count, memcache.get('versions_to_reload')))
def test_reload_preserves_sort_order(self):
self.be_admin_oauth_user()
self.post_package_version('1.2.3')
self.post_package_version('1.2.4')
self.post_package_version('1.2.4-pre')
self.be_admin_user()
self.testapp.post('/packages/versions/reload')
self.run_deferred_tasks()
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.3')
self.assertEqual(0, version.sort_order)
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.4-pre')
self.assertEqual(1, version.sort_order)
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.4')
self.assertEqual(2, version.sort_order)
def package_version(self, package, version, **additional_pubspec_fields):
"""Create a package version object.
This constructs the package archive based on the other information
passed in. The archive contains only the pubspec.
"""
pubspec = Pubspec(name=package.name, version=version)
pubspec.update(additional_pubspec_fields)
return PackageVersion.new(version=version,
package=package,
pubspec=pubspec,
uploader=package.uploaders[0])
def test_reload_preserves_sort_order(self):
self.be_admin_oauth_user()
self.post_package_version('1.2.3')
self.post_package_version('1.2.4')
self.post_package_version('1.2.4-pre')
self.be_admin_user()
self.testapp.post('/packages/versions/reload')
self.run_deferred_tasks()
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.3')
self.assertEqual(0, version.sort_order)
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.4-pre')
self.assertEqual(1, version.sort_order)
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.4')
self.assertEqual(2, version.sort_order)
def reload_status(self, format):
"""Return the status of the current package reload.
This is a JSON map. If the reload is finished, it will contain only a
'done' key with value true. If the reload is in progress, it will
contain 'count' and 'total' keys, indicating the total number of
packages to reload and the number that have been reloaded so far,
respectively.
"""
if not users.is_current_user_admin():
handlers.http_error(403, "Permission denied.")
reload_status = PackageVersion.get_reload_status()
return json.dumps(reload_status or {'done': True})
def reload_status(self, package_id, format):
"""Return the status of the current package reload.
This is a JSON map. If the reload is finished, it will contain only a
'done' key with value true. If the reload is in progress, it will
contain 'count' and 'total' keys, indicating the total number of
packages to reload and the number that have been reloaded so far,
respectively.
"""
if not users.is_current_user_admin():
handlers.http_error(403, "Permission denied.")
reload_status = PackageVersion.get_reload_status()
return json.dumps(reload_status or {'done': True})
def reload_status(self, package_id):
"""Return the status of the current package reload.
This is a JSON map. If the reload is finished, it will contain only a
'done' key with value true. If the reload is in progress, it will
contain 'count' and 'total' keys, indicating the total number of
packages to reload and the number that have been reloaded so far,
respectively.
"""
if not users.is_current_user_admin(): cherrypy.http_error(403)
cherrypy.response.headers['Content-Type'] = 'application/json'
reload_status = PackageVersion.get_reload_status()
return json.dumps(reload_status or {'done': True})
def test_reload_preserves_downloads(self):
self.be_admin_oauth_user()
self.post_package_version('1.2.3')
response = self.testapp.get(
'/packages/test-package/versions/1.2.3.tar.gz')
self.be_admin_user()
self.testapp.post('/packages/versions/reload')
self.run_deferred_tasks()
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.3')
self.assertEqual(1, version.downloads)
def package_version(self, package, version,
**additional_pubspec_fields):
"""Create a package version object.
This constructs the package archive based on the other information
passed in. The archive contains only the pubspec.
"""
pubspec = Pubspec(name=package.name, version=version)
pubspec.update(additional_pubspec_fields)
return PackageVersion.new(
version=version,
package=package,
pubspec=pubspec,
uploader=package.uploaders[0])
def test_imports_from_archive(self):
pubspec = {'name': 'test-package', 'version': '1.0.0'}
archive = self.tar_package(pubspec, {
'lib/foo.dart': '',
'lib/bar/foo.dart': '',
'lib/bar/src/foo.dart': '',
'lib/zip': '',
'lib/src/foo.dart': '',
})
version = PackageVersion.from_archive(StringIO(archive),
uploader=self.admin_user())
self.assertEqual(['bar/foo.dart', 'bar/src/foo.dart', 'foo.dart'],
version.libraries)
def test_reload_preserves_downloads(self):
self.be_admin_oauth_user()
self.post_package_version('1.2.3')
response = self.testapp.get(
'/packages/test-package/versions/1.2.3.tar.gz')
self.be_admin_user()
self.testapp.post('/packages/versions/reload')
self.run_deferred_tasks()
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.3')
self.assertEqual(1, version.downloads)
def test_reload_reloads_a_package_version(self):
self.be_admin_oauth_user()
self.post_package_version('1.2.3')
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.3')
version.libraries = ["wrong"]
version.put()
self.be_admin_user()
response = self.testapp.post('/packages/versions/reload', status=302)
self.assertEqual(response.headers['Location'],
'http://localhost:80/admin#tab-packages')
self.run_deferred_tasks()
version = PackageVersion.get_by_name_and_version(
'test-package', '1.2.3')
self.assertEqual([], version.libraries)
# The latest_version of the parent package should also be updated
version = Package.get_by_key_name('test-package').latest_version
self.assertEqual([], version.libraries)
self.assert_package_updated_is_latest_version_created()
def setUp(self):
super(SearchTest, self).setUp()
self.be_admin_user()
packages = [
"armadillo", "bandicoot", "cat", "dog", "elephant", "fox",
"gorilla", "hippo", "iguana", "jackal", "kangaroo", "lemur"
]
for package in packages:
self.create_package(package, "1.0.0")
packageVersion = PackageVersion.get_by_name_and_version(
package, "1.0.0")
packageVersion.pubspec.update(description="Description for " +
package)
packageVersion.put()
self.be_normal_user()
def admin(self):
"""Retrieve a page for performing administrative tasks."""
if not users.get_current_user():
raise cherrypy.HTTPRedirect(users.create_login_url(cherrypy.url()))
elif not users.is_current_user_admin():
raise handlers.http_error(403)
reload_status = PackageVersion.get_reload_status()
if reload_status is not None:
reload_status['percentage'] = '%d%%' % (
100.0 * reload_status['count'] / reload_status['total'])
return handlers.render('admin',
reload_status=reload_status,
private_keys_set=PrivateKey.get_oauth() is not None,
production=handlers.is_production(),
layout={'title': 'Admin Console'})
def package_version(self, version):
"""Load the current package version object.
This auto-detects the package name from the request parameters. If the
package version doesn't exist, throws a 404 error.
"""
if self._package_version: return self._package_version
package_name = self.request.params['package_id']
if not package_name:
http_error(403, "No package name found.")
self._package_version = PackageVersion.get_by_name_and_version(
package_name, version)
if self._package_version: return self._package_version
http_error(404, "\"%s\" version %s doesn't exist." %
(package_name, version))
def admin(self):
"""Retrieve a page for performing administrative tasks."""
if not users.get_current_user():
raise cherrypy.HTTPRedirect(users.create_login_url(cherrypy.url()))
elif not users.is_current_user_admin():
raise handlers.http_error(403)
reload_status = PackageVersion.get_reload_status()
if reload_status is not None:
reload_status['percentage'] = '%d%%' % (
100.0 * reload_status['count'] / reload_status['total'])
return handlers.render('admin',
reload_status=reload_status,
private_keys_set=PrivateKey.get_oauth() is not None,
production=handlers.is_production(),
layout={'title': 'Admin Console'})
def admin(self):
"""Retrieve a page for performing administrative tasks."""
if not users.get_current_user():
raise cherrypy.HTTPRedirect(users.create_login_url(cherrypy.url()))
elif not users.is_current_user_admin():
raise handlers.http_error(403)
reload_status = PackageVersion.get_reload_status()
if reload_status is not None:
reload_status["percentage"] = "%d%%" % (100.0 * reload_status["count"] / reload_status["total"])
return handlers.render(
"admin",
reload_status=reload_status,
private_key_set=PrivateKey.get() is not None,
production=handlers.is_production(),
layout={"title": "Admin Console"},
)
def setUp(self):
super(SearchTest, self).setUp()
self.be_admin_user()
packages = [
"armadillo", "bandicoot", "cat", "dog", "elephant",
"fox", "gorilla", "hippo", "iguana", "jackal",
"kangaroo", "lemur"
]
for package in packages:
self.create_package(package, "1.0.0")
packageVersion = PackageVersion.get_by_name_and_version(
package, "1.0.0")
packageVersion.pubspec.update(
description="Description for " + package)
packageVersion.put()
self.be_normal_user()
def get_package_version(self, version):
return PackageVersion.get_by_name_and_version('test-package', version)
def get_package_version(self, version):
return PackageVersion.get_by_name_and_version('test-package', version)
def create(self, package_id, id, format='html', **kwargs):
"""Create a new package version.
This creates a single package version. It will also create all the
package metadata if that doesn't already exist. The package archive is
uploaded to cloud storage separately.
If the user doesn't own the package or isn't logged in with admin
privileges, this will return a 403. If the package already has a version
with this number, or if the version is invalid, this will redirect to
the new version form.
Arguments:
id: The id of the package in cloud storage.
"""
try:
is_json = format == 'json'
route = handlers.request().route
if 'id' in route: del route['id']
package = handlers.request().maybe_package
if package and handlers.get_current_user() not in package.uploaders:
handlers.request().error(
403, "You aren't an uploader for package '%s'." %
package.name)
elif not handlers.is_current_user_admin():
handlers.request().error(
403, "Only admins may create packages.")
try:
with closing(cloud_storage.read('tmp/' + id)) as f:
version = PackageVersion.from_archive(
f, uploader=handlers.get_current_user())
except (KeyError, files.ExistenceError):
handlers.request().error(
403, "Package upload " + id + " does not exist.")
if version.package.is_saved():
if handlers.get_current_user() not in version.package.uploaders:
handlers.request().error(
403, "You aren't an uploader for package '%s'." %
version.package.name)
elif version.package.has_version(version.version):
message = 'Package "%s" already has version "%s".' % \
(version.package.name, version.version)
if is_json: handlers.json_error(400, message)
handlers.flash(message)
url = handlers.request().url(
action='new', package_id=version.package.name)
raise cherrypy.HTTPRedirect(url)
if self._should_update_latest_version(version):
version.package.latest_version = version
else:
version.package.latest_version = version
cloud_storage.modify_object(version.storage_path,
acl='public-read',
copy_source='tmp/' + id)
with models.transaction():
version.package.put()
version.put()
deferred.defer(self._compute_version_order, version.package.name)
message = '%s %s uploaded successfully.' % \
(version.package.name, version.version)
if is_json:
cherrypy.response.headers['Content-Type'] = 'application/json'
return json.dumps({"success": {"message": message}})
handlers.flash(message)
raise cherrypy.HTTPRedirect('/packages/%s' % version.package.name)
finally:
cloud_storage.delete_object('tmp/' + id)
