def test_new_requires_private_key(self):
self.be_admin_user()
PrivateKey.get_by_key_name("singleton").delete()
response = self.testapp.get("/packages/test-package/versions/new")
self.assertEqual(response.status_int, 302)
self.assertEqual(response.headers["Location"], "http://localhost:80/admin#tab-private-key")
def test_new_requires_private_key(self):
self.be_admin_oauth_user()
PrivateKey.get_by_key_name('singleton').delete()
response = self.testapp.get('/packages/test-package/versions/new.json',
status=500)
self.assert_json_error(response)
def test_api_new_requires_oauth_key(self):
self.be_admin_oauth_user()
PrivateKey.get_by_key_name('singleton').delete()
response = self.testapp.get('/api/packages/versions/new',
status=500)
self.assert_json_error(response)
def create(self, key):
"""Set the private key."""
if not users.is_current_user_admin():
handlers.http_error(403, "Only admins may set the private key.")
PrivateKey.set(key)
handlers.flash("Private key set successfully.")
raise cherrypy.HTTPRedirect("/admin#tab-private-key")
def create(self, key):
"""Set the private key."""
if not users.is_current_user_admin():
handlers.http_error(403, "Only admins may set the private key.")
PrivateKey.set(key)
handlers.flash("Private key set successfully.")
raise cherrypy.HTTPRedirect("/admin#tab-private-key")
def test_api_dartdoc_requires_oauth_key(self):
self.be_admin_oauth_user()
self.post_package_version('1.2.3')
PrivateKey.get_by_key_name('singleton').delete()
response = self.testapp.get(
'/api/packages/test-package/versions/1.2.3/new_dartdoc',
status=500)
self.assert_json_error(response)
def test_api_dartdoc_requires_private_key(self):
self.be_admin_oauth_user()
self.post_package_version('1.2.3')
PrivateKey.get_by_key_name('singleton').delete()
response = self.testapp.get(
'/api/packages/test-package/versions/1.2.3/new_dartdoc',
status=500)
self.assert_json_error(response)
def setUp(self):
"""Initialize stubs for testing.
This initializes the following fields:
self.testbed: An App Engine Testbed used for mocking App Engine
services. This is activated, and any necessary stubs are
initialized.
self.testapp: A webtest.TestApp wrapping the CherryPy application.
Subclasses that define their own setUp method should be sure to call
this one as well.
"""
self.__users = {}
self.__admins = {}
self.testbed = Testbed()
self.testbed.activate()
self.testbed.init_app_identity_stub()
self.testbed.init_blobstore_stub()
self.testbed.init_datastore_v3_stub()
self.testbed.init_files_stub()
self.testbed.init_memcache_stub()
self.testbed.init_taskqueue_stub()
self.testbed.init_user_stub()
self.testapp = webtest.TestApp(Application())
# Pretend we are running on AppEngine so that things like the error
# page that behave differently when run locally act like they do in
# production.
os.environ['SERVER_SOFTWARE'] = 'Google App Engine/TESTING'
# This private key is not actually used for anything other than testing.
PrivateKey.set('''-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: DES-EDE3-CBC,CEB8C6541017AC8B
q1SgqAfgHXK5cQvtdIF8rkSlbAS6a92T5tYMVKJIW24giF5cw+1++X7X6S5ECykC
/iECExP7WfVlPDVoJMZpWGsYLMPhxncnKfUDICbVgwsO4KKeEv8lYTrvkY1sCZIx
kSja/lGAWpyxBnmxwoLh0LbLJBGUxUgn2Uqv/8Iid+w0m3NlgrllV+kOo4gUYF9q
bestH4nEQj6F0CeI8VPW0FxzMwn0vreHFBT5hul6xbNdB1vnRcz6ed4FiGXoAB5K
/8/Q2qyy1UPe2Hr9IoC6wo4h2kXq7pmhy/rtzU9/gjsGPD33ByavbgHAen0ajY5p
RmCx0AGidK9T6/SNoyDD9CMq7ypL+0JWoCeVoAEw2aZqN4rMJNbKMgPH+ajgiAXe
AWuMVjWN6uTL5+QJmWQct6a6TF8GPKdTcOZfNIXU5U9drHB2hggLcy6XkCYGjVJv
MvLascE4oAtOHnwz7WhrpcmX6F6Fww+TPnFzjk+IGJrnBUnAArEcM13IjiDUgg9B
iLuKimwrqasdBBY3Ua3SRMoG8PJoNKdsk1hDGlpxLnqOVUCvPKZuz4QnFusnUJrR
kiv+aqVBpZYymMh2Q1MWcogA7rL7LIowAkyLzS8dNwDhyk9jbO+1uoFSHKr5BTNv
cMJfCVm8pqhXwCVx3uYnhUzvth7mcEseXh5Dcg1RHka5rCXUz4eVxTkj1u3FOy9o
9jgARPpnDYsXH1lESxmiNZucHa50qI/ezNvQx8CbNa1/+JWoZ77yqM9qnDlXUwDY
1Sxqx9+4kthG9oyCzzUwFvhf1kTEHd0RfIapgjJ16HBQmzLnEPtjPA==
-----END RSA PRIVATE KEY-----''')
self.dont_be_oauth_user()
def setUp(self):
"""Initialize stubs for testing.
This initializes the following fields:
self.testbed: An App Engine Testbed used for mocking App Engine
services. This is activated, and any necessary stubs are
initialized.
self.testapp: A webtest.TestApp wrapping the CherryPy application.
Subclasses that define their own setUp method should be sure to call
this one as well.
"""
self.__users = {}
self.__admins = {}
self.testbed = Testbed()
self.testbed.activate()
self.testbed.init_app_identity_stub()
self.testbed.init_blobstore_stub()
self.testbed.init_datastore_v3_stub()
self.testbed.init_files_stub()
self.testbed.init_memcache_stub()
self.testbed.init_taskqueue_stub()
self.testbed.init_user_stub()
self.testapp = webtest.TestApp(Application())
# These private keys are not used for anything other than testing.
PrivateKey.set_oauth('''-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: DES-EDE3-CBC,CEB8C6541017AC8B
q1SgqAfgHXK5cQvtdIF8rkSlbAS6a92T5tYMVKJIW24giF5cw+1++X7X6S5ECykC
/iECExP7WfVlPDVoJMZpWGsYLMPhxncnKfUDICbVgwsO4KKeEv8lYTrvkY1sCZIx
kSja/lGAWpyxBnmxwoLh0LbLJBGUxUgn2Uqv/8Iid+w0m3NlgrllV+kOo4gUYF9q
bestH4nEQj6F0CeI8VPW0FxzMwn0vreHFBT5hul6xbNdB1vnRcz6ed4FiGXoAB5K
/8/Q2qyy1UPe2Hr9IoC6wo4h2kXq7pmhy/rtzU9/gjsGPD33ByavbgHAen0ajY5p
RmCx0AGidK9T6/SNoyDD9CMq7ypL+0JWoCeVoAEw2aZqN4rMJNbKMgPH+ajgiAXe
AWuMVjWN6uTL5+QJmWQct6a6TF8GPKdTcOZfNIXU5U9drHB2hggLcy6XkCYGjVJv
MvLascE4oAtOHnwz7WhrpcmX6F6Fww+TPnFzjk+IGJrnBUnAArEcM13IjiDUgg9B
iLuKimwrqasdBBY3Ua3SRMoG8PJoNKdsk1hDGlpxLnqOVUCvPKZuz4QnFusnUJrR
kiv+aqVBpZYymMh2Q1MWcogA7rL7LIowAkyLzS8dNwDhyk9jbO+1uoFSHKr5BTNv
cMJfCVm8pqhXwCVx3uYnhUzvth7mcEseXh5Dcg1RHka5rCXUz4eVxTkj1u3FOy9o
9jgARPpnDYsXH1lESxmiNZucHa50qI/ezNvQx8CbNa1/+JWoZ77yqM9qnDlXUwDY
1Sxqx9+4kthG9oyCzzUwFvhf1kTEHd0RfIapgjJ16HBQmzLnEPtjPA==
-----END RSA PRIVATE KEY-----''')
PrivateKey.set_api('not a real API key')
self.dont_be_oauth_user()
def upload(self,
file,
key,
acl=None,
policy=None,
signature=None,
success_action_redirect=None,
**kwargs):
"""A development-only action for uploading a package archive.
In production, package archives are uploaded directly to cloud storage,
using a signed form for authentication. The signed form doesn't work for
the development server, since it uses a local database in place of cloud
storage, so this action emulates it by manually saving the file to the
development database.
"""
if handlers.is_production(): raise handlers.http_error(404)
if PrivateKey.sign(policy) != signature: raise handlers.http_error(403)
write_path = files.gs.create('/gs/' + key, acl=acl)
with files.open(write_path, 'a') as f:
f.write(file.file.read())
files.finalize(write_path)
if success_action_redirect:
raise cherrypy.HTTPRedirect(success_action_redirect)
cherrypy.response.status = 204
return ""
def new(self, package_id, format="html", **kwargs):
"""Retrieve the form for uploading a package version.
If the user isn't logged in, this presents a login screen. If they are
logged in but don't have admin priviliges or don't own the package, this
redirects to /packages/.
This accepts arbitrary keyword arguments to support OAuth.
"""
is_json = format == "json"
user = handlers.get_current_user()
package = handlers.request().maybe_package
if not user:
if is_json:
handlers.http_error(403, "OAuth authentication failed.")
else:
raise cherrypy.HTTPRedirect(users.create_login_url(cherrypy.url()))
elif package and user not in package.uploaders:
message = "You aren't an uploader for package '%s'." % package.name
if is_json:
handlers.http_error(403, message)
else:
handlers.flash(message)
raise cherrypy.HTTPRedirect("/packages/%s" % package.name)
elif not handlers.is_current_user_admin():
message = "Currently only admins may create packages."
if is_json:
handlers.http_error(403, message)
else:
handlers.flash(message)
raise cherrypy.HTTPRedirect("/packages")
elif PrivateKey.get() is None:
if is_json:
handlers.http_error(500, "No private key set.")
else:
raise cherrypy.HTTPRedirect("/admin#tab-private-key")
id = str(uuid4())
redirect_url = handlers.request().url(action="create", id=id)
upload = cloud_storage.Upload(
"tmp/" + id, acl="project-private", size_range=(0, Package.MAX_SIZE), success_redirect=redirect_url
)
# If the package hasn't been validated and moved out of tmp in five
# minutes, delete it. This could happen if the user uploads the package
# to cloud storage, but closes the browser before "create" is run.
deferred.defer(self._remove_tmp_package, id, _countdown=5 * 60)
if is_json:
return upload.to_json()
if package is not None:
title = "Upload a new version of %s" % package.name
else:
title = "Upload a new package"
return handlers.render(
"packages/versions/new", form=upload.to_form(), package=package, layout={"title": title}
)
def upload(
self, package_id, file, key, acl=None, policy=None, signature=None, success_action_redirect=None, **kwargs
):
"""A development-only action for uploading a package archive.
In production, package archives are uploaded directly to cloud storage,
using a signed form for authentication. The signed form doesn't work for
the development server, since it uses a local database in place of cloud
storage, so this action emulates it by manually saving the file to the
development database.
"""
if handlers.is_production():
raise handlers.http_error(404)
if PrivateKey.sign(policy) != signature:
raise handlers.http_error(403)
write_path = files.gs.create("/gs/" + key, acl=acl)
with files.open(write_path, "a") as f:
f.write(file.file.read())
files.finalize(write_path)
if success_action_redirect:
raise cherrypy.HTTPRedirect(success_action_redirect)
cherrypy.response.status = 204
return ""
def test_admin_creates_keys(self):
self.be_admin_user()
get_response = self.testapp.get('/admin')
self.assertEqual(get_response.status_int, 200)
form = get_response.forms["private-key"]
self.assertEqual(form.method, 'POST')
form['oauth_key'] = 'oauth'
form['api_key'] = 'api'
post_response = form.submit()
self.assertEqual(post_response.status_int, 302)
self.assertEqual(post_response.headers['Location'],
'http://localhost:80/admin#tab-private-keys')
self.assertEqual(PrivateKey.get_oauth(), 'oauth')
self.assertEqual(PrivateKey.get_api(), 'api')
def test_admin_creates_keys(self):
self.be_admin_user()
get_response = self.testapp.get('/admin')
self.assertEqual(get_response.status_int, 200)
form = get_response.forms["private-key"]
self.assertEqual(form.method, 'POST')
form['oauth_key'] = 'oauth'
form['api_key'] = 'api'
post_response = form.submit()
self.assertEqual(post_response.status_int, 302)
self.assertEqual(post_response.headers['Location'],
'http://localhost:80/admin#tab-private-keys')
self.assertEqual(PrivateKey.get_oauth(), 'oauth')
self.assertEqual(PrivateKey.get_api(), 'api')
def new_dartdoc(self, package_id, id, format):
"""Retrieve the form for uploading dartdoc for this package version."""
if PrivateKey.get() is None:
handlers.http_error(500, 'No private key set.')
version = handlers.request().package_version(id)
upload = cloud_storage.Upload(version.dartdoc_storage_path,
acl='public-read',
size_range=(0, Package.MAX_SIZE))
return upload.to_json()
def new_dartdoc(self, package_id, id, format):
"""Retrieve the form for uploading dartdoc for this package version."""
if PrivateKey.get() is None:
handlers.http_error(500, 'No private key set.')
version = handlers.request().package_version(id)
upload = cloud_storage.Upload(version.dartdoc_storage_path,
acl='public-read',
size_range=(0, Package.MAX_SIZE))
return upload.to_json()
def admin(self):
"""Retrieve a page for performing administrative tasks."""
if not users.get_current_user():
raise cherrypy.HTTPRedirect(users.create_login_url(cherrypy.url()))
elif not users.is_current_user_admin():
raise handlers.http_error(403)
reload_status = PackageVersion.get_reload_status()
if reload_status is not None:
reload_status['percentage'] = '%d%%' % (
100.0 * reload_status['count'] / reload_status['total'])
return handlers.render('admin',
reload_status=reload_status,
private_keys_set=PrivateKey.get_oauth() is not None,
production=handlers.is_production(),
layout={'title': 'Admin Console'})
def admin(self):
"""Retrieve a page for performing administrative tasks."""
if not users.get_current_user():
raise cherrypy.HTTPRedirect(users.create_login_url(cherrypy.url()))
elif not users.is_current_user_admin():
raise handlers.http_error(403)
reload_status = PackageVersion.get_reload_status()
if reload_status is not None:
reload_status['percentage'] = '%d%%' % (
100.0 * reload_status['count'] / reload_status['total'])
return handlers.render('admin',
reload_status=reload_status,
private_keys_set=PrivateKey.get_oauth() is not None,
production=handlers.is_production(),
layout={'title': 'Admin Console'})
def _query_search(self, query, page):
"""Call the Custom Search API with the given query and get the given
page of results.
Return a tuple. The first element is the list of search results for the
requested page. Each result is a map that can be passed to the search
result template. The second element is the total count of search
results. The count will always be the total count of all results
starting from the beginning, regardless of the page being requested.
For each search result, this looks up the actual package in the data
store and returns its live metadata. If the URL cannot be parsed or
the package cannot be found in the datastore, it is not included in the
results.
Arguments:
query: The search query to perform.
page: The page of results to return. If out of bounds, returns an
empty collection of results.
"""
global _search_service
if page < 1 or page > MAX_RESULTS / RESULTS_PER_PAGE:
handlers.http_error(400, "Page \"%d\" is out of bounds." % page)
start = (page - 1) * RESULTS_PER_PAGE + 1
results = []
if _mock_resource:
resource = _mock_resource
else:
if not _search_service:
_search_service = build("customsearch", "v1",
developerKey=PrivateKey.get_api())
resource = _search_service.cse().list(q=query, cx=CUSTOM_SEARCH_ID,
num=RESULTS_PER_PAGE, start=start).execute()
if "items" in resource:
for item in resource["items"]:
result = self._get_result(item)
if result: results.append(result)
count = int(resource["searchInformation"]["totalResults"])
return results, count
def admin(self):
"""Retrieve a page for performing administrative tasks."""
if not users.get_current_user():
raise cherrypy.HTTPRedirect(users.create_login_url(cherrypy.url()))
elif not users.is_current_user_admin():
raise handlers.http_error(403)
reload_status = PackageVersion.get_reload_status()
if reload_status is not None:
reload_status["percentage"] = "%d%%" % (100.0 * reload_status["count"] / reload_status["total"])
return handlers.render(
"admin",
reload_status=reload_status,
private_key_set=PrivateKey.get() is not None,
production=handlers.is_production(),
layout={"title": "Admin Console"},
)
def new(self, package_id, format='json', **kwargs):
"""Retrieve the form for uploading a package version.
This accepts arbitrary keyword arguments to support OAuth.
"""
if PrivateKey.get() is None:
handlers.http_error(500, 'No private key set.')
id = str(uuid4())
redirect_url = handlers.request().url(action="create", id=id)
upload = cloud_storage.Upload("tmp/" + id, acl="project-private",
size_range=(0, Package.MAX_SIZE),
success_redirect=redirect_url)
# If the package hasn't been validated and moved out of tmp in five
# minutes, delete it. This could happen if the user uploads the package
# to cloud storage, but closes the browser before "create" is run.
deferred.defer(self._remove_tmp_package, id, _countdown=5*60)
return upload.to_json()
def __init__(self, obj, lifetime=10*60, acl=None, cache_control=None,
content_disposition=None, content_encoding=None,
content_type=None, expires=None, success_redirect=None,
success_status=None, size_range=None, metadata={}):
"""Create a new Upload.
Most arguments are identical to the form fields listed in
https://developers.google.com/storage/docs/reference-methods#postobject, but
there are a few differences:
* The expires argument takes a number of seconds since the epoch.
* The key argument only specifies the key name, not the bucket.
* The metadata argument is a dictionary of metadata header names to values.
Each one is transformed into an x-goog-meta- field. The keys should not
include "x-goog-meta-". Null values are ignored.
* The policy document is automatically created and signed. It ensures that
all fields have the assigned values when they're submitted to Cloud
Storage.
The lifetime argument specifies how long the form is valid. It defaults to
ten minutes.
The size_range argument should be a tuple indicating the lower and upper
bounds on the size of the uploaded file, in bytes.
"""
obj = _object_path(obj)
metadata = {'x-goog-meta-' + key: value for key, value
in metadata.iteritems()}
if expires is not None: expires = _iso8601(expires)
policy = {}
policy['expiration'] = _iso8601(time.time() + lifetime)
policy['conditions'] = [{'key': obj}]
def _try_add_condition(name, value):
if value is not None: policy['conditions'].append({name: value})
_try_add_condition('acl', acl)
_try_add_condition('cache-control', cache_control)
_try_add_condition('content-disposition', content_disposition)
_try_add_condition('content-encoding', content_encoding)
_try_add_condition('content-type', content_type)
_try_add_condition('expires', expires)
_try_add_condition('success_action_redirect', success_redirect)
_try_add_condition('success_action_status', success_status)
for key, value in metadata.items(): _try_add_condition(key, value)
if size_range is not None:
policy['conditions'].append(
['content-length-range', size_range[0], size_range[1]])
policy = b64encode(json.dumps(policy))
signature = PrivateKey.sign(policy)
self._fields = {'key': obj,
'acl': acl,
'Cache-Control': cache_control,
'Content-Disposition': content_disposition,
'Content-Encoding': content_encoding,
'Content-Type': content_type,
'expires': expires,
'GoogleAccessId': _ACCESS_KEY,
'policy': policy,
'signature': signature,
'success_action_redirect': success_redirect,
'success_action_status': success_status}
self._fields.update(metadata)
if handlers.is_production():
self._url = "https://storage.googleapis.com"
else:
self._url = routes.url_for(controller="versions",
action="upload",
package_id=None,
qualified=True)
def requires_api_key(fn, *args, **kwargs):
"""A decorator for actions that require a Google API key to be set."""
if PrivateKey.get_api() is None:
http_error(500, 'No private Google API key set.')
return fn(*args, **kwargs)
def requires_oauth_key(fn, *args, **kwargs):
"""A decorator for actions that require an OAuth2 private key to be set."""
if PrivateKey.get_oauth() is None:
http_error(500, 'No OAuth2 private key set.')
return fn(*args, **kwargs)
def test_index_requires_api_key(self):
self.be_admin_oauth_user()
PrivateKey.get_by_key_name('api').delete()
response = self.testapp.get('/search?q=query', status=500)
self.assert_error_page(response)
def __init__(self,
obj,
lifetime=10 * 60,
acl=None,
cache_control=None,
content_disposition=None,
content_encoding=None,
content_type=None,
expires=None,
success_redirect=None,
success_status=None,
size_range=None,
metadata={}):
"""Create a new Upload.
Most arguments are identical to the form fields listed in
https://developers.google.com/storage/docs/reference-methods#postobject, but
there are a few differences:
* The expires argument takes a number of seconds since the epoch.
* The key argument only specifies the key name, not the bucket.
* The metadata argument is a dictionary of metadata header names to values.
Each one is transformed into an x-goog-meta- field. The keys should not
include "x-goog-meta-". Null values are ignored.
* The policy document is automatically created and signed. It ensures that
all fields have the assigned values when they're submitted to Cloud
Storage.
The lifetime argument specifies how long the form is valid. It defaults to
ten minutes.
The size_range argument should be a tuple indicating the lower and upper
bounds on the size of the uploaded file, in bytes.
"""
obj = _object_path(obj)
metadata = {
'x-goog-meta-' + key: value
for key, value in metadata.iteritems()
}
if expires is not None: expires = _iso8601(expires)
policy = {}
policy['expiration'] = _iso8601(time.time() + lifetime)
policy['conditions'] = [{'key': obj}]
def _try_add_condition(name, value):
if value is not None: policy['conditions'].append({name: value})
_try_add_condition('acl', acl)
_try_add_condition('cache-control', cache_control)
_try_add_condition('content-disposition', content_disposition)
_try_add_condition('content-encoding', content_encoding)
_try_add_condition('content-type', content_type)
_try_add_condition('expires', expires)
_try_add_condition('success_action_redirect', success_redirect)
_try_add_condition('success_action_status', success_status)
for key, value in metadata.items():
_try_add_condition(key, value)
if size_range is not None:
policy['conditions'].append(
['content-length-range', size_range[0], size_range[1]])
policy = b64encode(json.dumps(policy))
signature = PrivateKey.sign(policy)
self._fields = {
'key': obj,
'acl': acl,
'Cache-Control': cache_control,
'Content-Disposition': content_disposition,
'Content-Encoding': content_encoding,
'Content-Type': content_type,
'expires': expires,
'GoogleAccessId': _ACCESS_KEY,
'policy': policy,
'signature': signature,
'success_action_redirect': success_redirect,
'success_action_status': success_status
}
self._fields.update(metadata)
if handlers.is_production():
self._url = "https://storage.googleapis.com"
else:
self._url = routes.url_for(controller="api.versions",
action="upload",
qualified=True)
def test_index_requires_api_key(self):
self.be_admin_oauth_user()
PrivateKey.get_by_key_name('api').delete()
response = self.testapp.get('/search?q=query', status=500)
self.assert_error_page(response)