def sign(cls, string): """Returns the signature of a string. This signature is generated using the singleton private key, then base64-encoded. It's of the form expected by Google Cloud Storage query string authentication. See https://developers.google.com/storage/docs/accesscontrol#Signed-URLs. """ # All Google API keys have "notasecret" as their passphrase value = cls.get_oauth() if value is None: raise "Private key has not been set." if handlers.is_production(): # TODO(nweiz): This currently doesn't work on the development server # without adding 'AESCipher', 'blockalgo', and '_AES' to the # __CRYPTO_CIPHER_ALLOWED_MODULES constant in # google/appengine/tools/dev_appserver_import_hook.py. However, it # does work in production, so to make it work locally, we just do a # dumb hash of the private key and the string. # # See http://code.google.com/p/googleappengine/issues/detail?id=8188 key = RSA.importKey(value, passphrase='notasecret') return base64.b64encode( PKCS1_v1_5.new(key).sign(SHA256.new(string))) else: m = hashlib.md5() m.update(value) m.update(string) return base64.b64encode(m.digest())
def sign(cls, string): """Returns the signature of a string. This signature is generated using the singleton private key, then base64-encoded. It's of the form expected by Google Cloud Storage query string authentication. See https://developers.google.com/storage/docs/accesscontrol#Signed-URLs. """ # All Google API keys have "notasecret" as their passphrase value = cls.get_oauth() if value is None: raise "Private key has not been set." if handlers.is_production(): # TODO(nweiz): This currently doesn't work on the development server # without adding 'AESCipher', 'blockalgo', and '_AES' to the # __CRYPTO_CIPHER_ALLOWED_MODULES constant in # google/appengine/tools/dev_appserver_import_hook.py. However, it # does work in production, so to make it work locally, we just do a # dumb hash of the private key and the string. # # See http://code.google.com/p/googleappengine/issues/detail?id=8188 key = RSA.importKey(value, passphrase='notasecret') return base64.b64encode(PKCS1_v1_5.new(key).sign(SHA256.new(string))) else: m = hashlib.md5() m.update(value) m.update(string) return base64.b64encode(m.digest())
def upload(self, file, key, acl=None, policy=None, signature=None, success_action_redirect=None, **kwargs): """A development-only action for uploading a package archive. In production, package archives are uploaded directly to cloud storage, using a signed form for authentication. The signed form doesn't work for the development server, since it uses a local database in place of cloud storage, so this action emulates it by manually saving the file to the development database. """ if handlers.is_production(): raise handlers.http_error(404) if PrivateKey.sign(policy) != signature: raise handlers.http_error(403) write_path = files.gs.create('/gs/' + key, acl=acl) with files.open(write_path, 'a') as f: f.write(file.file.read()) files.finalize(write_path) if success_action_redirect: raise cherrypy.HTTPRedirect(success_action_redirect) cherrypy.response.status = 204 return ""
def upload( self, package_id, file, key, acl=None, policy=None, signature=None, success_action_redirect=None, **kwargs ): """A development-only action for uploading a package archive. In production, package archives are uploaded directly to cloud storage, using a signed form for authentication. The signed form doesn't work for the development server, since it uses a local database in place of cloud storage, so this action emulates it by manually saving the file to the development database. """ if handlers.is_production(): raise handlers.http_error(404) if PrivateKey.sign(policy) != signature: raise handlers.http_error(403) write_path = files.gs.create("/gs/" + key, acl=acl) with files.open(write_path, "a") as f: f.write(file.file.read()) files.finalize(write_path) if success_action_redirect: raise cherrypy.HTTPRedirect(success_action_redirect) cherrypy.response.status = 204 return ""
def serve(self, filename): """Serves a cloud storage file for the development server.""" if handlers.is_production(): return handlers.http_error(404) cherrypy.response.headers['Content-Type'] = 'application/octet-stream' cherrypy.response.headers['Content-Disposition'] = \ 'attachment; filename=%s' % os.path.basename(filename) try: with cloud_storage.open(filename) as f: return f.read() except KeyError, ExistenceError: handlers.http_error(404)
def admin(self): """Retrieve a page for performing administrative tasks.""" if not users.get_current_user(): raise cherrypy.HTTPRedirect(users.create_login_url(cherrypy.url())) elif not users.is_current_user_admin(): raise handlers.http_error(403) reload_status = PackageVersion.get_reload_status() if reload_status is not None: reload_status['percentage'] = '%d%%' % ( 100.0 * reload_status['count'] / reload_status['total']) return handlers.render('admin', reload_status=reload_status, private_keys_set=PrivateKey.get_oauth() is not None, production=handlers.is_production(), layout={'title': 'Admin Console'})
def admin(self): """Retrieve a page for performing administrative tasks.""" if not users.get_current_user(): raise cherrypy.HTTPRedirect(users.create_login_url(cherrypy.url())) elif not users.is_current_user_admin(): raise handlers.http_error(403) reload_status = PackageVersion.get_reload_status() if reload_status is not None: reload_status["percentage"] = "%d%%" % (100.0 * reload_status["count"] / reload_status["total"]) return handlers.render( "admin", reload_status=reload_status, private_key_set=PrivateKey.get() is not None, production=handlers.is_production(), layout={"title": "Admin Console"}, )
def modify_object(obj, content_encoding=None, content_type=None, content_disposition=None, acl=None, copy_source=None, copy_source_if_match=None, copy_source_if_none_match=None, copy_source_if_modified_since=None, copy_source_if_unmodified_since=None, copy_metadata=True, metadata={}): """Modifies or copies a cloud storage object. Most arguments are identical to the form fields listed in https://developers.google.com/storage/docs/reference-methods#putobject, but there are a few differences: * The copy_metadata argument can be True, indicating that the metadata should be copied, or False, indicating that it should be replaced. * The metadata argument is a dictionary of metadata header names to values. Each one is transformed into an x-goog-meta- field. The keys should not include "x-goog-meta-". Null values are ignored. """ if not handlers.is_production(): # The only way to modify an existing object using only the Python API # seems to be to copy it over itself. It's not a big deal since this is # only for development. if copy_source is None: copy_source = obj contents = None with files.open(_appengine_object_path(copy_source), 'r') as f: contents = f.read() if content_type is None: content_type = 'application/octet-stream' write_path = files.gs.create(_appengine_object_path(obj), mime_type=content_type, acl=acl, content_encoding=content_encoding, content_disposition=content_disposition, user_metadata=metadata) with files.open(write_path, 'a') as f: f.write(contents) files.finalize(write_path) return auth = "OAuth " + app_identity.get_access_token(_FULL_CONTROL_SCOPE)[0] headers = { "Authorization": auth, "Content-Encoding": content_encoding, "Content-Type": content_type, "Content-Disposition": content_disposition, "x-goog-api-version": "2", "x-goog-acl": acl, "x-goog-copy-source": _object_path(copy_source), "x-goog-copy-source-if-match": copy_source_if_match, "x-goog-copy-source-if-none-match": copy_source_if_none_match, "x-goog-copy-source-if-modified-since": copy_source_if_modified_since, "x-goog-copy-source-if-unmodified-since": copy_source_if_unmodified_since, "x-goog-copy-metadata-directive": "COPY" if copy_metadata else "REPLACE" } for (key, value) in metadata.iteritems(): headers["x-goog-meta-" + key] = value headers = {key: value for key, value in headers.iteritems() if value is not None} return urlfetch.fetch("https://storage.googleapis.com/" + urllib.quote(_object_path(obj)), method="PUT", headers=headers)
def __init__(self, obj, lifetime=10 * 60, acl=None, cache_control=None, content_disposition=None, content_encoding=None, content_type=None, expires=None, success_redirect=None, success_status=None, size_range=None, metadata={}): """Create a new Upload. Most arguments are identical to the form fields listed in https://developers.google.com/storage/docs/reference-methods#postobject, but there are a few differences: * The expires argument takes a number of seconds since the epoch. * The key argument only specifies the key name, not the bucket. * The metadata argument is a dictionary of metadata header names to values. Each one is transformed into an x-goog-meta- field. The keys should not include "x-goog-meta-". Null values are ignored. * The policy document is automatically created and signed. It ensures that all fields have the assigned values when they're submitted to Cloud Storage. The lifetime argument specifies how long the form is valid. It defaults to ten minutes. The size_range argument should be a tuple indicating the lower and upper bounds on the size of the uploaded file, in bytes. """ obj = _object_path(obj) metadata = { 'x-goog-meta-' + key: value for key, value in metadata.iteritems() } if expires is not None: expires = _iso8601(expires) policy = {} policy['expiration'] = _iso8601(time.time() + lifetime) policy['conditions'] = [{'key': obj}] def _try_add_condition(name, value): if value is not None: policy['conditions'].append({name: value}) _try_add_condition('acl', acl) _try_add_condition('cache-control', cache_control) _try_add_condition('content-disposition', content_disposition) _try_add_condition('content-encoding', content_encoding) _try_add_condition('content-type', content_type) _try_add_condition('expires', expires) _try_add_condition('success_action_redirect', success_redirect) _try_add_condition('success_action_status', success_status) for key, value in metadata.items(): _try_add_condition(key, value) if size_range is not None: policy['conditions'].append( ['content-length-range', size_range[0], size_range[1]]) policy = b64encode(json.dumps(policy)) signature = PrivateKey.sign(policy) self._fields = { 'key': obj, 'acl': acl, 'Cache-Control': cache_control, 'Content-Disposition': content_disposition, 'Content-Encoding': content_encoding, 'Content-Type': content_type, 'expires': expires, 'GoogleAccessId': _ACCESS_KEY, 'policy': policy, 'signature': signature, 'success_action_redirect': success_redirect, 'success_action_status': success_status } self._fields.update(metadata) if handlers.is_production(): self._url = "https://storage.googleapis.com" else: self._url = routes.url_for(controller="api.versions", action="upload", qualified=True)
def object_url(obj): """Returns the URL for an object in cloud storage.""" if handlers.is_production(): return 'https://commondatastorage.googleapis.com/' + _object_path(obj) else: return '/gs_/' + urllib.quote(obj)
def modify_object(obj, content_encoding=None, content_type=None, content_disposition=None, acl=None, copy_source=None, copy_source_if_match=None, copy_source_if_none_match=None, copy_source_if_modified_since=None, copy_source_if_unmodified_since=None, copy_metadata=True, metadata={}): """Modifies or copies a cloud storage object. Most arguments are identical to the form fields listed in https://developers.google.com/storage/docs/reference-methods#putobject, but there are a few differences: * The copy_metadata argument can be True, indicating that the metadata should be copied, or False, indicating that it should be replaced. * The metadata argument is a dictionary of metadata header names to values. Each one is transformed into an x-goog-meta- field. The keys should not include "x-goog-meta-". Null values are ignored. """ if not handlers.is_production(): # The only way to modify an existing object using only the Python API # seems to be to copy it over itself. It's not a big deal since this is # only for development. if copy_source is None: copy_source = obj contents = None with files.open(_appengine_object_path(copy_source), 'r') as f: contents = f.read() if content_type is None: content_type = 'application/octet-stream' write_path = files.gs.create(_appengine_object_path(obj), mime_type=content_type, acl=acl, content_encoding=content_encoding, content_disposition=content_disposition, user_metadata=metadata) with files.open(write_path, 'a') as f: f.write(contents) files.finalize(write_path) return auth = "OAuth " + app_identity.get_access_token(_FULL_CONTROL_SCOPE)[0] headers = { "Authorization": auth, "Content-Encoding": content_encoding, "Content-Type": content_type, "Content-Disposition": content_disposition, "x-goog-api-version": "2", "x-goog-acl": acl, "x-goog-copy-source": _object_path(copy_source), "x-goog-copy-source-if-match": copy_source_if_match, "x-goog-copy-source-if-none-match": copy_source_if_none_match, "x-goog-copy-source-if-modified-since": copy_source_if_modified_since, "x-goog-copy-source-if-unmodified-since": copy_source_if_unmodified_since, "x-goog-copy-metadata-directive": "COPY" if copy_metadata else "REPLACE" } for (key, value) in metadata.iteritems(): headers["x-goog-meta-" + key] = value headers = { key: value for key, value in headers.iteritems() if value is not None } response = urlfetch.fetch("https://storage.googleapis.com/" + urllib.quote(_object_path(obj)), method="PUT", headers=headers, follow_redirects=True) if response.status_code == 200: return xml = ElementTree.XML(response.content) raise handlers.http_error( 500, "Cloud storage %s error: %s\n%s" % (response.status_code, xml.find('Code').text, xml.find('Message').text))
def test_in_production_is_false_in_tests(self): # A little sanity check to make sure the tests don't run against # production. self.assertFalse(handlers.is_production())
def object_url(obj): """Returns the URL for an object in cloud storage.""" if handlers.is_production(): return 'http://commondatastorage.googleapis.com/' + _object_path(obj) else: return '/gs_/' + urllib.quote(obj)
def __init__(self, obj, lifetime=10*60, acl=None, cache_control=None, content_disposition=None, content_encoding=None, content_type=None, expires=None, success_redirect=None, success_status=None, size_range=None, metadata={}): """Create a new Upload. Most arguments are identical to the form fields listed in https://developers.google.com/storage/docs/reference-methods#postobject, but there are a few differences: * The expires argument takes a number of seconds since the epoch. * The key argument only specifies the key name, not the bucket. * The metadata argument is a dictionary of metadata header names to values. Each one is transformed into an x-goog-meta- field. The keys should not include "x-goog-meta-". Null values are ignored. * The policy document is automatically created and signed. It ensures that all fields have the assigned values when they're submitted to Cloud Storage. The lifetime argument specifies how long the form is valid. It defaults to ten minutes. The size_range argument should be a tuple indicating the lower and upper bounds on the size of the uploaded file, in bytes. """ obj = _object_path(obj) metadata = {'x-goog-meta-' + key: value for key, value in metadata.iteritems()} if expires is not None: expires = _iso8601(expires) policy = {} policy['expiration'] = _iso8601(time.time() + lifetime) policy['conditions'] = [{'key': obj}] def _try_add_condition(name, value): if value is not None: policy['conditions'].append({name: value}) _try_add_condition('acl', acl) _try_add_condition('cache-control', cache_control) _try_add_condition('content-disposition', content_disposition) _try_add_condition('content-encoding', content_encoding) _try_add_condition('content-type', content_type) _try_add_condition('expires', expires) _try_add_condition('success_action_redirect', success_redirect) _try_add_condition('success_action_status', success_status) for key, value in metadata.items(): _try_add_condition(key, value) if size_range is not None: policy['conditions'].append( ['content-length-range', size_range[0], size_range[1]]) policy = b64encode(json.dumps(policy)) signature = PrivateKey.sign(policy) self._fields = {'key': obj, 'acl': acl, 'Cache-Control': cache_control, 'Content-Disposition': content_disposition, 'Content-Encoding': content_encoding, 'Content-Type': content_type, 'expires': expires, 'GoogleAccessId': _ACCESS_KEY, 'policy': policy, 'signature': signature, 'success_action_redirect': success_redirect, 'success_action_status': success_status} self._fields.update(metadata) if handlers.is_production(): self._url = "https://storage.googleapis.com" else: self._url = routes.url_for(controller="versions", action="upload", package_id=None, qualified=True)