def __init__(self): """Constructor. Args: blob_storage: A BlobStorage instance. """ self.__cgi_handler = dev_appserver_upload.UploadCGIHandler( get_blob_storage())
def post(self, app_id="blob", session_id = "session"): global datastore_path #file = self.request.files['file'][0] db = datastore_distributed.DatastoreDistributed( app_id, datastore_path, False, False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db) os.environ['APPLICATION_ID'] = app_id # setup the app id in the datastore # Get session info and upload success path blob_session = get_session(session_id) if not blob_session: self.finish('Session has expired. Contact the owner of the app for support.\n\n') return success_path = blob_session["success_path"] server_host = success_path[:success_path.rfind("/",3)] if server_host.startswith("http://"): # strip off the beginging server_host = server_host[len("http://"):] server_host = server_host.split('/')[0] blob_storage = datastore_blob_storage.DatastoreBlobStorage("", app_id) uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage) datastore.Delete(blob_session) # This request is sent to the upload handler of the app # in the hope it returns a redirect to be forwarded to the user urlrequest = urllib2.Request(success_path) # Forward all relevant headers # Create data for request reqbody = self.request.body content_type = self.request.headers["Content-Type"] main, kv = split_content_type(content_type) boundary = None if "boundary" in kv: boundary = kv["boundary"] urlrequest.add_header("Content-Type",'multipart/form-data; boundary="%s"'%boundary) for name, value in self.request.headers.items(): if name.lower() not in STRIPPED_HEADERS: urlrequest.add_header(name, value) # Get correct redirect addresses, otherwise it will redirect back # to this port urlrequest.add_header("Host",server_host) form = MultiPartForm(boundary) creation = datetime.datetime.now() # Loop on all files in the form for filekey in self.request.files.keys(): file = self.request.files[filekey][0] body = file["body"] size = len(body) filetype = file["content_type"] filename = file["filename"] blob_entity = uploadhandler.StoreBlob(file, creation) blob_key = str(blob_entity.key().name()) if not blob_key: self.finish('Status: 500\n\n') return creation_formatted = blobstore._format_creation(creation) form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key, blobstore.BLOB_KEY_HEADER, size, creation_formatted) # Loop through form fields for fieldkey in self.request.arguments.keys(): form.add_field(fieldkey, self.request.arguments[fieldkey][0]) request_body = str(form) urlrequest.add_header("Content-Length",str(len(request_body))) urlrequest.add_data(request_body) opener = urllib2.build_opener(SmartRedirectHandler()) f = None redirect_path = None # We are catching the redirect error here # and extracting the Location to post the redirect try: f = opener.open(urlrequest) output = f.read() self.finish(output) except urllib2.HTTPError, e: if "Location" in e.hdrs: #catch any errors, use the success path to #get the ip and port, use the redirect path #for the path. We split redirect_path just in case #its a full path redirect_path = e.hdrs["Location"] success_path_toks = success_path.split('/') redirect_toks = redirect_path.split("/") final_redirect_path = success_path_toks[0] + '//' + success_path_toks[2] + '/' + redirect_toks[len(redirect_toks)-1] self.redirect(final_redirect_path) return else: self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e)) return
def post(self, session_id="session"): """ Handler a post request from a user uploading a blob. Args: session_id: Authentication token to validate the upload. """ app_id = self.request.headers.get('X-Appengine-Inbound-Appid', '') global datastore_path db = datastore_distributed.DatastoreDistributed(app_id, datastore_path) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db) os.environ['APPLICATION_ID'] = app_id # Setup the app id in the datastore. # Get session info and upload success path. blob_session = get_session(session_id) if not blob_session: self.finish('Session has expired. Contact the owner of the ' + \ 'app for support.\n\n') return success_path = blob_session["success_path"] if success_path.startswith('/'): success_path = urlparse.urljoin(self.request.full_url(), success_path) server_host = success_path[:success_path.rfind("/", 3)] if server_host.startswith("http://"): # Strip off the beginging of the server host server_host = server_host[len("http://"):] server_host = server_host.split('/')[0] blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id) uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage) datastore.Delete(blob_session) # This request is sent to the upload handler of the app # in the hope it returns a redirect to be forwarded to the user urlrequest = urllib2.Request(success_path) # Forward all relevant headers and create data for request content_type = self.request.headers["Content-Type"] kv = split_content_type(content_type) boundary = None if "boundary" in kv: boundary = kv["boundary"] urlrequest.add_header("Content-Type", 'application/x-www-form-urlencoded') urlrequest.add_header('X-AppEngine-BlobUpload', 'true') for name, value in self.request.headers.items(): if name.lower() not in STRIPPED_HEADERS: urlrequest.add_header(name, value) # Get correct redirect addresses, otherwise it will redirect back # to this port. urlrequest.add_header("Host", server_host) form = MultiPartForm(boundary) creation = datetime.datetime.now() # Loop on all files in the form. for filekey in self.request.files.keys(): data = {"blob_info_metadata": {filekey: []}} file = self.request.files[filekey][0] body = file["body"] size = len(body) filename = file["filename"] file_content_type = file["content_type"] gs_path = '' if 'gcs_bucket' in blob_session: gcs_config = {'scheme': 'https', 'port': 443} try: gcs_config.update(deployment_config.get_config('gcs')) except ConfigInaccessible: self.send_error('Unable to fetch GCS configuration.') return if 'host' not in gcs_config: self.send_error('GCS host is not defined.') return gcs_path = '{scheme}://{host}:{port}'.format(**gcs_config) gcs_bucket_name = blob_session['gcs_bucket'] gcs_url = '/'.join([gcs_path, gcs_bucket_name, filename]) response = requests.post(gcs_url, headers={'x-goog-resumable': 'start'}) if (response.status_code != 201 or GCS_UPLOAD_ID_HEADER not in response.headers): self.send_error( reason='Unable to start resumable GCS upload.') return upload_id = response.headers[GCS_UPLOAD_ID_HEADER] total_chunks = int(math.ceil(float(size) / GCS_CHUNK_SIZE)) for chunk_num in range(total_chunks): offset = GCS_CHUNK_SIZE * chunk_num current_chunk_size = min(GCS_CHUNK_SIZE, size - offset) end_byte = offset + current_chunk_size current_range = '{}-{}'.format(offset, end_byte - 1) content_range = 'bytes {}/{}'.format(current_range, size) response = requests.put( gcs_url, data=body[offset:end_byte], headers={'Content-Range': content_range}, params={'upload_id': upload_id}) if chunk_num == total_chunks - 1: if response.status_code != 200: self.send_error( reason='Unable to complete GCS upload.') return else: if response.status_code != 308: self.send_error( reason='Unable to continue GCS upload.') return gs_path = '/gs/{}/{}'.format(gcs_bucket_name, filename) blob_key = 'encoded_gs_key:' + base64.b64encode(gs_path) else: form_item = cgi.FieldStorage( headers={'content-type': file_content_type}) form_item.file = cStringIO.StringIO(body) form_item.filename = filename blob_entity = uploadhandler.StoreBlob(form_item, creation) blob_key = str(blob_entity.key().name()) if not blob_key: self.finish('Status: 500\n\n') return creation_formatted = blobstore._format_creation(creation) form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key, blobstore.BLOB_KEY_HEADER, size, creation_formatted) md5_handler = hashlib.md5(str(body)) blob_info = { "filename": filename, "creation-date": creation_formatted, "key": blob_key, "size": str(size), "content-type": file_content_type, "md5-hash": md5_handler.hexdigest() } if 'gcs_bucket' in blob_session: blob_info['gs-name'] = gs_path data["blob_info_metadata"][filekey].append(blob_info) # Loop through form fields for fieldkey in self.request.arguments.keys(): form.add_field(fieldkey, self.request.arguments[fieldkey][0]) data[fieldkey] = self.request.arguments[fieldkey][0] logger.debug("Callback data: \n{}".format(data)) data = urllib.urlencode(data) urlrequest.add_header("Content-Length", str(len(data))) urlrequest.add_data(data) # We are catching the redirect error here # and extracting the Location to post the redirect. try: response = urllib2.urlopen(urlrequest) output = response.read() if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(output) f = gzip.GzipFile(fileobj=buf) data = f.read() output = data self.finish(output) except urllib2.HTTPError, e: if "Location" in e.hdrs: # Catch any errors, use the success path to # get the ip and port, use the redirect path # for the path. We split redirect_path just in case # its a full path. redirect_path = e.hdrs["Location"] self.redirect(redirect_path) return else: self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e)) return
def post(self, app_id="blob", session_id="session"): """ Handler a post request from a user uploading a blob. Args: app_id: The application triggering the upload. session_id: Authentication token to validate the upload. """ global datastore_path db = datastore_distributed.DatastoreDistributed(app_id, datastore_path, require_indexes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db) os.environ['APPLICATION_ID'] = app_id # Setup the app id in the datastore. # Get session info and upload success path. blob_session = get_session(session_id) if not blob_session: self.finish('Session has expired. Contact the owner of the ' + \ 'app for support.\n\n') return success_path = blob_session["success_path"] server_host = success_path[:success_path.rfind("/", 3)] if server_host.startswith("http://"): # Strip off the beginging of the server host server_host = server_host[len("http://"):] server_host = server_host.split('/')[0] blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id) uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage) datastore.Delete(blob_session) # This request is sent to the upload handler of the app # in the hope it returns a redirect to be forwarded to the user urlrequest = urllib2.Request(success_path) # Forward all relevant headers and create data for request content_type = self.request.headers["Content-Type"] kv = split_content_type(content_type) boundary = None if "boundary" in kv: boundary = kv["boundary"] urlrequest.add_header("Content-Type", 'application/x-www-form-urlencoded') for name, value in self.request.headers.items(): if name.lower() not in STRIPPED_HEADERS: urlrequest.add_header(name, value) # Get correct redirect addresses, otherwise it will redirect back # to this port. urlrequest.add_header("Host", server_host) form = MultiPartForm(boundary) creation = datetime.datetime.now() # Loop on all files in the form. for filekey in self.request.files.keys(): data = {"blob_info_metadata": {filekey: []}} file = self.request.files[filekey][0] body = file["body"] size = len(body) filename = file["filename"] file_content_type = file["content_type"] blob_entity = uploadhandler.StoreBlob(file, creation) blob_key = str(blob_entity.key().name()) if not blob_key: self.finish('Status: 500\n\n') return creation_formatted = blobstore._format_creation(creation) form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key, blobstore.BLOB_KEY_HEADER, size, creation_formatted) md5_handler = hashlib.md5(str(body)) data["blob_info_metadata"][filekey].append({ "filename": filename, "creation-date": creation_formatted, "key": blob_key, "size": str(size), "content-type": file_content_type, "md5-hash": md5_handler.hexdigest() }) # Loop through form fields for fieldkey in self.request.arguments.keys(): form.add_field(fieldkey, self.request.arguments[fieldkey][0]) data[fieldkey] = self.request.arguments[fieldkey][0] logging.debug("Callback data: \n{}".format(data)) data = urllib.urlencode(data) urlrequest.add_header("Content-Length", str(len(data))) urlrequest.add_data(data) # We are catching the redirect error here # and extracting the Location to post the redirect. try: response = urllib2.urlopen(urlrequest) output = response.read() if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(output) f = gzip.GzipFile(fileobj=buf) data = f.read() output = data self.finish(output) except urllib2.HTTPError, e: if "Location" in e.hdrs: # Catch any errors, use the success path to # get the ip and port, use the redirect path # for the path. We split redirect_path just in case # its a full path. redirect_path = e.hdrs["Location"] self.redirect(redirect_path) return else: self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e)) return