def __init__(self):
"""Constructor.
Args:
blob_storage: A BlobStorage instance.
"""
self.__cgi_handler = dev_appserver_upload.UploadCGIHandler(
get_blob_storage())
def post(self, app_id="blob", session_id = "session"):
global datastore_path
#file = self.request.files['file'][0]
db = datastore_distributed.DatastoreDistributed(
app_id, datastore_path, False, False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
os.environ['APPLICATION_ID'] = app_id
# setup the app id in the datastore
# Get session info and upload success path
blob_session = get_session(session_id)
if not blob_session:
self.finish('Session has expired. Contact the owner of the app for support.\n\n')
return
success_path = blob_session["success_path"]
server_host = success_path[:success_path.rfind("/",3)]
if server_host.startswith("http://"):
# strip off the beginging
server_host = server_host[len("http://"):]
server_host = server_host.split('/')[0]
blob_storage = datastore_blob_storage.DatastoreBlobStorage("", app_id)
uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)
datastore.Delete(blob_session)
# This request is sent to the upload handler of the app
# in the hope it returns a redirect to be forwarded to the user
urlrequest = urllib2.Request(success_path)
# Forward all relevant headers
# Create data for request
reqbody = self.request.body
content_type = self.request.headers["Content-Type"]
main, kv = split_content_type(content_type)
boundary = None
if "boundary" in kv:
boundary = kv["boundary"]
urlrequest.add_header("Content-Type",'multipart/form-data; boundary="%s"'%boundary)
for name, value in self.request.headers.items():
if name.lower() not in STRIPPED_HEADERS:
urlrequest.add_header(name, value)
# Get correct redirect addresses, otherwise it will redirect back
# to this port
urlrequest.add_header("Host",server_host)
form = MultiPartForm(boundary)
creation = datetime.datetime.now()
# Loop on all files in the form
for filekey in self.request.files.keys():
file = self.request.files[filekey][0]
body = file["body"]
size = len(body)
filetype = file["content_type"]
filename = file["filename"]
blob_entity = uploadhandler.StoreBlob(file, creation)
blob_key = str(blob_entity.key().name())
if not blob_key:
self.finish('Status: 500\n\n')
return
creation_formatted = blobstore._format_creation(creation)
form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key,
blobstore.BLOB_KEY_HEADER, size, creation_formatted)
# Loop through form fields
for fieldkey in self.request.arguments.keys():
form.add_field(fieldkey, self.request.arguments[fieldkey][0])
request_body = str(form)
urlrequest.add_header("Content-Length",str(len(request_body)))
urlrequest.add_data(request_body)
opener = urllib2.build_opener(SmartRedirectHandler())
f = None
redirect_path = None
# We are catching the redirect error here
# and extracting the Location to post the redirect
try:
f = opener.open(urlrequest)
output = f.read()
self.finish(output)
except urllib2.HTTPError, e:
if "Location" in e.hdrs:
#catch any errors, use the success path to
#get the ip and port, use the redirect path
#for the path. We split redirect_path just in case
#its a full path
redirect_path = e.hdrs["Location"]
success_path_toks = success_path.split('/')
redirect_toks = redirect_path.split("/")
final_redirect_path = success_path_toks[0] + '//' + success_path_toks[2] + '/' + redirect_toks[len(redirect_toks)-1]
self.redirect(final_redirect_path)
return
else:
self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e))
return
def post(self, session_id="session"):
""" Handler a post request from a user uploading a blob.
Args:
session_id: Authentication token to validate the upload.
"""
app_id = self.request.headers.get('X-Appengine-Inbound-Appid', '')
global datastore_path
db = datastore_distributed.DatastoreDistributed(app_id, datastore_path)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
os.environ['APPLICATION_ID'] = app_id
# Setup the app id in the datastore.
# Get session info and upload success path.
blob_session = get_session(session_id)
if not blob_session:
self.finish('Session has expired. Contact the owner of the ' + \
'app for support.\n\n')
return
success_path = blob_session["success_path"]
if success_path.startswith('/'):
success_path = urlparse.urljoin(self.request.full_url(),
success_path)
server_host = success_path[:success_path.rfind("/", 3)]
if server_host.startswith("http://"):
# Strip off the beginging of the server host
server_host = server_host[len("http://"):]
server_host = server_host.split('/')[0]
blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id)
uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)
datastore.Delete(blob_session)
# This request is sent to the upload handler of the app
# in the hope it returns a redirect to be forwarded to the user
urlrequest = urllib2.Request(success_path)
# Forward all relevant headers and create data for request
content_type = self.request.headers["Content-Type"]
kv = split_content_type(content_type)
boundary = None
if "boundary" in kv:
boundary = kv["boundary"]
urlrequest.add_header("Content-Type",
'application/x-www-form-urlencoded')
urlrequest.add_header('X-AppEngine-BlobUpload', 'true')
for name, value in self.request.headers.items():
if name.lower() not in STRIPPED_HEADERS:
urlrequest.add_header(name, value)
# Get correct redirect addresses, otherwise it will redirect back
# to this port.
urlrequest.add_header("Host", server_host)
form = MultiPartForm(boundary)
creation = datetime.datetime.now()
# Loop on all files in the form.
for filekey in self.request.files.keys():
data = {"blob_info_metadata": {filekey: []}}
file = self.request.files[filekey][0]
body = file["body"]
size = len(body)
filename = file["filename"]
file_content_type = file["content_type"]
gs_path = ''
if 'gcs_bucket' in blob_session:
gcs_config = {'scheme': 'https', 'port': 443}
try:
gcs_config.update(deployment_config.get_config('gcs'))
except ConfigInaccessible:
self.send_error('Unable to fetch GCS configuration.')
return
if 'host' not in gcs_config:
self.send_error('GCS host is not defined.')
return
gcs_path = '{scheme}://{host}:{port}'.format(**gcs_config)
gcs_bucket_name = blob_session['gcs_bucket']
gcs_url = '/'.join([gcs_path, gcs_bucket_name, filename])
response = requests.post(gcs_url,
headers={'x-goog-resumable': 'start'})
if (response.status_code != 201
or GCS_UPLOAD_ID_HEADER not in response.headers):
self.send_error(
reason='Unable to start resumable GCS upload.')
return
upload_id = response.headers[GCS_UPLOAD_ID_HEADER]
total_chunks = int(math.ceil(float(size) / GCS_CHUNK_SIZE))
for chunk_num in range(total_chunks):
offset = GCS_CHUNK_SIZE * chunk_num
current_chunk_size = min(GCS_CHUNK_SIZE, size - offset)
end_byte = offset + current_chunk_size
current_range = '{}-{}'.format(offset, end_byte - 1)
content_range = 'bytes {}/{}'.format(current_range, size)
response = requests.put(
gcs_url,
data=body[offset:end_byte],
headers={'Content-Range': content_range},
params={'upload_id': upload_id})
if chunk_num == total_chunks - 1:
if response.status_code != 200:
self.send_error(
reason='Unable to complete GCS upload.')
return
else:
if response.status_code != 308:
self.send_error(
reason='Unable to continue GCS upload.')
return
gs_path = '/gs/{}/{}'.format(gcs_bucket_name, filename)
blob_key = 'encoded_gs_key:' + base64.b64encode(gs_path)
else:
form_item = cgi.FieldStorage(
headers={'content-type': file_content_type})
form_item.file = cStringIO.StringIO(body)
form_item.filename = filename
blob_entity = uploadhandler.StoreBlob(form_item, creation)
blob_key = str(blob_entity.key().name())
if not blob_key:
self.finish('Status: 500\n\n')
return
creation_formatted = blobstore._format_creation(creation)
form.add_file(filekey, filename, cStringIO.StringIO(blob_key),
blob_key, blobstore.BLOB_KEY_HEADER, size,
creation_formatted)
md5_handler = hashlib.md5(str(body))
blob_info = {
"filename": filename,
"creation-date": creation_formatted,
"key": blob_key,
"size": str(size),
"content-type": file_content_type,
"md5-hash": md5_handler.hexdigest()
}
if 'gcs_bucket' in blob_session:
blob_info['gs-name'] = gs_path
data["blob_info_metadata"][filekey].append(blob_info)
# Loop through form fields
for fieldkey in self.request.arguments.keys():
form.add_field(fieldkey, self.request.arguments[fieldkey][0])
data[fieldkey] = self.request.arguments[fieldkey][0]
logger.debug("Callback data: \n{}".format(data))
data = urllib.urlencode(data)
urlrequest.add_header("Content-Length", str(len(data)))
urlrequest.add_data(data)
# We are catching the redirect error here
# and extracting the Location to post the redirect.
try:
response = urllib2.urlopen(urlrequest)
output = response.read()
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(output)
f = gzip.GzipFile(fileobj=buf)
data = f.read()
output = data
self.finish(output)
except urllib2.HTTPError, e:
if "Location" in e.hdrs:
# Catch any errors, use the success path to
# get the ip and port, use the redirect path
# for the path. We split redirect_path just in case
# its a full path.
redirect_path = e.hdrs["Location"]
self.redirect(redirect_path)
return
else:
self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" +
str(e))
return
def post(self, app_id="blob", session_id="session"):
""" Handler a post request from a user uploading a blob.
Args:
app_id: The application triggering the upload.
session_id: Authentication token to validate the upload.
"""
global datastore_path
db = datastore_distributed.DatastoreDistributed(app_id,
datastore_path,
require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
os.environ['APPLICATION_ID'] = app_id
# Setup the app id in the datastore.
# Get session info and upload success path.
blob_session = get_session(session_id)
if not blob_session:
self.finish('Session has expired. Contact the owner of the ' + \
'app for support.\n\n')
return
success_path = blob_session["success_path"]
server_host = success_path[:success_path.rfind("/", 3)]
if server_host.startswith("http://"):
# Strip off the beginging of the server host
server_host = server_host[len("http://"):]
server_host = server_host.split('/')[0]
blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id)
uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)
datastore.Delete(blob_session)
# This request is sent to the upload handler of the app
# in the hope it returns a redirect to be forwarded to the user
urlrequest = urllib2.Request(success_path)
# Forward all relevant headers and create data for request
content_type = self.request.headers["Content-Type"]
kv = split_content_type(content_type)
boundary = None
if "boundary" in kv:
boundary = kv["boundary"]
urlrequest.add_header("Content-Type",
'application/x-www-form-urlencoded')
for name, value in self.request.headers.items():
if name.lower() not in STRIPPED_HEADERS:
urlrequest.add_header(name, value)
# Get correct redirect addresses, otherwise it will redirect back
# to this port.
urlrequest.add_header("Host", server_host)
form = MultiPartForm(boundary)
creation = datetime.datetime.now()
# Loop on all files in the form.
for filekey in self.request.files.keys():
data = {"blob_info_metadata": {filekey: []}}
file = self.request.files[filekey][0]
body = file["body"]
size = len(body)
filename = file["filename"]
file_content_type = file["content_type"]
blob_entity = uploadhandler.StoreBlob(file, creation)
blob_key = str(blob_entity.key().name())
if not blob_key:
self.finish('Status: 500\n\n')
return
creation_formatted = blobstore._format_creation(creation)
form.add_file(filekey, filename, cStringIO.StringIO(blob_key),
blob_key, blobstore.BLOB_KEY_HEADER, size,
creation_formatted)
md5_handler = hashlib.md5(str(body))
data["blob_info_metadata"][filekey].append({
"filename":
filename,
"creation-date":
creation_formatted,
"key":
blob_key,
"size":
str(size),
"content-type":
file_content_type,
"md5-hash":
md5_handler.hexdigest()
})
# Loop through form fields
for fieldkey in self.request.arguments.keys():
form.add_field(fieldkey, self.request.arguments[fieldkey][0])
data[fieldkey] = self.request.arguments[fieldkey][0]
logging.debug("Callback data: \n{}".format(data))
data = urllib.urlencode(data)
urlrequest.add_header("Content-Length", str(len(data)))
urlrequest.add_data(data)
# We are catching the redirect error here
# and extracting the Location to post the redirect.
try:
response = urllib2.urlopen(urlrequest)
output = response.read()
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(output)
f = gzip.GzipFile(fileobj=buf)
data = f.read()
output = data
self.finish(output)
except urllib2.HTTPError, e:
if "Location" in e.hdrs:
# Catch any errors, use the success path to
# get the ip and port, use the redirect path
# for the path. We split redirect_path just in case
# its a full path.
redirect_path = e.hdrs["Location"]
self.redirect(redirect_path)
return
else:
self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" +
str(e))
return
