Exemplo n.º 1
0
  def post(self, app_id="blob", session_id = "session"):
    """ Handler a post request from a user uploading a blob. 
    
    Args:
      app_id: The application triggering the upload.
      session_id: Authentication token to validate the upload.
    """
    global datastore_path
    db = datastore_distributed.DatastoreDistributed(
      app_id, datastore_path, require_indexes=False)
    apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
    os.environ['APPLICATION_ID'] = app_id

    # Setup the app id in the datastore.
    # Get session info and upload success path.
    blob_session = get_session(session_id)
    if not blob_session:
      self.finish('Session has expired. Contact the owner of the ' + \
                  'app for support.\n\n')
      return

    success_path = blob_session["success_path"]

    server_host = success_path[:success_path.rfind("/", 3)]
    if server_host.startswith("http://"):
      # Strip off the beginging of the server host
      server_host = server_host[len("http://"):]
    server_host = server_host.split('/')[0]

    blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id)
    uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)

    datastore.Delete(blob_session)

    # This request is sent to the upload handler of the app
    # in the hope it returns a redirect to be forwarded to the user
    urlrequest = urllib2.Request(success_path)

    # Forward all relevant headers and create data for request
    content_type = self.request.headers["Content-Type"]
    kv = split_content_type(content_type)
    boundary = None
    if "boundary" in kv:
      boundary = kv["boundary"]

    urlrequest.add_header("Content-Type",
                          'application/x-www-form-urlencoded')

    for name, value in self.request.headers.items():
      if name.lower() not in STRIPPED_HEADERS:
        urlrequest.add_header(name, value)
  
    # Get correct redirect addresses, otherwise it will redirect back
    # to this port.
    urlrequest.add_header("Host", server_host)

    form = MultiPartForm(boundary)
    creation = datetime.datetime.now()

    # Loop on all files in the form.
    for filekey in self.request.files.keys():
      data = {"blob_info_metadata": {filekey: []}}
      file = self.request.files[filekey][0] 
      body = file["body"]
      size = len(body)
      filename = file["filename"]
      file_content_type = file["content_type"]
     
      blob_entity = uploadhandler.StoreBlob(file, creation)

      blob_key = str(blob_entity.key().name())

      if not blob_key: 
        self.finish('Status: 500\n\n')
        return 
      creation_formatted = blobstore._format_creation(creation)
      form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key,
                    blobstore.BLOB_KEY_HEADER, size, creation_formatted)

      md5_handler = hashlib.md5(str(body))
      data["blob_info_metadata"][filekey].append( 
        {"filename": filename, "creation-date": creation_formatted, "key": blob_key, "size": str(size),
         "content-type": file_content_type, "md5-hash": md5_handler.hexdigest()})

    # Loop through form fields
    for fieldkey in self.request.arguments.keys():
      form.add_field(fieldkey, self.request.arguments[fieldkey][0])
      data[fieldkey] = self.request.arguments[fieldkey][0]

    logging.debug("Callback data: \n{}".format(data))
    data = urllib.urlencode(data)
    urlrequest.add_header("Content-Length", str(len(data)))
    urlrequest.add_data(data)

    # We are catching the redirect error here
    # and extracting the Location to post the redirect.
    try:
      response = urllib2.urlopen(urlrequest)
      output = response.read()
      if response.info().get('Content-Encoding') == 'gzip':
        buf = StringIO(output)
        f = gzip.GzipFile(fileobj=buf)
        data = f.read()
        output = data
      self.finish(output)
    except urllib2.HTTPError, e: 
      if "Location" in e.hdrs:
        # Catch any errors, use the success path to 
        # get the ip and port, use the redirect path
        # for the path. We split redirect_path just in case
        # its a full path.
        redirect_path = e.hdrs["Location"]
        self.redirect(redirect_path)
        return
      else:
        self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e))
        return         
Exemplo n.º 2
0
  def _GenerateMIMEMessage(self, form, boundary=None):
    """Generate a new post from original form.

    Also responsible for storing blobs in the datastore.

    Args:
      form: Instance of cgi.FieldStorage representing the whole form
        derived from original post data.
      boundary: Boundary to use for resulting form.  Used only in tests so
        that the boundary is always consistent.

    Returns:
      A MIMEMultipart instance representing the new HTTP post which should be
      forwarded to the developers actual CGI handler. DO NOT use the return
      value of this method to generate a string unless you know what you're
      doing and properly handle folding whitespace (from rfc822) properly.
    """
    message = multipart.MIMEMultipart('form-data', boundary)
    for name, value in form.headers.items():
      if name.lower() not in STRIPPED_HEADERS:
        message.add_header(name, value)

    def IterateForm():
      """Flattens form in to single sequence of cgi.FieldStorage instances.

      The resulting cgi.FieldStorage objects are a little bit irregular in
      their structure.  A single name can have mulitple sub-items.  In this
      case, the root FieldStorage object has a list associated with that field
      name.  Otherwise, the root FieldStorage object just refers to a single
      nested instance.

      Lists of FieldStorage instances occur when a form has multiple values
      for the same name.

      Yields:
        cgi.FieldStorage irrespective of their nesting level.
      """
      for key in sorted(form):
        form_item = form[key]
        if isinstance(form_item, list):
          for list_item in form_item:
            yield list_item
        else:
          yield form_item

    creation = self.__now_func()
    for form_item in IterateForm():

      disposition_parameters = {'name': form_item.name}

      if form_item.filename is None:
        variable = base.MIMEBase('text', 'plain')
        variable.set_payload(form_item.value)
      else:
        if not form_item.filename:
          continue

        disposition_parameters['filename'] = form_item.filename

        main_type, sub_type = _SplitMIMEType(form_item.type)

        blob_entity = self.StoreBlob(form_item, creation)

        variable = base.MIMEBase('message',
                                 'external-body',
                                 access_type=blobstore.BLOB_KEY_HEADER,
                                 blob_key=blob_entity.key().name())

        form_item.file.seek(0, 2)
        content_length = form_item.file.tell()
        form_item.file.seek(0)

        external = base.MIMEBase(main_type,
                                 sub_type,
                                 **form_item.type_options)
        headers = dict(form_item.headers)
        headers['Content-Length'] = str(content_length)
        headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = (
            blobstore._format_creation(creation))
        for key, value in headers.iteritems():
          external.add_header(key, value)

        external_disposition_parameters = dict(disposition_parameters)
        external_disposition_parameters['filename'] = form_item.filename
        if not external.get('Content-Disposition'):
          external.add_header('Content-Disposition',
                              'form-data',
                              **external_disposition_parameters)
        variable.set_payload([external])

      variable.add_header('Content-Disposition',
                          'form-data',
                          **disposition_parameters)
      message.attach(variable)

    return message
  def _GenerateMIMEMessage(self,
                           form,
                           boundary=None,
                           max_bytes_per_blob=None,
                           max_bytes_total=None,
                           bucket_name=None):
    """Generate a new post from original form.

    Also responsible for storing blobs in the datastore.

    Args:
      form: Instance of cgi.FieldStorage representing the whole form
        derived from original post data.
      boundary: Boundary to use for resulting form.  Used only in tests so
        that the boundary is always consistent.
      max_bytes_per_blob: The maximum size in bytes that any single blob
        in the form is allowed to be.
      max_bytes_total: The maximum size in bytes that the total of all blobs
        in the form is allowed to be.
      bucket_name: The name of the Google Storage bucket to uplad the file.

    Returns:
      A MIMEMultipart instance representing the new HTTP post which should be
      forwarded to the developers actual CGI handler. DO NOT use the return
      value of this method to generate a string unless you know what you're
      doing and properly handle folding whitespace (from rfc822) properly.

    Raises:
      UploadEntityTooLargeError: The upload exceeds either the
        max_bytes_per_blob or max_bytes_total limits.
      FilenameOrContentTypeTooLargeError: The filename or the content_type of
        the upload is larger than the allowed size for a string type in the
        datastore.
    """
    message = multipart.MIMEMultipart('form-data', boundary)
    for name, value in form.headers.items():
      if name.lower() not in STRIPPED_HEADERS:
        message.add_header(name, value)

    def IterateForm():
      """Flattens form in to single sequence of cgi.FieldStorage instances.

      The resulting cgi.FieldStorage objects are a little bit irregular in
      their structure.  A single name can have mulitple sub-items.  In this
      case, the root FieldStorage object has a list associated with that field
      name.  Otherwise, the root FieldStorage object just refers to a single
      nested instance.

      Lists of FieldStorage instances occur when a form has multiple values
      for the same name.

      Yields:
        cgi.FieldStorage irrespective of their nesting level.
      """


      for key in sorted(form):
        form_item = form[key]
        if isinstance(form_item, list):
          for list_item in form_item:
            yield list_item
        else:
          yield form_item

    creation = self.__now_func()
    total_bytes_uploaded = 0
    created_blobs = []
    upload_too_large = False
    filename_too_large = False
    content_type_too_large = False

    for form_item in IterateForm():








      disposition_parameters = {'name': form_item.name}

      if form_item.filename is None:

        variable = base.MIMEBase('text', 'plain')
        variable.set_payload(form_item.value)
      else:



        if not form_item.filename:
          continue

        disposition_parameters['filename'] = form_item.filename

        main_type, sub_type = _SplitMIMEType(form_item.type)


        form_item.file.seek(0, 2)
        content_length = form_item.file.tell()
        form_item.file.seek(0)

        total_bytes_uploaded += content_length

        if max_bytes_per_blob is not None:
          if max_bytes_per_blob < content_length:
            upload_too_large = True
            break
        if max_bytes_total is not None:
          if max_bytes_total < total_bytes_uploaded:
            upload_too_large = True
            break
        if form_item.filename is not None:
          if MAX_STRING_NAME_LENGTH < len(form_item.filename):
            filename_too_large = True
            break
        if form_item.type is not None:
          if MAX_STRING_NAME_LENGTH < len(form_item.type):
            content_type_too_large = True
            break


        blob_entity = self.StoreBlob(form_item, creation)


        created_blobs.append(blob_entity)

        variable = base.MIMEBase('message',
                                 'external-body',
                                 access_type=blobstore.BLOB_KEY_HEADER,
                                 blob_key=blob_entity.key().name())


        form_item.file.seek(0)
        digester = hashlib.md5()
        while True:
          block = form_item.file.read(1 << 20)
          if not block:
            break
          digester.update(block)

        blob_key = base64.urlsafe_b64encode(digester.hexdigest())
        form_item.file.seek(0)

        external = base.MIMEBase(main_type,
                                 sub_type,
                                 **form_item.type_options)
        headers = dict(form_item.headers)
        headers['Content-Length'] = str(content_length)
        headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = (
            blobstore._format_creation(creation))
        if bucket_name:
          headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = (
              '/gs/%s/fake-%s-%s' % (bucket_name, blob_entity.key().name(),
                                     blob_key))
        headers['Content-MD5'] = blob_key
        for key, value in headers.iteritems():
          external.add_header(key, value)


        external_disposition_parameters = dict(disposition_parameters)


        external_disposition_parameters['filename'] = form_item.filename
        if not external.get('Content-Disposition'):
          external.add_header('Content-Disposition',
                              'form-data',
                              **external_disposition_parameters)
        variable.set_payload([external])


      variable.add_header('Content-Disposition',
                          'form-data',
                          **disposition_parameters)
      message.attach(variable)

    if upload_too_large or filename_too_large or content_type_too_large:
      for blob in created_blobs:
        datastore.Delete(blob)
      if upload_too_large:
        raise UploadEntityTooLargeError()
      elif filename_too_large:
        raise FilenameOrContentTypeTooLargeError('filename')
      else:
        raise FilenameOrContentTypeTooLargeError('content-type')

    return message
Exemplo n.º 4
0
    def _GenerateMIMEMessage(self, form, boundary=None):
        """Generate a new post from original form.

    Also responsible for storing blobs in the datastore.

    Args:
      form: Instance of cgi.FieldStorage representing the whole form
        derived from original post data.
      boundary: Boundary to use for resulting form.  Used only in tests so
        that the boundary is always consistent.

    Returns:
      A MIMEMultipart instance representing the new HTTP post which should be
      forwarded to the developers actual CGI handler. DO NOT use the return
      value of this method to generate a string unless you know what you're
      doing and properly handle folding whitespace (from rfc822) properly.
    """
        message = multipart.MIMEMultipart('form-data', boundary)
        for name, value in form.headers.items():
            if name.lower() not in STRIPPED_HEADERS:
                message.add_header(name, value)

        def IterateForm():
            """Flattens form in to single sequence of cgi.FieldStorage instances.

      The resulting cgi.FieldStorage objects are a little bit irregular in
      their structure.  A single name can have mulitple sub-items.  In this
      case, the root FieldStorage object has a list associated with that field
      name.  Otherwise, the root FieldStorage object just refers to a single
      nested instance.

      Lists of FieldStorage instances occur when a form has multiple values
      for the same name.

      Yields:
        cgi.FieldStorage irrespective of their nesting level.
      """

            for key in sorted(form):
                form_item = form[key]
                if isinstance(form_item, list):
                    for list_item in form_item:
                        yield list_item
                else:
                    yield form_item

        creation = self.__now_func()
        for form_item in IterateForm():

            disposition_parameters = {'name': form_item.name}

            if form_item.filename is None:

                variable = base.MIMEBase('text', 'plain')
                variable.set_payload(form_item.value)
            else:

                if not form_item.filename:
                    continue

                disposition_parameters['filename'] = form_item.filename

                main_type, sub_type = _SplitMIMEType(form_item.type)

                blob_entity = self.StoreBlob(form_item, creation)

                variable = base.MIMEBase('message',
                                         'external-body',
                                         access_type=blobstore.BLOB_KEY_HEADER,
                                         blob_key=blob_entity.key().name())

                form_item.file.seek(0, 2)
                content_length = form_item.file.tell()
                form_item.file.seek(0)

                external = base.MIMEBase(main_type, sub_type,
                                         **form_item.type_options)
                headers = dict(form_item.headers)
                headers['Content-Length'] = str(content_length)
                headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = (
                    blobstore._format_creation(creation))
                for key, value in headers.iteritems():
                    external.add_header(key, value)

                external_disposition_parameters = dict(disposition_parameters)

                external_disposition_parameters[
                    'filename'] = form_item.filename
                if not external.get('Content-Disposition'):
                    external.add_header('Content-Disposition', 'form-data',
                                        **external_disposition_parameters)
                variable.set_payload([external])

            variable.add_header('Content-Disposition', 'form-data',
                                **disposition_parameters)
            message.attach(variable)

        return message
Exemplo n.º 5
0
  def _GenerateMIMEMessage(self,
                           form,
                           boundary=None,
                           max_bytes_per_blob=None,
                           max_bytes_total=None,
                           bucket_name=None):
    """Generate a new post from original form.

    Also responsible for storing blobs in the datastore.

    Args:
      form: Instance of cgi.FieldStorage representing the whole form
        derived from original post data.
      boundary: Boundary to use for resulting form.  Used only in tests so
        that the boundary is always consistent.
      max_bytes_per_blob: The maximum size in bytes that any single blob
        in the form is allowed to be.
      max_bytes_total: The maximum size in bytes that the total of all blobs
        in the form is allowed to be.
      bucket_name: The name of the Google Storage bucket to uplad the file.

    Returns:
      A MIMEMultipart instance representing the new HTTP post which should be
      forwarded to the developers actual CGI handler. DO NOT use the return
      value of this method to generate a string unless you know what you're
      doing and properly handle folding whitespace (from rfc822) properly.

    Raises:
      UploadEntityTooLargeError: The upload exceeds either the
        max_bytes_per_blob or max_bytes_total limits.
      FilenameOrContentTypeTooLargeError: The filename or the content_type of
        the upload is larger than the allowed size for a string type in the
        datastore.
    """
    message = multipart.MIMEMultipart('form-data', boundary)
    for name, value in form.headers.items():
      if name.lower() not in STRIPPED_HEADERS:
        message.add_header(name, value)

    def IterateForm():
      """Flattens form in to single sequence of cgi.FieldStorage instances.

      The resulting cgi.FieldStorage objects are a little bit irregular in
      their structure.  A single name can have mulitple sub-items.  In this
      case, the root FieldStorage object has a list associated with that field
      name.  Otherwise, the root FieldStorage object just refers to a single
      nested instance.

      Lists of FieldStorage instances occur when a form has multiple values
      for the same name.

      Yields:
        cgi.FieldStorage irrespective of their nesting level.
      """


      for key in sorted(form):
        form_item = form[key]
        if isinstance(form_item, list):
          for list_item in form_item:
            yield list_item
        else:
          yield form_item

    creation = self.__now_func()
    total_bytes_uploaded = 0
    created_blobs = []
    upload_too_large = False
    filename_too_large = False
    content_type_too_large = False

    for form_item in IterateForm():








      disposition_parameters = {'name': form_item.name}

      if form_item.filename is None:

        variable = base.MIMEBase('text', 'plain')
        variable.set_payload(form_item.value)
      else:



        if not form_item.filename:
          continue

        disposition_parameters['filename'] = form_item.filename

        main_type, sub_type = _SplitMIMEType(form_item.type)


        form_item.file.seek(0, 2)
        content_length = form_item.file.tell()
        form_item.file.seek(0)

        total_bytes_uploaded += content_length

        if max_bytes_per_blob is not None:
          if max_bytes_per_blob < content_length:
            upload_too_large = True
            break
        if max_bytes_total is not None:
          if max_bytes_total < total_bytes_uploaded:
            upload_too_large = True
            break
        if form_item.filename is not None:
          if MAX_STRING_NAME_LENGTH < len(form_item.filename):
            filename_too_large = True
            break
        if form_item.type is not None:
          if MAX_STRING_NAME_LENGTH < len(form_item.type):
            content_type_too_large = True
            break


        blob_entity = self.StoreBlob(form_item, creation)


        created_blobs.append(blob_entity)

        variable = base.MIMEBase('message',
                                 'external-body',
                                 access_type=blobstore.BLOB_KEY_HEADER,
                                 blob_key=blob_entity.key().name())


        form_item.file.seek(0)
        digester = hashlib.md5()
        while True:
          block = form_item.file.read(1 << 20)
          if not block:
            break
          digester.update(block)

        blob_key = base64.urlsafe_b64encode(digester.hexdigest())
        form_item.file.seek(0)

        external = base.MIMEBase(main_type,
                                 sub_type,
                                 **form_item.type_options)
        headers = dict(form_item.headers)
        headers['Content-Length'] = str(content_length)
        headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = (
            blobstore._format_creation(creation))
        if bucket_name:
          headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = (
              '/gs/%s/fake-%s-%s' % (bucket_name, blob_entity.key().name(),
                                     blob_key))
        headers['Content-MD5'] = blob_key
        for key, value in headers.iteritems():
          external.add_header(key, value)


        external_disposition_parameters = dict(disposition_parameters)


        external_disposition_parameters['filename'] = form_item.filename
        if not external.get('Content-Disposition'):
          external.add_header('Content-Disposition',
                              'form-data',
                              **external_disposition_parameters)
        variable.set_payload([external])


      variable.add_header('Content-Disposition',
                          'form-data',
                          **disposition_parameters)
      message.attach(variable)

    if upload_too_large or filename_too_large or content_type_too_large:
      for blob in created_blobs:
        datastore.Delete(blob)
      if upload_too_large:
        raise UploadEntityTooLargeError()
      elif filename_too_large:
        raise FilenameOrContentTypeTooLargeError('filename')
      else:
        raise FilenameOrContentTypeTooLargeError('content-type')

    return message
Exemplo n.º 6
0
    def post(self, session_id="session"):
        """ Handler a post request from a user uploading a blob. 
    
    Args:
      session_id: Authentication token to validate the upload.
    """
        app_id = self.request.headers.get('X-Appengine-Inbound-Appid', '')
        global datastore_path
        db = datastore_distributed.DatastoreDistributed(app_id, datastore_path)
        apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
        os.environ['APPLICATION_ID'] = app_id

        # Setup the app id in the datastore.
        # Get session info and upload success path.
        blob_session = get_session(session_id)
        if not blob_session:
            self.finish('Session has expired. Contact the owner of the ' + \
                        'app for support.\n\n')
            return

        success_path = blob_session["success_path"]
        if success_path.startswith('/'):
            success_path = urlparse.urljoin(self.request.full_url(),
                                            success_path)

        server_host = success_path[:success_path.rfind("/", 3)]
        if server_host.startswith("http://"):
            # Strip off the beginging of the server host
            server_host = server_host[len("http://"):]
        server_host = server_host.split('/')[0]

        blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id)
        uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)

        datastore.Delete(blob_session)

        # This request is sent to the upload handler of the app
        # in the hope it returns a redirect to be forwarded to the user
        urlrequest = urllib2.Request(success_path)

        # Forward all relevant headers and create data for request
        content_type = self.request.headers["Content-Type"]
        kv = split_content_type(content_type)
        boundary = None
        if "boundary" in kv:
            boundary = kv["boundary"]

        urlrequest.add_header("Content-Type",
                              'application/x-www-form-urlencoded')
        urlrequest.add_header('X-AppEngine-BlobUpload', 'true')

        for name, value in self.request.headers.items():
            if name.lower() not in STRIPPED_HEADERS:
                urlrequest.add_header(name, value)

        # Get correct redirect addresses, otherwise it will redirect back
        # to this port.
        urlrequest.add_header("Host", server_host)

        form = MultiPartForm(boundary)
        creation = datetime.datetime.now()

        # Loop on all files in the form.
        for filekey in self.request.files.keys():
            data = {"blob_info_metadata": {filekey: []}}
            file = self.request.files[filekey][0]
            body = file["body"]
            size = len(body)
            filename = file["filename"]
            file_content_type = file["content_type"]

            gs_path = ''
            if 'gcs_bucket' in blob_session:
                gcs_config = {'scheme': 'https', 'port': 443}
                try:
                    gcs_config.update(deployment_config.get_config('gcs'))
                except ConfigInaccessible:
                    self.send_error('Unable to fetch GCS configuration.')
                    return

                if 'host' not in gcs_config:
                    self.send_error('GCS host is not defined.')
                    return

                gcs_path = '{scheme}://{host}:{port}'.format(**gcs_config)
                gcs_bucket_name = blob_session['gcs_bucket']
                gcs_url = '/'.join([gcs_path, gcs_bucket_name, filename])
                response = requests.post(gcs_url,
                                         headers={'x-goog-resumable': 'start'})
                if (response.status_code != 201
                        or GCS_UPLOAD_ID_HEADER not in response.headers):
                    self.send_error(
                        reason='Unable to start resumable GCS upload.')
                    return
                upload_id = response.headers[GCS_UPLOAD_ID_HEADER]

                total_chunks = int(math.ceil(float(size) / GCS_CHUNK_SIZE))
                for chunk_num in range(total_chunks):
                    offset = GCS_CHUNK_SIZE * chunk_num
                    current_chunk_size = min(GCS_CHUNK_SIZE, size - offset)
                    end_byte = offset + current_chunk_size
                    current_range = '{}-{}'.format(offset, end_byte - 1)
                    content_range = 'bytes {}/{}'.format(current_range, size)
                    response = requests.put(
                        gcs_url,
                        data=body[offset:end_byte],
                        headers={'Content-Range': content_range},
                        params={'upload_id': upload_id})
                    if chunk_num == total_chunks - 1:
                        if response.status_code != 200:
                            self.send_error(
                                reason='Unable to complete GCS upload.')
                            return
                    else:
                        if response.status_code != 308:
                            self.send_error(
                                reason='Unable to continue GCS upload.')
                            return
                gs_path = '/gs/{}/{}'.format(gcs_bucket_name, filename)
                blob_key = 'encoded_gs_key:' + base64.b64encode(gs_path)
            else:
                form_item = cgi.FieldStorage(
                    headers={'content-type': file_content_type})
                form_item.file = cStringIO.StringIO(body)
                form_item.filename = filename

                blob_entity = uploadhandler.StoreBlob(form_item, creation)
                blob_key = str(blob_entity.key().name())

            if not blob_key:
                self.finish('Status: 500\n\n')
                return
            creation_formatted = blobstore._format_creation(creation)
            form.add_file(filekey, filename, cStringIO.StringIO(blob_key),
                          blob_key, blobstore.BLOB_KEY_HEADER, size,
                          creation_formatted)

            md5_handler = hashlib.md5(str(body))
            blob_info = {
                "filename": filename,
                "creation-date": creation_formatted,
                "key": blob_key,
                "size": str(size),
                "content-type": file_content_type,
                "md5-hash": md5_handler.hexdigest()
            }
            if 'gcs_bucket' in blob_session:
                blob_info['gs-name'] = gs_path
            data["blob_info_metadata"][filekey].append(blob_info)

        # Loop through form fields
        for fieldkey in self.request.arguments.keys():
            form.add_field(fieldkey, self.request.arguments[fieldkey][0])
            data[fieldkey] = self.request.arguments[fieldkey][0]

        logger.debug("Callback data: \n{}".format(data))
        data = urllib.urlencode(data)
        urlrequest.add_header("Content-Length", str(len(data)))
        urlrequest.add_data(data)

        # We are catching the redirect error here
        # and extracting the Location to post the redirect.
        try:
            response = urllib2.urlopen(urlrequest)
            output = response.read()
            if response.info().get('Content-Encoding') == 'gzip':
                buf = StringIO(output)
                f = gzip.GzipFile(fileobj=buf)
                data = f.read()
                output = data
            self.finish(output)
        except urllib2.HTTPError, e:
            if "Location" in e.hdrs:
                # Catch any errors, use the success path to
                # get the ip and port, use the redirect path
                # for the path. We split redirect_path just in case
                # its a full path.
                redirect_path = e.hdrs["Location"]
                self.redirect(redirect_path)
                return
            else:
                self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" +
                            str(e))
                return
Exemplo n.º 7
0
  def post(self, app_id="blob", session_id = "session"):
    global datastore_path
    #file = self.request.files['file'][0]
    db = datastore_distributed.DatastoreDistributed(
      app_id, datastore_path, False, False)
    apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
    os.environ['APPLICATION_ID'] = app_id

    # setup the app id in the datastore
    # Get session info and upload success path
    blob_session = get_session(session_id)
    if not blob_session:
      self.finish('Session has expired. Contact the owner of the app for support.\n\n')
      return
    success_path = blob_session["success_path"]

    server_host = success_path[:success_path.rfind("/",3)]
    if server_host.startswith("http://"):
      # strip off the beginging
      server_host = server_host[len("http://"):]
    server_host = server_host.split('/')[0]

    blob_storage = datastore_blob_storage.DatastoreBlobStorage("", app_id)
    uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)

    datastore.Delete(blob_session)
    # This request is sent to the upload handler of the app
    # in the hope it returns a redirect to be forwarded to the user
    urlrequest = urllib2.Request(success_path)
    # Forward all relevant headers
    # Create data for request

    reqbody = self.request.body
    content_type = self.request.headers["Content-Type"]
    main, kv = split_content_type(content_type)
    boundary = None
    if "boundary" in kv:
      boundary = kv["boundary"]

    urlrequest.add_header("Content-Type",'multipart/form-data; boundary="%s"'%boundary)

    for name, value in self.request.headers.items():
      if name.lower() not in STRIPPED_HEADERS:
        urlrequest.add_header(name, value)
  
    # Get correct redirect addresses, otherwise it will redirect back
    # to this port
    urlrequest.add_header("Host",server_host)

    form = MultiPartForm(boundary)
    creation = datetime.datetime.now()
    # Loop on all files in the form
    for filekey in self.request.files.keys():
      file = self.request.files[filekey][0] 
      body = file["body"]
      size = len(body)
      filetype = file["content_type"]
      filename = file["filename"]
     
      blob_entity = uploadhandler.StoreBlob(file, creation)

      blob_key = str(blob_entity.key().name())

      if not blob_key: 
        self.finish('Status: 500\n\n')
        return 
      creation_formatted = blobstore._format_creation(creation)
      form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key,
                    blobstore.BLOB_KEY_HEADER, size, creation_formatted) 
    # Loop through form fields
    for fieldkey in self.request.arguments.keys():
      form.add_field(fieldkey, self.request.arguments[fieldkey][0])
    request_body = str(form)
    urlrequest.add_header("Content-Length",str(len(request_body)))
    urlrequest.add_data(request_body)

    opener = urllib2.build_opener(SmartRedirectHandler())
    f = None
    redirect_path = None
    # We are catching the redirect error here
    # and extracting the Location to post the redirect
    try:
      f = opener.open(urlrequest)
      output = f.read()
      self.finish(output)
    except urllib2.HTTPError, e: 
      if "Location" in e.hdrs:
        #catch any errors, use the success path to 
        #get the ip and port, use the redirect path
        #for the path. We split redirect_path just in case
        #its a full path  
        redirect_path = e.hdrs["Location"]
        success_path_toks = success_path.split('/')
        redirect_toks = redirect_path.split("/")
        final_redirect_path = success_path_toks[0] + '//' + success_path_toks[2] + '/' + redirect_toks[len(redirect_toks)-1]
        self.redirect(final_redirect_path)
        return
      else:
        self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e))
        return         
Exemplo n.º 8
0
  def post(self, session_id = "session"):
    """ Handler a post request from a user uploading a blob. 
    
    Args:
      session_id: Authentication token to validate the upload.
    """
    app_id = self.request.headers.get('X-Appengine-Inbound-Appid', '')
    global datastore_path
    db = datastore_distributed.DatastoreDistributed(
      app_id, datastore_path)
    apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
    os.environ['APPLICATION_ID'] = app_id

    # Setup the app id in the datastore.
    # Get session info and upload success path.
    blob_session = get_session(session_id)
    if not blob_session:
      self.finish('Session has expired. Contact the owner of the ' + \
                  'app for support.\n\n')
      return

    success_path = blob_session["success_path"]
    if success_path.startswith('/'):
      success_path = urlparse.urljoin(self.request.full_url(), success_path)

    server_host = success_path[:success_path.rfind("/", 3)]
    if server_host.startswith("http://"):
      # Strip off the beginging of the server host
      server_host = server_host[len("http://"):]
    server_host = server_host.split('/')[0]

    blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id)
    uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)

    datastore.Delete(blob_session)

    # This request is sent to the upload handler of the app
    # in the hope it returns a redirect to be forwarded to the user
    urlrequest = urllib2.Request(success_path)

    # Forward all relevant headers and create data for request
    content_type = self.request.headers["Content-Type"]
    kv = split_content_type(content_type)
    boundary = None
    if "boundary" in kv:
      boundary = kv["boundary"]

    urlrequest.add_header("Content-Type",
                          'application/x-www-form-urlencoded')

    for name, value in self.request.headers.items():
      if name.lower() not in STRIPPED_HEADERS:
        urlrequest.add_header(name, value)
  
    # Get correct redirect addresses, otherwise it will redirect back
    # to this port.
    urlrequest.add_header("Host", server_host)

    form = MultiPartForm(boundary)
    creation = datetime.datetime.now()

    # Loop on all files in the form.
    for filekey in self.request.files.keys():
      data = {"blob_info_metadata": {filekey: []}}
      file = self.request.files[filekey][0] 
      body = file["body"]
      size = len(body)
      filename = file["filename"]
      file_content_type = file["content_type"]

      gs_path = ''
      if 'gcs_bucket' in blob_session:
        gcs_config = {'scheme': 'https', 'port': 443}
        try:
          gcs_config.update(deployment_config.get_config('gcs'))
        except ConfigInaccessible:
          self.send_error('Unable to fetch GCS configuration.')
          return

        if 'host' not in gcs_config:
          self.send_error('GCS host is not defined.')
          return

        gcs_path = '{scheme}://{host}:{port}'.format(**gcs_config)
        gcs_bucket_name = blob_session['gcs_bucket']
        gcs_url = '/'.join([gcs_path, gcs_bucket_name, filename])
        response = requests.post(gcs_url,
                                 headers={'x-goog-resumable': 'start'})
        if (response.status_code != 201 or
            GCS_UPLOAD_ID_HEADER not in response.headers):
          self.send_error(reason='Unable to start resumable GCS upload.')
          return
        upload_id = response.headers[GCS_UPLOAD_ID_HEADER]

        total_chunks = int(math.ceil(float(size) / GCS_CHUNK_SIZE))
        for chunk_num in range(total_chunks):
          offset = GCS_CHUNK_SIZE * chunk_num
          current_chunk_size = min(GCS_CHUNK_SIZE, size - offset)
          end_byte = offset + current_chunk_size
          current_range = '{}-{}'.format(offset, end_byte - 1)
          content_range = 'bytes {}/{}'.format(current_range, size)
          response = requests.put(gcs_url, data=body[offset:end_byte],
                                  headers={'Content-Range': content_range},
                                  params={'upload_id': upload_id})
          if chunk_num == total_chunks - 1:
            if response.status_code != 200:
              self.send_error(reason='Unable to complete GCS upload.')
              return
          else:
            if response.status_code != 308:
              self.send_error(reason='Unable to continue GCS upload.')
              return
        gs_path = '/gs/{}/{}'.format(gcs_bucket_name, filename)
        blob_key = 'encoded_gs_key:' + base64.b64encode(gs_path)
      else:
        form_item = cgi.FieldStorage(
          headers={'content-type': file_content_type})
        form_item.file = cStringIO.StringIO(body)
        form_item.filename = filename

        blob_entity = uploadhandler.StoreBlob(form_item, creation)
        blob_key = str(blob_entity.key().name())

      if not blob_key: 
        self.finish('Status: 500\n\n')
        return 
      creation_formatted = blobstore._format_creation(creation)
      form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key,
                    blobstore.BLOB_KEY_HEADER, size, creation_formatted)

      md5_handler = hashlib.md5(str(body))
      blob_info = {"filename": filename,
                   "creation-date": creation_formatted,
                   "key": blob_key,
                   "size": str(size),
                   "content-type": file_content_type,
                   "md5-hash": md5_handler.hexdigest()}
      if 'gcs_bucket' in blob_session:
        blob_info['gs-name'] = gs_path
      data["blob_info_metadata"][filekey].append(blob_info)

    # Loop through form fields
    for fieldkey in self.request.arguments.keys():
      form.add_field(fieldkey, self.request.arguments[fieldkey][0])
      data[fieldkey] = self.request.arguments[fieldkey][0]

    logger.debug("Callback data: \n{}".format(data))
    data = urllib.urlencode(data)
    urlrequest.add_header("Content-Length", str(len(data)))
    urlrequest.add_data(data)

    # We are catching the redirect error here
    # and extracting the Location to post the redirect.
    try:
      response = urllib2.urlopen(urlrequest)
      output = response.read()
      if response.info().get('Content-Encoding') == 'gzip':
        buf = StringIO(output)
        f = gzip.GzipFile(fileobj=buf)
        data = f.read()
        output = data
      self.finish(output)
    except urllib2.HTTPError, e: 
      if "Location" in e.hdrs:
        # Catch any errors, use the success path to 
        # get the ip and port, use the redirect path
        # for the path. We split redirect_path just in case
        # its a full path.
        redirect_path = e.hdrs["Location"]
        self.redirect(redirect_path)
        return
      else:
        self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e))
        return
Exemplo n.º 9
0
  def store_and_build_forward_message(self, form, boundary=None,
                                      max_bytes_per_blob=None,
                                      max_bytes_total=None,
                                      bucket_name=None):
    """Reads form data, stores blobs data and builds the forward request.

    This finds all of the file uploads in a set of form fields, converting them
    into blobs and storing them in the blobstore. It also generates the HTTP
    request to forward to the user's application.

    Args:
      form: cgi.FieldStorage instance representing the whole form derived from
        original POST data.
      boundary: The optional boundary to use for the resulting form. If omitted,
        one is randomly generated.
      max_bytes_per_blob: The maximum size in bytes that any single blob
        in the form is allowed to be.
      max_bytes_total: The maximum size in bytes that the total of all blobs
        in the form is allowed to be.
      bucket_name: The name of the Google Storage bucket to store the uploaded
                   files.

    Returns:
      A tuple (content_type, content_text), where content_type is the value of
      the Content-Type header, and content_text is a string containing the body
      of the HTTP request to forward to the application.

    Raises:
      webob.exc.HTTPException: The upload failed.
    """
    message = multipart.MIMEMultipart('form-data', boundary)

    creation = self._now_func()
    total_bytes_uploaded = 0
    created_blobs = []
    mime_type_error = None
    too_many_conflicts = False
    upload_too_large = False
    filename_too_large = False
    content_type_too_large = False

    # Extract all of the individual form items out of the FieldStorage.
    form_items = []
    # Sorting of forms is done merely to make testing a little easier since
    # it means blob-keys are generated in a predictable order.
    for key in sorted(form):
      form_item = form[key]
      if isinstance(form_item, list):
        form_items.extend(form_item)
      else:
        form_items.append(form_item)

    for form_item in form_items:
      disposition_parameters = {'name': form_item.name}

      variable = email.message.Message()

      if form_item.filename is None:
        # Copy as is
        variable.add_header('Content-Type', 'text/plain')
        variable.set_payload(form_item.value)
      else:
        # If there is no filename associated with this field it means that the
        # file form field was not filled in.  This blob should not be created
        # and forwarded to success handler.
        if not form_item.filename:
          continue

        disposition_parameters['filename'] = form_item.filename

        try:
          main_type, sub_type = _split_mime_type(form_item.type)
        except _InvalidMIMETypeFormatError, ex:
          mime_type_error = str(ex)
          break

        # Seek to the end of file and use the pos as the length.
        form_item.file.seek(0, os.SEEK_END)
        content_length = form_item.file.tell()
        form_item.file.seek(0)

        total_bytes_uploaded += content_length

        if max_bytes_per_blob is not None:
          if content_length > max_bytes_per_blob:
            upload_too_large = True
            break
        if max_bytes_total is not None:
          if total_bytes_uploaded > max_bytes_total:
            upload_too_large = True
            break
        if form_item.filename is not None:
          if len(form_item.filename) > _MAX_STRING_NAME_LENGTH:
            filename_too_large = True
            break
        if form_item.type is not None:
          if len(form_item.type) > _MAX_STRING_NAME_LENGTH:
            content_type_too_large = True
            break

        # Compute the MD5 hash of the upload.
        digester = hashlib.md5()
        while True:
          block = form_item.file.read(1 << 20)
          if not block:
            break
          digester.update(block)
        form_item.file.seek(0)

        # Create the external body message containing meta-data about the blob.
        external = email.message.Message()
        external.add_header('Content-Type', '%s/%s' % (main_type, sub_type),
                            **form_item.type_options)
        # NOTE: This is in violation of RFC 2616 (Content-MD5 should be the
        # base-64 encoding of the binary hash, not the hex digest), but it is
        # consistent with production.
        blob_key = base64.urlsafe_b64encode(digester.hexdigest())
        # Create header MIME message
        headers = dict(form_item.headers)
        for name in _STRIPPED_FILE_HEADERS:
          if name in headers:
            del headers[name]
        headers['Content-Length'] = str(content_length)
        headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = (
            blobstore._format_creation(creation))
        headers['Content-MD5'] = blob_key
        if bucket_name:
          headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = (
              '/gs/%s/fake-%s' % (bucket_name, blob_key))
        for key, value in headers.iteritems():
          external.add_header(key, value)
        # Add disposition parameters (a clone of the outer message's field).
        if not external.get('Content-Disposition'):
          external.add_header('Content-Disposition', 'form-data',
                              **disposition_parameters)

        # Store the actual contents in the blobstore.
        base64_encoding = (form_item.headers.get('Content-Transfer-Encoding') ==
                           'base64')
        try:
          blob_entity = self.store_blob(external['content-type'],
                                        form_item.filename, digester,
                                        form_item.file, creation,
                                        base64_encoding=base64_encoding)
        except _TooManyConflictsError:
          too_many_conflicts = True
          break

        # Track created blobs in case we need to roll them back.
        created_blobs.append(blob_entity)

        variable.add_header('Content-Type', 'message/external-body',
                            access_type=blobstore.BLOB_KEY_HEADER,
                            blob_key=blob_entity.key().name())
        variable.set_payload([external])

      # Set common information.
      variable.add_header('Content-Disposition', 'form-data',
                          **disposition_parameters)
      message.attach(variable)
Exemplo n.º 10
0
    def post(self, app_id="blob", session_id="session"):
        """ Handler a post request from a user uploading a blob. 
    
    Args:
      app_id: The application triggering the upload.
      session_id: Authentication token to validate the upload.
    """
        global datastore_path
        db = datastore_distributed.DatastoreDistributed(app_id,
                                                        datastore_path,
                                                        require_indexes=False)
        apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
        os.environ['APPLICATION_ID'] = app_id

        # Setup the app id in the datastore.
        # Get session info and upload success path.
        blob_session = get_session(session_id)
        if not blob_session:
            self.finish('Session has expired. Contact the owner of the ' + \
                        'app for support.\n\n')
            return

        success_path = blob_session["success_path"]

        server_host = success_path[:success_path.rfind("/", 3)]
        if server_host.startswith("http://"):
            # Strip off the beginging of the server host
            server_host = server_host[len("http://"):]
        server_host = server_host.split('/')[0]

        blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id)
        uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)

        datastore.Delete(blob_session)

        # This request is sent to the upload handler of the app
        # in the hope it returns a redirect to be forwarded to the user
        urlrequest = urllib2.Request(success_path)

        # Forward all relevant headers and create data for request
        content_type = self.request.headers["Content-Type"]
        kv = split_content_type(content_type)
        boundary = None
        if "boundary" in kv:
            boundary = kv["boundary"]

        urlrequest.add_header("Content-Type",
                              'application/x-www-form-urlencoded')

        for name, value in self.request.headers.items():
            if name.lower() not in STRIPPED_HEADERS:
                urlrequest.add_header(name, value)

        # Get correct redirect addresses, otherwise it will redirect back
        # to this port.
        urlrequest.add_header("Host", server_host)

        form = MultiPartForm(boundary)
        creation = datetime.datetime.now()

        # Loop on all files in the form.
        for filekey in self.request.files.keys():
            data = {"blob_info_metadata": {filekey: []}}
            file = self.request.files[filekey][0]
            body = file["body"]
            size = len(body)
            filename = file["filename"]
            file_content_type = file["content_type"]

            blob_entity = uploadhandler.StoreBlob(file, creation)

            blob_key = str(blob_entity.key().name())

            if not blob_key:
                self.finish('Status: 500\n\n')
                return
            creation_formatted = blobstore._format_creation(creation)
            form.add_file(filekey, filename, cStringIO.StringIO(blob_key),
                          blob_key, blobstore.BLOB_KEY_HEADER, size,
                          creation_formatted)

            md5_handler = hashlib.md5(str(body))
            data["blob_info_metadata"][filekey].append({
                "filename":
                filename,
                "creation-date":
                creation_formatted,
                "key":
                blob_key,
                "size":
                str(size),
                "content-type":
                file_content_type,
                "md5-hash":
                md5_handler.hexdigest()
            })

        # Loop through form fields
        for fieldkey in self.request.arguments.keys():
            form.add_field(fieldkey, self.request.arguments[fieldkey][0])
            data[fieldkey] = self.request.arguments[fieldkey][0]

        logging.debug("Callback data: \n{}".format(data))
        data = urllib.urlencode(data)
        urlrequest.add_header("Content-Length", str(len(data)))
        urlrequest.add_data(data)

        # We are catching the redirect error here
        # and extracting the Location to post the redirect.
        try:
            response = urllib2.urlopen(urlrequest)
            output = response.read()
            if response.info().get('Content-Encoding') == 'gzip':
                buf = StringIO(output)
                f = gzip.GzipFile(fileobj=buf)
                data = f.read()
                output = data
            self.finish(output)
        except urllib2.HTTPError, e:
            if "Location" in e.hdrs:
                # Catch any errors, use the success path to
                # get the ip and port, use the redirect path
                # for the path. We split redirect_path just in case
                # its a full path.
                redirect_path = e.hdrs["Location"]
                self.redirect(redirect_path)
                return
            else:
                self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" +
                            str(e))
                return
Exemplo n.º 11
0
    def store_and_build_forward_message(self,
                                        form,
                                        boundary=None,
                                        max_bytes_per_blob=None,
                                        max_bytes_total=None,
                                        bucket_name=None):
        """Reads form data, stores blobs data and builds the forward request.

    This finds all of the file uploads in a set of form fields, converting them
    into blobs and storing them in the blobstore. It also generates the HTTP
    request to forward to the user's application.

    Args:
      form: cgi.FieldStorage instance representing the whole form derived from
        original POST data.
      boundary: The optional boundary to use for the resulting form. If omitted,
        one is randomly generated.
      max_bytes_per_blob: The maximum size in bytes that any single blob
        in the form is allowed to be.
      max_bytes_total: The maximum size in bytes that the total of all blobs
        in the form is allowed to be.
      bucket_name: The name of the Google Storage bucket to store the uploaded
                   files.

    Returns:
      A tuple (content_type, content_text), where content_type is the value of
      the Content-Type header, and content_text is a string containing the body
      of the HTTP request to forward to the application.

    Raises:
      webob.exc.HTTPException: The upload failed.
    """
        message = multipart.MIMEMultipart('form-data', boundary)

        creation = self._now_func()
        total_bytes_uploaded = 0
        created_blobs = []
        mime_type_error = None
        too_many_conflicts = False
        upload_too_large = False
        filename_too_large = False
        content_type_too_large = False

        # Extract all of the individual form items out of the FieldStorage.
        form_items = []
        # Sorting of forms is done merely to make testing a little easier since
        # it means blob-keys are generated in a predictable order.
        for key in sorted(form):
            form_item = form[key]
            if isinstance(form_item, list):
                form_items.extend(form_item)
            else:
                form_items.append(form_item)

        for form_item in form_items:
            disposition_parameters = {'name': form_item.name}

            variable = email.message.Message()

            if form_item.filename is None:
                # Copy as is
                variable.add_header('Content-Type', 'text/plain')
                variable.set_payload(form_item.value)
            else:
                # If there is no filename associated with this field it means that the
                # file form field was not filled in.  This blob should not be created
                # and forwarded to success handler.
                if not form_item.filename:
                    continue

                disposition_parameters['filename'] = form_item.filename

                try:
                    main_type, sub_type = _split_mime_type(form_item.type)
                except _InvalidMIMETypeFormatError as ex:
                    mime_type_error = str(ex)
                    break

                # Seek to the end of file and use the pos as the length.
                form_item.file.seek(0, os.SEEK_END)
                content_length = form_item.file.tell()
                form_item.file.seek(0)

                total_bytes_uploaded += content_length

                if max_bytes_per_blob is not None:
                    if content_length > max_bytes_per_blob:
                        upload_too_large = True
                        break
                if max_bytes_total is not None:
                    if total_bytes_uploaded > max_bytes_total:
                        upload_too_large = True
                        break
                if form_item.filename is not None:
                    if len(form_item.filename) > _MAX_STRING_NAME_LENGTH:
                        filename_too_large = True
                        break
                if form_item.type is not None:
                    if len(form_item.type) > _MAX_STRING_NAME_LENGTH:
                        content_type_too_large = True
                        break

                # Compute the MD5 hash of the upload.
                digester = hashlib.md5()
                while True:
                    block = form_item.file.read(1 << 20)
                    if not block:
                        break
                    digester.update(block)
                form_item.file.seek(0)

                # Create the external body message containing meta-data about the blob.
                external = email.message.Message()
                external.add_header('Content-Type',
                                    '%s/%s' % (main_type, sub_type),
                                    **form_item.type_options)
                # NOTE: This is in violation of RFC 2616 (Content-MD5 should be the
                # base-64 encoding of the binary hash, not the hex digest), but it is
                # consistent with production.
                content_md5 = base64.urlsafe_b64encode(digester.hexdigest())
                # Create header MIME message
                headers = dict(form_item.headers)
                for name in _STRIPPED_FILE_HEADERS:
                    if name in headers:
                        del headers[name]
                headers['Content-Length'] = str(content_length)
                headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = (
                    blobstore._format_creation(creation))
                headers['Content-MD5'] = content_md5
                gs_filename = None
                if bucket_name:
                    random_key = str(self._generate_blob_key())
                    gs_filename = '%s/fake-%s' % (bucket_name, random_key)
                    headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = (
                        blobstore.GS_PREFIX + gs_filename)
                for key, value in six.iteritems(headers):
                    external.add_header(key, value)
                # Add disposition parameters (a clone of the outer message's field).
                if not external.get('Content-Disposition'):
                    external.add_header('Content-Disposition', 'form-data',
                                        **disposition_parameters)

                base64_encoding = (form_item.headers.get(
                    'Content-Transfer-Encoding') == 'base64')
                content_type, blob_file, filename = self._preprocess_data(
                    external['content-type'], form_item.file,
                    form_item.filename, base64_encoding)

                # Store the actual contents to storage.
                if gs_filename:
                    info_entity = self.store_gs_file(content_type, gs_filename,
                                                     blob_file, filename)
                else:
                    try:
                        info_entity = self.store_blob(content_type, filename,
                                                      digester, blob_file,
                                                      creation)
                    except _TooManyConflictsError:
                        too_many_conflicts = True
                        break

                # Track created blobs in case we need to roll them back.
                created_blobs.append(info_entity)

                variable.add_header('Content-Type',
                                    'message/external-body',
                                    access_type=blobstore.BLOB_KEY_HEADER,
                                    blob_key=info_entity.key().name())
                variable.set_payload([external])

            # Set common information.
            variable.add_header('Content-Disposition', 'form-data',
                                **disposition_parameters)
            message.attach(variable)

        if (mime_type_error or too_many_conflicts or upload_too_large
                or filename_too_large or content_type_too_large):
            for blob in created_blobs:
                datastore.Delete(blob)
            if mime_type_error:
                self.abort(400, detail=mime_type_error)
            elif too_many_conflicts:
                self.abort(500, detail='Could not generate a blob key.')
            elif upload_too_large:
                self.abort(413)
            else:
                if filename_too_large:
                    invalid_field = 'filename'
                elif content_type_too_large:
                    invalid_field = 'Content-Type'
                detail = 'The %s exceeds the maximum allowed length of %s.' % (
                    invalid_field, _MAX_STRING_NAME_LENGTH)
                self.abort(400, detail=detail)

        message_out = io.StringIO()
        gen = email.generator.Generator(message_out, maxheaderlen=0)
        gen.flatten(message, unixfrom=False)

        # Get the content text out of the message.
        message_text = message_out.getvalue()
        content_start = message_text.find('\n\n') + 2
        content_text = message_text[content_start:]
        content_text = content_text.replace('\n', '\r\n')

        return message.get('Content-Type'), content_text