def __ConstructRelativePath(self, method_config, request, relative_path=None): """Determine the relative path for request.""" path = relative_path or method_config.relative_path path = path.replace('+', '') for param in method_config.path_params: param_template = '{%s}' % param if param_template not in path: raise exceptions.InvalidUserInputError( 'Missing path parameter %s' % param) try: # TODO: Do we want to support some sophisticated # mapping here? value = getattr(request, param) except AttributeError: raise exceptions.InvalidUserInputError( 'Request missing required parameter %s' % param) if value is None: raise exceptions.InvalidUserInputError( 'Request missing required parameter %s' % param) try: if not isinstance(value, basestring): value = str(value) path = path.replace(param_template, urllib.quote(value.encode('utf_8'), '')) except TypeError as e: raise exceptions.InvalidUserInputError( 'Error setting required parameter %s to value %s: %s' % (param, value, e)) return path
def FromData(cls, stream, json_data, http, auto_transfer=None, **kwds): """Create a new Upload of stream from serialized json_data using http.""" info = json.loads(json_data) missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) if missing_keys: raise exceptions.InvalidDataError( 'Invalid serialization data, missing keys: %s' % (', '.join(missing_keys))) if 'total_size' in kwds: raise exceptions.InvalidUserInputError( 'Cannot override total_size on serialized Upload') upload = cls.FromStream(stream, info['mime_type'], total_size=info.get('total_size'), **kwds) if isinstance(stream, io.IOBase) and not stream.seekable(): raise exceptions.InvalidUserInputError( 'Cannot restart resumable upload on non-seekable stream') if auto_transfer is not None: upload.auto_transfer = auto_transfer else: upload.auto_transfer = info['auto_transfer'] upload.strategy = RESUMABLE_UPLOAD upload._Initialize(http, info['url']) # pylint: disable=protected-access upload.RefreshResumableUploadState() upload.EnsureInitialized() if upload.auto_transfer: upload.StreamInChunks() return upload
def ConfigureRequest(self, upload_config, http_request, url_builder): """Configure the request and url for this upload.""" # Validate total_size vs. max_size if (self.total_size and upload_config.max_size and self.total_size > upload_config.max_size): raise exceptions.InvalidUserInputError( 'Upload too big: %s larger than max size %s' % (self.total_size, upload_config.max_size)) # Validate mime type if not util.AcceptableMimeType(upload_config.accept, self.mime_type): raise exceptions.InvalidUserInputError( 'MIME type %s does not match any accepted MIME ranges %s' % (self.mime_type, upload_config.accept)) self.__SetDefaultUploadStrategy(upload_config, http_request) if self.strategy == SIMPLE_UPLOAD: url_builder.relative_path = upload_config.simple_path if http_request.body: url_builder.query_params['uploadType'] = 'multipart' self.__ConfigureMultipartRequest(http_request) else: url_builder.query_params['uploadType'] = 'media' self.__ConfigureMediaRequest(http_request) else: url_builder.relative_path = upload_config.resumable_path url_builder.query_params['uploadType'] = 'resumable' self.__ConfigureResumableRequest(http_request)
def InitializeUpload(self, http_request, http=None, client=None): """Initialize this upload from the given http_request.""" if self.strategy is None: raise exceptions.UserError( 'No upload strategy set; did you call ConfigureRequest?') if http is None and client is None: raise exceptions.UserError('Must provide client or http.') if self.strategy != _RESUMABLE_UPLOAD: return if self.total_size is None: raise exceptions.InvalidUserInputError( 'Cannot stream upload without total size') http = http or client.http if client is not None: http_request.url = client.FinalizeTransferUrl(http_request.url) self.EnsureUninitialized() http_response = http_wrapper.MakeRequest(http, http_request) if http_response.status_code != httplib.OK: raise exceptions.HttpError.FromResponse(http_response) self.__server_chunk_granularity = http_response.info.get( 'X-Goog-Upload-Chunk-Granularity') self.__ValidateChunksize() url = http_response.info['location'] if client is not None: url = client.FinalizeTransferUrl(url) self._Initialize(http, url) # Unless the user has requested otherwise, we want to just # go ahead and pump the bytes now. if self.auto_transfer: return self.StreamInChunks()
def FromFile(cls, filename, overwrite=False, auto_transfer=True, **kwds): """Create a new download object from a filename.""" path = os.path.expanduser(filename) if os.path.exists(path) and not overwrite: raise exceptions.InvalidUserInputError( 'File %s exists and overwrite not specified' % path) return cls(open(path, 'wb'), close_stream=True, auto_transfer=auto_transfer, **kwds)
def StreamInChunks(self, callback=None, finish_callback=None, additional_headers=None): """Send this (resumable) upload in chunks.""" if self.strategy != _RESUMABLE_UPLOAD: raise exceptions.InvalidUserInputError( 'Cannot stream non-resumable upload') if self.total_size is None: raise exceptions.InvalidUserInputError( 'Cannot stream upload without total size') callback = callback or self._ArgPrinter finish_callback = finish_callback or self._CompletePrinter # final_response is set if we resumed an already-completed upload. response = self.__final_response self.__ValidateChunksize(self.chunksize) self.EnsureInitialized() while not self.complete: response = self.__SendChunk(self.stream.tell(), additional_headers=additional_headers) if response.status_code in (httplib.OK, httplib.CREATED): self.__complete = True break self.__progress = self.__GetLastByte(response.info['range']) if self.progress + 1 != self.stream.tell(): # TODO: Add a better way to recover here. raise exceptions.CommunicationError( 'Failed to transfer all bytes in chunk, upload paused at byte ' '%d' % self.progress) self._ExecuteCallback(callback, response) if self.__complete: # TODO: Decide how to handle errors in the non-seekable case. current_pos = self.stream.tell() self.stream.seek(0, os.SEEK_END) end_pos = self.stream.tell() self.stream.seek(current_pos) if current_pos != end_pos: raise exceptions.TransferInvalidError( 'Upload complete with %s additional bytes left in stream' % (long(end_pos) - long(current_pos))) self._ExecuteCallback(finish_callback, response) return response
def FromStream(cls, stream, mime_type, total_size=None, auto_transfer=True): """Create a new Upload object from a stream.""" if mime_type is None: raise exceptions.InvalidUserInputError( 'No mime_type specified for stream') return cls(stream, mime_type, total_size=total_size, close_stream=False, auto_transfer=auto_transfer)
def FromFile(cls, filename, mime_type=None, auto_transfer=True): """Create a new Upload object from a filename.""" path = os.path.expanduser(filename) if not os.path.exists(path): raise exceptions.NotFoundError('Could not find file %s' % path) if not mime_type: mime_type, _ = mimetypes.guess_type(path) if mime_type is None: raise exceptions.InvalidUserInputError( 'Could not guess mime type for %s' % path) size = os.stat(path).st_size return cls(open(path, 'rb'), mime_type, total_size=size, close_stream=True, auto_transfer=auto_transfer)
def _EncodeUnknownFields(message): """Remap unknown fields in message out of message.source.""" source = _UNRECOGNIZED_FIELD_MAPPINGS.get(type(message)) if source is None: return message result = CopyProtoMessage(message) pairs_field = message.field_by_name(source) if not isinstance(pairs_field, messages.MessageField): raise exceptions.InvalidUserInputError('Invalid pairs field %s' % pairs_field) pairs_type = pairs_field.message_type value_variant = pairs_type.field_by_name('value').variant pairs = getattr(message, source) for pair in pairs: if value_variant == messages.Variant.MESSAGE: encoded_value = MessageToDict(pair.value) else: encoded_value = pair.value result.set_unrecognized_field(pair.key, encoded_value, value_variant) setattr(result, source, []) return result
def __init__(self, stream=None, *args, **kwds): if stream is None: raise apitools_exceptions.InvalidUserInputError( 'Cannot create HttpWithDownloadStream with no stream') self._stream = stream super(HttpWithDownloadStream, self).__init__(*args, **kwds)