def __ConstructRelativePath(self,
method_config,
request,
relative_path=None):
"""Determine the relative path for request."""
path = relative_path or method_config.relative_path
path = path.replace('+', '')
for param in method_config.path_params:
param_template = '{%s}' % param
if param_template not in path:
raise exceptions.InvalidUserInputError(
'Missing path parameter %s' % param)
try:
# TODO: Do we want to support some sophisticated
# mapping here?
value = getattr(request, param)
except AttributeError:
raise exceptions.InvalidUserInputError(
'Request missing required parameter %s' % param)
if value is None:
raise exceptions.InvalidUserInputError(
'Request missing required parameter %s' % param)
try:
if not isinstance(value, basestring):
value = str(value)
path = path.replace(param_template,
urllib.quote(value.encode('utf_8'), ''))
except TypeError as e:
raise exceptions.InvalidUserInputError(
'Error setting required parameter %s to value %s: %s' %
(param, value, e))
return path
def FromData(cls, stream, json_data, http, auto_transfer=None, **kwds):
"""Create a new Upload of stream from serialized json_data using http."""
info = json.loads(json_data)
missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys())
if missing_keys:
raise exceptions.InvalidDataError(
'Invalid serialization data, missing keys: %s' %
(', '.join(missing_keys)))
if 'total_size' in kwds:
raise exceptions.InvalidUserInputError(
'Cannot override total_size on serialized Upload')
upload = cls.FromStream(stream,
info['mime_type'],
total_size=info.get('total_size'),
**kwds)
if isinstance(stream, io.IOBase) and not stream.seekable():
raise exceptions.InvalidUserInputError(
'Cannot restart resumable upload on non-seekable stream')
if auto_transfer is not None:
upload.auto_transfer = auto_transfer
else:
upload.auto_transfer = info['auto_transfer']
upload.strategy = RESUMABLE_UPLOAD
upload._Initialize(http, info['url']) # pylint: disable=protected-access
upload.RefreshResumableUploadState()
upload.EnsureInitialized()
if upload.auto_transfer:
upload.StreamInChunks()
return upload
def ConfigureRequest(self, upload_config, http_request, url_builder):
"""Configure the request and url for this upload."""
# Validate total_size vs. max_size
if (self.total_size and upload_config.max_size
and self.total_size > upload_config.max_size):
raise exceptions.InvalidUserInputError(
'Upload too big: %s larger than max size %s' %
(self.total_size, upload_config.max_size))
# Validate mime type
if not util.AcceptableMimeType(upload_config.accept, self.mime_type):
raise exceptions.InvalidUserInputError(
'MIME type %s does not match any accepted MIME ranges %s' %
(self.mime_type, upload_config.accept))
self.__SetDefaultUploadStrategy(upload_config, http_request)
if self.strategy == SIMPLE_UPLOAD:
url_builder.relative_path = upload_config.simple_path
if http_request.body:
url_builder.query_params['uploadType'] = 'multipart'
self.__ConfigureMultipartRequest(http_request)
else:
url_builder.query_params['uploadType'] = 'media'
self.__ConfigureMediaRequest(http_request)
else:
url_builder.relative_path = upload_config.resumable_path
url_builder.query_params['uploadType'] = 'resumable'
self.__ConfigureResumableRequest(http_request)
def InitializeUpload(self, http_request, http=None, client=None):
"""Initialize this upload from the given http_request."""
if self.strategy is None:
raise exceptions.UserError(
'No upload strategy set; did you call ConfigureRequest?')
if http is None and client is None:
raise exceptions.UserError('Must provide client or http.')
if self.strategy != _RESUMABLE_UPLOAD:
return
if self.total_size is None:
raise exceptions.InvalidUserInputError(
'Cannot stream upload without total size')
http = http or client.http
if client is not None:
http_request.url = client.FinalizeTransferUrl(http_request.url)
self.EnsureUninitialized()
http_response = http_wrapper.MakeRequest(http, http_request)
if http_response.status_code != httplib.OK:
raise exceptions.HttpError.FromResponse(http_response)
self.__server_chunk_granularity = http_response.info.get(
'X-Goog-Upload-Chunk-Granularity')
self.__ValidateChunksize()
url = http_response.info['location']
if client is not None:
url = client.FinalizeTransferUrl(url)
self._Initialize(http, url)
# Unless the user has requested otherwise, we want to just
# go ahead and pump the bytes now.
if self.auto_transfer:
return self.StreamInChunks()
def FromFile(cls, filename, overwrite=False, auto_transfer=True, **kwds):
"""Create a new download object from a filename."""
path = os.path.expanduser(filename)
if os.path.exists(path) and not overwrite:
raise exceptions.InvalidUserInputError(
'File %s exists and overwrite not specified' % path)
return cls(open(path, 'wb'),
close_stream=True,
auto_transfer=auto_transfer,
**kwds)
def StreamInChunks(self,
callback=None,
finish_callback=None,
additional_headers=None):
"""Send this (resumable) upload in chunks."""
if self.strategy != _RESUMABLE_UPLOAD:
raise exceptions.InvalidUserInputError(
'Cannot stream non-resumable upload')
if self.total_size is None:
raise exceptions.InvalidUserInputError(
'Cannot stream upload without total size')
callback = callback or self._ArgPrinter
finish_callback = finish_callback or self._CompletePrinter
# final_response is set if we resumed an already-completed upload.
response = self.__final_response
self.__ValidateChunksize(self.chunksize)
self.EnsureInitialized()
while not self.complete:
response = self.__SendChunk(self.stream.tell(),
additional_headers=additional_headers)
if response.status_code in (httplib.OK, httplib.CREATED):
self.__complete = True
break
self.__progress = self.__GetLastByte(response.info['range'])
if self.progress + 1 != self.stream.tell():
# TODO: Add a better way to recover here.
raise exceptions.CommunicationError(
'Failed to transfer all bytes in chunk, upload paused at byte '
'%d' % self.progress)
self._ExecuteCallback(callback, response)
if self.__complete:
# TODO: Decide how to handle errors in the non-seekable case.
current_pos = self.stream.tell()
self.stream.seek(0, os.SEEK_END)
end_pos = self.stream.tell()
self.stream.seek(current_pos)
if current_pos != end_pos:
raise exceptions.TransferInvalidError(
'Upload complete with %s additional bytes left in stream' %
(long(end_pos) - long(current_pos)))
self._ExecuteCallback(finish_callback, response)
return response
def FromStream(cls,
stream,
mime_type,
total_size=None,
auto_transfer=True):
"""Create a new Upload object from a stream."""
if mime_type is None:
raise exceptions.InvalidUserInputError(
'No mime_type specified for stream')
return cls(stream,
mime_type,
total_size=total_size,
close_stream=False,
auto_transfer=auto_transfer)
def FromFile(cls, filename, mime_type=None, auto_transfer=True):
"""Create a new Upload object from a filename."""
path = os.path.expanduser(filename)
if not os.path.exists(path):
raise exceptions.NotFoundError('Could not find file %s' % path)
if not mime_type:
mime_type, _ = mimetypes.guess_type(path)
if mime_type is None:
raise exceptions.InvalidUserInputError(
'Could not guess mime type for %s' % path)
size = os.stat(path).st_size
return cls(open(path, 'rb'),
mime_type,
total_size=size,
close_stream=True,
auto_transfer=auto_transfer)
def _EncodeUnknownFields(message):
"""Remap unknown fields in message out of message.source."""
source = _UNRECOGNIZED_FIELD_MAPPINGS.get(type(message))
if source is None:
return message
result = CopyProtoMessage(message)
pairs_field = message.field_by_name(source)
if not isinstance(pairs_field, messages.MessageField):
raise exceptions.InvalidUserInputError('Invalid pairs field %s' %
pairs_field)
pairs_type = pairs_field.message_type
value_variant = pairs_type.field_by_name('value').variant
pairs = getattr(message, source)
for pair in pairs:
if value_variant == messages.Variant.MESSAGE:
encoded_value = MessageToDict(pair.value)
else:
encoded_value = pair.value
result.set_unrecognized_field(pair.key, encoded_value, value_variant)
setattr(result, source, [])
return result
def __init__(self, stream=None, *args, **kwds):
if stream is None:
raise apitools_exceptions.InvalidUserInputError(
'Cannot create HttpWithDownloadStream with no stream')
self._stream = stream
super(HttpWithDownloadStream, self).__init__(*args, **kwds)
