def _stream_data(self, response, iterator, chunked=False,
calculate_hash=True, chunk_size=None):
"""
Stream a data over an http connection.
@type response: C{RawResponse}
@param response: RawResponse object.
@type iterator: C{}
@param response: An object which implements an iterator interface
or a File like object with read method.
@type chunk_size: C{int}
@param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
@return C{tuple} First item is a boolean indicator of success, second
one is the uploaded data MD5 hash and the third one
is the number of transferred bytes.
"""
chunk_size = chunk_size or CHUNK_SIZE
data_hash = None
if calculate_hash:
data_hash = hashlib.md5()
generator = utils.read_in_chunks(iterator, chunk_size)
bytes_transferred = 0
try:
chunk = generator.next()
except StopIteration:
# No data?
return False, None, None
while len(chunk) > 0:
try:
if chunked:
response.connection.connection.send('%X\r\n' %
(len(chunk)))
response.connection.connection.send(chunk)
response.connection.connection.send('\r\n')
else:
response.connection.connection.send(chunk)
except Exception, e:
# Timeout, etc.
return False, None, bytes_transferred
bytes_transferred += len(chunk)
if calculate_hash:
data_hash.update(chunk)
try:
chunk = generator.next()
except StopIteration:
chunk = ''
def upload_object_via_stream(self, iterator, container, object_name,
extra=None):
if isinstance(iterator, file):
iterator = iter(iterator)
data_hash = hashlib.md5()
generator = utils.read_in_chunks(iterator, CHUNK_SIZE, True)
bytes_transferred = 0
try:
chunk = generator.next()
except StopIteration:
chunk = ''
path = self._namespace_path(container.name + '/' + object_name)
while True:
end = bytes_transferred + len(chunk) - 1
data_hash.update(chunk)
headers = {
'x-emc-meta': 'md5=' + data_hash.hexdigest(),
}
if len(chunk) > 0:
headers['Range'] = 'Bytes=%d-%d' % (bytes_transferred, end)
result = self.connection.request(path, method='PUT', data=chunk,
headers=headers)
bytes_transferred += len(chunk)
try:
chunk = generator.next()
except StopIteration:
break
if len(chunk) == 0:
break
data_hash = data_hash.hexdigest()
if extra is None:
meta_data = {}
else:
meta_data = extra.get('meta_data', {})
meta_data['md5'] = data_hash
user_meta = ', '.join([k + '=' + str(v) for k, v in meta_data.items()])
self.connection.request(path + '?metadata/user', method='POST',
headers={'x-emc-meta': user_meta})
result = self.connection.request(path + '?metadata/system')
meta = self._emc_meta(result)
extra = {
'object_id': meta['objectid'],
'meta_data': meta_data,
}
return Object(object_name, bytes_transferred, data_hash, extra,
meta_data, container, self)
def _stream_data(self,
response,
iterator,
chunked=False,
calculate_hash=True,
chunk_size=None):
"""
Stream a data over an http connection.
@type response: C{RawResponse}
@param response: RawResponse object.
@type iterator: C{}
@param response: An object which implements an iterator interface
or a File like object with read method.
@type chunk_size: C{int}
@param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
@return C{tuple} First item is a boolean indicator of success, second
one is the uploaded data MD5 hash and the third one
is the number of transferred bytes.
"""
chunk_size = chunk_size or CHUNK_SIZE
data_hash = None
if calculate_hash:
data_hash = hashlib.md5()
generator = utils.read_in_chunks(iterator, chunk_size)
bytes_transferred = 0
try:
chunk = generator.next()
except StopIteration:
# No data?
return False, None, None
while len(chunk) > 0:
try:
if chunked:
response.connection.connection.send('%X\r\n' %
(len(chunk)))
response.connection.connection.send(chunk)
response.connection.connection.send('\r\n')
else:
response.connection.connection.send(chunk)
except Exception:
# TODO: let this exception propagate
# Timeout, etc.
return False, None, bytes_transferred
bytes_transferred += len(chunk)
if calculate_hash:
data_hash.update(chunk)
try:
chunk = generator.next()
except StopIteration:
chunk = ''
if chunked:
response.connection.connection.send('0\r\n\r\n')
if calculate_hash:
data_hash = data_hash.hexdigest()
return True, data_hash, bytes_transferred
def _save_object(self,
response,
obj,
destination_path,
overwrite_existing=False,
delete_on_failure=True,
chunk_size=None):
"""
Save object to the provided path.
@type response: C{RawResponse}
@param response: RawResponse instance.
@type obj: C{Object}
@param obj: Object instance.
@type destination_path: C{Str}
@param destination_path: Destination directory.
@type delete_on_failure: C{bool}
@param delete_on_failure: True to delete partially downloaded object if
the download fails.
@type overwrite_existing: C{bool}
@param overwrite_existing: True to overwrite a local path if it already
exists.
@type chunk_size: C{int}
@param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
@return C{bool} True on success, False otherwise.
"""
chunk_size = chunk_size or CHUNK_SIZE
base_name = os.path.basename(destination_path)
if not base_name and not os.path.exists(destination_path):
raise LibcloudError(value='Path %s does not exist' %
(destination_path),
driver=self)
if not base_name:
file_path = pjoin(destination_path, obj.name)
else:
file_path = destination_path
if os.path.exists(file_path) and not overwrite_existing:
raise LibcloudError(value='File %s already exists, but ' %
(file_path) + 'overwrite_existing=False',
driver=self)
stream = utils.read_in_chunks(response, chunk_size)
try:
data_read = stream.next()
except StopIteration:
# Empty response?
return False
bytes_transferred = 0
with open(file_path, 'wb') as file_handle:
while len(data_read) > 0:
file_handle.write(data_read)
bytes_transferred += len(data_read)
try:
data_read = stream.next()
except StopIteration:
data_read = ''
if int(obj.size) != int(bytes_transferred):
# Transfer failed, support retry?
if delete_on_failure:
try:
os.unlink(file_path)
except Exception:
pass
return False
return True
def _save_object(self, response, obj, destination_path,
overwrite_existing=False, delete_on_failure=True,
chunk_size=None):
"""
Save object to the provided path.
@type response: C{RawResponse}
@param response: RawResponse instance.
@type obj: C{Object}
@param obj: Object instance.
@type destination_path: C{Str}
@param destination_path: Destination directory.
@type delete_on_failure: C{bool}
@param delete_on_failure: True to delete partially downloaded object if
the download fails.
@type overwrite_existing: C{bool}
@param overwrite_existing: True to overwrite a local path if it already
exists.
@type chunk_size: C{int}
@param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
@return C{bool} True on success, False otherwise.
"""
chunk_size = chunk_size or CHUNK_SIZE
base_name = os.path.basename(destination_path)
if not base_name and not os.path.exists(destination_path):
raise LibcloudError(value='Path %s does not exist' % (destination_path),
driver=self)
if not base_name:
file_path = pjoin(destination_path, obj.name)
else:
file_path = destination_path
if os.path.exists(file_path) and not overwrite_existing:
raise LibcloudError(value='File %s already exists, but ' % (file_path) +
'overwrite_existing=False',
driver=self)
stream = utils.read_in_chunks(response, chunk_size)
try:
data_read = stream.next()
except StopIteration:
# Empty response?
return False
bytes_transferred = 0
with open(file_path, 'wb') as file_handle:
while len(data_read) > 0:
file_handle.write(data_read)
bytes_transferred += len(data_read)
try:
data_read = stream.next()
except StopIteration:
data_read = ''
if obj.size != bytes_transferred:
# Transfer failed, support retry?
if delete_on_failure:
try:
os.unlink(file_path)
except Exception:
pass
return False
return True
def _stream_data(self, response, iterator, chunked=False,
calculate_hash=True, chunk_size=None):
"""
Stream a data over an http connection.
@type response: C{RawResponse}
@param response: RawResponse object.
@type iterator: C{}
@param response: An object which implements an iterator interface
or a File like object with read method.
@type chunked: C{boolean}
@param chunked: True if the chunked transfer encoding should be used
(defauls to False).
@type calculate_hash: C{boolean}
@param calculate_hash: True to calculate hash of the transfered data.
(defauls to True).
@type chunk_size: C{int}
@param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
@rtype: C{tuple}
@return: First item is a boolean indicator of success, second
one is the uploaded data MD5 hash and the third one
is the number of transferred bytes.
"""
chunk_size = chunk_size or CHUNK_SIZE
data_hash = None
if calculate_hash:
data_hash = self._get_hash_function()
generator = utils.read_in_chunks(iterator, chunk_size)
bytes_transferred = 0
try:
chunk = generator.next()
except StopIteration:
# Special case when StopIteration is thrown on the first iteration -
# create a 0-byte long object
chunk = ''
if chunked:
response.connection.connection.send('%X\r\n' %
(len(chunk)))
response.connection.connection.send(chunk)
response.connection.connection.send('\r\n')
response.connection.connection.send('0\r\n\r\n')
else:
response.connection.connection.send(chunk)
return True, data_hash.hexdigest(), bytes_transferred
while len(chunk) > 0:
try:
if chunked:
response.connection.connection.send('%X\r\n' %
(len(chunk)))
response.connection.connection.send(chunk)
response.connection.connection.send('\r\n')
else:
response.connection.connection.send(chunk)
except Exception:
# TODO: let this exception propagate
# Timeout, etc.
return False, None, bytes_transferred
bytes_transferred += len(chunk)
if calculate_hash:
data_hash.update(chunk)
try:
chunk = generator.next()
except StopIteration:
chunk = ''
if chunked:
response.connection.connection.send('0\r\n\r\n')
if calculate_hash:
data_hash = data_hash.hexdigest()
return True, data_hash, bytes_transferred
def _upload_object(self, object_name, content_type, upload_func,
upload_func_kwargs, request_path, request_method='PUT',
headers=None, file_path=None, iterator=None):
"""
Helper function for setting common request headers and calling the
passed in callback which uploads an object.
"""
headers = headers or {}
if file_path and not os.path.exists(file_path):
raise OSError('File %s does not exist' % (file_path))
if iterator is not None and not hasattr(iterator, 'next'):
raise AttributeError('iterator object must implement next() ' +
'method.')
if not content_type:
if file_path:
name = file_path
else:
name = object_name
content_type, _ = utils.guess_file_mime_type(name)
if not content_type:
raise AttributeError(
'File content-type could not be guessed and' +
' no content_type value provided')
file_size = None
if iterator:
if self.supports_chunked_encoding:
headers['Transfer-Encoding'] = 'chunked'
upload_func_kwargs['chunked'] = True
else:
# Chunked transfer encoding is not supported. Need to buffer all
# the data in memory so we can determine file size.
iterator = utils.read_in_chunks(iterator=iterator)
data = utils.exhaust_iterator(iterator=iterator)
file_size = len(data)
upload_func_kwargs['data'] = data
else:
file_size = os.path.getsize(file_path)
upload_func_kwargs['chunked'] = False
if file_size:
headers['Content-Length'] = file_size
headers['Content-Type'] = content_type
response = self.connection.request(request_path,
method=request_method, data=None,
headers=headers, raw=True)
upload_func_kwargs['response'] = response
success, data_hash, bytes_transferred = upload_func(**upload_func_kwargs)
if not success:
raise LibcloudError(value='Object upload failed, Perhaps a timeout?',
driver=self)
result_dict = { 'response': response, 'data_hash': data_hash,
'bytes_transferred': bytes_transferred }
return result_dict
def upload_object_via_stream(self,
iterator,
container,
object_name,
extra=None):
if isinstance(iterator, file):
iterator = iter(iterator)
data_hash = hashlib.md5()
generator = utils.read_in_chunks(iterator, CHUNK_SIZE, True)
bytes_transferred = 0
try:
chunk = generator.next()
except StopIteration:
chunk = ''
path = self._namespace_path(container.name + '/' + object_name)
while True:
end = bytes_transferred + len(chunk) - 1
data_hash.update(chunk)
headers = {
'x-emc-meta': 'md5=' + data_hash.hexdigest(),
}
if len(chunk) > 0:
headers['Range'] = 'Bytes=%d-%d' % (bytes_transferred, end)
result = self.connection.request(path,
method='PUT',
data=chunk,
headers=headers)
bytes_transferred += len(chunk)
try:
chunk = generator.next()
except StopIteration:
break
if len(chunk) == 0:
break
data_hash = data_hash.hexdigest()
if extra is None:
meta_data = {}
else:
meta_data = extra.get('meta_data', {})
meta_data['md5'] = data_hash
user_meta = ', '.join([k + '=' + str(v) for k, v in meta_data.items()])
self.connection.request(path + '?metadata/user',
method='POST',
headers={'x-emc-meta': user_meta})
result = self.connection.request(path + '?metadata/system')
meta = self._emc_meta(result)
extra = {
'object_id': meta['objectid'],
'meta_data': meta_data,
}
return Object(object_name, bytes_transferred, data_hash, extra,
meta_data, container, self)
def _stream_data(self,
response,
iterator,
chunked=False,
calculate_hash=True,
chunk_size=None):
"""
Stream a data over an http connection.
@type response: C{RawResponse}
@param response: RawResponse object.
@type iterator: C{}
@param response: An object which implements an iterator interface
or a File like object with read method.
@type chunked: C{boolean}
@param chunked: True if the chunked transfer encoding should be used
(defauls to False).
@type calculate_hash: C{boolean}
@param calculate_hash: True to calculate hash of the transfered data.
(defauls to True).
@type chunk_size: C{int}
@param chunk_size: Optional chunk size (defaults to CHUNK_SIZE)
@rtype: C{tuple}
@return: First item is a boolean indicator of success, second
one is the uploaded data MD5 hash and the third one
is the number of transferred bytes.
"""
chunk_size = chunk_size or CHUNK_SIZE
data_hash = None
if calculate_hash:
data_hash = self._get_hash_function()
generator = utils.read_in_chunks(iterator, chunk_size)
bytes_transferred = 0
try:
chunk = generator.next()
except StopIteration:
# Special case when StopIteration is thrown on the first iteration -
# create a 0-byte long object
chunk = ''
if chunked:
response.connection.connection.send('%X\r\n' % (len(chunk)))
response.connection.connection.send(chunk)
response.connection.connection.send('\r\n')
response.connection.connection.send('0\r\n\r\n')
else:
response.connection.connection.send(chunk)
return True, data_hash.hexdigest(), bytes_transferred
while len(chunk) > 0:
try:
if chunked:
response.connection.connection.send('%X\r\n' %
(len(chunk)))
response.connection.connection.send(chunk)
response.connection.connection.send('\r\n')
else:
response.connection.connection.send(chunk)
except Exception:
# TODO: let this exception propagate
# Timeout, etc.
return False, None, bytes_transferred
bytes_transferred += len(chunk)
if calculate_hash:
data_hash.update(chunk)
try:
chunk = generator.next()
except StopIteration:
chunk = ''
if chunked:
response.connection.connection.send('0\r\n\r\n')
if calculate_hash:
data_hash = data_hash.hexdigest()
return True, data_hash, bytes_transferred
def _upload_object(self,
object_name,
content_type,
upload_func,
upload_func_kwargs,
request_path,
request_method='PUT',
headers=None,
file_path=None,
iterator=None):
"""
Helper function for setting common request headers and calling the
passed in callback which uploads an object.
"""
headers = headers or {}
if file_path and not os.path.exists(file_path):
raise OSError('File %s does not exist' % (file_path))
if iterator is not None and not hasattr(iterator, 'next'):
raise AttributeError('iterator object must implement next() ' +
'method.')
if not content_type:
if file_path:
name = file_path
else:
name = object_name
content_type, _ = utils.guess_file_mime_type(name)
if not content_type:
raise AttributeError(
'File content-type could not be guessed and' +
' no content_type value provided')
file_size = None
if iterator:
if self.supports_chunked_encoding:
headers['Transfer-Encoding'] = 'chunked'
upload_func_kwargs['chunked'] = True
else:
# Chunked transfer encoding is not supported. Need to buffer all
# the data in memory so we can determine file size.
iterator = utils.read_in_chunks(iterator=iterator)
data = utils.exhaust_iterator(iterator=iterator)
file_size = len(data)
upload_func_kwargs['data'] = data
else:
file_size = os.path.getsize(file_path)
upload_func_kwargs['chunked'] = False
if file_size:
headers['Content-Length'] = file_size
headers['Content-Type'] = content_type
response = self.connection.request(request_path,
method=request_method,
data=None,
headers=headers,
raw=True)
upload_func_kwargs['response'] = response
success, data_hash, bytes_transferred = upload_func(
**upload_func_kwargs)
if not success:
raise LibcloudError(
value='Object upload failed, Perhaps a timeout?', driver=self)
result_dict = {
'response': response,
'data_hash': data_hash,
'bytes_transferred': bytes_transferred
}
return result_dict
