def _stream_data(self, response, iterator, chunked=False, calculate_hash=True, chunk_size=None): """ Stream a data over an http connection. @type response: C{RawResponse} @param response: RawResponse object. @type iterator: C{} @param response: An object which implements an iterator interface or a File like object with read method. @type chunk_size: C{int} @param chunk_size: Optional chunk size (defaults to CHUNK_SIZE) @return C{tuple} First item is a boolean indicator of success, second one is the uploaded data MD5 hash and the third one is the number of transferred bytes. """ chunk_size = chunk_size or CHUNK_SIZE data_hash = None if calculate_hash: data_hash = hashlib.md5() generator = utils.read_in_chunks(iterator, chunk_size) bytes_transferred = 0 try: chunk = generator.next() except StopIteration: # No data? return False, None, None while len(chunk) > 0: try: if chunked: response.connection.connection.send('%X\r\n' % (len(chunk))) response.connection.connection.send(chunk) response.connection.connection.send('\r\n') else: response.connection.connection.send(chunk) except Exception, e: # Timeout, etc. return False, None, bytes_transferred bytes_transferred += len(chunk) if calculate_hash: data_hash.update(chunk) try: chunk = generator.next() except StopIteration: chunk = ''
def upload_object_via_stream(self, iterator, container, object_name, extra=None): if isinstance(iterator, file): iterator = iter(iterator) data_hash = hashlib.md5() generator = utils.read_in_chunks(iterator, CHUNK_SIZE, True) bytes_transferred = 0 try: chunk = generator.next() except StopIteration: chunk = '' path = self._namespace_path(container.name + '/' + object_name) while True: end = bytes_transferred + len(chunk) - 1 data_hash.update(chunk) headers = { 'x-emc-meta': 'md5=' + data_hash.hexdigest(), } if len(chunk) > 0: headers['Range'] = 'Bytes=%d-%d' % (bytes_transferred, end) result = self.connection.request(path, method='PUT', data=chunk, headers=headers) bytes_transferred += len(chunk) try: chunk = generator.next() except StopIteration: break if len(chunk) == 0: break data_hash = data_hash.hexdigest() if extra is None: meta_data = {} else: meta_data = extra.get('meta_data', {}) meta_data['md5'] = data_hash user_meta = ', '.join([k + '=' + str(v) for k, v in meta_data.items()]) self.connection.request(path + '?metadata/user', method='POST', headers={'x-emc-meta': user_meta}) result = self.connection.request(path + '?metadata/system') meta = self._emc_meta(result) extra = { 'object_id': meta['objectid'], 'meta_data': meta_data, } return Object(object_name, bytes_transferred, data_hash, extra, meta_data, container, self)
def _stream_data(self, response, iterator, chunked=False, calculate_hash=True, chunk_size=None): """ Stream a data over an http connection. @type response: C{RawResponse} @param response: RawResponse object. @type iterator: C{} @param response: An object which implements an iterator interface or a File like object with read method. @type chunk_size: C{int} @param chunk_size: Optional chunk size (defaults to CHUNK_SIZE) @return C{tuple} First item is a boolean indicator of success, second one is the uploaded data MD5 hash and the third one is the number of transferred bytes. """ chunk_size = chunk_size or CHUNK_SIZE data_hash = None if calculate_hash: data_hash = hashlib.md5() generator = utils.read_in_chunks(iterator, chunk_size) bytes_transferred = 0 try: chunk = generator.next() except StopIteration: # No data? return False, None, None while len(chunk) > 0: try: if chunked: response.connection.connection.send('%X\r\n' % (len(chunk))) response.connection.connection.send(chunk) response.connection.connection.send('\r\n') else: response.connection.connection.send(chunk) except Exception: # TODO: let this exception propagate # Timeout, etc. return False, None, bytes_transferred bytes_transferred += len(chunk) if calculate_hash: data_hash.update(chunk) try: chunk = generator.next() except StopIteration: chunk = '' if chunked: response.connection.connection.send('0\r\n\r\n') if calculate_hash: data_hash = data_hash.hexdigest() return True, data_hash, bytes_transferred
def _save_object(self, response, obj, destination_path, overwrite_existing=False, delete_on_failure=True, chunk_size=None): """ Save object to the provided path. @type response: C{RawResponse} @param response: RawResponse instance. @type obj: C{Object} @param obj: Object instance. @type destination_path: C{Str} @param destination_path: Destination directory. @type delete_on_failure: C{bool} @param delete_on_failure: True to delete partially downloaded object if the download fails. @type overwrite_existing: C{bool} @param overwrite_existing: True to overwrite a local path if it already exists. @type chunk_size: C{int} @param chunk_size: Optional chunk size (defaults to CHUNK_SIZE) @return C{bool} True on success, False otherwise. """ chunk_size = chunk_size or CHUNK_SIZE base_name = os.path.basename(destination_path) if not base_name and not os.path.exists(destination_path): raise LibcloudError(value='Path %s does not exist' % (destination_path), driver=self) if not base_name: file_path = pjoin(destination_path, obj.name) else: file_path = destination_path if os.path.exists(file_path) and not overwrite_existing: raise LibcloudError(value='File %s already exists, but ' % (file_path) + 'overwrite_existing=False', driver=self) stream = utils.read_in_chunks(response, chunk_size) try: data_read = stream.next() except StopIteration: # Empty response? return False bytes_transferred = 0 with open(file_path, 'wb') as file_handle: while len(data_read) > 0: file_handle.write(data_read) bytes_transferred += len(data_read) try: data_read = stream.next() except StopIteration: data_read = '' if int(obj.size) != int(bytes_transferred): # Transfer failed, support retry? if delete_on_failure: try: os.unlink(file_path) except Exception: pass return False return True
def _save_object(self, response, obj, destination_path, overwrite_existing=False, delete_on_failure=True, chunk_size=None): """ Save object to the provided path. @type response: C{RawResponse} @param response: RawResponse instance. @type obj: C{Object} @param obj: Object instance. @type destination_path: C{Str} @param destination_path: Destination directory. @type delete_on_failure: C{bool} @param delete_on_failure: True to delete partially downloaded object if the download fails. @type overwrite_existing: C{bool} @param overwrite_existing: True to overwrite a local path if it already exists. @type chunk_size: C{int} @param chunk_size: Optional chunk size (defaults to CHUNK_SIZE) @return C{bool} True on success, False otherwise. """ chunk_size = chunk_size or CHUNK_SIZE base_name = os.path.basename(destination_path) if not base_name and not os.path.exists(destination_path): raise LibcloudError(value='Path %s does not exist' % (destination_path), driver=self) if not base_name: file_path = pjoin(destination_path, obj.name) else: file_path = destination_path if os.path.exists(file_path) and not overwrite_existing: raise LibcloudError(value='File %s already exists, but ' % (file_path) + 'overwrite_existing=False', driver=self) stream = utils.read_in_chunks(response, chunk_size) try: data_read = stream.next() except StopIteration: # Empty response? return False bytes_transferred = 0 with open(file_path, 'wb') as file_handle: while len(data_read) > 0: file_handle.write(data_read) bytes_transferred += len(data_read) try: data_read = stream.next() except StopIteration: data_read = '' if obj.size != bytes_transferred: # Transfer failed, support retry? if delete_on_failure: try: os.unlink(file_path) except Exception: pass return False return True
def _stream_data(self, response, iterator, chunked=False, calculate_hash=True, chunk_size=None): """ Stream a data over an http connection. @type response: C{RawResponse} @param response: RawResponse object. @type iterator: C{} @param response: An object which implements an iterator interface or a File like object with read method. @type chunked: C{boolean} @param chunked: True if the chunked transfer encoding should be used (defauls to False). @type calculate_hash: C{boolean} @param calculate_hash: True to calculate hash of the transfered data. (defauls to True). @type chunk_size: C{int} @param chunk_size: Optional chunk size (defaults to CHUNK_SIZE) @rtype: C{tuple} @return: First item is a boolean indicator of success, second one is the uploaded data MD5 hash and the third one is the number of transferred bytes. """ chunk_size = chunk_size or CHUNK_SIZE data_hash = None if calculate_hash: data_hash = self._get_hash_function() generator = utils.read_in_chunks(iterator, chunk_size) bytes_transferred = 0 try: chunk = generator.next() except StopIteration: # Special case when StopIteration is thrown on the first iteration - # create a 0-byte long object chunk = '' if chunked: response.connection.connection.send('%X\r\n' % (len(chunk))) response.connection.connection.send(chunk) response.connection.connection.send('\r\n') response.connection.connection.send('0\r\n\r\n') else: response.connection.connection.send(chunk) return True, data_hash.hexdigest(), bytes_transferred while len(chunk) > 0: try: if chunked: response.connection.connection.send('%X\r\n' % (len(chunk))) response.connection.connection.send(chunk) response.connection.connection.send('\r\n') else: response.connection.connection.send(chunk) except Exception: # TODO: let this exception propagate # Timeout, etc. return False, None, bytes_transferred bytes_transferred += len(chunk) if calculate_hash: data_hash.update(chunk) try: chunk = generator.next() except StopIteration: chunk = '' if chunked: response.connection.connection.send('0\r\n\r\n') if calculate_hash: data_hash = data_hash.hexdigest() return True, data_hash, bytes_transferred
def _upload_object(self, object_name, content_type, upload_func, upload_func_kwargs, request_path, request_method='PUT', headers=None, file_path=None, iterator=None): """ Helper function for setting common request headers and calling the passed in callback which uploads an object. """ headers = headers or {} if file_path and not os.path.exists(file_path): raise OSError('File %s does not exist' % (file_path)) if iterator is not None and not hasattr(iterator, 'next'): raise AttributeError('iterator object must implement next() ' + 'method.') if not content_type: if file_path: name = file_path else: name = object_name content_type, _ = utils.guess_file_mime_type(name) if not content_type: raise AttributeError( 'File content-type could not be guessed and' + ' no content_type value provided') file_size = None if iterator: if self.supports_chunked_encoding: headers['Transfer-Encoding'] = 'chunked' upload_func_kwargs['chunked'] = True else: # Chunked transfer encoding is not supported. Need to buffer all # the data in memory so we can determine file size. iterator = utils.read_in_chunks(iterator=iterator) data = utils.exhaust_iterator(iterator=iterator) file_size = len(data) upload_func_kwargs['data'] = data else: file_size = os.path.getsize(file_path) upload_func_kwargs['chunked'] = False if file_size: headers['Content-Length'] = file_size headers['Content-Type'] = content_type response = self.connection.request(request_path, method=request_method, data=None, headers=headers, raw=True) upload_func_kwargs['response'] = response success, data_hash, bytes_transferred = upload_func(**upload_func_kwargs) if not success: raise LibcloudError(value='Object upload failed, Perhaps a timeout?', driver=self) result_dict = { 'response': response, 'data_hash': data_hash, 'bytes_transferred': bytes_transferred } return result_dict
def _upload_object(self, object_name, content_type, upload_func, upload_func_kwargs, request_path, request_method='PUT', headers=None, file_path=None, iterator=None): """ Helper function for setting common request headers and calling the passed in callback which uploads an object. """ headers = headers or {} if file_path and not os.path.exists(file_path): raise OSError('File %s does not exist' % (file_path)) if iterator is not None and not hasattr(iterator, 'next'): raise AttributeError('iterator object must implement next() ' + 'method.') if not content_type: if file_path: name = file_path else: name = object_name content_type, _ = utils.guess_file_mime_type(name) if not content_type: raise AttributeError( 'File content-type could not be guessed and' + ' no content_type value provided') file_size = None if iterator: if self.supports_chunked_encoding: headers['Transfer-Encoding'] = 'chunked' upload_func_kwargs['chunked'] = True else: # Chunked transfer encoding is not supported. Need to buffer all # the data in memory so we can determine file size. iterator = utils.read_in_chunks(iterator=iterator) data = utils.exhaust_iterator(iterator=iterator) file_size = len(data) upload_func_kwargs['data'] = data else: file_size = os.path.getsize(file_path) upload_func_kwargs['chunked'] = False if file_size: headers['Content-Length'] = file_size headers['Content-Type'] = content_type response = self.connection.request(request_path, method=request_method, data=None, headers=headers, raw=True) upload_func_kwargs['response'] = response success, data_hash, bytes_transferred = upload_func( **upload_func_kwargs) if not success: raise LibcloudError( value='Object upload failed, Perhaps a timeout?', driver=self) result_dict = { 'response': response, 'data_hash': data_hash, 'bytes_transferred': bytes_transferred } return result_dict