def __put_to_s3(self, key, value): try: conn = boto.connect_s3( aws_access_key_id=self.__storage_client_config["access_key"], aws_secret_access_key=self. __storage_client_config["secret_key"], host=self.__storage_client_config["endpoint"], port=80, is_secure=False, calling_format=boto.s3.connection.OrdinaryCallingFormat()) bucket = conn.get_bucket(self.__storage_client_config["bucket"]) bucket.delete_key(key) store_key = bucket.new_key(key) #Get size of the bytestream source_size = sys.getsizeof(value) if source_size > self.__storage_client_config["size_limit"]: # Ceph throws EntityTooSmall Error when chunk size is less than 5 MB. if self.__storage_client_config["chunk_size"] < 5242880: raise Exception("chunk_size can't be smaller than 5MB.") chunk_size = self.__storage_client_config["chunk_size"] multipart_upload = bucket.initiate_multipart_upload(store_key) chunk_count = int(math.ceil(source_size / float(chunk_size))) chunks = lambda byte_array, chunk_size: [ byte_array[x:x + chunk_size] for x in xrange(0, len(byte_array), chunk_size) ] chunk_list = chunks(value, chunk_size) assert chunk_count == len(chunk_list) for idx in range(chunk_count): multipart_upload.upload_part_from_file(StringIO.StringIO( chunk_list[idx]), part_num=idx + 1) multipart_upload.complete_upload() else: store_key.set_contents_from_string(value) conn.close() except AWSConnectionError as e: raise AWSConnectionError("Unable to connect to AWS") except Exception as e: raise Exception("Exception occured" + str(e))
def __get_from_s3(self, key): try: conn = boto.connect_s3( aws_access_key_id=self.__storage_client_config["access_key"], aws_secret_access_key=self. __storage_client_config["secret_key"], host=self.__storage_client_config["endpoint"], port=80, is_secure=False, calling_format=boto.s3.connection.OrdinaryCallingFormat()) bucket = conn.get_bucket(self.__storage_client_config["bucket"]) store_key = bucket.get_key(key) result = store_key.get_contents_as_string() conn.close() return result except AWSConnectionError as e: raise AWSConnectionError("Unable to connect to AWS") except Exception as e: raise Exception("Exception occured" + str(e))
def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): self.proxy = proxy self.proxy_port = proxy_port self.proxy_user = proxy_user self.proxy_pass = proxy_pass if os.environ.has_key('http_proxy') and not self.proxy: pattern = re.compile( '(?:http://)?' \ '(?:(?P<user>\w+):(?P<pass>.*)@)?' \ '(?P<host>[\w\-\.]+)' \ '(?::(?P<port>\d+))?' ) match = pattern.match(os.environ['http_proxy']) if match: self.proxy = match.group('host') self.proxy_port = match.group('port') self.proxy_user = match.group('user') self.proxy_pass = match.group('pass') else: if not self.proxy: self.proxy = config.get_value('Boto', 'proxy', None) if not self.proxy_port: self.proxy_port = config.get_value('Boto', 'proxy_port', None) if not self.proxy_user: self.proxy_user = config.get_value('Boto', 'proxy_user', None) if not self.proxy_pass: self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) if not self.proxy_port and self.proxy: print "http_proxy environment variable does not specify " \ "a port, using default" self.proxy_port = self.port self.use_proxy = (self.proxy != None) if self.use_proxy and self.is_secure: raise AWSConnectionError( "Unable to provide secure connection through proxy")
def test_max_retry_after_aws_connect_error(self): self._test_max_retry_limit_causes_failure( AWSConnectionError( "Unable to provide secure connection through proxy"))
def test_retry_after_aws_connect_error(self): self._test_retry_after_limited_retry_error( AWSConnectionError( "Unable to provide secure connection through proxy"))
def testRetryOnConnectionError(self): # retryOnCloudWatchTransientError with retry on AWSConnectionError self._testRetryCommon(AWSConnectionError("Error Message"))