def __put_to_s3(self, key, value):
try:
conn = boto.connect_s3(
aws_access_key_id=self.__storage_client_config["access_key"],
aws_secret_access_key=self.
__storage_client_config["secret_key"],
host=self.__storage_client_config["endpoint"],
port=80,
is_secure=False,
calling_format=boto.s3.connection.OrdinaryCallingFormat())
bucket = conn.get_bucket(self.__storage_client_config["bucket"])
bucket.delete_key(key)
store_key = bucket.new_key(key)
#Get size of the bytestream
source_size = sys.getsizeof(value)
if source_size > self.__storage_client_config["size_limit"]:
# Ceph throws EntityTooSmall Error when chunk size is less than 5 MB.
if self.__storage_client_config["chunk_size"] < 5242880:
raise Exception("chunk_size can't be smaller than 5MB.")
chunk_size = self.__storage_client_config["chunk_size"]
multipart_upload = bucket.initiate_multipart_upload(store_key)
chunk_count = int(math.ceil(source_size / float(chunk_size)))
chunks = lambda byte_array, chunk_size: [
byte_array[x:x + chunk_size]
for x in xrange(0, len(byte_array), chunk_size)
]
chunk_list = chunks(value, chunk_size)
assert chunk_count == len(chunk_list)
for idx in range(chunk_count):
multipart_upload.upload_part_from_file(StringIO.StringIO(
chunk_list[idx]),
part_num=idx + 1)
multipart_upload.complete_upload()
else:
store_key.set_contents_from_string(value)
conn.close()
except AWSConnectionError as e:
raise AWSConnectionError("Unable to connect to AWS")
except Exception as e:
raise Exception("Exception occured" + str(e))
def __get_from_s3(self, key):
try:
conn = boto.connect_s3(
aws_access_key_id=self.__storage_client_config["access_key"],
aws_secret_access_key=self.
__storage_client_config["secret_key"],
host=self.__storage_client_config["endpoint"],
port=80,
is_secure=False,
calling_format=boto.s3.connection.OrdinaryCallingFormat())
bucket = conn.get_bucket(self.__storage_client_config["bucket"])
store_key = bucket.get_key(key)
result = store_key.get_contents_as_string()
conn.close()
return result
except AWSConnectionError as e:
raise AWSConnectionError("Unable to connect to AWS")
except Exception as e:
raise Exception("Exception occured" + str(e))
def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass):
self.proxy = proxy
self.proxy_port = proxy_port
self.proxy_user = proxy_user
self.proxy_pass = proxy_pass
if os.environ.has_key('http_proxy') and not self.proxy:
pattern = re.compile(
'(?:http://)?' \
'(?:(?P<user>\w+):(?P<pass>.*)@)?' \
'(?P<host>[\w\-\.]+)' \
'(?::(?P<port>\d+))?'
)
match = pattern.match(os.environ['http_proxy'])
if match:
self.proxy = match.group('host')
self.proxy_port = match.group('port')
self.proxy_user = match.group('user')
self.proxy_pass = match.group('pass')
else:
if not self.proxy:
self.proxy = config.get_value('Boto', 'proxy', None)
if not self.proxy_port:
self.proxy_port = config.get_value('Boto', 'proxy_port', None)
if not self.proxy_user:
self.proxy_user = config.get_value('Boto', 'proxy_user', None)
if not self.proxy_pass:
self.proxy_pass = config.get_value('Boto', 'proxy_pass', None)
if not self.proxy_port and self.proxy:
print "http_proxy environment variable does not specify " \
"a port, using default"
self.proxy_port = self.port
self.use_proxy = (self.proxy != None)
if self.use_proxy and self.is_secure:
raise AWSConnectionError(
"Unable to provide secure connection through proxy")
def test_max_retry_after_aws_connect_error(self):
self._test_max_retry_limit_causes_failure(
AWSConnectionError(
"Unable to provide secure connection through proxy"))
def test_retry_after_aws_connect_error(self):
self._test_retry_after_limited_retry_error(
AWSConnectionError(
"Unable to provide secure connection through proxy"))
def testRetryOnConnectionError(self):
# retryOnCloudWatchTransientError with retry on AWSConnectionError
self._testRetryCommon(AWSConnectionError("Error Message"))