示例#1
0
    def __put_to_s3(self, key, value):
        try:
            conn = boto.connect_s3(
                aws_access_key_id=self.__storage_client_config["access_key"],
                aws_secret_access_key=self.
                __storage_client_config["secret_key"],
                host=self.__storage_client_config["endpoint"],
                port=80,
                is_secure=False,
                calling_format=boto.s3.connection.OrdinaryCallingFormat())
            bucket = conn.get_bucket(self.__storage_client_config["bucket"])

            bucket.delete_key(key)
            store_key = bucket.new_key(key)

            #Get size of the bytestream
            source_size = sys.getsizeof(value)

            if source_size > self.__storage_client_config["size_limit"]:
                # Ceph throws EntityTooSmall Error when chunk size is less than 5 MB.
                if self.__storage_client_config["chunk_size"] < 5242880:
                    raise Exception("chunk_size can't be smaller than 5MB.")

                chunk_size = self.__storage_client_config["chunk_size"]
                multipart_upload = bucket.initiate_multipart_upload(store_key)
                chunk_count = int(math.ceil(source_size / float(chunk_size)))

                chunks = lambda byte_array, chunk_size: [
                    byte_array[x:x + chunk_size]
                    for x in xrange(0, len(byte_array), chunk_size)
                ]

                chunk_list = chunks(value, chunk_size)

                assert chunk_count == len(chunk_list)

                for idx in range(chunk_count):
                    multipart_upload.upload_part_from_file(StringIO.StringIO(
                        chunk_list[idx]),
                                                           part_num=idx + 1)

                multipart_upload.complete_upload()
            else:
                store_key.set_contents_from_string(value)
            conn.close()
        except AWSConnectionError as e:
            raise AWSConnectionError("Unable to connect to AWS")
        except Exception as e:
            raise Exception("Exception occured" + str(e))
示例#2
0
 def __get_from_s3(self, key):
     try:
         conn = boto.connect_s3(
             aws_access_key_id=self.__storage_client_config["access_key"],
             aws_secret_access_key=self.
             __storage_client_config["secret_key"],
             host=self.__storage_client_config["endpoint"],
             port=80,
             is_secure=False,
             calling_format=boto.s3.connection.OrdinaryCallingFormat())
         bucket = conn.get_bucket(self.__storage_client_config["bucket"])
         store_key = bucket.get_key(key)
         result = store_key.get_contents_as_string()
         conn.close()
         return result
     except AWSConnectionError as e:
         raise AWSConnectionError("Unable to connect to AWS")
     except Exception as e:
         raise Exception("Exception occured" + str(e))
示例#3
0
    def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass):
        self.proxy = proxy
        self.proxy_port = proxy_port
        self.proxy_user = proxy_user
        self.proxy_pass = proxy_pass
        if os.environ.has_key('http_proxy') and not self.proxy:
            pattern = re.compile(
                '(?:http://)?' \
                '(?:(?P<user>\w+):(?P<pass>.*)@)?' \
                '(?P<host>[\w\-\.]+)' \
                '(?::(?P<port>\d+))?'
            )
            match = pattern.match(os.environ['http_proxy'])
            if match:
                self.proxy = match.group('host')
                self.proxy_port = match.group('port')
                self.proxy_user = match.group('user')
                self.proxy_pass = match.group('pass')
        else:
            if not self.proxy:
                self.proxy = config.get_value('Boto', 'proxy', None)
            if not self.proxy_port:
                self.proxy_port = config.get_value('Boto', 'proxy_port', None)
            if not self.proxy_user:
                self.proxy_user = config.get_value('Boto', 'proxy_user', None)
            if not self.proxy_pass:
                self.proxy_pass = config.get_value('Boto', 'proxy_pass', None)

        if not self.proxy_port and self.proxy:
            print "http_proxy environment variable does not specify " \
                "a port, using default"
            self.proxy_port = self.port
        self.use_proxy = (self.proxy != None)
        if self.use_proxy and self.is_secure:
            raise AWSConnectionError(
                "Unable to provide secure connection through proxy")
示例#4
0
 def test_max_retry_after_aws_connect_error(self):
     self._test_max_retry_limit_causes_failure(
         AWSConnectionError(
             "Unable to provide secure connection through proxy"))
示例#5
0
 def test_retry_after_aws_connect_error(self):
     self._test_retry_after_limited_retry_error(
         AWSConnectionError(
             "Unable to provide secure connection through proxy"))
示例#6
0
 def testRetryOnConnectionError(self):
   # retryOnCloudWatchTransientError with retry on AWSConnectionError
   self._testRetryCommon(AWSConnectionError("Error Message"))