def __init__(self, client, config=None, osutil=None, executor_cls=None):
"""A transfer manager interface for Amazon S3
:param client: Client to be used by the manager
:param config: TransferConfig to associate specific configurations
:param osutil: OSUtils object to use for os-related behavior when
using with transfer manager.
:type executor_cls: s3transfer.futures.BaseExecutor
:param executor_cls: The class of executor to use with the transfer
manager. By default, concurrent.futures.ThreadPoolExecutor is used.
"""
self._client = client
self._config = config
if config is None:
self._config = TransferConfig()
self._osutil = osutil
if osutil is None:
self._osutil = OSUtils()
self._coordinator_controller = TransferCoordinatorController()
# A counter to create unique id's for each transfer submitted.
self._id_counter = 0
# The executor responsible for making S3 API transfer requests
self._request_executor = BoundedExecutor(
max_size=self._config.max_request_queue_size,
max_num_threads=self._config.max_request_concurrency,
tag_semaphores={
IN_MEMORY_UPLOAD_TAG:
TaskSemaphore(self._config.max_in_memory_upload_chunks),
IN_MEMORY_DOWNLOAD_TAG:
SlidingWindowSemaphore(
self._config.max_in_memory_download_chunks)
},
executor_cls=executor_cls)
# The executor responsible for submitting the necessary tasks to
# perform the desired transfer
self._submission_executor = BoundedExecutor(
max_size=self._config.max_submission_queue_size,
max_num_threads=self._config.max_submission_concurrency,
executor_cls=executor_cls)
# There is one thread available for writing to disk. It will handle
# downloads for all files.
self._io_executor = BoundedExecutor(
max_size=self._config.max_io_queue_size,
max_num_threads=1,
executor_cls=executor_cls)
# The component responsible for limiting bandwidth usage if it
# is configured.
self._bandwidth_limiter = None
if self._config.max_bandwidth is not None:
logger.debug('Setting max_bandwidth to %s',
self._config.max_bandwidth)
leaky_bucket = LeakyBucket(self._config.max_bandwidth)
self._bandwidth_limiter = BandwidthLimiter(leaky_bucket)
self._register_handlers()
def __init__(self, max_size, max_num_threads, tag_semaphores=None,
executor_cls=None):
"""An executor implentation that has a maximum queued up tasks
The executor will block if the number of tasks that have been
submitted and is currently working on is past its maximum.
:params max_size: The maximum number of inflight futures. An inflight
future means that the task is either queued up or is currently
being executed. A size of None or 0 means that the executor will
have no bound in terms of the number of inflight futures.
:params max_num_threads: The maximum number of threads the executor
uses.
:type tag_semaphores: dict
:params tag_semaphores: A dictionary where the key is the name of the
tag and the value is the semaphore to use when limiting the
number of tasks the executor is processing at a time.
:type executor_cls: BaseExecutor
:param underlying_executor_cls: The executor class that
get bounded by this executor. If None is provided, the
concurrent.futures.ThreadPoolExecutor class is used.
"""
self._max_num_threads = max_num_threads
if executor_cls is None:
executor_cls = self.EXECUTOR_CLS
self._executor = executor_cls(max_workers=self._max_num_threads)
self._semaphore = TaskSemaphore(max_size)
self._tag_semaphores = tag_semaphores
def setUp(self):
super(BaseSubmissionTaskTest, self).setUp()
self.config = TransferConfig()
self.osutil = OSUtils()
self.executor = BoundedExecutor(
1000, 1, {
IN_MEMORY_UPLOAD_TAG: TaskSemaphore(10),
IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(10)
})
class TestTaskSemaphore(unittest.TestCase):
def setUp(self):
self.semaphore = TaskSemaphore(1)
def test_should_block_at_max_capacity(self):
self.semaphore.acquire('a', blocking=False)
with self.assertRaises(NoResourcesAvailable):
self.semaphore.acquire('a', blocking=False)
def test_release_capacity(self):
acquire_token = self.semaphore.acquire('a', blocking=False)
self.semaphore.release('a', acquire_token)
try:
self.semaphore.acquire('a', blocking=False)
except NoResourcesAvailable:
self.fail('The release of the semaphore should have allowed for '
'the second acquire to not be blocked')
class TestTaskSemaphore(unittest.TestCase):
def setUp(self):
self.semaphore = TaskSemaphore(1)
def test_should_block_at_max_capacity(self):
self.semaphore.acquire('a', blocking=False)
with self.assertRaises(NoResourcesAvailable):
self.semaphore.acquire('a', blocking=False)
def test_release_capacity(self):
acquire_token = self.semaphore.acquire('a', blocking=False)
self.semaphore.release('a', acquire_token)
try:
self.semaphore.acquire('a', blocking=False)
except NoResourcesAvailable:
self.fail(
'The release of the semaphore should have allowed for '
'the second acquire to not be blocked'
)
def test_submit(self):
# Submit a callable to the transfer coordinator. It should submit it
# to the executor.
executor = RecordingExecutor(
BoundedExecutor(1, 1, {'my-tag': TaskSemaphore(1)}))
task = ReturnFooTask(self.transfer_coordinator)
future = self.transfer_coordinator.submit(executor, task, tag='my-tag')
executor.shutdown()
# Make sure the future got submit and executed as well by checking its
# result value which should include the provided future tag.
self.assertEqual(executor.submissions, [{
'block': True,
'tag': 'my-tag',
'task': task
}])
self.assertEqual(future.result(), 'foo')
def __init__(self, client, config=None, osutil=None):
"""A transfer manager interface for Amazon S3
:param client: Client to be used by the manager
:param config: TransferConfig to associate specific configurations
:param osutil: OSUtils object to use for os-related behavior when
using with transfer manager.
"""
self._client = client
self._config = config
if config is None:
self._config = TransferConfig()
self._osutil = osutil
if osutil is None:
self._osutil = OSUtils()
self._coordinator_controller = TransferCoordinatorController()
# A counter to create unique id's for each transfer submitted.
self._id_counter = 0
# The executor responsible for making S3 API transfer requests
self._request_executor = BoundedExecutor(
max_size=self._config.max_request_queue_size,
max_num_threads=self._config.max_request_concurrency,
tag_semaphores={
IN_MEMORY_UPLOAD_TAG: TaskSemaphore(
self._config.max_in_memory_upload_chunks),
IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(
self._config.max_in_memory_download_chunks)
}
)
# The executor responsible for submitting the necessary tasks to
# perform the desired transfer
self._submission_executor = BoundedExecutor(
max_size=self._config.max_submission_queue_size,
max_num_threads=self._config.max_submission_concurrency
)
# There is one thread available for writing to disk. It will handle
# downloads for all files.
self._io_executor = BoundedExecutor(
max_size=self._config.max_io_queue_size,
max_num_threads=1
)
self._register_handlers()
def __init__(self, max_size, max_num_threads, tag_semaphores=None):
"""An executor implentation that has a maximum queued up tasks
The executor will block if the number of tasks that have been
submitted and is currently working on is past its maximum.
:params max_size: The maximum number of inflight futures. An inflight
future means that the task is either queued up or is currently
being executed. A size of None or 0 means that the executor will
have no bound in terms of the number of inflight futures.
:params max_num_threads: The maximum number of threads the executor
uses.
:type tag_semaphores: dict
:params tag_semaphores: A dictionary where the key is the name of the
tag and the value is the semaphore to use when limiting the
number of tasks the executor is processing at a time.
"""
self._max_num_threads = max_num_threads
self._executor = self.EXECUTOR_CLS(max_workers=self._max_num_threads)
self._semaphore = TaskSemaphore(max_size)
self._tag_semaphores = tag_semaphores
def add_semaphore(self, task_tag, count):
self.tag_semaphores[task_tag] = TaskSemaphore(count)
def setUp(self):
self.semaphore = TaskSemaphore(1)
def setUp(self):
self.semaphore = TaskSemaphore(1)
