def __init__(self, client, config=None, osutil=None, executor_cls=None):
"""A transfer manager interface for Amazon S3
:param client: Client to be used by the manager
:param config: TransferConfig to associate specific configurations
:param osutil: OSUtils object to use for os-related behavior when
using with transfer manager.
:type executor_cls: ibm_s3transfer.futures.BaseExecutor
:param executor_cls: The class of executor to use with the transfer
manager. By default, concurrent.futures.ThreadPoolExecutor is used.
"""
self._client = client
self._config = config
if config is None:
self._config = TransferConfig()
self._osutil = osutil
if osutil is None:
self._osutil = OSUtils()
self._coordinator_controller = TransferCoordinatorController()
# A counter to create unique id's for each transfer submitted.
self._id_counter = 0
# The executor responsible for making S3 API transfer requests
self._request_executor = BoundedExecutor(
max_size=self._config.max_request_queue_size,
max_num_threads=self._config.max_request_concurrency,
tag_semaphores={
IN_MEMORY_UPLOAD_TAG:
TaskSemaphore(self._config.max_in_memory_upload_chunks),
IN_MEMORY_DOWNLOAD_TAG:
SlidingWindowSemaphore(
self._config.max_in_memory_download_chunks)
},
executor_cls=executor_cls)
# The executor responsible for submitting the necessary tasks to
# perform the desired transfer
self._submission_executor = BoundedExecutor(
max_size=self._config.max_submission_queue_size,
max_num_threads=self._config.max_submission_concurrency,
executor_cls=executor_cls)
# There is one thread available for writing to disk. It will handle
# downloads for all files.
self._io_executor = BoundedExecutor(
max_size=self._config.max_io_queue_size,
max_num_threads=1,
executor_cls=executor_cls)
# The component responsible for limiting bandwidth usage if it
# is configured.
self._bandwidth_limiter = None
if self._config.max_bandwidth is not None:
logger.debug('Setting max_bandwidth to %s',
self._config.max_bandwidth)
leaky_bucket = LeakyBucket(self._config.max_bandwidth)
self._bandwidth_limiter = BandwidthLimiter(leaky_bucket)
self._register_handlers()
def setUp(self):
super(TestDownloadSubmissionTask, self).setUp()
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.subscribers = []
# Create a stream to read from
self.content = b'my content'
self.stream = six.BytesIO(self.content)
# A list to keep track of all of the bodies sent over the wire
# and their order.
self.call_args = self.get_call_args()
self.transfer_future = self.get_transfer_future(self.call_args)
self.io_executor = BoundedExecutor(1000, 1)
self.submission_main_kwargs = {
'client': self.client,
'config': self.config,
'osutil': self.osutil,
'request_executor': self.executor,
'io_executor': self.io_executor,
'transfer_future': self.transfer_future
}
self.submission_task = self.get_download_submission_task()
def setUp(self):
super(BaseSubmissionTaskTest, self).setUp()
self.config = TransferConfig()
self.osutil = OSUtils()
self.executor = BoundedExecutor(
1000, 1, {
IN_MEMORY_UPLOAD_TAG: TaskSemaphore(10),
IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(10)
})
def setUp(self):
super(BaseDownloadOutputManagerTest, self).setUp()
self.osutil = OSUtils()
# Create a file to write to
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
self.call_args = CallArgs(fileobj=self.filename)
self.future = self.get_transfer_future(self.call_args)
self.io_executor = BoundedExecutor(1000, 1)
def setUp(self):
super(TestGetObjectTask, self).setUp()
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.callbacks = []
self.max_attempts = 5
self.io_executor = BoundedExecutor(1000, 1)
self.content = b'my content'
self.stream = six.BytesIO(self.content)
self.fileobj = WriteCollector()
self.osutil = OSUtils()
self.io_chunksize = 64 * (1024**2)
self.task_cls = GetObjectTask
self.download_output_manager = DownloadSeekableOutputManager(
self.osutil, self.transfer_coordinator, self.io_executor)
def test_submit(self):
# Submit a callable to the transfer coordinator. It should submit it
# to the executor.
executor = RecordingExecutor(
BoundedExecutor(1, 1, {'my-tag': TaskSemaphore(1)}))
task = ReturnFooTask(self.transfer_coordinator)
future = self.transfer_coordinator.submit(executor, task, tag='my-tag')
executor.shutdown()
# Make sure the future got submit and executed as well by checking its
# result value which should include the provided future tag.
self.assertEqual(executor.submissions, [{
'block': True,
'tag': 'my-tag',
'task': task
}])
self.assertEqual(future.result(), 'foo')
def test_association_and_disassociation_on_submit(self):
self.transfer_coordinator = RecordingTransferCoordinator()
# Submit a callable to the transfer coordinator.
executor = BoundedExecutor(1, 1)
task = ReturnFooTask(self.transfer_coordinator)
future = self.transfer_coordinator.submit(executor, task)
executor.shutdown()
# Make sure the future that got submitted was associated to the
# transfer future at some point.
self.assertEqual(
self.transfer_coordinator.all_transfer_futures_ever_associated,
set([future]))
# Make sure the future got disassociated once the future is now done
# by looking at the currently associated futures.
self.assertEqual(self.transfer_coordinator.associated_futures, set([]))
def test_submit_writes_from_internal_queue(self):
class FakeQueue(object):
def request_writes(self, offset, data):
return [
{
'offset': 0,
'data': 'foo'
},
{
'offset': 3,
'data': 'bar'
},
]
q = FakeQueue()
io_executor = BoundedExecutor(1000, 1)
manager = DownloadNonSeekableOutputManager(self.osutil,
self.transfer_coordinator,
io_executor=io_executor,
defer_queue=q)
fileobj = WriteCollector()
manager.queue_file_io_task(fileobj=fileobj, data='foo', offset=1)
io_executor.shutdown()
self.assertEqual(fileobj.writes, [(0, 'foo'), (3, 'bar')])
def setUp(self):
super(TestSubmissionTask, self).setUp()
self.executor = BoundedExecutor(1000, 5)
self.call_args = CallArgs(subscribers=[])
self.transfer_future = self.get_transfer_future(self.call_args)
self.main_kwargs = {'transfer_future': self.transfer_future}
def test_replace_underlying_executor(self):
mocked_executor_cls = mock.Mock(BaseExecutor)
executor = BoundedExecutor(10, 1, {}, mocked_executor_cls)
executor.submit(self.get_task(ReturnFooTask))
self.assertTrue(mocked_executor_cls.return_value.submit.called)
def get_executor(self, max_size=1, max_num_threads=1):
return BoundedExecutor(max_size, max_num_threads, self.tag_semaphores)