def test_acquire_release_basic_case(self):
sem = SlidingWindowSemaphore(1)
# Count is 1
num = sem.acquire('a', blocking=False)
self.assertEqual(num, 0)
sem.release('a', 0)
def test_acquire_release_basic_case(self):
sem = SlidingWindowSemaphore(1)
# Count is 1
num = sem.acquire('a', blocking=False)
self.assertEqual(num, 0)
sem.release('a', 0)
def test_can_check_in_partial_range(self):
sem = SlidingWindowSemaphore(4)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.release('a', 1)
sem.release('a', 3)
sem.release('a', 0)
self.assertEqual(sem.current_count(), 2)
def test_can_acquire_a_range(self):
sem = SlidingWindowSemaphore(3)
self.assertEqual(sem.acquire('a', blocking=False), 0)
self.assertEqual(sem.acquire('a', blocking=False), 1)
self.assertEqual(sem.acquire('a', blocking=False), 2)
sem.release('a', 0)
sem.release('a', 1)
sem.release('a', 2)
# Now we're reset so we should be able to acquire the same
# sequence again.
self.assertEqual(sem.acquire('a', blocking=False), 3)
self.assertEqual(sem.acquire('a', blocking=False), 4)
self.assertEqual(sem.acquire('a', blocking=False), 5)
self.assertEqual(sem.current_count(), 0)
def test_stress_invariants_random_order(self):
sem = SlidingWindowSemaphore(100)
for _ in range(10):
recorded = []
for _ in range(100):
recorded.append(sem.acquire('a', blocking=False))
# Release them in randomized order. As long as we
# eventually free all 100, we should have all the
# resources released.
random.shuffle(recorded)
for i in recorded:
sem.release('a', i)
# Everything's freed so should be back at count == 100
self.assertEqual(sem.current_count(), 100)
def test_is_error_to_double_release(self):
# This is different than other error tests because
# we're verifying we can reset the state after an
# acquire/release cycle.
sem = SlidingWindowSemaphore(2)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.release('a', 0)
sem.release('a', 1)
self.assertEqual(sem.current_count(), 2)
with self.assertRaises(ValueError):
sem.release('a', 0)
def __init__(self, client, config=None, osutil=None, executor_cls=None):
"""A transfer manager interface for Amazon S3
:param client: Client to be used by the manager
:param config: TransferConfig to associate specific configurations
:param osutil: OSUtils object to use for os-related behavior when
using with transfer manager.
:type executor_cls: s3transfer.futures.BaseExecutor
:param executor_cls: The class of executor to use with the transfer
manager. By default, concurrent.futures.ThreadPoolExecutor is used.
"""
self._client = client
self._config = config
if config is None:
self._config = TransferConfig()
self._osutil = osutil
if osutil is None:
self._osutil = OSUtils()
self._coordinator_controller = TransferCoordinatorController()
# A counter to create unique id's for each transfer submitted.
self._id_counter = 0
# The executor responsible for making S3 API transfer requests
self._request_executor = BoundedExecutor(
max_size=self._config.max_request_queue_size,
max_num_threads=self._config.max_request_concurrency,
tag_semaphores={
IN_MEMORY_UPLOAD_TAG:
TaskSemaphore(self._config.max_in_memory_upload_chunks),
IN_MEMORY_DOWNLOAD_TAG:
SlidingWindowSemaphore(
self._config.max_in_memory_download_chunks)
},
executor_cls=executor_cls)
# The executor responsible for submitting the necessary tasks to
# perform the desired transfer
self._submission_executor = BoundedExecutor(
max_size=self._config.max_submission_queue_size,
max_num_threads=self._config.max_submission_concurrency,
executor_cls=executor_cls)
# There is one thread available for writing to disk. It will handle
# downloads for all files.
self._io_executor = BoundedExecutor(
max_size=self._config.max_io_queue_size,
max_num_threads=1,
executor_cls=executor_cls)
# The component responsible for limiting bandwidth usage if it
# is configured.
self._bandwidth_limiter = None
if self._config.max_bandwidth is not None:
logger.debug('Setting max_bandwidth to %s',
self._config.max_bandwidth)
leaky_bucket = LeakyBucket(self._config.max_bandwidth)
self._bandwidth_limiter = BandwidthLimiter(leaky_bucket)
self._register_handlers()
def test_raises_error_when_count_is_zero(self):
sem = SlidingWindowSemaphore(3)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
# Count is now 0 so trying to acquire should fail.
with self.assertRaises(NoResourcesAvailable):
sem.acquire('a', blocking=False)
def test_can_acquire_release_multiple_times(self):
sem = SlidingWindowSemaphore(1)
num = sem.acquire('a', blocking=False)
self.assertEqual(num, 0)
sem.release('a', num)
num = sem.acquire('a', blocking=False)
self.assertEqual(num, 1)
sem.release('a', num)
def setUp(self):
super(BaseSubmissionTaskTest, self).setUp()
self.config = TransferConfig()
self.osutil = OSUtils()
self.executor = BoundedExecutor(
1000, 1, {
IN_MEMORY_UPLOAD_TAG: TaskSemaphore(10),
IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(10)
})
def test_can_track_multiple_tags(self):
sem = SlidingWindowSemaphore(3)
self.assertEqual(sem.acquire('a', blocking=False), 0)
self.assertEqual(sem.acquire('b', blocking=False), 0)
self.assertEqual(sem.acquire('a', blocking=False), 1)
# We're at our max of 3 even though 2 are for A and 1 is for B.
with self.assertRaises(NoResourcesAvailable):
sem.acquire('a', blocking=False)
with self.assertRaises(NoResourcesAvailable):
sem.acquire('b', blocking=False)
def test_is_error_to_double_release(self):
# This is different than other error tests because
# we're verifying we can reset the state after an
# acquire/release cycle.
sem = SlidingWindowSemaphore(2)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.release('a', 0)
sem.release('a', 1)
self.assertEqual(sem.current_count(), 2)
with self.assertRaises(ValueError):
sem.release('a', 0)
def test_can_acquire_release_multiple_times(self):
sem = SlidingWindowSemaphore(1)
num = sem.acquire('a', blocking=False)
self.assertEqual(num, 0)
sem.release('a', num)
num = sem.acquire('a', blocking=False)
self.assertEqual(num, 1)
sem.release('a', num)
def test_raises_error_when_count_is_zero(self):
sem = SlidingWindowSemaphore(3)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
# Count is now 0 so trying to acquire should fail.
with self.assertRaises(NoResourcesAvailable):
sem.acquire('a', blocking=False)
def test_acquire_blocks_until_release_is_called(self):
sem = SlidingWindowSemaphore(2)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
def acquire():
# This next call to acquire will block.
self.assertEqual(sem.acquire('a', blocking=True), 2)
t = threading.Thread(target=acquire)
self.threads.append(t)
# Starting the thread will block the sem.acquire()
# in the acquire function above.
t.start()
# This still will keep the thread blocked.
sem.release('a', 1)
# Releasing the min element will unblock the thread.
sem.release('a', 0)
t.join()
sem.release('a', 2)
def test_blocking_stress(self):
sem = SlidingWindowSemaphore(5)
num_threads = 10
num_iterations = 50
def acquire():
for _ in range(num_iterations):
num = sem.acquire('a', blocking=True)
time.sleep(0.001)
sem.release('a', num)
for i in range(num_threads):
t = threading.Thread(target=acquire)
self.threads.append(t)
self.start_threads()
self.join_threads()
# Should have all the available resources freed.
self.assertEqual(sem.current_count(), 5)
# Should have acquired num_threads * num_iterations
self.assertEqual(sem.acquire('a', blocking=False),
num_threads * num_iterations)
def test_blocking_stress(self):
sem = SlidingWindowSemaphore(5)
num_threads = 10
num_iterations = 50
def acquire():
for _ in range(num_iterations):
num = sem.acquire('a', blocking=True)
time.sleep(0.001)
sem.release('a', num)
for i in range(num_threads):
t = threading.Thread(target=acquire)
self.threads.append(t)
self.start_threads()
self.join_threads()
# Should have all the available resources freed.
self.assertEqual(sem.current_count(), 5)
# Should have acquired num_threads * num_iterations
self.assertEqual(sem.acquire('a', blocking=False),
num_threads * num_iterations)
def test_can_track_multiple_tags(self):
sem = SlidingWindowSemaphore(3)
self.assertEqual(sem.acquire('a', blocking=False), 0)
self.assertEqual(sem.acquire('b', blocking=False), 0)
self.assertEqual(sem.acquire('a', blocking=False), 1)
# We're at our max of 3 even though 2 are for A and 1 is for B.
with self.assertRaises(NoResourcesAvailable):
sem.acquire('a', blocking=False)
with self.assertRaises(NoResourcesAvailable):
sem.acquire('b', blocking=False)
def test_acquire_blocks_until_release_is_called(self):
sem = SlidingWindowSemaphore(2)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
def acquire():
# This next call to acquire will block.
self.assertEqual(sem.acquire('a', blocking=True), 2)
t = threading.Thread(target=acquire)
self.threads.append(t)
# Starting the thread will block the sem.acquire()
# in the acquire function above.
t.start()
# This still will keep the thread blocked.
sem.release('a', 1)
# Releasing the min element will unblock the thread.
sem.release('a', 0)
t.join()
sem.release('a', 2)
def __init__(self, client, config=None, osutil=None):
"""A transfer manager interface for Amazon S3
:param client: Client to be used by the manager
:param config: TransferConfig to associate specific configurations
:param osutil: OSUtils object to use for os-related behavior when
using with transfer manager.
"""
self._client = client
self._config = config
if config is None:
self._config = TransferConfig()
self._osutil = osutil
if osutil is None:
self._osutil = OSUtils()
self._coordinator_controller = TransferCoordinatorController()
# A counter to create unique id's for each transfer submitted.
self._id_counter = 0
# The executor responsible for making S3 API transfer requests
self._request_executor = BoundedExecutor(
max_size=self._config.max_request_queue_size,
max_num_threads=self._config.max_request_concurrency,
tag_semaphores={
IN_MEMORY_UPLOAD_TAG: TaskSemaphore(
self._config.max_in_memory_upload_chunks),
IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(
self._config.max_in_memory_download_chunks)
}
)
# The executor responsible for submitting the necessary tasks to
# perform the desired transfer
self._submission_executor = BoundedExecutor(
max_size=self._config.max_submission_queue_size,
max_num_threads=self._config.max_submission_concurrency
)
# There is one thread available for writing to disk. It will handle
# downloads for all files.
self._io_executor = BoundedExecutor(
max_size=self._config.max_io_queue_size,
max_num_threads=1
)
self._register_handlers()
def test_stress_invariants_random_order(self):
sem = SlidingWindowSemaphore(100)
for _ in range(10):
recorded = []
for _ in range(100):
recorded.append(sem.acquire('a', blocking=False))
# Release them in randomized order. As long as we
# eventually free all 100, we should have all the
# resources released.
random.shuffle(recorded)
for i in recorded:
sem.release('a', i)
# Everything's freed so should be back at count == 100
self.assertEqual(sem.current_count(), 100)
def test_error_to_release_unknown_tag(self):
sem = SlidingWindowSemaphore(3)
with self.assertRaises(ValueError):
sem.release('a', 0)
def test_can_handle_multiple_tags_released(self):
sem = SlidingWindowSemaphore(4)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('b', blocking=False)
sem.acquire('b', blocking=False)
sem.release('b', 1)
sem.release('a', 1)
self.assertEqual(sem.current_count(), 0)
sem.release('b', 0)
self.assertEqual(sem.acquire('a', blocking=False), 2)
sem.release('a', 0)
self.assertEqual(sem.acquire('b', blocking=False), 2)
def test_is_error_to_release_unknown_sequence_number(self):
sem = SlidingWindowSemaphore(3)
sem.acquire('a', blocking=False)
with self.assertRaises(ValueError):
sem.release('a', 1)
def test_can_check_in_partial_range(self):
sem = SlidingWindowSemaphore(4)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.release('a', 1)
sem.release('a', 3)
sem.release('a', 0)
self.assertEqual(sem.current_count(), 2)
def test_release_counters_can_increment_counter_repeatedly(self):
sem = SlidingWindowSemaphore(3)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
# These two releases don't increment the counter
# because we're waiting on 0.
sem.release('a', 1)
sem.release('a', 2)
self.assertEqual(sem.current_count(), 0)
# But as soon as we release 0, we free up 0, 1, and 2.
sem.release('a', 0)
self.assertEqual(sem.current_count(), 3)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
def test_is_error_to_release_unknown_sequence_number(self):
sem = SlidingWindowSemaphore(3)
sem.acquire('a', blocking=False)
with self.assertRaises(ValueError):
sem.release('a', 1)
def test_counter_release_only_on_min_element(self):
sem = SlidingWindowSemaphore(3)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
# The count only increases when we free the min
# element. This means if we're currently failing to
# acquire now:
with self.assertRaises(NoResourcesAvailable):
sem.acquire('a', blocking=False)
# Then freeing a non-min element:
sem.release('a', 1)
# doesn't change anything. We still fail to acquire.
with self.assertRaises(NoResourcesAvailable):
sem.acquire('a', blocking=False)
self.assertEqual(sem.current_count(), 0)
def test_can_handle_multiple_tags_released(self):
sem = SlidingWindowSemaphore(4)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('b', blocking=False)
sem.acquire('b', blocking=False)
sem.release('b', 1)
sem.release('a', 1)
self.assertEqual(sem.current_count(), 0)
sem.release('b', 0)
self.assertEqual(sem.acquire('a', blocking=False), 2)
sem.release('a', 0)
self.assertEqual(sem.acquire('b', blocking=False), 2)
def test_can_acquire_a_range(self):
sem = SlidingWindowSemaphore(3)
self.assertEqual(sem.acquire('a', blocking=False), 0)
self.assertEqual(sem.acquire('a', blocking=False), 1)
self.assertEqual(sem.acquire('a', blocking=False), 2)
sem.release('a', 0)
sem.release('a', 1)
sem.release('a', 2)
# Now we're reset so we should be able to acquire the same
# sequence again.
self.assertEqual(sem.acquire('a', blocking=False), 3)
self.assertEqual(sem.acquire('a', blocking=False), 4)
self.assertEqual(sem.acquire('a', blocking=False), 5)
self.assertEqual(sem.current_count(), 0)
def test_error_to_release_unknown_tag(self):
sem = SlidingWindowSemaphore(3)
with self.assertRaises(ValueError):
sem.release('a', 0)
def test_release_counters_can_increment_counter_repeatedly(self):
sem = SlidingWindowSemaphore(3)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
# These two releases don't increment the counter
# because we're waiting on 0.
sem.release('a', 1)
sem.release('a', 2)
self.assertEqual(sem.current_count(), 0)
# But as soon as we release 0, we free up 0, 1, and 2.
sem.release('a', 0)
self.assertEqual(sem.current_count(), 3)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
def test_counter_release_only_on_min_element(self):
sem = SlidingWindowSemaphore(3)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
sem.acquire('a', blocking=False)
# The count only increases when we free the min
# element. This means if we're currently failing to
# acquire now:
with self.assertRaises(NoResourcesAvailable):
sem.acquire('a', blocking=False)
# Then freeing a non-min element:
sem.release('a', 1)
# doesn't change anything. We still fail to acquire.
with self.assertRaises(NoResourcesAvailable):
sem.acquire('a', blocking=False)
self.assertEqual(sem.current_count(), 0)