def test_acquire_release_basic_case(self): sem = SlidingWindowSemaphore(1) # Count is 1 num = sem.acquire('a', blocking=False) self.assertEqual(num, 0) sem.release('a', 0)
def test_can_check_in_partial_range(self): sem = SlidingWindowSemaphore(4) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) sem.release('a', 1) sem.release('a', 3) sem.release('a', 0) self.assertEqual(sem.current_count(), 2)
def test_can_acquire_a_range(self): sem = SlidingWindowSemaphore(3) self.assertEqual(sem.acquire('a', blocking=False), 0) self.assertEqual(sem.acquire('a', blocking=False), 1) self.assertEqual(sem.acquire('a', blocking=False), 2) sem.release('a', 0) sem.release('a', 1) sem.release('a', 2) # Now we're reset so we should be able to acquire the same # sequence again. self.assertEqual(sem.acquire('a', blocking=False), 3) self.assertEqual(sem.acquire('a', blocking=False), 4) self.assertEqual(sem.acquire('a', blocking=False), 5) self.assertEqual(sem.current_count(), 0)
def test_stress_invariants_random_order(self): sem = SlidingWindowSemaphore(100) for _ in range(10): recorded = [] for _ in range(100): recorded.append(sem.acquire('a', blocking=False)) # Release them in randomized order. As long as we # eventually free all 100, we should have all the # resources released. random.shuffle(recorded) for i in recorded: sem.release('a', i) # Everything's freed so should be back at count == 100 self.assertEqual(sem.current_count(), 100)
def test_is_error_to_double_release(self): # This is different than other error tests because # we're verifying we can reset the state after an # acquire/release cycle. sem = SlidingWindowSemaphore(2) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) sem.release('a', 0) sem.release('a', 1) self.assertEqual(sem.current_count(), 2) with self.assertRaises(ValueError): sem.release('a', 0)
def __init__(self, client, config=None, osutil=None, executor_cls=None): """A transfer manager interface for Amazon S3 :param client: Client to be used by the manager :param config: TransferConfig to associate specific configurations :param osutil: OSUtils object to use for os-related behavior when using with transfer manager. :type executor_cls: s3transfer.futures.BaseExecutor :param executor_cls: The class of executor to use with the transfer manager. By default, concurrent.futures.ThreadPoolExecutor is used. """ self._client = client self._config = config if config is None: self._config = TransferConfig() self._osutil = osutil if osutil is None: self._osutil = OSUtils() self._coordinator_controller = TransferCoordinatorController() # A counter to create unique id's for each transfer submitted. self._id_counter = 0 # The executor responsible for making S3 API transfer requests self._request_executor = BoundedExecutor( max_size=self._config.max_request_queue_size, max_num_threads=self._config.max_request_concurrency, tag_semaphores={ IN_MEMORY_UPLOAD_TAG: TaskSemaphore(self._config.max_in_memory_upload_chunks), IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore( self._config.max_in_memory_download_chunks) }, executor_cls=executor_cls) # The executor responsible for submitting the necessary tasks to # perform the desired transfer self._submission_executor = BoundedExecutor( max_size=self._config.max_submission_queue_size, max_num_threads=self._config.max_submission_concurrency, executor_cls=executor_cls) # There is one thread available for writing to disk. It will handle # downloads for all files. self._io_executor = BoundedExecutor( max_size=self._config.max_io_queue_size, max_num_threads=1, executor_cls=executor_cls) # The component responsible for limiting bandwidth usage if it # is configured. self._bandwidth_limiter = None if self._config.max_bandwidth is not None: logger.debug('Setting max_bandwidth to %s', self._config.max_bandwidth) leaky_bucket = LeakyBucket(self._config.max_bandwidth) self._bandwidth_limiter = BandwidthLimiter(leaky_bucket) self._register_handlers()
def test_raises_error_when_count_is_zero(self): sem = SlidingWindowSemaphore(3) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) # Count is now 0 so trying to acquire should fail. with self.assertRaises(NoResourcesAvailable): sem.acquire('a', blocking=False)
def test_can_acquire_release_multiple_times(self): sem = SlidingWindowSemaphore(1) num = sem.acquire('a', blocking=False) self.assertEqual(num, 0) sem.release('a', num) num = sem.acquire('a', blocking=False) self.assertEqual(num, 1) sem.release('a', num)
def setUp(self): super(BaseSubmissionTaskTest, self).setUp() self.config = TransferConfig() self.osutil = OSUtils() self.executor = BoundedExecutor( 1000, 1, { IN_MEMORY_UPLOAD_TAG: TaskSemaphore(10), IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(10) })
def test_can_track_multiple_tags(self): sem = SlidingWindowSemaphore(3) self.assertEqual(sem.acquire('a', blocking=False), 0) self.assertEqual(sem.acquire('b', blocking=False), 0) self.assertEqual(sem.acquire('a', blocking=False), 1) # We're at our max of 3 even though 2 are for A and 1 is for B. with self.assertRaises(NoResourcesAvailable): sem.acquire('a', blocking=False) with self.assertRaises(NoResourcesAvailable): sem.acquire('b', blocking=False)
def test_acquire_blocks_until_release_is_called(self): sem = SlidingWindowSemaphore(2) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) def acquire(): # This next call to acquire will block. self.assertEqual(sem.acquire('a', blocking=True), 2) t = threading.Thread(target=acquire) self.threads.append(t) # Starting the thread will block the sem.acquire() # in the acquire function above. t.start() # This still will keep the thread blocked. sem.release('a', 1) # Releasing the min element will unblock the thread. sem.release('a', 0) t.join() sem.release('a', 2)
def test_blocking_stress(self): sem = SlidingWindowSemaphore(5) num_threads = 10 num_iterations = 50 def acquire(): for _ in range(num_iterations): num = sem.acquire('a', blocking=True) time.sleep(0.001) sem.release('a', num) for i in range(num_threads): t = threading.Thread(target=acquire) self.threads.append(t) self.start_threads() self.join_threads() # Should have all the available resources freed. self.assertEqual(sem.current_count(), 5) # Should have acquired num_threads * num_iterations self.assertEqual(sem.acquire('a', blocking=False), num_threads * num_iterations)
def __init__(self, client, config=None, osutil=None): """A transfer manager interface for Amazon S3 :param client: Client to be used by the manager :param config: TransferConfig to associate specific configurations :param osutil: OSUtils object to use for os-related behavior when using with transfer manager. """ self._client = client self._config = config if config is None: self._config = TransferConfig() self._osutil = osutil if osutil is None: self._osutil = OSUtils() self._coordinator_controller = TransferCoordinatorController() # A counter to create unique id's for each transfer submitted. self._id_counter = 0 # The executor responsible for making S3 API transfer requests self._request_executor = BoundedExecutor( max_size=self._config.max_request_queue_size, max_num_threads=self._config.max_request_concurrency, tag_semaphores={ IN_MEMORY_UPLOAD_TAG: TaskSemaphore( self._config.max_in_memory_upload_chunks), IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore( self._config.max_in_memory_download_chunks) } ) # The executor responsible for submitting the necessary tasks to # perform the desired transfer self._submission_executor = BoundedExecutor( max_size=self._config.max_submission_queue_size, max_num_threads=self._config.max_submission_concurrency ) # There is one thread available for writing to disk. It will handle # downloads for all files. self._io_executor = BoundedExecutor( max_size=self._config.max_io_queue_size, max_num_threads=1 ) self._register_handlers()
def test_error_to_release_unknown_tag(self): sem = SlidingWindowSemaphore(3) with self.assertRaises(ValueError): sem.release('a', 0)
def test_can_handle_multiple_tags_released(self): sem = SlidingWindowSemaphore(4) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) sem.acquire('b', blocking=False) sem.acquire('b', blocking=False) sem.release('b', 1) sem.release('a', 1) self.assertEqual(sem.current_count(), 0) sem.release('b', 0) self.assertEqual(sem.acquire('a', blocking=False), 2) sem.release('a', 0) self.assertEqual(sem.acquire('b', blocking=False), 2)
def test_is_error_to_release_unknown_sequence_number(self): sem = SlidingWindowSemaphore(3) sem.acquire('a', blocking=False) with self.assertRaises(ValueError): sem.release('a', 1)
def test_release_counters_can_increment_counter_repeatedly(self): sem = SlidingWindowSemaphore(3) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) # These two releases don't increment the counter # because we're waiting on 0. sem.release('a', 1) sem.release('a', 2) self.assertEqual(sem.current_count(), 0) # But as soon as we release 0, we free up 0, 1, and 2. sem.release('a', 0) self.assertEqual(sem.current_count(), 3) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False)
def test_counter_release_only_on_min_element(self): sem = SlidingWindowSemaphore(3) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) sem.acquire('a', blocking=False) # The count only increases when we free the min # element. This means if we're currently failing to # acquire now: with self.assertRaises(NoResourcesAvailable): sem.acquire('a', blocking=False) # Then freeing a non-min element: sem.release('a', 1) # doesn't change anything. We still fail to acquire. with self.assertRaises(NoResourcesAvailable): sem.acquire('a', blocking=False) self.assertEqual(sem.current_count(), 0)