Exemplo n.º 1
0
    def test_acquire_release_basic_case(self):
        sem = SlidingWindowSemaphore(1)
        # Count is 1

        num = sem.acquire('a', blocking=False)
        self.assertEqual(num, 0)
        sem.release('a', 0)
Exemplo n.º 2
0
    def test_acquire_release_basic_case(self):
        sem = SlidingWindowSemaphore(1)
        # Count is 1

        num = sem.acquire('a', blocking=False)
        self.assertEqual(num, 0)
        sem.release('a', 0)
Exemplo n.º 3
0
    def test_can_check_in_partial_range(self):
        sem = SlidingWindowSemaphore(4)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        sem.release('a', 1)
        sem.release('a', 3)
        sem.release('a', 0)
        self.assertEqual(sem.current_count(), 2)
Exemplo n.º 4
0
 def test_can_acquire_a_range(self):
     sem = SlidingWindowSemaphore(3)
     self.assertEqual(sem.acquire('a', blocking=False), 0)
     self.assertEqual(sem.acquire('a', blocking=False), 1)
     self.assertEqual(sem.acquire('a', blocking=False), 2)
     sem.release('a', 0)
     sem.release('a', 1)
     sem.release('a', 2)
     # Now we're reset so we should be able to acquire the same
     # sequence again.
     self.assertEqual(sem.acquire('a', blocking=False), 3)
     self.assertEqual(sem.acquire('a', blocking=False), 4)
     self.assertEqual(sem.acquire('a', blocking=False), 5)
     self.assertEqual(sem.current_count(), 0)
Exemplo n.º 5
0
    def test_stress_invariants_random_order(self):
        sem = SlidingWindowSemaphore(100)
        for _ in range(10):
            recorded = []
            for _ in range(100):
                recorded.append(sem.acquire('a', blocking=False))
            # Release them in randomized order.  As long as we
            # eventually free all 100, we should have all the
            # resources released.
            random.shuffle(recorded)
            for i in recorded:
                sem.release('a', i)

        # Everything's freed so should be back at count == 100
        self.assertEqual(sem.current_count(), 100)
Exemplo n.º 6
0
 def test_is_error_to_double_release(self):
     # This is different than other error tests because
     # we're verifying we can reset the state after an
     # acquire/release cycle.
     sem = SlidingWindowSemaphore(2)
     sem.acquire('a', blocking=False)
     sem.acquire('a', blocking=False)
     sem.release('a', 0)
     sem.release('a', 1)
     self.assertEqual(sem.current_count(), 2)
     with self.assertRaises(ValueError):
         sem.release('a', 0)
Exemplo n.º 7
0
    def __init__(self, client, config=None, osutil=None, executor_cls=None):
        """A transfer manager interface for Amazon S3

        :param client: Client to be used by the manager
        :param config: TransferConfig to associate specific configurations
        :param osutil: OSUtils object to use for os-related behavior when
            using with transfer manager.

        :type executor_cls: s3transfer.futures.BaseExecutor
        :param executor_cls: The class of executor to use with the transfer
            manager. By default, concurrent.futures.ThreadPoolExecutor is used.
        """
        self._client = client
        self._config = config
        if config is None:
            self._config = TransferConfig()
        self._osutil = osutil
        if osutil is None:
            self._osutil = OSUtils()
        self._coordinator_controller = TransferCoordinatorController()
        # A counter to create unique id's for each transfer submitted.
        self._id_counter = 0

        # The executor responsible for making S3 API transfer requests
        self._request_executor = BoundedExecutor(
            max_size=self._config.max_request_queue_size,
            max_num_threads=self._config.max_request_concurrency,
            tag_semaphores={
                IN_MEMORY_UPLOAD_TAG:
                TaskSemaphore(self._config.max_in_memory_upload_chunks),
                IN_MEMORY_DOWNLOAD_TAG:
                SlidingWindowSemaphore(
                    self._config.max_in_memory_download_chunks)
            },
            executor_cls=executor_cls)

        # The executor responsible for submitting the necessary tasks to
        # perform the desired transfer
        self._submission_executor = BoundedExecutor(
            max_size=self._config.max_submission_queue_size,
            max_num_threads=self._config.max_submission_concurrency,
            executor_cls=executor_cls)

        # There is one thread available for writing to disk. It will handle
        # downloads for all files.
        self._io_executor = BoundedExecutor(
            max_size=self._config.max_io_queue_size,
            max_num_threads=1,
            executor_cls=executor_cls)

        # The component responsible for limiting bandwidth usage if it
        # is configured.
        self._bandwidth_limiter = None
        if self._config.max_bandwidth is not None:
            logger.debug('Setting max_bandwidth to %s',
                         self._config.max_bandwidth)
            leaky_bucket = LeakyBucket(self._config.max_bandwidth)
            self._bandwidth_limiter = BandwidthLimiter(leaky_bucket)

        self._register_handlers()
Exemplo n.º 8
0
    def test_raises_error_when_count_is_zero(self):
        sem = SlidingWindowSemaphore(3)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        # Count is now 0 so trying to acquire should fail.
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('a', blocking=False)
Exemplo n.º 9
0
    def test_can_acquire_release_multiple_times(self):
        sem = SlidingWindowSemaphore(1)
        num = sem.acquire('a', blocking=False)
        self.assertEqual(num, 0)
        sem.release('a', num)

        num = sem.acquire('a', blocking=False)
        self.assertEqual(num, 1)
        sem.release('a', num)
Exemplo n.º 10
0
 def setUp(self):
     super(BaseSubmissionTaskTest, self).setUp()
     self.config = TransferConfig()
     self.osutil = OSUtils()
     self.executor = BoundedExecutor(
         1000, 1, {
             IN_MEMORY_UPLOAD_TAG: TaskSemaphore(10),
             IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(10)
         })
Exemplo n.º 11
0
    def test_can_track_multiple_tags(self):
        sem = SlidingWindowSemaphore(3)
        self.assertEqual(sem.acquire('a', blocking=False), 0)
        self.assertEqual(sem.acquire('b', blocking=False), 0)
        self.assertEqual(sem.acquire('a', blocking=False), 1)

        # We're at our max of 3 even though 2 are for A and 1 is for B.
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('a', blocking=False)
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('b', blocking=False)
Exemplo n.º 12
0
 def test_is_error_to_double_release(self):
     # This is different than other error tests because
     # we're verifying we can reset the state after an
     # acquire/release cycle.
     sem = SlidingWindowSemaphore(2)
     sem.acquire('a', blocking=False)
     sem.acquire('a', blocking=False)
     sem.release('a', 0)
     sem.release('a', 1)
     self.assertEqual(sem.current_count(), 2)
     with self.assertRaises(ValueError):
         sem.release('a', 0)
Exemplo n.º 13
0
    def test_can_acquire_release_multiple_times(self):
        sem = SlidingWindowSemaphore(1)
        num = sem.acquire('a', blocking=False)
        self.assertEqual(num, 0)
        sem.release('a', num)

        num = sem.acquire('a', blocking=False)
        self.assertEqual(num, 1)
        sem.release('a', num)
Exemplo n.º 14
0
    def test_raises_error_when_count_is_zero(self):
        sem = SlidingWindowSemaphore(3)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        # Count is now 0 so trying to acquire should fail.
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('a', blocking=False)
Exemplo n.º 15
0
    def test_acquire_blocks_until_release_is_called(self):
        sem = SlidingWindowSemaphore(2)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        def acquire():
            # This next call to acquire will block.
            self.assertEqual(sem.acquire('a', blocking=True), 2)

        t = threading.Thread(target=acquire)
        self.threads.append(t)
        # Starting the thread will block the sem.acquire()
        # in the acquire function above.
        t.start()
        # This still will keep the thread blocked.
        sem.release('a', 1)
        # Releasing the min element will unblock the thread.
        sem.release('a', 0)
        t.join()
        sem.release('a', 2)
Exemplo n.º 16
0
    def test_blocking_stress(self):
        sem = SlidingWindowSemaphore(5)
        num_threads = 10
        num_iterations = 50

        def acquire():
            for _ in range(num_iterations):
                num = sem.acquire('a', blocking=True)
                time.sleep(0.001)
                sem.release('a', num)

        for i in range(num_threads):
            t = threading.Thread(target=acquire)
            self.threads.append(t)
        self.start_threads()
        self.join_threads()
        # Should have all the available resources freed.
        self.assertEqual(sem.current_count(), 5)
        # Should have acquired num_threads * num_iterations
        self.assertEqual(sem.acquire('a', blocking=False),
                         num_threads * num_iterations)
Exemplo n.º 17
0
    def test_blocking_stress(self):
        sem = SlidingWindowSemaphore(5)
        num_threads = 10
        num_iterations = 50

        def acquire():
            for _ in range(num_iterations):
                num = sem.acquire('a', blocking=True)
                time.sleep(0.001)
                sem.release('a', num)

        for i in range(num_threads):
            t = threading.Thread(target=acquire)
            self.threads.append(t)
        self.start_threads()
        self.join_threads()
        # Should have all the available resources freed.
        self.assertEqual(sem.current_count(), 5)
        # Should have acquired num_threads * num_iterations
        self.assertEqual(sem.acquire('a', blocking=False),
                         num_threads * num_iterations)
Exemplo n.º 18
0
    def test_can_track_multiple_tags(self):
        sem = SlidingWindowSemaphore(3)
        self.assertEqual(sem.acquire('a', blocking=False), 0)
        self.assertEqual(sem.acquire('b', blocking=False), 0)
        self.assertEqual(sem.acquire('a', blocking=False), 1)

        # We're at our max of 3 even though 2 are for A and 1 is for B.
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('a', blocking=False)
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('b', blocking=False)
Exemplo n.º 19
0
    def test_acquire_blocks_until_release_is_called(self):
        sem = SlidingWindowSemaphore(2)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        def acquire():
            # This next call to acquire will block.
            self.assertEqual(sem.acquire('a', blocking=True), 2)

        t = threading.Thread(target=acquire)
        self.threads.append(t)
        # Starting the thread will block the sem.acquire()
        # in the acquire function above.
        t.start()
        # This still will keep the thread blocked.
        sem.release('a', 1)
        # Releasing the min element will unblock the thread.
        sem.release('a', 0)
        t.join()
        sem.release('a', 2)
Exemplo n.º 20
0
    def __init__(self, client, config=None, osutil=None):
        """A transfer manager interface for Amazon S3

        :param client: Client to be used by the manager
        :param config: TransferConfig to associate specific configurations
        :param osutil: OSUtils object to use for os-related behavior when
            using with transfer manager.
        """
        self._client = client
        self._config = config
        if config is None:
            self._config = TransferConfig()
        self._osutil = osutil
        if osutil is None:
            self._osutil = OSUtils()
        self._coordinator_controller = TransferCoordinatorController()
        # A counter to create unique id's for each transfer submitted.
        self._id_counter = 0

        # The executor responsible for making S3 API transfer requests
        self._request_executor = BoundedExecutor(
            max_size=self._config.max_request_queue_size,
            max_num_threads=self._config.max_request_concurrency,
            tag_semaphores={
                IN_MEMORY_UPLOAD_TAG: TaskSemaphore(
                    self._config.max_in_memory_upload_chunks),
                IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(
                    self._config.max_in_memory_download_chunks)
            }
        )

        # The executor responsible for submitting the necessary tasks to
        # perform the desired transfer
        self._submission_executor = BoundedExecutor(
            max_size=self._config.max_submission_queue_size,
            max_num_threads=self._config.max_submission_concurrency
        )

        # There is one thread available for writing to disk. It will handle
        # downloads for all files.
        self._io_executor = BoundedExecutor(
            max_size=self._config.max_io_queue_size,
            max_num_threads=1
        )
        self._register_handlers()
Exemplo n.º 21
0
    def test_stress_invariants_random_order(self):
        sem = SlidingWindowSemaphore(100)
        for _ in range(10):
            recorded = []
            for _ in range(100):
                recorded.append(sem.acquire('a', blocking=False))
            # Release them in randomized order.  As long as we
            # eventually free all 100, we should have all the
            # resources released.
            random.shuffle(recorded)
            for i in recorded:
                sem.release('a', i)

        # Everything's freed so should be back at count == 100
        self.assertEqual(sem.current_count(), 100)
Exemplo n.º 22
0
 def test_error_to_release_unknown_tag(self):
     sem = SlidingWindowSemaphore(3)
     with self.assertRaises(ValueError):
         sem.release('a', 0)
Exemplo n.º 23
0
    def test_can_handle_multiple_tags_released(self):
        sem = SlidingWindowSemaphore(4)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('b', blocking=False)
        sem.acquire('b', blocking=False)

        sem.release('b', 1)
        sem.release('a', 1)
        self.assertEqual(sem.current_count(), 0)

        sem.release('b', 0)
        self.assertEqual(sem.acquire('a', blocking=False), 2)

        sem.release('a', 0)
        self.assertEqual(sem.acquire('b', blocking=False), 2)
Exemplo n.º 24
0
 def test_is_error_to_release_unknown_sequence_number(self):
     sem = SlidingWindowSemaphore(3)
     sem.acquire('a', blocking=False)
     with self.assertRaises(ValueError):
         sem.release('a', 1)
Exemplo n.º 25
0
    def test_can_check_in_partial_range(self):
        sem = SlidingWindowSemaphore(4)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        sem.release('a', 1)
        sem.release('a', 3)
        sem.release('a', 0)
        self.assertEqual(sem.current_count(), 2)
Exemplo n.º 26
0
    def test_release_counters_can_increment_counter_repeatedly(self):
        sem = SlidingWindowSemaphore(3)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        # These two releases don't increment the counter
        # because we're waiting on 0.
        sem.release('a', 1)
        sem.release('a', 2)
        self.assertEqual(sem.current_count(), 0)
        # But as soon as we release 0, we free up 0, 1, and 2.
        sem.release('a', 0)
        self.assertEqual(sem.current_count(), 3)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
Exemplo n.º 27
0
 def test_is_error_to_release_unknown_sequence_number(self):
     sem = SlidingWindowSemaphore(3)
     sem.acquire('a', blocking=False)
     with self.assertRaises(ValueError):
         sem.release('a', 1)
Exemplo n.º 28
0
    def test_counter_release_only_on_min_element(self):
        sem = SlidingWindowSemaphore(3)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        # The count only increases when we free the min
        # element.  This means if we're currently failing to
        # acquire now:
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('a', blocking=False)

        # Then freeing a non-min element:
        sem.release('a', 1)

        # doesn't change anything.  We still fail to acquire.
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('a', blocking=False)
        self.assertEqual(sem.current_count(), 0)
Exemplo n.º 29
0
    def test_can_handle_multiple_tags_released(self):
        sem = SlidingWindowSemaphore(4)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('b', blocking=False)
        sem.acquire('b', blocking=False)

        sem.release('b', 1)
        sem.release('a', 1)
        self.assertEqual(sem.current_count(), 0)

        sem.release('b', 0)
        self.assertEqual(sem.acquire('a', blocking=False), 2)

        sem.release('a', 0)
        self.assertEqual(sem.acquire('b', blocking=False), 2)
Exemplo n.º 30
0
 def test_can_acquire_a_range(self):
     sem = SlidingWindowSemaphore(3)
     self.assertEqual(sem.acquire('a', blocking=False), 0)
     self.assertEqual(sem.acquire('a', blocking=False), 1)
     self.assertEqual(sem.acquire('a', blocking=False), 2)
     sem.release('a', 0)
     sem.release('a', 1)
     sem.release('a', 2)
     # Now we're reset so we should be able to acquire the same
     # sequence again.
     self.assertEqual(sem.acquire('a', blocking=False), 3)
     self.assertEqual(sem.acquire('a', blocking=False), 4)
     self.assertEqual(sem.acquire('a', blocking=False), 5)
     self.assertEqual(sem.current_count(), 0)
Exemplo n.º 31
0
 def test_error_to_release_unknown_tag(self):
     sem = SlidingWindowSemaphore(3)
     with self.assertRaises(ValueError):
         sem.release('a', 0)
Exemplo n.º 32
0
    def test_release_counters_can_increment_counter_repeatedly(self):
        sem = SlidingWindowSemaphore(3)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        # These two releases don't increment the counter
        # because we're waiting on 0.
        sem.release('a', 1)
        sem.release('a', 2)
        self.assertEqual(sem.current_count(), 0)
        # But as soon as we release 0, we free up 0, 1, and 2.
        sem.release('a', 0)
        self.assertEqual(sem.current_count(), 3)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
Exemplo n.º 33
0
    def test_counter_release_only_on_min_element(self):
        sem = SlidingWindowSemaphore(3)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)
        sem.acquire('a', blocking=False)

        # The count only increases when we free the min
        # element.  This means if we're currently failing to
        # acquire now:
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('a', blocking=False)

        # Then freeing a non-min element:
        sem.release('a', 1)

        # doesn't change anything.  We still fail to acquire.
        with self.assertRaises(NoResourcesAvailable):
            sem.acquire('a', blocking=False)
        self.assertEqual(sem.current_count(), 0)