Beispiel #1
0
    def testQuiescenceUnblocksMemoryConsumptionSemaphore(self):
        """Test that after a period of time you unblock (due to quiescence)."""
        _semaphore = utils.MemoryConsumptionSemaphore(
            system_available_buffer_bytes=1,
            single_proc_max_bytes=1,
            quiescence_time_seconds=2.0,
            unchecked_acquires=0)

        # Make large amount of memory available, but we expect quiescence
        # to block the second task.
        _semaphore._get_system_available = self.mock_get_system_available(
            2**64)
        start_time = time.time()
        self.assertEqual(_semaphore.acquire(ACQUIRE_TIMEOUT).result, True)
        _semaphore.release()

        # Get the lock or die trying. We spin fast here instead of ACQUIRE_TIMEOUT.
        while not _semaphore.acquire(1).result:
            continue
        _semaphore.release()

        # Check that the lock was acquired after quiescence_time_seconds.
        end_time = time.time()
        # Why 1.8? Because the clock isn't monotonic and we don't want to flake.
        self.assertGreaterEqual(end_time - start_time, 1.8)
Beispiel #2
0
    def testNoMemoryConsumptionSemaphore(self):
        """Tests that you can acquire a very little amount of memory."""
        # You should always get one byte.
        _semaphore = utils.MemoryConsumptionSemaphore(
            system_available_buffer_bytes=1,
            single_proc_max_bytes=1,
            quiescence_time_seconds=0.0)

        # Sure you can have two bytes.
        self.assertEqual(_semaphore.acquire(ACQUIRE_TIMEOUT).result, True)
        _semaphore.release()
Beispiel #3
0
    def testMassiveMemoryConsumptionSemaphore(self):
        """Tests that we block on not having enough memory."""
        # You should never get 2**64 bytes.
        _semaphore = utils.MemoryConsumptionSemaphore(
            system_available_buffer_bytes=2**64,
            single_proc_max_bytes=2**64,
            quiescence_time_seconds=0.0)

        # You can't get that much.
        self.assertEqual(
            _semaphore.acquire(ACQUIRE_SHOULD_BLOCK_TIMEOUT).result, False)
Beispiel #4
0
    def testUncheckedMemoryConsumptionSemaphore(self):
        """Tests that some acquires work unchecked."""
        # You should never get 2**64 bytes (i wish...).
        _semaphore = utils.MemoryConsumptionSemaphore(
            system_available_buffer_bytes=2**64,
            single_proc_max_bytes=2**64,
            quiescence_time_seconds=2.0,
            unchecked_acquires=2)

        # Nothing available, but we expect unchecked_acquires to allow it.
        _semaphore._get_system_available = self.mock_get_system_available(0)
        self.assertEqual(_semaphore.acquire(ACQUIRE_TIMEOUT).result, True)
        _semaphore.release()
        self.assertEqual(_semaphore.acquire(ACQUIRE_TIMEOUT).result, True)
        _semaphore.release()
Beispiel #5
0
 def testTotalMaxMemoryConsumptionSemaphore(self):
     """Tests that the total_max is respected."""
     _semaphore = utils.MemoryConsumptionSemaphore(
         system_available_buffer_bytes=0,
         single_proc_max_bytes=1,
         quiescence_time_seconds=0.0,
         total_max=3)
     # Look at all this memory.
     _semaphore._get_system_available = self.mock_get_system_available(
         2**64)
     # Sure you can have three.
     self.assertEqual(_semaphore.acquire(ACQUIRE_TIMEOUT).result, True)
     self.assertEqual(_semaphore.acquire(ACQUIRE_TIMEOUT).result, True)
     self.assertEqual(_semaphore.acquire(ACQUIRE_TIMEOUT).result, True)
     # Nope, you're now over max.
     self.assertEqual(_semaphore.acquire(1).result, False)
Beispiel #6
0
    def testQuiesceMemoryConsumptionSemaphore(self):
        """Tests that you wait for memory utilization to settle (quiesce)."""
        # All you want is two bytes.
        _semaphore = utils.MemoryConsumptionSemaphore(
            system_available_buffer_bytes=1,
            single_proc_max_bytes=1,
            quiescence_time_seconds=2.0)

        # Should want two bytes, have a whole lot.
        _semaphore._get_system_available = self.mock_get_system_available(
            2**64)
        self.assertEqual(_semaphore.acquire(ACQUIRE_TIMEOUT).result, True)
        _semaphore.release()

        # Should want two bytes, have a whole lot (but you'll block for 2 seconds).
        _semaphore._get_system_available = self.mock_get_system_available(
            2**64 - 2)
        self.assertEqual(_semaphore.acquire(ACQUIRE_TIMEOUT).result, True)
        _semaphore.release()
from chromite.lib.paygen import signer_payloads_client
from chromite.lib.paygen import urilib
from chromite.lib.paygen import utils

from chromite.scripts import build_dlc
from chromite.scripts import cros_set_lsb_release

DESCRIPTION_FILE_VERSION = 2

# See class docs for functionality information. Configured to reserve 6GB and
# consider each individual task as consuming 6GB. Therefore we won't start a
# new task unless available memory is about 12GB (including base utilization).
# We set a total max concurrency as a precaution. TODO(crbug.com/1016555)
_mem_semaphore = utils.MemoryConsumptionSemaphore(
    system_available_buffer_bytes=2**31 + 2**32,  # 6 GB
    single_proc_max_bytes=2**31 + 2**32,  # 6 GB
    quiescence_time_seconds=60.0,
    total_max=10,
    unchecked_acquires=4)


class Error(Exception):
    """Base class for payload generation errors."""


class UnexpectedSignerResultsError(Error):
    """This is raised when signer results don't match our expectations."""


class PayloadVerificationError(Error):
    """Raised when the generated payload fails to verify."""
Beispiel #8
0
    def testMultiProcessedMemoryConsumptionSemaphore(self):
        """Test many processes simultaneously using the Semaphore."""
        initial_memory = 6

        mem_avail = multiprocessing.Value('I', initial_memory, lock=True)
        good_process_exits = multiprocessing.Value('I', 0, lock=True)
        n_processes = 8

        # Currently executes in 45 seconds a 2 x Xeon Gold 6154 CPUs
        get_and_releases = 50

        def sub_mem():
            with mem_avail.get_lock():
                mem_avail.value -= 1
                self.assertGreaterEqual(mem_avail.value, 0)

        def add_mem():
            with mem_avail.get_lock():
                mem_avail.value += 1
                self.assertLessEqual(mem_avail.value, 6)

        def get_mem():
            with mem_avail.get_lock():
                return mem_avail.value

        # Ask for two bytes available each time.
        _semaphore = utils.MemoryConsumptionSemaphore(
            system_available_buffer_bytes=1,
            single_proc_max_bytes=1,
            quiescence_time_seconds=0.1,
            unchecked_acquires=1)

        _semaphore._get_system_available = get_mem

        def hammer_semaphore():
            for _ in range(get_and_releases):
                while not _semaphore.acquire(0.1).result:
                    continue
                # Simulate 'using the memory'.
                sub_mem()
                time.sleep(0.1)
                add_mem()
                _semaphore.release()
            with good_process_exits.get_lock():
                good_process_exits.value = good_process_exits.value + 1

        processes = [
            multiprocessing.Process(target=hammer_semaphore)
            for _ in range(n_processes)
        ]

        for p in processes:
            p.daemon = True
            p.start()

        for p in processes:
            p.join()

        # If we didn't get here a proc did not exit. This is fatal and may
        # indicate a deadlock has been introduced.
        self.assertEqual(initial_memory, get_mem())
        with good_process_exits.get_lock():
            self.assertEqual(good_process_exits.value, n_processes)
Beispiel #9
0
    def testThreadedMemoryConsumptionSemaphore(self):
        """Test many threads simultaneously using the Semaphore."""
        initial_memory = 6
        # These are lists so we can write nonlocal.
        mem_avail = [initial_memory]
        good_thread_exits = [0]
        mock_clock = TestUtils.MockClock()
        lock, exit_lock = threading.Lock(), threading.Lock()
        test_threads = 8

        # Currently executes in 1.6 seconds a 2 x Xeon Gold 6154 CPUs
        get_and_releases = 50

        def sub_mem():
            with lock:
                mem_avail[0] = mem_avail[0] - 1
                self.assertGreaterEqual(mem_avail[0], 0)

        def add_mem():
            with lock:
                mem_avail[0] = mem_avail[0] + 1
                self.assertGreaterEqual(mem_avail[0], 0)

        def get_mem():
            with lock:
                return mem_avail[0]

        # Ask for two bytes available each time.
        _semaphore = utils.MemoryConsumptionSemaphore(
            system_available_buffer_bytes=1,
            single_proc_max_bytes=1,
            quiescence_time_seconds=0.1,
            unchecked_acquires=1,
            clock=mock_clock)
        _semaphore._get_system_available = get_mem

        def hammer_semaphore():
            for _ in range(get_and_releases):
                while not _semaphore.acquire(0.1).result:
                    continue
                # Simulate 'using the memory'.
                sub_mem()
                time.sleep(0.1)
                add_mem()
                _semaphore.release()
            with exit_lock:
                good_thread_exits[0] = good_thread_exits[0] + 1

        threads = [
            threading.Thread(target=hammer_semaphore)
            for _ in range(test_threads)
        ]
        for x in threads:
            x.daemon = True
            x.start()

        # ~Maximum 600 seconds realtime, keeps clock ticking for overall timeout.
        for _ in range(60000):
            time.sleep(0.01)
            mock_clock.add_time(0.1)

            # Maybe we can break early? (and waste some time for other threads).
            threads_dead = [not x.isAlive() for x in threads]
            if all(threads_dead):
                break

        # If we didn't get here a thread did not exit. This is fatal and may
        # indicate a deadlock has been introduced.
        self.assertEqual(initial_memory, get_mem())
        self.assertEqual(good_thread_exits[0], test_threads)