def test_scenario_throws_exception_when_rate_drops(self): """ ReadRequestLoadScenario raises RequestRateTooLow if rate drops below the requested rate. Establish the requested rate by having the FakeFlockerClient respond to all requests, then lower the rate by dropping alternate requests. This should result in RequestRateTooLow being raised. """ c = Clock() cluster = self.make_cluster(RequestDroppingFakeFlockerClient) sample_size = 5 s = ReadRequestLoadScenario(c, cluster, sample_size=sample_size) s.start() # Advance the clock by `sample_size` seconds to establish the # requested rate. c.pump(repeat(1, sample_size)) cluster.get_control_service(c).drop_requests = True # Advance the clock by 2 seconds so that a request is dropped # and a new rate which is below the target can be established. c.advance(2) failure = self.failureResultOf(s.maintained()) self.assertIsInstance(failure.value, RequestRateTooLow)
def test_read_request_load_succeeds(self): """ ReadRequestLoadScenario starts and stops without collapsing. """ c = Clock() node1 = Node(uuid=uuid4(), public_address=IPAddress('10.0.0.1')) node2 = Node(uuid=uuid4(), public_address=IPAddress('10.0.0.2')) cluster = BenchmarkCluster( node1.public_address, lambda reactor: FakeFlockerClient([node1, node2]), {node1.public_address, node2.public_address}, default_volume_size=DEFAULT_VOLUME_SIZE) sample_size = 5 s = ReadRequestLoadScenario(c, cluster, sample_size=sample_size) d = s.start() # Request rate samples are recorded every second and we need to # collect enough samples to establish the rate which is defined # by `sample_size`. Therefore, advance the clock by # `sample_size` seconds to obtain enough samples. c.pump(repeat(1, sample_size)) s.maintained().addBoth(lambda x: self.fail()) d.addCallback(lambda ignored: s.stop()) self.successResultOf(d)
def test_scenario_throws_exception_if_requested_rate_not_reached(self): """ ReadRequestLoadScenario raises RequestRateNotReached if the target rate cannot be established within a given timeframe. """ c = Clock() cluster = self.make_cluster(RequestDroppingFakeFlockerClient) s = ReadRequestLoadScenario(c, cluster) cluster.get_control_service(c).drop_requests = True d = s.start() # Continue the clock for one second longer than the timeout # value to allow the timeout to be triggered. c.advance(s.timeout + 1) failure = self.failureResultOf(d) self.assertIsInstance(failure.value, RequestRateNotReached)
def test_scenario_throws_exception_if_overloaded(self): """ `ReadRequestLoadScenario` raises `RequestOverload` if the difference between sent requests and received requests exceeds the tolerated difference once we start monitoring the scenario. Note that, right now, the only way to make it fail is to generate this difference before we start monitoring the scenario. Once we implement some kind of tolerance, to allow fluctuations in the rate, we can update this tests to trigger the exception in a more realistic manner. """ # XXX Update this test when we add tolerance for rate fluctuations. c = Clock() cluster = self.make_cluster(RequestDroppingFakeFlockerClient) target_rate = 10 sample_size = 20 s = ReadRequestLoadScenario(c, cluster, request_rate=target_rate, sample_size=sample_size) dropped_rate = target_rate / 2 seconds_to_overload = s.max_outstanding / dropped_rate s.start() # Reach initial rate cluster.get_control_service(c).drop_requests = True # Initially, we generate enough dropped requests so that the scenario # is overloaded when we start monitoring. c.pump(repeat(1, seconds_to_overload + 1)) # We stop dropping requests cluster.get_control_service(c).drop_requests = False # Now we generate the initial rate to start monitoring the scenario c.pump(repeat(1, sample_size)) # We only need to advance one more second (first loop in the monitoring # loop) to trigger RequestOverload c.advance(1) failure = self.failureResultOf(s.maintained()) self.assertIsInstance(failure.value, RequestOverload)
def test_scenario_timeouts_if_requests_not_completed(self): """ `ReadRequestLoadScenario` should timeout if the outstanding requests for the scenarion do not complete within the specified time. """ c = Clock() cluster = self.make_cluster(RequestErrorFakeFlockerClient) sample_size = 5 s = ReadRequestLoadScenario(c, cluster, request_rate=10, sample_size=sample_size) # Set the delay for the requests to be longer than the scenario # timeout cluster.get_control_service(c).delay = s.timeout + 10 d = s.start() s.maintained().addBoth(lambda x: self.fail()) # Advance the clock by `sample_size` seconds to establish the # requested rate. c.pump(repeat(1, sample_size)) cluster.get_control_service(c).fail_requests = True c.advance(1) cluster.get_control_service(c).fail_requests = False d.addCallback(lambda ignored: s.stop()) # Advance the clock by the timeout value so it is triggered # before the requests complete. self.assertNoResult(d) c.advance(s.timeout + 1) self.assertTrue(s.rate_measurer.outstanding() > 0) self.successResultOf(d)
def test_scenario_stops_only_when_no_outstanding_requests(self): """ `ReadRequestLoadScenario` should only be considered as stopped when all outstanding requests made by it have completed. """ c = Clock() cluster = self.make_cluster(RequestErrorFakeFlockerClient) delay = 1 cluster.get_control_service(c).delay = delay sample_size = 5 s = ReadRequestLoadScenario(c, cluster, request_rate=10, sample_size=sample_size) d = s.start() s.maintained().addBoth(lambda x: self.fail()) # Advance the clock by `sample_size` seconds to establish the # requested rate. c.pump(repeat(1, sample_size)) # Force the control service to fail requests for one second. # These requests will fail after the delay period set in the # control service. cluster.get_control_service(c).fail_requests = True c.advance(1) cluster.get_control_service(c).fail_requests = False d.addCallback(lambda ignored: s.stop()) # The scenario should not successfully stop until after the # delay period for the failed requests. self.assertNoResult(d) c.advance(delay) self.successResultOf(d)