def test_scenario_throws_exception_when_rate_drops(self):
"""
ReadRequestLoadScenario raises RequestRateTooLow if rate
drops below the requested rate.
Establish the requested rate by having the FakeFlockerClient
respond to all requests, then lower the rate by dropping
alternate requests. This should result in RequestRateTooLow
being raised.
"""
c = Clock()
cluster = self.make_cluster(RequestDroppingFakeFlockerClient)
sample_size = 5
s = ReadRequestLoadScenario(c, cluster, sample_size=sample_size)
s.start()
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
cluster.get_control_service(c).drop_requests = True
# Advance the clock by 2 seconds so that a request is dropped
# and a new rate which is below the target can be established.
c.advance(2)
failure = self.failureResultOf(s.maintained())
self.assertIsInstance(failure.value, RequestRateTooLow)
def test_scenario_throws_exception_when_rate_drops(self):
"""
ReadRequestLoadScenario raises RequestRateTooLow if rate
drops below the requested rate.
Establish the requested rate by having the FakeFlockerClient
respond to all requests, then lower the rate by dropping
alternate requests. This should result in RequestRateTooLow
being raised.
"""
c = Clock()
cluster = self.make_cluster(RequestDroppingFakeFlockerClient)
sample_size = 5
s = ReadRequestLoadScenario(c, cluster, sample_size=sample_size)
s.start()
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
cluster.get_control_service(c).drop_requests = True
# Advance the clock by 2 seconds so that a request is dropped
# and a new rate which is below the target can be established.
c.advance(2)
failure = self.failureResultOf(s.maintained())
self.assertIsInstance(failure.value, RequestRateTooLow)
def test_read_request_load_succeeds(self):
"""
ReadRequestLoadScenario starts and stops without collapsing.
"""
c = Clock()
node1 = Node(uuid=uuid4(), public_address=IPAddress('10.0.0.1'))
node2 = Node(uuid=uuid4(), public_address=IPAddress('10.0.0.2'))
cluster = BenchmarkCluster(
node1.public_address,
lambda reactor: FakeFlockerClient([node1, node2]),
{node1.public_address, node2.public_address},
default_volume_size=DEFAULT_VOLUME_SIZE
)
sample_size = 5
s = ReadRequestLoadScenario(c, cluster, sample_size=sample_size)
d = s.start()
# Request rate samples are recorded every second and we need to
# collect enough samples to establish the rate which is defined
# by `sample_size`. Therefore, advance the clock by
# `sample_size` seconds to obtain enough samples.
c.pump(repeat(1, sample_size))
s.maintained().addBoth(lambda x: self.fail())
d.addCallback(lambda ignored: s.stop())
self.successResultOf(d)
def test_scenario_throws_exception_if_requested_rate_not_reached(self):
"""
ReadRequestLoadScenario raises RequestRateNotReached if the
target rate cannot be established within a given timeframe.
"""
c = Clock()
cluster = self.make_cluster(RequestDroppingFakeFlockerClient)
s = ReadRequestLoadScenario(c, cluster)
cluster.get_control_service(c).drop_requests = True
d = s.start()
# Continue the clock for one second longer than the timeout
# value to allow the timeout to be triggered.
c.advance(s.timeout + 1)
failure = self.failureResultOf(d)
self.assertIsInstance(failure.value, RequestRateNotReached)
def test_scenario_throws_exception_if_requested_rate_not_reached(self):
"""
ReadRequestLoadScenario raises RequestRateNotReached if the
target rate cannot be established within a given timeframe.
"""
c = Clock()
cluster = self.make_cluster(RequestDroppingFakeFlockerClient)
s = ReadRequestLoadScenario(c, cluster)
cluster.get_control_service(c).drop_requests = True
d = s.start()
# Continue the clock for one second longer than the timeout
# value to allow the timeout to be triggered.
c.advance(s.timeout + 1)
failure = self.failureResultOf(d)
self.assertIsInstance(failure.value, RequestRateNotReached)
def test_scenario_throws_exception_if_overloaded(self):
"""
`ReadRequestLoadScenario` raises `RequestOverload` if the
difference between sent requests and received requests exceeds
the tolerated difference once we start monitoring the scenario.
Note that, right now, the only way to make it fail is to generate
this difference before we start monitoring the scenario.
Once we implement some kind of tolerance, to allow fluctuations
in the rate, we can update this tests to trigger the exception
in a more realistic manner.
"""
# XXX Update this test when we add tolerance for rate fluctuations.
c = Clock()
cluster = self.make_cluster(RequestDroppingFakeFlockerClient)
target_rate = 10
sample_size = 20
s = ReadRequestLoadScenario(c,
cluster,
request_rate=target_rate,
sample_size=sample_size)
dropped_rate = target_rate / 2
seconds_to_overload = s.max_outstanding / dropped_rate
s.start()
# Reach initial rate
cluster.get_control_service(c).drop_requests = True
# Initially, we generate enough dropped requests so that the scenario
# is overloaded when we start monitoring.
c.pump(repeat(1, seconds_to_overload + 1))
# We stop dropping requests
cluster.get_control_service(c).drop_requests = False
# Now we generate the initial rate to start monitoring the scenario
c.pump(repeat(1, sample_size))
# We only need to advance one more second (first loop in the monitoring
# loop) to trigger RequestOverload
c.advance(1)
failure = self.failureResultOf(s.maintained())
self.assertIsInstance(failure.value, RequestOverload)
def test_read_request_load_succeeds(self):
"""
ReadRequestLoadScenario starts and stops without collapsing.
"""
c = Clock()
node1 = Node(uuid=uuid4(), public_address=IPAddress('10.0.0.1'))
node2 = Node(uuid=uuid4(), public_address=IPAddress('10.0.0.2'))
cluster = BenchmarkCluster(
node1.public_address,
lambda reactor: FakeFlockerClient([node1, node2]),
{node1.public_address, node2.public_address},
default_volume_size=DEFAULT_VOLUME_SIZE)
sample_size = 5
s = ReadRequestLoadScenario(c, cluster, sample_size=sample_size)
d = s.start()
# Request rate samples are recorded every second and we need to
# collect enough samples to establish the rate which is defined
# by `sample_size`. Therefore, advance the clock by
# `sample_size` seconds to obtain enough samples.
c.pump(repeat(1, sample_size))
s.maintained().addBoth(lambda x: self.fail())
d.addCallback(lambda ignored: s.stop())
self.successResultOf(d)
def test_scenario_throws_exception_if_overloaded(self):
"""
`ReadRequestLoadScenario` raises `RequestOverload` if the
difference between sent requests and received requests exceeds
the tolerated difference once we start monitoring the scenario.
Note that, right now, the only way to make it fail is to generate
this difference before we start monitoring the scenario.
Once we implement some kind of tolerance, to allow fluctuations
in the rate, we can update this tests to trigger the exception
in a more realistic manner.
"""
# XXX Update this test when we add tolerance for rate fluctuations.
c = Clock()
cluster = self.make_cluster(RequestDroppingFakeFlockerClient)
target_rate = 10
sample_size = 20
s = ReadRequestLoadScenario(c, cluster, request_rate=target_rate,
sample_size=sample_size)
dropped_rate = target_rate / 2
seconds_to_overload = s.max_outstanding / dropped_rate
s.start()
# Reach initial rate
cluster.get_control_service(c).drop_requests = True
# Initially, we generate enough dropped requests so that the scenario
# is overloaded when we start monitoring.
c.pump(repeat(1, seconds_to_overload+1))
# We stop dropping requests
cluster.get_control_service(c).drop_requests = False
# Now we generate the initial rate to start monitoring the scenario
c.pump(repeat(1, sample_size))
# We only need to advance one more second (first loop in the monitoring
# loop) to trigger RequestOverload
c.advance(1)
failure = self.failureResultOf(s.maintained())
self.assertIsInstance(failure.value, RequestOverload)
def test_scenario_timeouts_if_requests_not_completed(self):
"""
`ReadRequestLoadScenario` should timeout if the outstanding
requests for the scenarion do not complete within the specified
time.
"""
c = Clock()
cluster = self.make_cluster(RequestErrorFakeFlockerClient)
sample_size = 5
s = ReadRequestLoadScenario(
c, cluster, request_rate=10, sample_size=sample_size
)
# Set the delay for the requests to be longer than the scenario
# timeout
cluster.get_control_service(c).delay = s.timeout + 10
d = s.start()
s.maintained().addBoth(lambda x: self.fail())
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
cluster.get_control_service(c).fail_requests = True
c.advance(1)
cluster.get_control_service(c).fail_requests = False
d.addCallback(lambda ignored: s.stop())
# Advance the clock by the timeout value so it is triggered
# before the requests complete.
self.assertNoResult(d)
c.advance(s.timeout + 1)
self.assertTrue(s.rate_measurer.outstanding() > 0)
self.successResultOf(d)
def test_scenario_stops_only_when_no_outstanding_requests(self):
"""
`ReadRequestLoadScenario` should only be considered as stopped
when all outstanding requests made by it have completed.
"""
c = Clock()
cluster = self.make_cluster(RequestErrorFakeFlockerClient)
delay = 1
cluster.get_control_service(c).delay = delay
sample_size = 5
s = ReadRequestLoadScenario(
c, cluster, request_rate=10, sample_size=sample_size
)
d = s.start()
s.maintained().addBoth(lambda x: self.fail())
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
# Force the control service to fail requests for one second.
# These requests will fail after the delay period set in the
# control service.
cluster.get_control_service(c).fail_requests = True
c.advance(1)
cluster.get_control_service(c).fail_requests = False
d.addCallback(lambda ignored: s.stop())
# The scenario should not successfully stop until after the
# delay period for the failed requests.
self.assertNoResult(d)
c.advance(delay)
self.successResultOf(d)
def test_scenario_timeouts_if_requests_not_completed(self):
"""
`ReadRequestLoadScenario` should timeout if the outstanding
requests for the scenarion do not complete within the specified
time.
"""
c = Clock()
cluster = self.make_cluster(RequestErrorFakeFlockerClient)
sample_size = 5
s = ReadRequestLoadScenario(c,
cluster,
request_rate=10,
sample_size=sample_size)
# Set the delay for the requests to be longer than the scenario
# timeout
cluster.get_control_service(c).delay = s.timeout + 10
d = s.start()
s.maintained().addBoth(lambda x: self.fail())
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
cluster.get_control_service(c).fail_requests = True
c.advance(1)
cluster.get_control_service(c).fail_requests = False
d.addCallback(lambda ignored: s.stop())
# Advance the clock by the timeout value so it is triggered
# before the requests complete.
self.assertNoResult(d)
c.advance(s.timeout + 1)
self.assertTrue(s.rate_measurer.outstanding() > 0)
self.successResultOf(d)
def test_scenario_stops_only_when_no_outstanding_requests(self):
"""
`ReadRequestLoadScenario` should only be considered as stopped
when all outstanding requests made by it have completed.
"""
c = Clock()
cluster = self.make_cluster(RequestErrorFakeFlockerClient)
delay = 1
cluster.get_control_service(c).delay = delay
sample_size = 5
s = ReadRequestLoadScenario(c,
cluster,
request_rate=10,
sample_size=sample_size)
d = s.start()
s.maintained().addBoth(lambda x: self.fail())
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
# Force the control service to fail requests for one second.
# These requests will fail after the delay period set in the
# control service.
cluster.get_control_service(c).fail_requests = True
c.advance(1)
cluster.get_control_service(c).fail_requests = False
d.addCallback(lambda ignored: s.stop())
# The scenario should not successfully stop until after the
# delay period for the failed requests.
self.assertNoResult(d)
c.advance(delay)
self.successResultOf(d)
