def test_scenario_throws_exception_when_rate_drops(self):
"""
WriteRequestLoadScenario raises RequestRateTooLow if rate
drops below the requested rate.
Establish the requested rate by having the FakeFlockerClient
respond to all requests, then lower the rate by dropping
alternate requests. This should result in RequestRateTooLow
being raised.
"""
c = Clock()
control_service = self.get_dropping_flocker_client_instance()
cluster = self.make_cluster(control_service)
sample_size = 5
s = WriteRequestLoadScenario(c, cluster, sample_size=sample_size)
s.start()
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
control_service.drop_requests = True
# Advance the clock by 2 seconds so that a request is dropped
# and a new rate which is below the target can be established.
c.advance(2)
failure = self.failureResultOf(s.maintained())
self.assertIsInstance(failure.value, WRequestRateTooLow)
def test_scenario_throws_exception_when_rate_drops(self):
"""
WriteRequestLoadScenario raises RequestRateTooLow if rate
drops below the requested rate.
Establish the requested rate by having the FakeFlockerClient
respond to all requests, then lower the rate by dropping
alternate requests. This should result in RequestRateTooLow
being raised.
"""
c = Clock()
control_service = self.get_dropping_flocker_client_instance()
cluster = self.make_cluster(control_service)
sample_size = 5
s = WriteRequestLoadScenario(c, cluster, sample_size=sample_size)
s.start()
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
control_service.drop_requests = True
# Advance the clock by 2 seconds so that a request is dropped
# and a new rate which is below the target can be established.
c.advance(2)
failure = self.failureResultOf(s.maintained())
self.assertIsInstance(failure.value, WRequestRateTooLow)
def test_setup_timeout_when_datasat_not_created(self):
"""
`WriteRequestLoadScenario` should timeout if the setup the dataset
creation does not complete within the given time.
"""
c = Clock()
cluster = self.make_cluster(
self.get_unresponsive_flocker_client_instance())
s = WriteRequestLoadScenario(c, cluster, 5, sample_size=3)
d = s.start()
c.pump(repeat(1, s.timeout + 1))
failure = self.failureResultOf(d)
self.assertIsInstance(failure.value, DatasetCreationTimeout)
def test_setup_timeout_when_datasat_not_created(self):
"""
`WriteRequestLoadScenario` should timeout if the setup the dataset
creation does not complete within the given time.
"""
c = Clock()
cluster = self.make_cluster(
self.get_unresponsive_flocker_client_instance())
s = WriteRequestLoadScenario(c, cluster, 5, sample_size=3)
d = s.start()
c.pump(repeat(1, s.timeout+1))
failure = self.failureResultOf(d)
self.assertIsInstance(failure.value, DatasetCreationTimeout)
def test_scenario_throws_exception_if_requested_rate_not_reached(self):
"""
WriteRequestLoadScenario raises RequestRateNotReached if the
target rate cannot be established within a given timeframe.
"""
c = Clock()
control_service = self.get_dropping_flocker_client_instance()
cluster = self.make_cluster(control_service)
s = WriteRequestLoadScenario(c, cluster)
control_service.drop_requests = True
d = s.start()
# Continue the clock for one second longer than the timeout
# value to allow the timeout to be triggered.
c.advance(s.timeout + 1)
failure = self.failureResultOf(d)
self.assertIsInstance(failure.value, WRequestRateNotReached)
def test_scenario_throws_exception_if_requested_rate_not_reached(self):
"""
WriteRequestLoadScenario raises RequestRateNotReached if the
target rate cannot be established within a given timeframe.
"""
c = Clock()
control_service = self.get_dropping_flocker_client_instance()
cluster = self.make_cluster(control_service)
s = WriteRequestLoadScenario(c, cluster)
control_service.drop_requests = True
d = s.start()
# Continue the clock for one second longer than the timeout
# value to allow the timeout to be triggered.
c.advance(s.timeout + 1)
failure = self.failureResultOf(d)
self.assertIsInstance(failure.value, WRequestRateNotReached)
def test_setup_generates_dataset(self):
"""
`WriteRequestLoadScenario` starts and stops without collapsing.
"""
c = Clock()
cluster = self.make_cluster(self.get_fake_flocker_client_instance())
s = WriteRequestLoadScenario(c, cluster, 5, sample_size=3)
def assert_created(returned_datasets):
self.assertNotEqual(returned_datasets, [])
# Create a datasest and verify we get a success
d = s._create_dataset(self.node1)
self.successResultOf(d)
# Verify that a dataset is actually being created
d2 = s.control_service.list_datasets_configuration()
d2.addCallback(assert_created)
s.stop()
def test_write_request_load_succeeds(self):
"""
WriteRequestLoadScenario starts and stops without collapsing.
"""
c = Clock()
cluster = self.make_cluster(self.get_fake_flocker_client_instance())
sample_size = 5
s = WriteRequestLoadScenario(c, cluster, sample_size=sample_size)
d = s.start()
# Request rate samples are recorded every second and we need to
# collect enough samples to establish the rate which is defined
# by `sample_size`. Therefore, advance the clock by
# `sample_size` seconds to obtain enough samples.
c.pump(repeat(1, sample_size))
s.maintained().addBoth(lambda x: self.fail())
d.addCallback(lambda ignored: s.stop())
self.successResultOf(d)
def test_setup_generates_dataset(self):
"""
`WriteRequestLoadScenario` starts and stops without collapsing.
"""
c = Clock()
cluster = self.make_cluster(self.get_fake_flocker_client_instance())
s = WriteRequestLoadScenario(c, cluster, 5, sample_size=3)
def assert_created(returned_datasets):
self.assertNotEqual(returned_datasets, [])
# Create a datasest and verify we get a success
d = s._create_dataset(self.node1)
self.successResultOf(d)
# Verify that a dataset is actually being created
d2 = s.control_service.list_datasets_configuration()
d2.addCallback(assert_created)
s.stop()
def test_scenario_throws_exception_if_overloaded(self):
"""
`WriteRequestLoadScenario` raises `RequestOverload` if the
difference between sent requests and received requests exceeds
the tolerated difference once we start monitoring the scenario.
Note that, right now, the only way to make it fail is to generate
this difference before we start monitoring the scenario.
Once we implement some kind of tolerance, to allow fluctuations
in the rate, we can update this tests to trigger the exception
in a more realistic manner.
"""
# XXX Update this test when we add tolerance for rate fluctuations.
# See FLOC-3757.
c = Clock()
control_service = self.get_dropping_flocker_client_instance()
cluster = self.make_cluster(control_service)
target_rate = 10
sample_size = 20
s = WriteRequestLoadScenario(c,
cluster,
request_rate=target_rate,
sample_size=sample_size)
dropped_rate = target_rate / 2
seconds_to_overload = s.max_outstanding / dropped_rate
s.start()
# Reach initial rate
control_service.drop_requests = True
# Initially, we generate enough dropped requests so that the scenario
# is overloaded when we start monitoring.
c.pump(repeat(1, seconds_to_overload + 1))
# We stop dropping requests
control_service.drop_requests = False
# Now we generate the initial rate to start monitoring the scenario
c.pump(repeat(1, sample_size))
# We only need to advance one more second (first loop in the monitoring
# loop) to trigger RequestOverload
c.advance(1)
failure = self.failureResultOf(s.maintained())
self.assertIsInstance(failure.value, WRequestOverload)
def test_scenario_throws_exception_if_overloaded(self):
"""
`WriteRequestLoadScenario` raises `RequestOverload` if the
difference between sent requests and received requests exceeds
the tolerated difference once we start monitoring the scenario.
Note that, right now, the only way to make it fail is to generate
this difference before we start monitoring the scenario.
Once we implement some kind of tolerance, to allow fluctuations
in the rate, we can update this tests to trigger the exception
in a more realistic manner.
"""
# XXX Update this test when we add tolerance for rate fluctuations.
# See FLOC-3757.
c = Clock()
control_service = self.get_dropping_flocker_client_instance()
cluster = self.make_cluster(control_service)
target_rate = 10
sample_size = 20
s = WriteRequestLoadScenario(c, cluster, request_rate=target_rate,
sample_size=sample_size)
dropped_rate = target_rate / 2
seconds_to_overload = s.max_outstanding / dropped_rate
s.start()
# Reach initial rate
control_service.drop_requests = True
# Initially, we generate enough dropped requests so that the scenario
# is overloaded when we start monitoring.
c.pump(repeat(1, seconds_to_overload+1))
# We stop dropping requests
control_service.drop_requests = False
# Now we generate the initial rate to start monitoring the scenario
c.pump(repeat(1, sample_size))
# We only need to advance one more second (first loop in the monitoring
# loop) to trigger RequestOverload
c.advance(1)
failure = self.failureResultOf(s.maintained())
self.assertIsInstance(failure.value, WRequestOverload)
def test_scenario_timeouts_if_requests_not_completed(self):
"""
`WriteRequestLoadScenario` should timeout if the outstanding
requests for the scenarion do not complete within the specified
time.
"""
c = Clock()
control_service = self.get_error_response_client(c)
cluster = self.make_cluster(control_service)
sample_size = 5
s = WriteRequestLoadScenario(
c, cluster, request_rate=10, sample_size=sample_size
)
# Set the delay for the requests to be longer than the scenario
# timeout
control_service.delay = s.timeout + 10
d = s.start()
s.maintained().addBoth(lambda x: self.fail())
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
control_service.fail_requests = True
c.advance(1)
control_service.fail_requests = False
d.addCallback(lambda ignored: s.stop())
# Advance the clock by the timeout value so it is triggered
# before the requests complete.
self.assertNoResult(d)
c.advance(s.timeout + 1)
self.assertTrue(s.rate_measurer.outstanding() > 0)
self.successResultOf(d)
def test_scenario_stops_only_when_no_outstanding_requests(self):
"""
`WriteRequestLoadScenario` should only be considered as stopped
when all outstanding requests made by it have completed.
"""
c = Clock()
control_service = self.get_error_response_client(c)
cluster = self.make_cluster(control_service)
delay = 1
control_service.delay = delay
sample_size = 5
s = WriteRequestLoadScenario(
c, cluster, request_rate=10, sample_size=sample_size
)
d = s.start()
s.maintained().addBoth(lambda x: self.fail())
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
# Force the control service to fail requests for one second.
# These requests will fail after the delay period set in the
# control service.
control_service.fail_requests = True
c.advance(1)
control_service.fail_requests = False
d.addCallback(lambda ignored: s.stop())
# The scenario should not successfully stop until after the
# delay period for the failed requests.
self.assertNoResult(d)
c.advance(delay)
self.successResultOf(d)
def test_write_request_load_succeeds(self):
"""
WriteRequestLoadScenario starts and stops without collapsing.
"""
c = Clock()
cluster = self.make_cluster(self.get_fake_flocker_client_instance())
sample_size = 5
s = WriteRequestLoadScenario(c, cluster, sample_size=sample_size)
d = s.start()
# Request rate samples are recorded every second and we need to
# collect enough samples to establish the rate which is defined
# by `sample_size`. Therefore, advance the clock by
# `sample_size` seconds to obtain enough samples.
c.pump(repeat(1, sample_size))
s.maintained().addBoth(lambda x: self.fail())
d.addCallback(lambda ignored: s.stop())
self.successResultOf(d)
def test_scenario_timeouts_if_requests_not_completed(self):
"""
`WriteRequestLoadScenario` should timeout if the outstanding
requests for the scenarion do not complete within the specified
time.
"""
c = Clock()
control_service = self.get_error_response_client(c)
cluster = self.make_cluster(control_service)
sample_size = 5
s = WriteRequestLoadScenario(c,
cluster,
request_rate=10,
sample_size=sample_size)
# Set the delay for the requests to be longer than the scenario
# timeout
control_service.delay = s.timeout + 10
d = s.start()
s.maintained().addBoth(lambda x: self.fail())
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
control_service.fail_requests = True
c.advance(1)
control_service.fail_requests = False
d.addCallback(lambda ignored: s.stop())
# Advance the clock by the timeout value so it is triggered
# before the requests complete.
self.assertNoResult(d)
c.advance(s.timeout + 1)
self.assertTrue(s.rate_measurer.outstanding() > 0)
self.successResultOf(d)
def test_scenario_stops_only_when_no_outstanding_requests(self):
"""
`WriteRequestLoadScenario` should only be considered as stopped
when all outstanding requests made by it have completed.
"""
c = Clock()
control_service = self.get_error_response_client(c)
cluster = self.make_cluster(control_service)
delay = 1
control_service.delay = delay
sample_size = 5
s = WriteRequestLoadScenario(c,
cluster,
request_rate=10,
sample_size=sample_size)
d = s.start()
s.maintained().addBoth(lambda x: self.fail())
# Advance the clock by `sample_size` seconds to establish the
# requested rate.
c.pump(repeat(1, sample_size))
# Force the control service to fail requests for one second.
# These requests will fail after the delay period set in the
# control service.
control_service.fail_requests = True
c.advance(1)
control_service.fail_requests = False
d.addCallback(lambda ignored: s.stop())
# The scenario should not successfully stop until after the
# delay period for the failed requests.
self.assertNoResult(d)
c.advance(delay)
self.successResultOf(d)