def test_with_delays():
"""
This tests for service delays when starting. The service will delay start for 5
seconds, and the os will stop in 10 seconds.
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == services ==
error_handlers = [MockErrorHandler]
os.schedule_service(MockService,
ServiceMetaData("mock-service", delay=5, recovery_enabled=False),
error_handlers=error_handlers)
# now assert services have retries set to 0
assert os.scheduler.get_service_manager().get_service_meta("mock-service").starts == 0
def stop_os():
# by this time services have started and have registered retries == 1
assert os.scheduler.get_service_manager().get_service_meta("mock-service").starts == 1
assert os.scheduler.get_service_manager().get_service("mock-service").ack is True
os.shutdown()
# take timing from start to finish
t1 = time.time()
os_greenlet = gevent.spawn_later(os_stop_time, stop_os)
gevent.joinall([os_greenlet])
# == Test Validation/Verification ==
# Makes sure that the test timer works correctly
t2 = time.time()
t3 = t2 - t1
assert t3 > os_stop_time
assert os_greenlet.successful() # else look at the logs
def test_with_delay_that_never_starts():
"""
This tests if the service started with a delay and if it timed out during the
start within 2 seconds. The mock service being used has a sleep call for 5
seconds that hopefully doesn't complete as another greenlet will remove
the reference. This is effectively like a hard error killing off the greenlet
reference. This shouldn't happen but if it does it should get picked up by the
service manager. This test simulates that.
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == services ==
error_handlers = [MockErrorHandler]
os.schedule_service(
MockServiceBad,
ServiceMetaData("mock-service-bad", delay=1, start_timeout=2, recovery_enabled=False),
error_handlers=error_handlers,
)
# now assert services have retries set to 0
assert os.scheduler.get_service_manager().get_service_meta("mock-service-bad").starts == 0
def stop_os():
# by this time services have started and have registered retries == 1
meta = os.scheduler.get_service_manager().get_service_meta("mock-service-bad")
service = os.scheduler.get_service_manager().get_service("mock-service-bad")
assert meta.starts == 1
assert len(meta.exceptions) == 1
assert service.ack is False
os.shutdown()
# take timing from start to finish
t1 = time.time()
os_greenlet = gevent.spawn_later(os_stop_time, stop_os)
# remove_greenlet = gevent.spawn_later(2, remove_service)
# gevent.joinall([os_greenlet, remove_greenlet])
gevent.joinall([os_greenlet])
# == Test Validation/Verification ==
# Makes sure that the test timer works correctly
t2 = time.time()
t3 = t2 - t1
assert t3 > os_stop_time
# This os itself should not fail. Failed assertion will cause the os to fail.
# The below is done because assertions in the greenlet will not raise outside
# the gevent/greenlet scope.
assert os_greenlet.successful()
def test_with_delay_that_never_starts():
"""
This tests for service delays when starting but it won't ever start because
the OS is scheduled to stop in 10 seconds, while the service start delay is
set to start in 13 seconds (13 chosen because a time too close to the os
stop of 10 like 11 might be too close on a very very slow system). On those
systems operations take time, account for that. However anything past
a couple seconds is not worth it.
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == services ==
error_handlers = [MockErrorHandler]
os.schedule_service(MockService,
ServiceMetaData("mock-service", delay=13, recovery_enabled=False),
error_handlers=error_handlers)
# now assert services have retries set to 0
assert os.scheduler.get_service_manager().get_service_meta("mock-service").starts == 0
def stop_os():
# by this time services have started and have registered retries == 1
meta = os.scheduler.get_service_manager().get_service_meta("mock-service")
service = os.scheduler.get_service_manager().get_service("mock-service")
assert meta.starts == 1
# no exceptions occur as the system shutsdown before anything starts
assert len(meta.exceptions) == 0
assert service.ack is False
os.shutdown()
# take timing from start to finish
t1 = time.time()
os_greenlet = gevent.spawn_later(os_stop_time, stop_os)
gevent.joinall([os_greenlet])
# == Test Validation/Verification ==
# Makes sure that the test timer works correctly
t2 = time.time()
t3 = t2 - t1
assert t3 > os_stop_time
# This os itself should not fail. Failed assertion will cause the os to fail.
# The below is done because assertions in the greenlet will not raise outside
# the gevent/greenlet scope.
assert os_greenlet.successful()
def test_with_delay_that_never_starts():
"""
This tests for service delays when starting but it won't ever start because
the OS is scheduled to stop in 10 seconds, while the service start delay is
set to start in 13 seconds (13 chosen because a time too close to the os
stop of 10 like 11 might be too close on a very very slow system). On those
systems operations take time, account for that. However anything past
a couple seconds is not worth it.
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == services ==
error_handlers = [MockErrorHandler]
os.schedule_service(MockService,
ServiceMetaData("mock-service",
delay=13,
recovery_enabled=False),
error_handlers=error_handlers)
# now assert services have retries set to 0
assert os.scheduler.get_service_manager().get_service_meta(
"mock-service").starts == 0
def stop_os():
# by this time services have started and have registered retries == 1
meta = os.scheduler.get_service_manager().get_service_meta(
"mock-service")
service = os.scheduler.get_service_manager().get_service(
"mock-service")
assert meta.starts == 1
# no exceptions occur as the system shutsdown before anything starts
assert len(meta.exceptions) == 0
assert service.ack is False
os.shutdown()
# take timing from start to finish
t1 = time.time()
os_greenlet = gevent.spawn_later(os_stop_time, stop_os)
gevent.joinall([os_greenlet])
# == Test Validation/Verification ==
# Makes sure that the test timer works correctly
t2 = time.time()
t3 = t2 - t1
assert t3 > os_stop_time
# This os itself should not fail. Failed assertion will cause the os to fail.
# The below is done because assertions in the greenlet will not raise outside
# the gevent/greenlet scope.
assert os_greenlet.successful()
def test_with_delays():
"""
This tests for service delays when starting. The service will delay start for 5
seconds, and the os will stop in 10 seconds.
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == services ==
error_handlers = [MockErrorHandler]
os.schedule_service(MockService,
ServiceMetaData("mock-service",
delay=5,
recovery_enabled=False),
error_handlers=error_handlers)
# now assert services have retries set to 0
assert os.scheduler.get_service_manager().get_service_meta(
"mock-service").starts == 0
def stop_os():
# by this time services have started and have registered retries == 1
assert os.scheduler.get_service_manager().get_service_meta(
"mock-service").starts == 1
assert os.scheduler.get_service_manager().get_service(
"mock-service").ack is True
os.shutdown()
# take timing from start to finish
t1 = time.time()
os_greenlet = gevent.spawn_later(os_stop_time, stop_os)
gevent.joinall([os_greenlet])
# == Test Validation/Verification ==
# Makes sure that the test timer works correctly
t2 = time.time()
t3 = t2 - t1
assert t3 > os_stop_time
assert os_greenlet.successful() # else look at the logs
def main():
"""
Main
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == support services ==
os.schedule_service(DBService, "database-service", True)
os.schedule_service(QueueService, "queue-service", True)
# == action services ==
# reads either file or db and prims the queue - right now hard coded
os.schedule_service(InitializerService, "initializer-service", True)
# analyzer pulls from queue things to analyze, if a resource can
# be requested it is sent to the requestor queue to be requested
os.schedule_service(AnalyzerService, "analyzer-service", True)
# requestor pulls from the request queue and executes a request
# the response is then put on the publish queue
os.schedule_service(RequestorService, "requestor-service", True)
# response service will read from the publish queue and
# parse the response, updating the timings and doing other
# data intense tasks. One complete the resource is once again put
# on the analyze queue
os.schedule_service(ResponseParserService, "response-service", True)
# setup the freezer services, where each work in a different greenlet
# event loop with a sleep (wait idle).
os.schedule_service(Freezer50Service, "freezer-50", True)
os.schedule_service(Freezer250Service, "freezer-250", True)
os.schedule_service(Freezer500Service, "freezer-500", True)
os.schedule_service(Freezer1000Service, "freezer-1000", True)
def stop_os():
os.shutdown()
def stop():
return gevent.spawn_later(180, stop_os)
gevent.joinall([stop()])
def test_services_for_exceptions():
"""
This test simulates the RuntimeError "maximum recursion depth exceeded".
Ensure that this test passes without seeing this error.
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == support services ==
os.schedule_service(
DBService, ServiceMetaData("database-service", recovery_enabled=True))
os.schedule_service(
QueueService, ServiceMetaData("queue-service", recovery_enabled=True))
# == mock Init - will wait for analyzer to run ==
os.schedule_service(
MockInitializerService,
ServiceMetaData("mock-initializer-service", recovery_enabled=True))
os.schedule_service(
AnalyzerService,
ServiceMetaData("analyzer-service", recovery_enabled=True))
# == freezes - will wait until analyzer runs ==
os.schedule_service(Freezer50Service,
ServiceMetaData("freezer-50", recovery_enabled=True))
os.schedule_service(Freezer250Service,
ServiceMetaData("freezer-250", recovery_enabled=True))
os.schedule_service(Freezer500Service,
ServiceMetaData("freezer-500", recovery_enabled=True))
os.schedule_service(Freezer1000Service,
ServiceMetaData("freezer-1000", recovery_enabled=True))
# == error handlers for MockResponseService ==
error_handlers = [MockErrorHandler]
# == main services - in reverse order of execution so that
# analyzer runs last ==
os.schedule_service(MockResponseService,
ServiceMetaData("mock-response-service",
retries=2,
recovery_enabled=True),
error_handlers=error_handlers)
os.schedule_service(
MockRequestService,
ServiceMetaData("mock-requestor", recovery_enabled=True))
# Manually restart a service to register a retry count of +1
os.scheduler.restart_service("database-service")
assert os.scheduler.get_services_count() == 10
assert os.scheduler.get_service_manager().get_service_meta(
"database-service").starts == 1
# now assert other services have retries set to 0
assert os.scheduler.get_service_manager().get_service_meta(
"analyzer-service").starts == 0
assert os.scheduler.get_service_manager().get_service_meta(
"mock-requestor").starts == 0
assert os.scheduler.get_service_manager().get_service_meta(
"mock-initializer-service").starts == 0
def stop_os():
# by this time services have started and have registered retries == 1
assert os.scheduler.get_service_manager().get_service_meta(
"analyzer-service").starts == 1
assert os.scheduler.get_service_manager().get_service_meta(
"mock-requestor").starts == 1
assert os.scheduler.get_service_manager().get_service_meta(
"mock-initializer-service").starts == 1
# retry limit is set to 2, should only see three exceptions in the list
# since it will start three times (initial start + two retries) and fail each time.
assert len(os.scheduler.get_service_manager().get_service_meta(
"mock-response-service").exceptions) == 3
os.shutdown()
# take timing from start to finish
t1 = time.time()
os_greenlet = gevent.spawn_later(os_stop_time, stop_os)
gevent.joinall([os_greenlet])
# == Test Validation/Verification ==
# Makes sure that the test timer works correctly
t2 = time.time()
t3 = t2 - t1
assert t3 > os_stop_time
assert os_greenlet.successful()
def test_services_for_exceptions():
"""
This test simulates the RuntimeError "maximum recursion depth exceeded".
Ensure that this test passes without seeing this error.
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == support services ==
os.schedule_service(DBService, ServiceMetaData("database-service", recovery_enabled=True))
os.schedule_service(QueueService, ServiceMetaData("queue-service", recovery_enabled=True))
# == mock Init - will wait for analyzer to run ==
os.schedule_service(MockInitializerService, ServiceMetaData("mock-initializer-service", recovery_enabled=True))
os.schedule_service(AnalyzerService, ServiceMetaData("analyzer-service", recovery_enabled=True))
# == freezes - will wait until analyzer runs ==
os.schedule_service(Freezer50Service, ServiceMetaData("freezer-50", recovery_enabled=True))
os.schedule_service(Freezer250Service, ServiceMetaData("freezer-250", recovery_enabled=True))
os.schedule_service(Freezer500Service, ServiceMetaData("freezer-500", recovery_enabled=True))
os.schedule_service(Freezer1000Service, ServiceMetaData("freezer-1000", recovery_enabled=True))
# == error handlers for MockResponseService ==
error_handlers = [MockErrorHandler]
# == main services - in reverse order of execution so that
# analyzer runs last ==
os.schedule_service(MockResponseService,
ServiceMetaData("mock-response-service", retries=2, recovery_enabled=True),
error_handlers=error_handlers)
os.schedule_service(MockRequestService, ServiceMetaData("mock-requestor", recovery_enabled=True))
# Manually restart a service to register a retry count of +1
os.scheduler.restart_service("database-service")
assert os.scheduler.get_services_count() == 10
assert os.scheduler.get_service_manager().get_service_meta("database-service").starts == 1
# now assert other services have retries set to 0
assert os.scheduler.get_service_manager().get_service_meta("analyzer-service").starts == 0
assert os.scheduler.get_service_manager().get_service_meta("mock-requestor").starts == 0
assert os.scheduler.get_service_manager().get_service_meta("mock-initializer-service").starts == 0
def stop_os():
# by this time services have started and have registered retries == 1
assert os.scheduler.get_service_manager().get_service_meta("analyzer-service").starts == 1
assert os.scheduler.get_service_manager().get_service_meta("mock-requestor").starts == 1
assert os.scheduler.get_service_manager().get_service_meta("mock-initializer-service").starts == 1
# retry limit is set to 2, should only see three exceptions in the list
# since it will start three times (initial start + two retries) and fail each time.
assert len(os.scheduler.get_service_manager().get_service_meta("mock-response-service").exceptions) == 3
os.shutdown()
# take timing from start to finish
t1 = time.time()
os_greenlet = gevent.spawn_later(os_stop_time, stop_os)
gevent.joinall([os_greenlet])
# == Test Validation/Verification ==
# Makes sure that the test timer works correctly
t2 = time.time()
t3 = t2 - t1
assert t3 > os_stop_time
assert os_greenlet.successful()
def main():
"""
Main
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == support services ==
os.schedule_service(DBService, "database-service", True)
os.schedule_service(QueueService, "queue-service", True)
# == action services ==
# reads either file or db and prims the queue - right now hard coded
os.schedule_service(InitializerService, "initializer-service", True)
# analyzer pulls from queue things to analyze, if a resource can
# be requested it is sent to the requestor queue to be requested
os.schedule_service(AnalyzerService, "analyzer-service", True)
# requestor pulls from the request queue and executes a request
# the response is then put on the publish queue
os.schedule_service(RequestorService, "requestor-service", True)
# response service will read from the publish queue and
# parse the response, updating the timings and doing other
# data intense tasks. One complete the resource is once again put
# on the analyze queue
os.schedule_service(ResponseParserService, "response-service", True)
# setup the freezer services, where each work in a different greenlet
# event loop with a sleep (wait idle).
os.schedule_service(Freezer50Service, "freezer-50", True)
os.schedule_service(Freezer250Service, "freezer-250", True)
os.schedule_service(Freezer500Service, "freezer-500", True)
os.schedule_service(Freezer1000Service, "freezer-1000", True)
def stop_os():
os.shutdown()
def stop():
return gevent.spawn_later(180, stop_os)
gevent.joinall([stop()])
def test_freezer_services_for_max_recursion():
"""
This test simulates the RuntimeError "maximum recursion depth exceeded".
Ensure that this test passes without seeing this error.
:return:
"""
os = CannedOS("CannedOS")
os.bootup()
# == support services ==
os.schedule_service(DBService, ServiceMetaData("database-service", recovery_enabled=True))
os.schedule_service(QueueService, ServiceMetaData("queue-service", recovery_enabled=True))
# == mock Init - will wait for analyzer to run ==
os.schedule_service(MockInitializerService, ServiceMetaData("mock-initializer-service", recovery_enabled=True))
# == freezes - will wait until analyzer runs ==
os.schedule_service(Freezer50Service, ServiceMetaData("freezer-50", recovery_enabled=True))
os.schedule_service(Freezer250Service, ServiceMetaData("freezer-250", recovery_enabled=True))
os.schedule_service(Freezer500Service, ServiceMetaData("freezer-500", recovery_enabled=True))
os.schedule_service(Freezer1000Service, ServiceMetaData("freezer-1000", recovery_enabled=True))
# == main services - in reverse order of execution so that
# analyzer runs last ==
os.schedule_service(MockResponseService, ServiceMetaData("mock-response-service", recovery_enabled=True))
os.schedule_service(MockRequestService, ServiceMetaData("mock-requestor", recovery_enabled=True))
os.schedule_service(AnalyzerService, ServiceMetaData("analyzer-service", recovery_enabled=True))
# Manually restart a service to register a retry count of +1
# Of note here is that up until this point the database-service has not yet
# been started. It will register as such in the logs and then proceed to start.
# A log entry will be created saying the service was able to be restarted
# successfully. It's started count will be 1, and when the service manager
# loops the services it will notice that this service was already started,
# skipping it.
# Other systems would have errored on the stop but I believe it
# is the end state that is of importance. I would care not that the service
# wasn't running to be stopped as what I do care about is that it was
# restarted, even if that means it was not running in the first place.
# I issue restart because I want the service to be started.
os.scheduler.restart_service("database-service")
assert os.scheduler.get_services_count() == 10
assert os.scheduler.get_service_manager().get_service_meta("database-service").starts == 1
# now assert other services have retries set to 0
assert os.scheduler.get_service_manager().get_service_meta("analyzer-service").starts == 0
assert os.scheduler.get_service_manager().get_service_meta("mock-requestor").starts == 0
assert os.scheduler.get_service_manager().get_service_meta("mock-initializer-service").starts == 0
def stop_os():
# by this time services have started and have registered retries == 1
assert os.scheduler.get_service_manager().get_service_meta("analyzer-service").starts == 1
assert os.scheduler.get_service_manager().get_service_meta("mock-requestor").starts == 1
assert os.scheduler.get_service_manager().get_service_meta("mock-initializer-service").starts == 1
assert os.scheduler.get_service_manager().get_service_meta("database-service").starts == 1
os.shutdown()
def stop():
return gevent.spawn_later(os_stop_time, stop_os)
# take timing from start to finish
t1 = time.time()
gevent.joinall([stop()])
# == Test Validation/Verification ==
# Makes sure that the test timer works correctly
t2 = time.time()
t3 = t2 - t1
assert t3 > os_stop_time
def __init__(self, name):
CannedOS.__init__(self, name)
