def __init__(self, params):
"""Initialize.
:param params: Parameter data storage for Tester APIs.
:type params: :obj:`lib.params.Params`
"""
self.params = params
self.etos = ETOS("ETOS API", os.getenv("HOSTNAME"), "ETOS API")
self.etos.config.rabbitmq_publisher_from_environment()
self.etos.start_publisher()
def __init__(self):
"""Initialize ETOS library and start eiffel publisher."""
self.etos = ETOS("ETOS Test Runner", os.getenv("HOSTNAME"), "ETOS Test Runner")
self.etos.config.rabbitmq_publisher_from_environment()
# ETR will print the entire environment just before executing.
# Hide the password.
os.environ["RABBITMQ_PASSWORD"] = "******"
self.etos.start_publisher()
self.tests_url = os.getenv("SUB_SUITE_URL")
signal.signal(signal.SIGTERM, self.graceful_shutdown)
def __init__(self):
"""Initialize ESR by creating a rabbitmq publisher."""
self.logger = logging.getLogger("ESR")
self.etos = ETOS("ETOS Suite Runner", os.getenv("SOURCE_HOST"),
"ETOS Suite Runner")
signal.signal(signal.SIGTERM, self.graceful_exit)
self.params = ESRParameters(self.etos)
self.etos.config.rabbitmq_publisher_from_environment()
self.etos.start_publisher()
self.etos.config.set(
"WAIT_FOR_ENVIRONMENT_TIMEOUT",
int(os.getenv("ESR_WAIT_FOR_ENVIRONMENT_TIMEOUT")),
)
def __init__(self):
"""Initialize SuiteStarter by creating a rabbitmq publisher and subscriber."""
self.etos = ETOS("ETOS Suite Starter", os.getenv("HOSTNAME"),
"ETOS Suite Starter")
self._configure()
self.etos.config.rabbitmq_subscriber_from_environment()
self.etos.config.rabbitmq_publisher_from_environment()
self.etos.start_subscriber()
self.etos.start_publisher()
self.etos.subscriber.subscribe(
"EiffelTestExecutionRecipeCollectionCreatedEvent",
self.suite_runner_callback,
can_nack=True,
)
def test_get_configuration_missing(self):
"""Test that if a configuration is missing, a partial result is returned.
Approval criteria:
- The configure backend shall return a partial configuration if configuration
is missing.
Test steps:
1. Store a faulty configuration into the database.
2. Verify that it is possible to get the partial configuration.
"""
database = FakeDatabase()
test_suite_id = "ca51601e-6c9a-4b5d-8038-7dc2561283d2"
test_dataset = {"dataset": "test"}
self.logger.info(
"STEP: Store a faulty configuration into the database.")
database.writer.hset(f"EnvironmentProvider:{test_suite_id}", "Dataset",
json.dumps(test_dataset))
self.logger.info(
"STEP: Verify that it is possible to get the partial configuration."
)
registry = ProviderRegistry(ETOS("", "", ""), JsonTas(), database)
stored_configuration = get_configuration(registry, test_suite_id)
self.assertDictEqual(
stored_configuration,
{
"dataset": test_dataset,
"iut_provider": None,
"execution_space_provider": None,
"log_area_provider": None,
},
)
async def configure_environment_provider(
environment: ConfigureEnvironmentProviderRequest, ):
"""Configure environment provider request.
:param environment: Environment to configure.
:type environment: :obj:`etos_api.routers.etos.schemas.ConfigureEnvironmentProviderRequest`
"""
etos_library = ETOS("ETOS API", os.getenv("HOSTNAME"), "ETOS API")
end_time = time.time() + etos_library.debug.default_http_timeout
async with aiohttp.ClientSession() as session:
while time.time() < end_time:
try:
async with session.post(
f"{etos_library.debug.environment_provider}/configure",
json=environment.dict(),
headers={
"Content-Type": "application/json",
"Accept": "application/json",
},
) as response:
assert 200 <= response.status < 400
break
except AssertionError:
await asyncio.sleep(2)
else:
raise HTTPException(
status_code=400,
detail=
f"Unable to configure environment provider with '{environment.json()}'",
)
def test_register_log_area_provider(self):
"""Test that the register backend can register log area providers.
Approval criteria:
- The register backend shall be able to register a log area provider.
Test steps:
1. Register a log area provider with the register backend.
2. Verify that the log area provider was stored in the database.
"""
fake_database = FakeDatabase()
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
jsontas = JsonTas()
provider = {
"log": {
"id": "log_area_provider_test",
"list": {"available": [], "possible": []},
}
}
provider_registry = ProviderRegistry(etos, jsontas, fake_database)
self.logger.info(
"STEP: Register a log area provider with the register backend."
)
response = register(provider_registry, log_area_provider=provider)
self.logger.info(
"STEP: Verify that the log area provider was stored in the database."
)
stored_provider = json.loads(
fake_database.reader.hget(
"EnvironmentProvider:LogAreaProviders", "log_area_provider_test"
)
)
self.assertDictEqual(stored_provider, provider)
self.assertTrue(response)
def test_release_environment_no_task_result(self):
"""Test that it is not possible to release an environment without task results.
Approval criteria:
- The environment provider shall not attempt to release environments with results.
Test steps:
1. Attempt to release an environment without a task ID.
2. Verify that the release fails.
"""
database = FakeDatabase()
test_release_id = "ce63f53e-1797-42bb-ae72-861a0b6b7ef6"
worker = FakeCelery("thisdoesnotexist", "SUCCESS", {})
jsontas = JsonTas()
etos = ETOS("", "", "")
registry = ProviderRegistry(etos, jsontas, database)
self.logger.info(
"STEP: Attempt to release an environment without a task ID.")
success, _ = release_environment(
etos,
jsontas,
registry,
worker.AsyncResult(test_release_id),
test_release_id,
)
self.logger.info("STEP: Verify that the release fails.")
self.assertFalse(success)
def test_request_and_wait(self):
"""Test that the external log area provider can checkout log areas.
Approval criteria:
- The external log area provider shall request an external provider and
checkout log areas.
Test steps::
1. Initialize an external provider.
2. Send a checkout request via the external log area provider.
3. Verify that the provider returns a list of checked out log areas.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_LOG_AREA_TIMEOUT", 10)
jsontas = JsonTas()
identity = PackageURL.from_string("pkg:testing/etos")
jsontas.dataset.merge(
{
"identity": identity,
"artifact_id": "artifactid",
"artifact_created": "artifactcreated",
"artifact_published": "artifactpublished",
"tercc": "tercc",
"dataset": {},
"context": "context",
}
)
start_id = "1"
test_id = "logarea123"
provider_id = "test_request_and_wait"
# First request is 'start'.
# Second request is 'status'.
# Third request is 'stop' which should not be requested in this test.
with FakeServer(
["ok", "ok", "no_content"],
[
{"id": start_id},
{"log_areas": [{"test_id": test_id}], "status": "DONE"},
{},
],
) as server:
ruleset = {
"id": provider_id,
"status": {"host": server.host},
"start": {"host": server.host},
"stop": {"host": server.host},
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info(
"STEP: Send a checkout request via the external log area provider."
)
log_areas = provider.request_and_wait_for_log_areas()
self.logger.info(
"STEP: Verify that the provider returns a list of checked out log areas."
)
dict_log_areas = [log_area.as_dict for log_area in log_areas]
test_log_areas = [LogArea(provider_id=provider_id, test_id=test_id).as_dict]
self.assertEqual(dict_log_areas, test_log_areas)
def test_provider_stop_many(self):
"""Test that it is possible to checkin an external execution space provider with many
execution spaces.
Approval criteria:
- It shall be possible to send stop to an external execution space provider
with multiple execution spaces.
Test steps::
1. Initialize an external provider.
2. Send a stop request for multiple execution spaces.
3. Verify that the stop endpoint is called.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_EXECUTION_SPACE_TIMEOUT", 1)
jsontas = JsonTas()
execution_spaces = [
ExecutionSpace(test_execution_space=1),
ExecutionSpace(test_execution_space=2),
]
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
"execution_spaces":
execution_spaces,
})
dict_execution_spaces = [
execution_space.as_dict for execution_space in execution_spaces
]
with FakeServer("no_content", {}) as server:
ruleset = {
"id": "test_provider_stop_many",
"stop": {
"host": server.host
}
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info(
"STEP: Send a stop request for multiple execution spaces.")
provider.checkin_all()
self.logger.info("STEP: Verify that the stop endpoint is called.")
self.assertEqual(server.nbr_of_requests, 1)
self.assertEqual(server.requests, [dict_execution_spaces])
async def start_etos(etos: StartEtosRequest):
"""Start ETOS execution on post.
:param etos: ETOS pydantic model.
:type etos: :obj:`etos_api.routers.etos.schemas.StartEtosRequest`
:return: JSON dictionary with response.
:rtype: dict
"""
await SuiteValidator().validate(etos.test_suite_url)
etos_library = ETOS("ETOS API", os.getenv("HOSTNAME"), "ETOS API")
await sync_to_async(etos_library.config.rabbitmq_publisher_from_environment
)
artifact = await wait_for_artifact_created(etos_library,
etos.artifact_identity)
if artifact is None:
raise HTTPException(
status_code=400,
detail=
f"Unable to find artifact with identity '{etos.artifact_identity}'",
)
# There are assumptions here. Since "edges" list is already tested
# and we know that the return from GraphQL must be 'node'.'meta'.'id'
# if there are "edges", this is fine.
artifact_id = artifact[0]["node"]["meta"]["id"]
links = {"CAUSE": artifact_id}
data = {
"selectionStrategy": {
"tracker": "Suite Builder",
"id": str(uuid4())
},
"batchesUri": etos.test_suite_url,
}
tercc = EiffelTestExecutionRecipeCollectionCreatedEvent()
request = ConfigureEnvironmentProviderRequest(
suite_id=tercc.meta.event_id,
dataset=etos.dataset,
execution_space_provider=etos.execution_space_provider,
iut_provider=etos.iut_provider,
log_area_provider=etos.log_area_provider,
)
await configure_environment_provider(request)
await sync_to_async(etos_library.start_publisher)
async with aclosing(etos_library.publisher):
event = etos_library.events.send(tercc, links, data)
return {
"tercc": event.meta.event_id,
"event_repository": etos_library.debug.graphql_server,
}
def test_provider_status_http_exceptions(self):
"""Test that the wait method handles HTTP errors.
Approvial criteria:
- The wait method shall raise ExecutionSpaceNotAvailable on 404 errors.
- The wait method shall raise RuntimeError on 400 errors.
Test steps::
1. For status [400, 404]:
1.1 Initialize an external provider.
1.2 Send a status request for a started execution space provider.
1.3 Verify that the wait method raises the correct exception.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_EXECUTION_SPACE_TIMEOUT", 1)
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
self.logger.info("STEP: For status [400, 404]:")
for status, exception in (
("bad_request", RuntimeError),
("not_found", ExecutionSpaceNotAvailable),
):
with FakeServer(status, {"error": "failure"}) as server:
ruleset = {
"id": "test_provider_status_http_exceptions",
"status": {
"host": server.host
},
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info(
"STEP: Send a status request for a started execution space provider."
)
with self.assertRaises(exception):
self.logger.info(
"STEP: Verify that the wait method raises the correct exception."
)
provider.wait("1")
def test_provider_status_pending(self):
"""Test that the wait method waits on status PENDING.
Approvial criteria:
- The wait method shall call the status endpoint, waiting on PENDING.
Test steps::
1. Initialize an external provider.
2. Send a status request for a started execution space provider.
3. Verify that the wait method waits on PENDING.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_EXECUTION_SPACE_TIMEOUT", 10)
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
responses = [{
"status": "PENDING"
}, {
"status": "PENDING"
}, {
"status": "DONE"
}]
with FakeServer("ok", responses.copy()) as server:
ruleset = {
"id": "test_provider_status_pending",
"status": {
"host": server.host
},
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info(
"STEP: Send a status request for a started execution space provider."
)
provider.wait("1")
self.logger.info(
"STEP: Verify that the wait method waits on PENDING.")
self.assertEqual(server.nbr_of_requests, len(responses))
def test_provider_status_failed(self):
"""Test that the wait method raises ExecutionSpaceCheckoutFailed on FAILED status.
Approvial criteria:
- The wait method shall raise ExecutionSpaceCheckoutFailed on a FAILED status.
Test steps::
1. Initialize an external provider.
2. Send a status request for a started execution space provider.
3. Verify that the wait method raises ExecutionSpaceCheckoutFailed.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_EXECUTION_SPACE_TIMEOUT", 1)
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
description = "something failed!"
with FakeServer("ok", {
"status": "FAILED",
"description": description
}) as server:
ruleset = {
"id": "test_provider_status_failed",
"status": {
"host": server.host
},
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info(
"STEP: Send a status request for a started execution space provider."
)
with self.assertRaises(ExecutionSpaceCheckoutFailed):
self.logger.info(
"STEP: Verify that the wait method raises ExecutionSpaceCheckoutFailed."
)
provider.wait("1")
def test_provider_status(self):
"""Test that the wait method waits for status DONE and exits with response.
Approvial criteria:
- The wait method shall call the status endpoint and return on DONE.
Test steps::
1. Initialize an external provider.
2. Send a status request for a started execution space provider.
3. Verify that the wait method returns response on DONE.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_EXECUTION_SPACE_TIMEOUT", 1)
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
test_id = "123"
with FakeServer("ok", {
"status": "DONE",
"test_id": test_id
}) as server:
ruleset = {
"id": "test_provider_status",
"status": {
"host": server.host
}
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info(
"STEP: Send a status request for a started execution space provider."
)
response = provider.wait("1")
self.logger.info(
"STEP: Verify that the wait method return response on DONE.")
self.assertEqual(response.get("test_id"), test_id)
def test_provider_start_http_exception(self):
"""Test that the start method tries again if there's an HTTP error.
Approval criteria:
- The start method shall try again on HTTP errors.
Test steps::
1. Initialize an external provider.
2. Send a start request that fails.
3. Verify that the start method tries again on HTTP errors.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
expected_start_id = "123"
with FakeServer(["bad_request", "ok"], [{}, {
"id": expected_start_id
}]) as server:
ruleset = {
"id": "test_provider_start_http_exception",
"start": {
"host": server.host
},
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info("STEP: Send a start request that fails.")
start_id = provider.start(1, 2)
self.logger.info(
"STEP: Verify that the start method tries again on HTTP errors."
)
self.assertGreaterEqual(server.nbr_of_requests, 2)
self.assertEqual(start_id, expected_start_id)
def test_provider_stop_failed(self):
"""Test that the checkin method raises an ExecutionSpaceCheckinFailed exception.
Approval criteria:
- The checkin method shall fail with an ExecutionSpaceCheckinFailed exception.
Test steps::
1. Initialize an external provider.
2. Send a stop request that fails.
3. Verify that the checkin method raises ExecutionSpaceCheckinFailed exception.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_EXECUTION_SPACE_TIMEOUT", 1)
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
execution_space = ExecutionSpace(test_execution_space=1)
with FakeServer("bad_request", {"error": "no"}) as server:
ruleset = {
"id": "test_provider_stop_failed",
"stop": {
"host": server.host
}
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info("STEP: Send a stop request that fails.")
with self.assertRaises(ExecutionSpaceCheckinFailed):
self.logger.info(
"STEP: Verify that the checkin method raises ExecutionSpaceCheckinFailed "
"exception.")
provider.checkin(execution_space)
def test_provider_start_timeout(self):
"""Test that the start method raises a TimeoutError.
Approval criteria:
- The start method shall raise TimeoutError if the timeout is reached.
Test steps::
1. Initialize an external provider.
2. Send a start request which will never finish.
3. Verify that the start method raises TimeoutError.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
os.environ["ETOS_DEFAULT_HTTP_TIMEOUT"] = "1"
with FakeServer("bad_request", {}) as server:
ruleset = {
"id": "test_provider_start_timeout",
"start": {
"host": server.host
},
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info(
"STEP: Send a start request which will never finish.")
with self.assertRaises(TimeoutError):
self.logger.info(
"STEP: Verify that the start method raises TimeoutError.")
provider.start(1, 2)
def test_provider_stop(self):
"""Test that it is possible to checkin an external IUT provider.
Approval criteria:
- It shall be possible to send stop to an external IUT provider.
Test steps::
1. Initialize an external provider.
2. Send a stop request for a single IUT.
3. Verify that the stop endpoint is called.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_IUT_TIMEOUT", 1)
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
iut = Iut(test_iut=1)
with FakeServer("no_content", {}) as server:
ruleset = {
"id": "test_provider_stop",
"stop": {
"host": server.host
}
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info("STEP: Send a stop request for a single IUT.")
provider.checkin(iut)
self.logger.info("STEP: Verify that the stop endpoint is called.")
self.assertEqual(server.nbr_of_requests, 1)
self.assertEqual(server.requests, [[iut.as_dict]])
def test_provider_stop_timeout(self):
"""Test that the checkin method raises a TimeoutError when timed out.
Approval criteria:
- The checkin method shall raise TimeoutError when timed out.
Test steps::
1. Initialize an external provider.
2. Send a stop request for IUTs that times out.
3. Verify that the checkin method raises a TimeoutError.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_IUT_TIMEOUT", 1)
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
iut = Iut(test_iut=1)
with FakeServer("bad_request", {}) as server:
ruleset = {
"id": "test_provider_stop_timeout",
"stop": {
"host": server.host
},
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info("STEP: Send a stop request that fails.")
with self.assertRaises(TimeoutError):
self.logger.info(
"STEP: Verify that the checkin method raises a TimeoutError."
)
provider.checkin(iut)
def test_provider_start(self):
"""Test that it is possible to start an external execution space provider.
Approval criteria:
- It shall be possible to send start to an external execution space provider.
Test steps::
1. Initialize an external provider.
2. Send a start request.
3. Verify that the ID from the start request is returned.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
expected_start_id = "123"
with FakeServer("ok", {"id": expected_start_id}) as server:
ruleset = {
"id": "test_provider_start",
"start": {
"host": server.host
}
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info("STEP: Send a start request.")
start_id = provider.start(1, 2)
self.logger.info(
"STEP: Verify that the ID from the start request is returned.")
self.assertEqual(start_id, expected_start_id)
def test_provider_status_timeout(self):
"""Test that the wait method raises TimeoutError when timed out.
Approvial criteria:
- The wait method shall raise TimeoutError when timed out.
Test steps::
1. Initialize an external provider.
2. Send a status request that times out.
3. Verify that the wait method raises TimeoutError.
"""
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
etos.config.set("WAIT_FOR_EXECUTION_SPACE_TIMEOUT", 1)
jsontas = JsonTas()
jsontas.dataset.merge({
"identity":
PackageURL.from_string("pkg:testing/etos"),
"artifact_id":
"artifactid",
"artifact_created":
"artifactcreated",
"artifact_published":
"artifactpublished",
"tercc":
"tercc",
"dataset": {},
"context":
"context",
})
with FakeServer("internal_server_error", {}) as server:
ruleset = {
"id": "test_provider_status_timeout",
"status": {
"host": server.host
},
}
self.logger.info("STEP: Initialize an external provider.")
provider = Provider(etos, jsontas, ruleset)
self.logger.info("STEP: Send a status request that times out.")
with self.assertRaises(TimeoutError):
self.logger.info(
"STEP: Verify that the wait method raises TimeoutError.")
provider.wait("1")
def test_register_provider_none(self):
"""Test that the register backend return false if no provider is supplied.
Approval criteria:
- The register backend shall return False if no provider is supplied.
Test steps:
1. Register no provider with the register backend.
2. Verify that the register backend return False.
"""
fake_database = FakeDatabase()
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
jsontas = JsonTas()
provider_registry = ProviderRegistry(etos, jsontas, fake_database)
self.logger.info("STEP: Register no provider with the register backend.")
response = register(provider_registry)
self.logger.info("STEP: Verify that the register backend return False.")
self.assertFalse(response)
async def configure_environment_provider(
environment: ConfigureEnvironmentProviderRequest,
):
"""Configure environment provider request.
:param environment: Environment to configure.
:type environment: :obj:`etos_api.routers.etos.schemas.ConfigureEnvironmentProviderRequest`
"""
LOGGER.identifier.set(environment.suite_id)
LOGGER.info("Configuring environment provider using %r", environment)
etos_library = ETOS("ETOS API", os.getenv("HOSTNAME"), "ETOS API")
end_time = time.time() + etos_library.debug.default_http_timeout
LOGGER.debug("HTTP Timeout: %r", etos_library.debug.default_http_timeout)
async with aiohttp.ClientSession() as session:
while time.time() < end_time:
try:
async with session.post(
f"{etos_library.debug.environment_provider}/configure",
json=environment.dict(),
headers={
"Content-Type": "application/json",
"Accept": "application/json",
},
) as response:
assert 200 <= response.status < 400
break
except AssertionError:
LOGGER.warning(
"Configuration request failed: %r, %r",
response.status,
response.reason,
)
await asyncio.sleep(2)
else:
raise HTTPException(
status_code=400,
detail=f"Unable to configure environment provider with '{environment.json()}'",
)
await _wait_for_configuration(etos_library, environment)
def test_configure_missing_parameter(self):
"""Test that the configure backend does not configure if any parameter is missing.
Approval criteria:
- The configure backend shall return False and not configure if any parameter
is missing.
Test steps:
1. Attempt to configure the environment provider without any parameters.
2. Verify that False was returned and no configuration was made.
"""
database = FakeDatabase()
self.logger.info(
"STEP: Attempt to configure the environment provider without any parameters."
)
registry = ProviderRegistry(ETOS("", "", ""), JsonTas(), database)
success, _ = configure(registry, None, None, None, None, None)
self.logger.info(
"STEP: Verify that False was returned and no configuration was made."
)
self.assertFalse(success)
self.assertDictEqual(database.db_dict, {})
def test_register_execution_space_provider(self):
"""Test that the register backend can register execution space providers.
Approval criteria:
- The register backend shall be able to register an execution space provider.
Test steps:
1. Register an execution space provider with the register backend.
2. Verify that the execution space provider was stored in the database.
"""
fake_database = FakeDatabase()
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
jsontas = JsonTas()
provider = {
"execution_space": {
"id": "execution_space_provider_test",
"list": {"available": [{"identifier": "123"}], "possible": []},
}
}
provider_registry = ProviderRegistry(etos, jsontas, fake_database)
self.logger.info(
"STEP: Register an execution space provider with the register backend."
)
response = register(provider_registry, execution_space_provider=provider)
self.logger.info(
"STEP: Verify that the execution space provider was stored in the database."
)
stored_provider = json.loads(
fake_database.reader.hget(
"EnvironmentProvider:ExecutionSpaceProviders",
"execution_space_provider_test",
)
)
self.assertDictEqual(stored_provider, provider)
self.assertTrue(response)
class BaseTester:
"""Base tester API."""
__artifact_id = None
__gql_response = None
def __init__(self, params):
"""Initialize.
:param params: Parameter data storage for Tester APIs.
:type params: :obj:`lib.params.Params`
"""
self.params = params
self.etos = ETOS("ETOS API", os.getenv("HOSTNAME"), "ETOS API")
self.etos.config.rabbitmq_publisher_from_environment()
self.etos.start_publisher()
def configure_environment_provider(self, suite_id):
"""Configure the ETOS environment provider for a suite ID.
:param suite_id: Suite ID to configure the environment provider for.
:type suite_id: str
"""
params = {
"suite_id": suite_id,
"iut_provider": self.params.iut_provider,
"execution_space_provider": self.params.execution_space_provider,
"log_area_provider": self.params.log_area_provider,
"dataset": self.params.dataset,
}
generator = self.etos.http.retry(
"POST",
"{}/configure".format(self.etos.debug.environment_provider),
as_json=False,
json=params,
)
for response in generator:
print(response)
break
else:
raise Exception(
"Could not configure the Environment Provider with %r" %
params)
def handle(self): # pylint: disable=duplicate-code
"""Handle this tester."""
raise NotImplementedError
@property
def gql_response(self):
"""GQL response for artifact created and published events."""
if self.__gql_response is None:
request = "search: \"{{'data.identity': {{'$regex': '{}'}}}}\"".format(
self.params.artifact_identity)
wait_generator = self.etos.utils.wait(self.etos.graphql.execute,
query=ARTIFACT_REQUEST %
request)
for response in wait_generator:
self.__gql_response = response
break
return self.__gql_response
@property
def artifact_id(self):
"""Figure out Artifact event ID from event storage."""
if self.__artifact_id is None:
try:
artifact_node = self.gql_response["artifactCreated"]["edges"][
0]["node"]
self.__artifact_id = artifact_node["meta"]["id"]
except (KeyError, IndexError):
pass
return self.__artifact_id
def test_register_all_providers(self):
"""Test that the register backend can register all providers.
Approval criteria:
- The register backend shall be able to register all providers.
Test steps:
1. Register one of each provider with the register backend.
2. Verify that the providers were stored in the database.
"""
fake_database = FakeDatabase()
etos = ETOS("testing_etos", "testing_etos", "testing_etos")
jsontas = JsonTas()
test_iut_provider = {
"iut": {
"id": "iut_provider_test",
"list": {"available": [], "possible": []},
}
}
test_execution_space_provider = {
"execution_space": {
"id": "execution_space_provider_test",
"list": {"available": [{"identifier": "123"}], "possible": []},
}
}
test_log_area_provider = {
"log": {
"id": "log_area_provider_test",
"list": {"available": [], "possible": []},
}
}
provider_registry = ProviderRegistry(etos, jsontas, fake_database)
self.logger.info(
"STEP: Register one of each provider with the register backend."
)
response = register(
provider_registry,
iut_provider=test_iut_provider,
log_area_provider=test_log_area_provider,
execution_space_provider=test_execution_space_provider,
)
self.logger.info("STEP: Verify that the providers were stored in the database.")
stored_execution_space_provider = json.loads(
fake_database.reader.hget(
"EnvironmentProvider:ExecutionSpaceProviders",
"execution_space_provider_test",
)
)
self.assertDictEqual(
stored_execution_space_provider,
test_execution_space_provider,
)
stored_log_area_provider = json.loads(
fake_database.reader.hget(
"EnvironmentProvider:LogAreaProviders", "log_area_provider_test"
)
)
self.assertDictEqual(stored_log_area_provider, test_log_area_provider)
stored_iut_provider = json.loads(
fake_database.reader.hget(
"EnvironmentProvider:IUTProviders", "iut_provider_test"
)
)
self.assertDictEqual(stored_iut_provider, test_iut_provider)
self.assertTrue(response)
class ETR:
"""ETOS Test Runner."""
context = None
def __init__(self):
"""Initialize ETOS library and start eiffel publisher."""
self.etos = ETOS("ETOS Test Runner", os.getenv("HOSTNAME"),
"ETOS Test Runner")
self.etos.config.rabbitmq_publisher_from_environment()
# ETR will print the entire environment just before executing.
# Hide the password.
os.environ["RABBITMQ_PASSWORD"] = "******"
self.etos.start_publisher()
self.tests_url = os.getenv("SUB_SUITE_URL")
signal.signal(signal.SIGTERM, self.graceful_shutdown)
@staticmethod
def graceful_shutdown(*args):
"""Catch sigterm."""
raise Exception("ETR has been terminated.")
def download_and_load(self):
"""Download and load test json."""
generator = self.etos.http.wait_for_request(self.tests_url)
for response in generator:
json_config = response
break
self.etos.config.set("test_config", json_config)
self.etos.config.set("context", json_config.get("context"))
self.etos.config.set("artifact", json_config.get("artifact"))
def _run_tests(self):
"""Run tests in ETOS test runner.
:return: Results of test runner execution.
:rtype: bool
"""
iut = Iut(self.etos.config.get("test_config").get("iut"))
test_runner = TestRunner(iut, self.etos)
return test_runner.execute()
def run_etr(self):
"""Send activity events and run ETR.
:return: Result of testrunner execution.
:rtype: bool
"""
_LOGGER.info("Starting ETR.")
self.download_and_load()
try:
activity_name = self.etos.config.get("test_config").get("name")
triggered = self.etos.events.send_activity_triggered(activity_name)
self.etos.events.send_activity_started(triggered)
result = self._run_tests()
except Exception as exc: # pylint:disable=broad-except
self.etos.events.send_activity_finished(triggered, {
"conclusion": "FAILED",
"description": str(exc)
})
raise
self.etos.events.send_activity_finished(triggered,
{"conclusion": "SUCCESSFUL"})
_LOGGER.info("ETR finished.")
return result
def test_get_configuration(self):
"""Test that it is possible to get a stored configuration.
Approval criteria:
- It shall be possible to get a stored configuration.
Test steps:
1. Store a configuration into the database.
2. Verify that it is possible to get the stored configuration.
"""
database = FakeDatabase()
test_suite_id = "8d9344e3-a246-43ec-92b4-fc81ea31067a"
test_dataset = {"dataset": "test"}
test_iut_provider = OrderedDict({
"iut": {
"id": "iut_provider_test",
"list": {
"available": [],
"possible": []
},
}
})
test_execution_space_provider = OrderedDict({
"execution_space": {
"id": "execution_space_provider_test",
"list": {
"available": [{
"identifier": "123"
}],
"possible": []
},
}
})
test_log_area_provider = OrderedDict({
"log": {
"id": "log_area_provider_test",
"list": {
"available": [],
"possible": []
},
}
})
self.logger.info("STEP: Store a configuration into the database.")
database.writer.hset(f"EnvironmentProvider:{test_suite_id}", "Dataset",
json.dumps(test_dataset))
database.writer.hset(
f"EnvironmentProvider:{test_suite_id}",
"IUTProvider",
json.dumps(test_iut_provider),
)
database.writer.hset(
f"EnvironmentProvider:{test_suite_id}",
"ExecutionSpaceProvider",
json.dumps(test_execution_space_provider),
)
database.writer.hset(
f"EnvironmentProvider:{test_suite_id}",
"LogAreaProvider",
json.dumps(test_log_area_provider),
)
self.logger.info(
"STEP: Verify that it is possible to get the stored configuration."
)
registry = ProviderRegistry(ETOS("", "", ""), JsonTas(), database)
stored_configuration = get_configuration(registry, test_suite_id)
self.assertDictEqual(
stored_configuration,
{
"iut_provider":
test_iut_provider["iut"],
"execution_space_provider":
test_execution_space_provider["execution_space"],
"log_area_provider":
test_log_area_provider["log"],
"dataset":
test_dataset,
},
)
