async def instrumentation_message_handler(message: aio_pika.IncomingMessage,
app: web.Application) -> None:
data = json.loads(message.body)
if data["metrics"] == "service_started":
service_started(app,
**{key: data[key]
for key in SERVICE_STARTED_LABELS})
elif data["metrics"] == "service_stopped":
service_stopped(app,
**{key: data[key]
for key in SERVICE_STOPPED_LABELS})
await message.ack()
async def instrumentation_message_handler(message: aio_pika.IncomingMessage,
app: web.Application) -> None:
data = json.loads(message.body)
if data["metrics"] == "service_started":
service_started(
app,
**{key: value
for key, value in data.items() if key != "metrics"})
elif data["metrics"] == "service_stopped":
service_stopped(
app,
**{key: value
for key, value in data.items() if key != "metrics"})
await message.ack()
async def stop_service(app: web.Application, node_uuid: str,
save_state: bool) -> None:
log.debug("stopping service with uuid %s", node_uuid)
# get the docker client
async with docker_utils.docker_client() as client: # pylint: disable=not-async-context-manager
try:
list_running_services_with_uuid = await client.services.list(
filters={
"label": [
f"uuid={node_uuid}",
f"swarm_stack_name={config.SWARM_STACK_NAME}",
]
})
except aiodocker.exceptions.DockerError as err:
log.exception("Error while stopping container with uuid: %s",
node_uuid)
raise exceptions.GenericDockerError(
"Error while stopping container", err) from err
# error if no service with such an id exists
if not list_running_services_with_uuid:
raise exceptions.ServiceUUIDNotFoundError(node_uuid)
log.debug("found service(s) with uuid %s",
list_running_services_with_uuid)
# save the state of the main service if it can
service_details = await get_service_details(app, node_uuid)
# FIXME: the exception for the 3d-viewer shall be removed once the dy-sidecar comes in
service_host_name = "{}:{}{}".format(
service_details["service_host"],
service_details["service_port"]
if service_details["service_port"] else "80",
service_details["service_basepath"]
if not "3d-viewer" in service_details["service_host"] else "",
)
log.debug("saving state of service %s...", service_host_name)
if save_state:
try:
session = app[APP_CLIENT_SESSION_KEY]
service_url = "http://" + service_host_name + "/" + "state"
async with session.post(
service_url,
timeout=ServicesCommonSettings().
director_dynamic_service_save_timeout,
) as response:
if 199 < response.status < 300:
log.debug("service %s successfully saved its state",
service_host_name)
else:
log.warning(
"service %s does not allow saving state, answered %s",
service_host_name,
await response.text(),
)
except ClientConnectionError:
log.warning(
"service %s could not be contacted, state not saved",
service_host_name,
)
# remove the services
try:
log.debug("removing services...")
for service in list_running_services_with_uuid:
await client.services.delete(service["Spec"]["Name"])
log.debug("removed services, now removing network...")
except aiodocker.exceptions.DockerError as err:
raise exceptions.GenericDockerError(
"Error while removing services", err) from err
# remove network(s)
await _remove_overlay_network_of_swarm(client, node_uuid)
log.debug("removed network")
if config.MONITORING_ENABLED:
service_stopped(
app,
"undefined_user",
service_details["service_key"],
service_details["service_version"],
"DYNAMIC",
"SUCCESS",
)
async def stop_service(app: web.Application, node_uuid: str,
save_state: bool) -> None:
log.debug("stopping service with node_uuid=%s, save_state=%s", node_uuid,
save_state)
# get the docker client
async with docker_utils.docker_client() as client: # pylint: disable=not-async-context-manager
try:
list_running_services_with_uuid = await client.services.list(
filters={
"label": [
f"uuid={node_uuid}",
f"swarm_stack_name={config.SWARM_STACK_NAME}",
]
})
except aiodocker.exceptions.DockerError as err:
log.exception("Error while stopping container with uuid: %s",
node_uuid)
raise exceptions.GenericDockerError(
"Error while stopping container", err) from err
# error if no service with such an id exists
if not list_running_services_with_uuid:
raise exceptions.ServiceUUIDNotFoundError(node_uuid)
log.debug("found service(s) with uuid %s",
list_running_services_with_uuid)
# save the state of the main service if it can
service_details = await get_service_details(app, node_uuid)
# FIXME: the exception for the 3d-viewer shall be removed once the dy-sidecar comes in
service_host_name = "{}:{}{}".format(
service_details["service_host"],
service_details["service_port"]
if service_details["service_port"] else "80",
service_details["service_basepath"]
if not "3d-viewer" in service_details["service_host"] else "",
)
# If state save is enforced
if save_state:
log.debug("saving state of service %s...", service_host_name)
try:
await _save_service_state(service_host_name,
session=app[APP_CLIENT_SESSION_KEY])
except ClientResponseError as err:
raise ServiceStateSaveError(
node_uuid,
reason=
f"service {service_host_name} rejected to save state, "
f"responded {err.message} (status {err.status})."
"Aborting stop service to prevent data loss.",
) from err
except ClientError as err:
log.warning(
"Could not save state because {service_host_name} is unreachable [{err}]."
"Resuming stop_service.")
# remove the services
try:
log.debug("removing services ...")
for service in list_running_services_with_uuid:
log.debug("removing %s", service["Spec"]["Name"])
await client.services.delete(service["Spec"]["Name"])
except aiodocker.exceptions.DockerError as err:
raise exceptions.GenericDockerError(
"Error while removing services", err) from err
# remove network(s)
log.debug("removed services, now removing network...")
await _remove_overlay_network_of_swarm(client, node_uuid)
log.debug("removed network")
if config.MONITORING_ENABLED:
service_stopped(
app,
"undefined_user",
service_details["service_key"],
service_details["service_version"],
"DYNAMIC",
"SUCCESS",
)