def test_get_options(client: Client) -> None:
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:8761/eureka/apps/ECHO",
status=404,
)
with pytest.raises(ServiceNotFoundError, match="No service named <ECHO>"):
client.get_options("ECHO")
def test_transform_document_pipe_no_service(client: Client) -> None:
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:8761/eureka/apps/ECHO",
status=404,
)
with pytest.raises(ServiceNotFoundError, match="No service named <ECHO>"):
client.transform_document_pipe(request=TransformDocumentPipeRequest(
document="Hello World",
services=[PipeService(name="ECHO"),
PipeService(name="COUNT")],
))
def __init__(
self,
work: Optional[Worker],
config: "ServiceConfig",
options_type: Optional[Type[BaseModel]],
client: Optional[Client] = None,
) -> None:
self._work = work
self.config = config
if client is None:
self.client = Client(ClientConfig(registrar_url=config.registrar_url))
else:
self.client = client
self.options_type = options_type
log.info("initialized abc worker.")
def test_rest_transform_pipe(eureka_server):
print("start pipe test")
config = ClientConfig("http://127.0.0.1:8761/eureka")
client = Client(config)
response = client.transform_document_pipe(
TransformDocumentPipeRequest(
document="Hello World",
services=[
PipeService(name="TEST1"),
PipeService(name="TEST2"),
PipeService(name="TEST3"),
],
))
assert response.document == "Hello World,TEST1,TEST2,TEST3"
assert response.last_transformer == "TEST3"
def test_transform_document_pipe_with_instance_address(client: Client) -> None:
httpretty.register_uri(
method=httpretty.POST,
uri="http://localhost:50000/v1/document/transform-pipe",
body=json.dumps({
"document":
"Hello World!!",
"output": ["transforming document <simpletext.txt> ...", "lol"],
"error": ["Unknown Exceptions"],
"last_transformer":
"COUNT",
}),
status=200,
)
response = client.transform_document_pipe(
request=TransformDocumentPipeRequest(
document="Hello World",
services=[PipeService(name="ECHO"),
PipeService(name="COUNT")],
),
instance_address="localhost:50000",
)
assert response.document == "Hello World!!"
assert response.last_transformer == "COUNT"
assert not response.output is None
assert not response.error is None
assert len(response.output) == 2
assert len(response.error) == 1
def test_get_options_with_instance_address(client: Client) -> None:
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:50000/v1/service/options",
body=json.dumps({
"properties": {
"offset": {
"default": 0,
"title": "Offset",
"type": "string"
}
},
"title": "Options",
"type": "object",
}),
status=200,
)
response = client.get_options("ECHO", instance_address="localhost:50000")
assert response == {
"properties": {
"offset": {
"default": 0,
"title": "Offset",
"type": "string"
}
},
"title": "Options",
"type": "object",
}
def test_transform_document_pipe(client: Client) -> None:
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:8761/eureka/apps/ECHO",
body=EUREKA_GET_APPS_ECHO_RESPONSE,
status=200,
)
httpretty.register_uri(
method=httpretty.POST,
uri="http://localhost:50000/v1/document/transform-pipe",
body=json.dumps({
"document":
"Hello Worlds!",
"output": ["transforming document <simpletext.txt> ...", "lol"],
"error": ["Unknown Exceptions"],
"last_transformer":
"COUNT",
}),
status=200,
)
response = client.transform_document_pipe(
request=TransformDocumentPipeRequest(
document="Hello World",
services=[PipeService(name="ECHO"),
PipeService(name="COUNT")],
))
assert response.document == "Hello Worlds!"
assert response.last_transformer == "COUNT"
assert not response.output is None
assert not response.error is None
assert len(response.output) == 2
assert len(response.error) == 1
def test_transform_document_with_instance_address(client: Client) -> None:
httpretty.register_uri(
method=httpretty.POST,
uri="http://localhost:50000/v1/document/transform",
body=json.dumps({
"document":
"Hello Worlds!",
"output": ["transforming document <simpletext.txt> ...", "lol"],
"error": ["Unknown Exceptions"],
}),
status=200,
)
response = client.transform_document(
TransformDocumentRequest(
document="Hello world!",
service_name="ECHO",
file_name="simpletext.txt",
options={
"offset": 5,
"debug": True
},
),
instance_address="localhost:50000",
)
assert not response is None
assert response.document == "Hello Worlds!"
assert not response.output is None
assert len(response.output) == 2
assert response.output == [
"transforming document <simpletext.txt> ...", "lol"
]
assert response.error == ["Unknown Exceptions"]
def test_transform_document_pipe_with_instance_address_no_service(
monkeypatch, client: Client) -> None:
monkeypatch.setattr("morpho.client.requests.post", raise_connection_error)
with pytest.raises(requests.exceptions.ConnectionError):
client.transform_document(
TransformDocumentRequest(
document="Hello world!",
service_name="ECHO",
file_name="simpletext.txt",
options={
"offset": 5,
"debug": True
},
),
instance_address="localhost:50000",
)
def test_transform_document_no_service(client: Client) -> None:
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:8761/eureka/apps/ECHO",
status=404,
)
with pytest.raises(ServiceNotFoundError, match="No service named <ECHO>"):
client.transform_document(
TransformDocumentRequest(
document="Hello world!",
service_name="ECHO",
file_name="simpletext.txt",
options={
"offset": 5,
"debug": True
},
))
def test_transform_document_with_instance_address_no_service(
monkeypatch) -> None:
config = ClientConfig(registrar_url="http://localhost:8761/eureka")
client = Client(config)
monkeypatch.setattr("morpho.client.requests.post", raise_connection_error)
with pytest.raises(requests.exceptions.ConnectionError):
client.transform_document(
TransformDocumentRequest(
document="Hello world!",
service_name="ECHO",
file_name="simpletext.txt",
options={
"offset": 5,
"debug": True
},
),
instance_address="localhost:50000",
)
def test_list_services_with_instance_address(client: Client) -> None:
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:50000/v1/service/list",
body=json.dumps({"services": [{
"name": "ECHO"
}]}),
status=200,
)
response = client.list_services("ECHO", instance_address="localhost:50000")
assert len(response.services) == 1
service = response.services[0]
assert service.name == "ECHO"
def test_list_services(client: Client) -> None:
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:8761/eureka/apps/ECHO",
body=EUREKA_GET_APPS_ECHO_RESPONSE,
status=200,
)
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:50000/v1/service/list",
body=json.dumps({"services": [{
"name": "ECHO"
}]}),
status=200,
)
response = client.list_services("ECHO")
assert len(response.services) == 1
service = response.services[0]
assert service.name == "ECHO"
def __init__(
self,
work: Optional[Worker],
config: ServiceConfig,
options_type: Optional[Type[BaseModel]],
) -> None:
# consumers are only invoked with ServiceConfig
# so we need to cast to the RestGatewayServiceConfig
super().__init__(
work=work,
config=config,
options_type=options_type,
client=Client(
ClientConfig(
registrar_url=cast(RestGatewayServiceConfig, config).resolver_url
)
),
)
# TODO: somehow remove one of the config instances
# self.gateway_config = cast(config
# TODO: discuss: set implicit the default type to Gateway
self.config.type = DtaType.GATEWAY
def test_get_options(client: Client) -> None:
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:8761/eureka/apps/ECHO",
body=EUREKA_GET_APPS_ECHO_RESPONSE,
status=200,
)
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:50000/v1/service/options",
body=json.dumps({
"properties": {
"offset": {
"default": 0,
"title": "Offset",
"type": "integer"
}
},
"title": "Options",
"type": "object",
}),
status=200,
)
response = client.get_options("ECHO")
assert response == {
"properties": {
"offset": {
"default": 0,
"title": "Offset",
"type": "integer"
}
},
"title": "Options",
"type": "object",
}
def test_transform_document(client: Client) -> None:
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:8761/eureka/apps/ECHO",
body=EUREKA_GET_APPS_ECHO_RESPONSE,
status=200,
)
httpretty.register_uri(
method=httpretty.POST,
uri="http://localhost:50000/v1/document/transform",
body=json.dumps({
"document":
"Hello World!",
"output": ["transforming document <simpletext.txt> ..."],
"error": ["Unknown Exception"],
}),
status=200,
)
response = client.transform_document(
TransformDocumentRequest(
document="Hello world!",
service_name="ECHO",
file_name="simpletext.txt",
options={
"offset": 5,
"debug": True
},
))
assert not response is None
assert response.document == "Hello World!"
assert not response.output is None
assert len(response.output) == 1
assert response.output == ["transforming document <simpletext.txt> ..."]
assert response.error == ["Unknown Exception"]
from morpho.rest.models import TransformDocumentRequest
from morpho.client import Client
from morpho.client import ClientConfig
morpho = Client(ClientConfig("http://localhost:8761/eureka/"))
request = TransformDocumentRequest(document="This is a Document!",
service_name="Echo")
response = morpho.transform_document(request=request)
class WorkConsumer(ABC):
"""An abstract class which must be implemented by each ``work`` consumer.
Attributes:
work (Callable[[str], str]): Worker function which will executed to get the
transformed document.
config (ServiceConfig): Configuration for the given Server.
Note:
The ``work`` callback should be called once on a implemented work consumer after the
server received the request. After receiving the request the document should be already
be correctly marshalled.
"""
def __init__(
self,
work: Optional[Worker],
config: "ServiceConfig",
options_type: Optional[Type[BaseModel]],
client: Optional[Client] = None,
) -> None:
self._work = work
self.config = config
if client is None:
self.client = Client(ClientConfig(registrar_url=config.registrar_url))
else:
self.client = client
self.options_type = options_type
log.info("initialized abc worker.")
def _get_applications(self) -> Applications:
try:
return eureka_client.get_applications(self.config.registrar_url)
# TODO: add custom eureka not found error
except URLError:
log.error("no eureka instance is running at: %s", self.config.registrar_url)
exit(1)
def health(self) -> Health:
return self.config.health
def list_services(self) -> ListServicesResponse:
"""Lists all services from the eureka server of the provided ``ServiceConfig``.
Returns:
List[ListServicesResponse]: List of services.
"""
# return the service itself if the service is not registered at eureka
if not self.config.should_register:
return ListServicesResponse(
services=[
ServiceInfo(name=self.config.name.upper(), options=self.options())
]
)
applications = self._get_applications()
cached_applications: List[Tuple[str, str]] = []
# check if we can find a available gateway
for service in applications.applications:
instance = service.instances[0]
# skip self
if instance.app == self.config.name.upper():
continue
instance_address = f"{instance.ipAddr}:{instance.port.port}"
if instance.metadata:
try:
dta_type = DtaType(instance.metadata.get("dtaType"))
except ValueError:
dta_type = DtaType.UNKNOWN
if dta_type == dta_type.GATEWAY:
log.info(
"found gateway send list request directly to <%s (%s)>",
instance.app,
instance_address,
)
# TODO: gateways could cache the whole list of services
return self.client.list_services(
instance.app, instance_address=instance_address
)
# cache the list of instance_addresses
cached_applications.append((instance.app, instance_address))
services = [ServiceInfo(name=self.config.name.upper(), options=self.options())]
# iterate over cached applications and create ListServicesResponse
for cached_application in cached_applications:
app_name, instance_address = cached_application
# get options from service via rest call
options = self.client.get_options(
service_name=app_name, instance_address=instance_address
)
services.append(ServiceInfo(name=app_name, options=options))
return ListServicesResponse(services=services)
def options(self) -> Schema:
"""Lists available options of the service.
Returns:
Schema: A Schema representing the different available options. Returns an empty dictionary `{}` if no option is present.
"""
if self.options_type is None:
return {}
return self.options_type.schema()
def transform_document(
self, request: TransformDocumentRequest,
) -> TransformDocumentResponse:
document = None
# TODO: create a decorator for capturing stdout and stderr
# TODO: consider to move this into base class
captured_stdout = io.StringIO()
captured_stderr = io.StringIO()
with redirect_stderr(captured_stderr):
with redirect_stdout(captured_stdout):
options = (
self.options_type(**request.options) if self.options_type else None
)
if self._work is None:
raise NoWorkerFunctionError("No worker function specified!")
try:
if options is None:
document = self._work(request.document)
else:
document = self._work(request.document, options)
except BaseException: # pylint: disable=broad-except
traceback.print_exc()
error = captured_stderr.getvalue().splitlines()
output = captured_stdout.getvalue().splitlines()
captured_stderr.close()
captured_stdout.close()
log.info("-- after transform document --")
log.info("document: %s", document)
log.info("error: %s", error)
log.info("output: %s", output)
# TODO: test on none return type (if an error occurs) so the error
# will be still be transferred to the client
return TransformDocumentResponse(document=document, output=output, error=error)
def transform_document_pipe(
self, request: TransformDocumentPipeRequest
) -> TransformDocumentPipeResponse:
log.info("transform document pipe was called with: <%s>", request.document)
pipe_service = request.services.pop(0)
transform_response = self.transform_document(
TransformDocumentRequest(
document=request.document,
service_name=pipe_service.name,
file_name=request.file_name,
options=pipe_service.options,
)
)
# return a response if there are only 1 service left
# this means that the pipe is either finished or will
# be interupted because of an occured error
if request.services == [] or transform_response.error:
log.info(
"reached end of pipe returning response: %s",
transform_response.document,
)
return TransformDocumentPipeResponse(
document=transform_response.document,
output=transform_response.output,
last_transformer=self.config.name,
error=transform_response.error,
)
request.document = transform_response.document
response = self.client.transform_document_pipe(request=request)
# cancat outputs in the right order
response.output = transform_response.output + response.output
response.error = transform_response.error + response.error
return response
@abstractmethod
def start(self) -> None:
"""starts the implemented server instance.
Raises:
NotImplementedError: Must be implemented on the derived class.
Hint:
Can be used to instantiate a new ``Thread`` for the listening server of the Server.
Creating a new Thread with ``daemon=True`` helps the thread to destroy its self
which will then gracefully shutdown if the Server gets terminated
(preventing zombie threads).
Important:
This function should not block.
"""
raise NotImplementedError
def test_get_options_with_instance_address_no_service(monkeypatch,
client: Client) -> None:
monkeypatch.setattr("morpho.client.requests.get", raise_connection_error)
with pytest.raises(requests.exceptions.ConnectionError):
client.get_options("ECHO", instance_address="localhost:50000")
def client():
config = ClientConfig(registrar_url="http://localhost:8761/eureka")
yield Client(config)