def __init__(self, name,
local_address, frontend_address, user_engine,
service_sys_configuration, service_report):
"""A Service engine.
Every engine process a request in its own thread.
Args:
name: Service canonical name
local_address (ProxyAddress): Address info for local connection
frontend_address (ProxyAddress): Address info for FE connection
user_engine (Engine): User engine deployed
service_sys_configuration (ServiceSysConfig): Service system config
service_report (ServiceReport): Service report object
"""
super(ServiceEngine, self).__init__(name,
local_address.host,
local_address.pub_port,
frontend_address.host,
frontend_address.pub_port)
self._engine_object = user_engine
self._semaphore = Semaphore(1)
self._compiler = CCompiler(self.myname)
self._prev_composition = xMsgConstants.UNDEFINED
self._report = service_report
self._logger = ClaraLogger(repr(self))
self.execution_time = 0
self.sys_config = service_sys_configuration
def __init__(self, name, engine_class, engine_name, pool_size,
initial_state, local_address, frontend_address):
"""Create thread pool to run requests to this service
Create object pool to hold the engines this service. Object pool size
is set to be 2 in case it was requested to be 0 or negative number.
Args:
name (String): Service canonical name
engine_class (String): Engine class containing Engine
engine_name (String): Engine name
pool_size (int): Service object pool size
initial_state (String): Initial state
local_address (ProxyAddress): Address info for local proxy
connection
frontend_address (ProxyAddress): Address info for frontend
connection
"""
super(Service,
self).__init__(name.canonical_name(), local_address.host,
local_address.pub_port, frontend_address.host,
frontend_address.pub_port)
self._logger = ClaraLogger(repr(self))
# user provided engine class container class name
self._engine_class = engine_class
# actual engine class name
self._engine_name = engine_name
# pool size
self._pool_size = pool_size
# Create service executor objects and fill the pool
self._available_object_pool = dict()
# Dynamically loads service engine class
engine_instance = EngineLoader(engine_class, engine_name).load_engine()
self._service_sys_config = ServiceSysConfig(name.canonical_name,
initial_state)
self._report = ServiceReport(self, engine_instance)
self._engine_pool = []
for _ in range(self._pool_size):
self._engine_pool.append(
ServiceEngine(name.canonical_name(), local_address,
frontend_address, engine_instance,
self._service_sys_config, self._report))
# Get description defined in the service engine
self.description = engine_instance.get_description()
try:
# Subscribe messages addressed to this service container
self.subscription_handler = self.listen(self.myname,
_ServiceCallBack(self))
self._logger.log_info("service deployed")
except Exception as e:
self._logger.log_exception(str(e))
raise e
except KeyboardInterrupt:
return
def __init__(self, container_name, local_address, frontend_address):
"""Service container for clara
Args:
container_name (ContainerName): Container name
local_address (ProxyAddress): Local proxy address
frontend_address (ProxyAddress): Frontend address
"""
super(Container,
self).__init__(container_name.canonical_name(),
local_address.host, local_address.pub_port,
frontend_address.host, frontend_address.pub_port)
self._container_name = container_name
self._logger = ClaraLogger(repr(self))
self._logger.log_info("container deployed")
self._report = ContainerReport(self, getuser())
def __init__(self,
proxy_host,
frontend_host,
proxy_port,
frontend_port,
report_interval=5):
"""Dpe Constructor
Args:
proxy_host (String): local hostname
frontend_host (String): frontend hostname
proxy_port (int): proxy port, default is 7791
frontend_port (int): frontend port, default is 7791
report_interval (int): time interval in seconds for reporting
service to update the frontend
Returns:
Dpe: Dpe object
"""
if proxy_host == frontend_host and proxy_host == frontend_port:
proxy_host = xMsgUtil.host_to_ip(proxy_host)
frontend_host = proxy_host
_is_frontend = True
else:
proxy_host = xMsgUtil.host_to_ip(proxy_host)
frontend_host = xMsgUtil.host_to_ip(frontend_host)
_is_frontend = False
dpe_name = DpeName(str(proxy_host), proxy_port, ClaraLang.PYTHON)
super(Dpe, self).__init__(dpe_name.canonical_name(), proxy_host,
proxy_port, frontend_host, frontend_port)
self.dpe_name = dpe_name
self._is_frontend = _is_frontend
self._logger = ClaraLogger(repr(self))
self._print_logo()
if not self._is_frontend:
self._report = DpeReport(self, getuser())
self._report_control = Event()
self._report_service = _ReportingService(self._report_control,
report_interval, self)
self._report_service.start()
self.subscription_handler = None
self._start()
class _ServiceCallBack(xMsgCallBack):
def __init__(self, service):
self._service = service
self._logger = ClaraLogger(repr(service))
def callback(self, msg):
try:
if not msg.metadata.HasField('action'):
self._logger.log_info("received : SETUP")
self._service.setup(msg)
elif msg.metadata.action == xMsgMeta.EXECUTE:
self._logger.log_info("received : EXECUTE")
self._service.execute(msg)
elif msg.metadata.action == xMsgMeta.CONFIGURE:
self._logger.log_info("received : CONFIGURE")
self._service.configure(msg)
except Exception as e:
self._logger.log_exception(str(e))
finally:
return msg
class _DpeCallBack(xMsgCallBack):
def __init__(self, dpe):
super(_DpeCallBack, self).__init__()
self._dpe = dpe
self._logger = ClaraLogger(repr(dpe))
def callback(self, msg):
try:
parser = RequestParser.build_from_message(msg)
cmd = parser.next_string()
response = parser.request()
self._logger.log_info("received: %s" % cmd)
if cmd == CConstants.STOP_DPE:
self._dpe.exit()
elif cmd == CConstants.START_CONTAINER:
self._dpe.start_container(parser)
elif cmd == CConstants.STOP_CONTAINER:
self._dpe.stop_container(parser)
elif cmd == CConstants.START_SERVICE:
self._dpe.start_service(parser)
elif cmd == CConstants.STOP_SERVICE:
self._dpe.stop_service(parser)
else:
self._logger.log_error("received unknown command...")
if msg.has_reply_topic():
self._dpe.send_response(msg, xMsgMeta.INFO, response)
except Exception as e:
self._logger.log_exception(e.message)
raise e
finally:
return msg
class Dpe(ClaraBase):
"""Clara data processing environment. It can play the role of the Front-End
(FE), which is the static point of the entire cloud. It creates and manages
the registration database (local and case of being assigned as an FE: global
database). Note this is a copy of the subscribers database resident in the
xMsg registration database. This also creates a shared memory for
communicating Clara transient data objects between services within the same
process (this avoids data serialization and de-serialization).
"""
my_containers = dict()
subscription_handler = None
def __init__(self,
proxy_host,
frontend_host,
proxy_port,
frontend_port,
report_interval=5):
"""Dpe Constructor
Args:
proxy_host (String): local hostname
frontend_host (String): frontend hostname
proxy_port (int): proxy port, default is 7791
frontend_port (int): frontend port, default is 7791
report_interval (int): time interval in seconds for reporting
service to update the frontend
Returns:
Dpe: Dpe object
"""
if proxy_host == frontend_host and proxy_host == frontend_port:
proxy_host = xMsgUtil.host_to_ip(proxy_host)
frontend_host = proxy_host
_is_frontend = True
else:
proxy_host = xMsgUtil.host_to_ip(proxy_host)
frontend_host = xMsgUtil.host_to_ip(frontend_host)
_is_frontend = False
dpe_name = DpeName(str(proxy_host), proxy_port, ClaraLang.PYTHON)
super(Dpe, self).__init__(dpe_name.canonical_name(), proxy_host,
proxy_port, frontend_host, frontend_port)
self.dpe_name = dpe_name
self._is_frontend = _is_frontend
self._logger = ClaraLogger(repr(self))
self._print_logo()
if not self._is_frontend:
self._report = DpeReport(self, getuser())
self._report_control = Event()
self._report_service = _ReportingService(self._report_control,
report_interval, self)
self._report_service.start()
self.subscription_handler = None
self._start()
def _exit(self):
self._report_control.set()
self._logger.log_info("Gracefully quitting the dpe...")
for container in self.my_containers.itervalues():
container.exit()
container.destroy()
def _print_logo(self):
import platform
logo_width = 50
print "=" * logo_width
print " " * 20 + "CLARA DPE"
print "=" * logo_width
print ""
print " Name = %s" % self.myname
print " Date = %s" % xMsgUtil.current_time()
print " Version = %s" % clara_version
print " Lang = python-%s" % platform.python_version()
print ""
print " Proxy Host = %s" % self._proxy_address.host
print " Proxy Port = %d" % self._proxy_address.pub_port
print ""
if not self._is_frontend:
print " Frontend Host = %s" % self._fe_address.host
print " Frontend Port = %d" % self._fe_address.pub_port
print ""
print "=" * logo_width
print ""
def _start(self):
proxy_process = subprocess.Popen(['px_proxy'])
try:
topic = ClaraUtils.build_topic(CConstants.DPE, self.myname)
self.subscription_handler = self.listen(topic, _DpeCallBack(self))
xMsgUtil.keep_alive()
except KeyboardInterrupt:
self._exit()
self.stop_listening(self.subscription_handler)
os.kill(proxy_process.pid, signal.SIGINT)
def get_report(self):
"""Returns DPE report object
Returns:
DpeReport
"""
return self._report
def start_container(self, request):
"""Starts a Clara container.
Containers are required in order to launch a Clara service
Args:
request (RequestParser): Request received from Orchestrator to
create a Container
"""
container_name = request.next_string()
try:
if container_name in self.my_containers:
self._logger.log_warning("Container " + str(container_name) +
" already exists. No new container is"
" created")
else:
container = Container(
ContainerName(self.dpe_name, container_name),
self._proxy_address, self._fe_address)
self.my_containers[container_name] = container
self._report.add_container(container.get_report())
except Exception as e:
self._logger.log_exception(e.message)
raise e
def stop_container(self, request):
"""Removes a Clara container and its contained services
Args:
request (RequestParser): Request received from Orchestrator to
stop a Container
"""
container_name = request.next_string()
if container_name in self.my_containers:
container = self.my_containers.pop(container_name)
self._report.remove_container(container.get_report())
container.exit()
def start_service(self, request):
"""Starts a Clara service
Args:
request (RequestParser): Request received from Orchestrator to
start a Service
"""
try:
container_name = request.next_string()
engine_name = request.next_string()
engine_class = request.next_string()
pool_size = request.next_integer()
description = request.next_string()
initial_state = request.next_string()
if container_name in self.my_containers:
self.my_containers[container_name].add_service(
engine_name, engine_class, pool_size, initial_state)
except Exception as e:
self._logger.log_exception(e.message)
raise e
def stop_service(self, request):
"""Stops a running Clara service
Args:
request (RequestParser): Request received from Orchestrator to
stop a Service
"""
container_name = request.next_string()
engine_name = request.next_string()
service_name = ClaraUtils.form_service_name(container_name,
engine_name)
if container_name in self.my_containers:
try:
self.my_containers[container_name].remove_service(service_name)
except Exception as e:
raise Exception("Could not stop service %s: %s " %
(service_name, e))
else:
raise Exception("Could not stop service %s: missing container " %
service_name)
def __init__(self, dpe):
super(_DpeCallBack, self).__init__()
self._dpe = dpe
self._logger = ClaraLogger(repr(dpe))
class Container(ClaraBase):
my_services = dict()
def __init__(self, container_name, local_address, frontend_address):
"""Service container for clara
Args:
container_name (ContainerName): Container name
local_address (ProxyAddress): Local proxy address
frontend_address (ProxyAddress): Frontend address
"""
super(Container,
self).__init__(container_name.canonical_name(),
local_address.host, local_address.pub_port,
frontend_address.host, frontend_address.pub_port)
self._container_name = container_name
self._logger = ClaraLogger(repr(self))
self._logger.log_info("container deployed")
self._report = ContainerReport(self, getuser())
def add_service(self, engine_name, engine_class, service_pool_size,
initial_state):
"""Add a new service into the service container
Creates a new Clara service with the given parameters and attaches it
to this container. When container is destroyed all services are exited
also.
Args:
engine_name (String): User engine name
engine_class (String): Python class containing the engine to deploy
service_pool_size (int): Pool size for the deployed service
initial_state (String): Initial state for service
"""
service_name = ServiceName(self._container_name, engine_name)
if service_name.canonical_name() in self.my_services:
self._logger.log_warning("Service %s already exists. No new "
"service is deployed" % str(service_name))
else:
try:
service = Service(service_name, engine_class, engine_name,
service_pool_size, initial_state,
self._proxy_address, self._fe_address)
self.my_services[service_name.canonical_name()] = service
self._report.add_service(service.get_report())
except Exception as e:
self._logger.log_exception("%s: %s" % (str(service_name), e))
raise e
def exit(self):
"""Gracefully destroys this container"""
self._remove_services()
self._logger.log_info("container stopped")
def get_report(self):
"""Returns the Container report object
Returns:
ContainerReport
"""
return self._report
def remove_service(self, service_name):
"""Exits the given service
Args:
service_name (String) Service canonical name
"""
if service_name in self.my_services:
service = self.my_services.pop(service_name)
self._report.remove_service(service.get_report())
service.exit()
def _remove_services(self):
"""Exits all services"""
self._report.remove_services()
for service in self.my_services.itervalues():
service.exit()
def __init__(self, service):
self._service = service
self._logger = ClaraLogger(repr(service))
class Service(ClaraBase):
"""Service container/broker class. This creates ServiceMP object pool.
Handles subscriptions and callbacks. Calls service (service executor)
object's run method in a separate thread.
Attributes:
engine_class (String): the name of the python class containing
service engine class
engine_name (String): the name of the service engine python class
within the engine_container_class
pool_size (int): Service object pool size, i.e. number of parallel
services
"""
def __init__(self, name, engine_class, engine_name, pool_size,
initial_state, local_address, frontend_address):
"""Create thread pool to run requests to this service
Create object pool to hold the engines this service. Object pool size
is set to be 2 in case it was requested to be 0 or negative number.
Args:
name (String): Service canonical name
engine_class (String): Engine class containing Engine
engine_name (String): Engine name
pool_size (int): Service object pool size
initial_state (String): Initial state
local_address (ProxyAddress): Address info for local proxy
connection
frontend_address (ProxyAddress): Address info for frontend
connection
"""
super(Service,
self).__init__(name.canonical_name(), local_address.host,
local_address.pub_port, frontend_address.host,
frontend_address.pub_port)
self._logger = ClaraLogger(repr(self))
# user provided engine class container class name
self._engine_class = engine_class
# actual engine class name
self._engine_name = engine_name
# pool size
self._pool_size = pool_size
# Create service executor objects and fill the pool
self._available_object_pool = dict()
# Dynamically loads service engine class
engine_instance = EngineLoader(engine_class, engine_name).load_engine()
self._service_sys_config = ServiceSysConfig(name.canonical_name,
initial_state)
self._report = ServiceReport(self, engine_instance)
self._engine_pool = []
for _ in range(self._pool_size):
self._engine_pool.append(
ServiceEngine(name.canonical_name(), local_address,
frontend_address, engine_instance,
self._service_sys_config, self._report))
# Get description defined in the service engine
self.description = engine_instance.get_description()
try:
# Subscribe messages addressed to this service container
self.subscription_handler = self.listen(self.myname,
_ServiceCallBack(self))
self._logger.log_info("service deployed")
except Exception as e:
self._logger.log_exception(str(e))
raise e
except KeyboardInterrupt:
return
def configure(self, message):
"""Configures engine from the engine pool
Args:
message (xMsgMessage): Input data for service engine
"""
while True:
for engine in self._engine_pool:
if engine.try_acquire_semaphore():
try:
engine.configure(message)
except Exception as e:
self._logger.log_exception(e.message)
finally:
engine.release_semaphore()
return
def execute(self, message):
"""Executes engine from the engine pool
Args:
message (xMsgMessage): Input data for service engine
"""
while True:
for engine in self._engine_pool:
if engine.try_acquire_semaphore():
try:
engine.execute(message)
except Exception as e:
self._logger.log_exception(e.message)
finally:
engine.release_semaphore()
return
def exit(self):
"""Exits the service gracefully"""
self.stop_listening(self.subscription_handler)
self._logger.log_info("service stopped")
def get_engine_class(self):
"""Returns the engine class as string
Returns:
String
"""
return self._engine_class
def get_engine_name(self):
"""Returns engine name as string
Returns:
String
"""
return self._engine_name
def get_report(self):
"""Returns service report object
Returns:
ServiceReport
"""
return self._report
def _send_response(self, message, status, data):
response_message = xMsgMessage.from_string(message.topic, data)
response_message.metadata.status = status
self.send(response_message)
def setup(self, message):
"""Configure service reporting messages
Args:
message (xMsgMessage): message with setup request
"""
setup = RequestParser.build_from_message(message)
report = setup.next_string()
value = setup.next_integer()
for engine in self._engine_pool:
try:
if report == CConstants.SERVICE_REPORT_DONE:
engine.sys_config.done_request = True
engine.sys_config.done_report_threshold = value
engine.sys_config.reset_done_request_count()
elif report == CConstants.SERVICE_REPORT_DATA:
engine.sys_config.data_request = True
engine.sys_config.data_report_threshold = value
engine.sys_config.reset_data_request_count()
else:
self._logger.log_error("invalid report request: " +
str(report))
except Exception as e:
self._logger.log_exception(e.message)
return
if message.has_reply_topic():
self._send_response(message, xMsgMeta.INFO, setup)
class ServiceEngine(ClaraBase):
def __init__(self, name,
local_address, frontend_address, user_engine,
service_sys_configuration, service_report):
"""A Service engine.
Every engine process a request in its own thread.
Args:
name: Service canonical name
local_address (ProxyAddress): Address info for local connection
frontend_address (ProxyAddress): Address info for FE connection
user_engine (Engine): User engine deployed
service_sys_configuration (ServiceSysConfig): Service system config
service_report (ServiceReport): Service report object
"""
super(ServiceEngine, self).__init__(name,
local_address.host,
local_address.pub_port,
frontend_address.host,
frontend_address.pub_port)
self._engine_object = user_engine
self._semaphore = Semaphore(1)
self._compiler = CCompiler(self.myname)
self._prev_composition = xMsgConstants.UNDEFINED
self._report = service_report
self._logger = ClaraLogger(repr(self))
self.execution_time = 0
self.sys_config = service_sys_configuration
def configure(self, message):
"""Configure the deployed Engine
Args:
message (xMsgMessage): message containing engine configuration data
"""
input_data = None
outgoing_data = None
try:
input_data = self._get_engine_data(message)
outgoing_data = self._configure_engine(input_data)
except Exception as e:
self._logger.log_exception(e.message)
outgoing_data = self.build_system_error_data(message,
-4, e.message)
finally:
self._update_metadata(input_data.metadata, outgoing_data.metadata)
if message.has_reply_topic():
outgoing_message = self._put_engine_data(outgoing_data,
message.get_reply_topic())
self.send(outgoing_message)
else:
self._report_problem(outgoing_data)
def _configure_engine(self, engine_input_data):
output_data = self._engine_object.configure(engine_input_data)
if not output_data:
output_data = EngineData()
if output_data.get_data():
output_data.set_data(Mimetype.STRING, xMsgConstants.DONE)
return output_data
def _get_engine_data(self, message):
msg = self.de_serialize(message,
self._engine_object.get_input_data_types())
self._report.increment_bytes_received(sys.getsizeof(msg))
return msg
def _get_links(self, engine_input_data, engine_output_data):
owner_ss = ServiceState(engine_output_data.engine_name(),
engine_output_data.state)
input_ss = ServiceState(engine_input_data.engine_name(),
engine_input_data.state)
return self._compiler.get_links(owner_ss, input_ss)
def _update_metadata(self, in_meta, out_meta):
out_meta.author = self.myname
out_meta.version = self._engine_object.get_version()
if not out_meta.communicationId:
out_meta.communicationId = in_meta.communicationId
out_meta.composition = in_meta.composition
out_meta.executionTime = self.execution_time
out_meta.action = in_meta.action
def _parse_composition(self, engine_input_data):
current_composition = engine_input_data.composition
if current_composition != self._prev_composition:
self._compiler.compile(current_composition)
self._prev_composition = current_composition
def _put_engine_data(self, data, receiver):
topic = ClaraUtils.build_topic(CConstants.SERVICE, receiver)
msg = self.serialize(topic, data,
self._engine_object.get_output_data_types())
self._report.increment_bytes_sent(sys.getsizeof(msg))
return msg
def _report_problem(self, engine_data):
status = engine_data.status
if status == EngineStatus.ERROR:
self._report(xMsgConstants.ERROR, engine_data)
elif status == EngineStatus.WARNING:
self._report(xMsgConstants.WARNING, engine_data)
def _report(self, topic_prefix, engine_data):
topic = xMsgTopic.wrap(str(topic_prefix) + ":" + self.myname)
msg = self.serialize(topic, engine_data,
self._engine_object.get_output_data_types())
self.send_frontend(msg)
def _report_data(self, data):
self._report(xMsgConstants.DATA, data)
def _report_done(self, data):
self._report(xMsgConstants.DONE, data)
def _send_reports(self, outgoing_data):
if self.sys_config.data_request:
self._report_data(outgoing_data)
self.sys_config.reset_data_request_count()
if self.sys_config.done_request:
self._report_done(outgoing_data)
self.sys_config.reset_done_request_count()
def _send_response(self, outgoing_data, outgoing_links):
for link in outgoing_links:
msg = self._put_engine_data(outgoing_data, link)
self.send(msg)
def execute(self, message):
"""Executes the deployed engine with the given input data
Args:
message (xMsgMessage): message containing input data
"""
in_data = None
outgoing_data = None
self.sys_config.add_request()
self._report.increment_request_count()
try:
in_data = self._get_engine_data(message)
self._parse_composition(in_data)
start_time = time.time()
outgoing_data = self._execute_engine(in_data)
elapsed_time = time.time() - start_time
self._report.increment_execution_time(elapsed_time)
except Exception as e:
self._logger.log_exception(e.message)
self._report.increment_failure_count()
outgoing_data = self.build_system_error_data(message, -4,
e.message)
raise e
finally:
self._update_metadata(message.metadata, outgoing_data.metadata)
if message.has_reply_topic():
outgoing_message = self._put_engine_data(outgoing_data,
message.get_reply_topic())
self.send(outgoing_message)
self._send_reports(outgoing_data)
self._report_problem(outgoing_data)
self._send_response(outgoing_data, self._get_links(in_data,
outgoing_data))
def _execute_engine(self, engine_input_data):
out_data = self._engine_object.execute(engine_input_data)
if not out_data:
self._logger.log_exception("null engine result")
raise Exception("null engine result")
return out_data
def try_acquire_semaphore(self):
"""Returns true if service semaphore is available in a non blocking op
Returns:
boolean
"""
return self._semaphore.acquire(blocking=False)
def release_semaphore(self):
"""Releases engine semaphore
Returns:
boolean
"""
return self._semaphore.release()