def start(self):
# Create our own queue for container heartbeats and broadcasts
topic = get_safe(self._pd_core.pd_cfg, "aggregator.container_topic") or "bx_containers"
queue_name = "pd_aggregator_%s_%s" % (topic, create_valid_identifier(self.container.id, dot_sub="_"))
self.sub_cont = Subscriber(binding=topic, from_name=queue_name, auto_delete=True,
callback=self._receive_container_info)
self.sub_cont_gl = spawn(self.sub_cont.listen)
self.sub_cont.get_ready_event().wait()
self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent, callback=self._receive_event)
self.evt_sub.add_event_subscription(event_type=OT.ProcessLifecycleEvent)
self.evt_sub_gl = spawn(self.evt_sub.listen)
self.evt_sub.get_ready_event().wait()
log.info("PD Aggregator - event and heartbeat subscribers started")
def create_process(self, process_definition_id=''):
"""Create a process resource and process id. Does not yet start the process
@param process_definition_id str
@retval process_id str
@throws NotFound object with specified id does not exist
"""
if not process_definition_id:
raise NotFound('No process definition was provided')
process_definition = self.clients.resource_registry.read(process_definition_id)
# try to get a unique but still descriptive name
process_id = str(process_definition.name or "process") + uuid.uuid4().hex
process_id = create_valid_identifier(process_id, ws_sub='_')
# TODO: Create a resource object or directory entry here?
return process_id
def create_process(self, process_definition_id=''):
"""Create a process resource and process id. Does not yet start the process
@param process_definition_id str
@retval process_id str
@throws NotFound object with specified id does not exist
"""
if not process_definition_id:
raise NotFound('No process definition was provided')
process_definition = self.clients.resource_registry.read(
process_definition_id)
# try to get a unique but still descriptive name
process_id = str(process_definition.name
or "process") + uuid.uuid4().hex
process_id = create_valid_identifier(process_id, ws_sub='_')
# TODO: Create a resource object or directory entry here?
return process_id
def start(self):
# Create our own queue for container heartbeats and broadcasts
topic = get_safe(self._pd_core.pd_cfg,
"aggregator.container_topic") or "bx_containers"
queue_name = "pd_aggregator_%s_%s" % (
topic, create_valid_identifier(self.container.id, dot_sub="_"))
self.sub_cont = Subscriber(binding=topic,
from_name=queue_name,
auto_delete=True,
callback=self._receive_container_info)
self.sub_cont_gl = spawn(self.sub_cont.listen)
self.sub_cont.get_ready_event().wait()
self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent,
callback=self._receive_event)
self.evt_sub.add_event_subscription(
event_type=OT.ProcessLifecycleEvent)
self.evt_sub_gl = spawn(self.evt_sub.listen)
self.evt_sub.get_ready_event().wait()
log.info("PD Aggregator - event and heartbeat subscribers started")
def create_process(self, process_definition_id=''):
"""Create a process resource and process id. Does not yet start the process
@param process_definition_id str
@retval process_id str
@throws NotFound object with specified id does not exist
"""
if not process_definition_id:
raise NotFound('No process definition was provided')
process_definition = self.backend.read_definition(process_definition_id)
# try to get a unique but still descriptive name
process_id = str(process_definition.name or "process") + uuid.uuid4().hex
process_id = create_valid_identifier(process_id, ws_sub='_')
self.backend.create(process_id, process_definition_id)
try:
process = Process(process_id=process_id)
self.container.resource_registry.create(process, object_id=process_id)
except BadRequest:
log.debug("Tried to create Process %s, but already exists. This is normally ok.", process_id)
return process_id
class ProcessDispatcherService(BaseProcessDispatcherService):
# Implementation notes:
#
# Through Elaboration, the Process Dispatcher and other CEI services
# do not run as pyon services. Instead they run as standalone processes
# and communicate using simple AMQP messaging. However the Process
# Dispatcher needs to be called by the Transform Management Service via
# pyon messaging. To facilitate, this service acts as a bridge to the
# "real" CEI process dispatcher.
#
# Because the real process dispatcher will only be present in a CEI
# launch environment, this bridge service operates in two modes,
# detected based on a config value.
#
# 1. When a "process_dispatcher_bridge" config section is present, this
# service acts as a bridge to the real PD. The real PD must be running
# and some additional dependencies must be available.
#
# 2. Otherwise, processes are started directly in the local container.
# This mode is meant to support the developer and integration use of
# r2deploy.yml and other single-container test deployments.
#
# Note that this is a relatively short-term situation. The PD will soon
# natively run in the container and these tricks will be unnecessary.
def on_init(self):
#am I crazy or does self.CFG.get() not work?
try:
pd_conf = self.CFG.process_dispatcher_bridge
except AttributeError:
pd_conf = None
if pd_conf:
log.debug(
"Using Process Dispatcher Bridge backend -- requires running CEI services."
)
self.backend = PDBridgeBackend(pd_conf)
else:
log.debug(
"Using Process Dispatcher Local backend -- spawns processes in local container"
)
self.backend = PDLocalBackend(self.container)
def on_start(self):
self.backend.initialize()
def on_stop(self):
self.backend.shutdown()
def create_process_definition(self, process_definition=None):
"""Creates a Process Definition based on given object.
@param process_definition ProcessDefinition
@retval process_definition_id str
@throws BadRequest if object passed has _id or _rev attribute
"""
pd_id, version = self.clients.resource_registry.create(
process_definition)
return pd_id
def update_process_definition(self, process_definition=None):
"""Updates a Process Definition based on given object.
@param process_definition ProcessDefinition
@throws BadRequest if object does not have _id or _rev attribute
@throws NotFound object with specified id does not exist
@throws Conflict object not based on latest persisted object version
"""
self.clients.resource_registry.update(process_definition)
def read_process_definition(self, process_definition_id=''):
"""Returns a Process Definition as object.
@param process_definition_id str
@retval process_definition ProcessDefinition
@throws NotFound object with specified id does not exist
"""
pdef = self.clients.resource_registry.read(process_definition_id)
return pdef
def delete_process_definition(self, process_definition_id=''):
"""Deletes/retires a Process Definition.
@param process_definition_id str
@throws NotFound object with specified id does not exist
"""
self.clients.resource_registry.delete(process_definition_id)
def associate_execution_engine(self,
process_definition_id='',
execution_engine_definition_id=''):
"""Declare that the given process definition is compatible with the given execution engine.
@param process_definition_id str
@param execution_engine_definition_id str
@throws NotFound object with specified id does not exist
"""
self.clients.resource_registry.create_association(
process_definition_id, PRED.supportsExecutionEngine,
execution_engine_definition_id)
def dissociate_execution_engine(self,
process_definition_id='',
execution_engine_definition_id=''):
"""Remove the association of the process definition with an execution engine.
@param process_definition_id str
@param execution_engine_definition_id str
@throws NotFound object with specified id does not exist
"""
assoc = self.clients.resource_registry.get_association(
process_definition_id, PRED.supportsExecutionEngine,
execution_engine_definition_id)
self.clients.resource_registry.delete_association(assoc)
def create_process(self, process_definition_id=''):
"""Create a process resource and process id. Does not yet start the process
@param process_definition_id str
@retval process_id str
@throws NotFound object with specified id does not exist
"""
if not process_definition_id:
raise NotFound('No process definition was provided')
process_definition = self.clients.resource_registry.read(
process_definition_id)
# try to get a unique but still descriptive name
process_id = str(process_definition.name
or "process") + uuid.uuid4().hex
process_id = create_valid_identifier(process_id, ws_sub='_')
# TODO: Create a resource object or directory entry here?
return process_id
def schedule_process(self,
process_definition_id='',
schedule=None,
configuration=None,
process_id=''):
"""Schedule a process definition for execution on an Execution Engine. If no process id is given,
a new unique ID is generated.
@param process_definition_id str
@param schedule ProcessSchedule
@param configuration IngestionConfiguration
@param process_id str
@retval process_id str
@throws BadRequest if object passed has _id or _rev attribute
@throws NotFound object with specified id does not exist
"""
if not process_definition_id:
raise NotFound('No process definition was provided')
process_definition = self.clients.resource_registry.read(
process_definition_id)
# early validation before we pass definition through to backend
try:
module = process_definition.executable['module']
cls = process_definition.executable['class']
except KeyError, e:
raise BadRequest("Process definition incomplete. missing: %s", e)
if configuration is None:
configuration = {}
# If not provided, create a unique but still descriptive (valid) name
if not process_id:
process_id = str(process_definition.name
or "process") + uuid.uuid4().hex
process_id = create_valid_identifier(process_id, ws_sub='_')
return self.backend.spawn(process_id, process_definition, schedule,
configuration)
if configuration is None:
configuration = {}
else:
# push the config through a JSON serializer to ensure that the same
# config would work with the bridge backend
try:
json.dumps(configuration)
except TypeError, e:
raise BadRequest("bad configuration: " + str(e))
# If not provided, create a unique but still descriptive (valid) name
if not process_id:
process_id = str(process_definition.name or "process") + uuid.uuid4().hex
process_id = create_valid_identifier(process_id, ws_sub='_')
return self.backend.spawn(process_id, process_definition, schedule, configuration)
def cancel_process(self, process_id=''):
"""Cancels the execution of the given process id.
@param process_id str
@retval success bool
@throws NotFound object with specified id does not exist
"""
if not process_id:
raise NotFound('No process was provided')
return self.backend.cancel(process_id)
