def start(self): # Create our own queue for container heartbeats and broadcasts topic = get_safe(self._pd_core.pd_cfg, "aggregator.container_topic") or "bx_containers" queue_name = "pd_aggregator_%s_%s" % (topic, create_valid_identifier(self.container.id, dot_sub="_")) self.sub_cont = Subscriber(binding=topic, from_name=queue_name, auto_delete=True, callback=self._receive_container_info) self.sub_cont_gl = spawn(self.sub_cont.listen) self.sub_cont.get_ready_event().wait() self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent, callback=self._receive_event) self.evt_sub.add_event_subscription(event_type=OT.ProcessLifecycleEvent) self.evt_sub_gl = spawn(self.evt_sub.listen) self.evt_sub.get_ready_event().wait() log.info("PD Aggregator - event and heartbeat subscribers started")
def create_process(self, process_definition_id=''): """Create a process resource and process id. Does not yet start the process @param process_definition_id str @retval process_id str @throws NotFound object with specified id does not exist """ if not process_definition_id: raise NotFound('No process definition was provided') process_definition = self.clients.resource_registry.read(process_definition_id) # try to get a unique but still descriptive name process_id = str(process_definition.name or "process") + uuid.uuid4().hex process_id = create_valid_identifier(process_id, ws_sub='_') # TODO: Create a resource object or directory entry here? return process_id
def create_process(self, process_definition_id=''): """Create a process resource and process id. Does not yet start the process @param process_definition_id str @retval process_id str @throws NotFound object with specified id does not exist """ if not process_definition_id: raise NotFound('No process definition was provided') process_definition = self.clients.resource_registry.read( process_definition_id) # try to get a unique but still descriptive name process_id = str(process_definition.name or "process") + uuid.uuid4().hex process_id = create_valid_identifier(process_id, ws_sub='_') # TODO: Create a resource object or directory entry here? return process_id
def start(self): # Create our own queue for container heartbeats and broadcasts topic = get_safe(self._pd_core.pd_cfg, "aggregator.container_topic") or "bx_containers" queue_name = "pd_aggregator_%s_%s" % ( topic, create_valid_identifier(self.container.id, dot_sub="_")) self.sub_cont = Subscriber(binding=topic, from_name=queue_name, auto_delete=True, callback=self._receive_container_info) self.sub_cont_gl = spawn(self.sub_cont.listen) self.sub_cont.get_ready_event().wait() self.evt_sub = EventSubscriber(event_type=OT.ContainerLifecycleEvent, callback=self._receive_event) self.evt_sub.add_event_subscription( event_type=OT.ProcessLifecycleEvent) self.evt_sub_gl = spawn(self.evt_sub.listen) self.evt_sub.get_ready_event().wait() log.info("PD Aggregator - event and heartbeat subscribers started")
def create_process(self, process_definition_id=''): """Create a process resource and process id. Does not yet start the process @param process_definition_id str @retval process_id str @throws NotFound object with specified id does not exist """ if not process_definition_id: raise NotFound('No process definition was provided') process_definition = self.backend.read_definition(process_definition_id) # try to get a unique but still descriptive name process_id = str(process_definition.name or "process") + uuid.uuid4().hex process_id = create_valid_identifier(process_id, ws_sub='_') self.backend.create(process_id, process_definition_id) try: process = Process(process_id=process_id) self.container.resource_registry.create(process, object_id=process_id) except BadRequest: log.debug("Tried to create Process %s, but already exists. This is normally ok.", process_id) return process_id
class ProcessDispatcherService(BaseProcessDispatcherService): # Implementation notes: # # Through Elaboration, the Process Dispatcher and other CEI services # do not run as pyon services. Instead they run as standalone processes # and communicate using simple AMQP messaging. However the Process # Dispatcher needs to be called by the Transform Management Service via # pyon messaging. To facilitate, this service acts as a bridge to the # "real" CEI process dispatcher. # # Because the real process dispatcher will only be present in a CEI # launch environment, this bridge service operates in two modes, # detected based on a config value. # # 1. When a "process_dispatcher_bridge" config section is present, this # service acts as a bridge to the real PD. The real PD must be running # and some additional dependencies must be available. # # 2. Otherwise, processes are started directly in the local container. # This mode is meant to support the developer and integration use of # r2deploy.yml and other single-container test deployments. # # Note that this is a relatively short-term situation. The PD will soon # natively run in the container and these tricks will be unnecessary. def on_init(self): #am I crazy or does self.CFG.get() not work? try: pd_conf = self.CFG.process_dispatcher_bridge except AttributeError: pd_conf = None if pd_conf: log.debug( "Using Process Dispatcher Bridge backend -- requires running CEI services." ) self.backend = PDBridgeBackend(pd_conf) else: log.debug( "Using Process Dispatcher Local backend -- spawns processes in local container" ) self.backend = PDLocalBackend(self.container) def on_start(self): self.backend.initialize() def on_stop(self): self.backend.shutdown() def create_process_definition(self, process_definition=None): """Creates a Process Definition based on given object. @param process_definition ProcessDefinition @retval process_definition_id str @throws BadRequest if object passed has _id or _rev attribute """ pd_id, version = self.clients.resource_registry.create( process_definition) return pd_id def update_process_definition(self, process_definition=None): """Updates a Process Definition based on given object. @param process_definition ProcessDefinition @throws BadRequest if object does not have _id or _rev attribute @throws NotFound object with specified id does not exist @throws Conflict object not based on latest persisted object version """ self.clients.resource_registry.update(process_definition) def read_process_definition(self, process_definition_id=''): """Returns a Process Definition as object. @param process_definition_id str @retval process_definition ProcessDefinition @throws NotFound object with specified id does not exist """ pdef = self.clients.resource_registry.read(process_definition_id) return pdef def delete_process_definition(self, process_definition_id=''): """Deletes/retires a Process Definition. @param process_definition_id str @throws NotFound object with specified id does not exist """ self.clients.resource_registry.delete(process_definition_id) def associate_execution_engine(self, process_definition_id='', execution_engine_definition_id=''): """Declare that the given process definition is compatible with the given execution engine. @param process_definition_id str @param execution_engine_definition_id str @throws NotFound object with specified id does not exist """ self.clients.resource_registry.create_association( process_definition_id, PRED.supportsExecutionEngine, execution_engine_definition_id) def dissociate_execution_engine(self, process_definition_id='', execution_engine_definition_id=''): """Remove the association of the process definition with an execution engine. @param process_definition_id str @param execution_engine_definition_id str @throws NotFound object with specified id does not exist """ assoc = self.clients.resource_registry.get_association( process_definition_id, PRED.supportsExecutionEngine, execution_engine_definition_id) self.clients.resource_registry.delete_association(assoc) def create_process(self, process_definition_id=''): """Create a process resource and process id. Does not yet start the process @param process_definition_id str @retval process_id str @throws NotFound object with specified id does not exist """ if not process_definition_id: raise NotFound('No process definition was provided') process_definition = self.clients.resource_registry.read( process_definition_id) # try to get a unique but still descriptive name process_id = str(process_definition.name or "process") + uuid.uuid4().hex process_id = create_valid_identifier(process_id, ws_sub='_') # TODO: Create a resource object or directory entry here? return process_id def schedule_process(self, process_definition_id='', schedule=None, configuration=None, process_id=''): """Schedule a process definition for execution on an Execution Engine. If no process id is given, a new unique ID is generated. @param process_definition_id str @param schedule ProcessSchedule @param configuration IngestionConfiguration @param process_id str @retval process_id str @throws BadRequest if object passed has _id or _rev attribute @throws NotFound object with specified id does not exist """ if not process_definition_id: raise NotFound('No process definition was provided') process_definition = self.clients.resource_registry.read( process_definition_id) # early validation before we pass definition through to backend try: module = process_definition.executable['module'] cls = process_definition.executable['class'] except KeyError, e: raise BadRequest("Process definition incomplete. missing: %s", e) if configuration is None: configuration = {} # If not provided, create a unique but still descriptive (valid) name if not process_id: process_id = str(process_definition.name or "process") + uuid.uuid4().hex process_id = create_valid_identifier(process_id, ws_sub='_') return self.backend.spawn(process_id, process_definition, schedule, configuration)
if configuration is None: configuration = {} else: # push the config through a JSON serializer to ensure that the same # config would work with the bridge backend try: json.dumps(configuration) except TypeError, e: raise BadRequest("bad configuration: " + str(e)) # If not provided, create a unique but still descriptive (valid) name if not process_id: process_id = str(process_definition.name or "process") + uuid.uuid4().hex process_id = create_valid_identifier(process_id, ws_sub='_') return self.backend.spawn(process_id, process_definition, schedule, configuration) def cancel_process(self, process_id=''): """Cancels the execution of the given process id. @param process_id str @retval success bool @throws NotFound object with specified id does not exist """ if not process_id: raise NotFound('No process was provided') return self.backend.cancel(process_id)