Esempio n. 1
0
    def on_init(self):
        self.ingestion_profile = self.CFG.get_safe(CONFIG_KEY + ".ingestion_profile", "default")

        log.info("Ingestion starting using profile '%s'", self.ingestion_profile)
        self.exchange_name = "ingestion_process"

        self.ingestion_config = self.CFG.get_safe(CONFIG_KEY + ".profile_" + self.ingestion_profile) or {}
        if not self.ingestion_config:
            raise BadRequest("No config found for profile '%s'" % self.ingestion_profile)

        plugin_cls = get_safe(self.ingestion_config, "plugin")
        self.plugin = named_any(plugin_cls)(self)
        log.info("Started ingestion plugin '%s'", plugin_cls)

        self.persistence_formats = {}
        self.persistence_objects = {}
        self.default_persistence_format = get_safe(self.ingestion_config, "persist.persistence_format")
        self._require_persistence_layer(self.default_persistence_format)

        self.stream_sub = StreamSubscriber(process=self, exchange_name=self.exchange_name,
                                           callback=self.process_package)
        streams = get_safe(self.ingestion_config, "stream_subscriptions") or []
        for stream in streams:
            if isinstance(stream, list):
                stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1])

            log.info("Ingestion subscribed to stream '%s'", stream)
            self.stream_sub.add_stream_subscription(stream)

        self.plugin.on_init()

        self.stream_sub.start()
Esempio n. 2
0
    def on_init(self):
        log.info("Ingestion starting")
        self.exchange_name = "ingestion_process"

        plugin_cls = CFG.get_safe(CONFIG_KEY + ".plugin")
        self.plugin = named_any(plugin_cls)(self)
        log.info("Started ingestion plugin '%s'", plugin_cls)

        self.persistence_formats = {}
        self.persistence_objects = {}
        self.default_persistence_format = CFG.get_safe(CONFIG_KEY + ".persist.persistence_format")
        self._require_persistence_layer(self.default_persistence_format)

        self.stream_sub = StreamSubscriber(process=self, exchange_name=self.exchange_name,
                                           callback=self.process_package)
        streams = CFG.get_safe(CONFIG_KEY + ".stream_subscriptions") or []
        for stream in streams:
            if isinstance(stream, list):
                stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1])

            log.info("Ingestion subscribed to stream '%s'", stream)
            self.stream_sub.add_stream_subscription(stream)

        self.plugin.on_init()

        self.stream_sub.start()
Esempio n. 3
0
    def on_init(self):
        self.ingestion_profile = self.CFG.get_safe(CONFIG_KEY + ".ingestion_profile", "default")

        log.info("Ingestion starting using profile '%s'", self.ingestion_profile)
        self.exchange_name = "ingestion_process"

        self.ingestion_config = self.CFG.get_safe(CONFIG_KEY + ".profile_" + self.ingestion_profile) or {}
        if not self.ingestion_config:
            raise BadRequest("No config found for profile '%s'" % self.ingestion_profile)

        plugin_cls = get_safe(self.ingestion_config, "plugin")
        self.plugin = named_any(plugin_cls)(self)
        log.info("Started ingestion plugin '%s'", plugin_cls)

        self.persistence_formats = {}
        self.persistence_objects = {}
        self.default_persistence_format = get_safe(self.ingestion_config, "persist.persistence_format")
        self._require_persistence_layer(self.default_persistence_format)

        self.stream_sub = StreamSubscriber(
            process=self, exchange_name=self.exchange_name, callback=self.process_package
        )
        streams = get_safe(self.ingestion_config, "stream_subscriptions") or []
        for stream in streams:
            if isinstance(stream, list):
                stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1])

            log.info("Ingestion subscribed to stream '%s'", stream)
            self.stream_sub.add_stream_subscription(stream)

        self.plugin.on_init()

        self.stream_sub.start()
Esempio n. 4
0
    def on_init(self):

        self.http_server = None
        self.server_hostname = self.CFG.get_safe(CFG_PREFIX + '.web_server.hostname', DEFAULT_WEB_SERVER_HOSTNAME)
        self.server_port = self.CFG.get_safe(CFG_PREFIX + '.web_server.port', DEFAULT_WEB_SERVER_PORT)
        self.url_prefix = self.CFG.get_safe(CFG_PREFIX + '.url_prefix') or ""

        self.web_server_enabled = True
        self.logging = None
        self.interaction_observer = None
        self.plugin = None
        app.secret_key = self.__class__.__name__   # Enables sessions (for mscweb)

        #retain a pointer to this object for use in ProcessRPC calls
        global adminui_instance
        adminui_instance = self

        #Start the gevent web server unless disabled
        if self.web_server_enabled:
            self.start_service(self.server_hostname, self.server_port)

            plugin_cls = CFG.get_safe(CFG_PREFIX + '.plugin')
            if plugin_cls:
                cls = named_any(plugin_cls)
                self.plugin = cls(app, self)
Esempio n. 5
0
 def _get_object_class(self, objtype):
     if objtype in self.obj_classes:
         return self.obj_classes[objtype]
     try:
         obj_class = named_any("interface.objects.%s" % objtype)
         self.obj_classes[objtype] = obj_class
         return obj_class
     except Exception:
         log.error('failed to find class for type %s' % objtype)
Esempio n. 6
0
 def _get_object_class(self, objtype):
     if objtype in self.obj_classes:
         return self.obj_classes[objtype]
     try:
         obj_class = named_any("interface.objects.%s" % objtype)
         self.obj_classes[objtype] = obj_class
         return obj_class
     except Exception:
         log.error('failed to find class for type %s' % objtype)
Esempio n. 7
0
 def on_init(self):
     log.info("Start agent %s pid=%s resource_id=%s", self.__class__.__name__, self.id, self.resource_id)
     self.current_state = self.AGENTSTATE_INITIALIZED
     self.agent_config = self.CFG.get_safe("agent_config") or {}
     self.params = {}
     self.agent_plugin = None
     self.stream_name = self.agent_config.get("stream_name", None) or "stream_" + self.resource_id
     if "plugin" in self.agent_config:
         agent_plugin_cls = named_any(self.agent_config["plugin"])
         log.info("Instantiate agent plugin '%s'", self.agent_config["plugin"])
         self.agent_plugin = agent_plugin_cls(self, self.agent_config)
     if self.agent_config.get("auto_streaming", False) is True:
         self.connect()
         self.start_streaming()
Esempio n. 8
0
 def on_init(self):
     log.info("Start agent %s pid=%s resource_id=%s",
              self.__class__.__name__, self.id, self.resource_id)
     self.current_state = self.AGENTSTATE_INITIALIZED
     self.agent_config = self.CFG.get_safe("agent_config") or {}
     self.params = {}
     self.agent_plugin = None
     self.stream_name = self.agent_config.get(
         "stream_name", None) or "stream_" + self.resource_id
     if "plugin" in self.agent_config:
         agent_plugin_cls = named_any(self.agent_config["plugin"])
         log.info("Instantiate agent plugin '%s'",
                  self.agent_config["plugin"])
         self.agent_plugin = agent_plugin_cls(self, self.agent_config)
     if self.agent_config.get("auto_streaming", False) is True:
         self.connect()
         self.start_streaming()
Esempio n. 9
0
def _get_object_class(objtype):
    if objtype in obj_classes:
        return obj_classes[objtype]
    obj_class = named_any("interface.objects.%s" % objtype)
    obj_classes[objtype] = obj_class
    return obj_class
Esempio n. 10
0
def _get_object_class(objtype):
    if objtype in obj_classes:
        return obj_classes[objtype]
    obj_class = named_any("interface.objects.%s" % objtype)
    obj_classes[objtype] = obj_class
    return obj_class