def setup_interceptors(self, interceptor_cfg):
stack = interceptor_cfg["stack"]
defs = interceptor_cfg["interceptors"]
interceptors = defaultdict(list)
by_name_dict = {}
for type_and_direction in stack:
interceptor_names = stack[type_and_direction]
for name in interceptor_names:
if name in by_name_dict:
classinst = by_name_dict[name]
else:
interceptor_def = defs[name]
# Instantiate and put in by_name array
modpath, classname = interceptor_def['class'].rsplit(
'.', 1)
classinst = for_name(modpath, classname)
# Call configure
classinst.configure(
config=interceptor_def["config"] if "config" in
interceptor_def else None)
# Put in by_name_dict for possible re-use
by_name_dict[name] = classinst
interceptors[type_and_direction].append(classinst)
self.interceptors = dict(interceptors)
def _create_app_instance(self, process_id, name, module, cls, config, proc_attr):
"""
Creates an instance of a BaseService, representing the app logic of a ION process.
This is independent of the process type service, agent, standalone, etc.
"""
# APP INSTANCE.
app_instance = for_name(module, cls)
if not isinstance(app_instance, BaseService):
raise ContainerConfigError("Instantiated service not a BaseService %r" % app_instance)
# Set BaseService instance common attributes
app_instance.errcause = ""
app_instance.id = process_id
app_instance.container = self.container
app_instance.CFG = config
app_instance._proc_name = name
app_instance._proc_start_time = time.time()
for att, att_val in proc_attr.iteritems():
setattr(app_instance, att, att_val)
# Unless the process has been started as part of another Org, default to the container Org or the ION Org
if 'org_governance_name' in config:
app_instance.org_governance_name = config['org_governance_name']
else:
app_instance.org_governance_name = CFG.get_safe('container.org_name', CFG.get_safe('system.root_org', 'ION'))
# Add process state management, if applicable
self._add_process_state(app_instance)
# Check dependencies (RPC clients)
self._check_process_dependencies(app_instance)
return app_instance
def setup_interceptors(self, interceptor_cfg):
stack = interceptor_cfg["stack"]
defs = interceptor_cfg["interceptors"]
interceptors = defaultdict(list)
by_name_dict = {}
for type_and_direction in stack:
interceptor_names = stack[type_and_direction]
for name in interceptor_names:
if name in by_name_dict:
classinst = by_name_dict[name]
else:
interceptor_def = defs[name]
# Instantiate and put in by_name array
modpath, classname = interceptor_def['class'].rsplit('.', 1)
classinst = for_name(modpath, classname)
# Call configure
classinst.configure(config = interceptor_def["config"] if "config" in interceptor_def else None)
# Put in by_name_dict for possible re-use
by_name_dict[name] = classinst
interceptors[type_and_direction].append(classinst)
self.interceptors = dict(interceptors)
def _call_plugins(self, method, process, config, **kwargs):
bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
if bootstrap_plugins is None:
log.warn("Bootstrapper called without bootstrap_plugins config")
# Finding the system actor ID. If found, construct call context headers.
# This may be called very early in bootstrap with no system actor yet existing
system_actor = get_system_actor()
if system_actor:
actor_headers = get_system_actor_header(system_actor)
else:
# Use default actor headers, not roles.
actor_headers = build_actor_header()
# Set the call context of the current process
with process.push_context(actor_headers):
for plugin_info in bootstrap_plugins:
plugin_mod, plugin_cls = plugin_info.get(
"plugin", [None, None])
plugin_cfg = plugin_info.get("config", None)
plugin_cfg = dict_merge(
config, plugin_cfg) if plugin_cfg is not None else config
try:
log.info("Bootstrapping plugin %s.%s ...", plugin_mod,
plugin_cls)
plugin = for_name(plugin_mod, plugin_cls)
plugin_func = getattr(plugin, method)
plugin_func(process, plugin_cfg, **kwargs)
except AbortBootstrap as abort:
raise
except Exception as ex:
log.exception("Error bootstrapping plugin %s.%s",
plugin_mod, plugin_cls)
def _call_plugins(self, method, process, config, **kwargs):
bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
if bootstrap_plugins is None:
log.warn("Bootstrapper called without bootstrap_plugins config")
# Finding the system actor ID. If found, construct call context headers.
# This may be called very early in bootstrap with no system actor yet existing
system_actor, _ = process.container.resource_registry.find_resources(
RT.ActorIdentity, name=self.CFG.system.system_actor, id_only=True
)
system_actor_id = system_actor[0] if system_actor else "anonymous"
actor_headers = {
"ion-actor-id": system_actor_id,
"ion-actor-roles": {"ION": ["ION_MANAGER", "ORG_MANAGER"]} if system_actor else {},
}
# Set the call context of the current process
with process.push_context(actor_headers):
for plugin_info in bootstrap_plugins:
plugin_mod, plugin_cls = plugin_info.get("plugin", [None, None])
plugin_cfg = plugin_info.get("config", None)
plugin_cfg = dict_merge(config, plugin_cfg) if plugin_cfg is not None else config
try:
log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls)
plugin = for_name(plugin_mod, plugin_cls)
plugin_func = getattr(plugin, method)
plugin_func(process, plugin_cfg, **kwargs)
except AbortBootstrap as abort:
raise
except Exception as ex:
log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)
def _call_plugins(self, method, process, config, **kwargs):
bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
if bootstrap_plugins is None:
log.warn("Bootstrapper called without bootstrap_plugins config")
# Finding the system actor ID. If found, construct call context headers.
# This may be called very early in bootstrap with no system actor yet existing
system_actor, _ = process.container.resource_registry.find_resources(
RT.ActorIdentity, name=self.CFG.system.system_actor, id_only=False)
actor_headers = get_system_actor_header(
system_actor[0] if system_actor else None)
# Set the call context of the current process
with process.push_context(actor_headers):
for plugin_info in bootstrap_plugins:
plugin_mod, plugin_cls = plugin_info.get(
"plugin", [None, None])
plugin_cfg = plugin_info.get("config", None)
plugin_cfg = dict_merge(
config, plugin_cfg) if plugin_cfg is not None else config
try:
log.info("Bootstrapping plugin %s.%s ...", plugin_mod,
plugin_cls)
plugin = for_name(plugin_mod, plugin_cls)
plugin_func = getattr(plugin, method)
plugin_func(process, plugin_cfg, **kwargs)
except AbortBootstrap as abort:
raise
except Exception as ex:
log.exception("Error bootstrapping plugin %s.%s",
plugin_mod, plugin_cls)
def _call_plugins(self, method, process, config, **kwargs):
bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
if bootstrap_plugins is None:
log.warn("Bootstrapper called without bootstrap_plugins config")
# Finding the system actor ID. If found, construct call context headers.
# This may be called very early in bootstrap with no system actor yet existing
system_actor = get_system_actor()
if system_actor:
actor_headers = get_system_actor_header(system_actor)
else:
# Use default actor headers, not roles.
actor_headers = build_actor_header()
# Set the call context of the current process
with process.push_context(actor_headers):
for plugin_info in bootstrap_plugins:
plugin_mod, plugin_cls = plugin_info.get("plugin", [None,None])
plugin_cfg = plugin_info.get("config", None)
plugin_cfg = dict_merge(config, plugin_cfg) if plugin_cfg is not None else config
try:
log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls)
plugin = for_name(plugin_mod, plugin_cls)
plugin_func = getattr(plugin, method)
plugin_func(process, plugin_cfg, **kwargs)
except AbortBootstrap as abort:
raise
except Exception as ex:
log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)
def _create_service_instance(self, process_id, name, module, cls, config):
"""
Creates an instance of a "service", be it a Service, Agent, Stream, etc.
@rtype BaseService
@return An instance of a "service"
"""
# SERVICE INSTANCE.
service_instance = for_name(module, cls)
if not isinstance(service_instance, BaseService):
raise ContainerConfigError(
"Instantiated service not a BaseService %r" % service_instance)
# Prepare service instance
service_instance.errcause = ""
service_instance.id = process_id
service_instance.container = self.container
service_instance.CFG = config
service_instance._proc_name = name
service_instance._proc_start_time = time.time()
# start service dependencies (RPC clients)
self._start_service_dependencies(service_instance)
return service_instance
def _create_process_instance(self, process_id, name, module, cls, config):
"""
Creates an instance of a "service", be it a Service, Agent, Stream, etc.
@rtype BaseService
@return An instance of a "service"
"""
# SERVICE INSTANCE.
process_instance = for_name(module, cls)
if not isinstance(process_instance, BaseService):
raise ContainerConfigError("Instantiated service not a BaseService %r" % process_instance)
# Prepare service instance
process_instance.errcause = ""
process_instance.id = process_id
process_instance.container = self.container
process_instance.CFG = config
process_instance._proc_name = name
process_instance._proc_start_time = time.time()
# Unless the process has been started as part of another Org, default to the container Org or the ION Org
if config.has_key("org_name"):
process_instance.org_name = config["org_name"]
else:
process_instance.org_name = CFG.get_safe("container.org_name", CFG.get_safe("system.root_org", "ION"))
# Add stateful process operations
if hasattr(process_instance, "_flush_state"):
def _flush_state():
if not hasattr(process_instance, "_proc_state"):
process_instance._proc_state = {}
process_instance._proc_state_changed = False
return
process_instance.container.state_repository.put_state(process_instance.id, process_instance._proc_state)
process_instance._proc_state_changed = False
def _load_state():
if not hasattr(process_instance, "_proc_state"):
process_instance._proc_state = {}
try:
new_state = process_instance.container.state_repository.get_state(process_instance.id)
process_instance._proc_state.clear()
process_instance._proc_state.update(new_state)
process_instance._proc_state_changed = False
except Exception as ex:
log.warn("Process %s load state failed: %s", process_instance.id, str(ex))
process_instance._flush_state = _flush_state
process_instance._load_state = _load_state
process_start_mode = get_safe(config, "process.start_mode")
if process_start_mode == "RESTART":
if hasattr(process_instance, "_load_state"):
process_instance._load_state()
# start service dependencies (RPC clients)
self._start_process_dependencies(process_instance)
return process_instance
def _create_service_instance(self, process_id, name, module, cls, config):
# SERVICE INSTANCE.
service_instance = for_name(module, cls)
assert isinstance(service_instance, BaseService), "Instantiated service not a BaseService %r" % service_instance
# Prepare service instance
service_instance.errcause = ""
service_instance.id = "%s.%s" % (self.container.id, process_id)
service_instance.container = self.container
service_instance.CFG = config
service_instance._proc_name = name
# start service dependencies (RPC clients)
self._start_service_dependencies(service_instance)
return service_instance
def _create_service_instance(self, process_id, name, module, cls, config):
# SERVICE INSTANCE.
service_instance = for_name(module, cls)
if not isinstance(service_instance, BaseService):
raise ContainerConfigError(
"Instantiated service not a BaseService %r" % service_instance)
# Prepare service instance
service_instance.errcause = ""
service_instance.id = process_id
service_instance.container = self.container
service_instance.CFG = config
service_instance._proc_name = name
# start service dependencies (RPC clients)
self._start_service_dependencies(service_instance)
return service_instance
def _create_service_instance(self, process_id, name, module, cls, config):
"""
Creates an instance of a "service", be it a Service, Agent, Stream, etc.
@rtype BaseService
@return An instance of a "service"
"""
# SERVICE INSTANCE.
service_instance = for_name(module, cls)
if not isinstance(service_instance, BaseService):
raise ContainerConfigError("Instantiated service not a BaseService %r" % service_instance)
# Prepare service instance
service_instance.errcause = ""
service_instance.id = process_id
service_instance.container = self.container
service_instance.CFG = config
service_instance._proc_name = name
# start service dependencies (RPC clients)
self._start_service_dependencies(service_instance)
return service_instance
def _create_process_instance(self, process_id, name, module, cls, config):
"""
Creates an instance of a "service", be it a Service, Agent, Stream, etc.
@rtype BaseService
@return An instance of a "service"
"""
# SERVICE INSTANCE.
process_instance = for_name(module, cls)
if not isinstance(process_instance, BaseService):
raise ContainerConfigError(
"Instantiated service not a BaseService %r" % process_instance)
# Prepare service instance
process_instance.errcause = ""
process_instance.id = process_id
process_instance.container = self.container
process_instance.CFG = config
process_instance._proc_name = name
process_instance._proc_start_time = time.time()
#Unless the process has been started as part of another Org, default to the container Org or the ION Org
if config.has_key('org_name'):
process_instance.org_name = config['org_name']
else:
process_instance.org_name = CFG.get_safe(
'container.org_name', CFG.get_safe('system.root_org', 'ION'))
# Add stateful process operations
if hasattr(process_instance, "_flush_state"):
def _flush_state():
if not hasattr(process_instance, "_proc_state"):
process_instance._proc_state = {}
process_instance._proc_state_changed = False
return
process_instance.container.state_repository.put_state(
process_instance.id, process_instance._proc_state)
process_instance._proc_state_changed = False
def _load_state():
if not hasattr(process_instance, "_proc_state"):
process_instance._proc_state = {}
try:
new_state = process_instance.container.state_repository.get_state(
process_instance.id)
process_instance._proc_state.clear()
process_instance._proc_state.update(new_state)
process_instance._proc_state_changed = False
except Exception as ex:
log.warn("Process %s load state failed: %s",
process_instance.id, str(ex))
process_instance._flush_state = _flush_state
process_instance._load_state = _load_state
process_start_mode = get_safe(config, "process.start_mode")
if process_start_mode == "RESTART":
if hasattr(process_instance, "_load_state"):
process_instance._load_state()
# start service dependencies (RPC clients)
self._start_process_dependencies(process_instance)
return process_instance
def _create_process_instance(self, process_id, name, module, cls, config):
"""
Creates an instance of a "service", be it a Service, Agent, Stream, etc.
@rtype BaseService
@return An instance of a "service"
"""
# SERVICE INSTANCE.
process_instance = for_name(module, cls)
if not isinstance(process_instance, BaseService):
raise ContainerConfigError("Instantiated service not a BaseService %r" % process_instance)
# Prepare service instance
process_instance.errcause = ""
process_instance.id = process_id
process_instance.container = self.container
process_instance.CFG = config
process_instance._proc_name = name
process_instance._proc_start_time = time.time()
#Unless the process has been started as part of another Org, default to the container Org or the ION Org
if config.has_key('org_governance_name'):
process_instance.org_governance_name = config['org_governance_name']
else:
process_instance.org_governance_name = CFG.get_safe('container.org_name', CFG.get_safe('system.root_org', 'ION'))
# Add stateful process operations
if hasattr(process_instance, "_flush_state"):
def _flush_state():
with process_instance._state_lock:
state_obj = process_instance.container.state_repository.put_state(process_instance.id, process_instance._proc_state,
state_obj=process_instance._proc_state_obj)
state_obj.state = None # Make sure memory footprint is low for larger states
process_instance._proc_state_obj = state_obj
process_instance._proc_state_changed = False
def _load_state():
if not hasattr(process_instance, "_proc_state"):
process_instance._proc_state = {}
try:
with process_instance._state_lock:
new_state, state_obj = process_instance.container.state_repository.get_state(process_instance.id)
process_instance._proc_state.clear()
process_instance._proc_state.update(new_state)
process_instance._proc_state_obj = state_obj
process_instance._proc_state_changed = False
except NotFound as nf:
log.debug("No persisted state available for process %s", process_instance.id)
except Exception as ex:
log.warn("Process %s load state failed: %s", process_instance.id, str(ex))
process_instance._flush_state = _flush_state
process_instance._load_state = _load_state
process_instance._state_lock = RLock()
process_instance._proc_state = {}
process_instance._proc_state_obj = None
process_instance._proc_state_changed = False
# PROCESS RESTART: Need to check whether this process had persisted state.
# Note: This could happen anytime during a system run, not just on RESTART boot
log.debug("Loading persisted state for process %s", process_id)
process_instance._load_state()
# start service dependencies (RPC clients)
self._start_process_dependencies(process_instance)
return process_instance
def _transform_data(binding, data, module, cls, kwargs={}):
transform = for_name(module, cls)
validate_is_instance(
transform, TransformFunction,
'%s.%s is not a TransformFunction' % (module, cls))
return transform.execute(data, **kwargs)
def _transform_data(binding, data, module, cls, kwargs={}):
transform = for_name(module,cls)
validate_is_instance(transform,TransformFunction,'%s.%s is not a TransformFunction' % (module,cls))
return transform.execute(data,**kwargs)