class Loader(YomboLibrary, object):
"""
Responsible for loading libraries, and then delegating loading modules to
the modules library.
Libraries are never reloaded, however, during a reconfiguration,
modules are unloaded, and then reloaded after configurations are done
being downloaded.
"""
@property
def operation_mode(self):
return self._operation_mode
@operation_mode.setter
def operation_mode(self, val):
self.loadedLibraries['atoms']['loader.operation_mode'] = val
self._operation_mode = val
def __getitem__(self, component_requested):
"""
"""
logger.debug("looking for: {component_requested}", component_requested=component_requested)
if component_requested in self.loadedComponents:
logger.debug("found by loadedComponents! {component_requested}", component_requested=component_requested)
return self.loadedComponents[component_requested]
elif component_requested in self.loadedLibraries:
logger.debug("found by loadedLibraries! {component_requested}", component_requested=component_requested)
return self.loadedLibraries[component_requested]
elif component_requested in self._moduleLibrary:
logger.debug("found by self._moduleLibrary! {component_requested}", component_requested=self._moduleLibrary)
return self._moduleLibrary[component_requested]
else:
raise YomboWarning("Loader could not find requested component: {%s}"
% component_requested, '101', '__getitem__', 'loader')
def __init__(self, testing=False):
self.unittest = testing
self._moduleLibrary = None
YomboLibrary.__init__(self)
self.loadedComponents = FuzzySearch({self._FullName.lower(): self}, .95)
self.loadedLibraries = FuzzySearch({self._Name.lower(): self}, .95)
self.libraryNames = {}
self.__localModuleVars = {}
self._moduleLibrary = None
self._invoke_list_cache = {} # Store a list of hooks that exist or not. A cache.
self._operation_mode = None # One of: firstrun, config, run
self.sigint = False # will be set to true if SIGINT is received
self.hook_counts = OrderedDict() # keep track of hook names, and how many times it's called.
reactor.addSystemEventTrigger("before", "shutdown", self.shutdown)
# signal(SIGINT, self.shutdown2)
def shutdown(self):
"""
This is called if SIGINT (ctrl-c) was caught. Very useful incase it was called during startup.
:return:
"""
print "SIGINT received - twisted"
self.sigint = True
# def shutdown2(self, signum, frame):
# """
# This is called if SIGINT (ctrl-c) was caught. Very useful incase it was called during startup.
# :return:
# """
# print 'Signal handler called with signal %s' % signum
# print "WHAT! I was called - signal"
# self.sigint = True
# reactor.stop()
@inlineCallbacks
def start(self): #on startup, load libraried, then modules
"""
This is effectively the main start function.
This function is called when the gateway is to startup. In turn,
this function will load all the components and modules of the gateway.
"""
logger.info("Importing libraries, this can take a few moments.")
yield self.import_libraries() # import and init all libraries
# returnValue(None)
# if self.sigint:
# return
logger.debug("Calling load functions of libraries.")
for name, config in HARD_LOAD.iteritems():
# print "sigint: %s" % self.sigint
# if self.sigint:
# return
self._log_loader('debug', name, 'library', 'load', 'About to call _load_.')
if self.check_operation_mode(config['operation_mode']):
HARD_LOAD[name]['_load_'] = 'Starting'
libraryName = name.lower()
yield self.library_invoke(libraryName, "_load_", self)
HARD_LOAD[name]['_load_'] = True
else:
HARD_LOAD[name]['_load_'] = False
self._log_loader('debug', name, 'library', 'load', 'Finished call to _load_.')
self._moduleLibrary = self.loadedLibraries['modules']
# logger.debug("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1Calling start function of libraries.")
for name, config in HARD_LOAD.iteritems():
if self.sigint:
return
self._log_loader('debug', name, 'library', 'start', 'About to call _start_.')
if self.check_operation_mode(config['operation_mode']):
libraryName = name.lower()
yield self.library_invoke(libraryName, "_start_", self)
HARD_LOAD[name]['_start_'] = True
else:
HARD_LOAD[name]['_start_'] = False
for name, config in HARD_LOAD.iteritems():
if self.sigint:
return
self._log_loader('debug', name, 'library', 'started', 'About to call _started_.')
if self.check_operation_mode(config['operation_mode']):
libraryName = name.lower()
yield self.library_invoke(libraryName, "_started_", self)
HARD_LOAD[name]['_started_'] = True
else:
HARD_LOAD[name]['_started_'] = False
yield self._moduleLibrary.load_modules()
self.loadedLibraries['notifications'].add({'title': 'System started',
'message': 'System successfully started.', 'timeout': 300, 'source': 'Yombo Gateway System',
'persist': False,
'always_show': False,
})
@inlineCallbacks
def unload(self):
"""
Called when the gateway should stop. This will gracefully stop the gateway.
First, unload all modules, then unload all components.
"""
self.sigint = True # it's 99.999% true - usually only shutdown due to this.
if self._moduleLibrary is not None:
yield self._moduleLibrary.unload_modules()
yield self.unload_components()
def Times_i18n_atoms(self, **kwargs):
return [
{'loader.operation_mode': {
'en': 'One of: firstrun, run, config',
},
},
]
def check_component_status(self, name, function):
if name in HARD_LOAD:
if function in HARD_LOAD[name]:
return HARD_LOAD[name][function]
return None
def _log_loader(self, level, label, type, method, msg=""):
"""
A common log format for loading/unloading libraries and modules.
:param level: Log level - debug, info, warn...
:param label: Module label "x10", "messages"
:param type: Type of item being loaded: library, module
:param method: Method being called.
:param msg: Optional message to include.
:return:
"""
logit = func = getattr(logger, level)
logit("Loader: {label}({type})::{method} - {msg}", label=label, type=type, method=method, msg=msg)
@inlineCallbacks
def import_libraries(self):
"""
Import then "init" all libraries. Call "loadLibraries" when done.
"""
logger.debug("Importing server libraries.")
for name, config in HARD_LOAD.iteritems():
if self.sigint:
return
HARD_LOAD[name]['__init__'] = 'Starting'
pathName = "yombo.lib.%s" % name
self.import_component(pathName, name, 'library')
HARD_LOAD[name]['__init__'] = True
logger.debug("Calling init functions of libraries.")
for name, config in HARD_LOAD.iteritems():
if self.sigint:
return
if self.check_operation_mode(config['operation_mode']) is False:
HARD_LOAD[name]['_init_'] = False
continue
HARD_LOAD[name]['_init_'] = 'Starting'
self._log_loader('debug', name, 'library', 'init', 'About to call _init_.')
component = name.lower()
library = self.loadedLibraries[component]
library._AMQP = self.loadedLibraries['amqp']
library._Atoms = self.loadedLibraries['atoms']
library._Commands = self.loadedLibraries['commands']
library._Configs = self.loadedLibraries['configuration']
library._Devices = self.loadedLibraries['devices']
library._DeviceTypes = self.loadedLibraries['devicetypes']
library._InputTypes = self.loadedLibraries['inputtypes']
library._Libraries = self.loadedLibraries
library._Loader = self
library._Modules = self._moduleLibrary
library._Notifications = self.loadedLibraries['notifications']
library._Localize = self.loadedLibraries['localize']
library._MQTT = self.loadedLibraries['mqtt']
library._SQLDict = self.loadedLibraries['sqldict']
library._States = self.loadedLibraries['states']
library._Statistics = self.loadedLibraries['statistics']
library._YomboAPI = self.loadedLibraries['yomboapi']
library._Variables = self.loadedLibraries['variables']
if hasattr(library, '_init_') and callable(library._init_) \
and yombo.utils.get_method_definition_level(library._init_) != 'yombo.core.module.YomboModule':
d = yield maybeDeferred(library._init_)
self._log_loader('debug', name, 'library', 'init', 'Finished to call _init_.')
# try:
# d = yield maybeDeferred(library._init_, self)
# except YomboCritical, e:
# logger.error("---==(Critical Server Error in init function for library: {name})==----", name=name)
# logger.error("--------------------------------------------------------")
# logger.error("Error message: {e}", e=e)
# logger.error("--------------------------------------------------------")
# e.exit()
# except:
# logger.error("-------==(Error in init function for library: {name})==---------", name=name)
# logger.error("1:: {e}", e=sys.exc_info())
# logger.error("---------------==(Traceback)==--------------------------")
# logger.error("{e}", e=traceback.print_exc(file=sys.stdout))
# logger.error("--------------------------------------------------------")
HARD_LOAD[name]['_init_'] = True
else:
logger.error("----==(Library doesn't have init function: {name})==-----", name=name)
def check_operation_mode(self, allowed):
"""
Checks if something should be run based on the current operation_mode.
:param config: Either string or list or posible operation_modes
:return: True/False
"""
op_mode = self.operation_mode
if op_mode is None:
return True
def check_operation_mode_inside(mode, op_mode):
if mode == 'all':
return True
elif mode == op_mode:
return True
return False
if isinstance(allowed, basestring): # we have a string
return check_operation_mode_inside(allowed, op_mode)
else: # we have something else
for item in allowed:
if check_operation_mode_inside(item, op_mode):
return True
@inlineCallbacks
def library_invoke(self, requested_library, hook, called_by=None, **kwargs):
"""
Invokes a hook for a a given library. Passes kwargs in, returns the results to caller.
"""
if requested_library not in self.loadedLibraries:
returnValue(None)
if called_by is not None:
called_by = called_by._FullName
else:
called_by = 'Unknown'
cache_key = requested_library + hook
if cache_key in self._invoke_list_cache:
if self._invoke_list_cache[cache_key] is False:
# logger.warn("Cache hook ({cache_key})...SKIPPED", cache_key=cache_key)
returnValue(None) # skip. We already know function doesn't exist.
library = self.loadedLibraries[requested_library]
if requested_library == 'Loader':
returnValue(None)
if not (hook.startswith("_") and hook.endswith("_")):
hook = library._Name.lower() + "_" + hook
if hasattr(library, hook):
method = getattr(library, hook)
self._log_loader('debug', requested_library, 'library', 'library_invoke', 'About to call: %s' % hook)
if callable(method):
if library._Name not in self.hook_counts:
self.hook_counts[library._Name] = {}
if hook not in self.hook_counts:
self.hook_counts[library._Name][hook] = {'Total Count': {'count': 0}}
# print "hook counts: %s" % self.hook_counts
# print "hook counts: %s" % self.hook_counts[library._Name][hook]
if called_by not in self.hook_counts[library._Name][hook]:
self.hook_counts[library._Name][hook][called_by] = {'count': 0}
self.hook_counts[library._Name][hook][called_by]['count'] = self.hook_counts[library._Name][hook][called_by]['count'] + 1
self.hook_counts[library._Name][hook]['Total Count']['count'] = self.hook_counts[library._Name][hook]['Total Count']['count'] + 1
self._invoke_list_cache[cache_key] = True
results = yield maybeDeferred(method, **kwargs)
self._log_loader('debug', requested_library, 'library', 'library_invoke', 'Finished with call: %s' % hook)
returnValue(results)
else:
logger.warn("Cache library hook ({library}:{hook})...setting false", library=library._FullName, hook=hook)
logger.debug("----==(Library {library} doesn't have a callable function: {function})==-----", library=library._FullName, function=hook)
raise YomboWarning("Hook is not callable: %s" % hook)
else:
# logger.debug("Cache hook ({library}:{hook})...setting false", library=library._FullName, hook=hook)
self._invoke_list_cache[cache_key] = False
def library_invoke_all(self, hook, fullName=False, called_by=None, **kwargs):
"""
Calls library_invoke for all loaded libraries.
"""
results = {}
to_process = {}
if 'components' in kwargs:
to_process = kwargs['components']
else:
for library_name, library in self.loadedLibraries.iteritems():
# print "library %s" % library
label = library._FullName.lower() if fullName else library._Name.lower()
to_process[library_name] = label
for library_name, library in self.loadedLibraries.iteritems():
# logger.debug("invoke all:{libraryName} -> {hook}", libraryName=library_name, hook=hook )
try:
d = self.library_invoke(library_name, hook, called_by=called_by, **kwargs)
if isinstance(d, Deferred):
result = getattr(d, 'result', None)
if result is not None:
# logger.warn("1111aaa:: {libraryName} {hook} {result}", libraryName=libraryName, hook=hook, result=result)
results[library._FullName] = result
except YomboWarning:
pass
except YomboHookStopProcessing as e:
e.collected = results
e.by_who = label
raise
return results
def import_component(self, pathName, componentName, componentType, componentUUID=None):
"""
Load component of given name. Can be a core library, or a module.
"""
pymodulename = pathName.lower()
self._log_loader('debug', componentName, componentType, 'import', 'About to import.')
try:
pyclassname = ReSearch("(?<=\.)([^.]+)$", pathName).group(1)
except AttributeError:
self._log_loader('error', componentName, componentType, 'import', 'Not found. Path: %s' % pathName)
logger.error("Library or Module not found: {pathName}", pathName=pathName)
raise YomboCritical("Library or Module not found: %s", pathName)
try:
# print "pymodulename: %s" % pymodulename
module_root = __import__(pymodulename, globals(), locals(), [], 0)
pass
except ImportError as detail:
self._log_loader('error', componentName, componentType, 'import', 'Not found. Path: %s' % pathName)
logger.error("--------==(Error: Library or Module not found)==--------")
logger.error("----Name: {pathName}, Details: {detail}", pathName=pathName, detail=detail)
logger.error("--------------------------------------------------------")
logger.error("{error}", error=sys.exc_info())
logger.error("---------------==(Traceback)==--------------------------")
logger.error("{trace}", trace=traceback.print_exc(file=sys.stdout))
logger.error("--------------------------------------------------------")
return
module_tail = reduce(lambda p1, p2: getattr(p1, p2), [module_root, ]+pymodulename.split('.')[1:])
# print "module_tail: %s pyclassname: %s" % (module_tail, pyclassname)
klass = getattr(module_tail, pyclassname)
# print "klass: %s " % klass
# Put the component into various lists for mgmt
if not callable(klass):
logger.warn("Unable to start class '{classname}', it's not callable.", classname=pyclassname)
return
try:
# Instantiate the class
moduleinst = klass() # start the class, only libraries get the loader
if componentType == 'library':
if componentName.lower() == 'modules':
self._moduleLibrary = moduleinst
self.loadedComponents["yombo.gateway.lib." + str(componentName.lower())] = moduleinst
self.loadedLibraries[str(componentName.lower())] = moduleinst
# this is mostly for manhole module, but maybe useful elsewhere?
temp = componentName.split(".")
self.libraryNames[temp[-1]] = moduleinst
else:
self.loadedComponents["yombo.gateway.modules." + str(componentName.lower())] = moduleinst
self._moduleLibrary.add_imported_module(componentUUID, str(componentName.lower()), moduleinst)
except YomboCritical, e:
logger.debug("@!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
logger.debug("{e}", e=e)
logger.debug("@!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
e.exit()
raise
class VoiceCmds(YomboLibrary):
"""
Store all voice commands here and perfrom searches.
The purpose of this class is two fold: it provides a single repository for
all possible noun/verb combinations, and provides an ability to *not* have
to be 100% accurate when looking up noun/verb pairs.
Convert "voice_cmd" strings into voice commands.
Also, provides searching for voice commands.
"""
def __getitem__(self, voice_command_requested):
"""
Search for a voice command. this will return a dictionary: id, cmd object, device object.
You can use the device and command objects as desired.
>>> self._VoiceCmds['living room light on'] #by name
:param voicecmd_requested: Search all available voice commands for a phrase.
:type voicecmd_requested: string
:return: dict containing: 'id', 'cmd', 'device'
:rtype: dict
"""
return self.get(voice_command_requested)
def __len__(self):
return len(self.__yombocommands)
def __contains__(self, command_requested):
try:
self.get_command(command_requested)
return True
except:
return False
def _init_(self):
"""
Construct a new voice_cmds Instance
items is an dictionary to copy items from (optional)
limiter is the match ratio below which mathes should not be considered
"""
self.voice_command_strings = FuzzySearch(None, .80)
self.voice_command_data = {}
self.commandsByVoice = self._Libraries['commands'].get_commands_by_voice()
def _module_prestart_(self, **kwargs):
"""
Implements the _module_prestart_ and is called after _load_ is called for all the modules.
Expects a list of events to subscribe to.
**Hooks called**:
* _voicecmds_add_ : Expects a list of message subscription events to subscrib to.
**Usage**:
.. code-block:: python
def ModuleName_voice_cmds_load(self, **kwargs):
return ['status']
"""
voicecommands_to_add = global_invoke_all('_voicecmds_add_')
# logger.info("voicecommands_to_add: {voice_cmds}", voice_cmds=voicecommands_to_add)
for componentName, voice_cmds in voicecommands_to_add.iteritems():
if voice_cmds is None:
continue
for list in voice_cmds:
logger.debug("For module '{fullName}', adding voice_cmd: {voice_cmd}, order: {order}", voice_cmd=list['voice_cmd'], fullName=componentName, order=list['order'])
self.add_by_string(list['voice_cmd'], list['call_back'], list['device'], list['order'])
def get_all(self):
"""
Returns a list of voice commands.
:return:
"""
results = {}
for voice, voice_id in self.voice_command_strings.iteritems():
results[voice] = self.voice_command_data[voice_id]
return results
def get(self, voice_command_requested, limiter_override=None):
"""
Search for a voice command. this will return a dictionary: id, cmd object, device object.
You can use the device and command objects as desired.
>>> self._VoiceCmds['living room light on'] #by name
:param voicecmd_requested: Search all available voice commands for a phrase.
:type voicecmd_requested: string
:return: dict containing: 'id', 'cmd', 'device'
:rtype: dict
"""
try:
result = self.voice_commands.search2(voice_command_requested, limiter_override=limiter_override)
self._Statistics.increment("lib.voicecmds.search.found", bucket_time=30, anon=True)
return self.voice_command_data[result]
except:
self._Statistics.increment("lib.voicecmds.search.not_found", bucket_time=30, anon=True)
return None
def add_by_string(self, voice_string, call_back = None, device = None, order = 'devicecmd'):
"""
Adds a voice command by using a string like "desk lamp [on, off]". This will add the following voice commands
based on the value of 'order'. If both is provided, the following will be added:
* desklamp on
* desklamp off
* off desklamp
* on desklamp
You can specify 'order' as: both, devicecmd, cmddevice. This determines what ordering the voice commands
are added. In the above example, specifying 'devicecmd', then only the first two items are added.
Either a callback function must be provided or a device must be provided. Otherwise, the voice command
will not be added and a YomboWarning will be raised if either or both are defined.
:raises YomboWarning: If voiceString or destination is invalid.
:param voice_string: Voice command string to process: "desklamp [on, off]"
:type voice_string: string
:param call_back: A function to send the voice command id, device, and command objects to.
:type call_back: pointer to function
:param device: A device id or device label to search for a matching device.
:type device: string
:param order: The ordering in which to add voice command text lookup. Default: devicecmd
:type order: string
"""
logger.debug("Adding voice command: {voice_string}", voice_string=voice_string)
if call_back is None and device is None:
raise YomboWarning("'call_back' and 'device' are mising.", 1000, 'add_by_string', 'voicecmds')
if call_back is not None and device is not None:
raise YomboWarning("Either specifiy 'call_back' or 'device', not both.", 1001, 'add_by_string', 'voicecmds')
try:
tag_re = re.compile('(%s.*?%s)' % (re.escape('['), re.escape(']')))
string_parts = tag_re.split(voice_string)
string_parts = filter(None, string_parts) # remove empty bits
device_obj = None
if device is not None:
if isclass(device):
device_obj = device
else:
device_obj = self._Devices[device]
commands = []
except:
raise YomboWarning("Invalid format for 'voice_string'", 1010, 'add_by_string', 'voicecmds')
if len(string_parts) > 2:
raise YomboWarning("Invalid format for 'voice_string'", 1003, 'add_by_string', 'voicecmds')
for part in range(len(string_parts)):
string_parts[part] = string_parts[part].strip()
if string_parts[part].startswith("[") and string_parts[part].endswith("]"):
temp_commands = string_parts[part][1:-1].split(',')
for cmd in range(len(temp_commands)):
cmd_temp = temp_commands[cmd].strip()
if len(cmd_temp) > 0:
commands.append(cmd_temp)
else:
device_label = string_parts[part]
if len(commands) == 0:
raise YomboWarning("No commands found in voice_string.", 1003, 'add_by_string', 'voicecmds')
# logger.debug("commands by voice: {commandsByVoice}:{verb}", commandsByVoice=self.commandsByVoice, verb=verb)
for cmd in commands:
if cmd not in self.commandsByVoice:
continue
command = self.commandsByVoice[cmd]
vc_id = random_string(length=12)
if order == 'both':
self.voice_command_strings["%s %s" % (device_label, cmd)] = vc_id
self.voice_command_strings["%s %s" % (cmd, device_label)] = vc_id
elif order == 'devicecmd':
self.voice_command_strings["%s %s" % (device_label, cmd)] = vc_id
else:
self.voice_command_strings["%s %s" % (cmd, device_label)] = vc_id
self.voice_command_data[vc_id] = {
'id': vc_id,
'cmd': command,
'device': device_obj,
'call_back': call_back
}
