def module(unload=None):
from sys import modules
if unload is None:
return modules.keys()
if unload in modules.keys():
del modules[unload]
return "Module unload {} done.".format(unload)
return "Module unload {} failed.".format(unload)
def module(unload=None):
"""
List and unload LM modules
"""
from sys import modules
if unload is None:
return list(modules.keys())
if unload in modules.keys():
del modules[unload]
return "Module unload {} done.".format(unload)
return "Module unload {} failed.".format(unload)
def execute_LM_function_Core(argument_list, SocketServerObj=None):
"""
[1] module name (LM)
[2] function
[3...] parameters (separator: space)
NOTE: SocketServerObj is None from Interrupts and Hooks - shared functionality
"""
json_mode = False
# Check json mode for LM execution
if argument_list[-1] == '>json':
del argument_list[-1]
json_mode = True
if len(argument_list) >= 2:
LM_name, LM_function, LM_function_params = "LM_{}".format(argument_list[0]), argument_list[1], ', '.join(argument_list[2:])
try:
# --- LM LOAD & EXECUTE --- #
# [1] LOAD MODULE
if LM_name not in modules.keys():
exec("import {}".format(LM_name))
# [2] EXECUTE FUNCTION FROM MODULE - over SocketServerObj or /dev/null
lm_output = eval("{}.{}({})".format(LM_name, LM_function, LM_function_params))
if SocketServerObj is not None:
if not json_mode and isinstance(lm_output, dict):
# human readable format (not json mode) but dict
lm_output = '\n'.join(["{}: {}".format(key, value) for key, value in lm_output.items()])
SocketServerObj.reply_message(str(lm_output))
else:
# native return value (not dict) OR json mode raw dict output
SocketServerObj.reply_message(str(lm_output))
# ------------------------- #
except Exception as e:
# ERROR MSG: - over SocketServerObj or stdout
print("execute_LM_function {}->{}: {}".format(LM_name, LM_function, e))
if SocketServerObj is not None:
SocketServerObj.reply_message("execute_LM_function {}->{}: {}".format(LM_name, LM_function, e))
if 'memory allocation failed' in str(e) or 'is not defined' in str(e):
# UNLOAD MODULE IF MEMORY ERROR HAPPENED
if LM_name in modules.keys():
del modules[LM_name]
return False
# RETURN WITH HEALTH STATE - TRUE :) -> NO ACTION -or- FALSE :( -> RECOVERY ACTION
return True
# Syntax error show help msg
print("SHELL: Missing argument: [1](LM)module [2]function [3...]optional params")
if SocketServerObj is not None:
SocketServerObj.reply_message("SHELL: type help for single word commands (built-in)")
SocketServerObj.reply_message("SHELL: for LM exec: [1](LM)module [2]function [3...]optional params")
# RETURN WITH HEALTH STATE - TRUE :) -> NO ACTION -or- FALSE :( -> RECOVERY ACTION
return True
def unregister(self):
from sys import modules as sys_modules
for cls in reversed(self.prefs_props):
if getattr(cls, 'is_registered', False):
bpy.utils.unregister_class(cls)
self.prefs_props.remove(cls)
for cls in reversed(self.ordered_classes):
if getattr(cls, 'is_registered', False):
bpy.utils.unregister_class(cls)
for module in self.modules:
if module.__name__ == __name__:
continue
if hasattr(module, "unregister"):
try:
module.unregister()
except Exception as error:
print(f"Can't unregister module:\t{module.__name__}\n{error}")
for module in self.modules:
if module.__name__ == __name__:
continue
if module.__name__ in sys_modules:
del (sys_modules[module.__name__])
# Remove the remaining entries for the folder, zpy, and zpy.functions
for module_name in reversed(list(sys_modules.keys())):
# if module_name == __name__: # This should exist anyway
# continue
if module_name.startswith(self.__package__ + '.') or module_name == self.__package__:
del sys_modules[module_name]
self.keymaps.clear()
def unload_plugins(self):
self.execute_event('on_unload')
for module in [
module for module in modules.keys() if "plugins." in module
]:
del modules[module]
def func(self, lib, opts, args):
"""Command handler for the metasync function.
"""
pretend = opts.pretend
source = opts.source
query = ui.decargs(args)
sources = {}
for player in source:
__import__('beetsplug.metasync', fromlist=[str(player)])
module = 'beetsplug.metasync.' + player
if module not in modules.keys():
log.error(u'Unknown metadata source \'' + player + '\'')
continue
classes = inspect.getmembers(modules[module], inspect.isclass)
for entry in classes:
if entry[0].lower() == player:
sources[player] = entry[1]()
else:
continue
for item in lib.items(query):
for player in sources.values():
player.get_data(item)
changed = ui.show_model_changes(item)
if changed and not pretend:
item.store()
def get_moudles(self):
'''
get all imported modules
'''
x = [x.split(".")[0] for x in modules.keys() if not x.startswith("_")]
self.d = '(QBAnalyzer∞ proudly uses/depends on Docker, Python3, Bootstrap, Javascript, jquery, D3.js, JSON, Html, Sqlite3, Wikipedia, Linux\\MacOS\\Windows\\Android documentation, software77, MITRE ATT&CK™, sc0ty, hexacorn, radare2, dmg2img, font-awesome, flag-icon-css, {} and tons of researches ..) If i missed a reference/dependency, please let me know!'.format(
', '.join(list(set(x))))
def require(modulename, package=None):
"""
Load, or reload a module.
When under heavy development, a user's tool might consist of multiple
modules. If those are imported using the standard 'import' mechanism,
there is no guarantee that the Python implementation will re-read
and re-evaluate the module's Python code. In fact, it usually doesn't.
What should be done instead is 'reload()'-ing that module.
This is a simple helper function that will do just that: In case the
module doesn't exist, it 'import's it, and if it does exist,
'reload()'s it.
For more information, see: <http://www.hexblog.com/?p=749>.
"""
if modulename in modules.keys():
reload(modules[modulename])
else:
import importlib
import inspect
m = importlib.import_module(modulename, package)
frame_obj, filename, line_number, function_name, lines, index = inspect.stack(
)[1]
importer_module = inspect.getmodule(frame_obj)
if importer_module is None: # No importer module; called from command line
importer_module = modules['__main__']
setattr(importer_module, modulename, m)
modules[modulename] = m
def get_modules(name="proxy_lib"):
result = []
for module_name in modules.keys():
if name.lower() in module_name.lower():
result.append("\t> {:20} : {}".format(module_name,
modules[module_name]))
return result
def func(self, lib, opts, args):
"""Command handler for the metasync function.
"""
pretend = opts.pretend
source = opts.source
query = ui.decargs(args)
sources = {}
for player in source:
__import__('beetsplug.metasync', fromlist=[str(player)])
module = 'beetsplug.metasync.' + player
if module not in modules.keys():
log.error(u'Unknown metadata source \'' + player + '\'')
continue
classes = inspect.getmembers(modules[module], inspect.isclass)
for entry in classes:
if entry[0].lower() == player:
sources[player] = entry[1]()
else:
continue
for item in lib.items(query):
for player in sources.values():
player.get_data(item)
changed = ui.show_model_changes(item)
if changed and not pretend:
item.store()
def _process_modulename(self, modulename):
"""
Sets the modulename from input (if specified) or automatically.
"""
if modulename:
if modulename in modules.keys():
raise NameError(
"Module name has already been used in this instance of Python."
)
self._modulename = modulename
else:
while self._modulename in modules.keys():
self._modulename = count_up(self._modulename)
modulefile = self._tmpfile(self._modulename + ".so")
if path.isfile(modulefile):
raise OSError("Module file already exists.")
def reload():
import importlib
from sys import modules
keys = list(modules.keys())
for key in keys:
if key.startswith('WiertarBot.commands.'):
importlib.reload(modules[key])
def module(unload=None):
from sys import modules
if unload is None:
return modules.keys()
try:
del modules[unload]
return "Module unload {} done.".format(unload)
except Exception as e:
return "Module unload failed: {}".format(e)
def getpin(key='builtin'):
from sys import modules
from LogicalPins import get_pin_on_platform_by_key
return {
key:
get_pin_on_platform_by_key(key),
'pinmap':
', '.join((mdl.replace('LP_', '').split('.')[0]
for mdl in modules.keys() if mdl.startswith('LP_')))
}
def SettingsInputPressed(self, name):
if name == "Open Mods Folder":
os.startfile(ModsDirectory)
elif name == "Reload Mods":
for mod in list(modules.keys()):
if mod.startswith('Mods.'):
del modules[mod]
unrealsdk.Mods = []
modules["Mods"] = reload(modules["Mods"])
elif name == "Help":
webbrowser.open(
"https://github.com/bl-sdk/BL2-Python-Plugins/wiki")
def close():
if '.mockblok.' in modules:
mod_2_del = []
for mod in modules.keys():
if '.mockblok.' in mod:
mod_2_del.append(mod)
for mod in mod_2_del:
del modules[mod]
del BlokManager.bloks['mockblok']
BlokManager.ordered_bloks.remove('mockblok')
def tearDown(self):
super(TestImportManager, self).tearDown()
if '.mockblok.' in modules:
mod_2_del = []
for mod in modules.keys():
if '.mockblok.' in mod:
mod_2_del.append(mod)
for mod in mod_2_del:
del modules[mod]
del BlokManager.bloks['mockblok']
BlokManager.ordered_bloks.remove('mockblok')
def tearDown(self):
super(TestImportManager, self).tearDown()
if '.mockblok.' in modules:
mod_2_del = []
for mod in modules.keys():
if '.mockblok.' in mod:
mod_2_del.append(mod)
for mod in mod_2_del:
del modules[mod]
del BlokManager.bloks['mockblok']
BlokManager.ordered_bloks.remove('mockblok')
def __getuserloadedmodules__():
''' 3 types of modules; modules the system loads, system modules the
user specifically loads with import and user defined modules
this function creates a list of the last 2 types '''
python27builtin = ["copy_reg", "sre_compile", "_sre", "encodings", "site", \
"__builtin__", "sysconfig", "__main__", \
"encodings.encodings", "abc", "posixpath", "_weakrefset", \
"errno", "encodings.codecs", "sre_constants", "re", \
"_abcoll", "types", "_codecs", "encodings.__builtin__", \
"_warnings", "genericpath", "stat", "zipimport", \
"_sysconfigdata", "warnings", "UserDict", "encodings.utf_8", \
"sys", "codecs","_locale", "signal", \
"traceback", "linecache", "posix", "encodings.aliases", \
"exceptions", "sre_parse", "os", "_weakref"]
python27alwaysloaded = ['heapq', '_sha512', 'functools', 'random', 'unittest.types', \
'unittest.pprint', 'pprint', 'collections', 'unittest.sys', \
'unittest.main', 'unittest.functools',\
'unittest.util', 'unittest.StringIO', 'unittest.re', 'cStringIO',\
'unittest.difflib', 'math', 'unittest.case', 'unittest.suite',\
'unittest.runner', 'unittest.loader', '_functools', \
'StringIO', 'unittest.signal', 'weakref', 'itertools', 'marshal', \
'__future__', '_collections', 'unittest','operator', \
'_heapq', 'unittest.collections', 'binascii', '_sha256', \
'unittest.fnmatch', '_struct', 'hashlib', \
'keyword', 'unittest.weakref', 'fnmatch',
'_random', '_md5', 'unittest.result',
'unittest.time', '_sha', 'unittest.signals', 'difflib', 'unittest.warnings',
'time', 'unittest.traceback', 'unittest.os']
from sys import modules
mods = modules.keys()
for builtinmod in python27builtin:
try:
mods.remove(builtinmod)
except:
log.log(PRIORITY.INFO,
msg="info: could not pop builtin mod [" + builtinmod + "]")
for alwaysloaded in python27alwaysloaded:
try:
mods.remove(alwaysloaded)
except:
log.log(PRIORITY.INFO,
msg="info: could not pop alwaysloaded mo [" +
alwaysloaded + "]")
return (mods)
def __getuserloadedmodules__():
''' 3 types of modules; modules the system loads, system modules the
user specifically loads with import and user defined modules
this function creates a list of the last 2 types '''
python27builtin = ["copy_reg", "sre_compile", "_sre", "encodings", "site", \
"__builtin__", "sysconfig", "__main__", \
"encodings.encodings", "abc", "posixpath", "_weakrefset", \
"errno", "encodings.codecs", "sre_constants", "re", \
"_abcoll", "types", "_codecs", "encodings.__builtin__", \
"_warnings", "genericpath", "stat", "zipimport", \
"_sysconfigdata", "warnings", "UserDict", "encodings.utf_8", \
"sys", "codecs","_locale", "signal", \
"traceback", "linecache", "posix", "encodings.aliases", \
"exceptions", "sre_parse", "os", "_weakref"]
python27alwaysloaded = ['heapq', '_sha512', 'functools', 'random', 'unittest.types', \
'unittest.pprint', 'pprint', 'collections', 'unittest.sys', \
'unittest.main', 'unittest.functools',\
'unittest.util', 'unittest.StringIO', 'unittest.re', 'cStringIO',\
'unittest.difflib', 'math', 'unittest.case', 'unittest.suite',\
'unittest.runner', 'unittest.loader', '_functools', \
'StringIO', 'unittest.signal', 'weakref', 'itertools', 'marshal', \
'__future__', '_collections', 'unittest','operator', \
'_heapq', 'unittest.collections', 'binascii', '_sha256', \
'unittest.fnmatch', '_struct', 'hashlib', \
'keyword', 'unittest.weakref', 'fnmatch',
'_random', '_md5', 'unittest.result',
'unittest.time', '_sha', 'unittest.signals', 'difflib', 'unittest.warnings',
'time', 'unittest.traceback', 'unittest.os']
from sys import modules
mods = modules.keys()
for builtinmod in python27builtin:
try:
mods.remove(builtinmod)
except:
print "info: could not pop builtin mod",builtinmod
for alwaysloaded in python27alwaysloaded:
try:
mods.remove(alwaysloaded)
except:
print "info: could not pop alwaysloaded mod",alwaysloaded
return(mods)
def reload_package(self, pkg_name):
# disable and re-enabling the package
psettings = sublime.load_settings("Preferences.sublime-settings")
ignored_packages = psettings.get("ignored_packages", [])
ignored_packages.append(pkg_name)
psettings.set("ignored_packages", ignored_packages)
mods = list(modules.keys()).copy()
for mod in mods:
if mod.startswith(pkg_name + "."):
del modules[mod]
del modules[pkg_name]
ignored_packages.pop(-1)
psettings.set("ignored_packages", ignored_packages)
def __init__(self):
# get all loaded plugins from sys.modules and make them available as plugin_list
self.__plugin_list: Dict[str, Plugin] = {}
self.commands: Dict[str, PluginCommand] = {}
self.help_texts: Dict[str, str] = {}
self.hooks: Dict[str, List[PluginHook]] = {}
self.timers: List[Callable] = []
for key in modules.keys():
if match("^plugins\.\w*", key):
# TODO: this needs to catch exceptions
found_plugin: Plugin = modules[key].plugin
if isinstance(found_plugin, Plugin):
self.__plugin_list[found_plugin.name] = found_plugin
for plugin in self.__plugin_list.values():
"""assemble all valid commands and their respective methods"""
self.commands.update(plugin.get_commands())
"""assemble all hooks and their respective methods"""
event_type: str
plugin_hooks: List[PluginHook]
plugin_hook: PluginHook
for event_type, plugin_hooks in plugin.get_hooks().items():
if event_type in self.hooks.keys():
for plugin_hook in plugin_hooks:
self.hooks[event_type].append(plugin_hook)
else:
self.hooks[event_type] = plugin_hooks
"""assemble all timers and their respective methods"""
self.timers.extend(plugin.get_timers())
"""load the plugin's saved data"""
plugin.plugin_data = plugin.load_data()
logger.info(f"Loaded plugin {plugin.name}:")
if plugin.get_commands() != {}:
logger.info(
f" Commands: {', '.join([*plugin.get_commands().keys()])}"
)
if plugin.get_hooks() != {}:
logger.info(
f" Hooks: {', '.join([*plugin.get_hooks().keys()])}")
if plugin.get_timers():
timers: List[str] = []
for timer in plugin.get_timers():
timers.append(timer.__name__)
logger.info(f" Timers: {', '.join(timers)}")
def exec_lm_core(self, arg_list):
"""
[DUPLICATE] simplified copy of InterpreterCore.exec_lm_core
MAIN FUNCTION TO RUN STRING MODULE.FUNCTION EXECUTIONS
[1] module name (LM)
[2] function
[3...] parameters (separator: space)
NOTE: msgobj - must be a function with one input param (stdout/file/stream)
"""
# Check json mode for LM execution - skip mode
if arg_list[-1] == '>json':
del arg_list[-1]
# LoadModule execution
if len(arg_list) >= 2:
lm_mod, lm_func, lm_params = "LM_{}".format(
arg_list[0]), arg_list[1], ', '.join(arg_list[2:])
try:
# --- LM LOAD & EXECUTE --- #
# [1] LOAD MODULE
exec("import {}".format(lm_mod))
# [2] EXECUTE FUNCTION FROM MODULE - over msgobj (socket or stdout)
lm_output = eval("{}.{}({})".format(lm_mod, lm_func,
lm_params))
# Return LM exec result via msgobj
self.msg(str(lm_output))
return True
# ------------------------- #
except Exception as e:
self.msg("exec_lm_core {}->{}: {}".format(lm_mod, lm_func, e))
if 'memory allocation failed' in str(
e) or 'is not defined' in str(e):
# UNLOAD MODULE IF MEMORY ERROR HAPPENED
if lm_mod in modules.keys():
del modules[lm_mod]
# Exec FAIL -> recovery action in SocketServer
return False
self.msg("SHELL: type help for single word commands (built-in)")
self.msg(
"SHELL: for LM exec: [1](LM)module [2]function [3...]optional params"
)
# Exec OK
return True
def execute_LM_function_Core(argument_list, SocketServerObj=None):
"""
1. param. - LM name, i.e. LM_commands
2. param. - function call with parameters, i.e. a()
NOTE: SocketServerObj is None from Interrupts and Hooks - shared functionality
"""
# health - True [no action] - False [system soft recovery]
health = True
if len(argument_list) >= 2:
LM_name, LM_function, LM_function_call = "LM_{}".format(argument_list[0]), argument_list[1].split('(')[0], "".join(argument_list[1:])
if not LM_function_call.endswith(')'):
# Auto complete brackets "(" ")" with arguments
# ARG 0: LM_function
# ARG 1: LM function arguments (without '(' and/or ')')
LM_function_call = "{}({})".format(LM_function, str(" ".join(" ".join(argument_list[1:]).split('(')[1:])).replace(')', ''))
try:
# --- LM LOAD & EXECUTE --- #
# [1] LOAD MODULE
exec("from {} import {}".format(LM_name, LM_function))
# [2] EXECUTE FUNCTION FROM MODULE - over SocketServerObj or /dev/null
eval("{}".format(LM_function_call)) if SocketServerObj is None else SocketServerObj.reply_message(str(eval("{}".format(LM_function_call))))
# ------------------------- #
except Exception as e:
# ERROR MSG: - over SocketServerObj or stdout
print("execute_LM_function {}->{}: {}".format(LM_name, LM_function, e)) if SocketServerObj is None else SocketServerObj.reply_message("execute_LM_function {}->{}: {}".format(LM_name, LM_function, e))
if "memory allocation failed" in str(e):
# UNLOAD MODULE IF MEMORY ERROR HAPPENED
if LM_name in modules.keys():
del modules[LM_name]
health = False
else:
if SocketServerObj is not None:
SocketServerObj.reply_message("SHELL: type help for base (single word) commands")
SocketServerObj.reply_message("SHELL: for LM exec: [1](LM_)module [2]function[3](optional params.)")
else:
print("SHELL: Missing argument: [1](LM_)module [2]function[3](optional params.)")
# RETURN WITH HEALTH STATE - TRUE :) -> NO ACTION -or- FALSE :( -> RECOVERY ACTION
return health
def blockModule(self, modname):
''' Remplace the given module name by None. inhibits the module.
:param modname: Module name
:type modname: str
'''
if self._modules.get(modname) is None:
self.importModule(modname)
if modules[modname] is not None:
_mod = {}
_mod['mod'] = modules[modname]
_mod['submod'] = []
# Delete and block submodules
for mod in modules.keys():
if mod.find(modname) != -1:
_mod['submod'].append((mod, modules[mod]))
self._deleteModule(mod)
modules[mod] = None
# Delete and block the module
self._deleteModule(modname)
modules[modname] = None
self._modules[modname] = _mod
def blockModule(self, modname):
''' Remplace the given module name by None. inhibits the module.
:param modname: Module name
:type modname: str
'''
if self._modules.get(modname) is None:
self.importModule(modname)
if modules[modname] is not None:
_mod = {}
_mod['mod'] = modules[modname]
_mod['submod'] = []
# Delete and block submodules
for mod in modules.keys():
if mod.find(modname) != -1:
_mod['submod'].append((mod, modules[mod]))
self._deleteModule(mod)
modules[mod] = None
# Delete and block the module
self._deleteModule(modname)
modules[modname] = None
self._modules[modname] = _mod
def importExploits(self, files, route, refresh=False):
"""
Import all exploits that we found and want to import.
:param files: the files that will import
:param route: generic or specific
:param refresh: True if we want to re-import, False otherwise
:return: exploits imported
"""
exploits = []
try:
for exploit in files:
name = 'rexploit.modules.{0}.{1}'.format(route, exploit)
if refresh:
try:
modules.pop(name)
except KeyError:
break
# Reload all modules
if name in modules.keys():
# Get a module
e = modules.get(name)
else:
# Import a new module
e = import_module(
'.modules.{0}.{1}'.format(route, exploit), 'rexploit')
# Create an exploit object and we set IP and MAC
e = e.Exploit()
e.ip = self.__IP
e.mac = self.__MAC
exploits.append(e)
return exploits
except ImportError:
return exploits
def importExploits(self, files, route, refresh=False):
"""
Import all exploits that we found and want to import.
:param files: the files that will import
:param route: generic or specific
:param refresh: True if we want to re-import, False otherwise
:return: exploits imported
"""
exploits = []
try:
for exploit in files:
name = 'rexploit.modules.{0}.{1}'.format(route, exploit)
if refresh:
try:
modules.pop(name)
except KeyError:
break
# Reload all modules
if name in modules.keys():
# Get a module
e = modules.get(name)
else:
# Import a new module
e = import_module('.modules.{0}.{1}'.format(route, exploit), 'rexploit')
# Create an exploit object and we set IP and MAC
e = e.Exploit()
e.ip = self.__IP
e.mac = self.__MAC
exploits.append(e)
return exploits
except ImportError:
return exploits
async def reload_plugins(self):
self.execute_event('on_unload')
for module in [
module for module in listdir(self.cwd)
if '.py' in module and '__' not in module
]:
module_name = 'plugins.%s' % module.replace('.py', '')
if module_name in modules.keys():
current_module = modules[module_name]
reload(current_module)
else:
self.plugins.append(import_module(module_name))
self.execute_event('on_load')
await self._load_plugins()
print("[Plugin Manager] Successfully loaded [%d / %d] plugins." %
(len(self.plugins),
len([
module for module in listdir(self.cwd)
if '.py' in module and '__' not in module
])))
def ismodloaded(_module: str) -> bool:
"""Test if the named module is imported"""
if not isinstance(_module, str):
raise Exception('ismodloaded() only accepts strings - ismodloaded(\'_module\')')
return _module in modules.keys()
import api
from api import VacuumCleaner, MoveDirection, RotateDirection
from sys import modules as sys_modules
from os import environ as os_environ
os_environ["PYTHONPATH"] = "."
sys_modules['os'] = None
sys_modules['socket'] = None
# __builtins__.__dict__['__import__'] = None
if __name__ == '__main__':
cleaner = VacuumCleaner()
config = cleaner.get_config()
print(config)
if cleaner.send_debug_message(config):
print("Debug message sent!")
else:
print("Cannot send debug message")
print(sys_modules.keys())
def compile_C(
self,
extra_compile_args = DEFAULT_COMPILE_ARGS,
verbose = False,
modulename = None
):
"""
compiles the C code (using `Setuptools <http://pythonhosted.org/setuptools/>`_) and loads the compiled functions. If no C code exists, it is generated by calling `generate_f_C` and `generate_jac_C`.
Parameters
----------
extra_compile_args : list of strings
Arguments to be handed to the C compiler on top of what Setuptools chooses. In most situations, it’s best not to write your own list, but modify `DEFAULT_COMPILE_ARGS`, e.g., like this: `compile_C(extra_compile_args = DEFAULT_COMPILE_ARGS + ["--my-flag"])`.
verbose : boolean
Whether the compiler commands shall be shown. This is the same as Setuptools’ `verbose` setting.
modulename : string or `None`
The name used for the compiled module. If `None` or empty, the filename will be chosen by JiTCODE based on previously used filenames or default to `jitced.so`. The only reason why you may want to change this is if you want to save the module file for later use (with`save_compiled`). It is not possible to re-use a modulename for a given instance of `jitcode` (due to the limitations of Python’s import machinery).
Notes
-----
If you want to change the compiler, the intended way is your operating system’s `CC` flag, e.g., by calling `export CC=clang` in the terminal or `os.environ["CC"] = "clang"` in Python.
"""
self._generate_f_C()
self._generate_jac_C()
self._generate_helpers_C()
if modulename:
if modulename in modules.keys():
raise NameError("Module name has already been used in this instance of Python.")
self._modulename = modulename
else:
while self._modulename in modules.keys():
self._modulename = count_up(self._modulename)
sourcefile = self._tmpfile(self._modulename + ".c")
modulefile = self._tmpfile(self._modulename + ".so")
if path.isfile(modulefile):
raise OSError("Module file already exists.")
render_template(
"jitced_template.c",
sourcefile,
n = self.n,
has_Jacobian = self._jac_C_source,
module_name = self._modulename,
Python_version = version_info[0],
has_helpers = bool(self.helpers),
sparse_jac = self.sparse_jac if self._jac_C_source else None
)
setup(
name = self._modulename,
ext_modules = [Extension(
self._modulename,
sources = [sourcefile],
extra_compile_args = extra_compile_args
)],
script_args = [
"build_ext",
"--build-lib", self._tmpfile(),
"--build-temp", self._tmpfile(),
"--force",
#"clean" #, "--all"
],
verbose = verbose
)
self._jitced = find_and_load_module(self._modulename,self._tmpfile())
self.f = self._jitced.f
if self._jac_C_source:
self.jac = self._jitced.jac
if CL_DEBUG:
print '======CL_DEBUG=True=================================\n'
print "local set: ", __name__
print "settings path: ", __file__
print '\n===============================\nDirectories\n==============================='
print 'project path: ', PROJECT_ROOT
print 'rooibos path: ', ROOIBOS_ROOT
print 'media root: ', MEDIA_ROOT
print 'scratch: ', SCRATCH_DIR
print 'auto-storage: ', AUTO_STORAGE_DIR
print 'log file: ', LOGGING['handlers']['file']['filename']
if CL_DEBUG == 'Full':
print '=================CL_DEBUG set to \'Full\' - change to \'True\' to suppress the rest ==================\n'
print "os.getpid() =", getpid()
print "os.getcwd() =", getcwd()
print "os.curdir =", abspath(curdir)
print "sys.path =", repr(sys_path)
print '=================forks in sys.path==================\n'
for forks in sys_path:
print forks, '\n'
print '==================sys.modules.keys==================\n'
print "sys.modules.keys() =", repr(modules.keys())
print "sys.modules.has_key('rooibos') =", 'rooibos' in modules
if 'rooibos' in modules:
print "sys.modules['rooibos'].__name__ =", modules['rooibos'].__name__
print "sys.modules['rooibos'].__file__ =", modules['rooibos'].__file__
print "os.environ['DJANGO_SETTINGS_MODULE'] =", environ.get('DJANGO_SETTINGS_MODULE', None)
print '\n==================end=cl_debug======================\n'
def ismodloaded(_module: str) -> bool:
"""Test if the named module is imported"""
assert isinstance(_module, str), \
'ismodloaded() only accepts strings - ismodloaded(\'_module\')'
return _module in modules.keys()
def find_module(self, module_path):
if not module_path.endswith('.models'):
for path in modules.keys():
if path.endswith('.models') and module_path in path:
module_path = path
return modules[module_path]
## Asks the DAC polarity
def askDacPolarity(self):
self.query.send('GET_DAC_POLARITY')
## Asks the Firmware Version
def askFirmware(self):
self.query.send('GET_FW_VERSION')
try:
from sys import modules
CURRMOD = list(modules.keys())
try:
ENV = 'PyQt5'
CURRMOD.index(ENV)
from PyQt5.QtCore import QObject, pyqtSignal
except:
ENV = 'PyQt4'
CURRMOD.index(ENV)
from PyQt4.QtCore import QObject, pyqtSignal
class QtQuerist(DeafQuerist, QObject):
heardSomething = pyqtSignal(str, name='heardSomething')
def __init__(self, env, device=None, tagQ='REQPAR', tagA='SNDPAR'):
__author__ = "Sabuhi Shukurov"
__email__ = '*****@*****.**'
credits = [
"Sabuhi Shukurov", "Hasan Aliyev", "Tural Muradov", "vincentto13",
"Jeremy T. Hetzel"
]
__version__ = "0.2.0"
from sys import modules as imported_modules
if not "setuptools" in imported_modules.keys():
from fastapi_mqtt.fastmqtt import FastMQTT
from fastapi_mqtt.config import MQTTConfig, MQQTConfig
__all__ = ["FastMQTT", "MQTTConfig", "MQQTConfig"]
Returns a callable object that will set the key with the value of the obj
parameter of the dict object it is passed when called to be the value of
the args parameter.
'''
class AddLevelSpecs:
def __init__(self, o, args):
self.obj = o
self.specs = args
def __call__(self, filterParms):
filterParms[self.obj] = self.specs
return filterParms
return AddLevelSpecs(obj, args)
def setFormatter(formatter):
'''
Returns a callable object that will call setFormatter on the logging handler
object it is passed when called passing it the formatter parameter passed to
setFormatter.
'''
class SetFormatter:
def __init__(self, formatter):
self.formatter = formatter
def __call__(self, handler):
return handler.setFormatter(self.formatter)
return SetFormatter(formatter)
if (not handler.stdout) or (not handler.stderr):
# Create default known handlers+logging-level-filters+formatting
from sys import modules
if 'unittest' not in modules.keys():
handlers._initHandlers()
def __init__(self, timers_filepath: str):
# get all loaded plugins from sys.modules and make them available as plugin_list
self.__plugin_list: Dict[str, Plugin] = {}
self.commands: Dict[str, PluginCommand] = {}
self.help_texts: Dict[str, str] = {}
self.hooks: Dict[str, List[PluginHook]] = {}
"""load stored timers"""
self.timers_filepath: str = timers_filepath
self.timers: List[Timer] = []
stored_timers: List[Timer] = self.__load_timers()
for key in modules.keys():
if match(r"^plugins\.\w*(\.\w*)?", key):
if hasattr(modules[key], "plugin") and isinstance(
modules[key].plugin, Plugin):
self.__plugin_list[
modules[key].plugin.name] = modules[key].plugin
for plugin in self.__plugin_list.values():
"""assemble all valid commands and their respective methods"""
self.commands.update(plugin.get_commands())
"""assemble all hooks and their respective methods"""
event_type: str
plugin_hooks: List[PluginHook]
plugin_hook: PluginHook
for event_type, plugin_hooks in plugin.get_hooks().items():
if event_type in self.hooks.keys():
for plugin_hook in plugin_hooks:
self.hooks[event_type].append(plugin_hook)
else:
self.hooks[event_type] = plugin_hooks
"""
check if a timer of the same name exists already,
overwrite method if needed, keep last_execution from stored timer
non-existing timers will be added as is
"""
new_timer: Timer
stored_timer: Timer
for new_timer in plugin.get_timers():
for stored_timer in stored_timers:
if stored_timer.name == new_timer.name:
logger.debug(
f"Updated existing timer {stored_timer.name} {stored_timer.last_execution}"
)
self.timers.append(
Timer(new_timer.name, new_timer.method,
new_timer.frequency,
stored_timer.last_execution))
break
else:
# timer not found in stored timers
self.timers.append(new_timer)
logger.debug(f"Added new timer: {new_timer.name}")
"""load the plugin's saved data"""
plugin.plugin_data = plugin._load_data_from_file()
"""Display details about the loaded plugins, this does nothing else"""
logger.info(f"Loaded plugin {plugin.name}:")
if plugin.get_commands() != {}:
logger.info(
f" Commands: {', '.join([*plugin.get_commands().keys()])}"
)
if plugin.get_hooks() != {}:
logger.info(
f" Hooks: {', '.join([*plugin.get_hooks().keys()])}")
if plugin.get_timers():
timers: List[str] = []
for timer in plugin.get_timers():
timers.append(timer.name)
logger.info(f" Timers: {', '.join(timers)}")
def generate_f_C(self,
simplify=True,
do_cse=True,
chunk_size=100,
modulename=None,
verbose=False,
extra_compile_args=DEFAULT_COMPILE_ARGS):
"""
translates the derivative to C code using SymPy’s `C-code printer <http://docs.sympy.org/dev/modules/printing.html#module-sympy.printing.ccode>`_.
Parameters
----------
simplify : boolean
Whether the derivative should be `simplified <http://docs.sympy.org/dev/modules/simplify/simplify.html>`_ (with `ratio=1.0`) before translating to C code. The main reason why you could want to disable this is if your derivative is already optimised and so large that simplifying takes a considerable amount of time.
do_cse : boolean
Whether SymPy’s `common-subexpression detection <http://docs.sympy.org/dev/modules/rewriting.html#module-sympy.simplify.cse_main>`_ should be applied before translating to C code.
This is worthwile if your DDE contains the same delay more than once. Otherwise it is almost always better to let the compiler do this (unless you want to set the compiler optimisation to `-O2` or lower). As this requires all entries of `f` at once, it may void advantages gained from using generator functions as an input.
chunk_size : integer
If the number of instructions in the final C code exceeds this number, it will be split into chunks of this size. After the generation of each chunk, SymPy’s cache is cleared. See `large_systems` on why this is useful.
If there is an obvious grouping of your :math:`f`, the group size suggests itself for `chunk_size`. For example, if you want to simulate the dynamics of three-dimensional oscillators coupled onto a 40×40 lattice and if the differential equations are grouped first by oscillator and then by lattice row, a chunk size of 120 suggests itself.
If smaller than 1, no chunking will happen.
extra_compile_args : list of strings
Arguments to be handed to the C compiler on top of what Setuptools chooses. In most situations, it’s best not to write your own list, but modify `DEFAULT_COMPILE_ARGS`, e.g., like this: `compile_C(extra_compile_args = DEFAULT_COMPILE_ARGS + ["--my-flag"])`. However, if your compiler cannot handle one of the DEFAULT_COMPILE_ARGS, you best write your own arguments.
verbose : boolean
Whether the compiler commands shall be shown. This is the same as Setuptools’ `verbose` setting.
modulename : string or `None`
The name used for the compiled module. If `None` or empty, the filename will be chosen by JiTCDDE based on previously used filenames or default to `jitced.so`. The only reason why you may want to change this is if you want to save the module file for later use (with`save_compiled`). It is not possible to re-use a modulename for a given instance of Python (due to the limitations of Python’s import machinery).
"""
assert len(self.past) > 1, "You need to add the past first."
t, y, current_y, past_y, anchors = provide_advanced_symbols()
f_sym_wc = self.f_sym()
helpers_wc = self.helpers
if simplify:
f_sym_wc = (entry.simplify(ratio=1.0) for entry in f_sym_wc)
if do_cse:
additional_helper = sympy.Function("additional_helper")
_cse = sympy.cse(sympy.Matrix(list(f_sym_wc)),
symbols=(additional_helper(i) for i in count()))
helpers_wc.extend(_cse[0])
f_sym_wc = _cse[1][0]
if modulename:
warn(
"Setting the module name works, but saving and loading are not implemented yet. Your file will be located in %s."
% self._tmpfile())
arguments = [
("self", "dde_integrator * const"),
("t", "double const"),
("y", "double", self.n),
]
functions = ["current_y", "past_y", "anchors"]
helper_i = 0
anchor_i = 0
self.substitutions = []
converted_helpers = []
self.past_calls = 0
def finalise(expression):
expression = expression.subs(self.substitutions)
self.past_calls += expression.count(anchors)
return expression
if helpers_wc:
get_helper = sympy.Function("get_f_helper")
set_helper = sympy.Function("set_f_helper")
get_anchor = sympy.Function("get_f_anchor_helper")
set_anchor = sympy.Function("set_f_anchor_helper")
for helper in helpers_wc:
if helper[1].__class__ == anchors:
converted_helpers.append(
set_anchor(anchor_i, finalise(helper[1])))
self.substitutions.append(
(helper[0], get_anchor(anchor_i)))
anchor_i += 1
else:
converted_helpers.append(
set_helper(helper_i, finalise(helper[1])))
self.substitutions.append(
(helper[0], get_helper(helper_i)))
helper_i += 1
if helper_i:
arguments.append(("f_helper", "double", helper_i))
functions.extend(["get_f_helper", "set_f_helper"])
if anchor_i:
arguments.append(("f_anchor_helper", "anchor", anchor_i))
functions.extend(
["get_f_anchor_helper", "set_f_anchor_helper"])
render_and_write_code(converted_helpers,
self._tmpfile,
"helpers",
functions,
chunk_size=chunk_size,
arguments=arguments)
set_dy = sympy.Function("set_dy")
render_and_write_code(
(set_dy(i, finalise(entry)) for i, entry in enumerate(f_sym_wc)),
self._tmpfile,
"f",
functions=functions + ["set_dy"],
chunk_size=chunk_size,
arguments=arguments + [("dY", "double", self.n)])
if modulename:
if modulename in modules.keys():
raise NameError(
"Module name has already been used in this instance of Python."
)
self._modulename = modulename
else:
while self._modulename in modules.keys():
self._modulename = count_up(self._modulename)
sourcefile = self._tmpfile(self._modulename + ".c")
modulefile = self._tmpfile(self._modulename + ".so")
if path.isfile(modulefile):
raise OSError("Module file already exists.")
if not self.past_calls:
warn("Differential equation does not inclued a delay term.")
render_template("jitced_template.c",
sourcefile,
n=self.n,
module_name=self._modulename,
Python_version=version_info[0],
number_of_helpers=helper_i,
number_of_anchor_helpers=anchor_i,
has_any_helpers=anchor_i or helper_i,
anchor_mem_length=self.past_calls,
n_basic=self.n_basic)
setup(
name=self._modulename,
ext_modules=[
Extension(self._modulename,
sources=[sourcefile],
extra_compile_args=["-lm", "-I" + np.get_include()] +
extra_compile_args)
],
script_args=[
"build_ext",
"--build-lib",
self._tmpfile(),
"--build-temp",
self._tmpfile(),
"--force",
#"clean" #, "--all"
],
verbose=verbose)
jitced = find_and_load_module(self._modulename, self._tmpfile())
self.DDE = jitced.dde_integrator(self.past)
def lsmods() -> list:
"""List loaded modules"""
return list(modules.keys())
def exec_lm_core(arg_list, msgobj=None):
"""
MAIN FUNCTION TO RUN STRING MODULE.FUNCTION EXECUTIONS
[1] module name (LM)
[2] function
[3...] parameters (separator: space)
NOTE: msgobj - must be a function with one input param (stdout/file/stream)
"""
# Handle default msgobj >dev/null
if msgobj is None:
msgobj = lambda msg: None
# Dict output user format / jsonify
def __format_out(json_mode, lm_func, output):
if isinstance(output, dict):
if json_mode:
return dumps(output)
# Format dict output - human readable
return '\n'.join([
" {}: {}".format(key, value)
for key, value in lm_output.items()
])
# Handle output data stream
if lm_func == 'help':
if json_mode:
return dumps(output)
# Format help msg - human readable
return '\n'.join([' {},'.format(out) for out in output])
return output
# Check json mode for LM execution
json_mode = arg_list[-1] == '>json'
if json_mode:
del arg_list[-1]
# LoadModule execution
if len(arg_list) >= 2:
lm_mod, lm_func, lm_params = "LM_{}".format(
arg_list[0]), arg_list[1], ', '.join(arg_list[2:])
try:
# --- LM LOAD & EXECUTE --- #
# [1] LOAD MODULE
exec("import {}".format(lm_mod))
# [2] EXECUTE FUNCTION FROM MODULE - over msgobj (socket or stdout)
lm_output = eval("{}.{}({})".format(lm_mod, lm_func, lm_params))
# Handle output data stream
lm_output = __format_out(json_mode, lm_func, lm_output)
# Return LM exec result via msgobj
msgobj(str(lm_output))
return True
# ------------------------- #
except Exception as e:
msgobj("exec_lm_core {}->{}: {}".format(lm_mod, lm_func, e))
if 'memory allocation failed' in str(e) or 'is not defined' in str(
e):
# UNLOAD MODULE IF MEMORY ERROR HAPPENED
if lm_mod in modules.keys():
del modules[lm_mod]
# Exec FAIL -> recovery action in SocketServer
return False
msgobj("SHELL: type help for single word commands (built-in)")
msgobj(
"SHELL: for LM exec: [1](LM)module [2]function [3...]optional params")
# Exec OK
return True
