def _load_config(self,
path: Union[str, Path],
file_name: Optional[str] = None):
""" load a parameter from the config model """
file_name = file_name or self._config_file_name
path = Path(path)
# If a directory was provided look for expected file
if path.is_dir():
expected = Path(path) / file_name
# if file not found look for hidden
if not expected.exists() and not file_name.startswith('.'):
expected = Path(path / ("." + file_name))
if not expected.exists():
return {}
mod = file_name.replace(".py", "")
# If a path to a file was passed
elif path.is_file():
expected = path
mod = expected.name.replace(".py", "")
# Get the spec and import module
spec = spec_from_file_location(mod, expected)
mod = module_from_spec(spec)
spec.loader.exec_module(mod)
# pull out imported stuff and return the rest
out = {}
for item, value in mod.__dict__.items():
# skip other modules or built-in stuff
if inspect.ismodule(value) or item.startswith("__"):
continue
# skip classes defined in other modules
if getattr(value, "__module__", mod.__name__) != mod.__name__:
continue
out[item] = value
return out
async def search_extensions(cls):
if exists("mods"):
for mod in listdir("mods"):
if exists(f"mods/{mod}/{mod}.py"):
spec = spec_from_file_location("extension.module",
f"mods/{mod}/{mod}.py")
modu = module_from_spec(spec)
spec.loader.exec_module(modu)
del modu
def import_object(modname, objpath, objtype='', attrgetter=safe_getattr, warningiserror=False):
# type: (str, List[unicode], str, Callable[[Any, unicode], Any], bool) -> Any
if objpath:
logger.debug('[autodoc] from %s import %s', modname, '.'.join(objpath))
else:
logger.debug('[autodoc] import %s', modname)
try:
module = import_module(modname, warningiserror=warningiserror)
import os
if hasattr(module, '__file__') and os.path.isfile('{}i'.format(module.__file__)):
try:
from importlib._bootstrap import spec_from_loader
from importlib._bootstrap_external import SourceFileLoader
from importlib._bootstrap import module_from_spec
spec = spec_from_loader(modname, SourceFileLoader(modname, '{}i'.format(module.__file__)))
module = module_from_spec(spec)
spec.loader.exec_module(module)
except BaseException as e:
raise e
logger.debug('[autodoc] => %r', module)
obj = module
parent = None
object_name = None
for attrname in objpath:
parent = obj
logger.debug('[autodoc] getattr(_, %r)', attrname)
obj = attrgetter(obj, attrname)
logger.debug('[autodoc] => %r', obj)
object_name = attrname
return [module, parent, object_name, obj]
except (AttributeError, ImportError) as exc:
if objpath:
errmsg = ('autodoc: failed to import %s %r from module %r' %
(objtype, '.'.join(objpath), modname))
else:
errmsg = 'autodoc: failed to import %s %r' % (objtype, modname)
if isinstance(exc, ImportError):
# import_module() raises ImportError having real exception obj and
# traceback
real_exc, traceback_msg = exc.args
if isinstance(real_exc, SystemExit):
errmsg += ('; the module executes module level statement '
'and it might call sys.exit().')
elif isinstance(real_exc, ImportError) and real_exc.args:
errmsg += '; the following exception was raised:\n%s' % real_exc.args[0]
else:
errmsg += '; the following exception was raised:\n%s' % traceback_msg
else:
errmsg += '; the following exception was raised:\n%s' % traceback.format_exc()
if PY2:
errmsg = errmsg.decode('utf-8') # type: ignore
logger.debug(errmsg)
raise ImportError(errmsg)
def _find_strategies_modules(self) -> List[Tuple[str, str]]:
"""
Поиск модулей содержащих стратегии создания функций
"""
strategies_modules = []
checked_packed_paths = []
for app_name in settings.INSTALLED_APPS:
app_module = import_module(app_name)
app_path = app_module.__path__
# Если поиск уже осуществлялся по родительской директории,
# то проверку нужно пропустить
is_already_checked = False
for check_packed_path in checked_packed_paths:
if app_path in check_packed_path:
is_already_checked = True
break
if is_already_checked:
continue
application_path = Path(app_path[0])
strategies_file_patterns = [
'**/**/strategies.py',
]
for strategies_file_pattern in strategies_file_patterns:
strategies_modules_paths = application_path.glob(
strategies_file_pattern
)
for strategies_module_path in strategies_modules_paths:
strategies_module_path = str(strategies_module_path)
module_name = (
str(strategies_module_path).split('/')[-1].split('.')[0]
)
spec = spec_from_file_location(
name=module_name,
location=strategies_module_path,
)
strategies_module = module_from_spec(
spec=spec,
)
spec.loader.exec_module(strategies_module)
strategies_modules.append(
(
strategies_module,
strategies_module_path,
)
)
return strategies_modules
def load_plugin_classes(self, base_module_name: str, baseclass: Type):
# load the module
module_name = base_module_name + '.' + self.module
spec = spec_from_file_location(module_name, self.location.parent / (self.module + '.py'))
modu1e = module_from_spec(spec)
spec.loader.exec_module(modu1e)
sys.modules[module_name] = modu1e
# introspect the modules to find plugin classes
def is_plugin(member):
return inspect.isclass(member) and issubclass(member, baseclass) and member != baseclass
plugin_classes = inspect.getmembers(modu1e, is_plugin)
return plugin_classes
def import_module(modname, warningiserror=False):
module = original_import_module(modname, warningiserror)
if hasattr(module, '__file__') and os.path.isfile('{}i'.format(
module.__file__)):
# merge external spec into the imported module
from importlib._bootstrap import spec_from_loader
from importlib._bootstrap_external import SourceFileLoader
from importlib._bootstrap import module_from_spec
spec = spec_from_loader(
modname,
SourceFileLoader(modname, '{}i'.format(module.__file__)))
# print(spec.loader.get_code(modname).co_names)
module = module_from_spec(spec)
spec.loader.exec_module(module)
def load_plugin_classes(self, base_module_name: str, baseclass: Type):
# load the module
module_name = base_module_name + '.' + self.module
spec = spec_from_file_location(
module_name, self.location.parent / (self.module + '.py'))
modu1e = module_from_spec(spec)
spec.loader.exec_module(modu1e)
sys.modules[module_name] = modu1e
# introspect the modules to find plugin classes
def is_plugin(member):
return inspect.isclass(member) and issubclass(
member, baseclass) and member != baseclass
plugin_classes = inspect.getmembers(modu1e, is_plugin)
return plugin_classes
def discover_all_test_functions(tests_root_dir):
"""
Discover all defined test functions by dynamically loading all test modules.
e.g. of the returned dict:
{
"cloudwatch_logging": [
"test_cloudwatch_logging.py::test_cloudwatch_logging"
],
"dcv": [
"test_dcv.py::test_dcv_configuration",
"test_dcv.py::test_dcv_with_remote_access"
],
...
}
:param tests_root_dir: root dir where looking for test modules.
:return: a dict containing the discovered test modules and test functions.
"""
logging.info("Collecting all existing test functions")
test_module_paths = list(Path(tests_root_dir).rglob("test_*.py"))
discovered_test_functions = OrderedDict()
for module_path in test_module_paths:
module_filename = os.path.basename(str(module_path))
module_dirname = os.path.split(os.path.dirname(str(module_path)))[-1]
spec = spec_from_file_location(
os.path.splitext(module_filename)[0], module_path)
module = module_from_spec(spec)
spec.loader.exec_module(module)
test_functions = filter(
lambda func_name: func_name.startswith("test_"), dir(module))
test_functions_identifiers = [
f"{module_filename}::{test_function}"
for test_function in test_functions
]
discovered_test_functions[module_dirname] = (
discovered_test_functions.get(module_dirname, []) +
test_functions_identifiers)
logging.info("Discovered following test functions:\n%s",
json.dumps(discovered_test_functions, indent=2))
return discovered_test_functions
def start(self, until=0):
if self._running:
return 10
if until > 0:
self.end_ptr = utils.parse_ptr(until)
if self.end_ptr == 0:
# invalid end pointer
return 1
if self.context is None:
err = self.setup()
if err > 0:
return 200 + err
# calculate the start address
address = self._current_instruction
if address == 0:
if self.uc._arch == unicorn.UC_ARCH_ARM:
address = self.uc.reg_read(unicorn.arm_const.UC_ARM_REG_PC)
elif self.uc._arch == unicorn.UC_ARCH_ARM64:
address = self.uc.reg_read(unicorn.arm64_const.UC_ARM64_REG_PC)
else:
# unsupported arch
return 2
if until > 0:
self.log_to_ui('[*] start emulation from %s to %s' %
(hex(address), hex(self.end_ptr)))
else:
self.log_to_ui('[*] stepping %s' % hex(address))
self.dwarf.get_bus().emit('emulator_start')
if self.thumb:
address = address | 1
# invalidate prefs before start
self.invalida_configurations()
# load callbacks if needed
if self.callbacks_path is not None and self.callbacks_path != '':
try:
spec = spec_from_loader(
"callbacks",
SourceFileLoader("callbacks", self.callbacks_path))
self.callbacks = module_from_spec(spec)
spec.loader.exec_module(self.callbacks)
except Exception as e:
self.log_to_ui('[*] failed to load callbacks: %s' % str(e))
# reset callbacks path
self.dwarf.get_prefs().put(prefs.EMULATOR_CALLBACKS_PATH, '')
self.callbacks_path = ''
self.callbacks = None
else:
self.callbacks = None
# until is 0 (i.e we are stepping)
if until == 0:
self.stepping = [True, False]
self.end_ptr = address + (self.dwarf.pointer_size * 2)
else:
self.stepping = [False, False]
Thread(target=self.__start, args=(address, self.end_ptr)).start()
return 0
def emulate(self,
until=0,
step_mode=STEP_MODE_NONE,
user_arch=None,
user_mode=None,
cs_arch=None,
cs_mode=None):
if self.isRunning():
raise self.EmulatorAlreadyRunningError()
if isinstance(until, str):
try:
until = int(until, 16)
except ValueError:
until = 0
if until and isinstance(until, int):
self.end_ptr = utils.parse_ptr(until)
if self.end_ptr == 0:
# invalid end pointer
raise self.EmulatorSetupFailedError('Invalid EndPtr')
if self.context is None:
err = self.setup(user_arch=user_arch,
user_mode=user_mode,
cs_arch=cs_arch,
cs_mode=cs_mode)
if err > 0:
# make sure context is None if setup failed for any reason. we want a clean setup later
self.context = None
err_msg = 'unhandled error'
if err == self.ERR_INVALID_TID:
err_msg = 'invalid thread id'
elif err == self.ERR_INVALID_CONTEXT:
err_msg = 'invalid context'
raise self.EmulatorSetupFailedError('Setup failed: %s' %
err_msg)
# calculate the start address
address = self._next_instruction
if address == 0:
if self.uc._arch == unicorn.UC_ARCH_ARM:
address = self.uc.reg_read(unicorn.arm_const.UC_ARM_REG_PC)
elif self.uc._arch == unicorn.UC_ARCH_ARM64:
address = self.uc.reg_read(unicorn.arm64_const.UC_ARM64_REG_PC)
elif self.uc._arch == unicorn.UC_ARCH_X86 and self.uc._mode == unicorn.UC_MODE_32:
address = self.uc.reg_read(unicorn.x86_const.UC_X86_REG_EIP)
elif self.uc._arch == unicorn.UC_ARCH_X86 and self.uc._mode == unicorn.UC_MODE_64:
address = self.uc.reg_read(unicorn.x86_const.UC_X86_REG_RIP)
else:
raise self.EmulatorSetupFailedError('Unsupported arch')
if until > 0:
self.log_to_ui('[*] start emulation from %s to %s' %
(hex(address), hex(self.end_ptr)))
else:
if step_mode == STEP_MODE_NONE or step_mode == STEP_MODE_SINGLE:
self.log_to_ui('[*] stepping %s' % hex(address))
elif step_mode == STEP_MODE_FUNCTION:
self.log_to_ui('[*] stepping to next function call')
elif step_mode == STEP_MODE_JUMP:
self.log_to_ui('[*] stepping to next jump')
self.onEmulatorStart.emit()
# invalidate prefs before start
self.invalida_configurations()
# load callbacks if needed
if self.callbacks_path is not None and self.callbacks_path != '':
try:
spec = spec_from_loader(
"callbacks",
SourceFileLoader("callbacks", self.callbacks_path))
self.callbacks = module_from_spec(spec)
spec.loader.exec_module(self.callbacks)
except Exception as e:
self.log_to_ui('[*] failed to load callbacks: %s' % str(e))
# reset callbacks path
self._prefs.put(prefs.EMULATOR_CALLBACKS_PATH, '')
self.callbacks_path = ''
self.callbacks = None
else:
self.callbacks = None
# until is 0 (i.e we are stepping)
if until == 0 and step_mode == STEP_MODE_NONE:
self.step_mode = STEP_MODE_SINGLE
else:
self.step_mode = step_mode
self._start_address = address
if self.thumb:
if self._start_address % 2 == 0:
self._start_address = self._start_address | 1
else:
if self._start_address % 2 != 0:
self._start_address -= 1
self._end_address = self.end_ptr
self._setup_done = True
self.start()
def emulate(self, until=0):
if self.isRunning():
raise self.EmulatorAlreadyRunningError()
if isinstance(until, str):
try:
until = int(until, 16)
except ValueError:
until = 0
if until and isinstance(until, int):
self.end_ptr = utils.parse_ptr(until)
if self.end_ptr == 0:
# invalid end pointer
raise self.EmulatorSetupFailedError('Invalid EndPtr')
if self.context is None:
if not self.setup():
raise self.EmulatorSetupFailedError('Setup failed')
# calculate the start address
address = self._current_instruction
if address == 0:
if self.uc._arch == unicorn.UC_ARCH_ARM:
address = self.uc.reg_read(unicorn.arm_const.UC_ARM_REG_PC)
elif self.uc._arch == unicorn.UC_ARCH_ARM64:
address = self.uc.reg_read(unicorn.arm64_const.UC_ARM64_REG_PC)
elif self.uc._arch == unicorn.UC_ARCH_X86 and self.uc._mode == unicorn.UC_MODE_32:
address = self.uc.reg_read(unicorn.x86_const.UC_X86_REG_EIP)
elif self.uc._arch == unicorn.UC_ARCH_X86 and self.uc._mode == unicorn.UC_MODE_64:
address = self.uc.reg_read(unicorn.x86_const.UC_X86_REG_RIP)
else:
raise self.EmulatorSetupFailedError('Unsupported arch')
if until > 0:
self.log_to_ui('[*] start emulation from %s to %s' %
(hex(address), hex(self.end_ptr)))
else:
self.log_to_ui('[*] stepping %s' % hex(address))
self.onEmulatorStart.emit()
if self.thumb:
address = address | 1
# invalidate prefs before start
self.invalida_configurations()
# load callbacks if needed
if self.callbacks_path is not None and self.callbacks_path != '':
try:
spec = spec_from_loader(
"callbacks",
SourceFileLoader("callbacks", self.callbacks_path))
self.callbacks = module_from_spec(spec)
spec.loader.exec_module(self.callbacks)
except Exception as e:
self.log_to_ui('[*] failed to load callbacks: %s' % str(e))
# reset callbacks path
self._prefs.put(prefs.EMULATOR_CALLBACKS_PATH, '')
self.callbacks_path = ''
self.callbacks = None
else:
self.callbacks = None
# until is 0 (i.e we are stepping)
if until == 0:
self.stepping = [True, False]
# self.end_ptr = address + (self.dwarf.pointer_size * 2) stupid
else:
self.stepping = [False, False]
self._start_address = address
self._end_address = self.end_ptr
self._setup_done = True
self.start()