class DownloadManager(object):
"""DownloadManager will be used to download any kind of resource from a \
source system. DownloadManager will locate all plugins of\
type :class:`IDownloader` and will provide it back to the\
requesting object
"""
__single_download_manager = None
def __new__(cls, *args, **kwargs):
"""Singleton class constructor
"""
if cls != type(cls.__single_download_manager):
cls.__single_download_manager = object.__new__(cls, *args, **kwargs)
return cls.__single_download_manager
def __init__(self):
"""Constructor for DownloadManager
"""
self.plugins_path = constants.DOWNLOADER_PLUGINS_PATH
self.save_downloads_to = constants.SAVE_DOWNLOADS_TO_PATH
self.plugin_manager = PluginManager(categories_filter={constants.DOWNLOAD_PLUGIN_FILTER: iplugins.IDownloader})
self.plugin_manager.setPluginPlaces([self.plugins_path])
self.plugin_manager.locatePlugins()
self.plugin_manager.loadPlugins()
def downloaders_list(self):
"""Provides a list of all downloader plugins available
"""
return self.plugin_manager.getPluginsOfCategory(constants.DOWNLOAD_PLUGIN_FILTER)
def get_downloader(self, downloader_name=None):
"""Returns a downloader plugin instance by finding through name
"""
return self.plugin_manager.getPluginByName(downloader_name, constants.DOWNLOAD_PLUGIN_FILTER)
def load_plugins(parser, logger=None):
"""Load all plugins"""
# import yapsy if needed
try:
from yapsy.PluginManager import PluginManager
except ImportError:
parser.exit(status=3, message='PASTA plugins require yapsy.\n'
'See README file for more informations.\n'
'Alternatively, use the option --no-plugins to disable the'
' plugins\n')
from plugins import SingleConnectionAnalyser, \
InterConnectionsAnalyser
# create the plugin manager
plugin_manager = PluginManager(
categories_filter={
'SingleConnectionAnalyser': SingleConnectionAnalyser,
'InterConnectionsAnalyser': InterConnectionsAnalyser
},
directories_list = [os.path.join(os.path.dirname(sys.argv[0]),
'plugins')],
plugin_info_ext='plugin')
plugin_manager.locatePlugins()
def load_plugin(plugin):
"""A plugin is being loaded"""
if logger is not None:
logger.info('...plugin %s v.%s'
% (plugin.name, plugin.version))
plugin_manager.loadPlugins(load_plugin)
return plugin_manager
def testTwoStepsLoad(self): """ Test loading the plugins in two steps in order to collect more deltailed informations. """ spm = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # trigger the first step to look up for plugins spm.locatePlugins() # make full use of the "feedback" the loadPlugins can give # - set-up the callback function that will be called *before* # loading each plugin callback_infos = [] def preload_cbk(plugin_info): callback_infos.append(plugin_info) # - gather infos about the processed plugins (loaded or not) loadedPlugins = spm.loadPlugins(callback=preload_cbk) self.assertEqual(len(loadedPlugins),1) self.assertEqual(len(callback_infos),1) self.assertEqual(loadedPlugins[0].error,None) self.assertEqual(loadedPlugins[0],callback_infos[0]) # check that the getCategories works self.assertEqual(len(spm.getCategories()),1) sole_category = spm.getCategories()[0] # check the getPluginsOfCategory self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1) plugin_info = spm.getPluginsOfCategory(sole_category)[0] # try to remove it and check that is worked spm.removePluginFromCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),0) # now re-add this plugin the to same category spm.appendPluginToCategory(plugin_info,sole_category) self.assertEqual(len(spm.getPluginsOfCategory(sole_category)),1)
def __new__(cls):
if Utility.__INSTANCE is None:
Utility.__INSTANCE = object.__new__(cls)
_pm = PluginManager()
_pm.setPluginPlaces([PATHS.CONFIG_MGR])
_pm.locatePlugins()
configs = _pm.loadPlugins()
Utility.CONFIG = configs[0].plugin_object.CONFIG
return Utility.__INSTANCE
def get_all_plugins(self, path, p_categories_filter=None):
""" Returns all plugin objects as array available at the specified
path provided as argument
"""
_pm = PluginManager(categories_filter=p_categories_filter)
_pm.setPluginPlaces([path])
_pm.locatePlugins()
plugins = _pm.loadPlugins()
del _pm
return plugins
def get_plugin(self, path, p_categories_filter=None):
""" Returns a plugins available at specified path and of category
provided as argument
"""
_pm = PluginManager(categories_filter=p_categories_filter)
_pm.setPluginPlaces([path])
_pm.locatePlugins()
plugins = _pm.loadPlugins()
plugin = plugins[0].plugin_object
del _pm
del plugins
return plugin
def testTwoStepsLoadWithError(self):
"""
Test loading the plugins in two steps in order to collect more
deltailed informations and take care of an erroneous plugin.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
],
plugin_info_ext="yapsy-error-plugin")
# trigger the first step to look up for plugins
spm.locatePlugins()
# make full use of the "feedback" the loadPlugins can give
# - set-up the callback function that will be called *before*
# loading each plugin
callback_infos = []
def preload_cbk(i_plugin_info):
callback_infos.append(i_plugin_info)
callback_after_infos = []
def postload_cbk(i_plugin_info):
callback_after_infos.append(i_plugin_info)
# - gather infos about the processed plugins (loaded or not)
# and for the test, monkey patch the logger
originalLogLevel = log.getEffectiveLevel()
log.setLevel(logging.ERROR)
errorLogCallFlag = [False]
def errorMock(*args, **kwargs):
errorLogCallFlag[0] = True
originalErrorMethod = log.error
log.error = errorMock
try:
loadedPlugins = spm.loadPlugins(callback=preload_cbk,
callback_after=postload_cbk)
finally:
log.setLevel(originalLogLevel)
log.error = originalErrorMethod
self.assertTrue(errorLogCallFlag[0])
self.assertEqual(len(loadedPlugins), 1)
self.assertEqual(len(callback_infos), 1)
self.assertTrue(isinstance(callback_infos[0].error, tuple))
self.assertEqual(loadedPlugins[0], callback_infos[0])
self.assertTrue(issubclass(callback_infos[0].error[0], ImportError))
self.assertEqual(len(callback_after_infos), 0)
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
def __get_mgr(cls):
if not cls.__plugin_mgr:
analyzer = PluginFileAnalyzerMathingRegexWithInfoProperty(
'file_name_analyzer', r'^.*\.py$')
mgr = PluginManager(categories_filter=cls.__categories)
mgr.setPluginPlaces(["plugins"])
mgr.getPluginLocator().removeAllAnalyzer()
mgr.getPluginLocator().appendAnalyzer(analyzer)
mgr.locatePlugins()
mgr.loadPlugins()
cls.__plugin_mgr = mgr
return cls.__plugin_mgr
def load_plugins():
"""Loads plugins from the puglins folder"""
plugins = PluginManager()
plugins_folder = os.path.join(os.environ['LOOKDEVTOOLS'], 'python',
'ldtplugins')
print plugins_folder
plugins.setPluginPlaces([plugins_folder])
plugins.collectPlugins()
plugins.locatePlugins()
logger.info('Plugin candidates %s' % plugins.getPluginCandidates())
for pluginInfo in plugins.getAllPlugins():
plugins.activatePluginByName(pluginInfo.name)
logger.info('Plugin activated %s' %
plugins.activatePluginByName(pluginInfo.name))
return plugins
def testTwoStepsLoadWithError(self):
"""
Test loading the plugins in two steps in order to collect more
deltailed informations and take care of an erroneous plugin.
"""
spm = PluginManager(
directories_list=[os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")],
plugin_info_ext="yapsy-error-plugin",
)
# trigger the first step to look up for plugins
spm.locatePlugins()
# make full use of the "feedback" the loadPlugins can give
# - set-up the callback function that will be called *before*
# loading each plugin
callback_infos = []
def preload_cbk(i_plugin_info):
callback_infos.append(i_plugin_info)
# - gather infos about the processed plugins (loaded or not)
# and for the test, monkey patch the logger
originalLogLevel = log.getEffectiveLevel()
log.setLevel(logging.ERROR)
errorLogCallFlag = [False]
def errorMock(*args, **kwargs):
errorLogCallFlag[0] = True
originalErrorMethod = log.error
log.error = errorMock
try:
loadedPlugins = spm.loadPlugins(callback=preload_cbk)
finally:
log.setLevel(originalLogLevel)
log.error = originalErrorMethod
self.assertTrue(errorLogCallFlag[0])
self.assertEqual(len(loadedPlugins), 1)
self.assertEqual(len(callback_infos), 1)
self.assertTrue(isinstance(callback_infos[0].error, tuple))
self.assertEqual(loadedPlugins[0], callback_infos[0])
self.assertEqual(callback_infos[0].error[0], ImportError)
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
def testTwoStepsLoad(self):
"""
Test loading the plugins in two steps in order to collect more
deltailed informations.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# trigger the first step to look up for plugins
spm.locatePlugins()
# make full use of the "feedback" the loadPlugins can give
# - set-up the callback function that will be called *before*
# loading each plugin
callback_infos = []
def preload_cbk(plugin_info):
callback_infos.append(plugin_info)
callback_after_infos = []
def postload_cbk(plugin_info):
callback_after_infos.append(plugin_info)
# - gather infos about the processed plugins (loaded or not)
loadedPlugins = spm.loadPlugins(callback=preload_cbk,
callback_after=postload_cbk)
self.assertEqual(len(loadedPlugins), 1)
self.assertEqual(len(callback_infos), 1)
self.assertEqual(loadedPlugins[0].error, None)
self.assertEqual(loadedPlugins[0], callback_infos[0])
self.assertEqual(len(callback_after_infos), 1)
self.assertEqual(loadedPlugins[0], callback_infos[0])
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
plugin_info = spm.getPluginsOfCategory(sole_category)[0]
# try to remove it and check that is worked
spm.removePluginFromCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
# now re-add this plugin the to same category
spm.appendPluginToCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
class ClassifierManager():
def __init__(self):
self.manager = PluginManager(
categories_filter={"Classifiers": IClassifier})
self.manager.setPluginPlaces(["classifiers"])
self.manager.locatePlugins()
self.manager.loadPlugins()
self.classifiers = {}
def loadAll(self):
self.classifiers = {}
print("Loading classifiers...")
for classifier in self.manager.getAllPlugins():
print(classifier.plugin_object)
self.classifiers[classifier.plugin_object.__class__.
__name__] = classifier.plugin_object
def availableExtensions():
# setup the categories
categories = {'Extension': pluginTypes.IExtensionPlugin}
# Build the manager, set load location, and then collect them
extManager = PluginManager(categories_filter=categories)
#pluginManager.setPluginPlaces()
loc = extManager.getPluginLocator()
loc.setPluginPlaces([path + '\\extensions'])
extManager.locatePlugins()
extManager.loadPlugins()
extensions = startupExtensions(extManager)
return extensions
def testCandidatesManipulation(self): """ Test querying, removing and adding plugins from/to the lkist of plugins to load. """ spm = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # locate the plugins that should be loaded spm.locatePlugins() # check nb of candidatesx self.assertEqual(len(spm.getPluginCandidates()),1) # get the description of the plugin candidate candidate = spm.getPluginCandidates()[0] self.assertTrue(isinstance(candidate,tuple)) # try removing the candidate spm.removePluginCandidate(candidate) self.assertEqual(len(spm.getPluginCandidates()),0) # try re-adding it spm.appendPluginCandidate(candidate) self.assertEqual(len(spm.getPluginCandidates()),1)
def testCandidatesManipulation(self):
"""
Test querying, removing and adding plugins from/to the lkist
of plugins to load.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# locate the plugins that should be loaded
spm.locatePlugins()
# check nb of candidatesx
self.assertEqual(len(spm.getPluginCandidates()), 1)
# get the description of the plugin candidate
candidate = spm.getPluginCandidates()[0]
self.assertTrue(isinstance(candidate, tuple))
# try removing the candidate
spm.removePluginCandidate(candidate)
self.assertEqual(len(spm.getPluginCandidates()), 0)
# try re-adding it
spm.appendPluginCandidate(candidate)
self.assertEqual(len(spm.getPluginCandidates()), 1)
def availableInterfaces():
# setup the categories
categories = {
'Interface': pluginTypes.IInterfacePlugin,
'Techniques': pluginTypes.ITechniquePlugin
}
# Build the manager, set load location, and then collect them
interfaceManager = PluginManager(categories_filter=categories)
# pluginManager.setPluginPlaces()
loc = interfaceManager.getPluginLocator()
loc.setPluginPlaces([path + '\\interfaces'])
interfaceManager.locatePlugins()
interfaceManager.loadPlugins()
interfaces = startupInterfaces(interfaceManager)
return interfaces
def load_plugins():
""" Load plugin for environment. See lib/base.py
"""
# Create plugin manager
manager = PluginManager()
# Tell it the default place(s) where to find plugins
manager.setPluginPlaces(["pylons_yapsy_plugin/plugins/"])
# Define the various categories corresponding to the different
# kinds of plugins you have defined
manager.setCategoriesFilter({
"Menu" : menu.Menu,
"Inline" : inline.Inline,
})
manager.locatePlugins()
# Deactivate plugin
# Список деактивированных плагинов из БД
deactivatedPlugins = [plugin.name for plugin in\
s.query(DeactivatedPlugins).all()]
# Список кандидатов на загрузку
candidates = manager.getPluginCandidates()
# Список деактивированных плагинов в формате yapsy
deactivatedList = []
for candidate in candidates:
if candidate[2].name in deactivatedPlugins:
deactivatedList.append(candidate)
# Исключаем плагины которые указанны в БД
for plugin in deactivatedList:
manager.removePluginCandidate(plugin)
# Load plugins
manager.loadPlugins()
return manager, [plugin[2] for plugin in deactivatedList]
def testEnforcingPluginDirsDoesNotKeepDefaultDir(self):
"""
Test that providing the directories list override the default search directory
instead of extending the default list.
"""
class AcceptAllPluginFileAnalyzer(IPluginFileAnalyzer):
def __init__(self):
IPluginFileAnalyzer.__init__(self, "AcceptAll")
def isValidPlugin(self, filename):
return True
def getInfosDictFromPlugin(self, dirpath, filename):
return {"name": filename, "path": dirpath}, ConfigParser()
pluginLocator = PluginFileLocator()
pluginLocator.setAnalyzers([AcceptAllPluginFileAnalyzer()])
spm_default_dirs = PluginManager(plugin_locator=pluginLocator)
spm_default_dirs.locatePlugins()
candidates_in_default_dir = spm_default_dirs.getPluginCandidates()
candidates_files_in_default_dir = set(
[c[0] for c in candidates_in_default_dir])
pluginLocator = PluginFileLocator()
pluginLocator.setAnalyzers([AcceptAllPluginFileAnalyzer()])
spm = PluginManager(plugin_locator=pluginLocator,
directories_list=[
os.path.dirname(os.path.abspath(__file__)),
"does-not-exists"
])
spm.locatePlugins()
candidates = spm.getPluginCandidates()
candidates_files = set([c[0] for c in candidates])
self.assertFalse(
set(candidates_files_in_default_dir).issubset(
set(candidates_files)))
class Plugins():
"""
Class responsible to manage the plugins
"""
categories = [
[
"SitePreparsing", SitePreparsing,
"Site wide plugins that execute before the parsing start."
],
[
"SiteProcessor", SiteProcessor,
"Plugins that process the whole site once after parsing."
],
[
"SiteRendering", SiteRendering,
"Plugins that render additional pages after the rendering."
],
[
"PostProcessor", PostProcessor,
"Plugins that process each post after they are parsed"
],
[
"CollectionProcessor", CollectionProcessor,
"Plugins that process each collection after posts are parsed"
],
[
"TemplateFilter", TemplateFilter,
"Plugins that define jinja2 filters to be used in templates"
],
]
# for plugin info structure
PLUGIN_CAT = 0
PLUGIN_NAME = 1
PLUGIN_DESC = 2
PLUGIN_ENABLE = 3
PLUGIN_MODULE_NAME = 4
PLUGIN_VERSION = 5
def __init__(self, plugin_directories, debug_log_fname, plugins_config):
"Load plugins"
categories_filter = {}
for cat in self.categories:
categories_filter[cat[0]] = cat[1]
# single or multi-directory handling
if not isinstance(plugin_directories, list):
plugin_directories = [plugin_directories]
self.plugins = PluginManager(plugin_info_ext='sitefab-plugin',
categories_filter=categories_filter)
self.plugins.setPluginPlaces(plugin_directories)
self.plugins.locatePlugins()
self.plugins.loadPlugins()
self.plugins_config = plugins_config
# List of enabled plugins
self.plugins_enabled = {}
for pl in self.get_plugins_info():
if pl[self.PLUGIN_ENABLE]:
self.plugins_enabled[pl[self.PLUGIN_MODULE_NAME]] = 1
# list of plugins already executed. Used for dependencies tracking
# across stages
self.plugins_executed = {}
# FIXME: make sure it is working
# logging.basicConfig(filename=debug_log_fname, level=logging.DEBUG)
def get_plugins(self, category=None):
"""Return the list of plugins
:param str category: restrict to plugins of a given category
:rtype: list(iPlugin)
:return: list of plugins
"""
if category:
return self.plugins.getPluginsOfCategory(category)
else:
return self.plugins.getAllPlugins()
def get_num_plugins(self, category=None):
""" Return the number of plugins available.
:param str category: restrict to plugins of given category
:rtype: int
:return: number of plugins
"""
plugins = self.get_plugins(category)
return len(plugins)
def get_plugin_dir(self, plugin):
""" Return the directory where the plugin is stored
:param iPlugin plugin: the plugin requested
:rtype: str
:return: the module name
"""
module_path = Path(plugin.details.get("Core", "module"))
return module_path.parent
def get_plugin_default_configuration_filename(self, plugin):
""" Return the path to the plugin default configuration filename
"""
try:
fname = plugin.details.get("Configuration", "Filename")
except: # noqa
return ""
fname = fname.replace('"', '')
path = self.get_plugin_dir(plugin)
return path / fname
def get_plugin_documentation_filename(self, plugin):
""" Return the path to the plugin documentation
"""
try:
fname = plugin.details.get("Documentation", "Filename")
except: # noqa
return ""
path = self.get_plugin_dir(plugin)
return path / fname
def get_plugin_class_name(self, plugin):
""" Return the class of a given plugin
:param iPlugin plugin: the plugin requested
:rtype: str
:return: the module classname
"""
return plugin.categories[0]
def get_plugin_module_name(self, plugin):
""" Return the module name of a given plugin
:param iPlugin plugin: the plugin requested
:rtype: str
:return: the module name
"""
module_path = plugin.details.get("Core", "module")
path, filename = os.path.split(module_path)
return filename
def get_plugin_config(self, plugin):
""" Return the configuration of a given plugin
:param iPlugin plugin: the plugin requested
:rtype: dict
:return: plugin configuration
"""
module_name = self.get_plugin_module_name(plugin)
class_name = self.get_plugin_class_name(plugin)
try:
config = self.plugins_config[class_name][module_name]
except: # noqa
config = {}
return config
def get_plugin_dependencies(self, plugin):
""" Return the dependency of a given plugin
:param iPlugin plugin: the plugin requested
:rtype: list
:return: list of plugins name the plugin depend on
"""
# No dependencies
if not plugin.details.has_option("Core", "Dependencies"):
return set()
dependencies = set()
st = plugin.details.get("Core", "Dependencies")
if "," in st:
elts = st.split(",")
for elt in elts:
dependencies.add(elt.strip())
else:
dependencies.add(st)
return dependencies
def is_plugin_enabled(self, plugin):
config = self.get_plugin_config(plugin)
if config.get('enable'):
return True
else:
return False
def get_plugins_info(self, category=None):
"""Return the list of plugins available with their type
:param str category: restrict to plugins of a given category.
:rtype: list(str)
:return: list of plugins name
"""
pl = []
if category:
categories = [category]
else:
categories = self.plugins.getCategories()
for cat in categories:
for plugin in self.plugins.getPluginsOfCategory(cat):
enabled = self.is_plugin_enabled(plugin)
module_name = self.get_plugin_module_name(plugin)
try:
version = plugin.version
except: # noqa
version = "NA"
s = [
cat, plugin.name, plugin.description, enabled, module_name,
version
]
pl.append(s)
return pl
def display_execution_results(self, results, site):
""" Display execution summary
"""
cprint("|-Execution result", "magenta")
table_data = [['name', 'ok', 'skipped', 'errors']]
count = 0
for result in results:
row = []
plugin_name, stats = result
# plugin name
c = "cyan"
if count % 2:
c = "blue"
row.append(colored(plugin_name, c))
# ok
if stats[site.OK]:
if stats[site.OK] == 1:
val = 'v'
else:
val = stats[site.OK]
row.append(colored(val, 'green'))
else:
row.append(' ')
# warning
if stats[site.SKIPPED]:
row.append(colored(stats[site.SKIPPED], "yellow"))
else:
row.append(' ')
# error
if stats[site.ERROR]:
if stats[site.ERROR] == 1:
val = 'x'
else:
val = stats[site.ERROR]
row.append(colored(val, "red"))
else:
row.append(' ')
table_data.append(row)
count += 1
print(SingleTable(table_data).table)
def run_plugins(self, items, plugin_class, unit, site):
"""Execute a set of plugins on a given list of items
:param list items: list of items to process
:param str plugin_type: the plugin_class to use
:param str unit: the unit to use in the display
:param SiteFab site: pointer to the site object to be passed
to the plugins
:rtype: dict(dict(list))
:return: plugins execution statistics
"""
# dependencies map
dependencie_map = {}
# used to get back from the module name to the plugin
module_name_to_plugin = {}
plugins = self.plugins.getPluginsOfCategory(plugin_class)
# collecting plugins that are to be executed.
for plugin in plugins:
if self.is_plugin_enabled(plugin):
module_name = self.get_plugin_module_name(plugin)
module_name_to_plugin[module_name] = plugin
# dependencies computation.
# Due to potential dependencies on plugins from previous stage
# this must be computed after collecting which
# plugins were executed.
for plugin in module_name_to_plugin.values():
all_dependencies = self.get_plugin_dependencies(plugin)
dependencies = set() # topological sort requires use of set
module_name = self.get_plugin_module_name(plugin)
for dep_module_name in all_dependencies:
if dep_module_name not in self.plugins_enabled:
utils.error("Plugin:%s can't be executed because\
plugin %s is not enable" %
(module_name, dep_module_name))
# only add to the dependencies map the plugins
# that are from the same stage
if dep_module_name in module_name_to_plugin:
dependencies.add(dep_module_name)
else:
# check if already executed
if dep_module_name not in self.plugins_executed:
utils.error("Plugin:%s can't be executed because\
plugin %s was not executed in previous\
stage" % (module_name, dep_module_name))
dependencie_map[module_name] = dependencies
# print dependencie_map
# Topological sorting
try:
plugins_to_process = toposort_flatten(dependencie_map)
except Exception as e:
utils.error("Circular dependencies between plugins.\
Can't execute plugins:%s" % e)
s = "|-%s plugins" % (unit.strip().capitalize())
desc = colored(s, "magenta")
results = []
for module_name in tqdm(plugins_to_process,
unit=' plugin',
desc=desc,
leave=True):
if module_name in module_name_to_plugin:
plugin = module_name_to_plugin[module_name]
else:
raise Exception("The following plugin module name listed in\
dependencies don't exist % s " % module_name)
pclass = plugin_class.lower()
filename = "%s.%s.html" % (pclass, module_name)
log_id = site.logger.create_log(pclass, plugin.name, filename)
plugin_results = utils.dict_to_objdict({
site.OK: 0,
site.SKIPPED: 0,
site.ERROR: 0
})
config = self.get_plugin_config(plugin)
for item in tqdm(items, unit=unit, desc=plugin.name, leave=False):
result = plugin.plugin_object.process(item, site, config)
plugin_results[result[0]] += 1
severity = result[0]
name = result[1]
details = result[2]
site.logger.record_event(log_id, name, severity, details)
self.plugins_executed[module_name] = True
results.append([plugin.name, plugin_results])
site.logger.write_log(log_id)
return results
def get_template_filters(self):
"""Load template filters and return a dictionary list
Return:
dict: jinja filter functions
"""
template_filters = {}
filters = self.plugins.getPluginsOfCategory("TemplateFilter")
for flt in filters:
filter_name = self.get_plugin_module_name(flt)
template_filters[filter_name] = flt.plugin_object.myfilter
return template_filters
class Friday:
def __init__(self):
self.ai = ai_interface.API(apiai.ApiAI(settings['CLIENT_ACCESS_TOKEN'],
settings['SUBSCRIPTION_KEY']))
self.debugging = settings['debugging']
self.spoken_language = settings['spoken language']
self.input_system = settings['input system'] # google, local, text
self.output_system = settings['output system'] # both, audio, text
self.speech_file_location = settings['speech file location'] # .
self.speech_file_name = settings['speech file name'] # audio response
self.speak = settings['speak'] # True
# The question that the assistant hears
self.question = None
# The chosen, spoken response given to the user.
self.response = None
# Whether Friday is active
self.is_active = True
# What type of question is being asked.
self.request_type = None
if settings['input system'] != 'text':
self.recognizer = sr.Recognizer()
self.microphone = sr.Microphone(device_index=settings['DEVICE'],
sample_rate=settings['RATE'],
chunk_size=settings['CHUNK'])
if settings['input_system'] == 'google':
with self.microphone as source:
if settings['debugging']:
click.echo("Adjusting to ambient noise.")
# we only need to calibrate once, before we start listening
self.recognizer.adjust_for_ambient_noise(source)
# Build the manager
self.manager = PluginManager()
# Tell it the default place(s) where to find plugins
self.manager.setPluginPlaces(settings["plugin folder"])
# Load all plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
self.plugins = {}
# Activate all loaded plugins
for plugin in self.manager.getAllPlugins():
self.plugins[plugin.name] = plugin.plugin_object
def _local_stt(self):
pass
def _apiai_stt(self):
from math import log
import audioop
import pyaudio
import time
resampler = apiai.Resampler(source_samplerate=settings['RATE'])
request = self.ai.voice_request()
vad = apiai.VAD()
def callback(in_data, frame_count):
frames, data = resampler.resample(in_data, frame_count)
if settings.show_decibels:
decibel = 20 * log(audioop.rms(data, 2) + 1, 10)
click.echo(decibel)
state = vad.processFrame(frames)
request.send(data)
state_signal = pyaudio.paContinue if state == 1 else pyaudio.paComplete
return in_data, state_signal
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paInt32, input=True, output=False, stream_callback=callback,
channels=settings['CHANNELS'], rate=settings['RATE'], frames_per_buffer=settings['CHUNK'])
stream.start_stream()
click.echo("Speak!")
while stream.is_active():
time.sleep(0.1)
stream.stop_stream()
stream.close()
p.terminate()
def _google_stt(self):
"""
Uses Google's Speech to Text engine to
understand speech and convert it into
text.
:return: String, either the spoken text or error text.
"""
click.echo("Please speak now")
# Listen to the microphone set up in the __init__ file.
with self.microphone as mic_source:
audio = self.recognizer.listen(mic_source)
click.echo("Processing audio")
try:
# Try to recognize the text
return self.recognizer.recognize_google(audio, show_all=self.debugging)
except sr.UnknownValueError: # speech is unintelligible
return "Google could not understand the audio."
except sr.RequestError:
return "Could not request results from Google's speech recognition service."
def listen(self):
# Default value.
# TODO: Input plugin system goes here
# Plugins should be used for getting input so users can build their
# own custom front-end to the application.
# This will allow the speech to text functions to be moved out of the
# main class and into their own plugins.
if self.input_system == 'google':
self.question = self._google_stt()
else:
self.question = click.prompt("Input your query")
# TODO: Loading plugin system goes here
# Eventually I want people to be able to build custom behavior
# that executes when the response is being fetched. So there could
# be a loading screen that plays music or something for example.
click.echo("Wait for response...")
# Get a response from the AI using the api interface.
self.ai.get_response(self.question)
# Clean up the response and turn it into a Python object that can be read
self.response = self.ai.parse()
return self.response
def think(self, request):
# TODO: Move plugins to a priority system
# Eventually plugins should return an integer value which determines
# their ordering when responding to a request, in the event multiple
# plugins are able to respond to the same request.
# This may eventually allow for watchdog plugins which check for
# erroneous requests and alert the system by returning negative (Maybe)
# priorities, which can then be used to safeguard a user's system or
# settings.
return any(plugin.can_perform(self, request)
for name, plugin in self.plugins.items())
def perform(self, request):
# Trigger 'some request' from the loaded plugins
for name, plugin in self.plugins.items():
if plugin.can_perform(self, request):
# TODO: Allow plugins to fail
# Plugins will be able to return False in the event they fail,
# allowing the system to then apologize to the user.
plugin.perform(self, request)
return True
def refuse(self):
# May become unnecessary as the plugins could be built to refuse
# requests directly.
click.echo("Can't do that")
def apologize(self):
# May eventually become unnecessary as plugins could be built to
# return apologies directly.
click.echo("I failed to do that, sorry.")
def _print_response(self):
click.echo(self.response)
def _speak_response(self):
self.say(self.response, title=self.speech_file_name, speak_aloud=self.speak)
def respond(self):
# TODO: Output plugin system goes here
# Plugins should be used for providing output for the same reasons
# that plugins should be used for providing input.
if self.output_system == "audio":
self._speak_response()
elif self.output_system == "text":
self._print_response()
else:
self._print_response()
self._speak_response()
def say(self, message, speak_aloud=True, title='Speak'):
# May eventually be moved into its own plugin
message = str(message)
if not message.strip():
click.echo("No text to speak.")
return
computer_os = system()
folder = os.path.split(title)[0]
folder_path = os.path.join(self.speech_file_location,
folder)
if not os.path.exists(folder_path):
os.makedirs(folder_path)
home = settings['home']
if self.input_system == 'google':
# Create the MP3 file which will speak the text
title += '.wav'
path = os.path.join(home, title)
tts = gTTS(message, lang=self.spoken_language)
tts.save(path)
speech_file = "start /MIN {}"
if computer_os != 'Windows':
speech_file = "mpg123 {}"
speech_file = speech_file.format(os.path.join(home, title))
else:
# Create the Visual Basic code which will speak the text
title += '.vbs'
path = os.path.join(home, title)
message = re.sub('["\n\t]', '', message) # Strip out characters that would be spoken by the system.
if computer_os == "Windows":
with open(path, 'w') as file:
file.write(
"""
speaks="{}"
Dim speaks, speech
Set speech=CreateObject("sapi.spvoice")
speech.Speak speaks
""".format(message))
# Set up the file to be executed
speech_file = ["cscript.exe", title]
if computer_os != "Windows":
speech_file = ["espeak", message]
if speak_aloud:
try:
call(speech_file)
except FileNotFoundError:
if self.debugging:
click.echo("File not accessible:" + str(speech_file))
class ContentModelListener(ConnectionListener):
'''
classdocs
'''
def __init__(self,
content_models,
host='localhost',
port=61613,
user='',
passcode='',
fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' %
{
'host': host,
'port': port
})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" %
{'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url,
username=user,
password=passcode)
self.client = FedoraClient(self.fc)
# Create plugin manager
self.manager = PluginManager(
categories_filter={"FedoraMicroService": FedoraMicroService})
self.manager.setPluginPlaces(["plugins"])
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info(
"Loading plugin: %(name)s for content model %(cmodel)s." % {
'name': plugin.plugin_object.name,
'cmodel': plugin.plugin_object.content_model
})
plugin.plugin_object.config = config
if plugin.plugin_object.content_model in self.contentModels:
self.contentModels[plugin.plugin_object.content_model].append(
plugin.plugin_object)
else:
self.contentModels[plugin.plugin_object.content_model] = [
plugin.plugin_object
]
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
#logging.debug("\r \r", end='')
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
global TOPIC_PREFIX
self.__print_async('MESSAGE', headers, body)
f = feedparser.parse(body)
tags = f['entries'][0]['tags']
pid = [
tag['term'] for tag in tags if tag['scheme'] == 'fedora-types:pid'
][0]
dsID = [
tag['term'] for tag in tags if tag['scheme'] == 'fedora-types:dsID'
][0]
obj = self.client.getObject(pid)
content_model = headers['destination'][len(TOPIC_PREFIX):]
if content_model in self.contentModels:
logging.info('Running rules for %(pid)s from %(cmodel)s.' % {
'pid': obj.pid,
'cmodel': content_model
})
for plugin in self.contentModels[content_model]:
plugin.runRules(obj, dsID)
return
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers={'message-id': args[1]})
else:
self.conn.ack(headers={'message-id': args[1]},
transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, correlation_id, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination,
message=message,
headers={'correlation-id': correlation_id})
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
class MainWindow(QtGui.QMainWindow):
"""A configurable hex editor that supports binary templates and
scripting through use of the construct library
"""
def __init__(self, file_name=None):
"""Initializer"""
super(MainWindow, self).__init__()
#Flag set to ignore the next data change event
self._treeChangedData = False
self._isUntitled = True
self._curFile = ''
self._setCurrentFile('')
self.__mimeTypes = magic.Magic(mime=True)
# UI attribute definitions (populated in __initUI and the various
# __create* methods)
#ToolBar
self._fileToolBar = self.addToolBar("File")
#StatusBar
self._lbAddress = QtGui.QLabel()
self._lbAddressName = QtGui.QLabel()
self._lbSize = QtGui.QLabel()
self._lbSizeName = QtGui.QLabel()
self._lbOverwriteMode = QtGui.QLabel()
self._lbOverwriteModeName = QtGui.QLabel()
#Menus
self._fileMenu = self.menuBar().addMenu("&File")
self._editMenu = self.menuBar().addMenu("&Edit")
self._helpMenu = self.menuBar().addMenu("&Help")
#Action definitions
self._openAct = None
self._saveAct = None
self._saveAsAct = None
self._saveReadableAct = None
self._saveSelReadableAct = None
self._exitAct = None
self._undoAct = None
self._redoAct = None
self._aboutAct = None
self._optionsAct = None
#Other
self._hexEdit = QHexEdit()
self._treeDissected = QTreeWidget()
self._optionsDialog = OptionsDialog()
self.__initUI()
self.readSettings()
# Create plugin manager
# Which plugin types to load and which categories to put them in
category_mapping = {"FormatDissectors": FormatDissector}
self._manager = PluginManager(categories_filter=category_mapping)
self._manager.setPluginPlaces(["plugins"])
#Dissectors
self._dissector = None
self._availDissectors = {}
#load in the plugins
self.__reloadPlugins()
if file_name:
self.loadFile(file_name)
#############
# GUI SETUP #
#############
def about(self):
"""Display an 'About' dialog box describing the application"""
QtGui.QMessageBox.about(self, "About ParSlither",
"Parslither v0.1 (WIP)")
def closeEvent(self, event): # pylint: disable-msg=W0613
"""(PyQT event handler) the application is due to close"""
self.writeSettings()
del self._optionsDialog
self.close()
def __createActions(self):
"""Create actions for the menus and toolbars in the UI"""
self._openAct = QAction(QtGui.QIcon(':/images/open.png'),
"&Open...", self, shortcut=QKeySequence.Open,
statusTip="Open an existing file", triggered=self.dlgOpen)
self._saveAct = QAction(QtGui.QIcon(':/images/save.png'),
"&Save", self, shortcut=QKeySequence.Save,
statusTip="Save the document to disk", triggered=self.save)
self._saveAsAct = QAction("Save &As...", self,
shortcut=QKeySequence.SaveAs,
statusTip="Save the document under a new name",
triggered=self.dlgSaveAs)
self._saveReadableAct = QAction("Save as &Readable...", self,
statusTip="Save in a readable format",
triggered=self.dlgSaveToReadableFile)
self._saveSelReadableAct = QAction("Save Selection Readable...", self,
statusTip="Save selection in a readable format",
triggered=self.dlgSaveSelectionToReadableFile)
self._exitAct = QAction("E&xit", self, shortcut="Ctrl+Q",
statusTip="Exit the application", triggered=self.close)
self._undoAct = QAction("&Undo", self, shortcut=QKeySequence.Undo,
triggered=self._hexEdit.undo)
self._redoAct = QAction("&Redo", self, shortcut=QKeySequence.Redo,
triggered=self._hexEdit.redo)
self._aboutAct = QAction("&About", self,
statusTip="Show the application's About box",
triggered=self.about)
self._optionsAct = QAction("&Options", self,
statusTip="Show the options dialog",
triggered=self.showOptionsDialog)
def __initMenus(self):
"""Initialize menus for the UI"""
self._fileMenu.addAction(self._openAct)
self._fileMenu.addAction(self._saveAct)
self._fileMenu.addAction(self._saveAsAct)
self._fileMenu.addAction(self._saveReadableAct)
self._fileMenu.addSeparator()
self._fileMenu.addAction(self._exitAct)
self._editMenu.addAction(self._undoAct)
self._editMenu.addAction(self._redoAct)
self._editMenu.addAction(self._saveSelReadableAct)
self._editMenu.addSeparator()
self._editMenu.addAction(self._optionsAct)
self._helpMenu.addAction(self._aboutAct)
def __initStatusBar(self):
"""Initialize status bar for the UI"""
# Address Label
self._lbAddressName.setText("Address:")
self.statusBar().addPermanentWidget(self._lbAddressName)
self._lbAddress.setFrameShape(QtGui.QFrame.Panel)
self._lbAddress.setFrameShadow(QtGui.QFrame.Sunken)
self._lbAddress.setMinimumWidth(70)
self.statusBar().addPermanentWidget(self._lbAddress)
self._hexEdit.currentAddressChanged.connect(self.__setAddress)
# Address Size
self._lbSizeName.setText("Size:")
self.statusBar().addPermanentWidget(self._lbSizeName)
self._lbSize.setFrameShape(QtGui.QFrame.Panel)
self._lbSize.setFrameShadow(QtGui.QFrame.Sunken)
self._lbSize.setMinimumWidth(70)
self.statusBar().addPermanentWidget(self._lbSize)
self._hexEdit.currentSizeChanged.connect(self.__setSize)
# Overwrite Mode label
self._lbOverwriteModeName.setText("Mode:")
self.statusBar().addPermanentWidget(self._lbOverwriteModeName)
self._lbOverwriteMode.setFrameShape(QtGui.QFrame.Panel)
self._lbOverwriteMode.setFrameShadow(QtGui.QFrame.Sunken)
self._lbOverwriteMode.setMinimumWidth(70)
self.statusBar().addPermanentWidget(self._lbOverwriteMode)
self.setOverwriteMode(self._hexEdit.overwriteMode())
self.statusBar().showMessage("Ready")
def __initToolBars(self):
"""Initialize ToolBars for the UI"""
self._fileToolBar.addAction(self._openAct)
self._fileToolBar.addAction(self._saveAct)
def __initDockWindows(self):
"""Initialize Docked Windows for the UI"""
dock = QtGui.QDockWidget("Dissected", self)
dock.setFeatures(
QDockWidget.DockWidgetFeatures(QDockWidget.NoDockWidgetFeatures))
dock.setAllowedAreas(Qt.Qt.BottomDockWidgetArea)
dock.setWidget(self._treeDissected)
self.addDockWidget(Qt.Qt.BottomDockWidgetArea, dock)
def __initUI(self):
"""Initialize everything for the UI"""
self.setAttribute(QtCore.Qt.WA_DeleteOnClose)
self._optionsDialog.accepted.connect(self.__optionsAccepted)
self._hexEdit.overwriteModeChanged.connect(self.setOverwriteMode)
self._hexEdit.dataChanged.connect(self.__hexDataChanged)
self.setCentralWidget(self._hexEdit)
#we don't want to be able to sort by rows (keep serialized order)
self._treeDissected.setSortingEnabled(False)
tree_header = QTreeWidgetItem(["Name", "Value"])
self._treeDissected.setHeaderItem(tree_header)
self.__createActions()
self.__initMenus()
self.__initToolBars()
self.__initStatusBar()
self.__initDockWindows()
###########
# PLUGINS #
###########
def __reloadPlugins(self):
""" Load plugins """
self._manager.locatePlugins()
self._manager.loadPlugins()
self.__refreshDissectors()
def __refreshDissectors(self):
"""Refresh dissectors from the plugin manager"""
self._availDissectors = {}
for plugin in self._manager.getPluginsOfCategory("FormatDissectors"):
# plugin.plugin_object is an instance of the plugin
plug_obj = plugin.plugin_object
self._availDissectors[plug_obj.name] = plug_obj
# if we have a dissector loaded, reload it from the dict of
# available dissectors
if self._dissector and self._dissector.name in self._availDissectors:
self._dissector = self._availDissectors[self._dissector.name]
else:
self._dissector = None
############
# SETTINGS #
############
def readSettings(self):
"""Reload all settings for this application from storage"""
settings = QtCore.QSettings()
pos = settings.value('pos', QtCore.QPoint(200, 200)).toPoint()
size = settings.value('size', QtCore.QSize(610, 460)).toSize()
self.move(pos)
self.resize(size)
editor = self._hexEdit
editor.setAddressArea(settings.value("AddressArea").toBool())
editor.setAsciiArea(settings.value("AsciiArea").toBool())
editor.setHighlighting(settings.value("Highlighting").toBool())
editor.setOverwriteMode(settings.value("OverwriteMode").toBool())
editor.setReadOnly(settings.value("ReadOnly").toBool())
editor.setHighlightingColor(QColor(settings.value("HighlightingColor")))
editor.setAddressAreaColor(QColor(settings.value("AddressAreaColor")))
editor.setSelectionColor(QColor(settings.value("SelectionColor")))
default_font = QFont("Courier New", 10)
editor.setFont(QFont(settings.value("WidgetFont", default_font)))
editor.setAddressWidth(settings.value("AddressAreaWidth").toInt()[0])
def writeSettings(self):
"""Write all non-session settings to storage"""
settings = QtCore.QSettings()
settings.setValue('pos', self.pos())
settings.setValue('size', self.size())
def showOptionsDialog(self):
"""Show the options dialog"""
self._optionsDialog.show()
def __optionsAccepted(self):
"""(Callback) The user is ok with the changes to the settings"""
self.writeSettings()
self.readSettings()
#########################
# FILE LOADING / SAVING #
#########################
def save(self):
"""Save the entire hex editor buffer to a file as-is
If a file was already open, it will be saved to that file, otherwise a
file chooser will be presented and the user will be asked to choose a
file
"""
def write_whole(handle):
handle.write(self._hexEdit.data())
if self._isUntitled:
return self.dlgSaveAs()
else:
return self.writeFile(write_whole, self._curFile, as_is=True)
def dlgSaveAs(self):
"""Save the entire hex editor buffer to a file as-is"""
def write_whole(handle):
handle.write(self._hexEdit.data())
return self.writeFile(write_whole, as_is=True, op_name="Save As")
def dlgSaveToReadableFile(self):
"""Save the entire hex editor buffer to a file in a readable format"""
def write_readable(handle):
handle.write(self._hexEdit.toReadableString())
return self.writeFile(write_readable, op_name="Save To Readable File")
def dlgSaveSelectionToReadableFile(self):
"""Save the selected section to a file in a readable format"""
def write_sel_readable(handle):
handle.write(self._hexEdit.toReadableString())
return self.writeFile(write_sel_readable,
op_name="Save To Readable File")
def writeFile(self, write_func, file_name=None, as_is=False, op_name=""):
"""Try to save content to the specified file using the specified
write_func
Arguments:
write_func(handle) -- Function to save the desired content to the file
Keyword Arguments:
file_name -- Filename to save to instead of displaying a
file picker (Defaults to None)
as_is -- whether the hex editor is being saved without modification
(Defaults to False)
op_name -- (Defaults to "")
"""
if not file_name:
file_name = QtGui.QFileDialog.getSaveFileName(self,
op_name,
self._curFile)
if not file_name:
return False
file_handle = QtCore.QFile(file_name)
if not file_handle.open(QtCore.QFile.WriteOnly):
error_msg = "Cannot write file %s:\n%s." % \
(file_name, file_handle.errorString())
QtGui.QMessageBox.warning(self, "HexEdit", error_msg)
return False
# call the function to actually write to the file
write_func(file_handle)
# only set the current file to the saved filename if we're
# saving the whole file as-is
if as_is:
self._setCurrentFile(file_name)
self.statusBar().showMessage("File saved", 2000)
return True
def dlgOpen(self):
"""Display a file picker and ask the user to choose a file to open in
the hex editor"""
file_name = QtGui.QFileDialog.getOpenFileName(self)
if file_name:
self.loadFile(file_name)
def loadFile(self, file_name):
"""Load the specified file into the hex editor"""
file_handle = QtCore.QFile(file_name)
if not file_handle.open( QtCore.QFile.ReadOnly):
warning_msg = "Cannot read file %s:\n%s." % \
file_name, file_handle.errorString()
QtGui.QMessageBox.warning(self, "QHexEdit", warning_msg)
return
QtGui.QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
self._hexEdit.setData(file_handle.readAll())
QtGui.QApplication.restoreOverrideCursor()
self._setCurrentFile(file_name)
self.statusBar().showMessage("File loaded", 2000)
self.__autoLoadDissector(file_name)
self.__refreshDissectionTree()
def _setCurrentFile(self, file_name):
"""Set the current filename"""
self._curFile = file_name
self._isUntitled = (file_name == "")
self.setWindowModified(False)
window_title = "%s[*] - QHexEdit" % \
QtCore.QFileInfo(self._curFile).fileName()
self.setWindowTitle(window_title)
def __setAddress(self, address):
"""Set the address at the caret"""
self._lbAddress.setText('%x' % address)
def setOverwriteMode(self, mode):
"""Overwrite the nibble following the caret instead of inserting?"""
if mode:
self._lbOverwriteMode.setText("Overwrite")
else:
self._lbOverwriteMode.setText("Insert")
def __setSize(self, size):
"""Set the total size of the file in the hex editor"""
self._lbSize.setText('%d' % size)
def __hexDataChanged(self):
"""The data in the hex editor control changed"""
# don't refresh the dissection tree if the hex editor data was
# based on the data in there anyways
if not self._treeChangedData:
self.__refreshDissectionTree()
self._treeChangedData = False
def __autoLoadDissector(self, file_name):
"""Try to auto-assign an available dissector based on filename
and mimetype
"""
#don't use a dissector if we can't auto-assign one
self._dissector = None
#first try and assign a dissector by extension
for dissector in self._availDissectors.values():
for extension in dissector.file_exts:
if file_name.endsWith(extension):
self._dissector = dissector
return
#now try to assign a dissector by mimetype
file_mimetype = self.__mimeTypes.from_file(file_name)
if not file_mimetype:
return
for dissector in self._availDissectors.values():
for supp_mimetype in dissector.file_mimetypes:
if file_name == supp_mimetype:
self._dissector = dissector
return
def __refreshDissectionTree(self):
"""Refresh the tree of dissected data with data from the hex editor"""
self._treeDissected.clear()
#only refresh if we have data and a dissector
if self._dissector and self._hexEdit.data():
self.__addToDissectionTree(
self._dissector.dissect(self._hexEdit.data().data()))
def __addToDissectionTree(self, attr_container, parent=None):
"""Recursively add a Construct container and its children to the
dissected data tree widget
Arguments:
attr_container -- Construct container whose attributes to recursively
add to the tree
Keyword Arguments:
parent -- Reference to the tree item that represents the current
container (default None)
"""
def add_item_to_tree(child):
""" Add a tree item to the tree, with its parent item as the
parent if it has one """
if not parent:
self._treeDissected.addTopLevelItem(child)
else:
parent.addChild(child)
def add_container_to_tree(name, source_container):
""" Add a container to the dissection tree as a tree item and
handle its children """
container_item = QTreeWidgetItem([name, ""])
add_item_to_tree(container_item)
self.__addToDissectionTree(source_container, container_item)
#look through the container's attributes and add them to the tree
#as necessary
for attr_k in attr_container:
#skip private attributes if we were given any
if not attr_k.startswith("_"):
#get the value of this attribute
attr_v = attr_container[attr_k]
#value is a container
if isinstance(attr_v, construct.Container):
add_container_to_tree(attr_k, attr_v)
#value is list-like
elif isinstance(attr_v, (list, tuple)):
elem_idx = 0
for elem in attr_v:
elem_name = "%s[%d]" % (attr_k, elem_idx)
#list element is a container
if isinstance(elem, construct.Container):
add_container_to_tree(elem_name, elem)
#list element is a primitive or non-construct object
else:
new_item = QTreeWidgetItem([elem_name, str(elem)])
add_item_to_tree(new_item)
elem_idx += 1
#value is a primitive or a non-construct object
else:
add_item_to_tree(QTreeWidgetItem([attr_k, str(attr_v)]))
class app():
db = None # the sqlite database of plugins
plugins = None # Configured plugins
storage = None # The plugin to use for storage
PluginFolder = None # Folder where the plugins are
MinionFolder = None # Folder where the minions are
score = None # the plugin to use for scoring
classify = None # the clasification plugin
helper = None # The verum helper functions
loc = None # The verum lcoation
def __init__(self, PluginFolder=PluginFolder, MinionFolder=MinionFolder):
#global PluginFolder
self.PluginFolder = PluginFolder
#global MinionsFolder
self.MinionFolder = MinionFolder
# Load enrichments database
self.db = self.set_db()
# LOAD HELPER FROM SAME DIRECTORY
fp, pathname, description = imp.find_module("helper", [loc])
self.helper = imp.load_module("helper", fp, pathname, description)
# Save the verum location
self.loc = loc[:
-6] # -6 removed the trailing "verum/" from the location.
# Load the plugins Directory
if self.PluginFolder:
self.load_plugins()
else:
logging.warning(
"Plugin folder doesn't exist. Plugins not configured. Please run set_plugin_folder(<PluginFolder>) to set the plugin folder and then load_plugins() to load plugins."
)
## PLUGIN FUNCTIONS
def set_plugin_folder(self, PluginFolder):
self.PluginFolder = PluginFolder
def get_plugin_folder(self):
return self.PluginFolder
# Load the plugins from the plugin directory.
def load_plugins(self):
print "Configuring Plugin manager."
self.plugins = PluginManager()
if self.MinionFolder is None:
self.plugins.setPluginPlaces([self.PluginFolder])
else:
self.plugins.setPluginPlaces(
[self.PluginFolder, self.MinionFolder])
#self.plugins.collectPlugins()
self.plugins.locatePlugins()
self.plugins.loadPlugins()
print "Plugin manager configured."
# Loop round the plugins and print their names.
cur = self.db.cursor()
# Clear tables
cur.execute("""DELETE FROM enrichments""")
cur.execute("""DELETE FROM inputs""")
cur.execute("""DELETE FROM storage""")
cur.execute("""DELETE FROM score""")
cur.execute("""DELETE FROM minion""")
for plugin in self.plugins.getAllPlugins():
plugin_config = plugin.plugin_object.configure()
# Insert enrichment
if plugin_config[0] == 'enrichment': # type
cur.execute(
'''INSERT INTO enrichments VALUES (?, ?, ?, ?, ?)''',
(
plugin_config[2], # Name
int(plugin_config[1]), # Enabled
plugin_config[3], # Descripton
plugin_config[5], # Cost
plugin_config[6]) # Speed
)
for inp in plugin_config[4]: # inputs
# Insert into inputs table
cur.execute('''INSERT INTO inputs VALUES (?,?)''',
(plugin_config[2], inp))
self.db.commit()
elif plugin_config[0] == 'interface': # type
cur.execute('''INSERT INTO storage VALUES (?, ?)''',
(plugin_config[2], int(plugin_config[1])))
elif plugin_config[0] == 'score':
cur.execute(
'''INSERT INTO score VALUES (?, ?, ?, ?, ?)''',
(
plugin_config[2], # Name
int(plugin_config[1]), # Enabled
plugin_config[3], # Descripton
plugin_config[4], # Cost
plugin_config[5]) # Speed
)
if plugin_config[0] == 'minion':
plugin_config = plugin.plugin_object.configure(self)
cur.execute(
'''INSERT INTO minion VALUES (?, ?, ?, ?)''',
(
plugin_config[2], # Name
int(plugin_config[1]), # Enabled
plugin_config[3], # Descripton
plugin_config[4]) # Speed
)
if plugin.name == "classify": # Classify is a unique name. TODO: figure out if handling multiple 'classify' plugins is necessary
self.classify = plugin.plugin_object
print "Configured {2} plugin {0}. Success: {1}".format(
plugin.name, plugin_config[1], plugin_config[0])
def set_db(self):
"""
Sets up the enrichment sqlite in memory database
"""
conn = sqlite3.connect(":memory:")
cur = conn.cursor()
# Create enrichments table
cur.execute(
'''CREATE TABLE enrichments (name text NOT NULL PRIMARY KEY,
configured int,
description text,
cost int,
speed int);''')
# Create inputs table
cur.execute('''CREATE TABLE inputs (name text NOT NULL,
input text NOT NULL,
PRIMARY KEY (name, input),
FOREIGN KEY (name) REFERENCES enrichments(name));'''
)
# Create interface table
cur.execute('''CREATE TABLE storage (name text NOT NULL PRIMARY KEY,
configured int
);''')
# Create score table
cur.execute('''CREATE TABLE score (name text NOT NULL PRIMARY KEY,
configured int,
description text,
cost int,
speed int);''')
# Create minion table
cur.execute('''CREATE TABLE minion (name text NOT NULL PRIMARY KEY,
configured int,
description text,
cost int);''')
conn.commit()
return conn
## ENRICHMENT FUNCTIONS
def get_inputs(self):
""" NoneType -> list of strings
:return: A list of the potential enrichment inputs (ip, domain, etc)
"""
inputs = list()
cur = self.db.cursor()
for row in cur.execute('''SELECT DISTINCT input FROM inputs;'''):
inputs.append(row[0])
return inputs
def get_enrichments(self,
inputs,
cost=10000,
speed=10000,
configured=True):
"""
:param inputs: list of input types. (e.g. ["ip", "domain"]) All enrichments that match at least 1 input type will be returned.
:param cost: integer 1-10 of resource cost of running the enrichment. (1 = cheapest)
:param speed: integer 1-10 speed of enrichment. (1 = fastest)
:param enabled: Plugin is correctly configured. If false, plugin may not run correctly.
:return: list of tuples of (names, type) of enrichments matching the criteria
"""
cur = self.db.cursor()
if type(inputs) == str:
inputs = [inputs]
plugins = list()
names = list()
for row in cur.execute(
""" SELECT DISTINCT e.name, i.input
FROM enrichments e, inputs i
WHERE e.name = i.name
AND e.cost <= ?
AND e.speed <= ?
AND configured = ?
AND i.input IN ({0})""".format(
("?," * len(inputs))[:-1]),
[cost, speed, int(configured)] + inputs):
plugins.append(tuple(row))
return plugins
def run_enrichments(self,
topic,
topic_type,
names=None,
cost=10,
speed=10,
start_time=""):
"""
:param topic: topic to enrich (e.g. "1.1.1.1", "www.google.com")
:param topic_type: type of topic (e.g. "ip", "domain")
:param cost: integer 1-10 of resource cost of running the enrichment. (1 = cheapest)
:param speed: integer 1-10 speed of enrichment. (1 = fastest)
:param names: a name (as string) or a list of names of enrichments to use
:return: None if storage configured (networkx graph representing the enrichment of the topic
"""
enrichments = self.get_enrichments([topic_type],
cost,
speed,
configured=True)
enrichments = [e[0] for e in enrichments]
#print enrichments # DEBUG
g = nx.MultiDiGraph()
# IF a name(s) are given, subset to them
if names:
enrichments = set(enrichments).intersection(set(names))
for enrichment in enrichments:
# get the plugin
plugin = self.plugins.getPluginByName(enrichment)
# run the plugin
g2 = plugin.plugin_object.run(topic, start_time)
# merge the graphs
for node, props in g2.nodes(data=True):
g.add_node(node, props)
for edge in g2.edges(data=True):
g.add_edge(edge[0], edge[1], attr_dict=edge[2])
return g
## INTERFACE FUNCTIONS
def get_interfaces(self, configured=None):
"""
:return: list of strings of names of interface plugins
"""
cur = self.db.cursor()
interfaces = list()
if configured is None:
for row in cur.execute('''SELECT DISTINCT name FROM storage;'''):
interfaces.append(row[0])
else:
for row in cur.execute(
'''SELECT DISTINCT name from storage WHERE configured=?;''',
(int(configured), )):
interfaces.append(row[0])
return interfaces
def get_default_interface(self):
return self.storage
def set_interface(self, interface):
"""
:param interface: The name of the plugin to use for storage.
Sets the storage backend to use. It must have been configured through a plugin prior to setting.
"""
cur = self.db.cursor()
configured_storage = list()
for row in cur.execute(
'''SELECT DISTINCT name FROM storage WHERE configured=1;'''):
configured_storage.append(row[0])
if interface in configured_storage:
self.storage = interface
else:
raise ValueError(
"Requested interface {0} not configured. Options are {1}.".
format(interface, configured_storage))
def get_minions(self, cost=10000, configured=None):
"""
:param cost: a maximum cost of running the minion
:param configured: True, False, or None (for both).
:return: list of strings of tuples of (name, description) of minion plugins
"""
cur = self.db.cursor()
minions = list()
if configured is None:
for row in cur.execute(
'''SELECT DISTINCT name, description FROM minion WHERE cost <= ?;''',
[int(cost)]):
minions.append(tuple(row[0:2]))
else:
for row in cur.execute(
'''SELECT DISTINCT name, description FROM minion WHERE cost <= ? AND configured=?;''',
[int(cost), int(configured)]):
minions.append(tuple(row[0:2]))
return minions
def start_minions(self, names=None, cost=10000):
"""
:param names: a list of names of minions to run
:param cost: a maximum cost for minions
"""
minions = self.get_minions(cost=cost, configured=True)
minions = [m[0] for m in minions]
# IF a name(s) are given, subset to them
if names:
minions = set(minions).intersection(set(names))
for minion in minions:
# get the plugin
plugin = self.plugins.getPluginByName(minion)
# start the plugin
plugin.plugin_object.start()
def get_running_minions(self):
"""
:return: A set of names of minions which are running
"""
minions = self.get_minions(cost=10000, configured=True)
minions = [m[0] for m in minions]
running_minions = set()
# Iterate Through the minions
for minion in minions:
plugin = self.plugins.getPluginByName(minion)
if plugin.plugin_object.isAlive():
running_minions.add(minion)
return running_minions
def stop_minions(self, names=None):
minions = self.get_running_minions()
if names is not None:
minions = set(minions).intersection(set(names))
for minion in minions:
# get the plugin
plugin = self.plugins.getPluginByName(minion)
# start the plugin
plugin.plugin_object.stop()
def run_query(self,
topic,
max_depth=4,
dont_follow=['enrichment', 'classification'],
storage=None):
"""
:param storage: the storage plugin to use
:return: a networkx subgraph surrounded around the topic
"""
if not storage:
storage = self.storage
if not storage:
raise ValueError(
"No storage set. run set_storage() to set or provide directly. Storage must be a configured plugin."
)
else:
# get the plugin
plugin = self.plugins.getPluginByName(self.storage)
return plugin.plugin_object.query(topic,
max_depth=max_depth,
dont_follow=dont_follow)
def store_graph(self, g, storage=None):
"""
:param g: a networkx graph to merge with the set storage
"""
if not storage:
storage = self.storage
if not storage:
raise ValueError(
"No storage set. run set_storage() to set or provide directly. Storage must be a configured plugin."
)
else:
# get the plugin
plugin = self.plugins.getPluginByName(self.storage)
# merge the graph
plugin.plugin_object.enrich(g)
## SCORE FUNCTIONS
def get_scoring_plugins(self,
cost=10000,
speed=10000,
names=None,
configured=True):
"""
:param cost: integer 1-10 of resource cost of running the enrichment. (1 = cheapest)
:param speed: integer 1-10 speed of enrichment. (1 = fastest)
:param enabled: Plugin is correctly configured. If false, plugin may not run correctly.
:return: list of names of scoring plugins matching the criteria
"""
cur = self.db.cursor()
plugins = list()
if names is None:
for row in cur.execute(
'''SELECT DISTINCT name
FROM score
WHERE cost <= ?
AND speed <= ?
AND configured = ?''',
[cost, speed, int(configured)]):
plugins.append(row[0])
else:
for row in cur.execute(
'''SELECT DISTINCT name
FROM score
WHERE cost <= ?
AND speed <= ?
AND configured = ?
AND name IN ({0});'''.format(
("?," * len(names))[:-1]),
[cost, speed, int(configured)] + names):
plugins.append(row[0])
return plugins
def score_subgraph(self, topic, sg, plugin_name=None):
if plugin_name is None:
plugin_name = self.score
score_plugin = self.plugins.getPluginByName(plugin_name)
return score_plugin.plugin_object.score(sg, topic)
def set_scoring_plugin(self, plugin):
"""
:param interface: The name of the plugin to use for storage.
Sets the storage backend to use. It must have been configured through a plugin prior to setting.
"""
cur = self.db.cursor()
configured_scoring_plugins = list()
for row in cur.execute(
'''SELECT DISTINCT name FROM score WHERE configured=1;'''):
configured_scoring_plugins.append(row[0])
if plugin in configured_scoring_plugins:
self.score = plugin
else:
raise ValueError(
"Requested scoring plugin {0} is not configured. Options are {1}."
.format(plugin, configured_scoring_plugins))
def get_default_scoring_plugin(self):
return self.score
class ModuleManager():
def __init__(self, threads, kill_list, job_list):
self.threads = threads
self.kill_list = kill_list
self.job_list = job_list # Running jobs
self.pmanager = PluginManager()
self.pmanager.setPluginPlaces(["plugins"])
self.pmanager.collectPlugins()
self.pmanager.locatePlugins()
self.plugins = ['none']
num_plugins = len(self.pmanager.getAllPlugins())
if num_plugins == 0:
raise Exception("No Plugins Found!")
plugins = []
for plugin in self.pmanager.getAllPlugins():
plugin.threads = threads
self.plugins.append(plugin.name)
plugin.plugin_object.setname(plugin.name)
## Check for installed binaries
executable = ''
try:
settings = plugin.details.items('Settings')
for kv in settings:
executable = kv[1]
if executable.find(
'/') != -1: #Hackish "looks like a file"
if os.path.exists(executable):
logging.info("Found file: {}".format(executable))
break
else:
raise Exception()
## TODO detect binaries not in "executable" setting
except:
raise Exception(
'[ERROR]: {} -- Binary does not exist -- {}'.format(
plugin.name, executable))
plugins.append(plugin.name)
print "Plugins found [{}]: {}".format(num_plugins, sorted(plugins))
def run_module(self,
module,
job_data_orig,
tar=False,
all_data=False,
reads=False,
meta=False,
overrides=True):
"""
Keyword Arguments:
module -- name of plugin
job_data -- dict of job parameters
tar -- return tar of all output, rather than module.OUTPUT file
all_data -- return module.OUTPUT and list of all files in self.outdir
reads -- include files if module.OUTPUT == 'reads'
Not recommended for large read files.
"""
job_data = job_data_orig
# job_data = copy.deepcopy(job_data_orig)
# # Pass back orig file descriptor
# try:
# job_data['out_report'] = job_data_orig['out_report']
# except:
# pass
if not self.has_plugin(module):
raise Exception("No plugin named {}".format(module))
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
plugin.plugin_object.update_settings(job_data)
if meta:
output = plugin.plugin_object(settings, job_data, self, meta=True)
else:
output = plugin.plugin_object(settings, job_data, self)
log = plugin.plugin_object.out_module.name
if tar:
tarfile = plugin.plugin_object.tar_output(job_data['job_id'])
return output, tarfile, [], log
if all_data:
if not reads and plugin.plugin_object.OUTPUT == 'reads':
#Don't return all files from plugins that output reads
data = []
else:
data = plugin.plugin_object.get_all_output_files()
return output, data, log
return output, [], log
def output_type(self, module):
return self.pmanager.getPluginByName(module).plugin_object.OUTPUT
def input_type(self, module):
return self.pmanager.getPluginByName(module).plugin_object.INPUT
def get_short_name(self, module):
try:
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
for kv in settings:
if kv[0] == 'short_name':
sn = kv[1]
break
return sn
except:
return None
def get_executable(self, module):
try:
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
for kv in settings:
if kv[0] == 'short_name':
sn = kv[1]
break
return sn
except:
return None
def has_plugin(self, plugin):
if not plugin.lower() in self.plugins:
logging.error("{} plugin not found".format(plugin))
return False
return True
def valid_modules(self, l):
""" Return filtered list of available modules """
return [m for m in l if not m.startswith('?') and self.has_plugin(m)]
def validate_pipe(self, pipe):
for stage in pipe:
for word in stage.replace('+', ' ').split(' '):
if not (word.startswith('?') or self.has_plugin(word)):
raise Exception('Invalid pipeline command')
def split_pipe(self, l):
""" Splits a multi-module string in to bins
Ex: 'kiki ?k=29 velvet' -> [[kiki, ?k=29], [velvet]]
"""
bins = []
for word in l:
if not word.startswith('?'):
bins.append([word])
elif word.startswith('?'):
bins[-1].append(word)
return bins
def parse_input(self, pipe):
"""
Parses inital pipe and separates branching bins
Ex: ['sga', '?p=True', 'kiki ?k=31 velvet', 'sspace']
"""
stages = phelper.parse_branches(pipe)
return stages
def parse_pipe(self, pipe):
""" Returns the pipeline(s)z of modules.
Returns parameter overrides from string.
e.g Input: [sga_ec 'kiki ?k=31 velvet ?ins=500' sspace]
Output: [kiki, velvet, a5], [{k:31}, {ins:500}, {}]
"""
# Parse param overrides
overrides = []
pipeline = []
module_num = -1
for group in pipe:
for word in group.split('+'):
if word.lower() == 'none':
pass
elif not word.startswith('?') and self.has_plugin(
word): # is module
module_num = module_num + 1
pipeline.append(word)
overrides.append({})
elif word[1:-1].find('=') != -1: # is param
kv = word[1:].split('=')
overrides[module_num] = dict(
overrides[module_num].items() + dict([kv]).items())
return pipeline, overrides
class app():
db = None # the sqlite database of plugins
plugins = None # Configured plugins
storage = None # The plugin to use for storage
PluginFolder = None # Folder where the plugins are
MinionFolder = None # Folder where the minions are
score = None # the plugin to use for scoring
classify = None # the clasification plugin
helper = None # The verum helper functions
loc = None # The verum lcoation
def __init__(self, PluginFolder=PluginFolder, MinionFolder=MinionFolder):
#global PluginFolder
self.PluginFolder = PluginFolder
#global MinionsFolder
self.MinionFolder = MinionFolder
# Load enrichments database
self.db = self.set_db()
# LOAD HELPER FROM SAME DIRECTORY
fp, pathname, description = imp.find_module("helper", [loc])
self.helper = imp.load_module("helper", fp, pathname, description)
# Save the verum location
self.loc = loc[:-6] # -6 removed the trailing "verum/" from the location.
# Load the plugins Directory
if self.PluginFolder:
self.load_plugins()
else:
logging.warning("Plugin folder doesn't exist. Plugins not configured. Please run set_plugin_folder(<PluginFolder>) to set the plugin folder and then load_plugins() to load plugins.")
## PLUGIN FUNCTIONS
def set_plugin_folder(self, PluginFolder):
self.PluginFolder = PluginFolder
def get_plugin_folder(self):
return self.PluginFolder
# Load the plugins from the plugin directory.
def load_plugins(self):
print "Configuring Plugin manager."
self.plugins = PluginManager()
if self.MinionFolder is None:
self.plugins.setPluginPlaces([self.PluginFolder])
else:
self.plugins.setPluginPlaces([self.PluginFolder, self.MinionFolder])
#self.plugins.collectPlugins()
self.plugins.locatePlugins()
self.plugins.loadPlugins()
print "Plugin manager configured."
# Loop round the plugins and print their names.
cur = self.db.cursor()
# Clear tables
cur.execute("""DELETE FROM enrichments""")
cur.execute("""DELETE FROM inputs""")
cur.execute("""DELETE FROM storage""")
cur.execute("""DELETE FROM score""")
cur.execute("""DELETE FROM minion""")
for plugin in self.plugins.getAllPlugins():
plugin_config = plugin.plugin_object.configure()
# Insert enrichment
if plugin_config[0] == 'enrichment': # type
cur.execute('''INSERT INTO enrichments VALUES (?, ?, ?, ?, ?)''', (plugin_config[2], # Name
int(plugin_config[1]), # Enabled
plugin_config[3], # Descripton
plugin_config[5], # Cost
plugin_config[6]) # Speed
)
for inp in plugin_config[4]: # inputs
# Insert into inputs table
cur.execute('''INSERT INTO inputs VALUES (?,?)''', (plugin_config[2], inp))
self.db.commit()
elif plugin_config[0] == 'interface': # type
cur.execute('''INSERT INTO storage VALUES (?, ?)''', (plugin_config[2], int(plugin_config[1])))
elif plugin_config[0] == 'score':
cur.execute('''INSERT INTO score VALUES (?, ?, ?, ?, ?)''', (plugin_config[2], # Name
int(plugin_config[1]), # Enabled
plugin_config[3], # Descripton
plugin_config[4], # Cost
plugin_config[5]) # Speed
)
if plugin_config[0] == 'minion':
plugin_config = plugin.plugin_object.configure(self)
cur.execute('''INSERT INTO minion VALUES (?, ?, ?, ?)''', (plugin_config[2], # Name
int(plugin_config[1]), # Enabled
plugin_config[3], # Descripton
plugin_config[4]) # Speed
)
if plugin.name == "classify": # Classify is a unique name. TODO: figure out if handling multiple 'classify' plugins is necessary
self.classify = plugin.plugin_object
print "Configured {2} plugin {0}. Success: {1}".format(plugin.name, plugin_config[1], plugin_config[0])
def set_db(self):
"""
Sets up the enrichment sqlite in memory database
"""
conn = sqlite3.connect(":memory:")
cur = conn.cursor()
# Create enrichments table
cur.execute('''CREATE TABLE enrichments (name text NOT NULL PRIMARY KEY,
configured int,
description text,
cost int,
speed int);''')
# Create inputs table
cur.execute('''CREATE TABLE inputs (name text NOT NULL,
input text NOT NULL,
PRIMARY KEY (name, input),
FOREIGN KEY (name) REFERENCES enrichments(name));''')
# Create interface table
cur.execute('''CREATE TABLE storage (name text NOT NULL PRIMARY KEY,
configured int
);''')
# Create score table
cur.execute('''CREATE TABLE score (name text NOT NULL PRIMARY KEY,
configured int,
description text,
cost int,
speed int);''')
# Create minion table
cur.execute('''CREATE TABLE minion (name text NOT NULL PRIMARY KEY,
configured int,
description text,
cost int);''')
conn.commit()
return conn
## ENRICHMENT FUNCTIONS
def get_inputs(self):
""" NoneType -> list of strings
:return: A list of the potential enrichment inputs (ip, domain, etc)
"""
inputs = list()
cur = self.db.cursor()
for row in cur.execute('''SELECT DISTINCT input FROM inputs;'''):
inputs.append(row[0])
return inputs
def get_enrichments(self, inputs, cost=10000, speed=10000, configured=True):
"""
:param inputs: list of input types. (e.g. ["ip", "domain"]) All enrichments that match at least 1 input type will be returned.
:param cost: integer 1-10 of resource cost of running the enrichment. (1 = cheapest)
:param speed: integer 1-10 speed of enrichment. (1 = fastest)
:param enabled: Plugin is correctly configured. If false, plugin may not run correctly.
:return: list of tuples of (names, type) of enrichments matching the criteria
"""
cur = self.db.cursor()
if type(inputs) == str:
inputs = [inputs]
plugins = list()
names = list()
for row in cur.execute(""" SELECT DISTINCT e.name, i.input
FROM enrichments e, inputs i
WHERE e.name = i.name
AND e.cost <= ?
AND e.speed <= ?
AND configured = ?
AND i.input IN ({0})""".format(("?," * len(inputs))[:-1]),
[cost,
speed,
int(configured)] +
inputs
):
plugins.append(tuple(row))
return plugins
def run_enrichments(self, topic, topic_type, names=None, cost=10, speed=10, start_time=""):
"""
:param topic: topic to enrich (e.g. "1.1.1.1", "www.google.com")
:param topic_type: type of topic (e.g. "ip", "domain")
:param cost: integer 1-10 of resource cost of running the enrichment. (1 = cheapest)
:param speed: integer 1-10 speed of enrichment. (1 = fastest)
:param names: a name (as string) or a list of names of enrichments to use
:return: None if storage configured (networkx graph representing the enrichment of the topic
"""
enrichments = self.get_enrichments([topic_type], cost, speed, configured=True)
enrichments = [e[0] for e in enrichments]
#print enrichments # DEBUG
g = nx.MultiDiGraph()
# IF a name(s) are given, subset to them
if names:
enrichments = set(enrichments).intersection(set(names))
for enrichment in enrichments:
# get the plugin
plugin = self.plugins.getPluginByName(enrichment)
# run the plugin
g2 = plugin.plugin_object.run(topic, start_time)
# merge the graphs
for node, props in g2.nodes(data=True):
g.add_node(node, props)
for edge in g2.edges(data=True):
g.add_edge(edge[0], edge[1], attr_dict=edge[2])
return g
## INTERFACE FUNCTIONS
def get_interfaces(self, configured=None):
"""
:return: list of strings of names of interface plugins
"""
cur = self.db.cursor()
interfaces = list()
if configured is None:
for row in cur.execute('''SELECT DISTINCT name FROM storage;'''):
interfaces.append(row[0])
else:
for row in cur.execute('''SELECT DISTINCT name from storage WHERE configured=?;''', (int(configured),)):
interfaces.append(row[0])
return interfaces
def get_default_interface(self):
return self.storage
def set_interface(self, interface):
"""
:param interface: The name of the plugin to use for storage.
Sets the storage backend to use. It must have been configured through a plugin prior to setting.
"""
cur = self.db.cursor()
configured_storage = list()
for row in cur.execute('''SELECT DISTINCT name FROM storage WHERE configured=1;'''):
configured_storage.append(row[0])
if interface in configured_storage:
self.storage = interface
else:
raise ValueError("Requested interface {0} not configured. Options are {1}.".format(interface, configured_storage))
def get_minions(self, cost=10000, configured=None):
"""
:param cost: a maximum cost of running the minion
:param configured: True, False, or None (for both).
:return: list of strings of tuples of (name, description) of minion plugins
"""
cur = self.db.cursor()
minions = list()
if configured is None:
for row in cur.execute('''SELECT DISTINCT name, description FROM minion WHERE cost <= ?;''', [int(cost)]):
minions.append(tuple(row[0:2]))
else:
for row in cur.execute('''SELECT DISTINCT name, description FROM minion WHERE cost <= ? AND configured=?;''', [int(cost), int(configured)]):
minions.append(tuple(row[0:2]))
return minions
def start_minions(self, names=None, cost=10000):
"""
:param names: a list of names of minions to run
:param cost: a maximum cost for minions
"""
minions = self.get_minions(cost=cost, configured=True)
minions = [m[0] for m in minions]
# IF a name(s) are given, subset to them
if names:
minions = set(minions).intersection(set(names))
for minion in minions:
# get the plugin
plugin = self.plugins.getPluginByName(minion)
# start the plugin
plugin.plugin_object.start()
def get_running_minions(self):
"""
:return: A set of names of minions which are running
"""
minions = self.get_minions(cost=10000, configured=True)
minions = [m[0] for m in minions]
running_minions = set()
# Iterate Through the minions
for minion in minions:
plugin = self.plugins.getPluginByName(minion)
if plugin.plugin_object.isAlive():
running_minions.add(minion)
return running_minions
def stop_minions(self, names=None):
minions = self.get_running_minions()
if names is not None:
minions = set(minions).intersection(set(names))
for minion in minions:
# get the plugin
plugin = self.plugins.getPluginByName(minion)
# start the plugin
plugin.plugin_object.stop()
def run_query(self, topic, max_depth=4, dont_follow=['enrichment', 'classification'], storage=None):
"""
:param storage: the storage plugin to use
:return: a networkx subgraph surrounded around the topic
"""
if not storage:
storage = self.storage
if not storage:
raise ValueError("No storage set. run set_storage() to set or provide directly. Storage must be a configured plugin.")
else:
# get the plugin
plugin = self.plugins.getPluginByName(self.storage)
return plugin.plugin_object.query(topic, max_depth=max_depth, dont_follow=dont_follow)
def store_graph(self, g, storage=None):
"""
:param g: a networkx graph to merge with the set storage
"""
if not storage:
storage = self.storage
if not storage:
raise ValueError("No storage set. run set_storage() to set or provide directly. Storage must be a configured plugin.")
else:
# get the plugin
plugin = self.plugins.getPluginByName(self.storage)
# merge the graph
plugin.plugin_object.enrich(g)
## SCORE FUNCTIONS
def get_scoring_plugins(self, cost=10000, speed=10000, names=None, configured=True):
"""
:param cost: integer 1-10 of resource cost of running the enrichment. (1 = cheapest)
:param speed: integer 1-10 speed of enrichment. (1 = fastest)
:param enabled: Plugin is correctly configured. If false, plugin may not run correctly.
:return: list of names of scoring plugins matching the criteria
"""
cur = self.db.cursor()
plugins = list()
if names is None:
for row in cur.execute('''SELECT DISTINCT name
FROM score
WHERE cost <= ?
AND speed <= ?
AND configured = ?''',
[cost,
speed,
int(configured)]
):
plugins.append(row[0])
else:
for row in cur.execute('''SELECT DISTINCT name
FROM score
WHERE cost <= ?
AND speed <= ?
AND configured = ?
AND name IN ({0});'''.format(("?," * len(names))[:-1]),
[cost,
speed,
int(configured)] +
names
):
plugins.append(row[0])
return plugins
def score_subgraph(self, topic, sg, plugin_name=None):
if plugin_name is None:
plugin_name=self.score
score_plugin = self.plugins.getPluginByName(plugin_name)
return score_plugin.plugin_object.score(sg, topic)
def set_scoring_plugin(self, plugin):
"""
:param interface: The name of the plugin to use for storage.
Sets the storage backend to use. It must have been configured through a plugin prior to setting.
"""
cur = self.db.cursor()
configured_scoring_plugins = list()
for row in cur.execute('''SELECT DISTINCT name FROM score WHERE configured=1;'''):
configured_scoring_plugins.append(row[0])
if plugin in configured_scoring_plugins:
self.score = plugin
else:
raise ValueError("Requested scoring plugin {0} is not configured. Options are {1}.".format(plugin, configured_scoring_plugins))
def get_default_scoring_plugin(self):
return self.score
class MainWindow(QMainWindow):
kill_thread = Signal()
def __init__(self, app):
self.app = app
# self.qmp = QMP('localhost', 55555)
self.qmp = QMP()
self.qmp.stateChanged.connect(self.handle_pause_button)
self.qmp.connectionChange.connect(self.handle_connect_button)
self.paused = False
super().__init__()
self.init_ui()
self.qmp.timeUpdate.connect(self.update_time)
self.t = TimeThread(self.qmp)
self.time_mult = TimeMultiplier(self.qmp, self.kill_thread)
self.window = []
self.default_theme = QGuiApplication.palette()
def init_ui(self):
# Window Setup
self.setWindowTitle("QEMU Control")
self.setGeometry(100, 100, 275, 225)
self.setFixedSize(self.size())
# App Icon
icon = QIcon('package/icons/qemu-official.png')
self.setWindowIcon(icon)
# User Interface
self.menu_bar()
self.grid_layout()
self.show()
def menu_bar(self):
bar = self.menuBar()
# Menu Bar Actions
file_ = bar.addMenu("File")
edit = bar.addMenu("Edit")
run = bar.addMenu("Run")
tools = bar.addMenu("Tools")
help_ = bar.addMenu("Help")
# File Menu Options
open_ = QAction("Open Image", self)
file_.addAction(open_)
exit_ = QAction("Exit", self)
exit_.triggered.connect(self.close)
exit_.setShortcut('Ctrl+W')
file_.addAction(exit_)
# Edit Menu Options
prefs = QAction("Preferences", self, triggered=lambda:self.open_new_window(Preferences(self.app, self.default_theme, self.qmp, self.t)))
edit.addAction(prefs)
# Run Menu Options
pause = QAction("Pause", self, triggered=lambda:self.qmp.command('stop'))
run.addAction(pause)
play = QAction("Play", self, triggered=lambda:self.qmp.command('cont'))
run.addAction(play)
# Debug Menu Options
hexdmp = QAction("Memory Dump", self, triggered=(lambda: self.open_new_window(MemDumpWindow(self.qmp)) if self.qmp.isSockValid() else None))
tools.addAction(hexdmp)
asm = QAction("Assembly View", self, triggered=(lambda: self.open_new_window(AssemblyWindow(self.qmp)) if self.qmp.isSockValid() else None))
tools.addAction(asm)
registers = QAction("CPU Register View", self, triggered=(lambda: self.open_new_window(RegisterView(self.qmp)) if self.qmp.isSockValid() else None))
tools.addAction(registers)
errors = QAction("Logging View", self, triggered=lambda:self.open_new_window(LoggingWindow(self.qmp)))
tools.addAction(errors)
tree = QAction("Memory Tree", self, triggered=(lambda: self.open_new_window(MemTree(self.qmp, self)) if self.qmp.isSockValid() else None))
tools.addAction(tree)
mult = QAction("Time Multiplier", self, triggered=(lambda: self.time_mult.show() if self.qmp.isSockValid() else None))
tools.addAction(mult)
trace = QAction("Trace Event Viewer", self, triggered=lambda: self.open_new_window(TraceWindow(self.qmp)))
tools.addAction(trace)
self.addPlugins(tools)
# Help Menu Options
usage = QAction("Usage Guide", self)
help_.addAction(usage)
def addPlugins(self, menu):
plugins = menu.addMenu('Plugins')
self.manager = PluginManager()
self.manager.setPluginPlaces(['plugins'])
self.manager.locatePlugins()
self.manager.loadPlugins()
for plugin in self.manager.getAllPlugins():
plugins.addAction(QAction(plugin.name, self, triggered=(lambda: self.open_new_window(plugin.plugin_object.display(self.qmp)) if self.qmp.isSockValid() else None)))
def grid_layout(self):
grid = QVBoxLayout()
grid.setSpacing(15)
self.pause_button = QPushButton('■')
self.running_state = QLabel('Current State: <font color="grey">Inactive</font>')
def cont_sim():
self.pause_button.setText('■')
self.running_state.setText('Current State: <font color="green">Running</font>')
self.qmp.command('cont')
def stop_sim():
self.pause_button.setText('▶')
self.running_state.setText('Current State: <font color="red">Paused</font>')
self.qmp.command('stop')
subgrid = QHBoxLayout()
self.pause_button.clicked.connect(lambda: stop_sim() if not self.paused else cont_sim())
self.pause_button.setFixedSize(QSize(50, 50))
subgrid.addWidget(self.pause_button, 0)
# self.pause_button.setCheckable(True)
# self.handle_pause_button(False)
self.pause_button.setEnabled(False)
meatball = QLabel(self)
logo = QPixmap('package/icons/nasa.png')
logo = logo.scaled(75, 75, Qt.KeepAspectRatio)
meatball.setPixmap(logo)
subgrid.addWidget(meatball, 1)
grid.addLayout(subgrid, 0)
self.time = QLabel('Time: 00:00:00')
self.time.setFont(QFont('Courier New'))
grid.addWidget(self.time, 1)
grid.addWidget(self.running_state, 2)
self.banner = QLabel('<font color="grey">Connect to QMP to get started!</font>')
grid.addWidget(self.banner, 3)
conn_grid = QHBoxLayout()
self.connect_button = QPushButton("Connect")
self.connect_button.setCheckable(True)
self.connect_button.clicked.connect(self.qmp_start)
self.host = QLineEdit()
self.host.returnPressed.connect(lambda: self.connect_button.click() if not self.connect_button.isChecked() else None)
self.port = QLineEdit()
self.port.returnPressed.connect(lambda: self.connect_button.click() if not self.connect_button.isChecked() else None)
conn_grid.addWidget(self.host)
conn_grid.addWidget(self.port)
conn_grid.addWidget(self.connect_button)
grid.addLayout(conn_grid)
center = QWidget()
center.setLayout(grid)
self.setCentralWidget(center)
def throwError(self):
msgBox = QMessageBox(self)
msgBox.setText('Lost Connection to QMP!')
msgBox.show()
@Slot(bool)
def handle_pause_button(self, value):
# Catches signals from QMPWrapper
#print('recieved: ', value)
# time.sleep(0.05) # fix race condition
if value:
self.paused = False
self.pause_button.setText('■')
self.running_state.setText('Current State: <font color="green">Running</font>')
elif not value and value is not None:
self.paused = True
self.pause_button.setText('▶')
self.running_state.setText('Current State: <font color="red">Paused</font>')
def handle_connect_button(self, value):
self.connect_button.setChecked(value)
self.host.setReadOnly(value)
self.port.setReadOnly(value)
def open_new_window(self, new_window):
if self.qmp.isSockValid():
self.window.append(new_window)
def update_time(self, time):
date = datetime.fromtimestamp(time / 1000000000, timezone.utc)
self.time.setText(f'Time: {date.day - 1:02}:{date.hour:02}:{date.minute:02}:{date.second:02}') # -1 for day because it starts from 1
def qmp_start(self):
if self.qmp.isSockValid():
self.qmp.sock_disconnect()
self.kill_thread.emit()
self.banner.setText('<font color="grey">Connect to QMP to get started!</font>')
self.pause_button.setText('■')
self.running_state.setText('Current State: <font color="grey">Inactive</font>')
self.pause_button.setEnabled(False)
return
else:
s = self.port.text()
if s.isnumeric():
self.qmp.sock_connect(self.host.text(), int(s))
if self.qmp.isSockValid():
self.time_mult.start()
self.banner.setText('QEMU Version ' + str(self.qmp.banner['QMP']['version']['package']))
self.pause_button.setEnabled(True)
else:
self.host.setText('127.0.0.1')
self.port.setText('55555')
self.qmp.sock_connect('127.0.0.1', 55555)
if self.qmp.isSockValid():
self.time_mult.start()
self.banner.setText('QEMU Version ' + str(self.qmp.banner['QMP']['version']['package']))
self.pause_button.setEnabled(True)
# check if running initally
if self.qmp.running:
self.paused = False
self.pause_button.setText('■')
self.running_state.setText('Current State: <font color="green">Running</font>')
else:
self.paused = True
self.pause_button.setText('▶')
self.running_state.setText('Current State: <font color="red">Paused</font>')
if not self.qmp.isAlive():
self.qmp.start()
if not self.t.isAlive():
self.t.start()
def closeEvent(self, event):
self.kill_thread.emit()
event.accept()
def show_time_mult(self):
self.scene = QGraphicsScene()
self.view = QGraphicsView(self.scene)
self.scene.addItem(self.time_mult.chart)
self.view.show()
class binWidget(QtGui.QWidget, Observable):
scrolled = QtCore.pyqtSignal(int, name='scroll')
oldscrolled = QtCore.SIGNAL('scroll')
def __init__(self, parent, source):
super(binWidget, self).__init__()
Observable.__init__(self)
self.parent = parent
# offset for text window
#self.data = mapped
self.dataOffset = 0
self.dataModel = source
self.cursor = Cursor(0, 0)
# self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self),
# HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self)]
logging.basicConfig(level=logging.ERROR)
self.manager = PluginManager(categories_filter={ "FileFormat": FileFormat})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'format')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
Formats = []
for plugin in self.manager.getPluginsOfCategory("FileFormat"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.recognize(self.dataModel):
print '[+] ' + po.name
Formats.append(po)
# sort plugins by priority
Formats = sorted(Formats, key=lambda x: x.priority, reverse=True)
po = Formats[0]
print 'Choosed plugin: ' + po.name
#print QtGui.QFontDatabase.addApplicationFont(os.path.join('terminus-ttf-4.39', 'TerminusTTF-4.39.ttf'))
self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po),
HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po),
DisasmViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self, plugin=po)]
self.viewMode = self.multipleViewModes[0]
self.textDecorator = TextDecorator(self.viewMode)
self.viewMode.setTransformationEngine(self.textDecorator)
self.multipleViewModes[1].setTransformationEngine(self.textDecorator)
self.Banners = Banners()
#self.Banners.add(BottomBanner(self.dataModel, self.viewMode))
# self.Banners.add(TopBanner(self.dataModel, self.viewMode))
#self.Banners.add(self.banner)
# self.filebanner = FileAddrBanner(self.dataModel, self.viewMode)
#self.filebanner = PEBanner(self.dataModel, self.viewMode)
#self.Banners.add(PEBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
# self.offsetWindow_h = self.filebanner.getDesiredGeometry()[0] + 25
self.offsetWindow_h = 0
self.offsetWindow_v = 0
self.searchable = Searchable(self.dataModel, self.viewMode)
self.initUI()
[po.init(viewMode, parent=self) for viewMode in self.multipleViewModes]
for banner in po.getBanners():
self.Banners.add(banner)
po.registerShortcuts(self)
self.po = po
#self.scrolled = QtCore.pyqtSignal(int, name='scroll')
#self.scrolled.connect(self.scroll_from_outside)
self.searchWindow = SearchWindow(self, None, self.searchable)
self.addHandler(self.po)
self.addHandler(self.searchable)
self.addHandler(self.Banners)
self.notify(self.viewMode)
#self.connect(self, self.oldscrolled, self.scroll_from_outside)
#self.scrolled.emit(1)
#self.emit(QtCore.SIGNAL('scroll'), 1)
def scroll_from_outside(self, i):
#print 'slot-signal ' + str(i)
#self.scroll_pdown = True
self.update()
def initUI(self):
self.setMinimumSize(1, 30)
self.activateWindow()
self.setFocus()
#self.installEventFilter(self)
"""
# build thumbnail
dwidth = 100
dheight = 1200
factor = dheight/dwidth
import math
x = int(math.sqrt(len(self.data)/factor))
cols = x
pixThumb = QtGui.QPixmap(x*self.viewMode.fontWidth, factor*x*self.viewMode.fontHeight)
qp = QtGui.QPainter()
qp.begin(pixThumb)
qp.setFont(self.viewMode.font)
qp.setPen(self.viewMode.textPen)
for i,c in enumerate(self.data):
self.viewMode.transformText(qp, (i, c), self.data)
qp.drawText((i%cols)*self.viewMode.fontWidth, self.viewMode.fontHeight + (i/cols)*self.viewMode.fontHeight, c)
qp.end()
self.newqpix = pixThumb.scaled(dwidth, dheight, aspectRatioMode = QtCore.Qt.IgnoreAspectRatio, transformMode = QtCore.Qt.FastTransformation)
"""
"""
def getDisplayedData(self):
if self.dataOffset < 0:
self.dataOffset = 0
return False
chrlist = [unichr(cp437ToUnicode[ord(c)]) for c in self.data[self.dataOffset : self.dataOffset + self.viewMode.COLUMNS*self.viewMode.ROWS]]
self.text = "".join(chrlist)
return True
"""
def switchViewMode(self):
self.multipleViewModes = self.multipleViewModes[1:] + [self.multipleViewModes[0]]
self.viewMode = self.multipleViewModes[0]
# notify obervers
self.notify(self.viewMode)
def _resize(self):
self.Banners.resize(self.size().width() - self.offsetWindow_h, self.size().height() - self.offsetWindow_v)
# compute space ocupated by banners
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getBottomOffset() + self.Banners.getTopOffset()
# resize window, substract space ocupated by banners
self.viewMode.resize(self.size().width() - offsetLeft, self.size().height() - offsetBottom)
# event handlers
def resizeEvent(self, e):
self._resize()
def paintEvent(self, e):
qp = QtGui.QPainter()
qp.begin(self)
qp.setOpacity(1)
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getTopOffset()
#self.viewMode.draw2(qp, refresh=True)
#start = time()
qp.drawPixmap(offsetLeft, offsetBottom, self.viewMode.getPixmap())
#print 'Draw ' + str(time() - start)
self.Banners.draw(qp, self.offsetWindow_h, self.offsetWindow_v, self.size().height())
# qp.drawPixmap(self.offsetWindow_h, self.size().height() - 50, self.banner.getPixmap())
# qp.drawPixmap(20, 0, self.filebanner.getPixmap())
qp.end()
"""
def keyPressEvent(self, event):
if event.modifiers() & QtCore.Qt.ShiftModifier:
if self.viewMode.handleKeyPressEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
def keyReleaseEvent(self, event):
if event.key() == QtCore.Qt.Key_Shift:
if self.viewMode.handleKeyReleaseEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
"""
def eventFilter(self, watched, event):
if event.type() == QtCore.QEvent.KeyRelease:
key = event.key()
modifiers = event.modifiers()
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
self.update()
if event.type() == QtCore.QEvent.KeyPress:
#TODO: should we accept only certain keys ?
key = event.key()
modifiers = event.modifiers()
if key == QtCore.Qt.Key_F2:
if self.viewMode.isEditable():
if self.viewMode.isInEditMode():
self.viewMode.setEditMode(False)
else:
self.viewMode.setEditMode(True)
self.viewMode.draw(refresh=False)
# switch view mode
if key == QtCore.Qt.Key_Tab:
offs = self.viewMode.getCursorOffsetInPage()
base = self.viewMode.getDataModel().getOffset()
self.switchViewMode()
self._resize()
self.viewMode.goTo(base + offs)
self.update()
import pyperclip
if event.modifiers() & QtCore.Qt.ControlModifier:
if key == QtCore.Qt.Key_Insert:
if self.viewMode.selector.getCurrentSelection():
a, b = self.viewMode.selector.getCurrentSelection()
#print a, b
hx = ''
for s in self.dataModel.getStream(a, b):
hx += '{:02x}'.format(s)
pyperclip.copy(hx)
del pyperclip
#print pyperclip.paste()
# print 'coppied'
if event.modifiers() & QtCore.Qt.ShiftModifier:
if key == QtCore.Qt.Key_Insert:
import re
hx = pyperclip.paste()
#print hx
L = re.findall(r'.{1,2}', hx, re.DOTALL)
array = ''
for s in L:
array += chr(int(s, 16))
#print 'write '
#print 'write'
#print array
self.dataModel.write(0, array)
self.viewMode.draw(True)
del pyperclip
#print array
if key == QtCore.Qt.Key_F4:
self.unp = WUnpack(self, None)
self.unp.show()
if key == QtCore.Qt.Key_F10:
self.dataModel.flush()
import os
self.w = WHeaders(self, None)
self.w.show()
if not self.viewMode.isInEditMode():
if key == QtCore.Qt.Key_Slash:
self.searchWindow.show()
if key == QtCore.Qt.Key_N:
self.searchable.next(self.viewMode.getCursorAbsolutePosition() + 1)
if key == QtCore.Qt.Key_B:
self.searchable.previous(self.viewMode.getCursorAbsolutePosition())
# handle keys to view plugin
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
event.accept()
self.update()
return True
return False
def setTextViewport(self, qp):
qp.setViewport(self.offsetWindow_h, self.offsetWindow_v, self.size().width(), self.size().height())
qp.setWindow(0, 0, self.size().width(), self.size().height())
def needsSave(self):
return self.dataModel.isDirty()
def save(self):
return self.dataModel.flush()
self.build_pkg_hook_args=""
self.version=None
self.define_env_version=None
self.config_version_db_path="/var/tmp/dbversion.db"
self.config_version_db=None
plugin_dir = os.path.expanduser("~/.repacked/plugins")
if not os.path.exists(plugin_dir):
plugin_dir = os.path.join(os.path.dirname(__file__),'../../repacked/plugins')
pkg_plugins = {}
pluginMgr = PluginManager(plugin_info_ext="plugin")
pluginMgr.setPluginPlaces([plugin_dir])
pluginMgr.locatePlugins()
pluginMgr.loadPlugins()
for pluginInfo in pluginMgr.getAllPlugins():
pluginMgr.activatePluginByName(pluginInfo.name)
def parse_spec(filename):
"""
Loads the YAML file into a Python object for parsing
and returns it
"""
fp = open(filename, 'r')
spec = yaml.safe_load("\n".join(fp.readlines()))
return spec
class gEcrit(wx.Frame):
"""
Editor
This class is the entry point in the program.
It creates all the user interface and initializes
the required objects and classes.
The functions that cannot go into another objects
for diverse reasons go here.
"""
def dummy_tr(self, tr):
return tr
def __init__(self, id, parent):
"""
__init__
Creates the user interface.
Initializez the terminals if enabled.
Creates the required GUI and non GUI objects.
"""
BOTTOMPANEL_ID = 4002
SIDEPANEL_ID = 3999
try:
self.presLang = gettext.translation("gEcrit", "./locale")
self._ = self.presLang.ugettext
self.presLang.install()
except:
print("Translation for local language not found.")
self._ = self.dummy_tr
pathname = os.path.abspath(os.path.dirname((sys.argv)[0])) # Finding where
os.chdir(pathname) # gEcrit is running
#Setting up the plugin envirenment
self.general_plugins = {}
self.passive_plugins = {}
self.plugin_manager = PluginManager(
categories_filter={"General": General,
"Passives" : Passive})
#Sets YAPSY the plugin directory.
self.plugin_path = os.path.join(pathname, "data", "plugins")
self.plugin_manager.setPluginPlaces([self.plugin_path])
self.plugin_manager.locatePlugins()
#self.plugin_manager.collectPlugins()
self.plugin_manager.loadPlugins()
self.activated_plugins = Config.GetOption("ActivePlugins")
#populating the general plugin index
for f in self.plugin_manager.getPluginsOfCategory("General"):
if f.plugin_object.name in self.activated_plugins:
self.general_plugins[f.plugin_object.name] = f.plugin_object
#the passive plugins now
for p in self.plugin_manager.getPluginsOfCategory("Passives"):
if p.plugin_object.name in self.activated_plugins:
self.passive_plugins[p.plugin_object.name] = p.plugin_object
self.id_range = []
#getting the command line file argument
if "gEcrit.py" not in (sys.argv)[-1]:
target_file = os.path.normpath(os.path.realpath(sys.argv[-1]))
#no file was provided
else:
target_file = None
wx.Frame.__init__(self, parent, 1000, 'gEcrit', size=(700, 600))
self.Bind(wx.EVT_CLOSE, self.OnQuit)
#this object will handle layout and docking/undocking of widgets
self.aui_manager = wx.aui.AuiManager(self)
#creating the status bar
self.status_bar = self.CreateStatusBar()
self.status_bar.SetStatusText("Done")
self.status_bar.SetFieldsCount(3)
self.status_bar.SetId(999)
if not Config.GetOption("StatusBar"):
self.status_bar.Hide()
self.menubar = MainMenu(self)
self.SetMenuBar(self.menubar)
#setting the application icon
self.SetIcon(wx.Icon('icons/gEcrit.png', wx.BITMAP_TYPE_PNG))
#this variable is incremented each time we create a StcControl
self.text_id = 0
#finding the user home folder
self.HOMEDIR = os.path.expanduser('~')
os.chdir(os.path.abspath(self.HOMEDIR))
#creating a plugin manager instance
self.plugin_conf_manager = gEcritPluginManager(self)
#creating the left side notebook
self.side_notebook = wx.aui.AuiNotebook(self, id=SIDEPANEL_ID, size=(-1,-1),
style=wx.BORDER_SUNKEN|wx.aui.AUI_NB_TAB_SPLIT|wx.aui.AUI_NB_TAB_MOVE|wx.aui.AUI_NB_SCROLL_BUTTONS )
#creating the bottom side notebook
self.bottom_notebook = wx.aui.AuiNotebook(self, id=BOTTOMPANEL_ID, size=(-1,
-1), style=wx.BORDER_SUNKEN|wx.aui.AUI_NB_TAB_SPLIT|wx.aui.AUI_NB_TAB_MOVE|wx.aui.AUI_NB_SCROLL_BUTTONS )
#the aui notebook that will manage editor tabs
self.nb = AuiNoteBook(parent = self)
#going back to application running point
os.chdir(pathname)
#binding the menubar events
f = wx.FindWindowById
self.Bind(wx.EVT_MENU, lambda event: self.NewTab(event,
"New Document", "New Document"), id=500)
self.Bind(wx.EVT_MENU, lambda event: self.OnOpenFile(event), id=
501)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).Save(event),
id=502)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).SaveAs(event),
id=503)
self.Bind(wx.EVT_MENU, self.OnPrint,id=504)
self.Bind(wx.EVT_MENU, lambda event: self.OnMenuCloseTab(event,
(self.id_range)[self.nb.GetSelection()]), id=505)
self.Bind(wx.EVT_MENU, lambda event: self.OnQuit(event), id=506)
self.Bind(wx.EVT_MENU, self.SaveAll, id=563)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnReload(event),id = 507)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnUndo(event),
id=520)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnRedo(event),
id=521)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnCut(event),
id=522)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnCopy(event),
id=523)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnPaste(event),
id=524)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnSelectAll(event),
id=525)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnSelectCodeBlock(event),
id=562)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnInsertDate(event),
id=526)
self.Bind(wx.EVT_MENU, lambda event: self.OnPrefs(event), id=527)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnDedent(event),
id=528)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnIndent(event),
id=529)
self.Bind(wx.EVT_MENU, lambda event:f((self.id_range)[self.nb.GetSelection()]).OnComment(event),
id=559)
self.Bind(wx.EVT_MENU, lambda event:f((self.id_range)[self.nb.GetSelection()]).OnUnComment(event),
id=560)
self.Bind(wx.EVT_MENU, lambda event: FindRepl.FindDocText(event, (self.id_range)[self.nb.GetSelection()]),
id=530)
self.Bind(wx.EVT_MENU, lambda event: FindRepl.ReplaceDocText(event, (self.id_range)[self.nb.GetSelection()]),
id=531)
self.Bind(wx.EVT_MENU, lambda event: FindRepl.FindDocText(event, (self.id_range)[self.nb.GetSelection()],wx.stc.STC_FIND_REGEXP),
id=532)
self.Bind(wx.EVT_MENU, lambda event: FindRepl.ReplaceDocText(event ,(self.id_range)[self.nb.GetSelection()], wx.stc.STC_FIND_REGEXP),
id=533)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnStartRecordMacro(event), id=534)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnStopRecordMacro(event), id=542)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnMacroPlayback(event), id=543)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnZoomIn(event),
id=535)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnZoomOut(event),
id=536)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnResetZoom(event),
id=537)
self.Bind(wx.EVT_MENU, lambda event: Config.ChangeOption("LineNumbers",
self.menubar.IsChecked(538), self.id_range), id=538)
self.Bind(wx.EVT_MENU, lambda event: Config.ChangeOption("FoldMarks",
self.menubar.IsChecked(539), self.id_range), id=539)
self.Bind(wx.EVT_MENU, lambda event: Config.ChangeOption("Whitespace",
self.menubar.IsChecked(540), self.id_range), id=540)
self.Bind(wx.EVT_MENU, lambda event: Config.ChangeOption("IndetationGuides",
self.menubar.IsChecked(541), self.id_range), id=541)
self.Bind(wx.EVT_MENU, lambda event: Config.ChangeOption("EdgeLine",
self.menubar.IsChecked(546), self.id_range), id=546)
self.Bind(wx.EVT_MENU, lambda event: Config.ChangeOption("SyntaxHighlight",
self.menubar.IsChecked(547), self.id_range), id=547)
self.Bind(wx.EVT_MENU, lambda event: Config.ChangeOption("StatusBar",
self.menubar.IsChecked(545), self.id_range), id=545)
self.Bind(wx.EVT_MENU, self.OnFullScreen, id=557)
self.Bind(wx.EVT_MENU, self.ToggleSidePanel, id = 548)
self.Bind(wx.EVT_MENU, self.ToggleBottomPanel, id = 549)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).OnRemoveTrails(event),id=551)
self.Bind(wx.EVT_MENU, lambda event: self.OnRun(event,self.id_range[self.nb.GetSelection()]), id = 558)
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).Tabify(event), id = 552 )
self.Bind(wx.EVT_MENU, lambda event: f((self.id_range)[self.nb.GetSelection()]).UnTabify(event), id = 553 )
self.Bind(wx.EVT_MENU, self.SaveSessionFile , id = 554)
self.Bind(wx.EVT_MENU, gEcritSession.DeleteSessionFile , id = 555)
self.Bind(wx.EVT_MENU, lambda event: Config.ChangeOption("Session",self.menubar.IsChecked(556)) , id = 556)
self.Bind(wx.EVT_MENU, self.plugin_conf_manager.ShowMe, id = 564 )
self.Bind(wx.EVT_MENU, lambda event: self.OnAbout(event), id=550)
#setting up the toolbar
self.toolbar = MainToolbar(self, -1)
self.FontCtrl = wx.FontPickerCtrl(self.toolbar, 607, size=(100,
30))
self.Bind(wx.EVT_FONTPICKER_CHANGED, lambda event: ChangeFont(event,
self.FontCtrl.GetSelectedFont(), self.id_range))
#the goto line text box
self.toolbar.AddControl(self.FontCtrl)
self.toolbar.AddControl(wx.TextCtrl(self.toolbar, 608, size=(-1,
-1), style=wx.TE_PROCESS_ENTER))
#Binding toolbar events
self.Bind(wx.EVT_TOOL, lambda event: self.NewTab(event,
"New Document", "New Document"), id=600)
self.Bind(wx.EVT_TOOL, self.OnOpenFile, id=601)
self.Bind(wx.EVT_TOOL, lambda event: f((self.id_range)[self.nb.GetSelection()]).Save(event),
id=602)
self.Bind(wx.EVT_TOOL, lambda event: f((self.id_range)[self.nb.GetSelection()]).SaveAs(event),
id=603)
self.Bind(wx.EVT_TOOL, self.OnPrefs, id=604)
self.Bind(wx.EVT_TOOL, self.OnQuit, id=605)
self.Bind(wx.EVT_TEXT_ENTER, lambda event: self.OnGotoBox(event,
(self.id_range)[self.nb.GetSelection()]), id=608)
self.Bind(wx.EVT_TOOL, self.OnPrint, id=609)
self.Bind(wx.EVT_TOOL, lambda event: self.OnRun(event, (self.id_range)[self.nb.GetSelection()]),
id=610)
#Give the plugins a chance to set themselves in the system
#generals first
for g in self.general_plugins:
self.general_plugins[g].Init(self)
#passives now
for p in self.passive_plugins:
self.passive_plugins[p].Init(self)
#put it in the middle of the sceen
self.Centre()
#the preferences window
self.conf_win = ConfFrame = CfgFrame(self)
#addung the pane to the aui manager.
self.aui_manager.AddPane(self.toolbar, wx.aui.AuiPaneInfo().Name("toolbar").Caption(self._("Toolbar")).ToolbarPane().Top().CloseButton(False))
self.aui_manager.AddPane(self.nb, wx.aui.AuiPaneInfo().Name("editor tabs").Caption(self._("Tabs")).CenterPane())
self.aui_manager.AddPane(self.bottom_notebook, wx.aui.AuiPaneInfo().Name("bottom panel").Caption(self._("Assistants and others")).Bottom().BestSize((700,150)).PinButton(True).MaximizeButton(True))
self.aui_manager.AddPane(self.side_notebook, wx.aui.AuiPaneInfo().Name("left_side panel").Caption(self._("Toolbox")).Left().BestSize((150,400)).PinButton(True).MaximizeButton(True))
#loading saved session if any exists and if enabled
if Config.GetOption("Session"):
self.LoadSessionFile()
#make changes visible
self.aui_manager.Update()
if target_file: #load command line file path argument
self.NewTab(0, os.path.split(target_file)[-1], target_file)
def LoadSessionFile(self):
"""
LoadSessionFile
Loads the session file if it exists.
If it does not, creates an instance.
"""
try:
self.session = gEcritSession.LoadFromFile()
self.session.RestoreAppState(self)
self.SetStatus(0,self._ ( "Session file loaded."))
except Exceptions.NoSessionFile:
self.session = gEcritSession()
def SaveSessionFile(self, event):
"""
SaveSessionFile
Reccords the application state and saves it to disk via the
session instance.
"""
try: #testing if a session object exists
self.session
except AttributeError:
self.session = gEcritSession()
self.session.RecordAppState(self)
self.session.SaveToFile()
self.SetStatus(event, self._ ("Session saved."))
def OnFullScreen(self,event):
"""
OnFullScreen
Makes the main window fullscreen.
"""
self.ShowFullScreen(not self.IsFullScreen(),wx.FULLSCREEN_NOCAPTION)
def OnPrefs(self, event):
"""
OnPrefs
Shows the preferences window.
"""
self.conf_win.ShowMe(0)
def NewTab(self, event, nb, file_path):
"""
NewTab
Creates a new AUI NOTEBOOK tab, adds the contents,
initializez a STC object for it and binds some of its events.
Creates the sidebar, adds a notebook and adds its utilities
in its tabs.
"""
if not file_path:
return
#update recent file list
if file_path != "New Document" and file_path != "":
if not os.path.exists(file_path):
wx.MessageDialog(None, self._ ("Could not load file.\nThe file ")+file_path+self._ (" does not exists."),self._ ("Input Error") ,wx.OK).ShowModal()
return
lst = Config.GetOption("RecentFiles")
lst.append(file_path)
Config.ChangeOption("RecentFiles",lst)
self.menubar.UpdateRecentFiles()
#the parent of the StcControl
panel = wx.Panel(self)
panel.identifierTag = nb
#hiding self.text_id
text_id = self.text_id
#set up the editor
text_ctrl = StcTextCtrl(panel, self.text_id, file_path)
#the StcControl sizer
text_ctrl_sizer = wx.BoxSizer(wx.HORIZONTAL)
text_ctrl_sizer.Add(text_ctrl, 1, wx.EXPAND)
panel.SetSizer(text_ctrl_sizer)
panel.Fit()
#append the id of this StcControl to the id_range
self.id_range.append(text_id)
text_ctrl.SetBufferedDraw(True)
#apply the font
text_ctrl.StyleSetFont(0, self.FontCtrl.GetSelectedFont())
#add the panel as a new tab
self.nb.AddPage(panel, str(nb), select=True)
if file_path == "New Document" or file_path == "":
#notify plugins
for g in self.general_plugins:
self.general_plugins[g].NotifyNewTabOpened()
self.text_id += 1
return text_ctrl
def OnRun(self, event, text_id):
"""
Runs the current document in a xterm window, for testing.
"""
cur_doc = wx.FindWindowById(text_id)
cur_doc.Save(0)
os.system("xterm -e sh runner.sh "+cur_doc.GetFilePath())
def OnGotoBox(self, event, text_id):
"""
OnGotoBox
Finds the current document, and scrolls to the line indicated
by its input upon the Return key.
"""
cur_doc = wx.FindWindowById(text_id)
goto = wx.FindWindowById(608)
scroll_pos = int(goto.GetLineText(0))
cur_doc.ScrollToLine(scroll_pos - 1)
def OnPrint(self, event):
"""
OnPrint
Finds the document, sets the prints name, and calls the
wxPython toolkit to print the contents
"""
print_dlg = PrettyPrinter(self)
del print_dlg
def OnAbout(self, event):
"""
OnAbout
Shows the about window.
"""
#ShowAbout = AboutWindow
about_win = AboutWindow()
del about_win
def OnQuit(self, event):
"""
OnQuit
Closes the main window, stops the terminals, and kills the
application process.
It promps the user for confirmation.
"""
#warn the user
warn_dlg = wx.MessageDialog(None,
self._ ("Please make sure that your data is\
saved.\nAre you sure you want to quit?"),
self._ ("Are you sure?"), style=wx.YES_NO)
warn_dlg_val = warn_dlg.ShowModal()
if warn_dlg_val != 5104: #YES
#call the quit method to stop the terminals and the plugins
self.Quit()
def Quit(self):
#stop ond notify all plugins of application shutdown.
#generals now
for g in self.general_plugins:
self.general_plugins[g].Stop()
for p in self.passive_plugins:
self.passive_plugins[p].Stop()
#stop the shells if activated
if Config.GetOption("Session"):
self.SaveSessionFile(0)
#exit status 0, all ok
sys.exit(0)
def OnMenuCloseTab(self, event, text_id):
self.ManageCloseTab(False, text_id)
def ManageCloseTab(self, event, text_id):
"""
ManageCloseTab
Manages the process of closing a tab.
Checks if document is saved, prompts the user if not.
If this is the last tab in the application, it closes the
terminals, the window and kills the application.
If not, it decreases the number of tabs and delted the AUI
NETBOOK page.
"""
cur_doc = wx.FindWindowById(text_id)
current_text = cur_doc.GetText()
#check if the user saved the changes
if cur_doc.save_record != current_text:
#if not, notify him
save_prompt = wx.MessageDialog(None, self._ ("The file ") + os.path.split(cur_doc.GetFilePath())[-1] +
self._ (" is not saved.\n\
Do you wish to save it?"), "",
style=wx.CANCEL | wx.YES | wx.NO)
prompt_val_ = save_prompt.ShowModal()
if prompt_val_ == 5103: #YES
if not cur_doc.Save(0):
event.Veto()
return
else:
self.id_range.remove(text_id)
elif prompt_val_ == 5101: #CANCEL
event.Veto()
return
elif prompt_val_ == 5104: #NO
self.id_range.remove(text_id)
save_prompt.Destroy()
else:
self.id_range.remove(text_id)
# skip the event and let the AuiNotebook handle the deletion
cur_doc.Deactivate() # tell the StcTextCtrl to prepare for deletition
if not event: # check if it was fired from menu
self.nb.DeletePage(self.nb.GetSelection())
else:
event.Skip()
def OnOpenFile(self, event):
"""
OnOpenFile
Collects a path for a new file via a file dialog.
"""
open_file_dlg = wx.FileDialog(None, style=wx.OPEN | wx.FD_MULTIPLE)
if self.menubar.last_recent != "":
#go to the last accessed folder
open_file_dlg.SetDirectory(os.path.split(self.menubar.last_recent)[0])
else:
open_file_dlg.SetDirectory(self.HOMEDIR)
if open_file_dlg.ShowModal() == wx.ID_OK:
paths = open_file_dlg.GetPaths()
self.OpenFile(paths)
del open_file_dlg
def OpenFile(self, paths):
"""
OpenFile
Calls NewTab with the collected path.
Supports multiple path selection.
"""
# if paths is a list, open an StcContrel for each of them
if isinstance(paths, types.ListType):
for f in paths:
self.NewTab(0, os.path.split(f)[-1], f)
Log.AddLogEntry(self._ ("Opened file ") + f)
#if a string, open an StcControl for it
else:
self.NewTab(0, os.path.split(paths)[-1], paths)
Log.AddLogEntry(self._ ("Opened file ") + paths)
#notify general plugins
for t in self.general_plugins:
try: #insulate from possible plugin errors
self.general_plugins[t].NotifyDocumentOpened()
except: pass
AutoComp.UpdateCTagsFiles(self.id_range)
def SetStatus(self, event, text):
"""
ResetStatus
Sets the status of statusbar.
"""
self.status_bar.SetStatusText(text)
# event.Skip()
def ResetStatus(self, event):
"""
ResetStatus
Sets the status bar status to nothing.
"""
self.status_bar.SetStatusText("")
event.Skip()
def SaveAll(self, event):
"""
SaveAll
Saves all the current documents using the
objects Save function.
"""
for id in self.id_range:
cur_doc = wx.FindWindowById(id)
if cur_doc.GetFilePath() != "" and cur_doc.GetFilePath() != \
"New Document":
cur_doc.Save(0)
####################################################################
# PLUGIN INTERFACE #
####################################################################
def ToggleSidePanel(self, event):
pane = self.aui_manager.GetPane(self.side_notebook)
if pane.IsShown(): pane.Hide()
else: pane.Show()
self.aui_manager.Update()
def ToggleBottomPanel(self, event):
pane = self.aui_manager.GetPane(self.bottom_notebook)
if pane.IsShown(): pane.Hide()
else: pane.Show()
self.aui_manager.Update()
def GetCurrentDocument(self):
"""
GetCurrentDocument
Returns the selected active buffer object.
"""
try:
return wx.FindWindowById(self.id_range[self.nb.GetSelection()])
except IndexError:
return None
def GetAllDocuments(self):
"""
GetALlDocuments
Returns all existing buffers.
"""
docs = []
for d in self.id_range:
docs.append(wx.FindWindowById((d)))
return docs
def AddToMenuBar(self,label,menu):
"""
AddToMenuBar
@id The id of the new menu entry.
@label The label of the new menu entry.
@menu A wx.Menu object which will be added in the Plugins menu.
Adds a wx.Menu object to menubar.
"""
return self.menubar.plugins.AppendMenu(-1,label,menu)
def RemoveFromMenubar(self, menu):
"""
RemoveFromMenubar
Removes the supplied argument menu from the plugins submenu.
"""
self.menubar.plugins.RemoveItem(menu)
def BindMenubarEvent(self, item, function):
"""
BindMenuBarEvent
@item The menu entry object which to be bint.
@function The function the item to be bint to.
Binds a wx.EVT_MENU event to the suplied function.
"""
self.Bind(wx.EVT_MENU, function, id = item.GetId())
def GetBottomPanel(self):
"""
GetBottomPanel
Returns the lower notebook.
"""
return self.bottom_notebook
def AddToBottomPanel(self, panel, name):
"""
AddToBottomPanel
Adds the suplied panel to the lower notebook with tho supplied
name label.
"""
self.bottom_notebook.AddPage(panel, name)
def GetSidePanel(self):
"""
GetSidePanel
Returns the side notebook.
"""
return self.side_notebook
def AddToSidePanel(self, panel, name):
"""
AddToSidePanel
Adds the suplied panel to the side notebook with tho supplied
name label.
"""
self.side_notebook.AddPage(panel, name)
def DeleteBottomPage(self, name):
"""
DeleteBottomPage
Deletes the tab named name from the lower notebook.
"""
self.bottom_notebook.DeletePage(Config.GetTab(name,
self.bottom_notebook))
def DeleteSidePage(self, name):
"""
DeleteSidePage
Deletes the tab named name from the side notebook.
"""
self.side_notebook.DeletePage(Config.GetTab(name,
self.side_notebook))
def AddPaneToAui(self, widget ,pane_info):
"""
"AddPaneToAui
@widget the widget to be added
@pane needs to be an AuiPaneInfo object.
Adds the pane to the aui manager.
"""
self.aui_manager.AddPane(widget, pane_info)
def AddToolbarToAui(self, toolbar, pane_info):
"""
AddToosbartoAui
@toolbar the wx.Toolbar object
@pane_info needs to be a wx.AuiPaneInfo object with it's name and caption
defined.
"""
self.aui_manager.AddPane(toolbar, pane_info.ToolbarPane().Top().CloseButton(False))
def GetAuiManager(self):
"""
GetAuiManager
Returns the AuiManager that is responsable for window layout.
"""
return self.aui_manager
def GetTabManager(self):
"""
GetTabManager
Returns the AuiNoteBook that is resposible for tabs management.
"""
return self.nb
def CreateNewDocument(self, name):
"""
CreateNewDocument
@name a string to be given to the new document as a name.
Creates a new empty document.
Returns a reference to the now StcControl
"""
return self.NewTab(0, name, "")
class ModuleManager():
def __init__(self, threads, kill_list, job_list):
self.threads = threads
self.kill_list = kill_list
self.job_list = job_list # Running jobs
self.pmanager = PluginManager()
self.pmanager.setPluginPlaces(["plugins"])
self.pmanager.collectPlugins()
self.pmanager.locatePlugins()
self.plugins = ['none']
num_plugins = len(self.pmanager.getAllPlugins())
if num_plugins == 0:
raise Exception("No Plugins Found!")
plugins = []
for plugin in self.pmanager.getAllPlugins():
plugin.threads = threads
self.plugins.append(plugin.name)
plugin.plugin_object.setname(plugin.name)
## Check for installed binaries
executable = ''
try:
settings = plugin.details.items('Settings')
for kv in settings:
executable = kv[1]
if executable.find('/') != -1 : #Hackish "looks like a file"
if os.path.exists(executable):
logging.info("Found file: {}".format(executable))
break
else:
raise Exception()
## TODO detect binaries not in "executable" setting
except:
raise Exception('[ERROR]: {} -- Binary does not exist -- {}'.format(plugin.name, executable))
plugins.append(plugin.name)
print "Plugins found [{}]: {}".format(num_plugins, sorted(plugins))
def run_module(self, module, job_data_orig, tar=False,
all_data=False, reads=False, meta=False,
overrides=True):
"""
Keyword Arguments:
module -- name of plugin
job_data -- dict of job parameters
tar -- return tar of all output, rather than module.OUTPUT file
all_data -- return module.OUTPUT and list of all files in self.outdir
reads -- include files if module.OUTPUT == 'reads'
Not recommended for large read files.
"""
job_data = job_data_orig
# job_data = copy.deepcopy(job_data_orig)
# # Pass back orig file descriptor
# try:
# job_data['out_report'] = job_data_orig['out_report']
# except:
# pass
if not self.has_plugin(module):
raise Exception("No plugin named {}".format(module))
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
plugin.plugin_object.update_settings(job_data)
if meta:
output = plugin.plugin_object(settings, job_data, self, meta=True)
else:
output = plugin.plugin_object(settings, job_data, self)
log = plugin.plugin_object.out_module.name
if tar:
tarfile = plugin.plugin_object.tar_output(job_data['job_id'])
return output, tarfile, [], log
if all_data:
if not reads and plugin.plugin_object.OUTPUT == 'reads':
#Don't return all files from plugins that output reads
data = []
else:
data = plugin.plugin_object.get_all_output_files()
return output, data, log
return output, [], log
def output_type(self, module):
return self.pmanager.getPluginByName(module).plugin_object.OUTPUT
def input_type(self, module):
return self.pmanager.getPluginByName(module).plugin_object.INPUT
def get_short_name(self, module):
try:
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
for kv in settings:
if kv[0] == 'short_name':
sn = kv[1]
break
return sn
except:
return None
def get_executable(self, module):
try:
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
for kv in settings:
if kv[0] == 'short_name':
sn = kv[1]
break
return sn
except:
return None
def has_plugin(self, plugin):
if not plugin.lower() in self.plugins:
logging.error("{} plugin not found".format(plugin))
return False
return True
def valid_modules(self, l):
""" Return filtered list of available modules """
return [m for m in l if not m.startswith('?') and self.has_plugin(m)]
def validate_pipe(self, pipe):
for stage in pipe:
for word in stage.replace('+', ' ').split(' '):
if not (word.startswith('?') or self.has_plugin(word)):
raise Exception('Invalid pipeline command')
def split_pipe(self, l):
""" Splits a multi-module string in to bins
Ex: 'kiki ?k=29 velvet' -> [[kiki, ?k=29], [velvet]]
"""
bins = []
for word in l:
if not word.startswith('?'):
bins.append([word])
elif word.startswith('?'):
bins[-1].append(word)
return bins
def parse_input(self, pipe):
"""
Parses inital pipe and separates branching bins
Ex: ['sga', '?p=True', 'kiki ?k=31 velvet', 'sspace']
"""
stages = phelper.parse_branches(pipe)
return stages
def parse_pipe(self, pipe):
""" Returns the pipeline(s)z of modules.
Returns parameter overrides from string.
e.g Input: [sga_ec 'kiki ?k=31 velvet ?ins=500' sspace]
Output: [kiki, velvet, a5], [{k:31}, {ins:500}, {}]
"""
# Parse param overrides
overrides = []
pipeline = []
module_num = -1
for group in pipe:
for word in group.split('+'):
if word.lower() == 'none':
pass
elif not word.startswith('?') and self.has_plugin(word): # is module
module_num = module_num + 1
pipeline.append(word)
overrides.append({})
elif word[1:-1].find('=') != -1: # is param
kv = word[1:].split('=')
overrides[module_num] = dict(overrides[module_num].items() +
dict([kv]).items())
return pipeline, overrides
class binWidget(QtWidgets.QWidget, Observable):
scrolled = QtCore.pyqtSignal(int, name='scroll')
# oldscrolled = QtWidgets.SIGNAL('scroll')
def __init__(self, parent, source):
super(binWidget, self).__init__()
Observable.__init__(self)
self.parent = parent
# offset for text window
#self.data = mapped
self.dataOffset = 0
self.dataModel = source
self.cursor = Cursor(0, 0)
# self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self),
# HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self)]
logging.basicConfig(level=logging.ERROR)
self.manager = PluginManager(
categories_filter={"FileFormat": FileFormat})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'format')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
Formats = []
for plugin in self.manager.getPluginsOfCategory("FileFormat"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.recognize(self.dataModel):
print('[+] ' + po.name)
Formats.append(po)
# sort plugins by priority
Formats = sorted(Formats, key=lambda x: x.priority, reverse=True)
po = Formats[0]
print('Choosed plugin: ' + po.name)
#print QtGui.QFontDatabase.addApplicationFont(os.path.join('terminus-ttf-4.39', 'TerminusTTF-4.39.ttf'))
self.multipleViewModes = [
BinViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po),
HexViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po),
DisasmViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po)
]
self.viewMode = self.multipleViewModes[0]
self.textDecorator = TextDecorator(self.viewMode)
self.viewMode.setTransformationEngine(self.textDecorator)
self.multipleViewModes[1].setTransformationEngine(self.textDecorator)
self.Banners = Banners()
#self.Banners.add(BottomBanner(self.dataModel, self.viewMode))
# self.Banners.add(TopBanner(self.dataModel, self.viewMode))
#self.Banners.add(self.banner)
# self.filebanner = FileAddrBanner(self.dataModel, self.viewMode)
#self.filebanner = PEBanner(self.dataModel, self.viewMode)
#self.Banners.add(PEBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
# self.offsetWindow_h = self.filebanner.getDesiredGeometry()[0] + 25
self.offsetWindow_h = 0
self.offsetWindow_v = 0
self.searchable = Searchable(self.dataModel, self.viewMode)
self.initUI()
[po.init(viewMode, parent=self) for viewMode in self.multipleViewModes]
for banner in po.getBanners():
self.Banners.add(banner)
po.registerShortcuts(self)
self.po = po
#self.scrolled = QtCore.pyqtSignal(int, name='scroll')
#self.scrolled.connect(self.scroll_from_outside)
self.searchWindow = SearchWindow(self, None, self.searchable)
self.addHandler(self.po)
self.addHandler(self.searchable)
self.addHandler(self.Banners)
self.notify(self.viewMode)
#self.connect(self, self.oldscrolled, self.scroll_from_outside)
#self.scrolled.emit(1)
#self.emit(QtCore.SIGNAL('scroll'), 1)
def scroll_from_outside(self, i):
#print 'slot-signal ' + str(i)
#self.scroll_pdown = True
self.update()
def initUI(self):
self.setMinimumSize(1, 30)
self.activateWindow()
self.setFocus()
#self.installEventFilter(self)
"""
# build thumbnail
dwidth = 100
dheight = 1200
factor = dheight/dwidth
import math
x = int(math.sqrt(len(self.data)/factor))
cols = x
pixThumb = QtGui.QPixmap(x*self.viewMode.fontWidth, factor*x*self.viewMode.fontHeight)
qp = QtGui.QPainter()
qp.begin(pixThumb)
qp.setFont(self.viewMode.font)
qp.setPen(self.viewMode.textPen)
for i,c in enumerate(self.data):
self.viewMode.transformText(qp, (i, c), self.data)
qp.drawText((i%cols)*self.viewMode.fontWidth, self.viewMode.fontHeight + (i/cols)*self.viewMode.fontHeight, c)
qp.end()
self.newqpix = pixThumb.scaled(dwidth, dheight, aspectRatioMode = QtCore.Qt.IgnoreAspectRatio, transformMode = QtCore.Qt.FastTransformation)
"""
"""
def getDisplayedData(self):
if self.dataOffset < 0:
self.dataOffset = 0
return False
chrlist = [unichr(cp437ToUnicode[ord(c)]) for c in self.data[self.dataOffset : self.dataOffset + self.viewMode.COLUMNS*self.viewMode.ROWS]]
self.text = "".join(chrlist)
return True
"""
def switchViewMode(self):
self.multipleViewModes = self.multipleViewModes[1:] + [
self.multipleViewModes[0]
]
self.viewMode = self.multipleViewModes[0]
# notify obervers
self.notify(self.viewMode)
def _resize(self):
self.Banners.resize(self.size().width() - self.offsetWindow_h,
self.size().height() - self.offsetWindow_v)
# compute space ocupated by banners
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getBottomOffset(
) + self.Banners.getTopOffset()
# resize window, substract space ocupated by banners
self.viewMode.resize(self.size().width() - offsetLeft,
self.size().height() - offsetBottom)
# event handlers
def resizeEvent(self, e):
self._resize()
def paintEvent(self, e):
qp = QtGui.QPainter()
qp.begin(self)
qp.setOpacity(1)
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getTopOffset()
#self.viewMode.draw2(qp, refresh=True)
#start = time()
qp.drawPixmap(offsetLeft, offsetBottom, self.viewMode.getPixmap())
#print 'Draw ' + str(time() - start)
self.Banners.draw(qp, self.offsetWindow_h, self.offsetWindow_v,
self.size().height())
# qp.drawPixmap(self.offsetWindow_h, self.size().height() - 50, self.banner.getPixmap())
# qp.drawPixmap(20, 0, self.filebanner.getPixmap())
qp.end()
"""
def keyPressEvent(self, event):
if event.modifiers() & QtCore.Qt.ShiftModifier:
if self.viewMode.handleKeyPressEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
def keyReleaseEvent(self, event):
if event.key() == QtCore.Qt.Key_Shift:
if self.viewMode.handleKeyReleaseEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
"""
def eventFilter(self, watched, event):
if event.type() == QtCore.QEvent.KeyRelease:
key = event.key()
modifiers = event.modifiers()
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
self.update()
if event.type() == QtCore.QEvent.KeyPress:
#TODO: should we accept only certain keys ?
key = event.key()
modifiers = event.modifiers()
if key == QtCore.Qt.Key_F2:
if self.viewMode.isEditable():
if self.viewMode.isInEditMode():
self.viewMode.setEditMode(False)
else:
self.viewMode.setEditMode(True)
self.viewMode.draw(refresh=False)
# switch view mode
if key == QtCore.Qt.Key_Tab:
offs = self.viewMode.getCursorOffsetInPage()
base = self.viewMode.getDataModel().getOffset()
self.switchViewMode()
self._resize()
self.viewMode.goTo(base + offs)
self.update()
import pyperclip
if event.modifiers() & QtCore.Qt.ControlModifier:
if key == QtCore.Qt.Key_Insert:
if self.viewMode.selector.getCurrentSelection():
a, b = self.viewMode.selector.getCurrentSelection()
#print a, b
hx = ''
for s in self.dataModel.getStream(a, b):
hx += '{:02x}'.format(s)
pyperclip.copy(hx)
del pyperclip
#print pyperclip.paste()
# print 'coppied'
if event.modifiers() & QtCore.Qt.ShiftModifier:
if key == QtCore.Qt.Key_Insert:
raise Exception("Not implemented")
"""
import re
hx = pyperclip.paste()
#print hx
L = re.findall(r'.{1,2}', hx, re.DOTALL)
array = ''
for s in L:
array += chr(int(s, 16))
#print 'write '
#print 'write'
#print array
self.dataModel.write(0, array)
self.viewMode.draw(True)
del pyperclip
#print array
"""
if key == QtCore.Qt.Key_F4:
self.unp = WUnpack(self, None)
self.unp.show()
if key == QtCore.Qt.Key_F10:
self.dataModel.flush()
import os
self.w = WHeaders(self, None)
self.w.show()
if not self.viewMode.isInEditMode():
if key == QtCore.Qt.Key_Slash:
self.searchWindow.show()
if key == QtCore.Qt.Key_N:
self.searchable.next(
self.viewMode.getCursorAbsolutePosition() + 1)
if key == QtCore.Qt.Key_B:
self.searchable.previous(
self.viewMode.getCursorAbsolutePosition())
# handle keys to view plugin
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
event.accept()
self.update()
return True
return False
def setTextViewport(self, qp):
qp.setViewport(self.offsetWindow_h, self.offsetWindow_v,
self.size().width(),
self.size().height())
qp.setWindow(0, 0, self.size().width(), self.size().height())
def needsSave(self):
return self.dataModel.isDirty()
def save(self):
return self.dataModel.flush()
class WUnpack(QtGui.QDialog):
def __init__(self, parent, plugin):
super(WUnpack, self).__init__(parent)
self.parent = parent
self.plugin = plugin
self.oshow = super(WUnpack, self).show
root = os.path.dirname(sys.argv[0])
self.ui = PyQt4.uic.loadUi(os.path.join(root, 'unpack.ui'), baseinstance=self)
self.ui.setWindowTitle('Decrypt/Encrypt')
self.manager = PluginManager(categories_filter={ "UnpackPlugin": DecryptPlugin})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'unpack')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
self.Plugins = {}
Formats = []
for plugin in self.manager.getPluginsOfCategory("UnpackPlugin"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.init(self.parent.dataModel, self.parent.viewMode):
self.Plugins[plugin.name] = po
#self.ui.horizontalLayout.addWidget(po.getUI())
print '[+] ' + plugin.name
self.ui.listWidget.addItem(plugin.name)
#Formats.append(po)
self.ui.listWidget.currentItemChanged.connect(self.item_clicked)
self.ui.listWidget.setCurrentRow(0)
self.ui.connect(self.ui.proceed, PyQt4.QtCore.SIGNAL("clicked()"), self.handleProceed)
self.initUI()
def handleProceed(self):
item = str(self.ui.listWidget.currentItem().text())
self.Plugins[item].proceed()
#self.parent.update()
self.parent.viewMode.draw(refresh=True)
self.parent.update()
def item_clicked(self, current, previous):
#item = str(self.ui.listWidget.currentItem().text())
item = str(current.text())
if previous:
x = self.ui.horizontalLayout.takeAt(0)
while x:
x.widget().setParent(None)
x = self.ui.horizontalLayout.takeAt(0)
prev = str(previous.text())
#print prev
#self.ui.horizontalLayout.removeWidget(self.Plugins[prev].getUI())
if item:
#print item
po = self.Plugins[item]
self.ui.horizontalLayout.addWidget(po.getUI())
def show(self):
# TODO: remember position? resize plugin windows when parent resize?
pwidth = self.parent.parent.size().width()
pheight = self.parent.parent.size().height()
width = self.ui.size().width()+15
height = self.ui.size().height()+15
self.setGeometry(pwidth - width - 15, pheight - height, width, height)
self.setFixedSize(width, height)
self.oshow()
def initUI(self):
self.setSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
shortcut = QtGui.QShortcut(QtGui.QKeySequence("F4"), self, self.close, self.close)
#QtCore.QObject.connect(self.ui.ok, QtCore.SIGNAL('clicked()'), self.onClicked)
def onClicked(self):
dataModel = self.parent.dataModel
self.close()
class ContentModelListener(ConnectionListener):
'''
classdocs
'''
def __init__(self, content_models, host='localhost', port=61613, user='', passcode='', fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' % {'host': host, 'port': port})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" % {'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.username = user
self.password = passcode
# Create plugin manager
self.manager = PluginManager(categories_filter = {"FedoraMicroService": FedoraMicroService})
plugin_path = os.path.dirname(__file__)
self.manager.setPluginPlaces([plugin_path + "/plugins"])
logging.debug("Plugin path: " + plugin_path + "/plugins")
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info("Loading plugin: %(name)s for content model %(cmodel)s." % {'name': plugin.plugin_object.name, 'cmodel': plugin.plugin_object.content_model})
plugin.plugin_object.config = config
if type(plugin.plugin_object.content_model) == types.StringType:
content_models = [plugin.plugin_object.content_model]
else:
content_models = plugin.plugin_object.content_model
for content_model in content_models:
if content_model in self.contentModels:
self.contentModels[content_model].append(plugin.plugin_object)
else:
self.contentModels[content_model] = [plugin.plugin_object]
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
#logging.debug("\r \r", end='')
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
logging.error("lost connection reconnect in %d sec..." % reconnect_wait)
signal.alarm(reconnect_wait)
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
try:
global TOPIC_PREFIX
self.__print_async('MESSAGE', headers, body)
pid = headers['pid']
dsid = headers['dsid']
obj = self.client.getObject(pid)
content_model = headers['destination'][len(TOPIC_PREFIX):]
if content_model in self.contentModels:
logging.info('Running rules for %(pid)s from %(cmodel)s.' % {'pid': obj.pid, 'cmodel': content_model} )
for plugin in self.contentModels[content_model]:
plugin.runRules(obj, dsid, body)
except FedoraConnectionException:
logging.warning('Object %s was not found.' % (pid))
except:
logging.error("an exception occurred: " + str(sys.exc_info()[0]))
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers = { 'message-id' : args[1]})
else:
self.conn.ack(headers = { 'message-id' : args[1]}, transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, correlation_id, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination, message=message, headers={'correlation-id': correlation_id})
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url, username = self.username, password = self.password)
self.client = FedoraClient(self.fc)
class Bot(discord.Client):
def __init__(self):
super().__init__()
self.settingsChannelName = 'botsettings' # change this if you want your bot settings channel to have a different name
self.voiceChannel = None
self.voiceStarter = None
self.botCommands = botcommands.BotCommands()
# Setup logging for Discord.py
self.discordLogger = logging.getLogger('discord')
self.discordLogger.setLevel(logging.DEBUG)
self.discordHandler = logging.FileHandler(filename='discord.log',
encoding='utf-8',
mode='w')
self.discordHandler.setFormatter(
logging.Formatter(
'[%(asctime)s] [%(levelname)s] [%(name)s]: %(message)s'))
self.discordLogger.addHandler(self.discordHandler)
self.logger = logger
# Create Plugin Manager
self.pluginManager = PluginManager(
categories_filter={"Plugins": Plugin})
self.pluginManager.setPluginPlaces(["plugins"])
def isAdmin(self, user):
for role in user.roles:
if role.permissions.administrator:
return True
return False
async def reloadPlugins(self):
# Load Plugins
self.pluginManager.locatePlugins()
self.pluginManager.loadPlugins()
# Uncomment to see full command registry logged WARNING: There is a lot of text
#logger.debug('Registry: %s', str(plugincommand.registry))
# TODO: Make this a little more multi-server friendly
disabledplugins = []
settings = await self.getSettingsForTag(
list(self.servers)[0], botcommands.BotCommands.TAG)
if settings is not None and 'disabledplugins' in settings:
disabledplugins = settings['disabledplugins'].split(',')
if len(disabledplugins) == 1 and disabledplugins[0] is '':
disabledplugins = []
for plugin in self.pluginManager.getPluginsOfCategory("Plugins"):
# Give the plugin it's dictionary of commands (so it doesn't need to lookup later)
pluginCls = plugin.plugin_object.__class__.__name__
if pluginCls in plugincommand.registry:
plugin.plugin_object.parsedCommands = plugincommand.registry[
pluginCls]
else:
plugin.plugin_object.parsedCommands = {}
await plugin.plugin_object._init(plugin.name, self)
logger.debug('%s Initialized!', plugin.name)
for _, _disabledplugin in enumerate(disabledplugins):
if plugin.plugin_object.tag == _disabledplugin or plugin.plugin_object.shortTag == _disabledplugin:
plugin.plugin_object.isDisabled = True
plugin.plugin_object.isDisabledPermanently = True
self.commandCollisions = {}
self.getCommandCollisions()
def getCommandCollisions(self):
# Generate a dictionary of tags & commands that collide
for outer in self.pluginManager.getPluginsOfCategory("Plugins"):
for inner in self.pluginManager.getPluginsOfCategory("Plugins"):
if outer.name == inner.name:
continue
# This should NEVER happen
if outer.plugin_object.tag == inner.plugin_object.tag:
logger.error('Plugin Tag Collision! Tag: [%s]',
outer.plugin_object.tag)
# Check their commands to see if there is collision as well
if outer.plugin_object.shortTag == inner.plugin_object.shortTag:
logger.warning(
'Plugin Short Tag Collision! Short Tag: [%s]',
outer.plugin_object.shortTag)
for com in outer.plugin_object.parsedCommands:
if com in inner.plugin_object.parsedCommands:
logger.warning(
'Plugin Command Collision! Command: [%s]', com)
if outer.plugin_object.shortTag not in self.commandCollisions:
self.commandCollisions[
outer.plugin_object.shortTag] = []
self.commandCollisions[
outer.plugin_object.shortTag].append(com)
async def shutdownPlugins(self):
for plugin in self.pluginManager.getPluginsOfCategory("Plugins"):
await plugin.plugin_object.shutdown()
logger.debug('Shutdown %s', plugin.name)
if self.voiceChannel != None:
await self.voiceChannel.disconnect()
async def on_ready(self):
print('Logged in as')
print(self.user.name)
print(self.user.id)
print('-----------')
logger.debug('Logged in as [%s] [%s]', self.user.name, self.user.id)
botCmdCls = self.botCommands.__class__.__name__
if botCmdCls in botcommand.registry:
self.botCommands.parsedCommands = botcommand.registry[botCmdCls]
await self.botCommands._init(bot=self)
await self.reloadPlugins()
print('Plugins initialized!')
def isBotCommand(self, command):
return self.botCommands.isCommand(command)
async def on_message(self, message):
if message.author == self.user:
return
if not await self._executeBotCommand(message.content, message.server,
message.channel, message.author):
await self._readMessage(message)
if not await self._checkForCommandCollisions(
message.content, message.channel):
await self._executePluginCommand(message.content,
message.server,
message.channel,
message.author)
def _parseTextForCommandInfo(self, text):
results = {}
# If we have a command, extract the command and parameters (if any)
if text.startswith('!'):
tmp = text.split(' ', 1)
tmp[0] = tmp[0][1:] # get rid of !
content = ''
# Grab the rest of the string
if len(tmp) > 1:
content = tmp[1]
results['tag'] = tmp[0]
results['content'] = content
tmp = content.split(' ', 1)
results['command'] = tmp[0]
if len(tmp) > 1:
results['args'] = tmp[1]
else:
results['args'] = ''
return results
return None
async def _executeBotCommand(self, text, _server, _channel, _author,
**kwargs):
commandInfo = self._parseTextForCommandInfo(text)
if commandInfo is not None:
tag = commandInfo['tag']
content = commandInfo['content']
# If we have a bot command, execute it
if self.isBotCommand(tag):
index, args = self.botCommands.parseCommandArgs(tag, content)
if index == -1:
await self.send_message(
_channel,
'Incorrect use of command `{}`. Please see the usage to learn how to properly use this command.\nJust Type: `!usage {}`'
.format(tag, tag))
return True
await self.botCommands.executeCommand(index,
args,
**kwargs,
command=tag,
server=_server,
channel=_channel,
author=_author)
return True
# This isn't a bot command and there was nothing after it. This must be an unrecognized command.
elif content == '':
await self.send_message(
_channel,
'That is not a recognized command. For help, please try `!help`'
)
return True
# Otherwise, assume it is a plugin command
return False
async def _checkForCommandCollisions(self, text, channel):
# Check for tag/command collisions here and resolve before letting the plugins handle it
for key, array in self.commandCollisions.items():
for value in array:
if text.startswith('!' + key + ' ' + value):
logger.debug(
'There is more than one plugin with this short tag and command combination. Please use the full tag.'
)
await self.send_message(
channel,
'There is more than one plugin with this short tag and command combination. Please use the full tag.'
)
return True
return False
async def _readMessage(self, message):
# Go through each of the plugins and let them read the message
for plugin in self.pluginManager.getPluginsOfCategory("Plugins"):
if not plugin.plugin_object.isDisabled and plugin.plugin_object.isReadingMessages(
):
await plugin.plugin_object.readMessage(message)
async def _executePluginCommand(self, text, _server, _channel, _author,
**kwargs):
commandInfo = self._parseTextForCommandInfo(text)
if commandInfo is not None:
tag = commandInfo['tag']
command = commandInfo['command']
args = commandInfo['args']
found = False
# Go through each of the plugins and see if they can execute the command
for plugin in self.pluginManager.getPluginsOfCategory("Plugins"):
# Check if a plugin can handle the command and execute it if they can
if tag != None and command != None and plugin.plugin_object.isCommand(
tag, command):
if plugin.plugin_object.isDisabled:
found = True
await self.send_message(
_channel,
'This plugin is currently disabled. To use commands, please enable it first.'
)
continue
index, temp = plugin.plugin_object.parseCommandArgs(
command, args)
tag = plugin.plugin_object.tag # Update the tag for better feedback
if index == -1:
await self.send_message(
_channel,
'Incorrect use of command `{}`. Please see the usage to learn how to properly use this command.\nJust type: `!usage {} {}`'
.format(command, tag, command))
return
await plugin.plugin_object.executeCommand(index,
temp,
command=command,
server=_server,
channel=_channel,
author=_author,
**kwargs)
found = True
if not found and text.startswith('!'):
await self.send_message(
_channel,
'That is not a recognized command. For help, please try `!help`'
)
def download_image(self, imgUrl, filename):
try:
logger.debug('[download_image]: Opening url')
with urllib.request.urlopen(imgUrl) as imageOnWeb:
logger.debug('[download_image]: Checking if url is image')
if imageOnWeb.info()['Content-Type'].startswith('image'):
logger.debug('[download_image]: Reading Image')
buf = imageOnWeb.read()
logger.debug('[download_image]: Creating file [%s]',
os.getcwd() + '/' + filename)
downloadedImage = open(os.getcwd() + '/' + filename, 'wb')
logger.debug('[download_image]: Writing Image')
downloadedImage.write(buf)
downloadedImage.close()
imageOnWeb.close()
else:
logger.debug('[download_image]: Image URL is not an image')
return False
except:
logger.debug(
'[download_image]: Something failed while reading or writing the image'
)
return False
logger.debug('[download_image]: Successfully downloaded image')
return True
# Private helper for Settings API
async def _createSettingsChannel(self, server):
logger.debug('No settings found in server %s, creating settings.',
server)
return await self.create_channel(server, self.settingsChannelName)
# Private helper for Settings API
async def _getSettingsFromChannel(self, channel):
# Create a dictionary from the settings and return it
result = {}
async for message in self.logs_from(channel, limit=1000000):
if message.content.startswith('_'):
temp = message.content.split('=', 1)[0][1:]
tag = temp.split(':', 1)[0]
key = temp.split(':', 1)[1]
value = message.content.split('=', 1)[1]
if tag not in result:
result[tag] = {}
result[tag][key] = value
return result
# Private helper for Settings API
async def _getSettingsFromChannelForTag(self, channel, plugintag):
# Create a dictionary from the settings and return it
result = {}
async for message in self.logs_from(channel, limit=1000000):
if message.content.startswith('_'):
temp = message.content.split('=', 1)[0][1:]
tag = temp.split(':', 1)[0]
key = temp.split(':', 1)[1]
if tag != plugintag:
continue
value = message.content.split('=', 1)[1]
result[key] = value
return result
# Private helper for Settings API
async def _getMessageFromSettings(self, channel, tag, key):
async for message in self.logs_from(channel, limit=1000000):
if message.content.startswith('_' + tag + ':' + key):
return message
return None
# Private helper for Settings API
async def _getSettingsChannel(self, server):
channel = discord.utils.get(server.channels,
name=self.settingsChannelName,
type=discord.ChannelType.text)
if channel == None:
channel = await self._createSettingsChannel(server)
return channel
# Private helper for Settings API
async def _createSetting(self, channel, tag, key, value):
logger.debug('Creating Setting [%s:%s] with value [%s]', tag, key,
value)
await self.send_message(
channel, '_{}:{}={}'.format(str(tag), str(key), str(value)))
# Private helper for Settings API
async def _modifySetting(self, message, tag, key, value):
logger.debug('Modifying Setting [%s:%s] with value [%s]', tag, key,
value)
await self.edit_message(
message, '_{}:{}={}'.format(str(tag), str(key), str(value)))
# Private helper for Settings API
async def _deleteSetting(self, message):
logger.debug('Deleting Setting')
await self.delete_message(message)
# Gets the settings object from the server
# Settings object structure:
# object[plugintag][settingname] = settingvalue
async def getSettings(self, server):
for srv in self.servers:
if srv != server:
continue
channel = await self._getSettingsChannel(srv)
logger.debug('Settings constructed for server %s.', srv)
return await self._getSettingsFromChannel(channel)
logger.debug('The bot is not part of server %s!', server)
return None
# Gets the settings object from the server for a specific plugin tag
# Settings object structure:
# object[settingname] = settingvalue
async def getSettingsForTag(self, server, tag):
for srv in self.servers:
if srv != server:
continue
channel = await self._getSettingsChannel(srv)
logger.debug('Settings constructed for server %s.', srv)
return await self._getSettingsFromChannelForTag(channel, tag)
logger.debug('The bot is not part of server %s!', server)
return None
# Modifies the setting if it exists and creates it if it doesn't
async def modifySetting(self, server, tag, key, value):
for srv in self.servers:
if srv != server:
continue
channel = await self._getSettingsChannel(srv)
message = await self._getMessageFromSettings(channel, tag, key)
if message == None:
await self._createSetting(channel, tag, key, value)
else:
await self._modifySetting(message, tag, key, value)
return
logger.debug('The bot is not part of server %s!', server)
# Deletes the setting
async def deleteSetting(self, server, tag, key):
for srv in self.servers:
if srv != server:
continue
channel = await self._getSettingsChannel(srv)
message = await self._getMessageFromSettings(channel, tag, key)
if message != None:
await self._deleteSetting(message)
return
logger.debug('The bot is not part of server %s!', server)
# Returns whether the server has a specific setting
async def hasSetting(self, server, tag, key):
for srv in self.servers:
if srv != server:
continue
channel = await self._getSettingsChannel(srv)
message = await self._getMessageFromSettings(channel, tag, key)
if message == None:
return False
else:
return True
logger.debug('The bot is not part of server %s!', server)
return False
# Returns the user's object if it exists. The username can be their discord name or server nickname
def getUserFromName(self, server, username):
user = None
for member in server.members:
if member.display_name == username:
user = member
break
if not user:
user = discord.utils.get(server.members, name=username)
if not user:
return None
return user
class Cobiv(App):
root = None
observers = {}
logger = logging.getLogger(__name__)
def __init__(self, **kwargs):
super(Cobiv, self).__init__(**kwargs)
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["cobiv/modules"])
self.plugin_manager.setCategoriesFilter({
"View": View,
"Entity": Entity,
"Hud": Hud,
"TagReader": TagReader,
"Datasource": Datasource,
"SetManager": SetManager,
"BookManager": BookManager,
"Gesture": Gesture
})
self.plugin_manager.locatePlugins()
self.plugin_manager.loadPlugins()
for plugin in self.plugin_manager.getAllPlugins():
print("Plugin found : {} {} {}".format(plugin.plugin_object,
plugin.name,
plugin.categories))
config_path = os.path.join(os.path.expanduser('~'), '.cobiv')
if not os.path.exists(config_path):
os.makedirs(config_path)
def build_yaml_config(self):
if not os.path.exists('cobiv.yml'):
f = open('cobiv.yml', 'w')
data = self.root.build_yaml_main_config()
for plugin in self.plugin_manager.getAllPlugins():
plugin.plugin_object.build_yaml_config(data)
yaml.dump(data, f)
f.close()
f = open('cobiv.yml')
config = yaml.safe_load(f)
f.close()
self.root.configuration = config
self.root.read_yaml_main_config(config)
for plugin in self.plugin_manager.getAllPlugins():
plugin.plugin_object.read_yaml_config(config)
def build(self):
self.root = MainContainer()
for plugin in self.plugin_manager.getPluginsOfCategory("View"):
self.root.available_views[
plugin.plugin_object.get_name()] = plugin.plugin_object
self.build_yaml_config()
for plugin in self.plugin_manager.getAllPlugins():
plugin.plugin_object.ready()
print('plugin {} ready'.format(plugin.name))
self.logger.debug("plugin ready : " + str(plugin.name))
self.root.switch_view(self.get_config_value('startview', 'help'))
self.title = "COBIV"
self.root.ready()
return self.root
def on_start(self):
# self.profile = cProfile.Profile()
# self.profile.enable()
pass
def on_stop(self):
for plugin in self.plugin_manager.getAllPlugins():
plugin.plugin_object.on_application_quit()
# self.profile.disable()
# self.profile.dump_stats('cobiv.profile')
def lookup(self, name, category):
plugin = self.plugin_manager.getPluginByName(name, category=category)
return plugin.plugin_object if plugin is not None else None
def lookups(self, category):
return [
plugin.plugin_object
for plugin in self.plugin_manager.getPluginsOfCategory(category)
]
def register_event_observer(self, evt_name, callback):
if not evt_name in self.observers:
self.observers[evt_name] = [callback]
else:
self.observers[evt_name].append(callback)
def fire_event(self, evt_name, *args, **kwargs):
if evt_name in self.observers:
for callback in self.observers[evt_name]:
callback(*args, **kwargs)
def get_user_path(self, *args):
return os.path.join(os.path.expanduser('~'), '.cobiv', *args)
def get_config_value(self, key, default=None):
if self.root is None:
return default
keys = key.split('.')
cfg = self.root.configuration
for k in keys:
if k in cfg:
cfg = cfg.get(k)
else:
return default
return cfg
class WUnpack(QtWidgets.QDialog):
def __init__(self, parent, plugin):
super(WUnpack, self).__init__(parent)
self.parent = parent
self.plugin = plugin
self.oshow = super(WUnpack, self).show
root = os.path.dirname(sys.argv[0])
self.ui = PyQt5.uic.loadUi(os.path.join(root, 'unpack.ui'),
baseinstance=self)
self.ui.setWindowTitle('Decrypt/Encrypt')
self.manager = PluginManager(
categories_filter={"UnpackPlugin": DecryptPlugin})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'unpack')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
self.Plugins = {}
Formats = []
for plugin in self.manager.getPluginsOfCategory("UnpackPlugin"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.init(self.parent.dataModel, self.parent.viewMode):
self.Plugins[plugin.name] = po
#self.ui.horizontalLayout.addWidget(po.getUI())
print('[+] ' + plugin.name)
self.ui.listWidget.addItem(plugin.name)
#Formats.append(po)
self.ui.listWidget.currentItemChanged.connect(self.item_clicked)
self.ui.listWidget.setCurrentRow(0)
#self.ui.connect(self.ui.proceed, PyQt5.QtCore.SIGNAL("clicked()"), self.handleProceed)
self.ui.proceed.clicked.connect(lambda: self.handleProceed())
self.initUI()
def handleProceed(self):
item = str(self.ui.listWidget.currentItem().text())
self.Plugins[item].proceed()
#self.parent.update()
self.parent.viewMode.draw(refresh=True)
self.parent.update()
def item_clicked(self, current, previous):
#item = str(self.ui.listWidget.currentItem().text())
item = str(current.text())
if previous:
x = self.ui.horizontalLayout.takeAt(0)
while x:
x.widget().setParent(None)
x = self.ui.horizontalLayout.takeAt(0)
prev = str(previous.text())
#print prev
#self.ui.horizontalLayout.removeWidget(self.Plugins[prev].getUI())
if item:
#print item
po = self.Plugins[item]
self.ui.horizontalLayout.addWidget(po.getUI())
def show(self):
# TODO: remember position? resize plugin windows when parent resize?
pwidth = self.parent.parent.size().width()
pheight = self.parent.parent.size().height()
width = self.ui.size().width() + 15
height = self.ui.size().height() + 15
self.setGeometry(pwidth - width - 15, pheight - height, width, height)
self.setFixedSize(width, height)
self.oshow()
def initUI(self):
self.setSizePolicy(QtWidgets.QSizePolicy.Fixed,
QtWidgets.QSizePolicy.Fixed)
shortcut = QtWidgets.QShortcut(QtGui.QKeySequence("F4"), self,
self.close, self.close)
#QtCore.QObject.connect(self.ui.ok, QtCore.SIGNAL('clicked()'), self.onClicked)
def onClicked(self):
dataModel = self.parent.dataModel
self.close()
from bipy.core.constants import URLS, PATHS
from bipy.core.db.categories import SQLite
from yapsy.PluginManager import PluginManager
#----------- Create Plugin Mgr instance -------------------------
plugin_mgr = PluginManager(categories_filter={"SQLITE": SQLite})
#---------- Load Connection Mgr plugin for Warehouse ------------
plugin_mgr.setPluginPlaces([PATHS.CONNECTION_MANAGERS])
plugin_mgr.locatePlugins()
conns = plugin_mgr.loadPlugins()
conn_wh = conns[0].plugin_object
conn_wh
conn_wh.connect(URLS.TEST_DB)
#---------- Load Warehouse Browser plugin -----------------------
plugin_mgr.setPluginPlaces([PATHS.BROWSERS])
plugin_mgr.locatePlugins()
browsers = plugin_mgr.loadPlugins()
browser = browsers[0].plugin_object
browser
browser.connect(conn_wh)
#--------- Load Base Meta Generator plugin ----------------------
plugin_mgr.setPluginPlaces([PATHS.BASE_META_GEN])
plugin_mgr.locatePlugins()
base_meta_gen = plugin_mgr.loadPlugins()
meta_gen = base_meta_gen[0].plugin_object
meta_gen
#--------- Load Repository Manager plugin -----------------------
plugin_mgr.setPluginPlaces([PATHS.REPO_MGR])
plugin_mgr.locatePlugins()
repo_mgrs = plugin_mgr.loadPlugins()
repo_mgr = repo_mgrs[0].plugin_object
repo_mgr
def main(args: List[AnyStr] = None) -> int:
"""
prepare and start application
the main method is responsible for setting up the environment,
i.e. to load translations, set a logger, parse the command line arguments,
load configuration files and start the application.
:param args: command line arguments; default: None
:type args: List of strings
"""
if not args:
args = sys.argv
script_name = os.path.basename(sys.argv[0])
application = AppDirs(os.path.splitext(script_name)[0],
appauthor=__author__,
version=__version__,
roaming=True)
logging_path = application.user_data_dir
config_path = application.user_config_dir
# set locale
locale.setlocale(locale.LC_ALL, '')
_locale = locale.getlocale(locale.LC_ALL)[0]
locale_path = pkg_resources.resource_filename("servertools", "data/i18n")
translate = gettext.translation("servertools",
localedir=locale_path,
languages=[locale.getlocale()[0], 'en'])
translate.install()
if '_' not in globals():
_ = unicode
# setup logging
handlers = _log_default_handler(proc=script_name, log_dir=logging_path)
logging.basicConfig(level=4, format="%(message)s", handlers=handlers)
logger = logging.getLogger("{}_logger".format(
os.path.splitext(script_name)[0]))
logger.setLevel(logging.DEBUG)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.WARNING)
stream_handler.setFormatter(logging.Formatter("%(message)s"))
logger.addHandler(stream_handler)
# setup argument parser
parser = ArgumentParser(prog=script_name)
verbosity_parsergroup = parser.add_mutually_exclusive_group()
verbosity_parsergroup.add_argument("-v",
dest='verbosity',
action="count",
help=_("increase verbosity"),
default=0)
verbosity_parsergroup.add_argument(
"--silent",
action="store_true",
help=_("silent (no output to terminal)"))
parser.add_argument("--version",
action="version",
help=_("print version and exit"))
parser.add_argument("--log-file",
metavar='FILEPATH',
action="store",
help=_("overwrite path to log file"))
command_parsers = parser.add_subparsers(help="commands")
identify_parser = command_parsers.add_parser("identify",
add_help=False,
parents=[parser])
# load plugins for different server types
plugins_directories = [
pkg_resources.resource_filename("servertools", "plugins"),
os.path.join(application.user_data_dir, 'plugins'),
os.path.expanduser(
os.path.normpath(
'~/.{appname}/plugins'.format(appname=application.appname)))
]
sys.path.extend(plugins_directories)
plugin_manager = PluginManager(directories_list=plugins_directories,
plugin_info_ext='plugin')
plugin_manager.setCategoriesFilter({
"Server": Server,
})
plugin_manager.locatePlugins()
plugin_manager.collectPlugins()
# plugins to identify servers and fix problems that might happen on them
plugins = plugin_manager.getPluginsOfCategory(
'Server') # type: List[PluginInfo]
# choices for the command line flag `--look-for`
server_choices = [] # type: List[AnyStr]
# set valid values for --look-for from installed plugins
for plugin in plugins:
commandline_argument = plugin.name.lower().replace('server',
'').strip().replace(
' ', '-')
if len(commandline_argument) > 0:
server_choices.append(commandline_argument)
else:
raise ValueError(
_("Plugin {name} doesn't have a valid name!").format(
name=plugin.name))
identify_parser.add_argument("--look-for",
metavar="SERVER",
dest="look_for",
help=_("look for a specific server"),
choices=server_choices,
default='')
identify_parser.set_defaults(name="identify")
# fix_parser = command_parsers.add_parser('fix', help=_('apply fixes for a server'))
# parse arguments
args = parser.parse_args(args[1:])
# handle verbosity
verbosity = max(0, min(3, args.verbosity))
if args.silent:
verbosity = -1
logging_level = {
-1: logging.CRITICAL, # 50
0: logging.WARNING, # 30
1: logging.INFO, # 20
2: logging.INFO, # 20
3: logging.DEBUG # 10
}.get(verbosity, logging.WARNING)
stream_handler.setLevel(logging_level)
verbose = logger.info if verbosity > 1 else logger.debug
# start application
logger.debug(
_("{program} started at {started_at:%X} on Python {py_version} ({architecture})"
).format(program=script_name,
started_at=datetime.now(),
py_version=platform.python_version(),
architecture=platform.architecture()[0]))
logger.debug(
_("Verbosity Level at {verbosity}").format(verbosity=verbosity))
plugins_information = {}
for category in plugin_manager.getCategories():
plugins_information[category] = list()
for plugin in plugin_manager.getPluginsOfCategory(category):
plugins_information[category].append({
'name': plugin.name,
'author': plugin.author
})
# if verbosity > 1:
logger.info(_("Save logs to {path}").format(path=logging_path))
logger.info(_("Load configurations from {path}").format(path=config_path))
logger.info(
_('Look for plugins in:\n\t{path}').format(
path=',\n\t'.join(plugins_directories)))
verbose(
_('Include {amount} Plugin Categories.').format(
amount=len(plugin_manager.getCategories())))
logger.debug(', '.join(plugin_manager.getCategories()))
verbose(
_('{amount} Plugin(s) loaded.').format(
amount=len(plugins_information.keys())))
for category in plugins_information:
logger.debug('[{}]'.format(category))
for cat in plugins_information[category]:
logger.debug(
_('\t{name} (by {author})').format(name=cat['name'],
author=cat['author']))
logger.debug('')
for plugin in plugins:
logger.debug(_("Try to identify {name}").format(name=plugin.name))
command = plugin.name.lower().replace('server',
'').strip().replace(' ', '-'),
if args.look_for == command and plugin.plugin_object.identify():
logger.debug(
_("Successfully identified {name}").format(name=plugin.name))
print("{} - {}".format(plugin.details.get('Server Info', 'Host'),
plugin.details.get('Server Info',
'Slogan')))
break
else:
print(
_('Not {}').format(args.look_for)
if len(args.look_for) > 0 else _('Unknown platform'))
return 1
logger.debug("-" * 80)
return 0
class ModuleManager():
def __init__(self, threads, kill_list, kill_list_lock, job_list, binpath):
self.threads = threads
self.kill_list = kill_list
self.kill_list_lock = kill_list_lock
self.job_list = job_list # Running jobs
self.binpath = binpath
self.root_path = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', '..'))
self.module_bin_path = os.path.join(self.root_path, "module_bin")
self.plugin_path = os.path.join(self.root_path, "lib", "assembly", "plugins")
self.pmanager = PluginManager()
locator = self.pmanager.getPluginLocator()
locator.setPluginInfoExtension('asm-plugin')
self.pmanager.setPluginPlaces([ self.plugin_path ])
self.pmanager.collectPlugins()
self.pmanager.locatePlugins()
self.plugins = ['none']
num_plugins = len(self.pmanager.getAllPlugins())
if num_plugins == 0:
raise Exception("No Plugins Found!")
plugins = []
self.executables = {}
for plugin in self.pmanager.getAllPlugins():
plugin.threads = threads
self.plugins.append(plugin.name)
plugin.plugin_object.setname(plugin.name)
## Check for installed binaries
try:
version = plugin.details.get('Documentation', 'Version')
executables = plugin.details.items('Executables')
full_execs = [(k, self.get_executable_path(v)) for k,v in executables]
for binary in full_execs:
if not os.path.exists(binary[1]):
if float(version) < 1:
print '[Warning]: {} (v{}) -- Binary does not exist for beta plugin -- {}'.format(plugin.name, version, binary[1])
else:
raise Exception('[ERROR]: {} (v{})-- Binary does not exist -- {}'.format(plugin.name, version, binary[1]))
self.executables[plugin.name] = full_execs
except ConfigParser.NoSectionError: pass
plugins.append(plugin.name)
print "Plugins found [{}]: {}".format(num_plugins, sorted(plugins))
def run_proc(self, module, wlink, job_data, parameters):
""" Run module adapter for wasp interpreter
To support the Job_data mechanism, injects wlink
"""
if not self.has_plugin(module):
raise Exception("No plugin named {}".format(module))
plugin = self.pmanager.getPluginByName(module)
config_settings = plugin.details.items('Settings')
config_settings = update_settings(config_settings, parameters)
try:
settings = {k:v for k,v in self.executables[module]}
for k,v in config_settings: ## Don't override
if not k in settings:
settings[k] = v
settings = settings.items()
except:
# settings = config_settings
raise Exception("Plugin Config not updated: {}!".format(module))
#### Check input/output type compatibility
if wlink['link']:
for link in wlink['link']:
if not link:
continue
if link['module']:
try:
assert (self.output_type(link['module']) == self.input_type(module) or
self.output_type(link['module']) in self.input_type(module))
except AssertionError:
raise Exception('{} and {} have mismatched input/output types'.format(module, link['module']))
#### Run
job_data['wasp_chain'] = wlink
output = plugin.plugin_object.base_call(settings, job_data, self)
ot = self.output_type(module)
wlink.insert_output(output, ot,
plugin.name)
if not wlink.output:
raise Exception('"{}" module failed to produce {}'.format(module, ot))
def output_type(self, module):
return self.pmanager.getPluginByName(module).plugin_object.OUTPUT
def input_type(self, module):
return self.pmanager.getPluginByName(module).plugin_object.INPUT
def get_short_name(self, module):
try:
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
for kv in settings:
if kv[0] == 'short_name':
sn = kv[1]
break
return sn
except:
return None
def get_executable(self, module):
try:
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
for kv in settings:
if kv[0] == 'short_name':
sn = kv[1]
break
return sn
except:
return None
def verify_file(self, filename):
if not os.path.exists(filename):
raise Exception("File not found: %s" % filename)
def get_executable_path(self, filename, verify=False):
guess1 = os.path.join(self.module_bin_path, filename)
guess2 = os.path.join(self.binpath, filename)
fullname = guess1 if os.path.exists(guess1) else guess2
if verify: verify_file(fullname)
return fullname
def has_plugin(self, plugin):
if not plugin.lower() in self.plugins:
logging.error("{} plugin not found".format(plugin))
return False
return True
def valid_modules(self, l):
""" Return filtered list of available modules """
return [m for m in l if not m.startswith('?') and self.has_plugin(m)]
def validate_pipe(self, pipe):
for stage in pipe:
for word in stage.replace('+', ' ').split(' '):
if not (word.startswith('?') or self.has_plugin(word)):
raise Exception('Invalid pipeline command')
def split_pipe(self, l):
""" Splits a multi-module string in to bins
Ex: 'kiki ?k=29 velvet' -> [[kiki, ?k=29], [velvet]]
"""
bins = []
for word in l:
if not word.startswith('?'):
bins.append([word])
elif word.startswith('?'):
bins[-1].append(word)
return bins
def parse_input(self, pipe):
"""
Parses inital pipe and separates branching bins
Ex: ['sga', '?p=True', 'kiki ?k=31 velvet', 'sspace']
"""
stages = phelper.parse_branches(pipe)
return stages
class ModuleManager():
def __init__(self, threads, kill_list, kill_list_lock, job_list, binpath,
modulebin):
self.threads = threads
self.kill_list = kill_list
self.kill_list_lock = kill_list_lock
self.job_list = job_list # Running jobs
self.binpath = binpath
self.module_bin_path = modulebin
self.root_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..'))
self.plugin_path = os.path.join(self.root_path, "lib", "assembly",
"plugins")
self.pmanager = PluginManager()
locator = self.pmanager.getPluginLocator()
locator.setPluginInfoExtension('asm-plugin')
self.pmanager.setPluginPlaces([self.plugin_path])
self.pmanager.collectPlugins()
self.pmanager.locatePlugins()
self.plugins = ['none']
num_plugins = len(self.pmanager.getAllPlugins())
if num_plugins == 0:
raise Exception("No Plugins Found!")
plugins = []
self.executables = {}
for plugin in self.pmanager.getAllPlugins():
plugin.threads = threads
self.plugins.append(plugin.name)
plugin.plugin_object.setname(plugin.name)
## Check for installed binaries
try:
version = plugin.details.get('Documentation', 'Version')
executables = plugin.details.items('Executables')
full_execs = [(k, self.get_executable_path(v))
for k, v in executables]
for binary in full_execs:
if not os.path.exists(binary[1]):
if float(version) < 1:
logger.warn(
'Third-party binary does not exist for beta plugin: {} (v{}) -- {}'
.format(plugin.name, version, binary[1]))
else:
raise Exception(
'ERROR: Third-party binary does not exist for beta plugin: {} (v{}) -- {}'
.format(plugin.name, version, binary[1]))
self.executables[plugin.name] = full_execs
except ConfigParser.NoSectionError:
pass
plugins.append(plugin.name)
logger.info("Plugins found [{}]: {}".format(num_plugins,
sorted(plugins)))
def run_proc(self, module, wlink, job_data, parameters):
""" Run module adapter for wasp interpreter
To support the Job_data mechanism, injects wlink
"""
if not self.has_plugin(module):
raise Exception("No plugin named {}".format(module))
plugin = self.pmanager.getPluginByName(module)
config_settings = plugin.details.items('Settings')
config_settings = update_settings(config_settings, parameters)
try:
settings = {k: v for k, v in self.executables[module]}
for k, v in config_settings: ## Don't override
if not k in settings:
settings[k] = v
settings = settings.items()
except:
# settings = config_settings
raise Exception("Plugin Config not updated: {}!".format(module))
#### Check input/output type compatibility
if wlink['link']:
for link in wlink['link']:
if not link:
continue
if link['module']:
try:
assert (self.output_type(link['module'])
== self.input_type(module) or self.output_type(
link['module']) in self.input_type(module))
except AssertionError:
raise Exception(
'{} and {} have mismatched input/output types'.
format(module, link['module']))
#### Run
job_data['wasp_chain'] = wlink
output = plugin.plugin_object.base_call(settings, job_data, self)
ot = self.output_type(module)
wlink.insert_output(output, ot, plugin.name)
if not wlink.output:
raise Exception('"{}" module failed to produce {}'.format(
module, ot))
### Store any output values in job_data
data = {'module': module, 'module_output': output}
job_data['plugin_output'].append(data)
def output_type(self, module):
return self.pmanager.getPluginByName(module).plugin_object.OUTPUT
def input_type(self, module):
return self.pmanager.getPluginByName(module).plugin_object.INPUT
def get_short_name(self, module):
try:
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
for kv in settings:
if kv[0] == 'short_name':
sn = kv[1]
break
return sn
except:
return None
def get_executable(self, module):
try:
plugin = self.pmanager.getPluginByName(module)
settings = plugin.details.items('Settings')
for kv in settings:
if kv[0] == 'short_name':
sn = kv[1]
break
return sn
except:
return None
def verify_file(self, filename):
if not os.path.exists(filename):
raise Exception("File not found: %s" % filename)
def get_executable_path(self, filename, verify=False):
guess1 = os.path.join(self.module_bin_path, filename)
guess2 = os.path.join(self.binpath, filename)
fullname = guess1 if os.path.exists(guess1) else guess2
if verify: verify_file(fullname)
return fullname
def has_plugin(self, plugin):
if not plugin.lower() in self.plugins:
logger.error("{} plugin not found".format(plugin))
return False
return True
def valid_modules(self, l):
""" Return filtered list of available modules """
return [m for m in l if not m.startswith('?') and self.has_plugin(m)]
def validate_pipe(self, pipe):
for stage in pipe:
for word in stage.replace('+', ' ').split(' '):
if not (word.startswith('?') or self.has_plugin(word)):
raise Exception('Invalid pipeline command')
def split_pipe(self, l):
""" Splits a multi-module string in to bins
Ex: 'kiki ?k=29 velvet' -> [[kiki, ?k=29], [velvet]]
"""
bins = []
for word in l:
if not word.startswith('?'):
bins.append([word])
elif word.startswith('?'):
bins[-1].append(word)
return bins
def parse_input(self, pipe):
"""
Parses inital pipe and separates branching bins
Ex: ['sga', '?p=True', 'kiki ?k=31 velvet', 'sspace']
"""
stages = phelper.parse_branches(pipe)
return stages
class HandlerManager(QThread):
# 返回命令执行结果
# int: 0 -- 返回PlainText
# 1 -- 返回RichText
# str: 结果
outSignal = pyqtSignal(int, str)
def __init__(self):
super(HandlerManager, self).__init__()
# 创建工作队列，用户输入的命令按照顺序put进该队列
# 由CommandHandler在后台逐个处理
self.workQueue = Queue()
# 初始化插件功能
self.initPlugin()
def __del__(self):
self.wait()
def initPlugin(self):
'''
启动插件管理器，获取插件列表
'''
# 创建插件管理器
self.pluginManager = PluginManager()
# 设置插件接口为 DPlugin，所有插件必须继承categories.DPlugin
self.pluginManager.setCategoriesFilter({"DPlugin": DPlugin})
# 设置插件目录
self.pluginManager.setPluginPlaces(['plugins'])
# 加载插件到内存
self.pluginManager.locatePlugins()
self.pluginManager.loadPlugins()
def getAllPlugins(self):
'''获取插件列表'''
return self.pluginManager.getPluginsOfCategory('DPlugin')
def activateAllPlugins(self):
'''使能所有插件'''
for plugin in self.pluginManager.getPluginsOfCategory('DPlugin'):
plugin.plugin_object.activate()
def deactivateAllPlugins(self):
'''禁用所有插件'''
for plugin in self.pluginManager.getPluginsOfCategory('DPlugin'):
plugin.plugin_object.deactivate()
def activatePlugin(self, name):
'''使能特定插件'''
self.pluginManager.activatePluginByName(name, category="DPlugin")
def deactivatePlugin(self, name):
'''使能特定插件'''
self.pluginManager.deactivatePluginByName(name, category="DPlugin")
def processInput(self, userInput):
'''将命令放入队列'''
self.workQueue.put((False, userInput))
def processFile(self, filepath):
'''将待处理文件放入队列'''
self.workQueue.put((True, filepath))
def run(self):
'''不断从Queue中获取数据，然后执行解析命令'''
while True:
isFile, userInput = self.workQueue.get()
if isFile:
self.execFile(userInput)
else:
self.execCommand(userInput, None)
def execCommand(self, userInput, currentFile):
'''
解析命令
'''
if not len(userInput) or userInput.startswith('#'):
return
# 命令回显
self.writeCommand(userInput)
# 命令分割
command, *params = userInput.split(maxsplit=1)
if command == 'source':
if len(params) == 1:
filepath = params[0]
if not os.path.isabs(filepath):
basepath = os.path.dirname(currentFile)
filepath = os.path.normpath(filepath)
filepath = os.path.join(basepath, filepath)
self.execFile(filepath)
else:
self.writeStderr('syntax error: ' + userInput)
else:
self.dispatchCommand(userInput)
def execFile(self, filepath):
self.writeCommand('Processing file ' + filepath)
if os.path.isfile(filepath) and os.access(filepath, os.R_OK):
with open(filepath, 'r') as f:
for line in f.readlines():
line = line.strip()
self.execCommand(line, filepath)
else:
self.writeStderr('Cannot open file: ' + filepath)
def dispatchCommand(self, userInput):
'''将命令分发给各个插件'''
status = DPlugin.EXEC_NOTFOUND
for plugin in self.pluginManager.getPluginsOfCategory('DPlugin'):
ret, resultText = plugin.plugin_object.execCommand(userInput)
if ret == DPlugin.EXEC_SUCCESS:
self.writeStdout(resultText)
status = DPlugin.EXEC_SUCCESS
elif ret == DPlugin.EXEC_FAILED:
self.writeStderr(resultText)
status = DPlugin.EXEC_FAILED
if status == DPlugin.EXEC_NOTFOUND:
self.writeStderr(userInput + ': Command not found')
def writeCommand(self, text):
'''
输出原始命令，蓝色加粗显示，以示区分
'''
self.outSignal.emit(1, text)
def writeStdout(self, text):
'''
返回普通输出，文本后加换行
'''
self.outSignal.emit(0, text)
def writeStderr(self, text):
'''
返回错误输出，红色加粗显示，以示区分
'''
self.outSignal.emit(2, text)
