class ProfilesManager(object):
def __init__(self):
self.plugin_location = [PROFILE_MODULE_DIR]
self.pluginManager = PluginManager(plugin_info_ext="plugin")
self.pluginManager.setPluginPlaces(self.plugin_location)
self.pluginManager.collectPlugins()
def modules_info(self):
"""
Get information with regards to each modules loaded. It includes author, category, copyright, description,
details, name, version and website.
:return: information of all modules loaded
"""
modules_info = {}
for pluginInfo in self.pluginManager.getAllPlugins():
modules_info[pluginInfo.name] = pluginInfo
return modules_info
def configForms(self, device_serial, module_name=""):
"""
Get the configuration views of each modules to be displayed on web interface
:return: dictionary of pluginInfo as key and form as value
"""
if module_name:
plugin = self.pluginManager.getPluginByName(name=module_name)
configForms = plugin.plugin_object.get_view()
else:
configForms = {}
for pluginInfo in self.pluginManager.getAllPlugins():
form = pluginInfo.plugin_object.get_view()
configForms[pluginInfo] = form
return configForms
def run_simulation(self,option, profile_name, duration, device_serial, package_name, session):
"""
Run profile simulation script
:return:
"""
plugin = self.pluginManager.getPluginByName(name=profile_name)
if option == RANDOM_INTERACTION:
plugin.plugin_object.runSimulation(duration,package_name, random=True, device_serial=device_serial, session=session)
elif option == SCRIPTED_PROFILE_INTERACTION:
plugin.plugin_object.runSimulation(duration,package_name, random=False, device_serial=device_serial, session=session)
def setup_device(self,module, params, device_serial):
"""
install profile apk and profile app data onto device
:return:
"""
pluginInfo = self.pluginManager.getPluginByName(name=module)
if pluginInfo.name == module:
pluginInfo.plugin_object.prepare(params, device_serial)
return True
def run_output_plugins(**kwargs):
logger = logging.getLogger(__name__)
logger.info("Begin run_output_plugins")
simplePluginManager = PluginManager()
logging.getLogger('yapsy').setLevel(logging.DEBUG)
simplePluginManager.setCategoriesFilter({
"OutputResults": output_plugin
})
# Tell it the default place(s) where to find plugins
if logger:
logger.debug("Plugin directories: %s" % (kwargs['output_plugin_directories']))
simplePluginManager.setPluginPlaces(kwargs['output_plugin_directories'])
simplePluginManager.collectPlugins()
plugin_cnt = 0
plugin_start_time = time.time()
for plugin in simplePluginManager.getAllPlugins():
if logger:
logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(details=plugin.details):
plugin.plugin_object.emit(prediction_date=kwargs['prediction_date'].astimezone(timezone("US/Eastern")).strftime("%Y-%m-%d %H:%M:%S"),
execution_date=kwargs['prediction_run_date'].strftime("%Y-%m-%d %H:%M:%S"),
ensemble_tests=kwargs['site_model_ensemble'])
plugin_cnt += 1
else:
logger.error("Failed to initialize plugin: %s" % (plugin.name))
logger.debug("%d output plugins run in %f seconds" % (plugin_cnt, time.time() - plugin_start_time))
logger.info("Finished run_output_plugins")
def pane_data(self):
list = [] # Holds the filter tuples (name, activated=True)
# Get the default plugin directory, using XML
path = os.path.expanduser("~")
xml = xml_controller.Controller(path + "\.cxvrc.xml")
xml.load_file()
if os.path.exists(os.path.expanduser("~") + os.sep + "plugins"):
default_dir = os.path.expanduser("~") + os.sep + "plugins"
else:
default_dir = self.dicom_view.get_main_dir() + os.sep + "plugins"
if xml.get_plugin_directory() == "" or xml.get_plugin_directory() is None:
directory = [default_dir]
else:
directory = [default_dir, xml.get_plugin_directory()]
# Load the plugins from the default plugin directory.
manager = PluginManager()
manager.setPluginPlaces(directory)
manager.setPluginInfoExtension("plugin")
manager.collectPlugins()
# Append tuple with plugin name and enabled=True to the list
for plugin in manager.getAllPlugins():
list.append((plugin.name, True))
return list
def test_updatePluginPlaces(self): class SpecificLocator(IPluginLocator): pass pm = PluginManager() pm.setPluginPlaces(["bla/bli"]) pm.updatePluginPlaces(["mif/maf"]) self.assertEqual(set(["bla/bli","mif/maf"]),set(pm.getPluginLocator().plugins_places))
def _run_reporting(self, results, objfile):
#options
options = dict()
# Build the PluginManager
reportingPluginManager = PluginManager()
reportingPluginManager.setPluginPlaces(["reporting"])
reportingPluginManager.collectPlugins()
# Trigger run from the "Reporting" plugins
for pluginInfo in reportingPluginManager.getAllPlugins():
reportingModul = pluginInfo.plugin_object
reportingModul.set_task(self.task)
# Give it the the relevant reporting.conf section.
try:
options = self.cfgReporting.get(pluginInfo.name)
reportingModul.set_options(options)
except Exception:
log.error("Reporting module %s not found in configuration file", pluginInfo.name)
# If the processing module is disabled in the config, skip it.
if not options.enabled:
continue
log.debug("Run Reporting: " + pluginInfo.name)
try:
# Run the Reporting module
reportingModul.run(results, objfile)
except Exception as e:
log.exception("Failed to run the reporting module \"%s\":",
reportingModul.__class__.__name__)
class FurnivallApplication(tornado.web.Application): def __init__(self): """ Sets up the Tornado web server and loads all the init data """ # Init mongodb database self.dbconnection = Connection() self.db = self.dbconnection['furnivall'] # Init plugins self.plugin_manager = PluginManager() self.plugin_manager.setPluginPlaces(["./plugins"]) self.plugin_manager.collectPlugins() print "Loaded plugins:" for plugin_info in self.plugin_manager.getAllPlugins(): print ' * ' + plugin_info.name # Init routes urls = [ ] + Route.routes() settings = dict( static_path = os.path.join(data_dir, "static"), template_path = os.path.join(data_dir, "templates"), xsrf_cookies = False, cookie_secret = "11oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1ao/Vo=", ) tornado.web.Application.__init__(self, urls, **settings)
def group_dashboard(request, group_id):
# check user is allowed to access this
user = request.user
config_installed = 'config' in settings.INSTALLED_APPS
user_level = user.userprofile.level
machine_group = get_object_or_404(MachineGroup, pk=group_id)
business_unit = machine_group.business_unit
if business_unit not in user.businessunit_set.all():
if user_level != 'GA':
return redirect(index)
if user_level == 'GA' or user_level == 'RW':
is_editor = True
else:
is_editor = False
machines = machine_group.machine_set.all()
# Build the manager
manager = PluginManager()
# Tell it the default place(s) where to find plugins
manager.setPluginPlaces([settings.PLUGIN_DIR, os.path.join(settings.PROJECT_DIR, 'server/plugins')])
# Load all plugins
manager.collectPlugins()
output = []
for plugin in manager.getAllPlugins():
data = {}
data['name'] = plugin.name
(data['html'], data['width']) = plugin.plugin_object.show_widget('group_dashboard', machines, machine_group.id)
output.append(data)
output = utils.orderPluginOutput(output, 'group_dashboard', machine_group.id)
c = {'user': request.user, 'machine_group': machine_group, 'user_level': user_level, 'is_editor': is_editor, 'business_unit': business_unit, 'output':output, 'config_installed':config_installed, 'request':request}
return render_to_response('server/group_dashboard.html', c, context_instance=RequestContext(request))
def load_plugins(cls):
manager = PluginManager()
manager.setPluginPlaces([os.path.join(os.getcwd(), "plugins/")])
manager.collectPlugins()
ret = manager.getAllPlugins()
logger.info('Loaded {} plugins'.format(len(ret)))
return ret
def run():
logging.basicConfig(level=logging.INFO)
os.chdir('/home/mjolnir/git/PURIKURA')
# H A N D L E P L U G I N S
pm = PluginManager()
pm.setPluginPlaces(['./pyrikura/plugins'])
pm.collectPlugins()
for pi in pm.getAllPlugins():
logging.info('loading plugin %s', pi.name)
pm.activatePluginByName(pi.name)
brokers = {}
nodes = build()
head = nodes[0]
for node in nodes:
brokers[node] = node.load(pm)
for node, broker in brokers.items():
for other in node._listening:
broker.subscribe(brokers[other])
start = time.time()
last_time = 0
shots = 0
last_trigger = 0
for broker in itertools.cycle(brokers.values()):
broker.update()
class FakeSite(object):
def __init__(self):
self.template_system = self
self.config = {"DISABLED_PLUGINS": [], "EXTRA_PLUGINS": [], "DEFAULT_LANG": "en"}
self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"]
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
}
)
self.loghandlers = [STDERR_HANDLER]
self.plugin_manager.setPluginInfoExtension("plugin")
if sys.version_info[0] == 3:
places = [os.path.join(os.path.dirname(utils.__file__), "plugins")]
else:
places = [os.path.join(os.path.dirname(utils.__file__), utils.sys_encode("plugins"))]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.timeline = [FakePost(title="Fake post", slug="fake-post")]
def render_template(self, name, _, context):
return '<img src="IMG.jpg">'
def getBankRates():
# Load the plugins from the plugin directory.
manager = PluginManager()
# Server
# manager.setPluginPlaces(["./dev/liborWatch/server/plugins"])
# Dev
# manager.setPluginPlaces(["./plugins"])
# Local
manager.setPluginPlaces(["./dev/NextLevelApps/liborWatch/server/plugins"])
manager.collectPlugins()
today = datetime.date.today().strftime('%Y-%m-%d')
# Loop round the plugins and get the bank rates
for plugin in manager.getAllPlugins():
rates = plugin.plugin_object.getRates()
print(rates);
if(len(rates) != 10):
print('Error: We did not get 10 rates from the plugin ' + plugin.name + ' - ' + str(len(rates)))
return
newRate = []
iCnt = 0;
for rate in rates:
iCnt = iCnt + 1
newRate.append((plugin.name, today, iCnt, rate))
# print('execute insert statement ' + plugin.name, str(today), str(iCnt), str(rate))
con = sqlite3.connect('liborWatch.sqlite')
with con:
cur = con.cursor()
cur.execute("CREATE TABLE IF NOT EXISTS BankRate(id INTEGER PRIMARY KEY, bankName TEXT, date TEXT, fixedForYears INTEGER, rate REAL)")
cur.executemany("INSERT INTO BankRate VALUES(NULL, ?, ?, ?, ?)", newRate)
class AlgorithmManager():
def __init__(self):
self._pluginManager = PluginManager()
self._pluginManager.setPluginPlaces(config.PLUGIN_DIR)
self._pluginManager.collectPlugins()
for pluginInfo in self._pluginManager.getAllPlugins():
self._pluginManager.activatePluginByName(pluginInfo.name)
def get_alg_names(self):
for plugin in self._pluginManager.getAllPlugins():
plugin.plugin_object.print_name()
def execute_calc(self, data):
run = []
if not self.__is_sequence(data):
raise Exception('Data is no array!')
for alg in self._pluginManager.getAllPlugins():
result, prop = alg.plugin_object.calc(data)
run.append({alg.name : (result, prop)})
return run
def __is_sequence(self, arg):
return (not hasattr(arg, "strip") and
hasattr(arg, "__getitem__") or
hasattr(arg, "__iter__"))
def loadPlugins(root_folder, plugin_name, categories_filter):
from yapsy.PluginManager import PluginManager
plugin_dirs = []
plugin_dirs.append(configuration.fabfile_basedir + '/.fabalicious/plugins')
plugin_dirs.append(expanduser("~") + '/.fabalicious/plugins')
plugin_dirs.append(root_folder + '/plugins')
log.debug("Looking for %s-plugins in %s" % (plugin_name, ", ".join(plugin_dirs)))
manager = PluginManager()
manager.setPluginPlaces(plugin_dirs)
manager.setCategoriesFilter(categories_filter)
manager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in manager.getAllPlugins():
manager.activatePluginByName(pluginInfo.name)
result = {}
for plugin in manager.getAllPlugins():
if hasattr(plugin.plugin_object, 'aliases') and isinstance(plugin.plugin_object.aliases, list):
for alias in plugin.plugin_object.aliases:
result[alias] = plugin.plugin_object
elif hasattr(plugin.plugin_object, 'alias'):
result[plugin.plugin_object.alias] = plugin.plugin_object
else:
result[plugin.name] = plugin.plugin_object
return result
def main():
parser = argparse.ArgumentParser(description='Validates file format')
parser.add_argument('files', nargs='+',
help='files to be validated')
parser.add_argument('--verbose', action='store_true',
default=False,
help='Shows help about validation')
args = parser.parse_args()
simplePluginManager = PluginManager()
simplePluginManager.setPluginPlaces(["plugins"])
simplePluginManager.collectPlugins()
for filename in args.files:
supported = False
for plugin in simplePluginManager.getAllPlugins():
methods = plugin.plugin_object
if methods.should_manage(filename):
supported = True
try:
plugin.plugin_object.validate(filename)
print('%s sintax ok for %s' % (filename, plugin.name))
break
except Exception as e:
print('Invalid file %s for %s' % (filename, plugin.name))
if args.verbose:
print(e)
if not supported:
print('%s cannot be validated' % filename)
def collect_data(self, **kwargs):
self.logger.info("Begin collect_data")
simplePluginManager = PluginManager()
logging.getLogger('yapsy').setLevel(logging.DEBUG)
simplePluginManager.setCategoriesFilter({
"DataCollector": data_collector_plugin
})
# Tell it the default place(s) where to find plugins
self.logger.debug("Plugin directories: %s" % (kwargs['data_collector_plugin_directories']))
simplePluginManager.setPluginPlaces(kwargs['data_collector_plugin_directories'])
simplePluginManager.collectPlugins()
plugin_cnt = 0
plugin_start_time = time.time()
for plugin in simplePluginManager.getAllPlugins():
self.logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(details=plugin.details):
plugin.plugin_object.start()
else:
self.logger.error("Failed to initialize plugin: %s" % (plugin.name))
plugin_cnt += 1
#Wait for the plugings to finish up.
self.logger.info("Waiting for %d plugins to complete." % (plugin_cnt))
for plugin in simplePluginManager.getAllPlugins():
plugin.plugin_object.join()
self.logger.info("%d Plugins completed in %f seconds" % (plugin_cnt, time.time() - plugin_start_time))
def loadPlugins(pluginfldr,hivesdict):
"""
Enumerate plugins in the specified directory and return the populated
plugin manager and a list of the compatible plugins. Compatibility is
determined by checking if the required hives were loaded sucessfully.
"""
pluginmanager = PluginManager()
pluginmanager.setPluginPlaces([pluginfldr])
pluginmanager.collectPlugins()
print ""
logging.info("[PLUGINS IDENTIFIED: %s]" % len(pluginmanager.getAllPlugins()))
compat_plugins = list()
incompat_plugins = list()
for plugin in pluginmanager.getAllPlugins():
compat = 1
for req in plugin.plugin_object.getRequirements():
if (hivesdict[req][0] <> 1):
compat = 0
break
if compat:
compat_plugins.append(plugin.name)
else:
incompat_plugins.append(plugin.name)
logging.info(" [%s] Compatible Plugins:" % len(compat_plugins))
for plugin in compat_plugins:
logging.info(" - %s" % plugin)
logging.info(" [%s] Incompatible Plugins:" % len(incompat_plugins))
for plugin in incompat_plugins:
logging.info(" - %s" % plugin)
return pluginmanager, compat_plugins
def main():
# Read configuration
config = SafeConfigParser(
defaults = {'port':'15915',
'plugins_directory':'./plugins',
'plugins_enabled':'',
})
config.read(['./conf/cm15d.conf',
'/etc/cm15d.conf',
'/etc/cm15d/cm15d.conf',
'/etc/cm15d/conf.d/local.conf',
])
# Activate enabled plugins
plugins = PluginManager()
plugins.setPluginPlaces(config.get('cm15d', 'plugins_directory').split(','))
plugins.collectPlugins()
plugins_enabled = config.get('cm15d', 'plugins_enabled').split(',')
for plugin in plugins.getAllPlugins():
if plugin.name in plugins_enabled:
plugins.activatePluginByName(plugin.name)
print("Plugin %s enabled" % plugin.name)
# Start server
port = int(config.get('cm15d', 'port'))
endpoint = TCP4ServerEndpoint(reactor, port)
endpoint.listen(CM15DaemonFactory(plugins))
print("Server listening on port %s" % port)
reactor.run()
def runCrawler():
mapURL = {}
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf'))
# Build the PluginManager
crawlerPluginManager = PluginManager()
crawlerPluginManager.setPluginPlaces(["crawler"])
crawlerPluginManager.collectPlugins()
# Trigger run from the "Crawler" plugins
for pluginInfo in sorted(crawlerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name):
crawlerModul = pluginInfo.plugin_object
# Config for crawler module
try:
options = cfgCrawler.get(pluginInfo.name)
crawlerModul.set_options(options)
except Exception:
log.error("Crawler module %s not found in configuration file", pluginInfo.name)
# If the crawler module is disabled in the config, skip it.
if not options.enabled:
continue
try:
log.debug("Run Crawler: " + pluginInfo.name)
returnMap = crawlerModul.run()
mapURL.update(returnMap)
except Exception as e:
log.error('Error (%s) in %s', e, pluginInfo.name)
return mapURL
class FakeSite(object):
def __init__(self):
self.template_system = self
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(utils.__file__), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(utils.__file__), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
def render_template(self, name, _, context):
return('<img src="IMG.jpg">')
def __run_reporting_generator(self):
# options
options = dict()
# Build the PluginManager
reportingPluginManager = PluginManager()
reportingPluginManager.setPluginPlaces(["reporting"])
reportingPluginManager.collectPlugins()
# Trigger run from the "Reporting" plugins
for pluginInfo in sorted(reportingPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name):
reportingModulG = pluginInfo.plugin_object
reportingModulG.set_task(self.task)
# Give it the the relevant reporting.conf section.
try:
options = self.cfgReporting.get(pluginInfo.name)
reportingModulG.set_options(options)
except Exception:
log.error("Reporting module %s not found in configuration file", pluginInfo.name)
# If the processing module is disabled in the config, skip it.
if not options.enabled:
continue
log.debug("Run Reporting: " + pluginInfo.name)
yield reportingModulG
def init_manager():
anl = PluginFileAnalyzerMathingRegex('custom_res_handler_plugins', r'^[A-Za-z0-9]+\.py$')
res = PluginFileLocator(plugin_info_cls=CFCustomResourceHandler)
res.setAnalyzers([anl])
manager = PluginManager(plugin_locator=res, categories_filter={'CFHandlers' : CFCustomResourceHandler})
manager.setPluginPlaces([dirname(__file__) + '/plugins'])
manager.collectPlugins()
return manager
class BotInteraction():
def __init__(self, config):
self.channel = config['channel']
self.nickname = config['nickname']
self.prompt = config['prompt']
self.parts = ""
self.ping = ""
self.username = ""
self.message = ""
self.manager = PluginManager()
self.manager.setPluginPlaces(["plugins"])
self.manager.collectPlugins()
def handle_line(self, username, message, parts):
print(username + ": " + message)
if self.nickname not in parts[1] and message[0:1] == self.prompt:
message = message.replace(">", " ").split(None, 1)
for plugin in self.manager.getAllPlugins(): # Collect all plugins
try:
# Message[0] - search plugin_name
plugin_yapsy = self.manager.getPluginByName(message[0])
command = []
# If the message has arguments - message[1] add to command
if len(message) == 2:
command = message[1]
# To found plugin send arguments.
return plugin_yapsy.plugin_object.execute(self.channel,
username,
command)
except:
continue
def check_ping(self, line):
if line.find("PING") != -1: # If server pings then pong
line = string.split(line, " ")
self.ping = "PONG " + line[1]
else:
self.ping = ""
self.parts = string.split(line, ":")
if "QUIT" not in self.parts[1] and \
"JOIN" not in self.parts[1] and \
"PART" not in self.parts[1]:
try:
# Sets the message variable to the actual message sent
self.message = self.parts[2][:len(self.parts[2]) - 1]
except:
self.message = ""
# Sets the username variable to the actual username
usernamesplit = string.split(self.parts[1], "!")
self.username = usernamesplit[0]
def index(request):
# Get the current user's Business Units
user = request.user
# Count the number of users. If there is only one, they need to be made a GA
if User.objects.count() == 1:
# The first user created by syncdb won't have a profile. If there isn't one, make sure they get one.
try:
profile = UserProfile.objects.get(user=user)
except UserProfile.DoesNotExist:
profile = UserProfile(user=user)
profile.level = "GA"
profile.save()
user_level = user.userprofile.level
now = datetime.now()
hour_ago = now - timedelta(hours=1)
today = date.today()
week_ago = today - timedelta(days=7)
month_ago = today - timedelta(days=30)
three_months_ago = today - timedelta(days=90)
if user_level != "GA":
# user has many BU's display them all in a friendly manner
business_units = user.businessunit_set.all()
if user.businessunit_set.count() == 1:
# user only has one BU, redirect to it
for bu in user.businessunit_set.all():
return redirect("server.views.bu_dashboard", bu_id=bu.id)
break
else:
machines = Machine.objects.all()
# Build the manager
manager = PluginManager()
# Tell it the default place(s) where to find plugins
manager.setPluginPlaces([settings.PLUGIN_DIR, os.path.join(settings.PROJECT_DIR, "server/plugins")])
# Load all plugins
manager.collectPlugins()
output = []
# Loop round the plugins and print their names.
for plugin in manager.getAllPlugins():
data = {}
data["name"] = plugin.name
(data["html"], data["width"]) = plugin.plugin_object.show_widget("front", machines)
# output.append(plugin.plugin_object.show_widget('front'))
output.append(data)
output = utils.orderPluginOutput(output)
# get the user level - if they're a global admin, show all of the machines. If not, show only the machines they have access to
business_units = BusinessUnit.objects.all()
config_installed = "config" in settings.INSTALLED_APPS
c = {
"user": request.user,
"business_units": business_units,
"output": output,
"config_installed": config_installed,
}
return render_to_response("server/index.html", c, context_instance=RequestContext(request))
def main():
# Create plugin manager
manager = PluginManager()
manager.setPluginPlaces([PluginFolder])
# Collect plugins
manager.collectPlugins()
for plugin in manager.getAllPlugins():
plugin.plugin_object.say_hello()
def main():
# Load the plugins from the plugin directory.
manager = PluginManager(plugin_info_ext='cfg')
manager.setPluginPlaces(["../plugins"])
manager.collectPlugins()
# Loop round the plugins and print their names.
for plugin in manager.getAllPlugins():
plugin.plugin_object.print_name()
class Plugins():
'''
Plugins
'''
def __init__(self):
'''
Constructor
'''
super(Plugins, self).__init__()
self.write_tab_dock_plugin_dict = {}
self.write_panel_dock_plugin_dict = {}
# Build the manager
self._plugin_manager = PluginManager()
# List all sub-directories of "plugins"
plugin_path = os.path.sep.join([os.getcwd(), "plugins"])
plugin_places = [plugin_path]
for dirname, dirnames, filenames in os.walk(plugin_path):
# print path to all subdirectories first.
for subdirname in dirnames:
plugin_places.append(os.path.join(dirname, subdirname))
# Tell it the default place(s) where to find plugins
self._plugin_manager.setPluginPlaces(plugin_places)
sys.path.append(plugin_path)
# Define the various categories corresponding to the different
# kinds of plugins you have defined
self._plugin_manager.setCategoriesFilter({
"GuiWriteTabDockPlugin": GuiWriteTabDockPlugin,
"GuiWritePanelDockPlugin": GuiWritePanelDockPlugin
})
self._plugin_manager.collectPlugins()
self.load_plugins(
["GuiWriteTabDockPlugin", "GuiWritePanelDockPlugin"])
def load_plugins(self, categories):
'''
function:: load_plugins(categories)
:param categories: list
'''
for category in categories:
for pluginInfo in self._plugin_manager.getPluginsOfCategory(category):
setattr(self, pluginInfo.plugin_object.gui_class(
).__name__, pluginInfo.plugin_object.gui_class())
if category is "GuiWriteTabDockPlugin":
self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \
= pluginInfo.plugin_object.gui_class()
if category is "GuiWritePanelDockPlugin":
self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \
= pluginInfo.plugin_object.gui_class()
class PluginHandler:
def __init__(self):
self._plugin_manager = PluginManager()
self._category_active = {"Modifier": False, "Analyzer": False, "Comparator": False}
self._plugin_from_category = {"Modifier": [], "Analyzer": [], "Comparator": []}
self._modifier_plugins = []
self._analyzer_plugins = []
self._collect_all_plugins()
def _collect_all_plugins(self):
self._plugin_manager.setPluginPlaces(
["hugin/analyze/modifier", "hugin/analyze/analyzer", "hugin/analyze/comparator"]
)
# setting filter categories for pluginmanager
self._plugin_manager.setCategoriesFilter(
{
# movie metadata provider
"Modifier": IModifier,
# movie metadata provider
"Comparator": IComparator,
# sub metadata provider
"Analyzer": IAnalyzer,
}
)
self._plugin_manager.collectPlugins()
def activate_plugins_by_category(self, category):
self._toggle_activate_plugins_by_category(category)
def deactivate_plugins_by_category(self, category):
self._toggle_activate_plugins_by_category(category)
def _toggle_activate_plugins_by_category(self, category):
plugins = self._plugin_manager.getPluginsOfCategory(category)
is_active = self._category_active[category]
for pluginInfo in plugins:
if is_active:
self._plugin_manager.deactivatePluginByName(name=pluginInfo.name, category=category)
self._plugin_from_category[category].remove(pluginInfo)
else:
self._plugin_manager.activatePluginByName(name=pluginInfo.name, category=category)
self._plugin_from_category[category].append(pluginInfo)
self._category_active[category] = not is_active
def get_plugins_from_category(self, category):
plugins = []
for plugin in self._plugin_from_category[category]:
plugin.plugin_object.name = plugin.name
plugin.plugin_object.description = plugin.description
plugins.append(plugin.plugin_object)
return plugins
def is_activated(self, category):
""" True if category is activated. """
return self._category_active[category]
def plugins():
# Load the plugins from the plugin directory.
manager = PluginManager()
manager.setPluginPlaces(["plugins"])
manager.collectPlugins()
# Loop round the plugins and print their names.
for p in manager.getAllPlugins():
p.plugin_object.print_name()
p.plugin_object.activate()
def main():
"""docstring for main"""
# Load the plugins from the plugin directory.
manager = PluginManager()
manager.setPluginPlaces(["plugins"])
manager.collectPlugins()
# Loop round the plugins executing them.
for plugin in manager.getAllPlugins():
plugin.plugin_object.run()
def main():
# Load the plugins from the plugin directory.
manager = PluginManager()
manager.setPluginPlaces(["orthos/plugins"])
manager.collectPlugins()
# Loop round the plugins and print their names.
for plugin in manager.getAllPlugins():
print "Fo"
plugin.plugin_object.print_name()
class RetrieveResource(object):
def __init__(self, args, yaml):
self.simplePluginManager = PluginManager()
self.args = args
self.output_dir = args.output_dir if args.output_dir is not None else PIS_OUTPUT_DIR
self.yaml = yaml
# Warning the user about the gc credential needs for access to GC itself
def checks_gc_service_account(self):
if self.args.google_credential_key is None:
logger.info(
"Some of the steps might be not work properly due the lack of permissions to access to GCS. "
"Eg. Evidence")
else:
GoogleBucketResource.has_valid_auth_key(
self.args.google_credential_key)
# Copy the local files to the Google Storage
def copy_to_gs(self):
if self.args.google_bucket is not None:
Utils(self.yaml.config, self.yaml.outputs).gsutil_multi_copy_to(
self.args.google_bucket)
else:
logger.error("Destination bucket info missing")
# This function normalise the input inserted by the user. Lower and Upper cases can break the code if
# not managed. Eg. SO/so/So -> SO Plugin
def normalise_steps(self, steps, all_plugins_available):
normalise_steps = []
lowercase_steps = [each_step.lower() for each_step in steps]
for plugin in all_plugins_available:
if plugin.lower() in lowercase_steps:
normalise_steps.append(plugin)
lowercase_steps.remove(plugin.lower())
logger.info("Steps not found:\n" + ','.join(lowercase_steps))
return normalise_steps
# Extract and check the steps to run
def steps(self):
all_plugins_available = []
for plugin in self.simplePluginManager.getAllPlugins():
all_plugins_available.append(plugin.name)
steps_requested = self.normalise_steps(self.args.steps,
all_plugins_available)
excluded_requested = self.normalise_steps(self.args.exclude,
all_plugins_available)
if len(self.args.steps) == 0:
plugin_order = list(
set(all_plugins_available) - set(excluded_requested))
else:
plugin_order = list(set(steps_requested))
logger.info("Steps selected:\n" + ','.join(plugin_order))
return plugin_order
# Init yapsy plugin manager
def init_plugins(self):
# Tell it the default place(s) where to find plugins
self.simplePluginManager.setPluginPlaces(["plugins"])
# Load all plugins
self.simplePluginManager.collectPlugins()
# noinspection PyBroadException
# Given a list of steps to run, this procedure executes the selected plugins/step
def run_plugins(self):
steps_to_execute = self.steps()
for plugin_name in steps_to_execute:
plugin = self.simplePluginManager.getPluginByName(plugin_name)
try:
plugin.plugin_object.process(self.yaml[plugin_name.lower()],
self.yaml.outputs,
self.yaml.config)
except Exception as e:
logger.info(
"WARNING Plugin not available {}".format(plugin_name))
logger.info(e)
def create_output_structure(self, output_dir):
"""By default the directories prod and staging are created"""
remove_output_dir(
output_dir) if self.args.force_clean else logger.info(
"Warning: Output not deleted.")
self.yaml.outputs.prod_dir = create_output_dir(output_dir + '/prod')
self.yaml.outputs.staging_dir = create_output_dir(output_dir +
'/staging')
# Retrieve the resources requested.
def run(self):
self.create_output_structure(self.output_dir)
self.init_plugins()
self.checks_gc_service_account()
self.run_plugins()
self.copy_to_gs()
def index(request):
# Get the current user's Business Units
user = request.user
# Count the number of users. If there is only one, they need to be made a GA
if User.objects.count() == 1:
# The first user created by syncdb won't have a profile. If there isn't one, make sure they get one.
try:
profile = UserProfile.objects.get(user=user)
except UserProfile.DoesNotExist:
profile = UserProfile(user=user)
profile.level = 'GA'
profile.save()
user_level = user.userprofile.level
now = datetime.now()
hour_ago = now - timedelta(hours=1)
today = date.today()
week_ago = today - timedelta(days=7)
month_ago = today - timedelta(days=30)
three_months_ago = today - timedelta(days=90)
config_installed = 'config' in settings.INSTALLED_APPS
if user_level != 'GA':
# user has many BU's display them all in a friendly manner
business_units = user.businessunit_set.all()
if user.businessunit_set.count() == 0:
c = {'user': request.user, }
return render_to_response('server/no_access.html', c, context_instance=RequestContext(request))
if user.businessunit_set.count() == 1:
# user only has one BU, redirect to it
for bu in user.businessunit_set.all():
return redirect('server.views.bu_dashboard', bu_id=bu.id)
break
if user_level == 'GA':
machines = Machine.objects.all()
else:
machines = Machine.objects.none()
for business_unit in user.businessunit_set.all():
for group in business_unit.machinegroup_set.all():
machines = machines | group.machine_set.all()
# Load in the default plugins if needed
utils.loadDefaultPlugins()
# Build the manager
manager = PluginManager()
# Tell it the default place(s) where to find plugins
manager.setPluginPlaces([settings.PLUGIN_DIR, os.path.join(settings.PROJECT_DIR, 'server/plugins')])
# Load all plugins
manager.collectPlugins()
output = []
# Get all the enabled plugins
enabled_plugins = Plugin.objects.all().order_by('order')
for enabled_plugin in enabled_plugins:
# Loop round the plugins and print their names.
for plugin in manager.getAllPlugins():
if plugin.name == enabled_plugin.name:
data = {}
data['name'] = plugin.name
(data['html'], data['width']) = plugin.plugin_object.show_widget('front', machines)
output.append(data)
break
output = utils.orderPluginOutput(output)
# get the user level - if they're a global admin, show all of the machines. If not, show only the machines they have access to
if user_level == 'GA':
business_units = BusinessUnit.objects.all()
else:
business_units = user.businessunit_set.all()
c = {'user': request.user, 'business_units': business_units, 'output': output, }
return render_to_response('server/index.html', c, context_instance=RequestContext(request))
class Phong(object):
def __init__(self):
self._log = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
analyzer = PluginFileAnalyzerWithInfoFile('phong', 'phong-plugin')
self._plugins = PluginManager()
self._plugins.getPluginLocator().setAnalyzers([analyzer])
self._plugins.setPluginPlaces([
'./plugins/',
'/usr/lib/phong/plugins',
])
self._config = ConfigParser.ConfigParser()
self.loadConfig(
os.path.sep.join((os.path.dirname(__file__), 'defaults.cfg')))
self._wiki = None
self._commands = []
self._spiff = None
@property
def pluginManager(self):
return self._plugins()
@property
def config(self):
return self._config
@property
def spiff(self):
if spiff is not None and self._spiff is None:
self._spiff = spiff.API(self.config.get('phong', 'spiff-api'))
return self._spiff
@property
def wiki(self):
if self._wiki is None:
url = self._config.get('phong', 'mediawiki-url')
api = self._config.get('phong', 'mediawiki-api-url')
username = self._config.get('phong', 'mediawiki-username')
password = self._config.get('phong', 'mediawiki-password')
self._wiki = phong.wiki.Wiki(url, api, username, password)
return self._wiki
def buildContext(self):
context = {
'phong': templates.PhongTemplateAPI(self),
}
for plugin in self._plugins.getAllPlugins():
cxt = plugin.plugin_object.buildContext(self)
if isinstance(cxt, dict):
context.update(cxt)
return context
def getTemplate(self,
name,
defaultContext={},
buildContext=True,
prefix=None):
if buildContext:
cxt = self.buildContext()
cxt.update(defaultContext)
else:
cxt = defaultContext
engines = []
if self.config.has_option('phong', 'template-paths'):
for path in self.config.get('phong', 'template-paths').split(','):
engines.append(phong.templates.FileEngine(self, path.strip()))
engines.append(phong.templates.FileEngine(self, 'templates'))
engines.append(phong.templates.WikiEngine(self))
for e in engines:
if e.hasTemplate(name, prefix):
return e.getTemplate(name, prefix, cxt)
def renderTemplate(self, name, context={}):
cxt = self.buildContext()
cxt.update(context)
return self.getTemplate(name).render(context)
def loadPlugins(self):
self._plugins.collectPlugins()
for plugin in self._plugins.getAllPlugins():
self._plugins.activatePluginByName(plugin.name)
for cmd in plugin.plugin_object.availableCommands():
self._commands.append(cmd(self, plugin))
def loadConfig(self, path):
self._log.debug("Loading configuration from %s", path)
self._config.read(os.path.expanduser(path))
def getState(self, name):
dir = os.path.expanduser(self._config.get('phong', 'state-dir'))
return phong.state.State(
os.path.sep.join((dir, "%s.state.json" % (name))))
@property
def argv(self):
return self._argv
@property
def args(self):
return self._args
def main(self, argv):
parser = argparse.ArgumentParser(prog='phong', add_help=False)
parser.add_argument(
'-c',
'--config',
help=
'Configuration file to use. Can be specified multiple times. Later files override earlier ones.',
action='append',
default=[
"/etc/phong.cfg",
])
parser.add_argument('-d',
'--dry-run',
help='Dont actually do anything',
default=False,
action='store_true')
parser.add_argument('-D',
'--debug',
help='Print debug messages',
default=False,
action='store_true')
args = parser.parse_known_args(argv)
parser.add_argument('-h',
'--help',
help='show this help message and exit',
action='help')
if args[0].debug:
logging.basicConfig(level=logging.DEBUG)
for config in args[0].config:
self.loadConfig(config)
self.loadPlugins()
subparser = parser.add_subparsers(help='action')
for command in self._commands:
pluginArgs = subparser.add_parser(command.name(),
help=command.helpText())
pluginArgs.set_defaults(command_obj=command)
command.buildArgs(pluginArgs)
self._argv = argv
self._args = parser.parse_args(argv)
return self._args.command_obj.execute(self._args)
print ("Board type: OpenBCI Ganglion")
import openbci.ganglion as bci
else:
raise ValueError('Board type %r was not recognized. Known are 3 and 4' % args.board)
# Check AUTO port selection, a "None" parameter for the board API
if "AUTO" == args.port.upper():
print("Will try do auto-detect board's port. Set it manually with '--port' if it goes wrong.")
args.port = None
else:
print("Port: ", args.port)
plugins_paths = ["plugins"]
if args.plugins_path:
plugins_paths += args.plugins_path
manager.setPluginPlaces(plugins_paths)
manager.collectPlugins()
# Print list of available plugins and exit
if args.list:
print ("Available plugins:")
for plugin in manager.getAllPlugins():
print ("\t- " + plugin.name)
exit()
# User wants more info about a plugin...
if args.info:
plugin = manager.getPluginByName(args.info)
if plugin == None:
# eg: if an import fail inside a plugin, yapsy skip it
print ("Error: [ " + args.info + " ] not found or could not be loaded. Check name and requirements.")
@hug.get("/", output=hug.output_format.html)
def get_root(response):
return dashboard.on_get(persistence_mgr)
def plugins_callback(event_id, future):
"""Attach the value returned by a plugin to the associated event."""
return_val = future.result(0)
if return_val is not None:
persistence_mgr.attach_info(event_id, return_val)
mongodb_uri = config.config.get("server", "MongoDBUri")
persistence_mgr = persistence.PersistenceManager(mongodb_uri)
executor = ThreadPoolExecutor()
plugin_mgr = PluginManager()
plugin_mgr.setPluginPlaces(["./counterserver/plugins"])
plugin_mgr.setCategoriesFilter({
IEventReceiverPlugin.__name__:
IEventReceiverPlugin,
})
plugin_mgr.collectPlugins()
for p in plugin_mgr.getAllPlugins():
logging.info("Plugin '" + p.name + "' loaded")
if __name__ == '__main__':
hug.API(__name__).http.serve()
class Controller(EventEmitter):
"""Main controller of the class. Views+ui separated to different files."""
def __init__(self):
"""Constructor."""
# run the superclass constructor
EventEmitter.__init__(self)
# Model initialization code
self.sensors = []
self.tags = []
# define important directories for external (not program code) files
homedir = os.environ["HOME"]
ldrop_home = os.path.join(homedir, "Documents", "ldrop_data")
self.rootdir = ldrop_home
self.plugindir = os.path.join(ldrop_home, "plugins")
self.savedir = os.path.join(ldrop_home, "recordings")
# check that saving, experiment etc directories are present
utils.dircheck(self.savedir)
utils.dircheck(self.plugindir)
# put the plugins-directory
sys.path.append(self.plugindir)
# temporary? keyboard-contigency list
self.keyboard_contigency = []
# initialize plugin manager
self.pluginmanager = PluginManager(plugin_locator=LdropPluginLocator())
self.pluginmanager.setPluginPlaces([self.plugindir])
self.pluginmanager.collectPlugins()
self.gui = []
self.play_callback = None
self.stop_callback = None
self.continue_callback = None
self.data_callback = None
self.participant_id = ""
def add_model(self, model):
"""Add a model to listen for."""
#TODO: currently same add_model for different kinds of emitters
# (sensors and experiments), just counting they send different signals
model.on("tag", self.on_tag)
model.on("data", self.on_data)
model.on("close_controller", self.on_close_controller)
model.on("start_collecting_data", self.on_start_collecting_data)
model.on("stop_collecting_data", self.on_stop_collecting_data)
model.on("log_message", self.on_log_message)
model.on("query", self.on_query)
def add_sensor(self, sensor_name):
"""Callback for Add sensor -button."""
# TODO: Improve APIs for plugins
plugin_info = self.pluginmanager.getPluginByName(sensor_name)
if plugin_info is None:
print("Plugin " + sensor_name + " not found")
else:
plugin_info.plugin_object.get_sensor(self.rootdir,
self.on_sensor_created,
self.on_sensor_error)
def close(self):
"""Method that closes the drop controller."""
# disconnect all the sensors from the host
for sensor in self.sensors:
# TODO: this is done on stop_collecting data - unify
sensor.stop_recording()
sensor.disconnect()
self.remove_all_listeners()
self.ml.quit()
print("ldrop mainloop stopped.")
def close_gui(self):
"""Clear gui reference."""
self.gui = []
# run in what condition
self.close()
def continue_experiment(self):
"""Callback for continuebutton click."""
if self.continue_callback is not None:
self.continue_callback()
def enable_gui(self):
"""Initialize pygtk-view to be run when mainloop starts."""
self.gui.append(LDPV(self, self.savedir))
def get_participant_id(self):
"""Return participant_id."""
return self.participant_id
def get_sensors(self):
"""Return list of connected sensors."""
return self.sensors
def get_sensor_plugins(self):
"""Return list of available sensors."""
plugins = self.pluginmanager.getAllPlugins()
sensornames = []
for p in plugins:
sensornames.append(p.name)
return sensornames
def message_to_sensor(self, sensortype, msg):
"""
Callback for a message to sensor.
Sensor needs to support the msg.
"""
# find the right sensor(s) to forward the message to
for sensor in self.sensors:
if sensor.get_type == sensortype:
sensor.on_message(msg)
def on_close_controller(self):
"""Callback for signal close_controller."""
glib.idle_add(self.close)
def on_data(self, dp):
"""Callback for data-signal."""
if self.data_callback is not None:
glib.idle_add(self.data_callback, dp)
def on_experiment_completed(self):
"""Callback for experiment finished."""
# clear view references
for r in self.sensors:
self.exp_view.remove_model(r)
# self.exp_view = None
def on_keypress(self, keyname):
"""Callback for keypress."""
if keyname in self.keyboard_contigency:
self.keyboard_contigency = []
self.emit("continue")
tag = {
"id": keyname,
"secondary_id": "keypress",
"timestamp": self.timestamp()
}
self.on_tag("tag", tag)
def on_query(self, msg, title, buttons, callbacks, callback_args):
if len(self.gui) > 0:
for g in self.gui:
g.show_message_box(msg, title, buttons, callbacks,
callback_args)
def on_sensor_error(self, msg):
"""Sensor error-handler."""
self.emit("error", msg)
def on_sensor_created(self, shandle):
"""Callback for sensor initialization."""
self.sensors.append(shandle)
self.emit("sensorcount_changed")
# add model to hear calls from sensors, such as data_condition met
self.add_model(shandle)
def on_start_collecting_data(self, savesubdir, savefilestring):
"""A callback for start_collecting_data signal."""
self.start_collecting_data(savesubdir, savefilestring)
def on_stop_collecting_data(self):
"""A callback for stop_collecting_data signal."""
self.stop_collecting_data(None)
def on_tag(self, tag):
"""
Send a tag to all sensors.
Tag might not come instantly here so the
timestamp is taken in advance. The sensor must sync itself with the
computer. Tag consists of:
id = string, identifier of the tag
timestamp = timestamp in ms of the localtime clock
secondary_id = string, defines "start", "end", or "impulse", as a start
of perioid, end of it or single impulse
misc = other possible information (depends on the sensor how to use)
"""
for sensor in self.sensors:
# send a copy of the dict to each sensor
sensor.tag(tag.copy())
def on_log_message(self, logmsg):
"""Callback for log_append signal."""
self.emit("log_update", logmsg)
def play(self):
"""Start the experiment."""
# TODO: possibly change the "pipeline" of the drop-involvement in exp
if self.play_callback is not None:
glib.idle_add(self.play_callback)
def run(self):
"""Initialize controller start mainloop."""
# if no gui to control experiment is present, just start running the
# experiment
if len(self.gui) == 0 and self.play_callback is not None:
self.play()
self.ml = glib.MainLoop()
self.ml.run()
def remove_model(self, model):
"""Remove model listeners."""
model.remove_listener("tag", self.on_tag)
model.remove_listener("data", self.on_data)
model.remove_listener("close_controller", self.on_close_controller)
model.remove_listener("start_collecting_data",
self.on_start_collecting_data)
model.remove_listener("stop_collecting_data",
self.on_stop_collecting_data)
model.remove_listener("log_message", self.on_log_message)
model.remove_listener("query", self.on_query)
def remove_sensor(self, sensor_id):
"""Disconnect the sensor with the provided sensor_id."""
for sensor in self.sensors:
if sensor.get_sensor_id() == sensor_id:
sensor.disconnect()
self.sensors.remove(sensor)
self.emit("sensorcount_changed")
def set_callbacks(self, play_callback, stop_callback, continue_callback,
data_callback):
"""Experiment side callback-setter."""
self.play_callback = play_callback
self.stop_callback = stop_callback
self.continue_callback = continue_callback
self.data_callback = data_callback
def set_participant_id(self, pid):
"""Method for setting participant_id."""
self.participant_id = pid
self.emit("participant_id_updated")
def sensor_action(self, sensor_id, action_id):
"""Perform action that is listed on sensors control elements."""
for sensor in self.sensors:
if sensor.get_sensor_id() == sensor_id:
sensor.action(action_id)
def start_collecting_data(self, savesubdir, savefilestring):
"""Function starts data collection on all sensors."""
savepath = os.path.join(self.savedir, savesubdir)
for sensor in self.sensors:
sensor.start_recording(savepath, savefilestring)
# sensor.start_recording(savepath, self.participant_id,
# self.experiment_id)
def stop(self):
"""Callback for stopbutton click."""
if self.stop_callback is not None:
self.stop_callback()
def stop_collecting_data(self, callback):
"""Stop data collection on all sensors and run callback."""
for sensor in self.sensors:
sensor.stop_recording()
if callback is not None:
glib.idle_add(callback)
def timestamp(self):
"""Return a local timestamp in microsecond accuracy."""
return time.time()
def __del__(self):
"""Destructor."""
print("ldrop instance deleted.")
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {
'listings': 'listings'
},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension
})
self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
nikola.utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category(
"CompilerExtension")
self.timeline = [FakePost(title='Fake post', slug='fake-post')]
self.debug = True
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(
plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return ('<img src="IMG.jpg">')
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data,
self.shortcode_registry)
class Friday:
def __init__(self):
self.ai = ai_interface.API(
apiai.ApiAI(settings['CLIENT_ACCESS_TOKEN'],
settings['SUBSCRIPTION_KEY']))
self.debugging = settings['debugging']
self.spoken_language = settings['spoken language']
self.input_system = settings['input system'] # google, local, text
self.output_system = settings['output system'] # both, audio, text
self.speech_file_location = settings['speech file location'] # .
self.speech_file_name = settings['speech file name'] # audio response
self.speak = settings['speak'] # True
# The question that the assistant hears
self.question = None
# The chosen, spoken response given to the user.
self.response = None
# Whether Friday is active
self.is_active = True
# What type of question is being asked.
self.request_type = None
if settings['input system'] != 'text':
self.recognizer = sr.Recognizer()
self.microphone = sr.Microphone(device_index=settings['DEVICE'],
sample_rate=settings['RATE'],
chunk_size=settings['CHUNK'])
if settings['input_system'] == 'google':
with self.microphone as source:
if settings['debugging']:
click.echo("Adjusting to ambient noise.")
# we only need to calibrate once, before we start listening
self.recognizer.adjust_for_ambient_noise(source)
# Build the manager
self.manager = PluginManager()
# Tell it the default place(s) where to find plugins
self.manager.setPluginPlaces(settings["plugin folder"])
# Load all plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
self.plugins = {}
# Activate all loaded plugins
for plugin in self.manager.getAllPlugins():
self.plugins[plugin.name] = plugin.plugin_object
def _local_stt(self):
pass
def _apiai_stt(self):
from math import log
import audioop
import pyaudio
import time
resampler = apiai.Resampler(source_samplerate=settings['RATE'])
request = self.ai.voice_request()
vad = apiai.VAD()
def callback(in_data, frame_count):
frames, data = resampler.resample(in_data, frame_count)
if settings.show_decibels:
decibel = 20 * log(audioop.rms(data, 2) + 1, 10)
click.echo(decibel)
state = vad.processFrame(frames)
request.send(data)
state_signal = pyaudio.paContinue if state == 1 else pyaudio.paComplete
return in_data, state_signal
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paInt32,
input=True,
output=False,
stream_callback=callback,
channels=settings['CHANNELS'],
rate=settings['RATE'],
frames_per_buffer=settings['CHUNK'])
stream.start_stream()
click.echo("Speak!")
while stream.is_active():
time.sleep(0.1)
stream.stop_stream()
stream.close()
p.terminate()
def _google_stt(self):
"""
Uses Google's Speech to Text engine to
understand speech and convert it into
text.
:return: String, either the spoken text or error text.
"""
click.echo("Please speak now")
# Listen to the microphone set up in the __init__ file.
with self.microphone as mic_source:
audio = self.recognizer.listen(mic_source)
click.echo("Processing audio")
try:
# Try to recognize the text
return self.recognizer.recognize_google(audio,
show_all=self.debugging)
except sr.UnknownValueError: # speech is unintelligible
return "Google could not understand the audio."
except sr.RequestError:
return "Could not request results from Google's speech recognition service."
def listen(self):
# Default value.
# TODO: Input plugin system goes here
# Plugins should be used for getting input so users can build their
# own custom front-end to the application.
# This will allow the speech to text functions to be moved out of the
# main class and into their own plugins.
if self.input_system == 'google':
self.question = self._google_stt()
else:
self.question = click.prompt("Input your query")
# TODO: Loading plugin system goes here
# Eventually I want people to be able to build custom behavior
# that executes when the response is being fetched. So there could
# be a loading screen that plays music or something for example.
click.echo("Wait for response...")
# Get a response from the AI using the api interface.
self.ai.get_response(self.question)
# Clean up the response and turn it into a Python object that can be read
self.response = self.ai.parse()
return self.response
def think(self, request):
# TODO: Move plugins to a priority system
# Eventually plugins should return an integer value which determines
# their ordering when responding to a request, in the event multiple
# plugins are able to respond to the same request.
# This may eventually allow for watchdog plugins which check for
# erroneous requests and alert the system by returning negative (Maybe)
# priorities, which can then be used to safeguard a user's system or
# settings.
return any(
plugin.can_perform(self, request)
for name, plugin in self.plugins.items())
def perform(self, request):
# Trigger 'some request' from the loaded plugins
for name, plugin in self.plugins.items():
if plugin.can_perform(self, request):
# TODO: Allow plugins to fail
# Plugins will be able to return False in the event they fail,
# allowing the system to then apologize to the user.
plugin.perform(self, request)
return True
def refuse(self):
# May become unnecessary as the plugins could be built to refuse
# requests directly.
click.echo("Can't do that")
def apologize(self):
# May eventually become unnecessary as plugins could be built to
# return apologies directly.
click.echo("I failed to do that, sorry.")
def _print_response(self):
click.echo(self.response)
def _speak_response(self):
self.say(self.response,
title=self.speech_file_name,
speak_aloud=self.speak)
def respond(self):
# TODO: Output plugin system goes here
# Plugins should be used for providing output for the same reasons
# that plugins should be used for providing input.
if self.output_system == "audio":
self._speak_response()
elif self.output_system == "text":
self._print_response()
else:
self._print_response()
self._speak_response()
def say(self, message, speak_aloud=True, title='Speak'):
# May eventually be moved into its own plugin
message = str(message)
if not message.strip():
click.echo("No text to speak.")
return
computer_os = system()
folder = os.path.split(title)[0]
folder_path = os.path.join(self.speech_file_location, folder)
if not os.path.exists(folder_path):
os.makedirs(folder_path)
home = settings['home']
if self.input_system == 'google':
# Create the MP3 file which will speak the text
title += '.wav'
path = os.path.join(home, title)
tts = gTTS(message, lang=self.spoken_language)
tts.save(path)
speech_file = "start /MIN {}"
if computer_os != 'Windows':
speech_file = "mpg123 {}"
speech_file = speech_file.format(os.path.join(home, title))
else:
# Create the Visual Basic code which will speak the text
title += '.vbs'
path = os.path.join(home, title)
message = re.sub(
'["\n\t]', '', message
) # Strip out characters that would be spoken by the system.
if computer_os == "Windows":
with open(path, 'w') as file:
file.write("""
speaks="{}"
Dim speaks, speech
Set speech=CreateObject("sapi.spvoice")
speech.Speak speaks
""".format(message))
# Set up the file to be executed
speech_file = ["cscript.exe", title]
if computer_os != "Windows":
speech_file = ["espeak", message]
if speak_aloud:
try:
call(speech_file)
except FileNotFoundError:
if self.debugging:
click.echo("File not accessible:" + str(speech_file))
class Plugins():
"""
Class responsible to manage the plugins
"""
categories = [
[
"SitePreparsing", SitePreparsing,
"Site wide plugins that execute before the parsing start."
],
[
"SiteProcessor", SiteProcessor,
"Plugins that process the whole site once after parsing."
],
[
"SiteRendering", SiteRendering,
"Plugins that render additional pages after the rendering."
],
[
"PostProcessor", PostProcessor,
"Plugins that process each post after they are parsed"
],
[
"CollectionProcessor", CollectionProcessor,
"Plugins that process each collection after posts are parsed"
],
[
"TemplateFilter", TemplateFilter,
"Plugins that define jinja2 filters to be used in templates"
],
]
# for plugin info structure
PLUGIN_CAT = 0
PLUGIN_NAME = 1
PLUGIN_DESC = 2
PLUGIN_ENABLE = 3
PLUGIN_MODULE_NAME = 4
PLUGIN_VERSION = 5
def __init__(self, plugin_directories, debug_log_fname, plugins_config):
"Load plugins"
categories_filter = {}
for cat in self.categories:
categories_filter[cat[0]] = cat[1]
# single or multi-directory handling
if not isinstance(plugin_directories, list):
plugin_directories = [plugin_directories]
self.plugins = PluginManager(plugin_info_ext='sitefab-plugin',
categories_filter=categories_filter)
self.plugins.setPluginPlaces(plugin_directories)
self.plugins.locatePlugins()
self.plugins.loadPlugins()
self.plugins_config = plugins_config
# List of enabled plugins
self.plugins_enabled = {}
for pl in self.get_plugins_info():
if pl[self.PLUGIN_ENABLE]:
self.plugins_enabled[pl[self.PLUGIN_MODULE_NAME]] = 1
# list of plugins already executed. Used for dependencies tracking
# across stages
self.plugins_executed = {}
# FIXME: make sure it is working
# logging.basicConfig(filename=debug_log_fname, level=logging.DEBUG)
def get_plugins(self, category=None):
"""Return the list of plugins
:param str category: restrict to plugins of a given category
:rtype: list(iPlugin)
:return: list of plugins
"""
if category:
return self.plugins.getPluginsOfCategory(category)
else:
return self.plugins.getAllPlugins()
def get_num_plugins(self, category=None):
""" Return the number of plugins available.
:param str category: restrict to plugins of given category
:rtype: int
:return: number of plugins
"""
plugins = self.get_plugins(category)
return len(plugins)
def get_plugin_dir(self, plugin):
""" Return the directory where the plugin is stored
:param iPlugin plugin: the plugin requested
:rtype: str
:return: the module name
"""
module_path = Path(plugin.details.get("Core", "module"))
return module_path.parent
def get_plugin_default_configuration_filename(self, plugin):
""" Return the path to the plugin default configuration filename
"""
try:
fname = plugin.details.get("Configuration", "Filename")
except: # noqa
return ""
fname = fname.replace('"', '')
path = self.get_plugin_dir(plugin)
return path / fname
def get_plugin_documentation_filename(self, plugin):
""" Return the path to the plugin documentation
"""
try:
fname = plugin.details.get("Documentation", "Filename")
except: # noqa
return ""
path = self.get_plugin_dir(plugin)
return path / fname
def get_plugin_class_name(self, plugin):
""" Return the class of a given plugin
:param iPlugin plugin: the plugin requested
:rtype: str
:return: the module classname
"""
return plugin.categories[0]
def get_plugin_module_name(self, plugin):
""" Return the module name of a given plugin
:param iPlugin plugin: the plugin requested
:rtype: str
:return: the module name
"""
module_path = plugin.details.get("Core", "module")
path, filename = os.path.split(module_path)
return filename
def get_plugin_config(self, plugin):
""" Return the configuration of a given plugin
:param iPlugin plugin: the plugin requested
:rtype: dict
:return: plugin configuration
"""
module_name = self.get_plugin_module_name(plugin)
class_name = self.get_plugin_class_name(plugin)
try:
config = self.plugins_config[class_name][module_name]
except: # noqa
config = {}
return config
def get_plugin_dependencies(self, plugin):
""" Return the dependency of a given plugin
:param iPlugin plugin: the plugin requested
:rtype: list
:return: list of plugins name the plugin depend on
"""
# No dependencies
if not plugin.details.has_option("Core", "Dependencies"):
return set()
dependencies = set()
st = plugin.details.get("Core", "Dependencies")
if "," in st:
elts = st.split(",")
for elt in elts:
dependencies.add(elt.strip())
else:
dependencies.add(st)
return dependencies
def is_plugin_enabled(self, plugin):
config = self.get_plugin_config(plugin)
if config.get('enable'):
return True
else:
return False
def get_plugins_info(self, category=None):
"""Return the list of plugins available with their type
:param str category: restrict to plugins of a given category.
:rtype: list(str)
:return: list of plugins name
"""
pl = []
if category:
categories = [category]
else:
categories = self.plugins.getCategories()
for cat in categories:
for plugin in self.plugins.getPluginsOfCategory(cat):
enabled = self.is_plugin_enabled(plugin)
module_name = self.get_plugin_module_name(plugin)
try:
version = plugin.version
except: # noqa
version = "NA"
s = [
cat, plugin.name, plugin.description, enabled, module_name,
version
]
pl.append(s)
return pl
def display_execution_results(self, results, site):
""" Display execution summary
"""
cprint("|-Execution result", "magenta")
table_data = [['name', 'ok', 'skipped', 'errors']]
count = 0
for result in results:
row = []
plugin_name, stats = result
# plugin name
c = "cyan"
if count % 2:
c = "blue"
row.append(colored(plugin_name, c))
# ok
if stats[site.OK]:
if stats[site.OK] == 1:
val = 'v'
else:
val = stats[site.OK]
row.append(colored(val, 'green'))
else:
row.append(' ')
# warning
if stats[site.SKIPPED]:
row.append(colored(stats[site.SKIPPED], "yellow"))
else:
row.append(' ')
# error
if stats[site.ERROR]:
if stats[site.ERROR] == 1:
val = 'x'
else:
val = stats[site.ERROR]
row.append(colored(val, "red"))
else:
row.append(' ')
table_data.append(row)
count += 1
print(SingleTable(table_data).table)
def run_plugins(self, items, plugin_class, unit, site):
"""Execute a set of plugins on a given list of items
:param list items: list of items to process
:param str plugin_type: the plugin_class to use
:param str unit: the unit to use in the display
:param SiteFab site: pointer to the site object to be passed
to the plugins
:rtype: dict(dict(list))
:return: plugins execution statistics
"""
# dependencies map
dependencie_map = {}
# used to get back from the module name to the plugin
module_name_to_plugin = {}
plugins = self.plugins.getPluginsOfCategory(plugin_class)
# collecting plugins that are to be executed.
for plugin in plugins:
if self.is_plugin_enabled(plugin):
module_name = self.get_plugin_module_name(plugin)
module_name_to_plugin[module_name] = plugin
# dependencies computation.
# Due to potential dependencies on plugins from previous stage
# this must be computed after collecting which
# plugins were executed.
for plugin in module_name_to_plugin.values():
all_dependencies = self.get_plugin_dependencies(plugin)
dependencies = set() # topological sort requires use of set
module_name = self.get_plugin_module_name(plugin)
for dep_module_name in all_dependencies:
if dep_module_name not in self.plugins_enabled:
utils.error("Plugin:%s can't be executed because\
plugin %s is not enable" %
(module_name, dep_module_name))
# only add to the dependencies map the plugins
# that are from the same stage
if dep_module_name in module_name_to_plugin:
dependencies.add(dep_module_name)
else:
# check if already executed
if dep_module_name not in self.plugins_executed:
utils.error("Plugin:%s can't be executed because\
plugin %s was not executed in previous\
stage" % (module_name, dep_module_name))
dependencie_map[module_name] = dependencies
# print dependencie_map
# Topological sorting
try:
plugins_to_process = toposort_flatten(dependencie_map)
except Exception as e:
utils.error("Circular dependencies between plugins.\
Can't execute plugins:%s" % e)
s = "|-%s plugins" % (unit.strip().capitalize())
desc = colored(s, "magenta")
results = []
for module_name in tqdm(plugins_to_process,
unit=' plugin',
desc=desc,
leave=True):
if module_name in module_name_to_plugin:
plugin = module_name_to_plugin[module_name]
else:
raise Exception("The following plugin module name listed in\
dependencies don't exist % s " % module_name)
pclass = plugin_class.lower()
filename = "%s.%s.html" % (pclass, module_name)
log_id = site.logger.create_log(pclass, plugin.name, filename)
plugin_results = utils.dict_to_objdict({
site.OK: 0,
site.SKIPPED: 0,
site.ERROR: 0
})
config = self.get_plugin_config(plugin)
for item in tqdm(items, unit=unit, desc=plugin.name, leave=False):
result = plugin.plugin_object.process(item, site, config)
plugin_results[result[0]] += 1
severity = result[0]
name = result[1]
details = result[2]
site.logger.record_event(log_id, name, severity, details)
self.plugins_executed[module_name] = True
results.append([plugin.name, plugin_results])
site.logger.write_log(log_id)
return results
def get_template_filters(self):
"""Load template filters and return a dictionary list
Return:
dict: jinja filter functions
"""
template_filters = {}
filters = self.plugins.getPluginsOfCategory("TemplateFilter")
for flt in filters:
filter_name = self.get_plugin_module_name(flt)
template_filters[filter_name] = flt.plugin_object.myfilter
return template_filters
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.timeline = []
self.pages = []
self._scanned = False
# This is the default config
# TODO: fill it
self.config = {
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'DEFAULT_LANG': "en",
'OUTPUT_FOLDER': 'output',
'FILES_FOLDERS': {
'files': ''
},
'LISTINGS_FOLDER': 'listings',
'ADD_THIS_BUTTONS': True,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_TEASERS': False,
'MAX_IMAGE_SIZE': 1280,
'USE_FILENAME_AS_TITLE': True,
'SLUG_TAG_PATH': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'FILTERS': {},
'USE_BUNDLES': True,
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'default',
'post_compilers': {
"rest": ['.txt', '.rst'],
"markdown": ['.md', '.mdown', '.markdown'],
"html": ['.html', '.htm'],
},
}
self.config.update(config)
self.config['TRANSLATIONS'] = self.config.get(
'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''})
# FIXME: find way to achieve this with the plugins
#if self.config['USE_BUNDLES'] and not webassets:
#self.config['USE_BUNDLES'] = False
self.GLOBAL_CONTEXT = self.config.get('GLOBAL_CONTEXT', {})
self.THEMES = utils.get_theme_chain(self.config['THEME'])
self.MESSAGES = utils.load_messages(self.THEMES,
self.config['TRANSLATIONS'])
self.GLOBAL_CONTEXT['messages'] = self.MESSAGES
self.GLOBAL_CONTEXT['_link'] = self.link
self.GLOBAL_CONTEXT['rel_link'] = self.rel_link
self.GLOBAL_CONTEXT['abs_link'] = self.abs_link
self.GLOBAL_CONTEXT['exists'] = self.file_exists
self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[
'ADD_THIS_BUTTONS']
self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
})
self.plugin_manager.setPluginInfoExtension('plugin')
self.plugin_manager.setPluginPlaces([
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
])
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
pluginInfo.plugin_object.short_help = pluginInfo.description
self.commands[pluginInfo.name] = pluginInfo.plugin_object
# Activate all task plugins
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
self.plugin_manager.activatePluginByName(pluginInfo.name)
pluginInfo.plugin_object.set_site(self)
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name,
"TemplateSystem")
if pi is None:
sys.stderr.write("Error loading %s template system plugin\n" %
template_sys_name)
sys.exit(1)
self.template_system = pi.plugin_object
self.template_system.set_directories([
os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES
])
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for pluginInfo in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[pluginInfo.name] = \
pluginInfo.plugin_object.compile_html
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.post_compilers`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [
lang for lang, exts in self.config['post_compilers'].items()
if ext in exts
]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'post_compilers' in conf.py\n(The error is in"
"one of %s)" % ', '.join(langs))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("post_compilers in conf.py does not tell me how to "
"handle '%s' extensions." % ext)
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
data = self.template_system.render_template(template_name, None,
context,
self.GLOBAL_CONTEXT)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# This is to support windows paths
url_part = "/".join(url_part.split(os.sep))
src = urlparse.urljoin(self.config["BLOG_URL"], url_part)
parsed_src = urlparse.urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse.urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = '<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "w+") as post_file:
post_file.write(data)
def path(self, kind, name, lang, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
path = []
if kind == "tag_index":
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
'index.html'
])
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".html"
])
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".xml"
])
elif kind == "index":
if name > 0:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'],
'index-%s.html' % name
])
else:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], 'index.html'
])
elif kind == "rss":
path = filter(None, [
self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'],
'rss.xml'
])
elif kind == "archive":
if name:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'], name, 'index.html'
])
else:
path = filter(None, [
self.config['TRANSLATIONS'][lang],
self.config['ARCHIVE_PATH'],
self.config['ARCHIVE_FILENAME']
])
elif kind == "gallery":
path = filter(None,
[self.config['GALLERY_PATH'], name, 'index.html'])
elif kind == "listing":
path = filter(None,
[self.config['LISTINGS_FOLDER'], name + '.html'])
if is_link:
return '/' + ('/'.join(path))
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urlparse.urljoin(self.config['BLOG_URL'], dst)
return urlparse.urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urlparse.urljoin(self.config['BLOG_URL'], src)
dst = urlparse.urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlparse.urlsplit(src)
parsed_dst = urlparse.urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self):
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"):
for task in pluginInfo.plugin_object.gen_tasks():
yield task
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': 'all',
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if not self._scanned:
print "Scanning posts ",
targets = set([])
for wildcard, destination, _, use_in_feeds in \
self.config['post_pages']:
print ".",
for base_path in glob.glob(wildcard):
post = Post(base_path, destination, use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BLOG_URL'], self.MESSAGES)
for lang, langpath in self.config['TRANSLATIONS'].items():
dest = (destination, langpath, post.pagenames[lang])
if dest in targets:
raise Exception(
'Duplicated output path %r in post %r' %
(post.pagenames[lang], base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(
post.post_name)
for tag in post.tags:
self.posts_per_tag[tag].append(post.post_name)
else:
self.pages.append(post)
for name, post in self.global_data.items():
self.timeline.append(post)
self.timeline.sort(cmp=lambda a, b: cmp(a.date, b.date))
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print "done!"
def generic_page_renderer(self, lang, wildcard, template_name, destination,
filters):
"""Render post fragments to final HTML pages."""
for post in glob.glob(wildcard):
post_name = os.path.splitext(post)[0]
context = {}
post = self.global_data[post_name]
deps = post.deps(lang) + \
self.template_system.template_deps(template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
self.config['TRANSLATIONS'][lang],
destination,
post.pagenames[lang] + ".html")
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
task = {
'name':
output_name.encode('utf-8'),
'file_dep':
deps,
'targets': [output_name],
'actions':
[(self.render_template, [template_name, output_name,
context])],
'clean':
True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.titles[lang], p.permalink(lang))
for p in posts]
task = {
'name':
output_name.encode('utf8'),
'targets': [output_name],
'file_dep':
deps,
'actions':
[(self.render_template, [template_name, output_name, context])],
'clean':
True,
'uptodate': [config_changed(deps_context)]
}
yield utils.apply_filters(task, filters)
class DeepDecoder:
def __init__(self):
self.top_layer = None
self.plugin_manager = PluginManager()
self.plugin_locations = [path.join(expanduser("~"), '.phorcys', 'plugins')]
try:
environ_locations = os.environ['PLUGINS_DIR']
if environ_locations is not None and len(environ_locations) > 0:
dirs = environ_locations.split(';')
for d in dirs:
if os.path.isdir(d):
self.plugin_locations.append(d)
else:
pass
except KeyError:
pass
self.plugin_manager.setPluginPlaces(directories_list=self.plugin_locations)
self.plugin_manager.setCategoriesFilter(DecoderPlugin.filter())
self.plugin_manager.collectPlugins()
def get_loaded_plugins(self):
return self.plugin_manager.getPluginsOfCategory(DecoderPlugin.category())
def _complete_leaves(self):
for leaf in self.top_layer.leaves:
if leaf.parent is not None and leaf.parent.name == 'base64' and len(leaf.name) == 0:
leaf.parent.name = 'text'
leaf.parent.human_readable = True
leaf.parent.raw_data = ''.join(leaf.parent.lines)
leaf.parent.del_extracted_layer(leaf)
continue
if len(leaf.lines) == 0:
try:
leaf.lines = utils.to_hex_lines(bytes(leaf.raw_data))
leaf.name = 'raw'
except Exception as e:
pass
def decode(self, data, **kwargs) -> Layer:
parent = Layer()
parent.raw_data = data
parent.name = 'root'
to_visit = [parent]
while len(to_visit) != 0:
next = to_visit.pop()
layer = self.go_deeper(next, **kwargs)
if layer is not None:
layers = layer.extracted_layers
if layers is not None:
to_visit.extend(layers)
self.top_layer = parent
self._complete_leaves()
return parent
def inspect(self):
pass
def go_deeper(self, parent, **xargs) -> Optional[Layer]:
protocols = [
HttpHeaders(),
UrlEncoded(),
Json(),
Protobuf(),
Base64(),
Lzma(),
Zlib(),
Bzip(),
Gzip(),
Text(),
Html(),
Css(),
]
plugins = []
for plugin in self.plugin_manager.getPluginsOfCategory(DecoderPlugin.category()):
plugins.append(plugin.plugin_object)
plugins.extend(protocols)
protocols = plugins
for p in protocols:
try:
return p(parent, **xargs)
except Exception as e:
# print(traceback.format_exc())
# print('Not %s' % type(p))
# print(e)
pass
return None
class Statick(object):
"""Code analysis front-end."""
def __init__(self, user_paths):
"""Initialize Statick."""
self.resources = Resources(user_paths)
self.manager = PluginManager()
self.manager.setPluginPlaces(self.resources.get_plugin_paths())
self.manager.setCategoriesFilter({
"Discovery": DiscoveryPlugin,
"Tool": ToolPlugin,
"Reporting": ReportingPlugin
})
self.manager.collectPlugins()
self.discovery_plugins = {}
for plugin_info in self.manager.getPluginsOfCategory("Discovery"):
self.discovery_plugins[plugin_info.plugin_object.get_name()] = \
plugin_info.plugin_object
self.tool_plugins = {}
for plugin_info in self.manager.getPluginsOfCategory("Tool"):
self.tool_plugins[plugin_info.plugin_object.get_name()] = \
plugin_info.plugin_object
self.reporting_plugins = {}
for plugin_info in self.manager.getPluginsOfCategory("Reporting"):
self.reporting_plugins[plugin_info.plugin_object.get_name()] = \
plugin_info.plugin_object
self.config = []
self.exceptions = []
def get_config(self, args):
"""Get Statick configuration."""
config_filename = "config.yaml"
if args.config is not None:
config_filename = args.config
self.config = Config(self.resources.get_file(config_filename))
def get_exceptions(self, args):
"""Get Statick exceptions."""
exceptions_filename = "exceptions.yaml"
if args.exceptions is not None:
exceptions_filename = args.exceptions
self.exceptions = Exceptions(self.resources.get_file(exceptions_filename))
def get_ignore_packages(self):
"""Get packages to ignore during scan process."""
return self.exceptions.get_ignore_packages()
def gather_args(self, args):
"""Gather arguments."""
args.add_argument("--output-directory", dest="output_directory",
type=str, help="Directory to write output files to")
args.add_argument("--show-tool-output", dest="show_tool_output",
action="store_true", help="Show tool output")
args.add_argument("--config", dest="config",
type=str, help="Name of config yaml file")
args.add_argument("--profile", dest="profile",
type=str, help="Name of profile yaml file")
args.add_argument("--exceptions", dest="exceptions",
type=str, help="Name of exceptions yaml file")
args.add_argument("--force-tool-list", dest="force_tool_list",
type=str, help="Force only the given list of tools to run")
args.add_argument('--version', action='version',
version='%(prog)s {version}'.format(version=__version__))
args.add_argument('--mapping-file-suffix', dest="mapping_file_suffix",
type=str, help="Suffix to use when searching for CERT mapping files")
for _, plugin in list(self.discovery_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.tool_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.reporting_plugins.items()):
plugin.gather_args(args)
def get_level(self, path, args):
"""Get level to scan package at."""
path = os.path.abspath(path)
profile_filename = "profile.yaml"
if args.profile is not None:
profile_filename = args.profile
profile_resource = self.resources.get_file(profile_filename)
if profile_resource is None:
print("Could not find profile file {}!".format(profile_filename))
return None
try:
profile = Profile(profile_resource)
except OSError as ex:
# This isn't quite redundant with the profile_resource check: it's possible
# that something else triggers an OSError, like permissions
print("Failed to access profile file {}: {}".format(profile_filename, ex))
return None
except ValueError as ex:
print("Profile file {} has errors: {}".format(profile_filename, ex))
return None
package = Package(os.path.basename(path), path)
level = profile.get_package_level(package)
return level
def run(self, path, args): # pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements
"""Run scan tools against targets on path."""
success = True
path = os.path.abspath(path)
if not os.path.exists(path):
print("No package found at {}!".format(path))
return None, False
package = Package(os.path.basename(path), path)
level = self.get_level(path, args)
if not self.config or not self.config.has_level(level):
print("Can't find specified level {} in config!".format(level))
return None, False
orig_path = os.getcwd()
if args.output_directory:
if not os.path.isdir(args.output_directory):
print("Output directory not found at {}!".
format(args.output_directory))
return None, False
output_dir = os.path.join(args.output_directory,
package.name + "-" + level)
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
if not os.path.isdir(output_dir):
print("Unable to create output directory at {}!".format(
output_dir))
return None, False
print("Writing output to: {}".format(output_dir))
os.chdir(output_dir)
print("------")
print("Scanning package {} ({}) at level {}".format(package.name,
package.path,
level))
issues = {}
ignore_packages = self.get_ignore_packages()
if package.name in ignore_packages:
print("Package {} is configured to be ignored by Statick.".format(package.name))
return issues, True
plugin_context = PluginContext(args, self.resources, self.config)
print("---Discovery---")
if not DiscoveryPlugin.file_command_exists():
print("file command isn't available, discovery plugins will be less effective")
discovery_plugins = self.config.get_enabled_discovery_plugins(level)
if not discovery_plugins:
discovery_plugins = list(self.discovery_plugins.keys())
for plugin_name in discovery_plugins:
if plugin_name not in self.discovery_plugins:
print("Can't find specified discovery plugin {}!".format(plugin_name))
return None, False
plugin = self.discovery_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
print("Running {} discovery plugin...".format(plugin.get_name()))
plugin.scan(package, level, self.exceptions)
print("{} discovery plugin done.".format(plugin.get_name()))
print("---Discovery---")
print("---Tools---")
enabled_plugins = self.config.get_enabled_tool_plugins(level)
plugins_to_run = copy.copy(enabled_plugins)
plugins_ran = []
plugin_dependencies = []
while plugins_to_run:
plugin_name = plugins_to_run[0]
if plugin_name not in self.tool_plugins:
print("Can't find specified tool plugin {}!".format(plugin_name))
return None, False
if args.force_tool_list is not None:
force_tool_list = args.force_tool_list.split(",")
if plugin_name not in force_tool_list and plugin_name not in plugin_dependencies:
print("Skipping plugin not in force list {}!".format(plugin_name))
plugins_to_run.remove(plugin_name)
continue
plugin = self.tool_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
dependencies = plugin.get_tool_dependencies()
dependencies_met = True
for dependency_name in dependencies:
if dependency_name not in plugins_ran:
if dependency_name not in enabled_plugins:
print("Plugin {} depends on plugin {} which isn't "
"enabled!".format(plugin_name, dependency_name))
return None, False
plugin_dependencies.append(dependency_name)
plugins_to_run.remove(dependency_name)
plugins_to_run.insert(0, dependency_name)
dependencies_met = False
if not dependencies_met:
continue
print("Running {} tool plugin...".format(plugin.get_name()))
tool_issues = plugin.scan(package, level)
if tool_issues is not None:
issues[plugin_name] = tool_issues
print("{} tool plugin done.".format(plugin.get_name()))
else:
print("{} tool plugin failed".format(plugin.get_name()))
success = False
plugins_to_run.remove(plugin_name)
plugins_ran.append(plugin_name)
print("---Tools---")
issues = self.exceptions.filter_issues(package, issues)
os.chdir(orig_path)
print("---Reporting---")
reporting_plugins = self.config.get_enabled_reporting_plugins(level)
if not reporting_plugins:
reporting_plugins = self.reporting_plugins.keys()
for plugin_name in reporting_plugins:
if plugin_name not in self.reporting_plugins.keys():
print("Can't find specified reporting plugin {}!".format(plugin_name))
return None, False
plugin = self.reporting_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
print("Running {} reporting plugin...".format(plugin.get_name()))
plugin.report(package, issues, level)
print("{} reporting plugin done.".format(plugin.get_name()))
print("---Reporting---")
print("Done!")
return issues, success
from .ClusterProcessAlgorithmsClass import *
from .ConnectionClasses import MySQLDBConnectionOpts, SOLRCollection
from yapsy.PluginManager import PluginManager
from .PluginClasses import *
from .ResultClasses import *
from .compute_similarity import similarity_columns
# Url, Title, Fulltext, Plaintext
sim_weights = [0.05, 0.25, 0.2, 0.5]
# Initialize plugin manager and plugins
plugin_manager = PluginManager()
cur_dir = os.path.dirname(os.path.abspath(__file__))
plugin_manager.setPluginPlaces([
os.path.join(cur_dir, "cluster_plugins"),
os.path.join(cur_dir, "summary_plugins"),
os.path.join(cur_dir, "rank_plugins")
])
plugin_manager.setCategoriesFilter({
"cluster": IClusterPlugin,
"summary": ISummaryPlugin,
"rank": IRankPlugin,
})
plugin_manager.collectPlugins()
# Cosine similarity to cosine distance
def cos_sim_to_dist(sim_score):
return 1.0 - sim_score
from yapsy.PluginManager import PluginManager
import json, sys
# Load the plugins from the plugin directory.
manager = PluginManager()
manager.setPluginPlaces(["plugins"])
manager.collectPlugins()
def main(file):
# Loop round the plugins and print their names.
message = json.loads(open(file).read())
plugin = manager.getPluginByName(message['type'])
if plugin:
plugin.plugin_object.print_name()
plugin.plugin_object.retry(**message['target'], message = message.get('message'))
if __name__ == "__main__":
if len(sys.argv) > 1:
main(sys.argv[1])
long_name = all_props[name]["displaytext"]
else:
long_name = name
return long_name
# Helper class used to pass the necessary context information to the export plugin
PluginExportContext = namedtuple("PluginExportContext", [
"objectFeaturesSlot", "labelImageSlot", "rawImageSlot",
"additionalPluginArgumentsSlot"
])
###############
# the manager #
###############
pluginManager = PluginManager()
pluginManager.setPluginPlaces(plugin_paths)
pluginManager.setCategoriesFilter({
"ObjectFeatures":
ObjectFeaturesPlugin,
"TrackingExportFormats":
TrackingExportFormatPlugin
})
pluginManager.collectPlugins()
for pluginInfo in pluginManager.getAllPlugins():
pluginManager.activatePluginByName(pluginInfo.name)
class EfetchPluginManager(object):
"""This class manages and creates plugin objects"""
def __init__(self, plugins_file, curr_directory):
# Plugin Manager Setup
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces([curr_directory + u'/plugins/'])
self.plugins_file = plugins_file
self.reload_plugins()
def reload_plugins_file(self):
"""Reloads all plugins from the YAML file"""
self.config_file_plugins = self.load_plugin_config(self.plugins_file)
def reload_plugins(self):
"""Reloads all Yapsy and YAML file plugins"""
self.plugin_manager.collectPlugins()
for plugin in self.plugin_manager.getAllPlugins():
self.plugin_manager.activatePluginByName(plugin.name)
self.reload_plugins_file()
def load_plugin_config(self, plugins_file):
"""Loads the plugin config file"""
if not os.path.isfile(plugins_file):
logging.warn(u'Could not find Plugin Configuration File "' +
plugins_file + u'"')
return {}
with open(plugins_file, 'r') as stream:
try:
return yaml.load(stream)
except yaml.YAMLError as exc:
logging.error(u'Failed to parse Plugin Configuration File')
logging.error(exc)
return {}
def get_all_plugins(self):
"""Gets a list of all the plugins"""
plugins = []
for plugin in self.plugin_manager.getAllPlugins():
plugins.append(plugin.name)
for key in self.config_file_plugins:
plugins.append(key)
return plugins
def get_plugin_by_name(self, name):
"""Gets an Efetch plugin by name"""
plugin = self.plugin_manager.getPluginByName(str(name).lower())
if not plugin and name not in self.config_file_plugins:
logging.warn(u'Request made for unknown plugin "' + name + u'"')
elif not plugin:
plugin = self.config_file_plugins[name]
return Plugin(plugin.get('name', 'None'),
plugin.get('description', 'None'),
plugin.get('cache',
True), plugin.get('popularity', 5),
plugin.get('fast', False),
plugin.get('store', False),
map(str.lower, plugin.get('mimetypes', [])),
map(str.lower, plugin.get('extensions', [])),
map(str.lower, plugin.get('os', [])),
plugin.get('command', False),
plugin.get('format', 'Text'),
plugin.get('file', False),
plugin.get('openwith', False),
plugin.get('icon', 'fa-file-o'))
else:
return plugin.plugin_object
class Atomicapp():
debug = False
dryrun = False
atomicfile_data = None
params_data = None
answers_data = {GLOBAL_CONF: {}}
tmpdir = None
answers_file = None
provider = DEFAULT_PROVIDER
installed = False
plugins = None
recursive = True
update = False
app_path = None
target_path = None
app_id = None
app = None
def __init__(self,
answers,
app,
recursive=True,
update=False,
target_path=None,
dryrun=False,
debug=False):
run_path = os.path.dirname(os.path.realpath(__file__))
self.debug = debug
self.dryrun = dryrun
self.recursive = isTrue(recursive)
self.update = isTrue(update)
self.target_path = target_path
logger.info("Path for %s is %s" % (app, target_path))
if os.path.exists(app):
self.app_path = app
if not os.path.basename(app) == ATOMIC_FILE:
app = os.path.join(app, ATOMIC_FILE)
atomic_data = self._loadAtomicfile(app)
app = os.environ[
"IMAGE"] if "IMAGE" in os.environ else atomic_data["id"]
self.app_id = atomic_data["id"]
print("Setting path to %s" % self.app_path)
if not self.target_path:
if self.app_path:
self.target_path = self.app_path
else:
self.target_path = os.getcwd()
self.tmpdir = tempfile.mkdtemp(prefix="appent-%s" %
self._getComponentName(app))
logger.debug("Temporary dir: %s" % self.tmpdir)
self.app = app
self.answers_file = answers
self.plugins = PluginManager()
self.plugins.setPluginPlaces([os.path.join(run_path, "providers")])
self.plugins.collectPlugins()
def _sanitizeName(self, app):
return app.replace("/", "-")
def _mergeConfig(self):
config = self.params_data
if self.answers_data:
if config:
config = update(config, self.answers_data)
else:
config = self.answers_data
return config
def _mergeConfigComponent(self, component):
config = self._mergeConfig()
component_config = config[GLOBAL_CONF] if GLOBAL_CONF in config else {}
if component in config:
component_config = update(component_config, config[component])
return component_config
#LOAD FUNCTIONS
def _loadAtomicfile(self, path=None):
print(os.path.isfile(path))
if not os.path.exists(path):
print("Path: %s" % path)
logger.error("Atomicfile not found: %s" % path)
sys.exit(1)
with open(path, "r") as fp:
self.atomicfile_data = json.load(fp)
self.app_id = self.atomicfile_data["id"]
pprint(self.atomicfile_data)
return self.atomicfile_data
def _loadParams(self, path=None):
logger.debug("Loading %s" % path)
if not os.path.exists(path):
return None
config = ConfigParser.ConfigParser()
config.optionxform = str
data = {}
with open(path, "r") as fp:
config.readfp(fp)
for section in config.sections():
data[section] = dict(config.items(section))
if self.params_data:
self.params_data.update(data)
else:
self.params_data = data
return self.params_data
def _loadAnswers(self, path=None):
if not os.path.exists(path):
return None
config = ConfigParser.ConfigParser()
config.optionxform = str
data = {}
with open(path, "r") as fp:
config.readfp(fp)
for section in config.sections():
data[section] = dict(config.items(section))
self.answers_data = data
return self.answers_data
#GET_FUNCTIONS
def _getComponentDir(self, component):
return os.path.join(self.target_path, GRAPH_DIR,
self._getComponentName(component))
def _getProviderDir(self, component):
#FIXME add provider resolution by answers file
return os.path.join(self.target_path, GRAPH_DIR, component,
self.provider)
def _getComponentConf(self, component):
return os.path.join(self._getComponentDir(component), self.provider,
PARAMS_FILE)
def _getTmpAppDir(self):
return os.path.join(self.tmpdir, APP_ENT_PATH)
def _getGraphDir(self):
return os.path.join(self.target_path, GRAPH_DIR)
def _getComponentName(self, graph_item):
if type(graph_item) is str or type(graph_item) is unicode:
return os.path.basename(graph_item).split(":")[0]
elif type(graph_item) is dict:
return graph_item["name"].split(":")[0]
else:
return None
def _getComponentImageName(self, graph_item):
if type(graph_item) is str or type(graph_item) is unicode:
return graph_item
elif type(graph_item) is dict:
print(graph_item)
repo = ""
if "repository" in graph_item:
repo = graph_item["repository"]
print(repo)
return os.path.join(repo, graph_item["name"])
else:
return None
def _dispatchGraph(self):
if not "graph" in self.atomicfile_data:
raise Exception("Graph not specified in %s" % ATOMIC_FILE)
if not os.path.isdir(self._getGraphDir()):
raise Exception("Couldn't find %s directory" % GRAPH_DIR)
for graph_item in self.atomicfile_data["graph"]:
component = self._getComponentName(graph_item)
component_path = self._getComponentDir(component)
component_params = self._getComponentConf(component)
if os.path.isfile(component_params):
self._loadParams(component_params)
self._processComponent(component)
def _applyTemplate(self, data, component):
template = Template(data)
config = self._mergeConfigComponent(component)
return template.substitute(config)
def _getProvider(self):
for provider in self.plugins.getAllPlugins():
module_path = provider.details.get("Core", "Module")
if os.path.basename(module_path) == self.provider:
return provider.plugin_object
def _processComponent(self, component):
path = self._getProviderDir(component)
data = None
for artifact in os.listdir(path):
if artifact == PARAMS_FILE:
continue
with open(os.path.join(path, artifact), "r") as fp:
data = fp.read()
logger.debug("Templating artifact %s/%s" % (path, artifact))
data = self._applyTemplate(data, component)
dst_dir = os.path.join(self.tmpdir, component)
artifact_dst = os.path.join(dst_dir, artifact)
if not os.path.isdir(dst_dir):
os.makedirs(dst_dir)
with open(artifact_dst, "w") as fp:
fp.write(data)
provider = self._getProvider()
provider.init(self._mergeConfigComponent(component),
os.path.join(self.tmpdir, component), self.debug,
self.dryrun)
provider.deploy()
def _getImageURI(self, image):
config = self._mergeConfig()
if config and GLOBAL_CONF in config and "registry" in config[
GLOBAL_CONF]:
print("Adding registry %s for %s" %
(config[GLOBAL_CONF]["registry"], image))
image = os.path.join(config[GLOBAL_CONF]["registry"], image)
return image
def _pullApp(self, image):
image = self._getImageURI(image)
pull = ["docker", "pull", image]
if subprocess.call(pull) != 0:
print("Couldn't pull %s" % image)
sys.exit(1)
def _copyFromContainer(self, image):
image = self._getImageURI(image)
name = self._getComponentName(image)
create = ["docker", "create", "--name", name, image, "nop"]
subprocess.call(create)
cp = ["docker", "cp", "%s:/%s" % (name, APP_ENT_PATH), self.tmpdir]
if not subprocess.call(cp):
logger.debug("Application entity data copied to %s" % self.tmpdir)
rm = ["docker", "rm", name]
subprocess.call(rm)
def _populateMainApp(self, src=None, dst=None):
print("Copying app %s" % self._getComponentName(self.app))
if not src:
src = os.path.join(self.tmpdir, APP_ENT_PATH)
if not dst:
dst = self.target_path
distutils.dir_util.copy_tree(src, dst, update=(not self.update))
def _populateModule(self):
data_list = ["graph/%s/" % self.app_id, "Atomicfile", "params.conf"]
logger.info("Populating module %s" % self._getComponentName(self.app))
for item in data_list:
path = os.path.join(self.tmpdir, APP_ENT_PATH, item)
if os.path.isdir(path):
logger.debug("%s/%s/%s" %
(self.target_path, GRAPH_DIR, self.app_id))
distutils.dir_util.copy_tree(
path, self._getComponentDir(self.app_id))
else:
logger.debug("copy item %s > %s > %s > %s" %
(self.target_path, GRAPH_DIR, self.app_id, item))
distutils.file_util.copy_file(
path, os.path.join(self._getComponentDir(self.app_id),
item))
def run(self, level=AtomicappLevel.Main):
if not self.installed:
self.install(level)
if not self._loadAtomicfile(os.path.join(self.target_path,
ATOMIC_FILE)):
print("Failed to load %s" % ATOMIC_FILE)
return
if self.debug:
print(self.atomicfile_data)
if not self._loadParams(os.path.join(self.target_path, PARAMS_FILE)):
logger.error("Failed to load %s" % PARAMS_FILE)
return
else:
logger.debug("Loaded params: %s" % self.params_data)
config = self._mergeConfig()
if "provider" in config[GLOBAL_CONF]:
self.provider = config[GLOBAL_CONF]["provider"]
self._dispatchGraph()
def install(self, level=AtomicappLevel.Main):
if not self._loadAnswers(self.answers_file):
print("No %s file found, using defaults" % ANSWERS_FILE)
if self.app_path and not self.target_path == self.app_path:
logger.info("Copying content of directory %s to %s" %
(self.app_path, self.target_path))
self._populateMainApp(src=self.app_path)
atomicfile_path = os.path.join(self.target_path, ATOMIC_FILE)
if level == AtomicappLevel.Module:
atomicfile_path = os.path.join(self._getComponentDir(self.app),
ATOMIC_FILE)
logger.debug(
"Test: %s -> %s" %
(self.app,
(not self.app_path
and not os.path.exists(self._getComponentDir(self.app)))))
if not self.app_path and (self.update or not os.path.exists(
self._getComponentDir(self.app))):
self._pullApp(self.app)
self._copyFromContainer(self.app)
atomicfile_path = os.path.join(self._getTmpAppDir(), ATOMIC_FILE)
logger.debug("Atomicfile path for pulled image: %s" %
atomicfile_path)
self._loadAtomicfile(atomicfile_path)
logger.debug("App ID: %s" % self.app_id)
if level == AtomicappLevel.Main:
self._populateMainApp()
elif level == AtomicappLevel.Module:
self._populateModule()
else:
logger.info("Component data exist in %s, skipping population..." %
self._getComponentDir(self.app))
if not self.atomicfile_data:
self._loadAtomicfile(atomicfile_path)
if self.recursive:
self._installDependencies()
self.installed = True
return self.app_id
def _installDependencies(self):
for graph_item in self.atomicfile_data["graph"]:
component = self._getComponentName(graph_item)
component_path = self._getComponentDir(component)
logger.debug("Component path: %s" % component_path)
logger.debug("%s == %s -> %s" %
(component, self.app_id, component == self.app_id))
if not component == self.app_id and not os.path.isdir(
component_path) and (not self.app_path or self.update):
image_name = self._getComponentImageName(graph_item)
print("Pulling %s" % image_name)
component_atomicapp = Atomicapp(self.answers_file, image_name,
self.recursive, self.update,
self.target_path, self.dryrun,
self.debug)
component = component_atomicapp.install(AtomicappLevel.Module)
component_path = self._getComponentDir(component)
logger.info("Component installed into %s" % component_path)
disabledChecks = {
# "SemiemptyFields" : {"bbmri-eric:ID:NO_HUNT", "bbmri-eric:ID:NO_Janus"}
}
pp = pprint.PrettyPrinter(indent=4)
class ExtendAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest) or []
items.extend(values)
setattr(namespace, self.dest, items)
simplePluginManager = PluginManager()
simplePluginManager.setPluginPlaces(["checks"])
simplePluginManager.collectPlugins()
pluginList = []
for pluginInfo in simplePluginManager.getAllPlugins():
pluginList.append(os.path.basename(pluginInfo.path))
remoteCheckList = ['emails', 'geocoding', 'URLs']
cachesList = ['directory', 'emails', 'geocoding', 'URLs']
parser = argparse.ArgumentParser()
parser.register('action', 'extend', ExtendAction)
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='store_true',
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import unicode_literals
import logging
import os
from yapsy.PluginManager import PluginManager
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('yapsy').setLevel(logging.DEBUG)
# Yapsy Framework to load a plugin
# Load the plugins from the plugin directory.
manager = PluginManager()
# Location of consul plugin
here = os.path.dirname(__file__)
relative_path_of_plugin = os.path.join(here, 'storage_plugin')
manager.setPluginPlaces([relative_path_of_plugin])
manager.collectPlugins()
# Loop around the plugins and return the first plugin object.
for storage_plugin in manager.getAllPlugins():
plugin = storage_plugin.plugin_object
from yapsy.PluginManager import PluginManager
# Build the manager
simplePluginManager = PluginManager()
# Tell it the default place(s) where to find plugins
simplePluginManager.setPluginPlaces(["modules"])
# Load all plugins
simplePluginManager.collectPlugins()
def pluginsByPort():
global plugins
global tests_by_port
tests = {}
for i in plugins:
tests = i.plugin_object.get_test_list()
for t in tests:
p = t.get_port()
if p in tests:
tests[p].append(t)
tests_by_port = tests
def reloadPlugins():
global plugins
simplePluginManager = PluginManager()
simplePluginManager.setPluginPlaces(["modules"])
class Statick:
"""Code analysis front-end."""
def __init__(self, user_paths: List[str]) -> None:
"""Initialize Statick."""
self.resources = Resources(user_paths)
self.manager = PluginManager()
self.manager.setPluginPlaces(self.resources.get_plugin_paths())
self.manager.setCategoriesFilter(
{
"Discovery": DiscoveryPlugin,
"Tool": ToolPlugin,
"Reporting": ReportingPlugin,
}
)
self.manager.collectPlugins()
self.discovery_plugins: Dict[str, Any] = {}
for plugin_info in self.manager.getPluginsOfCategory("Discovery"):
self.discovery_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
self.tool_plugins: Dict[str, Any] = {}
for plugin_info in self.manager.getPluginsOfCategory("Tool"):
self.tool_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
self.reporting_plugins: Dict[str, Any] = {}
for plugin_info in self.manager.getPluginsOfCategory("Reporting"):
self.reporting_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
self.config: Optional[Config] = None
self.exceptions: Optional[Exceptions] = None
@staticmethod
def set_logging_level(args: argparse.Namespace) -> None:
"""Set the logging level to use for output.
Valid levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL. Specifying the level is
case-insensitive (both upper-case and lower-case are allowed).
"""
valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
log_level = args.log_level.upper()
if log_level not in valid_levels:
log_level = "WARNING"
args.log_level = log_level
logging.basicConfig(level=log_level)
logging.root.setLevel(log_level)
logging.info("Log level set to %s", args.log_level.upper())
def get_config(self, args: argparse.Namespace) -> None:
"""Get Statick configuration."""
base_config_filename = "config.yaml"
user_config_filename = ""
if args.config is not None:
user_config_filename = args.config
try:
self.config = Config(
self.resources.get_file(base_config_filename),
self.resources.get_file(user_config_filename),
)
except OSError as ex:
logging.error(
"Failed to access configuration file %s or %s: %s",
base_config_filename,
user_config_filename,
ex,
)
except ValueError as ex:
logging.error(
"Configuration file %s or %s has errors: %s",
base_config_filename,
user_config_filename,
ex,
)
def get_exceptions(self, args: argparse.Namespace) -> None:
"""Get Statick exceptions."""
exceptions_filename = "exceptions.yaml"
if args.exceptions is not None:
exceptions_filename = args.exceptions
try:
self.exceptions = Exceptions(self.resources.get_file(exceptions_filename))
except OSError as ex:
logging.error(
"Failed to access exceptions file %s: %s", exceptions_filename, ex
)
except ValueError as ex:
logging.error("Exceptions file %s has errors: %s", exceptions_filename, ex)
def get_ignore_packages(self) -> List[str]:
"""Get packages to ignore during scan process."""
if self.exceptions is None:
return []
return self.exceptions.get_ignore_packages()
def gather_args(self, args: argparse.ArgumentParser) -> None:
"""Gather arguments."""
args.add_argument(
"--output-directory",
dest="output_directory",
type=str,
help="Directory to write output files to",
)
args.add_argument(
"--log",
dest="log_level",
type=str,
default="WARNING",
help="Verbosity level of output to show (DEBUG, INFO, WARNING, ERROR"
", CRITICAL)",
)
args.add_argument(
"--check",
dest="check",
action="store_true",
help="Return the status. Return code 0 means there were no issues. \
Return code 1 means there were issues.",
)
args.add_argument(
"--config", dest="config", type=str, help="Name of config yaml file"
)
args.add_argument(
"--profile", dest="profile", type=str, help="Name of profile yaml file"
)
args.add_argument(
"--exceptions",
dest="exceptions",
type=str,
help="Name of exceptions yaml file",
)
args.add_argument(
"--force-tool-list",
dest="force_tool_list",
type=str,
help="Force only the given list of tools to run",
)
args.add_argument(
"--version",
action="version",
version=f"%(prog)s {__version__}",
)
args.add_argument(
"--mapping-file-suffix",
dest="mapping_file_suffix",
type=str,
help="Suffix to use when searching for CERT mapping files",
)
# statick workspace arguments
args.add_argument(
"-ws",
dest="workspace",
action="store_true",
help="Treat the path argument as a workspace of multiple packages",
)
args.add_argument(
"--max-procs",
dest="max_procs",
type=int,
default=int(multiprocessing.cpu_count() / 2),
help="Maximum number of CPU cores to use, only used when running on a"
"workspace",
)
args.add_argument(
"--packages-file",
dest="packages_file",
type=str,
help="File listing packages to scan, only used when running on a workspace",
)
args.add_argument(
"--list-packages",
dest="list_packages",
action="store_true",
help="List packages and levels, only used when running on a workspace",
)
for _, plugin in list(self.discovery_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.tool_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.reporting_plugins.items()):
plugin.gather_args(args)
def get_level(self, path: str, args: argparse.Namespace) -> Optional[str]:
"""Get level to scan package at."""
path = os.path.abspath(path)
profile_filename = "profile.yaml"
if args.profile is not None:
profile_filename = args.profile
profile_resource = self.resources.get_file(profile_filename)
if profile_resource is None:
logging.error("Could not find profile file %s!", profile_filename)
return None
try:
profile = Profile(profile_resource)
except OSError as ex:
# This isn't quite redundant with the profile_resource check: it's possible
# that something else triggers an OSError, like permissions.
logging.error("Failed to access profile file %s: %s", profile_filename, ex)
return None
except ValueError as ex:
logging.error("Profile file %s has errors: %s", profile_filename, ex)
return None
package = Package(os.path.basename(path), path)
level = profile.get_package_level(package)
return level
# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches
# pylint: disable=too-many-statements
def run(
self, path: str, args: argparse.Namespace
) -> Tuple[Optional[Dict[str, List[Issue]]], bool]:
"""Run scan tools against targets on path."""
success = True
path = os.path.abspath(path)
if not os.path.exists(path):
logging.error("No package found at %s!", path)
return None, False
package = Package(os.path.basename(path), path)
level: Optional[str] = self.get_level(path, args)
logging.info("level: %s", level)
if level is None:
logging.error("Level is not valid.")
return None, False
if not self.config or not self.config.has_level(level):
logging.error("Can't find specified level %s in config!", level)
return None, False
orig_path = os.getcwd()
if args.output_directory:
if not os.path.isdir(args.output_directory):
try:
os.mkdir(args.output_directory)
except OSError as ex:
logging.error(
"Unable to create output directory at %s: %s",
args.output_directory,
ex,
)
return None, False
output_dir = os.path.join(args.output_directory, package.name + "-" + level)
if not os.path.isdir(output_dir):
try:
os.mkdir(output_dir)
except OSError as ex:
logging.error(
"Unable to create output directory at %s: %s", output_dir, ex
)
return None, False
logging.info("Writing output to: %s", output_dir)
os.chdir(output_dir)
logging.info("------")
logging.info(
"Scanning package %s (%s) at level %s", package.name, package.path, level
)
issues: Dict[str, List[Issue]] = {}
ignore_packages = self.get_ignore_packages()
if package.name in ignore_packages:
logging.info(
"Package %s is configured to be ignored by Statick.", package.name
)
return issues, True
plugin_context = PluginContext(args, self.resources, self.config)
logging.info("---Discovery---")
if not DiscoveryPlugin.file_command_exists():
logging.info(
"file command isn't available, discovery plugins will be less effective"
)
discovery_plugins = self.config.get_enabled_discovery_plugins(level)
if not discovery_plugins:
discovery_plugins = list(self.discovery_plugins.keys())
plugins_ran: List[Any] = []
for plugin_name in discovery_plugins:
if plugin_name not in self.discovery_plugins:
logging.error("Can't find specified discovery plugin %s!", plugin_name)
return None, False
plugin = self.discovery_plugins[plugin_name]
dependencies = plugin.get_discovery_dependencies()
for dependency_name in dependencies:
dependency_plugin = self.discovery_plugins[dependency_name]
if dependency_plugin.get_name() in plugins_ran:
continue
dependency_plugin.set_plugin_context(plugin_context)
logging.info(
"Running %s discovery plugin...", dependency_plugin.get_name()
)
dependency_plugin.scan(package, level, self.exceptions)
logging.info("%s discovery plugin done.", dependency_plugin.get_name())
plugins_ran.append(dependency_plugin.get_name())
if plugin.get_name() not in plugins_ran:
plugin.set_plugin_context(plugin_context)
logging.info("Running %s discovery plugin...", plugin.get_name())
plugin.scan(package, level, self.exceptions)
logging.info("%s discovery plugin done.", plugin.get_name())
plugins_ran.append(plugin.get_name())
logging.info("---Discovery---")
logging.info("---Tools---")
enabled_plugins = self.config.get_enabled_tool_plugins(level)
plugins_to_run = copy.copy(enabled_plugins)
plugins_ran = []
plugin_dependencies: List[str] = []
while plugins_to_run:
plugin_name = plugins_to_run[0]
if plugin_name not in self.tool_plugins:
logging.error("Can't find specified tool plugin %s!", plugin_name)
return None, False
if args.force_tool_list is not None:
force_tool_list = args.force_tool_list.split(",")
if (
plugin_name not in force_tool_list
and plugin_name not in plugin_dependencies
):
logging.info("Skipping plugin not in force list %s!", plugin_name)
plugins_to_run.remove(plugin_name)
continue
plugin = self.tool_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
dependencies = plugin.get_tool_dependencies()
dependencies_met = True
for dependency_name in dependencies:
if dependency_name not in plugins_ran:
if dependency_name not in enabled_plugins:
logging.error(
"Plugin %s depends on plugin %s which isn't enabled!",
plugin_name,
dependency_name,
)
return None, False
plugin_dependencies.append(dependency_name)
if dependency_name in plugins_to_run:
plugins_to_run.remove(dependency_name)
plugins_to_run.insert(0, dependency_name)
dependencies_met = False
if not dependencies_met:
continue
logging.info("Running %s tool plugin...", plugin.get_name())
tool_issues = plugin.scan(package, level)
if tool_issues is not None:
issues[plugin_name] = tool_issues
logging.info("%s tool plugin done.", plugin.get_name())
else:
logging.error("%s tool plugin failed", plugin.get_name())
success = False
plugins_to_run.remove(plugin_name)
plugins_ran.append(plugin_name)
logging.info("---Tools---")
if self.exceptions is not None:
issues = self.exceptions.filter_issues(package, issues)
os.chdir(orig_path)
logging.info("---Reporting---")
reporting_plugins = self.config.get_enabled_reporting_plugins(level)
if not reporting_plugins:
reporting_plugins = self.reporting_plugins.keys() # type: ignore
for plugin_name in reporting_plugins:
if plugin_name not in self.reporting_plugins:
logging.error("Can't find specified reporting plugin %s!", plugin_name)
return None, False
plugin = self.reporting_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
logging.info("Running %s reporting plugin...", plugin.get_name())
plugin.report(package, issues, level)
logging.info("%s reporting plugin done.", plugin.get_name())
logging.info("---Reporting---")
logging.info("Done!")
return issues, success
def run_workspace(
self, parsed_args: argparse.Namespace
) -> Tuple[
Optional[Dict[str, List[Issue]]], bool
]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Run statick on a workspace.
--max-procs can be set to the desired number of CPUs to use for processing a
workspace.
This defaults to half the available CPUs.
Setting this to -1 will cause statick.run_workspace to use all available CPUs.
"""
max_cpus = multiprocessing.cpu_count()
if parsed_args.max_procs > max_cpus or parsed_args.max_procs == -1:
parsed_args.max_procs = max_cpus
elif parsed_args.max_procs <= 0:
parsed_args.max_procs = 1
if parsed_args.output_directory:
out_dir = parsed_args.output_directory
if not os.path.isdir(out_dir):
try:
os.mkdir(out_dir)
except OSError as ex:
logging.error(
"Unable to create output directory at %s: %s", out_dir, ex
)
return None, False
ignore_packages = self.get_ignore_packages()
ignore_files = ["AMENT_IGNORE", "CATKIN_IGNORE", "COLCON_IGNORE"]
package_indicators = ["package.xml", "setup.py", "pyproject.toml"]
packages = []
for root, dirs, files in os.walk(parsed_args.path):
if any(item in files for item in ignore_files):
dirs.clear()
continue
for sub_dir in dirs:
full_dir = os.path.join(root, sub_dir)
files = os.listdir(full_dir)
if any(item in package_indicators for item in files) and not any(
item in files for item in ignore_files
):
if ignore_packages and sub_dir in ignore_packages:
continue
packages.append(Package(sub_dir, full_dir))
if parsed_args.packages_file is not None:
packages_file_list = []
try:
packages_file = os.path.abspath(parsed_args.packages_file)
with open(packages_file, "r", encoding="utf8") as fname:
packages_file_list = [
package.strip()
for package in fname.readlines()
if package.strip() and package[0] != "#"
]
except OSError:
logging.error("Packages file not found")
return None, False
packages = [
package for package in packages if package.name in packages_file_list
]
if parsed_args.list_packages:
for package in packages:
logging.info(
"%s: %s", package.name, self.get_level(package.path, parsed_args)
)
return None, True
count = 0
total_issues = []
num_packages = len(packages)
mp_args = []
if multiprocessing.get_start_method() == "fork":
logging.info("-- Scanning %d packages --", num_packages)
for package in packages:
count += 1
mp_args.append((parsed_args, count, package, num_packages))
with multiprocessing.Pool(parsed_args.max_procs) as pool:
total_issues = pool.starmap(self.scan_package, mp_args)
else:
logging.warning(
"Statick's plugin manager does not currently support multiprocessing"
" without UNIX's fork function. Falling back to a single process."
)
logging.info("-- Scanning %d packages --", num_packages)
for package in packages:
count += 1
pkg_issues = self.scan_package(
parsed_args, count, package, num_packages
)
total_issues.append(pkg_issues)
logging.info("-- All packages run --")
logging.info("-- overall report --")
success = True
issues: Dict[str, List[Issue]] = {}
for issue in total_issues:
if issue is not None:
for key, value in list(issue.items()):
if key in issues:
issues[key] += value
if value:
success = False
else:
issues[key] = value
if value:
success = False
enabled_reporting_plugins: List[str] = []
available_reporting_plugins = {}
for plugin_info in self.manager.getPluginsOfCategory("Reporting"):
available_reporting_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
# Make a fake 'all' package for reporting
dummy_all_package = Package("all_packages", parsed_args.path)
level = self.get_level(dummy_all_package.path, parsed_args)
if level is not None and self.config is not None:
if not self.config or not self.config.has_level(level):
logging.error("Can't find specified level %s in config!", level)
enabled_reporting_plugins = list(available_reporting_plugins)
else:
enabled_reporting_plugins = self.config.get_enabled_reporting_plugins(
level
)
if not enabled_reporting_plugins:
enabled_reporting_plugins = list(available_reporting_plugins)
plugin_context = PluginContext(parsed_args, self.resources, self.config) # type: ignore
plugin_context.args.output_directory = parsed_args.output_directory
for plugin_name in enabled_reporting_plugins:
if plugin_name not in available_reporting_plugins:
logging.error("Can't find specified reporting plugin %s!", plugin_name)
continue
plugin = self.reporting_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
logging.info("Running %s reporting plugin...", plugin.get_name())
plugin.report(dummy_all_package, issues, level)
logging.info("%s reporting plugin done.", plugin.get_name())
return issues, success
def scan_package(
self,
parsed_args: argparse.Namespace,
count: int,
package: Package,
num_packages: int,
) -> Optional[Dict[str, List[Issue]]]:
"""Scan each package in a separate process while buffering output."""
logger = logging.getLogger()
old_handler = None
if logger.handlers[0]:
old_handler = logger.handlers[0]
handler = MemoryHandler(10000, flushLevel=logging.ERROR, target=old_handler)
logger.removeHandler(old_handler)
logger.addHandler(handler)
logging.info(
"-- Scanning package %s (%d of %d) --", package.name, count, num_packages
)
sio = io.StringIO()
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stdout = sio
sys.stderr = sio
issues, dummy = self.run(package.path, parsed_args)
sys.stdout = old_stdout
sys.stderr = old_stderr
logging.info(sio.getvalue())
if issues is not None:
logging.info(
"-- Done scanning package %s (%d of %d) --",
package.name,
count,
num_packages,
)
else:
logging.error("Failed to run statick on package %s!", package.name)
if old_handler is not None:
handler.flush()
logger.removeHandler(handler)
logger.addHandler(old_handler)
return issues
@staticmethod
def print_no_issues() -> None:
"""Print that no information about issues was found."""
logging.error(
"Something went wrong, no information about issues."
" Statick exiting with errors."
)
@staticmethod
def print_exit_status(status: bool) -> None:
"""Print Statick exit status."""
if status:
logging.info("Statick exiting with success.")
else:
logging.error("Statick exiting with errors.")
def reloadPluginsModel():
if settings.DEBUG:
logging.getLogger('yapsy').setLevel(logging.WARNING)
# Are there any plugin objects? If not, add in the defaults
plugin_objects = Plugin.objects.all().count()
if plugin_objects == 0:
order = 0
PLUGIN_ORDER = [
'Activity', 'Status', 'OperatingSystem', 'MunkiVersion', 'Uptime',
'Memory', 'DiskSpace', 'PendingAppleUpdates',
'Pending3rdPartyUpdates', 'PuppetStatus'
]
for item in PLUGIN_ORDER:
order = order + 1
plugin = Plugin(name=item, order=order)
plugin.save()
# Build the manager
manager = PluginManager()
# Tell it the default place(s) where to find plugins
manager.setPluginPlaces([
settings.PLUGIN_DIR,
os.path.join(settings.PROJECT_DIR, 'server/plugins')
])
# Load all plugins
manager.collectPlugins()
found = []
for plugin in manager.getAllPlugins():
found.append(plugin.name)
# Get all of the plugin objects - if it's in here not installed, remove it
all_plugins = Plugin.objects.all()
for plugin in all_plugins:
if plugin.name not in found:
plugin.delete()
# And go over again to update the plugin's type
for dbplugin in all_plugins:
for plugin in manager.getAllPlugins():
if plugin.name == dbplugin.name:
try:
dbplugin.type = plugin.plugin_object.plugin_type()
except:
dbplugin.type = 'builtin'
try:
dbplugin.description = plugin.plugin_object.get_description(
)
except:
pass
dbplugin.save()
all_plugins = Report.objects.all()
for plugin in all_plugins:
if plugin.name not in found:
plugin.delete()
# And go over again to update the plugin's type
for dbplugin in all_plugins:
for plugin in manager.getAllPlugins():
if plugin.name == dbplugin.name:
try:
dbplugin.type = plugin.plugin_object.plugin_type()
except:
dbplugin.type = 'builtin'
try:
dbplugin.description = plugin.plugin_object.get_description(
)
except:
pass
dbplugin.save()
storage_package = CONFIG['storage'].split('.', 2)[0]
storage_class = CONFIG['storage'].split('.', 2)[1]
m = __import__('telescope.storage.%s' % (storage_package, ))
m = getattr(m.storage, storage_package)
STORAGE = getattr(m, storage_class)(CONFIG['storage config'])
STORAGE.load_data()
# plugin system
from yapsy.PluginManager import PluginManager
# Build the manager
pluginManager = PluginManager()
# Tell it the default place(s) where to find plugins
myPath = os.path.dirname(__file__)
modulePath = os.path.join(myPath, "plugins")
pluginManager.setPluginPlaces([modulePath])
# Tell it about our categories
pluginManager.setCategoriesFilter({
'WebPlugin':
telescope.plugins.abstract.ITelescopeWebPlugin,
'AnnouncePlugin':
telescope.plugins.abstract.ITelescopeAnnouncePlugin
})
# Load all plugins
pluginManager.collectPlugins()
# Activate all loaded plugins
for pluginInfo in pluginManager.getAllPlugins():
pluginManager.activatePluginByName(pluginInfo.name)
PLUGIN_MANAGER = pluginManager
class PackageIndexManager(object):
"""Provides indexing functionality of packages in all of the package sources.
Note:
This is an singleton class and will be
shared between objects
"""
__single_package_index_manager = None
def __new__(cls, *args, **kwargs):
"""Singleton object creator
"""
if cls != type(cls.__single_package_index_manager):
cls.__single_package_index_manager = object.__new__(
cls, *args, **kwargs)
return cls.__single_package_index_manager
def __init__(self, db_connection, download_location):
"""Initialize :class:`PackageIndexManager` class
Args:
db_connection (object): An open connection to metadata database
download_location (str): location where packages will be downloaded
"""
self.__plugin_manager = PluginManager()
self.__plugin_manager.setPluginPlaces(
[constants.PKG_SOURCE_PLUGIN_PATH])
self.__plugin_manager.setCategoriesFilter(
{'PackageSourcePlugins': IPackageSourcePlugin})
self.__plugin_manager.collectPlugins()
self.__db_connection = db_connection
self._sources_table = db_connection.table('PackageSource')
self.__package_sources = self.get_all_sources()
self.__package_index_registry = {}
self.__source_validity_status = {}
self.__download_location = download_location
self.__get_index_thread = tasks.HybridThread(name='PackageIndexCoroThread',\
notify_on_all_done=self.__common_callback)
if len(self.__package_sources) <= 0:
warnings.warn(
'No package source are configured. :class:`PackageManager`\
will not be able to download any packages')
else:
self.__update_package_list()
def __common_callback(self, results, owner, _type):
"""The common coroutine callback, which will be called on completion \
of all coroutines with owner as one of the parameters
"""
if owner == 'ValidityStatusCheck':
self.__validity_status_callback(results, _type)
elif owner == 'FetchIndex':
self.__update_index_callback(results, _type)
async def __get_index(self, source):
"""Coroutine to get the index from source using async request
Args:
source (str): Name of the package source from where index needs to be fetched
"""
return await source.get_package_index()
def __update_index_callback(self, results, _type):
"""This function will be called once all coroutines :func:`__get_index` are done.
This callback will receive a list of results in random order as a \
tuple of two elements (source-name, source-index-list)
Args:
results (list): A 2 element tuple list
_type (int): function or coroutine
"""
for result in results:
_name = result[0]
_source_index = result[1]
self.__package_index_registry[_name] = _source_index
if self.__get_index_thread is not None:
self.__get_index_thread = None
def get_all_sources(self):
"""Returns an list of :class:`PackageSource` configured in current system
Returns:
sources (PackageSource): list of all sources configured
"""
_sources = {}
if self._sources_table is not None:
if self._sources_table.count() > 0:
for source in self._sources_table.all():
source_object = object()
if source.name is not None:
source_object = self.__plugin_manager.getPluginByName(
source.name, 'PackageSourcePlugin')
if source_object is not None and callable(
source_object):
self.__plugin_manager.activatePluginByName(
source_object)
_sources[source.name] = source_object
else:
_sources[source.name] = DefaultPackageSource\
(self.__download_location, \
self.__db_connection, source.name)
return _sources
def add_source(self, download_loc, name, uri=None, username=None, password=None, \
src_type='Web', modified_on=None, modified_by=None, security_id=None, \
class_type=None, class_in_module=None):
"""Add an new source system
Args:
name (str): Name of the source
uri (str): Uri of the new source i.e., url or folder path
username (str): Username to access source uri (optional)
password (str): Password to access source uri (optional)
src_type (str): Type of the source - Web, FileSystem, etc
modified_on (date): Datetime when new source is added
modified_by (str): Name of the user who added the source
security_id (uuid): The security rules applied to this source
class_type (str): Class type of the source \
(should be derived from :class:`PackageSource`)
class_in_module (str): Name of the module where the above \
'class_type' can be found
Returns:
status (bool): True or False
message (str): Failure reason
"""
new_source = DefaultPackageSource(download_loc, self.__db_connection, name, \
uri=uri, username=username, password=password, \
src_type=src_type, modified_on=modified_on, \
modified_by=modified_by, security_id=security_id, \
class_type=class_type, class_in_module=class_in_module)
status = new_source.save()
if status is not None and status[0] is True:
self.__package_sources[name] = new_source
return status
def update_source(self, name, attribute_name, value=None):
"""Update an source system
Args:
name (str): Name of the source
attribute_name (str): Name of the attribute that needs to be updated
value (str): New value of the attribute to be updated
Returns:
status (bool): True or False
message (str): Failure reason
"""
if self._sources_table is not None and len(self.__package_sources) > 0:
src_record = self.__package_sources[name]
_result = src_record.update(attribute_name, value)
return _result
return (False, 'No package source are yet configured')
def remove_source(self, source_name):
"""Removes source from system
Args:
source_name (str): Name of source that needs to be removed
Returns:
status (bool): True or False
message (str): Reason for failurer
"""
if source_name is not None:
result = self._sources_table.remove(Query()['Name'] == source_name)
if result is None or len(result) <= 0:
return (False, 'Can' 't delete source from the system')
return (True, 'Source has been deleted - {0}'.format(result))
async def __get_validity_status(self, source):
"""docstring for get_validity_status"""
return await source.get_validity_status()
def __validity_status_callback(self, results, _type):
"""Will be called once all :func:`__get_validity_status` coroutines are done
Args:
results (list): A list of two elements tuple
-[0] (str): source name
-[1] (bool): valid or not valid
"""
for result in results:
_src_name = result[0]
_is_src_valid = result[1]
self.__source_validity_status[_src_name] = _is_src_valid
if self.__get_index_thread is not None:
self.__get_index_thread = None
def refresh_index(self, percentage_completed_callback=None):
"""Refresh the local index from the backend index source
Args:
percentage_completed_callback (callable): A callable which be called \
back to percentage completed status (optional)
"""
self.__update_package_list(percentage_completed_callback)
self.__get_index_thread.wait_for_all_coroutines()
def __update_package_list(self, percentage_completed_callback=None):
"""This function will fetch a list of all packages under an source
Args:
percentage_completed_callback (func): Callback will be called on updation \
of each package and percentage completion will be passed to callback
"""
if len(self.__package_sources) > 0:
#Step1 - Get validity status of all current sources configured
self.__get_index_thread.set_owner('ValidityStatusCheck')
for source_name, source in self.__package_sources.items():
self.__get_index_thread.add_coroutine(
self.__get_validity_status, source)
if percentage_completed_callback is not None:
#return the percentage completion
self.__get_index_thread.register_coroutine_completed_percentage\
(percentage_completed_callback)
self.__get_index_thread.start_forever()
#Will wait for coroutines to finish as we need it for next step
self.__get_index_thread.wait_for_all_coroutines()
#Step2 - Get cached package index for sources having valid indexes
for source_name, is_valid in self.__source_validity_status.items():
if is_valid:
self.__package_index_registry[source_name] = \
self.__package_sources[source_name].get_cached_package_index()
#Step3 - Refresh cached index from source for those which are not valid
self.__get_index_thread.set_owner('FetchIndex')
for source_name, is_valid in self.__source_validity_status.items():
if not is_valid:
self.__get_index_thread\
.add_coroutine(self.__get_index, \
self.__package_sources[source_name])
if percentage_completed_callback is not None:
#return the percentage completion
self.__get_index_thread\
.register_coroutine_completed_percentage(percentage_completed_callback)
self.__get_index_thread.reschedule()
#No need to wait here as calling thread will handle it
def get_package_list(self,
refresh=False,
percentage_completed_callback=None):
"""Returns an list of all packages either from cache or downloading it from source
Args:
refresh (bool): Whether to have package indexes refreshed before \
returning the index list (default=False)
"""
if percentage_completed_callback is not None and callable(
percentage_completed_callback):
percentage_completed_callback(0)
if refresh:
self.__update_package_list(percentage_completed_callback)
if self.__get_index_thread is not None:
#wait for coroutines to finish
self.__get_index_thread.wait_for_all_coroutines()
if percentage_completed_callback is not None and callable(
percentage_completed_callback):
percentage_completed_callback(100)
return self.__package_index_registry
def find_package(self, package_name):
"""Finds an package in the index repository
Args:
package_name (str): Name of the package that needs to be searched
Returns:
result (tuple of 4 elements): Returns an tuple having below mentioned 4 elements
- status (bool): True if package found else False
- source_name (str): source name where package was found
- package_index (dict): dict of package index
- message (str): Error message if not able to find the package
"""
if package_name is not None:
for source_name, source in self.__package_sources.items():
if self.__package_index_registry[source_name].__contains__(
package_name):
return (True, source, \
self.__package_index_registry[source.name][package_name], '')
return (False, None, None,
'No such package found...{0}'.format(package_name))
return (False, None, None, 'Can'
't accept blank value for package name')
class Paratest(object):
def __init__(self, workspace_num, scripts, source_path, workspace_path,
output_path, test_pattern, persistence):
self.workspace_num = workspace_num
self.workspace_path = workspace_path
self.scripts = scripts
self.source_path = source_path
self.output_path = output_path
self.test_pattern = test_pattern
self._workers = []
self.pluginmgr = PluginManager()
self.pluginmgr.setPluginInfoExtension('paratest')
self.pluginmgr.setPluginPlaces(["plugins", ""])
self.pluginmgr.collectPlugins()
self.persistence = persistence
if not os.path.exists(self.source_path):
os.makedirs(self.source_path)
if not os.path.exists(self.output_path):
os.makedirs(self.output_path)
def list_plugins(self):
msg = "Available plugins are:\n"
for plugin in self.pluginmgr.getAllPlugins():
msg += " %s" % plugin.name
print(msg)
def run(self, plugin):
plugin = self.pluginmgr.getPluginByName(plugin)
pluginobj = plugin.plugin_object
self.run_script_setup()
test_number = self.queue_tests(pluginobj)
self.create_workers(pluginobj, self.num_of_workers(test_number))
self.start_workers()
self.wait_workers()
self.run_script_teardown()
self.assert_all_messages_were_processed()
def run_script_setup(self):
if run_script(self.scripts.setup, path=self.workspace_path):
raise Abort('The setup script failed. aborting.')
def run_script_teardown(self):
if run_script(self.scripts.teardown, path=self.workspace_path):
raise Abort('The teardown script failed, but nothing can be done.')
def queue_tests(self, pluginobj):
tids = 0
for tid in pluginobj.find(self.source_path, self.test_pattern):
shared_queue.put((self.persistence.get_priority(tid), tid))
tids += 1
return tids
def create_workers(self, pluginobj, workers):
for i in range(workers):
t = Worker(
pluginobj,
scripts=self.scripts,
workspace_path=self.workspace_path,
source_path=self.source_path,
output_path=self.output_path,
persistence=self.persistence,
name=str(i),
)
self._workers.append(t)
def num_of_workers(self, test_number):
return min(self.workspace_num, test_number)
def start_workers(self):
logger.debug("start workers")
for t in self._workers:
t.start()
shared_queue.put((INFINITE, FINISH))
def wait_workers(self):
logger.debug("wait for all workers to finish")
for t in self._workers:
t.join()
def assert_all_messages_were_processed(self):
if not shared_queue.empty():
raise Abort(
'There were unprocessed tests, but all workers are dead. Aborting.'
)
class MainWindow(QMainWindow):
"""MainWindow class
Attributes:
trace_data (TraceData): TraceData object
filtered_trace (list): Filtered trace
"""
def __init__(self, parent=None):
"""Inits MainWindow, UI and plugins"""
super(MainWindow, self).__init__(parent)
self.api = Api(self)
self.trace_data = TraceData()
self.filtered_trace = []
self.filter_text = ""
self.init_plugins()
self.init_ui()
if len(sys.argv) > 1:
self.open_trace(sys.argv[1])
def dragEnterEvent(self, event):
"""QMainWindow method reimplementation for file drag."""
event.setDropAction(Qt.MoveAction)
super().dragEnterEvent(event)
event.accept()
def dropEvent(self, event):
"""QMainWindow method reimplementation for file drop."""
super().dropEvent(event)
if event.mimeData().hasUrls():
for url in event.mimeData().urls():
local_file = url.toLocalFile()
if os.path.isfile(local_file):
self.open_trace(local_file)
def init_ui(self):
"""Inits UI"""
uic.loadUi("gui/mainwindow.ui", self)
self.setWindowTitle(prefs.PACKAGE_NAME)
# accept file drops
self.setAcceptDrops(True)
self.resize(prefs.WINDOW_WIDTH, prefs.WINDOW_HEIGHT)
# make trace table wider than regs&mem
self.splitter1.setSizes([1000, 100])
self.splitter2.setSizes([600, 100])
# Init trace table
self.trace_table.itemSelectionChanged.connect(
self.on_trace_table_row_changed)
self.trace_table.setColumnCount(len(prefs.TRACE_LABELS))
self.trace_table.setHorizontalHeaderLabels(prefs.TRACE_LABELS)
self.trace_table.horizontalHeader().setStretchLastSection(True)
self.trace_table.bookmarkCreated.connect(self.add_bookmark)
self.trace_table.commentEdited.connect(self.set_comment)
self.trace_table.printer = self.print
self.trace_table.set_row_height(prefs.TRACE_ROW_HEIGHT)
trace_font = QFont(prefs.TRACE_FONT)
trace_font.setPointSize(prefs.TRACE_FONT_SIZE)
self.trace_table.setFont(trace_font)
self.bookmark_table.setFont(trace_font)
self.trace_table.setShowGrid(prefs.TRACE_SHOW_GRID)
if prefs.USE_SYNTAX_HIGHLIGHT_IN_TRACE:
self.trace_table.init_syntax_highlight()
# trace pagination
if prefs.PAGINATION_ENABLED:
self.trace_pagination = PaginationWidget()
self.trace_pagination.pageChanged.connect(
self.trace_table.populate)
self.horizontalLayout.addWidget(self.trace_pagination)
self.trace_pagination.set_enabled(True)
self.trace_pagination.rows_per_page = prefs.PAGINATION_ROWS_PER_PAGE
self.trace_table.pagination = self.trace_pagination
self.horizontalLayout.setAlignment(self.trace_pagination,
Qt.AlignLeft)
# these are used to remember current pages & scroll values for both traces
self.trace_current_pages = [1, 1]
self.trace_scroll_values = [0, 0]
self.reg_table.create_context_menu()
self.reg_table.setColumnCount(len(prefs.REG_LABELS))
self.reg_table.setHorizontalHeaderLabels(prefs.REG_LABELS)
self.reg_table.horizontalHeader().setStretchLastSection(True)
self.reg_table.regCheckBoxChanged.connect(self.on_reg_checkbox_change)
self.reg_table.printer = self.print
if prefs.REG_FILTER_ENABLED:
self.reg_table.filtered_regs = prefs.REG_FILTER
if not prefs.USE_DARK_THEME:
trace_style = (
"QTableView { selection-background-color: #dddddd; selection-"
"color: #000000; border: 0px;} QTableWidget::item { padding: 0px; border: 0px}"
)
reg_style = (
"QTableView { selection-background-color: #eee864; selection"
"-color: #000000;}")
self.trace_table.setStyleSheet(trace_style)
self.bookmark_table.setStyleSheet(trace_style)
self.reg_table.setStyleSheet(reg_style)
# Init memory table
self.mem_table.setColumnCount(len(prefs.MEM_LABELS))
self.mem_table.setHorizontalHeaderLabels(prefs.MEM_LABELS)
self.mem_table.horizontalHeader().setStretchLastSection(True)
# Init bookmark table
self.bookmark_table.setColumnCount(len(prefs.BOOKMARK_LABELS))
self.bookmark_table.setHorizontalHeaderLabels(prefs.BOOKMARK_LABELS)
self.bookmark_table.setContextMenuPolicy(Qt.CustomContextMenu)
self.bookmark_table.customContextMenuRequested.connect(
self.bookmark_table_context_menu_event)
self.bookmark_table.delegate = SyntaxHighlightDelegate(self)
self.bookmark_table.delegate.disasm_columns = prefs.BOOKMARK_HL_DISASM_COLUMNS
self.bookmark_table.delegate.value_columns = prefs.BOOKMARK_HL_VALUE_COLUMNS
self.bookmark_table.setItemDelegate(self.bookmark_table.delegate)
self.bookmark_menu = QMenu(self)
go_action = QAction("Go to bookmark", self)
go_action.triggered.connect(self.go_to_bookmark_in_trace)
self.bookmark_menu.addAction(go_action)
delete_bookmarks_action = QAction("Delete bookmark(s)", self)
delete_bookmarks_action.triggered.connect(self.delete_bookmarks)
self.bookmark_menu.addAction(delete_bookmarks_action)
# Menu
exit_action = QAction("&Exit", self)
exit_action.setShortcut("Ctrl+Q")
exit_action.setStatusTip("Exit application")
exit_action.triggered.connect(self.close)
open_trace_action = QAction("&Open trace..", self)
open_trace_action.setStatusTip("Open trace")
open_trace_action.triggered.connect(self.dialog_open_trace)
self.save_trace_action = QAction("&Save trace", self)
self.save_trace_action.setStatusTip("Save trace")
self.save_trace_action.triggered.connect(self.save_trace)
self.save_trace_action.setEnabled(False)
save_trace_as_action = QAction("&Save trace as..", self)
save_trace_as_action.setStatusTip("Save trace as..")
save_trace_as_action.triggered.connect(self.dialog_save_trace_as)
save_trace_as_json_action = QAction("&Save trace as JSON..", self)
save_trace_as_json_action.setStatusTip("Save trace as JSON..")
save_trace_as_json_action.triggered.connect(
self.dialog_save_trace_as_json)
file_menu = self.menu_bar.addMenu("&File")
file_menu.addAction(open_trace_action)
file_menu.addAction(self.save_trace_action)
file_menu.addAction(save_trace_as_action)
file_menu.addAction(save_trace_as_json_action)
file_menu.addAction(exit_action)
self.plugins_topmenu = self.menu_bar.addMenu("&Plugins")
clear_bookmarks_action = QAction("&Clear bookmarks", self)
clear_bookmarks_action.setStatusTip("Clear bookmarks")
clear_bookmarks_action.triggered.connect(self.clear_bookmarks)
bookmarks_menu = self.menu_bar.addMenu("&Bookmarks")
bookmarks_menu.addAction(clear_bookmarks_action)
# Create right click menu for trace table
self.create_trace_table_menu()
# Create plugins menu on menu bar
self.create_plugins_menu()
about_action = QAction("&About", self)
about_action.triggered.connect(self.show_about_dialog)
about_menu = self.menu_bar.addMenu("&About")
about_menu.addAction(about_action)
if prefs.USE_SYNTAX_HIGHLIGHT_IN_LOG:
self.highlight = AsmHighlighter(self.log_text_edit.document())
# trace select
self.select_trace_combo_box.addItem("Full trace")
self.select_trace_combo_box.addItem("Filtered trace")
self.select_trace_combo_box.currentIndexChanged.connect(
self.on_trace_combo_box_index_changed)
self.filter_widget = FilterWidget()
self.filter_widget.filterBtnClicked.connect(self.on_filter_btn_clicked)
self.horizontalLayout.addWidget(self.filter_widget)
if prefs.SHOW_SAMPLE_FILTERS:
self.filter_widget.set_sample_filters(prefs.SAMPLE_FILTERS)
self.find_widget = FindWidget()
self.find_widget.findBtnClicked.connect(self.on_find_btn_clicked)
self.find_widget.set_fields(prefs.FIND_FIELDS)
self.horizontalLayout.addWidget(self.find_widget)
self.show()
def init_plugins(self):
"""Inits plugins"""
self.manager = PluginManager()
self.manager.setPluginPlaces(["plugins"])
self.manager.collectPlugins()
for plugin in self.manager.getAllPlugins():
print_debug(f"Plugin found: {plugin.name}")
def create_plugins_menu(self):
"""Creates plugins menu"""
self.plugins_topmenu.clear()
reload_action = QAction("Reload plugins", self)
reload_action.setShortcut("Ctrl+R")
func = functools.partial(self.reload_plugins)
reload_action.triggered.connect(func)
self.plugins_topmenu.addAction(reload_action)
self.plugins_topmenu.addSeparator()
plugins_menu = QMenu("Run plugin", self)
for plugin in self.manager.getAllPlugins():
action = QAction(plugin.name, self)
func = functools.partial(self.execute_plugin, plugin)
action.triggered.connect(func)
plugins_menu.addAction(action)
self.plugins_topmenu.addMenu(plugins_menu)
def create_trace_table_menu(self):
"""Creates right click menu for trace table"""
self.trace_table_menu = QMenu(self)
print_action = QAction("Print selected cells", self)
print_action.triggered.connect(self.trace_table.print_selected_cells)
self.trace_table_menu.addAction(print_action)
add_bookmark_action = QAction("Add Bookmark", self)
add_bookmark_action.triggered.connect(self.trace_table.create_bookmark)
self.trace_table_menu.addAction(add_bookmark_action)
plugins_menu = QMenu("Plugins", self)
for plugin in self.manager.getAllPlugins():
action = QAction(plugin.name, self)
func = functools.partial(self.execute_plugin, plugin)
action.triggered.connect(func)
plugins_menu.addAction(action)
self.trace_table_menu.addMenu(plugins_menu)
self.trace_table.menu = self.trace_table_menu
def reload_plugins(self):
"""Reloads plugins"""
self.init_plugins()
self.create_trace_table_menu()
self.create_plugins_menu()
def on_trace_table_row_changed(self):
"""Called when selected row changes"""
selected_row_ids = self.get_selected_row_ids(self.trace_table)
if not selected_row_ids:
return
row_id = selected_row_ids[0]
regs = self.trace_data.get_regs_and_values(row_id)
modified_regs = []
if prefs.HIGHLIGHT_MODIFIED_REGS:
modified_regs = self.trace_data.get_modified_regs(row_id)
self.reg_table.set_data(regs, modified_regs)
mem = []
if "mem" in self.trace_data.trace[row_id]:
mem = self.trace_data.trace[row_id]["mem"]
self.mem_table.set_data(mem)
self.update_status_bar()
def on_filter_btn_clicked(self, filter_text: str):
self.filter_text = filter_text
if self.trace_data is None:
return
try:
filtered_trace = filter_trace(
self.trace_data.trace,
self.trace_data.get_regs(),
filter_text,
)
except Exception as exc:
self.show_messagebox("Filter error", f"{exc}")
# print(traceback.format_exc())
else:
self.filtered_trace = filtered_trace
self.show_filtered_trace()
self.update_status_bar()
def on_find_btn_clicked(self, keyword: str, field_index: int,
direction: int):
"""Find next or prev button clicked"""
current_row = self.trace_table.currentRow()
if current_row < 0:
current_row = 0
if self.trace_table.pagination is not None:
pagination = self.trace_table.pagination
page = pagination.current_page
rows_per_page = pagination.rows_per_page
current_row += (page - 1) * rows_per_page
if field_index == 0:
field = TraceField.DISASM
elif field_index == 1:
field = TraceField.REGS
elif field_index == 2:
field = TraceField.MEM
elif field_index == 3:
field = TraceField.MEM_ADDR
elif field_index == 4:
field = TraceField.MEM_VALUE
elif field_index == 5:
field = TraceField.COMMENT
elif field_index == 6:
field = TraceField.ANY
try:
row_number = find(
trace=self.get_visible_trace(),
field=field,
keyword=keyword,
start_row=current_row + direction,
direction=direction,
)
except Exception as exc:
self.show_messagebox("Find error", f"{exc}")
print(traceback.format_exc())
self.print(traceback.format_exc())
return
if row_number is not None:
self.trace_table.go_to_row(row_number)
else:
print_debug(
f"{keyword} not found (row: {current_row}, direction: {direction})"
)
def get_visible_trace(self):
"""Returns the trace that is currently shown on trace table"""
index = self.select_trace_combo_box.currentIndex()
if self.trace_data is not None:
if index == 0:
return self.trace_data.trace
else:
return self.filtered_trace
return None
def bookmark_table_context_menu_event(self):
"""Context menu for bookmark table right click"""
self.bookmark_menu.popup(QCursor.pos())
def dialog_open_trace(self):
"""Shows dialog to open trace file"""
all_traces = "All traces (*.tvt *.trace32 *.trace64)"
all_files = "All files (*.*)"
filename = QFileDialog.getOpenFileName(
self, "Open trace", "", all_traces + ";; " + all_files)[0]
if filename:
self.open_trace(filename)
if self.trace_data:
self.save_trace_action.setEnabled(True)
def dialog_save_trace_as(self):
"""Shows a dialog to select a save file"""
filename = QFileDialog.getSaveFileName(
self, "Save trace as", "",
"Trace Viewer traces (*.tvt);; All files (*.*)")[0]
print_debug("Save trace as: " + filename)
if filename and trace_files.save_as_tv_trace(self.trace_data,
filename):
self.trace_data.filename = filename
self.save_trace_action.setEnabled(True)
def dialog_save_trace_as_json(self):
"""Shows a dialog to save trace to JSON file"""
filename = QFileDialog.getSaveFileName(
self, "Save as JSON", "",
"JSON files (*.txt);; All files (*.*)")[0]
print_debug("Save trace as: " + filename)
if filename:
trace_files.save_as_json(self.trace_data, filename)
def execute_plugin(self, plugin):
"""Executes a plugin and updates tables"""
print_debug(f"Executing a plugin: {plugin.name}")
try:
plugin.plugin_object.execute(self.api)
except Exception:
print("Error in plugin:")
print(traceback.format_exc())
self.print("Error in plugin:")
self.print(traceback.format_exc())
finally:
if prefs.USE_SYNTAX_HIGHLIGHT_IN_LOG:
self.highlight.rehighlight()
def show_filtered_trace(self):
"""Shows filtered_trace on trace_table"""
if self.select_trace_combo_box.currentIndex() == 0:
self.select_trace_combo_box.setCurrentIndex(1)
else:
self.trace_table.set_data(self.filtered_trace)
self.trace_table.populate()
def set_comment(self, row_id, comment):
"""Sets comment to row on full trace"""
self.trace_data.set_comment(row_id, comment)
def on_bookmark_table_cell_edited(self, item):
"""Called when any cell is edited on bookmark table"""
cell_type = item.whatsThis()
bookmarks = self.trace_data.get_bookmarks()
row = self.bookmark_table.currentRow()
if row < 0:
print_debug("Error, could not edit bookmark.")
return
if cell_type == "startrow":
bookmarks[row].startrow = int(item.text())
elif cell_type == "endrow":
bookmarks[row].endrow = int(item.text())
elif cell_type == "address":
bookmarks[row].addr = item.text()
elif cell_type == "disasm":
bookmarks[row].disasm = item.text()
elif cell_type == "comment":
bookmarks[row].comment = item.text()
else:
print_debug("Unknown field edited on bookmark table...")
def open_trace(self, filename):
"""Opens and reads a trace file"""
print_debug(f"Opening trace file: {filename}")
self.close_trace()
self.trace_data = trace_files.open_trace(filename)
if self.trace_data is None:
print_debug(f"Error, couldn't open trace file: {filename}")
else:
if prefs.PAGINATION_ENABLED:
self.trace_pagination.set_current_page(1, True)
self.trace_table.get_syntax_highlighter().reset()
self.trace_table.set_data(self.trace_data.trace)
self.trace_table.populate()
self.trace_table.selectRow(0)
self.setWindowTitle(
f"{filename.split('/')[-1]} - {prefs.PACKAGE_NAME}")
self.update_bookmark_table()
self.trace_table.update_column_widths()
def close_trace(self):
"""Clears trace and updates UI"""
self.trace_data = None
self.filtered_trace = []
self.trace_table.set_data([])
self.update_ui()
def update_ui(self):
"""Updates tables and status bar"""
self.trace_table.populate()
self.update_bookmark_table()
self.update_status_bar()
def save_trace(self):
"""Saves a trace file"""
filename = self.trace_data.filename
print_debug("Save trace: " + filename)
if filename:
trace_files.save_as_tv_trace(self.trace_data, filename)
def show_about_dialog(self):
"""Shows an about dialog"""
title = "About"
name = prefs.PACKAGE_NAME
version = prefs.PACKAGE_VERSION
copyrights = prefs.PACKAGE_COPYRIGHTS
url = prefs.PACKAGE_URL
text = f"{name} {version} \n {copyrights} \n {url}"
QMessageBox().about(self, title, text)
def update_column_widths(self, table):
"""Updates column widths of a TableWidget to match the content"""
table.setVisible(False) # fix ui glitch with column widths
table.resizeColumnsToContents()
table.horizontalHeader().setStretchLastSection(True)
table.setVisible(True)
def update_status_bar(self):
"""Updates status bar"""
if self.trace_data is None:
return
table = self.trace_table
row = table.currentRow()
row_count = table.rowCount()
row_info = f"{row}/{row_count - 1}"
msg = f"Row: {row_info} "
selected_row_id = 0
row_ids = self.trace_table.get_selected_row_ids()
if row_ids:
selected_row_id = row_ids[0]
msg += f" | {len(self.trace_data.trace)} rows in full trace."
if len(self.filter_text) > 0:
msg += f" | {len(self.filtered_trace)} rows in filtered trace."
bookmark = self.trace_data.get_bookmark_from_row(selected_row_id)
if bookmark:
msg += f" | Bookmark: {bookmark.disasm} ; {bookmark.comment}"
self.status_bar.showMessage(msg)
def on_reg_checkbox_change(self, reg: str, is_checked: bool):
"""Callback for register checkbox change"""
highlighter = self.trace_table.get_syntax_highlighter()
if highlighter:
highlighter.set_reg_highlight(reg, is_checked)
# force repaint, update() didn't work
self.trace_table.setVisible(False)
self.trace_table.setVisible(True)
else:
print_debug("Error, highlighter not found!")
def get_selected_row_ids(self, table):
"""Returns IDs of all selected rows of TableWidget.
Args:
table: PyQt TableWidget
returns:
list: Ordered list of row ids
"""
# use a set so we don't get duplicate ids
row_ids_set = set(
table.item(index.row(), 0).text()
for index in table.selectedIndexes())
try:
row_ids_list = [int(i) for i in row_ids_set]
except ValueError:
print_debug("Error. Values in the first column must be integers.")
return []
return sorted(row_ids_list)
def on_trace_combo_box_index_changed(self, index):
"""Trace selection combo box index changed"""
self.trace_table.set_data(self.get_visible_trace())
other_index = index ^ 1
if prefs.PAGINATION_ENABLED:
# save current page
self.trace_current_pages[
other_index] = self.trace_pagination.current_page
self.trace_pagination.set_current_page(
self.trace_current_pages[index], True)
# save scrollbar value
current_scroll = self.trace_table.verticalScrollBar().value()
self.trace_scroll_values[other_index] = current_scroll
next_value = self.trace_scroll_values[index]
self.trace_table.populate()
QApplication.processEvents() # this is needed to update the scrollbar
self.trace_table.verticalScrollBar().setValue(next_value)
def go_to_row_in_visible_trace(self, row):
"""Goes to given row in currently visible trace"""
self.trace_table.go_to_row(row)
self.tab_widget.setCurrentIndex(0)
def go_to_row_in_full_trace(self, row_id):
"""Switches to full trace and goes to given row"""
# make sure we are shown full trace, not filtered
if self.select_trace_combo_box.currentIndex() == 1:
self.select_trace_combo_box.setCurrentIndex(0)
self.go_to_row_in_visible_trace(row_id)
def go_to_bookmark_in_trace(self):
"""Goes to trace row of selected bookmark"""
selected_row_ids = self.get_selected_row_ids(self.bookmark_table)
if not selected_row_ids:
print_debug("Error. No bookmark selected.")
return
self.go_to_row_in_full_trace(selected_row_ids[0])
def clear_bookmarks(self):
"""Clears all bookmarks"""
self.trace_data.clear_bookmarks()
self.update_bookmark_table()
def delete_bookmarks(self):
"""Deletes selected bookmarks"""
selected = self.bookmark_table.selectedItems()
if not selected:
print_debug("Could not delete a bookmark. Nothing selected.")
return
selected_rows = sorted(set({sel.row() for sel in selected}))
for row in reversed(selected_rows):
self.trace_data.delete_bookmark(row)
self.bookmark_table.removeRow(row)
def get_selected_bookmarks(self):
"""Returns selected bookmarks"""
selected = self.bookmark_table.selectedItems()
if not selected:
print_debug("No bookmarks selected.")
return []
selected_rows = sorted(set({sel.row() for sel in selected}))
all_bookmarks = self.trace_data.get_bookmarks()
return [all_bookmarks[i] for i in selected_rows]
def add_bookmark(self, bookmark):
if prefs.ASK_FOR_BOOKMARK_COMMENT:
comment = self.get_string_from_user(
"Bookmark comment", "Give a comment for bookmark:")
if comment:
bookmark.comment = comment
self.trace_data.add_bookmark(bookmark)
self.update_bookmark_table()
def update_bookmark_table(self):
"""Updates bookmarks table from trace_data"""
if self.trace_data is None:
return
table = self.bookmark_table
try:
table.itemChanged.disconnect()
except Exception:
pass
bookmarks = self.trace_data.get_bookmarks()
table.setRowCount(len(bookmarks))
for i, bookmark in enumerate(bookmarks):
startrow = QTableWidgetItem(bookmark.startrow)
startrow.setData(Qt.DisplayRole, int(bookmark.startrow))
startrow.setWhatsThis("startrow")
table.setItem(i, 0, startrow)
endrow = QTableWidgetItem(bookmark.endrow)
endrow.setData(Qt.DisplayRole, int(bookmark.endrow))
endrow.setWhatsThis("endrow")
table.setItem(i, 1, endrow)
address = QTableWidgetItem(bookmark.addr)
address.setWhatsThis("address")
table.setItem(i, 2, address)
disasm = QTableWidgetItem(bookmark.disasm)
disasm.setWhatsThis("disasm")
table.setItem(i, 3, disasm)
comment = QTableWidgetItem(bookmark.comment)
comment.setWhatsThis("comment")
table.setItem(i, 4, comment)
table.setRowHeight(i, 14)
table.itemChanged.connect(self.on_bookmark_table_cell_edited)
self.update_column_widths(table)
def print(self, text):
"""Prints text to TextEdit on log tab"""
self.log_text_edit.appendPlainText(str(text))
def go_to_row(self, table, row):
"""Scrolls a table to the specified row"""
table.scrollToItem(table.item(row, 3),
QAbstractItemView.PositionAtCenter)
def ask_user(self, title, question):
"""Shows a messagebox with yes/no question
Args:
title (str): MessageBox title
question (str): MessageBox qustion label
Returns:
bool: True if user clicked yes, False otherwise
"""
answer = QMessageBox.question(
self,
title,
question,
QMessageBox.StandardButtons(QMessageBox.Yes | QMessageBox.No),
)
return bool(answer == QMessageBox.Yes)
def get_string_from_user(self, title, label):
"""Gets a string from user
Args:
title (str): Input dialog title
label (str): Input dialog label
Returns:
string: String given by user, empty string if user clicked cancel
"""
answer, ok_clicked = QInputDialog.getText(self, title, label,
QLineEdit.Normal, "")
if ok_clicked:
return answer
return ""
def get_values_from_user(self, title, data, on_ok_clicked=None):
"""Gets values from user
Args:
title (str): Input dialog title
data (list): List of dicts
on_ok_clicked (method): Callback function to e.g. check the input
Returns:
list: List of values given by user, empty list if user canceled
"""
input_dlg = InputDialog(self, title, data, on_ok_clicked)
input_dlg.exec_()
return input_dlg.get_data()
def show_messagebox(self, title, msg):
"""Shows a messagebox"""
alert = QMessageBox()
alert.setWindowTitle(title)
alert.setText(msg)
alert.exec_()
class FakeSite:
def __init__(self):
self.template_system = self
self.invariant = False
self.debug = True
self.config = {
"DISABLED_PLUGINS": [],
"EXTRA_PLUGINS": [],
"DEFAULT_LANG":
"en",
"MARKDOWN_EXTENSIONS": [
"markdown.extensions.fenced_code",
"markdown.extensions.codehilite",
],
"TRANSLATIONS_PATTERN":
"{path}.{lang}.{ext}",
"LISTINGS_FOLDERS": {
"listings": "listings"
},
"TRANSLATIONS": {
"en": ""
},
}
self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"]
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension,
})
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension("plugin")
places = [
os.path.join(os.path.dirname(nikola.utils.__file__), "plugins")
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category(
"CompilerExtension")
self.timeline = [FakePost(title="Fake post", slug="fake-post")]
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = "mako"
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get("DISABLED_PLUGINS"):
self.plugin_manager.removePluginFromCategory(
plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return '<img src="IMG.jpg">'
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warning('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data, *a, **kw):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data,
self.shortcode_registry,
**kw)
def apply_shortcodes_uuid(self, data, shortcodes, *a, **kw):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data,
self.shortcode_registry,
**kw)
class TestDef(object):
def __init__(self):
# set aside storage for options and cmd line args
self.options = {}
self.args = []
# record if we have loaded the plugins or
# not - this is just a bozo check to ensure
# someone doesn't tell us to do it twice
self.loaded = False
# set aside a spot for a logger object, and
# note that it hasn't yet been defined
self.logger = None
self.modcmd = None
self.execmd = None
self.harasser = None
self.config = None
self.stages = None
self.tools = None
self.utilities = None
self.defaults = None
self.log = {}
self.watchdog = None
self.plugin_trans_sem = Semaphore()
def setOptions(self, args):
self.options = vars(args)
self.args = args
# if they want us to clear the scratch, then do so
if self.options['clean'] and os.path.isdir(self.options['scratchdir']) :
shutil.rmtree(self.options['scratchdir'])
# setup the scratch directory
_mkdir_recursive(self.options['scratchdir'])
# private function to convert values
def __convert_value(self, opt, inval):
if opt is None or type(opt) is str:
return 0, inval
elif type(opt) is bool:
if type(inval) is bool:
return 0, inval
elif type(inval) is str:
if inval.lower() in ['true', '1', 't', 'y', 'yes']:
return 0, True
else:
return 0, False
elif type(inval) is int:
if 0 == inval:
return 0, False
else:
return 0, True
elif is_py2 and type(inval) is unicode:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is int:
if type(inval) is int:
return 0, inval
elif type(inval) is str:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is float:
if type(inval) is float:
return 0, inval
elif type(inval) is str or type(inval) is int:
return 0, float(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
else:
return 1, None
# scan the key-value pairs obtained from the configuration
# parser and compare them with the options defined for a
# given plugin. Generate an output dictionary that contains
# the updated set of option values, the default value for
# any option that wasn't included in the configuration file,
# and return an error status plus output identifying any
# keys in the configuration file that are not supported
# by the list of options
#
# @log [INPUT]
# - a dictionary that will return the status plus
# stderr containing strings identifying any
# provided keyvals that don't have a corresponding
# supported option
# @options [INPUT]
# - a dictionary of tuples, each consisting of three
# entries:
# (a) the default value
# (b) data type
# (c) a help-like description
# @keyvals [INPUT]
# - a dictionary of key-value pairs obtained from
# the configuration parser
# @target [OUTPUT]
# - the resulting dictionary of key-value pairs
def parseOptions(self, log, options, keyvals, target):
# parse the incoming keyvals dictionary against the source
# options. If a source option isn't provided, then
# copy it across to the target.
opts = list(options.keys())
kvkeys = list(keyvals.keys())
for opt in opts:
found = False
for kvkey in kvkeys:
if kvkey == opt:
# they provided us with an update, so
# pass this value into the target - expand
# any provided lists
if keyvals[kvkey] is None:
continue
st, outval = self.__convert_value(options[opt][0], keyvals[kvkey])
if 0 == st:
target[opt] = outval
else:
if len(keyvals[kvkey]) == 0:
# this indicates they do not want this option
found = True
break
if keyvals[kvkey][0][0] == "[":
# they provided a list - remove the brackets
val = keyvals[kvkey].replace('[','')
val = val.replace(']','')
# split the input to pickup sets of options
newvals = list(val)
# convert the values to specified type
i=0
for val in newvals:
st, newvals[i] = self.__convert_value(opt[0], val)
i = i + 1
target[opt] = newvals
else:
st, target[opt] = self.__convert_value(opt[0], keyvals[kvkey])
found = True
break
if not found:
# they didn't provide this one, so
# transfer only the value across
target[opt] = options[opt][0]
# add in any default settings that have not
# been overridden - anything set by this input
# stage will override the default
if self.defaults is not None:
keys = self.defaults.options.keys()
for key in keys:
if key not in target:
target[key] = self.defaults.options[key][0]
# now go thru in the reverse direction to see
# if any keyvals they provided aren't supported
# as this would be an error
unsupported_options = []
for kvkey in kvkeys:
# ignore some standard keys
if kvkey in ['section', 'plugin']:
continue
try:
if target[kvkey] is not None:
pass
except KeyError:
# some always need to be passed
if kvkey in ['parent', 'asis']:
target[kvkey] = keyvals[kvkey]
else:
unsupported_options.append(kvkey)
if unsupported_options:
sys.exit("ERROR: Unsupported options for section [%s]: %s" % (log['section'], ",".join(unsupported_options)))
log['status'] = 0
log['options'] = target
return
def loadPlugins(self, basedir, topdir):
if self.loaded:
print("Cannot load plugins multiple times")
exit(1)
self.loaded = True
# find the loader utility so we can bootstrap ourselves
try:
m = imp.load_source("LoadClasses", os.path.join(basedir, "LoadClasses.py"));
except ImportError:
print("ERROR: unable to load LoadClasses that must contain the class loader object")
exit(1)
cls = getattr(m, "LoadClasses")
a = cls()
# setup the loader object
self.loader = a.__class__();
# Setup the array of directories we will search for plugins
# Note that we always look at the topdir location by default
plugindirs = []
plugindirs.append(topdir)
if self.options['plugindir']:
# could be a comma-delimited list, so split on commas
x = self.options['plugindir'].split(',')
for y in x:
# prepend so we always look at the given
# location first in case the user wants
# to "overload/replace" a default MTT
# class definition
plugindirs.insert(0, y)
# Traverse the plugin directory tree and add all
# the class definitions we can find
for dirPath in plugindirs:
try:
filez = os.listdir(dirPath)
for file in filez:
file = os.path.join(dirPath, file)
if os.path.isdir(file):
self.loader.load(file)
except:
if not self.options['ignoreloadpatherrs']:
print("Plugin directory",dirPath,"not found")
sys.exit(1)
# Build the stages plugin manager
self.stages = PluginManager()
# set the location
self.stages.setPluginPlaces(plugindirs)
# Get a list of all the categories - this corresponds to
# the MTT stages that have been defined. Note that we
# don't need to formally define the stages here - anyone
# can add a new stage, or delete an old one, by simply
# adding or removing a plugin directory.
self.stages.setCategoriesFilter(self.loader.stages)
# Load all plugins we find there
self.stages.collectPlugins()
# Build the tools plugin manager - tools differ from sections
# in that they are plugins we will use to execute the various
# sections. For example, the TestRun section clearly needs the
# ability to launch jobs. There are many ways to launch jobs
# depending on the environment, and sometimes several ways to
# start jobs even within one environment (e.g., mpirun vs
# direct launch).
self.tools = PluginManager()
# location is the same
self.tools.setPluginPlaces(plugindirs)
# Get the list of tools - not every tool will be capable
# of executing. For example, a tool that supports direct launch
# against a specific resource manager cannot be used on a
# system being managed by a different RM.
self.tools.setCategoriesFilter(self.loader.tools)
# Load all the tool plugins
self.tools.collectPlugins()
# Tool plugins are required to provide a function we can
# probe to determine if they are capable of operating - check
# those now and prune those tools that cannot support this
# environment
# Build the utilities plugins
self.utilities = PluginManager()
# set the location
self.utilities.setPluginPlaces(plugindirs)
# Get the list of available utilities.
self.utilities.setCategoriesFilter(self.loader.utilities)
# Load all the utility plugins
self.utilities.collectPlugins()
# since we use these all over the place, find the
# ExecuteCmd and ModuleCmd plugins and record them
availUtil = list(self.loader.utilities.keys())
for util in availUtil:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if "ExecuteCmd" == pluginInfo.plugin_object.print_name():
self.execmd = pluginInfo.plugin_object
elif "ModuleCmd" == pluginInfo.plugin_object.print_name():
self.modcmd = pluginInfo.plugin_object
# initialize this module
self.modcmd.setCommand(self.options)
elif "Watchdog" == pluginInfo.plugin_object.print_name():
self.watchdog = pluginInfo.plugin_object
if self.execmd is not None and self.modcmd is not None and self.watchdog is not None:
break
if self.execmd is None:
print("ExecuteCmd plugin was not found")
print("This is a basic capability required")
print("for MTT operations - cannot continue")
sys.exit(1)
# Configure harasser plugin
print(self.tools.getPluginsOfCategory("Harasser"))
for pluginInfo in self.tools.getPluginsOfCategory("Harasser"):
print(pluginInfo.plugin_object.print_name())
if "Harasser" == pluginInfo.plugin_object.print_name():
self.harasser = pluginInfo.plugin_object
break
if self.harasser is None:
print("Harasser plugin was not found")
print("This is required for all TestRun plugins")
print("cannot continue")
sys.exit(1)
# similarly, capture the highest priority defaults stage here
pri = -1
for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"):
if pri < pluginInfo.plugin_object.priority():
self.defaults = pluginInfo.plugin_object
pri = pluginInfo.plugin_object.priority()
return
def printInfo(self):
# Print the available MTT sections out, if requested
if self.options['listsections']:
print("Supported MTT stages:")
# print them in the default order of execution
for stage in self.loader.stageOrder:
print(" " + stage)
exit(0)
# Print the detected plugins for a given stage
if self.options['listplugins']:
# if the list is '*', print the plugins for every stage
if self.options['listplugins'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['listplugins'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(section):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid stage name " + section)
print()
exit(1)
# Print the options for a given plugin
if self.options['liststageoptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['liststageoptions'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['liststageoptions'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(section):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid stage name " + section)
print()
exit(1)
# Print the available MTT tools out, if requested
if self.options['listtools']:
print("Available MTT tools:")
availTools = list(self.loader.tools.keys())
for tool in availTools:
print(" " + tool)
exit(0)
# Print the detected tool plugins for a given tool type
if self.options['listtoolmodules']:
# if the list is '*', print the plugins for every type
if self.options['listtoolmodules'] == "*":
print()
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtoolmodules'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid tool type name",tool)
print()
exit(1)
# Print the options for a given plugin
if self.options['listtooloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listtooloptions'] == "*":
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtooloptions'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid tool type name " + tool)
print()
exit(1)
# Print the available MTT utilities out, if requested
if self.options['listutils']:
print("Available MTT utilities:")
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
print(" " + util)
exit(0)
# Print the detected utility plugins for a given tool type
if self.options['listutilmodules']:
# if the list is '*', print the plugins for every type
if self.options['listutilmodules'] == "*":
print()
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutilitymodules'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid utility type name")
print()
exit(1)
# Print the options for a given plugin
if self.options['listutiloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listutiloptions'] == "*":
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutiloptions'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
print(" " + pluginInfo.plugin_object.print_name() + ":")
pluginInfo.plugin_object.print_options(self, " ")
except KeyError:
print(" Invalid utility type name " + util)
print()
exit(1)
# if they asked for the version info, print it and exit
if self.options['version']:
for pluginInfo in self.tools.getPluginsOfCategory("Version"):
print("MTT Base: " + pluginInfo.plugin_object.getVersion())
print("MTT Client: " + pluginInfo.plugin_object.getClientVersion())
sys.exit(0)
def openLogger(self):
# there must be a logger utility or we can't do
# anything useful
if not self.utilities.activatePluginByName("Logger", "Base"):
print("Required Logger plugin not found or could not be activated")
sys.exit(1)
# execute the provided test description
self.logger = self.utilities.getPluginByName("Logger", "Base").plugin_object
self.logger.open(self)
return
def fill_log_interpolation(self, basestr, sublog):
if isinstance(sublog, str):
self.config.set("LOG", basestr, sublog.replace("$","$$"))
elif isinstance(sublog, dict):
for k,v in sublog.items():
self.fill_log_interpolation("%s.%s" % (basestr, k), v)
elif isinstance(sublog, list):
if sum([((isinstance(t, list) or isinstance(t, tuple)) and len(t) == 2) for t in sublog]) == len(sublog):
self.fill_log_interpolation(basestr, {k:v for k,v in sublog})
else:
for i,v in enumerate(sublog):
self.fill_log_interpolation("%s.%d" % (basestr, i), v)
else:
self.fill_log_interpolation(basestr, str(sublog))
def configTest(self):
# setup the configuration parser
self.config = configparser.SafeConfigParser(interpolation=configparser.ExtendedInterpolation())
# Set the config parser to make option names case sensitive.
self.config.optionxform = str
# fill ENV section with environemt variables
self.config.add_section('ENV')
for k,v in os.environ.items():
self.config.set('ENV', k, v.replace("$","$$"))
# Add LOG section filled with log results of stages
self.config.add_section('LOG')
thefulllog = self.logger.getLog(None)
for e in thefulllog:
self.fill_log_interpolation(e['section'].replace(":","_"), e)
# log the list of files - note that the argument parser
# puts the input files in a list, with the first member
# being the list of input files
self.log['inifiles'] = self.args.ini_files[0]
# initialize the list of active sections
self.actives = []
# if they specified a list to execute, then use it
sections = []
if self.args.section:
sections = self.args.section.split(",")
skip = False
elif self.args.skipsections:
sections = self.args.skipsections.split(",")
skip = True
else:
sections = None
# cycle thru the input files
for testFile in self.log['inifiles']:
if not os.path.isfile(testFile):
print("Test description file",testFile,"not found!")
sys.exit(1)
self.config.read(self.log['inifiles'])
# Check for ENV input
required_env = []
all_file_contents = []
for testFile in self.log['inifiles']:
file_contents = open(testFile, "r").read()
file_contents = "\n".join(["%s %d: %s" % (testFile.split("/")[-1],i,l) for i,l in enumerate(file_contents.split("\n")) if not l.lstrip().startswith("#")])
all_file_contents.append(file_contents)
if "${ENV:" in file_contents:
required_env.extend([s.split("}")[0] for s in file_contents.split("${ENV:")[1:]])
env_not_found = set([e for e in required_env if e not in os.environ.keys()])
lines_with_env_not_found = []
for file_contents in all_file_contents:
lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \
for l in file_contents.split("\n") \
if sum(["${ENV:%s}"%e in l for e in env_not_found])])
if lines_with_env_not_found:
print("ERROR: Not all required environment variables are defined.")
print("ERROR: Still need:")
for l in lines_with_env_not_found:
print("ERROR: %s"%l)
sys.exit(1)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
# if we are to filter the sections, then do so
takeus = True
if sections is not None:
found = False
for sec in sections:
if sec == section:
found = True
sections.remove(sec)
if skip:
takeus = False
break
if not found and not skip:
takeus = False
if takeus:
self.actives.append(section)
if sections is not None and 0 != len(sections) and not skip:
print("ERROR: sections were specified for execution and not found:",sections)
sys.exit(1)
return
# Used with combinatorial executor, loads next .ini file to be run with the
# sequential executor
def configNewTest(self, file):
# clear the configuration parser
for section in self.config.sections():
self.config.remove_section(section)
# read in the file
self.config.read(file)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
return
def executeTest(self, enable_loop=True, executor="sequential"):
if not self.loaded:
print("Plugins have not been loaded - cannot execute test")
exit(1)
if self.config is None:
print("No test definition file was parsed - cannot execute test")
exit(1)
if not self.tools.getPluginByName(executor, "Executor"):
print("Specified executor %s not found" % executor)
exit(1)
# activate the specified plugin
self.tools.activatePluginByName(executor, "Executor")
# execute the provided test description
executor = self.tools.getPluginByName(executor, "Executor")
status = executor.plugin_object.execute(self)
if status == 0 and self.options['clean_after'] and os.path.isdir(self.options['scratchdir']):
self.logger.verbose_print("Cleaning up scratchdir after successful run")
shutil.rmtree(self.options['scratchdir'])
# Loop forever if specified
if enable_loop and self.options['loopforever']:
while True:
self.logger.reset()
self.executeTest(enable_loop=False, executor=executor)
return status
def executeCombinatorial(self, enable_loop=True):
return self.executeTest(enable_loop=enable_loop, executor="combinatorial")
def printOptions(self, options):
# if the options are empty, report that
if not options:
lines = ["None"]
return lines
# create the list of options
opts = []
vals = list(options.keys())
for val in vals:
opts.append(val)
if options[val][0] is None:
opts.append("None")
elif isinstance(options[val][0], bool):
if options[val][0]:
opts.append("True")
else:
opts.append("False")
elif isinstance(options[val][0], list):
opts.append(" ".join(options[val][0]))
elif isinstance(options[val][0], int):
opts.append(str(options[val][0]))
else:
opts.append(options[val][0])
opts.append(options[val][1])
# print the options, their default value, and
# the help description in 3 column format
max1 = 0
max2 = 0
for i in range(0,len(opts),3):
# we want all the columns to line up
# and left-justify, so first find out
# the max len of each of the first two
# column entries
if len(opts[i]) > max1:
max1 = len(opts[i])
if type(opts[i+1]) is not str:
optout = str(opts[i+1])
else:
optout = opts[i+1]
if len(optout) > max2:
max2 = len(optout)
# provide some spacing
max1 = max1 + 4
max2 = max2 + 4
# cycle thru again, padding each entry to
# align the columns
lines = []
sp = " "
for i in range(0,len(opts),3):
line = opts[i] + (max1-len(opts[i]))*sp
if type(opts[i+1]) is not str:
optout = str(opts[i+1])
else:
optout = opts[i+1]
line = line + optout + (max2-len(optout))*sp
# to make this more readable, we will wrap the line at
# 130 characters. First, see if the line is going to be
# too long
if 130 < (len(line) + len(opts[i+2])):
# split the remaining column into individual words
words = opts[i+2].split()
first = True
for word in words:
if (len(line) + len(word)) < 130:
if first:
line = line + word
first = False
else:
line = line + " " + word
else:
lines.append(line)
line = (max1 + max2)*sp + word
if 0 < len(line):
lines.append(line)
else:
# the line is fine - so just add the last piece
line = line + opts[i+2]
# append the result
lines.append(line)
# add one blank line
lines.append("")
return lines
def selectPlugin(self, name, category):
if category == "stage":
try:
availStages = list(self.loader.stages.keys())
for stage in availStages:
for pluginInfo in self.stages.getPluginsOfCategory(stage):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "tool":
try:
availTools = list(self.loader.tools.keys())
for tool in availTools:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "utility":
try:
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
else:
print("Unrecognized category:",category)
return None
class HandlerManager(QThread):
# 返回命令执行结果
# int: 0 -- 返回PlainText
# 1 -- 返回RichText
# str: 结果
outSignal = pyqtSignal(int, str)
def __init__(self):
super(HandlerManager, self).__init__()
# 创建工作队列，用户输入的命令按照顺序put进该队列
# 由CommandHandler在后台逐个处理
self.workQueue = Queue()
# 初始化插件功能
self.initPlugin()
def __del__(self):
self.wait()
def initPlugin(self):
'''
启动插件管理器，获取插件列表
'''
# 创建插件管理器
self.pluginManager = PluginManager()
# 设置插件接口为 DPlugin，所有插件必须继承categories.DPlugin
self.pluginManager.setCategoriesFilter({"DPlugin": DPlugin})
# 设置插件目录
self.pluginManager.setPluginPlaces(['plugins'])
# 加载插件到内存
self.pluginManager.locatePlugins()
self.pluginManager.loadPlugins()
def getAllPlugins(self):
'''获取插件列表'''
return self.pluginManager.getPluginsOfCategory('DPlugin')
def activateAllPlugins(self):
'''使能所有插件'''
for plugin in self.pluginManager.getPluginsOfCategory('DPlugin'):
plugin.plugin_object.activate()
def deactivateAllPlugins(self):
'''禁用所有插件'''
for plugin in self.pluginManager.getPluginsOfCategory('DPlugin'):
plugin.plugin_object.deactivate()
def activatePlugin(self, name):
'''使能特定插件'''
self.pluginManager.activatePluginByName(name, category="DPlugin")
def deactivatePlugin(self, name):
'''使能特定插件'''
self.pluginManager.deactivatePluginByName(name, category="DPlugin")
def processInput(self, userInput):
'''将命令放入队列'''
self.workQueue.put((False, userInput))
def processFile(self, filepath):
'''将待处理文件放入队列'''
self.workQueue.put((True, filepath))
def run(self):
'''不断从Queue中获取数据，然后执行解析命令'''
while True:
isFile, userInput = self.workQueue.get()
if isFile:
self.execFile(userInput)
else:
self.execCommand(userInput, None)
def execCommand(self, userInput, currentFile):
'''
解析命令
'''
if not len(userInput) or userInput.startswith('#'):
return
# 命令回显
self.writeCommand(userInput)
# 命令分割
command, *params = userInput.split(maxsplit=1)
if command == 'source':
if len(params) == 1:
filepath = params[0]
if not os.path.isabs(filepath):
basepath = os.path.dirname(currentFile)
filepath = os.path.normpath(filepath)
filepath = os.path.join(basepath, filepath)
self.execFile(filepath)
else:
self.writeStderr('syntax error: ' + userInput)
else:
self.dispatchCommand(userInput)
def execFile(self, filepath):
self.writeCommand('Processing file ' + filepath)
if os.path.isfile(filepath) and os.access(filepath, os.R_OK):
with open(filepath, 'r') as f:
for line in f.readlines():
line = line.strip()
self.execCommand(line, filepath)
else:
self.writeStderr('Cannot open file: ' + filepath)
def dispatchCommand(self, userInput):
'''将命令分发给各个插件'''
status = DPlugin.EXEC_NOTFOUND
for plugin in self.pluginManager.getPluginsOfCategory('DPlugin'):
ret, resultText = plugin.plugin_object.execCommand(userInput)
if ret == DPlugin.EXEC_SUCCESS:
self.writeStdout(resultText)
status = DPlugin.EXEC_SUCCESS
elif ret == DPlugin.EXEC_FAILED:
self.writeStderr(resultText)
status = DPlugin.EXEC_FAILED
if status == DPlugin.EXEC_NOTFOUND:
self.writeStderr(userInput + ': Command not found')
def writeCommand(self, text):
'''
输出原始命令，蓝色加粗显示，以示区分
'''
self.outSignal.emit(1, text)
def writeStdout(self, text):
'''
返回普通输出，文本后加换行
'''
self.outSignal.emit(0, text)
def writeStderr(self, text):
'''
返回错误输出，红色加粗显示，以示区分
'''
self.outSignal.emit(2, text)
