class ProfilesManager(object): def __init__(self): self.plugin_location = [PROFILE_MODULE_DIR] self.pluginManager = PluginManager(plugin_info_ext="plugin") self.pluginManager.setPluginPlaces(self.plugin_location) self.pluginManager.collectPlugins() def modules_info(self): """ Get information with regards to each modules loaded. It includes author, category, copyright, description, details, name, version and website. :return: information of all modules loaded """ modules_info = {} for pluginInfo in self.pluginManager.getAllPlugins(): modules_info[pluginInfo.name] = pluginInfo return modules_info def configForms(self, device_serial, module_name=""): """ Get the configuration views of each modules to be displayed on web interface :return: dictionary of pluginInfo as key and form as value """ if module_name: plugin = self.pluginManager.getPluginByName(name=module_name) configForms = plugin.plugin_object.get_view() else: configForms = {} for pluginInfo in self.pluginManager.getAllPlugins(): form = pluginInfo.plugin_object.get_view() configForms[pluginInfo] = form return configForms def run_simulation(self,option, profile_name, duration, device_serial, package_name, session): """ Run profile simulation script :return: """ plugin = self.pluginManager.getPluginByName(name=profile_name) if option == RANDOM_INTERACTION: plugin.plugin_object.runSimulation(duration,package_name, random=True, device_serial=device_serial, session=session) elif option == SCRIPTED_PROFILE_INTERACTION: plugin.plugin_object.runSimulation(duration,package_name, random=False, device_serial=device_serial, session=session) def setup_device(self,module, params, device_serial): """ install profile apk and profile app data onto device :return: """ pluginInfo = self.pluginManager.getPluginByName(name=module) if pluginInfo.name == module: pluginInfo.plugin_object.prepare(params, device_serial) return True
def run_output_plugins(**kwargs): logger = logging.getLogger(__name__) logger.info("Begin run_output_plugins") simplePluginManager = PluginManager() logging.getLogger('yapsy').setLevel(logging.DEBUG) simplePluginManager.setCategoriesFilter({ "OutputResults": output_plugin }) # Tell it the default place(s) where to find plugins if logger: logger.debug("Plugin directories: %s" % (kwargs['output_plugin_directories'])) simplePluginManager.setPluginPlaces(kwargs['output_plugin_directories']) simplePluginManager.collectPlugins() plugin_cnt = 0 plugin_start_time = time.time() for plugin in simplePluginManager.getAllPlugins(): if logger: logger.info("Starting plugin: %s" % (plugin.name)) if plugin.plugin_object.initialize_plugin(details=plugin.details): plugin.plugin_object.emit(prediction_date=kwargs['prediction_date'].astimezone(timezone("US/Eastern")).strftime("%Y-%m-%d %H:%M:%S"), execution_date=kwargs['prediction_run_date'].strftime("%Y-%m-%d %H:%M:%S"), ensemble_tests=kwargs['site_model_ensemble']) plugin_cnt += 1 else: logger.error("Failed to initialize plugin: %s" % (plugin.name)) logger.debug("%d output plugins run in %f seconds" % (plugin_cnt, time.time() - plugin_start_time)) logger.info("Finished run_output_plugins")
def pane_data(self): list = [] # Holds the filter tuples (name, activated=True) # Get the default plugin directory, using XML path = os.path.expanduser("~") xml = xml_controller.Controller(path + "\.cxvrc.xml") xml.load_file() if os.path.exists(os.path.expanduser("~") + os.sep + "plugins"): default_dir = os.path.expanduser("~") + os.sep + "plugins" else: default_dir = self.dicom_view.get_main_dir() + os.sep + "plugins" if xml.get_plugin_directory() == "" or xml.get_plugin_directory() is None: directory = [default_dir] else: directory = [default_dir, xml.get_plugin_directory()] # Load the plugins from the default plugin directory. manager = PluginManager() manager.setPluginPlaces(directory) manager.setPluginInfoExtension("plugin") manager.collectPlugins() # Append tuple with plugin name and enabled=True to the list for plugin in manager.getAllPlugins(): list.append((plugin.name, True)) return list
def test_updatePluginPlaces(self): class SpecificLocator(IPluginLocator): pass pm = PluginManager() pm.setPluginPlaces(["bla/bli"]) pm.updatePluginPlaces(["mif/maf"]) self.assertEqual(set(["bla/bli","mif/maf"]),set(pm.getPluginLocator().plugins_places))
def _run_reporting(self, results, objfile): #options options = dict() # Build the PluginManager reportingPluginManager = PluginManager() reportingPluginManager.setPluginPlaces(["reporting"]) reportingPluginManager.collectPlugins() # Trigger run from the "Reporting" plugins for pluginInfo in reportingPluginManager.getAllPlugins(): reportingModul = pluginInfo.plugin_object reportingModul.set_task(self.task) # Give it the the relevant reporting.conf section. try: options = self.cfgReporting.get(pluginInfo.name) reportingModul.set_options(options) except Exception: log.error("Reporting module %s not found in configuration file", pluginInfo.name) # If the processing module is disabled in the config, skip it. if not options.enabled: continue log.debug("Run Reporting: " + pluginInfo.name) try: # Run the Reporting module reportingModul.run(results, objfile) except Exception as e: log.exception("Failed to run the reporting module \"%s\":", reportingModul.__class__.__name__)
class FurnivallApplication(tornado.web.Application): def __init__(self): """ Sets up the Tornado web server and loads all the init data """ # Init mongodb database self.dbconnection = Connection() self.db = self.dbconnection['furnivall'] # Init plugins self.plugin_manager = PluginManager() self.plugin_manager.setPluginPlaces(["./plugins"]) self.plugin_manager.collectPlugins() print "Loaded plugins:" for plugin_info in self.plugin_manager.getAllPlugins(): print ' * ' + plugin_info.name # Init routes urls = [ ] + Route.routes() settings = dict( static_path = os.path.join(data_dir, "static"), template_path = os.path.join(data_dir, "templates"), xsrf_cookies = False, cookie_secret = "11oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1ao/Vo=", ) tornado.web.Application.__init__(self, urls, **settings)
def group_dashboard(request, group_id): # check user is allowed to access this user = request.user config_installed = 'config' in settings.INSTALLED_APPS user_level = user.userprofile.level machine_group = get_object_or_404(MachineGroup, pk=group_id) business_unit = machine_group.business_unit if business_unit not in user.businessunit_set.all(): if user_level != 'GA': return redirect(index) if user_level == 'GA' or user_level == 'RW': is_editor = True else: is_editor = False machines = machine_group.machine_set.all() # Build the manager manager = PluginManager() # Tell it the default place(s) where to find plugins manager.setPluginPlaces([settings.PLUGIN_DIR, os.path.join(settings.PROJECT_DIR, 'server/plugins')]) # Load all plugins manager.collectPlugins() output = [] for plugin in manager.getAllPlugins(): data = {} data['name'] = plugin.name (data['html'], data['width']) = plugin.plugin_object.show_widget('group_dashboard', machines, machine_group.id) output.append(data) output = utils.orderPluginOutput(output, 'group_dashboard', machine_group.id) c = {'user': request.user, 'machine_group': machine_group, 'user_level': user_level, 'is_editor': is_editor, 'business_unit': business_unit, 'output':output, 'config_installed':config_installed, 'request':request} return render_to_response('server/group_dashboard.html', c, context_instance=RequestContext(request))
def load_plugins(cls): manager = PluginManager() manager.setPluginPlaces([os.path.join(os.getcwd(), "plugins/")]) manager.collectPlugins() ret = manager.getAllPlugins() logger.info('Loaded {} plugins'.format(len(ret))) return ret
def run(): logging.basicConfig(level=logging.INFO) os.chdir('/home/mjolnir/git/PURIKURA') # H A N D L E P L U G I N S pm = PluginManager() pm.setPluginPlaces(['./pyrikura/plugins']) pm.collectPlugins() for pi in pm.getAllPlugins(): logging.info('loading plugin %s', pi.name) pm.activatePluginByName(pi.name) brokers = {} nodes = build() head = nodes[0] for node in nodes: brokers[node] = node.load(pm) for node, broker in brokers.items(): for other in node._listening: broker.subscribe(brokers[other]) start = time.time() last_time = 0 shots = 0 last_trigger = 0 for broker in itertools.cycle(brokers.values()): broker.update()
class FakeSite(object): def __init__(self): self.template_system = self self.config = {"DISABLED_PLUGINS": [], "EXTRA_PLUGINS": [], "DEFAULT_LANG": "en"} self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"] self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "RestExtension": RestExtension, } ) self.loghandlers = [STDERR_HANDLER] self.plugin_manager.setPluginInfoExtension("plugin") if sys.version_info[0] == 3: places = [os.path.join(os.path.dirname(utils.__file__), "plugins")] else: places = [os.path.join(os.path.dirname(utils.__file__), utils.sys_encode("plugins"))] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() self.timeline = [FakePost(title="Fake post", slug="fake-post")] def render_template(self, name, _, context): return '<img src="IMG.jpg">'
def getBankRates(): # Load the plugins from the plugin directory. manager = PluginManager() # Server # manager.setPluginPlaces(["./dev/liborWatch/server/plugins"]) # Dev # manager.setPluginPlaces(["./plugins"]) # Local manager.setPluginPlaces(["./dev/NextLevelApps/liborWatch/server/plugins"]) manager.collectPlugins() today = datetime.date.today().strftime('%Y-%m-%d') # Loop round the plugins and get the bank rates for plugin in manager.getAllPlugins(): rates = plugin.plugin_object.getRates() print(rates); if(len(rates) != 10): print('Error: We did not get 10 rates from the plugin ' + plugin.name + ' - ' + str(len(rates))) return newRate = [] iCnt = 0; for rate in rates: iCnt = iCnt + 1 newRate.append((plugin.name, today, iCnt, rate)) # print('execute insert statement ' + plugin.name, str(today), str(iCnt), str(rate)) con = sqlite3.connect('liborWatch.sqlite') with con: cur = con.cursor() cur.execute("CREATE TABLE IF NOT EXISTS BankRate(id INTEGER PRIMARY KEY, bankName TEXT, date TEXT, fixedForYears INTEGER, rate REAL)") cur.executemany("INSERT INTO BankRate VALUES(NULL, ?, ?, ?, ?)", newRate)
class AlgorithmManager(): def __init__(self): self._pluginManager = PluginManager() self._pluginManager.setPluginPlaces(config.PLUGIN_DIR) self._pluginManager.collectPlugins() for pluginInfo in self._pluginManager.getAllPlugins(): self._pluginManager.activatePluginByName(pluginInfo.name) def get_alg_names(self): for plugin in self._pluginManager.getAllPlugins(): plugin.plugin_object.print_name() def execute_calc(self, data): run = [] if not self.__is_sequence(data): raise Exception('Data is no array!') for alg in self._pluginManager.getAllPlugins(): result, prop = alg.plugin_object.calc(data) run.append({alg.name : (result, prop)}) return run def __is_sequence(self, arg): return (not hasattr(arg, "strip") and hasattr(arg, "__getitem__") or hasattr(arg, "__iter__"))
def loadPlugins(root_folder, plugin_name, categories_filter): from yapsy.PluginManager import PluginManager plugin_dirs = [] plugin_dirs.append(configuration.fabfile_basedir + '/.fabalicious/plugins') plugin_dirs.append(expanduser("~") + '/.fabalicious/plugins') plugin_dirs.append(root_folder + '/plugins') log.debug("Looking for %s-plugins in %s" % (plugin_name, ", ".join(plugin_dirs))) manager = PluginManager() manager.setPluginPlaces(plugin_dirs) manager.setCategoriesFilter(categories_filter) manager.collectPlugins() # Activate all loaded plugins for pluginInfo in manager.getAllPlugins(): manager.activatePluginByName(pluginInfo.name) result = {} for plugin in manager.getAllPlugins(): if hasattr(plugin.plugin_object, 'aliases') and isinstance(plugin.plugin_object.aliases, list): for alias in plugin.plugin_object.aliases: result[alias] = plugin.plugin_object elif hasattr(plugin.plugin_object, 'alias'): result[plugin.plugin_object.alias] = plugin.plugin_object else: result[plugin.name] = plugin.plugin_object return result
def main(): parser = argparse.ArgumentParser(description='Validates file format') parser.add_argument('files', nargs='+', help='files to be validated') parser.add_argument('--verbose', action='store_true', default=False, help='Shows help about validation') args = parser.parse_args() simplePluginManager = PluginManager() simplePluginManager.setPluginPlaces(["plugins"]) simplePluginManager.collectPlugins() for filename in args.files: supported = False for plugin in simplePluginManager.getAllPlugins(): methods = plugin.plugin_object if methods.should_manage(filename): supported = True try: plugin.plugin_object.validate(filename) print('%s sintax ok for %s' % (filename, plugin.name)) break except Exception as e: print('Invalid file %s for %s' % (filename, plugin.name)) if args.verbose: print(e) if not supported: print('%s cannot be validated' % filename)
def collect_data(self, **kwargs): self.logger.info("Begin collect_data") simplePluginManager = PluginManager() logging.getLogger('yapsy').setLevel(logging.DEBUG) simplePluginManager.setCategoriesFilter({ "DataCollector": data_collector_plugin }) # Tell it the default place(s) where to find plugins self.logger.debug("Plugin directories: %s" % (kwargs['data_collector_plugin_directories'])) simplePluginManager.setPluginPlaces(kwargs['data_collector_plugin_directories']) simplePluginManager.collectPlugins() plugin_cnt = 0 plugin_start_time = time.time() for plugin in simplePluginManager.getAllPlugins(): self.logger.info("Starting plugin: %s" % (plugin.name)) if plugin.plugin_object.initialize_plugin(details=plugin.details): plugin.plugin_object.start() else: self.logger.error("Failed to initialize plugin: %s" % (plugin.name)) plugin_cnt += 1 #Wait for the plugings to finish up. self.logger.info("Waiting for %d plugins to complete." % (plugin_cnt)) for plugin in simplePluginManager.getAllPlugins(): plugin.plugin_object.join() self.logger.info("%d Plugins completed in %f seconds" % (plugin_cnt, time.time() - plugin_start_time))
def loadPlugins(pluginfldr,hivesdict): """ Enumerate plugins in the specified directory and return the populated plugin manager and a list of the compatible plugins. Compatibility is determined by checking if the required hives were loaded sucessfully. """ pluginmanager = PluginManager() pluginmanager.setPluginPlaces([pluginfldr]) pluginmanager.collectPlugins() print "" logging.info("[PLUGINS IDENTIFIED: %s]" % len(pluginmanager.getAllPlugins())) compat_plugins = list() incompat_plugins = list() for plugin in pluginmanager.getAllPlugins(): compat = 1 for req in plugin.plugin_object.getRequirements(): if (hivesdict[req][0] <> 1): compat = 0 break if compat: compat_plugins.append(plugin.name) else: incompat_plugins.append(plugin.name) logging.info(" [%s] Compatible Plugins:" % len(compat_plugins)) for plugin in compat_plugins: logging.info(" - %s" % plugin) logging.info(" [%s] Incompatible Plugins:" % len(incompat_plugins)) for plugin in incompat_plugins: logging.info(" - %s" % plugin) return pluginmanager, compat_plugins
def main(): # Read configuration config = SafeConfigParser( defaults = {'port':'15915', 'plugins_directory':'./plugins', 'plugins_enabled':'', }) config.read(['./conf/cm15d.conf', '/etc/cm15d.conf', '/etc/cm15d/cm15d.conf', '/etc/cm15d/conf.d/local.conf', ]) # Activate enabled plugins plugins = PluginManager() plugins.setPluginPlaces(config.get('cm15d', 'plugins_directory').split(',')) plugins.collectPlugins() plugins_enabled = config.get('cm15d', 'plugins_enabled').split(',') for plugin in plugins.getAllPlugins(): if plugin.name in plugins_enabled: plugins.activatePluginByName(plugin.name) print("Plugin %s enabled" % plugin.name) # Start server port = int(config.get('cm15d', 'port')) endpoint = TCP4ServerEndpoint(reactor, port) endpoint.listen(CM15DaemonFactory(plugins)) print("Server listening on port %s" % port) reactor.run()
def runCrawler(): mapURL = {} cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')) # Build the PluginManager crawlerPluginManager = PluginManager() crawlerPluginManager.setPluginPlaces(["crawler"]) crawlerPluginManager.collectPlugins() # Trigger run from the "Crawler" plugins for pluginInfo in sorted(crawlerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name): crawlerModul = pluginInfo.plugin_object # Config for crawler module try: options = cfgCrawler.get(pluginInfo.name) crawlerModul.set_options(options) except Exception: log.error("Crawler module %s not found in configuration file", pluginInfo.name) # If the crawler module is disabled in the config, skip it. if not options.enabled: continue try: log.debug("Run Crawler: " + pluginInfo.name) returnMap = crawlerModul.run() mapURL.update(returnMap) except Exception as e: log.error('Error (%s) in %s', e, pluginInfo.name) return mapURL
class FakeSite(object): def __init__(self): self.template_system = self self.config = { 'DISABLED_PLUGINS': [], 'EXTRA_PLUGINS': [], } self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS'] self.plugin_manager = PluginManager(categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "RestExtension": RestExtension, }) self.plugin_manager.setPluginInfoExtension('plugin') if sys.version_info[0] == 3: places = [ os.path.join(os.path.dirname(utils.__file__), 'plugins'), ] else: places = [ os.path.join(os.path.dirname(utils.__file__), utils.sys_encode('plugins')), ] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() def render_template(self, name, _, context): return('<img src="IMG.jpg">')
def __run_reporting_generator(self): # options options = dict() # Build the PluginManager reportingPluginManager = PluginManager() reportingPluginManager.setPluginPlaces(["reporting"]) reportingPluginManager.collectPlugins() # Trigger run from the "Reporting" plugins for pluginInfo in sorted(reportingPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name): reportingModulG = pluginInfo.plugin_object reportingModulG.set_task(self.task) # Give it the the relevant reporting.conf section. try: options = self.cfgReporting.get(pluginInfo.name) reportingModulG.set_options(options) except Exception: log.error("Reporting module %s not found in configuration file", pluginInfo.name) # If the processing module is disabled in the config, skip it. if not options.enabled: continue log.debug("Run Reporting: " + pluginInfo.name) yield reportingModulG
def init_manager(): anl = PluginFileAnalyzerMathingRegex('custom_res_handler_plugins', r'^[A-Za-z0-9]+\.py$') res = PluginFileLocator(plugin_info_cls=CFCustomResourceHandler) res.setAnalyzers([anl]) manager = PluginManager(plugin_locator=res, categories_filter={'CFHandlers' : CFCustomResourceHandler}) manager.setPluginPlaces([dirname(__file__) + '/plugins']) manager.collectPlugins() return manager
class BotInteraction(): def __init__(self, config): self.channel = config['channel'] self.nickname = config['nickname'] self.prompt = config['prompt'] self.parts = "" self.ping = "" self.username = "" self.message = "" self.manager = PluginManager() self.manager.setPluginPlaces(["plugins"]) self.manager.collectPlugins() def handle_line(self, username, message, parts): print(username + ": " + message) if self.nickname not in parts[1] and message[0:1] == self.prompt: message = message.replace(">", " ").split(None, 1) for plugin in self.manager.getAllPlugins(): # Collect all plugins try: # Message[0] - search plugin_name plugin_yapsy = self.manager.getPluginByName(message[0]) command = [] # If the message has arguments - message[1] add to command if len(message) == 2: command = message[1] # To found plugin send arguments. return plugin_yapsy.plugin_object.execute(self.channel, username, command) except: continue def check_ping(self, line): if line.find("PING") != -1: # If server pings then pong line = string.split(line, " ") self.ping = "PONG " + line[1] else: self.ping = "" self.parts = string.split(line, ":") if "QUIT" not in self.parts[1] and \ "JOIN" not in self.parts[1] and \ "PART" not in self.parts[1]: try: # Sets the message variable to the actual message sent self.message = self.parts[2][:len(self.parts[2]) - 1] except: self.message = "" # Sets the username variable to the actual username usernamesplit = string.split(self.parts[1], "!") self.username = usernamesplit[0]
def index(request): # Get the current user's Business Units user = request.user # Count the number of users. If there is only one, they need to be made a GA if User.objects.count() == 1: # The first user created by syncdb won't have a profile. If there isn't one, make sure they get one. try: profile = UserProfile.objects.get(user=user) except UserProfile.DoesNotExist: profile = UserProfile(user=user) profile.level = "GA" profile.save() user_level = user.userprofile.level now = datetime.now() hour_ago = now - timedelta(hours=1) today = date.today() week_ago = today - timedelta(days=7) month_ago = today - timedelta(days=30) three_months_ago = today - timedelta(days=90) if user_level != "GA": # user has many BU's display them all in a friendly manner business_units = user.businessunit_set.all() if user.businessunit_set.count() == 1: # user only has one BU, redirect to it for bu in user.businessunit_set.all(): return redirect("server.views.bu_dashboard", bu_id=bu.id) break else: machines = Machine.objects.all() # Build the manager manager = PluginManager() # Tell it the default place(s) where to find plugins manager.setPluginPlaces([settings.PLUGIN_DIR, os.path.join(settings.PROJECT_DIR, "server/plugins")]) # Load all plugins manager.collectPlugins() output = [] # Loop round the plugins and print their names. for plugin in manager.getAllPlugins(): data = {} data["name"] = plugin.name (data["html"], data["width"]) = plugin.plugin_object.show_widget("front", machines) # output.append(plugin.plugin_object.show_widget('front')) output.append(data) output = utils.orderPluginOutput(output) # get the user level - if they're a global admin, show all of the machines. If not, show only the machines they have access to business_units = BusinessUnit.objects.all() config_installed = "config" in settings.INSTALLED_APPS c = { "user": request.user, "business_units": business_units, "output": output, "config_installed": config_installed, } return render_to_response("server/index.html", c, context_instance=RequestContext(request))
def main(): # Create plugin manager manager = PluginManager() manager.setPluginPlaces([PluginFolder]) # Collect plugins manager.collectPlugins() for plugin in manager.getAllPlugins(): plugin.plugin_object.say_hello()
def main(): # Load the plugins from the plugin directory. manager = PluginManager(plugin_info_ext='cfg') manager.setPluginPlaces(["../plugins"]) manager.collectPlugins() # Loop round the plugins and print their names. for plugin in manager.getAllPlugins(): plugin.plugin_object.print_name()
class Plugins(): ''' Plugins ''' def __init__(self): ''' Constructor ''' super(Plugins, self).__init__() self.write_tab_dock_plugin_dict = {} self.write_panel_dock_plugin_dict = {} # Build the manager self._plugin_manager = PluginManager() # List all sub-directories of "plugins" plugin_path = os.path.sep.join([os.getcwd(), "plugins"]) plugin_places = [plugin_path] for dirname, dirnames, filenames in os.walk(plugin_path): # print path to all subdirectories first. for subdirname in dirnames: plugin_places.append(os.path.join(dirname, subdirname)) # Tell it the default place(s) where to find plugins self._plugin_manager.setPluginPlaces(plugin_places) sys.path.append(plugin_path) # Define the various categories corresponding to the different # kinds of plugins you have defined self._plugin_manager.setCategoriesFilter({ "GuiWriteTabDockPlugin": GuiWriteTabDockPlugin, "GuiWritePanelDockPlugin": GuiWritePanelDockPlugin }) self._plugin_manager.collectPlugins() self.load_plugins( ["GuiWriteTabDockPlugin", "GuiWritePanelDockPlugin"]) def load_plugins(self, categories): ''' function:: load_plugins(categories) :param categories: list ''' for category in categories: for pluginInfo in self._plugin_manager.getPluginsOfCategory(category): setattr(self, pluginInfo.plugin_object.gui_class( ).__name__, pluginInfo.plugin_object.gui_class()) if category is "GuiWriteTabDockPlugin": self.write_tab_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \ = pluginInfo.plugin_object.gui_class() if category is "GuiWritePanelDockPlugin": self.write_panel_dock_plugin_dict[pluginInfo.plugin_object.gui_class().dock_name] \ = pluginInfo.plugin_object.gui_class()
class PluginHandler: def __init__(self): self._plugin_manager = PluginManager() self._category_active = {"Modifier": False, "Analyzer": False, "Comparator": False} self._plugin_from_category = {"Modifier": [], "Analyzer": [], "Comparator": []} self._modifier_plugins = [] self._analyzer_plugins = [] self._collect_all_plugins() def _collect_all_plugins(self): self._plugin_manager.setPluginPlaces( ["hugin/analyze/modifier", "hugin/analyze/analyzer", "hugin/analyze/comparator"] ) # setting filter categories for pluginmanager self._plugin_manager.setCategoriesFilter( { # movie metadata provider "Modifier": IModifier, # movie metadata provider "Comparator": IComparator, # sub metadata provider "Analyzer": IAnalyzer, } ) self._plugin_manager.collectPlugins() def activate_plugins_by_category(self, category): self._toggle_activate_plugins_by_category(category) def deactivate_plugins_by_category(self, category): self._toggle_activate_plugins_by_category(category) def _toggle_activate_plugins_by_category(self, category): plugins = self._plugin_manager.getPluginsOfCategory(category) is_active = self._category_active[category] for pluginInfo in plugins: if is_active: self._plugin_manager.deactivatePluginByName(name=pluginInfo.name, category=category) self._plugin_from_category[category].remove(pluginInfo) else: self._plugin_manager.activatePluginByName(name=pluginInfo.name, category=category) self._plugin_from_category[category].append(pluginInfo) self._category_active[category] = not is_active def get_plugins_from_category(self, category): plugins = [] for plugin in self._plugin_from_category[category]: plugin.plugin_object.name = plugin.name plugin.plugin_object.description = plugin.description plugins.append(plugin.plugin_object) return plugins def is_activated(self, category): """ True if category is activated. """ return self._category_active[category]
def plugins(): # Load the plugins from the plugin directory. manager = PluginManager() manager.setPluginPlaces(["plugins"]) manager.collectPlugins() # Loop round the plugins and print their names. for p in manager.getAllPlugins(): p.plugin_object.print_name() p.plugin_object.activate()
def main(): """docstring for main""" # Load the plugins from the plugin directory. manager = PluginManager() manager.setPluginPlaces(["plugins"]) manager.collectPlugins() # Loop round the plugins executing them. for plugin in manager.getAllPlugins(): plugin.plugin_object.run()
def main(): # Load the plugins from the plugin directory. manager = PluginManager() manager.setPluginPlaces(["orthos/plugins"]) manager.collectPlugins() # Loop round the plugins and print their names. for plugin in manager.getAllPlugins(): print "Fo" plugin.plugin_object.print_name()
class RetrieveResource(object): def __init__(self, args, yaml): self.simplePluginManager = PluginManager() self.args = args self.output_dir = args.output_dir if args.output_dir is not None else PIS_OUTPUT_DIR self.yaml = yaml # Warning the user about the gc credential needs for access to GC itself def checks_gc_service_account(self): if self.args.google_credential_key is None: logger.info( "Some of the steps might be not work properly due the lack of permissions to access to GCS. " "Eg. Evidence") else: GoogleBucketResource.has_valid_auth_key( self.args.google_credential_key) # Copy the local files to the Google Storage def copy_to_gs(self): if self.args.google_bucket is not None: Utils(self.yaml.config, self.yaml.outputs).gsutil_multi_copy_to( self.args.google_bucket) else: logger.error("Destination bucket info missing") # This function normalise the input inserted by the user. Lower and Upper cases can break the code if # not managed. Eg. SO/so/So -> SO Plugin def normalise_steps(self, steps, all_plugins_available): normalise_steps = [] lowercase_steps = [each_step.lower() for each_step in steps] for plugin in all_plugins_available: if plugin.lower() in lowercase_steps: normalise_steps.append(plugin) lowercase_steps.remove(plugin.lower()) logger.info("Steps not found:\n" + ','.join(lowercase_steps)) return normalise_steps # Extract and check the steps to run def steps(self): all_plugins_available = [] for plugin in self.simplePluginManager.getAllPlugins(): all_plugins_available.append(plugin.name) steps_requested = self.normalise_steps(self.args.steps, all_plugins_available) excluded_requested = self.normalise_steps(self.args.exclude, all_plugins_available) if len(self.args.steps) == 0: plugin_order = list( set(all_plugins_available) - set(excluded_requested)) else: plugin_order = list(set(steps_requested)) logger.info("Steps selected:\n" + ','.join(plugin_order)) return plugin_order # Init yapsy plugin manager def init_plugins(self): # Tell it the default place(s) where to find plugins self.simplePluginManager.setPluginPlaces(["plugins"]) # Load all plugins self.simplePluginManager.collectPlugins() # noinspection PyBroadException # Given a list of steps to run, this procedure executes the selected plugins/step def run_plugins(self): steps_to_execute = self.steps() for plugin_name in steps_to_execute: plugin = self.simplePluginManager.getPluginByName(plugin_name) try: plugin.plugin_object.process(self.yaml[plugin_name.lower()], self.yaml.outputs, self.yaml.config) except Exception as e: logger.info( "WARNING Plugin not available {}".format(plugin_name)) logger.info(e) def create_output_structure(self, output_dir): """By default the directories prod and staging are created""" remove_output_dir( output_dir) if self.args.force_clean else logger.info( "Warning: Output not deleted.") self.yaml.outputs.prod_dir = create_output_dir(output_dir + '/prod') self.yaml.outputs.staging_dir = create_output_dir(output_dir + '/staging') # Retrieve the resources requested. def run(self): self.create_output_structure(self.output_dir) self.init_plugins() self.checks_gc_service_account() self.run_plugins() self.copy_to_gs()
def index(request): # Get the current user's Business Units user = request.user # Count the number of users. If there is only one, they need to be made a GA if User.objects.count() == 1: # The first user created by syncdb won't have a profile. If there isn't one, make sure they get one. try: profile = UserProfile.objects.get(user=user) except UserProfile.DoesNotExist: profile = UserProfile(user=user) profile.level = 'GA' profile.save() user_level = user.userprofile.level now = datetime.now() hour_ago = now - timedelta(hours=1) today = date.today() week_ago = today - timedelta(days=7) month_ago = today - timedelta(days=30) three_months_ago = today - timedelta(days=90) config_installed = 'config' in settings.INSTALLED_APPS if user_level != 'GA': # user has many BU's display them all in a friendly manner business_units = user.businessunit_set.all() if user.businessunit_set.count() == 0: c = {'user': request.user, } return render_to_response('server/no_access.html', c, context_instance=RequestContext(request)) if user.businessunit_set.count() == 1: # user only has one BU, redirect to it for bu in user.businessunit_set.all(): return redirect('server.views.bu_dashboard', bu_id=bu.id) break if user_level == 'GA': machines = Machine.objects.all() else: machines = Machine.objects.none() for business_unit in user.businessunit_set.all(): for group in business_unit.machinegroup_set.all(): machines = machines | group.machine_set.all() # Load in the default plugins if needed utils.loadDefaultPlugins() # Build the manager manager = PluginManager() # Tell it the default place(s) where to find plugins manager.setPluginPlaces([settings.PLUGIN_DIR, os.path.join(settings.PROJECT_DIR, 'server/plugins')]) # Load all plugins manager.collectPlugins() output = [] # Get all the enabled plugins enabled_plugins = Plugin.objects.all().order_by('order') for enabled_plugin in enabled_plugins: # Loop round the plugins and print their names. for plugin in manager.getAllPlugins(): if plugin.name == enabled_plugin.name: data = {} data['name'] = plugin.name (data['html'], data['width']) = plugin.plugin_object.show_widget('front', machines) output.append(data) break output = utils.orderPluginOutput(output) # get the user level - if they're a global admin, show all of the machines. If not, show only the machines they have access to if user_level == 'GA': business_units = BusinessUnit.objects.all() else: business_units = user.businessunit_set.all() c = {'user': request.user, 'business_units': business_units, 'output': output, } return render_to_response('server/index.html', c, context_instance=RequestContext(request))
class Phong(object): def __init__(self): self._log = logging.getLogger( "%s.%s" % (self.__module__, self.__class__.__name__)) analyzer = PluginFileAnalyzerWithInfoFile('phong', 'phong-plugin') self._plugins = PluginManager() self._plugins.getPluginLocator().setAnalyzers([analyzer]) self._plugins.setPluginPlaces([ './plugins/', '/usr/lib/phong/plugins', ]) self._config = ConfigParser.ConfigParser() self.loadConfig( os.path.sep.join((os.path.dirname(__file__), 'defaults.cfg'))) self._wiki = None self._commands = [] self._spiff = None @property def pluginManager(self): return self._plugins() @property def config(self): return self._config @property def spiff(self): if spiff is not None and self._spiff is None: self._spiff = spiff.API(self.config.get('phong', 'spiff-api')) return self._spiff @property def wiki(self): if self._wiki is None: url = self._config.get('phong', 'mediawiki-url') api = self._config.get('phong', 'mediawiki-api-url') username = self._config.get('phong', 'mediawiki-username') password = self._config.get('phong', 'mediawiki-password') self._wiki = phong.wiki.Wiki(url, api, username, password) return self._wiki def buildContext(self): context = { 'phong': templates.PhongTemplateAPI(self), } for plugin in self._plugins.getAllPlugins(): cxt = plugin.plugin_object.buildContext(self) if isinstance(cxt, dict): context.update(cxt) return context def getTemplate(self, name, defaultContext={}, buildContext=True, prefix=None): if buildContext: cxt = self.buildContext() cxt.update(defaultContext) else: cxt = defaultContext engines = [] if self.config.has_option('phong', 'template-paths'): for path in self.config.get('phong', 'template-paths').split(','): engines.append(phong.templates.FileEngine(self, path.strip())) engines.append(phong.templates.FileEngine(self, 'templates')) engines.append(phong.templates.WikiEngine(self)) for e in engines: if e.hasTemplate(name, prefix): return e.getTemplate(name, prefix, cxt) def renderTemplate(self, name, context={}): cxt = self.buildContext() cxt.update(context) return self.getTemplate(name).render(context) def loadPlugins(self): self._plugins.collectPlugins() for plugin in self._plugins.getAllPlugins(): self._plugins.activatePluginByName(plugin.name) for cmd in plugin.plugin_object.availableCommands(): self._commands.append(cmd(self, plugin)) def loadConfig(self, path): self._log.debug("Loading configuration from %s", path) self._config.read(os.path.expanduser(path)) def getState(self, name): dir = os.path.expanduser(self._config.get('phong', 'state-dir')) return phong.state.State( os.path.sep.join((dir, "%s.state.json" % (name)))) @property def argv(self): return self._argv @property def args(self): return self._args def main(self, argv): parser = argparse.ArgumentParser(prog='phong', add_help=False) parser.add_argument( '-c', '--config', help= 'Configuration file to use. Can be specified multiple times. Later files override earlier ones.', action='append', default=[ "/etc/phong.cfg", ]) parser.add_argument('-d', '--dry-run', help='Dont actually do anything', default=False, action='store_true') parser.add_argument('-D', '--debug', help='Print debug messages', default=False, action='store_true') args = parser.parse_known_args(argv) parser.add_argument('-h', '--help', help='show this help message and exit', action='help') if args[0].debug: logging.basicConfig(level=logging.DEBUG) for config in args[0].config: self.loadConfig(config) self.loadPlugins() subparser = parser.add_subparsers(help='action') for command in self._commands: pluginArgs = subparser.add_parser(command.name(), help=command.helpText()) pluginArgs.set_defaults(command_obj=command) command.buildArgs(pluginArgs) self._argv = argv self._args = parser.parse_args(argv) return self._args.command_obj.execute(self._args)
print ("Board type: OpenBCI Ganglion") import openbci.ganglion as bci else: raise ValueError('Board type %r was not recognized. Known are 3 and 4' % args.board) # Check AUTO port selection, a "None" parameter for the board API if "AUTO" == args.port.upper(): print("Will try do auto-detect board's port. Set it manually with '--port' if it goes wrong.") args.port = None else: print("Port: ", args.port) plugins_paths = ["plugins"] if args.plugins_path: plugins_paths += args.plugins_path manager.setPluginPlaces(plugins_paths) manager.collectPlugins() # Print list of available plugins and exit if args.list: print ("Available plugins:") for plugin in manager.getAllPlugins(): print ("\t- " + plugin.name) exit() # User wants more info about a plugin... if args.info: plugin = manager.getPluginByName(args.info) if plugin == None: # eg: if an import fail inside a plugin, yapsy skip it print ("Error: [ " + args.info + " ] not found or could not be loaded. Check name and requirements.")
@hug.get("/", output=hug.output_format.html) def get_root(response): return dashboard.on_get(persistence_mgr) def plugins_callback(event_id, future): """Attach the value returned by a plugin to the associated event.""" return_val = future.result(0) if return_val is not None: persistence_mgr.attach_info(event_id, return_val) mongodb_uri = config.config.get("server", "MongoDBUri") persistence_mgr = persistence.PersistenceManager(mongodb_uri) executor = ThreadPoolExecutor() plugin_mgr = PluginManager() plugin_mgr.setPluginPlaces(["./counterserver/plugins"]) plugin_mgr.setCategoriesFilter({ IEventReceiverPlugin.__name__: IEventReceiverPlugin, }) plugin_mgr.collectPlugins() for p in plugin_mgr.getAllPlugins(): logging.info("Plugin '" + p.name + "' loaded") if __name__ == '__main__': hug.API(__name__).http.serve()
class Controller(EventEmitter): """Main controller of the class. Views+ui separated to different files.""" def __init__(self): """Constructor.""" # run the superclass constructor EventEmitter.__init__(self) # Model initialization code self.sensors = [] self.tags = [] # define important directories for external (not program code) files homedir = os.environ["HOME"] ldrop_home = os.path.join(homedir, "Documents", "ldrop_data") self.rootdir = ldrop_home self.plugindir = os.path.join(ldrop_home, "plugins") self.savedir = os.path.join(ldrop_home, "recordings") # check that saving, experiment etc directories are present utils.dircheck(self.savedir) utils.dircheck(self.plugindir) # put the plugins-directory sys.path.append(self.plugindir) # temporary? keyboard-contigency list self.keyboard_contigency = [] # initialize plugin manager self.pluginmanager = PluginManager(plugin_locator=LdropPluginLocator()) self.pluginmanager.setPluginPlaces([self.plugindir]) self.pluginmanager.collectPlugins() self.gui = [] self.play_callback = None self.stop_callback = None self.continue_callback = None self.data_callback = None self.participant_id = "" def add_model(self, model): """Add a model to listen for.""" #TODO: currently same add_model for different kinds of emitters # (sensors and experiments), just counting they send different signals model.on("tag", self.on_tag) model.on("data", self.on_data) model.on("close_controller", self.on_close_controller) model.on("start_collecting_data", self.on_start_collecting_data) model.on("stop_collecting_data", self.on_stop_collecting_data) model.on("log_message", self.on_log_message) model.on("query", self.on_query) def add_sensor(self, sensor_name): """Callback for Add sensor -button.""" # TODO: Improve APIs for plugins plugin_info = self.pluginmanager.getPluginByName(sensor_name) if plugin_info is None: print("Plugin " + sensor_name + " not found") else: plugin_info.plugin_object.get_sensor(self.rootdir, self.on_sensor_created, self.on_sensor_error) def close(self): """Method that closes the drop controller.""" # disconnect all the sensors from the host for sensor in self.sensors: # TODO: this is done on stop_collecting data - unify sensor.stop_recording() sensor.disconnect() self.remove_all_listeners() self.ml.quit() print("ldrop mainloop stopped.") def close_gui(self): """Clear gui reference.""" self.gui = [] # run in what condition self.close() def continue_experiment(self): """Callback for continuebutton click.""" if self.continue_callback is not None: self.continue_callback() def enable_gui(self): """Initialize pygtk-view to be run when mainloop starts.""" self.gui.append(LDPV(self, self.savedir)) def get_participant_id(self): """Return participant_id.""" return self.participant_id def get_sensors(self): """Return list of connected sensors.""" return self.sensors def get_sensor_plugins(self): """Return list of available sensors.""" plugins = self.pluginmanager.getAllPlugins() sensornames = [] for p in plugins: sensornames.append(p.name) return sensornames def message_to_sensor(self, sensortype, msg): """ Callback for a message to sensor. Sensor needs to support the msg. """ # find the right sensor(s) to forward the message to for sensor in self.sensors: if sensor.get_type == sensortype: sensor.on_message(msg) def on_close_controller(self): """Callback for signal close_controller.""" glib.idle_add(self.close) def on_data(self, dp): """Callback for data-signal.""" if self.data_callback is not None: glib.idle_add(self.data_callback, dp) def on_experiment_completed(self): """Callback for experiment finished.""" # clear view references for r in self.sensors: self.exp_view.remove_model(r) # self.exp_view = None def on_keypress(self, keyname): """Callback for keypress.""" if keyname in self.keyboard_contigency: self.keyboard_contigency = [] self.emit("continue") tag = { "id": keyname, "secondary_id": "keypress", "timestamp": self.timestamp() } self.on_tag("tag", tag) def on_query(self, msg, title, buttons, callbacks, callback_args): if len(self.gui) > 0: for g in self.gui: g.show_message_box(msg, title, buttons, callbacks, callback_args) def on_sensor_error(self, msg): """Sensor error-handler.""" self.emit("error", msg) def on_sensor_created(self, shandle): """Callback for sensor initialization.""" self.sensors.append(shandle) self.emit("sensorcount_changed") # add model to hear calls from sensors, such as data_condition met self.add_model(shandle) def on_start_collecting_data(self, savesubdir, savefilestring): """A callback for start_collecting_data signal.""" self.start_collecting_data(savesubdir, savefilestring) def on_stop_collecting_data(self): """A callback for stop_collecting_data signal.""" self.stop_collecting_data(None) def on_tag(self, tag): """ Send a tag to all sensors. Tag might not come instantly here so the timestamp is taken in advance. The sensor must sync itself with the computer. Tag consists of: id = string, identifier of the tag timestamp = timestamp in ms of the localtime clock secondary_id = string, defines "start", "end", or "impulse", as a start of perioid, end of it or single impulse misc = other possible information (depends on the sensor how to use) """ for sensor in self.sensors: # send a copy of the dict to each sensor sensor.tag(tag.copy()) def on_log_message(self, logmsg): """Callback for log_append signal.""" self.emit("log_update", logmsg) def play(self): """Start the experiment.""" # TODO: possibly change the "pipeline" of the drop-involvement in exp if self.play_callback is not None: glib.idle_add(self.play_callback) def run(self): """Initialize controller start mainloop.""" # if no gui to control experiment is present, just start running the # experiment if len(self.gui) == 0 and self.play_callback is not None: self.play() self.ml = glib.MainLoop() self.ml.run() def remove_model(self, model): """Remove model listeners.""" model.remove_listener("tag", self.on_tag) model.remove_listener("data", self.on_data) model.remove_listener("close_controller", self.on_close_controller) model.remove_listener("start_collecting_data", self.on_start_collecting_data) model.remove_listener("stop_collecting_data", self.on_stop_collecting_data) model.remove_listener("log_message", self.on_log_message) model.remove_listener("query", self.on_query) def remove_sensor(self, sensor_id): """Disconnect the sensor with the provided sensor_id.""" for sensor in self.sensors: if sensor.get_sensor_id() == sensor_id: sensor.disconnect() self.sensors.remove(sensor) self.emit("sensorcount_changed") def set_callbacks(self, play_callback, stop_callback, continue_callback, data_callback): """Experiment side callback-setter.""" self.play_callback = play_callback self.stop_callback = stop_callback self.continue_callback = continue_callback self.data_callback = data_callback def set_participant_id(self, pid): """Method for setting participant_id.""" self.participant_id = pid self.emit("participant_id_updated") def sensor_action(self, sensor_id, action_id): """Perform action that is listed on sensors control elements.""" for sensor in self.sensors: if sensor.get_sensor_id() == sensor_id: sensor.action(action_id) def start_collecting_data(self, savesubdir, savefilestring): """Function starts data collection on all sensors.""" savepath = os.path.join(self.savedir, savesubdir) for sensor in self.sensors: sensor.start_recording(savepath, savefilestring) # sensor.start_recording(savepath, self.participant_id, # self.experiment_id) def stop(self): """Callback for stopbutton click.""" if self.stop_callback is not None: self.stop_callback() def stop_collecting_data(self, callback): """Stop data collection on all sensors and run callback.""" for sensor in self.sensors: sensor.stop_recording() if callback is not None: glib.idle_add(callback) def timestamp(self): """Return a local timestamp in microsecond accuracy.""" return time.time() def __del__(self): """Destructor.""" print("ldrop instance deleted.")
class FakeSite(object): def __init__(self): self.template_system = self self.invariant = False self.config = { 'DISABLED_PLUGINS': [], 'EXTRA_PLUGINS': [], 'DEFAULT_LANG': 'en', 'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'], 'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}', 'LISTINGS_FOLDERS': { 'listings': 'listings' }, } self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS'] self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "CompilerExtension": CompilerExtension, "MarkdownExtension": MarkdownExtension, "RestExtension": RestExtension }) self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8 self.shortcode_registry = {} self.plugin_manager.setPluginInfoExtension('plugin') if sys.version_info[0] == 3: places = [ os.path.join(os.path.dirname(nikola.utils.__file__), 'plugins'), ] else: places = [ os.path.join(os.path.dirname(nikola.utils.__file__), nikola.utils.sys_encode('plugins')), ] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() self.compiler_extensions = self._activate_plugins_of_category( "CompilerExtension") self.timeline = [FakePost(title='Fake post', slug='fake-post')] self.debug = True self.rst_transforms = [] self.post_per_input_file = {} # This is to make plugin initialization happy self.template_system = self self.name = 'mako' def _activate_plugins_of_category(self, category): """Activate all the plugins of a given category and return them.""" # this code duplicated in nikola/nikola.py plugins = [] for plugin_info in self.plugin_manager.getPluginsOfCategory(category): if plugin_info.name in self.config.get('DISABLED_PLUGINS'): self.plugin_manager.removePluginFromCategory( plugin_info, category) else: self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugins.append(plugin_info) return plugins def render_template(self, name, _, context): return ('<img src="IMG.jpg">') # this code duplicated in nikola/nikola.py def register_shortcode(self, name, f): """Register function f to handle shortcode "name".""" if name in self.shortcode_registry: nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name) return self.shortcode_registry[name] = f def apply_shortcodes(self, data): """Apply shortcodes from the registry on data.""" return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry)
class Friday: def __init__(self): self.ai = ai_interface.API( apiai.ApiAI(settings['CLIENT_ACCESS_TOKEN'], settings['SUBSCRIPTION_KEY'])) self.debugging = settings['debugging'] self.spoken_language = settings['spoken language'] self.input_system = settings['input system'] # google, local, text self.output_system = settings['output system'] # both, audio, text self.speech_file_location = settings['speech file location'] # . self.speech_file_name = settings['speech file name'] # audio response self.speak = settings['speak'] # True # The question that the assistant hears self.question = None # The chosen, spoken response given to the user. self.response = None # Whether Friday is active self.is_active = True # What type of question is being asked. self.request_type = None if settings['input system'] != 'text': self.recognizer = sr.Recognizer() self.microphone = sr.Microphone(device_index=settings['DEVICE'], sample_rate=settings['RATE'], chunk_size=settings['CHUNK']) if settings['input_system'] == 'google': with self.microphone as source: if settings['debugging']: click.echo("Adjusting to ambient noise.") # we only need to calibrate once, before we start listening self.recognizer.adjust_for_ambient_noise(source) # Build the manager self.manager = PluginManager() # Tell it the default place(s) where to find plugins self.manager.setPluginPlaces(settings["plugin folder"]) # Load all plugins self.manager.locatePlugins() self.manager.loadPlugins() self.plugins = {} # Activate all loaded plugins for plugin in self.manager.getAllPlugins(): self.plugins[plugin.name] = plugin.plugin_object def _local_stt(self): pass def _apiai_stt(self): from math import log import audioop import pyaudio import time resampler = apiai.Resampler(source_samplerate=settings['RATE']) request = self.ai.voice_request() vad = apiai.VAD() def callback(in_data, frame_count): frames, data = resampler.resample(in_data, frame_count) if settings.show_decibels: decibel = 20 * log(audioop.rms(data, 2) + 1, 10) click.echo(decibel) state = vad.processFrame(frames) request.send(data) state_signal = pyaudio.paContinue if state == 1 else pyaudio.paComplete return in_data, state_signal p = pyaudio.PyAudio() stream = p.open(format=pyaudio.paInt32, input=True, output=False, stream_callback=callback, channels=settings['CHANNELS'], rate=settings['RATE'], frames_per_buffer=settings['CHUNK']) stream.start_stream() click.echo("Speak!") while stream.is_active(): time.sleep(0.1) stream.stop_stream() stream.close() p.terminate() def _google_stt(self): """ Uses Google's Speech to Text engine to understand speech and convert it into text. :return: String, either the spoken text or error text. """ click.echo("Please speak now") # Listen to the microphone set up in the __init__ file. with self.microphone as mic_source: audio = self.recognizer.listen(mic_source) click.echo("Processing audio") try: # Try to recognize the text return self.recognizer.recognize_google(audio, show_all=self.debugging) except sr.UnknownValueError: # speech is unintelligible return "Google could not understand the audio." except sr.RequestError: return "Could not request results from Google's speech recognition service." def listen(self): # Default value. # TODO: Input plugin system goes here # Plugins should be used for getting input so users can build their # own custom front-end to the application. # This will allow the speech to text functions to be moved out of the # main class and into their own plugins. if self.input_system == 'google': self.question = self._google_stt() else: self.question = click.prompt("Input your query") # TODO: Loading plugin system goes here # Eventually I want people to be able to build custom behavior # that executes when the response is being fetched. So there could # be a loading screen that plays music or something for example. click.echo("Wait for response...") # Get a response from the AI using the api interface. self.ai.get_response(self.question) # Clean up the response and turn it into a Python object that can be read self.response = self.ai.parse() return self.response def think(self, request): # TODO: Move plugins to a priority system # Eventually plugins should return an integer value which determines # their ordering when responding to a request, in the event multiple # plugins are able to respond to the same request. # This may eventually allow for watchdog plugins which check for # erroneous requests and alert the system by returning negative (Maybe) # priorities, which can then be used to safeguard a user's system or # settings. return any( plugin.can_perform(self, request) for name, plugin in self.plugins.items()) def perform(self, request): # Trigger 'some request' from the loaded plugins for name, plugin in self.plugins.items(): if plugin.can_perform(self, request): # TODO: Allow plugins to fail # Plugins will be able to return False in the event they fail, # allowing the system to then apologize to the user. plugin.perform(self, request) return True def refuse(self): # May become unnecessary as the plugins could be built to refuse # requests directly. click.echo("Can't do that") def apologize(self): # May eventually become unnecessary as plugins could be built to # return apologies directly. click.echo("I failed to do that, sorry.") def _print_response(self): click.echo(self.response) def _speak_response(self): self.say(self.response, title=self.speech_file_name, speak_aloud=self.speak) def respond(self): # TODO: Output plugin system goes here # Plugins should be used for providing output for the same reasons # that plugins should be used for providing input. if self.output_system == "audio": self._speak_response() elif self.output_system == "text": self._print_response() else: self._print_response() self._speak_response() def say(self, message, speak_aloud=True, title='Speak'): # May eventually be moved into its own plugin message = str(message) if not message.strip(): click.echo("No text to speak.") return computer_os = system() folder = os.path.split(title)[0] folder_path = os.path.join(self.speech_file_location, folder) if not os.path.exists(folder_path): os.makedirs(folder_path) home = settings['home'] if self.input_system == 'google': # Create the MP3 file which will speak the text title += '.wav' path = os.path.join(home, title) tts = gTTS(message, lang=self.spoken_language) tts.save(path) speech_file = "start /MIN {}" if computer_os != 'Windows': speech_file = "mpg123 {}" speech_file = speech_file.format(os.path.join(home, title)) else: # Create the Visual Basic code which will speak the text title += '.vbs' path = os.path.join(home, title) message = re.sub( '["\n\t]', '', message ) # Strip out characters that would be spoken by the system. if computer_os == "Windows": with open(path, 'w') as file: file.write(""" speaks="{}" Dim speaks, speech Set speech=CreateObject("sapi.spvoice") speech.Speak speaks """.format(message)) # Set up the file to be executed speech_file = ["cscript.exe", title] if computer_os != "Windows": speech_file = ["espeak", message] if speak_aloud: try: call(speech_file) except FileNotFoundError: if self.debugging: click.echo("File not accessible:" + str(speech_file))
class Plugins(): """ Class responsible to manage the plugins """ categories = [ [ "SitePreparsing", SitePreparsing, "Site wide plugins that execute before the parsing start." ], [ "SiteProcessor", SiteProcessor, "Plugins that process the whole site once after parsing." ], [ "SiteRendering", SiteRendering, "Plugins that render additional pages after the rendering." ], [ "PostProcessor", PostProcessor, "Plugins that process each post after they are parsed" ], [ "CollectionProcessor", CollectionProcessor, "Plugins that process each collection after posts are parsed" ], [ "TemplateFilter", TemplateFilter, "Plugins that define jinja2 filters to be used in templates" ], ] # for plugin info structure PLUGIN_CAT = 0 PLUGIN_NAME = 1 PLUGIN_DESC = 2 PLUGIN_ENABLE = 3 PLUGIN_MODULE_NAME = 4 PLUGIN_VERSION = 5 def __init__(self, plugin_directories, debug_log_fname, plugins_config): "Load plugins" categories_filter = {} for cat in self.categories: categories_filter[cat[0]] = cat[1] # single or multi-directory handling if not isinstance(plugin_directories, list): plugin_directories = [plugin_directories] self.plugins = PluginManager(plugin_info_ext='sitefab-plugin', categories_filter=categories_filter) self.plugins.setPluginPlaces(plugin_directories) self.plugins.locatePlugins() self.plugins.loadPlugins() self.plugins_config = plugins_config # List of enabled plugins self.plugins_enabled = {} for pl in self.get_plugins_info(): if pl[self.PLUGIN_ENABLE]: self.plugins_enabled[pl[self.PLUGIN_MODULE_NAME]] = 1 # list of plugins already executed. Used for dependencies tracking # across stages self.plugins_executed = {} # FIXME: make sure it is working # logging.basicConfig(filename=debug_log_fname, level=logging.DEBUG) def get_plugins(self, category=None): """Return the list of plugins :param str category: restrict to plugins of a given category :rtype: list(iPlugin) :return: list of plugins """ if category: return self.plugins.getPluginsOfCategory(category) else: return self.plugins.getAllPlugins() def get_num_plugins(self, category=None): """ Return the number of plugins available. :param str category: restrict to plugins of given category :rtype: int :return: number of plugins """ plugins = self.get_plugins(category) return len(plugins) def get_plugin_dir(self, plugin): """ Return the directory where the plugin is stored :param iPlugin plugin: the plugin requested :rtype: str :return: the module name """ module_path = Path(plugin.details.get("Core", "module")) return module_path.parent def get_plugin_default_configuration_filename(self, plugin): """ Return the path to the plugin default configuration filename """ try: fname = plugin.details.get("Configuration", "Filename") except: # noqa return "" fname = fname.replace('"', '') path = self.get_plugin_dir(plugin) return path / fname def get_plugin_documentation_filename(self, plugin): """ Return the path to the plugin documentation """ try: fname = plugin.details.get("Documentation", "Filename") except: # noqa return "" path = self.get_plugin_dir(plugin) return path / fname def get_plugin_class_name(self, plugin): """ Return the class of a given plugin :param iPlugin plugin: the plugin requested :rtype: str :return: the module classname """ return plugin.categories[0] def get_plugin_module_name(self, plugin): """ Return the module name of a given plugin :param iPlugin plugin: the plugin requested :rtype: str :return: the module name """ module_path = plugin.details.get("Core", "module") path, filename = os.path.split(module_path) return filename def get_plugin_config(self, plugin): """ Return the configuration of a given plugin :param iPlugin plugin: the plugin requested :rtype: dict :return: plugin configuration """ module_name = self.get_plugin_module_name(plugin) class_name = self.get_plugin_class_name(plugin) try: config = self.plugins_config[class_name][module_name] except: # noqa config = {} return config def get_plugin_dependencies(self, plugin): """ Return the dependency of a given plugin :param iPlugin plugin: the plugin requested :rtype: list :return: list of plugins name the plugin depend on """ # No dependencies if not plugin.details.has_option("Core", "Dependencies"): return set() dependencies = set() st = plugin.details.get("Core", "Dependencies") if "," in st: elts = st.split(",") for elt in elts: dependencies.add(elt.strip()) else: dependencies.add(st) return dependencies def is_plugin_enabled(self, plugin): config = self.get_plugin_config(plugin) if config.get('enable'): return True else: return False def get_plugins_info(self, category=None): """Return the list of plugins available with their type :param str category: restrict to plugins of a given category. :rtype: list(str) :return: list of plugins name """ pl = [] if category: categories = [category] else: categories = self.plugins.getCategories() for cat in categories: for plugin in self.plugins.getPluginsOfCategory(cat): enabled = self.is_plugin_enabled(plugin) module_name = self.get_plugin_module_name(plugin) try: version = plugin.version except: # noqa version = "NA" s = [ cat, plugin.name, plugin.description, enabled, module_name, version ] pl.append(s) return pl def display_execution_results(self, results, site): """ Display execution summary """ cprint("|-Execution result", "magenta") table_data = [['name', 'ok', 'skipped', 'errors']] count = 0 for result in results: row = [] plugin_name, stats = result # plugin name c = "cyan" if count % 2: c = "blue" row.append(colored(plugin_name, c)) # ok if stats[site.OK]: if stats[site.OK] == 1: val = 'v' else: val = stats[site.OK] row.append(colored(val, 'green')) else: row.append(' ') # warning if stats[site.SKIPPED]: row.append(colored(stats[site.SKIPPED], "yellow")) else: row.append(' ') # error if stats[site.ERROR]: if stats[site.ERROR] == 1: val = 'x' else: val = stats[site.ERROR] row.append(colored(val, "red")) else: row.append(' ') table_data.append(row) count += 1 print(SingleTable(table_data).table) def run_plugins(self, items, plugin_class, unit, site): """Execute a set of plugins on a given list of items :param list items: list of items to process :param str plugin_type: the plugin_class to use :param str unit: the unit to use in the display :param SiteFab site: pointer to the site object to be passed to the plugins :rtype: dict(dict(list)) :return: plugins execution statistics """ # dependencies map dependencie_map = {} # used to get back from the module name to the plugin module_name_to_plugin = {} plugins = self.plugins.getPluginsOfCategory(plugin_class) # collecting plugins that are to be executed. for plugin in plugins: if self.is_plugin_enabled(plugin): module_name = self.get_plugin_module_name(plugin) module_name_to_plugin[module_name] = plugin # dependencies computation. # Due to potential dependencies on plugins from previous stage # this must be computed after collecting which # plugins were executed. for plugin in module_name_to_plugin.values(): all_dependencies = self.get_plugin_dependencies(plugin) dependencies = set() # topological sort requires use of set module_name = self.get_plugin_module_name(plugin) for dep_module_name in all_dependencies: if dep_module_name not in self.plugins_enabled: utils.error("Plugin:%s can't be executed because\ plugin %s is not enable" % (module_name, dep_module_name)) # only add to the dependencies map the plugins # that are from the same stage if dep_module_name in module_name_to_plugin: dependencies.add(dep_module_name) else: # check if already executed if dep_module_name not in self.plugins_executed: utils.error("Plugin:%s can't be executed because\ plugin %s was not executed in previous\ stage" % (module_name, dep_module_name)) dependencie_map[module_name] = dependencies # print dependencie_map # Topological sorting try: plugins_to_process = toposort_flatten(dependencie_map) except Exception as e: utils.error("Circular dependencies between plugins.\ Can't execute plugins:%s" % e) s = "|-%s plugins" % (unit.strip().capitalize()) desc = colored(s, "magenta") results = [] for module_name in tqdm(plugins_to_process, unit=' plugin', desc=desc, leave=True): if module_name in module_name_to_plugin: plugin = module_name_to_plugin[module_name] else: raise Exception("The following plugin module name listed in\ dependencies don't exist % s " % module_name) pclass = plugin_class.lower() filename = "%s.%s.html" % (pclass, module_name) log_id = site.logger.create_log(pclass, plugin.name, filename) plugin_results = utils.dict_to_objdict({ site.OK: 0, site.SKIPPED: 0, site.ERROR: 0 }) config = self.get_plugin_config(plugin) for item in tqdm(items, unit=unit, desc=plugin.name, leave=False): result = plugin.plugin_object.process(item, site, config) plugin_results[result[0]] += 1 severity = result[0] name = result[1] details = result[2] site.logger.record_event(log_id, name, severity, details) self.plugins_executed[module_name] = True results.append([plugin.name, plugin_results]) site.logger.write_log(log_id) return results def get_template_filters(self): """Load template filters and return a dictionary list Return: dict: jinja filter functions """ template_filters = {} filters = self.plugins.getPluginsOfCategory("TemplateFilter") for flt in filters: filter_name = self.get_plugin_module_name(flt) template_filters[filter_name] = flt.plugin_object.myfilter return template_filters
class Nikola(object): """Class that handles site generation. Takes a site config as argument on creation. """ def __init__(self, **config): """Setup proper environment for running tasks.""" self.global_data = {} self.posts_per_year = defaultdict(list) self.posts_per_tag = defaultdict(list) self.timeline = [] self.pages = [] self._scanned = False # This is the default config # TODO: fill it self.config = { 'ARCHIVE_PATH': "", 'ARCHIVE_FILENAME': "archive.html", 'DEFAULT_LANG': "en", 'OUTPUT_FOLDER': 'output', 'FILES_FOLDERS': { 'files': '' }, 'LISTINGS_FOLDER': 'listings', 'ADD_THIS_BUTTONS': True, 'INDEX_DISPLAY_POST_COUNT': 10, 'INDEX_TEASERS': False, 'MAX_IMAGE_SIZE': 1280, 'USE_FILENAME_AS_TITLE': True, 'SLUG_TAG_PATH': False, 'INDEXES_TITLE': "", 'INDEXES_PAGES': "", 'FILTERS': {}, 'USE_BUNDLES': True, 'TAG_PAGES_ARE_INDEXES': False, 'THEME': 'default', 'post_compilers': { "rest": ['.txt', '.rst'], "markdown": ['.md', '.mdown', '.markdown'], "html": ['.html', '.htm'], }, } self.config.update(config) self.config['TRANSLATIONS'] = self.config.get( 'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''}) # FIXME: find way to achieve this with the plugins #if self.config['USE_BUNDLES'] and not webassets: #self.config['USE_BUNDLES'] = False self.GLOBAL_CONTEXT = self.config.get('GLOBAL_CONTEXT', {}) self.THEMES = utils.get_theme_chain(self.config['THEME']) self.MESSAGES = utils.load_messages(self.THEMES, self.config['TRANSLATIONS']) self.GLOBAL_CONTEXT['messages'] = self.MESSAGES self.GLOBAL_CONTEXT['_link'] = self.link self.GLOBAL_CONTEXT['rel_link'] = self.rel_link self.GLOBAL_CONTEXT['abs_link'] = self.abs_link self.GLOBAL_CONTEXT['exists'] = self.file_exists self.GLOBAL_CONTEXT['add_this_buttons'] = self.config[ 'ADD_THIS_BUTTONS'] self.GLOBAL_CONTEXT['index_display_post_count'] = self.config[ 'INDEX_DISPLAY_POST_COUNT'] self.GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES'] self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, }) self.plugin_manager.setPluginInfoExtension('plugin') self.plugin_manager.setPluginPlaces([ os.path.join(os.path.dirname(__file__), 'plugins'), os.path.join(os.getcwd(), 'plugins'), ]) self.plugin_manager.collectPlugins() self.commands = {} # Activate all command plugins for pluginInfo in self.plugin_manager.getPluginsOfCategory("Command"): self.plugin_manager.activatePluginByName(pluginInfo.name) pluginInfo.plugin_object.set_site(self) pluginInfo.plugin_object.short_help = pluginInfo.description self.commands[pluginInfo.name] = pluginInfo.plugin_object # Activate all task plugins for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"): self.plugin_manager.activatePluginByName(pluginInfo.name) pluginInfo.plugin_object.set_site(self) for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"): self.plugin_manager.activatePluginByName(pluginInfo.name) pluginInfo.plugin_object.set_site(self) # Load template plugin template_sys_name = utils.get_template_engine(self.THEMES) pi = self.plugin_manager.getPluginByName(template_sys_name, "TemplateSystem") if pi is None: sys.stderr.write("Error loading %s template system plugin\n" % template_sys_name) sys.exit(1) self.template_system = pi.plugin_object self.template_system.set_directories([ os.path.join(utils.get_theme_path(name), "templates") for name in self.THEMES ]) # Load compiler plugins self.compilers = {} self.inverse_compilers = {} for pluginInfo in self.plugin_manager.getPluginsOfCategory( "PageCompiler"): self.compilers[pluginInfo.name] = \ pluginInfo.plugin_object.compile_html def get_compiler(self, source_name): """Get the correct compiler for a post from `conf.post_compilers` To make things easier for users, the mapping in conf.py is compiler->[extensions], although this is less convenient for us. The majority of this function is reversing that dictionary and error checking. """ ext = os.path.splitext(source_name)[1] try: compile_html = self.inverse_compilers[ext] except KeyError: # Find the correct compiler for this files extension langs = [ lang for lang, exts in self.config['post_compilers'].items() if ext in exts ] if len(langs) != 1: if len(set(langs)) > 1: exit("Your file extension->compiler definition is" "ambiguous.\nPlease remove one of the file extensions" "from 'post_compilers' in conf.py\n(The error is in" "one of %s)" % ', '.join(langs)) elif len(langs) > 1: langs = langs[:1] else: exit("post_compilers in conf.py does not tell me how to " "handle '%s' extensions." % ext) lang = langs[0] compile_html = self.compilers[lang] self.inverse_compilers[ext] = compile_html return compile_html def render_template(self, template_name, output_name, context): data = self.template_system.render_template(template_name, None, context, self.GLOBAL_CONTEXT) assert output_name.startswith(self.config["OUTPUT_FOLDER"]) url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:] # This is to support windows paths url_part = "/".join(url_part.split(os.sep)) src = urlparse.urljoin(self.config["BLOG_URL"], url_part) parsed_src = urlparse.urlsplit(src) src_elems = parsed_src.path.split('/')[1:] def replacer(dst): # Refuse to replace links that are full URLs. dst_url = urlparse.urlparse(dst) if dst_url.netloc: if dst_url.scheme == 'link': # Magic link dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'), context['lang']) else: return dst # Normalize dst = urlparse.urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_dst = urlparse.urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute dst_elems = parsed_dst.path.split('/')[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break # Now i is the longest common prefix result = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) if not result: result = "." # Don't forget the fragment (anchor) part of the link if parsed_dst.fragment: result += "#" + parsed_dst.fragment assert result, (src, dst, i, src_elems, dst_elems) return result try: os.makedirs(os.path.dirname(output_name)) except: pass doc = lxml.html.document_fromstring(data) doc.rewrite_links(replacer) data = '<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8') with open(output_name, "w+") as post_file: post_file.write(data) def path(self, kind, name, lang, is_link=False): """Build the path to a certain kind of page. kind is one of: * tag_index (name is ignored) * tag (and name is the tag name) * tag_rss (name is the tag name) * archive (and name is the year, or None for the main archive index) * index (name is the number in index-number) * rss (name is ignored) * gallery (name is the gallery name) * listing (name is the source code file name) The returned value is always a path relative to output, like "categories/whatever.html" If is_link is True, the path is absolute and uses "/" as separator (ex: "/archive/index.html"). If is_link is False, the path is relative to output and uses the platform's separator. (ex: "archive\\index.html") """ path = [] if kind == "tag_index": path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'], 'index.html' ]) elif kind == "tag": if self.config['SLUG_TAG_PATH']: name = utils.slugify(name) path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'], name + ".html" ]) elif kind == "tag_rss": if self.config['SLUG_TAG_PATH']: name = utils.slugify(name) path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'], name + ".xml" ]) elif kind == "index": if name > 0: path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['INDEX_PATH'], 'index-%s.html' % name ]) else: path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['INDEX_PATH'], 'index.html' ]) elif kind == "rss": path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'], 'rss.xml' ]) elif kind == "archive": if name: path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['ARCHIVE_PATH'], name, 'index.html' ]) else: path = filter(None, [ self.config['TRANSLATIONS'][lang], self.config['ARCHIVE_PATH'], self.config['ARCHIVE_FILENAME'] ]) elif kind == "gallery": path = filter(None, [self.config['GALLERY_PATH'], name, 'index.html']) elif kind == "listing": path = filter(None, [self.config['LISTINGS_FOLDER'], name + '.html']) if is_link: return '/' + ('/'.join(path)) else: return os.path.join(*path) def link(self, *args): return self.path(*args, is_link=True) def abs_link(self, dst): # Normalize dst = urlparse.urljoin(self.config['BLOG_URL'], dst) return urlparse.urlparse(dst).path def rel_link(self, src, dst): # Normalize src = urlparse.urljoin(self.config['BLOG_URL'], src) dst = urlparse.urljoin(src, dst) # Avoid empty links. if src == dst: return "#" # Check that link can be made relative, otherwise return dest parsed_src = urlparse.urlsplit(src) parsed_dst = urlparse.urlsplit(dst) if parsed_src[:2] != parsed_dst[:2]: return dst # Now both paths are on the same site and absolute src_elems = parsed_src.path.split('/')[1:] dst_elems = parsed_dst.path.split('/')[1:] i = 0 for (i, s), d in zip(enumerate(src_elems), dst_elems): if s != d: break else: i += 1 # Now i is the longest common prefix return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:]) def file_exists(self, path, not_empty=False): """Returns True if the file exists. If not_empty is True, it also has to be not empty.""" exists = os.path.exists(path) if exists and not_empty: exists = os.stat(path).st_size > 0 return exists def gen_tasks(self): task_dep = [] for pluginInfo in self.plugin_manager.getPluginsOfCategory("Task"): for task in pluginInfo.plugin_object.gen_tasks(): yield task if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) for pluginInfo in self.plugin_manager.getPluginsOfCategory("LateTask"): for task in pluginInfo.plugin_object.gen_tasks(): yield task if pluginInfo.plugin_object.is_default: task_dep.append(pluginInfo.plugin_object.name) yield { 'name': 'all', 'actions': None, 'clean': True, 'task_dep': task_dep } def scan_posts(self): """Scan all the posts.""" if not self._scanned: print "Scanning posts ", targets = set([]) for wildcard, destination, _, use_in_feeds in \ self.config['post_pages']: print ".", for base_path in glob.glob(wildcard): post = Post(base_path, destination, use_in_feeds, self.config['TRANSLATIONS'], self.config['DEFAULT_LANG'], self.config['BLOG_URL'], self.MESSAGES) for lang, langpath in self.config['TRANSLATIONS'].items(): dest = (destination, langpath, post.pagenames[lang]) if dest in targets: raise Exception( 'Duplicated output path %r in post %r' % (post.pagenames[lang], base_path)) targets.add(dest) self.global_data[post.post_name] = post if post.use_in_feeds: self.posts_per_year[str(post.date.year)].append( post.post_name) for tag in post.tags: self.posts_per_tag[tag].append(post.post_name) else: self.pages.append(post) for name, post in self.global_data.items(): self.timeline.append(post) self.timeline.sort(cmp=lambda a, b: cmp(a.date, b.date)) self.timeline.reverse() post_timeline = [p for p in self.timeline if p.use_in_feeds] for i, p in enumerate(post_timeline[1:]): p.next_post = post_timeline[i] for i, p in enumerate(post_timeline[:-1]): p.prev_post = post_timeline[i + 1] self._scanned = True print "done!" def generic_page_renderer(self, lang, wildcard, template_name, destination, filters): """Render post fragments to final HTML pages.""" for post in glob.glob(wildcard): post_name = os.path.splitext(post)[0] context = {} post = self.global_data[post_name] deps = post.deps(lang) + \ self.template_system.template_deps(template_name) context['post'] = post context['lang'] = lang context['title'] = post.title(lang) context['description'] = post.description(lang) context['permalink'] = post.permalink(lang) context['page_list'] = self.pages output_name = os.path.join(self.config['OUTPUT_FOLDER'], self.config['TRANSLATIONS'][lang], destination, post.pagenames[lang] + ".html") deps_dict = copy(context) deps_dict.pop('post') if post.prev_post: deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)] if post.next_post: deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)] deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER'] deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS'] task = { 'name': output_name.encode('utf-8'), 'file_dep': deps, 'targets': [output_name], 'actions': [(self.render_template, [template_name, output_name, context])], 'clean': True, 'uptodate': [config_changed(deps_dict)], } yield utils.apply_filters(task, filters) def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context): """Renders pages with lists of posts.""" deps = self.template_system.template_deps(template_name) for post in posts: deps += post.deps(lang) context = {} context["posts"] = posts context["title"] = self.config['BLOG_TITLE'] context["description"] = self.config['BLOG_DESCRIPTION'] context["lang"] = lang context["prevlink"] = None context["nextlink"] = None context.update(extra_context) deps_context = copy(context) deps_context["posts"] = [(p.titles[lang], p.permalink(lang)) for p in posts] task = { 'name': output_name.encode('utf8'), 'targets': [output_name], 'file_dep': deps, 'actions': [(self.render_template, [template_name, output_name, context])], 'clean': True, 'uptodate': [config_changed(deps_context)] } yield utils.apply_filters(task, filters)
class DeepDecoder: def __init__(self): self.top_layer = None self.plugin_manager = PluginManager() self.plugin_locations = [path.join(expanduser("~"), '.phorcys', 'plugins')] try: environ_locations = os.environ['PLUGINS_DIR'] if environ_locations is not None and len(environ_locations) > 0: dirs = environ_locations.split(';') for d in dirs: if os.path.isdir(d): self.plugin_locations.append(d) else: pass except KeyError: pass self.plugin_manager.setPluginPlaces(directories_list=self.plugin_locations) self.plugin_manager.setCategoriesFilter(DecoderPlugin.filter()) self.plugin_manager.collectPlugins() def get_loaded_plugins(self): return self.plugin_manager.getPluginsOfCategory(DecoderPlugin.category()) def _complete_leaves(self): for leaf in self.top_layer.leaves: if leaf.parent is not None and leaf.parent.name == 'base64' and len(leaf.name) == 0: leaf.parent.name = 'text' leaf.parent.human_readable = True leaf.parent.raw_data = ''.join(leaf.parent.lines) leaf.parent.del_extracted_layer(leaf) continue if len(leaf.lines) == 0: try: leaf.lines = utils.to_hex_lines(bytes(leaf.raw_data)) leaf.name = 'raw' except Exception as e: pass def decode(self, data, **kwargs) -> Layer: parent = Layer() parent.raw_data = data parent.name = 'root' to_visit = [parent] while len(to_visit) != 0: next = to_visit.pop() layer = self.go_deeper(next, **kwargs) if layer is not None: layers = layer.extracted_layers if layers is not None: to_visit.extend(layers) self.top_layer = parent self._complete_leaves() return parent def inspect(self): pass def go_deeper(self, parent, **xargs) -> Optional[Layer]: protocols = [ HttpHeaders(), UrlEncoded(), Json(), Protobuf(), Base64(), Lzma(), Zlib(), Bzip(), Gzip(), Text(), Html(), Css(), ] plugins = [] for plugin in self.plugin_manager.getPluginsOfCategory(DecoderPlugin.category()): plugins.append(plugin.plugin_object) plugins.extend(protocols) protocols = plugins for p in protocols: try: return p(parent, **xargs) except Exception as e: # print(traceback.format_exc()) # print('Not %s' % type(p)) # print(e) pass return None
class Statick(object): """Code analysis front-end.""" def __init__(self, user_paths): """Initialize Statick.""" self.resources = Resources(user_paths) self.manager = PluginManager() self.manager.setPluginPlaces(self.resources.get_plugin_paths()) self.manager.setCategoriesFilter({ "Discovery": DiscoveryPlugin, "Tool": ToolPlugin, "Reporting": ReportingPlugin }) self.manager.collectPlugins() self.discovery_plugins = {} for plugin_info in self.manager.getPluginsOfCategory("Discovery"): self.discovery_plugins[plugin_info.plugin_object.get_name()] = \ plugin_info.plugin_object self.tool_plugins = {} for plugin_info in self.manager.getPluginsOfCategory("Tool"): self.tool_plugins[plugin_info.plugin_object.get_name()] = \ plugin_info.plugin_object self.reporting_plugins = {} for plugin_info in self.manager.getPluginsOfCategory("Reporting"): self.reporting_plugins[plugin_info.plugin_object.get_name()] = \ plugin_info.plugin_object self.config = [] self.exceptions = [] def get_config(self, args): """Get Statick configuration.""" config_filename = "config.yaml" if args.config is not None: config_filename = args.config self.config = Config(self.resources.get_file(config_filename)) def get_exceptions(self, args): """Get Statick exceptions.""" exceptions_filename = "exceptions.yaml" if args.exceptions is not None: exceptions_filename = args.exceptions self.exceptions = Exceptions(self.resources.get_file(exceptions_filename)) def get_ignore_packages(self): """Get packages to ignore during scan process.""" return self.exceptions.get_ignore_packages() def gather_args(self, args): """Gather arguments.""" args.add_argument("--output-directory", dest="output_directory", type=str, help="Directory to write output files to") args.add_argument("--show-tool-output", dest="show_tool_output", action="store_true", help="Show tool output") args.add_argument("--config", dest="config", type=str, help="Name of config yaml file") args.add_argument("--profile", dest="profile", type=str, help="Name of profile yaml file") args.add_argument("--exceptions", dest="exceptions", type=str, help="Name of exceptions yaml file") args.add_argument("--force-tool-list", dest="force_tool_list", type=str, help="Force only the given list of tools to run") args.add_argument('--version', action='version', version='%(prog)s {version}'.format(version=__version__)) args.add_argument('--mapping-file-suffix', dest="mapping_file_suffix", type=str, help="Suffix to use when searching for CERT mapping files") for _, plugin in list(self.discovery_plugins.items()): plugin.gather_args(args) for _, plugin in list(self.tool_plugins.items()): plugin.gather_args(args) for _, plugin in list(self.reporting_plugins.items()): plugin.gather_args(args) def get_level(self, path, args): """Get level to scan package at.""" path = os.path.abspath(path) profile_filename = "profile.yaml" if args.profile is not None: profile_filename = args.profile profile_resource = self.resources.get_file(profile_filename) if profile_resource is None: print("Could not find profile file {}!".format(profile_filename)) return None try: profile = Profile(profile_resource) except OSError as ex: # This isn't quite redundant with the profile_resource check: it's possible # that something else triggers an OSError, like permissions print("Failed to access profile file {}: {}".format(profile_filename, ex)) return None except ValueError as ex: print("Profile file {} has errors: {}".format(profile_filename, ex)) return None package = Package(os.path.basename(path), path) level = profile.get_package_level(package) return level def run(self, path, args): # pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements """Run scan tools against targets on path.""" success = True path = os.path.abspath(path) if not os.path.exists(path): print("No package found at {}!".format(path)) return None, False package = Package(os.path.basename(path), path) level = self.get_level(path, args) if not self.config or not self.config.has_level(level): print("Can't find specified level {} in config!".format(level)) return None, False orig_path = os.getcwd() if args.output_directory: if not os.path.isdir(args.output_directory): print("Output directory not found at {}!". format(args.output_directory)) return None, False output_dir = os.path.join(args.output_directory, package.name + "-" + level) if not os.path.isdir(output_dir): os.mkdir(output_dir) if not os.path.isdir(output_dir): print("Unable to create output directory at {}!".format( output_dir)) return None, False print("Writing output to: {}".format(output_dir)) os.chdir(output_dir) print("------") print("Scanning package {} ({}) at level {}".format(package.name, package.path, level)) issues = {} ignore_packages = self.get_ignore_packages() if package.name in ignore_packages: print("Package {} is configured to be ignored by Statick.".format(package.name)) return issues, True plugin_context = PluginContext(args, self.resources, self.config) print("---Discovery---") if not DiscoveryPlugin.file_command_exists(): print("file command isn't available, discovery plugins will be less effective") discovery_plugins = self.config.get_enabled_discovery_plugins(level) if not discovery_plugins: discovery_plugins = list(self.discovery_plugins.keys()) for plugin_name in discovery_plugins: if plugin_name not in self.discovery_plugins: print("Can't find specified discovery plugin {}!".format(plugin_name)) return None, False plugin = self.discovery_plugins[plugin_name] plugin.set_plugin_context(plugin_context) print("Running {} discovery plugin...".format(plugin.get_name())) plugin.scan(package, level, self.exceptions) print("{} discovery plugin done.".format(plugin.get_name())) print("---Discovery---") print("---Tools---") enabled_plugins = self.config.get_enabled_tool_plugins(level) plugins_to_run = copy.copy(enabled_plugins) plugins_ran = [] plugin_dependencies = [] while plugins_to_run: plugin_name = plugins_to_run[0] if plugin_name not in self.tool_plugins: print("Can't find specified tool plugin {}!".format(plugin_name)) return None, False if args.force_tool_list is not None: force_tool_list = args.force_tool_list.split(",") if plugin_name not in force_tool_list and plugin_name not in plugin_dependencies: print("Skipping plugin not in force list {}!".format(plugin_name)) plugins_to_run.remove(plugin_name) continue plugin = self.tool_plugins[plugin_name] plugin.set_plugin_context(plugin_context) dependencies = plugin.get_tool_dependencies() dependencies_met = True for dependency_name in dependencies: if dependency_name not in plugins_ran: if dependency_name not in enabled_plugins: print("Plugin {} depends on plugin {} which isn't " "enabled!".format(plugin_name, dependency_name)) return None, False plugin_dependencies.append(dependency_name) plugins_to_run.remove(dependency_name) plugins_to_run.insert(0, dependency_name) dependencies_met = False if not dependencies_met: continue print("Running {} tool plugin...".format(plugin.get_name())) tool_issues = plugin.scan(package, level) if tool_issues is not None: issues[plugin_name] = tool_issues print("{} tool plugin done.".format(plugin.get_name())) else: print("{} tool plugin failed".format(plugin.get_name())) success = False plugins_to_run.remove(plugin_name) plugins_ran.append(plugin_name) print("---Tools---") issues = self.exceptions.filter_issues(package, issues) os.chdir(orig_path) print("---Reporting---") reporting_plugins = self.config.get_enabled_reporting_plugins(level) if not reporting_plugins: reporting_plugins = self.reporting_plugins.keys() for plugin_name in reporting_plugins: if plugin_name not in self.reporting_plugins.keys(): print("Can't find specified reporting plugin {}!".format(plugin_name)) return None, False plugin = self.reporting_plugins[plugin_name] plugin.set_plugin_context(plugin_context) print("Running {} reporting plugin...".format(plugin.get_name())) plugin.report(package, issues, level) print("{} reporting plugin done.".format(plugin.get_name())) print("---Reporting---") print("Done!") return issues, success
from .ClusterProcessAlgorithmsClass import * from .ConnectionClasses import MySQLDBConnectionOpts, SOLRCollection from yapsy.PluginManager import PluginManager from .PluginClasses import * from .ResultClasses import * from .compute_similarity import similarity_columns # Url, Title, Fulltext, Plaintext sim_weights = [0.05, 0.25, 0.2, 0.5] # Initialize plugin manager and plugins plugin_manager = PluginManager() cur_dir = os.path.dirname(os.path.abspath(__file__)) plugin_manager.setPluginPlaces([ os.path.join(cur_dir, "cluster_plugins"), os.path.join(cur_dir, "summary_plugins"), os.path.join(cur_dir, "rank_plugins") ]) plugin_manager.setCategoriesFilter({ "cluster": IClusterPlugin, "summary": ISummaryPlugin, "rank": IRankPlugin, }) plugin_manager.collectPlugins() # Cosine similarity to cosine distance def cos_sim_to_dist(sim_score): return 1.0 - sim_score
from yapsy.PluginManager import PluginManager import json, sys # Load the plugins from the plugin directory. manager = PluginManager() manager.setPluginPlaces(["plugins"]) manager.collectPlugins() def main(file): # Loop round the plugins and print their names. message = json.loads(open(file).read()) plugin = manager.getPluginByName(message['type']) if plugin: plugin.plugin_object.print_name() plugin.plugin_object.retry(**message['target'], message = message.get('message')) if __name__ == "__main__": if len(sys.argv) > 1: main(sys.argv[1])
long_name = all_props[name]["displaytext"] else: long_name = name return long_name # Helper class used to pass the necessary context information to the export plugin PluginExportContext = namedtuple("PluginExportContext", [ "objectFeaturesSlot", "labelImageSlot", "rawImageSlot", "additionalPluginArgumentsSlot" ]) ############### # the manager # ############### pluginManager = PluginManager() pluginManager.setPluginPlaces(plugin_paths) pluginManager.setCategoriesFilter({ "ObjectFeatures": ObjectFeaturesPlugin, "TrackingExportFormats": TrackingExportFormatPlugin }) pluginManager.collectPlugins() for pluginInfo in pluginManager.getAllPlugins(): pluginManager.activatePluginByName(pluginInfo.name)
class EfetchPluginManager(object): """This class manages and creates plugin objects""" def __init__(self, plugins_file, curr_directory): # Plugin Manager Setup self.plugin_manager = PluginManager() self.plugin_manager.setPluginPlaces([curr_directory + u'/plugins/']) self.plugins_file = plugins_file self.reload_plugins() def reload_plugins_file(self): """Reloads all plugins from the YAML file""" self.config_file_plugins = self.load_plugin_config(self.plugins_file) def reload_plugins(self): """Reloads all Yapsy and YAML file plugins""" self.plugin_manager.collectPlugins() for plugin in self.plugin_manager.getAllPlugins(): self.plugin_manager.activatePluginByName(plugin.name) self.reload_plugins_file() def load_plugin_config(self, plugins_file): """Loads the plugin config file""" if not os.path.isfile(plugins_file): logging.warn(u'Could not find Plugin Configuration File "' + plugins_file + u'"') return {} with open(plugins_file, 'r') as stream: try: return yaml.load(stream) except yaml.YAMLError as exc: logging.error(u'Failed to parse Plugin Configuration File') logging.error(exc) return {} def get_all_plugins(self): """Gets a list of all the plugins""" plugins = [] for plugin in self.plugin_manager.getAllPlugins(): plugins.append(plugin.name) for key in self.config_file_plugins: plugins.append(key) return plugins def get_plugin_by_name(self, name): """Gets an Efetch plugin by name""" plugin = self.plugin_manager.getPluginByName(str(name).lower()) if not plugin and name not in self.config_file_plugins: logging.warn(u'Request made for unknown plugin "' + name + u'"') elif not plugin: plugin = self.config_file_plugins[name] return Plugin(plugin.get('name', 'None'), plugin.get('description', 'None'), plugin.get('cache', True), plugin.get('popularity', 5), plugin.get('fast', False), plugin.get('store', False), map(str.lower, plugin.get('mimetypes', [])), map(str.lower, plugin.get('extensions', [])), map(str.lower, plugin.get('os', [])), plugin.get('command', False), plugin.get('format', 'Text'), plugin.get('file', False), plugin.get('openwith', False), plugin.get('icon', 'fa-file-o')) else: return plugin.plugin_object
class Atomicapp(): debug = False dryrun = False atomicfile_data = None params_data = None answers_data = {GLOBAL_CONF: {}} tmpdir = None answers_file = None provider = DEFAULT_PROVIDER installed = False plugins = None recursive = True update = False app_path = None target_path = None app_id = None app = None def __init__(self, answers, app, recursive=True, update=False, target_path=None, dryrun=False, debug=False): run_path = os.path.dirname(os.path.realpath(__file__)) self.debug = debug self.dryrun = dryrun self.recursive = isTrue(recursive) self.update = isTrue(update) self.target_path = target_path logger.info("Path for %s is %s" % (app, target_path)) if os.path.exists(app): self.app_path = app if not os.path.basename(app) == ATOMIC_FILE: app = os.path.join(app, ATOMIC_FILE) atomic_data = self._loadAtomicfile(app) app = os.environ[ "IMAGE"] if "IMAGE" in os.environ else atomic_data["id"] self.app_id = atomic_data["id"] print("Setting path to %s" % self.app_path) if not self.target_path: if self.app_path: self.target_path = self.app_path else: self.target_path = os.getcwd() self.tmpdir = tempfile.mkdtemp(prefix="appent-%s" % self._getComponentName(app)) logger.debug("Temporary dir: %s" % self.tmpdir) self.app = app self.answers_file = answers self.plugins = PluginManager() self.plugins.setPluginPlaces([os.path.join(run_path, "providers")]) self.plugins.collectPlugins() def _sanitizeName(self, app): return app.replace("/", "-") def _mergeConfig(self): config = self.params_data if self.answers_data: if config: config = update(config, self.answers_data) else: config = self.answers_data return config def _mergeConfigComponent(self, component): config = self._mergeConfig() component_config = config[GLOBAL_CONF] if GLOBAL_CONF in config else {} if component in config: component_config = update(component_config, config[component]) return component_config #LOAD FUNCTIONS def _loadAtomicfile(self, path=None): print(os.path.isfile(path)) if not os.path.exists(path): print("Path: %s" % path) logger.error("Atomicfile not found: %s" % path) sys.exit(1) with open(path, "r") as fp: self.atomicfile_data = json.load(fp) self.app_id = self.atomicfile_data["id"] pprint(self.atomicfile_data) return self.atomicfile_data def _loadParams(self, path=None): logger.debug("Loading %s" % path) if not os.path.exists(path): return None config = ConfigParser.ConfigParser() config.optionxform = str data = {} with open(path, "r") as fp: config.readfp(fp) for section in config.sections(): data[section] = dict(config.items(section)) if self.params_data: self.params_data.update(data) else: self.params_data = data return self.params_data def _loadAnswers(self, path=None): if not os.path.exists(path): return None config = ConfigParser.ConfigParser() config.optionxform = str data = {} with open(path, "r") as fp: config.readfp(fp) for section in config.sections(): data[section] = dict(config.items(section)) self.answers_data = data return self.answers_data #GET_FUNCTIONS def _getComponentDir(self, component): return os.path.join(self.target_path, GRAPH_DIR, self._getComponentName(component)) def _getProviderDir(self, component): #FIXME add provider resolution by answers file return os.path.join(self.target_path, GRAPH_DIR, component, self.provider) def _getComponentConf(self, component): return os.path.join(self._getComponentDir(component), self.provider, PARAMS_FILE) def _getTmpAppDir(self): return os.path.join(self.tmpdir, APP_ENT_PATH) def _getGraphDir(self): return os.path.join(self.target_path, GRAPH_DIR) def _getComponentName(self, graph_item): if type(graph_item) is str or type(graph_item) is unicode: return os.path.basename(graph_item).split(":")[0] elif type(graph_item) is dict: return graph_item["name"].split(":")[0] else: return None def _getComponentImageName(self, graph_item): if type(graph_item) is str or type(graph_item) is unicode: return graph_item elif type(graph_item) is dict: print(graph_item) repo = "" if "repository" in graph_item: repo = graph_item["repository"] print(repo) return os.path.join(repo, graph_item["name"]) else: return None def _dispatchGraph(self): if not "graph" in self.atomicfile_data: raise Exception("Graph not specified in %s" % ATOMIC_FILE) if not os.path.isdir(self._getGraphDir()): raise Exception("Couldn't find %s directory" % GRAPH_DIR) for graph_item in self.atomicfile_data["graph"]: component = self._getComponentName(graph_item) component_path = self._getComponentDir(component) component_params = self._getComponentConf(component) if os.path.isfile(component_params): self._loadParams(component_params) self._processComponent(component) def _applyTemplate(self, data, component): template = Template(data) config = self._mergeConfigComponent(component) return template.substitute(config) def _getProvider(self): for provider in self.plugins.getAllPlugins(): module_path = provider.details.get("Core", "Module") if os.path.basename(module_path) == self.provider: return provider.plugin_object def _processComponent(self, component): path = self._getProviderDir(component) data = None for artifact in os.listdir(path): if artifact == PARAMS_FILE: continue with open(os.path.join(path, artifact), "r") as fp: data = fp.read() logger.debug("Templating artifact %s/%s" % (path, artifact)) data = self._applyTemplate(data, component) dst_dir = os.path.join(self.tmpdir, component) artifact_dst = os.path.join(dst_dir, artifact) if not os.path.isdir(dst_dir): os.makedirs(dst_dir) with open(artifact_dst, "w") as fp: fp.write(data) provider = self._getProvider() provider.init(self._mergeConfigComponent(component), os.path.join(self.tmpdir, component), self.debug, self.dryrun) provider.deploy() def _getImageURI(self, image): config = self._mergeConfig() if config and GLOBAL_CONF in config and "registry" in config[ GLOBAL_CONF]: print("Adding registry %s for %s" % (config[GLOBAL_CONF]["registry"], image)) image = os.path.join(config[GLOBAL_CONF]["registry"], image) return image def _pullApp(self, image): image = self._getImageURI(image) pull = ["docker", "pull", image] if subprocess.call(pull) != 0: print("Couldn't pull %s" % image) sys.exit(1) def _copyFromContainer(self, image): image = self._getImageURI(image) name = self._getComponentName(image) create = ["docker", "create", "--name", name, image, "nop"] subprocess.call(create) cp = ["docker", "cp", "%s:/%s" % (name, APP_ENT_PATH), self.tmpdir] if not subprocess.call(cp): logger.debug("Application entity data copied to %s" % self.tmpdir) rm = ["docker", "rm", name] subprocess.call(rm) def _populateMainApp(self, src=None, dst=None): print("Copying app %s" % self._getComponentName(self.app)) if not src: src = os.path.join(self.tmpdir, APP_ENT_PATH) if not dst: dst = self.target_path distutils.dir_util.copy_tree(src, dst, update=(not self.update)) def _populateModule(self): data_list = ["graph/%s/" % self.app_id, "Atomicfile", "params.conf"] logger.info("Populating module %s" % self._getComponentName(self.app)) for item in data_list: path = os.path.join(self.tmpdir, APP_ENT_PATH, item) if os.path.isdir(path): logger.debug("%s/%s/%s" % (self.target_path, GRAPH_DIR, self.app_id)) distutils.dir_util.copy_tree( path, self._getComponentDir(self.app_id)) else: logger.debug("copy item %s > %s > %s > %s" % (self.target_path, GRAPH_DIR, self.app_id, item)) distutils.file_util.copy_file( path, os.path.join(self._getComponentDir(self.app_id), item)) def run(self, level=AtomicappLevel.Main): if not self.installed: self.install(level) if not self._loadAtomicfile(os.path.join(self.target_path, ATOMIC_FILE)): print("Failed to load %s" % ATOMIC_FILE) return if self.debug: print(self.atomicfile_data) if not self._loadParams(os.path.join(self.target_path, PARAMS_FILE)): logger.error("Failed to load %s" % PARAMS_FILE) return else: logger.debug("Loaded params: %s" % self.params_data) config = self._mergeConfig() if "provider" in config[GLOBAL_CONF]: self.provider = config[GLOBAL_CONF]["provider"] self._dispatchGraph() def install(self, level=AtomicappLevel.Main): if not self._loadAnswers(self.answers_file): print("No %s file found, using defaults" % ANSWERS_FILE) if self.app_path and not self.target_path == self.app_path: logger.info("Copying content of directory %s to %s" % (self.app_path, self.target_path)) self._populateMainApp(src=self.app_path) atomicfile_path = os.path.join(self.target_path, ATOMIC_FILE) if level == AtomicappLevel.Module: atomicfile_path = os.path.join(self._getComponentDir(self.app), ATOMIC_FILE) logger.debug( "Test: %s -> %s" % (self.app, (not self.app_path and not os.path.exists(self._getComponentDir(self.app))))) if not self.app_path and (self.update or not os.path.exists( self._getComponentDir(self.app))): self._pullApp(self.app) self._copyFromContainer(self.app) atomicfile_path = os.path.join(self._getTmpAppDir(), ATOMIC_FILE) logger.debug("Atomicfile path for pulled image: %s" % atomicfile_path) self._loadAtomicfile(atomicfile_path) logger.debug("App ID: %s" % self.app_id) if level == AtomicappLevel.Main: self._populateMainApp() elif level == AtomicappLevel.Module: self._populateModule() else: logger.info("Component data exist in %s, skipping population..." % self._getComponentDir(self.app)) if not self.atomicfile_data: self._loadAtomicfile(atomicfile_path) if self.recursive: self._installDependencies() self.installed = True return self.app_id def _installDependencies(self): for graph_item in self.atomicfile_data["graph"]: component = self._getComponentName(graph_item) component_path = self._getComponentDir(component) logger.debug("Component path: %s" % component_path) logger.debug("%s == %s -> %s" % (component, self.app_id, component == self.app_id)) if not component == self.app_id and not os.path.isdir( component_path) and (not self.app_path or self.update): image_name = self._getComponentImageName(graph_item) print("Pulling %s" % image_name) component_atomicapp = Atomicapp(self.answers_file, image_name, self.recursive, self.update, self.target_path, self.dryrun, self.debug) component = component_atomicapp.install(AtomicappLevel.Module) component_path = self._getComponentDir(component) logger.info("Component installed into %s" % component_path)
disabledChecks = { # "SemiemptyFields" : {"bbmri-eric:ID:NO_HUNT", "bbmri-eric:ID:NO_Janus"} } pp = pprint.PrettyPrinter(indent=4) class ExtendAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest) or [] items.extend(values) setattr(namespace, self.dest, items) simplePluginManager = PluginManager() simplePluginManager.setPluginPlaces(["checks"]) simplePluginManager.collectPlugins() pluginList = [] for pluginInfo in simplePluginManager.getAllPlugins(): pluginList.append(os.path.basename(pluginInfo.path)) remoteCheckList = ['emails', 'geocoding', 'URLs'] cachesList = ['directory', 'emails', 'geocoding', 'URLs'] parser = argparse.ArgumentParser() parser.register('action', 'extend', ExtendAction) parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import unicode_literals import logging import os from yapsy.PluginManager import PluginManager logging.basicConfig(level=logging.DEBUG) logging.getLogger('yapsy').setLevel(logging.DEBUG) # Yapsy Framework to load a plugin # Load the plugins from the plugin directory. manager = PluginManager() # Location of consul plugin here = os.path.dirname(__file__) relative_path_of_plugin = os.path.join(here, 'storage_plugin') manager.setPluginPlaces([relative_path_of_plugin]) manager.collectPlugins() # Loop around the plugins and return the first plugin object. for storage_plugin in manager.getAllPlugins(): plugin = storage_plugin.plugin_object
from yapsy.PluginManager import PluginManager # Build the manager simplePluginManager = PluginManager() # Tell it the default place(s) where to find plugins simplePluginManager.setPluginPlaces(["modules"]) # Load all plugins simplePluginManager.collectPlugins() def pluginsByPort(): global plugins global tests_by_port tests = {} for i in plugins: tests = i.plugin_object.get_test_list() for t in tests: p = t.get_port() if p in tests: tests[p].append(t) tests_by_port = tests def reloadPlugins(): global plugins simplePluginManager = PluginManager() simplePluginManager.setPluginPlaces(["modules"])
class Statick: """Code analysis front-end.""" def __init__(self, user_paths: List[str]) -> None: """Initialize Statick.""" self.resources = Resources(user_paths) self.manager = PluginManager() self.manager.setPluginPlaces(self.resources.get_plugin_paths()) self.manager.setCategoriesFilter( { "Discovery": DiscoveryPlugin, "Tool": ToolPlugin, "Reporting": ReportingPlugin, } ) self.manager.collectPlugins() self.discovery_plugins: Dict[str, Any] = {} for plugin_info in self.manager.getPluginsOfCategory("Discovery"): self.discovery_plugins[ plugin_info.plugin_object.get_name() ] = plugin_info.plugin_object self.tool_plugins: Dict[str, Any] = {} for plugin_info in self.manager.getPluginsOfCategory("Tool"): self.tool_plugins[ plugin_info.plugin_object.get_name() ] = plugin_info.plugin_object self.reporting_plugins: Dict[str, Any] = {} for plugin_info in self.manager.getPluginsOfCategory("Reporting"): self.reporting_plugins[ plugin_info.plugin_object.get_name() ] = plugin_info.plugin_object self.config: Optional[Config] = None self.exceptions: Optional[Exceptions] = None @staticmethod def set_logging_level(args: argparse.Namespace) -> None: """Set the logging level to use for output. Valid levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL. Specifying the level is case-insensitive (both upper-case and lower-case are allowed). """ valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] log_level = args.log_level.upper() if log_level not in valid_levels: log_level = "WARNING" args.log_level = log_level logging.basicConfig(level=log_level) logging.root.setLevel(log_level) logging.info("Log level set to %s", args.log_level.upper()) def get_config(self, args: argparse.Namespace) -> None: """Get Statick configuration.""" base_config_filename = "config.yaml" user_config_filename = "" if args.config is not None: user_config_filename = args.config try: self.config = Config( self.resources.get_file(base_config_filename), self.resources.get_file(user_config_filename), ) except OSError as ex: logging.error( "Failed to access configuration file %s or %s: %s", base_config_filename, user_config_filename, ex, ) except ValueError as ex: logging.error( "Configuration file %s or %s has errors: %s", base_config_filename, user_config_filename, ex, ) def get_exceptions(self, args: argparse.Namespace) -> None: """Get Statick exceptions.""" exceptions_filename = "exceptions.yaml" if args.exceptions is not None: exceptions_filename = args.exceptions try: self.exceptions = Exceptions(self.resources.get_file(exceptions_filename)) except OSError as ex: logging.error( "Failed to access exceptions file %s: %s", exceptions_filename, ex ) except ValueError as ex: logging.error("Exceptions file %s has errors: %s", exceptions_filename, ex) def get_ignore_packages(self) -> List[str]: """Get packages to ignore during scan process.""" if self.exceptions is None: return [] return self.exceptions.get_ignore_packages() def gather_args(self, args: argparse.ArgumentParser) -> None: """Gather arguments.""" args.add_argument( "--output-directory", dest="output_directory", type=str, help="Directory to write output files to", ) args.add_argument( "--log", dest="log_level", type=str, default="WARNING", help="Verbosity level of output to show (DEBUG, INFO, WARNING, ERROR" ", CRITICAL)", ) args.add_argument( "--check", dest="check", action="store_true", help="Return the status. Return code 0 means there were no issues. \ Return code 1 means there were issues.", ) args.add_argument( "--config", dest="config", type=str, help="Name of config yaml file" ) args.add_argument( "--profile", dest="profile", type=str, help="Name of profile yaml file" ) args.add_argument( "--exceptions", dest="exceptions", type=str, help="Name of exceptions yaml file", ) args.add_argument( "--force-tool-list", dest="force_tool_list", type=str, help="Force only the given list of tools to run", ) args.add_argument( "--version", action="version", version=f"%(prog)s {__version__}", ) args.add_argument( "--mapping-file-suffix", dest="mapping_file_suffix", type=str, help="Suffix to use when searching for CERT mapping files", ) # statick workspace arguments args.add_argument( "-ws", dest="workspace", action="store_true", help="Treat the path argument as a workspace of multiple packages", ) args.add_argument( "--max-procs", dest="max_procs", type=int, default=int(multiprocessing.cpu_count() / 2), help="Maximum number of CPU cores to use, only used when running on a" "workspace", ) args.add_argument( "--packages-file", dest="packages_file", type=str, help="File listing packages to scan, only used when running on a workspace", ) args.add_argument( "--list-packages", dest="list_packages", action="store_true", help="List packages and levels, only used when running on a workspace", ) for _, plugin in list(self.discovery_plugins.items()): plugin.gather_args(args) for _, plugin in list(self.tool_plugins.items()): plugin.gather_args(args) for _, plugin in list(self.reporting_plugins.items()): plugin.gather_args(args) def get_level(self, path: str, args: argparse.Namespace) -> Optional[str]: """Get level to scan package at.""" path = os.path.abspath(path) profile_filename = "profile.yaml" if args.profile is not None: profile_filename = args.profile profile_resource = self.resources.get_file(profile_filename) if profile_resource is None: logging.error("Could not find profile file %s!", profile_filename) return None try: profile = Profile(profile_resource) except OSError as ex: # This isn't quite redundant with the profile_resource check: it's possible # that something else triggers an OSError, like permissions. logging.error("Failed to access profile file %s: %s", profile_filename, ex) return None except ValueError as ex: logging.error("Profile file %s has errors: %s", profile_filename, ex) return None package = Package(os.path.basename(path), path) level = profile.get_package_level(package) return level # pylint: disable=too-many-locals, too-many-return-statements, too-many-branches # pylint: disable=too-many-statements def run( self, path: str, args: argparse.Namespace ) -> Tuple[Optional[Dict[str, List[Issue]]], bool]: """Run scan tools against targets on path.""" success = True path = os.path.abspath(path) if not os.path.exists(path): logging.error("No package found at %s!", path) return None, False package = Package(os.path.basename(path), path) level: Optional[str] = self.get_level(path, args) logging.info("level: %s", level) if level is None: logging.error("Level is not valid.") return None, False if not self.config or not self.config.has_level(level): logging.error("Can't find specified level %s in config!", level) return None, False orig_path = os.getcwd() if args.output_directory: if not os.path.isdir(args.output_directory): try: os.mkdir(args.output_directory) except OSError as ex: logging.error( "Unable to create output directory at %s: %s", args.output_directory, ex, ) return None, False output_dir = os.path.join(args.output_directory, package.name + "-" + level) if not os.path.isdir(output_dir): try: os.mkdir(output_dir) except OSError as ex: logging.error( "Unable to create output directory at %s: %s", output_dir, ex ) return None, False logging.info("Writing output to: %s", output_dir) os.chdir(output_dir) logging.info("------") logging.info( "Scanning package %s (%s) at level %s", package.name, package.path, level ) issues: Dict[str, List[Issue]] = {} ignore_packages = self.get_ignore_packages() if package.name in ignore_packages: logging.info( "Package %s is configured to be ignored by Statick.", package.name ) return issues, True plugin_context = PluginContext(args, self.resources, self.config) logging.info("---Discovery---") if not DiscoveryPlugin.file_command_exists(): logging.info( "file command isn't available, discovery plugins will be less effective" ) discovery_plugins = self.config.get_enabled_discovery_plugins(level) if not discovery_plugins: discovery_plugins = list(self.discovery_plugins.keys()) plugins_ran: List[Any] = [] for plugin_name in discovery_plugins: if plugin_name not in self.discovery_plugins: logging.error("Can't find specified discovery plugin %s!", plugin_name) return None, False plugin = self.discovery_plugins[plugin_name] dependencies = plugin.get_discovery_dependencies() for dependency_name in dependencies: dependency_plugin = self.discovery_plugins[dependency_name] if dependency_plugin.get_name() in plugins_ran: continue dependency_plugin.set_plugin_context(plugin_context) logging.info( "Running %s discovery plugin...", dependency_plugin.get_name() ) dependency_plugin.scan(package, level, self.exceptions) logging.info("%s discovery plugin done.", dependency_plugin.get_name()) plugins_ran.append(dependency_plugin.get_name()) if plugin.get_name() not in plugins_ran: plugin.set_plugin_context(plugin_context) logging.info("Running %s discovery plugin...", plugin.get_name()) plugin.scan(package, level, self.exceptions) logging.info("%s discovery plugin done.", plugin.get_name()) plugins_ran.append(plugin.get_name()) logging.info("---Discovery---") logging.info("---Tools---") enabled_plugins = self.config.get_enabled_tool_plugins(level) plugins_to_run = copy.copy(enabled_plugins) plugins_ran = [] plugin_dependencies: List[str] = [] while plugins_to_run: plugin_name = plugins_to_run[0] if plugin_name not in self.tool_plugins: logging.error("Can't find specified tool plugin %s!", plugin_name) return None, False if args.force_tool_list is not None: force_tool_list = args.force_tool_list.split(",") if ( plugin_name not in force_tool_list and plugin_name not in plugin_dependencies ): logging.info("Skipping plugin not in force list %s!", plugin_name) plugins_to_run.remove(plugin_name) continue plugin = self.tool_plugins[plugin_name] plugin.set_plugin_context(plugin_context) dependencies = plugin.get_tool_dependencies() dependencies_met = True for dependency_name in dependencies: if dependency_name not in plugins_ran: if dependency_name not in enabled_plugins: logging.error( "Plugin %s depends on plugin %s which isn't enabled!", plugin_name, dependency_name, ) return None, False plugin_dependencies.append(dependency_name) if dependency_name in plugins_to_run: plugins_to_run.remove(dependency_name) plugins_to_run.insert(0, dependency_name) dependencies_met = False if not dependencies_met: continue logging.info("Running %s tool plugin...", plugin.get_name()) tool_issues = plugin.scan(package, level) if tool_issues is not None: issues[plugin_name] = tool_issues logging.info("%s tool plugin done.", plugin.get_name()) else: logging.error("%s tool plugin failed", plugin.get_name()) success = False plugins_to_run.remove(plugin_name) plugins_ran.append(plugin_name) logging.info("---Tools---") if self.exceptions is not None: issues = self.exceptions.filter_issues(package, issues) os.chdir(orig_path) logging.info("---Reporting---") reporting_plugins = self.config.get_enabled_reporting_plugins(level) if not reporting_plugins: reporting_plugins = self.reporting_plugins.keys() # type: ignore for plugin_name in reporting_plugins: if plugin_name not in self.reporting_plugins: logging.error("Can't find specified reporting plugin %s!", plugin_name) return None, False plugin = self.reporting_plugins[plugin_name] plugin.set_plugin_context(plugin_context) logging.info("Running %s reporting plugin...", plugin.get_name()) plugin.report(package, issues, level) logging.info("%s reporting plugin done.", plugin.get_name()) logging.info("---Reporting---") logging.info("Done!") return issues, success def run_workspace( self, parsed_args: argparse.Namespace ) -> Tuple[ Optional[Dict[str, List[Issue]]], bool ]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements """Run statick on a workspace. --max-procs can be set to the desired number of CPUs to use for processing a workspace. This defaults to half the available CPUs. Setting this to -1 will cause statick.run_workspace to use all available CPUs. """ max_cpus = multiprocessing.cpu_count() if parsed_args.max_procs > max_cpus or parsed_args.max_procs == -1: parsed_args.max_procs = max_cpus elif parsed_args.max_procs <= 0: parsed_args.max_procs = 1 if parsed_args.output_directory: out_dir = parsed_args.output_directory if not os.path.isdir(out_dir): try: os.mkdir(out_dir) except OSError as ex: logging.error( "Unable to create output directory at %s: %s", out_dir, ex ) return None, False ignore_packages = self.get_ignore_packages() ignore_files = ["AMENT_IGNORE", "CATKIN_IGNORE", "COLCON_IGNORE"] package_indicators = ["package.xml", "setup.py", "pyproject.toml"] packages = [] for root, dirs, files in os.walk(parsed_args.path): if any(item in files for item in ignore_files): dirs.clear() continue for sub_dir in dirs: full_dir = os.path.join(root, sub_dir) files = os.listdir(full_dir) if any(item in package_indicators for item in files) and not any( item in files for item in ignore_files ): if ignore_packages and sub_dir in ignore_packages: continue packages.append(Package(sub_dir, full_dir)) if parsed_args.packages_file is not None: packages_file_list = [] try: packages_file = os.path.abspath(parsed_args.packages_file) with open(packages_file, "r", encoding="utf8") as fname: packages_file_list = [ package.strip() for package in fname.readlines() if package.strip() and package[0] != "#" ] except OSError: logging.error("Packages file not found") return None, False packages = [ package for package in packages if package.name in packages_file_list ] if parsed_args.list_packages: for package in packages: logging.info( "%s: %s", package.name, self.get_level(package.path, parsed_args) ) return None, True count = 0 total_issues = [] num_packages = len(packages) mp_args = [] if multiprocessing.get_start_method() == "fork": logging.info("-- Scanning %d packages --", num_packages) for package in packages: count += 1 mp_args.append((parsed_args, count, package, num_packages)) with multiprocessing.Pool(parsed_args.max_procs) as pool: total_issues = pool.starmap(self.scan_package, mp_args) else: logging.warning( "Statick's plugin manager does not currently support multiprocessing" " without UNIX's fork function. Falling back to a single process." ) logging.info("-- Scanning %d packages --", num_packages) for package in packages: count += 1 pkg_issues = self.scan_package( parsed_args, count, package, num_packages ) total_issues.append(pkg_issues) logging.info("-- All packages run --") logging.info("-- overall report --") success = True issues: Dict[str, List[Issue]] = {} for issue in total_issues: if issue is not None: for key, value in list(issue.items()): if key in issues: issues[key] += value if value: success = False else: issues[key] = value if value: success = False enabled_reporting_plugins: List[str] = [] available_reporting_plugins = {} for plugin_info in self.manager.getPluginsOfCategory("Reporting"): available_reporting_plugins[ plugin_info.plugin_object.get_name() ] = plugin_info.plugin_object # Make a fake 'all' package for reporting dummy_all_package = Package("all_packages", parsed_args.path) level = self.get_level(dummy_all_package.path, parsed_args) if level is not None and self.config is not None: if not self.config or not self.config.has_level(level): logging.error("Can't find specified level %s in config!", level) enabled_reporting_plugins = list(available_reporting_plugins) else: enabled_reporting_plugins = self.config.get_enabled_reporting_plugins( level ) if not enabled_reporting_plugins: enabled_reporting_plugins = list(available_reporting_plugins) plugin_context = PluginContext(parsed_args, self.resources, self.config) # type: ignore plugin_context.args.output_directory = parsed_args.output_directory for plugin_name in enabled_reporting_plugins: if plugin_name not in available_reporting_plugins: logging.error("Can't find specified reporting plugin %s!", plugin_name) continue plugin = self.reporting_plugins[plugin_name] plugin.set_plugin_context(plugin_context) logging.info("Running %s reporting plugin...", plugin.get_name()) plugin.report(dummy_all_package, issues, level) logging.info("%s reporting plugin done.", plugin.get_name()) return issues, success def scan_package( self, parsed_args: argparse.Namespace, count: int, package: Package, num_packages: int, ) -> Optional[Dict[str, List[Issue]]]: """Scan each package in a separate process while buffering output.""" logger = logging.getLogger() old_handler = None if logger.handlers[0]: old_handler = logger.handlers[0] handler = MemoryHandler(10000, flushLevel=logging.ERROR, target=old_handler) logger.removeHandler(old_handler) logger.addHandler(handler) logging.info( "-- Scanning package %s (%d of %d) --", package.name, count, num_packages ) sio = io.StringIO() old_stdout = sys.stdout old_stderr = sys.stderr sys.stdout = sio sys.stderr = sio issues, dummy = self.run(package.path, parsed_args) sys.stdout = old_stdout sys.stderr = old_stderr logging.info(sio.getvalue()) if issues is not None: logging.info( "-- Done scanning package %s (%d of %d) --", package.name, count, num_packages, ) else: logging.error("Failed to run statick on package %s!", package.name) if old_handler is not None: handler.flush() logger.removeHandler(handler) logger.addHandler(old_handler) return issues @staticmethod def print_no_issues() -> None: """Print that no information about issues was found.""" logging.error( "Something went wrong, no information about issues." " Statick exiting with errors." ) @staticmethod def print_exit_status(status: bool) -> None: """Print Statick exit status.""" if status: logging.info("Statick exiting with success.") else: logging.error("Statick exiting with errors.")
def reloadPluginsModel(): if settings.DEBUG: logging.getLogger('yapsy').setLevel(logging.WARNING) # Are there any plugin objects? If not, add in the defaults plugin_objects = Plugin.objects.all().count() if plugin_objects == 0: order = 0 PLUGIN_ORDER = [ 'Activity', 'Status', 'OperatingSystem', 'MunkiVersion', 'Uptime', 'Memory', 'DiskSpace', 'PendingAppleUpdates', 'Pending3rdPartyUpdates', 'PuppetStatus' ] for item in PLUGIN_ORDER: order = order + 1 plugin = Plugin(name=item, order=order) plugin.save() # Build the manager manager = PluginManager() # Tell it the default place(s) where to find plugins manager.setPluginPlaces([ settings.PLUGIN_DIR, os.path.join(settings.PROJECT_DIR, 'server/plugins') ]) # Load all plugins manager.collectPlugins() found = [] for plugin in manager.getAllPlugins(): found.append(plugin.name) # Get all of the plugin objects - if it's in here not installed, remove it all_plugins = Plugin.objects.all() for plugin in all_plugins: if plugin.name not in found: plugin.delete() # And go over again to update the plugin's type for dbplugin in all_plugins: for plugin in manager.getAllPlugins(): if plugin.name == dbplugin.name: try: dbplugin.type = plugin.plugin_object.plugin_type() except: dbplugin.type = 'builtin' try: dbplugin.description = plugin.plugin_object.get_description( ) except: pass dbplugin.save() all_plugins = Report.objects.all() for plugin in all_plugins: if plugin.name not in found: plugin.delete() # And go over again to update the plugin's type for dbplugin in all_plugins: for plugin in manager.getAllPlugins(): if plugin.name == dbplugin.name: try: dbplugin.type = plugin.plugin_object.plugin_type() except: dbplugin.type = 'builtin' try: dbplugin.description = plugin.plugin_object.get_description( ) except: pass dbplugin.save()
storage_package = CONFIG['storage'].split('.', 2)[0] storage_class = CONFIG['storage'].split('.', 2)[1] m = __import__('telescope.storage.%s' % (storage_package, )) m = getattr(m.storage, storage_package) STORAGE = getattr(m, storage_class)(CONFIG['storage config']) STORAGE.load_data() # plugin system from yapsy.PluginManager import PluginManager # Build the manager pluginManager = PluginManager() # Tell it the default place(s) where to find plugins myPath = os.path.dirname(__file__) modulePath = os.path.join(myPath, "plugins") pluginManager.setPluginPlaces([modulePath]) # Tell it about our categories pluginManager.setCategoriesFilter({ 'WebPlugin': telescope.plugins.abstract.ITelescopeWebPlugin, 'AnnouncePlugin': telescope.plugins.abstract.ITelescopeAnnouncePlugin }) # Load all plugins pluginManager.collectPlugins() # Activate all loaded plugins for pluginInfo in pluginManager.getAllPlugins(): pluginManager.activatePluginByName(pluginInfo.name) PLUGIN_MANAGER = pluginManager
class PackageIndexManager(object): """Provides indexing functionality of packages in all of the package sources. Note: This is an singleton class and will be shared between objects """ __single_package_index_manager = None def __new__(cls, *args, **kwargs): """Singleton object creator """ if cls != type(cls.__single_package_index_manager): cls.__single_package_index_manager = object.__new__( cls, *args, **kwargs) return cls.__single_package_index_manager def __init__(self, db_connection, download_location): """Initialize :class:`PackageIndexManager` class Args: db_connection (object): An open connection to metadata database download_location (str): location where packages will be downloaded """ self.__plugin_manager = PluginManager() self.__plugin_manager.setPluginPlaces( [constants.PKG_SOURCE_PLUGIN_PATH]) self.__plugin_manager.setCategoriesFilter( {'PackageSourcePlugins': IPackageSourcePlugin}) self.__plugin_manager.collectPlugins() self.__db_connection = db_connection self._sources_table = db_connection.table('PackageSource') self.__package_sources = self.get_all_sources() self.__package_index_registry = {} self.__source_validity_status = {} self.__download_location = download_location self.__get_index_thread = tasks.HybridThread(name='PackageIndexCoroThread',\ notify_on_all_done=self.__common_callback) if len(self.__package_sources) <= 0: warnings.warn( 'No package source are configured. :class:`PackageManager`\ will not be able to download any packages') else: self.__update_package_list() def __common_callback(self, results, owner, _type): """The common coroutine callback, which will be called on completion \ of all coroutines with owner as one of the parameters """ if owner == 'ValidityStatusCheck': self.__validity_status_callback(results, _type) elif owner == 'FetchIndex': self.__update_index_callback(results, _type) async def __get_index(self, source): """Coroutine to get the index from source using async request Args: source (str): Name of the package source from where index needs to be fetched """ return await source.get_package_index() def __update_index_callback(self, results, _type): """This function will be called once all coroutines :func:`__get_index` are done. This callback will receive a list of results in random order as a \ tuple of two elements (source-name, source-index-list) Args: results (list): A 2 element tuple list _type (int): function or coroutine """ for result in results: _name = result[0] _source_index = result[1] self.__package_index_registry[_name] = _source_index if self.__get_index_thread is not None: self.__get_index_thread = None def get_all_sources(self): """Returns an list of :class:`PackageSource` configured in current system Returns: sources (PackageSource): list of all sources configured """ _sources = {} if self._sources_table is not None: if self._sources_table.count() > 0: for source in self._sources_table.all(): source_object = object() if source.name is not None: source_object = self.__plugin_manager.getPluginByName( source.name, 'PackageSourcePlugin') if source_object is not None and callable( source_object): self.__plugin_manager.activatePluginByName( source_object) _sources[source.name] = source_object else: _sources[source.name] = DefaultPackageSource\ (self.__download_location, \ self.__db_connection, source.name) return _sources def add_source(self, download_loc, name, uri=None, username=None, password=None, \ src_type='Web', modified_on=None, modified_by=None, security_id=None, \ class_type=None, class_in_module=None): """Add an new source system Args: name (str): Name of the source uri (str): Uri of the new source i.e., url or folder path username (str): Username to access source uri (optional) password (str): Password to access source uri (optional) src_type (str): Type of the source - Web, FileSystem, etc modified_on (date): Datetime when new source is added modified_by (str): Name of the user who added the source security_id (uuid): The security rules applied to this source class_type (str): Class type of the source \ (should be derived from :class:`PackageSource`) class_in_module (str): Name of the module where the above \ 'class_type' can be found Returns: status (bool): True or False message (str): Failure reason """ new_source = DefaultPackageSource(download_loc, self.__db_connection, name, \ uri=uri, username=username, password=password, \ src_type=src_type, modified_on=modified_on, \ modified_by=modified_by, security_id=security_id, \ class_type=class_type, class_in_module=class_in_module) status = new_source.save() if status is not None and status[0] is True: self.__package_sources[name] = new_source return status def update_source(self, name, attribute_name, value=None): """Update an source system Args: name (str): Name of the source attribute_name (str): Name of the attribute that needs to be updated value (str): New value of the attribute to be updated Returns: status (bool): True or False message (str): Failure reason """ if self._sources_table is not None and len(self.__package_sources) > 0: src_record = self.__package_sources[name] _result = src_record.update(attribute_name, value) return _result return (False, 'No package source are yet configured') def remove_source(self, source_name): """Removes source from system Args: source_name (str): Name of source that needs to be removed Returns: status (bool): True or False message (str): Reason for failurer """ if source_name is not None: result = self._sources_table.remove(Query()['Name'] == source_name) if result is None or len(result) <= 0: return (False, 'Can' 't delete source from the system') return (True, 'Source has been deleted - {0}'.format(result)) async def __get_validity_status(self, source): """docstring for get_validity_status""" return await source.get_validity_status() def __validity_status_callback(self, results, _type): """Will be called once all :func:`__get_validity_status` coroutines are done Args: results (list): A list of two elements tuple -[0] (str): source name -[1] (bool): valid or not valid """ for result in results: _src_name = result[0] _is_src_valid = result[1] self.__source_validity_status[_src_name] = _is_src_valid if self.__get_index_thread is not None: self.__get_index_thread = None def refresh_index(self, percentage_completed_callback=None): """Refresh the local index from the backend index source Args: percentage_completed_callback (callable): A callable which be called \ back to percentage completed status (optional) """ self.__update_package_list(percentage_completed_callback) self.__get_index_thread.wait_for_all_coroutines() def __update_package_list(self, percentage_completed_callback=None): """This function will fetch a list of all packages under an source Args: percentage_completed_callback (func): Callback will be called on updation \ of each package and percentage completion will be passed to callback """ if len(self.__package_sources) > 0: #Step1 - Get validity status of all current sources configured self.__get_index_thread.set_owner('ValidityStatusCheck') for source_name, source in self.__package_sources.items(): self.__get_index_thread.add_coroutine( self.__get_validity_status, source) if percentage_completed_callback is not None: #return the percentage completion self.__get_index_thread.register_coroutine_completed_percentage\ (percentage_completed_callback) self.__get_index_thread.start_forever() #Will wait for coroutines to finish as we need it for next step self.__get_index_thread.wait_for_all_coroutines() #Step2 - Get cached package index for sources having valid indexes for source_name, is_valid in self.__source_validity_status.items(): if is_valid: self.__package_index_registry[source_name] = \ self.__package_sources[source_name].get_cached_package_index() #Step3 - Refresh cached index from source for those which are not valid self.__get_index_thread.set_owner('FetchIndex') for source_name, is_valid in self.__source_validity_status.items(): if not is_valid: self.__get_index_thread\ .add_coroutine(self.__get_index, \ self.__package_sources[source_name]) if percentage_completed_callback is not None: #return the percentage completion self.__get_index_thread\ .register_coroutine_completed_percentage(percentage_completed_callback) self.__get_index_thread.reschedule() #No need to wait here as calling thread will handle it def get_package_list(self, refresh=False, percentage_completed_callback=None): """Returns an list of all packages either from cache or downloading it from source Args: refresh (bool): Whether to have package indexes refreshed before \ returning the index list (default=False) """ if percentage_completed_callback is not None and callable( percentage_completed_callback): percentage_completed_callback(0) if refresh: self.__update_package_list(percentage_completed_callback) if self.__get_index_thread is not None: #wait for coroutines to finish self.__get_index_thread.wait_for_all_coroutines() if percentage_completed_callback is not None and callable( percentage_completed_callback): percentage_completed_callback(100) return self.__package_index_registry def find_package(self, package_name): """Finds an package in the index repository Args: package_name (str): Name of the package that needs to be searched Returns: result (tuple of 4 elements): Returns an tuple having below mentioned 4 elements - status (bool): True if package found else False - source_name (str): source name where package was found - package_index (dict): dict of package index - message (str): Error message if not able to find the package """ if package_name is not None: for source_name, source in self.__package_sources.items(): if self.__package_index_registry[source_name].__contains__( package_name): return (True, source, \ self.__package_index_registry[source.name][package_name], '') return (False, None, None, 'No such package found...{0}'.format(package_name)) return (False, None, None, 'Can' 't accept blank value for package name')
class Paratest(object): def __init__(self, workspace_num, scripts, source_path, workspace_path, output_path, test_pattern, persistence): self.workspace_num = workspace_num self.workspace_path = workspace_path self.scripts = scripts self.source_path = source_path self.output_path = output_path self.test_pattern = test_pattern self._workers = [] self.pluginmgr = PluginManager() self.pluginmgr.setPluginInfoExtension('paratest') self.pluginmgr.setPluginPlaces(["plugins", ""]) self.pluginmgr.collectPlugins() self.persistence = persistence if not os.path.exists(self.source_path): os.makedirs(self.source_path) if not os.path.exists(self.output_path): os.makedirs(self.output_path) def list_plugins(self): msg = "Available plugins are:\n" for plugin in self.pluginmgr.getAllPlugins(): msg += " %s" % plugin.name print(msg) def run(self, plugin): plugin = self.pluginmgr.getPluginByName(plugin) pluginobj = plugin.plugin_object self.run_script_setup() test_number = self.queue_tests(pluginobj) self.create_workers(pluginobj, self.num_of_workers(test_number)) self.start_workers() self.wait_workers() self.run_script_teardown() self.assert_all_messages_were_processed() def run_script_setup(self): if run_script(self.scripts.setup, path=self.workspace_path): raise Abort('The setup script failed. aborting.') def run_script_teardown(self): if run_script(self.scripts.teardown, path=self.workspace_path): raise Abort('The teardown script failed, but nothing can be done.') def queue_tests(self, pluginobj): tids = 0 for tid in pluginobj.find(self.source_path, self.test_pattern): shared_queue.put((self.persistence.get_priority(tid), tid)) tids += 1 return tids def create_workers(self, pluginobj, workers): for i in range(workers): t = Worker( pluginobj, scripts=self.scripts, workspace_path=self.workspace_path, source_path=self.source_path, output_path=self.output_path, persistence=self.persistence, name=str(i), ) self._workers.append(t) def num_of_workers(self, test_number): return min(self.workspace_num, test_number) def start_workers(self): logger.debug("start workers") for t in self._workers: t.start() shared_queue.put((INFINITE, FINISH)) def wait_workers(self): logger.debug("wait for all workers to finish") for t in self._workers: t.join() def assert_all_messages_were_processed(self): if not shared_queue.empty(): raise Abort( 'There were unprocessed tests, but all workers are dead. Aborting.' )
class MainWindow(QMainWindow): """MainWindow class Attributes: trace_data (TraceData): TraceData object filtered_trace (list): Filtered trace """ def __init__(self, parent=None): """Inits MainWindow, UI and plugins""" super(MainWindow, self).__init__(parent) self.api = Api(self) self.trace_data = TraceData() self.filtered_trace = [] self.filter_text = "" self.init_plugins() self.init_ui() if len(sys.argv) > 1: self.open_trace(sys.argv[1]) def dragEnterEvent(self, event): """QMainWindow method reimplementation for file drag.""" event.setDropAction(Qt.MoveAction) super().dragEnterEvent(event) event.accept() def dropEvent(self, event): """QMainWindow method reimplementation for file drop.""" super().dropEvent(event) if event.mimeData().hasUrls(): for url in event.mimeData().urls(): local_file = url.toLocalFile() if os.path.isfile(local_file): self.open_trace(local_file) def init_ui(self): """Inits UI""" uic.loadUi("gui/mainwindow.ui", self) self.setWindowTitle(prefs.PACKAGE_NAME) # accept file drops self.setAcceptDrops(True) self.resize(prefs.WINDOW_WIDTH, prefs.WINDOW_HEIGHT) # make trace table wider than regs&mem self.splitter1.setSizes([1000, 100]) self.splitter2.setSizes([600, 100]) # Init trace table self.trace_table.itemSelectionChanged.connect( self.on_trace_table_row_changed) self.trace_table.setColumnCount(len(prefs.TRACE_LABELS)) self.trace_table.setHorizontalHeaderLabels(prefs.TRACE_LABELS) self.trace_table.horizontalHeader().setStretchLastSection(True) self.trace_table.bookmarkCreated.connect(self.add_bookmark) self.trace_table.commentEdited.connect(self.set_comment) self.trace_table.printer = self.print self.trace_table.set_row_height(prefs.TRACE_ROW_HEIGHT) trace_font = QFont(prefs.TRACE_FONT) trace_font.setPointSize(prefs.TRACE_FONT_SIZE) self.trace_table.setFont(trace_font) self.bookmark_table.setFont(trace_font) self.trace_table.setShowGrid(prefs.TRACE_SHOW_GRID) if prefs.USE_SYNTAX_HIGHLIGHT_IN_TRACE: self.trace_table.init_syntax_highlight() # trace pagination if prefs.PAGINATION_ENABLED: self.trace_pagination = PaginationWidget() self.trace_pagination.pageChanged.connect( self.trace_table.populate) self.horizontalLayout.addWidget(self.trace_pagination) self.trace_pagination.set_enabled(True) self.trace_pagination.rows_per_page = prefs.PAGINATION_ROWS_PER_PAGE self.trace_table.pagination = self.trace_pagination self.horizontalLayout.setAlignment(self.trace_pagination, Qt.AlignLeft) # these are used to remember current pages & scroll values for both traces self.trace_current_pages = [1, 1] self.trace_scroll_values = [0, 0] self.reg_table.create_context_menu() self.reg_table.setColumnCount(len(prefs.REG_LABELS)) self.reg_table.setHorizontalHeaderLabels(prefs.REG_LABELS) self.reg_table.horizontalHeader().setStretchLastSection(True) self.reg_table.regCheckBoxChanged.connect(self.on_reg_checkbox_change) self.reg_table.printer = self.print if prefs.REG_FILTER_ENABLED: self.reg_table.filtered_regs = prefs.REG_FILTER if not prefs.USE_DARK_THEME: trace_style = ( "QTableView { selection-background-color: #dddddd; selection-" "color: #000000; border: 0px;} QTableWidget::item { padding: 0px; border: 0px}" ) reg_style = ( "QTableView { selection-background-color: #eee864; selection" "-color: #000000;}") self.trace_table.setStyleSheet(trace_style) self.bookmark_table.setStyleSheet(trace_style) self.reg_table.setStyleSheet(reg_style) # Init memory table self.mem_table.setColumnCount(len(prefs.MEM_LABELS)) self.mem_table.setHorizontalHeaderLabels(prefs.MEM_LABELS) self.mem_table.horizontalHeader().setStretchLastSection(True) # Init bookmark table self.bookmark_table.setColumnCount(len(prefs.BOOKMARK_LABELS)) self.bookmark_table.setHorizontalHeaderLabels(prefs.BOOKMARK_LABELS) self.bookmark_table.setContextMenuPolicy(Qt.CustomContextMenu) self.bookmark_table.customContextMenuRequested.connect( self.bookmark_table_context_menu_event) self.bookmark_table.delegate = SyntaxHighlightDelegate(self) self.bookmark_table.delegate.disasm_columns = prefs.BOOKMARK_HL_DISASM_COLUMNS self.bookmark_table.delegate.value_columns = prefs.BOOKMARK_HL_VALUE_COLUMNS self.bookmark_table.setItemDelegate(self.bookmark_table.delegate) self.bookmark_menu = QMenu(self) go_action = QAction("Go to bookmark", self) go_action.triggered.connect(self.go_to_bookmark_in_trace) self.bookmark_menu.addAction(go_action) delete_bookmarks_action = QAction("Delete bookmark(s)", self) delete_bookmarks_action.triggered.connect(self.delete_bookmarks) self.bookmark_menu.addAction(delete_bookmarks_action) # Menu exit_action = QAction("&Exit", self) exit_action.setShortcut("Ctrl+Q") exit_action.setStatusTip("Exit application") exit_action.triggered.connect(self.close) open_trace_action = QAction("&Open trace..", self) open_trace_action.setStatusTip("Open trace") open_trace_action.triggered.connect(self.dialog_open_trace) self.save_trace_action = QAction("&Save trace", self) self.save_trace_action.setStatusTip("Save trace") self.save_trace_action.triggered.connect(self.save_trace) self.save_trace_action.setEnabled(False) save_trace_as_action = QAction("&Save trace as..", self) save_trace_as_action.setStatusTip("Save trace as..") save_trace_as_action.triggered.connect(self.dialog_save_trace_as) save_trace_as_json_action = QAction("&Save trace as JSON..", self) save_trace_as_json_action.setStatusTip("Save trace as JSON..") save_trace_as_json_action.triggered.connect( self.dialog_save_trace_as_json) file_menu = self.menu_bar.addMenu("&File") file_menu.addAction(open_trace_action) file_menu.addAction(self.save_trace_action) file_menu.addAction(save_trace_as_action) file_menu.addAction(save_trace_as_json_action) file_menu.addAction(exit_action) self.plugins_topmenu = self.menu_bar.addMenu("&Plugins") clear_bookmarks_action = QAction("&Clear bookmarks", self) clear_bookmarks_action.setStatusTip("Clear bookmarks") clear_bookmarks_action.triggered.connect(self.clear_bookmarks) bookmarks_menu = self.menu_bar.addMenu("&Bookmarks") bookmarks_menu.addAction(clear_bookmarks_action) # Create right click menu for trace table self.create_trace_table_menu() # Create plugins menu on menu bar self.create_plugins_menu() about_action = QAction("&About", self) about_action.triggered.connect(self.show_about_dialog) about_menu = self.menu_bar.addMenu("&About") about_menu.addAction(about_action) if prefs.USE_SYNTAX_HIGHLIGHT_IN_LOG: self.highlight = AsmHighlighter(self.log_text_edit.document()) # trace select self.select_trace_combo_box.addItem("Full trace") self.select_trace_combo_box.addItem("Filtered trace") self.select_trace_combo_box.currentIndexChanged.connect( self.on_trace_combo_box_index_changed) self.filter_widget = FilterWidget() self.filter_widget.filterBtnClicked.connect(self.on_filter_btn_clicked) self.horizontalLayout.addWidget(self.filter_widget) if prefs.SHOW_SAMPLE_FILTERS: self.filter_widget.set_sample_filters(prefs.SAMPLE_FILTERS) self.find_widget = FindWidget() self.find_widget.findBtnClicked.connect(self.on_find_btn_clicked) self.find_widget.set_fields(prefs.FIND_FIELDS) self.horizontalLayout.addWidget(self.find_widget) self.show() def init_plugins(self): """Inits plugins""" self.manager = PluginManager() self.manager.setPluginPlaces(["plugins"]) self.manager.collectPlugins() for plugin in self.manager.getAllPlugins(): print_debug(f"Plugin found: {plugin.name}") def create_plugins_menu(self): """Creates plugins menu""" self.plugins_topmenu.clear() reload_action = QAction("Reload plugins", self) reload_action.setShortcut("Ctrl+R") func = functools.partial(self.reload_plugins) reload_action.triggered.connect(func) self.plugins_topmenu.addAction(reload_action) self.plugins_topmenu.addSeparator() plugins_menu = QMenu("Run plugin", self) for plugin in self.manager.getAllPlugins(): action = QAction(plugin.name, self) func = functools.partial(self.execute_plugin, plugin) action.triggered.connect(func) plugins_menu.addAction(action) self.plugins_topmenu.addMenu(plugins_menu) def create_trace_table_menu(self): """Creates right click menu for trace table""" self.trace_table_menu = QMenu(self) print_action = QAction("Print selected cells", self) print_action.triggered.connect(self.trace_table.print_selected_cells) self.trace_table_menu.addAction(print_action) add_bookmark_action = QAction("Add Bookmark", self) add_bookmark_action.triggered.connect(self.trace_table.create_bookmark) self.trace_table_menu.addAction(add_bookmark_action) plugins_menu = QMenu("Plugins", self) for plugin in self.manager.getAllPlugins(): action = QAction(plugin.name, self) func = functools.partial(self.execute_plugin, plugin) action.triggered.connect(func) plugins_menu.addAction(action) self.trace_table_menu.addMenu(plugins_menu) self.trace_table.menu = self.trace_table_menu def reload_plugins(self): """Reloads plugins""" self.init_plugins() self.create_trace_table_menu() self.create_plugins_menu() def on_trace_table_row_changed(self): """Called when selected row changes""" selected_row_ids = self.get_selected_row_ids(self.trace_table) if not selected_row_ids: return row_id = selected_row_ids[0] regs = self.trace_data.get_regs_and_values(row_id) modified_regs = [] if prefs.HIGHLIGHT_MODIFIED_REGS: modified_regs = self.trace_data.get_modified_regs(row_id) self.reg_table.set_data(regs, modified_regs) mem = [] if "mem" in self.trace_data.trace[row_id]: mem = self.trace_data.trace[row_id]["mem"] self.mem_table.set_data(mem) self.update_status_bar() def on_filter_btn_clicked(self, filter_text: str): self.filter_text = filter_text if self.trace_data is None: return try: filtered_trace = filter_trace( self.trace_data.trace, self.trace_data.get_regs(), filter_text, ) except Exception as exc: self.show_messagebox("Filter error", f"{exc}") # print(traceback.format_exc()) else: self.filtered_trace = filtered_trace self.show_filtered_trace() self.update_status_bar() def on_find_btn_clicked(self, keyword: str, field_index: int, direction: int): """Find next or prev button clicked""" current_row = self.trace_table.currentRow() if current_row < 0: current_row = 0 if self.trace_table.pagination is not None: pagination = self.trace_table.pagination page = pagination.current_page rows_per_page = pagination.rows_per_page current_row += (page - 1) * rows_per_page if field_index == 0: field = TraceField.DISASM elif field_index == 1: field = TraceField.REGS elif field_index == 2: field = TraceField.MEM elif field_index == 3: field = TraceField.MEM_ADDR elif field_index == 4: field = TraceField.MEM_VALUE elif field_index == 5: field = TraceField.COMMENT elif field_index == 6: field = TraceField.ANY try: row_number = find( trace=self.get_visible_trace(), field=field, keyword=keyword, start_row=current_row + direction, direction=direction, ) except Exception as exc: self.show_messagebox("Find error", f"{exc}") print(traceback.format_exc()) self.print(traceback.format_exc()) return if row_number is not None: self.trace_table.go_to_row(row_number) else: print_debug( f"{keyword} not found (row: {current_row}, direction: {direction})" ) def get_visible_trace(self): """Returns the trace that is currently shown on trace table""" index = self.select_trace_combo_box.currentIndex() if self.trace_data is not None: if index == 0: return self.trace_data.trace else: return self.filtered_trace return None def bookmark_table_context_menu_event(self): """Context menu for bookmark table right click""" self.bookmark_menu.popup(QCursor.pos()) def dialog_open_trace(self): """Shows dialog to open trace file""" all_traces = "All traces (*.tvt *.trace32 *.trace64)" all_files = "All files (*.*)" filename = QFileDialog.getOpenFileName( self, "Open trace", "", all_traces + ";; " + all_files)[0] if filename: self.open_trace(filename) if self.trace_data: self.save_trace_action.setEnabled(True) def dialog_save_trace_as(self): """Shows a dialog to select a save file""" filename = QFileDialog.getSaveFileName( self, "Save trace as", "", "Trace Viewer traces (*.tvt);; All files (*.*)")[0] print_debug("Save trace as: " + filename) if filename and trace_files.save_as_tv_trace(self.trace_data, filename): self.trace_data.filename = filename self.save_trace_action.setEnabled(True) def dialog_save_trace_as_json(self): """Shows a dialog to save trace to JSON file""" filename = QFileDialog.getSaveFileName( self, "Save as JSON", "", "JSON files (*.txt);; All files (*.*)")[0] print_debug("Save trace as: " + filename) if filename: trace_files.save_as_json(self.trace_data, filename) def execute_plugin(self, plugin): """Executes a plugin and updates tables""" print_debug(f"Executing a plugin: {plugin.name}") try: plugin.plugin_object.execute(self.api) except Exception: print("Error in plugin:") print(traceback.format_exc()) self.print("Error in plugin:") self.print(traceback.format_exc()) finally: if prefs.USE_SYNTAX_HIGHLIGHT_IN_LOG: self.highlight.rehighlight() def show_filtered_trace(self): """Shows filtered_trace on trace_table""" if self.select_trace_combo_box.currentIndex() == 0: self.select_trace_combo_box.setCurrentIndex(1) else: self.trace_table.set_data(self.filtered_trace) self.trace_table.populate() def set_comment(self, row_id, comment): """Sets comment to row on full trace""" self.trace_data.set_comment(row_id, comment) def on_bookmark_table_cell_edited(self, item): """Called when any cell is edited on bookmark table""" cell_type = item.whatsThis() bookmarks = self.trace_data.get_bookmarks() row = self.bookmark_table.currentRow() if row < 0: print_debug("Error, could not edit bookmark.") return if cell_type == "startrow": bookmarks[row].startrow = int(item.text()) elif cell_type == "endrow": bookmarks[row].endrow = int(item.text()) elif cell_type == "address": bookmarks[row].addr = item.text() elif cell_type == "disasm": bookmarks[row].disasm = item.text() elif cell_type == "comment": bookmarks[row].comment = item.text() else: print_debug("Unknown field edited on bookmark table...") def open_trace(self, filename): """Opens and reads a trace file""" print_debug(f"Opening trace file: {filename}") self.close_trace() self.trace_data = trace_files.open_trace(filename) if self.trace_data is None: print_debug(f"Error, couldn't open trace file: {filename}") else: if prefs.PAGINATION_ENABLED: self.trace_pagination.set_current_page(1, True) self.trace_table.get_syntax_highlighter().reset() self.trace_table.set_data(self.trace_data.trace) self.trace_table.populate() self.trace_table.selectRow(0) self.setWindowTitle( f"{filename.split('/')[-1]} - {prefs.PACKAGE_NAME}") self.update_bookmark_table() self.trace_table.update_column_widths() def close_trace(self): """Clears trace and updates UI""" self.trace_data = None self.filtered_trace = [] self.trace_table.set_data([]) self.update_ui() def update_ui(self): """Updates tables and status bar""" self.trace_table.populate() self.update_bookmark_table() self.update_status_bar() def save_trace(self): """Saves a trace file""" filename = self.trace_data.filename print_debug("Save trace: " + filename) if filename: trace_files.save_as_tv_trace(self.trace_data, filename) def show_about_dialog(self): """Shows an about dialog""" title = "About" name = prefs.PACKAGE_NAME version = prefs.PACKAGE_VERSION copyrights = prefs.PACKAGE_COPYRIGHTS url = prefs.PACKAGE_URL text = f"{name} {version} \n {copyrights} \n {url}" QMessageBox().about(self, title, text) def update_column_widths(self, table): """Updates column widths of a TableWidget to match the content""" table.setVisible(False) # fix ui glitch with column widths table.resizeColumnsToContents() table.horizontalHeader().setStretchLastSection(True) table.setVisible(True) def update_status_bar(self): """Updates status bar""" if self.trace_data is None: return table = self.trace_table row = table.currentRow() row_count = table.rowCount() row_info = f"{row}/{row_count - 1}" msg = f"Row: {row_info} " selected_row_id = 0 row_ids = self.trace_table.get_selected_row_ids() if row_ids: selected_row_id = row_ids[0] msg += f" | {len(self.trace_data.trace)} rows in full trace." if len(self.filter_text) > 0: msg += f" | {len(self.filtered_trace)} rows in filtered trace." bookmark = self.trace_data.get_bookmark_from_row(selected_row_id) if bookmark: msg += f" | Bookmark: {bookmark.disasm} ; {bookmark.comment}" self.status_bar.showMessage(msg) def on_reg_checkbox_change(self, reg: str, is_checked: bool): """Callback for register checkbox change""" highlighter = self.trace_table.get_syntax_highlighter() if highlighter: highlighter.set_reg_highlight(reg, is_checked) # force repaint, update() didn't work self.trace_table.setVisible(False) self.trace_table.setVisible(True) else: print_debug("Error, highlighter not found!") def get_selected_row_ids(self, table): """Returns IDs of all selected rows of TableWidget. Args: table: PyQt TableWidget returns: list: Ordered list of row ids """ # use a set so we don't get duplicate ids row_ids_set = set( table.item(index.row(), 0).text() for index in table.selectedIndexes()) try: row_ids_list = [int(i) for i in row_ids_set] except ValueError: print_debug("Error. Values in the first column must be integers.") return [] return sorted(row_ids_list) def on_trace_combo_box_index_changed(self, index): """Trace selection combo box index changed""" self.trace_table.set_data(self.get_visible_trace()) other_index = index ^ 1 if prefs.PAGINATION_ENABLED: # save current page self.trace_current_pages[ other_index] = self.trace_pagination.current_page self.trace_pagination.set_current_page( self.trace_current_pages[index], True) # save scrollbar value current_scroll = self.trace_table.verticalScrollBar().value() self.trace_scroll_values[other_index] = current_scroll next_value = self.trace_scroll_values[index] self.trace_table.populate() QApplication.processEvents() # this is needed to update the scrollbar self.trace_table.verticalScrollBar().setValue(next_value) def go_to_row_in_visible_trace(self, row): """Goes to given row in currently visible trace""" self.trace_table.go_to_row(row) self.tab_widget.setCurrentIndex(0) def go_to_row_in_full_trace(self, row_id): """Switches to full trace and goes to given row""" # make sure we are shown full trace, not filtered if self.select_trace_combo_box.currentIndex() == 1: self.select_trace_combo_box.setCurrentIndex(0) self.go_to_row_in_visible_trace(row_id) def go_to_bookmark_in_trace(self): """Goes to trace row of selected bookmark""" selected_row_ids = self.get_selected_row_ids(self.bookmark_table) if not selected_row_ids: print_debug("Error. No bookmark selected.") return self.go_to_row_in_full_trace(selected_row_ids[0]) def clear_bookmarks(self): """Clears all bookmarks""" self.trace_data.clear_bookmarks() self.update_bookmark_table() def delete_bookmarks(self): """Deletes selected bookmarks""" selected = self.bookmark_table.selectedItems() if not selected: print_debug("Could not delete a bookmark. Nothing selected.") return selected_rows = sorted(set({sel.row() for sel in selected})) for row in reversed(selected_rows): self.trace_data.delete_bookmark(row) self.bookmark_table.removeRow(row) def get_selected_bookmarks(self): """Returns selected bookmarks""" selected = self.bookmark_table.selectedItems() if not selected: print_debug("No bookmarks selected.") return [] selected_rows = sorted(set({sel.row() for sel in selected})) all_bookmarks = self.trace_data.get_bookmarks() return [all_bookmarks[i] for i in selected_rows] def add_bookmark(self, bookmark): if prefs.ASK_FOR_BOOKMARK_COMMENT: comment = self.get_string_from_user( "Bookmark comment", "Give a comment for bookmark:") if comment: bookmark.comment = comment self.trace_data.add_bookmark(bookmark) self.update_bookmark_table() def update_bookmark_table(self): """Updates bookmarks table from trace_data""" if self.trace_data is None: return table = self.bookmark_table try: table.itemChanged.disconnect() except Exception: pass bookmarks = self.trace_data.get_bookmarks() table.setRowCount(len(bookmarks)) for i, bookmark in enumerate(bookmarks): startrow = QTableWidgetItem(bookmark.startrow) startrow.setData(Qt.DisplayRole, int(bookmark.startrow)) startrow.setWhatsThis("startrow") table.setItem(i, 0, startrow) endrow = QTableWidgetItem(bookmark.endrow) endrow.setData(Qt.DisplayRole, int(bookmark.endrow)) endrow.setWhatsThis("endrow") table.setItem(i, 1, endrow) address = QTableWidgetItem(bookmark.addr) address.setWhatsThis("address") table.setItem(i, 2, address) disasm = QTableWidgetItem(bookmark.disasm) disasm.setWhatsThis("disasm") table.setItem(i, 3, disasm) comment = QTableWidgetItem(bookmark.comment) comment.setWhatsThis("comment") table.setItem(i, 4, comment) table.setRowHeight(i, 14) table.itemChanged.connect(self.on_bookmark_table_cell_edited) self.update_column_widths(table) def print(self, text): """Prints text to TextEdit on log tab""" self.log_text_edit.appendPlainText(str(text)) def go_to_row(self, table, row): """Scrolls a table to the specified row""" table.scrollToItem(table.item(row, 3), QAbstractItemView.PositionAtCenter) def ask_user(self, title, question): """Shows a messagebox with yes/no question Args: title (str): MessageBox title question (str): MessageBox qustion label Returns: bool: True if user clicked yes, False otherwise """ answer = QMessageBox.question( self, title, question, QMessageBox.StandardButtons(QMessageBox.Yes | QMessageBox.No), ) return bool(answer == QMessageBox.Yes) def get_string_from_user(self, title, label): """Gets a string from user Args: title (str): Input dialog title label (str): Input dialog label Returns: string: String given by user, empty string if user clicked cancel """ answer, ok_clicked = QInputDialog.getText(self, title, label, QLineEdit.Normal, "") if ok_clicked: return answer return "" def get_values_from_user(self, title, data, on_ok_clicked=None): """Gets values from user Args: title (str): Input dialog title data (list): List of dicts on_ok_clicked (method): Callback function to e.g. check the input Returns: list: List of values given by user, empty list if user canceled """ input_dlg = InputDialog(self, title, data, on_ok_clicked) input_dlg.exec_() return input_dlg.get_data() def show_messagebox(self, title, msg): """Shows a messagebox""" alert = QMessageBox() alert.setWindowTitle(title) alert.setText(msg) alert.exec_()
class FakeSite: def __init__(self): self.template_system = self self.invariant = False self.debug = True self.config = { "DISABLED_PLUGINS": [], "EXTRA_PLUGINS": [], "DEFAULT_LANG": "en", "MARKDOWN_EXTENSIONS": [ "markdown.extensions.fenced_code", "markdown.extensions.codehilite", ], "TRANSLATIONS_PATTERN": "{path}.{lang}.{ext}", "LISTINGS_FOLDERS": { "listings": "listings" }, "TRANSLATIONS": { "en": "" }, } self.EXTRA_PLUGINS = self.config["EXTRA_PLUGINS"] self.plugin_manager = PluginManager( categories_filter={ "Command": Command, "Task": Task, "LateTask": LateTask, "TemplateSystem": TemplateSystem, "PageCompiler": PageCompiler, "TaskMultiplier": TaskMultiplier, "CompilerExtension": CompilerExtension, "MarkdownExtension": MarkdownExtension, "RestExtension": RestExtension, }) self.shortcode_registry = {} self.plugin_manager.setPluginInfoExtension("plugin") places = [ os.path.join(os.path.dirname(nikola.utils.__file__), "plugins") ] self.plugin_manager.setPluginPlaces(places) self.plugin_manager.collectPlugins() self.compiler_extensions = self._activate_plugins_of_category( "CompilerExtension") self.timeline = [FakePost(title="Fake post", slug="fake-post")] self.rst_transforms = [] self.post_per_input_file = {} # This is to make plugin initialization happy self.template_system = self self.name = "mako" def _activate_plugins_of_category(self, category): """Activate all the plugins of a given category and return them.""" # this code duplicated in nikola/nikola.py plugins = [] for plugin_info in self.plugin_manager.getPluginsOfCategory(category): if plugin_info.name in self.config.get("DISABLED_PLUGINS"): self.plugin_manager.removePluginFromCategory( plugin_info, category) else: self.plugin_manager.activatePluginByName(plugin_info.name) plugin_info.plugin_object.set_site(self) plugins.append(plugin_info) return plugins def render_template(self, name, _, context): return '<img src="IMG.jpg">' # this code duplicated in nikola/nikola.py def register_shortcode(self, name, f): """Register function f to handle shortcode "name".""" if name in self.shortcode_registry: nikola.utils.LOGGER.warning('Shortcode name conflict: %s', name) return self.shortcode_registry[name] = f def apply_shortcodes(self, data, *a, **kw): """Apply shortcodes from the registry on data.""" return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry, **kw) def apply_shortcodes_uuid(self, data, shortcodes, *a, **kw): """Apply shortcodes from the registry on data.""" return nikola.shortcodes.apply_shortcodes(data, self.shortcode_registry, **kw)
class TestDef(object): def __init__(self): # set aside storage for options and cmd line args self.options = {} self.args = [] # record if we have loaded the plugins or # not - this is just a bozo check to ensure # someone doesn't tell us to do it twice self.loaded = False # set aside a spot for a logger object, and # note that it hasn't yet been defined self.logger = None self.modcmd = None self.execmd = None self.harasser = None self.config = None self.stages = None self.tools = None self.utilities = None self.defaults = None self.log = {} self.watchdog = None self.plugin_trans_sem = Semaphore() def setOptions(self, args): self.options = vars(args) self.args = args # if they want us to clear the scratch, then do so if self.options['clean'] and os.path.isdir(self.options['scratchdir']) : shutil.rmtree(self.options['scratchdir']) # setup the scratch directory _mkdir_recursive(self.options['scratchdir']) # private function to convert values def __convert_value(self, opt, inval): if opt is None or type(opt) is str: return 0, inval elif type(opt) is bool: if type(inval) is bool: return 0, inval elif type(inval) is str: if inval.lower() in ['true', '1', 't', 'y', 'yes']: return 0, True else: return 0, False elif type(inval) is int: if 0 == inval: return 0, False else: return 0, True elif is_py2 and type(inval) is unicode: return 0, int(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None elif type(opt) is int: if type(inval) is int: return 0, inval elif type(inval) is str: return 0, int(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None elif type(opt) is float: if type(inval) is float: return 0, inval elif type(inval) is str or type(inval) is int: return 0, float(inval) else: # unknown conversion required print("Unknown conversion required for " + inval) return 1, None else: return 1, None # scan the key-value pairs obtained from the configuration # parser and compare them with the options defined for a # given plugin. Generate an output dictionary that contains # the updated set of option values, the default value for # any option that wasn't included in the configuration file, # and return an error status plus output identifying any # keys in the configuration file that are not supported # by the list of options # # @log [INPUT] # - a dictionary that will return the status plus # stderr containing strings identifying any # provided keyvals that don't have a corresponding # supported option # @options [INPUT] # - a dictionary of tuples, each consisting of three # entries: # (a) the default value # (b) data type # (c) a help-like description # @keyvals [INPUT] # - a dictionary of key-value pairs obtained from # the configuration parser # @target [OUTPUT] # - the resulting dictionary of key-value pairs def parseOptions(self, log, options, keyvals, target): # parse the incoming keyvals dictionary against the source # options. If a source option isn't provided, then # copy it across to the target. opts = list(options.keys()) kvkeys = list(keyvals.keys()) for opt in opts: found = False for kvkey in kvkeys: if kvkey == opt: # they provided us with an update, so # pass this value into the target - expand # any provided lists if keyvals[kvkey] is None: continue st, outval = self.__convert_value(options[opt][0], keyvals[kvkey]) if 0 == st: target[opt] = outval else: if len(keyvals[kvkey]) == 0: # this indicates they do not want this option found = True break if keyvals[kvkey][0][0] == "[": # they provided a list - remove the brackets val = keyvals[kvkey].replace('[','') val = val.replace(']','') # split the input to pickup sets of options newvals = list(val) # convert the values to specified type i=0 for val in newvals: st, newvals[i] = self.__convert_value(opt[0], val) i = i + 1 target[opt] = newvals else: st, target[opt] = self.__convert_value(opt[0], keyvals[kvkey]) found = True break if not found: # they didn't provide this one, so # transfer only the value across target[opt] = options[opt][0] # add in any default settings that have not # been overridden - anything set by this input # stage will override the default if self.defaults is not None: keys = self.defaults.options.keys() for key in keys: if key not in target: target[key] = self.defaults.options[key][0] # now go thru in the reverse direction to see # if any keyvals they provided aren't supported # as this would be an error unsupported_options = [] for kvkey in kvkeys: # ignore some standard keys if kvkey in ['section', 'plugin']: continue try: if target[kvkey] is not None: pass except KeyError: # some always need to be passed if kvkey in ['parent', 'asis']: target[kvkey] = keyvals[kvkey] else: unsupported_options.append(kvkey) if unsupported_options: sys.exit("ERROR: Unsupported options for section [%s]: %s" % (log['section'], ",".join(unsupported_options))) log['status'] = 0 log['options'] = target return def loadPlugins(self, basedir, topdir): if self.loaded: print("Cannot load plugins multiple times") exit(1) self.loaded = True # find the loader utility so we can bootstrap ourselves try: m = imp.load_source("LoadClasses", os.path.join(basedir, "LoadClasses.py")); except ImportError: print("ERROR: unable to load LoadClasses that must contain the class loader object") exit(1) cls = getattr(m, "LoadClasses") a = cls() # setup the loader object self.loader = a.__class__(); # Setup the array of directories we will search for plugins # Note that we always look at the topdir location by default plugindirs = [] plugindirs.append(topdir) if self.options['plugindir']: # could be a comma-delimited list, so split on commas x = self.options['plugindir'].split(',') for y in x: # prepend so we always look at the given # location first in case the user wants # to "overload/replace" a default MTT # class definition plugindirs.insert(0, y) # Traverse the plugin directory tree and add all # the class definitions we can find for dirPath in plugindirs: try: filez = os.listdir(dirPath) for file in filez: file = os.path.join(dirPath, file) if os.path.isdir(file): self.loader.load(file) except: if not self.options['ignoreloadpatherrs']: print("Plugin directory",dirPath,"not found") sys.exit(1) # Build the stages plugin manager self.stages = PluginManager() # set the location self.stages.setPluginPlaces(plugindirs) # Get a list of all the categories - this corresponds to # the MTT stages that have been defined. Note that we # don't need to formally define the stages here - anyone # can add a new stage, or delete an old one, by simply # adding or removing a plugin directory. self.stages.setCategoriesFilter(self.loader.stages) # Load all plugins we find there self.stages.collectPlugins() # Build the tools plugin manager - tools differ from sections # in that they are plugins we will use to execute the various # sections. For example, the TestRun section clearly needs the # ability to launch jobs. There are many ways to launch jobs # depending on the environment, and sometimes several ways to # start jobs even within one environment (e.g., mpirun vs # direct launch). self.tools = PluginManager() # location is the same self.tools.setPluginPlaces(plugindirs) # Get the list of tools - not every tool will be capable # of executing. For example, a tool that supports direct launch # against a specific resource manager cannot be used on a # system being managed by a different RM. self.tools.setCategoriesFilter(self.loader.tools) # Load all the tool plugins self.tools.collectPlugins() # Tool plugins are required to provide a function we can # probe to determine if they are capable of operating - check # those now and prune those tools that cannot support this # environment # Build the utilities plugins self.utilities = PluginManager() # set the location self.utilities.setPluginPlaces(plugindirs) # Get the list of available utilities. self.utilities.setCategoriesFilter(self.loader.utilities) # Load all the utility plugins self.utilities.collectPlugins() # since we use these all over the place, find the # ExecuteCmd and ModuleCmd plugins and record them availUtil = list(self.loader.utilities.keys()) for util in availUtil: for pluginInfo in self.utilities.getPluginsOfCategory(util): if "ExecuteCmd" == pluginInfo.plugin_object.print_name(): self.execmd = pluginInfo.plugin_object elif "ModuleCmd" == pluginInfo.plugin_object.print_name(): self.modcmd = pluginInfo.plugin_object # initialize this module self.modcmd.setCommand(self.options) elif "Watchdog" == pluginInfo.plugin_object.print_name(): self.watchdog = pluginInfo.plugin_object if self.execmd is not None and self.modcmd is not None and self.watchdog is not None: break if self.execmd is None: print("ExecuteCmd plugin was not found") print("This is a basic capability required") print("for MTT operations - cannot continue") sys.exit(1) # Configure harasser plugin print(self.tools.getPluginsOfCategory("Harasser")) for pluginInfo in self.tools.getPluginsOfCategory("Harasser"): print(pluginInfo.plugin_object.print_name()) if "Harasser" == pluginInfo.plugin_object.print_name(): self.harasser = pluginInfo.plugin_object break if self.harasser is None: print("Harasser plugin was not found") print("This is required for all TestRun plugins") print("cannot continue") sys.exit(1) # similarly, capture the highest priority defaults stage here pri = -1 for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"): if pri < pluginInfo.plugin_object.priority(): self.defaults = pluginInfo.plugin_object pri = pluginInfo.plugin_object.priority() return def printInfo(self): # Print the available MTT sections out, if requested if self.options['listsections']: print("Supported MTT stages:") # print them in the default order of execution for stage in self.loader.stageOrder: print(" " + stage) exit(0) # Print the detected plugins for a given stage if self.options['listplugins']: # if the list is '*', print the plugins for every stage if self.options['listplugins'] == "*": sections = self.loader.stageOrder else: sections = self.options['listplugins'].split(',') print() for section in sections: print(section + ":") try: for pluginInfo in self.stages.getPluginsOfCategory(section): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid stage name " + section) print() exit(1) # Print the options for a given plugin if self.options['liststageoptions']: # if the list is '*', print the options for every stage/plugin if self.options['liststageoptions'] == "*": sections = self.loader.stageOrder else: sections = self.options['liststageoptions'].split(',') print() for section in sections: print(section + ":") try: for pluginInfo in self.stages.getPluginsOfCategory(section): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options(self, " ") except KeyError: print(" Invalid stage name " + section) print() exit(1) # Print the available MTT tools out, if requested if self.options['listtools']: print("Available MTT tools:") availTools = list(self.loader.tools.keys()) for tool in availTools: print(" " + tool) exit(0) # Print the detected tool plugins for a given tool type if self.options['listtoolmodules']: # if the list is '*', print the plugins for every type if self.options['listtoolmodules'] == "*": print() availTools = list(self.loader.tools.keys()) else: availTools = self.options['listtoolmodules'].split(',') print() for tool in availTools: print(tool + ":") try: for pluginInfo in self.tools.getPluginsOfCategory(tool): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid tool type name",tool) print() exit(1) # Print the options for a given plugin if self.options['listtooloptions']: # if the list is '*', print the options for every stage/plugin if self.options['listtooloptions'] == "*": availTools = list(self.loader.tools.keys()) else: availTools = self.options['listtooloptions'].split(',') print() for tool in availTools: print(tool + ":") try: for pluginInfo in self.tools.getPluginsOfCategory(tool): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options(self, " ") except KeyError: print(" Invalid tool type name " + tool) print() exit(1) # Print the available MTT utilities out, if requested if self.options['listutils']: print("Available MTT utilities:") availUtils = list(self.loader.utilities.keys()) for util in availUtils: print(" " + util) exit(0) # Print the detected utility plugins for a given tool type if self.options['listutilmodules']: # if the list is '*', print the plugins for every type if self.options['listutilmodules'] == "*": print() availUtils = list(self.loader.utilities.keys()) else: availUtils = self.options['listutilitymodules'].split(',') print() for util in availUtils: print(util + ":") try: for pluginInfo in self.utilities.getPluginsOfCategory(util): print(" " + pluginInfo.plugin_object.print_name()) except KeyError: print(" Invalid utility type name") print() exit(1) # Print the options for a given plugin if self.options['listutiloptions']: # if the list is '*', print the options for every stage/plugin if self.options['listutiloptions'] == "*": availUtils = list(self.loader.utilities.keys()) else: availUtils = self.options['listutiloptions'].split(',') print() for util in availUtils: print(util + ":") try: for pluginInfo in self.utilities.getPluginsOfCategory(util): print(" " + pluginInfo.plugin_object.print_name() + ":") pluginInfo.plugin_object.print_options(self, " ") except KeyError: print(" Invalid utility type name " + util) print() exit(1) # if they asked for the version info, print it and exit if self.options['version']: for pluginInfo in self.tools.getPluginsOfCategory("Version"): print("MTT Base: " + pluginInfo.plugin_object.getVersion()) print("MTT Client: " + pluginInfo.plugin_object.getClientVersion()) sys.exit(0) def openLogger(self): # there must be a logger utility or we can't do # anything useful if not self.utilities.activatePluginByName("Logger", "Base"): print("Required Logger plugin not found or could not be activated") sys.exit(1) # execute the provided test description self.logger = self.utilities.getPluginByName("Logger", "Base").plugin_object self.logger.open(self) return def fill_log_interpolation(self, basestr, sublog): if isinstance(sublog, str): self.config.set("LOG", basestr, sublog.replace("$","$$")) elif isinstance(sublog, dict): for k,v in sublog.items(): self.fill_log_interpolation("%s.%s" % (basestr, k), v) elif isinstance(sublog, list): if sum([((isinstance(t, list) or isinstance(t, tuple)) and len(t) == 2) for t in sublog]) == len(sublog): self.fill_log_interpolation(basestr, {k:v for k,v in sublog}) else: for i,v in enumerate(sublog): self.fill_log_interpolation("%s.%d" % (basestr, i), v) else: self.fill_log_interpolation(basestr, str(sublog)) def configTest(self): # setup the configuration parser self.config = configparser.SafeConfigParser(interpolation=configparser.ExtendedInterpolation()) # Set the config parser to make option names case sensitive. self.config.optionxform = str # fill ENV section with environemt variables self.config.add_section('ENV') for k,v in os.environ.items(): self.config.set('ENV', k, v.replace("$","$$")) # Add LOG section filled with log results of stages self.config.add_section('LOG') thefulllog = self.logger.getLog(None) for e in thefulllog: self.fill_log_interpolation(e['section'].replace(":","_"), e) # log the list of files - note that the argument parser # puts the input files in a list, with the first member # being the list of input files self.log['inifiles'] = self.args.ini_files[0] # initialize the list of active sections self.actives = [] # if they specified a list to execute, then use it sections = [] if self.args.section: sections = self.args.section.split(",") skip = False elif self.args.skipsections: sections = self.args.skipsections.split(",") skip = True else: sections = None # cycle thru the input files for testFile in self.log['inifiles']: if not os.path.isfile(testFile): print("Test description file",testFile,"not found!") sys.exit(1) self.config.read(self.log['inifiles']) # Check for ENV input required_env = [] all_file_contents = [] for testFile in self.log['inifiles']: file_contents = open(testFile, "r").read() file_contents = "\n".join(["%s %d: %s" % (testFile.split("/")[-1],i,l) for i,l in enumerate(file_contents.split("\n")) if not l.lstrip().startswith("#")]) all_file_contents.append(file_contents) if "${ENV:" in file_contents: required_env.extend([s.split("}")[0] for s in file_contents.split("${ENV:")[1:]]) env_not_found = set([e for e in required_env if e not in os.environ.keys()]) lines_with_env_not_found = [] for file_contents in all_file_contents: lines_with_env_not_found.extend(["%s: %s"%(",".join([e for e in env_not_found if "${ENV:%s}"%e in l]),l) \ for l in file_contents.split("\n") \ if sum(["${ENV:%s}"%e in l for e in env_not_found])]) if lines_with_env_not_found: print("ERROR: Not all required environment variables are defined.") print("ERROR: Still need:") for l in lines_with_env_not_found: print("ERROR: %s"%l) sys.exit(1) for section in self.config.sections(): if section.startswith("SKIP") or section.startswith("skip"): # users often want to temporarily ignore a section # of their test definition file, but don't want to # remove it lest they forget what it did. So let # them just mark the section as "skip" to be ignored continue # if we are to filter the sections, then do so takeus = True if sections is not None: found = False for sec in sections: if sec == section: found = True sections.remove(sec) if skip: takeus = False break if not found and not skip: takeus = False if takeus: self.actives.append(section) if sections is not None and 0 != len(sections) and not skip: print("ERROR: sections were specified for execution and not found:",sections) sys.exit(1) return # Used with combinatorial executor, loads next .ini file to be run with the # sequential executor def configNewTest(self, file): # clear the configuration parser for section in self.config.sections(): self.config.remove_section(section) # read in the file self.config.read(file) for section in self.config.sections(): if section.startswith("SKIP") or section.startswith("skip"): # users often want to temporarily ignore a section # of their test definition file, but don't want to # remove it lest they forget what it did. So let # them just mark the section as "skip" to be ignored continue if self.logger is not None: self.logger.verbose_print("SECTION: " + section) self.logger.verbose_print(self.config.items(section)) return def executeTest(self, enable_loop=True, executor="sequential"): if not self.loaded: print("Plugins have not been loaded - cannot execute test") exit(1) if self.config is None: print("No test definition file was parsed - cannot execute test") exit(1) if not self.tools.getPluginByName(executor, "Executor"): print("Specified executor %s not found" % executor) exit(1) # activate the specified plugin self.tools.activatePluginByName(executor, "Executor") # execute the provided test description executor = self.tools.getPluginByName(executor, "Executor") status = executor.plugin_object.execute(self) if status == 0 and self.options['clean_after'] and os.path.isdir(self.options['scratchdir']): self.logger.verbose_print("Cleaning up scratchdir after successful run") shutil.rmtree(self.options['scratchdir']) # Loop forever if specified if enable_loop and self.options['loopforever']: while True: self.logger.reset() self.executeTest(enable_loop=False, executor=executor) return status def executeCombinatorial(self, enable_loop=True): return self.executeTest(enable_loop=enable_loop, executor="combinatorial") def printOptions(self, options): # if the options are empty, report that if not options: lines = ["None"] return lines # create the list of options opts = [] vals = list(options.keys()) for val in vals: opts.append(val) if options[val][0] is None: opts.append("None") elif isinstance(options[val][0], bool): if options[val][0]: opts.append("True") else: opts.append("False") elif isinstance(options[val][0], list): opts.append(" ".join(options[val][0])) elif isinstance(options[val][0], int): opts.append(str(options[val][0])) else: opts.append(options[val][0]) opts.append(options[val][1]) # print the options, their default value, and # the help description in 3 column format max1 = 0 max2 = 0 for i in range(0,len(opts),3): # we want all the columns to line up # and left-justify, so first find out # the max len of each of the first two # column entries if len(opts[i]) > max1: max1 = len(opts[i]) if type(opts[i+1]) is not str: optout = str(opts[i+1]) else: optout = opts[i+1] if len(optout) > max2: max2 = len(optout) # provide some spacing max1 = max1 + 4 max2 = max2 + 4 # cycle thru again, padding each entry to # align the columns lines = [] sp = " " for i in range(0,len(opts),3): line = opts[i] + (max1-len(opts[i]))*sp if type(opts[i+1]) is not str: optout = str(opts[i+1]) else: optout = opts[i+1] line = line + optout + (max2-len(optout))*sp # to make this more readable, we will wrap the line at # 130 characters. First, see if the line is going to be # too long if 130 < (len(line) + len(opts[i+2])): # split the remaining column into individual words words = opts[i+2].split() first = True for word in words: if (len(line) + len(word)) < 130: if first: line = line + word first = False else: line = line + " " + word else: lines.append(line) line = (max1 + max2)*sp + word if 0 < len(line): lines.append(line) else: # the line is fine - so just add the last piece line = line + opts[i+2] # append the result lines.append(line) # add one blank line lines.append("") return lines def selectPlugin(self, name, category): if category == "stage": try: availStages = list(self.loader.stages.keys()) for stage in availStages: for pluginInfo in self.stages.getPluginsOfCategory(stage): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None elif category == "tool": try: availTools = list(self.loader.tools.keys()) for tool in availTools: for pluginInfo in self.tools.getPluginsOfCategory(tool): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None elif category == "utility": try: availUtils = list(self.loader.utilities.keys()) for util in availUtils: for pluginInfo in self.utilities.getPluginsOfCategory(util): if name == pluginInfo.plugin_object.print_name(): return pluginInfo.plugin_object # didn't find it return None except: return None else: print("Unrecognized category:",category) return None
class HandlerManager(QThread): # 返回命令执行结果 # int: 0 -- 返回PlainText # 1 -- 返回RichText # str: 结果 outSignal = pyqtSignal(int, str) def __init__(self): super(HandlerManager, self).__init__() # 创建工作队列,用户输入的命令按照顺序put进该队列 # 由CommandHandler在后台逐个处理 self.workQueue = Queue() # 初始化插件功能 self.initPlugin() def __del__(self): self.wait() def initPlugin(self): ''' 启动插件管理器,获取插件列表 ''' # 创建插件管理器 self.pluginManager = PluginManager() # 设置插件接口为 DPlugin,所有插件必须继承categories.DPlugin self.pluginManager.setCategoriesFilter({"DPlugin": DPlugin}) # 设置插件目录 self.pluginManager.setPluginPlaces(['plugins']) # 加载插件到内存 self.pluginManager.locatePlugins() self.pluginManager.loadPlugins() def getAllPlugins(self): '''获取插件列表''' return self.pluginManager.getPluginsOfCategory('DPlugin') def activateAllPlugins(self): '''使能所有插件''' for plugin in self.pluginManager.getPluginsOfCategory('DPlugin'): plugin.plugin_object.activate() def deactivateAllPlugins(self): '''禁用所有插件''' for plugin in self.pluginManager.getPluginsOfCategory('DPlugin'): plugin.plugin_object.deactivate() def activatePlugin(self, name): '''使能特定插件''' self.pluginManager.activatePluginByName(name, category="DPlugin") def deactivatePlugin(self, name): '''使能特定插件''' self.pluginManager.deactivatePluginByName(name, category="DPlugin") def processInput(self, userInput): '''将命令放入队列''' self.workQueue.put((False, userInput)) def processFile(self, filepath): '''将待处理文件放入队列''' self.workQueue.put((True, filepath)) def run(self): '''不断从Queue中获取数据,然后执行解析命令''' while True: isFile, userInput = self.workQueue.get() if isFile: self.execFile(userInput) else: self.execCommand(userInput, None) def execCommand(self, userInput, currentFile): ''' 解析命令 ''' if not len(userInput) or userInput.startswith('#'): return # 命令回显 self.writeCommand(userInput) # 命令分割 command, *params = userInput.split(maxsplit=1) if command == 'source': if len(params) == 1: filepath = params[0] if not os.path.isabs(filepath): basepath = os.path.dirname(currentFile) filepath = os.path.normpath(filepath) filepath = os.path.join(basepath, filepath) self.execFile(filepath) else: self.writeStderr('syntax error: ' + userInput) else: self.dispatchCommand(userInput) def execFile(self, filepath): self.writeCommand('Processing file ' + filepath) if os.path.isfile(filepath) and os.access(filepath, os.R_OK): with open(filepath, 'r') as f: for line in f.readlines(): line = line.strip() self.execCommand(line, filepath) else: self.writeStderr('Cannot open file: ' + filepath) def dispatchCommand(self, userInput): '''将命令分发给各个插件''' status = DPlugin.EXEC_NOTFOUND for plugin in self.pluginManager.getPluginsOfCategory('DPlugin'): ret, resultText = plugin.plugin_object.execCommand(userInput) if ret == DPlugin.EXEC_SUCCESS: self.writeStdout(resultText) status = DPlugin.EXEC_SUCCESS elif ret == DPlugin.EXEC_FAILED: self.writeStderr(resultText) status = DPlugin.EXEC_FAILED if status == DPlugin.EXEC_NOTFOUND: self.writeStderr(userInput + ': Command not found') def writeCommand(self, text): ''' 输出原始命令,蓝色加粗显示,以示区分 ''' self.outSignal.emit(1, text) def writeStdout(self, text): ''' 返回普通输出,文本后加换行 ''' self.outSignal.emit(0, text) def writeStderr(self, text): ''' 返回错误输出,红色加粗显示,以示区分 ''' self.outSignal.emit(2, text)