def load_plugins(parser, logger=None):
"""Load all plugins"""
# import yapsy if needed
try:
from yapsy.PluginManager import PluginManager
except ImportError:
parser.exit(status=3, message='PASTA plugins require yapsy.\n'
'See README file for more informations.\n'
'Alternatively, use the option --no-plugins to disable the'
' plugins\n')
from plugins import SingleConnectionAnalyser, \
InterConnectionsAnalyser
# create the plugin manager
plugin_manager = PluginManager(
categories_filter={
'SingleConnectionAnalyser': SingleConnectionAnalyser,
'InterConnectionsAnalyser': InterConnectionsAnalyser
},
directories_list = [os.path.join(os.path.dirname(sys.argv[0]),
'plugins')],
plugin_info_ext='plugin')
plugin_manager.locatePlugins()
def load_plugin(plugin):
"""A plugin is being loaded"""
if logger is not None:
logger.info('...plugin %s v.%s'
% (plugin.name, plugin.version))
plugin_manager.loadPlugins(load_plugin)
return plugin_manager
class FurnivallApplication(tornado.web.Application): def __init__(self): """ Sets up the Tornado web server and loads all the init data """ # Init mongodb database self.dbconnection = Connection() self.db = self.dbconnection['furnivall'] # Init plugins self.plugin_manager = PluginManager() self.plugin_manager.setPluginPlaces(["./plugins"]) self.plugin_manager.collectPlugins() print "Loaded plugins:" for plugin_info in self.plugin_manager.getAllPlugins(): print ' * ' + plugin_info.name # Init routes urls = [ ] + Route.routes() settings = dict( static_path = os.path.join(data_dir, "static"), template_path = os.path.join(data_dir, "templates"), xsrf_cookies = False, cookie_secret = "11oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1ao/Vo=", ) tornado.web.Application.__init__(self, urls, **settings)
def __init__(self, cfg_p, registry,
categories_filter={"Default":IPlugin},
directories_list=None,
plugin_info_ext="yapsy-plugin"):
PluginManager.__init__(self, categories_filter, directories_list, plugin_info_ext)
self.setPluginInfoClass(EmpAttachmentInfo)
self.config = cfg_p
self.registry = registry
def init_manager():
anl = PluginFileAnalyzerMathingRegex('custom_res_handler_plugins', r'^[A-Za-z0-9]+\.py$')
res = PluginFileLocator(plugin_info_cls=CFCustomResourceHandler)
res.setAnalyzers([anl])
manager = PluginManager(plugin_locator=res, categories_filter={'CFHandlers' : CFCustomResourceHandler})
manager.setPluginPlaces([dirname(__file__) + '/plugins'])
manager.collectPlugins()
return manager
def init_plugin_manager():
global simplePluginManager
if not holder.plugin_manager:
logging.info("init plugin manager")
simplePluginManager = PluginManager(categories_filter={"bots": BotPlugin})
simplePluginManager.setPluginInfoExtension("plug")
holder.plugin_manager = simplePluginManager
else:
simplePluginManager = holder.plugin_manager
def init_plugin_manager():
global simplePluginManager
if not holder.plugin_manager:
logging.debug('init plugin manager')
simplePluginManager = PluginManager(categories_filter={"bots": BotPlugin})
simplePluginManager.setPluginInfoExtension('plug')
holder.plugin_manager = simplePluginManager
else:
simplePluginManager = holder.plugin_manager
class AlgorithmManager():
def __init__(self):
self._pluginManager = PluginManager()
self._pluginManager.setPluginPlaces(config.PLUGIN_DIR)
self._pluginManager.collectPlugins()
for pluginInfo in self._pluginManager.getAllPlugins():
self._pluginManager.activatePluginByName(pluginInfo.name)
def get_alg_names(self):
for plugin in self._pluginManager.getAllPlugins():
plugin.plugin_object.print_name()
def execute_calc(self, data):
run = []
if not self.__is_sequence(data):
raise Exception('Data is no array!')
for alg in self._pluginManager.getAllPlugins():
result, prop = alg.plugin_object.calc(data)
run.append({alg.name : (result, prop)})
return run
def __is_sequence(self, arg):
return (not hasattr(arg, "strip") and
hasattr(arg, "__getitem__") or
hasattr(arg, "__iter__"))
class ProfilesManager(object):
def __init__(self):
self.plugin_location = [PROFILE_MODULE_DIR]
self.pluginManager = PluginManager(plugin_info_ext="plugin")
self.pluginManager.setPluginPlaces(self.plugin_location)
self.pluginManager.collectPlugins()
def modules_info(self):
"""
Get information with regards to each modules loaded. It includes author, category, copyright, description,
details, name, version and website.
:return: information of all modules loaded
"""
modules_info = {}
for pluginInfo in self.pluginManager.getAllPlugins():
modules_info[pluginInfo.name] = pluginInfo
return modules_info
def configForms(self, device_serial, module_name=""):
"""
Get the configuration views of each modules to be displayed on web interface
:return: dictionary of pluginInfo as key and form as value
"""
if module_name:
plugin = self.pluginManager.getPluginByName(name=module_name)
configForms = plugin.plugin_object.get_view()
else:
configForms = {}
for pluginInfo in self.pluginManager.getAllPlugins():
form = pluginInfo.plugin_object.get_view()
configForms[pluginInfo] = form
return configForms
def run_simulation(self,option, profile_name, duration, device_serial, package_name, session):
"""
Run profile simulation script
:return:
"""
plugin = self.pluginManager.getPluginByName(name=profile_name)
if option == RANDOM_INTERACTION:
plugin.plugin_object.runSimulation(duration,package_name, random=True, device_serial=device_serial, session=session)
elif option == SCRIPTED_PROFILE_INTERACTION:
plugin.plugin_object.runSimulation(duration,package_name, random=False, device_serial=device_serial, session=session)
def setup_device(self,module, params, device_serial):
"""
install profile apk and profile app data onto device
:return:
"""
pluginInfo = self.pluginManager.getPluginByName(name=module)
if pluginInfo.name == module:
pluginInfo.plugin_object.prepare(params, device_serial)
return True
def collect_data(self, **kwargs):
self.logger.info("Begin collect_data")
simplePluginManager = PluginManager()
logging.getLogger('yapsy').setLevel(logging.DEBUG)
simplePluginManager.setCategoriesFilter({
"DataCollector": data_collector_plugin
})
# Tell it the default place(s) where to find plugins
self.logger.debug("Plugin directories: %s" % (kwargs['data_collector_plugin_directories']))
simplePluginManager.setPluginPlaces(kwargs['data_collector_plugin_directories'])
simplePluginManager.collectPlugins()
plugin_cnt = 0
plugin_start_time = time.time()
for plugin in simplePluginManager.getAllPlugins():
self.logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(details=plugin.details):
plugin.plugin_object.start()
else:
self.logger.error("Failed to initialize plugin: %s" % (plugin.name))
plugin_cnt += 1
#Wait for the plugings to finish up.
self.logger.info("Waiting for %d plugins to complete." % (plugin_cnt))
for plugin in simplePluginManager.getAllPlugins():
plugin.plugin_object.join()
self.logger.info("%d Plugins completed in %f seconds" % (plugin_cnt, time.time() - plugin_start_time))
def __init__( self ):
QObject.__init__( self )
PluginManager.__init__( self, None,
[ settingsDir + "plugins",
"/usr/share/codimension-plugins" ],
"cdmp" )
self.inactivePlugins = {} # Categorized inactive plugins
self.activePlugins = {} # Categorized active plugins
self.unknownPlugins = [] # Unknown plugins
return
def testCategoryManipulation(self): """ Test querying, removing and adding plugins from/to a category. """ spm = PluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")]) # load the plugins that may be found spm.collectPlugins() # check that the getCategories works self.assertEqual(len(spm.getCategories()),1)
def start_baseline(request):
manager = PluginManager()
plugin_name = request.data["plugin_name"]
plugin = manager.getPluginByName(plugin_name)
board = bci.OpenBCIBoard(port=args.port,
daisy=args.daisy,
filter_data=args.filtering,
scaled_output=True,
log=args.log)
if (plugin_name == "packets_to_csv"):
plugin.plugin_object.pre_activate({}, sample_rate=board.getSampleRate(), eeg_channels=board.getNbEEGChannels(), aux_channels=board.getNbAUXChannels())
def __init__(self,
categories_filter=None,
directories_list=None,
plugin_info_ext=None,
plugin_locator=None):
if categories_filter is None:
categories_filter = {"Default": IMultiprocessChildPlugin}
PluginManager.__init__(self,
categories_filter=categories_filter,
directories_list=directories_list,
plugin_info_ext=plugin_info_ext,
plugin_locator=plugin_locator)
def test_setPluginInfoClass_with_strategies(self):
class SpecificPluginInfo(PluginInfo):
pass
class SpecificLocator(IPluginLocator):
def setPluginInfoClass(self,cls,name):
if not hasattr(self,"icls"):
self.icls = {}
self.icls[name] = cls
loc = SpecificLocator()
pm = PluginManager(plugin_locator=loc)
pm.setPluginInfoClass(SpecificPluginInfo,["mouf","hop"])
self.assertEqual({"mouf":SpecificPluginInfo,"hop":SpecificPluginInfo},loc.icls)
def loadPlugins(self, callback = None):
PluginManager.loadPlugins(self, callback)
for pluginCategory in self.category_mapping.itervalues():
for plugin in pluginCategory:
name = os.path.basename(plugin.path)
plugin.plugin_object.logger = logging.getLogger('IslandoraListener.' + name)
initialized = plugin.plugin_object.initialize(plugin.config_parser)
delattr(plugin, 'config_parser')
if initialized:
logging.debug('Initialized %s' % name)
else:
logging.debug('Failed to initialize %s' % name)
pluginCategory.remove(plugin)
def __init__(self):
directory = os.path.dirname(os.path.abspath(__file__))
PluginManager.__init__(
self,
directories_list=[os.path.join(directory, 'analyzers'), os.path.join(directory, 'transcoders'), os.path.join(directory, 'repositories')],
categories_filter={
"Analyzer": IAnalyzer,
"Transcoder": ITranscoder,
"MetaDataStore": IMetaDataStore,
"Repository": IRepository,
},
plugin_info_ext=('analyzer', 'transcoder', 'repository',)
)
def main():
# Load and register all plugins
manager = PluginManager( directories_list = directories )
manager.collectPlugins()
plugins = manager.getAllPlugins()
plugins.sort( plugin_sort )
for plugin in plugins:
register_plugin( plugin )
# parse arguments and evaluate the current template
args = parser.parse_args()
template = templates[ args.which ]
template.do_work( args , default_replacements )
def run():
logging.basicConfig(level=logging.INFO)
os.chdir('/home/mjolnir/git/PURIKURA')
# H A N D L E P L U G I N S
pm = PluginManager()
pm.setPluginPlaces(['./pyrikura/plugins'])
pm.collectPlugins()
for pi in pm.getAllPlugins():
logging.info('loading plugin %s', pi.name)
pm.activatePluginByName(pi.name)
brokers = {}
nodes = build()
head = nodes[0]
for node in nodes:
brokers[node] = node.load(pm)
for node, broker in brokers.items():
for other in node._listening:
broker.subscribe(brokers[other])
start = time.time()
last_time = 0
shots = 0
last_trigger = 0
for broker in itertools.cycle(brokers.values()):
broker.update()
def test_plugin_locator_with_manager(self):
"""Test cases for plugin locator.
Uses custom PluginLocator inside PluginManager.
"""
from drop.plugins import DropPluginLocator
from yapsy.PluginManager import PluginManager
pm = PluginManager(plugin_locator=DropPluginLocator())
# TODO: Create some dummy plugins to be found
pm.collectPlugins()
pm.getAllPlugins()
def main():
# Read configuration
config = SafeConfigParser(
defaults = {'port':'15915',
'plugins_directory':'./plugins',
'plugins_enabled':'',
})
config.read(['./conf/cm15d.conf',
'/etc/cm15d.conf',
'/etc/cm15d/cm15d.conf',
'/etc/cm15d/conf.d/local.conf',
])
# Activate enabled plugins
plugins = PluginManager()
plugins.setPluginPlaces(config.get('cm15d', 'plugins_directory').split(','))
plugins.collectPlugins()
plugins_enabled = config.get('cm15d', 'plugins_enabled').split(',')
for plugin in plugins.getAllPlugins():
if plugin.name in plugins_enabled:
plugins.activatePluginByName(plugin.name)
print("Plugin %s enabled" % plugin.name)
# Start server
port = int(config.get('cm15d', 'port'))
endpoint = TCP4ServerEndpoint(reactor, port)
endpoint.listen(CM15DaemonFactory(plugins))
print("Server listening on port %s" % port)
reactor.run()
def pane_data(self):
list = [] # Holds the filter tuples (name, activated=True)
# Get the default plugin directory, using XML
path = os.path.expanduser("~")
xml = xml_controller.Controller(path + "\.cxvrc.xml")
xml.load_file()
if os.path.exists(os.path.expanduser("~") + os.sep + "plugins"):
default_dir = os.path.expanduser("~") + os.sep + "plugins"
else:
default_dir = self.dicom_view.get_main_dir() + os.sep + "plugins"
if xml.get_plugin_directory() == "" or xml.get_plugin_directory() is None:
directory = [default_dir]
else:
directory = [default_dir, xml.get_plugin_directory()]
# Load the plugins from the default plugin directory.
manager = PluginManager()
manager.setPluginPlaces(directory)
manager.setPluginInfoExtension("plugin")
manager.collectPlugins()
# Append tuple with plugin name and enabled=True to the list
for plugin in manager.getAllPlugins():
list.append((plugin.name, True))
return list
def run_output_plugins(**kwargs):
logger = logging.getLogger(__name__)
logger.info("Begin run_output_plugins")
simplePluginManager = PluginManager()
logging.getLogger('yapsy').setLevel(logging.DEBUG)
simplePluginManager.setCategoriesFilter({
"OutputResults": output_plugin
})
# Tell it the default place(s) where to find plugins
if logger:
logger.debug("Plugin directories: %s" % (kwargs['output_plugin_directories']))
simplePluginManager.setPluginPlaces(kwargs['output_plugin_directories'])
simplePluginManager.collectPlugins()
plugin_cnt = 0
plugin_start_time = time.time()
for plugin in simplePluginManager.getAllPlugins():
if logger:
logger.info("Starting plugin: %s" % (plugin.name))
if plugin.plugin_object.initialize_plugin(details=plugin.details):
plugin.plugin_object.emit(prediction_date=kwargs['prediction_date'].astimezone(timezone("US/Eastern")).strftime("%Y-%m-%d %H:%M:%S"),
execution_date=kwargs['prediction_run_date'].strftime("%Y-%m-%d %H:%M:%S"),
ensemble_tests=kwargs['site_model_ensemble'])
plugin_cnt += 1
else:
logger.error("Failed to initialize plugin: %s" % (plugin.name))
logger.debug("%d output plugins run in %f seconds" % (plugin_cnt, time.time() - plugin_start_time))
logger.info("Finished run_output_plugins")
def test_setPluginLocator_with_plugin_info_class(self): class SpecificLocator(IPluginLocator): def getPluginInfoClass(self): return self.picls def setPluginInfoClass(self,picls): self.picls = picls class SpecificPluginInfo(PluginInfo): pass pm = PluginManager() pm.setPluginLocator(SpecificLocator(),picls=SpecificPluginInfo) self.assertEqual(SpecificPluginInfo,pm.getPluginInfoClass())
def load_directory(plugin_folder):
"""Loads and imports a directory of plugins.
:param plugin_folder: The folder to look for plugin files.
:type plugin_folder: String
:rtype: Dictionary
:returns: A dictionary of imported plugins.
"""
manager = PluginManager()
manager.setPluginPlaces([plugin_folder])
manager.collectPlugins()
return manager
def plugins_init(self, is_reloading = False):
if is_reloading:
logging.info("Deactivating All Plugins")
for pluginInfo in self.pm.getAllPlugins():
self.pm.deactivatePluginByName(pluginInfo.name)
self.pm = PluginManager(
categories_filter = {
"BaseActions" : BaseActionPlugin,
"TextActions" : TextTriggerPlugin,
},
directories_list=["plugins"],)
self.pm.collectPlugins()
for pluginInfo in self.pm.getAllPlugins():
self.pm.activatePluginByName(pluginInfo.name)
logging.info("Plugin {0} activated".format(pluginInfo.name))
# TODO: create a list of localized ("_()") plugin triggers
# Create a dictionary of the names of all the plugins
self.action_plugins = {}
# List of the regexes and plugins
self.text_trigger_plugins = []
# TODO: specify categories of plugins with each trigger http://yapsy.sourceforge.net/PluginManager.html
for pluginInfo in self.pm.getPluginsOfCategory("BaseActions"):
self.action_plugins[pluginInfo.name] = pluginInfo.plugin_object
logging.debug("Action plugins: {0}".format(self.action_plugins))
for pluginInfo in self.pm.getPluginsOfCategory("TextActions"):
self.text_trigger_plugins.append((pluginInfo.plugin_object.trigger, pluginInfo.plugin_object))
logging.debug("Regex plugins: {0}".format(self.text_trigger_plugins))
def __init__(self):
self.plugin_manager = PluginManager(plugin_locator=PluginFileLocator(analyzers=[PluginFileAnalyzerMathingRegex("regex_matcher",('^.*%s%s$' % (PLUGIN_SUFFIX,PLUGIN_EXT)))]))
self.plugin_manager.setPluginPlaces([PLUGIN_PATH])
self.dummy_plugin = None
self.plugin_map = {}
self.init_plugins()
def __init__(self, content_models, host='localhost', port=61613, user='', passcode='', fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' % {'host': host, 'port': port})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" % {'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode)
self.client = FedoraClient(self.fc)
# Create plugin manager
self.manager = PluginManager(categories_filter = {"FedoraMicroService": FedoraMicroService})
self.manager.setPluginPlaces(["plugins"])
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info("Loading plugin: %(name)s for content model %(cmodel)s." % {'name': plugin.plugin_object.name, 'cmodel': plugin.plugin_object.content_model})
plugin.plugin_object.config = config
if plugin.plugin_object.content_model in self.contentModels:
self.contentModels[plugin.plugin_object.content_model].append(plugin.plugin_object)
else:
self.contentModels[plugin.plugin_object.content_model] = [plugin.plugin_object]
def __init__(self, config=None, extra_plugin_paths=[]):
'Initialize a new PluginSystem. Needs no arguments.'
# Build the manager
self._mngr = PluginManager(plugin_info_ext='plugin')
# Tell it the default place(s) where to find plugins
plugin_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'plugins')
self._mngr.setPluginPlaces([plugin_path] + extra_plugin_paths)
# find the categories specified in moosecat/plugins/__init__.py
self._categories = _get_interfaces_from_module()
LOGGER.debug('Discovered following plugins categories: ' + ', '.join(self.list_categories()))
# tell yapsy about our extra categories
self._mngr.setCategoriesFilter(self._categories)
try:
self._mngr.collectPlugins()
except SystemError as e:
LOGGER.exception('Some plugin could not be loaded.')
# Get a list of plugin names to load
load_these = [pluginInfo.name for pluginInfo in self._mngr.getAllPlugins()]
if config is not None:
config.add_defaults({'plugins_to_load': load_these})
load_these = config.get('plugins_to_load')
# Actually load them
for name in load_these:
self._mngr.activatePluginByName(name)
def __init__(self):
botdb.connect()
#botdb.drop_tables([User, Conversation, Message], safe=True)
botdb.create_tables([User, Conversation, Message], safe=True)
# Init plugins
print ("Looking for plugins...")
self._plugins = PluginManager(categories_filter={
"Command" : ICommandPlugin,
"Filter" : IFilterPlugin
})
self._plugins.setPluginPlaces(['bot/commands/', 'bot/filters/'])
self._plugins.collectPlugins()
# Load slash commands
self.__commands = {}
print ("╠ Commands")
for plugin in self._plugins.getPluginsOfCategory("Command"):
print ("║ * Found «%s»" % plugin.name)
for cmdname in plugin.plugin_object.aliases:
self.__commands[cmdname] = plugin.plugin_object.execute
# print ("using slash command '%s'" % cmdname)
# Load filters
print ("╚ Filters")
self.filters = self._plugins.getPluginsOfCategory("Filter")
for plugin in self.filters:
print (" * Found «%s»" % plugin.name)
print ("Done.\n")
def __init__(self):
self.template_system = self
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(utils.__file__), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(utils.__file__), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
def main():
"""Main program execution"""
if not path.isdir(inputdir):
print("Input directory doesn't exist")
exit(1)
if not path.isdir(outputdir):
print("Output directory doesn't exist")
exit(1)
inputdata = open(path.join(inputdir, "theme"), 'r')
inputdata = inputdata.read()
# Instantiate and fill the pack metadata
inputpack = EmotePack()
inputpack.path = inputdir
inputpack.output = outputdir
try:
inputpack.name = re.findall(r"Name=(.*)", inputdata)[-1]
except IndexError:
print("Couldn't find a name, skipping")
pass
try:
inputpack.desc = re.findall(r"Description=(.*)", inputdata)[-1]
except IndexError:
print("Couldn't find a description, skipping")
pass
try:
inputpack.author = re.findall(r"Author=(.*)", inputdata)[-1]
except IndexError:
print("Couldn't find an author, skipping")
pass
if opts.name:
inputpack.filename = opts.name
else:
inputpack.filename = inputpack.name
# Reopen the pack in line mode and skip the header lines
inputfile = open(path.join(inputdir, "theme"), 'r')
inputfile = inputfile.readlines()[6:]
# Fill the container with Emotes
for line in inputfile:
if verbose:
print(line)
if line.startswith('!'):
line = line.replace('!', '') # Strip off the leading !
line = line.strip().split() # Split into tokens
thisemote = Emote()
# Identify the file and its type, as required for some formats.
thisemote.filename = line[0]
thisemote.filetype = mimetypes.guess_type(line[0])[0]
thisemote.shortcuts = line[1:]
# Dimensions are required for phpBB
try:
im = Image.open(path.join(inputdir, thisemote.filename))
thisemote.width, thisemote.height = im.size
except IOError:
# If the file doesn't exist, lets not package it, either
continue
inputpack.emotelist.append(thisemote)
pm = PluginManager(
directories_list=["templates",
path.join(path.dirname(path.abspath(__file__)),
"templates")
],
plugin_info_ext="plug"
)
pm.collectPlugins()
for backend in pm.getAllPlugins():
backend.plugin_object.build(inputpack)
class Nikola(object):
"""Class that handles site generation.
Takes a site config as argument on creation.
"""
EXTRA_PLUGINS = [
'planetoid',
'ipynb',
'local_search',
'render_mustache',
]
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.posts_per_category = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
self._template_system = None
self._THEMES = None
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'BODY_END': "",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENT_SYSTEM': 'disqus',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'COMPILERS': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile', ),
"txt2tags": ('.t2t', ),
"bbcode": ('.bb', ),
"wiki": ('.wiki', ),
"ipynb": ('.ipynb', ),
"html": ('.html', '.htm')
},
'CONTENT_FOOTER': '',
'COPY_SOURCES': True,
'CREATE_MONTHLY_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'COMMENT_SYSTEM_ID': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FEED_LENGTH': 10,
'FILE_METADATA_REGEXP': None,
'ADDITIONAL_METADATA': {},
'FILES_FOLDERS': {
'files': ''
},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GZIP_FILES': False,
'GZIP_EXTENSIONS':
('.txt', '.htm', '.html', '.css', '.js', '.json'),
'HIDE_SOURCELINK': False,
'HIDE_UNTRANSLATED_POSTS': False,
'HYPHENATE': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEX_PATH': '',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'NAVIGATION_LINKS': None,
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'POSTS': (("posts/*.txt", "posts", "post.tmpl"), ),
'PAGES': (("stories/*.txt", "stories", "story.tmpl"), ),
'PRETTY_URLS': False,
'FUTURE_IS_NOW': False,
'READ_MORE_LINK':
'<p class="more"><a href="{link}">{read_more}…</a></p>',
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'SOCIAL_BUTTONS_CODE': SOCIAL_BUTTONS_CODE,
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'bootstrap',
'THEME_REVEAL_CONFIG_SUBTHEME': 'sky',
'THEME_REVEAL_CONFIG_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': None,
'DEPLOY_DRAFTS': True,
'DEPLOY_FUTURE': False,
'SCHEDULE_ALL': False,
'SCHEDULE_RULE': '',
'SCHEDULE_FORCE_TODAY': False
}
self.config.update(config)
# Make sure we have pyphen installed if we are using it
if self.config.get('HYPHENATE') and pyphen is None:
print('WARNING: To use the hyphenation, you have to install '
'the "pyphen" package.')
print('WARNING: Setting HYPHENATE to False.')
self.config['HYPHENATE'] = False
# Deprecating post_compilers
# TODO: remove on v7
if 'post_compilers' in config:
print(
"WARNING: The post_compilers option is deprecated, use COMPILERS instead."
)
if 'COMPILERS' in config:
print(
"WARNING: COMPILERS conflicts with post_compilers, ignoring post_compilers."
)
else:
self.config['COMPILERS'] = config['post_compilers']
# Deprecating post_pages
# TODO: remove on v7
if 'post_pages' in config:
print(
"WARNING: The post_pages option is deprecated, use POSTS and PAGES instead."
)
if 'POSTS' in config or 'PAGES' in config:
print(
"WARNING: POSTS and PAGES conflict with post_pages, ignoring post_pages."
)
else:
self.config['POSTS'] = [
item[:3] for item in config['post_pages'] if item[-1]
]
self.config['PAGES'] = [
item[:3] for item in config['post_pages'] if not item[-1]
]
# FIXME: Internally, we still use post_pages because it's a pain to change it
self.config['post_pages'] = []
for i1, i2, i3 in self.config['POSTS']:
self.config['post_pages'].append([i1, i2, i3, True])
for i1, i2, i3 in self.config['PAGES']:
self.config['post_pages'].append([i1, i2, i3, False])
# Deprecating DISQUS_FORUM
# TODO: remove on v7
if 'DISQUS_FORUM' in config:
print(
"WARNING: The DISQUS_FORUM option is deprecated, use COMMENT_SYSTEM_ID instead."
)
if 'COMMENT_SYSTEM_ID' in config:
print(
"WARNING: DISQUS_FORUM conflicts with COMMENT_SYSTEM_ID, ignoring DISQUS_FORUM."
)
else:
self.config['COMMENT_SYSTEM_ID'] = config['DISQUS_FORUM']
# Deprecating the ANALYTICS option
# TODO: remove on v7
if 'ANALYTICS' in config:
print(
"WARNING: The ANALYTICS option is deprecated, use BODY_END instead."
)
if 'BODY_END' in config:
print(
"WARNING: ANALYTICS conflicts with BODY_END, ignoring ANALYTICS."
)
else:
self.config['BODY_END'] = config['ANALYTICS']
# Deprecating the SIDEBAR_LINKS option
# TODO: remove on v7
if 'SIDEBAR_LINKS' in config:
print(
"WARNING: The SIDEBAR_LINKS option is deprecated, use NAVIGATION_LINKS instead."
)
if 'NAVIGATION_LINKS' in config:
print(
"WARNING: The SIDEBAR_LINKS conflicts with NAVIGATION_LINKS, ignoring SIDEBAR_LINKS."
)
else:
self.config['NAVIGATION_LINKS'] = config['SIDEBAR_LINKS']
# Compatibility alias
self.config['SIDEBAR_LINKS'] = self.config['NAVIGATION_LINKS']
if self.config['NAVIGATION_LINKS'] in (None, {}):
self.config['NAVIGATION_LINKS'] = {self.config['DEFAULT_LANG']: ()}
# Deprecating the ADD_THIS_BUTTONS option
# TODO: remove on v7
if 'ADD_THIS_BUTTONS' in config:
print(
"WARNING: The ADD_THIS_BUTTONS option is deprecated, use SOCIAL_BUTTONS_CODE instead."
)
if not config['ADD_THIS_BUTTONS']:
print(
"WARNING: Setting SOCIAL_BUTTONS_CODE to empty because ADD_THIS_BUTTONS is False."
)
self.config['SOCIAL_BUTTONS_CODE'] = ''
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
# TODO: remove on v7
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
print(
"WARNING: You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML"
)
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
if config.get('COPY_SOURCES') and not self.config['HIDE_SOURCELINK']:
self.config['HIDE_SOURCELINK'] = True
self.config['TRANSLATIONS'] = self.config.get(
'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''})
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
# TODO: remove on v7
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
print(
"WARNING: You should configure SITE_URL instead of BLOG_URL"
)
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
# BASE_URL should *always* end in /
if self.config['BASE_URL'] and self.config['BASE_URL'][-1] != '/':
print("WARNING: Your BASE_URL doesn't end in / -- adding it.")
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(__file__),
utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS
and plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(
task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS
and plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
if plugin_info.name in self.config["COMPILERS"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self._GLOBAL_CONTEXT = {}
self._GLOBAL_CONTEXT['_link'] = self.link
self._GLOBAL_CONTEXT['set_locale'] = s_l
self._GLOBAL_CONTEXT['rel_link'] = self.rel_link
self._GLOBAL_CONTEXT['abs_link'] = self.abs_link
self._GLOBAL_CONTEXT['exists'] = self.file_exists
self._GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config['SLUG_TAG_PATH']
self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self._GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self._GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
# TODO: remove fallback in v7
self._GLOBAL_CONTEXT['blog_url'] = self.config.get(
'SITE_URL', self.config.get('BLOG_URL'))
self._GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self._GLOBAL_CONTEXT['body_end'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['analytics'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['add_this_buttons'] = self.config.get(
'SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['social_buttons_code'] = self.config.get(
'SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self._GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self._GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self._GLOBAL_CONTEXT['comment_system'] = self.config.get(
'COMMENT_SYSTEM')
self._GLOBAL_CONTEXT['comment_system_id'] = self.config.get(
'COMMENT_SYSTEM_ID')
# TODO: remove in v7
self._GLOBAL_CONTEXT['disqus_forum'] = self.config.get(
'COMMENT_SYSTEM_ID')
self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self._GLOBAL_CONTEXT['subtheme'] = self.config.get(
'THEME_REVEAL_CONFIG_SUBTHEME')
self._GLOBAL_CONTEXT['transition'] = self.config.get(
'THEME_REVEAL_CONFIG_TRANSITION')
self._GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self._GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self._GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self._GLOBAL_CONTEXT['navigation_links'] = utils.Functionary(
list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('NAVIGATION_LINKS', {}).items():
self._GLOBAL_CONTEXT['navigation_links'][k] = v
# TODO: remove on v7
# Compatibility alias
self._GLOBAL_CONTEXT['sidebar_links'] = self._GLOBAL_CONTEXT[
'navigation_links']
self._GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self._GLOBAL_CONTEXT['hide_sourcelink'] = self.config.get(
'HIDE_SOURCELINK')
self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get(
'EXTRA_HEAD_DATA')
self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
def _get_themes(self):
if self._THEMES is None:
# Check for old theme names (Issue #650) TODO: remove in v7
theme_replacements = {
'site': 'bootstrap',
'orphan': 'base',
'default': 'oldfashioned',
}
if self.config['THEME'] in theme_replacements:
warnings.warn(
'You are using the old theme "{0}", using "{1}" instead.'.
format(self.config['THEME'],
theme_replacements[self.config['THEME']]))
self.config['THEME'] = theme_replacements[self.config['THEME']]
if self.config['THEME'] == 'oldfashioned':
warnings.warn(
'''You may need to install the "oldfashioned" theme '''
'''from themes.nikola.ralsina.com.ar because it's not '''
'''shipped by default anymore.''')
warnings.warn('Please change your THEME setting.')
try:
self._THEMES = utils.get_theme_chain(self.config['THEME'])
except Exception:
warnings.warn(
'''Can't load theme "{0}", using 'bootstrap' instead.'''.
format(self.config['THEME']))
self.config['THEME'] = 'bootstrap'
return self._get_themes()
# Check consistency of USE_CDN and the current THEME (Issue #386)
if self.config['USE_CDN']:
bootstrap_path = utils.get_asset_path(
os.path.join('assets', 'css', 'bootstrap.min.css'),
self._THEMES)
if bootstrap_path and bootstrap_path.split(
os.sep)[-4] not in ['bootstrap', 'bootstrap3']:
warnings.warn(
'The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.'
)
return self._THEMES
THEMES = property(_get_themes)
def _get_messages(self):
return utils.load_messages(self.THEMES, self.translations,
self.default_lang)
MESSAGES = property(_get_messages)
def _get_global_context(self):
"""Initialize some parts of GLOBAL_CONTEXT only when it's queried."""
if 'messages' not in self._GLOBAL_CONTEXT:
self._GLOBAL_CONTEXT['messages'] = self.MESSAGES
if 'has_custom_css' not in self._GLOBAL_CONTEXT:
# check if custom css exist and is not empty
custom_css_path = utils.get_asset_path(
'assets/css/custom.css', self.THEMES,
self.config['FILES_FOLDERS'])
if custom_css_path and self.file_exists(custom_css_path,
not_empty=True):
self._GLOBAL_CONTEXT['has_custom_css'] = True
else:
self._GLOBAL_CONTEXT['has_custom_css'] = False
return self._GLOBAL_CONTEXT
GLOBAL_CONTEXT = property(_get_global_context)
def _get_template_system(self):
if self._template_system is None:
# Load template plugin
template_sys_name = utils.get_template_engine(self.THEMES)
pi = self.plugin_manager.getPluginByName(template_sys_name,
"TemplateSystem")
if pi is None:
sys.stderr.write("Error loading {0} template system "
"plugin\n".format(template_sys_name))
sys.exit(1)
self._template_system = pi.plugin_object
lookup_dirs = ['templates'] + [
os.path.join(utils.get_theme_path(name), "templates")
for name in self.THEMES
]
self._template_system.set_directories(lookup_dirs,
self.config['CACHE_FOLDER'])
return self._template_system
template_system = property(_get_template_system)
def get_compiler(self, source_name):
"""Get the correct compiler for a post from `conf.COMPILERS`
To make things easier for users, the mapping in conf.py is
compiler->[extensions], although this is less convenient for us. The
majority of this function is reversing that dictionary and error
checking.
"""
ext = os.path.splitext(source_name)[1]
try:
compile_html = self.inverse_compilers[ext]
except KeyError:
# Find the correct compiler for this files extension
langs = [
lang for lang, exts in list(self.config['COMPILERS'].items())
if ext in exts
]
if len(langs) != 1:
if len(set(langs)) > 1:
exit("Your file extension->compiler definition is"
"ambiguous.\nPlease remove one of the file extensions"
"from 'COMPILERS' in conf.py\n(The error is in"
"one of {0})".format(', '.join(langs)))
elif len(langs) > 1:
langs = langs[:1]
else:
exit("COMPILERS in conf.py does not tell me how to "
"handle '{0}' extensions.".format(ext))
lang = langs[0]
compile_html = self.compilers[lang]
self.inverse_compilers[ext] = compile_html
return compile_html
def render_template(self, template_name, output_name, context):
local_context = {}
local_context["template_name"] = template_name
local_context.update(self.GLOBAL_CONTEXT)
local_context.update(context)
data = self.template_system.render_template(template_name, None,
local_context)
assert output_name.startswith(self.config["OUTPUT_FOLDER"])
url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
# Treat our site as if output/ is "/" and then make all URLs relative,
# making the site "relocatable"
src = os.sep + url_part
src = os.path.normpath(src)
# The os.sep is because normpath will change "/" to "\" on windows
src = "/".join(src.split(os.sep))
parsed_src = urlsplit(src)
src_elems = parsed_src.path.split('/')[1:]
def replacer(dst):
# Refuse to replace links that are full URLs.
dst_url = urlparse(dst)
if dst_url.netloc:
if dst_url.scheme == 'link': # Magic link
dst = self.link(dst_url.netloc, dst_url.path.lstrip('/'),
context['lang'])
else:
return dst
# Normalize
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
# Now i is the longest common prefix
result = '/'.join(['..'] * (len(src_elems) - i - 1) +
dst_elems[i:])
if not result:
result = "."
# Don't forget the fragment (anchor) part of the link
if parsed_dst.fragment:
result += "#" + parsed_dst.fragment
assert result, (src, dst, i, src_elems, dst_elems)
return result
try:
os.makedirs(os.path.dirname(output_name))
except:
pass
doc = lxml.html.document_fromstring(data)
doc.rewrite_links(replacer)
data = b'<!DOCTYPE html>' + lxml.html.tostring(doc, encoding='utf8')
with open(output_name, "wb+") as post_file:
post_file.write(data)
def current_lang(self): # FIXME: this is duplicated, turn into a mixin
"""Return the currently set locale, if it's one of the
available translations, or default_lang."""
lang = utils.LocaleBorg().current_lang
if lang:
if lang in self.translations:
return lang
lang = lang.split('_')[0]
if lang in self.translations:
return lang
# whatever
return self.default_lang
def path(self, kind, name, lang=None, is_link=False):
"""Build the path to a certain kind of page.
kind is one of:
* tag_index (name is ignored)
* tag (and name is the tag name)
* tag_rss (name is the tag name)
* category (and name is the category name)
* category_rss (and name is the category name)
* archive (and name is the year, or None for the main archive index)
* index (name is the number in index-number)
* rss (name is ignored)
* gallery (name is the gallery name)
* listing (name is the source code file name)
* post_path (name is 1st element in a POSTS/PAGES tuple)
The returned value is always a path relative to output, like
"categories/whatever.html"
If is_link is True, the path is absolute and uses "/" as separator
(ex: "/archive/index.html").
If is_link is False, the path is relative to output and uses the
platform's separator.
(ex: "archive\\index.html")
"""
if lang is None:
lang = self.current_lang()
path = []
if kind == "tag_index":
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
self.config['INDEX_FILE']
] if _f
]
elif kind == "tag":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".html"
] if _f
]
elif kind == "category":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
"cat_" + name + ".html"
] if _f
]
elif kind == "tag_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
name + ".xml"
] if _f
]
elif kind == "category_rss":
if self.config['SLUG_TAG_PATH']:
name = utils.slugify(name)
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['TAG_PATH'],
"cat_" + name + ".xml"
] if _f
]
elif kind == "index":
if name not in [None, 0]:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.
config['INDEX_PATH'], 'index-{0}.html'.format(name)
] if _f
]
else:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang],
self.config['INDEX_PATH'], self.config['INDEX_FILE']
] if _f
]
elif kind == "post_path":
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang],
os.path.dirname(name), self.config['INDEX_FILE']
] if _f
]
elif kind == "rss":
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.config['RSS_PATH'],
'rss.xml'
] if _f
]
elif kind == "archive":
if name:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.
config['ARCHIVE_PATH'], name, self.config['INDEX_FILE']
] if _f
]
else:
path = [
_f for _f in [
self.config['TRANSLATIONS'][lang], self.
config['ARCHIVE_PATH'], self.config['ARCHIVE_FILENAME']
] if _f
]
elif kind == "gallery":
path = [
_f for _f in
[self.config['GALLERY_PATH'], name, self.config['INDEX_FILE']]
if _f
]
elif kind == "listing":
path = [
_f for _f in [self.config['LISTINGS_FOLDER'], name + '.html']
if _f
]
if is_link:
link = '/' + ('/'.join(path))
index_len = len(self.config['INDEX_FILE'])
if self.config['STRIP_INDEXES'] and \
link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
return link[:-index_len]
else:
return link
else:
return os.path.join(*path)
def link(self, *args):
return self.path(*args, is_link=True)
def abs_link(self, dst):
# Normalize
dst = urljoin(self.config['BASE_URL'], dst)
return urlparse(dst).path
def rel_link(self, src, dst):
# Normalize
src = urljoin(self.config['BASE_URL'], src)
dst = urljoin(src, dst)
# Avoid empty links.
if src == dst:
return "#"
# Check that link can be made relative, otherwise return dest
parsed_src = urlsplit(src)
parsed_dst = urlsplit(dst)
if parsed_src[:2] != parsed_dst[:2]:
return dst
# Now both paths are on the same site and absolute
src_elems = parsed_src.path.split('/')[1:]
dst_elems = parsed_dst.path.split('/')[1:]
i = 0
for (i, s), d in zip(enumerate(src_elems), dst_elems):
if s != d:
break
else:
i += 1
# Now i is the longest common prefix
return '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
def file_exists(self, path, not_empty=False):
"""Returns True if the file exists. If not_empty is True,
it also has to be not empty."""
exists = os.path.exists(path)
if exists and not_empty:
exists = os.stat(path).st_size > 0
return exists
def gen_tasks(self, name, plugin_category):
def flatten(task):
if isinstance(task, dict):
yield task
else:
for t in task:
for ft in flatten(t):
yield ft
task_dep = []
for pluginInfo in self.plugin_manager.getPluginsOfCategory(
plugin_category):
for task in flatten(pluginInfo.plugin_object.gen_tasks()):
yield task
for multi in self.plugin_manager.getPluginsOfCategory(
"TaskMultiplier"):
flag = False
for task in multi.plugin_object.process(task, name):
flag = True
yield task
if flag:
task_dep.append('{0}_{1}'.format(
name, multi.plugin_object.name))
if pluginInfo.plugin_object.is_default:
task_dep.append(pluginInfo.plugin_object.name)
yield {
'name': name,
'actions': None,
'clean': True,
'task_dep': task_dep
}
def scan_posts(self):
"""Scan all the posts."""
if self._scanned:
return
seen = set([])
print("Scanning posts", end='')
tzinfo = None
if self.config['TIMEZONE'] is not None:
tzinfo = pytz.timezone(self.config['TIMEZONE'])
if self.config['FUTURE_IS_NOW']:
current_time = None
else:
current_time = utils.current_time(tzinfo)
targets = set([])
for wildcard, destination, template_name, use_in_feeds in \
self.config['post_pages']:
print(".", end='')
dirname = os.path.dirname(wildcard)
for dirpath, _, _ in os.walk(dirname):
dir_glob = os.path.join(dirpath, os.path.basename(wildcard))
dest_dir = os.path.normpath(
os.path.join(destination,
os.path.relpath(dirpath, dirname)))
full_list = glob.glob(dir_glob)
# Now let's look for things that are not in default_lang
for lang in self.config['TRANSLATIONS'].keys():
lang_glob = dir_glob + "." + lang
translated_list = glob.glob(lang_glob)
for fname in translated_list:
orig_name = os.path.splitext(fname)[0]
if orig_name in full_list:
continue
full_list.append(orig_name)
# We eliminate from the list the files inside any .ipynb folder
full_list = [
p for p in full_list
if not any([x.startswith('.') for x in p.split(os.sep)])
]
for base_path in full_list:
if base_path in seen:
continue
else:
seen.add(base_path)
post = Post(
base_path,
self.config['CACHE_FOLDER'],
dest_dir,
use_in_feeds,
self.config['TRANSLATIONS'],
self.config['DEFAULT_LANG'],
self.config['BASE_URL'],
self.MESSAGES,
template_name,
self.config['FILE_METADATA_REGEXP'],
self.config['STRIP_INDEXES'],
self.config['INDEX_FILE'],
tzinfo,
current_time,
self.config['HIDE_UNTRANSLATED_POSTS'],
self.config['PRETTY_URLS'],
self.config['HYPHENATE'],
)
for lang, langpath in list(
self.config['TRANSLATIONS'].items()):
dest = (destination, langpath, dir_glob,
post.meta[lang]['slug'])
if dest in targets:
raise Exception('Duplicated output path {0!r} '
'in post {1!r}'.format(
post.meta[lang]['slug'],
base_path))
targets.add(dest)
self.global_data[post.post_name] = post
if post.use_in_feeds:
self.posts_per_year[str(post.date.year)].append(
post.post_name)
self.posts_per_month['{0}/{1:02d}'.format(
post.date.year,
post.date.month)].append(post.post_name)
for tag in post.alltags:
self.posts_per_tag[tag].append(post.post_name)
self.posts_per_category[post.meta('category')].append(
post.post_name)
else:
self.pages.append(post)
if self.config['OLD_THEME_SUPPORT']:
post._add_old_metadata()
self.post_per_file[post.destination_path(lang=lang)] = post
self.post_per_file[post.destination_path(
lang=lang, extension=post.source_ext())] = post
for name, post in list(self.global_data.items()):
self.timeline.append(post)
self.timeline.sort(key=lambda p: p.date)
self.timeline.reverse()
post_timeline = [p for p in self.timeline if p.use_in_feeds]
for i, p in enumerate(post_timeline[1:]):
p.next_post = post_timeline[i]
for i, p in enumerate(post_timeline[:-1]):
p.prev_post = post_timeline[i + 1]
self._scanned = True
print("done!")
def generic_page_renderer(self, lang, post, filters):
"""Render post fragments to final HTML pages."""
context = {}
deps = post.deps(lang) + \
self.template_system.template_deps(post.template_name)
context['post'] = post
context['lang'] = lang
context['title'] = post.title(lang)
context['description'] = post.description(lang)
context['permalink'] = post.permalink(lang)
context['page_list'] = self.pages
if post.use_in_feeds:
context['enable_comments'] = True
else:
context['enable_comments'] = self.config['COMMENTS_IN_STORIES']
extension = self.get_compiler(post.source_path).extension()
output_name = os.path.join(self.config['OUTPUT_FOLDER'],
post.destination_path(lang, extension))
deps_dict = copy(context)
deps_dict.pop('post')
if post.prev_post:
deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
if post.next_post:
deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
deps_dict['global'] = self.GLOBAL_CONTEXT
deps_dict['comments'] = context['enable_comments']
if post:
deps_dict['post_translations'] = post.translated_to
task = {
'name':
os.path.normpath(output_name),
'file_dep':
deps,
'targets': [output_name],
'actions':
[(self.render_template, [post.template_name, output_name,
context])],
'clean':
True,
'uptodate': [config_changed(deps_dict)],
}
yield utils.apply_filters(task, filters)
def generic_post_list_renderer(self, lang, posts, output_name,
template_name, filters, extra_context):
"""Renders pages with lists of posts."""
deps = self.template_system.template_deps(template_name)
for post in posts:
deps += post.deps(lang)
context = {}
context["posts"] = posts
context["title"] = self.config['BLOG_TITLE']
context["description"] = self.config['BLOG_DESCRIPTION']
context["lang"] = lang
context["prevlink"] = None
context["nextlink"] = None
context.update(extra_context)
deps_context = copy(context)
deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang))
for p in posts]
deps_context["global"] = self.GLOBAL_CONTEXT
task = {
'name':
os.path.normpath(output_name),
'targets': [output_name],
'file_dep':
deps,
'actions':
[(self.render_template, [template_name, output_name, context])],
'clean':
True,
'uptodate': [config_changed(deps_context)]
}
return utils.apply_filters(task, filters)
def testMultipleCategoriesForASamePlugin(self):
"""
Test that associating a plugin to multiple categories works as expected.
"""
class AnotherPluginIfce(object):
def __init__(self):
pass
def activate(self):
pass
def deactivate(self):
pass
spm = PluginManager(categories_filter={
"Default": IPlugin,
"IP": IPlugin,
"Other": AnotherPluginIfce,
},
directories_list=[
os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"plugins")
])
# load the plugins that may be found
spm.collectPlugins()
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 3)
categories = spm.getCategories()
self.assertTrue("Default" in categories)
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory("Default")), 1)
plugin_info = spm.getPluginsOfCategory("Default")[0]
self.assertTrue("Default" in plugin_info.categories)
self.assertTrue("IP" in plugin_info.categories)
self.assertTrue("IP" in categories)
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1)
self.assertTrue("Other" in categories)
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory("Other")), 0)
# try to remove the plugin from one category and check the
# other category
spm.removePluginFromCategory(plugin_info, "Default")
self.assertEqual(len(spm.getPluginsOfCategory("Default")), 0)
self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1)
# now re-add this plugin the to same category
spm.appendPluginToCategory(plugin_info, "Default")
self.assertEqual(len(spm.getPluginsOfCategory("Default")), 1)
self.assertEqual(len(spm.getPluginsOfCategory("IP")), 1)
def testCandidatesManipulation(self):
"""
Test querying, removing and adding plugins from/to the lkist
of plugins to load.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# locate the plugins that should be loaded
spm.locatePlugins()
# check nb of candidatesx
self.assertEqual(len(spm.getPluginCandidates()), 1)
# get the description of the plugin candidate
candidate = spm.getPluginCandidates()[0]
self.assertTrue(isinstance(candidate, tuple))
# try removing the candidate
spm.removePluginCandidate(candidate)
self.assertEqual(len(spm.getPluginCandidates()), 0)
# try re-adding it
spm.appendPluginCandidate(candidate)
self.assertEqual(len(spm.getPluginCandidates()), 1)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Yapsy Framework to load a plugin
"""
import logging
import os
from yapsy.PluginManager import PluginManager
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('yapsy').setLevel(logging.DEBUG)
# Load the plugins from the plugin directory.
pluginManager = PluginManager()
# Location of consul plugin
RELATIVE_PATH_OF_PLUGIN = os.path.join('ConfigDB', 'api', 'storage_plugin')
pluginManager.setPluginPlaces([RELATIVE_PATH_OF_PLUGIN])
pluginManager.collectPlugins()
# FIXREQUIRED: No need of for loop here
# Loop around the plugins and return the first plugin object.
for storage_plugin in pluginManager.getAllPlugins():
plugin = storage_plugin.plugin_object
def index(request):
# Get the current user's Business Units
user = request.user
# Count the number of users. If there is only one, they need to be made a GA
if User.objects.count() == 1:
# The first user created by syncdb won't have a profile. If there isn't one, make sure they get one.
try:
profile = UserProfile.objects.get(user=user)
except UserProfile.DoesNotExist:
profile = UserProfile(user=user)
profile.level = 'GA'
profile.save()
else:
# Create profile for LDAP users if they need one.
try:
profile = UserProfile.objects.get(user=user)
if profile.level is None:
profile.level = 'SO'
profile.save()
except UserProfile.DoesNotExist:
profile = UserProfile(user=user)
profile.level = 'SO'
profile.save()
user_level = user.userprofile.level
now = datetime.now()
hour_ago = now - timedelta(hours=1)
today = date.today()
week_ago = today - timedelta(days=7)
month_ago = today - timedelta(days=30)
three_months_ago = today - timedelta(days=90)
if user_level != 'GA':
# user has many BU's display them all in a friendly manner
business_units = user.businessunit_set.all()
if user.businessunit_set.count() == 1:
# user only has one BU, redirect to it
for bu in user.businessunit_set.all():
return redirect('server.views.bu_dashboard', bu_id=bu.id)
break
else:
machines = Machine.objects.all()
# Build the manager
manager = PluginManager()
# Tell it the default place(s) where to find plugins
manager.setPluginPlaces([settings.PLUGIN_DIR, os.path.join(settings.PROJECT_DIR, 'server/plugins')])
# Load all plugins
manager.collectPlugins()
output = []
# Loop round the plugins and print their names.
for plugin in manager.getAllPlugins():
data = {}
data['name'] = plugin.name
(data['html'], data['width']) = plugin.plugin_object.show_widget('front', machines)
#output.append(plugin.plugin_object.show_widget('front'))
output.append(data)
output = utils.orderPluginOutput(output)
# get the user level - if they're a global admin, show all of the machines. If not, show only the machines they have access to
business_units = BusinessUnit.objects.all()
c = {'user': request.user, 'business_units': business_units, 'output': output}
return render_to_response('server/index.html', c, context_instance=RequestContext(request))
class Parser(object):
def __init__(
self,
arquivo,
provider_conf = {},
xsd_path=None,
plugins_path=None,
providers_path=None,
skeep=False
):
self.provider_conf = provider_conf
self.arquivo = arquivo
self.version = None
self.skeep_validation = skeep
self.plugins_path = plugins_path
self.providers_path = providers_path
self.xsd_path = xsd_path
self.xsd_valido = False
self.xsd_erros = []
self.valido = False
self.erros = {
'lote': {},
'guias': {},
}
self.providers = {}
self.arquivo_xsd = None
# analisa o arquivo
if not self.skeep_validation:
self.parse()
if self.arquivo_xsd:
# # descobre versao
self.get_version()
if self.version:
# valida o xsd
self.xsd_validate()
if self.xsd_valido:
# validação estrutural
self.valida_estrutura()
if self.valido:
# executa plugins de providers
self.executa_providers()
# validação do negócios em modelo de plugins
self.executa_plugins()
# se existir erros de guia, marca lote invalido
guias_invalidas = len(self.erros['guias'].items())
inconsistencias = 0
if guias_invalidas:
for guia in self.erros['guias'].items():
inconsistencias += len(guia[1])
self.valido = False
self.erros['lote']['_guias'] = u"%s Guias apresentaram %s inconsistências" % (
guias_invalidas,
inconsistencias
)
else:
self.erros['lote']['_estrutura'] = "Estrutura não é válida"
xml_errors = None
def parse(self):
#
# tenta identar o XML para apontar erro em linha
try:
conteudo = open(self.arquivo, encoding='iso-8859-1').read()
except TypeError:
# hack para python 2.7
conteudo = io.open(self.arquivo, encoding='iso-8859-1').read()
# remove o encoding do xml para deixar o lxml trabalhar
conteudo_sem = conteudo.replace('encoding="iso-8859-1"', '')
conteudo_sem = conteudo.replace('encoding="ISO-8859-1"', '')
root = etree.fromstring(conteudo_sem)
# conteudo bonito, identado
cb = etree.tostring(root, pretty_print=True).decode()
self.root = etree.fromstring(cb)
# checa se namespace valido.
# namespace errado da erro de interpretação.
try:
# se nao possuir o esquema da ans no namespace
# forçar definir hardcoded.
self.nsmap = self.root.nsmap
self.schema_ans_url = self.nsmap['ans']
except KeyError:
self.nsmap = {
'ans': 'http://www.ans.gov.br/padroes/tiss/schemas',
'ds': 'http://www.w3.org/2000/09/xmldsig#',
'xsi': 'http://www.w3.org/2001/XMLSchema-instance'
}
# devoler pro root o nsmap forçado
#self.root.nsmap = self.nsmap
# define o tipo de trasacao
self.tipo_transacao = self.get_xpath(
'//ans:tipoTransacao'
)[0].text.replace("\n", '').replace("\t", '')
# no momento so suporta envio de lote
if self.tipo_transacao == 'ENVIO_LOTE_GUIAS':
# neste caso, o arquivo xsd é o tiss, dentro da pasta xsd
self.arquivo_xsd = 'tiss'
# tambem existe um numero do lote
self.numero_lote = self.get_xpath("//ans:numeroLote")[0].text
# numero sequencial
self.numero_sequencial = self.get_xpath("//ans:sequencialTransacao")[0].text
#
# aqui precisa fazer testes em outros tipos de transação
# deve ser possivel/necessario filtrar isso nos plugins e providers
#
# arquivo xsd não encontrado.
if not self.arquivo_xsd:
self.valido = False
self.erros['lote']['_transacao'] = 'Tipo de Transação não identificado ou não suportado: %s' % self.tipo_transacao
# extrai as guias para utilizacao nos plugins
self.guias = self.root.xpath('//ans:guiasTISS', namespaces=self.nsmap)[0].getchildren()
# extrai o prestador do lote
#self.codigo_prestador = self.get_xpath(
# "//ans:cabecalho//ans:codigoPrestadorNaOperadora"
#)[0].text
def get_version(self):
'''retorna versao no formato N.NN.NN, conforme xml
'''
try:
# versao 3.0.1
versao301 = self.get_xpath(
'//ans:Padrao'
)
if versao301:
self.version = versao301[0].text
else:
self.version = self.get_xpath(
'//ans:versaoPadrao'
)[0].text.replace("\n", '').replace("\t", '')
except:
self.valid = False
self.erros['lote']['_versao'] = u"Erro ao detectar a versão do padrão TISS"
def xsd_validate(self):
if not self.xsd_path:
xsd_file = 'xsd/%sV%s.xsd' % (self.arquivo_xsd, self.version.replace('.', '_'))
self.xsd_path = os.path.join(os.path.dirname(__file__), xsd_file)
#f = open(self.xsd_path)
self.root_xsd = etree.parse(self.xsd_path)
try:
self.xsd_schema = etree.XMLSchema(self.root_xsd)
self.xsd_valido = True
except etree.XMLSchemaParseError as xsd_erros:
self.xsd_valido = False
self.xsd_erros = xsd_erros
i = 1
for erro in self.xsd_erros.error_log:
self.erros['lote']['_xsd_invalido%i' % i] = "Arquivo: %s, Linha %s, Coluna: %s: %s" % (
erro.filename, erro.line, erro.column, erro.message
)
i += 1
def valida_estrutura(self):
try:
self.xsd_schema.assertValid(self.root)
self.valido = True
self.schema_valido = True
self.calcula_hash()
self.valida_hash()
except etree.DocumentInvalid as xml_errors:
self.valid = False
self.schema_valido = False
self.schema_erros = xml_errors
i = 1
for erro in self.schema_erros.error_log:
self.erros['lote']['_schema_invalido%i' % i] = "Linha %s, Coluna: %s: %s" % (erro.line, erro.column, erro.message)
i += 1
def calcula_hash(self):
# remove epilogo do calculo
self.root_no_epilogo = self.root
self.epilogo = self.root_no_epilogo.xpath(
'//ans:epilogo', namespaces=self.nsmap)[0]
self.root_no_epilogo.remove(self.epilogo)
self.hash_fornecido = self.epilogo.getchildren()[0].text
reslist = list(self.root_no_epilogo.iter())
conteudos = []
for i in reslist:
if not i.getchildren():
conteudos.append(i.text)
self.conteudo = ''.join(conteudos)
cod = hashlib.md5()
cod.update(self.conteudo.encode('iso-8859-1'))
self.hash = cod.hexdigest()
self.parse()
def valida_hash(self):
if self.hash != self.hash_fornecido:
self.valido = False
self.erros['lote']['_hash'] = "Hash Inválido! Fornecido: %s, Calculado: %s" % (
self.hash_fornecido, self.hash)
def executa_plugins(self, plugin_path=None):
print("Executando plugins")
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginInfoExtension("tiss-plugin")
if not self.plugins_path:
self.plugins_path = os.path.join(os.path.dirname(__file__), 'extensoes/plugins')
self.plugin_manager.setPluginPlaces([self.plugins_path])
self.plugin_manager.collectPlugins()
for plugin in self.plugin_manager.getAllPlugins():
plugin.plugin_object.executa(objeto=self)
def executa_providers(self, providers_path=None):
print("Executando Providers")
self.provider_manager = PluginManager()
self.provider_manager.setPluginInfoExtension("tiss-provider")
if not self.providers_path:
self.providers_path = os.path.join(os.path.dirname(__file__), 'extensoes/providers')
self.provider_manager.setPluginPlaces([self.providers_path])
self.provider_manager.collectPlugins()
for plugin in self.provider_manager.getAllPlugins():
plugin.plugin_object.executa(objeto=self)
def registra_erro_guia(self, erro):
try:
self.erros['guias'][erro['numero']]
except KeyError:
self.erros['guias'][erro['numero']] = []
self.erros['guias'][erro['numero']].append(erro)
def registra_provider(self, provider_name, provider):
'''
registra os dados de uma provider
'''
try:
self.providers[provider_name]
except:
self.providers[provider_name] = {}
self.providers[provider_name] = [provider]
def get_xpath(self, xpath):
return self.root.xpath(xpath, namespaces=self.nsmap)
class binWidget(QtGui.QWidget, Observable):
scrolled = QtCore.pyqtSignal(int, name='scroll')
oldscrolled = QtCore.SIGNAL('scroll')
def __init__(self, parent, source):
super(binWidget, self).__init__()
Observable.__init__(self)
self.parent = parent
# offset for text window
#self.data = mapped
self.dataOffset = 0
self.dataModel = source
self.cursor = Cursor(0, 0)
# self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self),
# HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self)]
logging.basicConfig(level=logging.ERROR)
self.manager = PluginManager(
categories_filter={"FileFormat": FileFormat})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'format')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
Formats = []
for plugin in self.manager.getPluginsOfCategory("FileFormat"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.recognize(self.dataModel):
print '[+] ' + po.name
Formats.append(po)
# sort plugins by priority
Formats = sorted(Formats, key=lambda x: x.priority, reverse=True)
po = Formats[0]
print 'Choosed plugin: ' + po.name
#print QtGui.QFontDatabase.addApplicationFont(os.path.join('terminus-ttf-4.39', 'TerminusTTF-4.39.ttf'))
self.multipleViewModes = [
BinViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po),
HexViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po),
DisasmViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po)
]
self.viewMode = self.multipleViewModes[0]
self.textDecorator = TextDecorator(self.viewMode)
self.viewMode.setTransformationEngine(self.textDecorator)
self.multipleViewModes[1].setTransformationEngine(self.textDecorator)
self.Banners = Banners()
#self.Banners.add(BottomBanner(self.dataModel, self.viewMode))
# self.Banners.add(TopBanner(self.dataModel, self.viewMode))
#self.Banners.add(self.banner)
# self.filebanner = FileAddrBanner(self.dataModel, self.viewMode)
#self.filebanner = PEBanner(self.dataModel, self.viewMode)
#self.Banners.add(PEBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
# self.offsetWindow_h = self.filebanner.getDesiredGeometry()[0] + 25
self.offsetWindow_h = 0
self.offsetWindow_v = 0
self.searchable = Searchable(self.dataModel, self.viewMode)
self.initUI()
[po.init(viewMode, parent=self) for viewMode in self.multipleViewModes]
for banner in po.getBanners():
self.Banners.add(banner)
po.registerShortcuts(self)
self.po = po
#self.scrolled = QtCore.pyqtSignal(int, name='scroll')
#self.scrolled.connect(self.scroll_from_outside)
self.searchWindow = SearchWindow(self, None, self.searchable)
self.addHandler(self.po)
self.addHandler(self.searchable)
self.addHandler(self.Banners)
self.notify(self.viewMode)
#self.connect(self, self.oldscrolled, self.scroll_from_outside)
#self.scrolled.emit(1)
#self.emit(QtCore.SIGNAL('scroll'), 1)
def scroll_from_outside(self, i):
#print 'slot-signal ' + str(i)
#self.scroll_pdown = True
self.update()
def initUI(self):
self.setMinimumSize(1, 30)
self.activateWindow()
self.setFocus()
#self.installEventFilter(self)
"""
# build thumbnail
dwidth = 100
dheight = 1200
factor = dheight/dwidth
import math
x = int(math.sqrt(len(self.data)/factor))
cols = x
pixThumb = QtGui.QPixmap(x*self.viewMode.fontWidth, factor*x*self.viewMode.fontHeight)
qp = QtGui.QPainter()
qp.begin(pixThumb)
qp.setFont(self.viewMode.font)
qp.setPen(self.viewMode.textPen)
for i,c in enumerate(self.data):
self.viewMode.transformText(qp, (i, c), self.data)
qp.drawText((i%cols)*self.viewMode.fontWidth, self.viewMode.fontHeight + (i/cols)*self.viewMode.fontHeight, c)
qp.end()
self.newqpix = pixThumb.scaled(dwidth, dheight, aspectRatioMode = QtCore.Qt.IgnoreAspectRatio, transformMode = QtCore.Qt.FastTransformation)
"""
"""
def getDisplayedData(self):
if self.dataOffset < 0:
self.dataOffset = 0
return False
chrlist = [unichr(cp437ToUnicode[ord(c)]) for c in self.data[self.dataOffset : self.dataOffset + self.viewMode.COLUMNS*self.viewMode.ROWS]]
self.text = "".join(chrlist)
return True
"""
def switchViewMode(self):
self.multipleViewModes = self.multipleViewModes[1:] + [
self.multipleViewModes[0]
]
self.viewMode = self.multipleViewModes[0]
# notify obervers
self.notify(self.viewMode)
def _resize(self):
self.Banners.resize(self.size().width() - self.offsetWindow_h,
self.size().height() - self.offsetWindow_v)
# compute space ocupated by banners
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getBottomOffset(
) + self.Banners.getTopOffset()
# resize window, substract space ocupated by banners
self.viewMode.resize(self.size().width() - offsetLeft,
self.size().height() - offsetBottom)
# event handlers
def resizeEvent(self, e):
self._resize()
def paintEvent(self, e):
qp = QtGui.QPainter()
qp.begin(self)
qp.setOpacity(1)
offsetLeft = self.offsetWindow_h + self.Banners.getLeftOffset()
offsetBottom = self.offsetWindow_v + self.Banners.getTopOffset()
#self.viewMode.draw2(qp, refresh=True)
#start = time()
qp.drawPixmap(offsetLeft, offsetBottom, self.viewMode.getPixmap())
#print 'Draw ' + str(time() - start)
self.Banners.draw(qp, self.offsetWindow_h, self.offsetWindow_v,
self.size().height())
# qp.drawPixmap(self.offsetWindow_h, self.size().height() - 50, self.banner.getPixmap())
# qp.drawPixmap(20, 0, self.filebanner.getPixmap())
qp.end()
"""
def keyPressEvent(self, event):
if event.modifiers() & QtCore.Qt.ShiftModifier:
if self.viewMode.handleKeyPressEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
def keyReleaseEvent(self, event):
if event.key() == QtCore.Qt.Key_Shift:
if self.viewMode.handleKeyReleaseEvent(QtCore.Qt.ShiftModifier, event.key()):
self.update()
"""
def eventFilter(self, watched, event):
if event.type() == QtCore.QEvent.KeyRelease:
key = event.key()
modifiers = event.modifiers()
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
self.update()
if event.type() == QtCore.QEvent.KeyPress:
#TODO: should we accept only certain keys ?
key = event.key()
modifiers = event.modifiers()
if key == QtCore.Qt.Key_F2:
if self.viewMode.isEditable():
if self.viewMode.isInEditMode():
self.viewMode.setEditMode(False)
else:
self.viewMode.setEditMode(True)
self.viewMode.draw(refresh=False)
# switch view mode
if key == QtCore.Qt.Key_Tab:
offs = self.viewMode.getCursorOffsetInPage()
base = self.viewMode.getDataModel().getOffset()
self.switchViewMode()
self._resize()
self.viewMode.goTo(base + offs)
self.update()
import pyperclip
if event.modifiers() & QtCore.Qt.ControlModifier:
if key == QtCore.Qt.Key_Insert:
if self.viewMode.selector.getCurrentSelection():
a, b = self.viewMode.selector.getCurrentSelection()
#print a, b
hx = ''
for s in self.dataModel.getStream(a, b):
hx += '{:02x}'.format(s)
pyperclip.copy(hx)
del pyperclip
#print pyperclip.paste()
# print 'coppied'
if event.modifiers() & QtCore.Qt.ShiftModifier:
if key == QtCore.Qt.Key_Insert:
import re
hx = pyperclip.paste()
#print hx
L = re.findall(r'.{1,2}', hx, re.DOTALL)
array = ''
for s in L:
array += chr(int(s, 16))
#print 'write '
#print 'write'
#print array
self.dataModel.write(0, array)
self.viewMode.draw(True)
del pyperclip
#print array
if key == QtCore.Qt.Key_F4:
self.unp = WUnpack(self, None)
self.unp.show()
if key == QtCore.Qt.Key_F10:
self.dataModel.flush()
import os
self.w = WHeaders(self, None)
self.w.show()
if not self.viewMode.isInEditMode():
if key == QtCore.Qt.Key_Slash:
self.searchWindow.show()
if key == QtCore.Qt.Key_N:
self.searchable.next(
self.viewMode.getCursorAbsolutePosition() + 1)
if key == QtCore.Qt.Key_B:
self.searchable.previous(
self.viewMode.getCursorAbsolutePosition())
# handle keys to view plugin
if self.viewMode.handleKeyEvent(modifiers, key, event=event):
event.accept()
self.update()
return True
return False
def setTextViewport(self, qp):
qp.setViewport(self.offsetWindow_h, self.offsetWindow_v,
self.size().width(),
self.size().height())
qp.setWindow(0, 0, self.size().width(), self.size().height())
def needsSave(self):
return self.dataModel.isDirty()
def save(self):
return self.dataModel.flush()
class PluginsManagerService(object):
"""
Description: main class in order to manage all the plugin operations like loading and triggering
"""
def __init__(self):
self.plugin_manager = PluginManager()
self.loaded_plugin_list = []
self.loaded_plugin_objects = None
self.plugin_db = Plugins.Plugins(mongodb)
self.plugin_location = os.path.abspath(
os.path.join(os.getcwd(), 'Plugins'))
self.deleted_plugin_location = os.path.abspath(
os.path.join(os.getcwd(), 'Plugins_deleted'))
self.load_plugin()
def load_plugin(self):
"""loads all plugins from folder and activates the plugins
Returns:
:dict: active_plugins_dict_from_db
Raises:
no_value
no_active_plugins
"""
# Build the manager
# Tell it the default place(s) where to find plugins
print "###################"
print "STARTED PLUGIN LOAD "
print "###################"
if os.path.isdir(self.plugin_location) is True:
try:
# set plugins from folder
self.plugin_manager.setPluginPlaces([self.plugin_location])
# define interfaces to load you have defined
self.plugin_manager.setCategoriesFilter({
"PreDeploymentPlugin":
IPreDeploymentPlugin,
"PostDeploymentPlugin":
IPostDeploymentPlugin
})
# Load all plugins from plugin folder
self.plugin_manager.collectPlugins()
loaded_plugin_from_folder = [
x.name for x in self.plugin_manager.getAllPlugins()
]
# validate plugin if not valid deactivate
for plugin in loaded_plugin_from_folder:
if not self.is_valid_plugin(plugin):
self.deactivate(plugin)
print(
"the following plugin is deactivate cause is not valid: "
+ plugin)
# update DB with new plugins
self.detect_new_plugins()
# load all activated plugins from DB
active_plugins_dict_from_db = self.plugin_db.get_all_plugin(
status='active')
active_plugins_in_db = [
x['name']
for x in self.plugin_db.get_all_plugin(status='active')
]
# final active_plugins list
if active_plugins_in_db is None:
print "no plugins installed"
elif loaded_plugin_from_folder is None:
print "no plugins in the plugins folder"
else:
active_plugins = [
x for x in loaded_plugin_from_folder
if x in active_plugins_in_db
]
for plugin in active_plugins:
# validate plugin if not valid deactivate
if not self.is_valid_plugin(plugin):
print "plugin is not valid: " + plugin + " deactivate"
self.deactivate(plugin)
else:
# activate
self.plugin_manager.activatePluginByName(plugin)
print "loaded plugin name: " + plugin
ValueError("loaded plugin name: " + plugin)
print "###################"
print "COMPLETED PLUGIN LOAD "
print "###################"
return active_plugins_dict_from_db
except (Exception, ValueError) as error:
print error
raise ValueError("Unable to load plugins :", error)
elif os.path.isdir(self.plugin_location) is False:
raise ValueError("plugin folder is set to " +
self.plugin_location +
" missing or not configured well")
else:
raise ValueError("unknown err during plugin loading")
def detect_new_plugins(self):
"""
updates new plugins in db
:param loaded_plugin_from_folder:
:return:
"""
# load all from db
loaded_plugin_from_folder = self.plugin_manager.getAllPlugins()
plugins_in_db = [
x.get('name') for x in self.plugin_db.get_all_plugin()
]
if not plugins_in_db:
new_plugins = loaded_plugin_from_folder
else:
new_plugins = [
x for x in loaded_plugin_from_folder
if x.name not in plugins_in_db
]
for plugin_object in new_plugins:
# get plugin type
parent_class = type(plugin_object.plugin_object).__bases__
parent_class_name = parent_class[0].__name__
plugin_dict = {
'name': plugin_object.name,
'category': parent_class_name[1:],
'author': plugin_object.author,
'version': str(plugin_object.version),
'description': plugin_object.description,
'status': 'inactive'
}
try:
self.plugin_db.add_plugin(plugin_dict)
except Exception as error:
print error
raise RuntimeError("unable to insert the plugin to DB")
def install_plugin(self, plugin_name):
'''
activates the plugin and triggers the install method of the plugin
:param plugin_name: string
:return:
:raises
RuntimeError "unable to insert the plugin to DB"
'''
# set plugins from folder
# self.plugin_location = os.path.abspath(os.path.join(os.getcwd(), 'Plugins'))
# self.load_plugin()
plugin_details = self.plugin_db.get_plugin_by_name(plugin_name)
plugin_object = self.plugin_manager.getPluginByName(
plugin_name, plugin_details.get('category'))
if plugin_object:
try:
self.activate(plugin_name)
return plugin_object.plugin_object.install()
except Exception as error:
print error
raise RuntimeError("unable to insert the plugin to DB: ",
error)
else:
raise ValueError("no active plugin with this name")
def is_valid_plugin(self, plugin):
'''
validate that plugin has all the required methods and attributes
:param plugin:
:return:
True or False
'''
# mandatory element lists
plugin_mandatory_attributes = [
'author', 'name', 'version', 'description'
]
plugin_mandatory_methods = [
'preform', 'activate', 'deactivate', 'is_activated'
]
# get plugin
plugin = self.plugin_manager.getPluginByName(plugin,
'PreDeploymentPlugin')
# check plugin manifest
try:
for attribute in plugin_mandatory_attributes:
if str(getattr(plugin, attribute, None)) == "":
raise ValueError("plugin is missing value in " + attribute)
except (ValueError, AttributeError) as error:
print error
return False
# check plugin methods
try:
methods = [
x for x in plugin_mandatory_methods
if x not in dir(plugin.plugin_object)
]
if len(methods) > 0:
raise ValueError("folowwing methods are missing " +
str(methods))
except (ValueError, AttributeError) as error:
print error
return False
return True
def uninstall_plugin(self, plugin):
'''
triggers the uninstall method of the plugin by name
remove the db details
rename the plugin folder
:param plugin_name:
'''
try:
# get category form DB
plugin_details = self.plugin_db.get_plugin_by_name(plugin)
if plugin_details:
plugin_object = self.plugin_manager.getPluginByName(
plugin, plugin_details.get('category'))
# uninstal
plugin_object.plugin_object.uninstall()
# delete from db
self.plugin_db.delete_plugin_by_name(plugin)
# check if deleted folder exists
if os.path.isdir(self.deleted_plugin_location) is not True:
os.mkdir(self.deleted_plugin_location)
# delete folder
os.rename(
self.plugin_location + "/" + plugin,
self.deleted_plugin_location + "/" + plugin + ".deleted")
else:
raise ValueError('couldnt find plugin')
except (ValueError, Exception) as error:
print error
raise RuntimeError(" could not uninstall plugin " + plugin + " " +
error)
def activate(self, plugin_name):
'''
activates plugin by name
:param plugin_name:
:raise
RuntimeError "could not activate plugin"
'''
try:
# get category form DB
plugin_details = self.plugin_db.get_plugin_by_name(plugin_name)
# activate
self.plugin_manager.activatePluginByName(
plugin_name, plugin_details.get('category'))
# updateDB
self.plugin_db.update_plugin_status(plugin_name, 'active')
except Exception as error:
print error
raise RuntimeError(" could not activate plugin " + plugin_name)
def deactivate(self, plugin_name):
'''
deactivates plugin by name
:param plugin_name: string
:raise
RuntimeError "could not deactivate plugin"
'''
try:
# get category form DB
plugin_details = self.plugin_db.get_plugin_by_name(plugin_name)
# deactivate
self.plugin_manager.deactivatePluginByName(
plugin_name, plugin_details.get('category'))
# updateDB
self.plugin_db.update_plugin_status(plugin_name, 'inactive')
except Exception as error:
print error
raise RuntimeError(" could not deactivate plugin " + plugin_name)
def preform(self, category, plugin_name):
"""
execute the preform method of plugin in category
:param plugin_name: string
:param category: string
:raises
RuntimeError unable to trigger operation of plugin:
"""
try:
plugin = self.plugin_manager.getPluginByName(plugin_name, category)
plugin.plugin_object.preform()
except Exception as error:
print error
raise RuntimeError(" unable to trigger operation of plugin:" +
plugin_name)
def preform_all_in_category(self, category, **keyargs):
'''
triggers all preform methods of plugin in the provided category
:param category:
:return:
:raises
RuntimeError "unable to execute the plugin logic"
'''
for (key, value) in keyargs.iteritems():
print(key, value)
try:
for plugin in self.plugin_manager.getPluginsOfCategory(category):
print("preforming action of plugin " + plugin.name)
plugin.plugin_object.preform()
except (ValueError, Exception) as error:
print error
raise RuntimeError("unable to execute the plugin logic: " + error)
# ## test
def test(self):
# just for test
for p in self.plugin_manager.getPluginsOfCategory(
'PreDeploymentPlugin'):
self.install_plugin('TestPlugin')
class GeneManager():
"""
Merge data available in ?elasticsearch into proper json objects
plugin_paths is a collection of filesystem paths to search for potential plugins
plugin_names is an ordered collection of class names of plugins which determines
the order they are handled in
"""
def __init__(self,
loader,
r_server,
plugin_paths,
plugin_order):
self.loader = loader
self.r_server = r_server
self.genes = GeneSet()
self._logger = logging.getLogger(__name__)
self._logger.info("Preparing the plug in management system")
# Build the manager
self.simplePluginManager = PluginManager()
# Tell it the default place(s) where to find plugins
self.simplePluginManager.setPluginPlaces(plugin_paths)
for dir in plugin_paths:
self._logger.debug("Looking for plugins in %s", dir)
# Load all plugins
self.simplePluginManager.collectPlugins()
self.plugin_order = plugin_order
def merge_all(self, data_config, dry_run = False):
for plugin_name in self.plugin_order:
plugin = self.simplePluginManager.getPluginByName(plugin_name)
plugin.plugin_object.print_name()
plugin.plugin_object.merge_data(genes=self.genes,
loader=self.loader, r_server=self.r_server, data_config=data_config)
self._store_data(dry_run=dry_run)
def _store_data(self, dry_run = False):
if not dry_run:
self.loader.create_new_index(Const.ELASTICSEARCH_GENE_NAME_INDEX_NAME)
#need to directly get the versioned index name for this function
self.loader.prepare_for_bulk_indexing(
self.loader.get_versioned_index(Const.ELASTICSEARCH_GENE_NAME_INDEX_NAME))
for geneid, gene in self.genes.iterate():
gene.preprocess()
if not dry_run:
self.loader.put(Const.ELASTICSEARCH_GENE_NAME_INDEX_NAME,
Const.ELASTICSEARCH_GENE_NAME_DOC_NAME,
geneid, gene.to_json())
if not dry_run:
self.loader.flush_all_and_wait(Const.ELASTICSEARCH_GENE_NAME_INDEX_NAME)
#restore old pre-load settings
#note this automatically does all prepared indexes
self.loader.restore_after_bulk_indexing()
self._logger.info('all gene objects pushed to elasticsearch')
"""
Run a series of QC tests on EFO elasticsearch index. Returns a dictionary
of string test names and result objects
"""
def qc(self, esquery):
#number of gene entries
gene_count = 0
#Note: try to avoid doing this more than once!
for gene_entry in esquery.get_all_targets():
gene_count += 1
#put the metrics into a single dict
metrics = dict()
metrics["gene.count"] = gene_count
return metrics
def __init__(self, **config):
"""Setup proper environment for running tasks."""
self.global_data = {}
self.posts_per_year = defaultdict(list)
self.posts_per_month = defaultdict(list)
self.posts_per_tag = defaultdict(list)
self.posts_per_category = defaultdict(list)
self.post_per_file = {}
self.timeline = []
self.pages = []
self._scanned = False
self._template_system = None
self._THEMES = None
if not config:
self.configured = False
else:
self.configured = True
# This is the default config
self.config = {
'ADD_THIS_BUTTONS': True,
'ARCHIVE_PATH': "",
'ARCHIVE_FILENAME': "archive.html",
'BODY_END': "",
'CACHE_FOLDER': 'cache',
'CODE_COLOR_SCHEME': 'default',
'COMMENT_SYSTEM': 'disqus',
'COMMENTS_IN_GALLERIES': False,
'COMMENTS_IN_STORIES': False,
'COMPILERS': {
"rest": ('.txt', '.rst'),
"markdown": ('.md', '.mdown', '.markdown'),
"textile": ('.textile', ),
"txt2tags": ('.t2t', ),
"bbcode": ('.bb', ),
"wiki": ('.wiki', ),
"ipynb": ('.ipynb', ),
"html": ('.html', '.htm')
},
'CONTENT_FOOTER': '',
'COPY_SOURCES': True,
'CREATE_MONTHLY_ARCHIVE': False,
'DATE_FORMAT': '%Y-%m-%d %H:%M',
'DEFAULT_LANG': "en",
'DEPLOY_COMMANDS': [],
'DISABLED_PLUGINS': (),
'COMMENT_SYSTEM_ID': 'nikolademo',
'ENABLED_EXTRAS': (),
'EXTRA_HEAD_DATA': '',
'FAVICONS': {},
'FEED_LENGTH': 10,
'FILE_METADATA_REGEXP': None,
'ADDITIONAL_METADATA': {},
'FILES_FOLDERS': {
'files': ''
},
'FILTERS': {},
'GALLERY_PATH': 'galleries',
'GZIP_FILES': False,
'GZIP_EXTENSIONS':
('.txt', '.htm', '.html', '.css', '.js', '.json'),
'HIDE_SOURCELINK': False,
'HIDE_UNTRANSLATED_POSTS': False,
'HYPHENATE': False,
'INDEX_DISPLAY_POST_COUNT': 10,
'INDEX_FILE': 'index.html',
'INDEX_TEASERS': False,
'INDEXES_TITLE': "",
'INDEXES_PAGES': "",
'INDEX_PATH': '',
'LICENSE': '',
'LINK_CHECK_WHITELIST': [],
'LISTINGS_FOLDER': 'listings',
'NAVIGATION_LINKS': None,
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'MAX_IMAGE_SIZE': 1280,
'MATHJAX_CONFIG': '',
'OLD_THEME_SUPPORT': True,
'OUTPUT_FOLDER': 'output',
'POSTS': (("posts/*.txt", "posts", "post.tmpl"), ),
'PAGES': (("stories/*.txt", "stories", "story.tmpl"), ),
'PRETTY_URLS': False,
'FUTURE_IS_NOW': False,
'READ_MORE_LINK':
'<p class="more"><a href="{link}">{read_more}…</a></p>',
'REDIRECTIONS': [],
'RSS_LINK': None,
'RSS_PATH': '',
'RSS_TEASERS': True,
'SEARCH_FORM': '',
'SLUG_TAG_PATH': True,
'SOCIAL_BUTTONS_CODE': SOCIAL_BUTTONS_CODE,
'STORY_INDEX': False,
'STRIP_INDEXES': False,
'SITEMAP_INCLUDE_FILELESS_DIRS': True,
'TAG_PATH': 'categories',
'TAG_PAGES_ARE_INDEXES': False,
'THEME': 'bootstrap',
'THEME_REVEAL_CONFIG_SUBTHEME': 'sky',
'THEME_REVEAL_CONFIG_TRANSITION': 'cube',
'THUMBNAIL_SIZE': 180,
'USE_BUNDLES': True,
'USE_CDN': False,
'USE_FILENAME_AS_TITLE': True,
'TIMEZONE': None,
'DEPLOY_DRAFTS': True,
'DEPLOY_FUTURE': False,
'SCHEDULE_ALL': False,
'SCHEDULE_RULE': '',
'SCHEDULE_FORCE_TODAY': False
}
self.config.update(config)
# Make sure we have pyphen installed if we are using it
if self.config.get('HYPHENATE') and pyphen is None:
print('WARNING: To use the hyphenation, you have to install '
'the "pyphen" package.')
print('WARNING: Setting HYPHENATE to False.')
self.config['HYPHENATE'] = False
# Deprecating post_compilers
# TODO: remove on v7
if 'post_compilers' in config:
print(
"WARNING: The post_compilers option is deprecated, use COMPILERS instead."
)
if 'COMPILERS' in config:
print(
"WARNING: COMPILERS conflicts with post_compilers, ignoring post_compilers."
)
else:
self.config['COMPILERS'] = config['post_compilers']
# Deprecating post_pages
# TODO: remove on v7
if 'post_pages' in config:
print(
"WARNING: The post_pages option is deprecated, use POSTS and PAGES instead."
)
if 'POSTS' in config or 'PAGES' in config:
print(
"WARNING: POSTS and PAGES conflict with post_pages, ignoring post_pages."
)
else:
self.config['POSTS'] = [
item[:3] for item in config['post_pages'] if item[-1]
]
self.config['PAGES'] = [
item[:3] for item in config['post_pages'] if not item[-1]
]
# FIXME: Internally, we still use post_pages because it's a pain to change it
self.config['post_pages'] = []
for i1, i2, i3 in self.config['POSTS']:
self.config['post_pages'].append([i1, i2, i3, True])
for i1, i2, i3 in self.config['PAGES']:
self.config['post_pages'].append([i1, i2, i3, False])
# Deprecating DISQUS_FORUM
# TODO: remove on v7
if 'DISQUS_FORUM' in config:
print(
"WARNING: The DISQUS_FORUM option is deprecated, use COMMENT_SYSTEM_ID instead."
)
if 'COMMENT_SYSTEM_ID' in config:
print(
"WARNING: DISQUS_FORUM conflicts with COMMENT_SYSTEM_ID, ignoring DISQUS_FORUM."
)
else:
self.config['COMMENT_SYSTEM_ID'] = config['DISQUS_FORUM']
# Deprecating the ANALYTICS option
# TODO: remove on v7
if 'ANALYTICS' in config:
print(
"WARNING: The ANALYTICS option is deprecated, use BODY_END instead."
)
if 'BODY_END' in config:
print(
"WARNING: ANALYTICS conflicts with BODY_END, ignoring ANALYTICS."
)
else:
self.config['BODY_END'] = config['ANALYTICS']
# Deprecating the SIDEBAR_LINKS option
# TODO: remove on v7
if 'SIDEBAR_LINKS' in config:
print(
"WARNING: The SIDEBAR_LINKS option is deprecated, use NAVIGATION_LINKS instead."
)
if 'NAVIGATION_LINKS' in config:
print(
"WARNING: The SIDEBAR_LINKS conflicts with NAVIGATION_LINKS, ignoring SIDEBAR_LINKS."
)
else:
self.config['NAVIGATION_LINKS'] = config['SIDEBAR_LINKS']
# Compatibility alias
self.config['SIDEBAR_LINKS'] = self.config['NAVIGATION_LINKS']
if self.config['NAVIGATION_LINKS'] in (None, {}):
self.config['NAVIGATION_LINKS'] = {self.config['DEFAULT_LANG']: ()}
# Deprecating the ADD_THIS_BUTTONS option
# TODO: remove on v7
if 'ADD_THIS_BUTTONS' in config:
print(
"WARNING: The ADD_THIS_BUTTONS option is deprecated, use SOCIAL_BUTTONS_CODE instead."
)
if not config['ADD_THIS_BUTTONS']:
print(
"WARNING: Setting SOCIAL_BUTTONS_CODE to empty because ADD_THIS_BUTTONS is False."
)
self.config['SOCIAL_BUTTONS_CODE'] = ''
# STRIP_INDEX_HTML config has been replaces with STRIP_INDEXES
# Port it if only the oldef form is there
# TODO: remove on v7
if 'STRIP_INDEX_HTML' in config and 'STRIP_INDEXES' not in config:
print(
"WARNING: You should configure STRIP_INDEXES instead of STRIP_INDEX_HTML"
)
self.config['STRIP_INDEXES'] = config['STRIP_INDEX_HTML']
# PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
if config.get('PRETTY_URLS', False) and 'STRIP_INDEXES' not in config:
self.config['STRIP_INDEXES'] = True
if config.get('COPY_SOURCES') and not self.config['HIDE_SOURCELINK']:
self.config['HIDE_SOURCELINK'] = True
self.config['TRANSLATIONS'] = self.config.get(
'TRANSLATIONS', {self.config['DEFAULT_LANG']: ''})
# SITE_URL is required, but if the deprecated BLOG_URL
# is available, use it and warn
# TODO: remove on v7
if 'SITE_URL' not in self.config:
if 'BLOG_URL' in self.config:
print(
"WARNING: You should configure SITE_URL instead of BLOG_URL"
)
self.config['SITE_URL'] = self.config['BLOG_URL']
self.default_lang = self.config['DEFAULT_LANG']
self.translations = self.config['TRANSLATIONS']
# BASE_URL defaults to SITE_URL
if 'BASE_URL' not in self.config:
self.config['BASE_URL'] = self.config.get('SITE_URL')
# BASE_URL should *always* end in /
if self.config['BASE_URL'] and self.config['BASE_URL'][-1] != '/':
print("WARNING: Your BASE_URL doesn't end in / -- adding it.")
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"RestExtension": RestExtension,
})
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(__file__), 'plugins'),
os.path.join(os.getcwd(), 'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(__file__),
utils.sys_encode('plugins')),
os.path.join(os.getcwd(), utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.commands = {}
# Activate all command plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory("Command"):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS
and plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, "Command")
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugin_info.plugin_object.short_help = plugin_info.description
self.commands[plugin_info.name] = plugin_info.plugin_object
# Activate all task plugins
for task_type in ["Task", "LateTask"]:
for plugin_info in self.plugin_manager.getPluginsOfCategory(
task_type):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS and
plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all multiplier plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"TaskMultiplier"):
if (plugin_info.name in self.config['DISABLED_PLUGINS'] or
(plugin_info.name in self.EXTRA_PLUGINS
and plugin_info.name not in self.config['ENABLED_EXTRAS'])):
self.plugin_manager.removePluginFromCategory(
plugin_info, task_type)
continue
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# Activate all required compiler plugins
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
if plugin_info.name in self.config["COMPILERS"].keys():
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
# set global_context for template rendering
self._GLOBAL_CONTEXT = {}
self._GLOBAL_CONTEXT['_link'] = self.link
self._GLOBAL_CONTEXT['set_locale'] = s_l
self._GLOBAL_CONTEXT['rel_link'] = self.rel_link
self._GLOBAL_CONTEXT['abs_link'] = self.abs_link
self._GLOBAL_CONTEXT['exists'] = self.file_exists
self._GLOBAL_CONTEXT['SLUG_TAG_PATH'] = self.config['SLUG_TAG_PATH']
self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[
'INDEX_DISPLAY_POST_COUNT']
self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
self._GLOBAL_CONTEXT['date_format'] = self.config.get(
'DATE_FORMAT', '%Y-%m-%d %H:%M')
self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
self._GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
# TODO: remove fallback in v7
self._GLOBAL_CONTEXT['blog_url'] = self.config.get(
'SITE_URL', self.config.get('BLOG_URL'))
self._GLOBAL_CONTEXT['blog_desc'] = self.config.get('BLOG_DESCRIPTION')
self._GLOBAL_CONTEXT['body_end'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['analytics'] = self.config.get('BODY_END')
# TODO: remove in v7
self._GLOBAL_CONTEXT['add_this_buttons'] = self.config.get(
'SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['social_buttons_code'] = self.config.get(
'SOCIAL_BUTTONS_CODE')
self._GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
self._GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
self._GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
self._GLOBAL_CONTEXT['comment_system'] = self.config.get(
'COMMENT_SYSTEM')
self._GLOBAL_CONTEXT['comment_system_id'] = self.config.get(
'COMMENT_SYSTEM_ID')
# TODO: remove in v7
self._GLOBAL_CONTEXT['disqus_forum'] = self.config.get(
'COMMENT_SYSTEM_ID')
self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
'MATHJAX_CONFIG')
self._GLOBAL_CONTEXT['subtheme'] = self.config.get(
'THEME_REVEAL_CONFIG_SUBTHEME')
self._GLOBAL_CONTEXT['transition'] = self.config.get(
'THEME_REVEAL_CONFIG_TRANSITION')
self._GLOBAL_CONTEXT['content_footer'] = self.config.get(
'CONTENT_FOOTER')
self._GLOBAL_CONTEXT['rss_path'] = self.config.get('RSS_PATH')
self._GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
self._GLOBAL_CONTEXT['navigation_links'] = utils.Functionary(
list, self.config['DEFAULT_LANG'])
for k, v in self.config.get('NAVIGATION_LINKS', {}).items():
self._GLOBAL_CONTEXT['navigation_links'][k] = v
# TODO: remove on v7
# Compatibility alias
self._GLOBAL_CONTEXT['sidebar_links'] = self._GLOBAL_CONTEXT[
'navigation_links']
self._GLOBAL_CONTEXT['twitter_card'] = self.config.get(
'TWITTER_CARD', {})
self._GLOBAL_CONTEXT['hide_sourcelink'] = self.config.get(
'HIDE_SOURCELINK')
self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get(
'EXTRA_HEAD_DATA')
self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
# Load compiler plugins
self.compilers = {}
self.inverse_compilers = {}
for plugin_info in self.plugin_manager.getPluginsOfCategory(
"PageCompiler"):
self.compilers[plugin_info.name] = \
plugin_info.plugin_object
def main(package_name, output_dir, verbose, debug, output_json, input_list,
save_files):
# Normalize targeting options
targets = []
if package_name:
targets.append(package_name)
elif input_list:
targets = _decode_json_file(input_list)
# else:
# targets = _pull_from_queue()
if verbose and not output_json:
print("-> Loading scan plugins")
scan_plugins = PluginManager()
scan_plugins.setPluginPlaces(["plugins"])
scan_plugins.collectPlugins()
all_plugins = scan_plugins.getAllPlugins()
# Fire!
scan_errors = 0
for raw_input in targets:
scan_list = []
package_meta = {}
if verbose and not output_json:
print(f"-> Using pip to download {raw_input}")
if debug:
pprint(raw_input)
output = _pip_download(
raw_input=raw_input,
output_dir=output_dir,
verbose=verbose,
debug=debug,
output_json=output_json,
)
if verbose and not output_json:
print("-> Extracting archives and meta")
if debug:
pprint(output)
if output:
parsed_raw_dir_list, package_meta = _extract_archives(
output=output,
output_dir=output_dir,
package_meta=package_meta,
verbose=verbose,
debug=debug,
output_json=output_json,
)
if verbose and not output_json:
print("-> Parsing out the scan list")
if debug:
pprint(parsed_raw_dir_list)
if parsed_raw_dir_list:
scan_list, package_meta = _retrieve_directories_to_scan(
parsed_raw_dir_list=parsed_raw_dir_list,
package_meta=package_meta,
verbose=verbose,
debug=debug,
output_json=output_json,
)
if scan_list and package_meta:
responses = []
for plugin in all_plugins:
responses.append(
plugin.plugin_object.scan(
scan_list,
package_meta,
output_dir,
verbose,
debug,
output_json,
))
scan_errors += sum(responses)
if debug:
pprint(responses)
# @TODO add package cleanup routine
# package_meta["saved_file_name"]
if not save_files and package_meta:
if verbose and not output_json:
print("-> Cleaning up downloaded files")
if debug:
pprint(package_meta)
_clean_up_downloads(package_meta, output_dir, verbose,
debug, output_json)
else:
if verbose and not output_json:
print(f"! Pip download failed for {raw_input}")
scan_errors += 1
if verbose and not output_json:
print(f"Scan complete! {scan_errors} errors.")
class FakeSite(object):
def __init__(self):
self.template_system = self
self.invariant = False
self.config = {
'DISABLED_PLUGINS': [],
'EXTRA_PLUGINS': [],
'DEFAULT_LANG': 'en',
'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite'],
'TRANSLATIONS_PATTERN': '{path}.{lang}.{ext}',
'LISTINGS_FOLDERS': {
'listings': 'listings'
},
}
self.EXTRA_PLUGINS = self.config['EXTRA_PLUGINS']
self.plugin_manager = PluginManager(
categories_filter={
"Command": Command,
"Task": Task,
"LateTask": LateTask,
"TemplateSystem": TemplateSystem,
"PageCompiler": PageCompiler,
"TaskMultiplier": TaskMultiplier,
"CompilerExtension": CompilerExtension,
"MarkdownExtension": MarkdownExtension,
"RestExtension": RestExtension
})
self.loghandlers = nikola.utils.STDERR_HANDLER # TODO remove on v8
self.shortcode_registry = {}
self.plugin_manager.setPluginInfoExtension('plugin')
if sys.version_info[0] == 3:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
'plugins'),
]
else:
places = [
os.path.join(os.path.dirname(nikola.utils.__file__),
nikola.utils.sys_encode('plugins')),
]
self.plugin_manager.setPluginPlaces(places)
self.plugin_manager.collectPlugins()
self.compiler_extensions = self._activate_plugins_of_category(
"CompilerExtension")
self.timeline = [FakePost(title='Fake post', slug='fake-post')]
self.debug = True
self.rst_transforms = []
self.post_per_input_file = {}
# This is to make plugin initialization happy
self.template_system = self
self.name = 'mako'
def _activate_plugins_of_category(self, category):
"""Activate all the plugins of a given category and return them."""
# this code duplicated in nikola/nikola.py
plugins = []
for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
if plugin_info.name in self.config.get('DISABLED_PLUGINS'):
self.plugin_manager.removePluginFromCategory(
plugin_info, category)
else:
self.plugin_manager.activatePluginByName(plugin_info.name)
plugin_info.plugin_object.set_site(self)
plugins.append(plugin_info)
return plugins
def render_template(self, name, _, context):
return ('<img src="IMG.jpg">')
# this code duplicated in nikola/nikola.py
def register_shortcode(self, name, f):
"""Register function f to handle shortcode "name"."""
if name in self.shortcode_registry:
nikola.utils.LOGGER.warn('Shortcode name conflict: %s', name)
return
self.shortcode_registry[name] = f
def apply_shortcodes(self, data):
"""Apply shortcodes from the registry on data."""
return nikola.shortcodes.apply_shortcodes(data,
self.shortcode_registry)
def process_file(input_path, cfg):
""" """
source_format = cfg.get('input_format')
target_format = cfg.get('output_format')
# Initialize plugins
readers_dir = pkg_resources.resource_filename( 'syntool_ingestor'
, 'share/plugins/readers')
fmts_dir = pkg_resources.resource_filename( 'syntool_ingestor'
, 'share/plugins/formatters')
manager = PluginManager()
manager.setCategoriesFilter({ 'formatters': ifaces.IFormatterPlugin
, 'readers': ifaces.IReaderPlugin
})
manager.setPluginPlaces([fmts_dir, readers_dir])
manager.collectPlugins()
# Extract metadata and parse input data if needed
plugin_found = False
for plugin_wrapper in manager.getPluginsOfCategory('readers'):
plugin = plugin_wrapper.plugin_object
if plugin.can_handle(source_format):
plugin_found = True
break
if not plugin_found:
raise Exception('No plugin found for "{}"'.format(source_format))
input_generator = plugin.extract_from_dataset(input_path, cfg)
plugin_found = False
# Loop round the plugins to find a suitable one
for plugin_wrapper in manager.getPluginsOfCategory('formatters'):
plugin = plugin_wrapper.plugin_object
if plugin.can_handle(target_format):
plugin_found = True
break
if not plugin_found:
raise Exception('No plugin found for "{}"'.format(target_format))
metadata_list = []
for input_data in input_generator:
src_meta, src_data = input_data
src_meta['output_format'] = cfg['output_format']
src_meta['output_options'] = cfg['output_options']
produced_meta = {}
# Build output dir
workspace_root = cfg.get('workspace_root', os.getcwd())
output_id = plugin.get_output_id(cfg)
syntool_id = '{}_{}{}'.format( cfg['output_proj']
, src_meta['product']
, output_id)
syntool_id = syntool_id.replace(' ', '_')
src_meta['syntool_id'] = syntool_id
if 'datagroup' in src_meta:
workspace = os.path.join( workspace_root
, syntool_id
, src_meta['datagroup']
, src_meta['dataset'])
else:
workspace = os.path.join( workspace_root
, syntool_id
, src_meta['dataset'])
if not os.path.exists(workspace):
try:
os.makedirs(workspace)
except OSError, e:
if e.errno != errno.EEXIST:
raise
# Produce representation
produced_meta = plugin.create_representation(input_path, src_data, workspace, cfg, src_meta)
produced_meta['output_type'] = plugin.get_representation_type(cfg)
produced_meta['output_level'] = plugin_wrapper.details.get('Syntool', 'RepresentationLevel')
produced_meta['output_timeline'] = plugin_wrapper.details.get('Syntool', 'TimelineBehavior')
if 'output_timeline' in cfg:
produced_meta['output_timeline'] = cfg['output_timeline']
metadata = src_meta
metadata.update(produced_meta)
# Simplify shape given a tolerance (0. by default)
if not cfg['no_shape']:
shape_geom = ogr.CreateGeometryFromWkt(metadata['shape_str'])
shape_tolerance = float(cfg['output_options'].get('shape-tolerance', 0.))
simp_shape_geom = shape_geom.SimplifyPreserveTopology(shape_tolerance)
shape_wkt = simp_shape_geom.ExportToWkt()
shape_wkt = shape_wkt.replace('POLYGON (', 'POLYGON(')
logger.debug('Shape simplification tolerance: {}'.format(shape_tolerance))
logger.debug('Shape before simplification: {}'.format(metadata['shape_str']))
logger.debug('Shape after simplification: {}'.format(shape_wkt))
if simp_shape_geom.IsEmpty():
raise Exception('Empty simplified shape.')
if not simp_shape_geom.IsValid():
raise Exception('Not valid simplified shape.')
metadata['shape_str'] = shape_wkt
if cfg['create_feature_metadata']:
features_path = os.path.join(workspace, 'features')
try:
os.makedirs(features_path)
except OSError, e:
if e.errno != errno.EEXIST:
raise
metadata_ini_path = os.path.join(features_path, 'metadata.ini')
syntool_ingestor.metadata.write_ini(metadata_ini_path, metadata)
if isinstance(metadata['output_path'], list):
metadata['output_path'].append(features_path)
else:
metadata['output_path'] = [metadata['output_path'], features_path]
if plugin:
plugin_feature_names = {name: {}}
all_props = plugin.plugin_object.fill_properties(
plugin_feature_names) # fill in display name and such
if all_props:
long_name = all_props[name]["displaytext"]
else:
long_name = name
return long_name
###############
# the manager #
###############
pluginManager = PluginManager()
pluginManager.setPluginPlaces(plugin_paths)
pluginManager.setCategoriesFilter({
"ObjectFeatures":
ObjectFeaturesPlugin,
"TrackingExportFormats":
TrackingExportFormatPlugin
})
pluginManager.collectPlugins()
for pluginInfo in pluginManager.getAllPlugins():
pluginManager.activatePluginByName(pluginInfo.name)
def loadPlugins(self, basedir, topdir):
if self.loaded:
print("Cannot load plugins multiple times")
exit(1)
self.loaded = True
# find the loader utility so we can bootstrap ourselves
try:
m = imp.load_source("LoadClasses",
os.path.join(basedir, "LoadClasses.py"))
except ImportError:
print(
"ERROR: unable to load LoadClasses that must contain the class loader object"
)
exit(1)
cls = getattr(m, "LoadClasses")
a = cls()
# setup the loader object
self.loader = a.__class__()
# Setup the array of directories we will search for plugins
# Note that we always look at the topdir location by default
plugindirs = []
plugindirs.append(topdir)
if self.options['plugindir']:
# could be a comma-delimited list, so split on commas
x = self.options['plugindir'].split(',')
for y in x:
# prepend so we always look at the given
# location first in case the user wants
# to "overload/replace" a default MTT
# class definition
plugindirs.insert(0, y)
# Traverse the plugin directory tree and add all
# the class definitions we can find
for dirPath in plugindirs:
try:
filez = os.listdir(dirPath)
for file in filez:
file = os.path.join(dirPath, file)
if os.path.isdir(file):
self.loader.load(file)
except:
if not self.options['ignoreloadpatherrs']:
print("Plugin directory", dirPath, "not found")
sys.exit(1)
# Build the stages plugin manager
self.stages = PluginManager()
# set the location
self.stages.setPluginPlaces(plugindirs)
# Get a list of all the categories - this corresponds to
# the MTT stages that have been defined. Note that we
# don't need to formally define the stages here - anyone
# can add a new stage, or delete an old one, by simply
# adding or removing a plugin directory.
self.stages.setCategoriesFilter(self.loader.stages)
# Load all plugins we find there
self.stages.collectPlugins()
# Build the tools plugin manager - tools differ from sections
# in that they are plugins we will use to execute the various
# sections. For example, the TestRun section clearly needs the
# ability to launch jobs. There are many ways to launch jobs
# depending on the environment, and sometimes several ways to
# start jobs even within one environment (e.g., mpirun vs
# direct launch).
self.tools = PluginManager()
# location is the same
self.tools.setPluginPlaces(plugindirs)
# Get the list of tools - not every tool will be capable
# of executing. For example, a tool that supports direct launch
# against a specific resource manager cannot be used on a
# system being managed by a different RM.
self.tools.setCategoriesFilter(self.loader.tools)
# Load all the tool plugins
self.tools.collectPlugins()
# Tool plugins are required to provide a function we can
# probe to determine if they are capable of operating - check
# those now and prune those tools that cannot support this
# environment
# Build the utilities plugins
self.utilities = PluginManager()
# set the location
self.utilities.setPluginPlaces(plugindirs)
# Get the list of available utilities.
self.utilities.setCategoriesFilter(self.loader.utilities)
# Load all the utility plugins
self.utilities.collectPlugins()
# since we use these all over the place, find the
# ExecuteCmd and ModuleCmd plugins and record them
availUtil = list(self.loader.utilities.keys())
for util in availUtil:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if "ExecuteCmd" == pluginInfo.plugin_object.print_name():
self.execmd = pluginInfo.plugin_object
elif "ModuleCmd" == pluginInfo.plugin_object.print_name():
self.modcmd = pluginInfo.plugin_object
# initialize this module
self.modcmd.setCommand(self.options)
if self.execmd is not None and self.modcmd is not None:
break
if self.execmd is None:
print("ExecuteCmd plugin was not found")
print("This is a basic capability required")
print("for MTT operations - cannot continue")
sys.exit(1)
# similarly, capture the highest priority defaults stage here
pri = -1
for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"):
if pri < pluginInfo.plugin_object.priority():
self.defaults = pluginInfo.plugin_object
pri = pluginInfo.plugin_object.priority()
return
def testCategoryManipulation(self):
"""
Test querying, removing and adding plugins from/to a category.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# load the plugins that may be found
spm.collectPlugins()
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
plugin_info = spm.getPluginsOfCategory(sole_category)[0]
# try to remove it and check that is worked
spm.removePluginFromCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
# now re-add this plugin the to same category
spm.appendPluginToCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
from . import COLOURS, get_verify, get_host_overrides, tcptest
from .output import Output
FORMAT = '%(asctime)-15s %(name)s [%(levelname)s]: %(message)s'
THIS_DIR = os.path.dirname(os.path.realpath(__file__))
logging.basicConfig(format=FORMAT,
level=logging.ERROR,
datefmt="%Y-%m-%d %H:%M:%S")
LOG = logging.getLogger('smolder')
REQUESTS_LOG = logging.getLogger('requests')
REQUESTS_LOG.setLevel(logging.ERROR)
logging.getLogger('yapsy').setLevel(logging.INFO)
manager = PluginManager()
manager.setPluginPlaces([THIS_DIR, "~/.smolder_plugins"])
manager.collectPlugins()
OUTPUT_WIDTH = 108
SCHEMA = {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"uri": {
"type": "string",
"required": False
},
def testTwoStepsLoad(self):
"""
Test loading the plugins in two steps in order to collect more
deltailed informations.
"""
spm = PluginManager(directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins")
])
# trigger the first step to look up for plugins
spm.locatePlugins()
# make full use of the "feedback" the loadPlugins can give
# - set-up the callback function that will be called *before*
# loading each plugin
callback_infos = []
def preload_cbk(plugin_info):
callback_infos.append(plugin_info)
# - gather infos about the processed plugins (loaded or not)
loadedPlugins = spm.loadPlugins(callback=preload_cbk)
self.assertEqual(len(loadedPlugins), 1)
self.assertEqual(len(callback_infos), 1)
self.assertEqual(loadedPlugins[0].error, None)
self.assertEqual(loadedPlugins[0], callback_infos[0])
# check that the getCategories works
self.assertEqual(len(spm.getCategories()), 1)
sole_category = spm.getCategories()[0]
# check the getPluginsOfCategory
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
plugin_info = spm.getPluginsOfCategory(sole_category)[0]
# try to remove it and check that is worked
spm.removePluginFromCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 0)
# now re-add this plugin the to same category
spm.appendPluginToCategory(plugin_info, sole_category)
self.assertEqual(len(spm.getPluginsOfCategory(sole_category)), 1)
def main(args: List[AnyStr] = None) -> int:
"""
prepare and start application
the main method is responsible for setting up the environment,
i.e. to load translations, set a logger, parse the command line arguments,
load configuration files and start the application.
:param args: command line arguments; default: None
:type args: List of strings
"""
if not args:
args = sys.argv
script_name = os.path.basename(sys.argv[0])
application = AppDirs(os.path.splitext(script_name)[0],
appauthor=__author__,
version=__version__,
roaming=True)
logging_path = application.user_data_dir
config_path = application.user_config_dir
# set locale
locale.setlocale(locale.LC_ALL, '')
_locale = locale.getlocale(locale.LC_ALL)[0]
locale_path = pkg_resources.resource_filename("servertools", "data/i18n")
translate = gettext.translation("servertools",
localedir=locale_path,
languages=[locale.getlocale()[0], 'en'])
translate.install()
if '_' not in globals():
_ = unicode
# setup logging
handlers = _log_default_handler(proc=script_name, log_dir=logging_path)
logging.basicConfig(level=4, format="%(message)s", handlers=handlers)
logger = logging.getLogger("{}_logger".format(
os.path.splitext(script_name)[0]))
logger.setLevel(logging.DEBUG)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.WARNING)
stream_handler.setFormatter(logging.Formatter("%(message)s"))
logger.addHandler(stream_handler)
# setup argument parser
parser = ArgumentParser(prog=script_name)
verbosity_parsergroup = parser.add_mutually_exclusive_group()
verbosity_parsergroup.add_argument("-v",
dest='verbosity',
action="count",
help=_("increase verbosity"),
default=0)
verbosity_parsergroup.add_argument(
"--silent",
action="store_true",
help=_("silent (no output to terminal)"))
parser.add_argument("--version",
action="version",
help=_("print version and exit"))
parser.add_argument("--log-file",
metavar='FILEPATH',
action="store",
help=_("overwrite path to log file"))
command_parsers = parser.add_subparsers(help="commands")
identify_parser = command_parsers.add_parser("identify",
add_help=False,
parents=[parser])
# load plugins for different server types
plugins_directories = [
pkg_resources.resource_filename("servertools", "plugins"),
os.path.join(application.user_data_dir, 'plugins'),
os.path.expanduser(
os.path.normpath(
'~/.{appname}/plugins'.format(appname=application.appname)))
]
sys.path.extend(plugins_directories)
plugin_manager = PluginManager(directories_list=plugins_directories,
plugin_info_ext='plugin')
plugin_manager.setCategoriesFilter({
"Server": Server,
})
plugin_manager.locatePlugins()
plugin_manager.collectPlugins()
# plugins to identify servers and fix problems that might happen on them
plugins = plugin_manager.getPluginsOfCategory(
'Server') # type: List[PluginInfo]
# choices for the command line flag `--look-for`
server_choices = [] # type: List[AnyStr]
# set valid values for --look-for from installed plugins
for plugin in plugins:
commandline_argument = plugin.name.lower().replace('server',
'').strip().replace(
' ', '-')
if len(commandline_argument) > 0:
server_choices.append(commandline_argument)
else:
raise ValueError(
_("Plugin {name} doesn't have a valid name!").format(
name=plugin.name))
identify_parser.add_argument("--look-for",
metavar="SERVER",
dest="look_for",
help=_("look for a specific server"),
choices=server_choices,
default='')
identify_parser.set_defaults(name="identify")
# fix_parser = command_parsers.add_parser('fix', help=_('apply fixes for a server'))
# parse arguments
args = parser.parse_args(args[1:])
# handle verbosity
verbosity = max(0, min(3, args.verbosity))
if args.silent:
verbosity = -1
logging_level = {
-1: logging.CRITICAL, # 50
0: logging.WARNING, # 30
1: logging.INFO, # 20
2: logging.INFO, # 20
3: logging.DEBUG # 10
}.get(verbosity, logging.WARNING)
stream_handler.setLevel(logging_level)
verbose = logger.info if verbosity > 1 else logger.debug
# start application
logger.debug(
_("{program} started at {started_at:%X} on Python {py_version} ({architecture})"
).format(program=script_name,
started_at=datetime.now(),
py_version=platform.python_version(),
architecture=platform.architecture()[0]))
logger.debug(
_("Verbosity Level at {verbosity}").format(verbosity=verbosity))
plugins_information = {}
for category in plugin_manager.getCategories():
plugins_information[category] = list()
for plugin in plugin_manager.getPluginsOfCategory(category):
plugins_information[category].append({
'name': plugin.name,
'author': plugin.author
})
# if verbosity > 1:
logger.info(_("Save logs to {path}").format(path=logging_path))
logger.info(_("Load configurations from {path}").format(path=config_path))
logger.info(
_('Look for plugins in:\n\t{path}').format(
path=',\n\t'.join(plugins_directories)))
verbose(
_('Include {amount} Plugin Categories.').format(
amount=len(plugin_manager.getCategories())))
logger.debug(', '.join(plugin_manager.getCategories()))
verbose(
_('{amount} Plugin(s) loaded.').format(
amount=len(plugins_information.keys())))
for category in plugins_information:
logger.debug('[{}]'.format(category))
for cat in plugins_information[category]:
logger.debug(
_('\t{name} (by {author})').format(name=cat['name'],
author=cat['author']))
logger.debug('')
for plugin in plugins:
logger.debug(_("Try to identify {name}").format(name=plugin.name))
command = plugin.name.lower().replace('server',
'').strip().replace(' ', '-'),
if args.look_for == command and plugin.plugin_object.identify():
logger.debug(
_("Successfully identified {name}").format(name=plugin.name))
print("{} - {}".format(plugin.details.get('Server Info', 'Host'),
plugin.details.get('Server Info',
'Slogan')))
break
else:
print(
_('Not {}').format(args.look_for)
if len(args.look_for) > 0 else _('Unknown platform'))
return 1
logger.debug("-" * 80)
return 0
class Statick:
"""Code analysis front-end."""
def __init__(self, user_paths: List[str]) -> None:
"""Initialize Statick."""
self.resources = Resources(user_paths)
self.manager = PluginManager()
self.manager.setPluginPlaces(self.resources.get_plugin_paths())
self.manager.setCategoriesFilter(
{
"Discovery": DiscoveryPlugin,
"Tool": ToolPlugin,
"Reporting": ReportingPlugin,
}
)
self.manager.collectPlugins()
self.discovery_plugins = {} # type: Dict
for plugin_info in self.manager.getPluginsOfCategory("Discovery"):
self.discovery_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
self.tool_plugins = {} # type: Dict
for plugin_info in self.manager.getPluginsOfCategory("Tool"):
self.tool_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
self.reporting_plugins = {} # type: Dict
for plugin_info in self.manager.getPluginsOfCategory("Reporting"):
self.reporting_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
self.config = None # type: Optional[Config]
self.exceptions = None # type: Optional[Exceptions]
def get_config(self, args: argparse.Namespace) -> None:
"""Get Statick configuration."""
config_filename = "config.yaml"
if args.config is not None:
config_filename = args.config
self.config = Config(self.resources.get_file(config_filename))
def get_exceptions(self, args: argparse.Namespace) -> None:
"""Get Statick exceptions."""
exceptions_filename = "exceptions.yaml"
if args.exceptions is not None:
exceptions_filename = args.exceptions
self.exceptions = Exceptions(self.resources.get_file(exceptions_filename))
def get_ignore_packages(self) -> List[str]:
"""Get packages to ignore during scan process."""
assert self.exceptions
return self.exceptions.get_ignore_packages()
def gather_args(self, args: argparse.Namespace) -> None:
"""Gather arguments."""
args.add_argument(
"--output-directory",
dest="output_directory",
type=str,
help="Directory to write output files to",
)
args.add_argument(
"--show-tool-output",
dest="show_tool_output",
action="store_true",
help="Show tool output",
)
args.add_argument(
"--check",
dest="check",
action="store_true",
help="Return the status. Return code 0 means there were no issues. \
Return code 1 means there were issues.",
)
args.add_argument(
"--config", dest="config", type=str, help="Name of config yaml file"
)
args.add_argument(
"--profile", dest="profile", type=str, help="Name of profile yaml file"
)
args.add_argument(
"--exceptions",
dest="exceptions",
type=str,
help="Name of exceptions yaml file",
)
args.add_argument(
"--force-tool-list",
dest="force_tool_list",
type=str,
help="Force only the given list of tools to run",
)
args.add_argument(
"--version",
action="version",
version="%(prog)s {version}".format(version=__version__),
)
args.add_argument(
"--mapping-file-suffix",
dest="mapping_file_suffix",
type=str,
help="Suffix to use when searching for CERT mapping files",
)
for _, plugin in list(self.discovery_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.tool_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.reporting_plugins.items()):
plugin.gather_args(args)
def get_level(self, path: str, args: argparse.Namespace) -> Optional[str]:
"""Get level to scan package at."""
path = os.path.abspath(path)
profile_filename = "profile.yaml"
if args.profile is not None:
profile_filename = args.profile
profile_resource = self.resources.get_file(profile_filename)
if profile_resource is None:
print("Could not find profile file {}!".format(profile_filename))
return None
try:
profile = Profile(profile_resource)
except OSError as ex:
# This isn't quite redundant with the profile_resource check: it's possible
# that something else triggers an OSError, like permissions.
print("Failed to access profile file {}: {}".format(profile_filename, ex))
return None
except ValueError as ex:
print("Profile file {} has errors: {}".format(profile_filename, ex))
return None
package = Package(os.path.basename(path), path)
level = profile.get_package_level(package)
return level
def run( # pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements
self, path: str, args: argparse.Namespace
) -> Tuple[Optional[Dict], bool]:
"""Run scan tools against targets on path."""
success = True
path = os.path.abspath(path)
if not os.path.exists(path):
print("No package found at {}!".format(path))
return None, False
package = Package(os.path.basename(path), path)
level = self.get_level(path, args) # type: Optional[str]
assert level
if not self.config or not self.config.has_level(level):
print("Can't find specified level {} in config!".format(level))
return None, False
orig_path = os.getcwd()
if args.output_directory:
if not os.path.isdir(args.output_directory):
print("Output directory not found at {}!".format(args.output_directory))
return None, False
output_dir = os.path.join(args.output_directory, package.name + "-" + level)
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
if not os.path.isdir(output_dir):
print("Unable to create output directory at {}!".format(output_dir))
return None, False
print("Writing output to: {}".format(output_dir))
os.chdir(output_dir)
print("------")
print(
"Scanning package {} ({}) at level {}".format(
package.name, package.path, level
)
)
issues = {} # type: Dict[str, List[Issue]]
ignore_packages = self.get_ignore_packages()
if package.name in ignore_packages:
print(
"Package {} is configured to be ignored by Statick.".format(
package.name
)
)
return issues, True
plugin_context = PluginContext(args, self.resources, self.config)
print("---Discovery---")
if not DiscoveryPlugin.file_command_exists():
print(
"file command isn't available, discovery plugins will be less effective"
)
discovery_plugins = self.config.get_enabled_discovery_plugins(level)
if not discovery_plugins:
discovery_plugins = list(self.discovery_plugins.keys())
for plugin_name in discovery_plugins:
if plugin_name not in self.discovery_plugins:
print("Can't find specified discovery plugin {}!".format(plugin_name))
return None, False
plugin = self.discovery_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
print("Running {} discovery plugin...".format(plugin.get_name()))
plugin.scan(package, level, self.exceptions)
print("{} discovery plugin done.".format(plugin.get_name()))
print("---Discovery---")
print("---Tools---")
enabled_plugins = self.config.get_enabled_tool_plugins(level)
plugins_to_run = copy.copy(enabled_plugins)
plugins_ran = [] # type: List[Any]
plugin_dependencies = [] # type: List[str]
while plugins_to_run:
plugin_name = plugins_to_run[0]
if plugin_name not in self.tool_plugins:
print("Can't find specified tool plugin {}!".format(plugin_name))
return None, False
if args.force_tool_list is not None:
force_tool_list = args.force_tool_list.split(",")
if (
plugin_name not in force_tool_list
and plugin_name not in plugin_dependencies
):
print("Skipping plugin not in force list {}!".format(plugin_name))
plugins_to_run.remove(plugin_name)
continue
plugin = self.tool_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
dependencies = plugin.get_tool_dependencies()
dependencies_met = True
for dependency_name in dependencies:
if dependency_name not in plugins_ran:
if dependency_name not in enabled_plugins:
print(
"Plugin {} depends on plugin {} which isn't "
"enabled!".format(plugin_name, dependency_name)
)
return None, False
plugin_dependencies.append(dependency_name)
plugins_to_run.remove(dependency_name)
plugins_to_run.insert(0, dependency_name)
dependencies_met = False
if not dependencies_met:
continue
print("Running {} tool plugin...".format(plugin.get_name()))
tool_issues = plugin.scan(package, level)
if tool_issues is not None:
issues[plugin_name] = tool_issues
print("{} tool plugin done.".format(plugin.get_name()))
else:
print("{} tool plugin failed".format(plugin.get_name()))
success = False
plugins_to_run.remove(plugin_name)
plugins_ran.append(plugin_name)
print("---Tools---")
if self.exceptions is not None:
issues = self.exceptions.filter_issues(package, issues)
os.chdir(orig_path)
print("---Reporting---")
reporting_plugins = self.config.get_enabled_reporting_plugins(level)
if not reporting_plugins:
reporting_plugins = self.reporting_plugins.keys() # type: ignore
for plugin_name in reporting_plugins:
if plugin_name not in self.reporting_plugins.keys():
print("Can't find specified reporting plugin {}!".format(plugin_name))
return None, False
plugin = self.reporting_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
print("Running {} reporting plugin...".format(plugin.get_name()))
plugin.report(package, issues, level)
print("{} reporting plugin done.".format(plugin.get_name()))
print("---Reporting---")
print("Done!")
return issues, success
def reloadPluginsModel():
if settings.DEBUG:
logging.getLogger('yapsy').setLevel(logging.WARNING)
# Are there any plugin objects? If not, add in the defaults
plugin_objects = Plugin.objects.all().count()
if plugin_objects == 0:
order = 0
PLUGIN_ORDER = [
'Activity', 'Status', 'OperatingSystem', 'MunkiVersion', 'Uptime',
'Memory', 'DiskSpace', 'PendingAppleUpdates',
'Pending3rdPartyUpdates', 'PuppetStatus'
]
for item in PLUGIN_ORDER:
order = order + 1
plugin = Plugin(name=item, order=order)
plugin.save()
# Build the manager
manager = PluginManager()
# Tell it the default place(s) where to find plugins
manager.setPluginPlaces([
settings.PLUGIN_DIR,
os.path.join(settings.PROJECT_DIR, 'server/plugins')
])
# Load all plugins
manager.collectPlugins()
found = []
for plugin in manager.getAllPlugins():
found.append(plugin.name)
# Get all of the plugin objects - if it's in here not installed, remove it
all_plugins = Plugin.objects.all()
for plugin in all_plugins:
if plugin.name not in found:
plugin.delete()
# And go over again to update the plugin's type
for dbplugin in all_plugins:
for plugin in manager.getAllPlugins():
if plugin.name == dbplugin.name:
try:
dbplugin.type = plugin.plugin_object.plugin_type()
except:
dbplugin.type = 'builtin'
try:
dbplugin.description = plugin.plugin_object.get_description(
)
except:
pass
dbplugin.save()
all_plugins = Report.objects.all()
for plugin in all_plugins:
if plugin.name not in found:
plugin.delete()
# And go over again to update the plugin's type
for dbplugin in all_plugins:
for plugin in manager.getAllPlugins():
if plugin.name == dbplugin.name:
try:
dbplugin.type = plugin.plugin_object.plugin_type()
except:
dbplugin.type = 'builtin'
try:
dbplugin.description = plugin.plugin_object.get_description(
)
except:
pass
dbplugin.save()
class TestDef(object):
def __init__(self):
# set aside storage for options and cmd line args
self.options = {}
self.args = []
# record if we have loaded the plugins or
# not - this is just a bozo check to ensure
# someone doesn't tell us to do it twice
self.loaded = False
# set aside a spot for a logger object, and
# note that it hasn't yet been defined
self.logger = None
self.modcmd = None
self.execmd = None
self.config = None
self.stages = None
self.tools = None
self.utilities = None
self.defaults = None
self.log = {}
def setOptions(self, args):
self.options = vars(args)
self.args = args
# if they want us to clear the scratch, then do so
if self.options['clean'] and os.path.isdir(self.options['scratchdir']):
shutil.rmtree(self.options['scratchdir'])
# setup the scratch directory
_mkdir_recursive(self.options['scratchdir'])
# private function to convert values
def __convert_value(self, opt, inval):
if opt is None or type(opt) is str:
return 0, inval
elif type(opt) is bool:
if type(inval) is bool:
return 0, inval
elif type(inval) is unicode:
return 0, int(inval)
elif type(inval) is str:
if inval.lower() in ['true', '1', 't', 'y', 'yes']:
return 0, True
else:
return 0, False
elif type(inval) is int:
if 0 == inval:
return 0, False
else:
return 0, True
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is int:
if type(inval) is int:
return 0, inval
elif type(inval) is str:
return 0, int(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
elif type(opt) is float:
if type(inval) is float:
return 0, inval
elif type(inval) is str or type(inval) is int:
return 0, float(inval)
else:
# unknown conversion required
print("Unknown conversion required for " + inval)
return 1, None
else:
return 1, None
# scan the key-value pairs obtained from the configuration
# parser and compare them with the options defined for a
# given plugin. Generate an output dictionary that contains
# the updated set of option values, the default value for
# any option that wasn't included in the configuration file,
# and return an error status plus output identifying any
# keys in the configuration file that are not supported
# by the list of options
#
# @log [INPUT]
# - a dictionary that will return the status plus
# stderr containing strings identifying any
# provided keyvals that don't have a corresponding
# supported option
# @options [INPUT]
# - a dictionary of tuples, each consisting of three
# entries:
# (a) the default value
# (b) data type
# (c) a help-like description
# @keyvals [INPUT]
# - a dictionary of key-value pairs obtained from
# the configuration parser
# @target [OUTPUT]
# - the resulting dictionary of key-value pairs
def parseOptions(self, log, options, keyvals, target):
# parse the incoming keyvals dictionary against the source
# options. If a source option isn't provided, then
# copy it across to the target.
opts = list(options.keys())
kvkeys = list(keyvals.keys())
for opt in opts:
found = False
for kvkey in kvkeys:
if kvkey == opt:
# they provided us with an update, so
# pass this value into the target - expand
# any provided lists
if keyvals[kvkey] is None:
continue
st, outval = self.__convert_value(options[opt][0],
keyvals[kvkey])
if 0 == st:
target[opt] = outval
else:
if len(keyvals[kvkey]) == 0:
# this indicates they do not want this option
found = True
break
if keyvals[kvkey][0][0] == "[":
# they provided a list - remove the brackets
val = keyvals[kvkey].replace('[', '')
val = val.replace(']', '')
# split the input to pickup sets of options
newvals = list(val)
# convert the values to specified type
i = 0
for val in newvals:
st, newvals[i] = self.__convert_value(
opt[0], val)
i = i + 1
target[opt] = newvals
else:
st, target[opt] = self.__convert_value(
opt[0], keyvals[kvkey])
found = True
break
if not found:
# they didn't provide this one, so
# transfer only the value across
target[opt] = options[opt][0]
# add in any default settings that have not
# been overridden - anything set by this input
# stage will override the default
if self.defaults is not None:
keys = self.defaults.options.keys()
for key in keys:
if key not in target:
target[key] = self.defaults.options[key][0]
# now go thru in the reverse direction to see
# if any keyvals they provided aren't supported
# as this would be an error
stderr = []
for kvkey in kvkeys:
# ignore some standard keys
if kvkey in ['section', 'plugin']:
continue
try:
if target[kvkey] is not None:
pass
except KeyError:
# some always need to be passed
if kvkey in ['parent', 'asis']:
target[kvkey] = keyvals[kvkey]
else:
stderr.append("Option " + kvkey + " is not supported")
if stderr:
# mark the log with an error status
log['status'] = 1
# pass the errors back
log['stderr'] = stderr
else:
log['status'] = 0
log['options'] = target
return
def loadPlugins(self, basedir, topdir):
if self.loaded:
print("Cannot load plugins multiple times")
exit(1)
self.loaded = True
# find the loader utility so we can bootstrap ourselves
try:
m = imp.load_source("LoadClasses",
os.path.join(basedir, "LoadClasses.py"))
except ImportError:
print(
"ERROR: unable to load LoadClasses that must contain the class loader object"
)
exit(1)
cls = getattr(m, "LoadClasses")
a = cls()
# setup the loader object
self.loader = a.__class__()
# Setup the array of directories we will search for plugins
# Note that we always look at the topdir location by default
plugindirs = []
plugindirs.append(topdir)
if self.options['plugindir']:
# could be a comma-delimited list, so split on commas
x = self.options['plugindir'].split(',')
for y in x:
# prepend so we always look at the given
# location first in case the user wants
# to "overload/replace" a default MTT
# class definition
plugindirs.insert(0, y)
# Traverse the plugin directory tree and add all
# the class definitions we can find
for dirPath in plugindirs:
try:
filez = os.listdir(dirPath)
for file in filez:
file = os.path.join(dirPath, file)
if os.path.isdir(file):
self.loader.load(file)
except:
if not self.options['ignoreloadpatherrs']:
print("Plugin directory", dirPath, "not found")
sys.exit(1)
# Build the stages plugin manager
self.stages = PluginManager()
# set the location
self.stages.setPluginPlaces(plugindirs)
# Get a list of all the categories - this corresponds to
# the MTT stages that have been defined. Note that we
# don't need to formally define the stages here - anyone
# can add a new stage, or delete an old one, by simply
# adding or removing a plugin directory.
self.stages.setCategoriesFilter(self.loader.stages)
# Load all plugins we find there
self.stages.collectPlugins()
# Build the tools plugin manager - tools differ from sections
# in that they are plugins we will use to execute the various
# sections. For example, the TestRun section clearly needs the
# ability to launch jobs. There are many ways to launch jobs
# depending on the environment, and sometimes several ways to
# start jobs even within one environment (e.g., mpirun vs
# direct launch).
self.tools = PluginManager()
# location is the same
self.tools.setPluginPlaces(plugindirs)
# Get the list of tools - not every tool will be capable
# of executing. For example, a tool that supports direct launch
# against a specific resource manager cannot be used on a
# system being managed by a different RM.
self.tools.setCategoriesFilter(self.loader.tools)
# Load all the tool plugins
self.tools.collectPlugins()
# Tool plugins are required to provide a function we can
# probe to determine if they are capable of operating - check
# those now and prune those tools that cannot support this
# environment
# Build the utilities plugins
self.utilities = PluginManager()
# set the location
self.utilities.setPluginPlaces(plugindirs)
# Get the list of available utilities.
self.utilities.setCategoriesFilter(self.loader.utilities)
# Load all the utility plugins
self.utilities.collectPlugins()
# since we use these all over the place, find the
# ExecuteCmd and ModuleCmd plugins and record them
availUtil = list(self.loader.utilities.keys())
for util in availUtil:
for pluginInfo in self.utilities.getPluginsOfCategory(util):
if "ExecuteCmd" == pluginInfo.plugin_object.print_name():
self.execmd = pluginInfo.plugin_object
elif "ModuleCmd" == pluginInfo.plugin_object.print_name():
self.modcmd = pluginInfo.plugin_object
# initialize this module
self.modcmd.setCommand(self.options)
if self.execmd is not None and self.modcmd is not None:
break
if self.execmd is None:
print("ExecuteCmd plugin was not found")
print("This is a basic capability required")
print("for MTT operations - cannot continue")
sys.exit(1)
# similarly, capture the highest priority defaults stage here
pri = -1
for pluginInfo in self.stages.getPluginsOfCategory("MTTDefaults"):
if pri < pluginInfo.plugin_object.priority():
self.defaults = pluginInfo.plugin_object
pri = pluginInfo.plugin_object.priority()
return
def printInfo(self):
# Print the available MTT sections out, if requested
if self.options['listsections']:
print("Supported MTT stages:")
# print them in the default order of execution
for stage in self.loader.stageOrder:
print(" " + stage)
exit(0)
# Print the detected plugins for a given stage
if self.options['listplugins']:
# if the list is '*', print the plugins for every stage
if self.options['listplugins'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['listplugins'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(
section):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid stage name " + section)
print()
exit(1)
# Print the options for a given plugin
if self.options['liststageoptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['liststageoptions'] == "*":
sections = self.loader.stageOrder
else:
sections = self.options['liststageoptions'].split(',')
print()
for section in sections:
print(section + ":")
try:
for pluginInfo in self.stages.getPluginsOfCategory(
section):
print(" " + pluginInfo.plugin_object.print_name() +
":")
pluginInfo.plugin_object.print_options(
self, " ")
except KeyError:
print(" Invalid stage name " + section)
print()
exit(1)
# Print the available MTT tools out, if requested
if self.options['listtools']:
print("Available MTT tools:")
availTools = list(self.loader.tools.keys())
for tool in availTools:
print(" " + tool)
exit(0)
# Print the detected tool plugins for a given tool type
if self.options['listtoolmodules']:
# if the list is '*', print the plugins for every type
if self.options['listtoolmodules'] == "*":
print()
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtoolmodules'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid tool type name", tool)
print()
exit(1)
# Print the options for a given plugin
if self.options['listtooloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listtooloptions'] == "*":
availTools = list(self.loader.tools.keys())
else:
availTools = self.options['listtooloptions'].split(',')
print()
for tool in availTools:
print(tool + ":")
try:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
print(" " + pluginInfo.plugin_object.print_name() +
":")
pluginInfo.plugin_object.print_options(
self, " ")
except KeyError:
print(" Invalid tool type name " + tool)
print()
exit(1)
# Print the available MTT utilities out, if requested
if self.options['listutils']:
print("Available MTT utilities:")
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
print(" " + util)
exit(0)
# Print the detected utility plugins for a given tool type
if self.options['listutilmodules']:
# if the list is '*', print the plugins for every type
if self.options['listutilmodules'] == "*":
print()
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutilitymodules'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(
util):
print(" " + pluginInfo.plugin_object.print_name())
except KeyError:
print(" Invalid utility type name")
print()
exit(1)
# Print the options for a given plugin
if self.options['listutiloptions']:
# if the list is '*', print the options for every stage/plugin
if self.options['listutiloptions'] == "*":
availUtils = list(self.loader.utilities.keys())
else:
availUtils = self.options['listutiloptions'].split(',')
print()
for util in availUtils:
print(util + ":")
try:
for pluginInfo in self.utilities.getPluginsOfCategory(
util):
print(" " + pluginInfo.plugin_object.print_name() +
":")
pluginInfo.plugin_object.print_options(
self, " ")
except KeyError:
print(" Invalid utility type name " + util)
print()
exit(1)
# if they asked for the version info, print it and exit
if self.options['version']:
for pluginInfo in self.tools.getPluginsOfCategory("Version"):
print("MTT Base: " + pluginInfo.plugin_object.getVersion())
print("MTT Client: " +
pluginInfo.plugin_object.getClientVersion())
sys.exit(0)
def openLogger(self):
# there must be a logger utility or we can't do
# anything useful
if not self.utilities.activatePluginByName("Logger", "Base"):
print("Required Logger plugin not found or could not be activated")
sys.exit(1)
# execute the provided test description
self.logger = self.utilities.getPluginByName("Logger",
"Base").plugin_object
self.logger.open(self)
return
def configTest(self):
# setup the configuration parser
self.config = configparser.ConfigParser()
# Set the config parser to make option names case sensitive.
self.config.optionxform = str
# log the list of files - note that the argument parser
# puts the input files in a list, with the first member
# being the list of input files
self.log['inifiles'] = self.args.ini_files[0]
# initialize the list of active sections
self.actives = []
# if they specified a list to execute, then use it
sections = []
if self.args.section:
sections = self.args.section.split(",")
skip = False
elif self.args.skipsections:
sections = self.args.skipsections.split(",")
skip = True
else:
sections = None
# cycle thru the input files
for testFile in self.log['inifiles']:
if not os.path.isfile(testFile):
print("Test description file", testFile, "not found!")
sys.exit(1)
self.config.read(self.log['inifiles'])
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
# if we are to filter the sections, then do so
takeus = True
if sections is not None:
found = False
for sec in sections:
if sec == section:
found = True
sections.remove(sec)
if skip:
takeus = False
break
if not found and not skip:
takeus = False
if takeus:
self.actives.append(section)
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
if sections is not None and 0 != len(sections) and not skip:
print(
"ERROR: sections were specified for execution and not found:",
sections)
sys.exit(1)
return
# Used with combinatorial executor, loads next .ini file to be run with the
# sequential executor
def configNewTest(self, file):
# clear the configuration parser
for section in self.config.sections():
self.config.remove_section(section)
# read in the file
self.config.read(file)
for section in self.config.sections():
if section.startswith("SKIP") or section.startswith("skip"):
# users often want to temporarily ignore a section
# of their test definition file, but don't want to
# remove it lest they forget what it did. So let
# them just mark the section as "skip" to be ignored
continue
if self.logger is not None:
self.logger.verbose_print("SECTION: " + section)
self.logger.verbose_print(self.config.items(section))
return
def executeTest(self):
if not self.loaded:
print("Plugins have not been loaded - cannot execute test")
exit(1)
if self.config is None:
print("No test definition file was parsed - cannot execute test")
exit(1)
if not self.tools.getPluginByName("sequential", "Executor"):
print("Specified executor sequential not found")
exit(1)
# activate the specified plugin
self.tools.activatePluginByName("sequential", "Executor")
# execute the provided test description
executor = self.tools.getPluginByName("sequential", "Executor")
executor.plugin_object.execute(self)
return
def executeCombinatorial(self):
if not self.tools.getPluginByName("combinatorial", "Executor"):
print("Specified executor combinatorial not found")
exit(1)
# activate the specified plugin
self.tools.activatePluginByName("combinatorial", "Executor")
# execute the provided test description
executor = self.tools.getPluginByName("combinatorial", "Executor")
executor.plugin_object.execute(self)
return
def printOptions(self, options):
# if the options are empty, report that
if not options:
lines = ["None"]
return lines
# create the list of options
opts = []
vals = list(options.keys())
for val in vals:
opts.append(val)
if options[val][0] is None:
opts.append("None")
elif isinstance(options[val][0], bool):
if options[val][0]:
opts.append("True")
else:
opts.append("False")
elif isinstance(options[val][0], list):
opts.append(" ".join(options[val][0]))
elif isinstance(options[val][0], int):
opts.append(str(options[val][0]))
else:
opts.append(options[val][0])
opts.append(options[val][1])
# print the options, their default value, and
# the help description in 3 column format
max1 = 0
max2 = 0
for i in range(0, len(opts), 3):
# we want all the columns to line up
# and left-justify, so first find out
# the max len of each of the first two
# column entries
if len(opts[i]) > max1:
max1 = len(opts[i])
if type(opts[i + 1]) is not str:
optout = str(opts[i + 1])
else:
optout = opts[i + 1]
if len(optout) > max2:
max2 = len(optout)
# provide some spacing
max1 = max1 + 4
max2 = max2 + 4
# cycle thru again, padding each entry to
# align the columns
lines = []
sp = " "
for i in range(0, len(opts), 3):
line = opts[i] + (max1 - len(opts[i])) * sp
if type(opts[i + 1]) is not str:
optout = str(opts[i + 1])
else:
optout = opts[i + 1]
line = line + optout + (max2 - len(optout)) * sp
# to make this more readable, we will wrap the line at
# 130 characters. First, see if the line is going to be
# too long
if 130 < (len(line) + len(opts[i + 2])):
# split the remaining column into individual words
words = opts[i + 2].split()
first = True
for word in words:
if (len(line) + len(word)) < 130:
if first:
line = line + word
first = False
else:
line = line + " " + word
else:
lines.append(line)
line = (max1 + max2) * sp + word
if 0 < len(line):
lines.append(line)
else:
# the line is fine - so just add the last piece
line = line + opts[i + 2]
# append the result
lines.append(line)
# add one blank line
lines.append("")
return lines
def selectPlugin(self, name, category):
if category == "stage":
try:
availStages = list(self.loader.stages.keys())
for stage in availStages:
for pluginInfo in self.stages.getPluginsOfCategory(stage):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "tool":
try:
availTools = list(self.loader.tools.keys())
for tool in availTools:
for pluginInfo in self.tools.getPluginsOfCategory(tool):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
elif category == "utility":
try:
availUtils = list(self.loader.utilities.keys())
for util in availUtils:
for pluginInfo in self.utilities.getPluginsOfCategory(
util):
if name == pluginInfo.plugin_object.print_name():
return pluginInfo.plugin_object
# didn't find it
return None
except:
return None
else:
print("Unrecognized category:", category)
return None
disabledChecks = {
# "SemiemptyFields" : {"bbmri-eric:ID:NO_HUNT", "bbmri-eric:ID:NO_Janus"}
}
pp = pprint.PrettyPrinter(indent=4)
class ExtendAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest) or []
items.extend(values)
setattr(namespace, self.dest, items)
simplePluginManager = PluginManager()
simplePluginManager.setPluginPlaces(["checks"])
simplePluginManager.collectPlugins()
pluginList = []
for pluginInfo in simplePluginManager.getAllPlugins():
pluginList.append(os.path.basename(pluginInfo.path))
remoteCheckList = ['emails', 'geocoding', 'URLs']
cachesList = ['directory', 'emails', 'geocoding']
parser = argparse.ArgumentParser()
parser.register('action', 'extend', ExtendAction)
parser.add_argument('-v',
'--verbose',
dest='verbose',
from bipy.core.constants import URLS, PATHS
from bipy.core.db.categories import SQLite
from yapsy.PluginManager import PluginManager
#----------- Create Plugin Mgr instance -------------------------
plugin_mgr = PluginManager(categories_filter={"SQLITE": SQLite})
#---------- Load Connection Mgr plugin for Warehouse ------------
plugin_mgr.setPluginPlaces([PATHS.CONNECTION_MANAGERS])
plugin_mgr.locatePlugins()
conns = plugin_mgr.loadPlugins()
conn_wh = conns[0].plugin_object
conn_wh
conn_wh.connect(URLS.TEST_DB)
#---------- Load Warehouse Browser plugin -----------------------
plugin_mgr.setPluginPlaces([PATHS.BROWSERS])
plugin_mgr.locatePlugins()
browsers = plugin_mgr.loadPlugins()
browser = browsers[0].plugin_object
browser
browser.connect(conn_wh)
#--------- Load Base Meta Generator plugin ----------------------
plugin_mgr.setPluginPlaces([PATHS.BASE_META_GEN])
plugin_mgr.locatePlugins()
base_meta_gen = plugin_mgr.loadPlugins()
meta_gen = base_meta_gen[0].plugin_object
meta_gen
#--------- Load Repository Manager plugin -----------------------
plugin_mgr.setPluginPlaces([PATHS.REPO_MGR])
plugin_mgr.locatePlugins()
repo_mgrs = plugin_mgr.loadPlugins()
repo_mgr = repo_mgrs[0].plugin_object
repo_mgr
class WUnpack(QtGui.QDialog):
def __init__(self, parent, plugin):
super(WUnpack, self).__init__(parent)
self.parent = parent
self.plugin = plugin
self.oshow = super(WUnpack, self).show
root = os.path.dirname(sys.argv[0])
self.ui = PyQt4.uic.loadUi(os.path.join(root, 'unpack.ui'),
baseinstance=self)
self.ui.setWindowTitle('Decrypt/Encrypt')
self.manager = PluginManager(
categories_filter={"UnpackPlugin": DecryptPlugin})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'unpack')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
self.Plugins = {}
Formats = []
for plugin in self.manager.getPluginsOfCategory("UnpackPlugin"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.init(self.parent.dataModel, self.parent.viewMode):
self.Plugins[plugin.name] = po
#self.ui.horizontalLayout.addWidget(po.getUI())
print '[+] ' + plugin.name
self.ui.listWidget.addItem(plugin.name)
#Formats.append(po)
self.ui.listWidget.currentItemChanged.connect(self.item_clicked)
self.ui.listWidget.setCurrentRow(0)
self.ui.connect(self.ui.proceed, PyQt4.QtCore.SIGNAL("clicked()"),
self.handleProceed)
self.initUI()
def handleProceed(self):
item = str(self.ui.listWidget.currentItem().text())
self.Plugins[item].proceed()
#self.parent.update()
self.parent.viewMode.draw(refresh=True)
self.parent.update()
def item_clicked(self, current, previous):
#item = str(self.ui.listWidget.currentItem().text())
item = str(current.text())
if previous:
x = self.ui.horizontalLayout.takeAt(0)
while x:
x.widget().setParent(None)
x = self.ui.horizontalLayout.takeAt(0)
prev = str(previous.text())
#print prev
#self.ui.horizontalLayout.removeWidget(self.Plugins[prev].getUI())
if item:
#print item
po = self.Plugins[item]
self.ui.horizontalLayout.addWidget(po.getUI())
def show(self):
# TODO: remember position? resize plugin windows when parent resize?
pwidth = self.parent.parent.size().width()
pheight = self.parent.parent.size().height()
width = self.ui.size().width() + 15
height = self.ui.size().height() + 15
self.setGeometry(pwidth - width - 15, pheight - height, width, height)
self.setFixedSize(width, height)
self.oshow()
def initUI(self):
self.setSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed)
shortcut = QtGui.QShortcut(QtGui.QKeySequence("F4"), self, self.close,
self.close)
#QtCore.QObject.connect(self.ui.ok, QtCore.SIGNAL('clicked()'), self.onClicked)
def onClicked(self):
dataModel = self.parent.dataModel
self.close()
class WeaponSystem(rpyc.Service):
'''
RPC Services: This is the code that does the actual password cracking
and returns the results to orbital control. Currently only supports
cracking using rainbow tables (RCrackPy)
'''
is_initialized = False
mutex = Lock()
is_busy = False
job_id = None
def initialize(self):
''' Initializes variables, this should only be called once '''
logging.info("Weapon system initializing ...")
self.plugin_manager = PluginManager()
self.plugin_manager.setPluginPlaces(["plugins/"])
self.plugin_manager.setCategoriesFilter(FILTERS)
self.plugin_manager.collectPlugins()
self.plugins = {}
logging.info("Loaded %d plugin(s)" %
len(self.plugin_manager.getAllPlugins()))
self.__cpu__()
logging.info("Weapon system online, good hunting.")
@atomic
def on_connect(self):
''' Called when successfully connected '''
if not self.is_initialized:
self.initialize()
self.is_initialized = True
logging.info("Uplink to orbital control active")
def on_disconnect(self):
''' Called if the connection is lost/disconnected '''
logging.info("Disconnected from orbital command server.")
def __cpu__(self):
''' Detects the number of CPU cores on a system (including virtual cores) '''
if cpu_count is not None:
try:
self.cpu_cores = cpu_count()
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except NotImplementedError:
logging.error(
"Could not detect number of processors; assuming 1")
self.cpu_cores = 1
else:
try:
self.cpu_cores = int(sysconf("SC_NPROCESSORS_CONF"))
logging.info("Detected %d CPU core(s)" % self.cpu_cores)
except ValueError:
logging.error(
"Could not detect number of processors; assuming 1")
self.cpu_cores = 1
############################ [ EXPOSED METHODS ] ############################
@atomic
def exposed_crack(self, plugin_name, job_id, hashes, **kwargs):
''' Exposes plugins calls '''
assert plugin_name in self.plugins
self.is_busy = True
self.job_id = job_id
self.plugin_manager.activatePluginByName(plugin_name)
plugin = self.plugin_manager.getPluginByName(plugin_name)
results = plugin.execute(hashes, **kwargs)
self.plugin_manager.deactivatePluginByName(plugin_name)
self.job_id = None
self.is_busy = False
return results
def exposed_get_plugin_names(self):
''' Returns what algorithms can be cracked '''
logging.info("Method called: exposed_get_capabilities")
plugins = self.plugin_manager.getAllPlugins()
return [plugin.name for plugin in plugins]
def exposed_get_categories(self):
''' Return categories for which we have plugins '''
categories = []
for category in self.plugin_manager.getCategories():
if 0 < len(self.plugin_manager.getPluginsOfCategory(category)):
categories.append(category)
return categories
def exposed_get_category_plugins(self, category):
''' Get plugin names for a category '''
plugins = self.plugin_manager.getPluginsOfCategory(category)
return [plugin.name for plugin in plugins]
def exposed_get_plugin_details(self, category, plugin_name):
''' Get plugin based on name details '''
plugin = self.plugin_manager.getPluginByName(plugin_name, category)
info = {'name': plugin.name}
info['author'] = plugin.details.get('Documentation', 'author')
info['website'] = plugin.details.get('Documentation', 'website')
info['version'] = plugin.details.get('Documentation', 'version')
info['description'] = plugin.details.get('Documentation',
'description')
info['copyright'] = plugin.details.get('Documentation', 'copyright')
info['precomputation'] = plugin.details.getboolean(
'Core', 'precomputation')
return info
def exposed_ping(self):
''' Returns a pong message '''
return "PONG"
def exposed_is_busy(self):
''' Returns True/False if the current system is busy (thread safe) '''
return self.is_busy
def exposed_current_job_id(self):
''' Returns the current job id (thread safe) '''
return self.job_id
def exposed_cpu_count(self):
''' Returns the number of detected cpu cores '''
return self.cpu_cores
def main():
#Find and load plugins
pm = PluginManager()
libpath = '%s/OpenMesher/plugins' % (get_python_lib())
pm.setPluginPlaces([
"/usr/share/openmesher/plugins", "~/.openmesher/plugins",
"./OpenMesher/plugins", "./plugins", libpath
])
pm.setPluginInfoExtension('plugin')
pm.setCategoriesFilter({
'config': IOpenMesherConfigPlugin,
'package': IOpenMesherPackagePlugin,
'deploy': IOpenMesherDeployPlugin,
})
pm.collectPlugins()
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Configure, package, and deploy an OpenVPN mesh")
parser.add_argument('-r',
'--router',
action='append',
help='Adds a router that can be a client and server')
parser.add_argument(
'-s',
'--server',
action='append',
help='Adds a router that can only act as a server, not a client.')
parser.add_argument(
'-c',
'--client',
action='append',
help=
'Adds a router than can only act as a client. For example, a router that is behind NAT and not accessible by a public IP'
)
#BUG: Stupid argparse appends your switches to the default.
#parser.add_argument('-n', '--network', action='append', default=['10.99.99.0/24'])
parser.add_argument('-n', '--network', action='append', required=True)
portgroup = parser.add_mutually_exclusive_group()
portgroup.add_argument('-p',
'--ports',
action='append',
default=['7000-7999'])
portgroup.add_argument('-a', '--random', action='store_true')
parser.add_argument(
'-v',
'--verbose',
action='append_const',
const='verbose',
help='Specify multiple times to make things more verbose')
parser.add_argument('--version', action='version', version='v0.6.4')
pluginargsgroup = parser.add_argument_group('plugins')
for plugin in pm.getAllPlugins():
plugin.plugin_object.setupargs(pluginargsgroup)
arg = parser.parse_args()
for plugin in pm.getAllPlugins():
if plugin.plugin_object.__class__.__name__.lower() in arg:
if eval('arg.%s' %
(plugin.plugin_object.__class__.__name__.lower())):
logging.debug('Plugin enabled: %s' % (plugin.name))
pm.activatePluginByName(plugin.name)
else:
logging.debug('Plugin disabled: %s' % (plugin.name))
pm.deactivatePluginByName(plugin.name)
else:
logging.debug('Plugin disabled: %s' % (plugin.name))
pm.deactivatePluginByName(plugin.name)
l = logging.getLogger()
if arg.verbose:
if len(arg.verbose) == 1:
l.setLevel(logging.INFO)
if len(arg.verbose) >= 2:
l.setLevel(logging.DEBUG)
# Call activate() on all plugins so they prep themselves for use
for plugin in pm.getAllPlugins():
if eval('arg.%s' % (plugin.plugin_object.__class__.__name__.lower())):
logging.info('Enabled plugin: %s' % (plugin.name))
pm.activatePluginByName(plugin.name)
plugin.plugin_object.activate()
plugin.plugin_object._enabled = True
if len(arg.ports) > 1:
arg.ports.reverse()
arg.ports.pop()
port_list = []
if arg.random:
numdevs = 0
if arg.router:
numdevs += len(arg.router)
if arg.server:
numdevs += len(arg.server)
if arg.client:
numdevs += len(arg.client)
ports_needed = numdevs * (numdevs - 1) / 2
for i in range(0, ports_needed):
port_list.append(random.randrange(1025, 32767))
try:
if not arg.random:
# If we're not using random ports, pull whatever is in arg.ports
for portrange in arg.ports:
portstart, portstop = portrange.split('-')
port_list += range(int(portstart), int(portstop))
except ValueError as e:
print 'Invalid port range: %s' % (portrange)
raise ValueError('Invalid port range: %s' % (portrange))
linkmesh = create_link_mesh(routers=arg.router,
servers=arg.server,
clients=arg.client)
m = Mesh(linkmesh, port_list, arg.network)
files = None
# Run through config plugins
configPlugins = []
for plugin in pm.getPluginsOfCategory('config'):
if plugin.plugin_object._enabled:
plugin.plugin_object.process(m, arg)
configPlugins.append(plugin.plugin_object)
if files:
files = nested_dict_merge(files, plugin.plugin_object.files())
else:
files = plugin.plugin_object.files()
#Grab list of folders that need to be in the package root
includedirs = []
for f in files:
for fldr in files[f]:
rootfldr = fldr.split('/')[1]
if rootfldr not in includedirs:
includedirs.append(rootfldr)
logging.debug('The following folders will be packaged: %s' % (includedirs))
# Run through packaging plugins
packagePlugins = []
for plugin in pm.getPluginsOfCategory('package'):
if plugin.plugin_object._enabled:
#BUG: Services to restart may not necessarily be the same name as their config dir...
plugin.plugin_object.process(m,
include_dirs=includedirs,
restart_services=includedirs,
configPlugins=configPlugins,
cliargs=arg)
packagePlugins.append(plugin.plugin_object)
# Run through deployment plugins
for plugin in pm.getPluginsOfCategory('deploy'):
if plugin.plugin_object._enabled:
try:
plugin.plugin_object.deploy(packagePlugins=packagePlugins,
cliargs=arg,
stoponfailure=False)
except Exception as e:
print "Unable to deploy due to error: %s" % (e)
logging.info('OpenMesher run complete')
#!/usr/bin/env python2.7
import argparse # new in Python2.7
import os
import time
import string
import atexit
import threading
import logging
import sys
logging.basicConfig(level=logging.ERROR)
from yapsy.PluginManager import PluginManager
# Load the plugins from the plugin directory.
manager = PluginManager()
if __name__ == '__main__':
print ("------------user.py-------------")
parser = argparse.ArgumentParser(description="OpenBCI 'user'")
parser.add_argument('--board', default="cyton",
help="Choose between [cyton] and [ganglion] boards.")
parser.add_argument('-l', '--list', action='store_true',
help="List available plugins.")
parser.add_argument('-i', '--info', metavar='PLUGIN',
help="Show more information about a plugin.")
parser.add_argument('-p', '--port',
help="Port to connect to OpenBCI Dongle " +
"( ex /dev/ttyUSB0 or /dev/tty.usbserial-* ) or AUTO to attempt auto-dection.")
parser.set_defaults(port="AUTO")
def __init__(self, parent, source):
super(binWidget, self).__init__()
Observable.__init__(self)
self.parent = parent
# offset for text window
#self.data = mapped
self.dataOffset = 0
self.dataModel = source
self.cursor = Cursor(0, 0)
# self.multipleViewModes = [BinViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self),
# HexViewMode(self.size().width(), self.size().height(), self.dataModel, self.cursor, self)]
logging.basicConfig(level=logging.ERROR)
self.manager = PluginManager(
categories_filter={"FileFormat": FileFormat})
root = os.path.dirname(sys.argv[0])
self.manager.setPluginPlaces([os.path.join(root, 'plugins', 'format')])
#self.manager.setPluginPlaces(["plugins"])
# Load plugins
self.manager.locatePlugins()
self.manager.loadPlugins()
Formats = []
for plugin in self.manager.getPluginsOfCategory("FileFormat"):
# plugin.plugin_object is an instance of the plugin
po = plugin.plugin_object
if po.recognize(self.dataModel):
print '[+] ' + po.name
Formats.append(po)
# sort plugins by priority
Formats = sorted(Formats, key=lambda x: x.priority, reverse=True)
po = Formats[0]
print 'Choosed plugin: ' + po.name
#print QtGui.QFontDatabase.addApplicationFont(os.path.join('terminus-ttf-4.39', 'TerminusTTF-4.39.ttf'))
self.multipleViewModes = [
BinViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po),
HexViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po),
DisasmViewMode(self.size().width(),
self.size().height(),
self.dataModel,
self.cursor,
self,
plugin=po)
]
self.viewMode = self.multipleViewModes[0]
self.textDecorator = TextDecorator(self.viewMode)
self.viewMode.setTransformationEngine(self.textDecorator)
self.multipleViewModes[1].setTransformationEngine(self.textDecorator)
self.Banners = Banners()
#self.Banners.add(BottomBanner(self.dataModel, self.viewMode))
# self.Banners.add(TopBanner(self.dataModel, self.viewMode))
#self.Banners.add(self.banner)
# self.filebanner = FileAddrBanner(self.dataModel, self.viewMode)
#self.filebanner = PEBanner(self.dataModel, self.viewMode)
#self.Banners.add(PEBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
#self.Banners.add(FileAddrBanner(self.dataModel, self.viewMode))
# self.offsetWindow_h = self.filebanner.getDesiredGeometry()[0] + 25
self.offsetWindow_h = 0
self.offsetWindow_v = 0
self.searchable = Searchable(self.dataModel, self.viewMode)
self.initUI()
[po.init(viewMode, parent=self) for viewMode in self.multipleViewModes]
for banner in po.getBanners():
self.Banners.add(banner)
po.registerShortcuts(self)
self.po = po
#self.scrolled = QtCore.pyqtSignal(int, name='scroll')
#self.scrolled.connect(self.scroll_from_outside)
self.searchWindow = SearchWindow(self, None, self.searchable)
self.addHandler(self.po)
self.addHandler(self.searchable)
self.addHandler(self.Banners)
self.notify(self.viewMode)