Beispiel #1
0
    def setUpBackend():
        backendManager = PluginManager(plugin_info_ext='relo')
        backendManager.setPluginPlaces(["relo/core/backend"])
        backendManager.locatePlugins()
        backendManager.loadPlugins("<class 'relo.core.interfaces.Backend'>", ['redis'])

        for plugin in backendManager.getAllPlugins():
            backendManager.activatePluginByName(plugin.name)

        for plugin in backendManager.getAllPlugins():
            if plugin.name == conf.readConfig('core.index'):
                db = plugin.plugin_object
                db.init()
                return db
Beispiel #2
0
class IndexSearch(CustomSearch):
    def __init__(self, directory, key):
        self.directory = directory
        self.key = key
        self.setUpBackend()
        self.results = []
    def setUpBackend(self):
        self.backendManager = PluginManager(plugin_info_ext='relo')
        self.backendManager.setPluginPlaces(["relo/core/backend"])
        self.backendManager.locatePlugins()
        self.backendManager.loadPlugins("<class 'relo.core.interfaces.Backend'>", ['redis'])

        for plugin in self.backendManager.getAllPlugins():
            self.backendManager.activatePluginByName(plugin.name)

        for plugin in self.backendManager.getAllPlugins():
            if plugin.name == conf.readConfig('core.index'):
                self.db = plugin.plugin_object
                self.db.init()
    def loadFiles(self):
        return self.db.getSet(config.REDIS_KEY_DOCUMENTS % {"project_id": self.directory})
    def loadMetaphones(self):
        return self.db.getSet(config.REDIS_KEY_METAPHONES % {"project_id": self.directory})
    def nameSearch(self):
        files = self.loadFiles()
        for file in files:
            item = os.path.basename(file)
            if not item.find(self.key) == -1:
                self.results.append(file)
    def contentSearch(self):
        metaphones = self.loadMetaphones() ## do part of search in redis itself in the future so we dont hae to laod everything, keys *mp*
        # or just try to load metaphone and see what you get
        key_mps = double_metaphone(unicode(self.key, errors='ignore'))
        for mp in metaphones: # filter, map and reduce should help here in future
            for key_mp in key_mps:
                if key_mp == mp:
                    self.results.extend(self.db.getSet(config.REDIS_KEY_METAPHONE % {"project_id": self.directory, "metaphone":key_mp}))
    def printResult(self):
        if len(self.results) == 0:
            logger.item("No results found")
        else:
            for item in self.results:
                logger.item(item)
Beispiel #3
0
class Stats(object):
    """
    Main Stats class
    """
    def __init__(self, directory, module, hidden=False):
        self.directory = directory
        self.module = module

        self.manager = PluginManager(plugin_info_ext='relo')
        self.manager.setPluginPlaces(["relo/local/statistics"])
        self.manager.locatePlugins()
        self.manager.loadPlugins("<class 'relo.core.interfaces.Statistics'>", ['disk'])
        print "Plugin init done"
    def check(self):
        for plugin in self.manager.getAllPlugins():
            if self.module.upper() == plugin.name:
                return True
        return False
    def list(self):
        """
        Lists all available modules.
        """
        print "Available Statistic plugins:"

        for plugin in self.manager.getAllPlugins():
            print "    - " + plugin.name
    def go(self):
        for plugin in self.manager.getAllPlugins():
            self.manager.activatePluginByName(plugin.name)
            print plugin.name
            if plugin.name == self.module.upper():
                print "Found module"
                self.module = plugin.plugin_object
        self.module.init(self.directory)
        self.module.execute()
        self.module.end()
Beispiel #4
0
class Search(CustomSearch):
    def __init__(self, info=False, debug=False, all=False, hidden=False, filelog=False, content=False, recursive=False,
                 doctype=None, directory='./', key=''):
        """
        Main Relo class
        """
        self.name = "relo"
        self.info = info
        self.debug = debug
        self.filelog = filelog
        self.all = all
        self.hidden = hidden
        self.recursive = recursive
        self.doctype = doctype
        self.dir = directory
        self.key = key

        if content:
            self.type = "content Search"
        else:
            self.type = "fileName Search"

        logger.info("Relo: version %s" % core.config.get_version())
        if self.info:
            logger.info("Mode: Info")
        elif self.debug:
            logger.info("Mode: Debug")
        else:
            logger.info("Mode: Normal")
        logger.info("All Files: " + str(bool(self.all)))
        if self.doctype is None:
            logger.info("Special DocType: None")
        else:
            logger.info("Special DocType: " + self.doctype)
        logger.info("Hidden Files: " + str(bool(self.hidden)))
        logger.info("Recursive: " + str(bool(self.recursive)))
        logger.info("Search Type: " + self.type)
        logger.info("Directory: " + self.dir)
        logger.info("Searching for: " + self.key)

        self.filteredList = []
        self.extList = []
        self.total_size = 0
        self.fileList = []

        logger.head("Relo Search | " + self.dir + " | '" + self.key + "'")

        #Main Progress Bar
        self.mainWidgets = ['Searching: ', Percentage(), ' ', Bar('>'),
                   ' ', RotatingMarker()]
    def list(self):
        widgets = ["Listing directory content... ",
                   Bar('>'), ' ', RotatingMarker(), ' ', ReverseBar('<')]
        pbar = ProgressBar(widgets=widgets, maxval=100).start()
        pbar.update(0)
        time.sleep(0.5)
        if self.recursive:
            logger.debug("Listing directory content recursively...")
            pbar.update(20)
            time.sleep(1)
            self.total_size, self.fileList = util.recursiveListFiles(self.dir, self.hidden)
        else:
            logger.debug("Listing directory content...")
            pbar.update(20)
            time.sleep(1)
            self.total_size, self.fileList = util.listFiles(self.dir, self.hidden)
        pbar.update(100)
        pbar.finish()
        logger.debug("Supported File Types: " + repr(doctype.__all__))
        logger.info("Size of directory - %d" % self.total_size)

    def filter(self):
        if self.all:
            logger.info("No filter applied")
            self.filteredList = self.fileList
        elif not self.doctype is None:
            logger.log("Selecting DocType files...")
            self.filteredList = util.filterDocType(self.fileList, self.doctype)
        else:
            logger.log("Filtering file types...")
            self.filteredList = util.filterList(self.fileList)
        for itempath in self.filteredList:
            item = util.getFileType(itempath)
            if item not in self.extList:
                self.extList.append(item)

    def start(self):
        if len(self.filteredList) == 0:
            logger.error("No files in range.")
            return 0
        self.pbar = ProgressBar(widgets=self.mainWidgets, maxval=len(self.filteredList)).start()
        if 'content' in self.type:
            self.startContent()
        else:
            self.startName()
        self.pbar.finish()
        self.printResults()
    def startName(self):
        fileNameSearch(self.fileList, self.key)
    def startContent(self):
        self.setUpDocType(self.extList)
        i = 0
        self.results = {}
        for item in self.filteredList:
            self.pbar.update(i)
            content = self.load(item)
            self.search(content, item)
            i += 1
            self.pbar.update(i)
    def search(self, string, item):
        found = []
        for m in re.finditer(self.key, string):
            found.append(str(m.start()))
        self.results[item] = found
    def printResults(self):
        logger.head("Search Results | " + self.dir + " | '" + self.key + "'")
        for key, value in self.results.iteritems():
            if not value:
                continue
            logger.item(key)
            logger.subitem(repr(value))
    def setUpDocType(self, extList):
        self.extList = extList

        self.manager = PluginManager(plugin_info_ext='relo')
        self.manager.setPluginPlaces(["relo/core/doctype"])

        self.numPlugins = self.manager.locatePlugins()
        self.manager.loadPlugins("<class 'relo.core.interfaces.DocType'>", extList=extList)

        pluginList = []
        for plugin in self.manager.getAllPlugins():
            self.manager.activatePluginByName(plugin.name)
            pluginList.append(plugin.plugin_object.meta())
            #print pluginList

    def load(self, itempath):
        for plugin in self.manager.getAllPlugins():
            if plugin.name == util.getFileType(itempath).upper():
                return plugin.plugin_object.load(itempath)
        plugin = self.manager.getPluginByName("DEFAULT")
        return plugin.plugin_object.load(itempath)