def guess(inputstring, memorylist = []) -> set: from pydsl.Memory.Search.Searcher import MemorySearcher from pydsl.Memory.Storage.Directory.Grammar import GrammarDirStorage from pydsl.Memory.Storage.Dict import FileTypeDictStorage if not memorylist: try: dirname = resource_filename(Requirement.parse("pydsl_contrib"),"") except DistributionNotFound: pass else: memorylist.append(GrammarDirStorage(dirname + "grammar/")) memorylist.append(FileTypeDictStorage(dirname + "/dict/filetype.dict")) searcher = MemorySearcher([x.indexer() for x in memorylist]) result = set() for summary in searcher.search(): typ = None name = None try: for mem in memorylist: if summary["identifier"] in mem: name = summary["identifier"] typ = mem.load(name) break if typ.check(inputstring): result.add(str(name)) except TypeError: continue return result
def available_transforms(self): if self.functionlist == None: from pydsl.Memory.Search.Searcher import MemorySearcher from pydsl.Memory.Search.Indexer import Indexer from pydsl.Config import GLOBALCONFIG searcher = MemorySearcher([Indexer(x) for x in GLOBALCONFIG.memorylist]) resultlist = [] for element in self.guess(): resultlist += searcher.search({"input":{"$part":{"input":element}}}) self.functionlist = resultlist return self.functionlist
def available_transforms(content): from pydsl.Memory.Search.Searcher import MemorySearcher from pydsl.Memory.Search.Indexer import Indexer from pydsl.Config import GLOBALCONFIG searcher = MemorySearcher([Indexer(x) for x in GLOBALCONFIG.memorylist]) resultlist = [] if content.grammar: grammarlist = [content.grammar] else: grammarlist = content.available_grammars() for element in grammarlist: resultlist += searcher.search({"input":{"$part":{"input":element}}}) return [x['identifier'] for x in resultlist]
def __init__(self, dirpath:str, allowedextensions:list = []): self.identifier = dirpath from pydsl.Config import GLOBALCONFIG resultdirpath = [] self._allowedextensions = allowedextensions from pydsl.Memory.Search.Searcher import MemorySearcher self._searcher = MemorySearcher(self)
def __init__(self, dirpath, allowedextensions=(".py", ".bnf", ".re", ".board")): Memory.__init__(self) self.path = dirpath self._allowedextensions = allowedextensions from pydsl.Memory.Search.Searcher import MemorySearcher self._searcher = MemorySearcher(self)
class Guesser(object): def __init__(self, memorylist = None): from pydsl.Memory.Search.Searcher import MemorySearcher from pydsl.Memory.Directory import DirStorage if not memorylist: dirname = resource_filename("pydsl.contrib","") memorylist = [DirStorage(dirname + "/grammar/"), EncodingStorage(dirname + "/list/encoding.py")] self.memorylist = memorylist self.searcher = MemorySearcher([x.indexer() for x in memorylist]) def __call__(self, inputstring): #-> set: result = set() for summary in self.searcher.search(): typ = None name = None try: for mem in self.memorylist: if summary["identifier"] in mem: name = summary["identifier"] try: typ = mem.load(name) except: LOG.exception("Error while loading memory %s" % name) continue break else: continue # not found checker = load_checker(typ) if checker.check(inputstring): result.add(str(name)) except TypeError: continue return result
def __init__(self, memorylist = None): from pydsl.Memory.Search.Searcher import MemorySearcher from pydsl.Memory.Directory import DirStorage if not memorylist: dirname = resource_filename("pydsl.contrib","") memorylist = [DirStorage(dirname + "/grammar/"), EncodingStorage(dirname + "/list/encoding.py")] self.memorylist = memorylist self.searcher = MemorySearcher([x.indexer() for x in memorylist])
class DirStorage(Storage, metaclass = ABCMeta): """A collection of elements stored inside a directory""" def __init__(self, dirpath:str, allowedextensions:list = []): self.identifier = dirpath from pydsl.Config import GLOBALCONFIG resultdirpath = [] self._allowedextensions = allowedextensions from pydsl.Memory.Search.Searcher import MemorySearcher self._searcher = MemorySearcher(self) def __iter__(self): self.index = 0 self.cache = [] for filename in self.all_files(): try: self.cache.append(self.summary_from_filename(filename)) except AttributeError: pass return self def __next__(self): try: result = self.cache[self.index] except IndexError: raise StopIteration self.index += 1 return result def summary_from_filename(self, modulepath): (_, _, fileBaseName, ext) = getFileTuple(modulepath) result = None if _isRELFileName(modulepath): result = {"iclass":"re","identifier":fileBaseName, "filepath":modulepath} elif _isGDLFileName(modulepath): result = {"iclass":"BNFGrammar","identifier":fileBaseName, "filepath":modulepath} elif (modulepath).endswith(".board"): from pydsl.Function.Transformer.Board import Board result = {"iclass":"Board", "identifier":fileBaseName, "filepath":modulepath, "ancestors":Board.ancestors()} else: import imp moduleobject = imp.load_source(fileBaseName, modulepath) result = {"identifier":fileBaseName, "iclass":moduleobject.iclass, "filepath":modulepath} if hasattr(moduleobject, "title"): result["title"] = InmutableDict(moduleobject.title) if hasattr(moduleobject, "description"): result["description"] = InmutableDict(moduleobject.description) return InmutableDict(result) def _search_files(self, string: str, exact:bool = True): """Search through library""" for filename in self.all_files(): if exact: (_, _, fileBaseName, _) = getFileTuple(filename) if fileBaseName.strip() == string.strip(): yield filename else: if string.lower() in filename.lower(): yield filename def all_files(self): import glob if self._allowedextensions: for extension in self._allowedextensions: tmpresult = [] searchstring = self.identifier + "*" + extension tmpresult = glob.glob(searchstring) for result in tmpresult: if result.endswith("__init__.py"): continue yield result else: searchstring = self.identifier + "*" for result in glob.glob(searchstring): yield result def all_names(self): """Generates all Static Ids""" for fullname in self.all_files(): (_, _, fileBaseName, fileExtension) = getFileTuple(fullname) if self._allowedextensions and fileExtension not in self._allowedextensions: continue yield fileBaseName.split(".")[0] def _load_module_from_library(self, identifier): try: import imp moduleobject = imp.load_source(identifier, self.identifier + "/" + identifier + ".py") except (ImportError, IOError): pass else: return moduleobject raise ImportError def load(self, name, **kwargs): resultlist = self._searcher.search(name) if(len(resultlist) > 1): LOG.error("Found two or more matches, FIXME: processing the first, should raise exception") if len(resultlist) == 0: raise KeyError(self.__class__.__name__ + name) return load_python_file(list(resultlist)[0]["filepath"], **kwargs) def __contains__(self, key): return key in self.all_names() def provided_iclasses(self) -> set: return set([x['identifier'] for x in self])
if __name__ == "__main__": import argparse from pydsl.Config import VERSION, GLOBALCONFIG TUSAGE = "usage: %(prog)s [options] query" PARSER = argparse.ArgumentParser(usage = TUSAGE) PARSER.add_argument("search", metavar="search", help="Query to search") PARSER.add_argument('--version', action='version', version = VERSION) PARSER.add_argument('--filter', dest='myfilter',nargs='?', default=None, help="comma separated field list") PARSER.add_argument('-o', dest='outputformat',nargs='?', choices=["str","json"], default="str", help="output formats") ARGS = PARSER.parse_args() import sys from pydsl.Memory.Search.Searcher import MemorySearcher from pydsl.Memory.Search.Indexer import Indexer if ARGS.search: searcher = MemorySearcher([Indexer(x) for x in GLOBALCONFIG.memorylist]) myfilter = None if ARGS.myfilter: myfilter = ARGS.myfilter.split(',') if ARGS.outputformat == "str": print(search_pp(searcher.search(ARGS.search), myfilter)) elif ARGS.outputformat == "json": import json print(json.dumps(list(filterset(searcher.search(ARGS.search), myfilter)))) sys.exit(0) else: print(TUSAGE) sys.exit(0) sys.exit(0)
def __dir__(cls): from pydsl.Memory.Search.Searcher import MemorySearcher from pydsl.Memory.Search.Indexer import Indexer from pydsl.Config import GLOBALCONFIG searcher = MemorySearcher([Indexer(x) for x in GLOBALCONFIG.memorylist]) return [ x["identifier"] for x in searcher.search({"ancestors":{"$in":"Function"}})]
class DirStorage(Memory): """A collection of elements stored inside a directory""" def __init__(self, dirpath, allowedextensions=(".py", ".bnf", ".re", ".board")): Memory.__init__(self) self.path = dirpath self._allowedextensions = allowedextensions from pydsl.Memory.Search.Searcher import MemorySearcher self._searcher = MemorySearcher(self) def __iter__(self): self.index = 0 self.cache = [] for filename in self.all_files(): try: self.cache.append(self.summary_from_filename(filename)) except (AttributeError,ImportError) as e: LOG.debug("Error while loading %s file summary" % filename ) return self def next(self): try: result = self.cache[self.index] except IndexError: raise StopIteration self.index += 1 return result def summary_from_filename(self, modulepath): (_, _, fileBaseName, ext) = getFileTuple(modulepath) if _isRELFileName(modulepath): result = {"iclass":"re","identifier":fileBaseName, "filepath":modulepath} elif _isGDLFileName(modulepath): result = {"iclass":"BNFGrammar","identifier":fileBaseName, "filepath":modulepath} elif _isBoardFileName(modulepath): result = {"iclass":"Board", "identifier":fileBaseName, "filepath":modulepath} else: from pydsl.Memory.File.Python import summary_python_file result = summary_python_file(modulepath) return InmutableDict(result) def all_files(self): import glob if self._allowedextensions: for extension in self._allowedextensions: searchstring = self.path + "*" + extension tmpresult = glob.glob(searchstring) for result in tmpresult: if result.endswith("__init__.py"): continue yield result else: searchstring = self.path + "*" for result in glob.glob(searchstring): yield result def all_names(self): """Generates all Static Ids""" for fullname in self.all_files(): (_, _, fileBaseName, fileExtension) = getFileTuple(fullname) if self._allowedextensions and fileExtension not in self._allowedextensions: continue yield fileBaseName.split(".")[0] def _load_module_from_library(self, identifier): try: import imp moduleobject = imp.load_source(identifier, self.path + "/" + identifier + ".py") except (ImportError, IOError): pass else: return moduleobject raise ImportError def load(self, name, **kwargs): resultlist = self._searcher.search(name) if len(resultlist) > 1: LOG.error("Found two or more matches, FIXME: processing the first, should raise exception") if len(resultlist) == 0: raise KeyError(self.__class__.__name__ + name) filepath = list(resultlist)[0]["filepath"] if _isRELFileName(filepath): from pydsl.Memory.File.Regexp import load_re_from_file return load_re_from_file(filepath) if _isGDLFileName(filepath): from pydsl.Memory.File.BNF import load_bnf_file return load_bnf_file(filepath) if _isBoardFileName(filepath): from pydsl.Memory.File.Board import load_board_file return load_board_file(filepath) from pydsl.Memory.File.Python import load_python_file return load_python_file(filepath, **kwargs) def __contains__(self, key): return key in self.all_names() def provided_iclasses(self): return set([x['iclass'] for x in self])