def guess(inputstring, memorylist = []) -> set:
from pydsl.Memory.Search.Searcher import MemorySearcher
from pydsl.Memory.Storage.Directory.Grammar import GrammarDirStorage
from pydsl.Memory.Storage.Dict import FileTypeDictStorage
if not memorylist:
try:
dirname = resource_filename(Requirement.parse("pydsl_contrib"),"")
except DistributionNotFound:
pass
else:
memorylist.append(GrammarDirStorage(dirname + "grammar/"))
memorylist.append(FileTypeDictStorage(dirname + "/dict/filetype.dict"))
searcher = MemorySearcher([x.indexer() for x in memorylist])
result = set()
for summary in searcher.search():
typ = None
name = None
try:
for mem in memorylist:
if summary["identifier"] in mem:
name = summary["identifier"]
typ = mem.load(name)
break
if typ.check(inputstring):
result.add(str(name))
except TypeError:
continue
return result
def available_transforms(self):
if self.functionlist == None:
from pydsl.Memory.Search.Searcher import MemorySearcher
from pydsl.Memory.Search.Indexer import Indexer
from pydsl.Config import GLOBALCONFIG
searcher = MemorySearcher([Indexer(x) for x in GLOBALCONFIG.memorylist])
resultlist = []
for element in self.guess():
resultlist += searcher.search({"input":{"$part":{"input":element}}})
self.functionlist = resultlist
return self.functionlist
def available_transforms(content):
from pydsl.Memory.Search.Searcher import MemorySearcher
from pydsl.Memory.Search.Indexer import Indexer
from pydsl.Config import GLOBALCONFIG
searcher = MemorySearcher([Indexer(x) for x in GLOBALCONFIG.memorylist])
resultlist = []
if content.grammar:
grammarlist = [content.grammar]
else:
grammarlist = content.available_grammars()
for element in grammarlist:
resultlist += searcher.search({"input":{"$part":{"input":element}}})
return [x['identifier'] for x in resultlist]
def __init__(self, dirpath:str, allowedextensions:list = []):
self.identifier = dirpath
from pydsl.Config import GLOBALCONFIG
resultdirpath = []
self._allowedextensions = allowedextensions
from pydsl.Memory.Search.Searcher import MemorySearcher
self._searcher = MemorySearcher(self)
def __init__(self, dirpath, allowedextensions=(".py", ".bnf", ".re", ".board")):
Memory.__init__(self)
self.path = dirpath
self._allowedextensions = allowedextensions
from pydsl.Memory.Search.Searcher import MemorySearcher
self._searcher = MemorySearcher(self)
class Guesser(object):
def __init__(self, memorylist = None):
from pydsl.Memory.Search.Searcher import MemorySearcher
from pydsl.Memory.Directory import DirStorage
if not memorylist:
dirname = resource_filename("pydsl.contrib","")
memorylist = [DirStorage(dirname + "/grammar/"), EncodingStorage(dirname + "/list/encoding.py")]
self.memorylist = memorylist
self.searcher = MemorySearcher([x.indexer() for x in memorylist])
def __call__(self, inputstring): #-> set:
result = set()
for summary in self.searcher.search():
typ = None
name = None
try:
for mem in self.memorylist:
if summary["identifier"] in mem:
name = summary["identifier"]
try:
typ = mem.load(name)
except:
LOG.exception("Error while loading memory %s" % name)
continue
break
else:
continue # not found
checker = load_checker(typ)
if checker.check(inputstring):
result.add(str(name))
except TypeError:
continue
return result
def __init__(self, memorylist = None):
from pydsl.Memory.Search.Searcher import MemorySearcher
from pydsl.Memory.Directory import DirStorage
if not memorylist:
dirname = resource_filename("pydsl.contrib","")
memorylist = [DirStorage(dirname + "/grammar/"), EncodingStorage(dirname + "/list/encoding.py")]
self.memorylist = memorylist
self.searcher = MemorySearcher([x.indexer() for x in memorylist])
class DirStorage(Storage, metaclass = ABCMeta):
"""A collection of elements stored inside a directory"""
def __init__(self, dirpath:str, allowedextensions:list = []):
self.identifier = dirpath
from pydsl.Config import GLOBALCONFIG
resultdirpath = []
self._allowedextensions = allowedextensions
from pydsl.Memory.Search.Searcher import MemorySearcher
self._searcher = MemorySearcher(self)
def __iter__(self):
self.index = 0
self.cache = []
for filename in self.all_files():
try:
self.cache.append(self.summary_from_filename(filename))
except AttributeError:
pass
return self
def __next__(self):
try:
result = self.cache[self.index]
except IndexError:
raise StopIteration
self.index += 1
return result
def summary_from_filename(self, modulepath):
(_, _, fileBaseName, ext) = getFileTuple(modulepath)
result = None
if _isRELFileName(modulepath):
result = {"iclass":"re","identifier":fileBaseName, "filepath":modulepath}
elif _isGDLFileName(modulepath):
result = {"iclass":"BNFGrammar","identifier":fileBaseName, "filepath":modulepath}
elif (modulepath).endswith(".board"):
from pydsl.Function.Transformer.Board import Board
result = {"iclass":"Board", "identifier":fileBaseName, "filepath":modulepath, "ancestors":Board.ancestors()}
else:
import imp
moduleobject = imp.load_source(fileBaseName, modulepath)
result = {"identifier":fileBaseName, "iclass":moduleobject.iclass, "filepath":modulepath}
if hasattr(moduleobject, "title"):
result["title"] = InmutableDict(moduleobject.title)
if hasattr(moduleobject, "description"):
result["description"] = InmutableDict(moduleobject.description)
return InmutableDict(result)
def _search_files(self, string: str, exact:bool = True):
"""Search through library"""
for filename in self.all_files():
if exact:
(_, _, fileBaseName, _) = getFileTuple(filename)
if fileBaseName.strip() == string.strip():
yield filename
else:
if string.lower() in filename.lower():
yield filename
def all_files(self):
import glob
if self._allowedextensions:
for extension in self._allowedextensions:
tmpresult = []
searchstring = self.identifier + "*" + extension
tmpresult = glob.glob(searchstring)
for result in tmpresult:
if result.endswith("__init__.py"):
continue
yield result
else:
searchstring = self.identifier + "*"
for result in glob.glob(searchstring):
yield result
def all_names(self):
"""Generates all Static Ids"""
for fullname in self.all_files():
(_, _, fileBaseName, fileExtension) = getFileTuple(fullname)
if self._allowedextensions and fileExtension not in self._allowedextensions:
continue
yield fileBaseName.split(".")[0]
def _load_module_from_library(self, identifier):
try:
import imp
moduleobject = imp.load_source(identifier, self.identifier + "/" + identifier + ".py")
except (ImportError, IOError):
pass
else:
return moduleobject
raise ImportError
def load(self, name, **kwargs):
resultlist = self._searcher.search(name)
if(len(resultlist) > 1):
LOG.error("Found two or more matches, FIXME: processing the first, should raise exception")
if len(resultlist) == 0:
raise KeyError(self.__class__.__name__ + name)
return load_python_file(list(resultlist)[0]["filepath"], **kwargs)
def __contains__(self, key):
return key in self.all_names()
def provided_iclasses(self) -> set:
return set([x['identifier'] for x in self])
if __name__ == "__main__":
import argparse
from pydsl.Config import VERSION, GLOBALCONFIG
TUSAGE = "usage: %(prog)s [options] query"
PARSER = argparse.ArgumentParser(usage = TUSAGE)
PARSER.add_argument("search", metavar="search", help="Query to search")
PARSER.add_argument('--version', action='version', version = VERSION)
PARSER.add_argument('--filter', dest='myfilter',nargs='?', default=None, help="comma separated field list")
PARSER.add_argument('-o', dest='outputformat',nargs='?', choices=["str","json"], default="str", help="output formats")
ARGS = PARSER.parse_args()
import sys
from pydsl.Memory.Search.Searcher import MemorySearcher
from pydsl.Memory.Search.Indexer import Indexer
if ARGS.search:
searcher = MemorySearcher([Indexer(x) for x in GLOBALCONFIG.memorylist])
myfilter = None
if ARGS.myfilter:
myfilter = ARGS.myfilter.split(',')
if ARGS.outputformat == "str":
print(search_pp(searcher.search(ARGS.search), myfilter))
elif ARGS.outputformat == "json":
import json
print(json.dumps(list(filterset(searcher.search(ARGS.search), myfilter))))
sys.exit(0)
else:
print(TUSAGE)
sys.exit(0)
sys.exit(0)
def __dir__(cls):
from pydsl.Memory.Search.Searcher import MemorySearcher
from pydsl.Memory.Search.Indexer import Indexer
from pydsl.Config import GLOBALCONFIG
searcher = MemorySearcher([Indexer(x) for x in GLOBALCONFIG.memorylist])
return [ x["identifier"] for x in searcher.search({"ancestors":{"$in":"Function"}})]
class DirStorage(Memory):
"""A collection of elements stored inside a directory"""
def __init__(self, dirpath, allowedextensions=(".py", ".bnf", ".re", ".board")):
Memory.__init__(self)
self.path = dirpath
self._allowedextensions = allowedextensions
from pydsl.Memory.Search.Searcher import MemorySearcher
self._searcher = MemorySearcher(self)
def __iter__(self):
self.index = 0
self.cache = []
for filename in self.all_files():
try:
self.cache.append(self.summary_from_filename(filename))
except (AttributeError,ImportError) as e:
LOG.debug("Error while loading %s file summary" % filename )
return self
def next(self):
try:
result = self.cache[self.index]
except IndexError:
raise StopIteration
self.index += 1
return result
def summary_from_filename(self, modulepath):
(_, _, fileBaseName, ext) = getFileTuple(modulepath)
if _isRELFileName(modulepath):
result = {"iclass":"re","identifier":fileBaseName, "filepath":modulepath}
elif _isGDLFileName(modulepath):
result = {"iclass":"BNFGrammar","identifier":fileBaseName, "filepath":modulepath}
elif _isBoardFileName(modulepath):
result = {"iclass":"Board", "identifier":fileBaseName, "filepath":modulepath}
else:
from pydsl.Memory.File.Python import summary_python_file
result = summary_python_file(modulepath)
return InmutableDict(result)
def all_files(self):
import glob
if self._allowedextensions:
for extension in self._allowedextensions:
searchstring = self.path + "*" + extension
tmpresult = glob.glob(searchstring)
for result in tmpresult:
if result.endswith("__init__.py"):
continue
yield result
else:
searchstring = self.path + "*"
for result in glob.glob(searchstring):
yield result
def all_names(self):
"""Generates all Static Ids"""
for fullname in self.all_files():
(_, _, fileBaseName, fileExtension) = getFileTuple(fullname)
if self._allowedextensions and fileExtension not in self._allowedextensions:
continue
yield fileBaseName.split(".")[0]
def _load_module_from_library(self, identifier):
try:
import imp
moduleobject = imp.load_source(identifier, self.path + "/" + identifier + ".py")
except (ImportError, IOError):
pass
else:
return moduleobject
raise ImportError
def load(self, name, **kwargs):
resultlist = self._searcher.search(name)
if len(resultlist) > 1:
LOG.error("Found two or more matches, FIXME: processing the first, should raise exception")
if len(resultlist) == 0:
raise KeyError(self.__class__.__name__ + name)
filepath = list(resultlist)[0]["filepath"]
if _isRELFileName(filepath):
from pydsl.Memory.File.Regexp import load_re_from_file
return load_re_from_file(filepath)
if _isGDLFileName(filepath):
from pydsl.Memory.File.BNF import load_bnf_file
return load_bnf_file(filepath)
if _isBoardFileName(filepath):
from pydsl.Memory.File.Board import load_board_file
return load_board_file(filepath)
from pydsl.Memory.File.Python import load_python_file
return load_python_file(filepath, **kwargs)
def __contains__(self, key):
return key in self.all_names()
def provided_iclasses(self):
return set([x['iclass'] for x in self])
