def test_c_bnf_to_ebnf(self):
c_grammar_pth = path(__file__).dirname().joinpath("c.g")
c_token_pth = path(__file__).dirname().joinpath("token.g")
grammar = bnftranslator.convertbnf(c_grammar_pth, c_token_pth)
# The 'and_expression' rule is an example of a BNF -> EBNF rule conversion
self.assertTrue(grammar.find("and_expression: equality_expression ('&' equality_expression)*")>0)
def langlet_path(self, pth):
base = pth = path(pth).dirname()
while pth:
if pth.files("langlet.py"):
return pth
root = path(pth).dirname()
if root == pth:
return base
pth = root
raise RuntimeError("Unable to determine langlet path")
def create_files(self):
self.update_options()
if self.options["location"]:
loc = path(self.options["location"])
if loc.basename() != self.name:
pth_langlet = loc.joinpath(self.name)
else:
pth_langlet = loc
else:
pth_langlet = langlets_path.joinpath(self.name)
self.new_langlet_pth = pth_langlet
if pth_langlet.exists():
raise IOError("Langlet directory already exists: '%s'"%pth_langlet)
try:
pth_langlet_module = pth_langlet.joinpath("langlet.py")
count = self.create_new_langlet_counter()
lnlt_id = count*(10**4)
pth_template = parent_path.joinpath("langlet_template")
pth_conf = pth_template.joinpath("langlet_config.strtempl")
pth_langlet_py = pth_template.joinpath("langlet.strtempl")
langlet_py = ""
pth_template.copytree(pth_langlet)
if self.options["parent"]:
parent_langlet = path(loader.find_langlet(self.options["parent"])[1].__file__).dirname()
grammar_gen_path = parent_langlet.joinpath("parsedef", "GrammarGen.g")
grammar_gen_path.copy(pth_langlet.joinpath("parsedef", "GrammarBase.g"))
token_gen_path = parent_langlet.joinpath("lexdef", "TokenGen.g")
token_gen_path.copy(pth_langlet.joinpath("lexdef", "TokenBase.g"))
# operate on destination folder...
pth_conf = pth_langlet.joinpath("langlet_config.py")
pth_conf_template = pth_langlet.joinpath("langlet_config.strtempl")
with open(pth_conf,"w") as f_conf:
f_conf.write(self.create_langlet_config_py(lnlt_id, pth_conf_template))
pth_langlet.joinpath("run.py").rename(pth_langlet.joinpath("run_"+self.name+".py"))
pth_langlet.joinpath("langlet_config.strtempl").remove()
# update LANGLET_ID in lex_token.py
pth_lex_token = pth_langlet.joinpath("lexdef","lex_token.py")
pth_lex_token_template = path(pth_lex_token.replace(".py",".strtempl"))
with open(pth_lex_token, "w") as f_lex:
f_lex.write(self.update_lex_token(lnlt_id, pth_lex_token_template))
self.create_langlet_data(pth_conf)
pth_lex_token_template.remove()
except Exception, e:
print get_traceback()
self.new_langlet_pth.rmtree()
exit(2)
def pre_filter(self, fpth_mod):
if not path(fpth_mod).ext == u".py":
return True
if not self.is_langletmodule(fpth_mod):
return False
fpth_dir = path(fpth_mod.lower()).dirname()
if fpth_dir.basename() in ("lexdef", "parsedef", "langscape", "cstdef", "trail"):
return False
if fpth_mod.basename() in self.module_blacklist:
return False
return True
def compute_module_path(self):
'''
Computes module path from user input file-path fragment.
'''
self.fpth_mod_full = get_fs_path(self.fpth_mod_input)
if os.sep in self.fpth_mod_input:
module_path = []
self.module_name = path(self.fpth_mod_full.basename()).splitext()[0]
module_path.append(self.module_name)
k = len(self.fpth_mod_input.split(os.sep))-1
F = self.fpth_mod_full.dirname()
while k:
if F.joinpath("__init__.py").isfile():
module_path.insert(0, F.basename())
else:
if F not in sys.path:
sys.path.append(F)
break
F = F.dirname()
k-=1
else:
if F not in sys.path:
sys.path.append(F)
self.module_path = ".".join(module_path)
else:
F = self.fpth_mod_full.dirname()
if F not in sys.path:
sys.path.append(F)
self.module_path = self.module_name = self.fpth_mod_input.splitext()[0]
def __init__(self, langlet, name, *args, **kwd):
super(LSRecordedConsole, self).__init__(langlet, name, *args, **kwd)
self.last_output = LastOutput()
self.line_count = 0
self.report = self.acquire_session_report(langlet,name,kwd.get("recording"))
self.additional_header_info = " Creates session report " + path(self.report.name).basename()
self.user = RecordedUser(self.report)
def __init__(self, langlet, name, *args, **kwd):
super(LSReplayConsole, self).__init__(langlet, name, *args, **kwd)
self.last_output = LastOutput()
self.line_count = 0
self.ees_file = self.find_recorded_session(langlet,name,kwd.get("session"))
self.recorded_session = self.session(self.ees_file)
self.recorded_session.next() # closes report file
self.additional_header_info = " Replay session " + path(self.ees_file.name).basename()
self.user = ReplayedUser(self.session_protocol())
def find_module(self, mpth_mod, mpth_pack=None):
'''
Framework function find_module(). See
http://www.python.org/dev/peps/pep-0302/
'''
if mpth_pack and ".egg" in mpth_pack[
0]: # TODO - enable zipimport of langlet modules
return
if mpth_mod.startswith("langscape.base."):
mpth_mod = mpth_mod[15:]
if mpth_pack:
if mpth_pack[0].endswith("base"):
del mpth_pack[0]
package = ""
idx = mpth_mod.rfind(".") # maybe dotted module name ?
if idx > 0:
package, mpth_mod = mpth_mod[:idx], mpth_mod[idx + 1:]
mpth_pack = sys.modules[package].__path__
if mpth_pack and mpth_pack[0].endswith("encodings"):
return
if self.dbg:
dbg_import("input: module: `%s`" % mpth_mod)
dbg_import("input: package:`%s`" % mpth_pack)
moduledata = self.loader.find_module(mpth_mod, mpth_pack)
if isinstance(moduledata, zipimport.zipimporter):
if self.dbg:
dbg_import("zipimport: `%s`" % mpth_mod)
moduledata.load_module(mpth_mod)
return
if not moduledata:
if self.dbg:
dbg_import("no-data: `%s`, `%s`, `%s`" %
(mpth_mod, package, mpth_pack))
if mpth_pack:
raise ImportError("No module named %s found at %s." %
(mpth_mod, mpth_pack))
else:
raise ImportError("No module named %s found." % mpth_mod)
if self.dbg:
dbg_import("moduledata: `%s`\n" % (moduledata[1:], ))
if not moduledata[1]:
return
self.fpth_mod = path(moduledata[1])
self.mpth_mod = mpth_mod
# sys.stdout.write("DEBUG import_path: %s, module: %s\n"%(self.mpth_mod, self.fpth_mod))
if self.modulefilter.accept_module(self.fpth_mod):
if self.dbg:
dbg_import("accepted module:`%s`" % self.fpth_mod)
return self
def get_fs_path(path_fragment):
F = path(os.getcwd())
P = path(path_fragment)
FP = F.joinpath(P)
if FP.isfile():
return FP
if P.isfile():
return P
fragments = P.split(os.sep)
for i, item in enumerate(fragments):
if item in (".", ".."):
F = F.dirname()
elif i>0:
P = path(os.sep.join(fragments[i:]))
FP = F.joinpath(P)
if FP.isfile():
return FP
break
raise IOError("Can't open file '%s'"%path_fragment)
def __init__(self, langlet, name, *args, **kwd):
super(LSReplayConsole, self).__init__(langlet, name, *args, **kwd)
self.last_output = LastOutput()
self.line_count = 0
self.ees_file = self.find_recorded_session(langlet, name,
kwd.get("session"))
self.recorded_session = self.session(self.ees_file)
self.recorded_session.next() # closes report file
self.additional_header_info = " Replay session " + path(
self.ees_file.name).basename()
self.user = ReplayedUser(self.session_protocol())
def find_module(self, mpth_mod, mpth_pack = None):
'''
Framework function find_module(). See
http://www.python.org/dev/peps/pep-0302/
'''
if mpth_pack and ".egg" in mpth_pack[0]: # TODO - enable zipimport of langlet modules
return
if mpth_mod.startswith("langscape.base."):
mpth_mod = mpth_mod[15:]
if mpth_pack:
if mpth_pack[0].endswith("base"):
del mpth_pack[0]
package = ""
idx = mpth_mod.rfind(".") # maybe dotted module name ?
if idx>0:
package, mpth_mod = mpth_mod[:idx], mpth_mod[idx+1:]
mpth_pack = sys.modules[package].__path__
if mpth_pack and mpth_pack[0].endswith("encodings"):
return
if self.dbg:
dbg_import("input: module: `%s`"%mpth_mod)
dbg_import("input: package:`%s`"%mpth_pack)
moduledata = self.loader.find_module(mpth_mod, mpth_pack)
if isinstance(moduledata, zipimport.zipimporter):
if self.dbg:
dbg_import("zipimport: `%s`"%mpth_mod)
moduledata.load_module(mpth_mod)
return
if not moduledata:
if self.dbg:
dbg_import("no-data: `%s`, `%s`, `%s`"%(mpth_mod, package, mpth_pack))
if mpth_pack:
raise ImportError("No module named %s found at %s."%(mpth_mod, mpth_pack))
else:
raise ImportError("No module named %s found."%mpth_mod)
if self.dbg:
dbg_import("moduledata: `%s`\n"%(moduledata[1:],))
if not moduledata[1]:
return
self.fpth_mod = path(moduledata[1])
self.mpth_mod = mpth_mod
# sys.stdout.write("DEBUG import_path: %s, module: %s\n"%(self.mpth_mod, self.fpth_mod))
if self.modulefilter.accept_module(self.fpth_mod):
if self.dbg:
dbg_import("accepted module:`%s`"%self.fpth_mod)
return self
def __init__(self, langlet, modfilter = ModuleFilter):
'''
:param langlet: langlet module object
'''
self.langlet = langlet
self.fpth_langlet = self.langlet_path()
self.loader = self.module_loader()
self.dbg = langlet.options.get("debug_importer")
self.modulefilter = modfilter(langlet)
md = LangletModuleDescriptor()
md.fpth_mod_input = path(langlet.config.__file__)
md.compute_module_path()
self.module_descr = md
def autorun():
(options, args) = config.opt.parse_args()
langlet_obj = loader.load_langlet(config.langlet_name, **options.__dict__)
if langlet_obj.options["xml2p4d"]:
f = path(args[-1])
if not f.isfile():
if f[0] == '*':
ext = f.splitext()[1]
for fl in path(os.getcwd()).listdir():
if fl.ext == ext:
convert2p4d(langlet_obj, fl)
return
else:
f = path(os.getcwd()).joinpath(f)
convert2p4d(langlet_obj, f)
elif args:
module = args[-1]
langlet_obj.run_module(module)
else:
console = langlet_obj.console()
console.interact()
def __init__(self, langlet, modfilter=ModuleFilter):
'''
:param langlet: langlet module object
'''
self.langlet = langlet
self.fpth_langlet = self.langlet_path()
self.loader = self.module_loader()
self.dbg = langlet.options.get("debug_importer")
self.modulefilter = modfilter(langlet)
md = LangletModuleDescriptor()
md.fpth_mod_input = path(langlet.config.__file__)
md.compute_module_path()
self.module_descr = md
def autorun():
(options, args) = config.opt.parse_args()
langlet_obj = loader.load_langlet(config.langlet_name, **options.__dict__)
if langlet_obj.options["xml2p4d"]:
f = path(args[-1])
if not f.isfile():
if f[0] == '*':
ext = f.splitext()[1]
for fl in path(os.getcwd()).listdir():
if fl.ext == ext:
convert2p4d(langlet_obj, fl)
return
else:
f = path(os.getcwd()).joinpath(f)
convert2p4d(langlet_obj, f)
elif args:
module = args[-1]
langlet_obj.run_module(module)
else:
console = langlet_obj.console()
console.interact()
def run_module(self, cmdline_module_path):
'''
:param cmdline_module_path: command line module path data e.g. ``mod.py`` or ``tests/foo/mod.py``.
'''
try:
sys.argv = sys.argv[sys.argv.index(cmdline_module_path):]
except ValueError:
pass
md = LangletModuleDescriptor()
md.is_main = True
md.fpth_mod_input = path(cmdline_module_path)
md.compute_module_path()
self.importer.set_module_descriptor(md)
self.importer.register_importer()
self.target.register_excepthook(self)
__import__(md.module_path)
def run_module(self, cmdline_module_path):
'''
:param cmdline_module_path: command line module path data e.g. ``mod.py`` or ``tests/foo/mod.py``.
'''
try:
sys.argv = sys.argv[sys.argv.index(cmdline_module_path):]
except ValueError:
pass
md = LangletModuleDescriptor()
md.is_main = True
md.fpth_mod_input = path(cmdline_module_path)
md.compute_module_path()
self.importer.set_module_descriptor(md)
self.importer.register_importer()
self.target.register_excepthook(self)
__import__(md.module_path)
def run_tests(name="", exclude=()):
'''
This module provides limited test discovery. Unittest is not sufficient because it would reject
non-Python modules. The strategy compprises walking through the langscape subdirectories, searching
for directories named 'tests'. In those directories test_<name>.<suffix> files will be identified
and executed as scripts.
'''
remove_pyc()
testpaths = []
for P in path(langscape.__file__).dirname().walkdirs():
S = P.splitall()
if ".hg" in S:
continue
elif S[-1] == "tests":
if "langlets" in S:
testpaths.append((P, S[-2]))
else:
testpaths.append((P, ""))
langlet = langscape.load_langlet("python")
log = open("log.txt", "w")
for pth, nm in testpaths:
if (name and name != nm) or nm in exclude:
continue
for f in pth.files():
if nm and langlet.config.langlet_name != nm:
print "-" * 70
print "Load Langlet: ", nm
langlet = langscape.load_langlet(nm)
if f.basename().startswith("test_") and f.basename().endswith(
langlet.config.source_ext):
try:
print >> log, f
langlet.run_module(f)
except Exception, e:
print "Failed to run", langlet
print " " * 15, f
e = traceback.format_exc()
print e
return
def run_tests(name = "", exclude = ()):
'''
This module provides limited test discovery. Unittest is not sufficient because it would reject
non-Python modules. The strategy compprises walking through the langscape subdirectories, searching
for directories named 'tests'. In those directories test_<name>.<suffix> files will be identified
and executed as scripts.
'''
remove_pyc()
testpaths = []
for P in path(langscape.__file__).dirname().walkdirs():
S = P.splitall()
if ".hg" in S:
continue
elif S[-1] == "tests":
if "langlets" in S:
testpaths.append((P, S[-2]))
else:
testpaths.append((P,""))
langlet = langscape.load_langlet("python")
log = open("log.txt", "w")
for pth, nm in testpaths:
if (name and name != nm) or nm in exclude:
continue
for f in pth.files():
if nm and langlet.config.langlet_name!=nm:
print "-"*70
print "Load Langlet: ", nm
langlet = langscape.load_langlet(nm)
if f.basename().startswith("test_") and f.basename().endswith(langlet.config.source_ext):
try:
print >> log, f
langlet.run_module(f)
except Exception, e:
print "Failed to run", langlet
print " "*15, f
e = traceback.format_exc()
print e
return
def remove_pyc():
for f in path(langscape.__file__).dirname().walkfiles():
if f.ext in (".pcv", ".pyc"):
f.remove()
def __init__(self, langlet, name, *args, **kwd):
super(LSRecordedReplayConsole, self).__init__(langlet, name, *args, **kwd)
self.additional_header_info = " Reuses session report " + path(self.ees_file.name).basename()
self.user = RecordedReplayedUser(self)
self.user.report = file(self.ees_file.name,"w")
self.check_prefix_split = False
#! /usr/bin/env python
#
# URL: http://www.fiber-space.de
# Author: Kay Schluehr <*****@*****.**>
# Creation: 15 Oct 2009
from __future__ import with_statement
import os
import string
from langscape.util import get_traceback
from langscape.util.path import path
import langscape.base.loader as loader
own_path = path(__file__).dirname()
parent_path = own_path.dirname()
langlets_path = parent_path.joinpath("langlets")
class LangletGenerator(object):
def __init__(self, kwds):
self.name = kwds["langlet_name"].strip()
self.options = kwds
def update_options(self):
'''
Overrides empty options settings by parent options.
'''
if self.options["parent"]:
pth, module = loader.find_langlet(self.options["parent"])
if self.options["prompt"] == "> ":
self.options["prompt"] = module.config.prompt
def find_langlets():
for pth in LANGLETPATH:
pkg = __import__(pth, fromlist=[""])
for p in path(pkg.__file__).dirname().dirs():
if p.joinpath("langlet.py").isfile():
yield p, p.basename()
def get_path(self):
return path(self.config.__file__).dirname()
def remove_pyc():
for f in path(langscape.__file__).dirname().walkfiles():
if f.ext in (".pcv", ".pyc"):
f.remove()
def find_langlets():
for pth in LANGLETPATH:
pkg = __import__(pth, fromlist=[""])
for p in path(pkg.__file__).dirname().dirs():
if p.joinpath("langlet.py").isfile():
yield p, p.basename()
def get_path(self):
return path(self.config.__file__).dirname()
