def _atom(v):
is_symbol = '\\' == v[0]
v = Lissp.escape(v)
if is_symbol:
return munge(v)
try:
val = ast.literal_eval(v)
if isinstance(val, bytes): # bytes have their own literals.
return munge(v)
return val
except (ValueError, SyntaxError):
return munge(v)
def atom(v):
"""Preprocesses atoms. Handles escapes and munging."""
is_symbol = "\\" == v[0]
v = Lissp.escape(v)
if is_symbol:
return munge(v)
try:
val = ast.literal_eval(v)
if isinstance(val, bytes): # bytes have their own literals.
return munge(v)
return val
except (ValueError, SyntaxError):
return munge(v)
def _custom_macro(self, form, tag, extras):
assert tag.endswith("#")
tag = munge(self.escape(tag[:-1]))
if ".." in tag:
module, function = tag.split("..", 1)
m = reduce(getattr, function.split("."), import_module(module))
else:
try:
m = getattr(self.ns["_macro_"], tag + munge("#"))
except (AttributeError, KeyError):
raise SyntaxError(f"Unknown reader macro {tag!r}.",
self.position())
with self.compiler.macro_context():
args, kwargs = _parse_extras(extras)
return m(form, *args, **kwargs)
def test_munge_basic(self):
self.assertEqual(
munger.munge(r"""~!@#$%^&*()_+{}|:"<>?`-=[]\;',./"""),
"xTILDE_xBANG_xAT_xHASH_xDOLR_xPCENT_xCARET_xET_xSTAR_xPAREN_xTHESES_"
"_xPLUS_xCURLY_xBRACES_xBAR_xCOLON_x2QUOTE_xLT_xGT_xQUERY_xGRAVE_xH_xEQ_"
"xSQUARE_xBRACKETS_xBSLASH_xSCOLON_x1QUOTE_xCOMMA_.xSLASH_"
)
def parse_macro(self, tag: str, form):
if tag == "'":
return "quote", form
if tag == "`":
return self.template(form)
if tag == ",":
return _Unquote([":?", form])
if tag == ",@":
return _Unquote([":*", form])
assert tag.endswith("#")
tag = tag[:-1]
if tag == "_":
return DROP
if tag == "$":
return self.gensym(form)
if tag == ".":
return eval(readerless(form), {})
if ".." in tag and not tag.startswith(".."):
module, function = tag.split("..", 1)
function = munge(function)
if is_string(form):
form = form[1]
return reduce(getattr, function.split("."),
import_module(module))(form)
raise ValueError(f"Unknown reader macro {tag}")
def parse_tag(self, tag, form):
if tag == "_":
return DROP
if tag == "$":
return self.gensym(form)
if tag == ".":
return eval(readerless(form), {})
if is_string(form):
form = form[1]
tag = munge(self.escape(tag))
if ".." in tag and not tag.startswith(".."):
module, function = tag.split("..", 1)
return reduce(getattr, function.split("."),
import_module(module))(form)
try:
m = getattr(self.ns["_macro_"], tag)
except (AttributeError, KeyError):
raise SyntaxError(f"Unknown reader macro {tag}", self.position())
return m(form)
def _parse(self, tokens: Iterator[Token]) -> Iterator:
for k, v in tokens:
if k == "open":
depth = self.depth
self.depth += 1
yield (*self.parse(tokens), )
if self.depth != depth:
raise SyntaxError("Unclosed '('.")
elif k == "close":
self.depth -= 1
if self.depth < 0:
raise SyntaxError("Unopened ')'.")
return
elif k == "string":
yield "quote", ast.literal_eval(
v.replace("\\\n", "").replace("\n", r"\n")), {
":str": True
}
elif k in {"comment", "whitespace"}:
continue
elif k == "macro":
with {
"`": self.gensym_context,
",": self.unquote_context,
",@": self.unquote_context,
}.get(v, nullcontext)():
form = next(self.parse(tokens))
yield self.parse_macro(v, form)
elif k == "symbol":
try:
yield ast.literal_eval(v)
except (ValueError, SyntaxError):
yield munge(v)
else:
assert False, "unknown token: " + repr(k)
if self.depth:
SyntaxError("Ran out of tokens before completing form.")
def test_munge_symbol(self, s):
self.assertTrue(munger.munge(s).isidentifier())
def test_demunge(self, s: str):
x = munger.munge(s)
self.assertEqual(x, munger.munge(munger.demunge(x)))
if not s.startswith(":") and "x" not in s:
self.assertEqual(unicodedata.normalize("NFKC", s), munger.demunge(x))
def test_munge_basic(self):
self.assertEqual(
munger.munge(r"~!@#$%^&*+`-=|\:'<>?,/"),
"xTILDE_xBANG_xAT_xHASH_xDOLLAR_xPERCENT_xCARET_xET_xSTAR_xPLUS_xGRAVE_xH_"
"xEQ_xBAR_xBSLASH_xCOLON_xQUOTE_xLT_xGT_xQUERY_xCOMMA_xSLASH_",
)
def test_demunge(self, s: str):
x = munger.munge(s)
self.assertEqual(x, munger.munge(munger.demunge(x)))
if "x" not in s:
self.assertEqual(s, munger.demunge(x))
def test_munge_basic(self):
self.assertEqual(
munger.munge(r"""~!@#$%^&*()_+{}|:"<>?`-=[]\;',./"""),
"QzTILDE_QzBANG_QzAT_QzHASH_QzDOLR_QzPCENT_QzCARET_QzET_QzSTAR_QzLPAR_QzRPAR_"
"_QzPLUS_QzLCUB_QzRCUB_QzBAR_QzCOLON_QzQUOT_QzLT_QzGT_QzQUERY_QzGRAVE_Qz_QzEQ_"
"QzLSQB_QzRSQB_QzBSOL_QzSEMI_QzAPOS_QzCOMMA_.QzSOL_")