def parse_python_file(filename):
"""Parse file into chunks / objects."""
with open(filename, "rt", encoding="utf-8") as f:
tokens = list(tokenize.generate_tokens(f.readline))
tokens = normalize_tokens(tokens)
module = ChunkCode(tokens, 0, len(tokens))
module.parse()
print(module)
def main():
"""Executed when script is run as-is."""
# magic_files = {}
for filename in locate_files(ROOT_DIR):
print("Processing %s" % filename)
with open(filename, "rt") as f:
tokens = list(tokenize.generate_tokens(f.readline))
text1 = tokenize.untokenize(tokens)
ntokens = normalize_tokens(tokens)
text2 = tokenize.untokenize(ntokens)
assert text1 == text2
def find_magic_in_file(filename):
"""
Search the file for any magic incantations.
:param filename: file to search
:returns: a tuple containing the spell and then maybe some extra words (or None if no magic present)
"""
with open(filename, "rt", encoding="utf-8") as f:
for line in f:
if line.startswith("#"):
comment = line[1:].strip()
if comment.startswith("~~~~* ") or comment.startswith("----* ") or comment.startswith("====* "):
spell = comment[5:].strip()
return tuple(spell.split())
else:
break
return None # No magic found in file
