def _tokenize(self, path: Path) -> Iterator[tuple[str, str]]:
lexer = Lexer(yield_comments=True)
text = read_file(path, "rt", encoding="utf-8")
lexer.input(text)
code_tokens = ("ID", "NUMBER")
text_tokens = ("STRING", "LINE_COMMENT", "BLOCK_COMMENT")
for token in lexer:
if token.type in code_tokens:
yield "code", token.value
elif token.type in text_tokens:
yield "text", token.value
def main(inpath, tplpath, outpath, outsuffix, suffixes=frozenset()):
# type: (Path, Path, Path, str, FrozenSet[str]) -> None
tpl = read_file(tplpath, "rt")
for i, path in enumerate(path for path in inpath.iterdir()
if path.suffix in suffixes):
with open(outpath / path.with_suffix(outsuffix).name, "wt") as fw:
fw.write(
tpl.format(
filename=path.name,
efilename=esc(path.name),
path=path.path,
epath=esc(path.path),
stem=path.parent / path.stem, # path without suffix
i=i))
def _tokenize(self, path: Path) -> Iterator[tuple[str, str]]:
try:
lexer = self.cache[path.suffix]
except KeyError:
try:
lexer = self.cache[path.suffix] = get_lexer_for_filename(path.name)
except ClassNotFound:
raise NoLexerFound()
text = read_file(path, "rt", encoding="utf-8")
for tokentype, value in lexer.get_tokens(text):
if tokentype in Token.Name or tokentype in Token.Number:
yield "code", value
elif tokentype in Token.String or tokentype in Token.Comment:
yield "text", value
def main(inpath, tplpath, outpath, suffixes=frozenset(), prepend=""):
# type: (Path, Path, Path, FrozenSet[str], str) -> None
tpl = read_file(tplpath, "rt")
paths = [path for path in inpath.iterdir() if path.suffix in suffixes]
with open(outpath, "wt") as fw:
fw.write(prepend.format(num=len(paths)))
for i, path in enumerate(paths):
fw.write(
tpl.format(
filename=path.name,
efilename=esc(path.name),
path=path.path,
epath=esc(path.path),
stem=path.parent / path.stem, # path without suffix
i=i))
parser.add_argument("outpath",
type=future_file,
help="Path to output file.")
parser.add_argument("--suffixes",
nargs="+",
default=[],
type=suffix,
help="Extensions to filter for.")
prepend_group = parser.add_mutually_exclusive_group(required=False)
prepend_group.add_argument(
"--prepend",
default="",
help=
"String to prepend to the output file once. Use {num} to include number of entries."
)
prepend_group.add_argument(
"--prependfile",
type=is_file,
help=
"File to prepend to the output file once. Use {num} to include number of entries."
)
args = parser.parse_args()
if args.prependfile:
prepend = read_file(args.prependfile)
else:
prepend = args.prepend
main(args.inpath, args.tplpath, args.outpath, frozenset(args.suffixes),
prepend)