def defineTemplate(self, opts, block):
attributes = parseTagAttributes("template", opts, [ "name", "type" ])
if not "name" in attributes or not "type" in attributes:
fatal("Template definition requires both name and type attributes: " + opts)
name = attributes["name"]
type = attributes["type"]
dprint(1, "defining template name " + name + " of type " + type + ": " + str(block))
self.add(type, name, block)
return []
def formatFonts(fonts):
if len(fonts) == 0:
return []
block = []
for localname,googlename in fonts.items():
if getFontClass(localname):
continue
dprint(1, localname + ": " + googlename)
googlename = googlename.replace(' ', '+')
localFile = getGoogleFont(googlename)
block.append("@font-face {\n font-family: '" + localname + "';\n font-style: normal;\n src: url('" + localFile + "');\n}")
return block
def FNtoHtml(wb):
matchFN = re.compile("<fn\s+(.*?)/?>")
footnotes = {}
# footnote marks in text
i = 0
while i < len(wb):
off = 0
line = wb[i]
block = [ ]
while True:
m = matchFN.search(line, off)
if not m:
break
opts = m.group(1)
args = parseTagAttributes("fn", opts, [ "id", "target" ])
fmid = args["id"]
if not "target" in args:
fatal("Missing internal target in fn: " + line)
target = args["target"]
dprint(1, "id: " + fmid + ", target: " + target)
repl = "<sup><span style='font-size:0.9em'>" + fmid + "</span></sup>"
if target in noteMap:
# Note no link when we are co-locating the reference with the footnote
block.extend(noteMap[target])
del noteMap[target]
elif fmid in footnotes and footnotes[fmid] == target:
wprint('multifootnote', "warning: footnote id <fn id='" + fmid + "'> occurs multiple times. <footnote> link will be to the first. Line: >>>" + line + "<<<")
repl = "<a href='#f{0}' style='text-decoration:none'>{1}</a>".format(target, repl)
else:
footnotes[fmid] = target
repl = "<a id='r{0}'/><a href='#f{0}' style='text-decoration:none'>{1}</a>".format(target, repl)
l = line[0:m.start(0)] + repl
off = len(l) # Next loop
line = l + line[m.end(0):]
wb[i] = line
# Emit the footnote, right before the line with the footnote reference
# If the line with the footnote reference is a paragraph start, need
# to emit the footnote between the paragraph tag, and the rest of the
# text on the first line of the paragraph
if len(block) > 0:
m = re.match("(<p.*?>)", wb[i])
if m:
block.insert(0, m.group(1))
wb[i] = wb[i][len(m.group(1)):]
wb[i:i] = block
i += len(block)
i += 1
def footnotesToHtml(wb):
matchFN = re.compile("<fn\s+(.*?)/?>")
footnotes = {}
# footnote marks in text
i = 0
while i < len(wb):
off = 0
line = wb[i]
while True:
m = matchFN.search(line, off)
if not m:
break
opts = m.group(1)
args = parseTagAttributes("fn", opts, [ "id", "target" ])
fmid = args["id"]
if not "target" in args:
fatal("Missing internal target in fn: " + line)
target = args["target"]
dprint(1, "id: " + fmid + ", target: " + target)
if fmid in footnotes and footnotes[fmid] == target:
cprint("warning: footnote id <fn id='" + fmid + "'> occurs multiple times. <footnote> link will be to the first.")
repl = "<a href='#f{0}' style='text-decoration:none'><sup><span style='font-size:0.9em'>{1}</span></sup></a>".format(target, fmid)
else:
footnotes[fmid] = target
repl = "<a id='r{0}'/><a href='#f{0}' style='text-decoration:none'><sup><span style='font-size:0.9em'>{1}</span></sup></a>".format(target, fmid)
l = line[0:m.start(0)] + repl
off = len(l) # Next loop
line = l + line[m.end(0):]
wb[i] = line
i += 1
# footnote targets and text
i = 0
while i < len(wb):
m = re.match("<footnote\s+(.*?)>", wb[i])
if m:
opts = m.group(1)
args = parseTagAttributes("footnote", opts, [ "id", "target" ])
fnid = args["id"]
target = args["target"]
wb[i] = "<div id='f{0}'><a href='#r{0}'>{1}</a></div>".format(target, fnid)
while not re.match("<\/footnote>", wb[i]):
i += 1
wb[i] = "</div> <!-- footnote end -->"
i += 1
def getGoogleFont(name):
cssurl = "http://fonts.googleapis.com/css?family=" + name
try:
with urllib.request.urlopen(cssurl) as r:
contents = r.read().decode('utf-8')
#contents = """
#@font-face {
# font-family: 'Tangerine';
# font-style: normal;
# font-weight: 700;
# src: local('Tangerine Bold'), local('Tangerine-Bold'), url(http://fonts.gstatic.com/s/tangerine/v7/UkFsr-RwJB_d2l9fIWsx3onF5uFdDttMLvmWuJdhhgs.ttf) format('truetype');
#}"""
except urllib.error.URLError as e:
fatal("Cannot find google font " + name + ": " + e.reason)
dprint(1, str(contents))
m = re.search("url\((.*?)\)[ ;]", str(contents))
if not m:
fatal("Bad font file " + cssurl + " from google: " + str(contents))
url = m.group(1)
dprint(1, "Remote ttf: " + url)
with urllib.request.urlopen(url) as r:
ttf = r.read()
# Turn something like Tangerine:bold into Tangerine-bold
#basename = re.sub(":", "-", name)
basename = name.replace(":", "-")
localFile = "images/font-" + basename + ".ttf"
with open(localFile, "wb") as f:
f.write(ttf)
f.close()
dprint(1, "Brought google font into " + localFile)
return localFile
def processFile(options, bn):
# run Lint for every format specified
# user may have included conditional code blocks
if 't' in options.formats:
lint = Lint(options.infile, "", options.debug, 't')
lint.run()
# all HTML derivatives
if re.search('h|k|e|p', options.formats):
lint = Lint(options.infile, "", options.debug, 'h')
lint.run()
# generate desired output formats
if 't' in options.formats:
outfile = "{}.txt".format(bn)
tb = Text(options.infile, outfile, options.debug, 't')
print("creating UTF-8 text")
tb.run()
if 'h' in options.formats:
outfile = "{}.html".format(bn)
hb = HTML(options.infile, outfile, options.debug, 'h')
print("creating HTML")
hb.run()
madeEpub = False
if 'e' in options.formats:
outfile = "{}-e.html".format(bn)
hb = HTML(options.infile, outfile, options.debug, 'e')
print("creating Epub")
hb.run()
preserveMargins = (config.uopt.getopt('preserve-margins', 'false') == 'true')
args = getConvertArgs(OPT_EPUB_ARGS, outfile, "{}.epub".format(bn), hb)
if config.uopt.getopt('epub-margin-left') != "": # added 27-Mar-2014
args.append("--margin-left")
args.append("{}".format(config.uopt.getopt('epub-margin-left')))
elif preserveMargins:
args.append("--margin-left -1")
if config.uopt.getopt('epub-margin-right') != "": # added 27-Mar-2014
args.append("--margin-right")
args.append("{}".format(config.uopt.getopt('epub-margin-right')))
elif preserveMargins:
args.append("--margin-right -1")
# call(OPT_EPUB_ARGS, shell=False)
js = " ".join(args)
msgs.dprint(1, js)
os.system(js)
if not options.saveint:
os.remove(outfile)
madeEpub = True
if 'k' in options.formats:
print("creating Kindle")
# make epub as source for kindle
outfile = "{}-e2.html".format(bn)
hb = HTML(options.infile, outfile, options.debug, 'k')
hb.run()
preserveMargins = (config.uopt.getopt('preserve-margins', 'false') == 'true')
args = getConvertArgs(OPT_EPUB_ARGS, outfile, "{}-e2.epub".format(bn), hb)
if preserveMargins:
args.extend(OPT_PRESERVE_MARGINS)
# call(OPT_EPUB_ARGS, shell=False)
js = " ".join(args)
msgs.dprint(1, js)
os.system(js)
# generate mobi with Kindlegen based on epub made by ebook-convert
# os.system("kindlegen {0}-k.html -o {0}.mobi".format(bn))
msgs.dprint(1, "kindlegen {0}-e2.epub -o {0}.mobi".format(bn))
os.system("kindlegen {0}-e2.epub -o {0}.mobi".format(bn))
if not options.saveint:
os.remove("{0}-e2.epub".format(bn))
os.remove("{0}-e2.html".format(bn))
if 'p' in options.formats:
outfile = "{}-p.html".format(bn)
hb = HTML(options.infile, outfile, options.debug, 'p')
print("creating PDF")
hb.run()
preserveMargins = (config.uopt.getopt('preserve-margins', 'false') == 'true')
args = getConvertArgs(OPT_PDF_ARGS, outfile, "{}-a5.pdf".format(bn), hb)
if preserveMargins:
args.extend(OPT_PRESERVE_MARGINS)
args.append('--pdf-default-font-size')
args.append(config.uopt.getopt('pdf-default-font-size', "13"))
else:
args.extend(OPT_PDF_ARGS_RL)
# fpgen option -> [ebook-convert option, value]
for k,v in PDF_CONFIG_OPTS.items():
args.append(v[0])
args.append(config.uopt.getopt(k, v[1]))
extra = os.environ.get('FPGEN_EBOOK_CONVERT_EXTRA_ARGS_PDF')
if extra:
print("Extra pdf conversion args: " + extra)
args.append(extra)
# call(OPT_PDF_ARGS, shell=False)
js = " ".join(args)
msgs.dprint(0, js)
os.system(js)
if not options.saveint:
os.remove(outfile)
# Create a zip file with the ebook id, and all the output formats
# appropriately named
if options.ebookid != "":
epubid = options.ebookid
zipname = epubid + ".zip"
print("Writing zip file " + zipname)
zip = zipfile.ZipFile(zipname, "w", compression = zipfile.ZIP_DEFLATED)
print("Adding " + bn + "-src.txt")
zip.write(bn + "-src.txt")
for suffix in [ ".txt", ".html", ".mobi", ".epub", "-a5.pdf" ]:
src = bn + suffix
target = epubid + suffix
print("Adding " + src + " as " + target)
zip.write(src, target)
for dir, subdirs, files in os.walk("images"):
for file in files:
image = dir + "/" + file
print("Adding image: " + image)
zip.write(image)
zip.close()
def main():
# process command line
parser = OptionParser()
parser.add_option("-i", "--infile",
dest="infile", default="",
help="input file")
parser.add_option("-f", "--format",
dest="formats", default="th",
help="format=thkep (text,HTML,Kindle,Epub,PDF)")
parser.add_option("-d", "--debug",
dest="debug", default="0",
help="set debug mode level")
parser.add_option("", "--save",
action="store_true", dest="saveint", default=False,
help="save intermediate file")
parser.add_option("--unittest",
action="store_true", dest="unittest", default=False, help="run unittests")
parser.add_option("", "--ebookid",
dest="ebookid", default="",
help="Create fadedpage zip file")
(options, args) = parser.parse_args()
print("fpgen {}".format(config.VERSION))
if options.unittest:
sys.argv = sys.argv[:1]
l = unittest.TestLoader();
tests = []
from fpgen import TestParseTableColumn, TestMakeTable, TestTableCellFormat
from parse import TestParseTagAttributes, TestParsing
from drama import TestDrama, TestOneDramaBlockMethod
from testtext import TestTextInline, TestTextRewrap
from footnote import TestFootnote
from template import TestTemplate
for cl in [
TestParseTableColumn, TestMakeTable, TestDrama, TestParsing,
TestParseTagAttributes, TestOneDramaBlockMethod, TestTextRewrap,
TestTextInline, TestTableCellFormat, TestTemplate, TestFootnote
]:
tests.append(l.loadTestsFromTestCase(cl))
tests = l.suiteClass(tests)
unittest.TextTestRunner(verbosity=2).run(tests)
exit(0)
if options.ebookid != "":
options.formats = "thkep"
if not re.match("^20[01]\d[01]\d[0-9a-zA-Z][0-9a-zA-Z]$", options.ebookid):
print("Ebookid doesn't look correct: " + options.ebookid)
exit(1)
tmp = options.formats
tmp = re.sub('a|h|t|k|e|p', '', tmp)
if not tmp == '':
print("format option {} not supported".format(tmp))
exit(1)
# 'a' format is 'all'
if options.formats == 'a':
options.formats = "htpek"
# check input filename
m = re.match('(.*?)-src.txt', options.infile)
if not m:
print("source filename must end in \"-src.txt\".")
print("example: midnight-src.txt will generate midnight.html, midnight.txt")
exit(1)
else:
bn = m.group(1)
# run Lint for every format specified
# user may have included conditional code blocks
if 't' in options.formats:
lint = Lint(options.infile, "", options.debug, 't')
lint.run()
# all HTML derivatives
if re.search('h|k|e|p', options.formats):
lint = Lint(options.infile, "", options.debug, 'h')
lint.run()
# set defaults
# --remove-paragraph-spacing removed
# --remove-first-image removed
OPT_EPUB_ARGS = [
"ebook-convert",
"",
"",
"--cover", "\"images/cover.jpg\"",
#"--flow-size", "500",
"--change-justification", "\"left\"",
"--chapter-mark", "\"none\"",
"--disable-remove-fake-margins",
"--page-breaks-before", "\"//h:div[@style='page-break-before:always'] | //*[(name()='h1' or name()='h2') and not(@class='nobreak')]\"",
"--sr1-search", "\"<hr class=.pbk./>\"",
"--sr1-replace", "\"<div style='page-break-before:always'></div>\"",
"--sr1-search", "\"<br\/><br\/>\"",
"--sr1-replace", "—",
"--chapter", "\"//*[(name()='h1' or name()='h2')]\"",
"--level1-toc \"//h:h1\" --level2-toc \"//h:h2\""
]
# unused kindle options
# "--change-justification", "left", (destroys centered blocks)
# "--mobi-file-type", "new",
#
OPT_KINDLE_ARGS = [
"ebook-convert",
"",
"",
"--cover", "\"images/cover.jpg\"",
"--no-inline-toc",
"--sr1-search", "\"<hr class=.pbk./>\"",
"--sr1-replace", "\"<div style='page-break-before:always'></div>\"",
"--sr1-search", "\"<br\/><br\/>\"",
"--sr1-replace", "—",
"--chapter", "\"//*[(name()='h1' or name()='h2')]\""
]
OPT_PDF_ARGS = [
"ebook-convert",
"",
"",
"--cover", "\"images/cover.jpg\"",
"--paper-size", "\"a5\"",
"--pdf-default-font-size", "\"13\"",
"--margin-left", "\"20\"",
"--margin-right", "\"20\"",
"--margin-top", "\"20\"",
"--margin-bottom", "\"20\"",
"--chapter-mark", "\"none\"",
"--disable-remove-fake-margins",
"--page-breaks-before", "\"//h:div[@style='page-break-before:always'] | //*[(name()='h1' or name()='h2') and not(@class='nobreak')]\"",
"--sr1-search", "\"<hr class=.pbk./>\"",
"--sr1-replace", "\"<div style='page-break-before:always'></div>\"",
"--sr1-search", "\"<br\/><br\/>\"",
"--sr1-replace", "—"
]
# generate desired output formats
if 't' in options.formats:
outfile = "{}.txt".format(bn)
tb = Text(options.infile, outfile, options.debug, 't')
print("creating UTF-8 text")
tb.run()
if 'h' in options.formats:
outfile = "{}.html".format(bn)
hb = HTML(options.infile, outfile, options.debug, 'h')
print("creating HTML")
hb.run()
madeEpub = False
if 'e' in options.formats:
outfile = "{}-e.html".format(bn)
hb = HTML(options.infile, outfile, options.debug, 'e')
print("creating Epub")
hb.run()
OPT_EPUB_ARGS[1] = "{}-e.html".format(bn)
OPT_EPUB_ARGS[2] = "{}.epub".format(bn)
if config.pn_cover != "":
OPT_EPUB_ARGS[4] = config.pn_cover
if config.uopt.getopt('epub-margin-left') != "": # added 27-Mar-2014
OPT_EPUB_ARGS.append("--margin-left")
OPT_EPUB_ARGS.append("{}".format(config.uopt.getopt('epub-margin-left')))
if config.uopt.getopt('epub-margin-right') != "": # added 27-Mar-2014
OPT_EPUB_ARGS.append("--margin-right")
OPT_EPUB_ARGS.append("{}".format(config.uopt.getopt('epub-margin-right')))
# call(OPT_EPUB_ARGS, shell=False)
js = " ".join(OPT_EPUB_ARGS)
msgs.dprint(1, js)
os.system(js)
if not options.saveint:
os.remove(outfile)
madeEpub = True
if 'k' in options.formats:
print("creating Kindle")
# make epub as source for kindle
outfile = "{}-e2.html".format(bn)
hb = HTML(options.infile, outfile, options.debug, 'k')
hb.run()
OPT_EPUB_ARGS[1] = "{}-e2.html".format(bn)
OPT_EPUB_ARGS[2] = "{}-e2.epub".format(bn)
if config.pn_cover != "":
OPT_EPUB_ARGS[4] = config.pn_cover
# call(OPT_EPUB_ARGS, shell=False)
js = " ".join(OPT_EPUB_ARGS)
msgs.dprint(1, js)
os.system(js)
# generate mobi with Kindlegen based on epub made by ebook-convert
# os.system("kindlegen {0}-k.html -o {0}.mobi".format(bn))
msgs.dprint(1, "kindlegen {0}-e2.epub -o {0}.mobi".format(bn))
os.system("kindlegen {0}-e2.epub -o {0}.mobi".format(bn))
if not options.saveint:
os.remove("{0}-e2.epub".format(bn))
os.remove("{0}-e2.html".format(bn))
if 'p' in options.formats:
outfile = "{}-p.html".format(bn)
hb = HTML(options.infile, outfile, options.debug, 'p')
print("creating PDF")
hb.run()
OPT_PDF_ARGS[1] = "{}-p.html".format(bn)
OPT_PDF_ARGS[2] = "{}-a5.pdf".format(bn)
if config.uopt.getopt('pdf-default-font-size') != "":
OPT_PDF_ARGS.append("--pdf-default-font-size")
OPT_PDF_ARGS.append("{}".format(config.uopt.getopt('pdf-default-font-size')))
if config.uopt.getopt('pdf-margin-left') != "":
OPT_PDF_ARGS.append("--margin-left")
OPT_PDF_ARGS.append("{}".format(config.uopt.getopt('pdf-margin-left')))
if config.uopt.getopt('pdf-margin-right') != "":
OPT_PDF_ARGS.append("--margin-right")
OPT_PDF_ARGS.append("{}".format(config.uopt.getopt('pdf-margin-right')))
if config.pn_cover != "":
OPT_PDF_ARGS[4] = config.pn_cover
# call(OPT_PDF_ARGS, shell=False)
js = " ".join(OPT_PDF_ARGS)
msgs.dprint(1, js)
os.system(js)
if not options.saveint:
os.remove(outfile)
# Create a zip file with the ebook id, and all the output formats
# appropriately named
if options.ebookid != "":
epubid = options.ebookid
zipname = epubid + ".zip"
print("Writing zip file " + zipname)
zip = zipfile.ZipFile(zipname, "w", compression = zipfile.ZIP_DEFLATED)
print("Adding " + bn + "-src.txt")
zip.write(bn + "-src.txt")
for suffix in [ ".txt", ".html", ".mobi", ".epub", "-a5.pdf" ]:
src = bn + suffix
target = epubid + suffix
print("Adding " + src + " as " + target)
zip.write(src, target)
for dir, subdirs, files in os.walk("images"):
for file in files:
image = dir + "/" + file
print("Adding image: " + image)
zip.write(image)
zip.close()
def expandCondition(args, block):
dprint(1, "Expanding: " + args)
keyword = args.strip()
if keyword in keys:
return block
return [ ]
def chapterTemplates(self, lines, properties, meta):
# Figure out what template name to use.
if "template-chapter" in properties:
chapterTemplateName = properties["template-chapter"]
else:
chapterTemplateName = "default"
if "template-chapter-first" in properties:
chapterTemplateNameFirst = properties["template-chapter-first"]
else:
chapterTemplateNameFirst = "default-first"
dprint(1, "Chapter Template: Using first: " + chapterTemplateNameFirst + \
", subsequent: " + chapterTemplateName)
# Now we can set the globals, since we have now extracted all the metadata
self.setGlobals(meta)
# Figure out which templates we are going to use.
tFirst = self.get([ "chapter" ], chapterTemplateNameFirst)
t = self.get([ "chapter" ], chapterTemplateName)
regexMacro = re.compile("<expand-macro\s+(.*?)/?>")
i = 0
first = True
while i < len(lines):
line = lines[i]
if line.startswith("<chap-head"):
keys = {}
opts, keys["chap-head"] = parseLineEntry("chap-head", line)
j = i+1
while j < len(lines) and re.match(lines[j], "^\s*$"):
j += 1
if j == len(lines):
fatal("End of file after <chap-head>")
if opts != "":
attributes = parseTagAttributes("chap-head", opts, [ "vars" ])
dprint(1, "<chap-head> attributes: " + str(attributes))
if "vars" in attributes:
vars = parseOption("chap-head", attributes["vars"])
dprint(1, "<chap-head> vars: " + str(vars))
keys.update(vars)
line = lines[j]
if line.startswith("<sub-head"):
opts, keys["sub-head"] = parseLineEntry("sub-head", line)
else:
# Do not eat this line!
j -= 1
# If the first we've seen, it starts the book
if first:
templ = tFirst
first = False
else:
templ = t
dprint(1, "expand keys: " + str(keys))
replacement = templ.expand(keys)
dprint(2, "replace " + str(lines[i:j+1]) + " with " + str(replacement))
lines[i:j+1] = replacement
i += len(replacement)
continue
if line.startswith("<sub-head>"):
fatal("Found <sub-head> not after a <chap-head>: " + line)
# What about multiple macro expansions on a line? Or recursion?
# Make it simpler for now by just punting: if you expand, then we move on
# to the next line.
m = regexMacro.search(line)
if m:
opts = m.group(1)
attributes = parseTagAttributes("expand-macro", opts, [ "name", "vars" ])
if not "name" in attributes:
fatal("expand-macro: No macro name given: " + line)
template = self.get([ "macro" ], attributes["name"])
if "vars" in attributes:
keys = parseOption("expand-macro", attributes["vars"])
else:
keys = {}
replacement = template.expand(keys)
prefix = line[:m.start(0)]
suffix = line[m.end(0):]
if len(replacement) == 0:
# If the template returns nothing, then you end up with a single line of
# the prefix and suffix around the <expand-macro>
replacement = [ prefix + suffix ]
else:
# Otherwise the prefix goes on the first line; and the suffix at the end of
# the last; which might be the same single line.
replacement[0] = prefix + replacement[0]
replacement[-1] = replacement[-1] + suffix
lines[i:i+1] = replacement
i += len(replacement)
continue
i += 1
