def generate(config):
sitemap = []
target = config['target']
src = config['src']
if not os.path.exists(target):
os.mkdir(target)
gens = [klass(config) for klass in generators]
match = {}
for gen in gens:
for ext in gen.exts:
match[ext] = gen
cats = config.get('categories', '').split(',')
cats = [cat.strip() for cat in cats if cat.strip() != '']
config['categories'] = cats
for root, dirs, files in os.walk(src):
for file in files:
if file.startswith('_'):
continue
if file.endswith('.DS_Store'):
continue
ext = os.path.splitext(file)[-1]
path = os.path.join(root, file)
if ext in ('.mako', '.un~'):
continue
# configurable XXX
if (os.path.split(path)[0] == 'src/auteurs' and
file.endswith('.rst')):
continue
location = path[len(src) + 1:]
file_target = os.path.join(target, location)
target_dir = os.path.dirname(file_target)
file_target_name, ext = os.path.splitext(file_target)
url_target = file_target_name[len(target):] + '.html'
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# now calling the right generator
if ext in match:
try:
match[ext](path, file_target, url_target, config=config)
except Exception:
logger.info('Failed on %s' % path)
raise
sitemap.append((url_target, True))
else:
logger.info('Copying %r' % file_target)
shutil.copyfile(path, file_target)
# media
media = config['media']
if os.path.exists(media):
shutil.rmtree(media)
shutil.copytree('media', media)
# building category pages now
categories = defaultdict(list)
for key, index in get_index():
path, title = key.split(':')
for key, value in index.items():
if key != 'category':
continue
for cat in value:
categories[cat].append((path, title))
for wanted in config['categories']:
if wanted in categories:
continue
categories[wanted] = []
gen = Mako(config)
for cat, paths in categories.items():
logger.info('Generating category %r' % cat)
url_target = '/%s.html' % cat
file_target = os.path.join(target, cat + '.html')
gen(config['cats'], file_target, url_target, paths=paths,
title=cat.capitalize(), config=config,
category=cat)
sitemap.append((url_target, False))
# creating the authors index page
authors = {}
for key, index in get_index():
path, title = key.split(':')
for key, author_name in index.items():
if key != 'author':
continue
author_id = str2authorid(author_name)
if author_id in authors:
authors[author_id]['articles'].append((title, path))
else:
# should be configurable
link = '/auteurs/%s.html' % author_id
authors[author_id] = {'link': link,
'articles': [(title, path)],
'name': author_name}
authors = authors.items()
authors.sort()
# XXX should be configurable...
authors_template = os.path.join(src, 'auteurs', 'index.mako')
logger.info('Generating authors index')
url_target = '/auteurs/index.html'
file_target = os.path.join(target, 'auteurs', 'index.html')
gen(authors_template, file_target, url_target, authors=authors,
title="Auteurs", config=config)
sitemap.append((url_target, False))
# creating the author pages
gen = RestructuredText(config)
for author_id, data in authors:
template = os.path.join(src, 'auteurs', '%s.rst' % author_id)
if not os.path.exists(template):
logger.warning('Template not found for author %r' % author_id)
continue
# we're supposed to find an author .rst file in /auteur
url_target = '/auteurs/%s.html' % author_id
file_target = os.path.join(target, 'auteurs', '%s.html' % author_id)
fd, tmp = tempfile.mkstemp()
os.close(fd)
def _line(line):
return u'- `%s <%s>`_' % (line[0], line[1])
articles = AUTHOR_ARTICLES % '\n'.join([_line(data) for data in
data['articles']])
with codecs.open(template, encoding='utf8') as source_file:
with codecs.open(tmp, 'w', encoding='utf8') as target_file:
data = source_file.read()
data += articles + '\n'
target_file.write(data)
try:
gen(tmp, file_target, url_target, config=config)
finally:
os.remove(tmp)
sitemap.append((url_target, True))
# creating sitemap
sitemap_file = os.path.join(target, 'sitemap.json')
logger.info('Generating sitemap at %r' % sitemap_file)
now = datetime.datetime.now().isoformat()
urlset = [{'loc': loc, 'lastmod': now,
'changefreq': 'monthly',
'priority': 0.1,
'indexable': int(indexable)}
for loc, indexable in sitemap]
with open(sitemap_file, 'w') as f:
f.write(json.dumps({'urlset': urlset}))
# asking Trouvailles to index the web site
logger.info('Indexing the whole website')
url = config['search_server']
data = {'sitemap': config['sitemap']}
headers = {'Content-type': 'application/json'}
r = requests.post(url, data=json.dumps(data), headers=headers)
if r.status_code != 200:
logger.info('Indexation failed')
logger.info(r.status_code)
logger.info(r.content)
def _tree(node, document, title, config):
"""Renders a node in HTML.
"""
cdn = config['cdn']
text = []
klass = node.__class__.__name__
if klass == 'transition':
text.append('<hr/>')
elif klass == 'system_message':
pass
elif klass == 'block_quote':
text.append('<blockquote>')
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</blockquote>')
elif klass == 'paragraph':
text.append('<p>')
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</p>')
elif klass == 'Text':
text.append(node.astext())
elif klass == 'literal_block':
text.append('<div class="syntax rounded">')
text.append(hilite(node))
text.append('</div>')
elif klass == 'sidebar':
text.append('<div class="alert alert-info">')
text.append('<h4>%s</h4>' % node.children[0].astext())
for child in node.children[1:]:
text.append(_tree(child, document, title, config))
text.append('</div>')
elif klass == 'note':
node.attributes['class'] = 'well note'
text.extend(render_simple_tag(node, document, title, config,
'div', strip_child=False))
elif klass == 'warning':
text.append('<div class="alert">')
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</div>')
elif klass == 'table':
node.attributes['class'] = 'table'
text.extend(render_simple_tag(node, document, title, config,
'table', strip_child=True))
elif klass == 'image':
if node.get('uri').startswith('icon'):
return '<i class="%s"></i>' % node.get('uri')
nolegend = False
if node.hasattr('scale'):
span = 12. * (float(node['scale']) / 100.)
offset = int((12-span) / 2.) - 1
span = 'span%d' % int(span)
if offset > 0:
span += ' offset%d' % offset
else:
span = 'span12'
if node.hasattr('uri'):
uri = node['uri']
file_ = os.path.split(uri)[-1]
if file_ in config['icons']:
class_ = 'subst'
nolegend = True
else:
text.append('<div class="row-fluid">')
class_ = 'centered %s' % span
text.append('<img class="%s" src="%s">' % (class_, uri))
else:
text.append('<div class="row-fluid">')
text.append('<img class="centered %s">' % span)
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</img>')
if not nolegend and 'alt' in node:
text.append('<span class="legend %s">' % span)
text.append(node['alt'])
text.append('</span>')
text.append('</div>')
elif klass == 'figure':
if len(node['classes']) > 0:
floating = ' '.join(node['classes'])
else:
# let's use a row-fluid
floating = None
data = {}
for child in node.children:
klass = child.__class__.__name__
data[klass] = child
if 'image' not in data and 'reference' in data:
data['image'] = data['reference'].children[0]
# scaling
if 'scale' in data['image']:
scale = float(data['image']['scale'])
span = 12. * (scale / 100.)
offset = int((12-span) / 2.)
span = 'span%d' % int(span)
if offset > 0:
span += ' offset%d' % offset
else:
span = 'span12'
linked = 'reference' in data
# image
uri = data['image']['uri']
file_ = os.path.split(uri)[-1]
if file_ in config['icons']:
class_ = 'subst'
nolegend = True
else:
if floating is None:
text.append('<div class="row-fluid">')
# subdiv
if floating is None:
text.append('<div class="%s">' % span)
else:
text.append('<div class="%s">' % floating)
# url
if linked:
refuri = data['reference']['refuri']
if ('faitmain.org' not in refuri and not refuri.startswith('/')
and int(config.get('shorten', 1)) == 1):
refuri = shorten(refuri, config['shortener_server'],
config['shortener_key'],
config.get('amazon_tag'))
text.append('<a href="%s">' % refuri)
if not uri.startswith('http'):
uri = '/'.join([config['cdn'], document.split('/')[1], uri])
text.append('<img class="centered span12" src="%s"></img>' % uri)
# caption
if 'caption' in data:
text.append('<span class="legend">')
for child in data['caption'].children:
text.append(_tree(child, document, title, config))
text.append('</span>')
if linked:
text.append('</a>')
text.append('</div>')
if floating is None:
text.append('</div>')
elif klass == 'reference': # link
if node.hasattr('refid'):
text.append('<a href="#%s">' % node['refid'])
elif node.hasattr('refuri'):
refuri = node['refuri']
if 'wikipedia.org' in refuri:
text.append('<a href="%s" class="wikipedia">' % refuri)
else:
if ('faitmain.org' not in refuri and not refuri.startswith('/')
and int(config.get('shorten', 1)) == 1):
refuri = shorten(refuri, config['shortener_server'],
config['shortener_key'],
config.get('amazon_tag'))
text.append('<a href="%s">' % refuri)
else:
text.append('<a>')
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</a>')
elif klass == 'target':
# ??
pass
elif klass == 'section':
section_title = node.children[0][0].astext()
id = node.attributes['ids'][0]
index(document, title, 'sections', (section_title, id), append=True)
text.append('<div id="%s" class="section">' % id)
header = (u'<h2>%s <a class="headerlink" href="#%s"'
u'title="Lien vers cette section">\xb6</a></h2>')
header = header % (section_title, id)
text.append(header)
for child in node.children[1:]:
text.append(_tree(child, document, title, config))
text.append('</div>')
elif klass == 'substitution_definition':
#uri = node.children[0].attributes['uri']
#text.append('<img class="subst" src="%s"></img>' % uri)
pass
elif klass == 'docinfo':
# reading metadata
for child in node.children:
text.append(_tree(child, document, title, config))
elif klass == 'author':
value = node.astext()
index(document, title, 'author', value)
author_id = str2authorid(value)
text.append('<img class="subst" '
'src="%s/media/pen.png">' % cdn)
text.append('</img>')
text.append('<a href="%s/auteurs/%s.html">%s</a>' %
(config['siteurl'], author_id, value))
elif klass == 'date':
index(document, title, 'date', node.astext())
elif klass == 'field':
name = node.children[0].astext()
value = node.children[1].astext()
if name == 'category':
text.append('<img class="subst" '
'src="%s/media/info.png">' % cdn)
text.append('</img>')
cats = value.split(',')
index(document, title, name, cats)
cats = ['<a href="%s/%s.html">%s</a>' % (config['siteurl'],
cat, cat.capitalize())
for cat in cats]
text.append(' | '.join(cats))
elif name == 'level':
index(document, title, name, value)
text.append('<img class="subst" '
'src="%s/media/flash.png">' % cdn)
text.append('</img>')
text.append('<strong>Niveau</strong>: %s' % value.capitalize())
elif name == 'translator':
index(document, title, name, value)
text.append('<img class="subst" '
'src="%s/media/translation.png">' % cdn)
text.append('</img>')
author_id = str2authorid(value)
msg = ('<strong>Traduction</strong>: '
'<a href="%s/auteurs/%s.html">%s</a>')
text.append(msg % (config['siteurl'], author_id, value))
elif klass == 'colspec': # table colspec
pass
elif klass == 'entry': # table entry
tagname = 'td'
if node.parent.parent.tagname == 'thead':
tagname = 'th'
text.extend(render_simple_tag(node, document, title, config,
tagname, strip_child=True))
elif klass in SIMPLE_TAGS:
tagname, strip_child = SIMPLE_TAGS[klass]
text.extend(render_simple_tag(node, document, title, config,
tagname, strip_child=strip_child))
else:
raise NotImplementedError(node)
return ' '.join(text)
def generate(config):
sitemap = []
target = config["target"]
src = config["src"]
if not os.path.exists(target):
os.mkdir(target)
gens = [klass(config) for klass in generators]
match = {}
for gen in gens:
for ext in gen.exts:
match[ext] = gen
cats = config.get("categories", "").split(",")
cats = [cat.strip() for cat in cats if cat.strip() != ""]
config["categories"] = cats
for path in ("cnd", "siteurl"):
if os.path.exists(config[path]):
config[path] = os.path.abspath(config[path])
source_root = config.get("source_root")
for root, dirs, files in os.walk(src):
for file in files:
if file.startswith("_"):
continue
if file.endswith(".DS_Store"):
continue
ext = os.path.splitext(file)[-1]
path = os.path.join(root, file)
if ext in (".mako", ".un~"):
continue
# configurable XXX
if os.path.split(path)[0] == "src/auteurs" and file.endswith(".rst"):
continue
location = path[len(src) + 1 :]
file_target = os.path.join(target, location)
target_dir = os.path.dirname(file_target)
file_target_name, ext = os.path.splitext(file_target)
url_target = file_target_name[len(target) :] + ".html"
if source_root is not None:
config["github"] = os.path.join(source_root, location)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# now calling the right generator
if ext in match:
try:
match[ext](path, file_target, url_target, config=config)
except Exception:
logger.info("Failed on %s" % path)
raise
sitemap.append((url_target, True))
else:
logger.info("Copying %r" % file_target)
shutil.copyfile(path, file_target)
if "github" in config:
del config["github"]
# media
media = str(config["media"])
if os.path.exists(media):
shutil.rmtree(media)
shutil.copytree("media", media)
# building category pages now
categories = defaultdict(list)
for key, index in get_index():
path, title = key.split(":")
for key, value in index.items():
if key != "category":
continue
for cat in value:
categories[cat].append((path, title))
for wanted in config["categories"]:
if wanted in categories:
continue
categories[wanted] = []
gen = Mako(config)
for cat, paths in categories.items():
logger.info("Generating category %r" % cat)
url_target = "/%s.html" % cat
file_target = os.path.join(target, cat + ".html")
gen(config["cats"], file_target, url_target, paths=paths, title=cat.capitalize(), config=config, category=cat)
sitemap.append((url_target, False))
# creating the authors index page
authors = {}
for key, index in get_index():
path, title = key.split(":")
for key, authors_ in index.items():
if key != "author":
continue
for author_name in authors_:
author_id = str2authorid(author_name)
if author_id in authors:
authors[author_id]["articles"].append((title, path))
else:
# should be configurable
link = "%s/auteurs/%s.html" % (config["siteurl"], author_id)
authors[author_id] = {"link": link, "articles": [(title, path)], "name": author_name}
authors = authors.items()
authors.sort()
# XXX should be configurable...
authors_template = os.path.join(src, "auteurs", "index.mako")
logger.info("Generating authors index")
url_target = "/auteurs/index.html"
file_target = os.path.join(target, "auteurs", "index.html")
gen(authors_template, file_target, url_target, authors=authors, title="Auteurs", config=config)
sitemap.append((url_target, False))
# creating the author pages
gen = RestructuredText(config)
for author_id, data in authors:
template = os.path.join(src, "auteurs", "%s.rst" % author_id)
if not os.path.exists(template):
logger.warning("Template not found for author %r" % author_id)
continue
# we're supposed to find an author .rst file in /auteur
url_target = "/auteurs/%s.html" % author_id
file_target = os.path.join(target, "auteurs", "%s.html" % author_id)
fd, tmp = tempfile.mkstemp()
os.close(fd)
def _line(line):
title, path = line
path = "%s/%s" % (config["siteurl"].rstrip("/"), path)
return u"- `%s <%s>`_" % (title, path)
articles = AUTHOR_ARTICLES % "\n".join([_line(data) for data in data["articles"]])
with codecs.open(template, encoding="utf8") as source_file:
with codecs.open(tmp, "w", encoding="utf8") as target_file:
data = source_file.read()
data += articles + "\n"
target_file.write(data)
try:
gen(tmp, file_target, url_target, config=config)
finally:
os.remove(tmp)
sitemap.append((url_target, True))
# create the atom feed
siteurl = config.get("siteurl")
feed = Atom1Feed(
title="FaitMain.org", link=siteurl, feed_url=siteurl + "feed.xml", description=config.get("site-description")
)
for key, article in get_articles():
path, title = key.split(":")
feed.add_item(
title=title,
link="%s/%s" % (siteurl, path),
description=article["body"],
categories=article["category"],
author_name=article["author"],
pubdate=article["date"],
)
with open(os.path.join(target, "feed.xml"), "w") as f:
feed.write(f, "utf-8")
# creating sitemap
sitemap_file = os.path.join(target, "sitemap.json")
logger.info("Generating sitemap at %r" % sitemap_file)
now = datetime.datetime.now().isoformat()
urlset = [
{"loc": loc, "lastmod": now, "changefreq": "monthly", "priority": 0.1, "indexable": int(indexable)}
for loc, indexable in sitemap
]
with open(sitemap_file, "w") as f:
f.write(json.dumps({"urlset": urlset}))
# asking Trouvailles to index the web site
if int(config["indexing"]) == 0:
return
logger.info("Indexing the whole website")
url = config["search_server"]
data = {"sitemap": config["sitemap"]}
headers = {"Content-type": "application/json"}
r = requests.post(url, data=json.dumps(data), headers=headers)
if r.status_code != 200:
logger.info("Indexation failed")
logger.info(r.status_code)
logger.info(r.content)
def _tree(node, document, title, config):
"""Renders a node in HTML.
"""
cnd = config["cnd"]
text = []
klass = node.__class__.__name__
if klass == "transition":
text.append("<hr/>")
elif klass == "system_message":
pass
elif klass == "block_quote":
text.append("<blockquote>")
for child in node.children:
text.append(_tree(child, document, title, config))
text.append("</blockquote>")
elif klass == "paragraph":
text.append("<p>")
for child in node.children:
text.append(_tree(child, document, title, config))
text.append("</p>")
elif klass == "Text":
text.append(node.astext())
elif klass == "literal_block":
text.append('<div class="syntax rounded">')
text.append(hilite(node))
text.append("</div>")
elif klass == "sidebar":
text.append('<div class="alert alert-info">')
text.append("<h4>%s</h4>" % node.children[0].astext())
for child in node.children[1:]:
text.append(_tree(child, document, title, config))
text.append("</div>")
elif klass == "note":
node.attributes["class"] = "well note"
text.extend(render_simple_tag(node, document, title, config, "div", strip_child=False))
elif klass == "warning":
text.append('<div class="alert">')
for child in node.children:
text.append(_tree(child, document, title, config))
text.append("</div>")
elif klass == "table":
node.attributes["class"] = "table"
text.extend(render_simple_tag(node, document, title, config, "table", strip_child=True))
elif klass == "image":
if node.get("uri").startswith("icon"):
return '<i class="%s"></i>' % node.get("uri")
nolegend = False
if node.hasattr("scale"):
span = 12.0 * (float(node["scale"]) / 100.0)
offset = int((12 - span) / 2.0) - 1
span = "span%d" % int(span)
if offset > 0:
span += " offset%d" % offset
else:
span = "span12"
if node.hasattr("uri"):
uri = node["uri"]
file_ = os.path.split(uri)[-1]
if file_ in config["icons"]:
class_ = "subst"
nolegend = True
else:
text.append('<div class="row-fluid">')
class_ = "centered %s" % span
text.append('<img class="%s" src="%s">' % (class_, uri))
else:
text.append('<div class="row-fluid">')
text.append('<img class="centered %s">' % span)
for child in node.children:
text.append(_tree(child, document, title, config))
text.append("</img>")
if not nolegend and "alt" in node:
text.append('<span class="legend %s">' % span)
text.append(node["alt"])
text.append("</span>")
text.append("</div>")
elif klass == "figure":
if len(node["classes"]) > 0:
floating = " ".join(node["classes"])
else:
# let's use a row-fluid
floating = None
data = {}
for child in node.children:
klass = child.__class__.__name__
data[klass] = child
if "image" not in data and "reference" in data:
data["image"] = data["reference"].children[0]
# scaling
if "scale" in data["image"]:
scale = float(data["image"]["scale"])
span = 12.0 * (scale / 100.0)
offset = int((12 - span) / 2.0)
span = "span%d" % int(span)
if offset > 0:
span += " offset%d" % offset
else:
span = "span12"
linked = "reference" in data
# image
uri = data["image"]["uri"]
file_ = os.path.split(uri)[-1]
if file_ in config["icons"]:
class_ = "subst"
nolegend = True
else:
if floating is None:
text.append('<div class="row-fluid">')
# subdiv
if floating is None:
text.append('<div class="%s">' % span)
else:
text.append('<div class="%s">' % floating)
# url
if linked:
refuri = data["reference"]["refuri"]
if "faitmain.org" not in refuri and not refuri.startswith("/") and int(config.get("shorten", 1)) == 1:
refuri = shorten(refuri, config["shortener_server"], config["shortener_key"], config.get("amazon_tag"))
text.append('<a href="%s">' % refuri)
text.append('<img class="centered span12" src="%s"></img>' % uri)
# caption
if "caption" in data:
text.append('<span class="legend">')
for child in data["caption"].children:
text.append(_tree(child, document, title, config))
text.append("</span>")
if linked:
text.append("</a>")
text.append("</div>")
if floating is None:
text.append("</div>")
elif klass == "reference": # link
if node.hasattr("refid"):
text.append('<a href="#%s">' % node["refid"])
elif node.hasattr("refuri"):
refuri = node["refuri"]
if "wikipedia.org" in refuri:
text.append('<a href="%s" class="wikipedia">' % refuri)
else:
if "faitmain.org" not in refuri and not refuri.startswith("/") and int(config.get("shorten", 1)) == 1:
refuri = shorten(
refuri, config["shortener_server"], config["shortener_key"], config.get("amazon_tag")
)
text.append('<a href="%s">' % refuri)
else:
text.append("<a>")
for child in node.children:
text.append(_tree(child, document, title, config))
text.append("</a>")
elif klass == "target":
# ??
pass
elif klass == "section":
section_title = node.children[0][0].astext()
id = node.attributes["ids"][0]
index(document, title, "sections", (section_title, id), append=True)
text.append('<div id="%s" class="section">' % id)
header = u'<h2>%s <a class="headerlink" href="#%s"' u'title="Lien vers cette section">\xb6</a></h2>'
header = header % (section_title, id)
text.append(header)
for child in node.children[1:]:
text.append(_tree(child, document, title, config))
text.append("</div>")
elif klass == "substitution_definition":
# uri = node.children[0].attributes['uri']
# text.append('<img class="subst" src="%s"></img>' % uri)
pass
elif klass == "docinfo":
# reading metadata
for child in node.children:
text.append(_tree(child, document, title, config))
elif klass == "author":
value = node.astext()
index(document, title, "author", value)
author_id = str2authorid(value)
text.append('<img class="subst" ' 'src="%s/media/pen.png">' % cnd)
text.append("</img>")
text.append('<a href="/auteurs/%s.html">%s</a>' % (author_id, value))
elif klass == "date":
index(document, title, "date", node.astext())
elif klass == "field":
name = node.children[0].astext()
value = node.children[1].astext()
if name == "category":
text.append('<img class="subst" ' 'src="%s/media/info.png">' % cnd)
text.append("</img>")
cats = value.split(",")
index(document, title, name, cats)
cats = ['<a href="/%s.html">%s</a>' % (cat, cat.capitalize()) for cat in cats]
text.append(" | ".join(cats))
elif name == "level":
index(document, title, name, value)
text.append('<img class="subst" ' 'src="%s/media/flash.png">' % cnd)
text.append("</img>")
text.append("<strong>Niveau</strong>: %s" % value.capitalize())
elif name == "translator":
index(document, title, name, value)
text.append('<img class="subst" ' 'src="%s/media/translation.png">' % cnd)
text.append("</img>")
author_id = str2authorid(value)
msg = "<strong>Traduction</strong>: " '<a href="/auteurs/%s.html">%s</a>'
text.append(msg % (author_id, value))
elif klass == "colspec": # table colspec
pass
elif klass == "entry": # table entry
tagname = "td"
if node.parent.parent.tagname == "thead":
tagname = "th"
text.extend(render_simple_tag(node, document, title, config, tagname, strip_child=True))
elif klass in SIMPLE_TAGS:
tagname, strip_child = SIMPLE_TAGS[klass]
text.extend(render_simple_tag(node, document, title, config, tagname, strip_child=strip_child))
else:
raise NotImplementedError(node)
return " ".join(text)
def _tree(node, document, title, config):
"""Renders a node in HTML.
"""
global _LEVEL
_LEVEL += 1
cnd = config['cnd']
text = []
klass = node.__class__.__name__
if klass == 'transition':
text.append('<hr/>')
elif klass == 'system_message':
pass
elif klass == 'block_quote':
text.append('<blockquote>')
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</blockquote>')
elif klass == 'paragraph':
text.append('<p>')
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</p>')
elif klass == 'Text':
text.append(node.astext())
elif klass == 'literal_block':
text.append('<div class="syntax rounded">')
text.append(hilite(node))
text.append('</div>')
elif klass == 'sidebar':
text.append('<div class="alert alert-info">')
text.append('<h4>%s</h4>' % node.children[0].astext())
for child in node.children[1:]:
text.append(_tree(child, document, title, config))
text.append('</div>')
elif klass == 'note':
node.attributes['class'] = 'well note'
text.extend(
render_simple_tag(node,
document,
title,
config,
'div',
strip_child=False))
elif klass == 'warning':
text.append('<div class="alert">')
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</div>')
elif klass == 'table':
node.attributes['class'] = 'table'
text.extend(
render_simple_tag(node,
document,
title,
config,
'table',
strip_child=True))
elif klass == 'image':
if node.get('uri').startswith('icon'):
return '<i class="%s"></i>' % node.get('uri')
nolegend = False
if node.hasattr('scale'):
span = 12. * (float(node['scale']) / 100.)
offset = int((12 - span) / 2.) - 1
span = 'span%d' % int(span)
if offset > 0:
span += ' offset%d' % offset
else:
span = 'span12'
if node.hasattr('uri'):
uri = node['uri']
file_ = os.path.split(uri)[-1]
if file_ in config['icons']:
class_ = 'subst'
nolegend = True
else:
text.append('<div class="row-fluid">')
class_ = 'centered %s' % span
text.append('<img class="%s" src="%s">' % (class_, uri))
else:
text.append('<div class="row-fluid">')
text.append('<img class="centered %s">' % span)
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</img>')
if not nolegend and 'alt' in node:
text.append('<span class="legend %s">' % span)
text.append(node['alt'])
text.append('</span>')
text.append('</div>')
elif klass == 'figure':
if len(node['classes']) > 0:
floating = ' '.join(node['classes'])
else:
# let's use a row-fluid
floating = None
data = {}
for child in node.children:
klass = child.__class__.__name__
data[klass] = child
if 'image' not in data and 'reference' in data:
data['image'] = data['reference'].children[0]
# scaling
if 'scale' in data['image']:
scale = float(data['image']['scale'])
span = 12. * (scale / 100.)
offset = int((12 - span) / 2.)
span = 'span%d' % int(span)
if offset > 0:
span += ' offset%d' % offset
else:
span = 'span12'
linked = 'reference' in data
# image
uri = data['image']['uri']
file_ = os.path.split(uri)[-1]
if file_ in config['icons']:
class_ = 'subst'
nolegend = True
else:
if floating is None:
text.append('<div class="row-fluid">')
# subdiv
if floating is None:
text.append('<div class="%s">' % span)
else:
text.append('<div class="%s">' % floating)
# url
if linked:
refuri = data['reference']['refuri']
if ('faitmain.org' not in refuri and not refuri.startswith('/')
and int(config.get('shorten', 1)) == 1):
refuri = shorten(refuri, config['shortener_server'],
config['shortener_key'],
config.get('amazon_tag'))
text.append('<a href="%s">' % refuri)
if not uri.startswith('http'):
uri = '/'.join([config['cnd'], document.split('/')[1], uri])
text.append('<img class="centered span12" src="%s"></img>' % uri)
# caption
if 'caption' in data:
text.append('<span class="legend">')
for child in data['caption'].children:
text.append(_tree(child, document, title, config))
text.append('</span>')
if linked:
text.append('</a>')
text.append('</div>')
if floating is None:
text.append('</div>')
elif klass == 'reference': # link
if node.hasattr('refid'):
text.append('<a href="#%s">' % node['refid'])
elif node.hasattr('refuri'):
refuri = node['refuri']
if 'wikipedia.org' in refuri:
text.append('<a href="%s" class="wikipedia">' % refuri)
else:
if ('faitmain.org' not in refuri and not refuri.startswith('/')
and int(config.get('shorten', 1)) == 1):
refuri = shorten(refuri, config['shortener_server'],
config['shortener_key'],
config.get('amazon_tag'))
text.append('<a href="%s">' % refuri)
else:
text.append('<a>')
for child in node.children:
text.append(_tree(child, document, title, config))
text.append('</a>')
elif klass == 'target':
# ??
pass
elif klass == 'section':
if _LEVEL == 1:
tag = '<h2>%s</h2>'
elif _LEVEL == 2:
tag = '<h3>%s</h3>'
else:
tag = '<h4>%s</h4>'
section_title = node.children[0][0].astext()
id = node.attributes['ids'][0]
if _LEVEL == 1:
index(document,
title,
'sections', (section_title, id),
append=True)
text.append('<div id="%s" class="section">' % id)
header = (tag % u'%s <a class="headerlink" href="#%s"'
u'title="Lien vers cette section">\xb6</a>')
header = header % (section_title, id)
text.append(header)
for child in node.children[1:]:
text.append(_tree(child, document, title, config))
text.append('</div>')
elif klass == 'substitution_definition':
#uri = node.children[0].attributes['uri']
#text.append('<img class="subst" src="%s"></img>' % uri)
pass
elif klass == 'docinfo':
# reading metadata
for child in node.children:
text.append(_tree(child, document, title, config))
elif klass == 'author':
text.append('<img class="subst" ' 'src="%s/media/pen.png">' % cnd)
text.append('</img>')
authors = []
for value in node.astext().split(','):
value = value.strip()
if value == '':
continue
authors.append(value)
author_id = str2authorid(value)
text.append('<a href="%s/auteurs/%s.html">%s</a>' %
(config['siteurl'], author_id, value))
index(document, title, 'author', authors)
elif klass == 'date':
index(document, title, 'date', node.astext())
elif klass == 'field':
name = node.children[0].astext()
value = node.children[1].astext()
if name == 'category':
text.append('<img class="subst" ' 'src="%s/media/info.png">' % cnd)
text.append('</img>')
cats = value.split(',')
index(document, title, name, cats)
cats = [
'<a href="%s/%s.html">%s</a>' %
(config['siteurl'], cat, cat.capitalize()) for cat in cats
]
text.append(' | '.join(cats))
elif name == 'level':
index(document, title, name, value)
text.append('<img class="subst" '
'src="%s/media/flash.png">' % cnd)
text.append('</img>')
text.append('<strong>Niveau</strong>: %s' % value.capitalize())
elif name == 'translator':
index(document, title, name, value)
text.append('<img class="subst" '
'src="%s/media/translation.png">' % cnd)
text.append('</img>')
author_id = str2authorid(value)
msg = ('<strong>Traduction</strong>: '
'<a href="%s/auteurs/%s.html">%s</a>')
text.append(msg % (config['siteurl'], author_id, value))
elif name == 'licence':
text.append('<img class="subst" ' 'src="%s/media/cc.png">' % cnd)
text.append('</img>')
text.append(value)
elif klass == 'colspec': # table colspec
pass
elif klass == 'entry': # table entry
tagname = 'td'
if node.parent.parent.tagname == 'thead':
tagname = 'th'
text.extend(
render_simple_tag(node,
document,
title,
config,
tagname,
strip_child=True))
elif klass in SIMPLE_TAGS:
tagname, strip_child = SIMPLE_TAGS[klass]
text.extend(
render_simple_tag(node,
document,
title,
config,
tagname,
strip_child=strip_child))
else:
raise NotImplementedError(node)
_LEVEL -= 1
return ' '.join(text)
def generate(config):
sitemap = []
target = config['target']
src = config['src']
if not os.path.exists(target):
os.mkdir(target)
gens = [klass(config) for klass in generators]
match = {}
for gen in gens:
for ext in gen.exts:
match[ext] = gen
cats = config.get('categories', '').split(',')
cats = [cat.strip() for cat in cats if cat.strip() != '']
config['categories'] = cats
for path in ('cnd', 'siteurl'):
if os.path.exists(config[path]):
config[path] = os.path.abspath(config[path])
source_root = config.get('source_root')
for root, dirs, files in os.walk(src):
for file in files:
if file.startswith('_'):
continue
if file.endswith('.DS_Store'):
continue
ext = os.path.splitext(file)[-1]
path = os.path.join(root, file)
if ext in ('.mako', '.un~'):
continue
# configurable XXX
if (os.path.split(path)[0] == 'src/auteurs'
and file.endswith('.rst')):
continue
location = path[len(src) + 1:]
file_target = os.path.join(target, location)
target_dir = os.path.dirname(file_target)
file_target_name, ext = os.path.splitext(file_target)
url_target = file_target_name[len(target):] + '.html'
if source_root is not None:
config['github'] = os.path.join(source_root, location)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# now calling the right generator
if ext in match:
try:
match[ext](path, file_target, url_target, config=config)
except Exception:
logger.info('Failed on %s' % path)
raise
sitemap.append((url_target, True))
else:
logger.info('Copying %r' % file_target)
shutil.copyfile(path, file_target)
if 'github' in config:
del config['github']
# media
media = str(config['media'])
if os.path.exists(media):
shutil.rmtree(media)
shutil.copytree('media', media)
# building category pages now
categories = defaultdict(list)
for key, index in get_index():
path, title = key.split(':')
for key, value in index.items():
if key != 'category':
continue
for cat in value:
categories[cat].append((path, title))
for wanted in config['categories']:
if wanted in categories:
continue
categories[wanted] = []
gen = Mako(config)
for cat, paths in categories.items():
logger.info('Generating category %r' % cat)
url_target = '/%s.html' % cat
file_target = os.path.join(target, cat + '.html')
gen(config['cats'],
file_target,
url_target,
paths=paths,
title=cat.capitalize(),
config=config,
category=cat)
sitemap.append((url_target, False))
# creating the authors index page
authors = {}
for key, index in get_index():
path, title = key.split(':')
for key, authors_ in index.items():
if key != 'author':
continue
for author_name in authors_:
author_id = str2authorid(author_name)
if author_id in authors:
authors[author_id]['articles'].append((title, path))
else:
# should be configurable
link = '%s/auteurs/%s.html' % (config['siteurl'],
author_id)
authors[author_id] = {
'link': link,
'articles': [(title, path)],
'name': author_name
}
authors = authors.items()
authors.sort()
# XXX should be configurable...
authors_template = os.path.join(src, 'auteurs', 'index.mako')
logger.info('Generating authors index')
url_target = '/auteurs/index.html'
file_target = os.path.join(target, 'auteurs', 'index.html')
gen(authors_template,
file_target,
url_target,
authors=authors,
title="Auteurs",
config=config)
sitemap.append((url_target, False))
# creating the author pages
gen = RestructuredText(config)
for author_id, data in authors:
template = os.path.join(src, 'auteurs', '%s.rst' % author_id)
if not os.path.exists(template):
logger.warning('Template not found for author %r' % author_id)
continue
# we're supposed to find an author .rst file in /auteur
url_target = '/auteurs/%s.html' % author_id
file_target = os.path.join(target, 'auteurs', '%s.html' % author_id)
fd, tmp = tempfile.mkstemp()
os.close(fd)
def _line(line):
title, path = line
path = '%s/%s' % (config['siteurl'].rstrip('/'), path)
return u'- `%s <%s>`_' % (title, path)
articles = AUTHOR_ARTICLES % '\n'.join(
[_line(data) for data in data['articles']])
with codecs.open(template, encoding='utf8') as source_file:
with codecs.open(tmp, 'w', encoding='utf8') as target_file:
data = source_file.read()
data += articles + '\n'
target_file.write(data)
try:
gen(tmp, file_target, url_target, config=config)
finally:
os.remove(tmp)
sitemap.append((url_target, True))
# create the atom feed
siteurl = config.get('siteurl')
feed = Atom1Feed(title='FaitMain.org',
link=siteurl,
feed_url=siteurl + 'feed.xml',
description=config.get('site-description'))
for key, article in get_articles():
path, title = key.split(':')
feed.add_item(title=title,
link='%s/%s' % (siteurl, path),
description=article['body'],
categories=article['category'],
author_name=article['author'],
pubdate=article['date'])
with open(os.path.join(target, 'feed.xml'), 'w') as f:
feed.write(f, 'utf-8')
# creating sitemap
sitemap_file = os.path.join(target, 'sitemap.json')
logger.info('Generating sitemap at %r' % sitemap_file)
now = datetime.datetime.now().isoformat()
urlset = [{
'loc': loc,
'lastmod': now,
'changefreq': 'monthly',
'priority': 0.1,
'indexable': int(indexable)
} for loc, indexable in sitemap]
with open(sitemap_file, 'w') as f:
f.write(json.dumps({'urlset': urlset}))
# asking Trouvailles to index the web site
if int(config['indexing']) == 0:
return
logger.info('Indexing the whole website')
url = config['search_server']
data = {'sitemap': config['sitemap']}
headers = {'Content-type': 'application/json'}
r = requests.post(url, data=json.dumps(data), headers=headers)
if r.status_code != 200:
logger.info('Indexation failed')
logger.info(r.status_code)
logger.info(r.content)
