def generate(config): sitemap = [] target = config['target'] src = config['src'] if not os.path.exists(target): os.mkdir(target) gens = [klass(config) for klass in generators] match = {} for gen in gens: for ext in gen.exts: match[ext] = gen cats = config.get('categories', '').split(',') cats = [cat.strip() for cat in cats if cat.strip() != ''] config['categories'] = cats for root, dirs, files in os.walk(src): for file in files: if file.startswith('_'): continue if file.endswith('.DS_Store'): continue ext = os.path.splitext(file)[-1] path = os.path.join(root, file) if ext in ('.mako', '.un~'): continue # configurable XXX if (os.path.split(path)[0] == 'src/auteurs' and file.endswith('.rst')): continue location = path[len(src) + 1:] file_target = os.path.join(target, location) target_dir = os.path.dirname(file_target) file_target_name, ext = os.path.splitext(file_target) url_target = file_target_name[len(target):] + '.html' if not os.path.exists(target_dir): os.makedirs(target_dir) # now calling the right generator if ext in match: try: match[ext](path, file_target, url_target, config=config) except Exception: logger.info('Failed on %s' % path) raise sitemap.append((url_target, True)) else: logger.info('Copying %r' % file_target) shutil.copyfile(path, file_target) # media media = config['media'] if os.path.exists(media): shutil.rmtree(media) shutil.copytree('media', media) # building category pages now categories = defaultdict(list) for key, index in get_index(): path, title = key.split(':') for key, value in index.items(): if key != 'category': continue for cat in value: categories[cat].append((path, title)) for wanted in config['categories']: if wanted in categories: continue categories[wanted] = [] gen = Mako(config) for cat, paths in categories.items(): logger.info('Generating category %r' % cat) url_target = '/%s.html' % cat file_target = os.path.join(target, cat + '.html') gen(config['cats'], file_target, url_target, paths=paths, title=cat.capitalize(), config=config, category=cat) sitemap.append((url_target, False)) # creating the authors index page authors = {} for key, index in get_index(): path, title = key.split(':') for key, author_name in index.items(): if key != 'author': continue author_id = str2authorid(author_name) if author_id in authors: authors[author_id]['articles'].append((title, path)) else: # should be configurable link = '/auteurs/%s.html' % author_id authors[author_id] = {'link': link, 'articles': [(title, path)], 'name': author_name} authors = authors.items() authors.sort() # XXX should be configurable... authors_template = os.path.join(src, 'auteurs', 'index.mako') logger.info('Generating authors index') url_target = '/auteurs/index.html' file_target = os.path.join(target, 'auteurs', 'index.html') gen(authors_template, file_target, url_target, authors=authors, title="Auteurs", config=config) sitemap.append((url_target, False)) # creating the author pages gen = RestructuredText(config) for author_id, data in authors: template = os.path.join(src, 'auteurs', '%s.rst' % author_id) if not os.path.exists(template): logger.warning('Template not found for author %r' % author_id) continue # we're supposed to find an author .rst file in /auteur url_target = '/auteurs/%s.html' % author_id file_target = os.path.join(target, 'auteurs', '%s.html' % author_id) fd, tmp = tempfile.mkstemp() os.close(fd) def _line(line): return u'- `%s <%s>`_' % (line[0], line[1]) articles = AUTHOR_ARTICLES % '\n'.join([_line(data) for data in data['articles']]) with codecs.open(template, encoding='utf8') as source_file: with codecs.open(tmp, 'w', encoding='utf8') as target_file: data = source_file.read() data += articles + '\n' target_file.write(data) try: gen(tmp, file_target, url_target, config=config) finally: os.remove(tmp) sitemap.append((url_target, True)) # creating sitemap sitemap_file = os.path.join(target, 'sitemap.json') logger.info('Generating sitemap at %r' % sitemap_file) now = datetime.datetime.now().isoformat() urlset = [{'loc': loc, 'lastmod': now, 'changefreq': 'monthly', 'priority': 0.1, 'indexable': int(indexable)} for loc, indexable in sitemap] with open(sitemap_file, 'w') as f: f.write(json.dumps({'urlset': urlset})) # asking Trouvailles to index the web site logger.info('Indexing the whole website') url = config['search_server'] data = {'sitemap': config['sitemap']} headers = {'Content-type': 'application/json'} r = requests.post(url, data=json.dumps(data), headers=headers) if r.status_code != 200: logger.info('Indexation failed') logger.info(r.status_code) logger.info(r.content)
def _tree(node, document, title, config): """Renders a node in HTML. """ cdn = config['cdn'] text = [] klass = node.__class__.__name__ if klass == 'transition': text.append('<hr/>') elif klass == 'system_message': pass elif klass == 'block_quote': text.append('<blockquote>') for child in node.children: text.append(_tree(child, document, title, config)) text.append('</blockquote>') elif klass == 'paragraph': text.append('<p>') for child in node.children: text.append(_tree(child, document, title, config)) text.append('</p>') elif klass == 'Text': text.append(node.astext()) elif klass == 'literal_block': text.append('<div class="syntax rounded">') text.append(hilite(node)) text.append('</div>') elif klass == 'sidebar': text.append('<div class="alert alert-info">') text.append('<h4>%s</h4>' % node.children[0].astext()) for child in node.children[1:]: text.append(_tree(child, document, title, config)) text.append('</div>') elif klass == 'note': node.attributes['class'] = 'well note' text.extend(render_simple_tag(node, document, title, config, 'div', strip_child=False)) elif klass == 'warning': text.append('<div class="alert">') for child in node.children: text.append(_tree(child, document, title, config)) text.append('</div>') elif klass == 'table': node.attributes['class'] = 'table' text.extend(render_simple_tag(node, document, title, config, 'table', strip_child=True)) elif klass == 'image': if node.get('uri').startswith('icon'): return '<i class="%s"></i>' % node.get('uri') nolegend = False if node.hasattr('scale'): span = 12. * (float(node['scale']) / 100.) offset = int((12-span) / 2.) - 1 span = 'span%d' % int(span) if offset > 0: span += ' offset%d' % offset else: span = 'span12' if node.hasattr('uri'): uri = node['uri'] file_ = os.path.split(uri)[-1] if file_ in config['icons']: class_ = 'subst' nolegend = True else: text.append('<div class="row-fluid">') class_ = 'centered %s' % span text.append('<img class="%s" src="%s">' % (class_, uri)) else: text.append('<div class="row-fluid">') text.append('<img class="centered %s">' % span) for child in node.children: text.append(_tree(child, document, title, config)) text.append('</img>') if not nolegend and 'alt' in node: text.append('<span class="legend %s">' % span) text.append(node['alt']) text.append('</span>') text.append('</div>') elif klass == 'figure': if len(node['classes']) > 0: floating = ' '.join(node['classes']) else: # let's use a row-fluid floating = None data = {} for child in node.children: klass = child.__class__.__name__ data[klass] = child if 'image' not in data and 'reference' in data: data['image'] = data['reference'].children[0] # scaling if 'scale' in data['image']: scale = float(data['image']['scale']) span = 12. * (scale / 100.) offset = int((12-span) / 2.) span = 'span%d' % int(span) if offset > 0: span += ' offset%d' % offset else: span = 'span12' linked = 'reference' in data # image uri = data['image']['uri'] file_ = os.path.split(uri)[-1] if file_ in config['icons']: class_ = 'subst' nolegend = True else: if floating is None: text.append('<div class="row-fluid">') # subdiv if floating is None: text.append('<div class="%s">' % span) else: text.append('<div class="%s">' % floating) # url if linked: refuri = data['reference']['refuri'] if ('faitmain.org' not in refuri and not refuri.startswith('/') and int(config.get('shorten', 1)) == 1): refuri = shorten(refuri, config['shortener_server'], config['shortener_key'], config.get('amazon_tag')) text.append('<a href="%s">' % refuri) if not uri.startswith('http'): uri = '/'.join([config['cdn'], document.split('/')[1], uri]) text.append('<img class="centered span12" src="%s"></img>' % uri) # caption if 'caption' in data: text.append('<span class="legend">') for child in data['caption'].children: text.append(_tree(child, document, title, config)) text.append('</span>') if linked: text.append('</a>') text.append('</div>') if floating is None: text.append('</div>') elif klass == 'reference': # link if node.hasattr('refid'): text.append('<a href="#%s">' % node['refid']) elif node.hasattr('refuri'): refuri = node['refuri'] if 'wikipedia.org' in refuri: text.append('<a href="%s" class="wikipedia">' % refuri) else: if ('faitmain.org' not in refuri and not refuri.startswith('/') and int(config.get('shorten', 1)) == 1): refuri = shorten(refuri, config['shortener_server'], config['shortener_key'], config.get('amazon_tag')) text.append('<a href="%s">' % refuri) else: text.append('<a>') for child in node.children: text.append(_tree(child, document, title, config)) text.append('</a>') elif klass == 'target': # ?? pass elif klass == 'section': section_title = node.children[0][0].astext() id = node.attributes['ids'][0] index(document, title, 'sections', (section_title, id), append=True) text.append('<div id="%s" class="section">' % id) header = (u'<h2>%s <a class="headerlink" href="#%s"' u'title="Lien vers cette section">\xb6</a></h2>') header = header % (section_title, id) text.append(header) for child in node.children[1:]: text.append(_tree(child, document, title, config)) text.append('</div>') elif klass == 'substitution_definition': #uri = node.children[0].attributes['uri'] #text.append('<img class="subst" src="%s"></img>' % uri) pass elif klass == 'docinfo': # reading metadata for child in node.children: text.append(_tree(child, document, title, config)) elif klass == 'author': value = node.astext() index(document, title, 'author', value) author_id = str2authorid(value) text.append('<img class="subst" ' 'src="%s/media/pen.png">' % cdn) text.append('</img>') text.append('<a href="%s/auteurs/%s.html">%s</a>' % (config['siteurl'], author_id, value)) elif klass == 'date': index(document, title, 'date', node.astext()) elif klass == 'field': name = node.children[0].astext() value = node.children[1].astext() if name == 'category': text.append('<img class="subst" ' 'src="%s/media/info.png">' % cdn) text.append('</img>') cats = value.split(',') index(document, title, name, cats) cats = ['<a href="%s/%s.html">%s</a>' % (config['siteurl'], cat, cat.capitalize()) for cat in cats] text.append(' | '.join(cats)) elif name == 'level': index(document, title, name, value) text.append('<img class="subst" ' 'src="%s/media/flash.png">' % cdn) text.append('</img>') text.append('<strong>Niveau</strong>: %s' % value.capitalize()) elif name == 'translator': index(document, title, name, value) text.append('<img class="subst" ' 'src="%s/media/translation.png">' % cdn) text.append('</img>') author_id = str2authorid(value) msg = ('<strong>Traduction</strong>: ' '<a href="%s/auteurs/%s.html">%s</a>') text.append(msg % (config['siteurl'], author_id, value)) elif klass == 'colspec': # table colspec pass elif klass == 'entry': # table entry tagname = 'td' if node.parent.parent.tagname == 'thead': tagname = 'th' text.extend(render_simple_tag(node, document, title, config, tagname, strip_child=True)) elif klass in SIMPLE_TAGS: tagname, strip_child = SIMPLE_TAGS[klass] text.extend(render_simple_tag(node, document, title, config, tagname, strip_child=strip_child)) else: raise NotImplementedError(node) return ' '.join(text)
def generate(config): sitemap = [] target = config["target"] src = config["src"] if not os.path.exists(target): os.mkdir(target) gens = [klass(config) for klass in generators] match = {} for gen in gens: for ext in gen.exts: match[ext] = gen cats = config.get("categories", "").split(",") cats = [cat.strip() for cat in cats if cat.strip() != ""] config["categories"] = cats for path in ("cnd", "siteurl"): if os.path.exists(config[path]): config[path] = os.path.abspath(config[path]) source_root = config.get("source_root") for root, dirs, files in os.walk(src): for file in files: if file.startswith("_"): continue if file.endswith(".DS_Store"): continue ext = os.path.splitext(file)[-1] path = os.path.join(root, file) if ext in (".mako", ".un~"): continue # configurable XXX if os.path.split(path)[0] == "src/auteurs" and file.endswith(".rst"): continue location = path[len(src) + 1 :] file_target = os.path.join(target, location) target_dir = os.path.dirname(file_target) file_target_name, ext = os.path.splitext(file_target) url_target = file_target_name[len(target) :] + ".html" if source_root is not None: config["github"] = os.path.join(source_root, location) if not os.path.exists(target_dir): os.makedirs(target_dir) # now calling the right generator if ext in match: try: match[ext](path, file_target, url_target, config=config) except Exception: logger.info("Failed on %s" % path) raise sitemap.append((url_target, True)) else: logger.info("Copying %r" % file_target) shutil.copyfile(path, file_target) if "github" in config: del config["github"] # media media = str(config["media"]) if os.path.exists(media): shutil.rmtree(media) shutil.copytree("media", media) # building category pages now categories = defaultdict(list) for key, index in get_index(): path, title = key.split(":") for key, value in index.items(): if key != "category": continue for cat in value: categories[cat].append((path, title)) for wanted in config["categories"]: if wanted in categories: continue categories[wanted] = [] gen = Mako(config) for cat, paths in categories.items(): logger.info("Generating category %r" % cat) url_target = "/%s.html" % cat file_target = os.path.join(target, cat + ".html") gen(config["cats"], file_target, url_target, paths=paths, title=cat.capitalize(), config=config, category=cat) sitemap.append((url_target, False)) # creating the authors index page authors = {} for key, index in get_index(): path, title = key.split(":") for key, authors_ in index.items(): if key != "author": continue for author_name in authors_: author_id = str2authorid(author_name) if author_id in authors: authors[author_id]["articles"].append((title, path)) else: # should be configurable link = "%s/auteurs/%s.html" % (config["siteurl"], author_id) authors[author_id] = {"link": link, "articles": [(title, path)], "name": author_name} authors = authors.items() authors.sort() # XXX should be configurable... authors_template = os.path.join(src, "auteurs", "index.mako") logger.info("Generating authors index") url_target = "/auteurs/index.html" file_target = os.path.join(target, "auteurs", "index.html") gen(authors_template, file_target, url_target, authors=authors, title="Auteurs", config=config) sitemap.append((url_target, False)) # creating the author pages gen = RestructuredText(config) for author_id, data in authors: template = os.path.join(src, "auteurs", "%s.rst" % author_id) if not os.path.exists(template): logger.warning("Template not found for author %r" % author_id) continue # we're supposed to find an author .rst file in /auteur url_target = "/auteurs/%s.html" % author_id file_target = os.path.join(target, "auteurs", "%s.html" % author_id) fd, tmp = tempfile.mkstemp() os.close(fd) def _line(line): title, path = line path = "%s/%s" % (config["siteurl"].rstrip("/"), path) return u"- `%s <%s>`_" % (title, path) articles = AUTHOR_ARTICLES % "\n".join([_line(data) for data in data["articles"]]) with codecs.open(template, encoding="utf8") as source_file: with codecs.open(tmp, "w", encoding="utf8") as target_file: data = source_file.read() data += articles + "\n" target_file.write(data) try: gen(tmp, file_target, url_target, config=config) finally: os.remove(tmp) sitemap.append((url_target, True)) # create the atom feed siteurl = config.get("siteurl") feed = Atom1Feed( title="FaitMain.org", link=siteurl, feed_url=siteurl + "feed.xml", description=config.get("site-description") ) for key, article in get_articles(): path, title = key.split(":") feed.add_item( title=title, link="%s/%s" % (siteurl, path), description=article["body"], categories=article["category"], author_name=article["author"], pubdate=article["date"], ) with open(os.path.join(target, "feed.xml"), "w") as f: feed.write(f, "utf-8") # creating sitemap sitemap_file = os.path.join(target, "sitemap.json") logger.info("Generating sitemap at %r" % sitemap_file) now = datetime.datetime.now().isoformat() urlset = [ {"loc": loc, "lastmod": now, "changefreq": "monthly", "priority": 0.1, "indexable": int(indexable)} for loc, indexable in sitemap ] with open(sitemap_file, "w") as f: f.write(json.dumps({"urlset": urlset})) # asking Trouvailles to index the web site if int(config["indexing"]) == 0: return logger.info("Indexing the whole website") url = config["search_server"] data = {"sitemap": config["sitemap"]} headers = {"Content-type": "application/json"} r = requests.post(url, data=json.dumps(data), headers=headers) if r.status_code != 200: logger.info("Indexation failed") logger.info(r.status_code) logger.info(r.content)
def _tree(node, document, title, config): """Renders a node in HTML. """ cnd = config["cnd"] text = [] klass = node.__class__.__name__ if klass == "transition": text.append("<hr/>") elif klass == "system_message": pass elif klass == "block_quote": text.append("<blockquote>") for child in node.children: text.append(_tree(child, document, title, config)) text.append("</blockquote>") elif klass == "paragraph": text.append("<p>") for child in node.children: text.append(_tree(child, document, title, config)) text.append("</p>") elif klass == "Text": text.append(node.astext()) elif klass == "literal_block": text.append('<div class="syntax rounded">') text.append(hilite(node)) text.append("</div>") elif klass == "sidebar": text.append('<div class="alert alert-info">') text.append("<h4>%s</h4>" % node.children[0].astext()) for child in node.children[1:]: text.append(_tree(child, document, title, config)) text.append("</div>") elif klass == "note": node.attributes["class"] = "well note" text.extend(render_simple_tag(node, document, title, config, "div", strip_child=False)) elif klass == "warning": text.append('<div class="alert">') for child in node.children: text.append(_tree(child, document, title, config)) text.append("</div>") elif klass == "table": node.attributes["class"] = "table" text.extend(render_simple_tag(node, document, title, config, "table", strip_child=True)) elif klass == "image": if node.get("uri").startswith("icon"): return '<i class="%s"></i>' % node.get("uri") nolegend = False if node.hasattr("scale"): span = 12.0 * (float(node["scale"]) / 100.0) offset = int((12 - span) / 2.0) - 1 span = "span%d" % int(span) if offset > 0: span += " offset%d" % offset else: span = "span12" if node.hasattr("uri"): uri = node["uri"] file_ = os.path.split(uri)[-1] if file_ in config["icons"]: class_ = "subst" nolegend = True else: text.append('<div class="row-fluid">') class_ = "centered %s" % span text.append('<img class="%s" src="%s">' % (class_, uri)) else: text.append('<div class="row-fluid">') text.append('<img class="centered %s">' % span) for child in node.children: text.append(_tree(child, document, title, config)) text.append("</img>") if not nolegend and "alt" in node: text.append('<span class="legend %s">' % span) text.append(node["alt"]) text.append("</span>") text.append("</div>") elif klass == "figure": if len(node["classes"]) > 0: floating = " ".join(node["classes"]) else: # let's use a row-fluid floating = None data = {} for child in node.children: klass = child.__class__.__name__ data[klass] = child if "image" not in data and "reference" in data: data["image"] = data["reference"].children[0] # scaling if "scale" in data["image"]: scale = float(data["image"]["scale"]) span = 12.0 * (scale / 100.0) offset = int((12 - span) / 2.0) span = "span%d" % int(span) if offset > 0: span += " offset%d" % offset else: span = "span12" linked = "reference" in data # image uri = data["image"]["uri"] file_ = os.path.split(uri)[-1] if file_ in config["icons"]: class_ = "subst" nolegend = True else: if floating is None: text.append('<div class="row-fluid">') # subdiv if floating is None: text.append('<div class="%s">' % span) else: text.append('<div class="%s">' % floating) # url if linked: refuri = data["reference"]["refuri"] if "faitmain.org" not in refuri and not refuri.startswith("/") and int(config.get("shorten", 1)) == 1: refuri = shorten(refuri, config["shortener_server"], config["shortener_key"], config.get("amazon_tag")) text.append('<a href="%s">' % refuri) text.append('<img class="centered span12" src="%s"></img>' % uri) # caption if "caption" in data: text.append('<span class="legend">') for child in data["caption"].children: text.append(_tree(child, document, title, config)) text.append("</span>") if linked: text.append("</a>") text.append("</div>") if floating is None: text.append("</div>") elif klass == "reference": # link if node.hasattr("refid"): text.append('<a href="#%s">' % node["refid"]) elif node.hasattr("refuri"): refuri = node["refuri"] if "wikipedia.org" in refuri: text.append('<a href="%s" class="wikipedia">' % refuri) else: if "faitmain.org" not in refuri and not refuri.startswith("/") and int(config.get("shorten", 1)) == 1: refuri = shorten( refuri, config["shortener_server"], config["shortener_key"], config.get("amazon_tag") ) text.append('<a href="%s">' % refuri) else: text.append("<a>") for child in node.children: text.append(_tree(child, document, title, config)) text.append("</a>") elif klass == "target": # ?? pass elif klass == "section": section_title = node.children[0][0].astext() id = node.attributes["ids"][0] index(document, title, "sections", (section_title, id), append=True) text.append('<div id="%s" class="section">' % id) header = u'<h2>%s <a class="headerlink" href="#%s"' u'title="Lien vers cette section">\xb6</a></h2>' header = header % (section_title, id) text.append(header) for child in node.children[1:]: text.append(_tree(child, document, title, config)) text.append("</div>") elif klass == "substitution_definition": # uri = node.children[0].attributes['uri'] # text.append('<img class="subst" src="%s"></img>' % uri) pass elif klass == "docinfo": # reading metadata for child in node.children: text.append(_tree(child, document, title, config)) elif klass == "author": value = node.astext() index(document, title, "author", value) author_id = str2authorid(value) text.append('<img class="subst" ' 'src="%s/media/pen.png">' % cnd) text.append("</img>") text.append('<a href="/auteurs/%s.html">%s</a>' % (author_id, value)) elif klass == "date": index(document, title, "date", node.astext()) elif klass == "field": name = node.children[0].astext() value = node.children[1].astext() if name == "category": text.append('<img class="subst" ' 'src="%s/media/info.png">' % cnd) text.append("</img>") cats = value.split(",") index(document, title, name, cats) cats = ['<a href="/%s.html">%s</a>' % (cat, cat.capitalize()) for cat in cats] text.append(" | ".join(cats)) elif name == "level": index(document, title, name, value) text.append('<img class="subst" ' 'src="%s/media/flash.png">' % cnd) text.append("</img>") text.append("<strong>Niveau</strong>: %s" % value.capitalize()) elif name == "translator": index(document, title, name, value) text.append('<img class="subst" ' 'src="%s/media/translation.png">' % cnd) text.append("</img>") author_id = str2authorid(value) msg = "<strong>Traduction</strong>: " '<a href="/auteurs/%s.html">%s</a>' text.append(msg % (author_id, value)) elif klass == "colspec": # table colspec pass elif klass == "entry": # table entry tagname = "td" if node.parent.parent.tagname == "thead": tagname = "th" text.extend(render_simple_tag(node, document, title, config, tagname, strip_child=True)) elif klass in SIMPLE_TAGS: tagname, strip_child = SIMPLE_TAGS[klass] text.extend(render_simple_tag(node, document, title, config, tagname, strip_child=strip_child)) else: raise NotImplementedError(node) return " ".join(text)
def _tree(node, document, title, config): """Renders a node in HTML. """ global _LEVEL _LEVEL += 1 cnd = config['cnd'] text = [] klass = node.__class__.__name__ if klass == 'transition': text.append('<hr/>') elif klass == 'system_message': pass elif klass == 'block_quote': text.append('<blockquote>') for child in node.children: text.append(_tree(child, document, title, config)) text.append('</blockquote>') elif klass == 'paragraph': text.append('<p>') for child in node.children: text.append(_tree(child, document, title, config)) text.append('</p>') elif klass == 'Text': text.append(node.astext()) elif klass == 'literal_block': text.append('<div class="syntax rounded">') text.append(hilite(node)) text.append('</div>') elif klass == 'sidebar': text.append('<div class="alert alert-info">') text.append('<h4>%s</h4>' % node.children[0].astext()) for child in node.children[1:]: text.append(_tree(child, document, title, config)) text.append('</div>') elif klass == 'note': node.attributes['class'] = 'well note' text.extend( render_simple_tag(node, document, title, config, 'div', strip_child=False)) elif klass == 'warning': text.append('<div class="alert">') for child in node.children: text.append(_tree(child, document, title, config)) text.append('</div>') elif klass == 'table': node.attributes['class'] = 'table' text.extend( render_simple_tag(node, document, title, config, 'table', strip_child=True)) elif klass == 'image': if node.get('uri').startswith('icon'): return '<i class="%s"></i>' % node.get('uri') nolegend = False if node.hasattr('scale'): span = 12. * (float(node['scale']) / 100.) offset = int((12 - span) / 2.) - 1 span = 'span%d' % int(span) if offset > 0: span += ' offset%d' % offset else: span = 'span12' if node.hasattr('uri'): uri = node['uri'] file_ = os.path.split(uri)[-1] if file_ in config['icons']: class_ = 'subst' nolegend = True else: text.append('<div class="row-fluid">') class_ = 'centered %s' % span text.append('<img class="%s" src="%s">' % (class_, uri)) else: text.append('<div class="row-fluid">') text.append('<img class="centered %s">' % span) for child in node.children: text.append(_tree(child, document, title, config)) text.append('</img>') if not nolegend and 'alt' in node: text.append('<span class="legend %s">' % span) text.append(node['alt']) text.append('</span>') text.append('</div>') elif klass == 'figure': if len(node['classes']) > 0: floating = ' '.join(node['classes']) else: # let's use a row-fluid floating = None data = {} for child in node.children: klass = child.__class__.__name__ data[klass] = child if 'image' not in data and 'reference' in data: data['image'] = data['reference'].children[0] # scaling if 'scale' in data['image']: scale = float(data['image']['scale']) span = 12. * (scale / 100.) offset = int((12 - span) / 2.) span = 'span%d' % int(span) if offset > 0: span += ' offset%d' % offset else: span = 'span12' linked = 'reference' in data # image uri = data['image']['uri'] file_ = os.path.split(uri)[-1] if file_ in config['icons']: class_ = 'subst' nolegend = True else: if floating is None: text.append('<div class="row-fluid">') # subdiv if floating is None: text.append('<div class="%s">' % span) else: text.append('<div class="%s">' % floating) # url if linked: refuri = data['reference']['refuri'] if ('faitmain.org' not in refuri and not refuri.startswith('/') and int(config.get('shorten', 1)) == 1): refuri = shorten(refuri, config['shortener_server'], config['shortener_key'], config.get('amazon_tag')) text.append('<a href="%s">' % refuri) if not uri.startswith('http'): uri = '/'.join([config['cnd'], document.split('/')[1], uri]) text.append('<img class="centered span12" src="%s"></img>' % uri) # caption if 'caption' in data: text.append('<span class="legend">') for child in data['caption'].children: text.append(_tree(child, document, title, config)) text.append('</span>') if linked: text.append('</a>') text.append('</div>') if floating is None: text.append('</div>') elif klass == 'reference': # link if node.hasattr('refid'): text.append('<a href="#%s">' % node['refid']) elif node.hasattr('refuri'): refuri = node['refuri'] if 'wikipedia.org' in refuri: text.append('<a href="%s" class="wikipedia">' % refuri) else: if ('faitmain.org' not in refuri and not refuri.startswith('/') and int(config.get('shorten', 1)) == 1): refuri = shorten(refuri, config['shortener_server'], config['shortener_key'], config.get('amazon_tag')) text.append('<a href="%s">' % refuri) else: text.append('<a>') for child in node.children: text.append(_tree(child, document, title, config)) text.append('</a>') elif klass == 'target': # ?? pass elif klass == 'section': if _LEVEL == 1: tag = '<h2>%s</h2>' elif _LEVEL == 2: tag = '<h3>%s</h3>' else: tag = '<h4>%s</h4>' section_title = node.children[0][0].astext() id = node.attributes['ids'][0] if _LEVEL == 1: index(document, title, 'sections', (section_title, id), append=True) text.append('<div id="%s" class="section">' % id) header = (tag % u'%s <a class="headerlink" href="#%s"' u'title="Lien vers cette section">\xb6</a>') header = header % (section_title, id) text.append(header) for child in node.children[1:]: text.append(_tree(child, document, title, config)) text.append('</div>') elif klass == 'substitution_definition': #uri = node.children[0].attributes['uri'] #text.append('<img class="subst" src="%s"></img>' % uri) pass elif klass == 'docinfo': # reading metadata for child in node.children: text.append(_tree(child, document, title, config)) elif klass == 'author': text.append('<img class="subst" ' 'src="%s/media/pen.png">' % cnd) text.append('</img>') authors = [] for value in node.astext().split(','): value = value.strip() if value == '': continue authors.append(value) author_id = str2authorid(value) text.append('<a href="%s/auteurs/%s.html">%s</a>' % (config['siteurl'], author_id, value)) index(document, title, 'author', authors) elif klass == 'date': index(document, title, 'date', node.astext()) elif klass == 'field': name = node.children[0].astext() value = node.children[1].astext() if name == 'category': text.append('<img class="subst" ' 'src="%s/media/info.png">' % cnd) text.append('</img>') cats = value.split(',') index(document, title, name, cats) cats = [ '<a href="%s/%s.html">%s</a>' % (config['siteurl'], cat, cat.capitalize()) for cat in cats ] text.append(' | '.join(cats)) elif name == 'level': index(document, title, name, value) text.append('<img class="subst" ' 'src="%s/media/flash.png">' % cnd) text.append('</img>') text.append('<strong>Niveau</strong>: %s' % value.capitalize()) elif name == 'translator': index(document, title, name, value) text.append('<img class="subst" ' 'src="%s/media/translation.png">' % cnd) text.append('</img>') author_id = str2authorid(value) msg = ('<strong>Traduction</strong>: ' '<a href="%s/auteurs/%s.html">%s</a>') text.append(msg % (config['siteurl'], author_id, value)) elif name == 'licence': text.append('<img class="subst" ' 'src="%s/media/cc.png">' % cnd) text.append('</img>') text.append(value) elif klass == 'colspec': # table colspec pass elif klass == 'entry': # table entry tagname = 'td' if node.parent.parent.tagname == 'thead': tagname = 'th' text.extend( render_simple_tag(node, document, title, config, tagname, strip_child=True)) elif klass in SIMPLE_TAGS: tagname, strip_child = SIMPLE_TAGS[klass] text.extend( render_simple_tag(node, document, title, config, tagname, strip_child=strip_child)) else: raise NotImplementedError(node) _LEVEL -= 1 return ' '.join(text)
def generate(config): sitemap = [] target = config['target'] src = config['src'] if not os.path.exists(target): os.mkdir(target) gens = [klass(config) for klass in generators] match = {} for gen in gens: for ext in gen.exts: match[ext] = gen cats = config.get('categories', '').split(',') cats = [cat.strip() for cat in cats if cat.strip() != ''] config['categories'] = cats for path in ('cnd', 'siteurl'): if os.path.exists(config[path]): config[path] = os.path.abspath(config[path]) source_root = config.get('source_root') for root, dirs, files in os.walk(src): for file in files: if file.startswith('_'): continue if file.endswith('.DS_Store'): continue ext = os.path.splitext(file)[-1] path = os.path.join(root, file) if ext in ('.mako', '.un~'): continue # configurable XXX if (os.path.split(path)[0] == 'src/auteurs' and file.endswith('.rst')): continue location = path[len(src) + 1:] file_target = os.path.join(target, location) target_dir = os.path.dirname(file_target) file_target_name, ext = os.path.splitext(file_target) url_target = file_target_name[len(target):] + '.html' if source_root is not None: config['github'] = os.path.join(source_root, location) if not os.path.exists(target_dir): os.makedirs(target_dir) # now calling the right generator if ext in match: try: match[ext](path, file_target, url_target, config=config) except Exception: logger.info('Failed on %s' % path) raise sitemap.append((url_target, True)) else: logger.info('Copying %r' % file_target) shutil.copyfile(path, file_target) if 'github' in config: del config['github'] # media media = str(config['media']) if os.path.exists(media): shutil.rmtree(media) shutil.copytree('media', media) # building category pages now categories = defaultdict(list) for key, index in get_index(): path, title = key.split(':') for key, value in index.items(): if key != 'category': continue for cat in value: categories[cat].append((path, title)) for wanted in config['categories']: if wanted in categories: continue categories[wanted] = [] gen = Mako(config) for cat, paths in categories.items(): logger.info('Generating category %r' % cat) url_target = '/%s.html' % cat file_target = os.path.join(target, cat + '.html') gen(config['cats'], file_target, url_target, paths=paths, title=cat.capitalize(), config=config, category=cat) sitemap.append((url_target, False)) # creating the authors index page authors = {} for key, index in get_index(): path, title = key.split(':') for key, authors_ in index.items(): if key != 'author': continue for author_name in authors_: author_id = str2authorid(author_name) if author_id in authors: authors[author_id]['articles'].append((title, path)) else: # should be configurable link = '%s/auteurs/%s.html' % (config['siteurl'], author_id) authors[author_id] = { 'link': link, 'articles': [(title, path)], 'name': author_name } authors = authors.items() authors.sort() # XXX should be configurable... authors_template = os.path.join(src, 'auteurs', 'index.mako') logger.info('Generating authors index') url_target = '/auteurs/index.html' file_target = os.path.join(target, 'auteurs', 'index.html') gen(authors_template, file_target, url_target, authors=authors, title="Auteurs", config=config) sitemap.append((url_target, False)) # creating the author pages gen = RestructuredText(config) for author_id, data in authors: template = os.path.join(src, 'auteurs', '%s.rst' % author_id) if not os.path.exists(template): logger.warning('Template not found for author %r' % author_id) continue # we're supposed to find an author .rst file in /auteur url_target = '/auteurs/%s.html' % author_id file_target = os.path.join(target, 'auteurs', '%s.html' % author_id) fd, tmp = tempfile.mkstemp() os.close(fd) def _line(line): title, path = line path = '%s/%s' % (config['siteurl'].rstrip('/'), path) return u'- `%s <%s>`_' % (title, path) articles = AUTHOR_ARTICLES % '\n'.join( [_line(data) for data in data['articles']]) with codecs.open(template, encoding='utf8') as source_file: with codecs.open(tmp, 'w', encoding='utf8') as target_file: data = source_file.read() data += articles + '\n' target_file.write(data) try: gen(tmp, file_target, url_target, config=config) finally: os.remove(tmp) sitemap.append((url_target, True)) # create the atom feed siteurl = config.get('siteurl') feed = Atom1Feed(title='FaitMain.org', link=siteurl, feed_url=siteurl + 'feed.xml', description=config.get('site-description')) for key, article in get_articles(): path, title = key.split(':') feed.add_item(title=title, link='%s/%s' % (siteurl, path), description=article['body'], categories=article['category'], author_name=article['author'], pubdate=article['date']) with open(os.path.join(target, 'feed.xml'), 'w') as f: feed.write(f, 'utf-8') # creating sitemap sitemap_file = os.path.join(target, 'sitemap.json') logger.info('Generating sitemap at %r' % sitemap_file) now = datetime.datetime.now().isoformat() urlset = [{ 'loc': loc, 'lastmod': now, 'changefreq': 'monthly', 'priority': 0.1, 'indexable': int(indexable) } for loc, indexable in sitemap] with open(sitemap_file, 'w') as f: f.write(json.dumps({'urlset': urlset})) # asking Trouvailles to index the web site if int(config['indexing']) == 0: return logger.info('Indexing the whole website') url = config['search_server'] data = {'sitemap': config['sitemap']} headers = {'Content-type': 'application/json'} r = requests.post(url, data=json.dumps(data), headers=headers) if r.status_code != 200: logger.info('Indexation failed') logger.info(r.status_code) logger.info(r.content)