def render_posts(self):
"""Render posts using jinja2 templates."""
for post in tqdm(self.posts, unit=' pages', miniters=1, desc="Posts"):
template_name = "%s.html" % post.meta.template
template = self.jinja2.get_template(template_name)
rv = template.render(
content=post.html,
meta=post.meta,
year=datetime.today().year,
posts=self.posts,
plugin_data=self.plugin_data,
config=self.config,
categories=self.posts_by_category.get_as_dict(), # noqa: E501
tags=self.posts_by_tag.get_as_dict(),
templates=self.posts_by_template.get_as_dict(), # noqa: E501
microdata=self.posts_by_microdata.get_as_dict() # noqa: E501
)
# Linting
linter_results = self.linter.lint(post, rv, self)
# Are we stopping on linting errors?
if linter_results.has_errors and self.config.linter.stop_on_error:
print(post.filename)
for err in linter_results.info:
print("\t-%s:%s" % (err[0], err[1]))
sys.exit(-1)
perm_url = post.meta.permanent_url
if len(perm_url) and perm_url[0] == '/':
perm_url = perm_url[1:]
path = self.get_output_dir() / perm_url
files.write_file(path, 'index.html', rv)
def test_basic_file_flow(tmp_path):
"""testing all at once as it is hard to do in smaller chunk
write -> read -> list -> clean -> list again
"""
fname = 'a.txt'
content = 'this is a test'
path = Path(tmp_path) / fname
# write
write_file(tmp_path, fname, content)
assert path.exists()
# read
read_back = read_file(path)
assert read_back == content
# list
files_list = get_files_list(tmp_path, '*.txt')
assert path in files_list
# clean
clean_dir(tmp_path)
assert not path.exists()
# filelist again
files_list = get_files_list(path, '*.txt')
assert not files_list # must be empty
def generate_plugins_readme(site, output_fname):
"Generate plugins readme"
plugin_list_md = "# SiteFab plugins\n"
plugin_list_md += "List of available plugins\n\n"
plugin_list_md += "|Name | Description | dependencies|\n"
plugin_list_md += "|-----|:------------|:------------|\n"
plugins = site.plugins.get_plugins()
for plugin in plugins:
name = plugin.name
description = plugin.description
dependencies = site.plugins.get_plugin_dependencies(plugin)
if not len(dependencies):
dependencies = ""
else:
dependencies = ", ".join(dependencies)
doc_path = site.plugins.get_plugin_documentation_filename(plugin)
doc_path = str(doc_path)
if 'plugins' in doc_path:
doc_path = doc_path.replace('\\', '/').split('plugins')[1]
name = "[%s](tree/master%s)" % (name, doc_path)
plugin_list_md += "| %s | %s | %s |\n" % (name, description,
dependencies)
files.write_file('', output_fname, plugin_list_md)
def process(self, unused, site, config):
plugin_name = "js_posts"
js_filename = "posts.json"
# configuration
output_path = config.output_path
meta_fields_to_output = config.meta_fields_to_output
plugin_data_to_output = config.plugin_data_to_output
log_info = "base javascript: %s<br>ouput:%s%s<br>" % (
js_filename, output_path, js_filename)
log_info = "meta fields to outputs:%s" % (
", ".join(meta_fields_to_output))
# Reading the base JS
plugin_dir = os.path.dirname(__file__)
js_file = os.path.join(plugin_dir, js_filename)
js = files.read_file(js_file)
if not js:
return (SiteFab.ERROR, plugin_name,
"Base Javascript:%s not found." % js_file)
js_posts = {}
for post in site.posts:
js_post = {}
js_post['id'] = post.id
for field in meta_fields_to_output:
if field in post.meta:
if field == "permanent_url":
data = "%s/" % post.meta[field]
js_post[field] = data
else:
js_post[field] = post.meta[field]
if 'responsive_banner' in plugin_data_to_output and 'responsive_images' in site.plugin_data:
if post.meta.banner in site.plugin_data['responsive_images']:
js_post['banner_srcsets'] = site.plugin_data[
'responsive_images'][post.meta.banner]['srcsets']
js_post['banner_allsizes'] = site.plugin_data[
'responsive_images'][post.meta.banner]['allsizes']
if 'thumbnail_banner' in plugin_data_to_output and 'thumbnails' in site.plugin_data:
if post.meta.banner in site.plugin_data['thumbnails']:
js_post['thumbnails'] = site.plugin_data['thumbnails'][
post.meta.banner]
js_posts[post.id] = js_post
# replacing placeholder with post data
output_string = json.dumps(js_posts)
log_info += "output string:<br>%s" % output_string
js = str(js)
js = js.replace("JS_POSTS_PLUGIN_REPLACE", output_string)
# output
path = os.path.join(site.get_output_dir(), output_path)
log_info += "output directory: %s" % path
files.write_file(path, js_filename, js)
return (SiteFab.OK, plugin_name, log_info)
def process(self, unused, site, config):
template_name = config.template
# Loading template
if template_name not in site.jinja2.list_templates():
return SiteFab.ERROR, "sitemap", ("template %s not found" %
template_name)
template = site.jinja2.get_template(template_name)
# Rendering
post_list = []
for post in site.posts:
if post.meta.hidden:
continue
# add priority and frequency
post.meta.priority = 0.7
post.meta.frequency = "monthly"
if post.meta.microdata_type == "AboutPage": # about page
post.meta.priority = 0.8
if post.meta.microdata_type == "Blog": # Blog page
post.meta.priority = 1.0
post.meta.frequency = "daily"
if post.meta.microdata_type == "WebSite": # main page
post.meta.priority = 1.0
post.meta.frequency = "daily"
if post.meta.microdata_type == "CollectionPage": # tags
post.meta.priority = 0.8
post.meta.frequency = "daily"
if post.meta.banner:
post.meta.sitemap_banner_url = "%s%s" % (site.config.url,
post.meta.banner)
else:
post.meta.sitemap_banner_url = "%s/static/images/banner/default.png" % site.config.url # noqa
post_list.append(post)
for collection in site.posts_by_category.get_as_list():
# add priority and frequency
collection.meta.priority = 0.7
collection.meta.frequency = "daily"
collection.meta.sitemap_banner_url = "%s/static/images/banner/default.png" % site.config.url # noqa
collections = site.posts_by_category.get_as_list()
try:
rv = template.render(posts=post_list, collections=collections)
except Exception as e:
return (SiteFab.ERROR, "sitemap", e)
# output
path = site.get_output_dir()
files.write_file(path, 'sitemap.xml', rv)
log_info = "template used:%s<br>ouput:%ssitemap.xml" % (template_name,
path)
return SiteFab.OK, "sitemap", log_info
def process(self, unused, site, config):
plugin_name = "search"
js_filename = "search.js"
json_filename = "search.json"
output_path_js = config.output_path_js
output_path_json = config.output_path_json
num_terms = config.num_terms_per_post
# log_info = "base javascript: %s<br>ouput:%s%s<br>" % (
# js_filename, output_path, js_filename)
log_info = ""
# Reading the base JS
plugin_dir = os.path.dirname(__file__)
json_file = os.path.join(plugin_dir, json_filename)
jsondata = files.read_file(json_file)
jsdata = files.read_file(os.path.join(plugin_dir, js_filename))
# if not js or len(js) < 10:
# err = "Base Javascript:%s not found or too small." % js_file
# return (SiteFab.ERROR, plugin_name, err)
js_posts = {}
table_data = []
for post in site.posts:
terms = [t[0] for t in post.nlp.terms][:num_terms]
js_post = {
"id": post.id,
"template": post.meta.template,
"title": post.nlp.clean_fields.title,
"authors": post.nlp.clean_fields.authors,
"conference": "%s %s" % (post.nlp.clean_fields.conference_short_name, # noqa
post.nlp.clean_fields.conference_name), # noqa
"terms": terms
}
js_posts[post.id] = js_post
table_data.append([js_post['title'], js_post['terms']])
log_info += tabulate(table_data, tablefmt='html',
headers=['title', 'terms'])
# output
output_string = json.dumps(js_posts)
jsondata = jsondata.replace("SEARCH_DOC_PLUGIN_REPLACE", output_string)
path_json = os.path.join(site.get_output_dir(), output_path_json)
files.write_file(path_json, json_filename, jsondata)
path_js = os.path.join(site.get_output_dir(), output_path_js)
files.write_file(path_js, js_filename, jsdata)
return (SiteFab.OK, plugin_name, log_info)
def test_filelist_filter(tmp_path):
"making sure we get all the files which have a givem suffix"
fnames = ['😁.md', 'a.m', 'a.txt', 'b.md', 'a.md.md']
content = 'this is a test'
clean_dir(tmp_path)
for fname in fnames:
write_file(tmp_path, fname, content)
files_list = get_files_list(tmp_path, '*.md')
assert len(files_list) == 3
files_list = get_files_list(tmp_path, '*.txt')
assert len(files_list) == 1
clean_dir(tmp_path)
def test_unicode_support(tmp_path):
fname = '😁.txt'
content = 'this is a test 😁'
path = Path(tmp_path) / fname
# write
write_file(tmp_path, fname, content)
assert path.exists()
# read
read_back = read_file(path)
assert read_back == content
# cleanup
clean_dir(tmp_path)
def test_filelist_recursive(tmp_path):
clean_dir(tmp_path)
sub_path = tmp_path / 'newdir'
fname = 'test.md'
txt = 'hello'
write_file(sub_path, 'test.md', txt)
# ensure we have a sub dir and the file in it
assert sub_path.exists()
assert sub_path.is_dir()
read_text = read_file(sub_path / fname)
assert read_text == txt
# test we get a recursive listing
assert len(get_files_list(tmp_path)) == 1
# test that non recursive returns nothing
assert get_files_list(tmp_path, recursive=False) == []
def test_writing_subdirectory(tmp_path):
"make sure sub directory is created"
fname = 'myfile.md'
content = 'this is a test 😁'
subdir = 'tobecreated'
subdir_path = Path(tmp_path) / subdir
path = subdir_path / fname
# write
write_file(subdir_path, fname, content)
assert path.exists()
assert subdir_path.is_dir()
# read
read_back = read_file(path)
assert read_back == content
# cleanup
clean_dir(tmp_path)
assert not path.exists()
assert not subdir_path.exists()
def process(self, unused, site, config):
plugin_name = "autocomplete"
json_filename = "autocomplete.json"
js_filename = "autocomplete.js"
# configuration
output_path_js = config.output_path_js
output_path_json = config.output_path_json
num_suggestions = config.num_suggestions
excluded_terms = config.excluded_terms
log_info = ""
# log_info = "base javascript: %s<br>ouput:%s%s" % (
# js_filename, output_path_js, js_filename)
# Reading the base JS
plugin_dir = Path(__file__).parent
json_file = plugin_dir / json_filename
jsondata = files.read_file(json_file)
js_file = plugin_dir / js_filename
jsdata = files.read_file(js_file)
term_post_frequency = defaultdict(int)
term_score = defaultdict(float)
for post in site.posts:
# authors
for author in post.nlp.clean_fields.authors:
term_post_frequency[author] += 1
term_score[author] += 1 # ensure authors always first
for part in author.split(' '):
if len(part) < 2:
continue
term_post_frequency[part] += 1
term_score[part] += 1
# title terms
for term in post.nlp.title_terms:
if term in excluded_terms:
continue
term_post_frequency[term[0]] += 1
term_score[term[0]] += term[1] * 2
# other terms
for term in post.nlp.terms:
if term in excluded_terms:
continue
term_post_frequency[term[0]] += 1
term_score[term[0]] += term[1]
output = []
log_info += "num of terms considered: %s<br>" % len(term_score)
top_terms = sorted(term_score, key=term_score.get, reverse=True)
for term in top_terms[:num_suggestions]:
score = term_score[term]
post_frequency = term_post_frequency[term]
output.append([term, post_frequency, score])
# log results
log_info += tabulate(output,
headers=['term', 'post frequency', 'score'],
tablefmt='html')
# replacing placeholder with computation result
output_string = json.dumps(output)
jsondata = jsondata.replace("AUTOCOMPLETE_PLUGIN_REPLACE", output_string)
# output
path_js = site.get_output_dir() / output_path_js
path_json = site.get_output_dir() / output_path_json
log_info += "<br> output directory for js: %s" % path_js
log_info += "<br> output directory for json: %s" % path_json
# write js data file
files.write_file(path_js, js_filename, jsdata)
# write code file
files.write_file(path_json, json_filename, jsondata)
return (SiteFab.OK, plugin_name, log_info)
def process(self, unused, site, config):
template_name = config.template
config.banner = "%s%s" % (site.config.url, config.banner)
config.icon = "%s%s" % (site.config.url, config.icon)
config.logo_svg = "%s%s" % (site.config.url, config.logo_svg)
# rendering template
if template_name not in site.jinja2.list_templates():
return SiteFab.ERROR, "rss", ("template %s not found" %
template_name)
template = site.jinja2.get_template(str(template_name))
# custom parser
parser_tpl_path = Path(config.parser.template_dir)
config.parser.templates_path = (site.config.root_dir / parser_tpl_path)
config.parser = Parser.make_config(config.parser)
parser = Parser(config.parser, site)
# generating feed
rss_items = []
count = 0
posts = []
for post in site.posts:
posts.append(post)
# sort posts from newer to older
def k(x):
return x.meta.creation_date_ts
posts.sort(key=k, reverse=True)
for post in posts:
if (post.meta.hidden or
((post.meta.microdata_type != "BlogPosting") # noqa
and (post.meta.microdata_type != "ScholarlyArticle") # noqa
and
(post.meta.microdata_type != "PublicationEvent"))): # noqa
continue
# parse the post with a customized parser
file_content = files.read_file(post.filename)
parsed_post = parser.parse(file_content)
# adding the newly generated HTML as RSS
post.rss = parsed_post.html
formatted_rss_creation_date = datetime.fromtimestamp(
int(post.meta.creation_date_ts)).strftime(
'%a, %d %b %Y %H:%M:%S -0800')
if post.meta.update_date_ts:
formatted_rss_update_date = datetime.fromtimestamp(
int(post.meta.update_date_ts)).strftime(
'%a, %d %b %Y %H:%M:%S -0800')
else:
formatted_rss_update_date = formatted_rss_creation_date
post.meta.formatted_creation = formatted_rss_creation_date
post.meta.formatted_update = formatted_rss_update_date
# size of image
if post.meta.banner in site.plugin_data['image_info']:
post.meta.banner_size = site.plugin_data['image_info'][
post.meta.banner]['file_size']
post.meta.banner_mimetype = site.plugin_data['image_info'][
post.meta.banner]['mime_type']
post.meta.banner_fullurl = "%s%s" % (site.config.url,
post.meta.banner)
post.meta.author = post.meta.authors[0].replace(",", "")
rss_items.append(post)
count += 1
if count == config.num_posts:
break
if not len(rss_items):
return (SiteFab.ERROR, "rss", 'no RSS items')
config.formatted_update = rss_items[0].meta.formatted_update
try:
rv = template.render(site=site, rss=config, items=rss_items)
except Exception as e:
return (SiteFab.ERROR, "rss", e)
# output
path = site.get_output_dir()
files.write_file(path, 'rss.xml', rv)
log_info = "template used:%s<br>ouput:%srss.xml" % (template_name,
path)
return SiteFab.OK, "rss", log_info
def render_report(self):
"Create a linting report for all posts"
report = self.jinja2_template.render(results=self.results)
files.write_file(self.config.output_dir, "linter.html", report)