def sort_tags(site, filepaths, dry_run=False):
""" Sorts all the tags in the given list of posts.
$ nikola tags --sort posts/*.rst
The above command will sort all tags alphabetically, in all rst
posts. This command can be run on all posts, to clean up things.
"""
posts = [post for post in site.timeline if post.source_path in filepaths]
if len(posts) == 0:
LOGGER.error("Need at least one post.")
return
FMT = 'Tags for {0}:\n{1:>6} - {2}\n{3:>6} - {4}\n'
OLD = 'old'
NEW = 'new'
for post in posts:
new_tags = sorted(post.tags)
if dry_run:
print(FMT.format(
post.source_path, OLD, post.tags, NEW, new_tags)
)
elif new_tags != post.tags:
_replace_tags_line(post, new_tags)
return new_tags
def analyze(self, task, find_sources=False):
rv = False
self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']]
try:
filename = task.split(":")[-1]
d = lxml.html.fromstring(open(filename).read())
for l in d.iterlinks():
target = l[0].attrib[l[1]]
if target == "#":
continue
parsed = urlparse(target)
if parsed.scheme or target.startswith('//'):
continue
if parsed.fragment:
target = target.split('#')[0]
target_filename = os.path.abspath(
os.path.join(os.path.dirname(filename), unquote(target)))
if any(re.match(x, target_filename) for x in self.whitelist):
continue
elif target_filename not in self.existing_targets:
if os.path.exists(target_filename):
self.existing_targets.add(target_filename)
else:
rv = True
LOGGER.warn("Broken link in {0}: ".format(filename), target)
if find_sources:
LOGGER.warn("Possible sources:")
LOGGER.warn(os.popen('nikola list --deps ' + task, 'r').read())
LOGGER.warn("===============================\n")
except Exception as exc:
LOGGER.error("Error with:", filename, exc)
return rv
def _replace_tags_line(post, tags):
""" Replaces the line that lists the tags, with given tags. """
if post.is_two_file:
path = post.metadata_path
try:
if not post.newstylemeta:
LOGGER.error("{0} uses old-style metadata which is not supported by this plugin, skipping.".format(path))
return
except AttributeError:
# post.newstylemeta is not present in older versions. If the user
# has old-style meta files, it will crash or not do the job.
pass
else:
path = post.source_path
with codecs.open(path, 'r', 'utf-8') as f:
text = f.readlines()
tag_identifier = u'.. tags:'
new_tags = u'.. tags: %s\n' % ', '.join(tags)
for index, line in enumerate(text[:]):
if line.startswith(tag_identifier):
text[index] = new_tags
break
with codecs.open(path, 'w+', 'utf-8') as f:
f.writelines(text)
def _execute(self, options, args):
"""Start the watcher."""
try:
from livereload.server import start
except ImportError:
LOGGER.error('To use the auto command, you need to install the '
'"livereload" package.')
return
# Run an initial build so we are uptodate
subprocess.call(("nikola", "build"))
port = options and options.get('port')
# Create a Guardfile
with codecs.open("Guardfile", "wb+", "utf8") as guardfile:
l = ["conf.py", "themes", "templates", self.site.config['GALLERY_PATH']]
for item in self.site.config['post_pages']:
l.append(os.path.dirname(item[0]))
for item in self.site.config['FILES_FOLDERS']:
l.append(os.path.dirname(item))
data = GUARDFILE.format(json.dumps(l))
guardfile.write(data)
out_folder = self.site.config['OUTPUT_FOLDER']
os.chmod("Guardfile", 0o755)
start(port, out_folder, options and options.get('browser'))
def _execute(self, options, args):
"""Apply mincss the generated site."""
output_folder = self.site.config["OUTPUT_FOLDER"]
if Processor is None:
LOGGER.warn("To use the mincss command," ' you have to install the "mincss" package.')
return
p = Processor(preserve_remote_urls=False)
urls = []
css_files = {}
for root, dirs, files in os.walk(output_folder):
for f in files:
url = os.path.join(root, f)
if url.endswith(".css"):
fname = os.path.basename(url)
if fname in css_files:
LOGGER.error("You have two CSS files with the same name and that confuses me.")
sys.exit(1)
css_files[fname] = url
if not f.endswith(".html"):
continue
urls.append(url)
p.process(*urls)
for inline in p.links:
fname = os.path.basename(inline.href)
with open(css_files[fname], "wb+") as outf:
outf.write(inline.after)
def _replace_tags_line(post, tags):
""" Replaces the line that lists the tags, with given tags. """
source_path = post.source_path
if post.is_two_file:
# fixme: currently doesn't handle two post files.
LOGGER.error(
"Two file posts are not supported, currently."
"Skipping %s" % source_path
)
return
with codecs.open(source_path, 'r', 'utf-8') as f:
post_text = f.readlines()
tag_identifier = u'.. tags:'
new_tags = u'.. tags: %s\n' % ', '.join(tags)
for index, line in enumerate(post_text[:]):
if line.startswith(tag_identifier):
post_text[index] = new_tags
break
with codecs.open(source_path, 'w+', 'utf-8') as f:
f.writelines(post_text)
def plain(self):
"""Plain Python shell."""
from nikola import Nikola
try:
import conf
SITE = Nikola(**conf.__dict__)
SITE.scan_posts()
gl = {'conf': conf, 'SITE': SITE, 'Nikola': Nikola}
except ImportError:
LOGGER.error("No configuration found, cannot run the console.")
else:
import code
try:
import readline
except ImportError:
pass
else:
import rlcompleter
readline.set_completer(rlcompleter.Completer(gl).complete)
readline.parse_and_bind("tab:complete")
pythonrc = os.environ.get("PYTHONSTARTUP")
if pythonrc and os.path.isfile(pythonrc):
try:
execfile(pythonrc) # NOQA
except NameError:
pass
code.interact(local=gl, banner=self.header.format('Python'))
def _replace_tags_line(post, tags):
""" Replaces the line that lists the tags, with given tags. """
source_path = post.source_path
if post.is_two_file:
# fixme: currently doesn't handle two post files.
LOGGER.error("Two file posts are not supported, currently."
"Skipping %s" % source_path)
return
with codecs.open(source_path, 'r', 'utf-8') as f:
post_text = f.readlines()
tag_identifier = u'.. tags:'
new_tags = u'.. tags: %s\n' % ', '.join(tags)
for index, line in enumerate(post_text[:]):
if line.startswith(tag_identifier):
post_text[index] = new_tags
break
with codecs.open(source_path, 'w+', 'utf-8') as f:
f.writelines(post_text)
def compile_html(self, source, dest, is_two_file=True):
makedirs(os.path.dirname(dest))
try:
subprocess.check_call(('pandoc', '-o', dest, source))
except OSError as e:
if e.strreror == 'No such file or directory':
LOGGER.error('To use the pandoc compiler,'
' you have to install the "pandoc" Haskell package.')
raise Exception('Cannot compile {0} -- pandoc '
'missing'.format(source))
def _execute(self, options, args):
"""Start test server."""
out_dir = self.site.config['OUTPUT_FOLDER']
if not os.path.isdir(out_dir):
LOGGER.error("Missing '{0}' folder?".format(out_dir))
else:
os.chdir(out_dir)
httpd = HTTPServer((options['address'], options['port']),
OurHTTPRequestHandler)
sa = httpd.socket.getsockname()
LOGGER.notice("Serving HTTP on {0} port {1} ...".format(*sa))
httpd.serve_forever()
def run(self):
if 'alt' in self.options and self.ignore_alt:
LOGGER.warning("Graphviz: the :alt: option is ignored, it's better to set the title of your graph.")
if self.arguments:
if self.content:
LOGGER.warning("Graphviz: this directive can't have both content and a filename argument. Ignoring content.")
f_name = self.arguments[0]
# TODO: be smart about where exactly that file is located
with open(f_name, 'rb') as inf:
data = inf.read().decode('utf-8')
else:
data = '\n'.join(self.content)
node_list = []
try:
p = Popen([self.dot_path, '-Tsvg'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
svg_data, errors = p.communicate(input=data.encode('utf8'))
code = p.wait()
if code: # Some error
document = self.state.document
return [document.reporter.error(
'Error processing graph: {0}'.format(errors), line=self.lineno)]
if self.embed_graph: # SVG embedded in the HTML
if 'inline' in self.options:
svg_data = '<span class="graphviz">{0}</span>'.format(svg_data)
else:
svg_data = '<p class="graphviz">{0}</p>'.format(svg_data)
else: # External SVG file
# TODO: there is no reason why this branch needs to be a raw
# directive. It could generate regular docutils nodes and
# be useful for any writer.
makedirs(self.output_folder)
f_name = hashlib.md5(svg_data).hexdigest() + '.svg'
img_path = self.graph_path + f_name
f_path = os.path.join(self.output_folder, f_name)
alt = self.options.get('alt', '')
with open(f_path, 'wb+') as outf:
outf.write(svg_data)
self.state.document.settings.record_dependencies.add(f_path)
if 'inline' in self.options:
svg_data = '<span class="graphviz"><img src="{0}" alt="{1}"></span>'.format(img_path, alt)
else:
svg_data = '<p class="graphviz"><img src="{0}" alt="{1}"></p>'.format(img_path, alt)
node_list.append(nodes.raw('', svg_data, format='html'))
if 'caption' in self.options and 'inline' not in self.options:
node_list.append(
nodes.raw('', '<p class="caption">{0}</p>'.format(self.options['caption']),
format='html'))
return node_list
except OSError:
LOGGER.error("Can't execute 'dot'")
raise
def ipython(self):
"""IPython shell."""
from nikola import Nikola
try:
import conf
except ImportError:
LOGGER.error("No configuration found, cannot run the console.")
else:
import IPython
SITE = Nikola(**conf.__dict__)
SITE.scan_posts()
IPython.embed(header=self.header.format('IPython'))
def run(self):
if 'alt' in self.options and self.ignore_alt:
LOGGER.warning("Graphviz: the :alt: option is ignored, it's better to set the title of your graph.")
if self.arguments:
if self.content:
LOGGER.warning("Graphviz: this directive can't have both content and a filename argument. Ignoring content.")
f_name = self.arguments[0]
# TODO: be smart about where exactly that file is located
with open(f_name, 'rb') as inf:
data = inf.read().decode('utf-8')
else:
data = '\n'.join(self.content)
node_list = []
try:
p = Popen([self.dot_path, '-Tsvg'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
svg_data, errors = p.communicate(input=data.encode('utf8'))
code = p.wait()
if code: # Some error
document = self.state.document
return [document.reporter.error(
'Error processing graph: {0}'.format(errors), line=self.lineno)]
if self.embed_graph: # SVG embedded in the HTML
if 'inline' in self.options:
svg_data = '<span class="graphviz">{0}</span>'.format(svg_data.decode('utf8'))
else:
svg_data = '<p class="graphviz">{0}</p>'.format(svg_data.decode('utf8'))
else: # External SVG file
# TODO: there is no reason why this branch needs to be a raw
# directive. It could generate regular docutils nodes and
# be useful for any writer.
makedirs(self.output_folder)
f_name = hashlib.md5(svg_data).hexdigest() + '.svg'
img_path = self.graph_path + f_name
f_path = os.path.join(self.output_folder, f_name)
alt = self.options.get('alt', '')
with open(f_path, 'wb+') as outf:
outf.write(svg_data)
self.state.document.settings.record_dependencies.add(f_path)
if 'inline' in self.options:
svg_data = '<span class="graphviz"><img src="{0}" alt="{1}"></span>'.format(img_path, alt)
else:
svg_data = '<p class="graphviz"><img src="{0}" alt="{1}"></p>'.format(img_path, alt)
node_list.append(nodes.raw('', svg_data, format='html'))
if 'caption' in self.options and 'inline' not in self.options:
node_list.append(
nodes.raw('', '<p class="caption">{0}</p>'.format(self.options['caption']),
format='html'))
return node_list
except OSError:
LOGGER.error("Can't execute 'dot'")
raise
def bpython(self):
"""bpython shell."""
from nikola import Nikola
try:
import conf
except ImportError:
LOGGER.error("No configuration found, cannot run the console.")
else:
import bpython
SITE = Nikola(**conf.__dict__)
SITE.scan_posts()
gl = {'conf': conf, 'SITE': SITE, 'Nikola': Nikola}
bpython.embed(banner=self.header.format('bpython'), locals_=gl)
def doc_shortcode(*args, **kwargs):
"""Implement the doc shortcode."""
text = kwargs['data']
success, twin_slugs, title, permalink, slug = _doc_link(text, text, LOGGER)
if success:
if twin_slugs:
LOGGER.warning(
'More than one post with the same slug. Using "{0}" for doc shortcode'.format(permalink))
return '<a href="{0}">{1}</a>'.format(permalink, title)
else:
LOGGER.error(
'"{0}" slug doesn\'t exist.'.format(slug))
return '<span class="error text-error" style="color: red;">Invalid link: {0}</span>'.format(text)
def doc_shortcode(*args, **kwargs):
"""Implement the doc shortcode."""
text = kwargs['data']
success, twin_slugs, title, permalink, slug = _doc_link(text, text, LOGGER)
if success:
if twin_slugs:
LOGGER.warn(
'More than one post with the same slug. Using "{0}" for doc shortcode'.format(permalink))
return '<a href="{0}">{1}</a>'.format(permalink, title)
else:
LOGGER.error(
'"{0}" slug doesn\'t exist.'.format(slug))
return '<span class="error text-error" style="color: red;">Invalid link: {0}</span>'.format(text)
def tag(self, post, count=5):
""" Return a list of top tags, given a post.
post: can either be a post object or the source path
count: the number of tags to return
"""
if isinstance(post, (bytes_str, unicode_str)):
source_path = post
post = self._get_post_from_source_path(source_path)
if post is None:
LOGGER.error('No post found for path: %s' % source_path)
return
return self._find_top_scoring_tags(post, count)
def gen_tasks(self):
"""Build final pages from metadata and HTML fragments."""
kw = {
"post_pages": self.site.config["post_pages"],
"translations": self.site.config["TRANSLATIONS"],
"filters": self.site.config["FILTERS"],
"show_untranslated_posts":
self.site.config['SHOW_UNTRANSLATED_POSTS'],
"demote_headers": self.site.config['DEMOTE_HEADERS'],
}
self.site.scan_posts()
yield self.group_task()
index_paths = {}
for lang in kw["translations"]:
index_paths[lang] = False
if not self.site.config[
"DISABLE_INDEXES_PLUGIN_INDEX_AND_ATOM_FEED"]:
index_paths[lang] = os.path.normpath(
os.path.join(self.site.config['OUTPUT_FOLDER'],
self.site.path('index', '', lang=lang)))
for lang in kw["translations"]:
for post in self.site.timeline:
if not kw[
"show_untranslated_posts"] and not post.is_translation_available(
lang):
continue
if post.is_post:
context = {'pagekind': ['post_page']}
else:
context = {'pagekind': ['story_page', 'page_page']}
for task in self.site.generic_page_renderer(
lang, post, kw["filters"], context):
if task['name'] == index_paths[lang]:
# Issue 3022
LOGGER.error(
"Post {0!r}: output path ({1}) conflicts with the blog index ({2}). "
"Please change INDEX_PATH or disable index generation."
.format(post.source_path, task['name'],
index_paths[lang]))
task['uptodate'] = task['uptodate'] + [
config_changed(kw, 'nikola.plugins.task.pages')
]
task['basename'] = self.name
task['task_dep'] = ['render_posts']
yield task
def _execute(self, options, args):
"""Manage the tags on the site."""
try:
import conf
except ImportError:
LOGGER.error("No configuration found, cannot run the console.")
else:
_reload(conf)
nikola = Nikola(**conf.__dict__)
nikola.scan_posts()
if len(options['add']) > 0 and len(args) > 0:
add_tags(nikola, options['add'], args, options['dry-run'])
elif options['list']:
list_tags(nikola, options['list_sorting'])
elif options['merge'].count(',') > 0 and len(args) > 0:
merge_tags(nikola, options['merge'], args, options['dry-run'])
elif len(options['remove']) > 0 and len(args) > 0:
remove_tags(nikola, options['remove'], args,
options['dry-run'])
elif len(options['search']) > 0:
search_tags(nikola, options['search'])
elif options['tag'] and len(args) > 0:
tagger = _AutoTag(nikola)
for post in args:
tags = ','.join(tagger.tag(post))
add_tags(nikola, tags, [post], options['dry-run'])
elif options['sort'] and len(args) > 0:
sort_tags(nikola, args, options['dry-run'])
else:
print(self.help())
def handler(self, title=None, site=None, data=None, lang=None):
"""Create an inter-site link
Args:
title: optional argument to specify a different title from the post
Returns:
output HTML to replace the shortcode
"""
success, twin_slugs, title, permalink, slug = lancelot_link(
site, data, title)
if success:
if twin_slugs:
LOGGER.warning('More than one post with the same slug. '
f'Using "{permalink}" for lancelot shortcode')
output = f'<a href="{permalink}">{title}</a>'
else:
LOGGER.error(f'"{slug}" slug doesn\'t exist.')
output = ('<span class="error text-error" style="color: red;">'
f'Invalid link: {data}</span>')
return output, []
def _execute(self, options, args):
"""Manage the tags on the site."""
try:
import conf
except ImportError:
LOGGER.error("No configuration found, cannot run the console.")
else:
_reload(conf)
nikola = Nikola(**conf.__dict__)
nikola.scan_posts()
if len(options['add']) > 0 and len(args) > 0:
add_tags(nikola, options['add'], args, options['dry-run'])
elif options['list']:
list_tags(nikola, options['list_sorting'])
elif options['merge'].count(',') > 0 and len(args) > 0:
merge_tags(nikola, options['merge'], args, options['dry-run'])
elif len(options['remove']) > 0 and len(args) > 0:
remove_tags(nikola, options['remove'], args, options['dry-run'])
elif len(options['search']) > 0:
search_tags(nikola, options['search'])
elif options['tag'] and len(args) > 0:
tagger = _AutoTag(nikola)
for post in args:
tags = ','.join(tagger.tag(post))
add_tags(nikola, tags, [post], options['dry-run'])
elif options['sort'] and len(args) > 0:
sort_tags(nikola, args, options['dry-run'])
else:
print(self.help())
def update_feed(feed):
modified = feed.last_modified.timetuple()
etag = feed.etag
try:
parsed = feedparser.parse(
feed.url,
etag=etag,
modified=modified
)
feed.last_status = str(parsed.status)
except: # Probably a timeout
# TODO: log failure
return
if parsed.feed.get('title'):
LOGGER.notice(parsed.feed.title)
else:
LOGGER.notice(feed.url)
feed.etag = parsed.get('etag', 'foo')
modified = tuple(parsed.get('date_parsed', (1970, 1, 1)))[:6]
LOGGER.notice("==========>", modified)
modified = datetime.datetime(*modified)
feed.last_modified = modified
feed.save()
# No point in adding items from missinfg feeds
if parsed.status > 400:
# TODO log failure
return
for entry_data in parsed.entries:
LOGGER.notice("=========================================")
date = entry_data.get('published_parsed', None)
if date is None:
date = entry_data.get('updated_parsed', None)
if date is None:
LOGGER.error("Can't parse date from:\n", entry_data)
return False
LOGGER.notice("DATE:===>", date)
date = datetime.datetime(*(date[:6]))
title = "%s: %s" % (feed.name, entry_data.get('title', 'Sin título'))
content = entry_data.get('content', None)
if content:
content = content[0].value
if not content:
content = entry_data.get('description', None)
if not content:
content = entry_data.get('summary', 'Sin contenido')
guid = str(entry_data.get('guid', entry_data.link))
link = entry_data.link
LOGGER.notice(repr([date, title]))
e = list(Entry.select().where(Entry.guid == guid))
LOGGER.notice(
repr(dict(
date=date,
title=title,
content=content,
guid=guid,
feed=feed,
link=link,
))
)
if not e:
entry = Entry.create(
date=date,
title=title,
content=content,
guid=guid,
feed=feed,
link=link,
)
else:
entry = e[0]
entry.date = date
entry.title = title
entry.content = content
entry.link = link
entry.save()
