def sort_tags(site, filepaths, dry_run=False): """ Sorts all the tags in the given list of posts. $ nikola tags --sort posts/*.rst The above command will sort all tags alphabetically, in all rst posts. This command can be run on all posts, to clean up things. """ posts = [post for post in site.timeline if post.source_path in filepaths] if len(posts) == 0: LOGGER.error("Need at least one post.") return FMT = 'Tags for {0}:\n{1:>6} - {2}\n{3:>6} - {4}\n' OLD = 'old' NEW = 'new' for post in posts: new_tags = sorted(post.tags) if dry_run: print(FMT.format( post.source_path, OLD, post.tags, NEW, new_tags) ) elif new_tags != post.tags: _replace_tags_line(post, new_tags) return new_tags
def analyze(self, task, find_sources=False): rv = False self.whitelist = [re.compile(x) for x in self.site.config['LINK_CHECK_WHITELIST']] try: filename = task.split(":")[-1] d = lxml.html.fromstring(open(filename).read()) for l in d.iterlinks(): target = l[0].attrib[l[1]] if target == "#": continue parsed = urlparse(target) if parsed.scheme or target.startswith('//'): continue if parsed.fragment: target = target.split('#')[0] target_filename = os.path.abspath( os.path.join(os.path.dirname(filename), unquote(target))) if any(re.match(x, target_filename) for x in self.whitelist): continue elif target_filename not in self.existing_targets: if os.path.exists(target_filename): self.existing_targets.add(target_filename) else: rv = True LOGGER.warn("Broken link in {0}: ".format(filename), target) if find_sources: LOGGER.warn("Possible sources:") LOGGER.warn(os.popen('nikola list --deps ' + task, 'r').read()) LOGGER.warn("===============================\n") except Exception as exc: LOGGER.error("Error with:", filename, exc) return rv
def _replace_tags_line(post, tags): """ Replaces the line that lists the tags, with given tags. """ if post.is_two_file: path = post.metadata_path try: if not post.newstylemeta: LOGGER.error("{0} uses old-style metadata which is not supported by this plugin, skipping.".format(path)) return except AttributeError: # post.newstylemeta is not present in older versions. If the user # has old-style meta files, it will crash or not do the job. pass else: path = post.source_path with codecs.open(path, 'r', 'utf-8') as f: text = f.readlines() tag_identifier = u'.. tags:' new_tags = u'.. tags: %s\n' % ', '.join(tags) for index, line in enumerate(text[:]): if line.startswith(tag_identifier): text[index] = new_tags break with codecs.open(path, 'w+', 'utf-8') as f: f.writelines(text)
def _execute(self, options, args): """Start the watcher.""" try: from livereload.server import start except ImportError: LOGGER.error('To use the auto command, you need to install the ' '"livereload" package.') return # Run an initial build so we are uptodate subprocess.call(("nikola", "build")) port = options and options.get('port') # Create a Guardfile with codecs.open("Guardfile", "wb+", "utf8") as guardfile: l = ["conf.py", "themes", "templates", self.site.config['GALLERY_PATH']] for item in self.site.config['post_pages']: l.append(os.path.dirname(item[0])) for item in self.site.config['FILES_FOLDERS']: l.append(os.path.dirname(item)) data = GUARDFILE.format(json.dumps(l)) guardfile.write(data) out_folder = self.site.config['OUTPUT_FOLDER'] os.chmod("Guardfile", 0o755) start(port, out_folder, options and options.get('browser'))
def _execute(self, options, args): """Apply mincss the generated site.""" output_folder = self.site.config["OUTPUT_FOLDER"] if Processor is None: LOGGER.warn("To use the mincss command," ' you have to install the "mincss" package.') return p = Processor(preserve_remote_urls=False) urls = [] css_files = {} for root, dirs, files in os.walk(output_folder): for f in files: url = os.path.join(root, f) if url.endswith(".css"): fname = os.path.basename(url) if fname in css_files: LOGGER.error("You have two CSS files with the same name and that confuses me.") sys.exit(1) css_files[fname] = url if not f.endswith(".html"): continue urls.append(url) p.process(*urls) for inline in p.links: fname = os.path.basename(inline.href) with open(css_files[fname], "wb+") as outf: outf.write(inline.after)
def _replace_tags_line(post, tags): """ Replaces the line that lists the tags, with given tags. """ source_path = post.source_path if post.is_two_file: # fixme: currently doesn't handle two post files. LOGGER.error( "Two file posts are not supported, currently." "Skipping %s" % source_path ) return with codecs.open(source_path, 'r', 'utf-8') as f: post_text = f.readlines() tag_identifier = u'.. tags:' new_tags = u'.. tags: %s\n' % ', '.join(tags) for index, line in enumerate(post_text[:]): if line.startswith(tag_identifier): post_text[index] = new_tags break with codecs.open(source_path, 'w+', 'utf-8') as f: f.writelines(post_text)
def plain(self): """Plain Python shell.""" from nikola import Nikola try: import conf SITE = Nikola(**conf.__dict__) SITE.scan_posts() gl = {'conf': conf, 'SITE': SITE, 'Nikola': Nikola} except ImportError: LOGGER.error("No configuration found, cannot run the console.") else: import code try: import readline except ImportError: pass else: import rlcompleter readline.set_completer(rlcompleter.Completer(gl).complete) readline.parse_and_bind("tab:complete") pythonrc = os.environ.get("PYTHONSTARTUP") if pythonrc and os.path.isfile(pythonrc): try: execfile(pythonrc) # NOQA except NameError: pass code.interact(local=gl, banner=self.header.format('Python'))
def _replace_tags_line(post, tags): """ Replaces the line that lists the tags, with given tags. """ source_path = post.source_path if post.is_two_file: # fixme: currently doesn't handle two post files. LOGGER.error("Two file posts are not supported, currently." "Skipping %s" % source_path) return with codecs.open(source_path, 'r', 'utf-8') as f: post_text = f.readlines() tag_identifier = u'.. tags:' new_tags = u'.. tags: %s\n' % ', '.join(tags) for index, line in enumerate(post_text[:]): if line.startswith(tag_identifier): post_text[index] = new_tags break with codecs.open(source_path, 'w+', 'utf-8') as f: f.writelines(post_text)
def compile_html(self, source, dest, is_two_file=True): makedirs(os.path.dirname(dest)) try: subprocess.check_call(('pandoc', '-o', dest, source)) except OSError as e: if e.strreror == 'No such file or directory': LOGGER.error('To use the pandoc compiler,' ' you have to install the "pandoc" Haskell package.') raise Exception('Cannot compile {0} -- pandoc ' 'missing'.format(source))
def _execute(self, options, args): """Start test server.""" out_dir = self.site.config['OUTPUT_FOLDER'] if not os.path.isdir(out_dir): LOGGER.error("Missing '{0}' folder?".format(out_dir)) else: os.chdir(out_dir) httpd = HTTPServer((options['address'], options['port']), OurHTTPRequestHandler) sa = httpd.socket.getsockname() LOGGER.notice("Serving HTTP on {0} port {1} ...".format(*sa)) httpd.serve_forever()
def run(self): if 'alt' in self.options and self.ignore_alt: LOGGER.warning("Graphviz: the :alt: option is ignored, it's better to set the title of your graph.") if self.arguments: if self.content: LOGGER.warning("Graphviz: this directive can't have both content and a filename argument. Ignoring content.") f_name = self.arguments[0] # TODO: be smart about where exactly that file is located with open(f_name, 'rb') as inf: data = inf.read().decode('utf-8') else: data = '\n'.join(self.content) node_list = [] try: p = Popen([self.dot_path, '-Tsvg'], stdin=PIPE, stdout=PIPE, stderr=PIPE) svg_data, errors = p.communicate(input=data.encode('utf8')) code = p.wait() if code: # Some error document = self.state.document return [document.reporter.error( 'Error processing graph: {0}'.format(errors), line=self.lineno)] if self.embed_graph: # SVG embedded in the HTML if 'inline' in self.options: svg_data = '<span class="graphviz">{0}</span>'.format(svg_data) else: svg_data = '<p class="graphviz">{0}</p>'.format(svg_data) else: # External SVG file # TODO: there is no reason why this branch needs to be a raw # directive. It could generate regular docutils nodes and # be useful for any writer. makedirs(self.output_folder) f_name = hashlib.md5(svg_data).hexdigest() + '.svg' img_path = self.graph_path + f_name f_path = os.path.join(self.output_folder, f_name) alt = self.options.get('alt', '') with open(f_path, 'wb+') as outf: outf.write(svg_data) self.state.document.settings.record_dependencies.add(f_path) if 'inline' in self.options: svg_data = '<span class="graphviz"><img src="{0}" alt="{1}"></span>'.format(img_path, alt) else: svg_data = '<p class="graphviz"><img src="{0}" alt="{1}"></p>'.format(img_path, alt) node_list.append(nodes.raw('', svg_data, format='html')) if 'caption' in self.options and 'inline' not in self.options: node_list.append( nodes.raw('', '<p class="caption">{0}</p>'.format(self.options['caption']), format='html')) return node_list except OSError: LOGGER.error("Can't execute 'dot'") raise
def ipython(self): """IPython shell.""" from nikola import Nikola try: import conf except ImportError: LOGGER.error("No configuration found, cannot run the console.") else: import IPython SITE = Nikola(**conf.__dict__) SITE.scan_posts() IPython.embed(header=self.header.format('IPython'))
def run(self): if 'alt' in self.options and self.ignore_alt: LOGGER.warning("Graphviz: the :alt: option is ignored, it's better to set the title of your graph.") if self.arguments: if self.content: LOGGER.warning("Graphviz: this directive can't have both content and a filename argument. Ignoring content.") f_name = self.arguments[0] # TODO: be smart about where exactly that file is located with open(f_name, 'rb') as inf: data = inf.read().decode('utf-8') else: data = '\n'.join(self.content) node_list = [] try: p = Popen([self.dot_path, '-Tsvg'], stdin=PIPE, stdout=PIPE, stderr=PIPE) svg_data, errors = p.communicate(input=data.encode('utf8')) code = p.wait() if code: # Some error document = self.state.document return [document.reporter.error( 'Error processing graph: {0}'.format(errors), line=self.lineno)] if self.embed_graph: # SVG embedded in the HTML if 'inline' in self.options: svg_data = '<span class="graphviz">{0}</span>'.format(svg_data.decode('utf8')) else: svg_data = '<p class="graphviz">{0}</p>'.format(svg_data.decode('utf8')) else: # External SVG file # TODO: there is no reason why this branch needs to be a raw # directive. It could generate regular docutils nodes and # be useful for any writer. makedirs(self.output_folder) f_name = hashlib.md5(svg_data).hexdigest() + '.svg' img_path = self.graph_path + f_name f_path = os.path.join(self.output_folder, f_name) alt = self.options.get('alt', '') with open(f_path, 'wb+') as outf: outf.write(svg_data) self.state.document.settings.record_dependencies.add(f_path) if 'inline' in self.options: svg_data = '<span class="graphviz"><img src="{0}" alt="{1}"></span>'.format(img_path, alt) else: svg_data = '<p class="graphviz"><img src="{0}" alt="{1}"></p>'.format(img_path, alt) node_list.append(nodes.raw('', svg_data, format='html')) if 'caption' in self.options and 'inline' not in self.options: node_list.append( nodes.raw('', '<p class="caption">{0}</p>'.format(self.options['caption']), format='html')) return node_list except OSError: LOGGER.error("Can't execute 'dot'") raise
def bpython(self): """bpython shell.""" from nikola import Nikola try: import conf except ImportError: LOGGER.error("No configuration found, cannot run the console.") else: import bpython SITE = Nikola(**conf.__dict__) SITE.scan_posts() gl = {'conf': conf, 'SITE': SITE, 'Nikola': Nikola} bpython.embed(banner=self.header.format('bpython'), locals_=gl)
def doc_shortcode(*args, **kwargs): """Implement the doc shortcode.""" text = kwargs['data'] success, twin_slugs, title, permalink, slug = _doc_link(text, text, LOGGER) if success: if twin_slugs: LOGGER.warning( 'More than one post with the same slug. Using "{0}" for doc shortcode'.format(permalink)) return '<a href="{0}">{1}</a>'.format(permalink, title) else: LOGGER.error( '"{0}" slug doesn\'t exist.'.format(slug)) return '<span class="error text-error" style="color: red;">Invalid link: {0}</span>'.format(text)
def doc_shortcode(*args, **kwargs): """Implement the doc shortcode.""" text = kwargs['data'] success, twin_slugs, title, permalink, slug = _doc_link(text, text, LOGGER) if success: if twin_slugs: LOGGER.warn( 'More than one post with the same slug. Using "{0}" for doc shortcode'.format(permalink)) return '<a href="{0}">{1}</a>'.format(permalink, title) else: LOGGER.error( '"{0}" slug doesn\'t exist.'.format(slug)) return '<span class="error text-error" style="color: red;">Invalid link: {0}</span>'.format(text)
def tag(self, post, count=5): """ Return a list of top tags, given a post. post: can either be a post object or the source path count: the number of tags to return """ if isinstance(post, (bytes_str, unicode_str)): source_path = post post = self._get_post_from_source_path(source_path) if post is None: LOGGER.error('No post found for path: %s' % source_path) return return self._find_top_scoring_tags(post, count)
def gen_tasks(self): """Build final pages from metadata and HTML fragments.""" kw = { "post_pages": self.site.config["post_pages"], "translations": self.site.config["TRANSLATIONS"], "filters": self.site.config["FILTERS"], "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'], "demote_headers": self.site.config['DEMOTE_HEADERS'], } self.site.scan_posts() yield self.group_task() index_paths = {} for lang in kw["translations"]: index_paths[lang] = False if not self.site.config[ "DISABLE_INDEXES_PLUGIN_INDEX_AND_ATOM_FEED"]: index_paths[lang] = os.path.normpath( os.path.join(self.site.config['OUTPUT_FOLDER'], self.site.path('index', '', lang=lang))) for lang in kw["translations"]: for post in self.site.timeline: if not kw[ "show_untranslated_posts"] and not post.is_translation_available( lang): continue if post.is_post: context = {'pagekind': ['post_page']} else: context = {'pagekind': ['story_page', 'page_page']} for task in self.site.generic_page_renderer( lang, post, kw["filters"], context): if task['name'] == index_paths[lang]: # Issue 3022 LOGGER.error( "Post {0!r}: output path ({1}) conflicts with the blog index ({2}). " "Please change INDEX_PATH or disable index generation." .format(post.source_path, task['name'], index_paths[lang])) task['uptodate'] = task['uptodate'] + [ config_changed(kw, 'nikola.plugins.task.pages') ] task['basename'] = self.name task['task_dep'] = ['render_posts'] yield task
def _execute(self, options, args): """Manage the tags on the site.""" try: import conf except ImportError: LOGGER.error("No configuration found, cannot run the console.") else: _reload(conf) nikola = Nikola(**conf.__dict__) nikola.scan_posts() if len(options['add']) > 0 and len(args) > 0: add_tags(nikola, options['add'], args, options['dry-run']) elif options['list']: list_tags(nikola, options['list_sorting']) elif options['merge'].count(',') > 0 and len(args) > 0: merge_tags(nikola, options['merge'], args, options['dry-run']) elif len(options['remove']) > 0 and len(args) > 0: remove_tags(nikola, options['remove'], args, options['dry-run']) elif len(options['search']) > 0: search_tags(nikola, options['search']) elif options['tag'] and len(args) > 0: tagger = _AutoTag(nikola) for post in args: tags = ','.join(tagger.tag(post)) add_tags(nikola, tags, [post], options['dry-run']) elif options['sort'] and len(args) > 0: sort_tags(nikola, args, options['dry-run']) else: print(self.help())
def handler(self, title=None, site=None, data=None, lang=None): """Create an inter-site link Args: title: optional argument to specify a different title from the post Returns: output HTML to replace the shortcode """ success, twin_slugs, title, permalink, slug = lancelot_link( site, data, title) if success: if twin_slugs: LOGGER.warning('More than one post with the same slug. ' f'Using "{permalink}" for lancelot shortcode') output = f'<a href="{permalink}">{title}</a>' else: LOGGER.error(f'"{slug}" slug doesn\'t exist.') output = ('<span class="error text-error" style="color: red;">' f'Invalid link: {data}</span>') return output, []
def update_feed(feed): modified = feed.last_modified.timetuple() etag = feed.etag try: parsed = feedparser.parse( feed.url, etag=etag, modified=modified ) feed.last_status = str(parsed.status) except: # Probably a timeout # TODO: log failure return if parsed.feed.get('title'): LOGGER.notice(parsed.feed.title) else: LOGGER.notice(feed.url) feed.etag = parsed.get('etag', 'foo') modified = tuple(parsed.get('date_parsed', (1970, 1, 1)))[:6] LOGGER.notice("==========>", modified) modified = datetime.datetime(*modified) feed.last_modified = modified feed.save() # No point in adding items from missinfg feeds if parsed.status > 400: # TODO log failure return for entry_data in parsed.entries: LOGGER.notice("=========================================") date = entry_data.get('published_parsed', None) if date is None: date = entry_data.get('updated_parsed', None) if date is None: LOGGER.error("Can't parse date from:\n", entry_data) return False LOGGER.notice("DATE:===>", date) date = datetime.datetime(*(date[:6])) title = "%s: %s" % (feed.name, entry_data.get('title', 'Sin título')) content = entry_data.get('content', None) if content: content = content[0].value if not content: content = entry_data.get('description', None) if not content: content = entry_data.get('summary', 'Sin contenido') guid = str(entry_data.get('guid', entry_data.link)) link = entry_data.link LOGGER.notice(repr([date, title])) e = list(Entry.select().where(Entry.guid == guid)) LOGGER.notice( repr(dict( date=date, title=title, content=content, guid=guid, feed=feed, link=link, )) ) if not e: entry = Entry.create( date=date, title=title, content=content, guid=guid, feed=feed, link=link, ) else: entry = e[0] entry.date = date entry.title = title entry.content = content entry.link = link entry.save()