Example #1
0
    def __init__(self, source):
        self.source = path(source).abspath()
        """The absolute path to the source file for the post."""

        self.html_template_path = 'theme/post_detail.html'
        """The path to the template to use to transform the post into HTML."""

        self.markdown_template_path = 'core/post.md'
        """The path to the template to use to transform the post back into a :ref:`post source file <posts>`."""

        # This will get set to `True in _parse_source if the source file has 'fenced metadata' (like Jekyll)
        self._fence = False

        metadata, self._content_raw = self._parse_source()

        if not hasattr(self, 'content_preprocessed'):
            self.content_preprocessed = self.content_raw

        # Handle any preprocessor plugins
        for plugin in PostProcessor.plugins:
            plugin.preprocess(self, metadata)

        self.title = metadata.pop(
            'title',
            self.source.namebase.replace('-', ' ').replace('_', ' ').title())
        """The title of the post."""

        self.slug = metadata.pop('slug', slugify(self.title))
        """The slug for the post."""

        self.tags = wrap_list(metadata.pop('tags', []))
        """A list of strings representing the tags applied to the post."""

        self.link = metadata.pop('link', None)
        """The post's :ref:`external link <post link>`."""

        self.via = metadata.pop('via', None)
        """The post's attribution name."""

        self.via_link = metadata.pop('via-link',
                                     metadata.pop('via_link', None))
        """The post's attribution link."""

        try:
            self.status = Status(metadata.pop('status', Status.draft.name))
            """The status of the post (published or draft)."""

        except ValueError:
            logger.warning(
                "'%s': Invalid status value in metadata. Defaulting to 'draft'."
                % self.title)
            self.status = Status.draft

        self.timestamp = metadata.pop('timestamp', None)
        """The date/time the post was published or written."""

        if self.timestamp is None:
            self.timestamp = times.now()
            utctime = True
        else:
            utctime = False

        if not isinstance(self.timestamp, datetime):
            # looks like the timestamp from YAML wasn't directly convertible to a datetime, so we need to parse it
            self.timestamp = parser.parse(str(self.timestamp))

        if self.timestamp.tzinfo is not None:
            # parsed timestamp has an associated timezone, so convert it to UTC
            self.timestamp = times.to_universal(self.timestamp)
        elif not utctime:
            # convert to UTC assuming input time is in the DEFAULT_TIMEZONE
            self.timestamp = times.to_universal(self.timestamp,
                                                settings.POST_TIMEZONE)

        self.content = Post.convert_to_html(self.content_preprocessed)
        """The post's content in HTML format."""

        # determine the URL based on the HOME_URL and the PERMALINK_STYLE settings
        permalink = settings.PERMALINK_STYLE.format(
            year=unicode(self.timestamp_local.year),
            month=u'{0:02d}'.format(self.timestamp_local.month),
            day=u'{0:02d}'.format(self.timestamp_local.day),
            i_month=self.timestamp_local.month,
            i_day=self.timestamp_local.day,
            title=self.slug,  # for Jekyll compatibility
            slug=self.slug,
            timestamp=self.timestamp_local,
            post=self)
        if permalink.endswith('index.html'):
            permalink = permalink[:-10]
        elif permalink.endswith('.html') or permalink.endswith('/'):
            pass
        else:
            permalink += '.html'
        self._permalink = permalink

        # keep track of any remaining properties in the post metadata
        metadata.pop(
            'url',
            None)  # remove the url property from the metadata dict before copy
        self.custom_properties = copy(metadata)
        """A dict of any custom metadata properties specified in the post."""

        # handle any postprocessor plugins
        for plugin in PostProcessor.plugins:
            plugin.postprocess(self)

        # update cache
        settings.POST_CACHE[self.source] = self
Example #2
0
    def __init__(self, source):
        self.source = path(source).abspath()
        """The absolute path to the source file for the post."""

        self.html_template_path = 'theme/post_detail.html'
        """The path to the template to use to transform the post into HTML."""

        self.markdown_template_path = 'core/post.md'
        """The path to the template to use to transform the post back into a :ref:`post source file <posts>`."""

        # This will get set to `True in _parse_source if the source file has 'fenced metadata' (like Jekyll)
        self._fence = False

        metadata, self._content_raw = self._parse_source()

        if not hasattr(self, 'content_preprocessed'):
            self.content_preprocessed = self.content_raw

        # Handle any preprocessor plugins
        for plugin in PostProcessor.plugins:
            plugin.preprocess(self, metadata)

        self.title = metadata.pop('title', self.source.namebase.replace('-', ' ').replace('_', ' ').title())
        """The title of the post."""

        self.slug = metadata.pop('slug', slugify(self.title))
        """The slug for the post."""

        self.tags = wrap_list(metadata.pop('tags', []))
        """A list of strings representing the tags applied to the post."""

        self.link = metadata.pop('link', None)
        """The post's :ref:`external link <post link>`."""

        self.via = metadata.pop('via', None)
        """The post's attribution name."""

        self.via_link = metadata.pop('via-link', metadata.pop('via_link', None))
        """The post's attribution link."""

        try:
            self.status = Status(metadata.pop('status', Status.draft.name))
            """The status of the post (published or draft)."""

        except ValueError:
            logger.warning("'%s': Invalid status value in metadata. Defaulting to 'draft'." % self.title)
            self.status = Status.draft

        self.timestamp = metadata.pop('timestamp', None)
        """The date/time the post was published or written."""

        if self.timestamp is None:
            self.timestamp = times.now()
            utctime = True
        else:
            utctime = False

        if not isinstance(self.timestamp, datetime):
            # looks like the timestamp from YAML wasn't directly convertible to a datetime, so we need to parse it
            self.timestamp = parser.parse(str(self.timestamp))

        if self.timestamp.tzinfo is not None:
            # parsed timestamp has an associated timezone, so convert it to UTC
            self.timestamp = times.to_universal(self.timestamp)
        elif not utctime:
            # convert to UTC assuming input time is in the DEFAULT_TIMEZONE
            self.timestamp = times.to_universal(self.timestamp, settings.POST_TIMEZONE)

        self.content = Post.convert_to_html(self.content_preprocessed)
        """The post's content in HTML format."""

        # determine the URL based on the HOME_URL and the PERMALINK_STYLE settings
        permalink = settings.PERMALINK_STYLE.format(year=unicode(self.timestamp_local.year),
                                                    month=u'{0:02d}'.format(self.timestamp_local.month),
                                                    day=u'{0:02d}'.format(self.timestamp_local.day),
                                                    i_month=self.timestamp_local.month,
                                                    i_day=self.timestamp_local.day,
                                                    title=self.slug, # for Jekyll compatibility
                                                    slug=self.slug,
                                                    timestamp=self.timestamp_local,
                                                    post=self)
        if permalink.endswith('index.html'):
            permalink = permalink[:-10]
        elif permalink.endswith('.html') or permalink.endswith('/'):
            pass
        else:
            permalink += '.html'
        self._permalink = permalink

        # keep track of any remaining properties in the post metadata
        metadata.pop('url', None) # remove the url property from the metadata dict before copy
        self.custom_properties = copy(metadata)
        """A dict of any custom metadata properties specified in the post."""

        # handle any postprocessor plugins
        for plugin in PostProcessor.plugins:
            plugin.postprocess(self)

        # update cache
        settings.POST_CACHE[self.source] = self
Example #3
0
def build(args=None):
    """Builds an Engineer site using the settings specified in *args*."""
    from engineer.conf import settings
    from engineer.loaders import LocalLoader
    from engineer.log import get_file_handler
    from engineer.models import PostCollection, TemplatePage
    from engineer.themes import ThemeManager
    from engineer.util import mirror_folder, ensure_exists, slugify

    if args and args.clean:
        clean()

    settings.create_required_directories()

    logger = logging.getLogger('engineer.engine.build')
    logger.parent.addHandler(get_file_handler(settings.LOG_FILE))

    logger.debug("Starting build using configuration file %s." %
                 settings.SETTINGS_FILE)

    build_stats = {
        'time_run': times.now(),
        'counts': {
            'template_pages': 0,
            'new_posts': 0,
            'cached_posts': 0,
            'rollups': 0,
            'tag_pages': 0,
        },
        'files': {},
    }

    # Remove the output cache (not the post cache or the Jinja cache)
    # since we're rebuilding the site
    settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)

    # Copy static content to output dir
    logger.debug("Copying static files to output cache.")
    s = (settings.ENGINEER.STATIC_DIR / 'engineer').abspath()
    t = settings.OUTPUT_STATIC_DIR / 'engineer'
    mirror_folder(
        s, t,
        recurse=False)  # Copy only the files in this folder - don't recurse

    theme = ThemeManager.current_theme()
    # Copy Foundation files if used
    if theme.use_foundation:
        s = settings.ENGINEER.LIB_DIR / 'foundation'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR /
                          'engineer/lib/foundation')
        mirror_folder(s, t)
        logger.debug("Copied Foundation library files.")

    # Copy LESS js file if needed
    if theme.use_lesscss and not settings.PREPROCESS_LESS:
        s = settings.ENGINEER.LIB_DIR / 'less-1.3.1.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied LESS CSS files.")

    # Copy jQuery files if needed
    if theme.use_jquery or theme.use_tweet:
        s = settings.ENGINEER.LIB_DIR / 'jquery-1.7.1.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied jQuery files.")

    # Copy Tweet files if needed
    if theme.use_tweet:
        s = settings.ENGINEER.LIB_DIR / 'tweet'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/tweet')
        mirror_folder(s, t)
        logger.debug("Copied Tweet files.")

    # Copy modernizr files if needed
    if theme.use_modernizr:
        s = settings.ENGINEER.LIB_DIR / 'modernizr-2.5.3.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied Modernizr files.")

    logger.debug("Copied static files to %s." % relpath(t))

    # Copy 'raw' content to output cache - first pass
    # This first pass ensures that any static content - JS/LESS/CSS - that
    # is needed by site-specific pages (like template pages) is available
    # during the build
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Copy theme static content to output dir
    logger.debug("Copying theme static files to output cache.")
    try:
        s = theme.static_root.abspath()
    except ThemeNotFoundException as e:
        logger.critical(e.message)
        exit()
    t = (settings.OUTPUT_STATIC_DIR / 'theme').abspath()
    mirror_folder(s, t)
    logger.debug("Copied static files for theme to %s." % relpath(t))

    # Copy any theme additional content to output dir if needed
    if theme.content_mappings:
        logger.debug("Copying additional theme content to output cache.")
        for s, t in theme.content_mappings.iteritems():
            t = settings.OUTPUT_STATIC_DIR / 'theme' / t
            if s.isdir():
                mirror_folder(s, t)
            else:
                s.copy(ensure_exists(t))
            logger.debug("Copied additional files for theme to %s." %
                         relpath(t))

    # Load markdown input posts
    logger.info("Loading posts...")
    new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
    all_posts = PostCollection(new_posts + cached_posts)

    to_publish = PostCollection(all_posts.published)
    if settings.PUBLISH_DRAFTS:
        to_publish.extend(all_posts.drafts)
    if settings.PUBLISH_PENDING:
        to_publish.extend(all_posts.pending)
    if settings.PUBLISH_REVIEW:
        to_publish.extend(all_posts.review)

    if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
        logger.warning("This site contains the following pending posts:")
        for post in all_posts.pending:
            logger.warning("\t'%s' - publish time: %s, %s." %
                           (post.title, humanize.naturaltime(
                               post.timestamp), post.timestamp_local))
        logger.warning(
            "These posts won't be published until you build the site again after their publish time."
        )

    all_posts = PostCollection(
        sorted(to_publish, reverse=True, key=lambda post: post.timestamp))

    # Generate template pages
    if settings.TEMPLATE_PAGE_DIR.exists():
        logger.info("Generating template pages from %s." %
                    settings.TEMPLATE_PAGE_DIR)
        template_pages = []
        for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
            # We create all the TemplatePage objects first so we have all of the URLs to them in the template
            # environment. Without this step, template pages might have broken links if they link to a page that is
            # loaded after them, since the URL to the not-yet-loaded page will be missing.
            template_pages.append(TemplatePage(template))
        for page in template_pages:
            rendered_page = page.render_html(all_posts)
            ensure_exists(page.output_path)
            with open(page.output_path / page.output_file_name,
                      mode='wb',
                      encoding='UTF-8') as file:
                file.write(rendered_page)
                logger.debug("Output template page %s." % relpath(file.name))
                build_stats['counts']['template_pages'] += 1
        logger.info("Generated %s template pages." %
                    build_stats['counts']['template_pages'])

    # Generate individual post pages
    for post in all_posts:
        rendered_post = post.render_html(all_posts)
        ensure_exists(post.output_path)
        with open(post.output_path, mode='wb', encoding='UTF-8') as file:
            file.write(rendered_post)
            if post in new_posts:
                logger.console("Output new or modified post '%s'." %
                               post.title)
                build_stats['counts']['new_posts'] += 1
            elif post in cached_posts:
                build_stats['counts']['cached_posts'] += 1

    # Generate rollup pages
    num_posts = len(all_posts)
    num_slices = (
        num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0\
    else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1

    slice_num = 0
    for posts in all_posts.paginate():
        slice_num += 1
        has_next = slice_num < num_slices
        has_previous = 1 < slice_num <= num_slices
        rendered_page = posts.render_listpage_html(slice_num, has_next,
                                                   has_previous)
        ensure_exists(posts.output_path(slice_num))
        with open(posts.output_path(slice_num), mode='wb',
                  encoding='UTF-8') as file:
            file.write(rendered_page)
            logger.debug("Output rollup page %s." % relpath(file.name))
            build_stats['counts']['rollups'] += 1

        # Copy first rollup page to root of site - it's the homepage.
        if slice_num == 1:
            path.copyfile(posts.output_path(slice_num),
                          settings.OUTPUT_CACHE_DIR / 'index.html')
            logger.debug("Output '%s'." %
                         (settings.OUTPUT_CACHE_DIR / 'index.html'))

    # Generate archive page
    if num_posts > 0:
        archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
        ensure_exists(archive_output_path)

        rendered_archive = all_posts.render_archive_html(all_posts)

        with open(archive_output_path, mode='wb', encoding='UTF-8') as file:
            file.write(rendered_archive)
            logger.debug("Output %s." % relpath(file.name))

    # Generate tag pages
    if num_posts > 0:
        tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
        for tag in all_posts.all_tags:
            rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
            tag_path = ensure_exists(tags_output_path / slugify(tag) /
                                     'index.html')
            with open(tag_path, mode='wb', encoding='UTF-8') as file:
                file.write(rendered_tag_page)
                build_stats['counts']['tag_pages'] += 1
                logger.debug("Output %s." % relpath(file.name))

    # Generate feeds
    feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                     'feeds/rss.xml')
    feed_content = settings.JINJA_ENV.get_or_select_template([
        'rss.xml', 'theme/rss.xml', 'core/rss.xml'
    ]).render(post_list=all_posts[:settings.FEED_ITEM_LIMIT],
              build_date=all_posts[0].timestamp)
    with open(feed_output_path, mode='wb', encoding='UTF-8') as file:
        file.write(feed_content)
        logger.debug("Output %s." % relpath(file.name))

    # Generate sitemap
    sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                        'sitemap.xml.gz')
    sitemap_content = settings.JINJA_ENV.get_or_select_template(
        ['sitemap.xml', 'theme/sitemap.xml',
         'core/sitemap.xml']).render(post_list=all_posts)
    with gzip.open(sitemap_output_path, mode='wb') as file:
        file.write(sitemap_content)
        logger.debug("Output %s." % relpath(file.name))

    # Copy 'raw' content to output cache - second/final pass
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Compress all files marked for compression
    for file, compression_type in settings.COMPRESS_FILE_LIST:
        if file not in settings.COMPRESSION_CACHE:
            with open(file, mode='rb') as input:
                output = compress(input.read(), compression_type)
                logger.debug("Compressed %s." % relpath(file))
            settings.COMPRESSION_CACHE[file] = output
        else:
            logger.debug("Found pre-compressed file in cache: %s." %
                         relpath(file))
            output = settings.COMPRESSION_CACHE[file]
        with open(file, mode='wb') as f:
            f.write(output)

    # Remove LESS files if LESS preprocessing is being done
    if settings.PREPROCESS_LESS:
        logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
        for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
            logger.debug("Deleting file: %s." % relpath(f))
            f.remove_p()

    logger.debug("Synchronizing output directory with output cache.")
    build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR,
                                         settings.OUTPUT_DIR)
    from pprint import pformat

    logger.debug("Folder mirroring report: %s" % pformat(build_stats['files']))
    logger.console('')
    logger.console("Site: '%s' output to %s." %
                   (settings.SITE_TITLE, settings.OUTPUT_DIR))
    logger.console("Posts: %s (%s new or updated)" %
                   ((build_stats['counts']['new_posts'] +
                     build_stats['counts']['cached_posts']),
                    build_stats['counts']['new_posts']))
    logger.console(
        "Post rollup pages: %s (%s posts per page)" %
        (build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
    logger.console("Template pages: %s" %
                   build_stats['counts']['template_pages'])
    logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
    logger.console("%s new items, %s modified items, and %s deleted items." %
                   (len(build_stats['files']['new']),
                    len(build_stats['files']['overwritten']),
                    len(build_stats['files']['deleted'])))
    logger.console('')
    logger.console("Full build log at %s." % settings.LOG_FILE)

    with open(settings.BUILD_STATS_FILE, mode='wb') as file:
        pickle.dump(build_stats, file)
    settings.CACHE.close()
    return build_stats
Example #4
0
 def tag(name):
     page_path = urljoin('tag', slugify(name))
     page_path = urljoin(self.HOME_URL, page_path)
     return page_path
Example #5
0
def build(args=None):
    """Builds an Engineer site using the settings specified in *args*."""
    from engineer.conf import settings
    from engineer.loaders import LocalLoader
    from engineer.log import get_file_handler
    from engineer.models import PostCollection, TemplatePage
    from engineer.themes import ThemeManager
    from engineer.util import mirror_folder, ensure_exists, slugify

    if args and args.clean:
        clean()

    settings.create_required_directories()

    logger = logging.getLogger('engineer.engine.build')
    logger.parent.addHandler(get_file_handler(settings.LOG_FILE))

    logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE)

    build_stats = {
        'time_run': times.now(),
        'counts': {
            'template_pages': 0,
            'new_posts': 0,
            'cached_posts': 0,
            'rollups': 0,
            'tag_pages': 0,
        },
        'files': {},
    }

    # Remove the output cache (not the post cache or the Jinja cache)
    # since we're rebuilding the site
    settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)

    # Copy static content to output dir
    logger.debug("Copying static files to output cache.")
    s = (settings.ENGINEER.STATIC_DIR / 'engineer').abspath()
    t = settings.OUTPUT_STATIC_DIR / 'engineer'
    mirror_folder(s, t, recurse=False) # Copy only the files in this folder - don't recurse

    theme = ThemeManager.current_theme()
    # Copy Foundation files if used
    if theme.use_foundation:
        s = settings.ENGINEER.LIB_DIR / 'foundation'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/foundation')
        mirror_folder(s, t)
        logger.debug("Copied Foundation library files.")

    # Copy LESS js file if needed
    if theme.use_lesscss and not settings.PREPROCESS_LESS:
        s = settings.ENGINEER.LIB_DIR / 'less-1.3.1.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied LESS CSS files.")

    # Copy jQuery files if needed
    if theme.use_jquery or theme.use_tweet:
        s = settings.ENGINEER.LIB_DIR / 'jquery-1.7.1.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied jQuery files.")

    # Copy Tweet files if needed
    if theme.use_tweet:
        s = settings.ENGINEER.LIB_DIR / 'tweet'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/tweet')
        mirror_folder(s, t)
        logger.debug("Copied Tweet files.")

    # Copy modernizr files if needed
    if theme.use_modernizr:
        s = settings.ENGINEER.LIB_DIR / 'modernizr-2.5.3.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied Modernizr files.")

    logger.debug("Copied static files to %s." % relpath(t))

    # Copy 'raw' content to output cache - first pass
    # This first pass ensures that any static content - JS/LESS/CSS - that
    # is needed by site-specific pages (like template pages) is available
    # during the build
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Copy theme static content to output dir
    logger.debug("Copying theme static files to output cache.")
    try:
        s = theme.static_root.abspath()
    except ThemeNotFoundException as e:
        logger.critical(e.message)
        exit()
    t = (settings.OUTPUT_STATIC_DIR / 'theme').abspath()
    mirror_folder(s, t)
    logger.debug("Copied static files for theme to %s." % relpath(t))

    # Copy any theme additional content to output dir if needed
    if theme.content_mappings:
        logger.debug("Copying additional theme content to output cache.")
        for s, t in theme.content_mappings.iteritems():
            t = settings.OUTPUT_STATIC_DIR / 'theme' / t
            if s.isdir():
                mirror_folder(s, t)
            else:
                s.copy(ensure_exists(t))
            logger.debug("Copied additional files for theme to %s." % relpath(t))

    # Load markdown input posts
    logger.info("Loading posts...")
    new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
    all_posts = PostCollection(new_posts + cached_posts)

    to_publish = PostCollection(all_posts.published)
    if settings.PUBLISH_DRAFTS:
        to_publish.extend(all_posts.drafts)
    if settings.PUBLISH_PENDING:
        to_publish.extend(all_posts.pending)
    if settings.PUBLISH_REVIEW:
        to_publish.extend(all_posts.review)

    if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
        logger.warning("This site contains the following pending posts:")
        for post in all_posts.pending:
            logger.warning("\t'%s' - publish time: %s, %s." % (post.title,
                                                               naturaltime(post.timestamp),
                                                               post.timestamp_local))
        logger.warning("These posts won't be published until you build the site again after their publish time.")

    all_posts = PostCollection(
        sorted(to_publish, reverse=True, key=lambda post: post.timestamp))

    # Generate template pages
    if settings.TEMPLATE_PAGE_DIR.exists():
        logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR)
        template_pages = []
        for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
            # We create all the TemplatePage objects first so we have all of the URLs to them in the template
            # environment. Without this step, template pages might have broken links if they link to a page that is
            # loaded after them, since the URL to the not-yet-loaded page will be missing.
            template_pages.append(TemplatePage(template))
        for page in template_pages:
            rendered_page = page.render_html(all_posts)
            ensure_exists(page.output_path)
            with open(page.output_path / page.output_file_name, mode='wb',
                      encoding='UTF-8') as file:
                file.write(rendered_page)
                logger.debug("Output template page %s." % relpath(file.name))
                build_stats['counts']['template_pages'] += 1
        logger.info("Generated %s template pages." % build_stats['counts']['template_pages'])

    # Generate individual post pages
    for post in all_posts:
        rendered_post = post.render_html(all_posts)
        ensure_exists(post.output_path)
        with open(post.output_path, mode='wb',
                  encoding='UTF-8') as file:
            file.write(rendered_post)
            if post in new_posts:
                logger.console("Output new or modified post '%s'." % post.title)
                build_stats['counts']['new_posts'] += 1
            elif post in cached_posts:
                build_stats['counts']['cached_posts'] += 1

    # Generate rollup pages
    num_posts = len(all_posts)
    num_slices = (
        num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0\
    else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1

    slice_num = 0
    for posts in all_posts.paginate():
        slice_num += 1
        has_next = slice_num < num_slices
        has_previous = 1 < slice_num <= num_slices
        rendered_page = posts.render_listpage_html(slice_num, has_next,
                                                   has_previous)
        ensure_exists(posts.output_path(slice_num))
        with open(posts.output_path(slice_num), mode='wb',
                  encoding='UTF-8') as file:
            file.write(rendered_page)
            logger.debug("Output rollup page %s." % relpath(file.name))
            build_stats['counts']['rollups'] += 1

        # Copy first rollup page to root of site - it's the homepage.
        if slice_num == 1:
            path.copyfile(posts.output_path(slice_num),
                          settings.OUTPUT_CACHE_DIR / 'index.html')
            logger.debug(
                "Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html'))

    # Generate archive page
    if num_posts > 0:
        archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
        ensure_exists(archive_output_path)

        rendered_archive = all_posts.render_archive_html(all_posts)

        with open(archive_output_path, mode='wb', encoding='UTF-8') as file:
            file.write(rendered_archive)
            logger.debug("Output %s." % relpath(file.name))

    # Generate tag pages
    if num_posts > 0:
        tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
        for tag in all_posts.all_tags:
            rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
            tag_path = ensure_exists(
                tags_output_path / slugify(tag) / 'index.html')
            with open(tag_path, mode='wb', encoding='UTF-8') as file:
                file.write(rendered_tag_page)
                build_stats['counts']['tag_pages'] += 1
                logger.debug("Output %s." % relpath(file.name))

    # Generate feeds
    feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml')
    feed_content = settings.JINJA_ENV.get_or_select_template(['rss.xml',
                                                              'theme/rss.xml',
                                                              'core/rss.xml']).render(
        post_list=all_posts[:settings.FEED_ITEM_LIMIT],
        build_date=all_posts[0].timestamp)
    with open(feed_output_path, mode='wb', encoding='UTF-8') as file:
        file.write(feed_content)
        logger.debug("Output %s." % relpath(file.name))

    # Generate sitemap
    sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'sitemap.xml.gz')
    sitemap_content = settings.JINJA_ENV.get_or_select_template(['sitemap.xml',
                                                                 'theme/sitemap.xml',
                                                                 'core/sitemap.xml']).render(post_list=all_posts)
    with gzip.open(sitemap_output_path, mode='wb') as file:
        file.write(sitemap_content)
        logger.debug("Output %s." % relpath(file.name))

    # Copy 'raw' content to output cache - second/final pass
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Compress all files marked for compression
    for file, compression_type in settings.COMPRESS_FILE_LIST:
        if file not in settings.COMPRESSION_CACHE:
            with open(file, mode='rb') as input:
                output = compress(input.read(), compression_type)
                logger.debug("Compressed %s." % relpath(file))
            settings.COMPRESSION_CACHE[file] = output
        else:
            logger.debug("Found pre-compressed file in cache: %s." % relpath(file))
            output = settings.COMPRESSION_CACHE[file]
        with open(file, mode='wb') as f:
            f.write(output)

    # Remove LESS files if LESS preprocessing is being done
    if settings.PREPROCESS_LESS:
        logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
        for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
            logger.debug("Deleting file: %s." % relpath(f))
            f.remove_p()

    logger.debug("Synchronizing output directory with output cache.")
    build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR,
                                         settings.OUTPUT_DIR)
    from pprint import pformat

    logger.debug("Folder mirroring report: %s" % pformat(build_stats['files']))
    logger.console('')
    logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR))
    logger.console("Posts: %s (%s new or updated)" % (
        (build_stats['counts']['new_posts'] + build_stats['counts'][
                                              'cached_posts']),
        build_stats['counts']['new_posts']))
    logger.console("Post rollup pages: %s (%s posts per page)" % (
        build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
    logger.console("Template pages: %s" % build_stats['counts']['template_pages'])
    logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
    logger.console("%s new items, %s modified items, and %s deleted items." % (
        len(build_stats['files']['new']),
        len(build_stats['files']['overwritten']),
        len(build_stats['files']['deleted'])))
    logger.console('')
    logger.console("Full build log at %s." % settings.LOG_FILE)

    with open(settings.BUILD_STATS_FILE, mode='wb') as file:
        pickle.dump(build_stats, file)
    settings.CACHE.close()
    return build_stats
Example #6
0
def build(args=None):
    """Builds an Engineer site using the settings specified in *args*."""
    from engineer.conf import settings
    from engineer.loaders import LocalLoader
    from engineer.log import get_file_handler
    from engineer.models import PostCollection, TemplatePage
    from engineer.themes import ThemeManager
    from engineer.util import mirror_folder, ensure_exists, slugify

    if args and args.clean:
        clean()

    settings.create_required_directories()

    logger = logging.getLogger('engineer.engine.build')
    logger.parent.addHandler(get_file_handler(settings.LOG_FILE))

    logger.debug("Starting build using configuration file %s." %
                 settings.SETTINGS_FILE)

    build_stats = {
        'time_run': times.now(),
        'counts': {
            'template_pages': 0,
            'new_posts': 0,
            'cached_posts': 0,
            'rollups': 0,
            'tag_pages': 0,
        },
        'files': {},
    }

    # Remove the output cache (not the post cache or the Jinja cache)
    # since we're rebuilding the site
    settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)

    theme = ThemeManager.current_theme()
    engineer_lib = (settings.OUTPUT_STATIC_DIR / 'engineer/lib/').abspath()
    ensure_exists(engineer_lib)
    # Copy Foundation files if used
    if theme.use_foundation:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.FOUNDATION_CSS
        t = ensure_exists(engineer_lib / settings.ENGINEER.FOUNDATION_CSS)
        mirror_folder(s, t)
        logger.debug("Copied Foundation library files.")

    # Copy LESS js file if needed
    if theme.use_lesscss and not settings.PREPROCESS_LESS:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.LESS_JS
        s.copy(engineer_lib)
        logger.debug("Copied LESS CSS files.")

    # Copy jQuery files if needed
    if theme.use_jquery:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.JQUERY
        s.copy(engineer_lib)
        logger.debug("Copied jQuery files.")

    # Copy modernizr files if needed
    if theme.use_modernizr:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.MODERNIZR
        s.copy(engineer_lib)
        logger.debug("Copied Modernizr files.")

    # Copy normalize.css if needed
    if theme.use_normalize_css:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.NORMALIZE_CSS
        s.copy(engineer_lib)
        logger.debug("Copied normalize.css.")

    # Copy 'raw' content to output cache - first pass
    # This first pass ensures that any static content - JS/LESS/CSS - that
    # is needed by site-specific pages (like template pages) is available
    # during the build
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Copy theme static content to output dir
    theme_output_dir = settings.OUTPUT_STATIC_DIR / 'theme'
    logger.debug("Copying theme static files to output cache.")
    theme.copy_content(theme_output_dir)
    logger.debug("Copied static files for theme to %s." %
                 relpath(theme_output_dir))

    # Copy any theme additional content to output dir if needed
    if theme.content_mappings:
        logger.debug("Copying additional theme content to output cache.")
        theme.copy_related_content(theme_output_dir)
        logger.debug("Copied additional files for theme to %s." %
                     relpath(theme_output_dir))

    # Load markdown input posts
    logger.info("Loading posts...")
    new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
    all_posts = PostCollection(new_posts + cached_posts)

    to_publish = PostCollection(all_posts.published)
    if settings.PUBLISH_DRAFTS:
        to_publish.extend(all_posts.drafts)
    if settings.PUBLISH_PENDING:
        to_publish.extend(all_posts.pending)
    if settings.PUBLISH_REVIEW:
        to_publish.extend(all_posts.review)

    if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
        logger.warning("This site contains the following pending posts:")
        for post in all_posts.pending:
            logger.warning("\t'%s' - publish time: %s, %s." %
                           (post.title, naturaltime(
                               post.timestamp), post.timestamp_local))
        logger.warning(
            "These posts won't be published until you build the site again after their publish time."
        )

    all_posts = PostCollection(
        sorted(to_publish, reverse=True, key=lambda p: p.timestamp))

    # Generate template pages
    if settings.TEMPLATE_PAGE_DIR.exists():
        logger.info("Generating template pages from %s." %
                    settings.TEMPLATE_PAGE_DIR)
        template_pages = []
        for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
            # We create all the TemplatePage objects first so we have all of the URLs to them in the template
            # environment. Without this step, template pages might have broken links if they link to a page that is
            # loaded after them, since the URL to the not-yet-loaded page will be missing.
            template_pages.append(TemplatePage(template))
        for page in template_pages:
            rendered_page = page.render_html(all_posts)
            ensure_exists(page.output_path)
            with open(page.output_path / page.output_file_name,
                      mode='wb',
                      encoding='UTF-8') as the_file:
                the_file.write(rendered_page)
                logger.info("Output template page %s." %
                            relpath(the_file.name))
                build_stats['counts']['template_pages'] += 1
        logger.info("Generated %s template pages." %
                    build_stats['counts']['template_pages'])

    # Generate individual post pages
    for post in all_posts:
        rendered_post = post.render_html(all_posts)
        ensure_exists(post.output_path)
        with open(post.output_path, mode='wb', encoding='UTF-8') as the_file:
            the_file.write(rendered_post)
            if post in new_posts:
                logger.console("Output new or modified post '%s'." %
                               post.title)
                build_stats['counts']['new_posts'] += 1
            elif post in cached_posts:
                build_stats['counts']['cached_posts'] += 1

    # Generate rollup pages
    num_posts = len(all_posts)
    num_slices = (
        num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0 \
        else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1

    slice_num = 0
    for posts in all_posts.paginate():
        slice_num += 1
        has_next = slice_num < num_slices
        has_previous = 1 < slice_num <= num_slices
        rendered_page = posts.render_listpage_html(slice_num, has_next,
                                                   has_previous)
        ensure_exists(posts.output_path(slice_num))
        with open(posts.output_path(slice_num), mode='wb',
                  encoding='UTF-8') as the_file:
            the_file.write(rendered_page)
            logger.debug("Output rollup page %s." % relpath(the_file.name))
            build_stats['counts']['rollups'] += 1

        # Copy first rollup page to root of site - it's the homepage.
        if slice_num == 1:
            path.copyfile(posts.output_path(slice_num),
                          settings.OUTPUT_CACHE_DIR / 'index.html')
            logger.debug("Output '%s'." %
                         (settings.OUTPUT_CACHE_DIR / 'index.html'))

    # Generate archive page
    if num_posts > 0:
        archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
        ensure_exists(archive_output_path)

        rendered_archive = all_posts.render_archive_html(all_posts)

        with open(archive_output_path, mode='wb',
                  encoding='UTF-8') as the_file:
            the_file.write(rendered_archive)
            logger.debug("Output %s." % relpath(the_file.name))

    # Generate tag pages
    if num_posts > 0:
        tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
        for tag in all_posts.all_tags:
            rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
            tag_path = ensure_exists(tags_output_path / slugify(tag) /
                                     'index.html')
            with open(tag_path, mode='wb', encoding='UTF-8') as the_file:
                the_file.write(rendered_tag_page)
                build_stats['counts']['tag_pages'] += 1
                logger.debug("Output %s." % relpath(the_file.name))

    # Generate feeds
    rss_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                         'feeds/rss.xml')
    atom_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                          'feeds/atom.xml')
    rss_feed = Rss201rev2Feed(title=settings.FEED_TITLE,
                              link=settings.SITE_URL,
                              description=settings.FEED_DESCRIPTION,
                              feed_url=settings.FEED_URL)

    atom_feed = Atom1Feed(title=settings.FEED_TITLE,
                          link=settings.SITE_URL,
                          description=settings.FEED_DESCRIPTION,
                          feed_url=settings.FEED_URL)

    for feed in (rss_feed, atom_feed):
        for post in all_posts[:settings.FEED_ITEM_LIMIT]:
            title = settings.JINJA_ENV.get_template(
                'core/feeds/title.jinja2').render(post=post)
            link = settings.JINJA_ENV.get_template(
                'core/feeds/link.jinja2').render(post=post)
            content = settings.JINJA_ENV.get_template(
                'core/feeds/content.jinja2').render(post=post)
            feed.add_item(title=title,
                          link=link,
                          description=content,
                          pubdate=post.timestamp,
                          unique_id=post.absolute_url)

    with open(rss_feed_output_path, mode='wb') as the_file:
        rss_feed.write(the_file, 'UTF-8')
        logger.debug("Output %s." % relpath(the_file.name))

    with open(atom_feed_output_path, mode='wb') as the_file:
        atom_feed.write(the_file, 'UTF-8')
        logger.debug("Output %s." % relpath(the_file.name))

    # Generate sitemap
    sitemap_file_name = 'sitemap.xml.gz'
    sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                        sitemap_file_name)
    sitemap_content = settings.JINJA_ENV.get_or_select_template(
        ['sitemap.xml', 'theme/sitemap.xml',
         'core/sitemap.xml']).render(post_list=all_posts)
    with gzip.open(sitemap_output_path, mode='wb') as the_file:
        the_file.write(sitemap_content)
        logger.debug("Output %s." % relpath(the_file.name))

    # Copy 'raw' content to output cache - second/final pass
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Compress all files marked for compression
    for the_file, compression_type in settings.COMPRESS_FILE_LIST:
        if the_file not in settings.COMPRESSION_CACHE:
            with open(the_file, mode='rb') as input:
                output = compress(input.read(), compression_type)
                logger.debug("Compressed %s." % relpath(the_file))
            settings.COMPRESSION_CACHE[the_file] = output
        else:
            logger.debug("Found pre-compressed file in cache: %s." %
                         relpath(the_file))
            output = settings.COMPRESSION_CACHE[the_file]
        with open(the_file, mode='wb') as f:
            f.write(output)

    # Remove LESS files if LESS preprocessing is being done
    if settings.PREPROCESS_LESS:
        logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
        for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
            logger.debug("Deleting file: %s." % relpath(f))
            f.remove_p()

    # Check if anything has changed other than the sitemap
    have_changes = False
    compare = filecmp.dircmp(settings.OUTPUT_CACHE_DIR,
                             settings.OUTPUT_DIR,
                             ignore=settings.OUTPUT_DIR_IGNORE)

    # The algorithm below takes advantage of the fact that once we've determined that there is more than one file
    # that's different, or if the first item returned by the generator is not the sitemap, then we can break out of
    # the generator loop early. This is also advantageous because it doesn't require us to completely exhaust the
    # generator. In the case of a fresh site build, for example, the generator will return a lot more data. So the
    # other approach here of expanding the generator into a list with a list comprehension would be inefficient
    # in many cases. This approach performs equally well in all cases at the cost of some unusual-looking code.
    diff_file_count = 0
    if not has_files(settings.OUTPUT_DIR):
        have_changes = True
    else:
        for file_path in diff_dir(compare):
            diff_file_count += 1
            if file_path != sitemap_output_path:
                have_changes = True
                break
            if diff_file_count > 1:
                have_changes = True
                break

    if not have_changes:
        logger.console('')
        logger.console("No site changes to publish.")
    else:
        logger.debug("Synchronizing output directory with output cache.")
        build_stats['files'] = mirror_folder(
            settings.OUTPUT_CACHE_DIR,
            settings.OUTPUT_DIR,
            ignore_list=settings.OUTPUT_DIR_IGNORE)
        from pprint import pformat

        logger.debug("Folder mirroring report: %s" %
                     pformat(build_stats['files']))
        logger.console('')
        logger.console("Site: '%s' output to %s." %
                       (settings.SITE_TITLE, settings.OUTPUT_DIR))
        logger.console("Posts: %s (%s new or updated)" %
                       ((build_stats['counts']['new_posts'] +
                         build_stats['counts']['cached_posts']),
                        build_stats['counts']['new_posts']))
        logger.console(
            "Post rollup pages: %s (%s posts per page)" %
            (build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
        logger.console("Template pages: %s" %
                       build_stats['counts']['template_pages'])
        logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
        logger.console(
            "%s new items, %s modified items, and %s deleted items." %
            (len(build_stats['files']['new']),
             len(build_stats['files']['overwritten']),
             len(build_stats['files']['deleted'])))

    logger.console('')
    logger.console("Full build log at %s." % settings.LOG_FILE)
    logger.console('')

    with open(settings.BUILD_STATS_FILE, mode='wb') as the_file:
        pickle.dump(build_stats, the_file)
    settings.CACHE.close()
    return build_stats
Example #7
0
 def tag(name):
     page_path = urljoin('tag', slugify(name))
     page_path = urljoin(self.HOME_URL, page_path)
     return page_path
Example #8
0
def build(args=None):
    """Builds an Engineer site using the settings specified in *args*."""
    from engineer.conf import settings
    from engineer.loaders import LocalLoader
    from engineer.log import get_file_handler
    from engineer.models import PostCollection, TemplatePage
    from engineer.themes import ThemeManager
    from engineer.util import mirror_folder, ensure_exists, slugify

    if args and args.clean:
        clean()

    settings.create_required_directories()

    logger = logging.getLogger('engineer.engine.build')
    logger.parent.addHandler(get_file_handler(settings.LOG_FILE))

    logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE)

    build_stats = {
        'time_run': times.now(),
        'counts': {
            'template_pages': 0,
            'new_posts': 0,
            'cached_posts': 0,
            'rollups': 0,
            'tag_pages': 0,
        },
        'files': {},
    }

    # Remove the output cache (not the post cache or the Jinja cache)
    # since we're rebuilding the site
    settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)

    theme = ThemeManager.current_theme()
    engineer_lib = (settings.OUTPUT_STATIC_DIR / 'engineer/lib/').abspath()
    ensure_exists(engineer_lib)
    # Copy Foundation files if used
    if theme.use_foundation:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.FOUNDATION_CSS
        t = ensure_exists(engineer_lib / settings.ENGINEER.FOUNDATION_CSS)
        mirror_folder(s, t)
        logger.debug("Copied Foundation library files.")

    # Copy LESS js file if needed
    if theme.use_lesscss and not settings.PREPROCESS_LESS:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.LESS_JS
        s.copy(engineer_lib)
        logger.debug("Copied LESS CSS files.")

    # Copy jQuery files if needed
    if theme.use_jquery:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.JQUERY
        s.copy(engineer_lib)
        logger.debug("Copied jQuery files.")

    # Copy modernizr files if needed
    if theme.use_modernizr:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.MODERNIZR
        s.copy(engineer_lib)
        logger.debug("Copied Modernizr files.")

    # Copy normalize.css if needed
    if theme.use_normalize_css:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.NORMALIZE_CSS
        s.copy(engineer_lib)
        logger.debug("Copied normalize.css.")

    # Copy 'raw' content to output cache - first pass
    # This first pass ensures that any static content - JS/LESS/CSS - that
    # is needed by site-specific pages (like template pages) is available
    # during the build
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Copy theme static content to output dir
    theme_output_dir = settings.OUTPUT_STATIC_DIR / 'theme'
    logger.debug("Copying theme static files to output cache.")
    theme.copy_content(theme_output_dir)
    logger.debug("Copied static files for theme to %s." % relpath(theme_output_dir))

    # Copy any theme additional content to output dir if needed
    if theme.content_mappings:
        logger.debug("Copying additional theme content to output cache.")
        theme.copy_related_content(theme_output_dir)
        logger.debug("Copied additional files for theme to %s." % relpath(theme_output_dir))

    # Load markdown input posts
    logger.info("Loading posts...")
    new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
    all_posts = PostCollection(new_posts + cached_posts)

    to_publish = PostCollection(all_posts.published)
    if settings.PUBLISH_DRAFTS:
        to_publish.extend(all_posts.drafts)
    if settings.PUBLISH_PENDING:
        to_publish.extend(all_posts.pending)
    if settings.PUBLISH_REVIEW:
        to_publish.extend(all_posts.review)

    if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
        logger.warning("This site contains the following pending posts:")
        for post in all_posts.pending:
            logger.warning("\t'%s' - publish time: %s, %s." % (post.title,
                                                               naturaltime(post.timestamp),
                                                               post.timestamp_local))
        logger.warning("These posts won't be published until you build the site again after their publish time.")

    all_posts = PostCollection(
        sorted(to_publish, reverse=True, key=lambda p: p.timestamp))

    # Generate template pages
    if settings.TEMPLATE_PAGE_DIR.exists():
        logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR)
        template_pages = []
        for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
            # We create all the TemplatePage objects first so we have all of the URLs to them in the template
            # environment. Without this step, template pages might have broken links if they link to a page that is
            # loaded after them, since the URL to the not-yet-loaded page will be missing.
            template_pages.append(TemplatePage(template))
        for page in template_pages:
            rendered_page = page.render_html(all_posts)
            ensure_exists(page.output_path)
            with open(page.output_path / page.output_file_name, mode='wb',
                      encoding='UTF-8') as the_file:
                the_file.write(rendered_page)
                logger.info("Output template page %s." % relpath(the_file.name))
                build_stats['counts']['template_pages'] += 1
        logger.info("Generated %s template pages." % build_stats['counts']['template_pages'])

    # Generate individual post pages
    for post in all_posts:
        rendered_post = post.render_html(all_posts)
        ensure_exists(post.output_path)
        with open(post.output_path, mode='wb',
                  encoding='UTF-8') as the_file:
            the_file.write(rendered_post)
            if post in new_posts:
                logger.console("Output new or modified post '%s'." % post.title)
                build_stats['counts']['new_posts'] += 1
            elif post in cached_posts:
                build_stats['counts']['cached_posts'] += 1

    # Generate rollup pages
    num_posts = len(all_posts)
    num_slices = (
        num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0 \
        else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1

    slice_num = 0
    for posts in all_posts.paginate():
        slice_num += 1
        has_next = slice_num < num_slices
        has_previous = 1 < slice_num <= num_slices
        rendered_page = posts.render_listpage_html(slice_num, has_next,
                                                   has_previous)
        ensure_exists(posts.output_path(slice_num))
        with open(posts.output_path(slice_num), mode='wb',
                  encoding='UTF-8') as the_file:
            the_file.write(rendered_page)
            logger.debug("Output rollup page %s." % relpath(the_file.name))
            build_stats['counts']['rollups'] += 1

        # Copy first rollup page to root of site - it's the homepage.
        if slice_num == 1:
            path.copyfile(posts.output_path(slice_num),
                          settings.OUTPUT_CACHE_DIR / 'index.html')
            logger.debug(
                "Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html'))

    # Generate archive page
    if num_posts > 0:
        archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
        ensure_exists(archive_output_path)

        rendered_archive = all_posts.render_archive_html(all_posts)

        with open(archive_output_path, mode='wb', encoding='UTF-8') as the_file:
            the_file.write(rendered_archive)
            logger.debug("Output %s." % relpath(the_file.name))

    # Generate tag pages
    if num_posts > 0:
        tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
        for tag in all_posts.all_tags:
            rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
            tag_path = ensure_exists(
                tags_output_path / slugify(tag) / 'index.html')
            with open(tag_path, mode='wb', encoding='UTF-8') as the_file:
                the_file.write(rendered_tag_page)
                build_stats['counts']['tag_pages'] += 1
                logger.debug("Output %s." % relpath(the_file.name))

    # Generate feeds
    rss_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml')
    atom_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/atom.xml')
    rss_feed = Rss201rev2Feed(
        title=settings.FEED_TITLE,
        link=settings.SITE_URL,
        description=settings.FEED_DESCRIPTION,
        feed_url=settings.FEED_URL
    )

    atom_feed = Atom1Feed(
        title=settings.FEED_TITLE,
        link=settings.SITE_URL,
        description=settings.FEED_DESCRIPTION,
        feed_url=settings.FEED_URL
    )

    for feed in (rss_feed, atom_feed):
        for post in all_posts[:settings.FEED_ITEM_LIMIT]:
            title = settings.JINJA_ENV.get_template('core/feeds/title.jinja2').render(post=post)
            link = settings.JINJA_ENV.get_template('core/feeds/link.jinja2').render(post=post)
            content = settings.JINJA_ENV.get_template('core/feeds/content.jinja2').render(post=post)
            feed.add_item(
                title=title,
                link=link,
                description=content,
                pubdate=post.timestamp,
                unique_id=post.absolute_url)

    with open(rss_feed_output_path, mode='wb') as the_file:
        rss_feed.write(the_file, 'UTF-8')
        logger.debug("Output %s." % relpath(the_file.name))

    with open(atom_feed_output_path, mode='wb') as the_file:
        atom_feed.write(the_file, 'UTF-8')
        logger.debug("Output %s." % relpath(the_file.name))

    # Generate sitemap
    sitemap_file_name = 'sitemap.xml.gz'
    sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / sitemap_file_name)
    sitemap_content = settings.JINJA_ENV.get_or_select_template(['sitemap.xml',
                                                                 'theme/sitemap.xml',
                                                                 'core/sitemap.xml']).render(post_list=all_posts)
    with gzip.open(sitemap_output_path, mode='wb') as the_file:
        the_file.write(sitemap_content)
        logger.debug("Output %s." % relpath(the_file.name))

    # Copy 'raw' content to output cache - second/final pass
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Compress all files marked for compression
    for the_file, compression_type in settings.COMPRESS_FILE_LIST:
        if the_file not in settings.COMPRESSION_CACHE:
            with open(the_file, mode='rb') as input:
                output = compress(input.read(), compression_type)
                logger.debug("Compressed %s." % relpath(the_file))
            settings.COMPRESSION_CACHE[the_file] = output
        else:
            logger.debug("Found pre-compressed file in cache: %s." % relpath(the_file))
            output = settings.COMPRESSION_CACHE[the_file]
        with open(the_file, mode='wb') as f:
            f.write(output)

    # Remove LESS files if LESS preprocessing is being done
    if settings.PREPROCESS_LESS:
        logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
        for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
            logger.debug("Deleting file: %s." % relpath(f))
            f.remove_p()

    # Check if anything has changed other than the sitemap
    have_changes = False
    compare = filecmp.dircmp(settings.OUTPUT_CACHE_DIR,
                             settings.OUTPUT_DIR,
                             ignore=settings.OUTPUT_DIR_IGNORE)

    # The algorithm below takes advantage of the fact that once we've determined that there is more than one file
    # that's different, or if the first item returned by the generator is not the sitemap, then we can break out of
    # the generator loop early. This is also advantageous because it doesn't require us to completely exhaust the
    # generator. In the case of a fresh site build, for example, the generator will return a lot more data. So the
    # other approach here of expanding the generator into a list with a list comprehension would be inefficient
    # in many cases. This approach performs equally well in all cases at the cost of some unusual-looking code.
    diff_file_count = 0
    if not has_files(settings.OUTPUT_DIR):
        have_changes = True
    else:
        for file_path in diff_dir(compare):
            diff_file_count += 1
            if file_path != sitemap_output_path:
                have_changes = True
                break
            if diff_file_count > 1:
                have_changes = True
                break

    if not have_changes:
        logger.console('')
        logger.console("No site changes to publish.")
    else:
        logger.debug("Synchronizing output directory with output cache.")
        build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR,
                                             settings.OUTPUT_DIR,
                                             ignore_list=settings.OUTPUT_DIR_IGNORE)
        from pprint import pformat

        logger.debug("Folder mirroring report: %s" % pformat(build_stats['files']))
        logger.console('')
        logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR))
        logger.console("Posts: %s (%s new or updated)" % (
            (build_stats['counts']['new_posts'] + build_stats['counts']['cached_posts']),
            build_stats['counts']['new_posts']))
        logger.console("Post rollup pages: %s (%s posts per page)" % (
            build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
        logger.console("Template pages: %s" % build_stats['counts']['template_pages'])
        logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
        logger.console("%s new items, %s modified items, and %s deleted items." % (
            len(build_stats['files']['new']),
            len(build_stats['files']['overwritten']),
            len(build_stats['files']['deleted'])))

    logger.console('')
    logger.console("Full build log at %s." % settings.LOG_FILE)
    logger.console('')

    with open(settings.BUILD_STATS_FILE, mode='wb') as the_file:
        pickle.dump(build_stats, the_file)
    settings.CACHE.close()
    return build_stats