Exemple #1
0
    def setUp(self):
        from engineer.conf import settings

        bootstrap()  #bootstrap logging infrastructure
        load_plugins()  #load plugins
        self.source_path = test_data_root
        os.chdir(self.copied_data_path)
        self.post_tests_dir = self.copied_data_path / 'post_tests'

        settings.reload(self.post_tests_dir / 'settings.yaml')
        settings.create_required_directories()
Exemple #2
0
    def setUp(self):
        from engineer.conf import settings

        bootstrap()  #bootstrap logging infrastructure
        load_plugins()  #load plugins
        self.source_path = test_data_root
        os.chdir(self.copied_data_path)
        self.post_tests_dir = self.copied_data_path / 'post_tests'

        settings.reload(self.post_tests_dir / 'settings.yaml')
        settings.create_required_directories()
Exemple #3
0
    def test_force_fenced_metadata(self):
        from engineer.conf import settings
        from engineer.models import Post

        settings.reload('finalization_fenced.yaml')
        settings.create_required_directories()
        post = Post('posts/finalization_draft.md')
        self.assertEqual(post.title, "Finalization Draft")

        expected_output = '---\n\n' + finalization_draft_output
        with open(post.source, mode='rb') as post_file:
            content = post_file.read()
        self.assertEqual(content, expected_output)
    def test_finalization_unfenced_post(self):
        from engineer.conf import settings
        from engineer.models import Post

        settings.reload('finalization_unfenced.yaml')
        settings.create_required_directories()
        post = Post('posts/finalization_fenced.md')
        self.assertEqual(post.title, "Finalization Fenced")

        expected_output = ''.join(finalization_fenced_output.splitlines(True)[2:])
        with open(post.source, mode='rb') as post_file:
            content = post_file.read()
        self.assertEqual(content, expected_output)
    def test_force_fenced_metadata(self):
        from engineer.conf import settings
        from engineer.models import Post

        settings.reload('finalization_fenced.yaml')
        settings.create_required_directories()
        post = Post('posts/finalization_draft.md')
        self.assertEqual(post.title, "Finalization Draft")

        expected_output = '---\n\n' + finalization_draft_output
        with open(post.source, mode='rb') as post_file:
            content = post_file.read()
        self.assertEqual(content, expected_output)
Exemple #6
0
    def test_finalization_unfenced_post(self):
        from engineer.conf import settings
        from engineer.models import Post

        settings.reload('finalization_unfenced.yaml')
        settings.create_required_directories()
        post = Post('posts/finalization_fenced.md')
        self.assertEqual(post.title, "Finalization Fenced")

        expected_output = ''.join(
            finalization_fenced_output.splitlines(True)[2:])
        with open(post.source, mode='rb') as post_file:
            content = post_file.read()
        self.assertEqual(content, expected_output)
    def test_finalization_draft(self):
        from engineer.conf import settings
        from engineer.models import Post

        settings.reload('finalization.yaml')
        settings.create_required_directories()
        post = Post('posts/finalization_draft.md')
        self.assertEqual(post.title, "Finalization Draft")

        expected_output = """title: Finalization Draft
status: draft
slug: finalization-draft
tags:
- tag

---

This is a finalization test post.
""".format(year=post.timestamp_local.year, month=post.timestamp_local.month, day=post.timestamp_local.day)

        with open(post.source, mode='rb') as post_file:
            content = post_file.read()
        self.assertEqual(content, expected_output)
Exemple #8
0
    def test_finalization_draft(self):
        from engineer.conf import settings
        from engineer.models import Post

        settings.reload('finalization.yaml')
        settings.create_required_directories()
        post = Post('posts/finalization_draft.md')
        self.assertEqual(post.title, "Finalization Draft")

        expected_output = """title: Finalization Draft
status: draft
url: /{year:02d}/{month:02d}/{day:02d}/finalization-draft/
slug: finalization-draft
tags:
- tag

---

This is a finalization test post.
""".format(year=post.timestamp_local.year, month=post.timestamp_local.month, day=post.timestamp_local.day)

        with open(post.source, mode='rb') as post_file:
            content = post_file.read()
        self.assertEqual(content, expected_output)
Exemple #9
0
def build(args=None):
    """Builds an Engineer site using the settings specified in *args*."""
    from engineer.conf import settings
    from engineer.loaders import LocalLoader
    from engineer.log import get_file_handler
    from engineer.models import PostCollection, TemplatePage
    from engineer.themes import ThemeManager
    from engineer.util import mirror_folder, ensure_exists, slugify

    if args and args.clean:
        clean()

    settings.create_required_directories()

    logger = logging.getLogger('engineer.engine.build')
    logger.parent.addHandler(get_file_handler(settings.LOG_FILE))

    logger.debug("Starting build using configuration file %s." %
                 settings.SETTINGS_FILE)

    build_stats = {
        'time_run': times.now(),
        'counts': {
            'template_pages': 0,
            'new_posts': 0,
            'cached_posts': 0,
            'rollups': 0,
            'tag_pages': 0,
        },
        'files': {},
    }

    # Remove the output cache (not the post cache or the Jinja cache)
    # since we're rebuilding the site
    settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)

    # Copy static content to output dir
    logger.debug("Copying static files to output cache.")
    s = (settings.ENGINEER.STATIC_DIR / 'engineer').abspath()
    t = settings.OUTPUT_STATIC_DIR / 'engineer'
    mirror_folder(
        s, t,
        recurse=False)  # Copy only the files in this folder - don't recurse

    theme = ThemeManager.current_theme()
    # Copy Foundation files if used
    if theme.use_foundation:
        s = settings.ENGINEER.LIB_DIR / 'foundation'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR /
                          'engineer/lib/foundation')
        mirror_folder(s, t)
        logger.debug("Copied Foundation library files.")

    # Copy LESS js file if needed
    if theme.use_lesscss and not settings.PREPROCESS_LESS:
        s = settings.ENGINEER.LIB_DIR / 'less-1.3.1.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied LESS CSS files.")

    # Copy jQuery files if needed
    if theme.use_jquery or theme.use_tweet:
        s = settings.ENGINEER.LIB_DIR / 'jquery-1.7.1.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied jQuery files.")

    # Copy Tweet files if needed
    if theme.use_tweet:
        s = settings.ENGINEER.LIB_DIR / 'tweet'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/tweet')
        mirror_folder(s, t)
        logger.debug("Copied Tweet files.")

    # Copy modernizr files if needed
    if theme.use_modernizr:
        s = settings.ENGINEER.LIB_DIR / 'modernizr-2.5.3.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied Modernizr files.")

    logger.debug("Copied static files to %s." % relpath(t))

    # Copy 'raw' content to output cache - first pass
    # This first pass ensures that any static content - JS/LESS/CSS - that
    # is needed by site-specific pages (like template pages) is available
    # during the build
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Copy theme static content to output dir
    logger.debug("Copying theme static files to output cache.")
    try:
        s = theme.static_root.abspath()
    except ThemeNotFoundException as e:
        logger.critical(e.message)
        exit()
    t = (settings.OUTPUT_STATIC_DIR / 'theme').abspath()
    mirror_folder(s, t)
    logger.debug("Copied static files for theme to %s." % relpath(t))

    # Copy any theme additional content to output dir if needed
    if theme.content_mappings:
        logger.debug("Copying additional theme content to output cache.")
        for s, t in theme.content_mappings.iteritems():
            t = settings.OUTPUT_STATIC_DIR / 'theme' / t
            if s.isdir():
                mirror_folder(s, t)
            else:
                s.copy(ensure_exists(t))
            logger.debug("Copied additional files for theme to %s." %
                         relpath(t))

    # Load markdown input posts
    logger.info("Loading posts...")
    new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
    all_posts = PostCollection(new_posts + cached_posts)

    to_publish = PostCollection(all_posts.published)
    if settings.PUBLISH_DRAFTS:
        to_publish.extend(all_posts.drafts)
    if settings.PUBLISH_PENDING:
        to_publish.extend(all_posts.pending)
    if settings.PUBLISH_REVIEW:
        to_publish.extend(all_posts.review)

    if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
        logger.warning("This site contains the following pending posts:")
        for post in all_posts.pending:
            logger.warning("\t'%s' - publish time: %s, %s." %
                           (post.title, humanize.naturaltime(
                               post.timestamp), post.timestamp_local))
        logger.warning(
            "These posts won't be published until you build the site again after their publish time."
        )

    all_posts = PostCollection(
        sorted(to_publish, reverse=True, key=lambda post: post.timestamp))

    # Generate template pages
    if settings.TEMPLATE_PAGE_DIR.exists():
        logger.info("Generating template pages from %s." %
                    settings.TEMPLATE_PAGE_DIR)
        template_pages = []
        for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
            # We create all the TemplatePage objects first so we have all of the URLs to them in the template
            # environment. Without this step, template pages might have broken links if they link to a page that is
            # loaded after them, since the URL to the not-yet-loaded page will be missing.
            template_pages.append(TemplatePage(template))
        for page in template_pages:
            rendered_page = page.render_html(all_posts)
            ensure_exists(page.output_path)
            with open(page.output_path / page.output_file_name,
                      mode='wb',
                      encoding='UTF-8') as file:
                file.write(rendered_page)
                logger.debug("Output template page %s." % relpath(file.name))
                build_stats['counts']['template_pages'] += 1
        logger.info("Generated %s template pages." %
                    build_stats['counts']['template_pages'])

    # Generate individual post pages
    for post in all_posts:
        rendered_post = post.render_html(all_posts)
        ensure_exists(post.output_path)
        with open(post.output_path, mode='wb', encoding='UTF-8') as file:
            file.write(rendered_post)
            if post in new_posts:
                logger.console("Output new or modified post '%s'." %
                               post.title)
                build_stats['counts']['new_posts'] += 1
            elif post in cached_posts:
                build_stats['counts']['cached_posts'] += 1

    # Generate rollup pages
    num_posts = len(all_posts)
    num_slices = (
        num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0\
    else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1

    slice_num = 0
    for posts in all_posts.paginate():
        slice_num += 1
        has_next = slice_num < num_slices
        has_previous = 1 < slice_num <= num_slices
        rendered_page = posts.render_listpage_html(slice_num, has_next,
                                                   has_previous)
        ensure_exists(posts.output_path(slice_num))
        with open(posts.output_path(slice_num), mode='wb',
                  encoding='UTF-8') as file:
            file.write(rendered_page)
            logger.debug("Output rollup page %s." % relpath(file.name))
            build_stats['counts']['rollups'] += 1

        # Copy first rollup page to root of site - it's the homepage.
        if slice_num == 1:
            path.copyfile(posts.output_path(slice_num),
                          settings.OUTPUT_CACHE_DIR / 'index.html')
            logger.debug("Output '%s'." %
                         (settings.OUTPUT_CACHE_DIR / 'index.html'))

    # Generate archive page
    if num_posts > 0:
        archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
        ensure_exists(archive_output_path)

        rendered_archive = all_posts.render_archive_html(all_posts)

        with open(archive_output_path, mode='wb', encoding='UTF-8') as file:
            file.write(rendered_archive)
            logger.debug("Output %s." % relpath(file.name))

    # Generate tag pages
    if num_posts > 0:
        tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
        for tag in all_posts.all_tags:
            rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
            tag_path = ensure_exists(tags_output_path / slugify(tag) /
                                     'index.html')
            with open(tag_path, mode='wb', encoding='UTF-8') as file:
                file.write(rendered_tag_page)
                build_stats['counts']['tag_pages'] += 1
                logger.debug("Output %s." % relpath(file.name))

    # Generate feeds
    feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                     'feeds/rss.xml')
    feed_content = settings.JINJA_ENV.get_or_select_template([
        'rss.xml', 'theme/rss.xml', 'core/rss.xml'
    ]).render(post_list=all_posts[:settings.FEED_ITEM_LIMIT],
              build_date=all_posts[0].timestamp)
    with open(feed_output_path, mode='wb', encoding='UTF-8') as file:
        file.write(feed_content)
        logger.debug("Output %s." % relpath(file.name))

    # Generate sitemap
    sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                        'sitemap.xml.gz')
    sitemap_content = settings.JINJA_ENV.get_or_select_template(
        ['sitemap.xml', 'theme/sitemap.xml',
         'core/sitemap.xml']).render(post_list=all_posts)
    with gzip.open(sitemap_output_path, mode='wb') as file:
        file.write(sitemap_content)
        logger.debug("Output %s." % relpath(file.name))

    # Copy 'raw' content to output cache - second/final pass
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Compress all files marked for compression
    for file, compression_type in settings.COMPRESS_FILE_LIST:
        if file not in settings.COMPRESSION_CACHE:
            with open(file, mode='rb') as input:
                output = compress(input.read(), compression_type)
                logger.debug("Compressed %s." % relpath(file))
            settings.COMPRESSION_CACHE[file] = output
        else:
            logger.debug("Found pre-compressed file in cache: %s." %
                         relpath(file))
            output = settings.COMPRESSION_CACHE[file]
        with open(file, mode='wb') as f:
            f.write(output)

    # Remove LESS files if LESS preprocessing is being done
    if settings.PREPROCESS_LESS:
        logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
        for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
            logger.debug("Deleting file: %s." % relpath(f))
            f.remove_p()

    logger.debug("Synchronizing output directory with output cache.")
    build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR,
                                         settings.OUTPUT_DIR)
    from pprint import pformat

    logger.debug("Folder mirroring report: %s" % pformat(build_stats['files']))
    logger.console('')
    logger.console("Site: '%s' output to %s." %
                   (settings.SITE_TITLE, settings.OUTPUT_DIR))
    logger.console("Posts: %s (%s new or updated)" %
                   ((build_stats['counts']['new_posts'] +
                     build_stats['counts']['cached_posts']),
                    build_stats['counts']['new_posts']))
    logger.console(
        "Post rollup pages: %s (%s posts per page)" %
        (build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
    logger.console("Template pages: %s" %
                   build_stats['counts']['template_pages'])
    logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
    logger.console("%s new items, %s modified items, and %s deleted items." %
                   (len(build_stats['files']['new']),
                    len(build_stats['files']['overwritten']),
                    len(build_stats['files']['deleted'])))
    logger.console('')
    logger.console("Full build log at %s." % settings.LOG_FILE)

    with open(settings.BUILD_STATS_FILE, mode='wb') as file:
        pickle.dump(build_stats, file)
    settings.CACHE.close()
    return build_stats
Exemple #10
0
def build(args=None):
    """Builds an Engineer site using the settings specified in *args*."""
    from engineer.conf import settings
    from engineer.loaders import LocalLoader
    from engineer.log import get_file_handler
    from engineer.models import PostCollection, TemplatePage
    from engineer.themes import ThemeManager
    from engineer.util import mirror_folder, ensure_exists, slugify

    if args and args.clean:
        clean()

    settings.create_required_directories()

    logger = logging.getLogger('engineer.engine.build')
    logger.parent.addHandler(get_file_handler(settings.LOG_FILE))

    logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE)

    build_stats = {
        'time_run': times.now(),
        'counts': {
            'template_pages': 0,
            'new_posts': 0,
            'cached_posts': 0,
            'rollups': 0,
            'tag_pages': 0,
        },
        'files': {},
    }

    # Remove the output cache (not the post cache or the Jinja cache)
    # since we're rebuilding the site
    settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)

    # Copy static content to output dir
    logger.debug("Copying static files to output cache.")
    s = (settings.ENGINEER.STATIC_DIR / 'engineer').abspath()
    t = settings.OUTPUT_STATIC_DIR / 'engineer'
    mirror_folder(s, t, recurse=False) # Copy only the files in this folder - don't recurse

    theme = ThemeManager.current_theme()
    # Copy Foundation files if used
    if theme.use_foundation:
        s = settings.ENGINEER.LIB_DIR / 'foundation'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/foundation')
        mirror_folder(s, t)
        logger.debug("Copied Foundation library files.")

    # Copy LESS js file if needed
    if theme.use_lesscss and not settings.PREPROCESS_LESS:
        s = settings.ENGINEER.LIB_DIR / 'less-1.3.1.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied LESS CSS files.")

    # Copy jQuery files if needed
    if theme.use_jquery or theme.use_tweet:
        s = settings.ENGINEER.LIB_DIR / 'jquery-1.7.1.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied jQuery files.")

    # Copy Tweet files if needed
    if theme.use_tweet:
        s = settings.ENGINEER.LIB_DIR / 'tweet'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/tweet')
        mirror_folder(s, t)
        logger.debug("Copied Tweet files.")

    # Copy modernizr files if needed
    if theme.use_modernizr:
        s = settings.ENGINEER.LIB_DIR / 'modernizr-2.5.3.min.js'
        t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
        s.copy(t)
        logger.debug("Copied Modernizr files.")

    logger.debug("Copied static files to %s." % relpath(t))

    # Copy 'raw' content to output cache - first pass
    # This first pass ensures that any static content - JS/LESS/CSS - that
    # is needed by site-specific pages (like template pages) is available
    # during the build
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Copy theme static content to output dir
    logger.debug("Copying theme static files to output cache.")
    try:
        s = theme.static_root.abspath()
    except ThemeNotFoundException as e:
        logger.critical(e.message)
        exit()
    t = (settings.OUTPUT_STATIC_DIR / 'theme').abspath()
    mirror_folder(s, t)
    logger.debug("Copied static files for theme to %s." % relpath(t))

    # Copy any theme additional content to output dir if needed
    if theme.content_mappings:
        logger.debug("Copying additional theme content to output cache.")
        for s, t in theme.content_mappings.iteritems():
            t = settings.OUTPUT_STATIC_DIR / 'theme' / t
            if s.isdir():
                mirror_folder(s, t)
            else:
                s.copy(ensure_exists(t))
            logger.debug("Copied additional files for theme to %s." % relpath(t))

    # Load markdown input posts
    logger.info("Loading posts...")
    new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
    all_posts = PostCollection(new_posts + cached_posts)

    to_publish = PostCollection(all_posts.published)
    if settings.PUBLISH_DRAFTS:
        to_publish.extend(all_posts.drafts)
    if settings.PUBLISH_PENDING:
        to_publish.extend(all_posts.pending)
    if settings.PUBLISH_REVIEW:
        to_publish.extend(all_posts.review)

    if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
        logger.warning("This site contains the following pending posts:")
        for post in all_posts.pending:
            logger.warning("\t'%s' - publish time: %s, %s." % (post.title,
                                                               naturaltime(post.timestamp),
                                                               post.timestamp_local))
        logger.warning("These posts won't be published until you build the site again after their publish time.")

    all_posts = PostCollection(
        sorted(to_publish, reverse=True, key=lambda post: post.timestamp))

    # Generate template pages
    if settings.TEMPLATE_PAGE_DIR.exists():
        logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR)
        template_pages = []
        for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
            # We create all the TemplatePage objects first so we have all of the URLs to them in the template
            # environment. Without this step, template pages might have broken links if they link to a page that is
            # loaded after them, since the URL to the not-yet-loaded page will be missing.
            template_pages.append(TemplatePage(template))
        for page in template_pages:
            rendered_page = page.render_html(all_posts)
            ensure_exists(page.output_path)
            with open(page.output_path / page.output_file_name, mode='wb',
                      encoding='UTF-8') as file:
                file.write(rendered_page)
                logger.debug("Output template page %s." % relpath(file.name))
                build_stats['counts']['template_pages'] += 1
        logger.info("Generated %s template pages." % build_stats['counts']['template_pages'])

    # Generate individual post pages
    for post in all_posts:
        rendered_post = post.render_html(all_posts)
        ensure_exists(post.output_path)
        with open(post.output_path, mode='wb',
                  encoding='UTF-8') as file:
            file.write(rendered_post)
            if post in new_posts:
                logger.console("Output new or modified post '%s'." % post.title)
                build_stats['counts']['new_posts'] += 1
            elif post in cached_posts:
                build_stats['counts']['cached_posts'] += 1

    # Generate rollup pages
    num_posts = len(all_posts)
    num_slices = (
        num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0\
    else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1

    slice_num = 0
    for posts in all_posts.paginate():
        slice_num += 1
        has_next = slice_num < num_slices
        has_previous = 1 < slice_num <= num_slices
        rendered_page = posts.render_listpage_html(slice_num, has_next,
                                                   has_previous)
        ensure_exists(posts.output_path(slice_num))
        with open(posts.output_path(slice_num), mode='wb',
                  encoding='UTF-8') as file:
            file.write(rendered_page)
            logger.debug("Output rollup page %s." % relpath(file.name))
            build_stats['counts']['rollups'] += 1

        # Copy first rollup page to root of site - it's the homepage.
        if slice_num == 1:
            path.copyfile(posts.output_path(slice_num),
                          settings.OUTPUT_CACHE_DIR / 'index.html')
            logger.debug(
                "Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html'))

    # Generate archive page
    if num_posts > 0:
        archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
        ensure_exists(archive_output_path)

        rendered_archive = all_posts.render_archive_html(all_posts)

        with open(archive_output_path, mode='wb', encoding='UTF-8') as file:
            file.write(rendered_archive)
            logger.debug("Output %s." % relpath(file.name))

    # Generate tag pages
    if num_posts > 0:
        tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
        for tag in all_posts.all_tags:
            rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
            tag_path = ensure_exists(
                tags_output_path / slugify(tag) / 'index.html')
            with open(tag_path, mode='wb', encoding='UTF-8') as file:
                file.write(rendered_tag_page)
                build_stats['counts']['tag_pages'] += 1
                logger.debug("Output %s." % relpath(file.name))

    # Generate feeds
    feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml')
    feed_content = settings.JINJA_ENV.get_or_select_template(['rss.xml',
                                                              'theme/rss.xml',
                                                              'core/rss.xml']).render(
        post_list=all_posts[:settings.FEED_ITEM_LIMIT],
        build_date=all_posts[0].timestamp)
    with open(feed_output_path, mode='wb', encoding='UTF-8') as file:
        file.write(feed_content)
        logger.debug("Output %s." % relpath(file.name))

    # Generate sitemap
    sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'sitemap.xml.gz')
    sitemap_content = settings.JINJA_ENV.get_or_select_template(['sitemap.xml',
                                                                 'theme/sitemap.xml',
                                                                 'core/sitemap.xml']).render(post_list=all_posts)
    with gzip.open(sitemap_output_path, mode='wb') as file:
        file.write(sitemap_content)
        logger.debug("Output %s." % relpath(file.name))

    # Copy 'raw' content to output cache - second/final pass
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Compress all files marked for compression
    for file, compression_type in settings.COMPRESS_FILE_LIST:
        if file not in settings.COMPRESSION_CACHE:
            with open(file, mode='rb') as input:
                output = compress(input.read(), compression_type)
                logger.debug("Compressed %s." % relpath(file))
            settings.COMPRESSION_CACHE[file] = output
        else:
            logger.debug("Found pre-compressed file in cache: %s." % relpath(file))
            output = settings.COMPRESSION_CACHE[file]
        with open(file, mode='wb') as f:
            f.write(output)

    # Remove LESS files if LESS preprocessing is being done
    if settings.PREPROCESS_LESS:
        logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
        for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
            logger.debug("Deleting file: %s." % relpath(f))
            f.remove_p()

    logger.debug("Synchronizing output directory with output cache.")
    build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR,
                                         settings.OUTPUT_DIR)
    from pprint import pformat

    logger.debug("Folder mirroring report: %s" % pformat(build_stats['files']))
    logger.console('')
    logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR))
    logger.console("Posts: %s (%s new or updated)" % (
        (build_stats['counts']['new_posts'] + build_stats['counts'][
                                              'cached_posts']),
        build_stats['counts']['new_posts']))
    logger.console("Post rollup pages: %s (%s posts per page)" % (
        build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
    logger.console("Template pages: %s" % build_stats['counts']['template_pages'])
    logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
    logger.console("%s new items, %s modified items, and %s deleted items." % (
        len(build_stats['files']['new']),
        len(build_stats['files']['overwritten']),
        len(build_stats['files']['deleted'])))
    logger.console('')
    logger.console("Full build log at %s." % settings.LOG_FILE)

    with open(settings.BUILD_STATS_FILE, mode='wb') as file:
        pickle.dump(build_stats, file)
    settings.CACHE.close()
    return build_stats
Exemple #11
0
def build(args=None):
    """Builds an Engineer site using the settings specified in *args*."""
    from engineer.conf import settings
    from engineer.loaders import LocalLoader
    from engineer.log import get_file_handler
    from engineer.models import PostCollection, TemplatePage
    from engineer.themes import ThemeManager
    from engineer.util import mirror_folder, ensure_exists, slugify

    if args and args.clean:
        clean()

    settings.create_required_directories()

    logger = logging.getLogger('engineer.engine.build')
    logger.parent.addHandler(get_file_handler(settings.LOG_FILE))

    logger.debug("Starting build using configuration file %s." %
                 settings.SETTINGS_FILE)

    build_stats = {
        'time_run': times.now(),
        'counts': {
            'template_pages': 0,
            'new_posts': 0,
            'cached_posts': 0,
            'rollups': 0,
            'tag_pages': 0,
        },
        'files': {},
    }

    # Remove the output cache (not the post cache or the Jinja cache)
    # since we're rebuilding the site
    settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)

    theme = ThemeManager.current_theme()
    engineer_lib = (settings.OUTPUT_STATIC_DIR / 'engineer/lib/').abspath()
    ensure_exists(engineer_lib)
    # Copy Foundation files if used
    if theme.use_foundation:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.FOUNDATION_CSS
        t = ensure_exists(engineer_lib / settings.ENGINEER.FOUNDATION_CSS)
        mirror_folder(s, t)
        logger.debug("Copied Foundation library files.")

    # Copy LESS js file if needed
    if theme.use_lesscss and not settings.PREPROCESS_LESS:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.LESS_JS
        s.copy(engineer_lib)
        logger.debug("Copied LESS CSS files.")

    # Copy jQuery files if needed
    if theme.use_jquery:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.JQUERY
        s.copy(engineer_lib)
        logger.debug("Copied jQuery files.")

    # Copy modernizr files if needed
    if theme.use_modernizr:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.MODERNIZR
        s.copy(engineer_lib)
        logger.debug("Copied Modernizr files.")

    # Copy normalize.css if needed
    if theme.use_normalize_css:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.NORMALIZE_CSS
        s.copy(engineer_lib)
        logger.debug("Copied normalize.css.")

    # Copy 'raw' content to output cache - first pass
    # This first pass ensures that any static content - JS/LESS/CSS - that
    # is needed by site-specific pages (like template pages) is available
    # during the build
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Copy theme static content to output dir
    theme_output_dir = settings.OUTPUT_STATIC_DIR / 'theme'
    logger.debug("Copying theme static files to output cache.")
    theme.copy_content(theme_output_dir)
    logger.debug("Copied static files for theme to %s." %
                 relpath(theme_output_dir))

    # Copy any theme additional content to output dir if needed
    if theme.content_mappings:
        logger.debug("Copying additional theme content to output cache.")
        theme.copy_related_content(theme_output_dir)
        logger.debug("Copied additional files for theme to %s." %
                     relpath(theme_output_dir))

    # Load markdown input posts
    logger.info("Loading posts...")
    new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
    all_posts = PostCollection(new_posts + cached_posts)

    to_publish = PostCollection(all_posts.published)
    if settings.PUBLISH_DRAFTS:
        to_publish.extend(all_posts.drafts)
    if settings.PUBLISH_PENDING:
        to_publish.extend(all_posts.pending)
    if settings.PUBLISH_REVIEW:
        to_publish.extend(all_posts.review)

    if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
        logger.warning("This site contains the following pending posts:")
        for post in all_posts.pending:
            logger.warning("\t'%s' - publish time: %s, %s." %
                           (post.title, naturaltime(
                               post.timestamp), post.timestamp_local))
        logger.warning(
            "These posts won't be published until you build the site again after their publish time."
        )

    all_posts = PostCollection(
        sorted(to_publish, reverse=True, key=lambda p: p.timestamp))

    # Generate template pages
    if settings.TEMPLATE_PAGE_DIR.exists():
        logger.info("Generating template pages from %s." %
                    settings.TEMPLATE_PAGE_DIR)
        template_pages = []
        for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
            # We create all the TemplatePage objects first so we have all of the URLs to them in the template
            # environment. Without this step, template pages might have broken links if they link to a page that is
            # loaded after them, since the URL to the not-yet-loaded page will be missing.
            template_pages.append(TemplatePage(template))
        for page in template_pages:
            rendered_page = page.render_html(all_posts)
            ensure_exists(page.output_path)
            with open(page.output_path / page.output_file_name,
                      mode='wb',
                      encoding='UTF-8') as the_file:
                the_file.write(rendered_page)
                logger.info("Output template page %s." %
                            relpath(the_file.name))
                build_stats['counts']['template_pages'] += 1
        logger.info("Generated %s template pages." %
                    build_stats['counts']['template_pages'])

    # Generate individual post pages
    for post in all_posts:
        rendered_post = post.render_html(all_posts)
        ensure_exists(post.output_path)
        with open(post.output_path, mode='wb', encoding='UTF-8') as the_file:
            the_file.write(rendered_post)
            if post in new_posts:
                logger.console("Output new or modified post '%s'." %
                               post.title)
                build_stats['counts']['new_posts'] += 1
            elif post in cached_posts:
                build_stats['counts']['cached_posts'] += 1

    # Generate rollup pages
    num_posts = len(all_posts)
    num_slices = (
        num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0 \
        else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1

    slice_num = 0
    for posts in all_posts.paginate():
        slice_num += 1
        has_next = slice_num < num_slices
        has_previous = 1 < slice_num <= num_slices
        rendered_page = posts.render_listpage_html(slice_num, has_next,
                                                   has_previous)
        ensure_exists(posts.output_path(slice_num))
        with open(posts.output_path(slice_num), mode='wb',
                  encoding='UTF-8') as the_file:
            the_file.write(rendered_page)
            logger.debug("Output rollup page %s." % relpath(the_file.name))
            build_stats['counts']['rollups'] += 1

        # Copy first rollup page to root of site - it's the homepage.
        if slice_num == 1:
            path.copyfile(posts.output_path(slice_num),
                          settings.OUTPUT_CACHE_DIR / 'index.html')
            logger.debug("Output '%s'." %
                         (settings.OUTPUT_CACHE_DIR / 'index.html'))

    # Generate archive page
    if num_posts > 0:
        archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
        ensure_exists(archive_output_path)

        rendered_archive = all_posts.render_archive_html(all_posts)

        with open(archive_output_path, mode='wb',
                  encoding='UTF-8') as the_file:
            the_file.write(rendered_archive)
            logger.debug("Output %s." % relpath(the_file.name))

    # Generate tag pages
    if num_posts > 0:
        tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
        for tag in all_posts.all_tags:
            rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
            tag_path = ensure_exists(tags_output_path / slugify(tag) /
                                     'index.html')
            with open(tag_path, mode='wb', encoding='UTF-8') as the_file:
                the_file.write(rendered_tag_page)
                build_stats['counts']['tag_pages'] += 1
                logger.debug("Output %s." % relpath(the_file.name))

    # Generate feeds
    rss_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                         'feeds/rss.xml')
    atom_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                          'feeds/atom.xml')
    rss_feed = Rss201rev2Feed(title=settings.FEED_TITLE,
                              link=settings.SITE_URL,
                              description=settings.FEED_DESCRIPTION,
                              feed_url=settings.FEED_URL)

    atom_feed = Atom1Feed(title=settings.FEED_TITLE,
                          link=settings.SITE_URL,
                          description=settings.FEED_DESCRIPTION,
                          feed_url=settings.FEED_URL)

    for feed in (rss_feed, atom_feed):
        for post in all_posts[:settings.FEED_ITEM_LIMIT]:
            title = settings.JINJA_ENV.get_template(
                'core/feeds/title.jinja2').render(post=post)
            link = settings.JINJA_ENV.get_template(
                'core/feeds/link.jinja2').render(post=post)
            content = settings.JINJA_ENV.get_template(
                'core/feeds/content.jinja2').render(post=post)
            feed.add_item(title=title,
                          link=link,
                          description=content,
                          pubdate=post.timestamp,
                          unique_id=post.absolute_url)

    with open(rss_feed_output_path, mode='wb') as the_file:
        rss_feed.write(the_file, 'UTF-8')
        logger.debug("Output %s." % relpath(the_file.name))

    with open(atom_feed_output_path, mode='wb') as the_file:
        atom_feed.write(the_file, 'UTF-8')
        logger.debug("Output %s." % relpath(the_file.name))

    # Generate sitemap
    sitemap_file_name = 'sitemap.xml.gz'
    sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
                                        sitemap_file_name)
    sitemap_content = settings.JINJA_ENV.get_or_select_template(
        ['sitemap.xml', 'theme/sitemap.xml',
         'core/sitemap.xml']).render(post_list=all_posts)
    with gzip.open(sitemap_output_path, mode='wb') as the_file:
        the_file.write(sitemap_content)
        logger.debug("Output %s." % relpath(the_file.name))

    # Copy 'raw' content to output cache - second/final pass
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Compress all files marked for compression
    for the_file, compression_type in settings.COMPRESS_FILE_LIST:
        if the_file not in settings.COMPRESSION_CACHE:
            with open(the_file, mode='rb') as input:
                output = compress(input.read(), compression_type)
                logger.debug("Compressed %s." % relpath(the_file))
            settings.COMPRESSION_CACHE[the_file] = output
        else:
            logger.debug("Found pre-compressed file in cache: %s." %
                         relpath(the_file))
            output = settings.COMPRESSION_CACHE[the_file]
        with open(the_file, mode='wb') as f:
            f.write(output)

    # Remove LESS files if LESS preprocessing is being done
    if settings.PREPROCESS_LESS:
        logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
        for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
            logger.debug("Deleting file: %s." % relpath(f))
            f.remove_p()

    # Check if anything has changed other than the sitemap
    have_changes = False
    compare = filecmp.dircmp(settings.OUTPUT_CACHE_DIR,
                             settings.OUTPUT_DIR,
                             ignore=settings.OUTPUT_DIR_IGNORE)

    # The algorithm below takes advantage of the fact that once we've determined that there is more than one file
    # that's different, or if the first item returned by the generator is not the sitemap, then we can break out of
    # the generator loop early. This is also advantageous because it doesn't require us to completely exhaust the
    # generator. In the case of a fresh site build, for example, the generator will return a lot more data. So the
    # other approach here of expanding the generator into a list with a list comprehension would be inefficient
    # in many cases. This approach performs equally well in all cases at the cost of some unusual-looking code.
    diff_file_count = 0
    if not has_files(settings.OUTPUT_DIR):
        have_changes = True
    else:
        for file_path in diff_dir(compare):
            diff_file_count += 1
            if file_path != sitemap_output_path:
                have_changes = True
                break
            if diff_file_count > 1:
                have_changes = True
                break

    if not have_changes:
        logger.console('')
        logger.console("No site changes to publish.")
    else:
        logger.debug("Synchronizing output directory with output cache.")
        build_stats['files'] = mirror_folder(
            settings.OUTPUT_CACHE_DIR,
            settings.OUTPUT_DIR,
            ignore_list=settings.OUTPUT_DIR_IGNORE)
        from pprint import pformat

        logger.debug("Folder mirroring report: %s" %
                     pformat(build_stats['files']))
        logger.console('')
        logger.console("Site: '%s' output to %s." %
                       (settings.SITE_TITLE, settings.OUTPUT_DIR))
        logger.console("Posts: %s (%s new or updated)" %
                       ((build_stats['counts']['new_posts'] +
                         build_stats['counts']['cached_posts']),
                        build_stats['counts']['new_posts']))
        logger.console(
            "Post rollup pages: %s (%s posts per page)" %
            (build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
        logger.console("Template pages: %s" %
                       build_stats['counts']['template_pages'])
        logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
        logger.console(
            "%s new items, %s modified items, and %s deleted items." %
            (len(build_stats['files']['new']),
             len(build_stats['files']['overwritten']),
             len(build_stats['files']['deleted'])))

    logger.console('')
    logger.console("Full build log at %s." % settings.LOG_FILE)
    logger.console('')

    with open(settings.BUILD_STATS_FILE, mode='wb') as the_file:
        pickle.dump(build_stats, the_file)
    settings.CACHE.close()
    return build_stats
Exemple #12
0
def build(args=None):
    """Builds an Engineer site using the settings specified in *args*."""
    from engineer.conf import settings
    from engineer.loaders import LocalLoader
    from engineer.log import get_file_handler
    from engineer.models import PostCollection, TemplatePage
    from engineer.themes import ThemeManager
    from engineer.util import mirror_folder, ensure_exists, slugify

    if args and args.clean:
        clean()

    settings.create_required_directories()

    logger = logging.getLogger('engineer.engine.build')
    logger.parent.addHandler(get_file_handler(settings.LOG_FILE))

    logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE)

    build_stats = {
        'time_run': times.now(),
        'counts': {
            'template_pages': 0,
            'new_posts': 0,
            'cached_posts': 0,
            'rollups': 0,
            'tag_pages': 0,
        },
        'files': {},
    }

    # Remove the output cache (not the post cache or the Jinja cache)
    # since we're rebuilding the site
    settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)

    theme = ThemeManager.current_theme()
    engineer_lib = (settings.OUTPUT_STATIC_DIR / 'engineer/lib/').abspath()
    ensure_exists(engineer_lib)
    # Copy Foundation files if used
    if theme.use_foundation:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.FOUNDATION_CSS
        t = ensure_exists(engineer_lib / settings.ENGINEER.FOUNDATION_CSS)
        mirror_folder(s, t)
        logger.debug("Copied Foundation library files.")

    # Copy LESS js file if needed
    if theme.use_lesscss and not settings.PREPROCESS_LESS:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.LESS_JS
        s.copy(engineer_lib)
        logger.debug("Copied LESS CSS files.")

    # Copy jQuery files if needed
    if theme.use_jquery:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.JQUERY
        s.copy(engineer_lib)
        logger.debug("Copied jQuery files.")

    # Copy modernizr files if needed
    if theme.use_modernizr:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.MODERNIZR
        s.copy(engineer_lib)
        logger.debug("Copied Modernizr files.")

    # Copy normalize.css if needed
    if theme.use_normalize_css:
        s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.NORMALIZE_CSS
        s.copy(engineer_lib)
        logger.debug("Copied normalize.css.")

    # Copy 'raw' content to output cache - first pass
    # This first pass ensures that any static content - JS/LESS/CSS - that
    # is needed by site-specific pages (like template pages) is available
    # during the build
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Copy theme static content to output dir
    theme_output_dir = settings.OUTPUT_STATIC_DIR / 'theme'
    logger.debug("Copying theme static files to output cache.")
    theme.copy_content(theme_output_dir)
    logger.debug("Copied static files for theme to %s." % relpath(theme_output_dir))

    # Copy any theme additional content to output dir if needed
    if theme.content_mappings:
        logger.debug("Copying additional theme content to output cache.")
        theme.copy_related_content(theme_output_dir)
        logger.debug("Copied additional files for theme to %s." % relpath(theme_output_dir))

    # Load markdown input posts
    logger.info("Loading posts...")
    new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
    all_posts = PostCollection(new_posts + cached_posts)

    to_publish = PostCollection(all_posts.published)
    if settings.PUBLISH_DRAFTS:
        to_publish.extend(all_posts.drafts)
    if settings.PUBLISH_PENDING:
        to_publish.extend(all_posts.pending)
    if settings.PUBLISH_REVIEW:
        to_publish.extend(all_posts.review)

    if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
        logger.warning("This site contains the following pending posts:")
        for post in all_posts.pending:
            logger.warning("\t'%s' - publish time: %s, %s." % (post.title,
                                                               naturaltime(post.timestamp),
                                                               post.timestamp_local))
        logger.warning("These posts won't be published until you build the site again after their publish time.")

    all_posts = PostCollection(
        sorted(to_publish, reverse=True, key=lambda p: p.timestamp))

    # Generate template pages
    if settings.TEMPLATE_PAGE_DIR.exists():
        logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR)
        template_pages = []
        for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
            # We create all the TemplatePage objects first so we have all of the URLs to them in the template
            # environment. Without this step, template pages might have broken links if they link to a page that is
            # loaded after them, since the URL to the not-yet-loaded page will be missing.
            template_pages.append(TemplatePage(template))
        for page in template_pages:
            rendered_page = page.render_html(all_posts)
            ensure_exists(page.output_path)
            with open(page.output_path / page.output_file_name, mode='wb',
                      encoding='UTF-8') as the_file:
                the_file.write(rendered_page)
                logger.info("Output template page %s." % relpath(the_file.name))
                build_stats['counts']['template_pages'] += 1
        logger.info("Generated %s template pages." % build_stats['counts']['template_pages'])

    # Generate individual post pages
    for post in all_posts:
        rendered_post = post.render_html(all_posts)
        ensure_exists(post.output_path)
        with open(post.output_path, mode='wb',
                  encoding='UTF-8') as the_file:
            the_file.write(rendered_post)
            if post in new_posts:
                logger.console("Output new or modified post '%s'." % post.title)
                build_stats['counts']['new_posts'] += 1
            elif post in cached_posts:
                build_stats['counts']['cached_posts'] += 1

    # Generate rollup pages
    num_posts = len(all_posts)
    num_slices = (
        num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0 \
        else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1

    slice_num = 0
    for posts in all_posts.paginate():
        slice_num += 1
        has_next = slice_num < num_slices
        has_previous = 1 < slice_num <= num_slices
        rendered_page = posts.render_listpage_html(slice_num, has_next,
                                                   has_previous)
        ensure_exists(posts.output_path(slice_num))
        with open(posts.output_path(slice_num), mode='wb',
                  encoding='UTF-8') as the_file:
            the_file.write(rendered_page)
            logger.debug("Output rollup page %s." % relpath(the_file.name))
            build_stats['counts']['rollups'] += 1

        # Copy first rollup page to root of site - it's the homepage.
        if slice_num == 1:
            path.copyfile(posts.output_path(slice_num),
                          settings.OUTPUT_CACHE_DIR / 'index.html')
            logger.debug(
                "Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html'))

    # Generate archive page
    if num_posts > 0:
        archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
        ensure_exists(archive_output_path)

        rendered_archive = all_posts.render_archive_html(all_posts)

        with open(archive_output_path, mode='wb', encoding='UTF-8') as the_file:
            the_file.write(rendered_archive)
            logger.debug("Output %s." % relpath(the_file.name))

    # Generate tag pages
    if num_posts > 0:
        tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
        for tag in all_posts.all_tags:
            rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
            tag_path = ensure_exists(
                tags_output_path / slugify(tag) / 'index.html')
            with open(tag_path, mode='wb', encoding='UTF-8') as the_file:
                the_file.write(rendered_tag_page)
                build_stats['counts']['tag_pages'] += 1
                logger.debug("Output %s." % relpath(the_file.name))

    # Generate feeds
    rss_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml')
    atom_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/atom.xml')
    rss_feed = Rss201rev2Feed(
        title=settings.FEED_TITLE,
        link=settings.SITE_URL,
        description=settings.FEED_DESCRIPTION,
        feed_url=settings.FEED_URL
    )

    atom_feed = Atom1Feed(
        title=settings.FEED_TITLE,
        link=settings.SITE_URL,
        description=settings.FEED_DESCRIPTION,
        feed_url=settings.FEED_URL
    )

    for feed in (rss_feed, atom_feed):
        for post in all_posts[:settings.FEED_ITEM_LIMIT]:
            title = settings.JINJA_ENV.get_template('core/feeds/title.jinja2').render(post=post)
            link = settings.JINJA_ENV.get_template('core/feeds/link.jinja2').render(post=post)
            content = settings.JINJA_ENV.get_template('core/feeds/content.jinja2').render(post=post)
            feed.add_item(
                title=title,
                link=link,
                description=content,
                pubdate=post.timestamp,
                unique_id=post.absolute_url)

    with open(rss_feed_output_path, mode='wb') as the_file:
        rss_feed.write(the_file, 'UTF-8')
        logger.debug("Output %s." % relpath(the_file.name))

    with open(atom_feed_output_path, mode='wb') as the_file:
        atom_feed.write(the_file, 'UTF-8')
        logger.debug("Output %s." % relpath(the_file.name))

    # Generate sitemap
    sitemap_file_name = 'sitemap.xml.gz'
    sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / sitemap_file_name)
    sitemap_content = settings.JINJA_ENV.get_or_select_template(['sitemap.xml',
                                                                 'theme/sitemap.xml',
                                                                 'core/sitemap.xml']).render(post_list=all_posts)
    with gzip.open(sitemap_output_path, mode='wb') as the_file:
        the_file.write(sitemap_content)
        logger.debug("Output %s." % relpath(the_file.name))

    # Copy 'raw' content to output cache - second/final pass
    if settings.CONTENT_DIR.exists():
        mirror_folder(settings.CONTENT_DIR,
                      settings.OUTPUT_CACHE_DIR,
                      delete_orphans=False)

    # Compress all files marked for compression
    for the_file, compression_type in settings.COMPRESS_FILE_LIST:
        if the_file not in settings.COMPRESSION_CACHE:
            with open(the_file, mode='rb') as input:
                output = compress(input.read(), compression_type)
                logger.debug("Compressed %s." % relpath(the_file))
            settings.COMPRESSION_CACHE[the_file] = output
        else:
            logger.debug("Found pre-compressed file in cache: %s." % relpath(the_file))
            output = settings.COMPRESSION_CACHE[the_file]
        with open(the_file, mode='wb') as f:
            f.write(output)

    # Remove LESS files if LESS preprocessing is being done
    if settings.PREPROCESS_LESS:
        logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
        for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
            logger.debug("Deleting file: %s." % relpath(f))
            f.remove_p()

    # Check if anything has changed other than the sitemap
    have_changes = False
    compare = filecmp.dircmp(settings.OUTPUT_CACHE_DIR,
                             settings.OUTPUT_DIR,
                             ignore=settings.OUTPUT_DIR_IGNORE)

    # The algorithm below takes advantage of the fact that once we've determined that there is more than one file
    # that's different, or if the first item returned by the generator is not the sitemap, then we can break out of
    # the generator loop early. This is also advantageous because it doesn't require us to completely exhaust the
    # generator. In the case of a fresh site build, for example, the generator will return a lot more data. So the
    # other approach here of expanding the generator into a list with a list comprehension would be inefficient
    # in many cases. This approach performs equally well in all cases at the cost of some unusual-looking code.
    diff_file_count = 0
    if not has_files(settings.OUTPUT_DIR):
        have_changes = True
    else:
        for file_path in diff_dir(compare):
            diff_file_count += 1
            if file_path != sitemap_output_path:
                have_changes = True
                break
            if diff_file_count > 1:
                have_changes = True
                break

    if not have_changes:
        logger.console('')
        logger.console("No site changes to publish.")
    else:
        logger.debug("Synchronizing output directory with output cache.")
        build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR,
                                             settings.OUTPUT_DIR,
                                             ignore_list=settings.OUTPUT_DIR_IGNORE)
        from pprint import pformat

        logger.debug("Folder mirroring report: %s" % pformat(build_stats['files']))
        logger.console('')
        logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR))
        logger.console("Posts: %s (%s new or updated)" % (
            (build_stats['counts']['new_posts'] + build_stats['counts']['cached_posts']),
            build_stats['counts']['new_posts']))
        logger.console("Post rollup pages: %s (%s posts per page)" % (
            build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
        logger.console("Template pages: %s" % build_stats['counts']['template_pages'])
        logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
        logger.console("%s new items, %s modified items, and %s deleted items." % (
            len(build_stats['files']['new']),
            len(build_stats['files']['overwritten']),
            len(build_stats['files']['deleted'])))

    logger.console('')
    logger.console("Full build log at %s." % settings.LOG_FILE)
    logger.console('')

    with open(settings.BUILD_STATS_FILE, mode='wb') as the_file:
        pickle.dump(build_stats, the_file)
    settings.CACHE.close()
    return build_stats