def build(args=None):
"""Builds an Engineer site using the settings specified in *args*."""
from engineer.conf import settings
from engineer.loaders import LocalLoader
from engineer.log import get_file_handler
from engineer.models import PostCollection, TemplatePage
from engineer.themes import ThemeManager
from engineer.util import mirror_folder, ensure_exists, slugify
if args and args.clean:
clean()
settings.create_required_directories()
logger = logging.getLogger('engineer.engine.build')
logger.parent.addHandler(get_file_handler(settings.LOG_FILE))
logger.debug("Starting build using configuration file %s." %
settings.SETTINGS_FILE)
build_stats = {
'time_run': times.now(),
'counts': {
'template_pages': 0,
'new_posts': 0,
'cached_posts': 0,
'rollups': 0,
'tag_pages': 0,
},
'files': {},
}
# Remove the output cache (not the post cache or the Jinja cache)
# since we're rebuilding the site
settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)
# Copy static content to output dir
logger.debug("Copying static files to output cache.")
s = (settings.ENGINEER.STATIC_DIR / 'engineer').abspath()
t = settings.OUTPUT_STATIC_DIR / 'engineer'
mirror_folder(
s, t,
recurse=False) # Copy only the files in this folder - don't recurse
theme = ThemeManager.current_theme()
# Copy Foundation files if used
if theme.use_foundation:
s = settings.ENGINEER.LIB_DIR / 'foundation'
t = ensure_exists(settings.OUTPUT_STATIC_DIR /
'engineer/lib/foundation')
mirror_folder(s, t)
logger.debug("Copied Foundation library files.")
# Copy LESS js file if needed
if theme.use_lesscss and not settings.PREPROCESS_LESS:
s = settings.ENGINEER.LIB_DIR / 'less-1.3.1.min.js'
t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
s.copy(t)
logger.debug("Copied LESS CSS files.")
# Copy jQuery files if needed
if theme.use_jquery or theme.use_tweet:
s = settings.ENGINEER.LIB_DIR / 'jquery-1.7.1.min.js'
t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
s.copy(t)
logger.debug("Copied jQuery files.")
# Copy Tweet files if needed
if theme.use_tweet:
s = settings.ENGINEER.LIB_DIR / 'tweet'
t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/tweet')
mirror_folder(s, t)
logger.debug("Copied Tweet files.")
# Copy modernizr files if needed
if theme.use_modernizr:
s = settings.ENGINEER.LIB_DIR / 'modernizr-2.5.3.min.js'
t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
s.copy(t)
logger.debug("Copied Modernizr files.")
logger.debug("Copied static files to %s." % relpath(t))
# Copy 'raw' content to output cache - first pass
# This first pass ensures that any static content - JS/LESS/CSS - that
# is needed by site-specific pages (like template pages) is available
# during the build
if settings.CONTENT_DIR.exists():
mirror_folder(settings.CONTENT_DIR,
settings.OUTPUT_CACHE_DIR,
delete_orphans=False)
# Copy theme static content to output dir
logger.debug("Copying theme static files to output cache.")
try:
s = theme.static_root.abspath()
except ThemeNotFoundException as e:
logger.critical(e.message)
exit()
t = (settings.OUTPUT_STATIC_DIR / 'theme').abspath()
mirror_folder(s, t)
logger.debug("Copied static files for theme to %s." % relpath(t))
# Copy any theme additional content to output dir if needed
if theme.content_mappings:
logger.debug("Copying additional theme content to output cache.")
for s, t in theme.content_mappings.iteritems():
t = settings.OUTPUT_STATIC_DIR / 'theme' / t
if s.isdir():
mirror_folder(s, t)
else:
s.copy(ensure_exists(t))
logger.debug("Copied additional files for theme to %s." %
relpath(t))
# Load markdown input posts
logger.info("Loading posts...")
new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
all_posts = PostCollection(new_posts + cached_posts)
to_publish = PostCollection(all_posts.published)
if settings.PUBLISH_DRAFTS:
to_publish.extend(all_posts.drafts)
if settings.PUBLISH_PENDING:
to_publish.extend(all_posts.pending)
if settings.PUBLISH_REVIEW:
to_publish.extend(all_posts.review)
if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
logger.warning("This site contains the following pending posts:")
for post in all_posts.pending:
logger.warning("\t'%s' - publish time: %s, %s." %
(post.title, humanize.naturaltime(
post.timestamp), post.timestamp_local))
logger.warning(
"These posts won't be published until you build the site again after their publish time."
)
all_posts = PostCollection(
sorted(to_publish, reverse=True, key=lambda post: post.timestamp))
# Generate template pages
if settings.TEMPLATE_PAGE_DIR.exists():
logger.info("Generating template pages from %s." %
settings.TEMPLATE_PAGE_DIR)
template_pages = []
for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
# We create all the TemplatePage objects first so we have all of the URLs to them in the template
# environment. Without this step, template pages might have broken links if they link to a page that is
# loaded after them, since the URL to the not-yet-loaded page will be missing.
template_pages.append(TemplatePage(template))
for page in template_pages:
rendered_page = page.render_html(all_posts)
ensure_exists(page.output_path)
with open(page.output_path / page.output_file_name,
mode='wb',
encoding='UTF-8') as file:
file.write(rendered_page)
logger.debug("Output template page %s." % relpath(file.name))
build_stats['counts']['template_pages'] += 1
logger.info("Generated %s template pages." %
build_stats['counts']['template_pages'])
# Generate individual post pages
for post in all_posts:
rendered_post = post.render_html(all_posts)
ensure_exists(post.output_path)
with open(post.output_path, mode='wb', encoding='UTF-8') as file:
file.write(rendered_post)
if post in new_posts:
logger.console("Output new or modified post '%s'." %
post.title)
build_stats['counts']['new_posts'] += 1
elif post in cached_posts:
build_stats['counts']['cached_posts'] += 1
# Generate rollup pages
num_posts = len(all_posts)
num_slices = (
num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0\
else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1
slice_num = 0
for posts in all_posts.paginate():
slice_num += 1
has_next = slice_num < num_slices
has_previous = 1 < slice_num <= num_slices
rendered_page = posts.render_listpage_html(slice_num, has_next,
has_previous)
ensure_exists(posts.output_path(slice_num))
with open(posts.output_path(slice_num), mode='wb',
encoding='UTF-8') as file:
file.write(rendered_page)
logger.debug("Output rollup page %s." % relpath(file.name))
build_stats['counts']['rollups'] += 1
# Copy first rollup page to root of site - it's the homepage.
if slice_num == 1:
path.copyfile(posts.output_path(slice_num),
settings.OUTPUT_CACHE_DIR / 'index.html')
logger.debug("Output '%s'." %
(settings.OUTPUT_CACHE_DIR / 'index.html'))
# Generate archive page
if num_posts > 0:
archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
ensure_exists(archive_output_path)
rendered_archive = all_posts.render_archive_html(all_posts)
with open(archive_output_path, mode='wb', encoding='UTF-8') as file:
file.write(rendered_archive)
logger.debug("Output %s." % relpath(file.name))
# Generate tag pages
if num_posts > 0:
tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
for tag in all_posts.all_tags:
rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
tag_path = ensure_exists(tags_output_path / slugify(tag) /
'index.html')
with open(tag_path, mode='wb', encoding='UTF-8') as file:
file.write(rendered_tag_page)
build_stats['counts']['tag_pages'] += 1
logger.debug("Output %s." % relpath(file.name))
# Generate feeds
feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
'feeds/rss.xml')
feed_content = settings.JINJA_ENV.get_or_select_template([
'rss.xml', 'theme/rss.xml', 'core/rss.xml'
]).render(post_list=all_posts[:settings.FEED_ITEM_LIMIT],
build_date=all_posts[0].timestamp)
with open(feed_output_path, mode='wb', encoding='UTF-8') as file:
file.write(feed_content)
logger.debug("Output %s." % relpath(file.name))
# Generate sitemap
sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
'sitemap.xml.gz')
sitemap_content = settings.JINJA_ENV.get_or_select_template(
['sitemap.xml', 'theme/sitemap.xml',
'core/sitemap.xml']).render(post_list=all_posts)
with gzip.open(sitemap_output_path, mode='wb') as file:
file.write(sitemap_content)
logger.debug("Output %s." % relpath(file.name))
# Copy 'raw' content to output cache - second/final pass
if settings.CONTENT_DIR.exists():
mirror_folder(settings.CONTENT_DIR,
settings.OUTPUT_CACHE_DIR,
delete_orphans=False)
# Compress all files marked for compression
for file, compression_type in settings.COMPRESS_FILE_LIST:
if file not in settings.COMPRESSION_CACHE:
with open(file, mode='rb') as input:
output = compress(input.read(), compression_type)
logger.debug("Compressed %s." % relpath(file))
settings.COMPRESSION_CACHE[file] = output
else:
logger.debug("Found pre-compressed file in cache: %s." %
relpath(file))
output = settings.COMPRESSION_CACHE[file]
with open(file, mode='wb') as f:
f.write(output)
# Remove LESS files if LESS preprocessing is being done
if settings.PREPROCESS_LESS:
logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
logger.debug("Deleting file: %s." % relpath(f))
f.remove_p()
logger.debug("Synchronizing output directory with output cache.")
build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR,
settings.OUTPUT_DIR)
from pprint import pformat
logger.debug("Folder mirroring report: %s" % pformat(build_stats['files']))
logger.console('')
logger.console("Site: '%s' output to %s." %
(settings.SITE_TITLE, settings.OUTPUT_DIR))
logger.console("Posts: %s (%s new or updated)" %
((build_stats['counts']['new_posts'] +
build_stats['counts']['cached_posts']),
build_stats['counts']['new_posts']))
logger.console(
"Post rollup pages: %s (%s posts per page)" %
(build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
logger.console("Template pages: %s" %
build_stats['counts']['template_pages'])
logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
logger.console("%s new items, %s modified items, and %s deleted items." %
(len(build_stats['files']['new']),
len(build_stats['files']['overwritten']),
len(build_stats['files']['deleted'])))
logger.console('')
logger.console("Full build log at %s." % settings.LOG_FILE)
with open(settings.BUILD_STATS_FILE, mode='wb') as file:
pickle.dump(build_stats, file)
settings.CACHE.close()
return build_stats
def build(args=None):
"""Builds an Engineer site using the settings specified in *args*."""
from engineer.conf import settings
from engineer.loaders import LocalLoader
from engineer.log import get_file_handler
from engineer.models import PostCollection, TemplatePage
from engineer.themes import ThemeManager
from engineer.util import mirror_folder, ensure_exists, slugify
if args and args.clean:
clean()
settings.create_required_directories()
logger = logging.getLogger('engineer.engine.build')
logger.parent.addHandler(get_file_handler(settings.LOG_FILE))
logger.debug("Starting build using configuration file %s." %
settings.SETTINGS_FILE)
build_stats = {
'time_run': times.now(),
'counts': {
'template_pages': 0,
'new_posts': 0,
'cached_posts': 0,
'rollups': 0,
'tag_pages': 0,
},
'files': {},
}
# Remove the output cache (not the post cache or the Jinja cache)
# since we're rebuilding the site
settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)
theme = ThemeManager.current_theme()
engineer_lib = (settings.OUTPUT_STATIC_DIR / 'engineer/lib/').abspath()
ensure_exists(engineer_lib)
# Copy Foundation files if used
if theme.use_foundation:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.FOUNDATION_CSS
t = ensure_exists(engineer_lib / settings.ENGINEER.FOUNDATION_CSS)
mirror_folder(s, t)
logger.debug("Copied Foundation library files.")
# Copy LESS js file if needed
if theme.use_lesscss and not settings.PREPROCESS_LESS:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.LESS_JS
s.copy(engineer_lib)
logger.debug("Copied LESS CSS files.")
# Copy jQuery files if needed
if theme.use_jquery:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.JQUERY
s.copy(engineer_lib)
logger.debug("Copied jQuery files.")
# Copy modernizr files if needed
if theme.use_modernizr:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.MODERNIZR
s.copy(engineer_lib)
logger.debug("Copied Modernizr files.")
# Copy normalize.css if needed
if theme.use_normalize_css:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.NORMALIZE_CSS
s.copy(engineer_lib)
logger.debug("Copied normalize.css.")
# Copy 'raw' content to output cache - first pass
# This first pass ensures that any static content - JS/LESS/CSS - that
# is needed by site-specific pages (like template pages) is available
# during the build
if settings.CONTENT_DIR.exists():
mirror_folder(settings.CONTENT_DIR,
settings.OUTPUT_CACHE_DIR,
delete_orphans=False)
# Copy theme static content to output dir
theme_output_dir = settings.OUTPUT_STATIC_DIR / 'theme'
logger.debug("Copying theme static files to output cache.")
theme.copy_content(theme_output_dir)
logger.debug("Copied static files for theme to %s." %
relpath(theme_output_dir))
# Copy any theme additional content to output dir if needed
if theme.content_mappings:
logger.debug("Copying additional theme content to output cache.")
theme.copy_related_content(theme_output_dir)
logger.debug("Copied additional files for theme to %s." %
relpath(theme_output_dir))
# Load markdown input posts
logger.info("Loading posts...")
new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
all_posts = PostCollection(new_posts + cached_posts)
to_publish = PostCollection(all_posts.published)
if settings.PUBLISH_DRAFTS:
to_publish.extend(all_posts.drafts)
if settings.PUBLISH_PENDING:
to_publish.extend(all_posts.pending)
if settings.PUBLISH_REVIEW:
to_publish.extend(all_posts.review)
if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
logger.warning("This site contains the following pending posts:")
for post in all_posts.pending:
logger.warning("\t'%s' - publish time: %s, %s." %
(post.title, naturaltime(
post.timestamp), post.timestamp_local))
logger.warning(
"These posts won't be published until you build the site again after their publish time."
)
all_posts = PostCollection(
sorted(to_publish, reverse=True, key=lambda p: p.timestamp))
# Generate template pages
if settings.TEMPLATE_PAGE_DIR.exists():
logger.info("Generating template pages from %s." %
settings.TEMPLATE_PAGE_DIR)
template_pages = []
for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
# We create all the TemplatePage objects first so we have all of the URLs to them in the template
# environment. Without this step, template pages might have broken links if they link to a page that is
# loaded after them, since the URL to the not-yet-loaded page will be missing.
template_pages.append(TemplatePage(template))
for page in template_pages:
rendered_page = page.render_html(all_posts)
ensure_exists(page.output_path)
with open(page.output_path / page.output_file_name,
mode='wb',
encoding='UTF-8') as the_file:
the_file.write(rendered_page)
logger.info("Output template page %s." %
relpath(the_file.name))
build_stats['counts']['template_pages'] += 1
logger.info("Generated %s template pages." %
build_stats['counts']['template_pages'])
# Generate individual post pages
for post in all_posts:
rendered_post = post.render_html(all_posts)
ensure_exists(post.output_path)
with open(post.output_path, mode='wb', encoding='UTF-8') as the_file:
the_file.write(rendered_post)
if post in new_posts:
logger.console("Output new or modified post '%s'." %
post.title)
build_stats['counts']['new_posts'] += 1
elif post in cached_posts:
build_stats['counts']['cached_posts'] += 1
# Generate rollup pages
num_posts = len(all_posts)
num_slices = (
num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0 \
else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1
slice_num = 0
for posts in all_posts.paginate():
slice_num += 1
has_next = slice_num < num_slices
has_previous = 1 < slice_num <= num_slices
rendered_page = posts.render_listpage_html(slice_num, has_next,
has_previous)
ensure_exists(posts.output_path(slice_num))
with open(posts.output_path(slice_num), mode='wb',
encoding='UTF-8') as the_file:
the_file.write(rendered_page)
logger.debug("Output rollup page %s." % relpath(the_file.name))
build_stats['counts']['rollups'] += 1
# Copy first rollup page to root of site - it's the homepage.
if slice_num == 1:
path.copyfile(posts.output_path(slice_num),
settings.OUTPUT_CACHE_DIR / 'index.html')
logger.debug("Output '%s'." %
(settings.OUTPUT_CACHE_DIR / 'index.html'))
# Generate archive page
if num_posts > 0:
archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
ensure_exists(archive_output_path)
rendered_archive = all_posts.render_archive_html(all_posts)
with open(archive_output_path, mode='wb',
encoding='UTF-8') as the_file:
the_file.write(rendered_archive)
logger.debug("Output %s." % relpath(the_file.name))
# Generate tag pages
if num_posts > 0:
tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
for tag in all_posts.all_tags:
rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
tag_path = ensure_exists(tags_output_path / slugify(tag) /
'index.html')
with open(tag_path, mode='wb', encoding='UTF-8') as the_file:
the_file.write(rendered_tag_page)
build_stats['counts']['tag_pages'] += 1
logger.debug("Output %s." % relpath(the_file.name))
# Generate feeds
rss_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
'feeds/rss.xml')
atom_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
'feeds/atom.xml')
rss_feed = Rss201rev2Feed(title=settings.FEED_TITLE,
link=settings.SITE_URL,
description=settings.FEED_DESCRIPTION,
feed_url=settings.FEED_URL)
atom_feed = Atom1Feed(title=settings.FEED_TITLE,
link=settings.SITE_URL,
description=settings.FEED_DESCRIPTION,
feed_url=settings.FEED_URL)
for feed in (rss_feed, atom_feed):
for post in all_posts[:settings.FEED_ITEM_LIMIT]:
title = settings.JINJA_ENV.get_template(
'core/feeds/title.jinja2').render(post=post)
link = settings.JINJA_ENV.get_template(
'core/feeds/link.jinja2').render(post=post)
content = settings.JINJA_ENV.get_template(
'core/feeds/content.jinja2').render(post=post)
feed.add_item(title=title,
link=link,
description=content,
pubdate=post.timestamp,
unique_id=post.absolute_url)
with open(rss_feed_output_path, mode='wb') as the_file:
rss_feed.write(the_file, 'UTF-8')
logger.debug("Output %s." % relpath(the_file.name))
with open(atom_feed_output_path, mode='wb') as the_file:
atom_feed.write(the_file, 'UTF-8')
logger.debug("Output %s." % relpath(the_file.name))
# Generate sitemap
sitemap_file_name = 'sitemap.xml.gz'
sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR /
sitemap_file_name)
sitemap_content = settings.JINJA_ENV.get_or_select_template(
['sitemap.xml', 'theme/sitemap.xml',
'core/sitemap.xml']).render(post_list=all_posts)
with gzip.open(sitemap_output_path, mode='wb') as the_file:
the_file.write(sitemap_content)
logger.debug("Output %s." % relpath(the_file.name))
# Copy 'raw' content to output cache - second/final pass
if settings.CONTENT_DIR.exists():
mirror_folder(settings.CONTENT_DIR,
settings.OUTPUT_CACHE_DIR,
delete_orphans=False)
# Compress all files marked for compression
for the_file, compression_type in settings.COMPRESS_FILE_LIST:
if the_file not in settings.COMPRESSION_CACHE:
with open(the_file, mode='rb') as input:
output = compress(input.read(), compression_type)
logger.debug("Compressed %s." % relpath(the_file))
settings.COMPRESSION_CACHE[the_file] = output
else:
logger.debug("Found pre-compressed file in cache: %s." %
relpath(the_file))
output = settings.COMPRESSION_CACHE[the_file]
with open(the_file, mode='wb') as f:
f.write(output)
# Remove LESS files if LESS preprocessing is being done
if settings.PREPROCESS_LESS:
logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
logger.debug("Deleting file: %s." % relpath(f))
f.remove_p()
# Check if anything has changed other than the sitemap
have_changes = False
compare = filecmp.dircmp(settings.OUTPUT_CACHE_DIR,
settings.OUTPUT_DIR,
ignore=settings.OUTPUT_DIR_IGNORE)
# The algorithm below takes advantage of the fact that once we've determined that there is more than one file
# that's different, or if the first item returned by the generator is not the sitemap, then we can break out of
# the generator loop early. This is also advantageous because it doesn't require us to completely exhaust the
# generator. In the case of a fresh site build, for example, the generator will return a lot more data. So the
# other approach here of expanding the generator into a list with a list comprehension would be inefficient
# in many cases. This approach performs equally well in all cases at the cost of some unusual-looking code.
diff_file_count = 0
if not has_files(settings.OUTPUT_DIR):
have_changes = True
else:
for file_path in diff_dir(compare):
diff_file_count += 1
if file_path != sitemap_output_path:
have_changes = True
break
if diff_file_count > 1:
have_changes = True
break
if not have_changes:
logger.console('')
logger.console("No site changes to publish.")
else:
logger.debug("Synchronizing output directory with output cache.")
build_stats['files'] = mirror_folder(
settings.OUTPUT_CACHE_DIR,
settings.OUTPUT_DIR,
ignore_list=settings.OUTPUT_DIR_IGNORE)
from pprint import pformat
logger.debug("Folder mirroring report: %s" %
pformat(build_stats['files']))
logger.console('')
logger.console("Site: '%s' output to %s." %
(settings.SITE_TITLE, settings.OUTPUT_DIR))
logger.console("Posts: %s (%s new or updated)" %
((build_stats['counts']['new_posts'] +
build_stats['counts']['cached_posts']),
build_stats['counts']['new_posts']))
logger.console(
"Post rollup pages: %s (%s posts per page)" %
(build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
logger.console("Template pages: %s" %
build_stats['counts']['template_pages'])
logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
logger.console(
"%s new items, %s modified items, and %s deleted items." %
(len(build_stats['files']['new']),
len(build_stats['files']['overwritten']),
len(build_stats['files']['deleted'])))
logger.console('')
logger.console("Full build log at %s." % settings.LOG_FILE)
logger.console('')
with open(settings.BUILD_STATS_FILE, mode='wb') as the_file:
pickle.dump(build_stats, the_file)
settings.CACHE.close()
return build_stats
def build(args=None):
"""Builds an Engineer site using the settings specified in *args*."""
from engineer.conf import settings
from engineer.loaders import LocalLoader
from engineer.log import get_file_handler
from engineer.models import PostCollection, TemplatePage
from engineer.themes import ThemeManager
from engineer.util import mirror_folder, ensure_exists, slugify
if args and args.clean:
clean()
settings.create_required_directories()
logger = logging.getLogger('engineer.engine.build')
logger.parent.addHandler(get_file_handler(settings.LOG_FILE))
logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE)
build_stats = {
'time_run': times.now(),
'counts': {
'template_pages': 0,
'new_posts': 0,
'cached_posts': 0,
'rollups': 0,
'tag_pages': 0,
},
'files': {},
}
# Remove the output cache (not the post cache or the Jinja cache)
# since we're rebuilding the site
settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)
# Copy static content to output dir
logger.debug("Copying static files to output cache.")
s = (settings.ENGINEER.STATIC_DIR / 'engineer').abspath()
t = settings.OUTPUT_STATIC_DIR / 'engineer'
mirror_folder(s, t, recurse=False) # Copy only the files in this folder - don't recurse
theme = ThemeManager.current_theme()
# Copy Foundation files if used
if theme.use_foundation:
s = settings.ENGINEER.LIB_DIR / 'foundation'
t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/foundation')
mirror_folder(s, t)
logger.debug("Copied Foundation library files.")
# Copy LESS js file if needed
if theme.use_lesscss and not settings.PREPROCESS_LESS:
s = settings.ENGINEER.LIB_DIR / 'less-1.3.1.min.js'
t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
s.copy(t)
logger.debug("Copied LESS CSS files.")
# Copy jQuery files if needed
if theme.use_jquery or theme.use_tweet:
s = settings.ENGINEER.LIB_DIR / 'jquery-1.7.1.min.js'
t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
s.copy(t)
logger.debug("Copied jQuery files.")
# Copy Tweet files if needed
if theme.use_tweet:
s = settings.ENGINEER.LIB_DIR / 'tweet'
t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/tweet')
mirror_folder(s, t)
logger.debug("Copied Tweet files.")
# Copy modernizr files if needed
if theme.use_modernizr:
s = settings.ENGINEER.LIB_DIR / 'modernizr-2.5.3.min.js'
t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/')
s.copy(t)
logger.debug("Copied Modernizr files.")
logger.debug("Copied static files to %s." % relpath(t))
# Copy 'raw' content to output cache - first pass
# This first pass ensures that any static content - JS/LESS/CSS - that
# is needed by site-specific pages (like template pages) is available
# during the build
if settings.CONTENT_DIR.exists():
mirror_folder(settings.CONTENT_DIR,
settings.OUTPUT_CACHE_DIR,
delete_orphans=False)
# Copy theme static content to output dir
logger.debug("Copying theme static files to output cache.")
try:
s = theme.static_root.abspath()
except ThemeNotFoundException as e:
logger.critical(e.message)
exit()
t = (settings.OUTPUT_STATIC_DIR / 'theme').abspath()
mirror_folder(s, t)
logger.debug("Copied static files for theme to %s." % relpath(t))
# Copy any theme additional content to output dir if needed
if theme.content_mappings:
logger.debug("Copying additional theme content to output cache.")
for s, t in theme.content_mappings.iteritems():
t = settings.OUTPUT_STATIC_DIR / 'theme' / t
if s.isdir():
mirror_folder(s, t)
else:
s.copy(ensure_exists(t))
logger.debug("Copied additional files for theme to %s." % relpath(t))
# Load markdown input posts
logger.info("Loading posts...")
new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
all_posts = PostCollection(new_posts + cached_posts)
to_publish = PostCollection(all_posts.published)
if settings.PUBLISH_DRAFTS:
to_publish.extend(all_posts.drafts)
if settings.PUBLISH_PENDING:
to_publish.extend(all_posts.pending)
if settings.PUBLISH_REVIEW:
to_publish.extend(all_posts.review)
if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
logger.warning("This site contains the following pending posts:")
for post in all_posts.pending:
logger.warning("\t'%s' - publish time: %s, %s." % (post.title,
naturaltime(post.timestamp),
post.timestamp_local))
logger.warning("These posts won't be published until you build the site again after their publish time.")
all_posts = PostCollection(
sorted(to_publish, reverse=True, key=lambda post: post.timestamp))
# Generate template pages
if settings.TEMPLATE_PAGE_DIR.exists():
logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR)
template_pages = []
for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
# We create all the TemplatePage objects first so we have all of the URLs to them in the template
# environment. Without this step, template pages might have broken links if they link to a page that is
# loaded after them, since the URL to the not-yet-loaded page will be missing.
template_pages.append(TemplatePage(template))
for page in template_pages:
rendered_page = page.render_html(all_posts)
ensure_exists(page.output_path)
with open(page.output_path / page.output_file_name, mode='wb',
encoding='UTF-8') as file:
file.write(rendered_page)
logger.debug("Output template page %s." % relpath(file.name))
build_stats['counts']['template_pages'] += 1
logger.info("Generated %s template pages." % build_stats['counts']['template_pages'])
# Generate individual post pages
for post in all_posts:
rendered_post = post.render_html(all_posts)
ensure_exists(post.output_path)
with open(post.output_path, mode='wb',
encoding='UTF-8') as file:
file.write(rendered_post)
if post in new_posts:
logger.console("Output new or modified post '%s'." % post.title)
build_stats['counts']['new_posts'] += 1
elif post in cached_posts:
build_stats['counts']['cached_posts'] += 1
# Generate rollup pages
num_posts = len(all_posts)
num_slices = (
num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0\
else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1
slice_num = 0
for posts in all_posts.paginate():
slice_num += 1
has_next = slice_num < num_slices
has_previous = 1 < slice_num <= num_slices
rendered_page = posts.render_listpage_html(slice_num, has_next,
has_previous)
ensure_exists(posts.output_path(slice_num))
with open(posts.output_path(slice_num), mode='wb',
encoding='UTF-8') as file:
file.write(rendered_page)
logger.debug("Output rollup page %s." % relpath(file.name))
build_stats['counts']['rollups'] += 1
# Copy first rollup page to root of site - it's the homepage.
if slice_num == 1:
path.copyfile(posts.output_path(slice_num),
settings.OUTPUT_CACHE_DIR / 'index.html')
logger.debug(
"Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html'))
# Generate archive page
if num_posts > 0:
archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
ensure_exists(archive_output_path)
rendered_archive = all_posts.render_archive_html(all_posts)
with open(archive_output_path, mode='wb', encoding='UTF-8') as file:
file.write(rendered_archive)
logger.debug("Output %s." % relpath(file.name))
# Generate tag pages
if num_posts > 0:
tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
for tag in all_posts.all_tags:
rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
tag_path = ensure_exists(
tags_output_path / slugify(tag) / 'index.html')
with open(tag_path, mode='wb', encoding='UTF-8') as file:
file.write(rendered_tag_page)
build_stats['counts']['tag_pages'] += 1
logger.debug("Output %s." % relpath(file.name))
# Generate feeds
feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml')
feed_content = settings.JINJA_ENV.get_or_select_template(['rss.xml',
'theme/rss.xml',
'core/rss.xml']).render(
post_list=all_posts[:settings.FEED_ITEM_LIMIT],
build_date=all_posts[0].timestamp)
with open(feed_output_path, mode='wb', encoding='UTF-8') as file:
file.write(feed_content)
logger.debug("Output %s." % relpath(file.name))
# Generate sitemap
sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'sitemap.xml.gz')
sitemap_content = settings.JINJA_ENV.get_or_select_template(['sitemap.xml',
'theme/sitemap.xml',
'core/sitemap.xml']).render(post_list=all_posts)
with gzip.open(sitemap_output_path, mode='wb') as file:
file.write(sitemap_content)
logger.debug("Output %s." % relpath(file.name))
# Copy 'raw' content to output cache - second/final pass
if settings.CONTENT_DIR.exists():
mirror_folder(settings.CONTENT_DIR,
settings.OUTPUT_CACHE_DIR,
delete_orphans=False)
# Compress all files marked for compression
for file, compression_type in settings.COMPRESS_FILE_LIST:
if file not in settings.COMPRESSION_CACHE:
with open(file, mode='rb') as input:
output = compress(input.read(), compression_type)
logger.debug("Compressed %s." % relpath(file))
settings.COMPRESSION_CACHE[file] = output
else:
logger.debug("Found pre-compressed file in cache: %s." % relpath(file))
output = settings.COMPRESSION_CACHE[file]
with open(file, mode='wb') as f:
f.write(output)
# Remove LESS files if LESS preprocessing is being done
if settings.PREPROCESS_LESS:
logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
logger.debug("Deleting file: %s." % relpath(f))
f.remove_p()
logger.debug("Synchronizing output directory with output cache.")
build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR,
settings.OUTPUT_DIR)
from pprint import pformat
logger.debug("Folder mirroring report: %s" % pformat(build_stats['files']))
logger.console('')
logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR))
logger.console("Posts: %s (%s new or updated)" % (
(build_stats['counts']['new_posts'] + build_stats['counts'][
'cached_posts']),
build_stats['counts']['new_posts']))
logger.console("Post rollup pages: %s (%s posts per page)" % (
build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
logger.console("Template pages: %s" % build_stats['counts']['template_pages'])
logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
logger.console("%s new items, %s modified items, and %s deleted items." % (
len(build_stats['files']['new']),
len(build_stats['files']['overwritten']),
len(build_stats['files']['deleted'])))
logger.console('')
logger.console("Full build log at %s." % settings.LOG_FILE)
with open(settings.BUILD_STATS_FILE, mode='wb') as file:
pickle.dump(build_stats, file)
settings.CACHE.close()
return build_stats
def build(args=None):
"""Builds an Engineer site using the settings specified in *args*."""
from engineer.conf import settings
from engineer.loaders import LocalLoader
from engineer.log import get_file_handler
from engineer.models import PostCollection, TemplatePage
from engineer.themes import ThemeManager
from engineer.util import mirror_folder, ensure_exists, slugify
if args and args.clean:
clean()
settings.create_required_directories()
logger = logging.getLogger('engineer.engine.build')
logger.parent.addHandler(get_file_handler(settings.LOG_FILE))
logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE)
build_stats = {
'time_run': times.now(),
'counts': {
'template_pages': 0,
'new_posts': 0,
'cached_posts': 0,
'rollups': 0,
'tag_pages': 0,
},
'files': {},
}
# Remove the output cache (not the post cache or the Jinja cache)
# since we're rebuilding the site
settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True)
theme = ThemeManager.current_theme()
engineer_lib = (settings.OUTPUT_STATIC_DIR / 'engineer/lib/').abspath()
ensure_exists(engineer_lib)
# Copy Foundation files if used
if theme.use_foundation:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.FOUNDATION_CSS
t = ensure_exists(engineer_lib / settings.ENGINEER.FOUNDATION_CSS)
mirror_folder(s, t)
logger.debug("Copied Foundation library files.")
# Copy LESS js file if needed
if theme.use_lesscss and not settings.PREPROCESS_LESS:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.LESS_JS
s.copy(engineer_lib)
logger.debug("Copied LESS CSS files.")
# Copy jQuery files if needed
if theme.use_jquery:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.JQUERY
s.copy(engineer_lib)
logger.debug("Copied jQuery files.")
# Copy modernizr files if needed
if theme.use_modernizr:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.MODERNIZR
s.copy(engineer_lib)
logger.debug("Copied Modernizr files.")
# Copy normalize.css if needed
if theme.use_normalize_css:
s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.NORMALIZE_CSS
s.copy(engineer_lib)
logger.debug("Copied normalize.css.")
# Copy 'raw' content to output cache - first pass
# This first pass ensures that any static content - JS/LESS/CSS - that
# is needed by site-specific pages (like template pages) is available
# during the build
if settings.CONTENT_DIR.exists():
mirror_folder(settings.CONTENT_DIR,
settings.OUTPUT_CACHE_DIR,
delete_orphans=False)
# Copy theme static content to output dir
theme_output_dir = settings.OUTPUT_STATIC_DIR / 'theme'
logger.debug("Copying theme static files to output cache.")
theme.copy_content(theme_output_dir)
logger.debug("Copied static files for theme to %s." % relpath(theme_output_dir))
# Copy any theme additional content to output dir if needed
if theme.content_mappings:
logger.debug("Copying additional theme content to output cache.")
theme.copy_related_content(theme_output_dir)
logger.debug("Copied additional files for theme to %s." % relpath(theme_output_dir))
# Load markdown input posts
logger.info("Loading posts...")
new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR)
all_posts = PostCollection(new_posts + cached_posts)
to_publish = PostCollection(all_posts.published)
if settings.PUBLISH_DRAFTS:
to_publish.extend(all_posts.drafts)
if settings.PUBLISH_PENDING:
to_publish.extend(all_posts.pending)
if settings.PUBLISH_REVIEW:
to_publish.extend(all_posts.review)
if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0:
logger.warning("This site contains the following pending posts:")
for post in all_posts.pending:
logger.warning("\t'%s' - publish time: %s, %s." % (post.title,
naturaltime(post.timestamp),
post.timestamp_local))
logger.warning("These posts won't be published until you build the site again after their publish time.")
all_posts = PostCollection(
sorted(to_publish, reverse=True, key=lambda p: p.timestamp))
# Generate template pages
if settings.TEMPLATE_PAGE_DIR.exists():
logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR)
template_pages = []
for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'):
# We create all the TemplatePage objects first so we have all of the URLs to them in the template
# environment. Without this step, template pages might have broken links if they link to a page that is
# loaded after them, since the URL to the not-yet-loaded page will be missing.
template_pages.append(TemplatePage(template))
for page in template_pages:
rendered_page = page.render_html(all_posts)
ensure_exists(page.output_path)
with open(page.output_path / page.output_file_name, mode='wb',
encoding='UTF-8') as the_file:
the_file.write(rendered_page)
logger.info("Output template page %s." % relpath(the_file.name))
build_stats['counts']['template_pages'] += 1
logger.info("Generated %s template pages." % build_stats['counts']['template_pages'])
# Generate individual post pages
for post in all_posts:
rendered_post = post.render_html(all_posts)
ensure_exists(post.output_path)
with open(post.output_path, mode='wb',
encoding='UTF-8') as the_file:
the_file.write(rendered_post)
if post in new_posts:
logger.console("Output new or modified post '%s'." % post.title)
build_stats['counts']['new_posts'] += 1
elif post in cached_posts:
build_stats['counts']['cached_posts'] += 1
# Generate rollup pages
num_posts = len(all_posts)
num_slices = (
num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0 \
else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1
slice_num = 0
for posts in all_posts.paginate():
slice_num += 1
has_next = slice_num < num_slices
has_previous = 1 < slice_num <= num_slices
rendered_page = posts.render_listpage_html(slice_num, has_next,
has_previous)
ensure_exists(posts.output_path(slice_num))
with open(posts.output_path(slice_num), mode='wb',
encoding='UTF-8') as the_file:
the_file.write(rendered_page)
logger.debug("Output rollup page %s." % relpath(the_file.name))
build_stats['counts']['rollups'] += 1
# Copy first rollup page to root of site - it's the homepage.
if slice_num == 1:
path.copyfile(posts.output_path(slice_num),
settings.OUTPUT_CACHE_DIR / 'index.html')
logger.debug(
"Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html'))
# Generate archive page
if num_posts > 0:
archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html'
ensure_exists(archive_output_path)
rendered_archive = all_posts.render_archive_html(all_posts)
with open(archive_output_path, mode='wb', encoding='UTF-8') as the_file:
the_file.write(rendered_archive)
logger.debug("Output %s." % relpath(the_file.name))
# Generate tag pages
if num_posts > 0:
tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag'
for tag in all_posts.all_tags:
rendered_tag_page = all_posts.render_tag_html(tag, all_posts)
tag_path = ensure_exists(
tags_output_path / slugify(tag) / 'index.html')
with open(tag_path, mode='wb', encoding='UTF-8') as the_file:
the_file.write(rendered_tag_page)
build_stats['counts']['tag_pages'] += 1
logger.debug("Output %s." % relpath(the_file.name))
# Generate feeds
rss_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml')
atom_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/atom.xml')
rss_feed = Rss201rev2Feed(
title=settings.FEED_TITLE,
link=settings.SITE_URL,
description=settings.FEED_DESCRIPTION,
feed_url=settings.FEED_URL
)
atom_feed = Atom1Feed(
title=settings.FEED_TITLE,
link=settings.SITE_URL,
description=settings.FEED_DESCRIPTION,
feed_url=settings.FEED_URL
)
for feed in (rss_feed, atom_feed):
for post in all_posts[:settings.FEED_ITEM_LIMIT]:
title = settings.JINJA_ENV.get_template('core/feeds/title.jinja2').render(post=post)
link = settings.JINJA_ENV.get_template('core/feeds/link.jinja2').render(post=post)
content = settings.JINJA_ENV.get_template('core/feeds/content.jinja2').render(post=post)
feed.add_item(
title=title,
link=link,
description=content,
pubdate=post.timestamp,
unique_id=post.absolute_url)
with open(rss_feed_output_path, mode='wb') as the_file:
rss_feed.write(the_file, 'UTF-8')
logger.debug("Output %s." % relpath(the_file.name))
with open(atom_feed_output_path, mode='wb') as the_file:
atom_feed.write(the_file, 'UTF-8')
logger.debug("Output %s." % relpath(the_file.name))
# Generate sitemap
sitemap_file_name = 'sitemap.xml.gz'
sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / sitemap_file_name)
sitemap_content = settings.JINJA_ENV.get_or_select_template(['sitemap.xml',
'theme/sitemap.xml',
'core/sitemap.xml']).render(post_list=all_posts)
with gzip.open(sitemap_output_path, mode='wb') as the_file:
the_file.write(sitemap_content)
logger.debug("Output %s." % relpath(the_file.name))
# Copy 'raw' content to output cache - second/final pass
if settings.CONTENT_DIR.exists():
mirror_folder(settings.CONTENT_DIR,
settings.OUTPUT_CACHE_DIR,
delete_orphans=False)
# Compress all files marked for compression
for the_file, compression_type in settings.COMPRESS_FILE_LIST:
if the_file not in settings.COMPRESSION_CACHE:
with open(the_file, mode='rb') as input:
output = compress(input.read(), compression_type)
logger.debug("Compressed %s." % relpath(the_file))
settings.COMPRESSION_CACHE[the_file] = output
else:
logger.debug("Found pre-compressed file in cache: %s." % relpath(the_file))
output = settings.COMPRESSION_CACHE[the_file]
with open(the_file, mode='wb') as f:
f.write(output)
# Remove LESS files if LESS preprocessing is being done
if settings.PREPROCESS_LESS:
logger.debug("Deleting LESS files since PREPROCESS_LESS is True.")
for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"):
logger.debug("Deleting file: %s." % relpath(f))
f.remove_p()
# Check if anything has changed other than the sitemap
have_changes = False
compare = filecmp.dircmp(settings.OUTPUT_CACHE_DIR,
settings.OUTPUT_DIR,
ignore=settings.OUTPUT_DIR_IGNORE)
# The algorithm below takes advantage of the fact that once we've determined that there is more than one file
# that's different, or if the first item returned by the generator is not the sitemap, then we can break out of
# the generator loop early. This is also advantageous because it doesn't require us to completely exhaust the
# generator. In the case of a fresh site build, for example, the generator will return a lot more data. So the
# other approach here of expanding the generator into a list with a list comprehension would be inefficient
# in many cases. This approach performs equally well in all cases at the cost of some unusual-looking code.
diff_file_count = 0
if not has_files(settings.OUTPUT_DIR):
have_changes = True
else:
for file_path in diff_dir(compare):
diff_file_count += 1
if file_path != sitemap_output_path:
have_changes = True
break
if diff_file_count > 1:
have_changes = True
break
if not have_changes:
logger.console('')
logger.console("No site changes to publish.")
else:
logger.debug("Synchronizing output directory with output cache.")
build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR,
settings.OUTPUT_DIR,
ignore_list=settings.OUTPUT_DIR_IGNORE)
from pprint import pformat
logger.debug("Folder mirroring report: %s" % pformat(build_stats['files']))
logger.console('')
logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR))
logger.console("Posts: %s (%s new or updated)" % (
(build_stats['counts']['new_posts'] + build_stats['counts']['cached_posts']),
build_stats['counts']['new_posts']))
logger.console("Post rollup pages: %s (%s posts per page)" % (
build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE))
logger.console("Template pages: %s" % build_stats['counts']['template_pages'])
logger.console("Tag pages: %s" % build_stats['counts']['tag_pages'])
logger.console("%s new items, %s modified items, and %s deleted items." % (
len(build_stats['files']['new']),
len(build_stats['files']['overwritten']),
len(build_stats['files']['deleted'])))
logger.console('')
logger.console("Full build log at %s." % settings.LOG_FILE)
logger.console('')
with open(settings.BUILD_STATS_FILE, mode='wb') as the_file:
pickle.dump(build_stats, the_file)
settings.CACHE.close()
return build_stats