def updatePages( self ): """ Generates static pages based on content in contentPath Places these pages in webPath """ # Load paths rootDir = path( self.webPath ) contentDir = path( self.contentPath ) webDir = path( self.webPath ) # Delete files in webRoot for page in rootDir.walk(): if page.isfile(): page.remove() # Create new files from content for page in contentDir.files(): # Copy CSS files to css/ without parsing if page.ext == '.css': path.copyfile( self.contentPath + page.basename(),self.webPath + 'css/' + page.basename() ) # Compile LESS stylesheets # elif page.ext == '.less': # Compile SASS stylesheets # elif page.ext == '.sass': # Copy CSS files to css/ without parsing if page.ext == '.js': path.copyfile( self.contentPath + page.basename(),self.webPath + 'js/' + page.basename() ) # Generate HTML and/or PHP pages from content and template elif page.ext == '.html' or page.ext == '.php': pageHTML = BeautifulSoup( open( self.rootPath + 'data/template.php' ) ) # Insert content # Handle tags with placeholders for parser contentString = page.text().replace( '<?php','<php>').replace( '?>','</php>' ) contentHTML = BeautifulSoup( contentString ) # Insert extra <head> arguments for element in filter( lambda a: a != '\n', contentHTML.head.contents ): pageHTML.head.append( element ) # Replace <content> pageHTML.content.replaceWith( contentHTML.content ) # Generate Nav navHTML = BeautifulSoup( self.makeNav( page.basename() ) ) pageHTML.nav.replaceWith( navHTML ) # Create page file pageString = unicode( pageHTML ).replace( '<php>','<?php').replace( '</php>','?>' ) pageFile = open( self.webPath + page.basename(), 'wb' ) pageFile.write( pageString ) pageFile.close() self.log( "Completed" ) return
def build(args=None): """Builds an Engineer site using the settings specified in *args*.""" from engineer.conf import settings from engineer.loaders import LocalLoader from engineer.log import get_file_handler from engineer.models import PostCollection, TemplatePage from engineer.themes import ThemeManager from engineer.util import mirror_folder, ensure_exists, slugify if args and args.clean: clean() settings.create_required_directories() logger = logging.getLogger('engineer.engine.build') logger.parent.addHandler(get_file_handler(settings.LOG_FILE)) logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE) build_stats = { 'time_run': times.now(), 'counts': { 'template_pages': 0, 'new_posts': 0, 'cached_posts': 0, 'rollups': 0, 'tag_pages': 0, }, 'files': {}, } # Remove the output cache (not the post cache or the Jinja cache) # since we're rebuilding the site settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True) # Copy static content to output dir logger.debug("Copying static files to output cache.") s = (settings.ENGINEER.STATIC_DIR / 'engineer').abspath() t = settings.OUTPUT_STATIC_DIR / 'engineer' mirror_folder( s, t, recurse=False) # Copy only the files in this folder - don't recurse theme = ThemeManager.current_theme() # Copy Foundation files if used if theme.use_foundation: s = settings.ENGINEER.LIB_DIR / 'foundation' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/foundation') mirror_folder(s, t) logger.debug("Copied Foundation library files.") # Copy LESS js file if needed if theme.use_lesscss and not settings.PREPROCESS_LESS: s = settings.ENGINEER.LIB_DIR / 'less-1.3.1.min.js' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/') s.copy(t) logger.debug("Copied LESS CSS files.") # Copy jQuery files if needed if theme.use_jquery or theme.use_tweet: s = settings.ENGINEER.LIB_DIR / 'jquery-1.7.1.min.js' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/') s.copy(t) logger.debug("Copied jQuery files.") # Copy Tweet files if needed if theme.use_tweet: s = settings.ENGINEER.LIB_DIR / 'tweet' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/tweet') mirror_folder(s, t) logger.debug("Copied Tweet files.") # Copy modernizr files if needed if theme.use_modernizr: s = settings.ENGINEER.LIB_DIR / 'modernizr-2.5.3.min.js' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/') s.copy(t) logger.debug("Copied Modernizr files.") logger.debug("Copied static files to %s." % relpath(t)) # Copy 'raw' content to output cache - first pass # This first pass ensures that any static content - JS/LESS/CSS - that # is needed by site-specific pages (like template pages) is available # during the build if settings.CONTENT_DIR.exists(): mirror_folder(settings.CONTENT_DIR, settings.OUTPUT_CACHE_DIR, delete_orphans=False) # Copy theme static content to output dir logger.debug("Copying theme static files to output cache.") try: s = theme.static_root.abspath() except ThemeNotFoundException as e: logger.critical(e.message) exit() t = (settings.OUTPUT_STATIC_DIR / 'theme').abspath() mirror_folder(s, t) logger.debug("Copied static files for theme to %s." % relpath(t)) # Copy any theme additional content to output dir if needed if theme.content_mappings: logger.debug("Copying additional theme content to output cache.") for s, t in theme.content_mappings.iteritems(): t = settings.OUTPUT_STATIC_DIR / 'theme' / t if s.isdir(): mirror_folder(s, t) else: s.copy(ensure_exists(t)) logger.debug("Copied additional files for theme to %s." % relpath(t)) # Load markdown input posts logger.info("Loading posts...") new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR) all_posts = PostCollection(new_posts + cached_posts) to_publish = PostCollection(all_posts.published) if settings.PUBLISH_DRAFTS: to_publish.extend(all_posts.drafts) if settings.PUBLISH_PENDING: to_publish.extend(all_posts.pending) if settings.PUBLISH_REVIEW: to_publish.extend(all_posts.review) if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0: logger.warning("This site contains the following pending posts:") for post in all_posts.pending: logger.warning("\t'%s' - publish time: %s, %s." % (post.title, humanize.naturaltime( post.timestamp), post.timestamp_local)) logger.warning( "These posts won't be published until you build the site again after their publish time." ) all_posts = PostCollection( sorted(to_publish, reverse=True, key=lambda post: post.timestamp)) # Generate template pages if settings.TEMPLATE_PAGE_DIR.exists(): logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR) template_pages = [] for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'): # We create all the TemplatePage objects first so we have all of the URLs to them in the template # environment. Without this step, template pages might have broken links if they link to a page that is # loaded after them, since the URL to the not-yet-loaded page will be missing. template_pages.append(TemplatePage(template)) for page in template_pages: rendered_page = page.render_html(all_posts) ensure_exists(page.output_path) with open(page.output_path / page.output_file_name, mode='wb', encoding='UTF-8') as file: file.write(rendered_page) logger.debug("Output template page %s." % relpath(file.name)) build_stats['counts']['template_pages'] += 1 logger.info("Generated %s template pages." % build_stats['counts']['template_pages']) # Generate individual post pages for post in all_posts: rendered_post = post.render_html(all_posts) ensure_exists(post.output_path) with open(post.output_path, mode='wb', encoding='UTF-8') as file: file.write(rendered_post) if post in new_posts: logger.console("Output new or modified post '%s'." % post.title) build_stats['counts']['new_posts'] += 1 elif post in cached_posts: build_stats['counts']['cached_posts'] += 1 # Generate rollup pages num_posts = len(all_posts) num_slices = ( num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0\ else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1 slice_num = 0 for posts in all_posts.paginate(): slice_num += 1 has_next = slice_num < num_slices has_previous = 1 < slice_num <= num_slices rendered_page = posts.render_listpage_html(slice_num, has_next, has_previous) ensure_exists(posts.output_path(slice_num)) with open(posts.output_path(slice_num), mode='wb', encoding='UTF-8') as file: file.write(rendered_page) logger.debug("Output rollup page %s." % relpath(file.name)) build_stats['counts']['rollups'] += 1 # Copy first rollup page to root of site - it's the homepage. if slice_num == 1: path.copyfile(posts.output_path(slice_num), settings.OUTPUT_CACHE_DIR / 'index.html') logger.debug("Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html')) # Generate archive page if num_posts > 0: archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html' ensure_exists(archive_output_path) rendered_archive = all_posts.render_archive_html(all_posts) with open(archive_output_path, mode='wb', encoding='UTF-8') as file: file.write(rendered_archive) logger.debug("Output %s." % relpath(file.name)) # Generate tag pages if num_posts > 0: tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag' for tag in all_posts.all_tags: rendered_tag_page = all_posts.render_tag_html(tag, all_posts) tag_path = ensure_exists(tags_output_path / slugify(tag) / 'index.html') with open(tag_path, mode='wb', encoding='UTF-8') as file: file.write(rendered_tag_page) build_stats['counts']['tag_pages'] += 1 logger.debug("Output %s." % relpath(file.name)) # Generate feeds feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml') feed_content = settings.JINJA_ENV.get_or_select_template([ 'rss.xml', 'theme/rss.xml', 'core/rss.xml' ]).render(post_list=all_posts[:settings.FEED_ITEM_LIMIT], build_date=all_posts[0].timestamp) with open(feed_output_path, mode='wb', encoding='UTF-8') as file: file.write(feed_content) logger.debug("Output %s." % relpath(file.name)) # Generate sitemap sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'sitemap.xml.gz') sitemap_content = settings.JINJA_ENV.get_or_select_template( ['sitemap.xml', 'theme/sitemap.xml', 'core/sitemap.xml']).render(post_list=all_posts) with gzip.open(sitemap_output_path, mode='wb') as file: file.write(sitemap_content) logger.debug("Output %s." % relpath(file.name)) # Copy 'raw' content to output cache - second/final pass if settings.CONTENT_DIR.exists(): mirror_folder(settings.CONTENT_DIR, settings.OUTPUT_CACHE_DIR, delete_orphans=False) # Compress all files marked for compression for file, compression_type in settings.COMPRESS_FILE_LIST: if file not in settings.COMPRESSION_CACHE: with open(file, mode='rb') as input: output = compress(input.read(), compression_type) logger.debug("Compressed %s." % relpath(file)) settings.COMPRESSION_CACHE[file] = output else: logger.debug("Found pre-compressed file in cache: %s." % relpath(file)) output = settings.COMPRESSION_CACHE[file] with open(file, mode='wb') as f: f.write(output) # Remove LESS files if LESS preprocessing is being done if settings.PREPROCESS_LESS: logger.debug("Deleting LESS files since PREPROCESS_LESS is True.") for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"): logger.debug("Deleting file: %s." % relpath(f)) f.remove_p() logger.debug("Synchronizing output directory with output cache.") build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR, settings.OUTPUT_DIR) from pprint import pformat logger.debug("Folder mirroring report: %s" % pformat(build_stats['files'])) logger.console('') logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR)) logger.console("Posts: %s (%s new or updated)" % ((build_stats['counts']['new_posts'] + build_stats['counts']['cached_posts']), build_stats['counts']['new_posts'])) logger.console( "Post rollup pages: %s (%s posts per page)" % (build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE)) logger.console("Template pages: %s" % build_stats['counts']['template_pages']) logger.console("Tag pages: %s" % build_stats['counts']['tag_pages']) logger.console("%s new items, %s modified items, and %s deleted items." % (len(build_stats['files']['new']), len(build_stats['files']['overwritten']), len(build_stats['files']['deleted']))) logger.console('') logger.console("Full build log at %s." % settings.LOG_FILE) with open(settings.BUILD_STATS_FILE, mode='wb') as file: pickle.dump(build_stats, file) settings.CACHE.close() return build_stats
def build(args=None): """Builds an Engineer site using the settings specified in *args*.""" from engineer.conf import settings from engineer.loaders import LocalLoader from engineer.log import get_file_handler from engineer.models import PostCollection, TemplatePage from engineer.themes import ThemeManager from engineer.util import mirror_folder, ensure_exists, slugify if args and args.clean: clean() settings.create_required_directories() logger = logging.getLogger('engineer.engine.build') logger.parent.addHandler(get_file_handler(settings.LOG_FILE)) logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE) build_stats = { 'time_run': times.now(), 'counts': { 'template_pages': 0, 'new_posts': 0, 'cached_posts': 0, 'rollups': 0, 'tag_pages': 0, }, 'files': {}, } # Remove the output cache (not the post cache or the Jinja cache) # since we're rebuilding the site settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True) # Copy static content to output dir logger.debug("Copying static files to output cache.") s = (settings.ENGINEER.STATIC_DIR / 'engineer').abspath() t = settings.OUTPUT_STATIC_DIR / 'engineer' mirror_folder(s, t, recurse=False) # Copy only the files in this folder - don't recurse theme = ThemeManager.current_theme() # Copy Foundation files if used if theme.use_foundation: s = settings.ENGINEER.LIB_DIR / 'foundation' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/foundation') mirror_folder(s, t) logger.debug("Copied Foundation library files.") # Copy LESS js file if needed if theme.use_lesscss and not settings.PREPROCESS_LESS: s = settings.ENGINEER.LIB_DIR / 'less-1.3.1.min.js' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/') s.copy(t) logger.debug("Copied LESS CSS files.") # Copy jQuery files if needed if theme.use_jquery or theme.use_tweet: s = settings.ENGINEER.LIB_DIR / 'jquery-1.7.1.min.js' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/') s.copy(t) logger.debug("Copied jQuery files.") # Copy Tweet files if needed if theme.use_tweet: s = settings.ENGINEER.LIB_DIR / 'tweet' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/tweet') mirror_folder(s, t) logger.debug("Copied Tweet files.") # Copy modernizr files if needed if theme.use_modernizr: s = settings.ENGINEER.LIB_DIR / 'modernizr-2.5.3.min.js' t = ensure_exists(settings.OUTPUT_STATIC_DIR / 'engineer/lib/') s.copy(t) logger.debug("Copied Modernizr files.") logger.debug("Copied static files to %s." % relpath(t)) # Copy 'raw' content to output cache - first pass # This first pass ensures that any static content - JS/LESS/CSS - that # is needed by site-specific pages (like template pages) is available # during the build if settings.CONTENT_DIR.exists(): mirror_folder(settings.CONTENT_DIR, settings.OUTPUT_CACHE_DIR, delete_orphans=False) # Copy theme static content to output dir logger.debug("Copying theme static files to output cache.") try: s = theme.static_root.abspath() except ThemeNotFoundException as e: logger.critical(e.message) exit() t = (settings.OUTPUT_STATIC_DIR / 'theme').abspath() mirror_folder(s, t) logger.debug("Copied static files for theme to %s." % relpath(t)) # Copy any theme additional content to output dir if needed if theme.content_mappings: logger.debug("Copying additional theme content to output cache.") for s, t in theme.content_mappings.iteritems(): t = settings.OUTPUT_STATIC_DIR / 'theme' / t if s.isdir(): mirror_folder(s, t) else: s.copy(ensure_exists(t)) logger.debug("Copied additional files for theme to %s." % relpath(t)) # Load markdown input posts logger.info("Loading posts...") new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR) all_posts = PostCollection(new_posts + cached_posts) to_publish = PostCollection(all_posts.published) if settings.PUBLISH_DRAFTS: to_publish.extend(all_posts.drafts) if settings.PUBLISH_PENDING: to_publish.extend(all_posts.pending) if settings.PUBLISH_REVIEW: to_publish.extend(all_posts.review) if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0: logger.warning("This site contains the following pending posts:") for post in all_posts.pending: logger.warning("\t'%s' - publish time: %s, %s." % (post.title, naturaltime(post.timestamp), post.timestamp_local)) logger.warning("These posts won't be published until you build the site again after their publish time.") all_posts = PostCollection( sorted(to_publish, reverse=True, key=lambda post: post.timestamp)) # Generate template pages if settings.TEMPLATE_PAGE_DIR.exists(): logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR) template_pages = [] for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'): # We create all the TemplatePage objects first so we have all of the URLs to them in the template # environment. Without this step, template pages might have broken links if they link to a page that is # loaded after them, since the URL to the not-yet-loaded page will be missing. template_pages.append(TemplatePage(template)) for page in template_pages: rendered_page = page.render_html(all_posts) ensure_exists(page.output_path) with open(page.output_path / page.output_file_name, mode='wb', encoding='UTF-8') as file: file.write(rendered_page) logger.debug("Output template page %s." % relpath(file.name)) build_stats['counts']['template_pages'] += 1 logger.info("Generated %s template pages." % build_stats['counts']['template_pages']) # Generate individual post pages for post in all_posts: rendered_post = post.render_html(all_posts) ensure_exists(post.output_path) with open(post.output_path, mode='wb', encoding='UTF-8') as file: file.write(rendered_post) if post in new_posts: logger.console("Output new or modified post '%s'." % post.title) build_stats['counts']['new_posts'] += 1 elif post in cached_posts: build_stats['counts']['cached_posts'] += 1 # Generate rollup pages num_posts = len(all_posts) num_slices = ( num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0\ else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1 slice_num = 0 for posts in all_posts.paginate(): slice_num += 1 has_next = slice_num < num_slices has_previous = 1 < slice_num <= num_slices rendered_page = posts.render_listpage_html(slice_num, has_next, has_previous) ensure_exists(posts.output_path(slice_num)) with open(posts.output_path(slice_num), mode='wb', encoding='UTF-8') as file: file.write(rendered_page) logger.debug("Output rollup page %s." % relpath(file.name)) build_stats['counts']['rollups'] += 1 # Copy first rollup page to root of site - it's the homepage. if slice_num == 1: path.copyfile(posts.output_path(slice_num), settings.OUTPUT_CACHE_DIR / 'index.html') logger.debug( "Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html')) # Generate archive page if num_posts > 0: archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html' ensure_exists(archive_output_path) rendered_archive = all_posts.render_archive_html(all_posts) with open(archive_output_path, mode='wb', encoding='UTF-8') as file: file.write(rendered_archive) logger.debug("Output %s." % relpath(file.name)) # Generate tag pages if num_posts > 0: tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag' for tag in all_posts.all_tags: rendered_tag_page = all_posts.render_tag_html(tag, all_posts) tag_path = ensure_exists( tags_output_path / slugify(tag) / 'index.html') with open(tag_path, mode='wb', encoding='UTF-8') as file: file.write(rendered_tag_page) build_stats['counts']['tag_pages'] += 1 logger.debug("Output %s." % relpath(file.name)) # Generate feeds feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml') feed_content = settings.JINJA_ENV.get_or_select_template(['rss.xml', 'theme/rss.xml', 'core/rss.xml']).render( post_list=all_posts[:settings.FEED_ITEM_LIMIT], build_date=all_posts[0].timestamp) with open(feed_output_path, mode='wb', encoding='UTF-8') as file: file.write(feed_content) logger.debug("Output %s." % relpath(file.name)) # Generate sitemap sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'sitemap.xml.gz') sitemap_content = settings.JINJA_ENV.get_or_select_template(['sitemap.xml', 'theme/sitemap.xml', 'core/sitemap.xml']).render(post_list=all_posts) with gzip.open(sitemap_output_path, mode='wb') as file: file.write(sitemap_content) logger.debug("Output %s." % relpath(file.name)) # Copy 'raw' content to output cache - second/final pass if settings.CONTENT_DIR.exists(): mirror_folder(settings.CONTENT_DIR, settings.OUTPUT_CACHE_DIR, delete_orphans=False) # Compress all files marked for compression for file, compression_type in settings.COMPRESS_FILE_LIST: if file not in settings.COMPRESSION_CACHE: with open(file, mode='rb') as input: output = compress(input.read(), compression_type) logger.debug("Compressed %s." % relpath(file)) settings.COMPRESSION_CACHE[file] = output else: logger.debug("Found pre-compressed file in cache: %s." % relpath(file)) output = settings.COMPRESSION_CACHE[file] with open(file, mode='wb') as f: f.write(output) # Remove LESS files if LESS preprocessing is being done if settings.PREPROCESS_LESS: logger.debug("Deleting LESS files since PREPROCESS_LESS is True.") for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"): logger.debug("Deleting file: %s." % relpath(f)) f.remove_p() logger.debug("Synchronizing output directory with output cache.") build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR, settings.OUTPUT_DIR) from pprint import pformat logger.debug("Folder mirroring report: %s" % pformat(build_stats['files'])) logger.console('') logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR)) logger.console("Posts: %s (%s new or updated)" % ( (build_stats['counts']['new_posts'] + build_stats['counts'][ 'cached_posts']), build_stats['counts']['new_posts'])) logger.console("Post rollup pages: %s (%s posts per page)" % ( build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE)) logger.console("Template pages: %s" % build_stats['counts']['template_pages']) logger.console("Tag pages: %s" % build_stats['counts']['tag_pages']) logger.console("%s new items, %s modified items, and %s deleted items." % ( len(build_stats['files']['new']), len(build_stats['files']['overwritten']), len(build_stats['files']['deleted']))) logger.console('') logger.console("Full build log at %s." % settings.LOG_FILE) with open(settings.BUILD_STATS_FILE, mode='wb') as file: pickle.dump(build_stats, file) settings.CACHE.close() return build_stats
def build(args=None): """Builds an Engineer site using the settings specified in *args*.""" from engineer.conf import settings from engineer.loaders import LocalLoader from engineer.log import get_file_handler from engineer.models import PostCollection, TemplatePage from engineer.themes import ThemeManager from engineer.util import mirror_folder, ensure_exists, slugify if args and args.clean: clean() settings.create_required_directories() logger = logging.getLogger('engineer.engine.build') logger.parent.addHandler(get_file_handler(settings.LOG_FILE)) logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE) build_stats = { 'time_run': times.now(), 'counts': { 'template_pages': 0, 'new_posts': 0, 'cached_posts': 0, 'rollups': 0, 'tag_pages': 0, }, 'files': {}, } # Remove the output cache (not the post cache or the Jinja cache) # since we're rebuilding the site settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True) theme = ThemeManager.current_theme() engineer_lib = (settings.OUTPUT_STATIC_DIR / 'engineer/lib/').abspath() ensure_exists(engineer_lib) # Copy Foundation files if used if theme.use_foundation: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.FOUNDATION_CSS t = ensure_exists(engineer_lib / settings.ENGINEER.FOUNDATION_CSS) mirror_folder(s, t) logger.debug("Copied Foundation library files.") # Copy LESS js file if needed if theme.use_lesscss and not settings.PREPROCESS_LESS: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.LESS_JS s.copy(engineer_lib) logger.debug("Copied LESS CSS files.") # Copy jQuery files if needed if theme.use_jquery: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.JQUERY s.copy(engineer_lib) logger.debug("Copied jQuery files.") # Copy modernizr files if needed if theme.use_modernizr: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.MODERNIZR s.copy(engineer_lib) logger.debug("Copied Modernizr files.") # Copy normalize.css if needed if theme.use_normalize_css: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.NORMALIZE_CSS s.copy(engineer_lib) logger.debug("Copied normalize.css.") # Copy 'raw' content to output cache - first pass # This first pass ensures that any static content - JS/LESS/CSS - that # is needed by site-specific pages (like template pages) is available # during the build if settings.CONTENT_DIR.exists(): mirror_folder(settings.CONTENT_DIR, settings.OUTPUT_CACHE_DIR, delete_orphans=False) # Copy theme static content to output dir theme_output_dir = settings.OUTPUT_STATIC_DIR / 'theme' logger.debug("Copying theme static files to output cache.") theme.copy_content(theme_output_dir) logger.debug("Copied static files for theme to %s." % relpath(theme_output_dir)) # Copy any theme additional content to output dir if needed if theme.content_mappings: logger.debug("Copying additional theme content to output cache.") theme.copy_related_content(theme_output_dir) logger.debug("Copied additional files for theme to %s." % relpath(theme_output_dir)) # Load markdown input posts logger.info("Loading posts...") new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR) all_posts = PostCollection(new_posts + cached_posts) to_publish = PostCollection(all_posts.published) if settings.PUBLISH_DRAFTS: to_publish.extend(all_posts.drafts) if settings.PUBLISH_PENDING: to_publish.extend(all_posts.pending) if settings.PUBLISH_REVIEW: to_publish.extend(all_posts.review) if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0: logger.warning("This site contains the following pending posts:") for post in all_posts.pending: logger.warning("\t'%s' - publish time: %s, %s." % (post.title, naturaltime( post.timestamp), post.timestamp_local)) logger.warning( "These posts won't be published until you build the site again after their publish time." ) all_posts = PostCollection( sorted(to_publish, reverse=True, key=lambda p: p.timestamp)) # Generate template pages if settings.TEMPLATE_PAGE_DIR.exists(): logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR) template_pages = [] for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'): # We create all the TemplatePage objects first so we have all of the URLs to them in the template # environment. Without this step, template pages might have broken links if they link to a page that is # loaded after them, since the URL to the not-yet-loaded page will be missing. template_pages.append(TemplatePage(template)) for page in template_pages: rendered_page = page.render_html(all_posts) ensure_exists(page.output_path) with open(page.output_path / page.output_file_name, mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_page) logger.info("Output template page %s." % relpath(the_file.name)) build_stats['counts']['template_pages'] += 1 logger.info("Generated %s template pages." % build_stats['counts']['template_pages']) # Generate individual post pages for post in all_posts: rendered_post = post.render_html(all_posts) ensure_exists(post.output_path) with open(post.output_path, mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_post) if post in new_posts: logger.console("Output new or modified post '%s'." % post.title) build_stats['counts']['new_posts'] += 1 elif post in cached_posts: build_stats['counts']['cached_posts'] += 1 # Generate rollup pages num_posts = len(all_posts) num_slices = ( num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0 \ else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1 slice_num = 0 for posts in all_posts.paginate(): slice_num += 1 has_next = slice_num < num_slices has_previous = 1 < slice_num <= num_slices rendered_page = posts.render_listpage_html(slice_num, has_next, has_previous) ensure_exists(posts.output_path(slice_num)) with open(posts.output_path(slice_num), mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_page) logger.debug("Output rollup page %s." % relpath(the_file.name)) build_stats['counts']['rollups'] += 1 # Copy first rollup page to root of site - it's the homepage. if slice_num == 1: path.copyfile(posts.output_path(slice_num), settings.OUTPUT_CACHE_DIR / 'index.html') logger.debug("Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html')) # Generate archive page if num_posts > 0: archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html' ensure_exists(archive_output_path) rendered_archive = all_posts.render_archive_html(all_posts) with open(archive_output_path, mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_archive) logger.debug("Output %s." % relpath(the_file.name)) # Generate tag pages if num_posts > 0: tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag' for tag in all_posts.all_tags: rendered_tag_page = all_posts.render_tag_html(tag, all_posts) tag_path = ensure_exists(tags_output_path / slugify(tag) / 'index.html') with open(tag_path, mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_tag_page) build_stats['counts']['tag_pages'] += 1 logger.debug("Output %s." % relpath(the_file.name)) # Generate feeds rss_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml') atom_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/atom.xml') rss_feed = Rss201rev2Feed(title=settings.FEED_TITLE, link=settings.SITE_URL, description=settings.FEED_DESCRIPTION, feed_url=settings.FEED_URL) atom_feed = Atom1Feed(title=settings.FEED_TITLE, link=settings.SITE_URL, description=settings.FEED_DESCRIPTION, feed_url=settings.FEED_URL) for feed in (rss_feed, atom_feed): for post in all_posts[:settings.FEED_ITEM_LIMIT]: title = settings.JINJA_ENV.get_template( 'core/feeds/title.jinja2').render(post=post) link = settings.JINJA_ENV.get_template( 'core/feeds/link.jinja2').render(post=post) content = settings.JINJA_ENV.get_template( 'core/feeds/content.jinja2').render(post=post) feed.add_item(title=title, link=link, description=content, pubdate=post.timestamp, unique_id=post.absolute_url) with open(rss_feed_output_path, mode='wb') as the_file: rss_feed.write(the_file, 'UTF-8') logger.debug("Output %s." % relpath(the_file.name)) with open(atom_feed_output_path, mode='wb') as the_file: atom_feed.write(the_file, 'UTF-8') logger.debug("Output %s." % relpath(the_file.name)) # Generate sitemap sitemap_file_name = 'sitemap.xml.gz' sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / sitemap_file_name) sitemap_content = settings.JINJA_ENV.get_or_select_template( ['sitemap.xml', 'theme/sitemap.xml', 'core/sitemap.xml']).render(post_list=all_posts) with gzip.open(sitemap_output_path, mode='wb') as the_file: the_file.write(sitemap_content) logger.debug("Output %s." % relpath(the_file.name)) # Copy 'raw' content to output cache - second/final pass if settings.CONTENT_DIR.exists(): mirror_folder(settings.CONTENT_DIR, settings.OUTPUT_CACHE_DIR, delete_orphans=False) # Compress all files marked for compression for the_file, compression_type in settings.COMPRESS_FILE_LIST: if the_file not in settings.COMPRESSION_CACHE: with open(the_file, mode='rb') as input: output = compress(input.read(), compression_type) logger.debug("Compressed %s." % relpath(the_file)) settings.COMPRESSION_CACHE[the_file] = output else: logger.debug("Found pre-compressed file in cache: %s." % relpath(the_file)) output = settings.COMPRESSION_CACHE[the_file] with open(the_file, mode='wb') as f: f.write(output) # Remove LESS files if LESS preprocessing is being done if settings.PREPROCESS_LESS: logger.debug("Deleting LESS files since PREPROCESS_LESS is True.") for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"): logger.debug("Deleting file: %s." % relpath(f)) f.remove_p() # Check if anything has changed other than the sitemap have_changes = False compare = filecmp.dircmp(settings.OUTPUT_CACHE_DIR, settings.OUTPUT_DIR, ignore=settings.OUTPUT_DIR_IGNORE) # The algorithm below takes advantage of the fact that once we've determined that there is more than one file # that's different, or if the first item returned by the generator is not the sitemap, then we can break out of # the generator loop early. This is also advantageous because it doesn't require us to completely exhaust the # generator. In the case of a fresh site build, for example, the generator will return a lot more data. So the # other approach here of expanding the generator into a list with a list comprehension would be inefficient # in many cases. This approach performs equally well in all cases at the cost of some unusual-looking code. diff_file_count = 0 if not has_files(settings.OUTPUT_DIR): have_changes = True else: for file_path in diff_dir(compare): diff_file_count += 1 if file_path != sitemap_output_path: have_changes = True break if diff_file_count > 1: have_changes = True break if not have_changes: logger.console('') logger.console("No site changes to publish.") else: logger.debug("Synchronizing output directory with output cache.") build_stats['files'] = mirror_folder( settings.OUTPUT_CACHE_DIR, settings.OUTPUT_DIR, ignore_list=settings.OUTPUT_DIR_IGNORE) from pprint import pformat logger.debug("Folder mirroring report: %s" % pformat(build_stats['files'])) logger.console('') logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR)) logger.console("Posts: %s (%s new or updated)" % ((build_stats['counts']['new_posts'] + build_stats['counts']['cached_posts']), build_stats['counts']['new_posts'])) logger.console( "Post rollup pages: %s (%s posts per page)" % (build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE)) logger.console("Template pages: %s" % build_stats['counts']['template_pages']) logger.console("Tag pages: %s" % build_stats['counts']['tag_pages']) logger.console( "%s new items, %s modified items, and %s deleted items." % (len(build_stats['files']['new']), len(build_stats['files']['overwritten']), len(build_stats['files']['deleted']))) logger.console('') logger.console("Full build log at %s." % settings.LOG_FILE) logger.console('') with open(settings.BUILD_STATS_FILE, mode='wb') as the_file: pickle.dump(build_stats, the_file) settings.CACHE.close() return build_stats
def _copy_file(self, src, dest): from path import path test_dir = dest.parent test_dir.makedirs_p() path.copyfile(src, dest)
def build(args=None): """Builds an Engineer site using the settings specified in *args*.""" from engineer.conf import settings from engineer.loaders import LocalLoader from engineer.log import get_file_handler from engineer.models import PostCollection, TemplatePage from engineer.themes import ThemeManager from engineer.util import mirror_folder, ensure_exists, slugify if args and args.clean: clean() settings.create_required_directories() logger = logging.getLogger('engineer.engine.build') logger.parent.addHandler(get_file_handler(settings.LOG_FILE)) logger.debug("Starting build using configuration file %s." % settings.SETTINGS_FILE) build_stats = { 'time_run': times.now(), 'counts': { 'template_pages': 0, 'new_posts': 0, 'cached_posts': 0, 'rollups': 0, 'tag_pages': 0, }, 'files': {}, } # Remove the output cache (not the post cache or the Jinja cache) # since we're rebuilding the site settings.OUTPUT_CACHE_DIR.rmtree(ignore_errors=True) theme = ThemeManager.current_theme() engineer_lib = (settings.OUTPUT_STATIC_DIR / 'engineer/lib/').abspath() ensure_exists(engineer_lib) # Copy Foundation files if used if theme.use_foundation: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.FOUNDATION_CSS t = ensure_exists(engineer_lib / settings.ENGINEER.FOUNDATION_CSS) mirror_folder(s, t) logger.debug("Copied Foundation library files.") # Copy LESS js file if needed if theme.use_lesscss and not settings.PREPROCESS_LESS: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.LESS_JS s.copy(engineer_lib) logger.debug("Copied LESS CSS files.") # Copy jQuery files if needed if theme.use_jquery: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.JQUERY s.copy(engineer_lib) logger.debug("Copied jQuery files.") # Copy modernizr files if needed if theme.use_modernizr: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.MODERNIZR s.copy(engineer_lib) logger.debug("Copied Modernizr files.") # Copy normalize.css if needed if theme.use_normalize_css: s = settings.ENGINEER.LIB_DIR / settings.ENGINEER.NORMALIZE_CSS s.copy(engineer_lib) logger.debug("Copied normalize.css.") # Copy 'raw' content to output cache - first pass # This first pass ensures that any static content - JS/LESS/CSS - that # is needed by site-specific pages (like template pages) is available # during the build if settings.CONTENT_DIR.exists(): mirror_folder(settings.CONTENT_DIR, settings.OUTPUT_CACHE_DIR, delete_orphans=False) # Copy theme static content to output dir theme_output_dir = settings.OUTPUT_STATIC_DIR / 'theme' logger.debug("Copying theme static files to output cache.") theme.copy_content(theme_output_dir) logger.debug("Copied static files for theme to %s." % relpath(theme_output_dir)) # Copy any theme additional content to output dir if needed if theme.content_mappings: logger.debug("Copying additional theme content to output cache.") theme.copy_related_content(theme_output_dir) logger.debug("Copied additional files for theme to %s." % relpath(theme_output_dir)) # Load markdown input posts logger.info("Loading posts...") new_posts, cached_posts = LocalLoader.load_all(input=settings.POST_DIR) all_posts = PostCollection(new_posts + cached_posts) to_publish = PostCollection(all_posts.published) if settings.PUBLISH_DRAFTS: to_publish.extend(all_posts.drafts) if settings.PUBLISH_PENDING: to_publish.extend(all_posts.pending) if settings.PUBLISH_REVIEW: to_publish.extend(all_posts.review) if not settings.PUBLISH_PENDING and len(all_posts.pending) > 0: logger.warning("This site contains the following pending posts:") for post in all_posts.pending: logger.warning("\t'%s' - publish time: %s, %s." % (post.title, naturaltime(post.timestamp), post.timestamp_local)) logger.warning("These posts won't be published until you build the site again after their publish time.") all_posts = PostCollection( sorted(to_publish, reverse=True, key=lambda p: p.timestamp)) # Generate template pages if settings.TEMPLATE_PAGE_DIR.exists(): logger.info("Generating template pages from %s." % settings.TEMPLATE_PAGE_DIR) template_pages = [] for template in settings.TEMPLATE_PAGE_DIR.walkfiles('*.html'): # We create all the TemplatePage objects first so we have all of the URLs to them in the template # environment. Without this step, template pages might have broken links if they link to a page that is # loaded after them, since the URL to the not-yet-loaded page will be missing. template_pages.append(TemplatePage(template)) for page in template_pages: rendered_page = page.render_html(all_posts) ensure_exists(page.output_path) with open(page.output_path / page.output_file_name, mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_page) logger.info("Output template page %s." % relpath(the_file.name)) build_stats['counts']['template_pages'] += 1 logger.info("Generated %s template pages." % build_stats['counts']['template_pages']) # Generate individual post pages for post in all_posts: rendered_post = post.render_html(all_posts) ensure_exists(post.output_path) with open(post.output_path, mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_post) if post in new_posts: logger.console("Output new or modified post '%s'." % post.title) build_stats['counts']['new_posts'] += 1 elif post in cached_posts: build_stats['counts']['cached_posts'] += 1 # Generate rollup pages num_posts = len(all_posts) num_slices = ( num_posts / settings.ROLLUP_PAGE_SIZE) if num_posts % settings.ROLLUP_PAGE_SIZE == 0 \ else (num_posts / settings.ROLLUP_PAGE_SIZE) + 1 slice_num = 0 for posts in all_posts.paginate(): slice_num += 1 has_next = slice_num < num_slices has_previous = 1 < slice_num <= num_slices rendered_page = posts.render_listpage_html(slice_num, has_next, has_previous) ensure_exists(posts.output_path(slice_num)) with open(posts.output_path(slice_num), mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_page) logger.debug("Output rollup page %s." % relpath(the_file.name)) build_stats['counts']['rollups'] += 1 # Copy first rollup page to root of site - it's the homepage. if slice_num == 1: path.copyfile(posts.output_path(slice_num), settings.OUTPUT_CACHE_DIR / 'index.html') logger.debug( "Output '%s'." % (settings.OUTPUT_CACHE_DIR / 'index.html')) # Generate archive page if num_posts > 0: archive_output_path = settings.OUTPUT_CACHE_DIR / 'archives/index.html' ensure_exists(archive_output_path) rendered_archive = all_posts.render_archive_html(all_posts) with open(archive_output_path, mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_archive) logger.debug("Output %s." % relpath(the_file.name)) # Generate tag pages if num_posts > 0: tags_output_path = settings.OUTPUT_CACHE_DIR / 'tag' for tag in all_posts.all_tags: rendered_tag_page = all_posts.render_tag_html(tag, all_posts) tag_path = ensure_exists( tags_output_path / slugify(tag) / 'index.html') with open(tag_path, mode='wb', encoding='UTF-8') as the_file: the_file.write(rendered_tag_page) build_stats['counts']['tag_pages'] += 1 logger.debug("Output %s." % relpath(the_file.name)) # Generate feeds rss_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/rss.xml') atom_feed_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / 'feeds/atom.xml') rss_feed = Rss201rev2Feed( title=settings.FEED_TITLE, link=settings.SITE_URL, description=settings.FEED_DESCRIPTION, feed_url=settings.FEED_URL ) atom_feed = Atom1Feed( title=settings.FEED_TITLE, link=settings.SITE_URL, description=settings.FEED_DESCRIPTION, feed_url=settings.FEED_URL ) for feed in (rss_feed, atom_feed): for post in all_posts[:settings.FEED_ITEM_LIMIT]: title = settings.JINJA_ENV.get_template('core/feeds/title.jinja2').render(post=post) link = settings.JINJA_ENV.get_template('core/feeds/link.jinja2').render(post=post) content = settings.JINJA_ENV.get_template('core/feeds/content.jinja2').render(post=post) feed.add_item( title=title, link=link, description=content, pubdate=post.timestamp, unique_id=post.absolute_url) with open(rss_feed_output_path, mode='wb') as the_file: rss_feed.write(the_file, 'UTF-8') logger.debug("Output %s." % relpath(the_file.name)) with open(atom_feed_output_path, mode='wb') as the_file: atom_feed.write(the_file, 'UTF-8') logger.debug("Output %s." % relpath(the_file.name)) # Generate sitemap sitemap_file_name = 'sitemap.xml.gz' sitemap_output_path = ensure_exists(settings.OUTPUT_CACHE_DIR / sitemap_file_name) sitemap_content = settings.JINJA_ENV.get_or_select_template(['sitemap.xml', 'theme/sitemap.xml', 'core/sitemap.xml']).render(post_list=all_posts) with gzip.open(sitemap_output_path, mode='wb') as the_file: the_file.write(sitemap_content) logger.debug("Output %s." % relpath(the_file.name)) # Copy 'raw' content to output cache - second/final pass if settings.CONTENT_DIR.exists(): mirror_folder(settings.CONTENT_DIR, settings.OUTPUT_CACHE_DIR, delete_orphans=False) # Compress all files marked for compression for the_file, compression_type in settings.COMPRESS_FILE_LIST: if the_file not in settings.COMPRESSION_CACHE: with open(the_file, mode='rb') as input: output = compress(input.read(), compression_type) logger.debug("Compressed %s." % relpath(the_file)) settings.COMPRESSION_CACHE[the_file] = output else: logger.debug("Found pre-compressed file in cache: %s." % relpath(the_file)) output = settings.COMPRESSION_CACHE[the_file] with open(the_file, mode='wb') as f: f.write(output) # Remove LESS files if LESS preprocessing is being done if settings.PREPROCESS_LESS: logger.debug("Deleting LESS files since PREPROCESS_LESS is True.") for f in settings.OUTPUT_STATIC_DIR.walkfiles(pattern="*.less"): logger.debug("Deleting file: %s." % relpath(f)) f.remove_p() # Check if anything has changed other than the sitemap have_changes = False compare = filecmp.dircmp(settings.OUTPUT_CACHE_DIR, settings.OUTPUT_DIR, ignore=settings.OUTPUT_DIR_IGNORE) # The algorithm below takes advantage of the fact that once we've determined that there is more than one file # that's different, or if the first item returned by the generator is not the sitemap, then we can break out of # the generator loop early. This is also advantageous because it doesn't require us to completely exhaust the # generator. In the case of a fresh site build, for example, the generator will return a lot more data. So the # other approach here of expanding the generator into a list with a list comprehension would be inefficient # in many cases. This approach performs equally well in all cases at the cost of some unusual-looking code. diff_file_count = 0 if not has_files(settings.OUTPUT_DIR): have_changes = True else: for file_path in diff_dir(compare): diff_file_count += 1 if file_path != sitemap_output_path: have_changes = True break if diff_file_count > 1: have_changes = True break if not have_changes: logger.console('') logger.console("No site changes to publish.") else: logger.debug("Synchronizing output directory with output cache.") build_stats['files'] = mirror_folder(settings.OUTPUT_CACHE_DIR, settings.OUTPUT_DIR, ignore_list=settings.OUTPUT_DIR_IGNORE) from pprint import pformat logger.debug("Folder mirroring report: %s" % pformat(build_stats['files'])) logger.console('') logger.console("Site: '%s' output to %s." % (settings.SITE_TITLE, settings.OUTPUT_DIR)) logger.console("Posts: %s (%s new or updated)" % ( (build_stats['counts']['new_posts'] + build_stats['counts']['cached_posts']), build_stats['counts']['new_posts'])) logger.console("Post rollup pages: %s (%s posts per page)" % ( build_stats['counts']['rollups'], settings.ROLLUP_PAGE_SIZE)) logger.console("Template pages: %s" % build_stats['counts']['template_pages']) logger.console("Tag pages: %s" % build_stats['counts']['tag_pages']) logger.console("%s new items, %s modified items, and %s deleted items." % ( len(build_stats['files']['new']), len(build_stats['files']['overwritten']), len(build_stats['files']['deleted']))) logger.console('') logger.console("Full build log at %s." % settings.LOG_FILE) logger.console('') with open(settings.BUILD_STATS_FILE, mode='wb') as the_file: pickle.dump(build_stats, the_file) settings.CACHE.close() return build_stats