def run(self, ctx): out_dir = (ctx.args.output or os.path.join(ctx.app.root_dir, '_counter')) success = True ctx.timers = {} start_time = time.perf_counter() try: # Bake the site sources. if not ctx.args.assets_only: success = success & self._bakeSources(ctx, out_dir) # Bake the assets. if not ctx.args.html_only: success = success & self._bakeAssets(ctx, out_dir) # Show merged timers. if ctx.args.show_timers: logger.info("-------------------") logger.info("Timing information:") _show_timers(ctx.timers) # All done. logger.info('-------------------------') logger.info(format_timed(start_time, 'done baking')) return 0 if success else 1 except Exception as ex: if ctx.app.debug: logger.exception(ex) else: logger.error(str(ex)) return 1
def _unsafeRun(self, job): start_time = time.clock() entry = job.record_entry try: self._page_baker.bake(job.factory, job.route, entry) except BakingError as ex: logger.debug("Got baking error. Adding it to the record.") while ex: entry.errors.append(str(ex)) ex = ex.__cause__ has_error = False for e in entry.getAllErrors(): has_error = True logger.error(e) if has_error: return False if entry.was_any_sub_baked: first_sub = entry.subs[0] friendly_uri = first_sub.out_uri if friendly_uri == '': friendly_uri = '[main page]' friendly_count = '' if entry.num_subs > 1: friendly_count = ' (%d pages)' % entry.num_subs logger.info(format_timed( start_time, '[%d] %s%s' % (self.wid, friendly_uri, friendly_count))) return True
def run(self, ctx): out_dir = (ctx.args.output or os.path.join(ctx.app.root_dir, '_counter')) success = True ctx.stats = {} start_time = time.perf_counter() try: # Bake the site sources. if not ctx.args.assets_only: success = success & self._bakeSources(ctx, out_dir) # Bake the assets. if not ctx.args.html_only: success = success & self._bakeAssets(ctx, out_dir) # Show merged stats. if ctx.args.show_stats: logger.info("-------------------") logger.info("Timing information:") _show_stats(ctx.stats) # All done. logger.info('-------------------------') logger.info(format_timed(start_time, 'done baking')) return 0 if success else 1 except Exception as ex: if ctx.app.debug: logger.exception(ex) else: logger.error(str(ex)) return 1
def _handleCacheValidity(self, record): start_time = time.perf_counter() reason = None if self.force: reason = "ordered to" elif not self.app.config.get('__cache_valid'): # The configuration file was changed, or we're running a new # version of the app. reason = "not valid anymore" elif (not record.previous.bake_time or not record.previous.hasLatestVersion()): # We have no valid previous bake record. reason = "need bake record regeneration" else: # Check if any template has changed since the last bake. Since # there could be some advanced conditional logic going on, we'd # better just force a bake from scratch if that's the case. max_time = 0 for d in self.app.templates_dirs: for dpath, _, filenames in os.walk(d): for fn in filenames: full_fn = os.path.join(dpath, fn) max_time = max(max_time, os.path.getmtime(full_fn)) if max_time >= record.previous.bake_time: reason = "templates modified" if reason is not None: # We have to bake everything from scratch. self.app.cache.clearCaches(except_names=['app']) self.force = True record.incremental_count = 0 record.clearPrevious() logger.info( format_timed(start_time, "cleaned cache (reason: %s)" % reason)) return False else: record.incremental_count += 1 logger.debug( format_timed(start_time, "cache is assumed valid", colored=False)) return True
def _bakeTaxonomies(self, record, pool): logger.debug("Baking taxonomy pages...") with format_timed_scope(logger, 'built taxonomy buckets', level=logging.DEBUG, colored=False): buckets = self._buildTaxonomyBuckets(record) start_time = time.perf_counter() page_count = self._bakeTaxonomyBuckets(record, pool, buckets) logger.info(format_timed(start_time, "baked %d taxonomy pages." % page_count))
def _handleCacheValidity(self, record): start_time = time.perf_counter() reason = None if self.force: reason = "ordered to" elif not self.app.config.get('__cache_valid'): # The configuration file was changed, or we're running a new # version of the app. reason = "not valid anymore" elif (not record.previous.bake_time or not record.previous.hasLatestVersion()): # We have no valid previous bake record. reason = "need bake record regeneration" else: # Check if any template has changed since the last bake. Since # there could be some advanced conditional logic going on, we'd # better just force a bake from scratch if that's the case. max_time = 0 for d in self.app.templates_dirs: for dpath, _, filenames in os.walk(d): for fn in filenames: full_fn = os.path.join(dpath, fn) max_time = max(max_time, os.path.getmtime(full_fn)) if max_time >= record.previous.bake_time: reason = "templates modified" if reason is not None: # We have to bake everything from scratch. self.app.cache.clearCaches(except_names=['app']) self.force = True record.incremental_count = 0 record.clearPrevious() logger.info(format_timed( start_time, "cleaned cache (reason: %s)" % reason)) return False else: record.incremental_count += 1 logger.debug(format_timed( start_time, "cache is assumed valid", colored=False)) return True
def _bakeTaxonomies(self, record, pool): logger.debug("Baking taxonomy pages...") with format_timed_scope(logger, 'built taxonomy buckets', level=logging.DEBUG, colored=False): buckets = self._buildTaxonomyBuckets(record) start_time = time.perf_counter() page_count = self._bakeTaxonomyBuckets(record, pool, buckets) logger.info( format_timed(start_time, "baked %d taxonomy pages." % page_count))
def run(self, ctx): from piecrust.chefutil import format_timed from piecrust.environment import ExecutionStats out_dir = (ctx.args.output or os.path.join(ctx.app.root_dir, '_counter')) success = True avg_stats = ExecutionStats() avg_stats.registerTimer('Total') start_time = time.perf_counter() num_iter = 1 if ctx.args.profile > 0: num_iter = ctx.args.profile for i in range(num_iter): iter_start_time = time.perf_counter() if num_iter > 1: import gc gc.collect() logger.info("---- %d/%d ----" % (i + 1, num_iter)) # Don't cheat -- the app instance caches a bunch of stuff # so we need to create a fresh one. ctx.app = ctx.appfactory.create() try: records = self._doBake(ctx, out_dir) except Exception as ex: if ctx.app.debug: logger.exception(ex) else: logger.error(str(ex)) return 1 success = success and records.success avg_stats.mergeStats(records.stats) avg_stats.stepTimerSince('Total', iter_start_time) # Show merged stats. if ctx.args.show_stats: if num_iter > 1: _average_stats(avg_stats, num_iter) logger.info("-------------------") logger.info("Timing information:") _show_stats(avg_stats) # All done. logger.info('-------------------------') logger.info(format_timed(start_time, 'done baking')) return 0 if success else 1
def run(self, ctx): if ctx.args.foodtruck: if ctx.args.html_only: raise Exception("`--foodtruck` and `--html-only` can't be " "both specified.") if ctx.args.output: raise Exception("`--foodtruck` and `--output` can't be " "both specified.") ctx.args.assets_only = True ctx.args.output = os.path.join(ctx.app.root_dir, CACHE_DIR, 'foodtruck', 'server') out_dir = (ctx.args.output or os.path.join(ctx.app.root_dir, '_counter')) success = True ctx.timers = {} start_time = time.perf_counter() try: # Bake the site sources. if not ctx.args.assets_only: success = success & self._bakeSources(ctx, out_dir) # Bake the assets. if not ctx.args.html_only: success = success & self._bakeAssets(ctx, out_dir) # Show merged timers. if ctx.args.show_timers: logger.info("-------------------") logger.info("Timing information:") _show_timers(ctx.timers) # All done. logger.info('-------------------------') logger.info(format_timed(start_time, 'done baking')) return 0 if success else 1 except Exception as ex: if ctx.app.debug: logger.exception(ex) else: logger.error(str(ex)) return 1
def bake(self, ctx): if not self.page_ref.exists: logger.debug( "No page found at '%s', skipping taxonomy '%s'." % (self.page_ref, self.taxonomy.name)) return logger.debug("Baking %s pages...", self.taxonomy.name) with format_timed_scope(logger, 'gathered taxonomy terms', level=logging.DEBUG, colored=False): all_terms, dirty_terms = self._buildDirtyTaxonomyTerms(ctx) start_time = time.perf_counter() page_count = self._bakeTaxonomyTerms(ctx, all_terms, dirty_terms) if page_count > 0: logger.info(format_timed( start_time, "baked %d %s pages for %s." % ( page_count, self.taxonomy.term_name, self.source_name)))
def processNode(self, node): full_path = self._getNodePath(node) proc = node.getProcessor() if proc.is_bypassing_structured_processing: try: start_time = time.perf_counter() with proc.app.env.stats.timerScope(proc.__class__.__name__): proc.process(full_path, self.out_dir) print_node( node, format_timed(start_time, "(bypassing structured processing)", colored=False)) return True except Exception as e: raise ProcessorError(proc.PROCESSOR_NAME, full_path) from e # All outputs of a node must go to the same directory, so we can get # the output directory off of the first output. base_out_dir = self._getNodeBaseDir(node.outputs[0]) rel_out_dir = os.path.dirname(node.path) out_dir = os.path.join(base_out_dir, rel_out_dir) if not os.path.isdir(out_dir): try: os.makedirs(out_dir, 0o755, exist_ok=True) except OSError: pass try: start_time = time.perf_counter() with proc.app.env.stats.timerScope(proc.__class__.__name__): proc_res = proc.process(full_path, out_dir) if proc_res is None: raise Exception("Processor '%s' didn't return a boolean " "result value." % proc) if proc_res: print_node(node, "-> %s" % out_dir) return True else: print_node(node, "-> %s [clean]" % out_dir) return False except Exception as e: raise ProcessorError(proc.PROCESSOR_NAME, full_path) from e
def processNode(self, node): full_path = self._getNodePath(node) proc = node.getProcessor() if proc.is_bypassing_structured_processing: try: start_time = time.perf_counter() with proc.app.env.timerScope(proc.__class__.__name__): proc.process(full_path, self.out_dir) print_node( node, format_timed( start_time, "(bypassing structured processing)", colored=False)) return True except Exception as e: raise ProcessorError(proc.PROCESSOR_NAME, full_path) from e # All outputs of a node must go to the same directory, so we can get # the output directory off of the first output. base_out_dir = self._getNodeBaseDir(node.outputs[0]) rel_out_dir = os.path.dirname(node.path) out_dir = os.path.join(base_out_dir, rel_out_dir) if not os.path.isdir(out_dir): try: os.makedirs(out_dir, 0o755, exist_ok=True) except OSError: pass try: start_time = time.perf_counter() with proc.app.env.timerScope(proc.__class__.__name__): proc_res = proc.process(full_path, out_dir) if proc_res is None: raise Exception("Processor '%s' didn't return a boolean " "result value." % proc) if proc_res: print_node(node, "-> %s" % out_dir) return True else: print_node(node, "-> %s [clean]" % out_dir) return False except Exception as e: raise ProcessorError(proc.PROCESSOR_NAME, full_path) from e
def build(self, path): start_time = time.clock() tree_root = ProcessingTreeNode(path, list(self.processors)) loop_guard = 100 walk_stack = [tree_root] while len(walk_stack) > 0: loop_guard -= 1 if loop_guard <= 0: raise ProcessingTreeError("Infinite loop detected!") cur_node = walk_stack.pop() proc = cur_node.getProcessor() # If the root tree node (and only that one) wants to bypass this # whole tree business, so be it. if proc.is_bypassing_structured_processing: if cur_node != tree_root: raise ProcessingTreeError("Only root processors can " "bypass structured processing.") break # Get the destination directory and output files. rel_dir, basename = os.path.split(cur_node.path) out_names = proc.getOutputFilenames(basename) if out_names is None: continue for n in out_names: out_node = ProcessingTreeNode( os.path.join(rel_dir, n), list(cur_node.available_procs), cur_node.level + 1) cur_node.outputs.append(out_node) if proc.PROCESSOR_NAME != 'copy': walk_stack.append(out_node) logger.debug(format_timed( start_time, "Built processing tree for: %s" % path, colored=False)) return tree_root
def _bakeRealm(self, record, pool, realm, srclist): start_time = time.perf_counter() try: record.current.baked_count[realm] = 0 all_factories = [] for source in srclist: factories = source.getPageFactories() all_factories += [f for f in factories if f.path not in self.taxonomy_pages] self._loadRealmPages(record, pool, all_factories) self._renderRealmPages(record, pool, all_factories) self._bakeRealmPages(record, pool, realm, all_factories) finally: page_count = record.current.baked_count[realm] logger.info(format_timed( start_time, "baked %d %s pages." % (page_count, REALM_NAMES[realm].lower())))
def _bakeRealm(self, pool, ppmngr, record_histories, pp_pass_num, realm, pplist): start_time = time.perf_counter() job_count = 0 job_descs = {} realm_name = REALM_NAMES[realm].lower() pool.userdata.cur_pass = pp_pass_num for ppinfo in pplist: src = ppinfo.source pp = ppinfo.pipeline jcctx = PipelineJobCreateContext(pp_pass_num, pp.record_name, record_histories) jobs, job_desc = pp.createJobs(jcctx) if jobs is not None: new_job_count = len(jobs) job_count += new_job_count pool.queueJobs(jobs) if job_desc: job_descs.setdefault(job_desc, []).append(src.name) else: new_job_count = 0 logger.debug( "Queued %d jobs for source '%s' using pipeline '%s' " "(%s)." % (new_job_count, src.name, pp.PIPELINE_NAME, realm_name)) if job_count == 0: logger.debug("No jobs queued! Bailing out of this bake pass.") return pool.wait() logger.info(format_timed( start_time, "%d jobs completed (%s)." % (job_count, ', '.join( ['%s %s' % (d, ', '.join(sn)) for d, sn in job_descs.items()]))))
def runQueue(self, *, only_task=None, clear_queue=True): start_time = time.perf_counter() tasks = list(self.getTasks(only_task=only_task)) for path, task_type, task_data in tasks: if not task_type: logger.error("Got task with no type: %s" % path) continue runner = self._getRunner(task_type) if runner is None: logger.error("No task runner for type: %s" % task_type) continue ctx = TaskContext() runner.runTask(task_data, ctx) if clear_queue: os.remove(path) logger.info(format_timed(start_time, "Ran %d tasks." % len(tasks)))
def _bakeRealm(self, record, pool, realm, srclist): start_time = time.perf_counter() try: record.current.baked_count[realm] = 0 all_factories = [] for source in srclist: factories = source.getPageFactories() all_factories += [ f for f in factories if f.path not in self.taxonomy_pages ] self._loadRealmPages(record, pool, all_factories) self._renderRealmPages(record, pool, all_factories) self._bakeRealmPages(record, pool, realm, all_factories) finally: page_count = record.current.baked_count[realm] logger.info( format_timed( start_time, "baked %d %s pages." % (page_count, REALM_NAMES[realm].lower())))
def runQueue(self, *, only_task=None, clear_queue=True): start_time = time.perf_counter() tasks = list(self.getTasks(only_task=only_task)) for path, task_type, task_data in tasks: if not task_type: logger.error("Got task with no type: %s" % path) continue runner = self._getRunner(task_type) if runner is None: logger.error("No task runner for type: %s" % task_type) continue ctx = TaskContext() runner.runTask(task_data, ctx) if clear_queue: os.remove(path) logger.info(format_timed( start_time, "Ran %d tasks." % len(tasks)))
def bake(self): logger.debug(" Bake Output: %s" % self.out_dir) logger.debug(" Root URL: %s" % self.app.config.get('site/root')) # Get into bake mode. start_time = time.clock() self.app.config.set('baker/is_baking', True) self.app.env.base_asset_url_format = '%uri%' # Make sure the output directory exists. if not os.path.isdir(self.out_dir): os.makedirs(self.out_dir, 0o755) # Load/create the bake record. record = TransitionalBakeRecord() record_cache = self.app.cache.getCache('baker') record_id = hashlib.md5(self.out_dir.encode('utf8')).hexdigest() record_name = record_id + '.record' if not self.force and record_cache.has(record_name): t = time.clock() record.loadPrevious(record_cache.getCachePath(record_name)) logger.debug(format_timed( t, 'loaded previous bake record', colored=False)) record.current.success = True # Figure out if we need to clean the cache because important things # have changed. self._handleCacheValidity(record) # Gather all sources by realm -- we're going to bake each realm # separately so we can handle "overlaying" (i.e. one realm overrides # another realm's pages). sources_by_realm = {} for source in self.app.sources: srclist = sources_by_realm.setdefault(source.realm, []) srclist.append(source) # Bake the realms. realm_list = [REALM_USER, REALM_THEME] for realm in realm_list: srclist = sources_by_realm.get(realm) if srclist is not None: self._bakeRealm(record, realm, srclist) # Bake taxonomies. self._bakeTaxonomies(record) # Delete files from the output. self._handleDeletetions(record) # Backup previous records. for i in range(8, -1, -1): suffix = '' if i == 0 else '.%d' % i record_path = record_cache.getCachePath( '%s%s.record' % (record_id, suffix)) if os.path.exists(record_path): record_path_next = record_cache.getCachePath( '%s.%s.record' % (record_id, i + 1)) if os.path.exists(record_path_next): os.remove(record_path_next) os.rename(record_path, record_path_next) # Save the bake record. t = time.clock() record.current.bake_time = time.time() record.current.out_dir = self.out_dir record.saveCurrent(record_cache.getCachePath(record_name)) logger.debug(format_timed(t, 'saved bake record', colored=False)) # All done. self.app.config.set('baker/is_baking', False) logger.debug(format_timed(start_time, 'done baking')) return record.detach()
def run(self, target, force=False, preview=False, extra_args=None, log_file=None, log_debug_info=False, append_log_file=False): start_time = time.perf_counter() # Get publisher for this target. pub = self.app.getPublisher(target) if pub is None: raise InvalidPublishTargetError( "No such publish target: %s" % target) # Will we need to bake first? bake_first = pub.config.get('bake', True) # Setup logging stuff. hdlr = None root_logger = logging.getLogger() if log_file and not preview: logger.debug("Adding file handler for: %s" % log_file) mode = 'w' if append_log_file: mode = 'a' hdlr = logging.FileHandler(log_file, mode=mode, encoding='utf8') root_logger.addHandler(hdlr) if log_debug_info: _log_debug_info(target, force, preview, extra_args) if not preview: logger.info("Deploying to %s" % target) else: logger.info("Previewing deployment to %s" % target) # Bake first is necessary. records = None was_baked = False bake_out_dir = os.path.join(self.app.root_dir, '_pub', target) if bake_first: if not preview: bake_start_time = time.perf_counter() logger.debug("Baking first to: %s" % bake_out_dir) from piecrust.baking.baker import Baker baker = Baker( self.appfactory, self.app, bake_out_dir, force=force) records = baker.bake() was_baked = True if not records.success: raise Exception( "Error during baking, aborting publishing.") logger.info(format_timed(bake_start_time, "Baked website.")) else: logger.info("Would bake to: %s" % bake_out_dir) # Publish! logger.debug( "Running publish target '%s' with publisher: %s" % (target, pub.PUBLISHER_NAME)) pub_start_time = time.perf_counter() success = False ctx = PublishingContext() ctx.bake_out_dir = bake_out_dir ctx.bake_records = records ctx.was_baked = was_baked ctx.preview = preview ctx.args = extra_args try: success = pub.run(ctx) except Exception as ex: raise PublishingError( "Error publishing to target: %s" % target) from ex finally: if hdlr: root_logger.removeHandler(hdlr) hdlr.close() logger.info(format_timed( pub_start_time, "Ran publisher %s" % pub.PUBLISHER_NAME)) if success: logger.info(format_timed(start_time, 'Deployed to %s' % target)) return 0 else: logger.error(format_timed(start_time, 'Failed to deploy to %s' % target)) return 1
def run(self, target, force=False, preview=False, extra_args=None, log_file=None, log_debug_info=False, append_log_file=False): start_time = time.perf_counter() # Get publisher for this target. pub = self.app.getPublisher(target) if pub is None: raise InvalidPublishTargetError("No such publish target: %s" % target) # Will we need to bake first? bake_first = pub.config.get('bake', True) # Setup logging stuff. hdlr = None root_logger = logging.getLogger() if log_file and not preview: logger.debug("Adding file handler for: %s" % log_file) mode = 'w' if append_log_file: mode = 'a' hdlr = logging.FileHandler(log_file, mode=mode, encoding='utf8') root_logger.addHandler(hdlr) if log_debug_info: _log_debug_info(target, force, preview, extra_args) if not preview: logger.info("Deploying to %s" % target) else: logger.info("Previewing deployment to %s" % target) # Bake first is necessary. records = None was_baked = False bake_out_dir = os.path.join(self.app.root_dir, '_pub', target) if bake_first: if not preview: bake_start_time = time.perf_counter() logger.debug("Baking first to: %s" % bake_out_dir) from piecrust.baking.baker import Baker baker = Baker(self.appfactory, self.app, bake_out_dir, force=force) records = baker.bake() was_baked = True if not records.success: raise Exception( "Error during baking, aborting publishing.") logger.info(format_timed(bake_start_time, "Baked website.")) else: logger.info("Would bake to: %s" % bake_out_dir) # Publish! logger.debug("Running publish target '%s' with publisher: %s" % (target, pub.PUBLISHER_NAME)) pub_start_time = time.perf_counter() success = False ctx = PublishingContext() ctx.bake_out_dir = bake_out_dir ctx.bake_records = records ctx.was_baked = was_baked ctx.preview = preview ctx.args = extra_args try: success = pub.run(ctx) except Exception as ex: raise PublishingError("Error publishing to target: %s" % target) from ex finally: if hdlr: root_logger.removeHandler(hdlr) hdlr.close() logger.info( format_timed(pub_start_time, "Ran publisher %s" % pub.PUBLISHER_NAME)) if success: logger.info(format_timed(start_time, 'Deployed to %s' % target)) return 0 else: logger.error( format_timed(start_time, 'Failed to deploy to %s' % target)) return 1
def _run_chef(pre_args, argv): # Setup the app. start_time = time.perf_counter() root = None if pre_args.root: root = os.path.expanduser(pre_args.root) else: try: root = find_app_root() except SiteNotFoundError: root = None if not root: app = NullPieCrust() else: app = PieCrust(root, cache=(not pre_args.no_cache), debug=pre_args.debug) # Build a hash for a custom cache directory. cache_key = 'default' # Handle custom configurations. if pre_args.config_variant is not None and not root: raise SiteNotFoundError("Can't apply any variant.") apply_variant_and_values(app, pre_args.config_variant, pre_args.config_values) # Adjust the cache key. if pre_args.config_variant is not None: cache_key += ',variant=%s' % pre_args.config_variant if pre_args.config_values: for name, value in pre_args.config_values: cache_key += ',%s=%s' % (name, value) # Setup the arg parser. parser = argparse.ArgumentParser( prog='chef', description="The PieCrust chef manages your website.", formatter_class=argparse.RawDescriptionHelpFormatter) _setup_main_parser_arguments(parser) commands = sorted(app.plugin_loader.getCommands(), key=lambda c: c.name) subparsers = parser.add_subparsers(title='list of commands') for c in commands: p = subparsers.add_parser(c.name, help=c.description) c.setupParser(p, app) p.set_defaults(func=c.checkedRun) p.set_defaults(cache_name=c.cache_name) help_cmd = next(filter(lambda c: c.name == 'help', commands), None) if help_cmd and help_cmd.has_topics: with io.StringIO() as epilog: epilog.write("additional help topics:\n") for name, desc in help_cmd.getTopics(): print_help_item(epilog, name, desc) parser.epilog = epilog.getvalue() # Parse the command line. result = parser.parse_args(argv) logger.debug(format_timed(start_time, 'initialized PieCrust', colored=False)) # Print the help if no command was specified. if not hasattr(result, 'func'): parser.print_help() return 0 # Use a customized cache for the command and current config. if result.cache_name != 'default' or cache_key != 'default': app.useSubCache(result.cache_name, cache_key) # Run the command! ctx = CommandContext(app, parser, result) ctx.config_variant = pre_args.config_variant ctx.config_values = pre_args.config_values exit_code = result.func(ctx) if exit_code is None: return 0 if not isinstance(exit_code, int): logger.error("Got non-integer exit code: %s" % exit_code) return -1 return exit_code
def _unsafeRun(self, job): start_time = time.clock() pipeline = self.ctx.pipeline record = self.ctx.record rel_path = os.path.relpath(job.path, job.base_dir) previous_entry = record.getPreviousEntry(rel_path) record_entry = ProcessorPipelineRecordEntry(job.base_dir, rel_path) record.addEntry(record_entry) # Figure out if a previously processed file is overriding this one. # This can happen if a theme file (processed via a mount point) # is overridden in the user's website. if record.current.hasOverrideEntry(rel_path): record_entry.flags |= FLAG_OVERRIDEN logger.info( format_timed(start_time, '%s [not baked, overridden]' % rel_path)) return True processors = pipeline.getFilteredProcessors( job.mount_info['processors']) try: builder = ProcessingTreeBuilder(processors) tree_root = builder.build(rel_path) record_entry.flags |= FLAG_PREPARED except ProcessingTreeError as ex: msg = str(ex) logger.error("Error preparing %s:\n%s" % (rel_path, msg)) while ex: record_entry.errors.append(str(ex)) ex = ex.__cause__ return False print_node(tree_root, recursive=True) leaves = tree_root.getLeaves() record_entry.rel_outputs = [l.path for l in leaves] record_entry.proc_tree = get_node_name_tree(tree_root) if tree_root.getProcessor().is_bypassing_structured_processing: record_entry.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING force = (pipeline.force or previous_entry is None or not previous_entry.was_processed_successfully) if not force: force = re_matchany(rel_path, pipeline.force_patterns) if force: tree_root.setState(STATE_DIRTY, True) try: runner = ProcessingTreeRunner(job.base_dir, pipeline.tmp_dir, pipeline.out_dir, self.ctx.pipeline_lock) if runner.processSubTree(tree_root): record_entry.flags |= FLAG_PROCESSED logger.info( format_timed(start_time, "[%d] %s" % (self.wid, rel_path))) return True except ProcessingTreeError as ex: msg = str(ex) if isinstance(ex, ProcessorError): msg = str(ex.__cause__) logger.error("Error processing %s:\n%s" % (rel_path, msg)) while ex: msg = re_ansicolors.sub('', str(ex)) record_entry.errors.append(msg) ex = ex.__cause__ return False
def _computeNodeState(self, node): if node.state != STATE_UNKNOWN: return proc = node.getProcessor() if (proc.is_bypassing_structured_processing or not proc.is_delegating_dependency_check): # This processor wants to handle things on its own... node.setState(STATE_DIRTY, False) return start_time = time.perf_counter() # Get paths and modification times for the input path and # all dependencies (if any). base_dir = self._getNodeBaseDir(node) full_path = os.path.join(base_dir, node.path) in_mtime = (full_path, os.path.getmtime(full_path)) force_build = False try: deps = proc.getDependencies(full_path) if deps == FORCE_BUILD: force_build = True elif deps is not None: for dep in deps: dep_mtime = os.path.getmtime(dep) if dep_mtime > in_mtime[1]: in_mtime = (dep, dep_mtime) except Exception as e: logger.warning("%s -- Will force-bake: %s" % (e, node.path)) node.setState(STATE_DIRTY, True) return if force_build: # Just do what the processor told us to do. node.setState(STATE_DIRTY, True) message = "Processor requested a forced build." print_node(node, message) else: # Get paths and modification times for the outputs. message = None for o in node.outputs: full_out_path = self._getNodePath(o) if not os.path.isfile(full_out_path): message = "Output '%s' doesn't exist." % o.path break o_mtime = os.path.getmtime(full_out_path) if o_mtime < in_mtime[1]: message = "Input '%s' is newer than output '%s'." % ( in_mtime[0], o.path) break if message is not None: node.setState(STATE_DIRTY, True) message += " Re-processing sub-tree." print_node(node, message) else: node.setState(STATE_CLEAN, False) state = "dirty" if node.state == STATE_DIRTY else "clean" logger.debug(format_timed(start_time, "Computed node dirtyness: %s" % state, indent_level=node.level, colored=False))
def run(self, src_dir_or_file=None, *, delete=True, previous_record=None, save_record=True): start_time = time.perf_counter() # Get the list of processors for this run. processors = self.app.plugin_loader.getProcessors() if self.enabled_processors is not None: logger.debug("Filtering processors to: %s" % self.enabled_processors) processors = get_filtered_processors(processors, self.enabled_processors) if self.additional_processors_factories is not None: logger.debug("Adding %s additional processors." % len(self.additional_processors_factories)) for proc_fac in self.additional_processors_factories: proc = proc_fac() self.app.env.registerTimer(proc.__class__.__name__, raise_if_registered=False) proc.initialize(self.app) processors.append(proc) # Invoke pre-processors. pipeline_ctx = PipelineContext(-1, self.app, self.out_dir, self.tmp_dir, self.force) for proc in processors: proc.onPipelineStart(pipeline_ctx) # Pre-processors can define additional ignore patterns. self.ignore_patterns += make_re( pipeline_ctx._additional_ignore_patterns) # Create the pipeline record. record = TransitionalProcessorPipelineRecord() record_cache = self.app.cache.getCache('proc') record_name = ( hashlib.md5(self.out_dir.encode('utf8')).hexdigest() + '.record') if previous_record: record.setPrevious(previous_record) elif not self.force and record_cache.has(record_name): with format_timed_scope(logger, 'loaded previous bake record', level=logging.DEBUG, colored=False): record.loadPrevious(record_cache.getCachePath(record_name)) logger.debug("Got %d entries in process record." % len(record.previous.entries)) record.current.success = True record.current.processed_count = 0 # Work! def _handler(res): entry = record.getCurrentEntry(res.path) assert entry is not None entry.flags = res.flags entry.proc_tree = res.proc_tree entry.rel_outputs = res.rel_outputs if entry.flags & FLAG_PROCESSED: record.current.processed_count += 1 if res.errors: entry.errors += res.errors record.current.success = False rel_path = os.path.relpath(res.path, self.app.root_dir) logger.error("Errors found in %s:" % rel_path) for e in entry.errors: logger.error(" " + e) jobs = [] self._process(src_dir_or_file, record, jobs) pool = self._createWorkerPool() ar = pool.queueJobs(jobs, handler=_handler) ar.wait() # Shutdown the workers and get timing information from them. reports = pool.close() record.current.timers = {} for i in range(len(reports)): timers = reports[i] if timers is None: continue worker_name = 'PipelineWorker_%d' % i record.current.timers[worker_name] = {} for name, val in timers['data'].items(): main_val = record.current.timers.setdefault(name, 0) record.current.timers[name] = main_val + val record.current.timers[worker_name][name] = val # Invoke post-processors. pipeline_ctx.record = record.current for proc in processors: proc.onPipelineEnd(pipeline_ctx) # Handle deletions. if delete: for path, reason in record.getDeletions(): logger.debug("Removing '%s': %s" % (path, reason)) try: os.remove(path) except FileNotFoundError: pass logger.info('[delete] %s' % path) # Finalize the process record. record.current.process_time = time.time() record.current.out_dir = self.out_dir record.collapseRecords() # Save the process record. if save_record: with format_timed_scope(logger, 'saved bake record', level=logging.DEBUG, colored=False): record.saveCurrent(record_cache.getCachePath(record_name)) logger.info(format_timed( start_time, "processed %d assets." % record.current.processed_count)) return record.detach()
def bake(self): logger.debug(" Bake Output: %s" % self.out_dir) logger.debug(" Root URL: %s" % self.app.config.get('site/root')) # Get into bake mode. start_time = time.perf_counter() self.app.config.set('baker/is_baking', True) self.app.env.base_asset_url_format = '%uri%' # Make sure the output directory exists. if not os.path.isdir(self.out_dir): os.makedirs(self.out_dir, 0o755) # Load/create the bake record. record = TransitionalBakeRecord() record_cache = self.app.cache.getCache('baker') record_id = hashlib.md5(self.out_dir.encode('utf8')).hexdigest() record_name = record_id + '.record' previous_record_path = None if not self.force and record_cache.has(record_name): with format_timed_scope(logger, "loaded previous bake record", level=logging.DEBUG, colored=False): previous_record_path = record_cache.getCachePath(record_name) record.loadPrevious(previous_record_path) record.current.success = True # Figure out if we need to clean the cache because important things # have changed. is_cache_valid = self._handleCacheValidity(record) if not is_cache_valid: previous_record_path = None # Pre-create all caches. for cache_name in ['app', 'baker', 'pages', 'renders']: self.app.cache.getCache(cache_name) # Gather all sources by realm -- we're going to bake each realm # separately so we can handle "overriding" (i.e. one realm overrides # another realm's pages, like the user realm overriding the theme # realm). sources_by_realm = {} for source in self.app.sources: srclist = sources_by_realm.setdefault(source.realm, []) srclist.append(source) # Create the worker processes. pool = self._createWorkerPool(previous_record_path) # Bake the realms. realm_list = [REALM_USER, REALM_THEME] for realm in realm_list: srclist = sources_by_realm.get(realm) if srclist is not None: self._bakeRealm(record, pool, realm, srclist) # Bake taxonomies. self._bakeTaxonomies(record, pool) # All done with the workers. Close the pool and get timing reports. reports = pool.close() record.current.timers = {} for i in range(len(reports)): timers = reports[i] if timers is None: continue worker_name = 'BakeWorker_%d' % i record.current.timers[worker_name] = {} for name, val in timers['data'].items(): main_val = record.current.timers.setdefault(name, 0) record.current.timers[name] = main_val + val record.current.timers[worker_name][name] = val # Delete files from the output. self._handleDeletetions(record) # Backup previous records. for i in range(8, -1, -1): suffix = '' if i == 0 else '.%d' % i record_path = record_cache.getCachePath('%s%s.record' % (record_id, suffix)) if os.path.exists(record_path): record_path_next = record_cache.getCachePath( '%s.%s.record' % (record_id, i + 1)) if os.path.exists(record_path_next): os.remove(record_path_next) os.rename(record_path, record_path_next) # Save the bake record. with format_timed_scope(logger, "saved bake record.", level=logging.DEBUG, colored=False): record.current.bake_time = time.time() record.current.out_dir = self.out_dir record.saveCurrent(record_cache.getCachePath(record_name)) # All done. self.app.config.set('baker/is_baking', False) logger.debug(format_timed(start_time, 'done baking')) return record.detach()
def run(self, src_dir_or_file=None, *, delete=True, previous_record=None, save_record=True): # Invoke pre-processors. for proc in self.processors: proc.onPipelineStart(self) # Sort our processors again in case the pre-process step involved # patching the processors with some new ones. self.processors.sort(key=lambda p: p.priority) # Create the pipeline record. record = TransitionalProcessorPipelineRecord() record_cache = self.app.cache.getCache('proc') record_name = ( hashlib.md5(self.out_dir.encode('utf8')).hexdigest() + '.record') if previous_record: record.setPrevious(previous_record) elif not self.force and record_cache.has(record_name): t = time.clock() record.loadPrevious(record_cache.getCachePath(record_name)) logger.debug(format_timed(t, 'loaded previous bake record', colored=False)) logger.debug("Got %d entries in process record." % len(record.previous.entries)) # Create the workers. pool = [] queue = Queue() abort = threading.Event() pipeline_lock = threading.Lock() for i in range(self.num_workers): ctx = ProcessingWorkerContext(self, record, queue, abort, pipeline_lock) worker = ProcessingWorker(i, ctx) worker.start() pool.append(worker) if src_dir_or_file is not None: # Process only the given path. # Find out what mount point this is in. for name, info in self.mounts.items(): path = info['path'] if src_dir_or_file[:len(path)] == path: base_dir = path mount_info = info break else: known_roots = [i['path'] for i in self.mounts.values()] raise Exception("Input path '%s' is not part of any known " "mount point: %s" % (src_dir_or_file, known_roots)) ctx = ProcessingContext(base_dir, mount_info, queue, record) logger.debug("Initiating processing pipeline on: %s" % src_dir_or_file) if os.path.isdir(src_dir_or_file): self.processDirectory(ctx, src_dir_or_file) elif os.path.isfile(src_dir_or_file): self.processFile(ctx, src_dir_or_file) else: # Process everything. for name, info in self.mounts.items(): path = info['path'] ctx = ProcessingContext(path, info, queue, record) logger.debug("Initiating processing pipeline on: %s" % path) self.processDirectory(ctx, path) # Wait on all workers. record.current.success = True for w in pool: w.join() record.current.success &= w.success if abort.is_set(): raise Exception("Worker pool was aborted.") # Handle deletions. if delete: for path, reason in record.getDeletions(): logger.debug("Removing '%s': %s" % (path, reason)) try: os.remove(path) except FileNotFoundError: pass logger.info('[delete] %s' % path) # Invoke post-processors. for proc in self.processors: proc.onPipelineEnd(self) # Finalize the process record. record.current.process_time = time.time() record.current.out_dir = self.out_dir record.collapseRecords() # Save the process record. if save_record: t = time.clock() record.saveCurrent(record_cache.getCachePath(record_name)) logger.debug(format_timed(t, 'saved bake record', colored=False)) return record.detach()
def _unsafeRun(self, job): start_time = time.clock() pipeline = self.ctx.pipeline record = self.ctx.record rel_path = os.path.relpath(job.path, job.base_dir) previous_entry = record.getPreviousEntry(rel_path) record_entry = ProcessorPipelineRecordEntry(job.base_dir, rel_path) record.addEntry(record_entry) # Figure out if a previously processed file is overriding this one. # This can happen if a theme file (processed via a mount point) # is overridden in the user's website. if record.current.hasOverrideEntry(rel_path): record_entry.flags |= FLAG_OVERRIDEN logger.info(format_timed(start_time, '%s [not baked, overridden]' % rel_path)) return True processors = pipeline.getFilteredProcessors( job.mount_info['processors']) try: builder = ProcessingTreeBuilder(processors) tree_root = builder.build(rel_path) record_entry.flags |= FLAG_PREPARED except ProcessingTreeError as ex: msg = str(ex) logger.error("Error preparing %s:\n%s" % (rel_path, msg)) while ex: record_entry.errors.append(str(ex)) ex = ex.__cause__ return False print_node(tree_root, recursive=True) leaves = tree_root.getLeaves() record_entry.rel_outputs = [l.path for l in leaves] record_entry.proc_tree = get_node_name_tree(tree_root) if tree_root.getProcessor().is_bypassing_structured_processing: record_entry.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING force = (pipeline.force or previous_entry is None or not previous_entry.was_processed_successfully) if not force: force = re_matchany(rel_path, pipeline.force_patterns) if force: tree_root.setState(STATE_DIRTY, True) try: runner = ProcessingTreeRunner( job.base_dir, pipeline.tmp_dir, pipeline.out_dir, self.ctx.pipeline_lock) if runner.processSubTree(tree_root): record_entry.flags |= FLAG_PROCESSED logger.info(format_timed( start_time, "[%d] %s" % (self.wid, rel_path))) return True except ProcessingTreeError as ex: msg = str(ex) if isinstance(ex, ProcessorError): msg = str(ex.__cause__) logger.error("Error processing %s:\n%s" % (rel_path, msg)) while ex: msg = re_ansicolors.sub('', str(ex)) record_entry.errors.append(msg) ex = ex.__cause__ return False
def _run_chef(pre_args, argv): # Setup the app. start_time = time.clock() root = pre_args.root if root is None: try: root = find_app_root() except SiteNotFoundError: root = None if not root: app = NullPieCrust() else: app = PieCrust(root, cache=pre_args.cache, debug=pre_args.debug) # Build a hash for a custom cache directory. cache_key = 'default' # Handle custom configurations. if pre_args.config_variant is not None and not root: raise SiteNotFoundError("Can't apply any variant.") apply_variant_and_values(app, pre_args.config_variant, pre_args.config_values) # Adjust the cache key. if pre_args.config_variant is not None: cache_key += ',variant=%s' % pre_args.config_variant for name, value in pre_args.config_values: cache_key += ',%s=%s' % (name, value) # Setup the arg parser. parser = argparse.ArgumentParser( prog='chef', description="The PieCrust chef manages your website.", formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '--version', action='version', version=('%(prog)s ' + APP_VERSION)) parser.add_argument( '--root', help="The root directory of the website.") parser.add_argument( '--config', help="The configuration variant to use for this command.") parser.add_argument( '--config-set', help="Sets a specific site configuration setting.") parser.add_argument( '--debug', help="Show debug information.", action='store_true') parser.add_argument( '--no-cache', help="When applicable, disable caching.", action='store_true') parser.add_argument( '--quiet', help="Print only important information.", action='store_true') parser.add_argument( '--log', help="Send log messages to the specified file.") parser.add_argument( '--log-debug', help="Log debug messages to the log file.", action='store_true') commands = sorted(app.plugin_loader.getCommands(), key=lambda c: c.name) subparsers = parser.add_subparsers(title='list of commands') for c in commands: p = subparsers.add_parser(c.name, help=c.description) c.setupParser(p, app) p.set_defaults(func=c.checkedRun) p.set_defaults(cache_name=c.cache_name) help_cmd = next(filter(lambda c: c.name == 'help', commands), None) if help_cmd and help_cmd.has_topics: with io.StringIO() as epilog: epilog.write("additional help topics:\n") for name, desc in help_cmd.getTopics(): print_help_item(epilog, name, desc) parser.epilog = epilog.getvalue() # Parse the command line. result = parser.parse_args(argv) logger.debug(format_timed(start_time, 'initialized PieCrust', colored=False)) # Print the help if no command was specified. if not hasattr(result, 'func'): parser.print_help() return 0 # Use a customized cache for the command and current config. if result.cache_name != 'default' or cache_key != 'default': app.useSubCache(result.cache_name, cache_key) # Run the command! ctx = CommandContext(app, parser, result) ctx.config_variant = pre_args.config_variant ctx.config_values = pre_args.config_values exit_code = result.func(ctx) if exit_code is None: return 0 if not isinstance(exit_code, int): logger.error("Got non-integer exit code: %s" % exit_code) return -1 return exit_code
def bake(self): start_time = time.perf_counter() # Setup baker. logger.debug(" Bake Output: %s" % self.out_dir) logger.debug(" Root URL: %s" % self.app.config.get('site/root')) # Get into bake mode. self.app.config.set('baker/is_baking', True) self.app.config.set('site/asset_url_format', '%page_uri%/%filename%') stats = self.app.env.stats stats.registerTimer('LoadSourceContents', raise_if_registered=False) stats.registerTimer('CacheTemplates', raise_if_registered=False) # Make sure the output directory exists. if not os.path.isdir(self.out_dir): os.makedirs(self.out_dir, 0o755) # Load/create the bake records. records_path = get_bake_records_path( self.app, self.out_dir) if not self.force and os.path.isfile(records_path): with format_timed_scope(logger, "loaded previous bake records", level=logging.DEBUG, colored=False): previous_records = load_records(records_path) else: previous_records = MultiRecord() current_records = MultiRecord() # Figure out if we need to clean the cache because important things # have changed. is_cache_valid = self._handleCacheValidity(previous_records, current_records) if not is_cache_valid: previous_records = MultiRecord() # Create the bake records history which tracks what's up-to-date # or not since last time we baked to the given output folder. record_histories = MultiRecordHistory( previous_records, current_records) # Pre-create all caches. for cache_name in ['app', 'baker', 'pages', 'renders']: self.app.cache.getCache(cache_name) # Create the pipelines. ppmngr = self._createPipelineManager(record_histories) # Done with all the setup, let's start the actual work. logger.info(format_timed(start_time, "setup baker")) # Load all sources, pre-cache templates. load_start_time = time.perf_counter() self._populateTemplateCaches() logger.info(format_timed(load_start_time, "cache templates")) # Create the worker processes. pool_userdata = _PoolUserData(self, ppmngr) pool = self._createWorkerPool(records_path, pool_userdata) # Bake the realms. self._bakeRealms(pool, ppmngr, record_histories) # Handle deletions, collapse records, etc. ppmngr.postJobRun() ppmngr.deleteStaleOutputs() ppmngr.collapseRecords(self.keep_unused_records) # All done with the workers. Close the pool and get reports. pool_stats = pool.close() current_records.stats = _merge_execution_stats(stats, *pool_stats) # Shutdown the pipelines. ppmngr.shutdownPipelines() # Backup previous records, save the current ones. current_records.bake_time = time.time() current_records.out_dir = self.out_dir _save_bake_records(current_records, records_path, rotate_previous=self.rotate_bake_records) # All done. self.app.config.set('baker/is_baking', False) logger.debug(format_timed(start_time, 'done baking')) return current_records
def _computeNodeState(self, node): if node.state != STATE_UNKNOWN: return proc = node.getProcessor() if (proc.is_bypassing_structured_processing or not proc.is_delegating_dependency_check): # This processor wants to handle things on its own... node.setState(STATE_DIRTY, False) return start_time = time.perf_counter() # Get paths and modification times for the input path and # all dependencies (if any). base_dir = self._getNodeBaseDir(node) full_path = os.path.join(base_dir, node.path) in_mtime = (full_path, os.path.getmtime(full_path)) force_build = False try: deps = proc.getDependencies(full_path) if deps == FORCE_BUILD: force_build = True elif deps is not None: for dep in deps: dep_mtime = os.path.getmtime(dep) if dep_mtime > in_mtime[1]: in_mtime = (dep, dep_mtime) except Exception as e: logger.warning("%s -- Will force-bake: %s" % (e, node.path)) node.setState(STATE_DIRTY, True) return if force_build: # Just do what the processor told us to do. node.setState(STATE_DIRTY, True) message = "Processor requested a forced build." print_node(node, message) else: # Get paths and modification times for the outputs. message = None for o in node.outputs: full_out_path = self._getNodePath(o) if not os.path.isfile(full_out_path): message = "Output '%s' doesn't exist." % o.path break o_mtime = os.path.getmtime(full_out_path) if o_mtime < in_mtime[1]: message = "Input '%s' is newer than output '%s'." % ( in_mtime[0], o.path) break if message is not None: node.setState(STATE_DIRTY, True) message += " Re-processing sub-tree." print_node(node, message) else: node.setState(STATE_CLEAN, False) if node.state == STATE_DIRTY: state = "dirty" elif node.state == STATE_CLEAN: state = "clean" else: state = "unknown" logger.debug( format_timed(start_time, "Computed node dirtyness: %s" % state, indent_level=node.level, colored=False))
def _run_chef(pre_args): # Setup the app. start_time = time.clock() root = pre_args.root if root is None: try: root = find_app_root() except SiteNotFoundError: root = None if not root: app = NullPieCrust() else: app = PieCrust(root, cache=pre_args.cache, debug=pre_args.debug) # Build a hash for a custom cache directory. cache_key = 'default' # Handle a configuration variant. if pre_args.config_variant is not None: if not root: raise SiteNotFoundError("Can't apply any variant.") app.config.applyVariant('variants/' + pre_args.config_variant) cache_key += ',variant=%s' % pre_args.config_variant for name, value in pre_args.config_values: logger.debug("Setting configuration '%s' to: %s" % (name, value)) app.config.set(name, value) cache_key += ',%s=%s' % (name, value) # Setup the arg parser. parser = argparse.ArgumentParser( prog='chef', description="The PieCrust chef manages your website.", formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--version', action='version', version=('%(prog)s ' + APP_VERSION)) parser.add_argument('--root', help="The root directory of the website.") parser.add_argument( '--config', help="The configuration variant to use for this command.") parser.add_argument('--config-set', help="Sets a specific site configuration setting.") parser.add_argument('--debug', help="Show debug information.", action='store_true') parser.add_argument('--no-cache', help="When applicable, disable caching.", action='store_true') parser.add_argument('--quiet', help="Print only important information.", action='store_true') parser.add_argument('--log', help="Send log messages to the specified file.") parser.add_argument('--log-debug', help="Log debug messages to the log file.", action='store_true') commands = sorted(app.plugin_loader.getCommands(), key=lambda c: c.name) subparsers = parser.add_subparsers(title='list of commands') for c in commands: p = subparsers.add_parser(c.name, help=c.description) c.setupParser(p, app) p.set_defaults(func=c.checkedRun) p.set_defaults(cache_name=c.cache_name) help_cmd = next(filter(lambda c: c.name == 'help', commands), None) if help_cmd and help_cmd.has_topics: with io.StringIO() as epilog: epilog.write("additional help topics:\n") for name, desc in help_cmd.getTopics(): print_help_item(epilog, name, desc) parser.epilog = epilog.getvalue() # Parse the command line. result = parser.parse_args() logger.debug( format_timed(start_time, 'initialized PieCrust', colored=False)) # Print the help if no command was specified. if not hasattr(result, 'func'): parser.print_help() return 0 # Use a customized cache for the command and current config. if result.cache_name != 'default' or cache_key != 'default': app.useSubCache(result.cache_name, cache_key) # Run the command! ctx = CommandContext(app, parser, result) exit_code = result.func(ctx) if exit_code is None: return 0 if not isinstance(exit_code, int): logger.error("Got non-integer exit code: %s" % exit_code) return -1 return exit_code
def run(self, target, force=False, preview=False, extra_args=None, log_file=None, applied_config_variant=None, applied_config_values=None): start_time = time.perf_counter() # Get publisher for this target. pub = self.app.getPublisher(target) if pub is None: raise InvalidPublishTargetError( "No such publish target: %s" % target) # Will we need to bake first? bake_first = True if not pub.has_url_config: bake_first = pub.getConfigValue('bake', True) # Setup logging stuff. hdlr = None root_logger = logging.getLogger() if log_file and not preview: logger.debug("Adding file handler for: %s" % log_file) hdlr = logging.FileHandler(log_file, mode='w', encoding='utf8') root_logger.addHandler(hdlr) if not preview: logger.info("Deploying to %s" % target) else: logger.info("Previewing deployment to %s" % target) # Bake first is necessary. rec1 = None rec2 = None was_baked = False bake_out_dir = os.path.join(self.app.root_dir, '_pub', target) if bake_first: if not preview: bake_start_time = time.perf_counter() logger.debug("Baking first to: %s" % bake_out_dir) from piecrust.baking.baker import Baker baker = Baker( self.app, bake_out_dir, applied_config_variant=applied_config_variant, applied_config_values=applied_config_values) rec1 = baker.bake() from piecrust.processing.pipeline import ProcessorPipeline proc = ProcessorPipeline( self.app, bake_out_dir, applied_config_variant=applied_config_variant, applied_config_values=applied_config_values) rec2 = proc.run() was_baked = True if not rec1.success or not rec2.success: raise Exception( "Error during baking, aborting publishing.") logger.info(format_timed(bake_start_time, "Baked website.")) else: logger.info("Would bake to: %s" % bake_out_dir) # Publish! logger.debug( "Running publish target '%s' with publisher: %s" % (target, pub.PUBLISHER_NAME)) pub_start_time = time.perf_counter() ctx = PublishingContext() ctx.bake_out_dir = bake_out_dir ctx.bake_record = rec1 ctx.processing_record = rec2 ctx.was_baked = was_baked ctx.preview = preview ctx.args = extra_args try: pub.run(ctx) except Exception as ex: raise PublishingError( "Error publishing to target: %s" % target) from ex finally: if hdlr: root_logger.removeHandler(hdlr) hdlr.close() logger.info(format_timed( pub_start_time, "Ran publisher %s" % pub.PUBLISHER_NAME)) logger.info(format_timed(start_time, 'Deployed to %s' % target))
def bake(self): logger.debug(" Bake Output: %s" % self.out_dir) logger.debug(" Root URL: %s" % self.app.config.get('site/root')) # Get into bake mode. start_time = time.perf_counter() self.app.config.set('baker/is_baking', True) self.app.env.base_asset_url_format = '%uri%' # Make sure the output directory exists. if not os.path.isdir(self.out_dir): os.makedirs(self.out_dir, 0o755) # Load/create the bake record. record = TransitionalBakeRecord() record_cache = self.app.cache.getCache('baker') record_id = hashlib.md5(self.out_dir.encode('utf8')).hexdigest() record_name = record_id + '.record' previous_record_path = None if not self.force and record_cache.has(record_name): with format_timed_scope(logger, "loaded previous bake record", level=logging.DEBUG, colored=False): previous_record_path = record_cache.getCachePath(record_name) record.loadPrevious(previous_record_path) record.current.success = True # Figure out if we need to clean the cache because important things # have changed. is_cache_valid = self._handleCacheValidity(record) if not is_cache_valid: previous_record_path = None # Pre-create all caches. for cache_name in ['app', 'baker', 'pages', 'renders']: self.app.cache.getCache(cache_name) # Gather all sources by realm -- we're going to bake each realm # separately so we can handle "overriding" (i.e. one realm overrides # another realm's pages, like the user realm overriding the theme # realm). sources_by_realm = {} for source in self.app.sources: srclist = sources_by_realm.setdefault(source.realm, []) srclist.append(source) # Create the worker processes. pool = self._createWorkerPool(previous_record_path) # Bake the realms. realm_list = [REALM_USER, REALM_THEME] for realm in realm_list: srclist = sources_by_realm.get(realm) if srclist is not None: self._bakeRealm(record, pool, realm, srclist) # Bake taxonomies. self._bakeTaxonomies(record, pool) # All done with the workers. Close the pool and get timing reports. reports = pool.close() record.current.timers = {} for i in range(len(reports)): timers = reports[i] if timers is None: continue worker_name = 'BakeWorker_%d' % i record.current.timers[worker_name] = {} for name, val in timers['data'].items(): main_val = record.current.timers.setdefault(name, 0) record.current.timers[name] = main_val + val record.current.timers[worker_name][name] = val # Delete files from the output. self._handleDeletetions(record) # Backup previous records. for i in range(8, -1, -1): suffix = '' if i == 0 else '.%d' % i record_path = record_cache.getCachePath( '%s%s.record' % (record_id, suffix)) if os.path.exists(record_path): record_path_next = record_cache.getCachePath( '%s.%s.record' % (record_id, i + 1)) if os.path.exists(record_path_next): os.remove(record_path_next) os.rename(record_path, record_path_next) # Save the bake record. with format_timed_scope(logger, "saved bake record.", level=logging.DEBUG, colored=False): record.current.bake_time = time.time() record.current.out_dir = self.out_dir record.saveCurrent(record_cache.getCachePath(record_name)) # All done. self.app.config.set('baker/is_baking', False) logger.debug(format_timed(start_time, 'done baking')) return record.detach()
def run(self, src_dir_or_file=None, *, delete=True, previous_record=None, save_record=True): start_time = time.perf_counter() # Get the list of processors for this run. processors = self.app.plugin_loader.getProcessors() if self.enabled_processors is not None: logger.debug("Filtering processors to: %s" % self.enabled_processors) processors = get_filtered_processors(processors, self.enabled_processors) if self.additional_processors_factories is not None: logger.debug("Adding %s additional processors." % len(self.additional_processors_factories)) for proc_fac in self.additional_processors_factories: proc = proc_fac() self.app.env.registerTimer(proc.__class__.__name__, raise_if_registered=False) proc.initialize(self.app) processors.append(proc) # Invoke pre-processors. pipeline_ctx = PipelineContext(-1, self.app, self.out_dir, self.tmp_dir, self.force) for proc in processors: proc.onPipelineStart(pipeline_ctx) # Pre-processors can define additional ignore patterns. self.ignore_patterns += make_re( pipeline_ctx._additional_ignore_patterns) # Create the pipeline record. record = TransitionalProcessorPipelineRecord() record_cache = self.app.cache.getCache('proc') record_name = (hashlib.md5(self.out_dir.encode('utf8')).hexdigest() + '.record') if previous_record: record.setPrevious(previous_record) elif not self.force and record_cache.has(record_name): with format_timed_scope(logger, 'loaded previous bake record', level=logging.DEBUG, colored=False): record.loadPrevious(record_cache.getCachePath(record_name)) logger.debug("Got %d entries in process record." % len(record.previous.entries)) record.current.success = True record.current.processed_count = 0 # Work! def _handler(res): entry = record.getCurrentEntry(res.path) assert entry is not None entry.flags = res.flags entry.proc_tree = res.proc_tree entry.rel_outputs = res.rel_outputs if entry.flags & FLAG_PROCESSED: record.current.processed_count += 1 if res.errors: entry.errors += res.errors record.current.success = False rel_path = os.path.relpath(res.path, self.app.root_dir) logger.error("Errors found in %s:" % rel_path) for e in entry.errors: logger.error(" " + e) jobs = [] self._process(src_dir_or_file, record, jobs) pool = self._createWorkerPool() ar = pool.queueJobs(jobs, handler=_handler) ar.wait() # Shutdown the workers and get timing information from them. reports = pool.close() record.current.timers = {} for i in range(len(reports)): timers = reports[i] if timers is None: continue worker_name = 'PipelineWorker_%d' % i record.current.timers[worker_name] = {} for name, val in timers['data'].items(): main_val = record.current.timers.setdefault(name, 0) record.current.timers[name] = main_val + val record.current.timers[worker_name][name] = val # Invoke post-processors. pipeline_ctx.record = record.current for proc in processors: proc.onPipelineEnd(pipeline_ctx) # Handle deletions. if delete: for path, reason in record.getDeletions(): logger.debug("Removing '%s': %s" % (path, reason)) try: os.remove(path) except FileNotFoundError: pass logger.info('[delete] %s' % path) # Finalize the process record. record.current.process_time = time.time() record.current.out_dir = self.out_dir record.collapseRecords() # Save the process record. if save_record: with format_timed_scope(logger, 'saved bake record', level=logging.DEBUG, colored=False): record.saveCurrent(record_cache.getCachePath(record_name)) logger.info( format_timed( start_time, "processed %d assets." % record.current.processed_count)) return record.detach()
def _run_chef(pre_args, argv): # Setup the app. root = None if pre_args.root: root = os.path.expanduser(pre_args.root) else: try: root = find_app_root(theme=pre_args.theme) except SiteNotFoundError: root = None # Can't apply custom configuration stuff if there's no website. if (pre_args.config_variants or pre_args.config_values) and not root: raise SiteNotFoundError( "Can't apply any configuration variant or value overrides, " "there is no website here.") if root: cache_key = None if not pre_args.no_cache: cache_key = _build_cache_key(pre_args) appfactory = PieCrustFactory( root, theme_site=pre_args.theme, cache=(not pre_args.no_cache), cache_key=cache_key, debug=pre_args.debug, config_variants=pre_args.config_variants, config_values=pre_args.config_values) app = appfactory.create() else: appfactory = None app = NullPieCrust( theme_site=pre_args.theme) # Setup the arg parser. parser = argparse.ArgumentParser( prog='chef', description="The PieCrust chef manages your website.", formatter_class=argparse.RawDescriptionHelpFormatter) _setup_main_parser_arguments(parser) commands = sorted(app.plugin_loader.getCommands(), key=lambda c: c.name) subparsers = parser.add_subparsers(title='list of commands') for c in commands: p = subparsers.add_parser(c.name, help=c.description) c.setupParser(p, app) p.set_defaults(func=c.checkedRun) p.set_defaults(cache_name=c.cache_name) help_cmd = next(filter(lambda c: c.name == 'help', commands), None) if help_cmd and help_cmd.has_topics: with io.StringIO() as epilog: epilog.write("additional help topics:\n") for name, desc in help_cmd.getTopics(): print_help_item(epilog, name, desc) parser.epilog = epilog.getvalue() # Parse the command line. result = parser.parse_args(argv) logger.debug(format_timed(_chef_start_time, 'initialized PieCrust', colored=False)) # Print the help if no command was specified. if not hasattr(result, 'func'): parser.print_help() return 0 # Do any custom setup the user wants. custom_env = app.config.get('chef/env') if custom_env: _setup_app_environment(app, custom_env) # Add some timing information. if app.env: app.env.stats.registerTimer('ChefStartup') app.env.stats.stepTimerSince('ChefStartup', _chef_start_time) # Run the command! ctx = CommandContext(appfactory, app, parser, result) exit_code = result.func(ctx) if exit_code is None: return 0 if not isinstance(exit_code, int): logger.error("Got non-integer exit code: %s" % exit_code) return -1 return exit_code
def run(self, src_dir_or_file=None, *, delete=True, previous_record=None, save_record=True): # Invoke pre-processors. for proc in self.processors: proc.onPipelineStart(self) # Sort our processors again in case the pre-process step involved # patching the processors with some new ones. self.processors.sort(key=lambda p: p.priority) # Create the pipeline record. record = TransitionalProcessorPipelineRecord() record_cache = self.app.cache.getCache('proc') record_name = (hashlib.md5(self.out_dir.encode('utf8')).hexdigest() + '.record') if previous_record: record.setPrevious(previous_record) elif not self.force and record_cache.has(record_name): t = time.clock() record.loadPrevious(record_cache.getCachePath(record_name)) logger.debug( format_timed(t, 'loaded previous bake record', colored=False)) logger.debug("Got %d entries in process record." % len(record.previous.entries)) # Create the workers. pool = [] queue = Queue() abort = threading.Event() pipeline_lock = threading.Lock() for i in range(self.num_workers): ctx = ProcessingWorkerContext(self, record, queue, abort, pipeline_lock) worker = ProcessingWorker(i, ctx) worker.start() pool.append(worker) if src_dir_or_file is not None: # Process only the given path. # Find out what mount point this is in. for name, info in self.mounts.items(): path = info['path'] if src_dir_or_file[:len(path)] == path: base_dir = path mount_info = info break else: known_roots = [i['path'] for i in self.mounts.values()] raise Exception("Input path '%s' is not part of any known " "mount point: %s" % (src_dir_or_file, known_roots)) ctx = ProcessingContext(base_dir, mount_info, queue, record) logger.debug("Initiating processing pipeline on: %s" % src_dir_or_file) if os.path.isdir(src_dir_or_file): self.processDirectory(ctx, src_dir_or_file) elif os.path.isfile(src_dir_or_file): self.processFile(ctx, src_dir_or_file) else: # Process everything. for name, info in self.mounts.items(): path = info['path'] ctx = ProcessingContext(path, info, queue, record) logger.debug("Initiating processing pipeline on: %s" % path) self.processDirectory(ctx, path) # Wait on all workers. record.current.success = True for w in pool: w.join() record.current.success &= w.success if abort.is_set(): raise Exception("Worker pool was aborted.") # Handle deletions. if delete: for path, reason in record.getDeletions(): logger.debug("Removing '%s': %s" % (path, reason)) try: os.remove(path) except FileNotFoundError: pass logger.info('[delete] %s' % path) # Invoke post-processors. for proc in self.processors: proc.onPipelineEnd(self) # Finalize the process record. record.current.process_time = time.time() record.current.out_dir = self.out_dir record.collapseRecords() # Save the process record. if save_record: t = time.clock() record.saveCurrent(record_cache.getCachePath(record_name)) logger.debug(format_timed(t, 'saved bake record', colored=False)) return record.detach()