def test_build_simple_tree(): builder = ProcessingTreeBuilder(mock_processors) root = builder.build('/foo.mock') assert root is not None assert root.getProcessor() == mock_processors[IDX_MOCK] assert not root.is_leaf assert len(root.outputs) == 1 out = root.outputs[0] assert out.getProcessor() == mock_processors[IDX_COPY]
def process(self, job): result = ProcessingWorkerResult(job.path) processors = get_filtered_processors(self.processors, job.mount_info['processors']) # Build the processing tree for this job. rel_path = os.path.relpath(job.path, job.base_dir) try: with self.app.env.timerScope('BuildProcessingTree'): builder = ProcessingTreeBuilder(processors) tree_root = builder.build(rel_path) result.flags |= FLAG_PREPARED except ProcessingTreeError as ex: result.errors = _get_errors(ex) return result # Prepare and run the tree. print_node(tree_root, recursive=True) leaves = tree_root.getLeaves() result.rel_outputs = [l.path for l in leaves] result.proc_tree = get_node_name_tree(tree_root) if tree_root.getProcessor().is_bypassing_structured_processing: result.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING if job.force: tree_root.setState(STATE_DIRTY, True) try: with self.app.env.timerScope('RunProcessingTree'): runner = ProcessingTreeRunner(job.base_dir, self.ctx.tmp_dir, self.ctx.out_dir) if runner.processSubTree(tree_root): result.flags |= FLAG_PROCESSED except ProcessingTreeError as ex: if isinstance(ex, ProcessorError): ex = ex.__cause__ # Need to strip out colored errors from external processes. result.errors = _get_errors(ex, strip_colors=True) return result
def _unsafeRun(self, job): start_time = time.clock() pipeline = self.ctx.pipeline record = self.ctx.record rel_path = os.path.relpath(job.path, job.base_dir) previous_entry = record.getPreviousEntry(rel_path) record_entry = ProcessorPipelineRecordEntry(job.base_dir, rel_path) record.addEntry(record_entry) # Figure out if a previously processed file is overriding this one. # This can happen if a theme file (processed via a mount point) # is overridden in the user's website. if record.current.hasOverrideEntry(rel_path): record_entry.flags |= FLAG_OVERRIDEN logger.info( format_timed(start_time, '%s [not baked, overridden]' % rel_path)) return True processors = pipeline.getFilteredProcessors( job.mount_info['processors']) try: builder = ProcessingTreeBuilder(processors) tree_root = builder.build(rel_path) record_entry.flags |= FLAG_PREPARED except ProcessingTreeError as ex: msg = str(ex) logger.error("Error preparing %s:\n%s" % (rel_path, msg)) while ex: record_entry.errors.append(str(ex)) ex = ex.__cause__ return False print_node(tree_root, recursive=True) leaves = tree_root.getLeaves() record_entry.rel_outputs = [l.path for l in leaves] record_entry.proc_tree = get_node_name_tree(tree_root) if tree_root.getProcessor().is_bypassing_structured_processing: record_entry.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING force = (pipeline.force or previous_entry is None or not previous_entry.was_processed_successfully) if not force: force = re_matchany(rel_path, pipeline.force_patterns) if force: tree_root.setState(STATE_DIRTY, True) try: runner = ProcessingTreeRunner(job.base_dir, pipeline.tmp_dir, pipeline.out_dir, self.ctx.pipeline_lock) if runner.processSubTree(tree_root): record_entry.flags |= FLAG_PROCESSED logger.info( format_timed(start_time, "[%d] %s" % (self.wid, rel_path))) return True except ProcessingTreeError as ex: msg = str(ex) if isinstance(ex, ProcessorError): msg = str(ex.__cause__) logger.error("Error processing %s:\n%s" % (rel_path, msg)) while ex: msg = re_ansicolors.sub('', str(ex)) record_entry.errors.append(msg) ex = ex.__cause__ return False