Esempio n. 1
0
def test_build_simple_tree():
    builder = ProcessingTreeBuilder(mock_processors)
    root = builder.build('/foo.mock')
    assert root is not None
    assert root.getProcessor() == mock_processors[IDX_MOCK]
    assert not root.is_leaf
    assert len(root.outputs) == 1
    out = root.outputs[0]
    assert out.getProcessor() == mock_processors[IDX_COPY]
Esempio n. 2
0
    def process(self, job):
        result = ProcessingWorkerResult(job.path)

        processors = get_filtered_processors(self.processors,
                                             job.mount_info['processors'])

        # Build the processing tree for this job.
        rel_path = os.path.relpath(job.path, job.base_dir)
        try:
            with self.app.env.timerScope('BuildProcessingTree'):
                builder = ProcessingTreeBuilder(processors)
                tree_root = builder.build(rel_path)
                result.flags |= FLAG_PREPARED
        except ProcessingTreeError as ex:
            result.errors = _get_errors(ex)
            return result

        # Prepare and run the tree.
        print_node(tree_root, recursive=True)
        leaves = tree_root.getLeaves()
        result.rel_outputs = [l.path for l in leaves]
        result.proc_tree = get_node_name_tree(tree_root)
        if tree_root.getProcessor().is_bypassing_structured_processing:
            result.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING

        if job.force:
            tree_root.setState(STATE_DIRTY, True)

        try:
            with self.app.env.timerScope('RunProcessingTree'):
                runner = ProcessingTreeRunner(job.base_dir, self.ctx.tmp_dir,
                                              self.ctx.out_dir)
                if runner.processSubTree(tree_root):
                    result.flags |= FLAG_PROCESSED
        except ProcessingTreeError as ex:
            if isinstance(ex, ProcessorError):
                ex = ex.__cause__
            # Need to strip out colored errors from external processes.
            result.errors = _get_errors(ex, strip_colors=True)

        return result
Esempio n. 3
0
    def process(self, job):
        result = ProcessingWorkerResult(job.path)

        processors = get_filtered_processors(
                self.processors, job.mount_info['processors'])

        # Build the processing tree for this job.
        rel_path = os.path.relpath(job.path, job.base_dir)
        try:
            with self.app.env.timerScope('BuildProcessingTree'):
                builder = ProcessingTreeBuilder(processors)
                tree_root = builder.build(rel_path)
                result.flags |= FLAG_PREPARED
        except ProcessingTreeError as ex:
            result.errors = _get_errors(ex)
            return result

        # Prepare and run the tree.
        print_node(tree_root, recursive=True)
        leaves = tree_root.getLeaves()
        result.rel_outputs = [l.path for l in leaves]
        result.proc_tree = get_node_name_tree(tree_root)
        if tree_root.getProcessor().is_bypassing_structured_processing:
            result.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING

        if job.force:
            tree_root.setState(STATE_DIRTY, True)

        try:
            with self.app.env.timerScope('RunProcessingTree'):
                runner = ProcessingTreeRunner(
                        job.base_dir, self.ctx.tmp_dir, self.ctx.out_dir)
                if runner.processSubTree(tree_root):
                    result.flags |= FLAG_PROCESSED
        except ProcessingTreeError as ex:
            if isinstance(ex, ProcessorError):
                ex = ex.__cause__
            # Need to strip out colored errors from external processes.
            result.errors = _get_errors(ex, strip_colors=True)

        return result
Esempio n. 4
0
    def _unsafeRun(self, job):
        start_time = time.clock()
        pipeline = self.ctx.pipeline
        record = self.ctx.record

        rel_path = os.path.relpath(job.path, job.base_dir)
        previous_entry = record.getPreviousEntry(rel_path)

        record_entry = ProcessorPipelineRecordEntry(job.base_dir, rel_path)
        record.addEntry(record_entry)

        # Figure out if a previously processed file is overriding this one.
        # This can happen if a theme file (processed via a mount point)
        # is overridden in the user's website.
        if record.current.hasOverrideEntry(rel_path):
            record_entry.flags |= FLAG_OVERRIDEN
            logger.info(
                format_timed(start_time,
                             '%s [not baked, overridden]' % rel_path))
            return True

        processors = pipeline.getFilteredProcessors(
            job.mount_info['processors'])
        try:
            builder = ProcessingTreeBuilder(processors)
            tree_root = builder.build(rel_path)
            record_entry.flags |= FLAG_PREPARED
        except ProcessingTreeError as ex:
            msg = str(ex)
            logger.error("Error preparing %s:\n%s" % (rel_path, msg))
            while ex:
                record_entry.errors.append(str(ex))
                ex = ex.__cause__
            return False

        print_node(tree_root, recursive=True)
        leaves = tree_root.getLeaves()
        record_entry.rel_outputs = [l.path for l in leaves]
        record_entry.proc_tree = get_node_name_tree(tree_root)
        if tree_root.getProcessor().is_bypassing_structured_processing:
            record_entry.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING

        force = (pipeline.force or previous_entry is None
                 or not previous_entry.was_processed_successfully)
        if not force:
            force = re_matchany(rel_path, pipeline.force_patterns)

        if force:
            tree_root.setState(STATE_DIRTY, True)

        try:
            runner = ProcessingTreeRunner(job.base_dir, pipeline.tmp_dir,
                                          pipeline.out_dir,
                                          self.ctx.pipeline_lock)
            if runner.processSubTree(tree_root):
                record_entry.flags |= FLAG_PROCESSED
                logger.info(
                    format_timed(start_time, "[%d] %s" % (self.wid, rel_path)))
            return True
        except ProcessingTreeError as ex:
            msg = str(ex)
            if isinstance(ex, ProcessorError):
                msg = str(ex.__cause__)
            logger.error("Error processing %s:\n%s" % (rel_path, msg))
            while ex:
                msg = re_ansicolors.sub('', str(ex))
                record_entry.errors.append(msg)
                ex = ex.__cause__
            return False
Esempio n. 5
0
    def _unsafeRun(self, job):
        start_time = time.clock()
        pipeline = self.ctx.pipeline
        record = self.ctx.record

        rel_path = os.path.relpath(job.path, job.base_dir)
        previous_entry = record.getPreviousEntry(rel_path)

        record_entry = ProcessorPipelineRecordEntry(job.base_dir, rel_path)
        record.addEntry(record_entry)

        # Figure out if a previously processed file is overriding this one.
        # This can happen if a theme file (processed via a mount point)
        # is overridden in the user's website.
        if record.current.hasOverrideEntry(rel_path):
            record_entry.flags |= FLAG_OVERRIDEN
            logger.info(format_timed(start_time,
                    '%s [not baked, overridden]' % rel_path))
            return True

        processors = pipeline.getFilteredProcessors(
                job.mount_info['processors'])
        try:
            builder = ProcessingTreeBuilder(processors)
            tree_root = builder.build(rel_path)
            record_entry.flags |= FLAG_PREPARED
        except ProcessingTreeError as ex:
            msg = str(ex)
            logger.error("Error preparing %s:\n%s" % (rel_path, msg))
            while ex:
                record_entry.errors.append(str(ex))
                ex = ex.__cause__
            return False

        print_node(tree_root, recursive=True)
        leaves = tree_root.getLeaves()
        record_entry.rel_outputs = [l.path for l in leaves]
        record_entry.proc_tree = get_node_name_tree(tree_root)
        if tree_root.getProcessor().is_bypassing_structured_processing:
            record_entry.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING

        force = (pipeline.force or previous_entry is None or
                 not previous_entry.was_processed_successfully)
        if not force:
            force = re_matchany(rel_path, pipeline.force_patterns)

        if force:
            tree_root.setState(STATE_DIRTY, True)

        try:
            runner = ProcessingTreeRunner(
                    job.base_dir, pipeline.tmp_dir,
                    pipeline.out_dir, self.ctx.pipeline_lock)
            if runner.processSubTree(tree_root):
                record_entry.flags |= FLAG_PROCESSED
                logger.info(format_timed(
                    start_time, "[%d] %s" % (self.wid, rel_path)))
            return True
        except ProcessingTreeError as ex:
            msg = str(ex)
            if isinstance(ex, ProcessorError):
                msg = str(ex.__cause__)
            logger.error("Error processing %s:\n%s" % (rel_path, msg))
            while ex:
                msg = re_ansicolors.sub('', str(ex))
                record_entry.errors.append(msg)
                ex = ex.__cause__
            return False