def test_build_simple_tree():
builder = ProcessingTreeBuilder(mock_processors)
root = builder.build('/foo.mock')
assert root is not None
assert root.getProcessor() == mock_processors[IDX_MOCK]
assert not root.is_leaf
assert len(root.outputs) == 1
out = root.outputs[0]
assert out.getProcessor() == mock_processors[IDX_COPY]
def process(self, job):
result = ProcessingWorkerResult(job.path)
processors = get_filtered_processors(self.processors,
job.mount_info['processors'])
# Build the processing tree for this job.
rel_path = os.path.relpath(job.path, job.base_dir)
try:
with self.app.env.timerScope('BuildProcessingTree'):
builder = ProcessingTreeBuilder(processors)
tree_root = builder.build(rel_path)
result.flags |= FLAG_PREPARED
except ProcessingTreeError as ex:
result.errors = _get_errors(ex)
return result
# Prepare and run the tree.
print_node(tree_root, recursive=True)
leaves = tree_root.getLeaves()
result.rel_outputs = [l.path for l in leaves]
result.proc_tree = get_node_name_tree(tree_root)
if tree_root.getProcessor().is_bypassing_structured_processing:
result.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING
if job.force:
tree_root.setState(STATE_DIRTY, True)
try:
with self.app.env.timerScope('RunProcessingTree'):
runner = ProcessingTreeRunner(job.base_dir, self.ctx.tmp_dir,
self.ctx.out_dir)
if runner.processSubTree(tree_root):
result.flags |= FLAG_PROCESSED
except ProcessingTreeError as ex:
if isinstance(ex, ProcessorError):
ex = ex.__cause__
# Need to strip out colored errors from external processes.
result.errors = _get_errors(ex, strip_colors=True)
return result
def process(self, job):
result = ProcessingWorkerResult(job.path)
processors = get_filtered_processors(
self.processors, job.mount_info['processors'])
# Build the processing tree for this job.
rel_path = os.path.relpath(job.path, job.base_dir)
try:
with self.app.env.timerScope('BuildProcessingTree'):
builder = ProcessingTreeBuilder(processors)
tree_root = builder.build(rel_path)
result.flags |= FLAG_PREPARED
except ProcessingTreeError as ex:
result.errors = _get_errors(ex)
return result
# Prepare and run the tree.
print_node(tree_root, recursive=True)
leaves = tree_root.getLeaves()
result.rel_outputs = [l.path for l in leaves]
result.proc_tree = get_node_name_tree(tree_root)
if tree_root.getProcessor().is_bypassing_structured_processing:
result.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING
if job.force:
tree_root.setState(STATE_DIRTY, True)
try:
with self.app.env.timerScope('RunProcessingTree'):
runner = ProcessingTreeRunner(
job.base_dir, self.ctx.tmp_dir, self.ctx.out_dir)
if runner.processSubTree(tree_root):
result.flags |= FLAG_PROCESSED
except ProcessingTreeError as ex:
if isinstance(ex, ProcessorError):
ex = ex.__cause__
# Need to strip out colored errors from external processes.
result.errors = _get_errors(ex, strip_colors=True)
return result
def _unsafeRun(self, job):
start_time = time.clock()
pipeline = self.ctx.pipeline
record = self.ctx.record
rel_path = os.path.relpath(job.path, job.base_dir)
previous_entry = record.getPreviousEntry(rel_path)
record_entry = ProcessorPipelineRecordEntry(job.base_dir, rel_path)
record.addEntry(record_entry)
# Figure out if a previously processed file is overriding this one.
# This can happen if a theme file (processed via a mount point)
# is overridden in the user's website.
if record.current.hasOverrideEntry(rel_path):
record_entry.flags |= FLAG_OVERRIDEN
logger.info(
format_timed(start_time,
'%s [not baked, overridden]' % rel_path))
return True
processors = pipeline.getFilteredProcessors(
job.mount_info['processors'])
try:
builder = ProcessingTreeBuilder(processors)
tree_root = builder.build(rel_path)
record_entry.flags |= FLAG_PREPARED
except ProcessingTreeError as ex:
msg = str(ex)
logger.error("Error preparing %s:\n%s" % (rel_path, msg))
while ex:
record_entry.errors.append(str(ex))
ex = ex.__cause__
return False
print_node(tree_root, recursive=True)
leaves = tree_root.getLeaves()
record_entry.rel_outputs = [l.path for l in leaves]
record_entry.proc_tree = get_node_name_tree(tree_root)
if tree_root.getProcessor().is_bypassing_structured_processing:
record_entry.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING
force = (pipeline.force or previous_entry is None
or not previous_entry.was_processed_successfully)
if not force:
force = re_matchany(rel_path, pipeline.force_patterns)
if force:
tree_root.setState(STATE_DIRTY, True)
try:
runner = ProcessingTreeRunner(job.base_dir, pipeline.tmp_dir,
pipeline.out_dir,
self.ctx.pipeline_lock)
if runner.processSubTree(tree_root):
record_entry.flags |= FLAG_PROCESSED
logger.info(
format_timed(start_time, "[%d] %s" % (self.wid, rel_path)))
return True
except ProcessingTreeError as ex:
msg = str(ex)
if isinstance(ex, ProcessorError):
msg = str(ex.__cause__)
logger.error("Error processing %s:\n%s" % (rel_path, msg))
while ex:
msg = re_ansicolors.sub('', str(ex))
record_entry.errors.append(msg)
ex = ex.__cause__
return False
def _unsafeRun(self, job):
start_time = time.clock()
pipeline = self.ctx.pipeline
record = self.ctx.record
rel_path = os.path.relpath(job.path, job.base_dir)
previous_entry = record.getPreviousEntry(rel_path)
record_entry = ProcessorPipelineRecordEntry(job.base_dir, rel_path)
record.addEntry(record_entry)
# Figure out if a previously processed file is overriding this one.
# This can happen if a theme file (processed via a mount point)
# is overridden in the user's website.
if record.current.hasOverrideEntry(rel_path):
record_entry.flags |= FLAG_OVERRIDEN
logger.info(format_timed(start_time,
'%s [not baked, overridden]' % rel_path))
return True
processors = pipeline.getFilteredProcessors(
job.mount_info['processors'])
try:
builder = ProcessingTreeBuilder(processors)
tree_root = builder.build(rel_path)
record_entry.flags |= FLAG_PREPARED
except ProcessingTreeError as ex:
msg = str(ex)
logger.error("Error preparing %s:\n%s" % (rel_path, msg))
while ex:
record_entry.errors.append(str(ex))
ex = ex.__cause__
return False
print_node(tree_root, recursive=True)
leaves = tree_root.getLeaves()
record_entry.rel_outputs = [l.path for l in leaves]
record_entry.proc_tree = get_node_name_tree(tree_root)
if tree_root.getProcessor().is_bypassing_structured_processing:
record_entry.flags |= FLAG_BYPASSED_STRUCTURED_PROCESSING
force = (pipeline.force or previous_entry is None or
not previous_entry.was_processed_successfully)
if not force:
force = re_matchany(rel_path, pipeline.force_patterns)
if force:
tree_root.setState(STATE_DIRTY, True)
try:
runner = ProcessingTreeRunner(
job.base_dir, pipeline.tmp_dir,
pipeline.out_dir, self.ctx.pipeline_lock)
if runner.processSubTree(tree_root):
record_entry.flags |= FLAG_PROCESSED
logger.info(format_timed(
start_time, "[%d] %s" % (self.wid, rel_path)))
return True
except ProcessingTreeError as ex:
msg = str(ex)
if isinstance(ex, ProcessorError):
msg = str(ex.__cause__)
logger.error("Error processing %s:\n%s" % (rel_path, msg))
while ex:
msg = re_ansicolors.sub('', str(ex))
record_entry.errors.append(msg)
ex = ex.__cause__
return False