def run(self, src_dir_or_file=None, *,
delete=True, previous_record=None, save_record=True):
start_time = time.perf_counter()
# Get the list of processors for this run.
processors = self.app.plugin_loader.getProcessors()
if self.enabled_processors is not None:
logger.debug("Filtering processors to: %s" %
self.enabled_processors)
processors = get_filtered_processors(processors,
self.enabled_processors)
if self.additional_processors_factories is not None:
logger.debug("Adding %s additional processors." %
len(self.additional_processors_factories))
for proc_fac in self.additional_processors_factories:
proc = proc_fac()
self.app.env.registerTimer(proc.__class__.__name__,
raise_if_registered=False)
proc.initialize(self.app)
processors.append(proc)
# Invoke pre-processors.
pipeline_ctx = PipelineContext(-1, self.app, self.out_dir,
self.tmp_dir, self.force)
for proc in processors:
proc.onPipelineStart(pipeline_ctx)
# Pre-processors can define additional ignore patterns.
self.ignore_patterns += make_re(
pipeline_ctx._additional_ignore_patterns)
# Create the pipeline record.
record = TransitionalProcessorPipelineRecord()
record_cache = self.app.cache.getCache('proc')
record_name = (
hashlib.md5(self.out_dir.encode('utf8')).hexdigest() +
'.record')
if previous_record:
record.setPrevious(previous_record)
elif not self.force and record_cache.has(record_name):
with format_timed_scope(logger, 'loaded previous bake record',
level=logging.DEBUG, colored=False):
record.loadPrevious(record_cache.getCachePath(record_name))
logger.debug("Got %d entries in process record." %
len(record.previous.entries))
record.current.success = True
record.current.processed_count = 0
# Work!
def _handler(res):
entry = record.getCurrentEntry(res.path)
assert entry is not None
entry.flags = res.flags
entry.proc_tree = res.proc_tree
entry.rel_outputs = res.rel_outputs
if entry.flags & FLAG_PROCESSED:
record.current.processed_count += 1
if res.errors:
entry.errors += res.errors
record.current.success = False
rel_path = os.path.relpath(res.path, self.app.root_dir)
logger.error("Errors found in %s:" % rel_path)
for e in entry.errors:
logger.error(" " + e)
jobs = []
self._process(src_dir_or_file, record, jobs)
pool = self._createWorkerPool()
ar = pool.queueJobs(jobs, handler=_handler)
ar.wait()
# Shutdown the workers and get timing information from them.
reports = pool.close()
record.current.timers = {}
for i in range(len(reports)):
timers = reports[i]
if timers is None:
continue
worker_name = 'PipelineWorker_%d' % i
record.current.timers[worker_name] = {}
for name, val in timers['data'].items():
main_val = record.current.timers.setdefault(name, 0)
record.current.timers[name] = main_val + val
record.current.timers[worker_name][name] = val
# Invoke post-processors.
pipeline_ctx.record = record.current
for proc in processors:
proc.onPipelineEnd(pipeline_ctx)
# Handle deletions.
if delete:
for path, reason in record.getDeletions():
logger.debug("Removing '%s': %s" % (path, reason))
try:
os.remove(path)
except FileNotFoundError:
pass
logger.info('[delete] %s' % path)
# Finalize the process record.
record.current.process_time = time.time()
record.current.out_dir = self.out_dir
record.collapseRecords()
# Save the process record.
if save_record:
with format_timed_scope(logger, 'saved bake record',
level=logging.DEBUG, colored=False):
record.saveCurrent(record_cache.getCachePath(record_name))
logger.info(format_timed(
start_time,
"processed %d assets." % record.current.processed_count))
return record.detach()
def run(self,
src_dir_or_file=None,
*,
delete=True,
previous_record=None,
save_record=True):
# Invoke pre-processors.
for proc in self.processors:
proc.onPipelineStart(self)
# Sort our processors again in case the pre-process step involved
# patching the processors with some new ones.
self.processors.sort(key=lambda p: p.priority)
# Create the pipeline record.
record = TransitionalProcessorPipelineRecord()
record_cache = self.app.cache.getCache('proc')
record_name = (hashlib.md5(self.out_dir.encode('utf8')).hexdigest() +
'.record')
if previous_record:
record.setPrevious(previous_record)
elif not self.force and record_cache.has(record_name):
t = time.clock()
record.loadPrevious(record_cache.getCachePath(record_name))
logger.debug(
format_timed(t, 'loaded previous bake record', colored=False))
logger.debug("Got %d entries in process record." %
len(record.previous.entries))
# Create the workers.
pool = []
queue = Queue()
abort = threading.Event()
pipeline_lock = threading.Lock()
for i in range(self.num_workers):
ctx = ProcessingWorkerContext(self, record, queue, abort,
pipeline_lock)
worker = ProcessingWorker(i, ctx)
worker.start()
pool.append(worker)
if src_dir_or_file is not None:
# Process only the given path.
# Find out what mount point this is in.
for name, info in self.mounts.items():
path = info['path']
if src_dir_or_file[:len(path)] == path:
base_dir = path
mount_info = info
break
else:
known_roots = [i['path'] for i in self.mounts.values()]
raise Exception("Input path '%s' is not part of any known "
"mount point: %s" %
(src_dir_or_file, known_roots))
ctx = ProcessingContext(base_dir, mount_info, queue, record)
logger.debug("Initiating processing pipeline on: %s" %
src_dir_or_file)
if os.path.isdir(src_dir_or_file):
self.processDirectory(ctx, src_dir_or_file)
elif os.path.isfile(src_dir_or_file):
self.processFile(ctx, src_dir_or_file)
else:
# Process everything.
for name, info in self.mounts.items():
path = info['path']
ctx = ProcessingContext(path, info, queue, record)
logger.debug("Initiating processing pipeline on: %s" % path)
self.processDirectory(ctx, path)
# Wait on all workers.
record.current.success = True
for w in pool:
w.join()
record.current.success &= w.success
if abort.is_set():
raise Exception("Worker pool was aborted.")
# Handle deletions.
if delete:
for path, reason in record.getDeletions():
logger.debug("Removing '%s': %s" % (path, reason))
try:
os.remove(path)
except FileNotFoundError:
pass
logger.info('[delete] %s' % path)
# Invoke post-processors.
for proc in self.processors:
proc.onPipelineEnd(self)
# Finalize the process record.
record.current.process_time = time.time()
record.current.out_dir = self.out_dir
record.collapseRecords()
# Save the process record.
if save_record:
t = time.clock()
record.saveCurrent(record_cache.getCachePath(record_name))
logger.debug(format_timed(t, 'saved bake record', colored=False))
return record.detach()
def run(self,
src_dir_or_file=None,
*,
delete=True,
previous_record=None,
save_record=True):
start_time = time.perf_counter()
# Get the list of processors for this run.
processors = self.app.plugin_loader.getProcessors()
if self.enabled_processors is not None:
logger.debug("Filtering processors to: %s" %
self.enabled_processors)
processors = get_filtered_processors(processors,
self.enabled_processors)
if self.additional_processors_factories is not None:
logger.debug("Adding %s additional processors." %
len(self.additional_processors_factories))
for proc_fac in self.additional_processors_factories:
proc = proc_fac()
self.app.env.registerTimer(proc.__class__.__name__,
raise_if_registered=False)
proc.initialize(self.app)
processors.append(proc)
# Invoke pre-processors.
pipeline_ctx = PipelineContext(-1, self.app, self.out_dir,
self.tmp_dir, self.force)
for proc in processors:
proc.onPipelineStart(pipeline_ctx)
# Pre-processors can define additional ignore patterns.
self.ignore_patterns += make_re(
pipeline_ctx._additional_ignore_patterns)
# Create the pipeline record.
record = TransitionalProcessorPipelineRecord()
record_cache = self.app.cache.getCache('proc')
record_name = (hashlib.md5(self.out_dir.encode('utf8')).hexdigest() +
'.record')
if previous_record:
record.setPrevious(previous_record)
elif not self.force and record_cache.has(record_name):
with format_timed_scope(logger,
'loaded previous bake record',
level=logging.DEBUG,
colored=False):
record.loadPrevious(record_cache.getCachePath(record_name))
logger.debug("Got %d entries in process record." %
len(record.previous.entries))
record.current.success = True
record.current.processed_count = 0
# Work!
def _handler(res):
entry = record.getCurrentEntry(res.path)
assert entry is not None
entry.flags = res.flags
entry.proc_tree = res.proc_tree
entry.rel_outputs = res.rel_outputs
if entry.flags & FLAG_PROCESSED:
record.current.processed_count += 1
if res.errors:
entry.errors += res.errors
record.current.success = False
rel_path = os.path.relpath(res.path, self.app.root_dir)
logger.error("Errors found in %s:" % rel_path)
for e in entry.errors:
logger.error(" " + e)
jobs = []
self._process(src_dir_or_file, record, jobs)
pool = self._createWorkerPool()
ar = pool.queueJobs(jobs, handler=_handler)
ar.wait()
# Shutdown the workers and get timing information from them.
reports = pool.close()
record.current.timers = {}
for i in range(len(reports)):
timers = reports[i]
if timers is None:
continue
worker_name = 'PipelineWorker_%d' % i
record.current.timers[worker_name] = {}
for name, val in timers['data'].items():
main_val = record.current.timers.setdefault(name, 0)
record.current.timers[name] = main_val + val
record.current.timers[worker_name][name] = val
# Invoke post-processors.
pipeline_ctx.record = record.current
for proc in processors:
proc.onPipelineEnd(pipeline_ctx)
# Handle deletions.
if delete:
for path, reason in record.getDeletions():
logger.debug("Removing '%s': %s" % (path, reason))
try:
os.remove(path)
except FileNotFoundError:
pass
logger.info('[delete] %s' % path)
# Finalize the process record.
record.current.process_time = time.time()
record.current.out_dir = self.out_dir
record.collapseRecords()
# Save the process record.
if save_record:
with format_timed_scope(logger,
'saved bake record',
level=logging.DEBUG,
colored=False):
record.saveCurrent(record_cache.getCachePath(record_name))
logger.info(
format_timed(
start_time,
"processed %d assets." % record.current.processed_count))
return record.detach()
def run(self, src_dir_or_file=None, *,
delete=True, previous_record=None, save_record=True):
# Invoke pre-processors.
for proc in self.processors:
proc.onPipelineStart(self)
# Sort our processors again in case the pre-process step involved
# patching the processors with some new ones.
self.processors.sort(key=lambda p: p.priority)
# Create the pipeline record.
record = TransitionalProcessorPipelineRecord()
record_cache = self.app.cache.getCache('proc')
record_name = (
hashlib.md5(self.out_dir.encode('utf8')).hexdigest() +
'.record')
if previous_record:
record.setPrevious(previous_record)
elif not self.force and record_cache.has(record_name):
t = time.clock()
record.loadPrevious(record_cache.getCachePath(record_name))
logger.debug(format_timed(t, 'loaded previous bake record',
colored=False))
logger.debug("Got %d entries in process record." %
len(record.previous.entries))
# Create the workers.
pool = []
queue = Queue()
abort = threading.Event()
pipeline_lock = threading.Lock()
for i in range(self.num_workers):
ctx = ProcessingWorkerContext(self, record,
queue, abort, pipeline_lock)
worker = ProcessingWorker(i, ctx)
worker.start()
pool.append(worker)
if src_dir_or_file is not None:
# Process only the given path.
# Find out what mount point this is in.
for name, info in self.mounts.items():
path = info['path']
if src_dir_or_file[:len(path)] == path:
base_dir = path
mount_info = info
break
else:
known_roots = [i['path'] for i in self.mounts.values()]
raise Exception("Input path '%s' is not part of any known "
"mount point: %s" %
(src_dir_or_file, known_roots))
ctx = ProcessingContext(base_dir, mount_info, queue, record)
logger.debug("Initiating processing pipeline on: %s" % src_dir_or_file)
if os.path.isdir(src_dir_or_file):
self.processDirectory(ctx, src_dir_or_file)
elif os.path.isfile(src_dir_or_file):
self.processFile(ctx, src_dir_or_file)
else:
# Process everything.
for name, info in self.mounts.items():
path = info['path']
ctx = ProcessingContext(path, info, queue, record)
logger.debug("Initiating processing pipeline on: %s" % path)
self.processDirectory(ctx, path)
# Wait on all workers.
record.current.success = True
for w in pool:
w.join()
record.current.success &= w.success
if abort.is_set():
raise Exception("Worker pool was aborted.")
# Handle deletions.
if delete:
for path, reason in record.getDeletions():
logger.debug("Removing '%s': %s" % (path, reason))
try:
os.remove(path)
except FileNotFoundError:
pass
logger.info('[delete] %s' % path)
# Invoke post-processors.
for proc in self.processors:
proc.onPipelineEnd(self)
# Finalize the process record.
record.current.process_time = time.time()
record.current.out_dir = self.out_dir
record.collapseRecords()
# Save the process record.
if save_record:
t = time.clock()
record.saveCurrent(record_cache.getCachePath(record_name))
logger.debug(format_timed(t, 'saved bake record', colored=False))
return record.detach()