def _bakeAssets(self, ctx, out_dir):
proc = ProcessorPipeline(
ctx.app, out_dir,
force=ctx.args.force)
record = proc.run()
_merge_timers(record.timers, ctx.timers)
return record.success
def _bakeAssets(self, ctx, out_dir):
proc = ProcessorPipeline(
ctx.app, out_dir,
force=ctx.args.force,
applied_config_variant=ctx.config_variant,
applied_config_values=ctx.config_values)
record = proc.run()
_merge_stats(record.stats, ctx.stats)
return record.success
def _initPipeline(self):
# Create the app and pipeline.
self.app = PieCrust(root_dir=self.root_dir, debug=self.debug)
if self.sub_cache_dir:
self.app._useSubCacheDir(self.sub_cache_dir)
self.pipeline = ProcessorPipeline(self.app, self.out_dir)
# Get the list of assets directories.
self._roots = list(self.pipeline.mounts.keys())
# The 'assets' folder may not be in the mounts list if it doesn't
# exist yet, but we want to monitor for when the user creates it.
default_root = os.path.join(self.app.root_dir, 'assets')
self._monitor_assets_root = (default_root not in self._roots)
# Build the list of initial asset files.
self._paths = set()
for root in self._roots:
for dirpath, dirnames, filenames in os.walk(root):
self._paths |= set([os.path.join(dirpath, f)
for f in filenames])
def runtest(self):
fs = self._prepareMockFs()
from piecrust.processing.pipeline import ProcessorPipeline
with mock_fs_scope(fs):
out_dir = fs.path('kitchen/_counter')
app = fs.getApp()
pipeline = ProcessorPipeline(app, out_dir)
proc_names = self.spec.get('processors')
if proc_names:
pipeline.enabled_processors = proc_names
record = pipeline.run()
if not record.success:
errors = []
for e in record.entries:
errors += e.errors
raise PipelineError(errors)
check_expected_outputs(self.spec, fs, ExpectedPipelineOutputError)
def runtest(self):
fs = self._prepareMockFs()
from piecrust.processing.pipeline import ProcessorPipeline
with mock_fs_scope(fs):
out_dir = fs.path('kitchen/_counter')
app = fs.getApp()
pipeline = ProcessorPipeline(app, out_dir)
proc_names = self.spec.get('processors')
if proc_names:
pipeline.enabled_processors = proc_names
record = pipeline.run()
if not record.success:
errors = []
for e in record.entries:
errors += e.errors
raise PipelineError(errors)
check_expected_outputs(self.spec, fs, ExpectedPipelineOutputError)
def _initPipeline(self):
# Create the app and pipeline.
self.app = self.appfactory.create()
self.pipeline = ProcessorPipeline(self.app, self.out_dir)
# Get the list of assets directories.
self._roots = list(self.pipeline.mounts.keys())
# The 'assets' folder may not be in the mounts list if it doesn't
# exist yet, but we want to monitor for when the user creates it.
default_root = os.path.join(self.app.root_dir, 'assets')
self._monitor_assets_root = (default_root not in self._roots)
# Build the list of initial asset files.
self._paths = set()
for root in self._roots:
for dirpath, dirnames, filenames in os.walk(root):
self._paths |= set([os.path.join(dirpath, f)
for f in filenames])
def _initPipeline(self):
# Create the app and pipeline.
self.app = PieCrust(root_dir=self.root_dir, debug=self.debug)
if self.sub_cache_dir:
self.app._useSubCacheDir(self.sub_cache_dir)
self.pipeline = ProcessorPipeline(self.app, self.out_dir)
# Get the list of assets directories.
self._roots = list(self.pipeline.mounts.keys())
# The 'assets' folder may not be in the mounts list if it doesn't
# exist yet, but we want to monitor for when the user creates it.
default_root = os.path.join(self.app.root_dir, "assets")
self._monitor_assets_root = default_root not in self._roots
# Build the list of initial asset files.
self._paths = set()
for root in self._roots:
for dirpath, dirnames, filenames in os.walk(root):
self._paths |= set([os.path.join(dirpath, f) for f in filenames])
class ProcessingLoop(threading.Thread):
def __init__(self, root_dir, out_dir, sub_cache_dir=None, debug=False):
super(ProcessingLoop, self).__init__(
name='pipeline-reloader', daemon=True)
self.root_dir = root_dir
self.out_dir = out_dir
self.sub_cache_dir = sub_cache_dir
self.debug = debug
self.last_status_id = 0
self.interval = 1
self.app = None
self._roots = []
self._monitor_assets_root = False
self._paths = set()
self._config_path = os.path.join(root_dir, 'config.yml')
self._record = None
self._last_bake = 0
self._last_config_mtime = 0
self._obs = []
self._obs_lock = threading.Lock()
def addObserver(self, obs):
with self._obs_lock:
self._obs.append(obs)
def removeObserver(self, obs):
with self._obs_lock:
self._obs.remove(obs)
def run(self):
self._initPipeline()
self._last_bake = time.time()
self._last_config_mtime = os.path.getmtime(self._config_path)
self._record = self.pipeline.run()
while True:
cur_config_time = os.path.getmtime(self._config_path)
if self._last_config_mtime < cur_config_time:
logger.info("Site configuration changed, reloading pipeline.")
self._last_config_mtime = cur_config_time
self._initPipeline()
for root in self._roots:
self._runPipeline(root)
continue
if self._monitor_assets_root:
assets_dir = os.path.join(self.app.root_dir, 'assets')
if os.path.isdir(assets_dir):
logger.info("Assets directory was created, reloading "
"pipeline.")
self._initPipeline()
self._runPipeline(assets_dir)
continue
for root in self._roots:
# For each mount root we try to find the first new or
# modified file. If any, we just run the pipeline on
# that mount.
found_new_or_modified = False
for dirpath, dirnames, filenames in os.walk(root):
for filename in filenames:
path = os.path.join(dirpath, filename)
if path not in self._paths:
logger.debug("Found new asset: %s" % path)
self._paths.add(path)
found_new_or_modified = True
break
if os.path.getmtime(path) > self._last_bake:
logger.debug("Found modified asset: %s" % path)
found_new_or_modified = True
break
if found_new_or_modified:
break
if found_new_or_modified:
self._runPipeline(root)
time.sleep(self.interval)
def _initPipeline(self):
# Create the app and pipeline.
self.app = PieCrust(root_dir=self.root_dir, debug=self.debug)
if self.sub_cache_dir:
self.app._useSubCacheDir(self.sub_cache_dir)
self.pipeline = ProcessorPipeline(self.app, self.out_dir)
# Get the list of assets directories.
self._roots = list(self.pipeline.mounts.keys())
# The 'assets' folder may not be in the mounts list if it doesn't
# exist yet, but we want to monitor for when the user creates it.
default_root = os.path.join(self.app.root_dir, 'assets')
self._monitor_assets_root = (default_root not in self._roots)
# Build the list of initial asset files.
self._paths = set()
for root in self._roots:
for dirpath, dirnames, filenames in os.walk(root):
self._paths |= set([os.path.join(dirpath, f)
for f in filenames])
def _runPipeline(self, root):
self._last_bake = time.time()
try:
self._record = self.pipeline.run(
root,
previous_record=self._record,
save_record=False)
status_id = self.last_status_id + 1
self.last_status_id += 1
if self._record.success:
changed = filter(
lambda i: not i.was_collapsed_from_last_run,
self._record.entries)
changed = itertools.chain.from_iterable(
map(lambda i: i.rel_outputs, changed))
changed = list(changed)
item = {
'id': status_id,
'type': 'pipeline_success',
'assets': changed}
self._notifyObservers(item)
else:
item = {
'id': status_id,
'type': 'pipeline_error',
'assets': []}
for entry in self._record.entries:
if entry.errors:
asset_item = {
'path': entry.path,
'errors': list(entry.errors)}
item['assets'].append(asset_item)
self._notifyObservers(item)
except Exception as ex:
logger.exception(ex)
def _notifyObservers(self, item):
with self._obs_lock:
observers = list(self._obs)
for obs in observers:
obs.addBuildEvent(item)
def run(self, target,
force=False, preview=False, extra_args=None, log_file=None,
applied_config_variant=None, applied_config_values=None):
start_time = time.perf_counter()
# Get publisher for this target.
pub = self.app.getPublisher(target)
if pub is None:
raise InvalidPublishTargetError(
"No such publish target: %s" % target)
# Will we need to bake first?
bake_first = True
if not pub.has_url_config:
bake_first = pub.getConfigValue('bake', True)
# Setup logging stuff.
hdlr = None
root_logger = logging.getLogger()
if log_file and not preview:
logger.debug("Adding file handler for: %s" % log_file)
hdlr = logging.FileHandler(log_file, mode='w', encoding='utf8')
root_logger.addHandler(hdlr)
if not preview:
logger.info("Deploying to %s" % target)
else:
logger.info("Previewing deployment to %s" % target)
# Bake first is necessary.
rec1 = None
rec2 = None
was_baked = False
bake_out_dir = os.path.join(self.app.root_dir, '_pub', target)
if bake_first:
if not preview:
bake_start_time = time.perf_counter()
logger.debug("Baking first to: %s" % bake_out_dir)
from piecrust.baking.baker import Baker
baker = Baker(
self.app, bake_out_dir,
applied_config_variant=applied_config_variant,
applied_config_values=applied_config_values)
rec1 = baker.bake()
from piecrust.processing.pipeline import ProcessorPipeline
proc = ProcessorPipeline(
self.app, bake_out_dir,
applied_config_variant=applied_config_variant,
applied_config_values=applied_config_values)
rec2 = proc.run()
was_baked = True
if not rec1.success or not rec2.success:
raise Exception(
"Error during baking, aborting publishing.")
logger.info(format_timed(bake_start_time, "Baked website."))
else:
logger.info("Would bake to: %s" % bake_out_dir)
# Publish!
logger.debug(
"Running publish target '%s' with publisher: %s" %
(target, pub.PUBLISHER_NAME))
pub_start_time = time.perf_counter()
ctx = PublishingContext()
ctx.bake_out_dir = bake_out_dir
ctx.bake_record = rec1
ctx.processing_record = rec2
ctx.was_baked = was_baked
ctx.preview = preview
ctx.args = extra_args
try:
pub.run(ctx)
except Exception as ex:
raise PublishingError(
"Error publishing to target: %s" % target) from ex
finally:
if hdlr:
root_logger.removeHandler(hdlr)
hdlr.close()
logger.info(format_timed(
pub_start_time, "Ran publisher %s" % pub.PUBLISHER_NAME))
logger.info(format_timed(start_time, 'Deployed to %s' % target))
class ProcessingLoop(threading.Thread):
def __init__(self, appfactory, out_dir):
super(ProcessingLoop, self).__init__(
name='pipeline-reloader', daemon=True)
self.appfactory = appfactory
self.out_dir = out_dir
self.last_status_id = 0
self.interval = 1
self.app = None
self._roots = []
self._monitor_assets_root = False
self._paths = set()
self._record = None
self._last_bake = 0
self._last_config_mtime = 0
self._obs = []
self._obs_lock = threading.Lock()
config_name = (
THEME_CONFIG_PATH if appfactory.theme_site else CONFIG_PATH)
self._config_path = os.path.join(appfactory.root_dir, config_name)
def addObserver(self, obs):
with self._obs_lock:
self._obs.append(obs)
def removeObserver(self, obs):
with self._obs_lock:
self._obs.remove(obs)
def run(self):
self._initPipeline()
self._last_bake = time.time()
self._last_config_mtime = os.path.getmtime(self._config_path)
self._record = self.pipeline.run()
while True:
cur_config_time = os.path.getmtime(self._config_path)
if self._last_config_mtime < cur_config_time:
logger.info("Site configuration changed, reloading pipeline.")
self._last_config_mtime = cur_config_time
self._initPipeline()
for root in self._roots:
self._runPipeline(root)
continue
if self._monitor_assets_root:
assets_dir = os.path.join(self.app.root_dir, 'assets')
if os.path.isdir(assets_dir):
logger.info("Assets directory was created, reloading "
"pipeline.")
self._initPipeline()
self._runPipeline(assets_dir)
continue
for root in self._roots:
# For each mount root we try to find the first new or
# modified file. If any, we just run the pipeline on
# that mount.
found_new_or_modified = False
for dirpath, dirnames, filenames in os.walk(root):
for filename in filenames:
path = os.path.join(dirpath, filename)
if path not in self._paths:
logger.debug("Found new asset: %s" % path)
self._paths.add(path)
found_new_or_modified = True
break
if os.path.getmtime(path) > self._last_bake:
logger.debug("Found modified asset: %s" % path)
found_new_or_modified = True
break
if found_new_or_modified:
break
if found_new_or_modified:
self._runPipeline(root)
time.sleep(self.interval)
def _initPipeline(self):
# Create the app and pipeline.
self.app = self.appfactory.create()
self.pipeline = ProcessorPipeline(self.app, self.out_dir)
# Get the list of assets directories.
self._roots = list(self.pipeline.mounts.keys())
# The 'assets' folder may not be in the mounts list if it doesn't
# exist yet, but we want to monitor for when the user creates it.
default_root = os.path.join(self.app.root_dir, 'assets')
self._monitor_assets_root = (default_root not in self._roots)
# Build the list of initial asset files.
self._paths = set()
for root in self._roots:
for dirpath, dirnames, filenames in os.walk(root):
self._paths |= set([os.path.join(dirpath, f)
for f in filenames])
def _runPipeline(self, root):
self._last_bake = time.time()
try:
self._record = self.pipeline.run(
root,
previous_record=self._record,
save_record=False)
status_id = self.last_status_id + 1
self.last_status_id += 1
if self._record.success:
changed = filter(
lambda i: not i.was_collapsed_from_last_run,
self._record.entries)
changed = itertools.chain.from_iterable(
map(lambda i: i.rel_outputs, changed))
changed = list(changed)
item = {
'id': status_id,
'type': 'pipeline_success',
'assets': changed}
self._notifyObservers(item)
else:
item = {
'id': status_id,
'type': 'pipeline_error',
'assets': []}
for entry in self._record.entries:
if entry.errors:
asset_item = {
'path': entry.path,
'errors': list(entry.errors)}
item['assets'].append(asset_item)
self._notifyObservers(item)
except Exception as ex:
logger.exception(ex)
def _notifyObservers(self, item):
with self._obs_lock:
observers = list(self._obs)
for obs in observers:
obs.addBuildEvent(item)
def _bakeAssets(self, ctx, out_dir):
proc = ProcessorPipeline(ctx.app, out_dir, force=ctx.args.force)
record = proc.run()
_merge_timers(record.timers, ctx.timers)
return record.success
def _get_pipeline(fs, app=None):
app = app or fs.getApp()
return ProcessorPipeline(app, fs.path('counter'))
