def initialize(self):
# Create the app local to this worker.
app = PieCrust(self.ctx.root_dir, debug=self.ctx.debug)
app._useSubCacheDir(self.ctx.sub_cache_dir)
app.config.set('baker/is_baking', True)
app.env.base_asset_url_format = '%uri%'
app.env.fs_cache_only_for_main_page = True
app.env.registerTimer("BakeWorker_%d_Total" % self.wid)
app.env.registerTimer("BakeWorkerInit")
app.env.registerTimer("JobReceive")
apply_variant_and_values(app, self.ctx.config_variant,
self.ctx.config_values)
self.ctx.app = app
# Load previous record
if self.ctx.previous_record_path:
self.ctx.previous_record = BakeRecord.load(
self.ctx.previous_record_path)
self.ctx.previous_record_index = {}
for e in self.ctx.previous_record.entries:
key = _get_transition_key(e.path, e.taxonomy_info)
self.ctx.previous_record_index[key] = e
# Create the job handlers.
job_handlers = {
JOB_LOAD: LoadJobHandler(self.ctx),
JOB_RENDER_FIRST: RenderFirstSubJobHandler(self.ctx),
JOB_BAKE: BakeJobHandler(self.ctx)}
for jt, jh in job_handlers.items():
app.env.registerTimer(type(jh).__name__)
self.job_handlers = job_handlers
app.env.stepTimerSince("BakeWorkerInit", self.work_start_time)
def initialize(self):
# Create the app local to this worker.
app = PieCrust(self.ctx.root_dir, debug=self.ctx.debug)
app._useSubCacheDir(self.ctx.sub_cache_dir)
app.config.set('baker/is_baking', True)
app.config.set('baker/worker_id', self.wid)
app.env.base_asset_url_format = '%uri%'
app.env.fs_cache_only_for_main_page = True
app.env.registerTimer("BakeWorker_%d_Total" % self.wid)
app.env.registerTimer("BakeWorkerInit")
app.env.registerTimer("JobReceive")
apply_variant_and_values(app, self.ctx.config_variant,
self.ctx.config_values)
self.ctx.app = app
# Load previous record
if self.ctx.previous_record_path:
self.ctx.previous_record = BakeRecord.load(
self.ctx.previous_record_path)
self.ctx.previous_record_index = {}
for e in self.ctx.previous_record.entries:
key = _get_transition_key(e.path, e.taxonomy_info)
self.ctx.previous_record_index[key] = e
# Create the job handlers.
job_handlers = {
JOB_LOAD: LoadJobHandler(self.ctx),
JOB_RENDER_FIRST: RenderFirstSubJobHandler(self.ctx),
JOB_BAKE: BakeJobHandler(self.ctx)}
for jt, jh in job_handlers.items():
app.env.registerTimer(type(jh).__name__)
self.job_handlers = job_handlers
app.env.stepTimerSince("BakeWorkerInit", self.work_start_time)
def get_app_for_server(root_dir, debug=False, sub_cache_dir=None):
app = PieCrust(root_dir=root_dir, debug=debug)
if sub_cache_dir:
app._useSubCacheDir(sub_cache_dir)
app.config.set('site/root', '/')
app.config.set('server/is_serving', True)
return app
def get_app_for_server(root_dir, debug=False, sub_cache_dir=None,
root_url='/'):
app = PieCrust(root_dir=root_dir, debug=debug)
if sub_cache_dir:
app._useSubCacheDir(sub_cache_dir)
app.config.set('site/root', root_url)
app.config.set('server/is_serving', True)
return app
def _try_run_request(self, environ, start_response):
request = Request(environ)
# We don't support anything else than GET requests since we're
# previewing something that will be static later.
if self.static_preview and request.method != 'GET':
logger.error("Only GET requests are allowed, got %s" %
request.method)
raise MethodNotAllowed()
# Handle special requests right away.
response = self._try_special_request(environ, request)
if response is not None:
return response(environ, start_response)
# Also handle requests to a pipeline-built asset right away.
response = self._try_serve_asset(environ, request)
if response is not None:
return response(environ, start_response)
# Create the app for this request.
app = PieCrust(root_dir=self.root_dir, debug=self.debug)
if self.sub_cache_dir:
app._useSubCacheDir(self.sub_cache_dir)
app.config.set('site/root', '/')
app.config.set('server/is_serving', True)
if (app.config.get('site/enable_debug_info') and
self.enable_debug_info and
'!debug' in request.args):
app.config.set('site/show_debug_info', True)
# We'll serve page assets directly from where they are.
app.env.base_asset_url_format = '/_asset/%path%'
# Let's see if it can be a page asset.
response = self._try_serve_page_asset(app, environ, request)
if response is not None:
return response(environ, start_response)
# Nope. Let's see if it's an actual page.
try:
response = self._try_serve_page(app, environ, request)
return response(environ, start_response)
except (RouteNotFoundError, SourceNotFoundError) as ex:
raise NotFound(str(ex)) from ex
except HTTPException:
raise
except Exception as ex:
if app.debug:
logger.exception(ex)
raise
msg = str(ex)
logger.error(msg)
raise InternalServerError(msg) from ex
def _try_run_request(self, environ, start_response):
request = Request(environ)
# We don't support anything else than GET requests since we're
# previewing something that will be static later.
if self.static_preview and request.method != 'GET':
logger.error("Only GET requests are allowed, got %s" %
request.method)
raise MethodNotAllowed()
# Handle special requests right away.
response = self._try_special_request(environ, request)
if response is not None:
return response(environ, start_response)
# Also handle requests to a pipeline-built asset right away.
response = self._try_serve_asset(environ, request)
if response is not None:
return response(environ, start_response)
# Create the app for this request.
app = PieCrust(root_dir=self.root_dir, debug=self.debug)
app._useSubCacheDir(self.sub_cache_dir)
app.config.set('site/root', '/')
app.config.set('server/is_serving', True)
if (app.config.get('site/enable_debug_info') and self.enable_debug_info
and '!debug' in request.args):
app.config.set('site/show_debug_info', True)
# We'll serve page assets directly from where they are.
app.env.base_asset_url_format = '/_asset/%path%'
# Let's see if it can be a page asset.
response = self._try_serve_page_asset(app, environ, request)
if response is not None:
return response(environ, start_response)
# Nope. Let's see if it's an actual page.
try:
response = self._try_serve_page(app, environ, request)
return response(environ, start_response)
except (RouteNotFoundError, SourceNotFoundError) as ex:
raise NotFound(str(ex)) from ex
except HTTPException:
raise
except Exception as ex:
if app.debug:
logger.exception(ex)
raise
msg = str(ex)
logger.error(msg)
raise InternalServerError(msg) from ex
def getWsgiApp(self):
# Bake all the assets so we know what we have, and so we can serve
# them to the client. We need a temp app for this.
app = PieCrust(root_dir=self.root_dir, debug=self.debug)
app._useSubCacheDir(self.sub_cache_dir)
self._out_dir = os.path.join(app.sub_cache_dir, 'server')
self._page_record = ServeRecord()
if not self.run_sse_check or self.run_sse_check():
# When using a server with code reloading, some implementations
# use process forking and we end up going here twice. We only want
# to start the pipeline loop in the inner process most of the
# time so we let the implementation tell us if this is OK.
from piecrust.processing.base import ProcessorPipeline
from piecrust.serving.procloop import ProcessingLoop
pipeline = ProcessorPipeline(app, self._out_dir)
self._proc_loop = ProcessingLoop(pipeline)
self._proc_loop.start()
# Run the WSGI app.
wsgi_wrapper = WsgiServerWrapper(self)
return wsgi_wrapper
def getWsgiApp(self):
# Bake all the assets so we know what we have, and so we can serve
# them to the client. We need a temp app for this.
app = PieCrust(root_dir=self.root_dir, debug=self.debug)
if self.sub_cache_dir:
app._useSubCacheDir(self.sub_cache_dir)
self._out_dir = os.path.join(app.sub_cache_dir, 'server')
if not self.run_sse_check or self.run_sse_check():
# When using a server with code reloading, some implementations
# use process forking and we end up going here twice. We only want
# to start the pipeline loop in the inner process most of the
# time so we let the implementation tell us if this is OK.
from piecrust.processing.pipeline import ProcessorPipeline
from piecrust.serving.procloop import ProcessingLoop
pipeline = ProcessorPipeline(app, self._out_dir)
self._proc_loop = ProcessingLoop(pipeline)
self._proc_loop.start()
# Run the WSGI app.
wsgi_wrapper = WsgiServerWrapper(self)
return wsgi_wrapper
class ProcessingLoop(threading.Thread):
def __init__(self, root_dir, out_dir, sub_cache_dir=None, debug=False):
super(ProcessingLoop, self).__init__(
name='pipeline-reloader', daemon=True)
self.root_dir = root_dir
self.out_dir = out_dir
self.sub_cache_dir = sub_cache_dir
self.debug = debug
self.last_status_id = 0
self.interval = 1
self.app = None
self._roots = []
self._monitor_assets_root = False
self._paths = set()
self._config_path = os.path.join(root_dir, 'config.yml')
self._record = None
self._last_bake = 0
self._last_config_mtime = 0
self._obs = []
self._obs_lock = threading.Lock()
def addObserver(self, obs):
with self._obs_lock:
self._obs.append(obs)
def removeObserver(self, obs):
with self._obs_lock:
self._obs.remove(obs)
def run(self):
self._initPipeline()
self._last_bake = time.time()
self._last_config_mtime = os.path.getmtime(self._config_path)
self._record = self.pipeline.run()
while True:
cur_config_time = os.path.getmtime(self._config_path)
if self._last_config_mtime < cur_config_time:
logger.info("Site configuration changed, reloading pipeline.")
self._last_config_mtime = cur_config_time
self._initPipeline()
for root in self._roots:
self._runPipeline(root)
continue
if self._monitor_assets_root:
assets_dir = os.path.join(self.app.root_dir, 'assets')
if os.path.isdir(assets_dir):
logger.info("Assets directory was created, reloading "
"pipeline.")
self._initPipeline()
self._runPipeline(assets_dir)
continue
for root in self._roots:
# For each mount root we try to find the first new or
# modified file. If any, we just run the pipeline on
# that mount.
found_new_or_modified = False
for dirpath, dirnames, filenames in os.walk(root):
for filename in filenames:
path = os.path.join(dirpath, filename)
if path not in self._paths:
logger.debug("Found new asset: %s" % path)
self._paths.add(path)
found_new_or_modified = True
break
if os.path.getmtime(path) > self._last_bake:
logger.debug("Found modified asset: %s" % path)
found_new_or_modified = True
break
if found_new_or_modified:
break
if found_new_or_modified:
self._runPipeline(root)
time.sleep(self.interval)
def _initPipeline(self):
# Create the app and pipeline.
self.app = PieCrust(root_dir=self.root_dir, debug=self.debug)
if self.sub_cache_dir:
self.app._useSubCacheDir(self.sub_cache_dir)
self.pipeline = ProcessorPipeline(self.app, self.out_dir)
# Get the list of assets directories.
self._roots = list(self.pipeline.mounts.keys())
# The 'assets' folder may not be in the mounts list if it doesn't
# exist yet, but we want to monitor for when the user creates it.
default_root = os.path.join(self.app.root_dir, 'assets')
self._monitor_assets_root = (default_root not in self._roots)
# Build the list of initial asset files.
self._paths = set()
for root in self._roots:
for dirpath, dirnames, filenames in os.walk(root):
self._paths |= set([os.path.join(dirpath, f)
for f in filenames])
def _runPipeline(self, root):
self._last_bake = time.time()
try:
self._record = self.pipeline.run(
root,
previous_record=self._record,
save_record=False)
status_id = self.last_status_id + 1
self.last_status_id += 1
if self._record.success:
changed = filter(
lambda i: not i.was_collapsed_from_last_run,
self._record.entries)
changed = itertools.chain.from_iterable(
map(lambda i: i.rel_outputs, changed))
changed = list(changed)
item = {
'id': status_id,
'type': 'pipeline_success',
'assets': changed}
self._notifyObservers(item)
else:
item = {
'id': status_id,
'type': 'pipeline_error',
'assets': []}
for entry in self._record.entries:
if entry.errors:
asset_item = {
'path': entry.path,
'errors': list(entry.errors)}
item['assets'].append(asset_item)
self._notifyObservers(item)
except Exception as ex:
logger.exception(ex)
def _notifyObservers(self, item):
with self._obs_lock:
observers = list(self._obs)
for obs in observers:
obs.addBuildEvent(item)
class ProcessingLoop(threading.Thread):
def __init__(self, root_dir, out_dir, sub_cache_dir=None, debug=False):
super(ProcessingLoop, self).__init__(name="pipeline-reloader", daemon=True)
self.root_dir = root_dir
self.out_dir = out_dir
self.sub_cache_dir = sub_cache_dir
self.debug = debug
self.last_status_id = 0
self.interval = 1
self.app = None
self._roots = []
self._monitor_assets_root = False
self._paths = set()
self._config_path = os.path.join(root_dir, "config.yml")
self._record = None
self._last_bake = 0
self._last_config_mtime = 0
self._obs = []
self._obs_lock = threading.Lock()
def addObserver(self, obs):
with self._obs_lock:
self._obs.append(obs)
def removeObserver(self, obs):
with self._obs_lock:
self._obs.remove(obs)
def run(self):
self._initPipeline()
self._last_bake = time.time()
self._last_config_mtime = os.path.getmtime(self._config_path)
self._record = self.pipeline.run()
while True:
cur_config_time = os.path.getmtime(self._config_path)
if self._last_config_mtime < cur_config_time:
logger.info("Site configuration changed, reloading pipeline.")
self._last_config_mtime = cur_config_time
self._initPipeline()
for root in self._roots:
self._runPipeline(root)
continue
if self._monitor_assets_root:
assets_dir = os.path.join(self.app.root_dir, "assets")
if os.path.isdir(assets_dir):
logger.info("Assets directory was created, reloading " "pipeline.")
self._initPipeline()
self._runPipeline(assets_dir)
continue
for root in self._roots:
# For each mount root we try to find the first new or
# modified file. If any, we just run the pipeline on
# that mount.
found_new_or_modified = False
for dirpath, dirnames, filenames in os.walk(root):
for filename in filenames:
path = os.path.join(dirpath, filename)
if path not in self._paths:
logger.debug("Found new asset: %s" % path)
self._paths.add(path)
found_new_or_modified = True
break
if os.path.getmtime(path) > self._last_bake:
logger.debug("Found modified asset: %s" % path)
found_new_or_modified = True
break
if found_new_or_modified:
break
if found_new_or_modified:
self._runPipeline(root)
time.sleep(self.interval)
def _initPipeline(self):
# Create the app and pipeline.
self.app = PieCrust(root_dir=self.root_dir, debug=self.debug)
if self.sub_cache_dir:
self.app._useSubCacheDir(self.sub_cache_dir)
self.pipeline = ProcessorPipeline(self.app, self.out_dir)
# Get the list of assets directories.
self._roots = list(self.pipeline.mounts.keys())
# The 'assets' folder may not be in the mounts list if it doesn't
# exist yet, but we want to monitor for when the user creates it.
default_root = os.path.join(self.app.root_dir, "assets")
self._monitor_assets_root = default_root not in self._roots
# Build the list of initial asset files.
self._paths = set()
for root in self._roots:
for dirpath, dirnames, filenames in os.walk(root):
self._paths |= set([os.path.join(dirpath, f) for f in filenames])
def _runPipeline(self, root):
self._last_bake = time.time()
try:
self._record = self.pipeline.run(root, previous_record=self._record, save_record=False)
status_id = self.last_status_id + 1
self.last_status_id += 1
if self._record.success:
changed = filter(lambda i: not i.was_collapsed_from_last_run, self._record.entries)
changed = itertools.chain.from_iterable(map(lambda i: i.rel_outputs, changed))
changed = list(changed)
item = {"id": status_id, "type": "pipeline_success", "assets": changed}
self._notifyObservers(item)
else:
item = {"id": status_id, "type": "pipeline_error", "assets": []}
for entry in self._record.entries:
if entry.errors:
asset_item = {"path": entry.path, "errors": list(entry.errors)}
item["assets"].append(asset_item)
self._notifyObservers(item)
except Exception as ex:
logger.exception(ex)
def _notifyObservers(self, item):
with self._obs_lock:
observers = list(self._obs)
for obs in observers:
obs.addBuildEvent(item)
