def __init__(self, wid, ctx):
super(BakeWorker, self).__init__(name=('worker%d' % wid))
self.wid = wid
self.ctx = ctx
self.abort_exception = None
self.success = True
self._page_baker = PageBaker(
ctx.app, ctx.out_dir, ctx.force,
ctx.record)
class BakeJobHandler(JobHandler):
def __init__(self, ctx):
super(BakeJobHandler, self).__init__(ctx)
self.page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force)
def shutdown(self):
self.page_baker.shutdown()
def handleJob(self, job):
# Actually bake the page and all its sub-pages to the output folder.
fac = load_factory(self.app, job['factory_info'])
self.app.env.addManifestEntry('BakeJobs', fac.ref_spec)
route_index = job['route_index']
route_metadata = job['route_metadata']
route = self.app.routes[route_index]
gen_name = job['generator_name']
gen_key = job['generator_record_key']
dirty_source_names = job['dirty_source_names']
page = fac.buildPage()
qp = QualifiedPage(page, route, route_metadata)
result = {
'path': fac.path,
'generator_name': gen_name,
'generator_record_key': gen_key,
'sub_entries': None,
'errors': None}
if job.get('needs_config', False):
result['config'] = page.config.getAll()
previous_entry = None
if self.ctx.previous_record_index is not None:
key = _get_transition_key(fac.path, gen_key)
previous_entry = self.ctx.previous_record_index.get(key)
logger.debug("Baking page: %s" % fac.ref_spec)
logger.debug("With route metadata: %s" % route_metadata)
try:
sub_entries = self.page_baker.bake(
qp, previous_entry, dirty_source_names, gen_name)
result['sub_entries'] = sub_entries
except BakingError as ex:
logger.debug("Got baking error. Sending it to master.")
result['errors'] = _get_errors(ex)
if self.ctx.app.debug:
logger.exception(ex)
return result
def test_get_output_path(uri, pretty, expected):
app = get_mock_app()
if pretty:
app.config.set('site/pretty_urls', True)
assert app.config.get('site/pretty_urls') == pretty
for site_root in ['/', '/whatever/', '/~johndoe/']:
app.config.set('site/root', urllib.parse.quote(site_root))
baker = PageBaker(app, '/destination')
path = baker.getOutputPath(urllib.parse.quote(site_root) + uri)
expected = os.path.normpath(
os.path.join('/destination', expected))
assert expected == path
class BakeJobHandler(JobHandler):
def __init__(self, ctx):
super(BakeJobHandler, self).__init__(ctx)
self.page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force)
def handleJob(self, job):
# Actually bake the page and all its sub-pages to the output folder.
fac = load_factory(self.app, job['factory_info'])
route_metadata = job['route_metadata']
tax_info = job['taxonomy_info']
if tax_info is not None:
route = self.app.getTaxonomyRoute(tax_info.taxonomy_name,
tax_info.source_name)
else:
route = self.app.getRoute(fac.source.name, route_metadata,
skip_taxonomies=True)
assert route is not None
page = fac.buildPage()
qp = QualifiedPage(page, route, route_metadata)
result = {
'path': fac.path,
'taxonomy_info': tax_info,
'sub_entries': None,
'errors': None}
dirty_source_names = job['dirty_source_names']
previous_entry = None
if self.ctx.previous_record_index is not None:
key = _get_transition_key(fac.path, tax_info)
previous_entry = self.ctx.previous_record_index.get(key)
logger.debug("Baking page: %s" % fac.ref_spec)
try:
sub_entries = self.page_baker.bake(
qp, previous_entry, dirty_source_names, tax_info)
result['sub_entries'] = sub_entries
except BakingError as ex:
logger.debug("Got baking error. Sending it to master.")
result['errors'] = _get_errors(ex)
if self.ctx.debug:
logger.exception(ex)
return result
class BakeJobHandler(JobHandler):
def __init__(self, ctx):
super(BakeJobHandler, self).__init__(ctx)
self.page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force)
def handleJob(self, job):
# Actually bake the page and all its sub-pages to the output folder.
fac = load_factory(self.app, job['factory_info'])
route_metadata = job['route_metadata']
tax_info = job['taxonomy_info']
if tax_info is not None:
route = self.app.getTaxonomyRoute(tax_info.taxonomy_name,
tax_info.source_name)
else:
route = self.app.getRoute(fac.source.name, route_metadata,
skip_taxonomies=True)
assert route is not None
page = fac.buildPage()
qp = QualifiedPage(page, route, route_metadata)
result = {
'path': fac.path,
'taxonomy_info': tax_info,
'sub_entries': None,
'errors': None}
dirty_source_names = job['dirty_source_names']
previous_entry = None
if self.ctx.previous_record_index is not None:
key = _get_transition_key(fac.path, tax_info)
previous_entry = self.ctx.previous_record_index.get(key)
logger.debug("Baking page: %s" % fac.ref_spec)
try:
sub_entries = self.page_baker.bake(
qp, previous_entry, dirty_source_names, tax_info)
result['sub_entries'] = sub_entries
except BakingError as ex:
logger.debug("Got baking error. Sending it to master.")
result['errors'] = _get_errors(ex)
if self.ctx.debug:
logger.exception(ex)
return result
def __init__(self, ctx):
super(BakeJobHandler, self).__init__(ctx)
self.page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force)
def __init__(self, ctx):
super(BakeJobHandler, self).__init__(ctx)
self.page_baker = PageBaker(ctx.app, ctx.out_dir, ctx.force)
class BakeWorker(threading.Thread):
def __init__(self, wid, ctx):
super(BakeWorker, self).__init__(name=('worker%d' % wid))
self.wid = wid
self.ctx = ctx
self.abort_exception = None
self.success = True
self._page_baker = PageBaker(
ctx.app, ctx.out_dir, ctx.force,
ctx.record)
def run(self):
while(not self.ctx.abort_event.is_set()):
try:
job = self.ctx.work_queue.getNextJob(wait_timeout=1)
if job is None:
logger.debug(
"[%d] No more work... shutting down." %
self.wid)
break
success = self._unsafeRun(job)
logger.debug("[%d] Done with page." % self.wid)
self.ctx.work_queue.onJobFinished(job)
self.success &= success
except Exception as ex:
self.ctx.abort_event.set()
self.abort_exception = ex
self.success = False
logger.debug("[%d] Critical error, aborting." % self.wid)
if self.ctx.app.debug:
logger.exception(ex)
break
def _unsafeRun(self, job):
start_time = time.clock()
entry = job.record_entry
try:
self._page_baker.bake(job.factory, job.route, entry)
except BakingError as ex:
logger.debug("Got baking error. Adding it to the record.")
while ex:
entry.errors.append(str(ex))
ex = ex.__cause__
has_error = False
for e in entry.getAllErrors():
has_error = True
logger.error(e)
if has_error:
return False
if entry.was_any_sub_baked:
first_sub = entry.subs[0]
friendly_uri = first_sub.out_uri
if friendly_uri == '':
friendly_uri = '[main page]'
friendly_count = ''
if entry.num_subs > 1:
friendly_count = ' (%d pages)' % entry.num_subs
logger.info(format_timed(
start_time, '[%d] %s%s' %
(self.wid, friendly_uri, friendly_count)))
return True
