def _validate_site(v, values, cache):
sources = v.get('sources')
if not sources:
raise ConfigurationError("No sources were defined.")
routes = v.get('routes')
if not routes:
raise ConfigurationError("No routes were defined.")
taxonomies = v.get('taxonomies')
if taxonomies is None:
v['taxonomies'] = {}
return v
def _parse_config_date(page_date):
if page_date is None:
return None
if isinstance(page_date, str):
try:
parsed_d = dateutil.parser.parse(page_date)
except Exception as ex:
raise ConfigurationError("Invalid date: %s" % page_date) from ex
return datetime.date(year=parsed_d.year,
month=parsed_d.month,
day=parsed_d.day)
raise ConfigurationError("Invalid date: %s" % page_date)
def _validate_site_pagination_suffix(v, values, cache):
if len(v) == 0 or v[0] != '/':
raise ConfigurationError("The 'site/pagination_suffix' setting "
"must start with a slash.")
if '%num%' not in v:
raise ConfigurationError("The 'site/pagination_suffix' setting "
"must contain the '%num%' placeholder.")
pgn_suffix_fmt = v.replace('%num%', '%(num)d')
cache.write('pagination_suffix_format', pgn_suffix_fmt)
pgn_suffix_re = re.escape(v)
pgn_suffix_re = (pgn_suffix_re.replace("\\%num\\%", "(?P<num>\\d+)") + '$')
cache.write('pagination_suffix_re', pgn_suffix_re)
return v
def publishers(self):
defs_by_name = {}
defs_by_scheme = {}
for cls in self.plugin_loader.getPublishers():
defs_by_name[cls.PUBLISHER_NAME] = cls
if cls.PUBLISHER_SCHEME:
defs_by_scheme[cls.PUBLISHER_SCHEME] = cls
tgts = []
publish_config = self.config.get('publish')
if publish_config is None:
return tgts
for n, t in publish_config.items():
pub_class = None
if isinstance(t, dict):
pub_type = t.get('type')
pub_class = defs_by_name[pub_type]
pub_cfg = t
elif isinstance(t, str):
comps = urllib.parse.urlparse(t)
pub_type = comps.scheme
pub_class = defs_by_scheme[pub_type]
pub_cfg = None
if pub_class is None:
raise ConfigurationError("No such publisher: %s" % pub_type)
tgt = pub_class(self, n, pub_cfg)
if pub_cfg is None:
tgt.parseUrlTarget(comps)
tgts.append(tgt)
return tgts
def _validate_site_plugins(v, values, cache):
if isinstance(v, str):
v = v.split(',')
elif not isinstance(v, list):
raise ConfigurationError(
"The 'site/plugins' setting must be an array, or a "
"comma-separated list.")
return v
def get_pipeline_name_for_source(source):
pname = source.config['pipeline']
if not pname:
pname = source.DEFAULT_PIPELINE_NAME
if not pname:
raise ConfigurationError("Source '%s' doesn't specify any pipeline." %
source.name)
return pname
def __init__(self, app, name, config):
super().__init__(app, name, config)
config.setdefault('data_type', 'page_iterator')
self.capture_mode = config.get('capture_mode', 'path')
if self.capture_mode not in ['path', 'dirname', 'filename']:
raise ConfigurationError("Capture mode in source '%s' must be "
"one of: path, dirname, filename" % name)
def _validate_site_taxonomies(v, values, cache):
if not isinstance(v, dict):
raise ConfigurationError(
"The 'site/taxonomies' setting must be a mapping.")
for tn, tc in v.items():
tc.setdefault('multiple', False)
tc.setdefault('term', tn)
tc.setdefault('page', '_%s.%%ext%%' % tc['term'])
return v
def _validate_site_auto_formats(v, values, cache):
if not isinstance(v, dict):
raise ConfigurationError("The 'site/auto_formats' setting must be "
"a dictionary.")
v.setdefault('html', values['site']['default_format'])
auto_formats_re = r"\.(%s)$" % ('|'.join(
[re.escape(i) for i in list(v.keys())]))
cache.write('auto_formats_re', auto_formats_re)
return v
def buildDataProvider(self, page, user_data):
if self._provider_type is None:
cls = next((pt for pt in self.app.plugin_loader.getDataProviders()
if pt.PROVIDER_NAME == self.data_type), None)
if cls is None:
raise ConfigurationError("Unknown data provider type: %s" %
self.data_type)
self._provider_type = cls
return self._provider_type(self, page, user_data)
def __init__(self, app, name, config):
super().__init__(app, name, config)
source_name = config.get('source')
if source_name is None:
raise ConfigurationError(
"Taxonomy source '%s' requires an inner source." % name)
self._inner_source_name = source_name
self._raw_item = ''
self._raw_item_time = time.time()
def _validate_site_routes(v, values, cache):
if not v:
raise ConfigurationError("There are no routes defined.")
if not isinstance(v, list):
raise ConfigurationError("The 'site/routes' setting must be a "
"list.")
# Check routes are referencing correct sources, have default
# values, etc.
used_sources = set()
source_configs = values['site']['sources']
existing_sources = set(source_configs.keys())
for rc in v:
if not isinstance(rc, dict):
raise ConfigurationError("All routes in 'site/routes' must be "
"dictionaries.")
rc_url = rc.get('url')
if not rc_url:
raise ConfigurationError("All routes in 'site/routes' must "
"have an 'url'.")
if rc_url[0] != '/':
raise ConfigurationError("Route URLs must start with '/'.")
r_source = rc.get('source')
if r_source is None:
raise ConfigurationError("Routes must specify a source.")
if r_source not in existing_sources:
raise ConfigurationError("Route is referencing unknown "
"source: %s" % r_source)
if r_source in used_sources:
raise ConfigurationError("Source '%s' already has a route." %
r_source)
used_sources.add(r_source)
rc.setdefault('pass', 1)
rc.setdefault('page_suffix', '/%num%')
# Raise errors about non-asset sources that have no URL routes.
sources_with_no_route = list(
filter(lambda s: source_configs[s].get('pipeline') != 'asset',
existing_sources.difference(used_sources)))
if sources_with_no_route:
raise ConfigurationError("The following sources have no routes: %s" %
', '.join(sources_with_no_route))
return v
def _validate_site_root(v, values, cache):
url_bits = urllib.parse.urlparse(v)
if url_bits.params or url_bits.query or url_bits.fragment:
raise ConfigurationError("Root URL is invalid: %s" % v)
path = url_bits.path.rstrip('/') + '/'
if '%' not in path:
path = urllib.parse.quote(path)
root_url = urllib.parse.urlunparse(
(url_bits.scheme, url_bits.netloc, path, '', '', ''))
return root_url
def _parse_config_time(page_time):
if page_time is None:
return None
if isinstance(page_time, datetime.timedelta):
return page_time
if isinstance(page_time, str):
try:
parsed_t = dateutil.parser.parse(page_time)
except Exception as ex:
raise ConfigurationError("Invalid time: %s" % page_time) from ex
return datetime.timedelta(hours=parsed_t.hour,
minutes=parsed_t.minute,
seconds=parsed_t.second)
if isinstance(page_time, int):
# Total seconds... convert to a time struct.
return datetime.timedelta(seconds=page_time)
raise ConfigurationError("Invalid time: %s" % page_time)
def __init__(self, app, name, config):
super(AutoConfigSourceBase, self).__init__(app, name, config)
self.fs_endpoint = config.get('fs_endpoint', name)
self.fs_endpoint_path = os.path.join(self.root_dir, self.fs_endpoint)
self.supported_extensions = list(
app.config.get('site/auto_formats').keys())
self.default_auto_format = app.config.get('site/default_auto_format')
self.capture_mode = config.get('capture_mode', 'path')
if self.capture_mode not in ['path', 'dirname', 'filename']:
raise ConfigurationError("Capture mode in source '%s' must be "
"one of: path, dirname, filename" % name)
def build_data_provider(provider_type, source, page):
if not provider_type:
raise Exception("No data provider type specified.")
for p in page.app.plugin_loader.getDataProviders():
if p.PROVIDER_NAME == provider_type:
pclass = p
break
else:
raise ConfigurationError("Unknown data provider type: %s" %
provider_type)
return pclass(source, page)
def _validate_site_sources(v, values, cache):
# Basic checks.
if not v:
raise ConfigurationError("There are no sources defined.")
if not isinstance(v, dict):
raise ConfigurationError("The 'site/sources' setting must be a "
"dictionary.")
# Sources have the `default` scanner by default, duh. Also, a bunch
# of other default values for other configuration stuff.
reserved_endpoints = set([
'piecrust', 'site', 'page', 'route', 'assets', 'pagination',
'siblings', 'family'
])
for sn, sc in v.items():
if not isinstance(sc, dict):
raise ConfigurationError("All sources in 'site/sources' must "
"be dictionaries.")
sc.setdefault('type', 'default')
sc.setdefault('fs_endpoint', sn)
sc.setdefault('ignore_missing_dir', False)
sc.setdefault('data_endpoint', None)
sc.setdefault('data_type', None)
sc.setdefault('default_layout', 'default')
sc.setdefault('item_name', sn)
sc.setdefault('items_per_page', 5)
sc.setdefault('date_format', DEFAULT_DATE_FORMAT)
sc.setdefault('realm', REALM_USER)
sc.setdefault('pipeline', None)
# Validate endpoints.
endpoint = sc['data_endpoint']
if endpoint in reserved_endpoints:
raise ConfigurationError(
"Source '%s' is using a reserved endpoint name: %s" %
(sn, endpoint))
return v
def __init__(self, app, name, config):
super().__init__(app, name, config)
tax_name = config.get('taxonomy')
if tax_name is None:
raise ConfigurationError(
"Taxonomy source '%s' requires a taxonomy name." % name)
self.taxonomy = _get_taxonomy(app, tax_name)
sm = config.get('slugify_mode')
self.slugifier = _get_slugifier(app, self.taxonomy, sm)
tpl_name = config.get('template', '_%s.html' % tax_name)
self._raw_item = _taxonomy_index % {'template': tpl_name}
def sources(self):
defs = {}
for cls in self.plugin_loader.getSources():
defs[cls.SOURCE_NAME] = cls
sources = []
for n, s in self.config.get('site/sources').items():
cls = defs.get(s['type'])
if cls is None:
raise ConfigurationError("No such page source type: %s" %
s['type'])
src = cls(self, n, s)
sources.append(src)
return sources
def _load(self):
if self._dict is not None:
return
self._dict = {}
for source in self._page.app.sources:
pname = source.config.get('data_type') or 'page_iterator'
pendpoint = source.config.get('data_endpoint')
if not pname or not pendpoint:
continue
endpoint_bits = re_endpoint_sep.split(pendpoint)
endpoint = self._dict
for e in endpoint_bits[:-1]:
if e not in endpoint:
endpoint[e] = {}
endpoint = endpoint[e]
existing = endpoint.get(endpoint_bits[-1])
if existing is None:
provider = build_data_provider(pname, source, self._page)
endpoint[endpoint_bits[-1]] = provider
elif isinstance(existing, DataProvider):
existing_source = existing._sources[0]
if (existing.PROVIDER_NAME != pname
or existing_source.SOURCE_NAME != source.SOURCE_NAME):
raise ConfigurationError(
"Can't combine data providers '%s' and '%' "
"(using sources '%s' and '%s') "
"on endpoint '%s'." % (existing.PROVIDER_NAME, pname,
existing_source.SOURCE_NAME,
source.SOURCE_NAME, pendpoint))
existing._addSource(source)
else:
raise ConfigurationError(
"Endpoint '%s' can't be used for a data provider because "
"it's already used for something else." % pendpoint)
def _validateAll(self, values):
# Put all the defaults in the `site` section.
default_sitec = collections.OrderedDict({
'title':
"Untitled PieCrust website",
'root':
'/',
'default_format':
DEFAULT_FORMAT,
'default_template_engine':
DEFAULT_TEMPLATE_ENGINE,
'enable_gzip':
True,
'pretty_urls':
False,
'trailing_slash':
False,
'date_format':
DEFAULT_DATE_FORMAT,
'auto_formats':
collections.OrderedDict([('html', ''), ('md', 'markdown'),
('textile', 'textile')]),
'default_auto_format':
'md',
'pagination_suffix':
'/%num%',
'plugins':
None,
'themes_sources': [DEFAULT_THEME_SOURCE],
'cache_time':
28800,
'enable_debug_info':
True,
'show_debug_info':
False,
'use_default_content':
True
})
sitec = values.get('site')
if sitec is None:
sitec = collections.OrderedDict()
for key, val in default_sitec.items():
sitec.setdefault(key, val)
values['site'] = sitec
# Add a section for our cached information.
cachec = collections.OrderedDict()
values['__cache'] = cachec
# Make sure the site root starts and ends with a slash.
if not sitec['root'].startswith('/'):
raise ConfigurationError("The `site/root` setting must start "
"with a slash.")
sitec['root'] = sitec['root'].rstrip('/') + '/'
# Cache auto-format regexes.
if not isinstance(sitec['auto_formats'], dict):
raise ConfigurationError("The 'site/auto_formats' setting must be "
"a dictionary.")
html_auto_format = sitec['auto_formats']
if not html_auto_format:
sitec['auto_formats']['html'] = sitec['default_format']
cachec['auto_formats_re'] = r"\.(%s)$" % ('|'.join(
[re.escape(i) for i in list(sitec['auto_formats'].keys())]))
if sitec['default_auto_format'] not in sitec['auto_formats']:
raise ConfigurationError("Default auto-format '%s' is not "
"declared." %
sitec['default_auto_format'])
# Cache pagination suffix regex and format.
pgn_suffix = sitec['pagination_suffix']
if len(pgn_suffix) == 0 or pgn_suffix[0] != '/':
raise ConfigurationError("The 'site/pagination_suffix' setting "
"must start with a slash.")
if '%num%' not in pgn_suffix:
raise ConfigurationError("The 'site/pagination_suffix' setting "
"must contain the '%num%' placeholder.")
pgn_suffix_fmt = pgn_suffix.replace('%num%', '%(num)d')
cachec['pagination_suffix_format'] = pgn_suffix_fmt
pgn_suffix_re = re.escape(pgn_suffix)
pgn_suffix_re = (pgn_suffix_re.replace("\\%num\\%", "(?P<num>\\d+)") +
'$')
cachec['pagination_suffix_re'] = pgn_suffix_re
# Make sure theme sources is a list.
if not isinstance(sitec['themes_sources'], list):
sitec['themes_sources'] = [sitec['themes_sources']]
# Figure out if we need to validate sources/routes, or auto-generate
# them from simple blog settings.
orig_sources = sitec.get('sources')
orig_routes = sitec.get('routes')
orig_taxonomies = sitec.get('taxonomies')
use_default_content = sitec.get('use_default_content')
if (orig_sources is None or orig_routes is None
or orig_taxonomies is None or use_default_content):
# Setup defaults for various settings.
posts_fs = sitec.setdefault('posts_fs', DEFAULT_POSTS_FS)
blogsc = sitec.setdefault('blogs', ['posts'])
g_page_layout = sitec.get('default_page_layout', 'default')
g_post_layout = sitec.get('default_post_layout', 'post')
g_post_url = sitec.get('post_url', '%year%/%month%/%day%/%slug%')
g_tag_url = sitec.get('tag_url', 'tag/%tag%')
g_category_url = sitec.get('category_url', '%category%')
g_posts_per_page = sitec.get('posts_per_page', 5)
g_posts_filters = sitec.get('posts_filters')
g_date_format = sitec.get('date_format', DEFAULT_DATE_FORMAT)
# The normal pages and tags/categories.
sourcesc = collections.OrderedDict()
sourcesc['pages'] = {
'type': 'default',
'ignore_missing_dir': True,
'data_endpoint': 'site.pages',
'default_layout': g_page_layout,
'item_name': 'page'
}
sitec['sources'] = sourcesc
routesc = []
routesc.append({
'url': '/%path:slug%',
'source': 'pages',
'func': 'pcurl(slug)'
})
sitec['routes'] = routesc
taxonomiesc = collections.OrderedDict()
taxonomiesc['tags'] = {'multiple': True, 'term': 'tag'}
taxonomiesc['categories'] = {'term': 'category'}
sitec['taxonomies'] = taxonomiesc
# Setup sources/routes/taxonomies for each blog.
for blog_name in blogsc:
blogc = values.get(blog_name, {})
url_prefix = blog_name + '/'
fs_endpoint = 'posts/%s' % blog_name
data_endpoint = blog_name
item_name = '%s-post' % blog_name
items_per_page = blogc.get('posts_per_page', g_posts_per_page)
items_filters = blogc.get('posts_filters', g_posts_filters)
date_format = blogc.get('date_format', g_date_format)
if len(blogsc) == 1:
url_prefix = ''
fs_endpoint = 'posts'
data_endpoint = 'blog'
item_name = 'post'
sourcesc[blog_name] = {
'type': 'posts/%s' % posts_fs,
'fs_endpoint': fs_endpoint,
'data_endpoint': data_endpoint,
'ignore_missing_dir': True,
'data_type': 'blog',
'item_name': item_name,
'items_per_page': items_per_page,
'items_filters': items_filters,
'date_format': date_format,
'default_layout': g_post_layout
}
tax_page_prefix = ''
if len(blogsc) > 1:
tax_page_prefix = blog_name + '/'
sourcesc[blog_name]['taxonomy_pages'] = {
'tags': ('pages:%s_tag.%%ext%%;'
'theme_pages:_tag.%%ext%%' % tax_page_prefix),
'categories':
('pages:%s_category.%%ext%%;'
'theme_pages:_category.%%ext%%' % tax_page_prefix)
}
post_url = blogc.get('post_url', url_prefix + g_post_url)
post_url = '/' + post_url.lstrip('/')
tag_url = blogc.get('tag_url', url_prefix + g_tag_url)
tag_url = '/' + tag_url.lstrip('/')
category_url = blogc.get('category_url',
url_prefix + g_category_url)
category_url = '/' + category_url.lstrip('/')
routesc.append({
'url': post_url,
'source': blog_name,
'func': 'pcposturl(year,month,day,slug)'
})
routesc.append({
'url': tag_url,
'source': blog_name,
'taxonomy': 'tags',
'func': 'pctagurl(tag)'
})
routesc.append({
'url': category_url,
'source': blog_name,
'taxonomy': 'categories',
'func': 'pccaturl(category)'
})
# If the user defined some additional sources/routes/taxonomies,
# add them to the default ones. For routes, the order matters,
# though, so we make sure to add the user routes at the front
# of the list so they're evaluated first.
if orig_sources:
sourcesc.update(orig_sources)
sitec['sources'] = sourcesc
if orig_routes:
routesc = orig_routes + routesc
sitec['routes'] = routesc
if orig_taxonomies:
taxonomiesc.update(orig_taxonomies)
sitec['taxonomies'] = taxonomiesc
# Validate sources/routes.
sourcesc = sitec.get('sources')
routesc = sitec.get('routes')
if not sourcesc:
raise ConfigurationError("There are no sources defined.")
if not routesc:
raise ConfigurationError("There are no routes defined.")
if not isinstance(sourcesc, dict):
raise ConfigurationError("The 'site/sources' setting must be a "
"dictionary.")
if not isinstance(routesc, list):
raise ConfigurationError("The 'site/routes' setting must be a "
"list.")
# Add the theme page source if no sources were defined in the theme
# configuration itself.
has_any_theme_source = False
for sn, sc in sourcesc.items():
if sc.get('realm') == REALM_THEME:
has_any_theme_source = True
break
if not has_any_theme_source:
sitec['sources']['theme_pages'] = {
'theme_source': True,
'fs_endpoint': 'pages',
'data_endpoint': 'site/pages',
'item_name': 'page',
'realm': REALM_THEME
}
sitec['routes'].append({
'url': '/%path:slug%',
'source': 'theme_pages',
'func': 'pcurl(slug)'
})
# Sources have the `default` scanner by default, duh. Also, a bunch
# of other default values for other configuration stuff.
for sn, sc in sourcesc.items():
if not isinstance(sc, dict):
raise ConfigurationError("All sources in 'site/sources' must "
"be dictionaries.")
sc.setdefault('type', 'default')
sc.setdefault('fs_endpoint', sn)
sc.setdefault('ignore_missing_dir', False)
sc.setdefault('data_endpoint', sn)
sc.setdefault('data_type', 'iterator')
sc.setdefault('item_name', sn)
sc.setdefault('items_per_page', 5)
sc.setdefault('date_format', DEFAULT_DATE_FORMAT)
sc.setdefault('realm', REALM_USER)
# Check routes are referencing correct routes, have default
# values, etc.
for rc in routesc:
if not isinstance(rc, dict):
raise ConfigurationError("All routes in 'site/routes' must be "
"dictionaries.")
rc_url = rc.get('url')
if not rc_url:
raise ConfigurationError("All routes in 'site/routes' must "
"have an 'url'.")
if rc_url[0] != '/':
raise ConfigurationError("Route URLs must start with '/'.")
if rc.get('source') is None:
raise ConfigurationError("Routes must specify a source.")
if rc['source'] not in list(sourcesc.keys()):
raise ConfigurationError("Route is referencing unknown "
"source: %s" % rc['source'])
rc.setdefault('taxonomy', None)
rc.setdefault('page_suffix', '/%num%')
# Validate taxonomies.
sitec.setdefault('taxonomies', {})
taxonomiesc = sitec.get('taxonomies')
for tn, tc in taxonomiesc.items():
tc.setdefault('multiple', False)
tc.setdefault('term', tn)
tc.setdefault('page', '_%s.%%ext%%' % tc['term'])
# Validate endpoints, and make sure the theme has a default source.
reserved_endpoints = set([
'piecrust', 'site', 'page', 'route', 'assets', 'pagination',
'siblings', 'family'
])
for name, src in sitec['sources'].items():
endpoint = src['data_endpoint']
if endpoint in reserved_endpoints:
raise ConfigurationError(
"Source '%s' is using a reserved endpoint name: %s" %
(name, endpoint))
# Make sure the `plugins` setting is a list.
user_plugins = sitec.get('plugins')
if user_plugins:
if isinstance(user_plugins, str):
sitec['plugins'] = user_plugins.split(',')
elif not isinstance(user_plugins, list):
raise ConfigurationError(
"The 'site/plugins' setting must be an array, or a "
"comma-separated list.")
# Done validating!
return values
def _get_taxonomy(app, tax_name):
tax_config = app.config.get('site/taxonomies/' + tax_name)
if tax_config is None:
raise ConfigurationError("No such taxonomy: %s" % tax_name)
return Taxonomy(tax_name, tax_config)
def _validate_site_default_auto_format(v, values, cache):
if v not in values['site']['auto_formats']:
raise ConfigurationError("Default auto-format '%s' is not declared." %
v)
return v