def webpack_settings_from_settings(registry_settings, prefixes=None):
prefixes = prefixes or ['webpack.']
settings = {}
for k, v in registry_settings.items():
for prefix in prefixes:
slicelen = len(prefix)
if k.startswith(prefix):
setting_name = k[slicelen:]
try:
default = DEFAULT_SETTINGS[setting_name]
except KeyError:
toggle = prefix + 'errorOnInvalidSetting'
if toggle in registry_settings:
default = ''
else:
raise ConfigurationError('Invalid webpack setting %s' %
setting_name)
# jinja2_webpack exposes a DEFAULT_SETTINGS dict which
# contains the default value for all the settings.
# Use the type of the default to process the setting from ini.
if type(default) == bool:
v = asbool(v)
elif type(default) == FunctionType:
v = resolve_dotted(v)
elif setting_name == 'renderByExt':
v = parse_renderByExt(v)
settings[setting_name] = v
return settings
def parse_renderByExt(val):
# resolves dotted names
result = parse_multiline(val)
resolved_result = {}
for k, v in result.items():
resolved_result[k] = resolve_dotted(v)
return resolved_result
def fetch_inventory_cmd(store, config_file, force_overwrite, data_dir):
"""Build up list of wines available on a store's website.
Arguments:
STORE: Store from which to fetch inventory.
Supported options: kenaston, lc
Options documentation:
-c, --config: Path to config file.
-d, --data_dir: Location to save results to.
-f, --force_overwrite: Overwrite file specified by --saved_urls. Useful if
you want to refresh the list - new wines added, etc.
"""
saved_urls = os.path.join(data_dir, 'inventory', store, 'urls.txt')
config = ConfigParser()
print(f'Reading config from {config_file}')
config.read(config_file)
all_wine_urls = []
if os.path.exists(saved_urls) and not force_overwrite:
print(f'URLs file {saved_urls} already exists and force overwrite '
'was not specified. Use -f or --force-overwrite if you want to '
'overwrite this file.\nExiting.')
sys.exit(0)
if os.path.exists(saved_urls) and force_overwrite:
print(f'URLs file {saved_urls} exists but force_overwrite was '
'specified. Beginning URL fetch.')
elif not os.path.exists(saved_urls):
print(f'Beginning URL fetch (this can take some time)...')
# Setup for processing the first page
next_page_url = config[store]["first_query_url"]
# Initialize list parser defined for the given store
list_parser = resolve_dotted(config[store]['list_parser'])()
while next_page_url:
# TODO: debug/verbose when logging set up.
print(f'Processing {next_page_url}')
req = requests.get(config[store]["base_url"] + next_page_url)
wine_urls, next_page_url = list_parser(req)
all_wine_urls.extend(wine_urls)
time.sleep(0.1)
print(f'Done fetching urls. Writing results to {saved_urls}')
target_dir = os.path.dirname(saved_urls)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
with open(saved_urls, 'w') as f:
for url in all_wine_urls:
f.write(config[store]['base_url'] + url + '\n')
def dump(self, sql_type):
log.info("Dumping schemas for SQL type%r" % sql_type)
# SQL type names are prefixed with an underscore to distinguish
# them from actual portal_types
dottedname = sql_type.lstrip('_')
main_klass = resolve_dotted(dottedname)
schemas = []
schema_dumper = SQLSchemaDumper()
# Also consider columns from base classes (inheritance).
# We only consider one level of inheritance though, we don't look
# up base classes recursively.
bases = list(filter(_is_mapped_class, main_klass.__bases__))
mapper_args = getattr(main_klass, '__mapper_args__', {})
if bases and 'polymorphic_identity' not in mapper_args:
raise NotImplementedError(
"Unexpected inheritance for %r: Mapped base classes, but "
"no polymorphic_identity found!" % main_klass)
for base_class in bases:
if list(filter(_is_mapped_class, base_class.__bases__)) != []:
raise NotImplementedError(
"More than one level of inheritance currently not"
"supported when dumping SQL schemas.")
for cls in [main_klass] + bases:
schema = schema_dumper.dump(cls)
schemas.append(schema)
type_dump = OrderedDict((
('portal_type', sql_type),
('title', main_klass.__name__),
('schemas', schemas),
))
return type_dump
def dump(self, sql_type):
log.info("Dumping schemas for SQL type%r" % sql_type)
# SQL type names are prefixed with an underscore to distinguish
# them from actual portal_types
dottedname = sql_type.lstrip('_')
main_klass = resolve_dotted(dottedname)
schemas = []
schema_dumper = SQLSchemaDumper()
# Also consider columns from base classes (inheritance).
# We only consider one level of inheritance though, we don't look
# up base classes recursively.
bases = list(filter(_is_mapped_class, main_klass.__bases__))
mapper_args = getattr(main_klass, '__mapper_args__', {})
if bases and 'polymorphic_identity' not in mapper_args:
raise NotImplementedError(
"Unexpected inheritance for %r: Mapped base classes, but "
"no polymorphic_identity found!" % main_klass)
for base_class in bases:
if list(filter(_is_mapped_class, base_class.__bases__)) != []:
raise NotImplementedError(
"More than one level of inheritance currently not"
"supported when dumping SQL schemas.")
for cls in [main_klass] + bases:
schema = schema_dumper.dump(cls)
schemas.append(schema)
type_dump = OrderedDict((
('portal_type', sql_type),
('title', main_klass.__name__),
('schemas', schemas),
))
return type_dump
def update_models_cmd(store, config_file, data_dir):
saved_urls = os.path.join(data_dir, 'inventory', store, 'urls.txt')
if not os.path.exists(saved_urls):
print(f'Inventory list {saved_urls} does not exist. Please run the '
f'fetch_inventory command for store {store} and then try again.')
sys.exit(1)
config = ConfigParser()
print(f'Reading config from {config_file}')
config.read(config_file)
parser = resolve_dotted(config[store]['parser'])()
db = setup_db(config['default']['db'])
with open(saved_urls) as f:
for url in f:
url = url.strip()
print(f'Fetching and scraping {url}')
req = requests.get(url)
data = parser(req)
return
print(f'Adding {data.name} to DB')
# TODO: replace with tinydb update
db.insert(data.to_native())
def install(self):
"""
Recipe install function.
"""
# Helper functions
def split(s):
"""
Template filter splitting on any whitespace.
"""
return re.split("\s+", s.strip())
def as_bool(s):
"""
Template filter which translates the given string into a boolean.
"""
return s.lower() in ("yes", "true", "1", "on")
def strip_dict(d):
"""
Strips the values of a dictionary in place. All values are assumed
to be strings. The same dictionary object is returned.
"""
for k, v in d.iteritems():
d[k] = v.strip()
return d
# Validate template and target lists
template_file_option = self.options.get(
"template-file",
self.options.get("input")
)
target_file_option = self.options.get(
"target-file",
self.options.get("output")
)
template_files = split(template_file_option)
target_files = split(target_file_option)
if len(template_files) != len(target_files):
raise zc.buildout.UserError(
"The number of template and target files must match")
# Validate and normalise target executable option
target_executables = split(self.options.get("target-executable",
"false"))
target_executables = [as_bool(v) for v in target_executables]
if len(target_executables) == 1:
value = target_executables[0]
target_executables = (value for i in xrange(len(template_files)))
else:
if len(target_executables) != len(template_files):
raise zc.buildout.UserError("The number of target executables"
"must 0, 1 or match the number of template files")
# Assemble lists
files = zip(template_files, target_files, target_executables)
# Assemble template context
context = strip_dict(dict(self.options))
# Handle eggs specially
if "eggs" in context:
log.info("Making working set out of the eggs")
eggs = Eggs(self.buildout, self.options["recipe"], self.options)
names, eggs = eggs.working_set()
context["eggs"] = eggs
# Make recursive dict to be enable access dashed values.
context['context'] = context
# Make options from other parts available.
part_options = dict(self.buildout)
if 'parts' not in context.keys():
context.update({'parts': part_options})
else:
log.error("You should not use parts as a name of a variable,"
" since it is used internally by this receipe")
raise zc.buildout.UserError("parts used as a variable in %s"
% self.name)
filters = self.options.get('jinja2_filters')
if filters:
jinja2_filters = {}
filters = filters.split()
for filter_ in filters:
try:
jinja2_filters[filter_.split('.')[-1]] = resolve_dotted(filter_)
except ImportError, e:
raise zc.buildout.UserError("Filter '%s' not found.\n%s"
% (filter_, e))
def load_environment(self):
"""Load an :class:`Environment` instance defined in the YAML file.
Expects the following format:
.. code-block:: yaml
directory: ../static
url: /media
debug: True
updater: timestamp
filters:
- my_custom_package.my_filter
config:
compass_bin: /opt/compass
another_custom_config_value: foo
bundles:
# ...
All values, including ``directory`` and ``url`` are optional. The
syntax for defining bundles is the same as for
:meth:`~.YAMLLoader.load_bundles`.
Sample usage::
from webassets.loaders import YAMLLoader
loader = YAMLLoader('asset.yml')
env = loader.load_environment()
env['some-bundle'].urls()
"""
f, filename = self._open()
try:
obj = self.yaml.load(f) or {}
env = Environment()
# Load environment settings
for setting in ('debug', 'cache', 'versions', 'url_expire',
'auto_build', 'url', 'directory', 'manifest', 'load_path',
'cache_file_mode',
# TODO: The deprecated values; remove at some point
'expire', 'updater'):
if setting in obj:
setattr(env, setting, obj[setting])
# Treat the 'directory' option special, make it relative to the
# path of the YAML file, if we know it.
if filename and 'directory' in env.config:
env.directory = path.normpath(
path.join(path.dirname(filename),
env.config['directory']))
# Treat the 'filters' option special, it should resolve the
# entries as classes and register them to the environment
if 'filters' in obj:
try:
resolve_dotted = self._get_import_resolver()
except ImportError:
raise EnvironmentError(
"In order to use custom filters in the YAMLLoader "
"you must install the zope.dottedname package")
for filter_class in obj['filters']:
try:
cls = resolve_dotted(filter_class)
except ImportError:
raise LoaderError("Unable to resolve class %s" % filter_class)
if inspect.isclass(cls):
register_filter(cls)
else:
raise LoaderError("Custom filters must be classes "
"not modules or functions")
# Load custom config options
if 'config' in obj:
env.config.update(obj['config'])
# Load bundles
bundles = self._get_bundles(obj.get('bundles', {}))
for name, bundle in six.iteritems(bundles):
env.register(name, bundle)
return env
finally:
f.close()
def load_environment(self):
"""Load an :class:`Environment` instance defined in the YAML file.
Expects the following format:
.. code-block:: yaml
directory: ../static
url: /media
debug: True
updater: timestamp
filters:
- my_custom_package.my_filter
config:
compass_bin: /opt/compass
another_custom_config_value: foo
bundles:
# ...
All values, including ``directory`` and ``url`` are optional. The
syntax for defining bundles is the same as for
:meth:`~.YAMLLoader.load_bundles`.
Sample usage::
from webassets.loaders import YAMLLoader
loader = YAMLLoader('asset.yml')
env = loader.load_environment()
env['some-bundle'].urls()
"""
f, filename = self._open()
try:
obj = self.yaml.load(f) or {}
env = Environment()
# Load environment settings
for setting in (
'debug',
'cache',
'versions',
'url_expire',
'auto_build',
'url',
'directory',
'manifest',
'load_path',
'cache_file_mode',
# TODO: The deprecated values; remove at some point
'expire',
'updater'):
if setting in obj:
setattr(env, setting, obj[setting])
# Treat the 'directory' option special, make it relative to the
# path of the YAML file, if we know it.
if filename and 'directory' in env.config:
env.directory = path.normpath(
path.join(path.dirname(filename), env.config['directory']))
# Treat the 'filters' option special, it should resolve the
# entries as classes and register them to the environment
if 'filters' in obj:
try:
resolve_dotted = self._get_import_resolver()
except ImportError:
raise EnvironmentError(
"In order to use custom filters in the YAMLLoader "
"you must install the zope.dottedname package")
for filter_class in obj['filters']:
try:
cls = resolve_dotted(filter_class)
except ImportError:
raise LoaderError("Unable to resolve class %s" %
filter_class)
if inspect.isclass(cls):
register_filter(cls)
else:
raise LoaderError("Custom filters must be classes "
"not modules or functions")
# Load custom config options
if 'config' in obj:
env.config.update(obj['config'])
# Load bundles
bundles = self._get_bundles(obj.get('bundles', {}))
for name, bundle in six.iteritems(bundles):
env.register(name, bundle)
return env
finally:
f.close()
def install(self):
"""
Recipe install function.
"""
# Helper functions
def split(s):
"""
Template filter splitting on any whitespace.
"""
return re.split(r'\s+', s.strip())
def as_bool(s):
"""
Template filter which translates the given string into a boolean.
"""
return s.lower() in ("yes", "true", "1", "on")
def strip_dict(d):
"""
Strips the values of a dictionary in place. All values are assumed
to be strings. The same dictionary object is returned.
"""
for k, v in d.items():
d[k] = v.strip()
return d
# Validate template and target lists
template_file_option = self.options.get(
"template-file",
self.options.get("input"),
)
target_file_option = self.options.get(
"target-file",
self.options.get("output"),
)
template_files = split(template_file_option)
target_files = split(target_file_option)
if len(template_files) != len(target_files):
raise zc.buildout.UserError(
"The number of template and target files must match")
# Validate and normalise target executable option
target_executables = split(
self.options.get("target-executable", "false"))
target_executables = [as_bool(v) for v in target_executables]
if len(target_executables) == 1:
value = target_executables[0]
target_executables = (value for i in range(len(template_files)))
else:
if len(target_executables) != len(template_files):
raise zc.buildout.UserError(
"The number of target executables"
"must 0, 1 or match the number of template files")
# Assemble lists
files = zip(template_files, target_files, target_executables)
# Assemble template context
context = strip_dict(dict(self.options))
# Handle eggs specially
if "eggs" in context:
log.info("Making working set out of the eggs")
eggs = Eggs(self.buildout, self.options["recipe"], self.options)
names, eggs = eggs.working_set()
context["eggs"] = eggs
# Make recursive dict to be enable access dashed values.
context['context'] = context
# Make options from other parts available.
part_options = SafeBuildout(self.buildout)
if 'parts' not in context.keys():
context.update({'parts': part_options})
else:
log.error("You should not use parts as a name of a variable,"
" since it is used internally by this recipe")
raise zc.buildout.UserError("parts used as a variable in %s" %
self.name)
filters = self.options.get('jinja2_filters')
if filters:
jinja2_filters = {}
filters = filters.split()
for filter_ in filters:
try:
jinja2_filters[filter_.split('.')[-1]] = resolve_dotted(
filter_)
except ImportError as e:
raise zc.buildout.UserError("Filter '%s' not found.\n%s" %
(filter_, e))
else:
jinja2_filters = {}
filters = {
"split": split,
"as_bool": as_bool,
"type": type,
"eval": eval,
"re_escape": re.escape,
}
filters.update(jinja2_filters)
# Set up jinja2 environment
jinja2_env = self._jinja2_env(filters=filters)
# Load, render, and save files
for template_file, target_file, executable in files:
template = self._load_template(jinja2_env, template_file)
output = template.render(**context)
# Make target file
target_file = os.path.abspath(target_file)
self._ensure_dir(os.path.dirname(target_file))
fp = open(target_file, "wt")
fp.write(output)
fp.close()
# Chmod target file
if executable:
os.chmod(target_file, 0o755)
self.options.created(target_file)
return self.options.created()