def _options(self, dataset, args, path_args, path_kwargs, key):
"""For each dataset, prepare the arguments."""
if self.request.body:
content_type = self.request.headers.get('Content-Type', '')
if content_type == 'application/json':
args.update(json.loads(self.request.body))
filter_kwargs = AttrDict(dataset)
filter_kwargs.pop('modify', None)
prepare = filter_kwargs.pop('prepare', None)
queryfunction = filter_kwargs.pop('queryfunction', None)
filter_kwargs['transform_kwargs'] = {'handler': self}
# Use default arguments
defaults = {
k: v if isinstance(v, list) else [v]
for k, v in filter_kwargs.pop('default', {}).items()
}
# /(.*)/(.*) become 2 path arguments _0 and _1
defaults.update({'_%d' % k: [v] for k, v in enumerate(path_args)})
# /(?P<x>\d+)/(?P<y>\d+) become 2 keyword arguments x and y
defaults.update({k: [v] for k, v in path_kwargs.items()})
args = merge(namespaced_args(args, key), defaults, mode='setdefault')
if callable(prepare):
result = prepare(args=args, key=key, handler=self)
if result is not None:
args = result
if callable(queryfunction):
filter_kwargs['query'] = queryfunction(args=args, key=key, handler=self)
return AttrDict(
fmt=args.pop('_format', ['json'])[0],
download=args.pop('_download', [''])[0],
args=args,
meta_header=args.pop('_meta', [''])[0],
filter_kwargs=filter_kwargs,
)
def setup_error(cls, error):
'''
Sample configuration::
error:
404:
path: template.json # Use a template
autoescape: false # with no autoescape
whitespace: single # as a single line
headers:
Content-Type: application/json
500:
function: module.fn
args: [=status_code, =kwargs, =handler]
'''
if not error:
return
if not isinstance(error, dict):
return app_log.error('url:%s.error is not a dict', cls.name)
# Compile all errors handlers
cls.error = {}
for error_code, error_config in error.items():
try:
error_code = int(error_code)
if error_code < 100 or error_code > 1000:
raise ValueError()
except ValueError:
app_log.error('url.%s.error code %s is not a number (100 - 1000)',
cls.name, error_code)
continue
if not isinstance(error_config, dict):
return app_log.error('url:%s.error.%d is not a dict', cls.name, error_code)
# Make a copy of the original. When we add headers, etc, it shouldn't affect original
error_config = AttrDict(error_config)
error_path, error_function = error_config.get('path'), error_config.get('function')
if error_function:
if error_path:
error_config.pop('path')
app_log.warning('url.%s.error.%d has function: AND path:. Ignoring path:',
cls.name, error_code)
cls.error[error_code] = {'function': build_transform(
error_config,
vars=AttrDict((('status_code', None), ('kwargs', None), ('handler', None))),
filename='url:%s.error.%d' % (cls.name, error_code)
)}
elif error_path:
encoding = error_config.get('encoding', 'utf-8')
cls.error[error_code] = {'function': cls._error_fn(error_code, error_config)}
mime_type, encoding = mimetypes.guess_type(error_path, strict=False)
if mime_type:
error_config.setdefault('headers', {}).setdefault('Content-Type', mime_type)
else:
app_log.error('url.%s.error.%d must have a path or function key',
cls.name, error_code)
# Add the error configuration for reference
if error_code in cls.error:
cls.error[error_code]['conf'] = error_config
cls._write_error, cls.write_error = cls.write_error, cls._write_custom_error
def load():
state.PROGRAMS = Tree()
# Add configured players
for pcls in [Player, Downloader, Postprocessor, ShellCommand]:
ptype = camel_to_snake(pcls.__name__)
cfgkey = ptype + "s"
for name, cfg in config.settings.profile[cfgkey].items():
if not cfg:
cfg = AttrDict()
path = cfg.pop("path", None) or cfg.get(
"command",
distutils.spawn.find_executable(name)
)
if not path:
logger.warning(f"couldn't find command for {name}")
continue
# First, try to find by "type" config value, if present
try:
klass = next(
c for c in Program.SUBCLASSES[ptype].values()
if c.__name__.lower().replace(ptype, "")
== cfg.get("type", "").replace("-", "").lower()
)
except StopIteration:
# Next, try to find by config name matching class name
try:
klass = next(
c for c in Program.SUBCLASSES[ptype].values()
if c.cmd == name
)
except StopIteration:
# Give up and make it a generic program
klass = pcls
if cfg.get("disabled") == True:
continue
state.PROGRAMS[ptype][name] = ProgramDef(
cls=klass,
name=name,
path=path,
cfg = AttrDict(cfg)
)
# Try to find any players not configured
for ptype in Program.SUBCLASSES.keys():
cfgkey = ptype + "s"
for name, klass in Program.SUBCLASSES[ptype].items():
cfg = config.settings.profile[cfgkey][name]
if name in state.PROGRAMS[ptype] or (cfg and cfg.disabled == True):
continue
path = distutils.spawn.find_executable(name)
if path:
state.PROGRAMS[ptype][name] = ProgramDef(
cls=klass,
name=name,
path=path,
cfg = AttrDict()
)
def load(cls):
global PROGRAMS
# Add configured players
for ptype in [Player, Helper, Downloader]:
cfgkey = ptype.__name__.lower() + "s"
for name, cfg in config.settings.profile[cfgkey].items():
if not cfg:
cfg = AttrDict()
path = cfg.pop("path", None) or cfg.get(
"command", distutils.spawn.find_executable(name))
try:
# raise Exception(cls.SUBCLASSES[ptype])
klass = next(c for c in cls.SUBCLASSES[ptype].values()
if c.cmd == name)
except StopIteration:
klass = ptype
if cfg.get("disabled") == True:
logger.info(f"player {name} is disabled")
continue
PROGRAMS[ptype][name] = ProgramDef(cls=klass,
name=name,
path=path,
cfg=AttrDict(cfg))
# Try to find any players not configured
for ptype in cls.SUBCLASSES.keys():
cfgkey = ptype.__name__.lower() + "s"
for name, klass in cls.SUBCLASSES[ptype].items():
cfg = config.settings.profile[cfgkey][name]
if name in PROGRAMS[ptype] or cfg.disabled == True:
continue
path = distutils.spawn.find_executable(name)
if path:
PROGRAMS[ptype][name] = ProgramDef(cls=klass,
name=name,
path=path,
cfg=AttrDict())
def load_imports(config, source, warn=None):
'''
Post-process a config for imports.
``config`` is the data to process. ``source`` is the path where it was
loaded from.
If ``config`` has an ``import:`` key, treat all values below that as YAML
files (specified relative to ``source``) and import them in sequence.
Return a list of imported paths as :func:_pathstat objects. (This includes
``source``.)
For example, if the ``source`` is ``base.yaml`` (which has the below
configuration) and is loaded into ``config``::
app:
port: 20
start: true
path: /
import: update*.yaml # Can be any glob, e.g. */gramex.yaml
... and ``update.yaml`` looks like this::
app:
port: 30
new: yes
... then after this function is called, ``config`` looks like this::
app:
port: 20 # From base.yaml. NOT updated by update.yaml
start: true # From base.yaml
new: yes # From update.yaml
path: / # From base.yaml
The ``import:`` keys are deleted. The return value contains :func:_pathstat
values for ``base.yaml`` and ``update.yaml`` in that order.
Multiple ``import:`` values can be specified as a dictionary::
import:
first-app: app1/*.yaml
next-app: app2/*.yaml
To import sub-keys as namespaces, use::
import:
app: {path: */gramex.yaml, namespace: 'url'}
This prefixes all keys under ``url:``. Here are more examples::
namespace: True # Add namespace to all top-level keys
namespace: url # Add namespace to url.*
namespace: log.loggers # Add namespace to log.loggers.*
namespace: [True, url] # Add namespace to top level keys and url.*
By default, the prefix is the relative path of the imported YAML file
(relative to the importer).
``warn=`` is an optional list of key paths. Any conflict on dictionaries
matching any of these paths is logged as a warning. For example,
``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key
has a conflict.
'''
imported_paths = [_pathstat(source)]
root = source.absolute().parent
for key, value, node in list(walk(config)):
if isinstance(key,
six.string_types) and key.startswith('import.merge'):
# Strip the top level key(s) from import.merge values
if isinstance(value, dict):
for name, conf in value.items():
node[name] = conf
elif value:
raise ValueError('import.merge: must be dict, not %s at %s' %
(repr(value), source))
# Delete the import key
del node[key]
elif key == 'import':
# Convert "import: path" to "import: {app: path}"
if isinstance(value, six.string_types):
value = {'apps': value}
# Allow "import: [path, path]" to "import: {app0: path, app1: path}"
elif isinstance(value, list):
value = OrderedDict(
(('app%d' % i, conf) for i, conf in enumerate(value)))
# By now, import: should be a dict
elif not isinstance(value, dict):
raise ValueError(
'import: must be string/list/dict, not %s at %s' %
(repr(value), source))
# If already a dict with a single import via 'path', convert to dict of apps
if 'path' in value:
value = {'app': value}
for name, conf in value.items():
if not isinstance(conf, dict):
conf = AttrDict(path=conf)
if 'path' not in conf:
raise ValueError('import: has no conf at %s' % source)
paths = conf.pop('path')
paths = paths if isinstance(paths, list) else [paths]
globbed_paths = []
for path in paths:
globbed_paths += sorted(
root.glob(path)) if '*' in path else [Path(path)]
ns = conf.pop('namespace', None)
for path in globbed_paths:
abspath = root.joinpath(path)
new_conf = _yaml_open(abspath, **conf)
if ns is not None:
prefix = Path(path).as_posix()
new_conf = _add_ns(new_conf, ns, name + ':' + prefix)
imported_paths += load_imports(new_conf, source=abspath)
merge(old=node, new=new_conf, mode='setdefault', warn=warn)
# Delete the import key
del node[key]
return imported_paths
