def _options(self, dataset, args, path_args, path_kwargs, key): """For each dataset, prepare the arguments.""" if self.request.body: content_type = self.request.headers.get('Content-Type', '') if content_type == 'application/json': args.update(json.loads(self.request.body)) filter_kwargs = AttrDict(dataset) filter_kwargs.pop('modify', None) prepare = filter_kwargs.pop('prepare', None) queryfunction = filter_kwargs.pop('queryfunction', None) filter_kwargs['transform_kwargs'] = {'handler': self} # Use default arguments defaults = { k: v if isinstance(v, list) else [v] for k, v in filter_kwargs.pop('default', {}).items() } # /(.*)/(.*) become 2 path arguments _0 and _1 defaults.update({'_%d' % k: [v] for k, v in enumerate(path_args)}) # /(?P<x>\d+)/(?P<y>\d+) become 2 keyword arguments x and y defaults.update({k: [v] for k, v in path_kwargs.items()}) args = merge(namespaced_args(args, key), defaults, mode='setdefault') if callable(prepare): result = prepare(args=args, key=key, handler=self) if result is not None: args = result if callable(queryfunction): filter_kwargs['query'] = queryfunction(args=args, key=key, handler=self) return AttrDict( fmt=args.pop('_format', ['json'])[0], download=args.pop('_download', [''])[0], args=args, meta_header=args.pop('_meta', [''])[0], filter_kwargs=filter_kwargs, )
def setup_error(cls, error): ''' Sample configuration:: error: 404: path: template.json # Use a template autoescape: false # with no autoescape whitespace: single # as a single line headers: Content-Type: application/json 500: function: module.fn args: [=status_code, =kwargs, =handler] ''' if not error: return if not isinstance(error, dict): return app_log.error('url:%s.error is not a dict', cls.name) # Compile all errors handlers cls.error = {} for error_code, error_config in error.items(): try: error_code = int(error_code) if error_code < 100 or error_code > 1000: raise ValueError() except ValueError: app_log.error('url.%s.error code %s is not a number (100 - 1000)', cls.name, error_code) continue if not isinstance(error_config, dict): return app_log.error('url:%s.error.%d is not a dict', cls.name, error_code) # Make a copy of the original. When we add headers, etc, it shouldn't affect original error_config = AttrDict(error_config) error_path, error_function = error_config.get('path'), error_config.get('function') if error_function: if error_path: error_config.pop('path') app_log.warning('url.%s.error.%d has function: AND path:. Ignoring path:', cls.name, error_code) cls.error[error_code] = {'function': build_transform( error_config, vars=AttrDict((('status_code', None), ('kwargs', None), ('handler', None))), filename='url:%s.error.%d' % (cls.name, error_code) )} elif error_path: encoding = error_config.get('encoding', 'utf-8') cls.error[error_code] = {'function': cls._error_fn(error_code, error_config)} mime_type, encoding = mimetypes.guess_type(error_path, strict=False) if mime_type: error_config.setdefault('headers', {}).setdefault('Content-Type', mime_type) else: app_log.error('url.%s.error.%d must have a path or function key', cls.name, error_code) # Add the error configuration for reference if error_code in cls.error: cls.error[error_code]['conf'] = error_config cls._write_error, cls.write_error = cls.write_error, cls._write_custom_error
def load(): state.PROGRAMS = Tree() # Add configured players for pcls in [Player, Downloader, Postprocessor, ShellCommand]: ptype = camel_to_snake(pcls.__name__) cfgkey = ptype + "s" for name, cfg in config.settings.profile[cfgkey].items(): if not cfg: cfg = AttrDict() path = cfg.pop("path", None) or cfg.get( "command", distutils.spawn.find_executable(name) ) if not path: logger.warning(f"couldn't find command for {name}") continue # First, try to find by "type" config value, if present try: klass = next( c for c in Program.SUBCLASSES[ptype].values() if c.__name__.lower().replace(ptype, "") == cfg.get("type", "").replace("-", "").lower() ) except StopIteration: # Next, try to find by config name matching class name try: klass = next( c for c in Program.SUBCLASSES[ptype].values() if c.cmd == name ) except StopIteration: # Give up and make it a generic program klass = pcls if cfg.get("disabled") == True: continue state.PROGRAMS[ptype][name] = ProgramDef( cls=klass, name=name, path=path, cfg = AttrDict(cfg) ) # Try to find any players not configured for ptype in Program.SUBCLASSES.keys(): cfgkey = ptype + "s" for name, klass in Program.SUBCLASSES[ptype].items(): cfg = config.settings.profile[cfgkey][name] if name in state.PROGRAMS[ptype] or (cfg and cfg.disabled == True): continue path = distutils.spawn.find_executable(name) if path: state.PROGRAMS[ptype][name] = ProgramDef( cls=klass, name=name, path=path, cfg = AttrDict() )
def load(cls): global PROGRAMS # Add configured players for ptype in [Player, Helper, Downloader]: cfgkey = ptype.__name__.lower() + "s" for name, cfg in config.settings.profile[cfgkey].items(): if not cfg: cfg = AttrDict() path = cfg.pop("path", None) or cfg.get( "command", distutils.spawn.find_executable(name)) try: # raise Exception(cls.SUBCLASSES[ptype]) klass = next(c for c in cls.SUBCLASSES[ptype].values() if c.cmd == name) except StopIteration: klass = ptype if cfg.get("disabled") == True: logger.info(f"player {name} is disabled") continue PROGRAMS[ptype][name] = ProgramDef(cls=klass, name=name, path=path, cfg=AttrDict(cfg)) # Try to find any players not configured for ptype in cls.SUBCLASSES.keys(): cfgkey = ptype.__name__.lower() + "s" for name, klass in cls.SUBCLASSES[ptype].items(): cfg = config.settings.profile[cfgkey][name] if name in PROGRAMS[ptype] or cfg.disabled == True: continue path = distutils.spawn.find_executable(name) if path: PROGRAMS[ptype][name] = ProgramDef(cls=klass, name=name, path=path, cfg=AttrDict())
def load_imports(config, source, warn=None): ''' Post-process a config for imports. ``config`` is the data to process. ``source`` is the path where it was loaded from. If ``config`` has an ``import:`` key, treat all values below that as YAML files (specified relative to ``source``) and import them in sequence. Return a list of imported paths as :func:_pathstat objects. (This includes ``source``.) For example, if the ``source`` is ``base.yaml`` (which has the below configuration) and is loaded into ``config``:: app: port: 20 start: true path: / import: update*.yaml # Can be any glob, e.g. */gramex.yaml ... and ``update.yaml`` looks like this:: app: port: 30 new: yes ... then after this function is called, ``config`` looks like this:: app: port: 20 # From base.yaml. NOT updated by update.yaml start: true # From base.yaml new: yes # From update.yaml path: / # From base.yaml The ``import:`` keys are deleted. The return value contains :func:_pathstat values for ``base.yaml`` and ``update.yaml`` in that order. Multiple ``import:`` values can be specified as a dictionary:: import: first-app: app1/*.yaml next-app: app2/*.yaml To import sub-keys as namespaces, use:: import: app: {path: */gramex.yaml, namespace: 'url'} This prefixes all keys under ``url:``. Here are more examples:: namespace: True # Add namespace to all top-level keys namespace: url # Add namespace to url.* namespace: log.loggers # Add namespace to log.loggers.* namespace: [True, url] # Add namespace to top level keys and url.* By default, the prefix is the relative path of the imported YAML file (relative to the importer). ``warn=`` is an optional list of key paths. Any conflict on dictionaries matching any of these paths is logged as a warning. For example, ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key has a conflict. ''' imported_paths = [_pathstat(source)] root = source.absolute().parent for key, value, node in list(walk(config)): if isinstance(key, six.string_types) and key.startswith('import.merge'): # Strip the top level key(s) from import.merge values if isinstance(value, dict): for name, conf in value.items(): node[name] = conf elif value: raise ValueError('import.merge: must be dict, not %s at %s' % (repr(value), source)) # Delete the import key del node[key] elif key == 'import': # Convert "import: path" to "import: {app: path}" if isinstance(value, six.string_types): value = {'apps': value} # Allow "import: [path, path]" to "import: {app0: path, app1: path}" elif isinstance(value, list): value = OrderedDict( (('app%d' % i, conf) for i, conf in enumerate(value))) # By now, import: should be a dict elif not isinstance(value, dict): raise ValueError( 'import: must be string/list/dict, not %s at %s' % (repr(value), source)) # If already a dict with a single import via 'path', convert to dict of apps if 'path' in value: value = {'app': value} for name, conf in value.items(): if not isinstance(conf, dict): conf = AttrDict(path=conf) if 'path' not in conf: raise ValueError('import: has no conf at %s' % source) paths = conf.pop('path') paths = paths if isinstance(paths, list) else [paths] globbed_paths = [] for path in paths: globbed_paths += sorted( root.glob(path)) if '*' in path else [Path(path)] ns = conf.pop('namespace', None) for path in globbed_paths: abspath = root.joinpath(path) new_conf = _yaml_open(abspath, **conf) if ns is not None: prefix = Path(path).as_posix() new_conf = _add_ns(new_conf, ns, name + ':' + prefix) imported_paths += load_imports(new_conf, source=abspath) merge(old=node, new=new_conf, mode='setdefault', warn=warn) # Delete the import key del node[key] return imported_paths