def from_settings(cls, settings): classes = settings.get('MODULES', {}) modules = [] for clspath, delay in classes.iteritems(): clsobj = load_object(clspath) if hasattr(clsobj, 'from_settings'): obj = clsobj.from_settings(settings, delay) else: obj = clsobj(delay) modules.append(obj) enabled = [x.__class__.__name__ for x in modules] log.msg("Enabled modules: %s" %(", ".join(enabled), )) #load extensions classes = settings.get('EXTENSIONS', ()) extensions = [] for clspath in classes: clsobj = load_object(clspath) if hasattr(clsobj, 'from_settings'): obj = clsobj.from_settings(settings) else: obj = clsobj() extensions.append(obj) enabled_exts = [x.__class__.__name__ for x in extensions] log.msg("Enable extensions: %s" % ", ".join(enabled_exts)) poolsize = settings.get('MAX_PROCESS_NUM', 10) return cls(modules, extensions, poolsize)
def _load_before_strategy(self): coms = self.settings.get("BEFORE_STRATEGIES") for com in coms: comcls = load_object(com) como = comcls(self.settings, self) self.before_strategies.append(como) self.before_strategies_dict[como.name] = como
def _load_after_strategy(self): coms = self.settings.get("AFTER_STRATEGIES") for com in coms: comcls = load_object(com) como = comcls(self.settings, self) self.after_strategies.append(como) self.after_strategies_dict[como.name] = como
def get_redis_from_settings(conf): params = REDIS_PARAMS.copy() params.update(usage.getdict(conf, 'REDIS_URL')) for source, dest in SETTINGS_PARAMS_MAP.items(): val = usage.get(conf, source) if val: params[dest] = val if isinstance(params.get('redis_cls'), six.string_types): params['redis_cls'] = load_object(params['redis_cls']) return get_redis(**params)
def inject(self, period='start'): if not isinstance(self.settings, dict): raise TypeError('[settings], Except to get dict type, but get %s' % type(self.settings).__name__) if period == 'start': sort_settings = dict( sorted(self.settings.items(), key=lambda d: d[1])) else: sort_settings = dict( sorted(self.settings.items(), key=lambda d: d[1], reverse=True)) for middleware, status in sort_settings.items(): if not status: continue downloader = load_object(middleware)() if not self.process_enable(downloader, period): break
def _load_component(self): coms = self.settings.get("COMPONENTS") for com, attrs in coms.items(): comcls = load_object(com) self.components[comcls.name] = comcls(self.settings, self.input_data)