def from_settings(cls, settings):
classes = settings.get('MODULES', {})
modules = []
for clspath, delay in classes.iteritems():
clsobj = load_object(clspath)
if hasattr(clsobj, 'from_settings'):
obj = clsobj.from_settings(settings, delay)
else:
obj = clsobj(delay)
modules.append(obj)
enabled = [x.__class__.__name__ for x in modules]
log.msg("Enabled modules: %s" %(", ".join(enabled), ))
#load extensions
classes = settings.get('EXTENSIONS', ())
extensions = []
for clspath in classes:
clsobj = load_object(clspath)
if hasattr(clsobj, 'from_settings'):
obj = clsobj.from_settings(settings)
else:
obj = clsobj()
extensions.append(obj)
enabled_exts = [x.__class__.__name__ for x in extensions]
log.msg("Enable extensions: %s" % ", ".join(enabled_exts))
poolsize = settings.get('MAX_PROCESS_NUM', 10)
return cls(modules, extensions, poolsize)
def _load_before_strategy(self):
coms = self.settings.get("BEFORE_STRATEGIES")
for com in coms:
comcls = load_object(com)
como = comcls(self.settings, self)
self.before_strategies.append(como)
self.before_strategies_dict[como.name] = como
def _load_after_strategy(self):
coms = self.settings.get("AFTER_STRATEGIES")
for com in coms:
comcls = load_object(com)
como = comcls(self.settings, self)
self.after_strategies.append(como)
self.after_strategies_dict[como.name] = como
def _load_after_strategy(self):
coms = self.settings.get("AFTER_STRATEGIES")
for com in coms:
comcls = load_object(com)
como = comcls(self.settings, self)
self.after_strategies.append(como)
self.after_strategies_dict[como.name] = como
def _load_before_strategy(self):
coms = self.settings.get("BEFORE_STRATEGIES")
for com in coms:
comcls = load_object(com)
como = comcls(self.settings, self)
self.before_strategies.append(como)
self.before_strategies_dict[como.name] = como
def get_redis_from_settings(conf):
params = REDIS_PARAMS.copy()
params.update(usage.getdict(conf, 'REDIS_URL'))
for source, dest in SETTINGS_PARAMS_MAP.items():
val = usage.get(conf, source)
if val:
params[dest] = val
if isinstance(params.get('redis_cls'), six.string_types):
params['redis_cls'] = load_object(params['redis_cls'])
return get_redis(**params)
def inject(self, period='start'):
if not isinstance(self.settings, dict):
raise TypeError('[settings], Except to get dict type, but get %s' %
type(self.settings).__name__)
if period == 'start':
sort_settings = dict(
sorted(self.settings.items(), key=lambda d: d[1]))
else:
sort_settings = dict(
sorted(self.settings.items(), key=lambda d: d[1],
reverse=True))
for middleware, status in sort_settings.items():
if not status:
continue
downloader = load_object(middleware)()
if not self.process_enable(downloader, period):
break
def _load_component(self):
coms = self.settings.get("COMPONENTS")
for com, attrs in coms.items():
comcls = load_object(com)
self.components[comcls.name] = comcls(self.settings, self.input_data)
def _load_component(self):
coms = self.settings.get("COMPONENTS")
for com, attrs in coms.items():
comcls = load_object(com)
self.components[comcls.name] = comcls(self.settings,
self.input_data)