if hasattr(settings, 'PROXY_PROVIDER'): proxy_provider = getattr( importlib.import_module(settings.PROXY_PROVIDER[0]), settings.PROXY_PROVIDER[1]) request_engine.setup_proxy_provider( proxy_provider(**settings.PROXY_PROVIDER_ARGUMENTS)) logger.info('Set up proxy provider') else: request_engine.setup_proxy_provider(CustomProxyProvider()) if hasattr(settings, 'UA_PROVIDER'): ua_provider = getattr(importlib.import_module(settings.UA_PROVIDER[0]), settings.UA_PROVIDER[1]) request_engine.setup_user_agent_provider(ua_provider()) logger.info('Set up UA provider') else: request_engine.setup_user_agent_provider(UserAgentProvider()) request_engine.setup_request_queue(env.request_queue) env.downloader = request_engine # Processors processors = importlib.import_module('app.processors') for f in glob.glob(os.path.dirname(processors.__file__) + "/*.py"): __import__('app.processors.' + os.path.basename(f)[:-3]) def start(): env.downloader.request()