def launch(hosted_handlers):
signal.signal(signal.SIGTERM, HostedHandlerManager.stop)
signal.signal(signal.SIGINT, HostedHandlerManager.stop) # for ctrl-c
common_settings.mqclient().set_stop_condition(stop_condition)
common_settings.cache_client().set_stop_condition(stop_condition)
for handler_name, handler_config in hosted_handlers.items():
handler_type = common_settings.mq_settings["handler_configs"][handler_name]["type"]
# TODO get settings module from config
handler_settings = common_settings.mq_settings["handler_configs"][handler_name].get("settings", None)
if handler_settings is not None:
handler_settings = __import__(handler_settings, {}, {}, [''])
common_settings.override_settings(handler_settings)
concurrency = int(handler_config.get("concurrency", "0"))
HostedHandlerManager.register_handler(handler_name, handler_type, concurrency)
HostedHandlerManager.start()
'''
Created on 2012-7-18
@author: baiwu
'''
import sys
import ccrawler.static_crawler.settings as settings
import ccrawler.common.settings as common_settings
common_settings.override_settings(settings)
from ccrawler.handler.hosted_handler_manager import HostedHandlerManager
if len(sys.argv) > 1:
if len(sys.argv) > 2:
concurrency = sys.argv[2]
else:
concurrency = "1"
common_settings.hosted_handlers = {sys.argv[1] : {"concurrency" : concurrency}}
HostedHandlerManager.launch()