def __init__(self, param):
# 1.拿到secret的值，用来生成sign签名
self.url, self.app_key, self.secret = Conf().api_conf()
# 2.对入参进行处理
self.param = self.param_fix(param)
def __init__(self, level=logging.NOTSET):
"""Setting up msyql handler, initializing mysql database connection via ."""
base_dispatcher_config_path = os.path.split(
os.path.realpath(__file__))[0] + "/config.conf"
self.__config = Conf([base_dispatcher_config_path])
logging.Handler.__init__(self, level)
pass
class Scheduler(object):
handlers = {}
def __init__(self):
base_dispatcher_config_path = os.path.split(
os.path.realpath(__file__))[0] + "/recever.conf"
self.config = Conf([base_dispatcher_config_path])
self.__rabbitmq = RabbitMq(self.config)
def load(self, project_name):
if project_name is None:
return None
# 1加载配置文件
project_config_path = os.path.split(
os.path.realpath(__file__)
)[0] + '/handler/' + project_name + '/' + project_name + '.conf'
self.config.do([project_config_path])
# 2动态加载处理类
handler = importlib.import_module('recever.handler.' + project_name +
'.' + project_name + '_handler')
project_handler = handler.Handler(self.config)
# 3handlers添加处理实例
self.handlers[str(project_name)] = project_handler
# 4返回处理实例
return project_handler
def handler(self, ch, method, properties, body):
result = json.loads(body)
project_name = result.get("project_name")
t_hander = self.handlers.get(project_name)
if t_hander is None:
t_hander = self.load(project_name)
if t_hander is not None:
t_hander.handle(result)
# 调用rabbitmq的回调函数方法启动接收信息
def start(self):
self.__rabbitmq.get(self.handler)
class Scheduler(object):
distributes = {}
def __init__(self,project_name):
base_dispatcher_config_path = os.path.split(os.path.realpath(__file__))[0] + "/dispatcher.conf"
self.config = Conf([base_dispatcher_config_path])
self.__distribute = self.load(project_name)
def load(self, project_name):
if project_name is None:
return None
# 1加载配置文件
project_config_path = os.path.split(os.path.realpath(__file__))[0] + '/distribute/' + project_name + '/' + project_name + '.conf'
self.config.do([project_config_path])
# 2动态加载处理类
distribute = importlib.import_module('dispatcher.distribute.' + project_name + '.' + project_name + '_distribute')
project_distribute = distribute.Distribute(self.config)
# 4返回处理实例
return project_distribute
# 调用rabbitmq的回调函数方法启动接收信息
def start(self):
self.__distribute.dispatch()
# raise Exception("invalid project name,you must give params like '-n ent'")
pn = 'bulletin'
# 初始化配置文件
conf_list = []
# 加载标准配置文件
basePath = os.path.split(os.path.realpath(__file__))[0] + '/worker.conf'
#加载对应项目的配置文件
projectPath = os.path.split(
os.path.realpath(__file__))[0] + '/parse/' + pn + '/' + pn + '.conf'
conf_list.append(basePath)
conf_list.append(projectPath)
config = Conf(conf_list)
thread_count = config.get("base", "thread_count")
# 启动线程调用解析
for i in range(int(thread_count)):
Scheduler(config, pn).start()
# 传递数据结构
# task{
# data(字典，任务条件)
# project_name(str, 标识项目名)
# nodeToken(int, 标识列表页，详情页)
# }
#
# result{
# infos（list,解析的结果）
def __init__(self):
base_dispatcher_config_path = os.path.split(
os.path.realpath(__file__))[0] + "/recever.conf"
self.config = Conf([base_dispatcher_config_path])
self.__rabbitmq = RabbitMq(self.config)
from config.config import Conf
def __init__(self,project_name):
base_dispatcher_config_path = os.path.split(os.path.realpath(__file__))[0] + "/dispatcher.conf"
self.config = Conf([base_dispatcher_config_path])
self.__distribute = self.load(project_name)
import yaml