def run(): input_end = RabbitmqInputEndpoint('http_callback:input', **RABBITMQ_CONF) server = pipeflow.Server() group = server.add_group('main', MAX_WORKERS) group.add_input_endpoint('input', input_end) group.set_handle(handle_worker) server.run()
def run(): input_end = RabbitmqInputEndpoint('amz_qa:input', **RABBITMQ_CONF) output_end = RabbitmqOutputEndpoint( ['amz_qa:input', 'amz_qa:output', 'amz_ip_ban:input'], **RABBITMQ_CONF) queue = asyncio.Queue() notify_input_end = pipeflow.QueueInputEndpoint(queue) notify_output_end = pipeflow.QueueOutputEndpoint(queue) queue = asyncio.Queue() inner_input_end = pipeflow.QueueInputEndpoint(queue) inner_output_end = pipeflow.QueueOutputEndpoint(queue) server = pipeflow.Server() task_group = server.add_group('task', 1) task_group.set_handle(handle_task) task_group.add_input_endpoint('input', input_end) task_group.add_input_endpoint('notify', notify_input_end) task_group.add_output_endpoint('input_back', output_end, 'amz_qa:input') task_group.add_output_endpoint('inner_output', inner_output_end) worker_group = server.add_group('work', MAX_WORKERS) worker_group.set_handle(handle_worker) worker_group.add_input_endpoint('inner_input', inner_input_end) worker_group.add_output_endpoint('output', output_end, 'amz_qa:output') worker_group.add_output_endpoint('inner_output', inner_output_end) worker_group.add_output_endpoint('notify', notify_output_end) worker_group.add_output_endpoint('ban', output_end, 'amz_ip_ban:input') server.run()
def run(): input_end = RabbitmqInputEndpoint('statistic:input', **RABBITMQ_CONF) server = pipeflow.Server() server.add_worker(auto_flush) group = server.add_group('main', MAX_WORKERS) group.add_input_endpoint('input', input_end) group.set_handle(handle_worker) server.run()
def run(): server = pipeflow.Server() server.add_routine_worker(cash_reward, interval=3, immediately=True) server.add_routine_worker(checkin_faker, interval=3, immediately=True) # server.add_routine_worker(update_checkin_result, interval=5, immediately=True) server.add_routine_worker(checkin_user_reward, interval=59, immediately=True) server.run()
def run(): get_popt() input_end = RabbitmqInputEndpoint('amz_bsr_result:input', **RABBITMQ_CONF) output_end = pipeflow.RedisOutputEndpoint('amz_product:output:bsr', **BSR_REDIS_CONF) server = pipeflow.Server() group = server.add_group('main', 1) group.set_handle(handle_worker) group.add_input_endpoint('input', input_end) group.add_output_endpoint('output', output_end) server.run()
def run(): input_end = NsqInputEndpoint(TOPIC_NAME, 'ql_callback_worker', WORKER_NUMBER, **INPUT_NSQ_CONF) logger.info('连接nsq成功,topic_name = {}, nsq_address={}'.format(TOPIC_NAME, INPUT_NSQ_CONF)) server = pipeflow.Server() logger.info("pipeflow开始工作") group = server.add_group('main', WORKER_NUMBER) logger.info("抓取任务") group.set_handle(callback_handle) logger.info("处理任务") group.add_input_endpoint('input', input_end) # server.add_routine_worker(ebay_maintain_task, interval=5, immediately=True) server.run()
def run(): ban_input_end = RabbitmqInputEndpoint('amz_ip_ban:input', **RABBITMQ_CONF) release_ip_end = TimeInputEndpoint('amz_banned_ip', **IP_REDIS_CONF) ban_ip_end = TimeOutputEndpoint([('amz_banned_ip', 2880)], **IP_REDIS_CONF) server = pipeflow.Server() group = server.add_group('remove', 1) group.add_input_endpoint('input', ban_input_end) group.add_output_endpoint('output', ban_ip_end) group.set_handle(remove_proxy) group = server.add_group('realse', 1) group.add_input_endpoint('input', release_ip_end) group.set_handle(release_proxy) server.run()
def run(): routine_input_end = pipeflow.RedisInputEndpoint('amz_product:routine_input', host='192.168.0.10', port=6379, db=0, password=None) b_routine_input_end = pipeflow.RedisOutputEndpoint('amz_product:routine_input', host='192.168.0.10', port=6379, db=0, password=None) input_end = pipeflow.RedisInputEndpoint('amz_product:input', host='192.168.0.10', port=6379, db=0, password=None) b_input_end = pipeflow.RedisOutputEndpoint('amz_product:input', host='192.168.0.10', port=6379, db=0, password=None) output_end = pipeflow.RedisOutputEndpoint('amz_product:output', host='192.168.0.10', port=6379, db=0, password=None) server = pipeflow.Server() server.add_worker(change_ip) group = server.add_group('main', MAX_WORKERS) group.set_handle(handle_worker) group.add_input_endpoint('routine_input', routine_input_end) group.add_output_endpoint('routine_input_back', b_routine_input_end) group.add_input_endpoint('input', input_end) group.add_output_endpoint('input_back', b_input_end) group.add_output_endpoint('output', output_end) server.run()
def run(): server = pipeflow.Server() # ebay 报表任务处理 ebay_end = NsqInputEndpoint(EBAY_REPORT_TASK_TOPIC, 'ebay_analysis', WORKER_NUMBER, **INPUT_NSQ_CONF) group = server.add_group('ebay_report', WORKER_NUMBER) group.set_handle(ebay_handle) group.add_input_endpoint('input', ebay_end) # shopee 报表处理 shopee_end = NsqInputEndpoint(SHOPEE_REPORT_TASK_TOPIC, 'shopee_analysis', WORKER_NUMBER, **INPUT_NSQ_CONF) group = server.add_group('shopee_report', WORKER_NUMBER) group.set_handle(shopee_handle) group.add_input_endpoint('input', shopee_end) # 处理 Amazon报表任务 amazon_input_end = NsqInputEndpoint(AMAZON_REPORT_TASK_TOPIC, 'amazon_analysis', WORKER_NUMBER, **INPUT_NSQ_CONF) amazon_report_group = server.add_group('amazon_report', WORKER_NUMBER) amazon_report_group.set_handle(amazon_handle) amazon_report_group.add_input_endpoint("input", amazon_input_end) # 处理 wish 报表任务 wish_input_end = NsqInputEndpoint(WISH_REPORT_TASK_TOPIC, 'wish_analysis', WORKER_NUMBER, **INPUT_NSQ_CONF) wish_report_group = server.add_group('wish_report', WORKER_NUMBER) wish_report_group.set_handle(wish_handle) wish_report_group.add_input_endpoint("input", wish_input_end) # 处理 walmart 报表任务 walmart_input_end = NsqInputEndpoint(WALMART_REPORT_TASK_TOPIC, 'walmart_analysis', WORKER_NUMBER, **INPUT_NSQ_CONF) wish_report_group = server.add_group('walmart_report', WORKER_NUMBER) wish_report_group.set_handle(walmart_handle) wish_report_group.add_input_endpoint("input", walmart_input_end) # 把数据库任务 推到NSQ中 task_save_to_nsq = TaskSaveToNsqScheduler(db_session_mk) server.add_worker(task_save_to_nsq.schedule) server.run()
def run(): input_end = RabbitmqInputEndpoint('amz_product:input', **RABBITMQ_CONF) input5_end = RabbitmqInputEndpoint('amz_product:5:input', **RABBITMQ_CONF) output_end = RabbitmqOutputEndpoint([ 'amz_product:input', 'amz_product:5:input', 'amz_product:output', 'amz_ip_ban:input' ], **RABBITMQ_CONF) server = pipeflow.Server() group = server.add_group('main', MAX_WORKERS) group.set_handle(handle_worker) group.add_input_endpoint('input', input_end) group.add_input_endpoint('input5_end', input5_end) group.add_output_endpoint('input_back', output_end, 'amz_product:input') group.add_output_endpoint('input_5_back', output_end, 'amz_product:5:input') group.add_output_endpoint('output', output_end, 'amz_product:output') group.add_output_endpoint('ban', output_end, 'amz_ip_ban:input') server.run()
def run(): refresh_conf() server = pipeflow.Server() server.add_worker(refresh_routine) group = server.add_group('main', MAX_WORKERS) for i in range(len(flow_conf['merge_ls'])): conf = flow_conf['merge_ls'][i] queue_ls = flow_conf['merge_dct'][i] if conf['type'] == 'redis': ep = pipeflow.RedisOutputEndpoint(queue_ls, host=conf['host'], port=conf['port'], db=conf['db'], password=conf['password']) for queue in queue_ls: group.add_output_endpoint(queue, ep, queue) elif conf['type'] == 'rabbitmq': ep = pipeflow.RabbitmqOutputEndpoint(queue_ls, host=conf['host'], port=conf['port'], virtualhost=conf['virtualhost'], heartbeat_interval=conf['heartbeat'], login=conf['login'], password=conf['password']) for queue in queue_ls: group.add_output_endpoint(queue, ep, queue) for node in flow_conf[FLOW_NODE_CONF]: if 'o' in flow_conf[FLOW_NODE_CONF][node]: for conf in flow_conf[FLOW_NODE_CONF][node]['o']: if conf['type'] == 'redis': ep = pipeflow.RedisInputEndpoint(conf['queue'], host=conf['host'], port=conf['port'], db=conf['db'], password=conf['password']) group.add_input_endpoint(conf['queue'], ep) elif conf['type'] == 'rabbitmq': ep = pipeflow.RabbitmqInputEndpoint(conf['queue'], no_ack=True, qos=MAX_WORKERS, host=conf['host'], port=conf['port'], virtualhost=conf['virtualhost'], heartbeat_interval=conf['heartbeat'], login=conf['login'], password=conf['password']) group.add_input_endpoint(conf['queue'], ep) group.set_handle(handle_worker) server.run()