def run():
input_end = RabbitmqInputEndpoint('http_callback:input', **RABBITMQ_CONF)
server = pipeflow.Server()
group = server.add_group('main', MAX_WORKERS)
group.add_input_endpoint('input', input_end)
group.set_handle(handle_worker)
server.run()
def run():
input_end = RabbitmqInputEndpoint('amz_qa:input', **RABBITMQ_CONF)
output_end = RabbitmqOutputEndpoint(
['amz_qa:input', 'amz_qa:output', 'amz_ip_ban:input'], **RABBITMQ_CONF)
queue = asyncio.Queue()
notify_input_end = pipeflow.QueueInputEndpoint(queue)
notify_output_end = pipeflow.QueueOutputEndpoint(queue)
queue = asyncio.Queue()
inner_input_end = pipeflow.QueueInputEndpoint(queue)
inner_output_end = pipeflow.QueueOutputEndpoint(queue)
server = pipeflow.Server()
task_group = server.add_group('task', 1)
task_group.set_handle(handle_task)
task_group.add_input_endpoint('input', input_end)
task_group.add_input_endpoint('notify', notify_input_end)
task_group.add_output_endpoint('input_back', output_end, 'amz_qa:input')
task_group.add_output_endpoint('inner_output', inner_output_end)
worker_group = server.add_group('work', MAX_WORKERS)
worker_group.set_handle(handle_worker)
worker_group.add_input_endpoint('inner_input', inner_input_end)
worker_group.add_output_endpoint('output', output_end, 'amz_qa:output')
worker_group.add_output_endpoint('inner_output', inner_output_end)
worker_group.add_output_endpoint('notify', notify_output_end)
worker_group.add_output_endpoint('ban', output_end, 'amz_ip_ban:input')
server.run()
def run():
input_end = RabbitmqInputEndpoint('statistic:input', **RABBITMQ_CONF)
server = pipeflow.Server()
server.add_worker(auto_flush)
group = server.add_group('main', MAX_WORKERS)
group.add_input_endpoint('input', input_end)
group.set_handle(handle_worker)
server.run()
def run():
server = pipeflow.Server()
server.add_routine_worker(cash_reward, interval=3, immediately=True)
server.add_routine_worker(checkin_faker, interval=3, immediately=True)
# server.add_routine_worker(update_checkin_result, interval=5, immediately=True)
server.add_routine_worker(checkin_user_reward,
interval=59,
immediately=True)
server.run()
def run():
get_popt()
input_end = RabbitmqInputEndpoint('amz_bsr_result:input', **RABBITMQ_CONF)
output_end = pipeflow.RedisOutputEndpoint('amz_product:output:bsr',
**BSR_REDIS_CONF)
server = pipeflow.Server()
group = server.add_group('main', 1)
group.set_handle(handle_worker)
group.add_input_endpoint('input', input_end)
group.add_output_endpoint('output', output_end)
server.run()
def run():
input_end = NsqInputEndpoint(TOPIC_NAME, 'ql_callback_worker', WORKER_NUMBER, **INPUT_NSQ_CONF)
logger.info('连接nsq成功,topic_name = {}, nsq_address={}'.format(TOPIC_NAME, INPUT_NSQ_CONF))
server = pipeflow.Server()
logger.info("pipeflow开始工作")
group = server.add_group('main', WORKER_NUMBER)
logger.info("抓取任务")
group.set_handle(callback_handle)
logger.info("处理任务")
group.add_input_endpoint('input', input_end)
# server.add_routine_worker(ebay_maintain_task, interval=5, immediately=True)
server.run()
def run():
ban_input_end = RabbitmqInputEndpoint('amz_ip_ban:input', **RABBITMQ_CONF)
release_ip_end = TimeInputEndpoint('amz_banned_ip', **IP_REDIS_CONF)
ban_ip_end = TimeOutputEndpoint([('amz_banned_ip', 2880)], **IP_REDIS_CONF)
server = pipeflow.Server()
group = server.add_group('remove', 1)
group.add_input_endpoint('input', ban_input_end)
group.add_output_endpoint('output', ban_ip_end)
group.set_handle(remove_proxy)
group = server.add_group('realse', 1)
group.add_input_endpoint('input', release_ip_end)
group.set_handle(release_proxy)
server.run()
def run():
routine_input_end = pipeflow.RedisInputEndpoint('amz_product:routine_input', host='192.168.0.10', port=6379, db=0, password=None)
b_routine_input_end = pipeflow.RedisOutputEndpoint('amz_product:routine_input', host='192.168.0.10', port=6379, db=0, password=None)
input_end = pipeflow.RedisInputEndpoint('amz_product:input', host='192.168.0.10', port=6379, db=0, password=None)
b_input_end = pipeflow.RedisOutputEndpoint('amz_product:input', host='192.168.0.10', port=6379, db=0, password=None)
output_end = pipeflow.RedisOutputEndpoint('amz_product:output', host='192.168.0.10', port=6379, db=0, password=None)
server = pipeflow.Server()
server.add_worker(change_ip)
group = server.add_group('main', MAX_WORKERS)
group.set_handle(handle_worker)
group.add_input_endpoint('routine_input', routine_input_end)
group.add_output_endpoint('routine_input_back', b_routine_input_end)
group.add_input_endpoint('input', input_end)
group.add_output_endpoint('input_back', b_input_end)
group.add_output_endpoint('output', output_end)
server.run()
def run():
server = pipeflow.Server()
# ebay 报表任务处理
ebay_end = NsqInputEndpoint(EBAY_REPORT_TASK_TOPIC, 'ebay_analysis',
WORKER_NUMBER, **INPUT_NSQ_CONF)
group = server.add_group('ebay_report', WORKER_NUMBER)
group.set_handle(ebay_handle)
group.add_input_endpoint('input', ebay_end)
# shopee 报表处理
shopee_end = NsqInputEndpoint(SHOPEE_REPORT_TASK_TOPIC, 'shopee_analysis',
WORKER_NUMBER, **INPUT_NSQ_CONF)
group = server.add_group('shopee_report', WORKER_NUMBER)
group.set_handle(shopee_handle)
group.add_input_endpoint('input', shopee_end)
# 处理 Amazon报表任务
amazon_input_end = NsqInputEndpoint(AMAZON_REPORT_TASK_TOPIC,
'amazon_analysis', WORKER_NUMBER,
**INPUT_NSQ_CONF)
amazon_report_group = server.add_group('amazon_report', WORKER_NUMBER)
amazon_report_group.set_handle(amazon_handle)
amazon_report_group.add_input_endpoint("input", amazon_input_end)
# 处理 wish 报表任务
wish_input_end = NsqInputEndpoint(WISH_REPORT_TASK_TOPIC, 'wish_analysis',
WORKER_NUMBER, **INPUT_NSQ_CONF)
wish_report_group = server.add_group('wish_report', WORKER_NUMBER)
wish_report_group.set_handle(wish_handle)
wish_report_group.add_input_endpoint("input", wish_input_end)
# 处理 walmart 报表任务
walmart_input_end = NsqInputEndpoint(WALMART_REPORT_TASK_TOPIC,
'walmart_analysis', WORKER_NUMBER,
**INPUT_NSQ_CONF)
wish_report_group = server.add_group('walmart_report', WORKER_NUMBER)
wish_report_group.set_handle(walmart_handle)
wish_report_group.add_input_endpoint("input", walmart_input_end)
# 把数据库任务 推到NSQ中
task_save_to_nsq = TaskSaveToNsqScheduler(db_session_mk)
server.add_worker(task_save_to_nsq.schedule)
server.run()
def run():
input_end = RabbitmqInputEndpoint('amz_product:input', **RABBITMQ_CONF)
input5_end = RabbitmqInputEndpoint('amz_product:5:input', **RABBITMQ_CONF)
output_end = RabbitmqOutputEndpoint([
'amz_product:input', 'amz_product:5:input', 'amz_product:output',
'amz_ip_ban:input'
], **RABBITMQ_CONF)
server = pipeflow.Server()
group = server.add_group('main', MAX_WORKERS)
group.set_handle(handle_worker)
group.add_input_endpoint('input', input_end)
group.add_input_endpoint('input5_end', input5_end)
group.add_output_endpoint('input_back', output_end, 'amz_product:input')
group.add_output_endpoint('input_5_back', output_end,
'amz_product:5:input')
group.add_output_endpoint('output', output_end, 'amz_product:output')
group.add_output_endpoint('ban', output_end, 'amz_ip_ban:input')
server.run()
def run():
refresh_conf()
server = pipeflow.Server()
server.add_worker(refresh_routine)
group = server.add_group('main', MAX_WORKERS)
for i in range(len(flow_conf['merge_ls'])):
conf = flow_conf['merge_ls'][i]
queue_ls = flow_conf['merge_dct'][i]
if conf['type'] == 'redis':
ep = pipeflow.RedisOutputEndpoint(queue_ls, host=conf['host'],
port=conf['port'], db=conf['db'],
password=conf['password'])
for queue in queue_ls:
group.add_output_endpoint(queue, ep, queue)
elif conf['type'] == 'rabbitmq':
ep = pipeflow.RabbitmqOutputEndpoint(queue_ls, host=conf['host'],
port=conf['port'], virtualhost=conf['virtualhost'],
heartbeat_interval=conf['heartbeat'],
login=conf['login'], password=conf['password'])
for queue in queue_ls:
group.add_output_endpoint(queue, ep, queue)
for node in flow_conf[FLOW_NODE_CONF]:
if 'o' in flow_conf[FLOW_NODE_CONF][node]:
for conf in flow_conf[FLOW_NODE_CONF][node]['o']:
if conf['type'] == 'redis':
ep = pipeflow.RedisInputEndpoint(conf['queue'], host=conf['host'],
port=conf['port'], db=conf['db'],
password=conf['password'])
group.add_input_endpoint(conf['queue'], ep)
elif conf['type'] == 'rabbitmq':
ep = pipeflow.RabbitmqInputEndpoint(conf['queue'], no_ack=True, qos=MAX_WORKERS, host=conf['host'],
port=conf['port'], virtualhost=conf['virtualhost'],
heartbeat_interval=conf['heartbeat'],
login=conf['login'], password=conf['password'])
group.add_input_endpoint(conf['queue'], ep)
group.set_handle(handle_worker)
server.run()