def main():
words = ['贸易战', '世界杯'] * 50
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
p1 = Pipe(
urls,
HttpLoader(),
Branch(get_all_items, collect),
Branch(get_all_page_url, HttpLoader(), get_all_items, collect),
)
Pipe(Timer(delay=delay), show_progress)
BotFlow.run()
def test_routetype_count3():
BotFlow.reset()
a_counter = Counter()
b_counter = Counter()
c_counter = Counter()
p = Pipe(
[A(), B(), A(), C(), C()],
Branch(lambda i: isinstance(i, (A, C)), route_type=[A, C]),
Branch(Branch(only_c, c_counter, route_type=C),
share=False,
route_type=[A, C]),
)
BotFlow.run()
assert (c_counter.count == 2)
def main():
Pipe(
Timer(delay=1, max_time=10),
Branch(very_slow),
print,
)
BotFlow.run()
def main():
words = ['贸易战', '世界杯']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
outputfile=AioFile('ex_output/baidu.txt')
Pipe(
urls,
HttpLoader(),
Branch(get_all_items,outputfile),
Branch(get_all_page_url, HttpLoader(), get_all_items, outputfile),
)
#生成流程图
BotFlow.debug_print()
BotFlow.render('ex_output/baiduspider')
BotFlow.run()
def test_routetype_count2():
BotFlow.reset()
b_counter = Counter()
b1_counter = Counter()
counter = Counter('count2')
counter1 = Counter('count1')
p = Pipe([A(), B(), A()],
Branch(only_b, counter1, route_type=B, join=True, share=True),
counter,
Branch(only_a,
a_to_b,
only_b,
b1_counter,
route_type=A,
share=False,
join=True), only_b, b_counter)
BotFlow.run()
assert (counter1.count == 1)
assert (counter.count == 4)
assert (b1_counter.count == 2)
assert (b_counter.count == 4)
def test_routetype_count():
BotFlow.reset()
b_counter = Counter()
a_counter = Counter()
Pipe([A(), B(), A()],
Branch(only_a,
a_counter,
a_to_b,
route_type=A,
share=False,
join=True), only_b, b_counter)
BotFlow.run()
assert (b_counter.count == 3)
assert (a_counter.count == 2)
def main():
Pipe([A(), B(), A(), A(), B()], Branch(process_A,
share=False,
route_type=A), process_B)
BotFlow.run()
def test_routetype_no_shared():
BotFlow.reset()
Pipe([A(), B(), A()], Branch(only_a, route_type=A, share=False), only_b)
BotFlow.run()
def test_routetype():
BotFlow.reset()
Pipe([A(), B(), A()], Branch(only_a, route_type=A))
BotFlow.run()
def main():
Pipe(range(10), range(10), Branch(op_sum, print))
BotFlow.run()
print(op_sum)