def main():
words = ['贸易战']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
insert = Insert(
"insert into test.baidu (id,name ,url,page_rank,page_no)values('{id}','{name}' ,'{url}',{page_rank},{page_no})",
**dbconf)
p = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, join=True),
Branch(get_all_page_url,
HttpLoader(),
get_all_items,
share=False,
join=True,
route_type=HttpResponse),
insert,
)
Pipe(Timer(delay=2, until=p.finished), show_info)
BotFrame.render('ex_output/baiduspider')
BotFrame.run()
def test_routetype_count2(self):
b_counter = Counter()
b1_counter = Counter()
counter = Counter('count2')
counter1 = Counter('count1')
p = Pipe(
Loop([A(), B(), A()]),
Branch(self.only_b, counter1, route_type=B, join=True, share=True),
counter,
Branch(self.only_a,
self.a_to_b,
self.only_b,
b1_counter,
route_type=A,
share=False,
join=True), self.only_b, b_counter)
self.assertFalse(p.finished())
BotFrame.run()
self.assertEqual(counter1.count, 1)
self.assertEqual(counter.count, 4)
self.assertEqual(b1_counter.count, 2)
self.assertEqual(b_counter.count, 4)
self.assertTrue(p.finished())
def main():
words = ['贸易战', '世界杯'] * 50
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
p1 = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, collect),
Branch(get_all_page_url, HttpLoader(), get_all_items, collect),
)
Pipe(Timer(delay=delay, until=p1.finished), show_progress)
BotFrame.run()
def main():
words = ['贸易战', '世界杯']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
outputfile = aiofile('ex_output/baidu.txt')
Pipe(
urls,
HttpLoader(),
Branch(get_all_items, outputfile),
Branch(get_all_page_url, HttpLoader(), get_all_items, outputfile),
)
#生成流程图
BotFrame.render('ex_output/baiduspider')
BotFrame.run()
def test_routetype_count3(self):
self.a_count = 0
self.b_count = 0
self.c_count = 0
p = Pipe(
[A(), B(), A(), C(), C()],
Branch(lambda i: isinstance(i, (A, C)),
self.assertTrue,
route_type=[A, C]),
Branch(Branch(self.only_c, route_type=C),
share=False,
route_type=[A, C]),
)
BotFrame.run()
self.assertEqual(self.c_count, 2)
def test_routetype_count2(self):
self.a_count = 0
self.b_count = 0
p = Pipe(
Loop([A(), B(), A()]), Branch(self.only_b, route_type=B,
join=True),
Branch(self.only_a,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b)
self.assertFalse(p.finished())
BotFrame.run()
self.assertEqual(self.b_count, 5)
self.assertEqual(self.a_count, 2)
self.assertTrue(p.finished())
def test_boost(self):
import time
def very_slow(a):
time.sleep(10)
Pipe(
Timer(delay=1),
Branch(very_slow),
print,
)
def main():
Pipe(Loop([A(), B(), A(), A(), B()]),
Branch(process_A, share=False, route_type=A), process_B)
BotFrame.run()
print('----ex2')
Pipe(Loop([A(), B(), A(), A(), B()]),
Fork(process_A, route_type=A, share=False), process_B, print)
BotFrame.run()
def test_routetype_count(self):
self.a_count = 0
self.b_count = 0
Pipe(
Loop([A(), B(), A()]),
Branch(self.only_a,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b)
BotFrame.run()
self.assertTrue(self.b_count == 3)
self.assertTrue(self.a_count == 2)
def test_routetype_count(self):
b_counter = Counter()
a_counter = Counter()
Pipe(
Loop([A(), B(), A()]),
Branch(self.only_a,
a_counter,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b, b_counter)
BotFrame.run()
self.assertTrue(b_counter.count == 3)
self.assertTrue(a_counter.count == 2)
def test_routetype_no_shared(self):
Pipe(Loop([A(), B(), A()]),
Branch(self.only_a, route_type=A, share=False), self.only_b)
BotFrame.run()
def test_routetype(self):
Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A))
BotFrame.run()
def main():
Pipe(Loop(range(10)), Loop(range(10)), Branch(op_sum, print))
BotFrame.run()
print(op_sum)
def main():
Pipe(range(100000), Branch(op_sum, print))
BotFrame.run()
print(op_sum)
