def main(): words = ['贸易战'] baidu_url = 'https://www.baidu.com/s?wd=%s' urls = [baidu_url % (word) for word in words] # make data flow net insert = Insert( "insert into test.baidu (id,name ,url,page_rank,page_no)values('{id}','{name}' ,'{url}',{page_rank},{page_no})", **dbconf) p = Pipe( Loop(urls), HttpLoader(), Branch(get_all_items, join=True), Branch(get_all_page_url, HttpLoader(), get_all_items, share=False, join=True, route_type=HttpResponse), insert, ) Pipe(Timer(delay=2, until=p.finished), show_info) BotFrame.render('ex_output/baiduspider') BotFrame.run()
def test_routetype_count2(self): b_counter = Counter() b1_counter = Counter() counter = Counter('count2') counter1 = Counter('count1') p = Pipe( Loop([A(), B(), A()]), Branch(self.only_b, counter1, route_type=B, join=True, share=True), counter, Branch(self.only_a, self.a_to_b, self.only_b, b1_counter, route_type=A, share=False, join=True), self.only_b, b_counter) self.assertFalse(p.finished()) BotFrame.run() self.assertEqual(counter1.count, 1) self.assertEqual(counter.count, 4) self.assertEqual(b1_counter.count, 2) self.assertEqual(b_counter.count, 4) self.assertTrue(p.finished())
def main(): words = ['贸易战', '世界杯'] * 50 baidu_url = 'https://www.baidu.com/s?wd=%s' urls = [baidu_url % (word) for word in words] # make data flow net p1 = Pipe( Loop(urls), HttpLoader(), Branch(get_all_items, collect), Branch(get_all_page_url, HttpLoader(), get_all_items, collect), ) Pipe(Timer(delay=delay, until=p1.finished), show_progress) BotFrame.run()
def main(): words = ['贸易战', '世界杯'] baidu_url = 'https://www.baidu.com/s?wd=%s' urls = [baidu_url % (word) for word in words] outputfile = aiofile('ex_output/baidu.txt') Pipe( urls, HttpLoader(), Branch(get_all_items, outputfile), Branch(get_all_page_url, HttpLoader(), get_all_items, outputfile), ) #生成流程图 BotFrame.render('ex_output/baiduspider') BotFrame.run()
def test_routetype_count3(self): self.a_count = 0 self.b_count = 0 self.c_count = 0 p = Pipe( [A(), B(), A(), C(), C()], Branch(lambda i: isinstance(i, (A, C)), self.assertTrue, route_type=[A, C]), Branch(Branch(self.only_c, route_type=C), share=False, route_type=[A, C]), ) BotFrame.run() self.assertEqual(self.c_count, 2)
def test_routetype_count2(self): self.a_count = 0 self.b_count = 0 p = Pipe( Loop([A(), B(), A()]), Branch(self.only_b, route_type=B, join=True), Branch(self.only_a, self.a_to_b, route_type=A, share=False, join=True), self.only_b) self.assertFalse(p.finished()) BotFrame.run() self.assertEqual(self.b_count, 5) self.assertEqual(self.a_count, 2) self.assertTrue(p.finished())
def test_boost(self): import time def very_slow(a): time.sleep(10) Pipe( Timer(delay=1), Branch(very_slow), print, )
def main(): Pipe(Loop([A(), B(), A(), A(), B()]), Branch(process_A, share=False, route_type=A), process_B) BotFrame.run() print('----ex2') Pipe(Loop([A(), B(), A(), A(), B()]), Fork(process_A, route_type=A, share=False), process_B, print) BotFrame.run()
def test_routetype_count(self): self.a_count = 0 self.b_count = 0 Pipe( Loop([A(), B(), A()]), Branch(self.only_a, self.a_to_b, route_type=A, share=False, join=True), self.only_b) BotFrame.run() self.assertTrue(self.b_count == 3) self.assertTrue(self.a_count == 2)
def test_routetype_count(self): b_counter = Counter() a_counter = Counter() Pipe( Loop([A(), B(), A()]), Branch(self.only_a, a_counter, self.a_to_b, route_type=A, share=False, join=True), self.only_b, b_counter) BotFrame.run() self.assertTrue(b_counter.count == 3) self.assertTrue(a_counter.count == 2)
def test_routetype_no_shared(self): Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A, share=False), self.only_b) BotFrame.run()
def test_routetype(self): Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A)) BotFrame.run()
def main(): Pipe(Loop(range(10)), Loop(range(10)), Branch(op_sum, print)) BotFrame.run() print(op_sum)
def main(): Pipe(range(100000), Branch(op_sum, print)) BotFrame.run() print(op_sum)