def test_routetype_count2(self): b_counter = Counter() b1_counter = Counter() counter = Counter('count2') counter1 = Counter('count1') p = Pipe( Loop([A(), B(), A()]), Branch(self.only_b, counter1, route_type=B, join=True, share=True), counter, Branch(self.only_a, self.a_to_b, self.only_b, b1_counter, route_type=A, share=False, join=True), self.only_b, b_counter) self.assertFalse(p.finished()) BotFrame.run() self.assertEqual(counter1.count, 1) self.assertEqual(counter.count, 4) self.assertEqual(b1_counter.count, 2) self.assertEqual(b_counter.count, 4) self.assertTrue(p.finished())
def main(): words = ['贸易战'] baidu_url = 'https://www.baidu.com/s?wd=%s' urls = [baidu_url % (word) for word in words] # make data flow net insert = Insert( "insert into test.baidu (id,name ,url,page_rank,page_no)values('{id}','{name}' ,'{url}',{page_rank},{page_no})", **dbconf) p = Pipe( Loop(urls), HttpLoader(), Branch(get_all_items, join=True), Branch(get_all_page_url, HttpLoader(), get_all_items, share=False, join=True, route_type=HttpResponse), insert, ) Pipe(Timer(delay=2, until=p.finished), show_info) BotFrame.render('ex_output/baiduspider') BotFrame.run()
def main(): Pipe(Loop([A(), B(), A(), A(), B()]), Branch(process_A, share=False, route_type=A), process_B) BotFrame.run() print('----ex2') Pipe(Loop([A(), B(), A(), A(), B()]), Fork(process_A, route_type=A, share=False), process_B, print) BotFrame.run()
def main(): words = ['贸易战', '世界杯'] * 50 baidu_url = 'https://www.baidu.com/s?wd=%s' urls = [baidu_url % (word) for word in words] # make data flow net p1 = Pipe( Loop(urls), HttpLoader(), Branch(get_all_items, collect), Branch(get_all_page_url, HttpLoader(), get_all_items, collect), ) Pipe(Timer(delay=delay, until=p1.finished), show_progress) BotFrame.run()
def test_routetype_count2(self): self.a_count = 0 self.b_count = 0 p = Pipe( Loop([A(), B(), A()]), Branch(self.only_b, route_type=B, join=True), Branch(self.only_a, self.a_to_b, route_type=A, share=False, join=True), self.only_b) self.assertFalse(p.finished()) BotFrame.run() self.assertEqual(self.b_count, 5) self.assertEqual(self.a_count, 2) self.assertTrue(p.finished())
def test_filter3(self): Pipe([A(), B(), C()], Filter(route_type=[A, B], route_func=lambda r: isinstance(r, (A, C))), self.only_a) BotFrame.run()
def test_fork(self): self.a_count = 0 self.b_count = 0 p = Pipe(Loop([A(), A()]), Fork(self.a_to_b, self.a_to_b, share=False, join=True), self.only_b) BotFrame.run()
def test_loop_double(self): count = 0 def sum(x): nonlocal count count += x p = Pipe(Loop(range(10)), Loop(range(10)), sum) BotFrame.run() self.assertEqual(count, 45 * 10)
def main(): print('ex1----') Pipe( Loop(range(1)), Query('select * from mysql.user limit 10 ', **dbconf, map_class=User), Fork(print), Insert('insert into jk.new_table(user,host)values ("{user}","{host}")', **dbconf), ) BotFrame.run()
def test_boost(self): import time def very_slow(a): time.sleep(10) Pipe( Timer(delay=1), Branch(very_slow), print, )
def main(): Pipe( Timer(delay=2, max_time=5), "http://api.coindesk.com/v1/bpi/currentprice.json", HttpLoader(), lambda r: r.json['bpi']['USD']['rate_float'], print, ) BotFrame.render('simple_bitcoin_price') BotFrame.run()
def main(): Pipe( Timer(delay=2), # send timer data to pipe every 2 sen "http://api.coindesk.com/v1/bpi/currentprice.json", # send url to pipe when timer trigger HttpLoader(), # read url and load http response lambda r: r.json['bpi']['USD'][ 'rate_float'], # read http response and parese as json print, # print out ) BotFrame.render('simple_bitcoin_price') BotFrame.run()
def test_blockedjoin(self): from databot.flow import BlockedJoin def check(r): self.assertEqual(len(r), 2) self.assertTrue(isinstance(r[0], B)) self.assertTrue(isinstance(r[0], B)) p = Pipe(Loop([A()]), BlockedJoin(self.a_to_b, self.a_to_b), check) BotFrame.run()
def test_routetype_count(self): self.a_count = 0 self.b_count = 0 Pipe( Loop([A(), B(), A()]), Branch(self.only_a, self.a_to_b, route_type=A, share=False, join=True), self.only_b) BotFrame.run() self.assertTrue(self.b_count == 3) self.assertTrue(self.a_count == 2)
def test_routetype_count(self): b_counter = Counter() a_counter = Counter() Pipe( Loop([A(), B(), A()]), Branch(self.only_a, a_counter, self.a_to_b, route_type=A, share=False, join=True), self.only_b, b_counter) BotFrame.run() self.assertTrue(b_counter.count == 3) self.assertTrue(a_counter.count == 2)
def main(): print('ex1----') Pipe(Loop(range(1)), Query('select * from mysql.user limit 10 ', **dbconf, map_class=User), print) BotFrame.run() print('ex2----param bind with inout') Pipe(Loop(range(10)), Query('select %s ', **dbconf), print) BotFrame.run() print('ex3----param bind with dict') u = User() u.user = '******' Pipe( Loop([{ 'user': '******' }, u.__dict__]), Query('select host,user from mysql.user where user="******" ', **dbconf), print) BotFrame.run()
def main(): words = ['贸易战', '世界杯'] baidu_url = 'https://www.baidu.com/s?wd=%s' urls = [baidu_url % (word) for word in words] outputfile = aiofile('ex_output/baidu.txt') Pipe( urls, HttpLoader(), Branch(get_all_items, outputfile), Branch(get_all_page_url, HttpLoader(), get_all_items, outputfile), ) #生成流程图 BotFrame.render('ex_output/baiduspider') BotFrame.run()
def test_routetype_count3(self): self.a_count = 0 self.b_count = 0 self.c_count = 0 p = Pipe( [A(), B(), A(), C(), C()], Branch(lambda i: isinstance(i, (A, C)), self.assertTrue, route_type=[A, C]), Branch(Branch(self.only_c, route_type=C), share=False, route_type=[A, C]), ) BotFrame.run() self.assertEqual(self.c_count, 2)
def main(): hget = HttpLoader(timeout=2) Pipe( flow.Timer(delay=3, max_time=5), Join( Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD", hget, parse_kraken), Return( "https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC", hget, parse_bittrex), ), print, ) BotFrame.render('ex_output/bitcoin_arbitrage') BotFrame.run()
def test_blockedjoin_exception2(self): from databot.flow import BlockedJoin from databot.config import config config.exception_policy = config.Exception_pipein def raise_exception(a): raise Exception() def check(r): self.assertEqual(len(r), 2) self.assertTrue(isinstance(r[0], Exception)) self.assertTrue(isinstance(r[1], B)) p = Pipe(Loop([A()]), BlockedJoin(raise_exception, self.a_to_b), check) BotFrame.run() config.exception_policy = config.Exception_default
def main(): httpload=HttpLoader(timeout=2) Pipe( Timer(delay=10,max_time=5), BlockedJoin( Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD",httpload , parse_kraken), Return("https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC", httpload, parse_bittrex), Return("https://www.bitstamp.net/api/ticker/", httpload, parse_bitstamp), Return("https://api.bitfinex.com/v1/ticker/btcusd", httpload, parse_bitfinex), Return("https://bitpay.com/api/rates", httpload, parse_bitpay), Return("http://api.coindesk.com/v1/bpi/currentprice.json", httpload, parse_coindesk), ), print, ) BotFrame.render('ex_output/bitcoin_arbitrage') BotFrame.run()
def main(): Pipe(Timer(delay=0.5, max_time=10), FileSaver('a.txt')) BotFrame.run()
def main(): Pipe(range(10), BlockedJoin(double, triple), print) BotFrame.render('ex_output/blockedjoin') BotFrame.run()
def test_routetype_no_shared(self): Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A, share=False), self.only_b) BotFrame.run()
def test_routetype(self): Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A)) BotFrame.run()
def main(): Pipe(Loop(range(10)), double, triple, print) BotFrame.run()
def test_filter2(self): Pipe([A(), B(), C()], Filter(route_func=lambda r: isinstance(r, A)), self.only_a)
def test_filter(self): Pipe([A(), B(), C()], Filter(route_type=A), self.only_a)
def main(): Pipe(Loop(range(10)), Loop(range(10)), Branch(op_sum, print)) BotFrame.run() print(op_sum)
def main(): Pipe(Loop(range(1000000)), Fork(op_sum), print) BotFrame.run() print(op_sum)