def test_filter3(self): Pipe([A(), B(), C()], Filter(route_type=[A, B], route_func=lambda r: isinstance(r, (A, C))), self.only_a) BotFrame.run()
def test_routetype_count2(self): b_counter = Counter() b1_counter = Counter() counter = Counter('count2') counter1 = Counter('count1') p = Pipe( Loop([A(), B(), A()]), Branch(self.only_b, counter1, route_type=B, join=True, share=True), counter, Branch(self.only_a, self.a_to_b, self.only_b, b1_counter, route_type=A, share=False, join=True), self.only_b, b_counter) self.assertFalse(p.finished()) BotFrame.run() self.assertEqual(counter1.count, 1) self.assertEqual(counter.count, 4) self.assertEqual(b1_counter.count, 2) self.assertEqual(b_counter.count, 4) self.assertTrue(p.finished())
def main(): words = ['贸易战'] baidu_url = 'https://www.baidu.com/s?wd=%s' urls = [baidu_url % (word) for word in words] # make data flow net insert = Insert( "insert into test.baidu (id,name ,url,page_rank,page_no)values('{id}','{name}' ,'{url}',{page_rank},{page_no})", **dbconf) p = Pipe( Loop(urls), HttpLoader(), Branch(get_all_items, join=True), Branch(get_all_page_url, HttpLoader(), get_all_items, share=False, join=True, route_type=HttpResponse), insert, ) Pipe(Timer(delay=2, until=p.finished), show_info) BotFrame.render('ex_output/baiduspider') BotFrame.run()
def test_fork(self): self.a_count = 0 self.b_count = 0 p = Pipe(Loop([A(), A()]), Fork(self.a_to_b, self.a_to_b, share=False, join=True), self.only_b) BotFrame.run()
def main(): Pipe(Loop([A(), B(), A(), A(), B()]), Branch(process_A, share=False, route_type=A), process_B) BotFrame.run() print('----ex2') Pipe(Loop([A(), B(), A(), A(), B()]), Fork(process_A, route_type=A, share=False), process_B, print) BotFrame.run()
def main(): Pipe( Timer(delay=2, max_time=5), "http://api.coindesk.com/v1/bpi/currentprice.json", HttpLoader(), lambda r: r.json['bpi']['USD']['rate_float'], print, ) BotFrame.render('simple_bitcoin_price') BotFrame.run()
def main(): print('ex1----') Pipe( Loop(range(1)), Query('select * from mysql.user limit 10 ', **dbconf, map_class=User), Fork(print), Insert('insert into jk.new_table(user,host)values ("{user}","{host}")', **dbconf), ) BotFrame.run()
def test_loop_double(self): count = 0 def sum(x): nonlocal count count += x p = Pipe(Loop(range(10)), Loop(range(10)), sum) BotFrame.run() self.assertEqual(count, 45 * 10)
def test_blockedjoin(self): from databot.flow import BlockedJoin def check(r): self.assertEqual(len(r), 2) self.assertTrue(isinstance(r[0], B)) self.assertTrue(isinstance(r[0], B)) p = Pipe(Loop([A()]), BlockedJoin(self.a_to_b, self.a_to_b), check) BotFrame.run()
def main(): Pipe( Timer(delay=2), # send timer data to pipe every 2 sen "http://api.coindesk.com/v1/bpi/currentprice.json", # send url to pipe when timer trigger HttpLoader(), # read url and load http response lambda r: r.json['bpi']['USD'][ 'rate_float'], # read http response and parese as json print, # print out ) BotFrame.render('simple_bitcoin_price') BotFrame.run()
def get_q_by_bot_id_list(cls, iid, oid): q_of_writer = set() q_of_reader = set() for i in iid: for q in BotFrame.get_botinfo_by_id(i).oq: q_of_writer.add(q) for i in oid: for q in BotFrame.get_botinfo_by_id(i).iq: q_of_reader.add(q) r = q_of_writer & q_of_reader return r.pop()
def test_routetype_count(self): self.a_count = 0 self.b_count = 0 Pipe( Loop([A(), B(), A()]), Branch(self.only_a, self.a_to_b, route_type=A, share=False, join=True), self.only_b) BotFrame.run() self.assertTrue(self.b_count == 3) self.assertTrue(self.a_count == 2)
def make_route_bot(self, iq, oq): if self.joined: q_o = oq else: q_o = queue.NullQueue() self.start_q = [] self.output_q = oq #parallel in sub network not in node for func in self.args: q_i = asyncio.Queue() self.start_q.append(q_i) BotFrame.make_bot(q_i, q_o, func)
def main(): words = ['贸易战', '世界杯'] * 50 baidu_url = 'https://www.baidu.com/s?wd=%s' urls = [baidu_url % (word) for word in words] # make data flow net p1 = Pipe( Loop(urls), HttpLoader(), Branch(get_all_items, collect), Branch(get_all_page_url, HttpLoader(), get_all_items, collect), ) Pipe(Timer(delay=delay, until=p1.finished), show_progress) BotFrame.run()
def test_routetype_count(self): b_counter = Counter() a_counter = Counter() Pipe( Loop([A(), B(), A()]), Branch(self.only_a, a_counter, self.a_to_b, route_type=A, share=False, join=True), self.only_b, b_counter) BotFrame.run() self.assertTrue(b_counter.count == 3) self.assertTrue(a_counter.count == 2)
def main(): words = ['贸易战', '世界杯'] baidu_url = 'https://www.baidu.com/s?wd=%s' urls = [baidu_url % (word) for word in words] outputfile = aiofile('ex_output/baidu.txt') Pipe( urls, HttpLoader(), Branch(get_all_items, outputfile), Branch(get_all_page_url, HttpLoader(), get_all_items, outputfile), ) #生成流程图 BotFrame.render('ex_output/baiduspider') BotFrame.run()
def __init__(self, *args): q_o = queue.GodQueue() # get this pip own inside bot self.start_index = len(BotFrame.bots) self.q_start = q_o self.joined = False for func in args: q_i = q_o if func == args[-1]: q_o = queue.NullQueue() else: if config.replay_mode: q_o = queue.CachedQueue() else: q_o = queue.DataQueue() bis = BotFrame.make_bot(q_i, q_o, func) for b in bis: b.flow = 'main' if isinstance(func, Route): if hasattr(func, 'joined') and func.joined: self.joined = True self.end_index = len(BotFrame.bots) for i in range(self.start_index, self.end_index): BotFrame.bots[i].pipeline = str(self) self.q_end = q_o if self.joined or config.joined_network: self.check_joined_node()
def make_route_bot(self, iq, oq): self.output_q = oq q_o = queue.DataQueue() self.start_q = [q_o] for func in self.args: q_i = q_o if self.is_last_one(self.args, func): if self.joined: q_o = oq else: q_o = queue.NullQueue() else: q_o = queue.DataQueue() BotFrame.make_bot(q_i, q_o, func)
def test_routetype_count2(self): self.a_count = 0 self.b_count = 0 p = Pipe( Loop([A(), B(), A()]), Branch(self.only_b, route_type=B, join=True), Branch(self.only_a, self.a_to_b, route_type=A, share=False, join=True), self.only_b) self.assertFalse(p.finished()) BotFrame.run() self.assertEqual(self.b_count, 5) self.assertEqual(self.a_count, 2) self.assertTrue(p.finished())
def test_routetype_count3(self): self.a_count = 0 self.b_count = 0 self.c_count = 0 p = Pipe( [A(), B(), A(), C(), C()], Branch(lambda i: isinstance(i, (A, C)), self.assertTrue, route_type=[A, C]), Branch(Branch(self.only_c, route_type=C), share=False, route_type=[A, C]), ) BotFrame.run() self.assertEqual(self.c_count, 2)
def make_route_bot(self, oq): q_o = queue.DataQueue() self.start_q = [q_o] self.output_q = oq # if self.share: # self.start_q.append(oq) for idx, func in enumerate(self.args): q_i = q_o if idx == len(self.args) - 1: if self.joined: q_o = self.output_q else: q_o = queue.NullQueue() else: q_o = queue.DataQueue() BotFrame.make_bot(q_i, q_o, func)
def main(): hget = HttpLoader(timeout=2) Pipe( flow.Timer(delay=3, max_time=5), Join( Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD", hget, parse_kraken), Return( "https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC", hget, parse_bittrex), ), print, ) BotFrame.render('ex_output/bitcoin_arbitrage') BotFrame.run()
def test_blockedjoin_exception2(self): from databot.flow import BlockedJoin from databot.config import config config.exception_policy = config.Exception_pipein def raise_exception(a): raise Exception() def check(r): self.assertEqual(len(r), 2) self.assertTrue(isinstance(r[0], Exception)) self.assertTrue(isinstance(r[1], B)) p = Pipe(Loop([A()]), BlockedJoin(raise_exception, self.a_to_b), check) BotFrame.run() config.exception_policy = config.Exception_default
def main(): httpload=HttpLoader(timeout=2) Pipe( Timer(delay=10,max_time=5), BlockedJoin( Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD",httpload , parse_kraken), Return("https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC", httpload, parse_bittrex), Return("https://www.bitstamp.net/api/ticker/", httpload, parse_bitstamp), Return("https://api.bitfinex.com/v1/ticker/btcusd", httpload, parse_bitfinex), Return("https://bitpay.com/api/rates", httpload, parse_bitpay), Return("http://api.coindesk.com/v1/bpi/currentprice.json", httpload, parse_coindesk), ), print, ) BotFrame.render('ex_output/bitcoin_arbitrage') BotFrame.run()
def make_route_bot(self, iq, oq): self.start_q = [] self.tmp_output_q = [] self.output_q = oq self.share = False self.joined = True self.route_type = [object] self.start_index = len(BotFrame.bots) for func in self.args: i_q = asyncio.Queue(maxsize=1) o_q = queue.DataQueue() self.start_q.append(i_q) self.tmp_output_q.append(o_q) BotFrame.make_bot(i_q, o_q, func) self.end_index = len(BotFrame.bots)
def make_route_bot(self, oq): self.start_q = [] self.output_q = oq self.inner_output_q = queue.DataQueue() self.share = False self.joined = True self.route_type = [object] self.joined_result = {} self.start_index = len(BotFrame.bots) for func in self.args: i_q = asyncio.Queue(maxsize=1) self.start_q.append(i_q) BotFrame.make_bot(i_q, self.inner_output_q, func) BotFrame.make_bot(self.inner_output_q, self.output_q, self.join_merge, raw_bdata=True) self.end_index = len(BotFrame.bots)
def main(): print('ex1----') Pipe(Loop(range(1)), Query('select * from mysql.user limit 10 ', **dbconf, map_class=User), print) BotFrame.run() print('ex2----param bind with inout') Pipe(Loop(range(10)), Query('select %s ', **dbconf), print) BotFrame.run() print('ex3----param bind with dict') u = User() u.user = '******' Pipe( Loop([{ 'user': '******' }, u.__dict__]), Query('select host,user from mysql.user where user="******" ', **dbconf), print) BotFrame.run()
def show_info(i): BotFrame.debug()
def test_routetype_no_shared(self): Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A, share=False), self.only_b) BotFrame.run()
def test_routetype(self): Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A)) BotFrame.run()