def test_filter3(self):
Pipe([A(), B(), C()],
Filter(route_type=[A, B],
route_func=lambda r: isinstance(r, (A, C))), self.only_a)
BotFrame.run()
def test_routetype_count2(self):
b_counter = Counter()
b1_counter = Counter()
counter = Counter('count2')
counter1 = Counter('count1')
p = Pipe(
Loop([A(), B(), A()]),
Branch(self.only_b, counter1, route_type=B, join=True, share=True),
counter,
Branch(self.only_a,
self.a_to_b,
self.only_b,
b1_counter,
route_type=A,
share=False,
join=True), self.only_b, b_counter)
self.assertFalse(p.finished())
BotFrame.run()
self.assertEqual(counter1.count, 1)
self.assertEqual(counter.count, 4)
self.assertEqual(b1_counter.count, 2)
self.assertEqual(b_counter.count, 4)
self.assertTrue(p.finished())
def main():
words = ['贸易战']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
insert = Insert(
"insert into test.baidu (id,name ,url,page_rank,page_no)values('{id}','{name}' ,'{url}',{page_rank},{page_no})",
**dbconf)
p = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, join=True),
Branch(get_all_page_url,
HttpLoader(),
get_all_items,
share=False,
join=True,
route_type=HttpResponse),
insert,
)
Pipe(Timer(delay=2, until=p.finished), show_info)
BotFrame.render('ex_output/baiduspider')
BotFrame.run()
def test_fork(self):
self.a_count = 0
self.b_count = 0
p = Pipe(Loop([A(), A()]),
Fork(self.a_to_b, self.a_to_b, share=False, join=True),
self.only_b)
BotFrame.run()
def main():
Pipe(Loop([A(), B(), A(), A(), B()]),
Branch(process_A, share=False, route_type=A), process_B)
BotFrame.run()
print('----ex2')
Pipe(Loop([A(), B(), A(), A(), B()]),
Fork(process_A, route_type=A, share=False), process_B, print)
BotFrame.run()
def main():
Pipe(
Timer(delay=2, max_time=5),
"http://api.coindesk.com/v1/bpi/currentprice.json",
HttpLoader(),
lambda r: r.json['bpi']['USD']['rate_float'],
print,
)
BotFrame.render('simple_bitcoin_price')
BotFrame.run()
def main():
print('ex1----')
Pipe(
Loop(range(1)),
Query('select * from mysql.user limit 10 ', **dbconf, map_class=User),
Fork(print),
Insert('insert into jk.new_table(user,host)values ("{user}","{host}")',
**dbconf),
)
BotFrame.run()
def test_loop_double(self):
count = 0
def sum(x):
nonlocal count
count += x
p = Pipe(Loop(range(10)), Loop(range(10)), sum)
BotFrame.run()
self.assertEqual(count, 45 * 10)
def test_blockedjoin(self):
from databot.flow import BlockedJoin
def check(r):
self.assertEqual(len(r), 2)
self.assertTrue(isinstance(r[0], B))
self.assertTrue(isinstance(r[0], B))
p = Pipe(Loop([A()]), BlockedJoin(self.a_to_b, self.a_to_b), check)
BotFrame.run()
def main():
Pipe(
Timer(delay=2), # send timer data to pipe every 2 sen
"http://api.coindesk.com/v1/bpi/currentprice.json", # send url to pipe when timer trigger
HttpLoader(), # read url and load http response
lambda r: r.json['bpi']['USD'][
'rate_float'], # read http response and parese as json
print, # print out
)
BotFrame.render('simple_bitcoin_price')
BotFrame.run()
def get_q_by_bot_id_list(cls, iid, oid):
q_of_writer = set()
q_of_reader = set()
for i in iid:
for q in BotFrame.get_botinfo_by_id(i).oq:
q_of_writer.add(q)
for i in oid:
for q in BotFrame.get_botinfo_by_id(i).iq:
q_of_reader.add(q)
r = q_of_writer & q_of_reader
return r.pop()
def test_routetype_count(self):
self.a_count = 0
self.b_count = 0
Pipe(
Loop([A(), B(), A()]),
Branch(self.only_a,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b)
BotFrame.run()
self.assertTrue(self.b_count == 3)
self.assertTrue(self.a_count == 2)
def make_route_bot(self, iq, oq):
if self.joined:
q_o = oq
else:
q_o = queue.NullQueue()
self.start_q = []
self.output_q = oq
#parallel in sub network not in node
for func in self.args:
q_i = asyncio.Queue()
self.start_q.append(q_i)
BotFrame.make_bot(q_i, q_o, func)
def main():
words = ['贸易战', '世界杯'] * 50
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
p1 = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, collect),
Branch(get_all_page_url, HttpLoader(), get_all_items, collect),
)
Pipe(Timer(delay=delay, until=p1.finished), show_progress)
BotFrame.run()
def test_routetype_count(self):
b_counter = Counter()
a_counter = Counter()
Pipe(
Loop([A(), B(), A()]),
Branch(self.only_a,
a_counter,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b, b_counter)
BotFrame.run()
self.assertTrue(b_counter.count == 3)
self.assertTrue(a_counter.count == 2)
def main():
words = ['贸易战', '世界杯']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
outputfile = aiofile('ex_output/baidu.txt')
Pipe(
urls,
HttpLoader(),
Branch(get_all_items, outputfile),
Branch(get_all_page_url, HttpLoader(), get_all_items, outputfile),
)
#生成流程图
BotFrame.render('ex_output/baiduspider')
BotFrame.run()
def __init__(self, *args):
q_o = queue.GodQueue()
# get this pip own inside bot
self.start_index = len(BotFrame.bots)
self.q_start = q_o
self.joined = False
for func in args:
q_i = q_o
if func == args[-1]:
q_o = queue.NullQueue()
else:
if config.replay_mode:
q_o = queue.CachedQueue()
else:
q_o = queue.DataQueue()
bis = BotFrame.make_bot(q_i, q_o, func)
for b in bis:
b.flow = 'main'
if isinstance(func, Route):
if hasattr(func, 'joined') and func.joined:
self.joined = True
self.end_index = len(BotFrame.bots)
for i in range(self.start_index, self.end_index):
BotFrame.bots[i].pipeline = str(self)
self.q_end = q_o
if self.joined or config.joined_network:
self.check_joined_node()
def make_route_bot(self, iq, oq):
self.output_q = oq
q_o = queue.DataQueue()
self.start_q = [q_o]
for func in self.args:
q_i = q_o
if self.is_last_one(self.args, func):
if self.joined:
q_o = oq
else:
q_o = queue.NullQueue()
else:
q_o = queue.DataQueue()
BotFrame.make_bot(q_i, q_o, func)
def test_routetype_count2(self):
self.a_count = 0
self.b_count = 0
p = Pipe(
Loop([A(), B(), A()]), Branch(self.only_b, route_type=B,
join=True),
Branch(self.only_a,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b)
self.assertFalse(p.finished())
BotFrame.run()
self.assertEqual(self.b_count, 5)
self.assertEqual(self.a_count, 2)
self.assertTrue(p.finished())
def test_routetype_count3(self):
self.a_count = 0
self.b_count = 0
self.c_count = 0
p = Pipe(
[A(), B(), A(), C(), C()],
Branch(lambda i: isinstance(i, (A, C)),
self.assertTrue,
route_type=[A, C]),
Branch(Branch(self.only_c, route_type=C),
share=False,
route_type=[A, C]),
)
BotFrame.run()
self.assertEqual(self.c_count, 2)
def make_route_bot(self, oq):
q_o = queue.DataQueue()
self.start_q = [q_o]
self.output_q = oq
# if self.share:
# self.start_q.append(oq)
for idx, func in enumerate(self.args):
q_i = q_o
if idx == len(self.args) - 1:
if self.joined:
q_o = self.output_q
else:
q_o = queue.NullQueue()
else:
q_o = queue.DataQueue()
BotFrame.make_bot(q_i, q_o, func)
def main():
hget = HttpLoader(timeout=2)
Pipe(
flow.Timer(delay=3, max_time=5),
Join(
Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD", hget,
parse_kraken),
Return(
"https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC",
hget, parse_bittrex),
),
print,
)
BotFrame.render('ex_output/bitcoin_arbitrage')
BotFrame.run()
def test_blockedjoin_exception2(self):
from databot.flow import BlockedJoin
from databot.config import config
config.exception_policy = config.Exception_pipein
def raise_exception(a):
raise Exception()
def check(r):
self.assertEqual(len(r), 2)
self.assertTrue(isinstance(r[0], Exception))
self.assertTrue(isinstance(r[1], B))
p = Pipe(Loop([A()]), BlockedJoin(raise_exception, self.a_to_b), check)
BotFrame.run()
config.exception_policy = config.Exception_default
def main():
httpload=HttpLoader(timeout=2)
Pipe(
Timer(delay=10,max_time=5),
BlockedJoin(
Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD",httpload , parse_kraken),
Return("https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC", httpload, parse_bittrex),
Return("https://www.bitstamp.net/api/ticker/", httpload, parse_bitstamp),
Return("https://api.bitfinex.com/v1/ticker/btcusd", httpload, parse_bitfinex),
Return("https://bitpay.com/api/rates", httpload, parse_bitpay),
Return("http://api.coindesk.com/v1/bpi/currentprice.json", httpload, parse_coindesk),
),
print,
)
BotFrame.render('ex_output/bitcoin_arbitrage')
BotFrame.run()
def make_route_bot(self, iq, oq):
self.start_q = []
self.tmp_output_q = []
self.output_q = oq
self.share = False
self.joined = True
self.route_type = [object]
self.start_index = len(BotFrame.bots)
for func in self.args:
i_q = asyncio.Queue(maxsize=1)
o_q = queue.DataQueue()
self.start_q.append(i_q)
self.tmp_output_q.append(o_q)
BotFrame.make_bot(i_q, o_q, func)
self.end_index = len(BotFrame.bots)
def make_route_bot(self, oq):
self.start_q = []
self.output_q = oq
self.inner_output_q = queue.DataQueue()
self.share = False
self.joined = True
self.route_type = [object]
self.joined_result = {}
self.start_index = len(BotFrame.bots)
for func in self.args:
i_q = asyncio.Queue(maxsize=1)
self.start_q.append(i_q)
BotFrame.make_bot(i_q, self.inner_output_q, func)
BotFrame.make_bot(self.inner_output_q,
self.output_q,
self.join_merge,
raw_bdata=True)
self.end_index = len(BotFrame.bots)
def main():
print('ex1----')
Pipe(Loop(range(1)),
Query('select * from mysql.user limit 10 ', **dbconf, map_class=User),
print)
BotFrame.run()
print('ex2----param bind with inout')
Pipe(Loop(range(10)), Query('select %s ', **dbconf), print)
BotFrame.run()
print('ex3----param bind with dict')
u = User()
u.user = '******'
Pipe(
Loop([{
'user': '******'
}, u.__dict__]),
Query('select host,user from mysql.user where user="******" ',
**dbconf), print)
BotFrame.run()
def show_info(i):
BotFrame.debug()
def test_routetype_no_shared(self):
Pipe(Loop([A(), B(), A()]),
Branch(self.only_a, route_type=A, share=False), self.only_b)
BotFrame.run()
def test_routetype(self):
Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A))
BotFrame.run()
