def test_loop_double(self):
count = 0
def sum(x):
nonlocal count
count += x
p = Pipe(Loop(range(10)), Loop(range(10)), sum)
BotFrame.run()
self.assertEqual(count, 45 * 10)
def main():
Pipe(Loop([A(), B(), A(), A(), B()]),
Branch(process_A, share=False, route_type=A), process_B)
BotFrame.run()
print('----ex2')
Pipe(Loop([A(), B(), A(), A(), B()]),
Fork(process_A, route_type=A, share=False), process_B, print)
BotFrame.run()
def test_routetype_count2(self):
b_counter = Counter()
b1_counter = Counter()
counter = Counter('count2')
counter1 = Counter('count1')
p = Pipe(
Loop([A(), B(), A()]),
Branch(self.only_b, counter1, route_type=B, join=True, share=True),
counter,
Branch(self.only_a,
self.a_to_b,
self.only_b,
b1_counter,
route_type=A,
share=False,
join=True), self.only_b, b_counter)
self.assertFalse(p.finished())
BotFrame.run()
self.assertEqual(counter1.count, 1)
self.assertEqual(counter.count, 4)
self.assertEqual(b1_counter.count, 2)
self.assertEqual(b_counter.count, 4)
self.assertTrue(p.finished())
def main():
words = ['贸易战']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
insert = Insert(
"insert into test.baidu (id,name ,url,page_rank,page_no)values('{id}','{name}' ,'{url}',{page_rank},{page_no})",
**dbconf)
p = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, join=True),
Branch(get_all_page_url,
HttpLoader(),
get_all_items,
share=False,
join=True,
route_type=HttpResponse),
insert,
)
Pipe(Timer(delay=2, until=p.finished), show_info)
BotFrame.render('ex_output/baiduspider')
BotFrame.run()
def test_fork(self):
self.a_count = 0
self.b_count = 0
p = Pipe(Loop([A(), A()]),
Fork(self.a_to_b, self.a_to_b, share=False, join=True),
self.only_b)
BotFrame.run()
def main():
print('ex1----')
Pipe(
Loop(range(1)),
Query('select * from mysql.user limit 10 ', **dbconf, map_class=User),
Fork(print),
Insert('insert into jk.new_table(user,host)values ("{user}","{host}")',
**dbconf),
)
BotFrame.run()
def test_blockedjoin(self):
from databot.flow import BlockedJoin
def check(r):
self.assertEqual(len(r), 2)
self.assertTrue(isinstance(r[0], B))
self.assertTrue(isinstance(r[0], B))
p = Pipe(Loop([A()]), BlockedJoin(self.a_to_b, self.a_to_b), check)
BotFrame.run()
def test_routetype_count(self):
self.a_count = 0
self.b_count = 0
Pipe(
Loop([A(), B(), A()]),
Branch(self.only_a,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b)
BotFrame.run()
self.assertTrue(self.b_count == 3)
self.assertTrue(self.a_count == 2)
def main():
words = ['贸易战', '世界杯'] * 50
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
p1 = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, collect),
Branch(get_all_page_url, HttpLoader(), get_all_items, collect),
)
Pipe(Timer(delay=delay, until=p1.finished), show_progress)
BotFrame.run()
def main():
print('ex1----')
Pipe(Loop(range(1)),
Query('select * from mysql.user limit 10 ', **dbconf, map_class=User),
print)
BotFrame.run()
print('ex2----param bind with inout')
Pipe(Loop(range(10)), Query('select %s ', **dbconf), print)
BotFrame.run()
print('ex3----param bind with dict')
u = User()
u.user = '******'
Pipe(
Loop([{
'user': '******'
}, u.__dict__]),
Query('select host,user from mysql.user where user="******" ',
**dbconf), print)
BotFrame.run()
def test_routetype_count(self):
b_counter = Counter()
a_counter = Counter()
Pipe(
Loop([A(), B(), A()]),
Branch(self.only_a,
a_counter,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b, b_counter)
BotFrame.run()
self.assertTrue(b_counter.count == 3)
self.assertTrue(a_counter.count == 2)
def test_routetype_count2(self):
self.a_count = 0
self.b_count = 0
p = Pipe(
Loop([A(), B(), A()]), Branch(self.only_b, route_type=B,
join=True),
Branch(self.only_a,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b)
self.assertFalse(p.finished())
BotFrame.run()
self.assertEqual(self.b_count, 5)
self.assertEqual(self.a_count, 2)
self.assertTrue(p.finished())
def test_blockedjoin_exception2(self):
from databot.flow import BlockedJoin
from databot.config import config
config.exception_policy = config.Exception_pipein
def raise_exception(a):
raise Exception()
def check(r):
self.assertEqual(len(r), 2)
self.assertTrue(isinstance(r[0], Exception))
self.assertTrue(isinstance(r[1], B))
p = Pipe(Loop([A()]), BlockedJoin(raise_exception, self.a_to_b), check)
BotFrame.run()
config.exception_policy = config.Exception_default
def test_routetype_no_shared(self):
Pipe(Loop([A(), B(), A()]),
Branch(self.only_a, route_type=A, share=False), self.only_b)
BotFrame.run()
def test_routetype(self):
Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A))
BotFrame.run()
def main():
Pipe(Loop(range(10)), BlockedJoin(double, triple), print)
BotFrame.render('abc')
BotFrame.run()
def main():
Pipe(Loop(range(1000000)), Fork(op_sum), print)
BotFrame.run()
print(op_sum)
def main():
Pipe(Loop(range(10)), double, triple, print)
BotFrame.run()
def main():
Pipe(Loop(range(10)), Loop(range(10)), Branch(op_sum, print))
BotFrame.run()
print(op_sum)