def main():
words = ['贸易战']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
insert = Insert(
"insert into test.baidu (id,name ,url,page_rank,page_no)values('{id}','{name}' ,'{url}',{page_rank},{page_no})",
**dbconf)
p = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, join=True),
Branch(get_all_page_url,
HttpLoader(),
get_all_items,
share=False,
join=True,
route_type=HttpResponse),
insert,
)
Pipe(Timer(delay=2, until=p.finished), show_info)
BotFrame.render('ex_output/baiduspider')
BotFrame.run()
def main():
Pipe(Loop([A(), B(), A(), A(), B()]),
Branch(process_A, share=False, route_type=A), process_B)
BotFrame.run()
print('----ex2')
Pipe(Loop([A(), B(), A(), A(), B()]),
Fork(process_A, route_type=A, share=False), process_B, print)
BotFrame.run()
def main():
words = ['贸易战', '世界杯'] * 50
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
p1 = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, collect),
Branch(get_all_page_url, HttpLoader(), get_all_items, collect),
)
Pipe(Timer(delay=delay, until=p1.finished), show_progress)
BotFrame.run()
def test_routetype_count2(self):
b_counter = Counter()
b1_counter = Counter()
counter = Counter('count2')
counter1 = Counter('count1')
p = Pipe(
Loop([A(), B(), A()]),
Branch(self.only_b, counter1, route_type=B, join=True, share=True),
counter,
Branch(self.only_a,
self.a_to_b,
self.only_b,
b1_counter,
route_type=A,
share=False,
join=True), self.only_b, b_counter)
self.assertFalse(p.finished())
BotFrame.run()
self.assertEqual(counter1.count, 1)
self.assertEqual(counter.count, 4)
self.assertEqual(b1_counter.count, 2)
self.assertEqual(b_counter.count, 4)
self.assertTrue(p.finished())
def test_filter3(self):
Pipe([A(), B(), C()],
Filter(route_type=[A, B],
route_func=lambda r: isinstance(r, (A, C))), self.only_a)
BotFrame.run()
def test_fork(self):
self.a_count = 0
self.b_count = 0
p = Pipe(Loop([A(), A()]),
Fork(self.a_to_b, self.a_to_b, share=False, join=True),
self.only_b)
BotFrame.run()
def test_boost(self):
import time
def very_slow(a):
time.sleep(10)
Pipe(
Timer(delay=1),
Branch(very_slow),
print,
)
def main():
Pipe(
Timer(delay=2, max_time=5),
"http://api.coindesk.com/v1/bpi/currentprice.json",
HttpLoader(),
lambda r: r.json['bpi']['USD']['rate_float'],
print,
)
BotFrame.render('simple_bitcoin_price')
BotFrame.run()
def main():
print('ex1----')
Pipe(
Loop(range(1)),
Query('select * from mysql.user limit 10 ', **dbconf, map_class=User),
Fork(print),
Insert('insert into jk.new_table(user,host)values ("{user}","{host}")',
**dbconf),
)
BotFrame.run()
def test_loop_double(self):
count = 0
def sum(x):
nonlocal count
count += x
p = Pipe(Loop(range(10)), Loop(range(10)), sum)
BotFrame.run()
self.assertEqual(count, 45 * 10)
def main():
Pipe(
Timer(delay=2), # send timer data to pipe every 2 sen
"http://api.coindesk.com/v1/bpi/currentprice.json", # send url to pipe when timer trigger
HttpLoader(), # read url and load http response
lambda r: r.json['bpi']['USD'][
'rate_float'], # read http response and parese as json
print, # print out
)
BotFrame.render('simple_bitcoin_price')
BotFrame.run()
def test_blockedjoin(self):
from databot.flow import BlockedJoin
def check(r):
self.assertEqual(len(r), 2)
self.assertTrue(isinstance(r[0], B))
self.assertTrue(isinstance(r[0], B))
p = Pipe(Loop([A()]), BlockedJoin(self.a_to_b, self.a_to_b), check)
BotFrame.run()
def test_routetype_count(self):
self.a_count = 0
self.b_count = 0
Pipe(
Loop([A(), B(), A()]),
Branch(self.only_a,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b)
BotFrame.run()
self.assertTrue(self.b_count == 3)
self.assertTrue(self.a_count == 2)
def test_routetype_count(self):
b_counter = Counter()
a_counter = Counter()
Pipe(
Loop([A(), B(), A()]),
Branch(self.only_a,
a_counter,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b, b_counter)
BotFrame.run()
self.assertTrue(b_counter.count == 3)
self.assertTrue(a_counter.count == 2)
def main():
print('ex1----')
Pipe(Loop(range(1)),
Query('select * from mysql.user limit 10 ', **dbconf, map_class=User),
print)
BotFrame.run()
print('ex2----param bind with inout')
Pipe(Loop(range(10)), Query('select %s ', **dbconf), print)
BotFrame.run()
print('ex3----param bind with dict')
u = User()
u.user = '******'
Pipe(
Loop([{
'user': '******'
}, u.__dict__]),
Query('select host,user from mysql.user where user="******" ',
**dbconf), print)
BotFrame.run()
def main():
words = ['贸易战', '世界杯']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
outputfile = aiofile('ex_output/baidu.txt')
Pipe(
urls,
HttpLoader(),
Branch(get_all_items, outputfile),
Branch(get_all_page_url, HttpLoader(), get_all_items, outputfile),
)
#生成流程图
BotFrame.render('ex_output/baiduspider')
BotFrame.run()
def test_routetype_count2(self):
self.a_count = 0
self.b_count = 0
p = Pipe(
Loop([A(), B(), A()]), Branch(self.only_b, route_type=B,
join=True),
Branch(self.only_a,
self.a_to_b,
route_type=A,
share=False,
join=True), self.only_b)
self.assertFalse(p.finished())
BotFrame.run()
self.assertEqual(self.b_count, 5)
self.assertEqual(self.a_count, 2)
self.assertTrue(p.finished())
def test_routetype_count3(self):
self.a_count = 0
self.b_count = 0
self.c_count = 0
p = Pipe(
[A(), B(), A(), C(), C()],
Branch(lambda i: isinstance(i, (A, C)),
self.assertTrue,
route_type=[A, C]),
Branch(Branch(self.only_c, route_type=C),
share=False,
route_type=[A, C]),
)
BotFrame.run()
self.assertEqual(self.c_count, 2)
def main():
hget = HttpLoader(timeout=2)
Pipe(
flow.Timer(delay=3, max_time=5),
Join(
Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD", hget,
parse_kraken),
Return(
"https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC",
hget, parse_bittrex),
),
print,
)
BotFrame.render('ex_output/bitcoin_arbitrage')
BotFrame.run()
def test_blockedjoin_exception2(self):
from databot.flow import BlockedJoin
from databot.config import config
config.exception_policy = config.Exception_pipein
def raise_exception(a):
raise Exception()
def check(r):
self.assertEqual(len(r), 2)
self.assertTrue(isinstance(r[0], Exception))
self.assertTrue(isinstance(r[1], B))
p = Pipe(Loop([A()]), BlockedJoin(raise_exception, self.a_to_b), check)
BotFrame.run()
config.exception_policy = config.Exception_default
def main():
httpload=HttpLoader(timeout=2)
Pipe(
Timer(delay=10,max_time=5),
BlockedJoin(
Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD",httpload , parse_kraken),
Return("https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC", httpload, parse_bittrex),
Return("https://www.bitstamp.net/api/ticker/", httpload, parse_bitstamp),
Return("https://api.bitfinex.com/v1/ticker/btcusd", httpload, parse_bitfinex),
Return("https://bitpay.com/api/rates", httpload, parse_bitpay),
Return("http://api.coindesk.com/v1/bpi/currentprice.json", httpload, parse_coindesk),
),
print,
)
BotFrame.render('ex_output/bitcoin_arbitrage')
BotFrame.run()
def main():
Pipe(Timer(delay=0.5, max_time=10), FileSaver('a.txt'))
BotFrame.run()
def main():
Pipe(Loop(range(10)), Loop(range(10)), Branch(op_sum, print))
BotFrame.run()
print(op_sum)
def test_routetype_no_shared(self):
Pipe(Loop([A(), B(), A()]),
Branch(self.only_a, route_type=A, share=False), self.only_b)
BotFrame.run()
def test_routetype(self):
Pipe(Loop([A(), B(), A()]), Branch(self.only_a, route_type=A))
BotFrame.run()
def test_filter2(self):
Pipe([A(), B(), C()], Filter(route_func=lambda r: isinstance(r, A)),
self.only_a)
def test_filter(self):
Pipe([A(), B(), C()], Filter(route_type=A), self.only_a)
def main():
Pipe(range(10), BlockedJoin(double, triple), print)
BotFrame.render('ex_output/blockedjoin')
BotFrame.run()
def main():
Pipe(Loop(range(10)), double, triple, print)
BotFrame.run()
def main():
Pipe(Loop(range(1000000)), Fork(op_sum), print)
BotFrame.run()
print(op_sum)
