def main():
words = ['贸易战']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
insert = Insert(
"insert into test.baidu (id,name ,url,page_rank,page_no)values('{id}','{name}' ,'{url}',{page_rank},{page_no})",
**dbconf)
p = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, join=True),
Branch(get_all_page_url,
HttpLoader(),
get_all_items,
share=False,
join=True,
route_type=HttpResponse),
insert,
)
Pipe(Timer(delay=2, until=p.finished), show_info)
BotFrame.render('ex_output/baiduspider')
BotFrame.run()
def main():
words = ['贸易战', '世界杯'] * 50
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
# make data flow net
p1 = Pipe(
Loop(urls),
HttpLoader(),
Branch(get_all_items, collect),
Branch(get_all_page_url, HttpLoader(), get_all_items, collect),
)
Pipe(Timer(delay=delay, until=p1.finished), show_progress)
BotFrame.run()
def main():
words = ['贸易战', '世界杯']
baidu_url = 'https://www.baidu.com/s?wd=%s'
urls = [baidu_url % (word) for word in words]
outputfile = aiofile('ex_output/baidu.txt')
Pipe(
urls,
HttpLoader(),
Branch(get_all_items, outputfile),
Branch(get_all_page_url, HttpLoader(), get_all_items, outputfile),
)
#生成流程图
BotFrame.render('ex_output/baiduspider')
BotFrame.run()
def main():
Pipe(
Timer(delay=2, max_time=5),
"http://api.coindesk.com/v1/bpi/currentprice.json",
HttpLoader(),
lambda r: r.json['bpi']['USD']['rate_float'],
print,
)
BotFrame.render('simple_bitcoin_price')
BotFrame.run()
def main():
Pipe(
Timer(delay=2), # send timer data to pipe every 2 sen
"http://api.coindesk.com/v1/bpi/currentprice.json", # send url to pipe when timer trigger
HttpLoader(), # read url and load http response
lambda r: r.json['bpi']['USD'][
'rate_float'], # read http response and parese as json
print, # print out
)
BotFrame.render('simple_bitcoin_price')
BotFrame.run()
def main():
hget = HttpLoader(timeout=2)
Pipe(
flow.Timer(delay=3, max_time=5),
Join(
Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD", hget,
parse_kraken),
Return(
"https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC",
hget, parse_bittrex),
),
print,
)
BotFrame.render('ex_output/bitcoin_arbitrage')
BotFrame.run()
def main():
httpload=HttpLoader(timeout=2)
Pipe(
Timer(delay=10,max_time=5),
BlockedJoin(
Return("https://api.kraken.com/0/public/Ticker?pair=XBTUSD",httpload , parse_kraken),
Return("https://bittrex.com/api/v1.1/public/getticker?market=USD-BTC", httpload, parse_bittrex),
Return("https://www.bitstamp.net/api/ticker/", httpload, parse_bitstamp),
Return("https://api.bitfinex.com/v1/ticker/btcusd", httpload, parse_bitfinex),
Return("https://bitpay.com/api/rates", httpload, parse_bitpay),
Return("http://api.coindesk.com/v1/bpi/currentprice.json", httpload, parse_coindesk),
),
print,
)
BotFrame.render('ex_output/bitcoin_arbitrage')
BotFrame.run()