def crawl_usa_stock_data():
# crawl the stock list
process_crawl(AmericaListSpider, {})
# crawl the kdata
for _, security_item in get_security_list(security_type='stock', exchanges=['nasdaq'],
codes=US_STOCK_CODES).iterrows():
process_crawl(AmericaStockKdataSpider, {"security_item": security_item})
def fetch_kdata(exchange_str='bitstamp'):
ccxt_exchange = eval("ccxt.{}()".format(exchange_str))
if ccxt_exchange.has['fetchOHLCV']:
for _, security_item in get_security_list(security_type='cryptocurrency', exchanges=[exchange_str]).iterrows():
try:
if security_item['name'] not in CRYPTOCURRENCY_PAIR:
continue
start_date, df = get_latest_download_trading_date(security_item)
# 日K线只抓到昨天
end_date = pd.Timestamp.today() - pd.DateOffset(1)
if start_date and (start_date > end_date):
logger.info("{} kdata is ok".format(security_item['code']))
continue
try:
kdatas = ccxt_exchange.fetch_ohlcv(security_item['name'], timeframe='1d')
# for rateLimit
time.sleep(5)
except Exception as e:
logger.exception("fetch_kdata for {} {} failed".format(exchange_str, security_item['name']), e)
continue
for kdata in kdatas:
timestamp = pd.Timestamp.fromtimestamp(int(kdata[0] / 1000))
if is_same_date(timestamp, pd.Timestamp.today()):
continue
kdata_json = {
'timestamp': to_time_str(timestamp),
'code': security_item['code'],
'name': security_item['name'],
'open': kdata[1],
'high': kdata[2],
'low': kdata[3],
'close': kdata[4],
'volume': kdata[5],
'securityId': security_item['id'],
'preClose': None,
'change': None,
'changePct': None
}
df = df.append(kdata_json, ignore_index=True)
if not df.empty:
df = df.loc[:, KDATA_COMMON_COL]
kdata_df_save(df, get_kdata_path(security_item), calculate_change=True)
logger.info(
"fetch_kdata for exchange:{} security:{} success".format(exchange_str, security_item['name']))
except Exception as e:
logger.info(
"fetch_kdata for exchange:{} security:{} failed".format(exchange_str, security_item['name'], e))
else:
logger.warning("exchange:{} not support fetchOHLCV".format(exchange_str))
def fetch_ticks(exchange_str, pairs=None):
if not pairs:
df = get_security_list(security_type=SECURITY_TYPE_CRYPTO,
exchanges=exchange_str)
pairs = set(df.loc[:, 'name'].tolist()) & set(CRYPTOCURRENCY_PAIR)
if not pairs:
logger.warning("{} not support pair:{}".format(
exchange_str, CRYPTOCURRENCY_PAIR))
return
else:
logger.info("{} get tick for paris:{}".format(exchange_str, pairs))
exchange = eval("ccxt.{}()".format(exchange_str))
if exchange.has['fetchTrades']:
# verify one trade at first
pairs = [pair for pair in pairs if _check_fetch_trades(exchange, pair)]
logger.info("after check {} get tick for paris:{}".format(
exchange_str, pairs))
while True:
for pair in pairs:
trades = exchange.fetch_trades(symbol=pair)
trade = trades[-1]
code = pair.replace('/', '-')
tick = {
'securityId':
"{}_{}_{}".format("cryptocurrency", exchange_str, code),
'code':
code,
'name':
pair,
'timestamp':
int(trade['timestamp'] / 1000),
'id':
trade['id'],
'price':
trade['price'],
'volume':
trade['amount']
}
yield tick
rate_limit = 5
time.sleep(rate_limit)
logger.info("fetch_tickers exchange:{} pairs:{} sleep:{}".format(
exchange_str, pairs, rate_limit))