def trade_cal(force=False):
'''
http://218.244.146.57/static/calAll.csv
交易日历
isOpen=1是交易日，isOpen=0为休市
'''
dst = os.path.join(CURDIR, 'trade.cal.csv')
if os.path.exists(dst) and not force:
df = pd.read_csv(dst, index_col=0)
df = df.set_index('calendarDate')
return df
def date2int(date):
y, m, d = date.split("/")
return int(y) * 10000 + int(m) * 100 + int(d)
for _ in range(5):
try:
df = pd.read_csv('http://218.244.146.57/static/calAll.csv')
df['calendarDate'] = df['calendarDate'].map(date2int)
df.to_csv(dst)
df = df.set_index('calendarDate')
ytrack.info('update trade_cal from network..')
return df
except Exception as e:
ytrack.error(traceback.format_exc())
def get_sina_goods():
from codes import add_params, GOODS_URL, goods, get_page, parse_df
for good in goods:
dst = "%s/sina_goods/%s.csv" % (CURDIR, good['breed'])
if os.path.exists(dst):
ytrack.error("%s exists.." % dst)
continue
url = add_params(GOODS_URL, good)
page = get_page(url)
if not page:
ytrack.fail('%s page is None' % good['name'])
continue
data = pd.DataFrame()
for i in range(1, page + 1):
good.update({"page": i})
url = add_params(GOODS_URL, good)
df = parse_df(url)
if df is None or len(df) == 0:
continue
else:
data = data.append(df, ignore_index=True)
if len(data) > 0:
data.columns = ['date', 'close', 'open', 'high', 'low', 'volume']
data.to_csv(dst)
ytrack.info("%s finished.." % good['name'])
else:
ytrack.error("%s len is 0")
ytrack.show()
def _get_basics(force=False):
fle = os.path.join(CURDIR, 'stock.basics.csv')
if os.path.exists(fle) and not force:
df = pd.read_csv(fle, dtype=str)
df = df.set_index('code')
return df
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_stock_basics()
df.to_csv(fle, encoding="utf-8")
ytrack.info('update 股票列表 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
ytrack.info('成功 股票基本信息 from network..')
return df
def get_all_ohlcs():
ytrack.info("Start get_all_ohlcs ..")
for code in basics_df.index.values[:3]:
dst = os.path.join(CURDIR, 'ohlc_daily/%s.txt' % code)
if os.path.exists(dst):
ytrack.info("%s exists.." % dst)
continue
df = get_hfq_data(code)
if df is None:
continue
if 'code' not in df.columns:
df.insert(0, 'code', code)
df.to_csv(dst, date_format="%Y%m%d")
ytrack.info("%s finished.." % dst)
ytrack.info("End get_all_ohlcs ..")
def get_hs_indexs():
import codes
ytrack.info("Start get hs indexs ..")
for code in codes.hsindexs:
dst = os.path.join(CURDIR, "hs_ohlc_daily/%s.txt" % code)
if os.path.exists(dst):
ytrack.info("%s exists.." % dst)
continue
df = get_hfq_data(code,
index=True,
start='1989-01-01',
end='2016-12-04')
if df is None:
continue
if 'code' not in df.columns:
df.insert(0, 'code', code)
df.to_csv(dst, date_format="%Y%m%d")
ytrack.info("%s finished.." % dst)
ytrack.info("End get_hs_indexs ..")
ytrack.show()
def update_stock_info():
trade_cal(True)
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_stock_basics()
fle = os.path.join(CURDIR, 'stock.basics.csv')
df.to_csv(fle, encoding="utf-8")
df = df.set_index(df.index.astype(int))
df.to_sql('stock_list',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update 股票列表 save to sql')
time.sleep(0.1)
df = ts.get_today_all()
fle = os.path.join(CURDIR, 'stock.today.csv')
df.to_csv(fle, encoding="utf-8")
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_today',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update stock_today')
break
except Exception as e:
ytrack.error(traceback.format_exc())
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_industry_classified()
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_industry',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update 股票行业 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
# ytrack.info('成功 股票基本信息 from network..')
# ynotice.send(ytrack.get_logs(), style='stock', title='更新股票基础数据和交易日期')
# return
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_concept_classified()
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_concept',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update 股票概念 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_area_classified()
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_area',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update 股票地域 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_sme_classified()
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_sme',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update 中小板股票 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_gem_classified()
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_gem',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update 创业板股票 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_st_classified()
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_st',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update st股票 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_hs300s()
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_hs300',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update hs300股票 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_sz50s()
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_sz50s',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update sz50s股票 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
for _ in range(5):
try:
time.sleep(0.1)
df = ts.get_zz500s()
df = df.set_index('code')
df = df.set_index(df.index.astype(int))
df.to_sql('stock_zz500s',
engine,
if_exists='replace',
index=True,
index_label='code')
ytrack.info('update zz500s股票 save to sql')
break
except Exception as e:
ytrack.error(traceback.format_exc())
ytrack.info('成功 股票基本信息 from network..')
ynotice.send(ytrack.get_logs(), style='stock', title='更新股票基础数据和交易日期')