def __addCompanyList(list, dataset):
df = dataset
for i in range(0, len(df)):
item = stock.Stock_company()
item.ts_code = df.iat[i, 0]
item.exchange = df.iat[i, 1]
item.chairman = df.iat[i, 2]
item.manager = df.iat[i, 3]
item.secretary = df.iat[i, 4]
item.re_capital = util.toFloat(df.iat[i, 5])
item.setup_date = util.date_cn2us(df.iat[i, 6])
item.province = df.iat[i, 7]
item.city = df.iat[i, 8]
item.introduction = df.iat[i, 9]
item.website = df.iat[i, 10]
item.email = df.iat[i, 11]
item.office = df.iat[i, 12]
#it is an problem of tushare https://github.com/waditu/tushare/issues/1111
item.employees = util.toInt(df.iat[i, 14])
#item.main_business = df.iat[i,13]
item.main_business = "-"
#end of the problem.
item.business_scope = df.iat[i, 15]
list.append(item)
return list
def insert_cnstock_fina():
daystr = util.get_today_datestr()
cndaystr = util.date_us2cn(daystr)
stkdao = cdao.cnstock_item_dao()
stocklist = stkdao.get_all_tscode()
list = ctu.read_cnstock_fina(stocklist, cndaystr)
fina_dao = cdao.dao_cnstock_fina()
fina_dao.add_oneItemEachTime(list)
def insert_cnstock_fina2():
daystr = util.get_today_datestr()
cndaystr = util.date_us2cn(daystr)
stkdao = cdao.cnstock_item_dao()
sfadao = cdao.dao_cnstock_fina()
stocklist = stkdao.get_all_tscode()
existinglist = sfadao.get_existing_tscodelist()
print('start fina2 insert...')
for ts_code in stocklist:
ts_code = ts_code.strip()
if ts_code not in existinglist:
list = ctu.read_cnstock_finaone(ts_code, cndaystr)
sfadao.add_cnstock_fina(list)
def update_cnstock_basic2(list, trade_date):
pro = get_tushare_api()
dt_basic = pro.daily_basic(
ts_code='',
trade_date=trade_date,
fields='ts_code, trade_date, turnover_rate, volume_ratio, pe')
for i in range(0, len(dt_basic)):
for item in list:
if item.ts_code == dt_basic.iat[
i, 0] and item.trade_date == util.date_cn2us(
dt_basic.iat[i, 1]):
item.turnover_rate = util.toFloat(dt_basic.iat[i, 2])
item.volume_ratio = util.toFloat(dt_basic.iat[i, 3])
item.pe = util.toFloat(dt_basic.iat[i, 4])
return list
def read_cnstocklist():
pro = get_tushare_api()
data = pro.stock_basic(
exchange='',
list_status='L',
fields=
'ts_code, name, area, industry, enname, market, exchange, list_status, list_date, is_hs'
)
if data.empty:
print("stock basic list is empty. tushare.")
list = []
for i in range(0, len(data)):
item = stock.Stock_item()
item.ts_code = data.iat[i, 0]
item.name = data.iat[i, 1]
item.area = data.iat[i, 2]
item.industry = data.iat[i, 3]
item.enname = data.iat[i, 4]
item.market = data.iat[i, 5]
item.exchange = data.iat[i, 6]
item.list_status = data.iat[i, 7]
item.list_date = util.date_cn2us(data.iat[i, 8])
item.is_hs = data.iat[i, 9]
list.append(item)
return list
def ann_predict_cn(filepath):
#model = ann.create_gru_model()
#model.load_weights(filepath)
model = ann.load_mymodel(filepath)
rs = ann.predict_model_cn(model)
pred = rs[0]
sym = rs[1]
list = []
for i in range(0, len(pred)):
item = cstk.Stock_predict()
item.symbol = str(sym[i])
item.trade_date = get_latestdatefromdaily_cn(item.symbol)
item.cal_date = util.get_today_datestr()
item.result = util.toFloat(pred[i])
item.comment = 'GRU drop 0.4'
list.append(item)
insert_predictresult_cn(list)
def set_dailydata(self, inputdata, symbol):
self.openlist = np.array(inputdata['open'], dtype=float)
self.closelist = np.array(inputdata['close'], dtype=float)
self.highlist = np.array(inputdata['high'], dtype=float)
self.lowlist = np.array(inputdata['low'], dtype=float)
self.vollist = np.array(inputdata['volume'], dtype=float)
self.length = util.toFloat(len(inputdata['close']))
self.symbol = symbol
def data_generator(min_index,
max_index,
shuffle=False,
batch_size=32,
country='us'):
#global lock
if country == 'us':
symbollist = get_symbolists_us()
else:
symbollist = get_symbolists_cn()
rowcnt = len(symbollist)
if max_index is None:
max_index = rowcnt - 1
if max_index > rowcnt:
max_index = rowcnt
if min_index < 0:
min_index = 0
if min_index >= max_index:
min_index = 0
i = min_index
while True:
if shuffle:
rows = np.random.randint(min_index, max_index, size=batch_size)
else:
if i + batch_size >= max_index:
i = min_index
rows = np.arange(i, min(i + batch_size, max_index))
i += len(rows)
if lock.acquire(1):
if country == 'us':
list = get_samplelists_us(symbollist, rows)
else:
list = get_samplelists_cn(symbollist, rows)
samples = np.zeros(
(len(list), scal.SAMPLE_DATASIZE, scal.FEATURE_NUM))
targets = np.zeros((len(list), ))
for j in range(0, len(list)):
try:
samples[j] = np.array(list[j][0])
targets[j] = util.toInt(list[j][1])
except ValueError as err:
print(
"Cannot broadcast input array into shape(60,26): %s" %
str(err))
samples = toStandard(samples)
lock.release()
yield samples, targets
def create_item_fromdataset(dt_data):
dt_daily = dt_data
list = []
for i in range(0, len(dt_daily)):
item = stock.Stock_daily()
item.ts_code = dt_daily.iat[i, 0]
item.trade_date = util.date_cn2us(dt_daily.iat[i, 1])
item.open = util.toFloat(dt_daily.iat[i, 2])
item.high = util.toFloat(dt_daily.iat[i, 3])
item.low = util.toFloat(dt_daily.iat[i, 4])
item.close = util.toFloat(dt_daily.iat[i, 5])
item.pct_chg = util.toFloat(dt_daily.iat[i, 6])
item.vol = util.toFloat(dt_daily.iat[i, 7])
item.volume_ratio = 0
item.turnover_rate = 0
item.pe = 0
list.append(item)
return list
def insert_alldaily_oneday_cn(daystr):
st_dao = cdao.cnstock_item_dao()
sd_dao = cdao.cnstock_daily_dao()
done_list = sd_dao.get_existing_symbollist2(daystr)
print("the length of donelist: %d" %len(done_list))
daystr = util.date_us2cn(daystr)
print("reading %s data from tushare..."%daystr)
dl = ctu.read_cnstock_daily2(daystr)
print("the length of read data: %d" %len(dl))
i = 0
to_add = []
for item in dl:
if item.ts_code not in done_list:
to_add.append(item)
else:
print("%s already exists in database...skipped." %item.ts_code)
i+=1
print("totally %d skipped." %i)
print("totally to be add: %d"%len(to_add))
print("insert into data base...")
sd_dao.insert_newlist(to_add)
def get_turnover(self, country):
if country == 'us':
stdb = udbo.dao_ussstock_item()
rs = stdb.get_marketcap(self.symbol)
marketcap = util.cmplx_toFloat(rs)
size = len(self.closelist)
tr = np.zeros(size, dtype=float)
for i in range(0, size):
if marketcap != 0:
tr[i] = (self.closelist[i] * self.vollist[i]) / marketcap
else:
tr[i] = 0
return tr
else:
stdb = cdbo.cnstock_daily_dao()
list = stdb.get_turnoverratio_list(self.symbol)
if len(list) == len(self.closelist):
return list
else:
print("turnover rate leng != datasize")
tr = np.zeros(len(self.closelist), dtype=float)
return tr
def get_onedaily(symbol, trade_date):
client = get_TiingoClient()
pr = client.get_ticker_price(symbol, startDate=trade_date, endDate=trade_date,frequency='daily')
list = []
for row in pr:
item = stock.Usstock_daily()
item.symbol = symbol
item.trade_date = trade_date
item.close = util.toFloat(row['close'])
item.open = util.toFloat(row['open'])
item.high = util.toFloat(row['high'])
item.low = util.toFloat(row['low'])
item.adj_close = util.toFloat(row['adjClose'])
item.volume = util.toFloat(row['volume'])
list.append(item)
print("read data...%s"%(symbol+str(trade_date)))
return list
def get_daily_bysymbol(smybol):
df, msg = usstock.get_daily(smybol)
if df is None or df.empty:
print("%s Daily data empty." %smybol)
return None
df = df.tail(cfg.CONSTANT.DATASIZE)
data_list = []
print("reading... %s %s" %(smybol,str(len(df))))
for i in range(0, len(df)):
item = stock.Usstock_daily()
item.symbol = smybol
item.trade_date = datetime.strptime(df.iat[i,0], '%Y-%m-%d')
item.open = util.toFloat(df.iat[i,1])
item.high = util.toFloat(df.iat[i,2])
item.low = util.toFloat(df.iat[i,3])
item.close = util.toFloat(df.iat[i,4])
item.adj_close = util.toFloat(df.iat[i,5])
item.volume = util.toFloat(df.iat[i,6])
data_list.append(item)
return data_list
def us_add_today():
todaystr = util.get_today_datestr()
serv.insert_alldaily_oneday_us(todaystr)
def save_model2(model, filename):
mdf = filename+"_"+util.get_today_datestr()+"_model.h5"
wtf = filename+"_"+util.get_today_datestr()+"_weight.h5"
model.save_weights(wtf, overwrite=True)
model.save(mdf)
def insert_alldaily_today():
todaystr = util.get_today_datestr()
insert_alldaily_oneday_us(todaystr)
insert_alldaily_oneday_cn(todaystr)
def read_cnstock_finaone(ts_code, endday_cnstr):
pro = get_tushare_api()
print("reading %s financial indicators from Tushare. " % ts_code)
df = pro.query('fina_indicator',
ts_code=ts_code,
start_date='20170101',
end_date=endday_cnstr,
fields=TUSHARE_FINAFIELDS)
list = []
for i in range(0, len(df)):
item = stock.Stock_fina()
item.ts_code = df.iat[i, 0]
item.ann_date = util.date_cn2us(df.iat[i, 1])
item.end_date = util.date_cn2us(df.iat[i, 2])
item.eps = util.toFloat(df.iat[i, 3])
item.dt_eps = util.toFloat(df.iat[i, 4])
item.total_revenue_ps = util.toFloat(df.iat[i, 5])
item.revenue_ps = util.toFloat(df.iat[i, 6])
item.extra_item = util.toFloat(df.iat[i, 7])
item.profit_dedt = util.toFloat(df.iat[i, 8])
item.gross_margin = util.toFloat(df.iat[i, 9])
item.current_ratio = util.toFloat(df.iat[i, 10])
item.quick_ratio = util.toFloat(df.iat[i, 11])
item.cash_ratio = util.toFloat(df.iat[i, 12])
item.assets_turn = util.toFloat(df.iat[i, 13])
item.interst_income = util.toFloat(df.iat[i, 14])
item.daa = util.toFloat(df.iat[i, 15])
item.ebit = util.toFloat(df.iat[i, 16])
item.ebitda = util.toFloat(df.iat[i, 17])
item.netdebt = util.toFloat(df.iat[i, 18])
item.bps = util.toFloat(df.iat[i, 19])
item.roe = util.toFloat(df.iat[i, 20])
item.roa = util.toFloat(df.iat[i, 21])
item.npta = util.toFloat(df.iat[i, 22])
item.debt_to_assets = util.toFloat(df.iat[i, 23])
list.append(item)
return list
def get_oneFina(symbol):
client = get_TiingoClient()
tk = yf.Ticker(symbol)
print(symbol)
df = tk.financials
nc = len(df.columns)
rows, cols = df.shape
list = []
for i in range(0, nc):
item = stock.Usstock_fina()
item.end_date = util.date_yf2us(df.columns.values[i])
item.symbol = symbol
item.total_rev = util.toFloat(df.iat[0, i])
item.cost_of_rev = util.toFloat(df.iat[1,i])
item.gross_profit = util.toFloat(df.iat[2,i])
item.ops_exp = util.toFloat(df.iat[3,i])
item.r_n_d = util.toFloat(df.iat[4,i])
item.selling_ga = util.toFloat(df.iat[5,i])
item.non_rec = util.toFloat(df.iat[6,i])
item.others = util.toFloat(df.iat[7,i])
item.total_ops = util.toFloat(df.iat[8,i])
item.ops_il = util.toFloat(df.iat[9,i])
item.income_cops = util.toFloat(df.iat[10,i])
item.total_other = util.toFloat(df.iat[11,i])
item.ebit = util.toFloat(df.iat[12,i])
item.interest_exp = util.toFloat(df.iat[13,i])
item.income_before_tax = util.toFloat(df.iat[14,i])
item.income_tax_expense = util.toFloat(df.iat[15,i])
item.min_int = util.toFloat(df.iat[16,i])
item.net_income_cs = util.toFloat(df.iat[17,i])
item.disc_ops = util.toFloat(df.iat[19,i])
item.extra_items = util.toFloat(df.iat[20,i])
item.effect_ofac = util.toFloat(df.iat[21,i])
item.other_items = util.toFloat(df.iat[22,i])
item.net_income = util.toFloat(df.iat[23,i])
item.pre_stk = util.toFloat(df.iat[25,i])
item.net_income_to_cs = util.toFloat(df.iat[26, i])
list.append(item)
return list