def read_data(self, needStore = False):
if os.path.exists('i_data.json'):
with open('k_data.json', 'r') as f:
k_data = pd.read_json(f.read(), orient = 'records', lines = True)
k_data.date = k_data.date.dt.strftime('%Y-%m-%d')
with open('d_data.json', 'r') as f:
d_data = pd.read_json(f.read(), orient = 'records', lines = True)
with open('i_data.json', 'r') as f:
i_data = pd.read_json(f.read(), orient = 'records', lines = True)
i_data.date = i_data.date.dt.strftime('%Y-%m-%d')
else:
obj = CStock(self.code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1')
k_data = obj.get_k_data()
k_data.date = pd.to_datetime(k_data.date).dt.strftime('%Y-%m-%d')
d_data = obj.get_chip_distribution()
iobj = CIndex(self.index_code, dbinfo = ct.OUT_DB_INFO, redis_host = '127.0.0.1')
i_data = iobj.get_k_data()
cdates = k_data.date.tolist()
i_data = i_data.loc[i_data.date.isin(cdates)]
i_data = i_data.reset_index(drop = True)
k_data = k_data[['date', 'open', 'high', 'low', 'close', 'volume', 'amount', 'outstanding', 'totals', 'adj', 'aprice', 'uprice', 'sprice', 'mprice', 'lprice', 'profit']]
k_data = k_data.rename(columns = {"date": "time"})
i_data = i_data[['date', 'open', 'high', 'low', 'close', 'volume', 'amount']]
i_data = i_data.rename(columns = {"date": "time"})
if needStore:
with open('k_data.json', 'w') as f:
f.write(k_data.to_json(orient='records', lines=True))
with open('d_data.json', 'w') as f:
f.write(d_data.to_json(orient='records', lines=True))
with open('i_data.json', 'w') as f:
f.write(i_data.to_json(orient='records', lines=True))
return k_data, d_data, i_data
def create_stock_figure_column(code, start_date, end_date):
obj = CStock(code)
delta_days = (end_date - start_date).days
if delta_days <= 0: return None
start_date = start_date.strftime('%Y-%m-%d')
end_date = end_date.strftime('%Y-%m-%d')
df = obj.get_k_data_in_range(start_date, end_date)
if df is None: return None
df['date'] = df['date'].apply(
lambda x: str_to_datetime(x, dformat="%Y-%m-%d"))
source = ColumnDataSource(df)
mapper = linear_cmap(field_name='pchange',
palette=['red', 'green'],
low=0,
high=0,
low_color='green',
high_color='red')
p = figure(plot_height=500,
plot_width=1300,
tools="",
toolbar_location=None,
sizing_mode="scale_both",
x_range=(0, len(df)))
p.xaxis.axis_label = "时间"
p.yaxis.axis_label = "点数"
p.segment(x0='index',
y0='low',
x1='index',
y1='high',
line_width=2,
color='black',
source=source)
p.vbar(x='index',
bottom='open',
top='close',
width=50 / delta_days,
color=mapper,
source=source)
p.xaxis.major_label_overrides = {
i: mdate.strftime('%Y-%m-%d')
for i, mdate in enumerate(df["date"])
}
volume_p = figure(plot_height=150,
plot_width=1300,
tools="",
toolbar_location=None,
sizing_mode="scale_both")
volume_p.x_range = p.x_range
volume_p.vbar(x='index',
top='volume',
width=50 / delta_days,
color=mapper,
source=source)
volume_p.xaxis.major_label_overrides = {
i: mdate.strftime('%Y-%m-%d')
for i, mdate in enumerate(df["date"])
}
return column(p, volume_p)
def test_get(self, mock_create, mock_create_info, mock_mysql_get):
mock_create.return_value = True
mock_create_info.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'testa')
mock_mysql_get.return_value = pd.DataFrame(
{'timeToMarket': Series(['20170923'])})
self.assertEqual(cs.get('timeToMarket'), '20170923')
def choose_stock(code_list, start_date, end_date):
state_dict = dict()
state_dict[ct.FL] = list()
state_dict[ct.JL] = list()
state_dict[ct.QL] = list()
state_dict[ct.KL] = list()
good_code_list = list()
#stock_info = DataFrame(columns={'code', 'ppercent', 'npercent', 'sai', 'sri', 'pchange'})
stock_info = DataFrame()
for code in code_list:
cstock_obj = CStock(code, redis_host = '127.0.0.1')
df_profit = cstock_obj.get_base_floating_profit_in_range(start_date, end_date)
if df_profit.profit.mean() > 2:
state_dict[ct.FL].append(code)
elif df_profit.profit.mean() < -2:
state_dict[ct.KL].append(code)
elif df_profit.profit.mean() >= -2 and df_profit.profit.mean() <= 0:
state_dict[ct.QL].append(code)
else:
state_dict[ct.JL].append(code)
df = cstock_obj.get_k_data_in_range(start_date, end_date)
if df is None or df.empty or len(df) < 55: continue
if average_amount_volume(df) and large_down_time(df) and max_turnover(df):
good_code_list.append(code)
return state_dict, good_code_list
def scallback(event):
global dist_source, dist_fig, roe_fig, profit_fig
code = code_text.value
sobj = CStock(code)
mdate = stock_source.data['date'][int(event.x)]
print(code, mdate)
ddf = sobj.get_chip_distribution(mdate)
dist_source = ColumnDataSource(ddf)
dist_fig = create_dist_figure(dist_source)
layout.children[4] = gridplot([[stock_fig, dist_fig],
[profit_fig, roe_fig]])
def test_init(self, mock_create, mock_create_info, mock_get_k_data,
mock_ts_get_data, mock_mysql_set):
mock_create.return_value = True
mock_create_info.return_value = True
mock_get_k_data.return_value = pd.DataFrame(
{'date': Series(['20170922'])})
mock_ts_get_data.return_value = pd.DataFrame(
{'date': Series(['20170923'])})
_data = pd.DataFrame({
'date': Series(['20170922', '20170923'])
}).reset_index(drop=True)
cs = CStock(ct.DB_INFO, '111111', 'testb')
cs.init()
mock_mysql_set.assert_called_once()
def create_stock_obj(self, code):
try:
CStock(code, should_create_influxdb = True, should_create_mysqldb = True)
return (code, True)
except Exception as e:
logger.info(e)
return (code, False)
def init_real_stock_info(self):
concerned_list = self.get_concerned_list()
for code_id in concerned_list:
ret = self.subscriber.subscribe_tick(add_prifix(code_id), CStock)
if 0 == ret:
if code_id not in self.stock_objs:
self.stock_objs[code_id] = CStock(self.dbinfo, code_id)
def _set_base_float_profit(code_id):
if CStock(code_id).set_base_floating_profit():
self.logger.info("%s set base float profit success" % code_id)
return (code_id, True)
else:
self.logger.error("%s set base float profit failed" % code_id)
return (code_id, False)
def init_all_stock_tick(self):
start_date = '2015-01-01'
_today = datetime.now().strftime('%Y-%m-%d')
num_days = delta_days(start_date, _today)
start_date_dmy_format = time.strftime(
"%m/%d/%Y", time.strptime(start_date, "%Y-%m-%d"))
data_times = pd.date_range(start_date_dmy_format,
periods=num_days,
freq='D')
date_only_array = np.vectorize(lambda s: s.strftime('%Y-%m-%d'))(
data_times.to_pydatetime())
date_only_array = date_only_array[::-1]
obj_pool = Pool(4)
df = self.stock_info_client.get()
for _, code_id in df.code.iteritems():
_obj = self.stock_objs[
code_id] if code_id in self.stock_objs else CStock(
self.dbinfo, code_id)
for _date in date_only_array:
if self.cal_client.is_trading_day(_date):
try:
if obj_pool.full(): obj_pool.join()
obj_pool.spawn(_obj.set_ticket, _date)
except Exception as e:
logger.info(e)
obj_pool.join()
obj_pool.kill()
def compute(self):
code_list = self.get_code_list()
df = pd.DataFrame()
for code in code_list:
df_byte = self.redis.get(CStock.get_redis_name(code))
if df_byte is None: continue
df = df.append(_pickle.loads(df_byte))
num = len(df)
if 0 == num: return pd.DataFrame()
_price = df.price.astype(float).sum() / num
_volume = df.volume.astype(float).sum() / num
_amount = df.turnover.astype(float).sum() / num
ctime = datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
data = {
'code': [self.code],
'name': [self.get('name')],
'time': [ctime],
'price': [_price],
'amount': [_amount],
'volume': [_volume]
}
df = pd.DataFrame(data)
df.time = pd.to_datetime(df.time)
df = df.set_index('time')
return df
def get_time_to_market(self, code):
"""获取沪深股股票上市时间"""
file_name = "%s%s.csv" % (CStock.get_pre_str(code), code)
file_path = os.path.join(self.stocks_dir, file_name)
if not os.path.exists(file_path): return 0
with open(file_path, 'r') as f:
lines = f.readlines()
return int(lines[1].split(',')[2])
def plate_momentum(mode=ct.PAPER_TRADING,
start_date='2018-03-01',
end_date='2018-10-28'):
if mode == ct.PAPER_TRADING:
threshold = 100000
feed = dataFramefeed.Feed()
instruments = choose_plate()
if len(instruments): return 0
for code in instruments:
data = CStock(code,
should_create_influxdb=False,
should_create_mysqldb=False).get_k_data_in_range(
start_date, end_date)
data = data.set_index('date')
feed.addBarsFromDataFrame(code, data)
# broker setting
# broker commission类设置
broker_commission = broker.backtesting.TradePercentage(0.002)
# fill strategy设置
fill_stra = broker.fillstrategy.DefaultStrategy(volumeLimit=1.0)
sli_stra = broker.slippage.NoSlippage()
fill_stra.setSlippageModel(sli_stra)
# 完善broker类
brk = broker.backtesting.Broker(threshold * len(instruments), feed,
broker_commission)
brk.setFillStrategy(fill_stra)
myStrategy = PlateMomentumStrategy(feed, instruments, brk, threshold)
# Attach a returns analyzers to the strategy.
returnsAnalyzer = returns.Returns()
myStrategy.attachAnalyzer(returnsAnalyzer)
# Attach the plotter to the strategy.
plt = plotter.StrategyPlotter(myStrategy)
# Plot the simple returns on each bar.
plt.getOrCreateSubplot("returns").addDataSeries(
"Simple returns", returnsAnalyzer.getReturns())
myStrategy.run()
myStrategy.info("Final portfolio value: $%.2f" % myStrategy.getResult())
plt.plot()
def _set_stock_info(_date, bonus_info, index_info, code_id):
try:
if CStock(code_id).set_k_data(bonus_info, index_info, _date):
self.logger.info("%s set k data success" % code_id)
return (code_id, True)
else:
self.logger.error("%s set k data failed" % code_id)
return (code_id, False)
except Exception as e:
self.logger.error("%s set k data exception:%s" % (code_id, e))
return (code_id, False)
def test_is_subnew(self, mock_create, mock_get, mock_createInfo):
mock_createInfo.return_value = True
mock_create.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'test')
self.assertEqual(cs.is_subnew(time2Market='0'), False)
mock_createInfo.return_value = True
mock_create.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'test')
mock_get.return_value = '20131221'
self.assertEqual(cs.is_subnew(), False)
mock_createInfo.return_value = True
mock_create.return_value = True
mock_get.return_value = '20171221'
cs = CStock(ct.DB_INFO, '111111', 'test')
self.assertEqual(cs.is_subnew(time2Market=None), True)
def update_stock(attr, old, new):
code = code_text.value
if code is None: return
sobj = CStock(code)
sdf = sobj.get_k_data()
if sdf is None: return
vdf = get_val_data(code)
global stock_fig, profit_fig, dist_fig, roe_fig, stock_source, dist_source, val_source
mdate = mmap_pckr.value
mdate = mdate.strftime('%Y-%m-%d')
ddf = sobj.get_chip_distribution(mdate)
stock_source = ColumnDataSource(sdf)
dist_source = ColumnDataSource(ddf)
val_source = ColumnDataSource(vdf)
stock_fig = create_stock_figure(stock_source)
profit_fig = create_profit_figure(stock_source)
dist_fig = create_dist_figure(dist_source)
roe_fig = create_roe_figure(val_source)
stock_fig.on_event(DoubleTap, scallback)
layout.children[4] = gridplot([[stock_fig, dist_fig],
[profit_fig, roe_fig]])
def init_real_stock_info(self):
concerned_list = self.comb_info_client.get_concerned_list()
prefix_concerned_list = [add_prifix(code) for code in concerned_list]
ret = self.subscriber.subscribe(prefix_concerned_list, SubType.TICKER,
self.ticker_handler)
if 0 == ret:
for code in concerned_list:
if code not in self.stock_objs:
self.stock_objs[code] = CStock(code,
self.dbinfo,
should_create_influxdb=True,
should_create_mysqldb=True)
return ret
def _set_stock_info(mdate, bonus_info, index_info, code_id):
try:
if CStock(code_id).set_k_data(bonus_info, index_info, mdate):
self.logger.info("%s set k data success for date:%s",
code_id, mdate)
return (code_id, True)
else:
self.logger.error("%s set k data failed for date:%s",
code_id, mdate)
return (code_id, False)
except Exception as e:
self.logger.error("%s set k data for date %s exception:%s",
code_id, mdate, e)
return (code_id, False)
def init(self):
df = ts.get_stock_basics()
if df is None: return False
df = df.reset_index(drop = False)
failed_list = list()
for _, code_id in df['code'].iteritems():
dbname = CStock.get_dbname(code_id)
if dbname not in self.mysql_dbs:
if not self.mysql_client.create_db(dbname): failed_list.append(code_id)
if len(failed_list) > 0:
logger.error("%s create failed" % failed_list)
return False
df['limitUpNum'] = 0
df['limitDownNum'] = 0
return self.redis.set(ct.STOCK_INFO, _pickle.dumps(df, 2))
def test_create_static(self, mock_create, mock_get_all_tables,
mock_createInfo, mock_mysql_create):
mock_create.return_value = True
mock_createInfo.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'test')
mock_get_all_tables.return_value = ['111111_D']
self.assertEqual(cs.create_static(), True)
mock_create.return_value = True
mock_createInfo.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'test')
mock_get_all_tables.return_value = []
mock_mysql_create.return_value = True
self.assertEqual(cs.create_static(), True)
mock_create.return_value = True
mock_createInfo.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'test')
mock_get_all_tables.return_value = []
mock_mysql_create.return_value = False
self.assertEqual(cs.create_static(), False)
def test_create_realtime(self, mock_create, mock_create_info,
mock_get_all_tables, mock_mysql_create):
mock_create.return_value = True
mock_create_info.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'test')
mock_get_all_tables.return_value = []
mock_mysql_create.return_value = False
self.assertEqual(cs.create_realtime(), False)
mock_create.return_value = True
mock_create_info.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'test')
mock_get_all_tables.return_value = []
mock_mysql_create.return_value = True
self.assertEqual(cs.create_realtime(), True)
mock_create.return_value = True
mock_create_info.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'test')
mock_get_all_tables.return_value = [cs.realtime_table]
mock_mysql_create.return_value = False
self.assertEqual(cs.create_realtime(), True)
def init_today_stock_tick(self):
_date = datetime.now().strftime('%Y-%m-%d')
obj_pool = Pool(50)
df = self.stock_info_client.get()
if self.cal_client.is_trading_day(_date):
for _, code_id in df.code.iteritems():
_obj = self.stock_objs[
code_id] if code_id in self.stock_objs else CStock(
self.dbinfo, code_id)
try:
if obj_pool.full(): obj_pool.join()
obj_pool.spawn(_obj.set_ticket, _date)
obj_pool.spawn(_obj.set_k_data)
except Exception as e:
logger.info(e)
obj_pool.join()
obj_pool.kill()
def read_stock_csv(stock_dir):
for filename in os.listdir(stock_dir):
#logger.info("start :%s" % filename)
code = filename.split('.')[0]
d_table_name = "%s_D" % code
stock = CStock(ct.DB_INFO, code)
#logger.info("created :%s" % filename)
try:
df = pd.read_csv(os.path.join(stock_dir, filename))
except pd.errors.EmptyDataError:
logger.info("%s:empty data" % code)
continue
df.columns = [
'cdate', 'open', 'high', 'low', 'close', 'volume', 'amount'
]
#logger.info("readcsv :%s" % filename)
if not stock.mysql_client.set(df, d_table_name):
logger.info("failed :%s" % filename)
succeed_list.append(filename)
logger.info("succeed :%s" % filename)
def max_amounts(self, code, price, max_qty):
"https://trade.cgws.com/cgi-bin/stock/EntrustQuery?function=ajaxMaxAmount&market=0&secuid=0121056913&stkcode=300762&bsflag=B&price=16.280&rand=1551774639858"
randNum = str(int(time.time())) + "".join(map(lambda x:random.choice(string.digits), range(3)))
market_id = CStock.get_market(code)
post_data = {
"function": "ajaxMaxAmount",
"market": market_id,
"secuid": self.secuids[market_id],
"stkcode": code,
"bsflag": "B",
"price": price,
"rand": randNum
}
(ret, response) = self.session_client.post(STOCK_MONUT, post_data)
if ret != 0:
self.log.warn("get to url fail: ret=%d" % ret)
return -5, "get order url failed"
try:
stock_info = response.json()
return int(stock_info[0]['errorCode']), int(stock_info[0]['maxstkqty'])
except:
self.log.warn("stock info can not be json.")
return -6, "stock info can not be json."
def get_stock_data(self, cdate, code):
return (code, CStock(code).get_k_data(cdate))
def _set_base_float_profit(code_id):
return (code_id,
True) if CStock(code_id).set_base_floating_profit() else (
code_id, False)
def cget(mdate, code):
return code, CStock(code).get_val_data(mdate)
def myfunc(code, mdate):
tmp_df = df.loc[(df.code == code) & (df.date == mdate)]
tmp_df = tmp_df.reset_index(drop=True)
CStock(code).set_val_data(tmp_df, fpath="/data/valuation/cstocks")
def test_get_k_data(self, mock_create_info, mock_create, mock_sql_get):
mock_create.return_value = True
mock_create_info.return_value = True
cs = CStock(ct.DB_INFO, '111111', 'testa')
cs.get_k_data('2017-3-18')
mock_sql_get.assert_called_with(
"select * from `111111` where date=\"2017-3-18\"")
cs = CStock(ct.DB_INFO, '111111', 'testa')
cs.get_k_data()
mock_sql_get.assert_called_with("select * from `111111`")
cs = CStock(ct.DB_INFO, '111111a', 'testa')
cs.get_k_data()
mock_sql_get.assert_called_with("select * from 111111a")
cs = CStock(ct.DB_INFO, '111111a', 'testa')
cs.get_k_data('2017-3-19')
mock_sql_get.assert_called_with(
"select * from 111111a where date=\"2017-3-19\"")
if tmp_df.volume.sum() != outstanding:
raise Exception(
"tmp_df.volume.sum() is not equal to outstanding")
tmp_df = tmp_df.reset_index(drop=True)
pre_outstanding = outstanding
df = df.append(tmp_df)
df = df[df.volume != 0]
df = df.reset_index(drop=True)
return df
if __name__ == '__main__':
from cindex import CIndex
from cstock import CStock
cdate = None
cstock = CStock('601318')
index_info = CIndex('000001').get_k_data(cdate)
bonus_info = pd.read_csv("/data/tdx/base/bonus.csv",
sep=',',
dtype={
'code': str,
'market': int,
'type': int,
'money': float,
'price': float,
'count': float,
'rate': float,
'date': int
})
quantity_change_info, price_change_info = cstock.collect_right_info(
bonus_info)
