def SU_update_stock_day(client=ms.client):
def save_stock_day(code, start, end, coll):
util_log_info('##JOB01 Now Updating STOCK_DAY==== %s' % (str(code)))
data = util_to_json_from_pandas(
fetch_get_stock_day(str(code), start, end, '00'))
if len(data) > 0:
coll.insert_many(data)
else:
pass
coll_stock_day = client.flyshare.stock_day
for item in fetch_get_stock_time_to_market().index:
if coll_stock_day.find({'code': str(item)[0:6]}).count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = str(coll_stock_day.find({'code': str(item)[0:6]})[
coll_stock_day.find({'code': str(item)[0:6]}).count() - 1]['date'])
print('*' * 20)
end_date = str(now_time())[0:10]
start_date = trade_date_sse[trade_date_sse.index(
start_date) + 1]
util_log_info(' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' %
(item, start_date, end_date))
save_stock_day(item, start_date, end_date, coll_stock_day)
else:
save_stock_day(item, '1990-01-01',
str(now_time())[0:10], coll_stock_day)
util_log_info('Done == \n')
def __saving_work(code, __coll):
util_log_info('##JOB07 Now Saving ETF_MIN ==== %s' % (str(code)))
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
ref_ = __coll.find({'code': str(code)[0:6], 'type': type})
end_time = str(now_time())[0:19]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['datetime']
else:
start_time = '2015-01-01'
util_log_info(
'##JOB07.%s Now Saving %s from %s to %s ==%s ' %
(['1min', '5min', '15min', '30min', '60min'
].index(type), str(code), start_time, end_time, type))
if start_time != end_time:
__data = fetch_get_index_min(str(code), start_time,
end_time, type)
if len(__data) > 1:
__coll.insert_many(
util_to_json_from_pandas(__data[1::]))
except:
__err.append(code)
def data_stock_to_fq(__data, type_='01'):
def __fetch_stock_xdxr(code,
format_='pd',
collections=ms.client.flyshare.stock_xdxr):
'获取股票除权信息/数据库'
try:
data = pd.DataFrame([
item for item in collections.find({'code': code})
]).drop(['_id'], axis=1)
data['date'] = pd.to_datetime(data['date'])
return data.set_index(['date', 'code'], drop=False)
except:
return pd.DataFrame(columns=[
'category', 'category_meaning', 'code', 'date', 'fenhong',
'fenshu', 'liquidity_after', 'liquidity_before', 'name',
'peigu', 'peigujia', 'shares_after', 'shares_before',
'songzhuangu', 'suogu', 'xingquanjia'
])
'股票 日线/分钟线 动态复权接口'
if type_ in ['01', 'qfq']:
#print(data_make_qfq(__data, __fetch_stock_xdxr(__data['code'][0])))
return data_make_qfq(__data, __fetch_stock_xdxr(__data['code'][0]))
elif type_ in ['02', 'hfq']:
return data_make_hfq(__data, __fetch_stock_xdxr(__data['code'][0]))
else:
util_log_info('wrong fq type! Using qfq')
return data_make_qfq(__data, __fetch_stock_xdxr(__data['code'][0]))
def fetch_stock_full(date_, format_='numpy', collections=ms.client.flyshare.stock_day):
'获取全市场的某一日的数据'
#__start = str(__start)[0:10]
Date = str(date_)[0:10]
if util_date_valid(Date) == True:
__data = []
for item in collections.find({
"date_stamp": {
"$lte": util_date_stamp(Date),
"$gte": util_date_stamp(Date)}}):
__data.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['volume']), item['date']])
# 多种数据格式
if format_ in ['n', 'N', 'numpy']:
__data = numpy.asarray(__data)
elif format_ in ['list', 'l', 'L']:
__data = __data
elif format_ in ['P', 'p', 'pandas', 'pd']:
__data = DataFrame(__data, columns=[
'code', 'open', 'high', 'low', 'close', 'volume', 'date'])
__data['date'] = pd.to_datetime(__data['date'])
__data = __data.set_index('date', drop=True)
return __data
else:
util_log_info('something wrong with date')
def save_stock_day_all_bfq(client=ms.client):
df = ts.get_stock_basics()
__coll = client.flyshare.stock_day_bfq
__coll.ensure_index('code')
def saving_work(i):
util_log_info('Now Saving ==== %s' % (i))
try:
data_json = fetch_get_stock_day(i,
startDate='1990-01-01',
if_fq='00')
__coll.insert_many(data_json)
except:
util_log_info('error in saving ==== %s' % str(i))
for i_ in range(len(df.index)):
util_log_info('The %s of Total %s' % (i_, len(df.index)))
util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(i_ / len(df.index) * 100))[0:4] + '%')
saving_work(df.index[i_])
saving_work('hs300')
saving_work('sz50')
def fetch_index_day(code, __start, __end, format_='numpy', collections=ms.client.flyshare.index_day):
'获取指数日线'
__start = str(__start)[0:10]
__end = str(__end)[0:10]
if util_date_valid(__end) == True:
__data = []
for item in collections.find({
'code': str(code)[0:6], "date_stamp": {
"$lte": util_date_stamp(__end),
"$gte": util_date_stamp(__start)}}):
__data.append([str(item['code']), float(item['open']), float(item['high']), float(
item['low']), float(item['close']), float(item['vol']), item['date']])
# 多种数据格式
if format_ in ['n', 'N', 'numpy']:
__data = numpy.asarray(__data)
elif format_ in ['list', 'l', 'L']:
__data = __data
elif format_ in ['P', 'p', 'pandas', 'pd']:
__data = DataFrame(__data, columns=[
'code', 'open', 'high', 'low', 'close', 'volume', 'date'])
__data['date'] = pd.to_datetime(__data['date'])
__data = __data.set_index('date', drop=False)
return __data
else:
util_log_info('something wrong with date')
def fetch_get_stock_financial(name, startDate, endDate):
try:
from WindPy import w
except:
util_log_info('No WindPY Module!')
w.start()
pass
def saving_work(i):
util_log_info('Now Saving ==== %s' % (i))
try:
data_json = fetch_get_stock_day(i, startDate='1990-01-01')
__coll.insert_many(data_json)
except:
util_log_info('error in saving ==== %s' % str(i))
def user_sign_in(name, password, clients=pymongo.MongoClient()):
coll = clients.flyshare.user_list
if (coll.find({'username': name, 'password': password}).count() > 0):
util_log_info('success login! your username is:' + str(name))
return True
else:
util_log_info('Failed to login,please check your password: username='******',password=' + password)
return False
def __saving_work(code, __coll):
util_log_info('##JOB02 Now Saving XDXR INFO ==== %s' % (str(code)))
try:
__coll.insert_many(
util_to_json_from_pandas(fetch_get_stock_xdxr(str(code))))
except:
__err.append(str(code))
def select_best_ip():
util_log_info('Selecting the Best Server IP of TDX')
listx = vars.SLIST
conn_times = [ping(x) for x in listx]
best_ip = listx[conn_times.index(min(conn_times))]
util_log_info('===The BEST SERVER is : %s ===' % (best_ip))
ac.TDX_BEST_IP = best_ip
return best_ip
def user_sign_up(name, password, clients=pymongo.MongoClient()):
coll = clients.flyshare.user_list
if (coll.find({'username': name}).count() > 0):
util_log_info('user name is already exist')
return False
else:
coll.insert({'username': name, 'password': password})
util_log_info('Success sign in! please login ')
return True
def fetch_stock_info(code, format_='pd', collections=ms.client.flyshare.stock_info):
try:
data = pd.DataFrame([item for item in collections.find(
{'code': code})]).drop(['_id'], axis=1)
#data['date'] = pd.to_datetime(data['date'])
return data.set_index('code', drop=False)
except Exception as e:
util_log_info(e)
return None
def save_stock_day(code, start, end, coll):
util_log_info('##JOB01 Now Updating STOCK_DAY==== %s' % (str(code)))
data = util_to_json_from_pandas(
fetch_get_stock_day(str(code), start, end, '00'))
if len(data) > 0:
coll.insert_many(data)
else:
pass
def to_hfq(self):
if self.if_fq is 'bfq':
data = DataStruct_Stock_day(pd.concat(list(map(lambda x: data_stock_to_fq(
self.data[self.data['code'] == x], '01'), self.code))))
data.if_fq = 'hfq'
return data
else:
util_log_info(
'none support type for qfq Current type is: %s' % self.if_fq)
return self
def SU_save_stock_list(client=ms.client):
client.flyshare.drop_collection('stock_list')
__coll = client.flyshare.stock_list
__coll.create_index('code')
__err = []
try:
util_log_info('##JOB08 Now Saving STOCK_LIST ====')
__coll.insert_many(util_to_json_from_pandas(fetch_get_stock_list()))
except:
pass
def saving_work(i):
util_log_info('Now Saving ==== %s' % (i))
try:
data_hfq = fetch_get_stock_day(i,
startDate='1990-01-01',
if_fq='02',
type_='pd')
data_json = util_to_json_from_pandas(data_hfq)
__coll.insert_many(data_json)
except:
util_log_info('error in saving ==== %s' % str(i))
def __saving_work(code):
util_log_info('##JOB10 Now Saving STOCK_TRANSACTION ==== %s' %
(str(code)))
try:
__coll.insert_many(
util_to_json_from_pandas(
fetch_get_stock_transaction(str(code),
str(__stock_list[code]),
str(now_time())[0:10])))
except:
__err.append(str(code))
def setting_login(self):
self.username = self.setting_user_name
self.password = self.setting_user_password
util_log_info('username:'******'username'] = self.username
self.user['password'] = self.password
self.user['login'] = True
return self.user
else:
util_log_info('failed to login')
def SU_save_stock_block(client=ms.client):
client.flyshare.drop_collection('stock_block')
__coll = client.flyshare.stock_block
__coll.create_index('code')
__err = []
try:
util_log_info('##JOB09 Now Saving STOCK_BlOCK ====')
__coll.insert_many(
util_to_json_from_pandas(fetch_get_stock_block('tdx')))
__coll.insert_many(
util_to_json_from_pandas(fetch_get_stock_block('ths')))
except:
pass
def SU_save_stock_transaction(client=ms.client):
__stock_list = fetch_get_stock_time_to_market()
__coll = client.flyshare.stock_transaction
__coll.create_index('code', pymongo.ASCENDING)
__err = []
def __saving_work(code):
util_log_info('##JOB10 Now Saving STOCK_TRANSACTION ==== %s' %
(str(code)))
try:
__coll.insert_many(
util_to_json_from_pandas(
fetch_get_stock_transaction(str(code),
str(__stock_list[code]),
str(now_time())[0:10])))
except:
__err.append(str(code))
for i_ in range(len(__stock_list)):
#__saving_work('000001')
util_log_info('The %s of Total %s' % (i_, len(__stock_list)))
util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(i_ / len(__stock_list) * 100))[0:4] + '%')
__saving_work(__stock_list.index[i_])
util_log_info('ERROR CODE \n ')
util_log_info(__err)
def SU_save_stock_xdxr(client=ms.client):
client.flyshare.drop_collection('stock_xdxr')
__stock_list = fetch_get_stock_time_to_market()
__coll = client.flyshare.stock_xdxr
__coll.create_index([('code', pymongo.ASCENDING),
('date', pymongo.ASCENDING)])
__err = []
def __saving_work(code, __coll):
util_log_info('##JOB02 Now Saving XDXR INFO ==== %s' % (str(code)))
try:
__coll.insert_many(
util_to_json_from_pandas(fetch_get_stock_xdxr(str(code))))
except:
__err.append(str(code))
for i_ in range(len(__stock_list)):
#__saving_work('000001')
util_log_info('The %s of Total %s' % (i_, len(__stock_list)))
util_log_info('DOWNLOAD PROGRESS %s ' %
str(float(i_ / len(__stock_list) * 100))[0:4] + '%')
__saving_work(__stock_list.index[i_], __coll)
util_log_info('ERROR CODE \n ')
util_log_info(__err)
def __confirm_bid(__bid):
if isinstance(__bid.price, str):
if __bid.price == 'market_price':
return __bid
elif __bid.price == 'close_price':
return __bid
elif __bid.price == 'strict' or 'strict_model' or 'strict_price':
__bid.price = 'strict_price'
return __bid
else:
util_log_info('unsupport type:' + __bid.price)
return __bid
else:
return __bid
def fetch_get_stock_day(name, startDate, endDate, if_fq='01'):
try:
from WindPy import w
except:
util_log_info('No WindPY Module!')
w.start()
if (util_date_valid(endDate) == False):
util_log_info("wrong date")
else:
if if_fq in ['00', 'bfq']:
data = w.wsd(name, "sec_name,pre_close,open,high,low,close,volume",
startDate, endDate)
elif if_fq in ['01', 'qfq']:
data = w.wsd(name, "sec_name,pre_close,open,high,low,close,volume",
startDate, endDate, "PriceAdj=F")
elif if_fq in ['02', 'hfq']:
data = w.wsd(name, "sec_name,pre_close,open,high,low,close,volume",
startDate, endDate, "PriceAdj=B")
else:
util_log_info('wrong fq factor! using qfq')
data = w.wsd(name, "sec_name,pre_close,open,high,low,close,volume",
startDate, endDate, "PriceAdj=B")
if (data.ErrorCode == 0):
util_log_info("Connent to Wind successfully")
return pd.DataFrame(np.asarray(data.Data).T,
columns=data.Fields,
index=data.Times)
def fetch_get_stock_list(date):
try:
from WindPy import w
except:
util_log_info('No WindPY Module!')
w.start()
if (util_date_valid(date) == False):
util_log_info("wrong date")
else:
awgs = 'date=' + date + ';sectorid=a001010100000000'
data = w.wset("sectorconstituent", awgs)
return pd.DataFrame(np.asarray(data.Data).T,
columns=data.Fields,
index=data.Times)
def to_qfq(self):
if self.if_fq is 'bfq':
if len(self.code) < 20:
data = DataStruct_Stock_min(pd.concat(list(map(lambda x: data_stock_to_fq(
self.data[self.data['code'] == x]), self.code))).set_index(['datetime', 'code'], drop=False))
data.if_fq = 'qfq'
return data
else:
data = DataStruct_Stock_min(
self.data.groupby('code').apply(data_stock_to_fq))
return data
else:
util_log_info(
'none support type for qfq Current type is:%s' % self.if_fq)
return self
def fetch_get_stock_block(ip=ac.TDX_BEST_IP, port=7709):
'板块数据'
api = TdxHq_API()
with api.connect(ip, port):
data = pd.concat([api.to_df(api.get_and_parse_block_info("block_gn.dat")).assign(type='gn'),
api.to_df(api.get_and_parse_block_info(
"block.dat")).assign(type='yb'),
api.to_df(api.get_and_parse_block_info(
"block_zs.dat")).assign(type='zs'),
api.to_df(api.get_and_parse_block_info("block_fg.dat")).assign(type='fg')])
if len(data) > 10:
return data.assign(source='tdx').drop(['block_type', 'code_index'], axis=1).set_index('code', drop=False, inplace=False).drop_duplicates()
else:
util_log_info('Wrong with fetch block ')
def fetch_stock_day_adv(code,
__start,
__end,
if_drop_index=False,
collections=ms.client.flyshare.stock_day):
'获取股票日线'
__start = str(__start)[0:10]
__end = str(__end)[0:10]
if isinstance(code, str):
if util_date_valid(__end) == True:
__data = []
for item in collections.find({
'code': str(code)[0:6],
"date_stamp": {
"$lte": util_date_stamp(__end),
"$gte": util_date_stamp(__start)
}
}):
__data.append([
str(item['code']),
float(item['open']),
float(item['high']),
float(item['low']),
float(item['close']),
float(item['vol']),
float(item['amount']), item['date']
])
__data = DataFrame(__data,
columns=[
'code', 'open', 'high', 'low', 'close',
'volume', 'amount', 'date'
])
__data['date'] = pd.to_datetime(__data['date'])
return DataStruct_Stock_day(
__data.query('volume>1').set_index(['date', 'code'],
drop=if_drop_index))
else:
util_log_info('something wrong with date')
elif isinstance(code, list):
return DataStruct_Stock_day(
pd.concat(fetch_stocklist_day(
code, [__start, __end])).query('volume>1').set_index(
['date', 'code'], drop=if_drop_index))
def fetch_get_stock_list_special(date, id):
try:
from WindPy import w
except:
util_log_info('No WindPY Module!')
w.start()
if (util_date_valid(date) == False):
util_log_info("wrong date")
else:
if id in [
'big', 'small', 'cixin', 'yujing', 'rzrq', 'rq', 'yj', 'st',
'sst'
]:
awgs = 'date=' + date + ';sectorid=' + \
data_list.wind_stock_list_special_id[id]
data = w.wset("sectorconstituent", awgs)
return pd.DataFrame(np.asarray(data.Data).T,
columns=data.Fields,
index=data.Times)
def fetch_get_trade_date(endDate, exchange):
try:
from WindPy import w
except:
util_log_info('No WindPY Module!')
w.start()
supportExchanges = ["SSE", "SZSE", "CFFEX", "SHFE", "DCE", "CZCE"]
if (exchange in supportExchanges):
#"SSE","SZSE","CFFEX","SHFE","DCE","CZCE"
# 上海股票交易所,深圳股票交易所,中国金融期货交易所,上海期货交易所,大连商品交易所,郑州期货交易所
exchanges = "TradingCalendar=" + exchange
data = w.tdays("1990-01-01", endDate, exchanges)
# util_log_info(data.Data)
dates = pd.DataFrame(np.asarray(data.Data).T,
columns=data.Fields,
index=data.Times)
else:
util_log_info("exchange name problem")
return dates