def SU_save_stock_block(client=STOCKDATA, ui_log=None, ui_progress=None):
"""save stock_block
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
client.drop_collection('stock_block')
coll = client.stock_block
coll.create_index('code')
try:
util_log_info('##JOB09 Now Saving STOCK_BlOCK ====', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=5000)
coll.insert_many(util_to_json_from_pandas(
fetch_get_stock_block('tdx')))
util_log_info('tdx Block ====', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=5000)
# 🛠todo fixhere here 获取同花顺板块， 还是调用tdx的
coll.insert_many(util_to_json_from_pandas(
fetch_get_stock_block('ths')))
util_log_info('ths Block ====', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=8000)
util_log_info('完成股票板块获取=', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=10000)
except Exception as e:
util_log_info(e, ui_log=ui_log)
print(" Error save_tdx.SU_save_stock_block exception!")
pass
def __saving_work(code, coll):
try:
ref_ = coll.find({'code': str(code)[0:6]})
end_time = str(now_time())[0:10]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['date']
util_log_info('##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'.format
(code, start_time, end_time), ui_log=ui_log)
if start_time != end_time:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_index_day(str(code), util_get_next_day(start_time), end_time)))
else:
start_time = '1990-01-01'
util_log_info('##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'.format
(code, start_time, end_time), ui_log=ui_log)
if start_time != end_time:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_index_day(str(code), start_time, end_time)))
except:
err.append(str(code))
def __saving_work(code, coll_option_day):
try:
util_log_info('##JOB12 Now Saving OPTION_DAY==== {}'.format(
str(code)), ui_log=ui_log)
# 首选查找数据库 是否 有 这个代码的数据
# 期权代码 从 10000001 开始编码 10001228
ref = coll_option_day.find({'code': str(code)[0:8]})
end_date = str(now_time())[0:10]
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date = ref[ref.count() - 1]['date']
util_log_info(' 上次获取期权日线数据的最后日期是 {}'.format(
start_date), ui_log=ui_log)
util_log_info('UPDATE_OPTION_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'.format(
code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
start_date0 = util_get_next_day(start_date)
df0 = fetch_get_option_day(code=code, start_date=start_date0, end_date=end_date,
frequence='day', ip=None, port=None)
retCount = df0.iloc[:, 0].size
util_log_info("日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库"
.format(start_date0, end_date, code, retCount), ui_log=ui_log)
coll_option_day.insert_many(
util_to_json_from_pandas(df0))
else:
util_log_info("^已经获取过这天的数据了^ {}".format(
start_date), ui_log=ui_log)
else:
start_date = '1990-01-01'
util_log_info('UPDATE_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'.format
(code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
df0 = fetch_get_option_day(code=code, start_date=start_date, end_date=end_date,
frequence='day', ip=None, port=None)
retCount = df0.iloc[:, 0].size
util_log_info("日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ "
.format(start_date, end_date, code, retCount),
ui_log=ui_log)
coll_option_day.insert_many(
util_to_json_from_pandas(df0))
else:
util_log_info(
"*已经获取过这天的数据了* {}".format(start_date), ui_log=ui_log)
except Exception as error0:
print(error0)
err.append(str(code))
def SU_save_future_list(client=STOCKDATA, ui_log=None, ui_progress=None):
future_list = fetch_get_future_list()
coll_future_list = client.future_list
coll_future_list.create_index("code", unique=True)
try:
coll_future_list.insert_many(
util_to_json_from_pandas(future_list), ordered=False)
except:
pass
def __saving_work(code):
util_log_info(
'##JOB11 Now Saving STOCK_TRANSACTION ==== {}'.format(str(code)), ui_log=ui_log)
try:
coll.insert_many(
util_to_json_from_pandas(
# 🛠todo str(stock_list[code]) 参数不对？
fetch_get_stock_transaction(str(code), '1990-01-01', str(now_time())[0:10])))
except:
err.append(str(code))
def __saving_work(code, coll):
util_log_info(
'##JOB010 Now Saving STOCK INFO ==== {}'.format(str(code)), ui_log=ui_log)
try:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_stock_info(str(code))))
except:
err.append(str(code))
def __saving_work(code, coll):
util_log_info(
'##JOB05 Now Saving Index_MIN ==== {}'.format(str(code)), ui_log=ui_log)
try:
for type in ['1min', '5min', '15min', '30min', '60min']:
ref_ = coll.find(
{'code': str(code)[0:6], 'type': type})
end_time = str(now_time())[0:19]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['datetime']
util_log_info(
'##JOB05.{} Now Saving {} from {} to {} =={} '.
format(['1min', '5min', '15min', '30min', '60min'].
index(type), str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_index_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
util_to_json_from_pandas(__data[1::]))
else:
start_time = '2015-01-01'
util_log_info(
'##JOB05.{} Now Saving {} from {} to {} =={} '.
format(['1min', '5min', '15min', '30min', '60min'].
index(type), str(code), start_time, end_time, type),
ui_log=ui_log)
if start_time != end_time:
__data = fetch_get_index_min(
str(code), start_time, end_time, type)
if len(__data) > 1:
coll.insert_many(
util_to_json_from_pandas(__data))
except:
err.append(code)
def __init__(self, market_data):
"""转化成dict模式
Arguments:
market_data {[type]} -- [description]
"""
if isinstance(market_data, dict):
self.data = market_data
elif isinstance(market_data, pd.DataFrame):
self.data = util_to_json_from_pandas(market_data)
def __saving_work(code, coll):
util_log_info('##JOB02 Now Saving XDXR INFO ==== {}'.format(
str(code)), ui_log=ui_log)
try:
coll.insert_many(
util_to_json_from_pandas(
fetch_get_stock_xdxr(str(code))), ordered=False)
except:
err.append(str(code))
def SU_save_stock_list(client=STOCKDATA, ui_log=None, ui_progress=None):
"""save stock_list
Keyword Arguments:
client {[type]} -- [description] (default: {STOCKDATA})
"""
client.drop_collection('stock_list')
coll = client.stock_list
coll.create_index('code')
try:
# 🛠todo 这个应该是第一个任务 JOB01， 先更新股票列表！！
util_log_info('##JOB08 Now Saving STOCK_LIST ====', ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=5000)
stock_list_from_tdx = fetch_get_stock_list()
pandas_data = util_to_json_from_pandas(stock_list_from_tdx)
coll.insert_many(pandas_data)
util_log_info("完成股票列表获取", ui_log=ui_log,
ui_progress=ui_progress, ui_progress_int_value=10000)
except Exception as e:
util_log_info(e, ui_log=ui_log)
print(" Error save_tdx.SU_save_stock_list exception!")
pass
def to_json(self):
"""
转换DataStruct为json
"""
return util_to_json_from_pandas(self.data.reset_index())