def _saving_work(code, coll_stock_day, ui_log=None, err=[]):
try:
QA_util_log_info(
'##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)),
ui_log)
# 首选查找数据库 是否 有 这个代码的数据
ref = coll_stock_day.find({'ts_code': str(code)},
sort=[('trade_date', 1)])
end_date = now_time()
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date_new_format = ref[ref.count() - 1]['trade_date']
start_date = ref[ref.count() - 1]['date']
if date_conver_to_new_format(
QA_util_get_next_day(start_date, n=100)) < end_date:
QA_util_log_info(
'LAST STOCK_DAY 100 days ago {}, skip. To resume please delete {}'
.format(start_date_new_format, code), ui_log)
else:
QA_util_log_info(
'UPDATE_BOND_DAY \n Trying updating {} from {} to {}'.
format(code, start_date_new_format, end_date), ui_log)
if start_date_new_format != end_date:
pandas_data = QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(
str(code),
date_conver_to_new_format(
QA_util_get_next_day(start_date)),
end_date, 'bfq'))
if len(pandas_data) > 0:
coll_stock_day.insert_many(pandas_data)
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
else:
start_date = '19900101'
QA_util_log_info(
'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.
format(code, start_date, end_date), ui_log)
if start_date != end_date:
pandas_data = QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date, end_date,
'bfq'))
if len(pandas_data) > 0:
coll_stock_day.insert_many(pandas_data)
except Exception as e:
print(e)
err.append(str(code))
def __saving_work(code, coll):
try:
ref_ = coll.find({'code': str(code)[0:6]})
end_time = str(now_time())[0:10]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['date']
QA_util_log_info(
'##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'
.format(code, start_time, end_time))
if start_time != end_time:
coll.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_index_day(
str(code), QA_util_get_next_day(start_time),
end_time)))
else:
start_time = '1990-01-01'
QA_util_log_info(
'##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'
.format(code, start_time, end_time))
if start_time != end_time:
coll.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_index_day(str(code), start_time,
end_time)))
except:
err.append(str(code))
def __saving_work(code, coll_stock_day):
try:
QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== %s' %
(str(code)))
ref = coll_stock_day.find({'code': str(code)[0:6]})
end_date = str(now_time())[0:10]
if ref.count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = ref[ref.count() - 1]['date']
QA_util_log_info(
' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' %
(code, start_date, end_date))
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(
str(code), QA_util_get_next_day(start_date),
end_date, '00')))
else:
start_date = '1990-01-01'
QA_util_log_info(
' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' %
(code, start_date, end_date))
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date,
end_date, '00')))
except:
err.append(str(code))
def __saving_work(code, coll):
# try:
ref_ = coll.find({'code': str(code)[0:6]})
end_time = str(now_time())[0:10]
if ref_.count() > 0:
start_time = ref_[ref_.count() - 1]['date']
QA_util_log_info(
'##JOB04 Now Saving INDEX_DAY==== \n Trying updating %s from %s to %s'
% (code, start_time, end_time))
if start_time != end_time:
coll.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_index_day(
str(code), QA_util_get_next_day(start_time),
end_time)))
else:
start_time = '1990-01-01'
QA_util_log_info(
'##JOB04 Now Saving INDEX_DAY==== \n Trying updating %s from %s to %s'
% (code, start_time, end_time))
coll.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_index_day(str(code), start_time, end_time)))
def saving_work(code, coll_stock_day):
try:
print("entering saving_work {}".format(code))
ref = coll_stock_day.find({'code': str(code)[:6]})
end_date = str(now_time())[:10]
if ref.count() > 0:
start_date = ref[ref.count() - 1]['date']
QA_util_log_info(
'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.
format(code, start_date, end_date))
if start_date != end_date:
start_date = QA_util_get_next_day(start_date)
data_json = QA_fetch_get_stock_day(str(code),
start=start_date,
end=end_date,
type_='json')
coll_stock_day.insert_many(data_json)
else:
start_date = '1990-01-01'
data_json = QA_fetch_get_stock_day(str(code),
start=start_date,
type_='json')
coll_stock_day.insert_many(data_json)
print("done saving_work {}".format(code))
except Exception as e:
QA_util_log_info('error in saving ==== %s' % str(code))
print("The exception is {}".format(str(e)))
def __saving_work(code, coll_stock_day):
try:
QA_util_log_info(
'##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)), ui_log)
# 首选查找数据库 是否 有 这个代码的数据
ref = coll_stock_day.find({'code': str(code)[0:6]})
end_date = str(now_time())[0:10]
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date = ref[ref.count() - 1]['date']
QA_util_log_info('UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.format(
code, start_date, end_date), ui_log)
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), QA_util_get_next_day(start_date), end_date, '00')))
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
else:
start_date = '1990-01-01'
QA_util_log_info('UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.format
(code, start_date, end_date), ui_log)
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date, end_date, '00')))
except Exception as error0:
print(error0)
err.append(str(code))
def __saving_work(code, coll_stock_year):
try:
QA_util_log_info(
'##JOB01 Now Saving STOCK_YEAR==== {}'.format(str(code)), ui_log=ui_log)
ref = coll_stock_year.find({'code': str(code)[0:6]})
end_date = str(now_time())[0:10]
if ref.count() > 0:
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
start_date = ref[ref.count() - 1]['date']
QA_util_log_info('UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'.format
(code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
coll_stock_year.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), QA_util_get_next_day(start_date), end_date, '00', frequence='year')))
else:
start_date = '1990-01-01'
QA_util_log_info('UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'.format
(code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
coll_stock_year.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date, end_date, '00', frequence='year')))
except:
err.append(str(code))
def __saving_work(code, coll):
try:
QA_util_log_info(logs='##{} Now Saving {}==== {}'.format(
job_id, type_, str(code)),
ui_log=ui_log)
# 首选查找数据库 是否 有 这个代码的数据
ref = coll.find({'code': str(code)[0:6]})
end_date = str(now_time())[0:10]
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date = ref[ref.count() - 1]['date']
else:
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
start_date = beginning_date
QA_util_log_info(
logs='UPDATE_{} \n Trying updating {} from {} to {}'.format(
type_, code, start_date, end_date),
ui_log=ui_log)
if start_date != end_date:
start_date = QA_util_get_next_day(
start_date
) if start_date != beginning_date else beginning_date
coll.insert_many(
QA_util_to_json_from_pandas(config['fetch'](
code=str(code),
start_date=start_date,
end_date=end_date,
frequence=frequence)))
except Exception as error0:
print(error0)
err.append(str(code))
def __saving_work(code, coll_option_day):
try:
QA_util_log_info('##JOB12 Now Saving OPTION_DAY==== {}'.format(
str(code)), ui_log=ui_log)
# 首选查找数据库 是否 有 这个代码的数据
# 期权代码 从 10000001 开始编码 10001228
ref = coll_option_day.find({'code': str(code)[0:8]})
end_date = str(now_time())[0:10]
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date = ref[ref.count() - 1]['date']
QA_util_log_info(' 上次获取期权日线数据的最后日期是 {}'.format(
start_date), ui_log=ui_log)
QA_util_log_info('UPDATE_OPTION_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'.format(
code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
start_date0 = QA_util_get_next_day(start_date)
df0 = QA_fetch_get_option_day(code=code, start_date=start_date0, end_date=end_date,
frequence='day', ip=None, port=None)
retCount = df0.iloc[:, 0].size
QA_util_log_info("日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库"
.format(start_date0, end_date, code, retCount), ui_log=ui_log)
coll_option_day.insert_many(
QA_util_to_json_from_pandas(df0))
else:
QA_util_log_info("^已经获取过这天的数据了^ {}".format(
start_date), ui_log=ui_log)
else:
start_date = '1990-01-01'
QA_util_log_info('UPDATE_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'.format
(code, start_date, end_date), ui_log=ui_log)
if start_date != end_date:
df0 = QA_fetch_get_option_day(code=code, start_date=start_date, end_date=end_date,
frequence='day', ip=None, port=None)
retCount = df0.iloc[:, 0].size
QA_util_log_info("日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ "
.format(start_date, end_date, code, retCount),
ui_log=ui_log)
coll_option_day.insert_many(
QA_util_to_json_from_pandas(df0))
else:
QA_util_log_info(
"*已经获取过这天的数据了* {}".format(start_date), ui_log=ui_log)
except Exception as error0:
print(error0)
err.append(str(code))
def __init__(self,
featuredata,
feature_name=None,
stock_data=None,
returnday=5,
host=clickhouse_ip,
port=clickhouse_port,
user=clickhouse_user,
password=clickhouse_password) -> None:
self.feature = featuredata
self.featurename = featuredata.name if feature_name is None else feature_name
self.feature.name = self.featurename
self.codelist = self.feature.index.levels[1].unique().tolist()
self.start = str(self.feature.index.levels[0][0])[0:10]
self.end = str(self.feature.index.levels[0][-1])[0:10]
self._host = host
self._port = port
self._user = user
self._password = password
self.factorclient = clickhouse_driver.Client(host=self._host,
port=self._port,
user=self._user,
password=self._password,
database='factor')
self.dataclient = QACKClient(host=self._host,
port=self._port,
user=self._user,
password=self._password)
if stock_data is None:
self.stock_data = self.dataclient.get_stock_day_qfq_adv(
self.codelist, self.start,
QA_util_get_next_day(self.end, returnday))
else:
self.stock_data = stock_data
self.returns = self.make_ret(self.stock_data.data).dropna()
def _saving_work(code, coll_stock_tech_indicator):
try:
#logging.debug("process {} now working on stock {}".format(os.getpid(), code))
QA_util_log_info('Now Processing ==== %s' % (code))
ref = coll_stock_tech_indicator.find({'code': str(code)[:6]})
end_date = str(datetime.date.today())
if ref.count() > 0:
start_date = ref[ref.count() - 1]['date']
if start_date != end_date:
start_date = QA_util_get_next_day(start_date)
df_indicator = QA_SU_calculate_stock_tech_indicator(
code, start_date, end_date)
if len(df_indicator) > 0:
df_indicator['date'] = df_indicator.index.strftime(
'%Y-%m-%d')
df_indicator['date_stamp'] = df_indicator[
'date'].apply(lambda x: QA_util_date_stamp(x))
indicator_json = QA_util_to_json_from_pandas(
df_indicator)
coll_stock_tech_indicator.insert_many(indicator_json)
#logging.debug("df_indicator successfully inserted into DB")
else:
start_date = '1990-01-01'
df_indicator = QA_SU_calculate_stock_tech_indicator(
code, start_date, end_date)
if len(df_indicator) > 0:
df_indicator['date'] = df_indicator.index.strftime(
'%Y-%m-%d')
df_indicator['date_stamp'] = df_indicator['date'].apply(
lambda x: QA_util_date_stamp(x))
indicator_json = QA_util_to_json_from_pandas(df_indicator)
coll_stock_tech_indicator.insert_many(indicator_json)
#logging.debug("df_indicator successfully inserted into DB")
except Exception as e:
QA_util_log_info('error in processing ==== %s' % str(code))
err.append(code)
print("The exception is {}".format(str(e)))
def __saving_work(self, code):
def __QA_log_info(code, end_time, start_time):
def loginfo(prefix='', astr='', listCounts=5):
if len(self._loginfolist) < listCounts:
self._loginfolist.append(astr)
else:
str = ''
for i in range(len(self._loginfolist)):
str += self._loginfolist[i] + ' '
str += astr
QA_util_log_info(prefix.format(str), self.ui_log)
self._loginfolist.clear()
index_or_etf = self.get_index_or_etf_from_code(code)
prefix = '##JOB04 Saving {}_DAY ==== Trying updating\n{}'.format(
index_or_etf, '{}')
loginfo(prefix, ' {} from {} to {}'.format(code, start_time,
end_time))
# log_info = '##JOB04 Saving {}_DAY====\nTrying updating {} from {} to {}'.format(
# index_or_etf,
# code,
# start_time,
# end_time
# )
# QA_util_log_info(
# log_info,
# ui_log=self.ui_log
# )
try:
search_cond = {'code': str(code)[0:6]}
ref_ = get_coll().find(search_cond)
ref_count = get_coll().count_documents(search_cond)
end_time = str(now_time())[0:10]
if ref_count > 0:
start_time = ref_[ref_count - 1]['date']
__QA_log_info(code, end_time, start_time)
if start_time != end_time:
get_coll().insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_index_day(
str(code), QA_util_get_next_day(start_time),
end_time)))
else:
try:
start_time = '1990-01-01'
__QA_log_info(code, end_time, start_time)
get_coll().insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_index_day(str(code), start_time,
end_time)))
except Exception as e:
start_time = '2009-01-01'
__QA_log_info(code, end_time, start_time)
get_coll().insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_index_day(str(code), start_time,
end_time)))
except Exception as e:
QA_util_log_info(e, ui_log=self.ui_log)
self.err.append(str(code))
QA_util_log_info(self.err, ui_log=self.ui_log)
def __saving_work(self, code):
try:
QA_util_log_info(
'##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)),
self.ui_log)
time.sleep(0.2)
# 首选查找数据库 是否 有 这个代码的数据
search_cond = {'code': str(code)[0:6]}
ref = get_coll(self.client).find(search_cond)
ref_count = get_coll(self.client).count_documents(search_cond)
end_date = now_time()
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref_count > 0:
# 接着上次获取的日期继续更新
start_date_new_format = ref[ref_count - 1]['trade_date']
start_date = ref[ref_count - 1]['date']
QA_util_log_info(
'UPDATE_STOCK_DAY: Trying updating {} {} from {} to {}'.
format(ref_count, code, start_date_new_format,
end_date), self.ui_log)
if start_date_new_format != end_date:
df = QA_fetch_get_stock_day(
str(code),
date_conver_to_new_format(
QA_util_get_next_day(start_date)), end_date, 'bfq')
if not (df is None) and len(df) > 0:
get_coll(self.client).insert_many(
QA_util_to_json_from_pandas(df))
else:
QA_util_log_info(
'UPDATE_STOCK_DAY: Trying updating {} {} from {} to {} ---- No Data'
.format(ref_count, code, start_date_new_format,
end_date), self.ui_log)
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
# 一次只返回 5000 条，所以有些股票得获取两次
else:
start_date = '19900101'
QA_util_log_info(
'FETCH_ALL_STOCK_DAY: Trying fetching all data {} from {} to {}'
.format(code, start_date, end_date), self.ui_log)
if start_date != end_date:
# 第一次获取
df = QA_fetch_get_stock_day(str(code), start_date,
end_date, 'bfq')
if not (df is None):
if len(df) >= 5000:
again_date = QA_util_get_last_day(df['date'].min())
QA_util_log_info(
'FETCH_ALL_STOCK_DAY: Trying updating again {} from {} to {}'
.format(code, start_date,
again_date), self.ui_log)
df2 = QA_fetch_get_stock_day(
str(code), start_date, again_date, 'bfq')
df = df.append(df2)
get_coll(self.client).insert_many(
QA_util_to_json_from_pandas(df))
else:
QA_util_log_info(
'FETCH_ALL_STOCK_DAY: Trying fetching all data {} from {} to {} ---- No Data'
.format(code, start_date, end_date), self.ui_log)
except Exception as e:
print(e)
self.err.append(str(code))
#%%
data = QA.QAFetch.QATdx.QA_fetch_get_stock_transaction('000001','2015-06-05','2015-06-06')
data.index = pd.to_datetime(data.index)
type(data.index[0])('2019-01-01')
#%%
data = QA.QAFetch.QATdx.QA_fetch_get_future_list()
data['signal'] = data['code'].apply(lambda x:x[-2:])
data[data['signal'].isin(['L8','L9'])]
1300/8200
assert False
STOCK_DAY = (['code', 'open', 'high', 'low','close', 'volume', 'amount', 'date'],'date')
['code','open'] in
from QUANTAXIS.QAUtil import (DATABASE, QA_Setting, QA_util_date_stamp,
QA_util_date_valid, QA_util_dict_remove_key,
QA_util_log_info, QA_util_code_tostr, QA_util_date_str2int, QA_util_date_int2str,
QA_util_sql_mongo_sort_DESCENDING,
QA_util_time_stamp, QA_util_to_json_from_pandas,
trade_date_sse,QA_util_dateordatetime_valid,QA_util_to_anyformat_from_pandas,
QA_util_get_last_day,QA_util_get_next_day,QA_util_get_real_date)
QA_util_time_stamp('2016-01-01 09:00:00')
QA_util_time_stamp('2016-01-01')
QA_util_time_stamp('2016-01-01 00:00:00')
QA_util_get_real_date(QA_util_get_last_day('2019-07-01'), towards = -1)
QA_util_get_real_date(QA_util_get_next_day('2019-07-01'), towards = 1)