def saving_work(code, coll_stock_day):
try:
print("entering saving_work {}".format(code))
ref = coll_stock_day.find({'code': str(code)[:6]})
end_date = str(now_time())[:10]
if ref.count() > 0:
start_date = ref[ref.count() - 1]['date']
QA_util_log_info(
'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.
format(code, start_date, end_date))
if start_date != end_date:
start_date = QA_util_get_next_day(start_date)
data_json = QA_fetch_get_stock_day(str(code),
start=start_date,
end=end_date,
type_='json')
coll_stock_day.insert_many(data_json)
else:
start_date = '1990-01-01'
data_json = QA_fetch_get_stock_day(str(code),
start=start_date,
type_='json')
coll_stock_day.insert_many(data_json)
print("done saving_work {}".format(code))
except Exception as e:
QA_util_log_info('error in saving ==== %s' % str(code))
print("The exception is {}".format(str(e)))
def saving_work(i):
QA_util_log_info('Now Saving ==== %s' % (i))
try:
df = QA_fetch_get_stock_day(i, start='1990-01-01', if_fq='bfq')
__coll.insert_many(json.loads(df.to_json(orient='records')))
except Exception as e:
print(e)
QA_util_log_info('error in saving ==== %s' % str(i))
def saving_work(i):
QA_util_log_info('Now Saving ==== %s' % (i))
try:
df = QA_fetch_get_stock_day(i, start='1990-01-01', if_fq='bfq')
__coll.insert_many(json.loads(df.to_json(orient='records')))
except Exception as e:
print(e)
QA_util_log_info('error in saving ==== %s' % str(i))
def _saving_work(code, coll_stock_day, ui_log=None, err=[]):
try:
QA_util_log_info(
'##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)),
ui_log)
# 首选查找数据库 是否 有 这个代码的数据
ref = coll_stock_day.find({'ts_code': str(code)},
sort=[('trade_date', 1)])
end_date = now_time()
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date_new_format = ref[ref.count() - 1]['trade_date']
start_date = ref[ref.count() - 1]['date']
if date_conver_to_new_format(
QA_util_get_next_day(start_date, n=100)) < end_date:
QA_util_log_info(
'LAST STOCK_DAY 100 days ago {}, skip. To resume please delete {}'
.format(start_date_new_format, code), ui_log)
else:
QA_util_log_info(
'UPDATE_BOND_DAY \n Trying updating {} from {} to {}'.
format(code, start_date_new_format, end_date), ui_log)
if start_date_new_format != end_date:
pandas_data = QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(
str(code),
date_conver_to_new_format(
QA_util_get_next_day(start_date)),
end_date, 'bfq'))
if len(pandas_data) > 0:
coll_stock_day.insert_many(pandas_data)
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
else:
start_date = '19900101'
QA_util_log_info(
'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.
format(code, start_date, end_date), ui_log)
if start_date != end_date:
pandas_data = QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date, end_date,
'bfq'))
if len(pandas_data) > 0:
coll_stock_day.insert_many(pandas_data)
except Exception as e:
print(e)
err.append(str(code))
def saving_work(i):
QA_util_log_info('Now Saving ==== %s' % (i))
try:
data_json = QA_fetch_get_stock_day(i, startDate='1990-01-01')
__coll.insert_many(data_json)
except:
QA_util_log_info('error in saving ==== %s' % str(i))
def saving_work(i):
QA_util_log_info('Now Saving ==== %s' % (i))
try:
data_hfq = QA_fetch_get_stock_day(
i, start='1990-01-01', if_fq='02', type_='pd')
data_json = QA_util_to_json_from_pandas(data_hfq)
__coll.insert_many(data_json)
except:
QA_util_log_info('error in saving ==== %s' % str(i))
def _saving_work(code, coll_stock_day, ui_log=None, err=[]):
try:
QA_util_log_info(
'##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)), ui_log)
# 首选查找数据库 是否 有 这个代码的数据
ref = coll_stock_day.find({'code': str(code)[0:9]})
end_date = now_time()
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref.count() > 0:
# 接着上次获取的日期继续更新
start_date_new_format = ref[ref.count() - 1]['trade_date']
start_date = ref[ref.count() - 1]['date']
QA_util_log_info(
'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.format(
code, start_date, end_date), ui_log)
if start_date_new_format != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(
str(code),
date_conver_to_new_format(
QA_util_get_next_day(start_date)), end_date,
'qfq')))
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
else:
start_date = '19900101'
QA_util_log_info(
'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.format(
code, start_date, end_date), ui_log)
if start_date != end_date:
coll_stock_day.insert_many(
QA_util_to_json_from_pandas(
QA_fetch_get_stock_day(str(code), start_date, end_date,
'qfq')))
except Exception as e:
print(e)
err.append(str(code))
def __saving_work(self, code):
try:
QA_util_log_info(
'##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)),
self.ui_log)
time.sleep(0.2)
# 首选查找数据库 是否 有 这个代码的数据
search_cond = {'code': str(code)[0:6]}
ref = get_coll(self.client).find(search_cond)
ref_count = get_coll(self.client).count_documents(search_cond)
end_date = now_time()
# 当前数据库已经包含了这个代码的数据， 继续增量更新
# 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现
if ref_count > 0:
# 接着上次获取的日期继续更新
start_date_new_format = ref[ref_count - 1]['trade_date']
start_date = ref[ref_count - 1]['date']
QA_util_log_info(
'UPDATE_STOCK_DAY: Trying updating {} {} from {} to {}'.
format(ref_count, code, start_date_new_format,
end_date), self.ui_log)
if start_date_new_format != end_date:
df = QA_fetch_get_stock_day(
str(code),
date_conver_to_new_format(
QA_util_get_next_day(start_date)), end_date, 'bfq')
if not (df is None) and len(df) > 0:
get_coll(self.client).insert_many(
QA_util_to_json_from_pandas(df))
else:
QA_util_log_info(
'UPDATE_STOCK_DAY: Trying updating {} {} from {} to {} ---- No Data'
.format(ref_count, code, start_date_new_format,
end_date), self.ui_log)
# 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据
# 一次只返回 5000 条，所以有些股票得获取两次
else:
start_date = '19900101'
QA_util_log_info(
'FETCH_ALL_STOCK_DAY: Trying fetching all data {} from {} to {}'
.format(code, start_date, end_date), self.ui_log)
if start_date != end_date:
# 第一次获取
df = QA_fetch_get_stock_day(str(code), start_date,
end_date, 'bfq')
if not (df is None):
if len(df) >= 5000:
again_date = QA_util_get_last_day(df['date'].min())
QA_util_log_info(
'FETCH_ALL_STOCK_DAY: Trying updating again {} from {} to {}'
.format(code, start_date,
again_date), self.ui_log)
df2 = QA_fetch_get_stock_day(
str(code), start_date, again_date, 'bfq')
df = df.append(df2)
get_coll(self.client).insert_many(
QA_util_to_json_from_pandas(df))
else:
QA_util_log_info(
'FETCH_ALL_STOCK_DAY: Trying fetching all data {} from {} to {} ---- No Data'
.format(code, start_date, end_date), self.ui_log)
except Exception as e:
print(e)
self.err.append(str(code))