def analysis_day_kline_change_percent(self, temp_kline_tuple, price_avg_pre):
"""
股票日K涨跌幅计算方法
:param temp_kline_tuple: 相邻两个日K数据的列表
:return:
"""
try:
# 需要处理的涨跌幅的交易日，即第二个元组的the_date
the_date = temp_kline_tuple[1][0]
# 前一日收盘价
close1 = temp_kline_tuple[0][1]
# 当前收盘价
close2 = temp_kline_tuple[1][1]
# 前一日交易额
amount1 = temp_kline_tuple[0][2]
# 当前交易额
amount2 = temp_kline_tuple[1][2]
# 前一日交易量
vol1 = temp_kline_tuple[0][3]
# 当前交易量
vol2 = temp_kline_tuple[1][3]
if price_avg_pre is None :
price_avg_pre = temp_kline_tuple[0][4]
# 涨跌幅(百分比)计算:当日(收盘价-前一日收盘价)/前一日收盘价 * 100
close_chg = None
if close1 is not None and close1 != 0:
close_chg = Utils.base_round(Utils.division_zero((close2 - close1), close1) * 100, 2)
else:
close1 = 0
close_chg = 0
amount_chg = None
if amount1 is not None and amount1 != 0:
amount_chg = Utils.base_round(Utils.division_zero((amount2 - amount1), amount1) * 100, 2)
else:
amount1 = 0
amount_chg = 0
vol_chg = None
if vol1 is not None and vol1 != 0:
vol_chg = Utils.base_round(Utils.division_zero((vol2 - vol1), vol1) * 100, 2)
else:
vol1 = 0
vol_chg = 0
price_avg = Utils.base_round_zero(Utils.division_zero(amount2, vol2), 2)
close_price_avg_chg = Utils.base_round_zero(Utils.division_zero(close2 - price_avg, price_avg) * 100, 2)
if price_avg_pre is None :
price_avg_pre = 0
price_avg_chg = Utils.base_round_zero(Utils.division_zero(price_avg - price_avg_pre, price_avg_pre) * 100, 2)
return [the_date, close1, close_chg, amount1, amount_chg, vol1, vol_chg, price_avg, close_price_avg_chg, price_avg_chg]
except Exception:
log_list = [self.now(), self.error(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append(traceback.format_exc())
self.print_log(log_list)
return None
def processing_day_kline_chg_calculate(dict_item1, dict_item2,
dict_pre_data):
"""
计算股票日K涨跌幅指标，即相邻2天的变动幅度
:param dict_item1: 前一天股票数据，字典类型
:param dict_item2: 后一天股票数据，字典类型
:return: 返回后一天股票涨跌幅数据，字典类型
"""
vol1 = dict_item1['vol']
vol2 = dict_item2['vol']
close1 = dict_item1['close']
close2 = dict_item2['close']
amount2 = dict_item2['amount']
vol_chg = Utils.base_round(
Utils.division_zero((vol2 - vol1), vol1) * 100, 2)
close_chg = Utils.base_round(
Utils.division_zero((close2 - close1), close1) * 100, 2)
price_avg = Utils.base_round(Utils.division_zero(amount2, vol2), 2)
price_avg_pre = dict_pre_data['price_avg_pre']
if price_avg_pre is not None:
price_avg_chg = Utils.base_round(
Utils.division_zero(
(price_avg - price_avg_pre), price_avg_pre) * 100, 2)
else:
price_avg_chg = 0
close_price_avg_chg = Utils.base_round_zero(
Utils.division_zero(close2 - price_avg, price_avg) * 100, 2)
open2 = dict_item2['open']
close_open_chg = Utils.base_round_zero(
Utils.division_zero(close2 - open2, open2) * 100, 2)
return {
'vol_chg': vol_chg,
'close_chg': close_chg,
'price_avg': price_avg,
'price_avg_chg': price_avg_chg,
'close_price_avg_chg': close_price_avg_chg,
'close_open_chg': close_open_chg
}
def analysis_columns_day_kline(day_kline, idx):
"""
股票日K数据处理方法
:param day_kline: 日K的DataFrame
:param idx: DataFrame的索引
:return:
"""
try:
the_date = (str(idx))[0:10]
amount = day_kline.at[idx, 'amount']
# amount的类型为numpy.ndarray，是一个多维数组，可能包含多个值，其他的字段也是一样，测试的时候发现有异常抛出
if isinstance(amount, numpy.ndarray) and amount.size > 1:
amount = amount.tolist()[0]
amount = Utils.base_round(amount, 2)
vol = day_kline.at[idx, 'vol']
if isinstance(vol, numpy.ndarray) and vol.size > 1:
vol = vol.tolist()[0]
vol = Utils.base_round(vol, 2)
open = day_kline.at[idx, 'open']
if isinstance(open, numpy.ndarray) and open.size > 1:
open = open.tolist()[0]
open = Utils.base_round(open, 2)
high = day_kline.at[idx, 'high']
if isinstance(high, numpy.ndarray) and high.size > 1:
high = high.tolist()[0]
high = Utils.base_round(high, 2)
low = day_kline.at[idx, 'low']
if isinstance(low, numpy.ndarray) and low.size > 1:
low = low.tolist()[0]
low = Utils.base_round(low, 2)
close = day_kline.at[idx, 'close']
if isinstance(close, numpy.ndarray) and close.size > 1:
close = close.tolist()[0]
close = Utils.base_round(close, 2)
return [the_date, amount, vol, open, high, low, close]
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
line_no = traceback.extract_stack()[-2][1]
error_log_list = ['StockOneStopProcessor']
error_log_list.append(StockOneStopProcessor.get_method_name())
error_log_list.append('line_no')
error_log_list.append(line_no)
error_log_list.append(exc_type)
error_log_list.append(exc_value)
error_log_list.append(exc_traceback)
r.lpush(queue, error_log_list, logging.ERROR)
return None
def analysis_columns_day_kline(self, day_kline, idx):
"""
股票日K数据处理方法
:param day_kline: 日K的DataFrame
:param idx: DataFrame的索引
:return:
"""
try:
the_date = (str(idx))[0:10]
amount = day_kline.at[idx, 'amount']
# amount的类型为numpy.ndarray，是一个多维数组，可能包含多个值，其他的字段也是一样，测试的时候发现有异常抛出
if isinstance(amount, numpy.ndarray) and amount.size > 1:
amount = amount.tolist()[0]
amount = Utils.base_round(amount, 2)
vol = day_kline.at[idx, 'vol']
if isinstance(vol, numpy.ndarray) and vol.size > 1:
vol = vol.tolist()[0]
vol = Utils.base_round(vol, 2)
open = day_kline.at[idx, 'open']
if isinstance(open, numpy.ndarray) and open.size > 1:
open = open.tolist()[0]
open = Utils.base_round(open, 2)
high = day_kline.at[idx, 'high']
if isinstance(high, numpy.ndarray) and high.size > 1:
high = high.tolist()[0]
high = Utils.base_round(high, 2)
low = day_kline.at[idx, 'low']
if isinstance(low, numpy.ndarray) and low.size > 1:
low = low.tolist()[0]
low = Utils.base_round(low, 2)
close = day_kline.at[idx, 'close']
if isinstance(close, numpy.ndarray) and close.size > 1:
close = close.tolist()[0]
close = Utils.base_round(close, 2)
return [the_date, amount, vol, open, high, low, close]
except Exception:
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append(traceback.format_exc())
self.print_log(log_list)
return None
def analysis_average_line_avg(self, ma, temp_line_tuple):
"""
均线数据涨跌幅平均计算方法
:param ma: 均线类型
:param temp_line_tuple: 均线数据ma切片列表
:return:
"""
# temp_line_tuple中的数据为the_date, close_avg_chg, amount_avg_chg,
# vol_avg_chg, price_avg_chg, amount_flow_chg, vol_flow_chg
# 当日the_date为正序排序最后一天的the_date，第一个元素
the_date = temp_line_tuple[ma - 1][0]
# 将元组元素转换为列表元素
# temp_items = [item for item in temp_line_tuple[0:]]
# ma日均收盘价涨跌幅均=sum(前ma日(含)均收盘价涨跌幅)/ma
close_avg_chg_list = [close_avg_chg for close_avg_chg in [item[1] for item in temp_line_tuple]]
close_avg_chg_avg = Utils.base_round(Utils.average(close_avg_chg_list), 2)
# ma日均成交额涨跌幅均=sum(前ma日(含)均成交额涨跌幅)/ma
amount_avg_chg_list = [amount_avg_chg for amount_avg_chg in [item[2] for item in temp_line_tuple]]
amount_avg_chg_avg = Utils.base_round(Utils.average(amount_avg_chg_list), 2)
# ma日均成交量涨跌幅均=sum(前ma日(含)均成交量涨跌幅)/ma
vol_avg_chg_list = [vol_avg_chg for vol_avg_chg in [item[3] for item in temp_line_tuple]]
vol_avg_chg_avg = Utils.base_round(Utils.average(vol_avg_chg_list), 2)
# ma日均成交价涨跌幅均=sum(前ma日(含)均成交价涨跌幅)/ma
price_avg_chg_list = [price_avg_chg for price_avg_chg in [item[4] for item in temp_line_tuple]]
price_avg_chg_avg = Utils.base_round(Utils.average(price_avg_chg_list), 2)
# 日金钱流向涨跌幅均=sum(前ma日(含)金钱流向涨跌幅)/ma
amount_flow_chg_list = [amount_flow_chg for amount_flow_chg in [item[5] for item in temp_line_tuple]]
amount_flow_chg_avg = Utils.base_round(Utils.average(amount_flow_chg_list), 2)
# 日成交量流向涨跌幅均=sum(前ma日(含)成交量流向涨跌幅)/ma
vol_flow_chg_list = [vol_flow_chg for vol_flow_chg in [item[5] for item in temp_line_tuple]]
vol_flow_chg_avg = Utils.base_round(Utils.average(vol_flow_chg_list), 2)
return [the_date,
close_avg_chg_avg, amount_avg_chg_avg, vol_avg_chg_avg,
price_avg_chg_avg, amount_flow_chg_avg, vol_flow_chg_avg]
low_min = None
open_first = None
close_last = None
total_amount = 0
total_vol = 0
idx_datetime = None
date0930 = datetime.datetime.now()
date0930 = date0930.replace(year=2017, month=4, day=7, hour=9, minute=30, second=00, microsecond=0)
print('date0930', date0930)
for idx in indexes_values:
idx_datetime = datetime.datetime.utcfromtimestamp(idx.astype('O')/1e9)
if idx_datetime >= date0930:
amount = day_kline.at[idx, 'amount']
if isinstance(amount, numpy.ndarray) and amount.size > 1:
amount = amount.tolist()[0]
amount = Utils.base_round(amount, 2)
total_amount += amount
vol = day_kline.at[idx, 'vol']
if isinstance(vol, numpy.ndarray) and vol.size > 1:
vol = vol.tolist()[0]
vol = Utils.base_round(vol, 2)
total_vol += Utils.base_round(vol, 2)
high = day_kline.at[idx, 'high']
if isinstance(high, numpy.ndarray) and high.size > 1:
high = high.tolist()[0]
high = Utils.base_round(high, 2)
if high_max is None:
high_max = high
elif high > high_max:
def processing_average_line_avg(ma, security_code, r, queue):
"""
股票均线数据涨跌幅平均数据处理方法
:param ma: 均线类型
:return:
"""
average_line_avg_max_the_date = StockOneStopProcessor.dbService.get_average_line_avg_max_the_date(
ma, security_code)
decline_ma_the_date = StockOneStopProcessor.dbService.get_average_line_avg_decline_max_the_date(
ma, average_line_avg_max_the_date)
r.lpush(queue, [
'StockOneStopProcessor',
StockOneStopProcessor.get_method_name(), 'ma', ma, security_code,
'average_line_avg_max_the_date', average_line_avg_max_the_date,
'decline_ma_the_date', decline_ma_the_date
])
result = StockOneStopProcessor.dbService.get_average_line(
ma, security_code, decline_ma_the_date)
len_result = len(result)
if len_result < ma:
return
try:
if result is not None and len_result > 0:
# 开始解析股票日K数据, the_date, close
# 临时存储批量更新sql的列表
upsert_sql_list = []
# 需要处理的单只股票进度计数
add_up = 0
# 需要处理的单只股票进度打印字符
process_line = '='
# 循环处理security_code的股票日K数据
i = 0
while i < len_result:
add_up += 1
# 如果切片的下标是元祖的最后一个元素，则退出，因为已经处理完毕
if (i + ma) > len_result:
add_up -= 1
break
temp_line_tuple = result[i:(i + ma)]
# 返回值list_data list [the_date,
# close_avg_chg_avg, amount_avg_chg_avg, vol_avg_chg_avg,
# price_avg_chg_avg, amount_flow_chg_avg, vol_flow_chg_avg]
list_data = StockOneStopProcessor.analysis_average_line_avg(
ma, temp_line_tuple)
upsert_sql = StockOneStopProcessor.upsert_average_line_avg.format(
security_code=Utils.quotes_surround(security_code),
the_date=Utils.quotes_surround(
list_data[0].strftime('%Y-%m-%d')),
ma=ma,
close_avg_chg_avg=list_data[1],
amount_avg_chg_avg=list_data[2],
vol_avg_chg_avg=list_data[3],
price_avg_chg_avg=list_data[4],
amount_flow_chg_avg=list_data[5],
vol_flow_chg_avg=list_data[6])
# print(upsert_sql)
# 批量(100)提交数据更新
if len(upsert_sql_list) == 1000:
StockOneStopProcessor.dbService.insert_many(
upsert_sql_list)
process_line += '='
upsert_sql_list = []
upsert_sql_list.append(upsert_sql)
if len_result == ma:
progress = Utils.base_round(1 * 100, 2)
else:
progress = Utils.base_round(
Utils.division_zero(add_up,
(len_result - ma + 1)) *
100, 2)
progress_log_list = ['StockOneStopProcessor']
progress_log_list.append(
StockOneStopProcessor.get_method_name())
progress_log_list.append('ma')
progress_log_list.append(ma)
progress_log_list.append('security_code')
progress_log_list.append(security_code)
progress_log_list.append('progress')
progress_log_list.append(add_up)
progress_log_list.append((len_result - ma + 1))
progress_log_list.append(process_line)
progress_log_list.append(progress)
r.lpush(queue, progress_log_list)
else:
if upsert_sql is not None:
upsert_sql_list.append(upsert_sql)
i += 1
# 处理最后一批security_code的更新语句
if len(upsert_sql_list) > 0:
StockOneStopProcessor.dbService.insert_many(
upsert_sql_list)
process_line += '='
if len_result == ma:
progress = Utils.base_round(1 * 100, 2)
else:
progress = Utils.base_round(
Utils.division_zero(add_up,
(len_result - ma + 1)) * 100, 2)
progress_log_list = ['StockOneStopProcessor']
progress_log_list.append(
StockOneStopProcessor.get_method_name())
progress_log_list.append('ma')
progress_log_list.append(ma)
progress_log_list.append('security_code')
progress_log_list.append(security_code)
progress_log_list.append('progress')
progress_log_list.append(add_up)
progress_log_list.append((len_result - ma + 1))
progress_log_list.append(process_line)
progress_log_list.append(progress)
r.lpush(queue, progress_log_list)
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
line_no = traceback.extract_stack()[-2][1]
error_log_list = ['StockOneStopProcessor']
error_log_list.append(StockOneStopProcessor.get_method_name())
error_log_list.append('ma')
error_log_list.append(ma)
error_log_list.append('security_code')
error_log_list.append(security_code)
error_log_list.append('line_no')
error_log_list.append(line_no)
error_log_list.append(exc_type)
error_log_list.append(exc_value)
error_log_list.append(exc_traceback)
r.lpush(queue, error_log_list, logging.ERROR)
def processing_day_kline(security_code, r, queue):
"""
股票日K数据处理，分全量还是增量
:return:
"""
try:
recent_few_days = StockOneStopProcessor.dbService.get_day_kline_recentdays(
security_code)
r.lpush(queue, [
'StockOneStopProcessor',
StockOneStopProcessor.get_method_name(), security_code,
'recent_few_days', recent_few_days
])
if recent_few_days is not None and recent_few_days > 0:
result = tt.get_last_n_daybar(security_code, recent_few_days,
'qfq')
else:
result = tt.get_all_daybar(security_code, 'qfq')
# r.lpush(queue, ['result', result])
if result.empty == False:
# 索引值为日期
indexes_values = result.index.values
# 临时存储批量更新sql的列表
upsert_sql_list = []
# 需要处理的单只股票进度计数
add_up = 0
# 需要处理的单只股票进度打印字符
process_line = '='
# 循环处理security_code的股票日K数据
if indexes_values is not None:
len_indexes = len(indexes_values)
for idx in indexes_values:
add_up += 1
# 解析股票日K数据（每行）
# 解析每行的返回值格式为list [the_date, amount, vol, open, high, low, close]
list_data = StockOneStopProcessor.analysis_columns_day_kline(
result, idx)
if list_data is not None:
upsert_sql = StockOneStopProcessor.upsert_day_kline.format(
security_code=Utils.quotes_surround(
security_code),
the_date=Utils.quotes_surround(list_data[0]),
amount=list_data[1],
vol=list_data[2],
open=list_data[3],
high=list_data[4],
low=list_data[5],
close=list_data[6])
else:
continue
# 批量(100)提交数据更新
if len(upsert_sql_list) == 3000:
StockOneStopProcessor.dbService.insert_many(
upsert_sql_list)
process_line += '='
upsert_sql_list = []
if upsert_sql is not None:
upsert_sql_list.append(upsert_sql)
progress = Utils.base_round(
Utils.division_zero(add_up, len_indexes) * 100,
2)
progress_log_list = ['StockOneStopProcessor']
progress_log_list.append(
StockOneStopProcessor.get_method_name())
progress_log_list.append('security_code')
progress_log_list.append(security_code)
progress_log_list.append('progress')
progress_log_list.append(add_up)
progress_log_list.append(len_indexes)
progress_log_list.append(process_line)
progress_log_list.append(str(progress) + '%')
r.lpush(queue, progress_log_list)
else:
upsert_sql_list.append(upsert_sql)
# 处理最后一批security_code的更新语句
if len(upsert_sql_list) > 0:
StockOneStopProcessor.dbService.insert_many(
upsert_sql_list)
process_line += '='
progress = Utils.base_round(
Utils.division_zero(add_up, len(indexes_values)) * 100,
2)
progress_log_list = ['StockOneStopProcessor']
progress_log_list.append(
StockOneStopProcessor.get_method_name())
progress_log_list.append('security_code')
progress_log_list.append(security_code)
progress_log_list.append('progress')
progress_log_list.append(add_up)
progress_log_list.append(len_indexes)
progress_log_list.append(process_line)
progress_log_list.append(progress)
r.lpush(queue, progress_log_list)
else:
warn_log_list = ['StockOneStopProcessor']
warn_log_list.append(
StockOneStopProcessor.get_method_name())
warn_log_list.append('security_code')
warn_log_list.append(security_code)
warn_log_list.append('result`s indexes_values is None')
r.lpush(queue, warn_log_list, logging.WARNING)
else:
warn_log_list = ['StockOneStopProcessor']
warn_log_list.append(StockOneStopProcessor.get_method_name())
warn_log_list.append('security_code')
warn_log_list.append(security_code)
warn_log_list.append(
'tt.get_all_daybar or tt.get_last_n_daybar result is None')
r.lpush(queue, warn_log_list, logging.WARNING)
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
line_no = traceback.extract_stack()[-2][1]
error_log_list = ['StockOneStopProcessor']
error_log_list.append(StockOneStopProcessor.get_method_name())
error_log_list.append('security_code')
error_log_list.append(security_code)
error_log_list.append('line_no')
error_log_list.append(line_no)
error_log_list.append(exc_type)
error_log_list.append(exc_value)
error_log_list.append(exc_traceback)
r.lpush(queue, error_log_list, logging.ERROR)
def analysis_day_kline_change_percent(temp_kline_tuple, price_avg_pre):
"""
股票日K涨跌幅计算方法
:param temp_kline_tuple: 相邻两个日K数据的列表
:return:
"""
try:
# 需要处理的涨跌幅的交易日，即第二个元组的the_date
the_date = temp_kline_tuple[1][0]
# 前一日收盘价
close1 = temp_kline_tuple[0][1]
# 当前收盘价
close2 = temp_kline_tuple[1][1]
# 前一日交易额
amount1 = temp_kline_tuple[0][2]
# 当前交易额
amount2 = temp_kline_tuple[1][2]
# 前一日交易量
vol1 = temp_kline_tuple[0][3]
# 当前交易量
vol2 = temp_kline_tuple[1][3]
if price_avg_pre is None:
price_avg_pre = temp_kline_tuple[0][4]
# 涨跌幅(百分比)计算:当日(收盘价-前一日收盘价)/前一日收盘价 * 100
close_chg = None
if close1 is not None and close1 != 0:
close_chg = Utils.base_round(
Utils.division_zero((close2 - close1), close1) * 100, 2)
else:
close1 = 0
close_chg = 0
amount_chg = None
if amount1 is not None and amount1 != 0:
amount_chg = Utils.base_round(
Utils.division_zero((amount2 - amount1), amount1) * 100, 2)
else:
amount1 = 0
amount_chg = 0
vol_chg = None
if vol1 is not None and vol1 != 0:
vol_chg = Utils.base_round(
Utils.division_zero((vol2 - vol1), vol1) * 100, 2)
else:
vol1 = 0
vol_chg = 0
price_avg = Utils.base_round_zero(
Utils.division_zero(amount2, vol2), 2)
close_price_avg_chg = Utils.base_round_zero(
Utils.division_zero(close2 - price_avg, price_avg) * 100, 2)
if price_avg_pre is None:
price_avg_pre = 0
price_avg_chg = Utils.base_round_zero(
Utils.division_zero(price_avg - price_avg_pre, price_avg_pre) *
100, 2)
return [
the_date, close1, close_chg, amount1, amount_chg, vol1,
vol_chg, price_avg, close_price_avg_chg, price_avg_chg
]
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
line_no = traceback.extract_stack()[-2][1]
error_log_list = ['StockOneStopProcessor']
error_log_list.append(StockOneStopProcessor.get_method_name())
error_log_list.append('line_no')
error_log_list.append(line_no)
error_log_list.append(exc_type)
error_log_list.append(exc_value)
error_log_list.append(exc_traceback)
r.lpush(queue, error_log_list, logging.ERROR)
return None
def processing_day_kline_change_percent(security_code, r, queue):
"""
股票日K数据涨跌幅处理方法
:return:
"""
day_kline_max_the_date = StockOneStopProcessor.dbService.get_day_kline_max_the_date(
security_code)
r.lpush(queue, [
'StockOneStopProcessor',
StockOneStopProcessor.get_method_name(), security_code,
'day_kline_max_the_date', day_kline_max_the_date
])
result = StockOneStopProcessor.dbService.get_stock_day_kline(
security_code, day_kline_max_the_date)
len_result = len(result)
# print('result', result)
if len_result == 0:
return
# 临时存储批量更新sql的列表
upsert_sql_list = []
# 需要处理的单只股票进度计数
add_up = 0
# 需要处理的单只股票进度打印字符
process_line = ''
# 循环处理security_code的股票日K数据
i = 0
price_avg_pre = None
while i < len_result:
# 切片元组，每相连的2个一组
section_idx = i + 2
if section_idx > len_result:
i += 1
break
temp_kline_tuple = result[i:section_idx]
# 返回值格式list [the_date, close1, close_chg, amount1, amount_chg, vol1, vol_chg,
# price_avg, close_price_avg_chg, price_avg_chg]
list_data = StockOneStopProcessor.analysis_day_kline_change_percent(
temp_kline_tuple, price_avg_pre)
# print(temp_kline_tuple)
# print(price_avg_pre)
if list_data is not None:
"""
日K涨跌幅数据入库
"""
upsert_sql = 'insert into tquant_stock_day_kline (security_code, the_date, ' \
'close_pre, close_chg, ' \
'amount_pre, amount_chg, ' \
'vol_pre, vol_chg, ' \
'price_avg, close_price_avg_chg, price_avg_chg) ' \
'values ({security_code}, {the_date}, ' \
'{close_pre}, {close_chg},' \
'{amount_pre}, {amount_chg}, {vol_pre}, {vol_chg}, ' \
'{price_avg}, {close_price_avg_chg}, {price_avg_chg}) ' \
'on duplicate key update ' \
'close_pre=values(close_pre), close_chg=values(close_chg), ' \
'amount_pre=values(amount_pre), amount_chg=values(amount_chg), ' \
'vol_pre=values(vol_pre), vol_chg=values(vol_chg), ' \
'price_avg=values(price_avg), close_price_avg_chg=values(close_price_avg_chg), price_avg_chg=values(price_avg_chg)'
upsert_sql = upsert_sql.format(
security_code=Utils.quotes_surround(security_code),
the_date=Utils.quotes_surround(
list_data[0].strftime('%Y-%m-%d')),
close_pre=list_data[1],
close_chg=list_data[2],
amount_pre=list_data[3],
amount_chg=list_data[4],
vol_pre=list_data[5],
vol_chg=list_data[6],
price_avg=list_data[7],
close_price_avg_chg=list_data[8],
price_avg_chg=list_data[9])
price_avg_pre = list_data[7]
else:
upsert_sql = None
# 批量(100)提交数据更新
if len(upsert_sql_list) == 1000:
StockOneStopProcessor.dbService.insert_many(upsert_sql_list)
process_line += '='
upsert_sql_list = []
if upsert_sql is not None:
upsert_sql_list.append(upsert_sql)
# 这个地方为什么要add_up + 1？因为第一条数据不会被处理，所以总数会少一条，所以在计算进度的时候要+1
progress = Utils.base_round(
Utils.division_zero((add_up + 1), len_result) * 100, 2)
progress_log_list = ['StockOneStopProcessor']
progress_log_list.append(
StockOneStopProcessor.get_method_name())
progress_log_list.append('security_code')
progress_log_list.append(security_code)
progress_log_list.append('progress')
progress_log_list.append(add_up + 1)
progress_log_list.append(len_result)
progress_log_list.append(process_line)
progress_log_list.append(progress)
r.lpush(queue, progress_log_list)
else:
if upsert_sql is not None:
upsert_sql_list.append(upsert_sql)
i += 1
add_up += 1
# 处理最后一批security_code的更新语句
if len(upsert_sql_list) > 0:
StockOneStopProcessor.dbService.insert_many(upsert_sql_list)
process_line += '='
progress = Utils.base_round(
Utils.division_zero((add_up + 1), len_result) * 100, 2)
progress_log_list = ['StockOneStopProcessor']
progress_log_list.append(StockOneStopProcessor.get_method_name())
progress_log_list.append('security_code')
progress_log_list.append(security_code)
progress_log_list.append('progress')
progress_log_list.append(add_up + 1)
progress_log_list.append(len_result)
progress_log_list.append(process_line)
progress_log_list.append(progress)
r.lpush(queue, progress_log_list)
def analysis_real_time_kline(day_kline, start_date, security_code, r,
queue):
"""
解析单只股票的实时行情，并入库
:param day_kline:
:param start_date:
:return:
"""
if day_kline.empty == False:
indexes_values = day_kline.index.values
the_date = None
high_max = None
low_min = None
open_first = None
close_last = None
total_amount = 0
total_vol = 0
if indexes_values is None or len(indexes_values) == 0:
warn_log_list = ['StockOneStopProcessor']
warn_log_list.append(StockOneStopProcessor.get_method_name())
warn_log_list.append('security_code')
warn_log_list.append(security_code)
warn_log_list.append('start_date')
warn_log_list.append(start_date)
warn_log_list.append('tt.get_stock_bar(security_code, 1)')
warn_log_list.append('indexes_values is None')
r.lpush(queue, warn_log_list, logging.WARNING)
return
for idx in indexes_values:
idx_datetime = datetime.datetime.utcfromtimestamp(
idx.astype('O') / 1e9)
if idx_datetime >= start_date:
amount = day_kline.at[idx, 'amount']
if isinstance(amount, numpy.ndarray) and amount.size > 1:
amount = amount.tolist()[0]
amount = Utils.base_round(amount, 2)
total_amount += amount
vol = day_kline.at[idx, 'vol']
if isinstance(vol, numpy.ndarray) and vol.size > 1:
vol = vol.tolist()[0]
vol = Utils.base_round(vol, 2)
total_vol += Utils.base_round(vol, 2)
high = day_kline.at[idx, 'high']
if isinstance(high, numpy.ndarray) and high.size > 1:
high = high.tolist()[0]
high = Utils.base_round(high, 2)
if high_max is None:
high_max = high
elif high > high_max:
high_max = high
low = day_kline.at[idx, 'low']
if isinstance(low, numpy.ndarray) and low.size > 1:
low = low.tolist()[0]
low = Utils.base_round(low, 2)
if low_min is None:
low_min = low
elif low < low_min:
low_min = low
open = day_kline.at[idx, 'open']
if isinstance(open, numpy.ndarray) and open.size > 1:
open = open.tolist()[0]
open = Utils.base_round(open, 2)
if open_first is None:
open_first = open
close = day_kline.at[idx, 'close']
if isinstance(close, numpy.ndarray) and close.size > 1:
close = close.tolist()[0]
close = Utils.base_round(close, 2)
close_last = close
the_date = (idx_datetime.strftime('%Y-%m-%d'))
if the_date is not None:
total_amount = total_amount * 100
total_vol = total_vol * 100
upsert_sql = StockOneStopProcessor.upsert_day_kline.format(
security_code=Utils.quotes_surround(security_code),
the_date=Utils.quotes_surround(the_date),
amount=total_amount,
vol=total_vol,
open=open_first,
high=high_max,
low=low_min,
close=close_last)
StockOneStopProcessor.dbService.insert(upsert_sql)
def processing_day_kline(self):
"""
股票日K数据处理，分全量还是增量
:return:
"""
try:
recent_few_days = self.dbService.get_day_kline_recentdays(self.security_code)
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append('recent_few_days')
log_list.append(recent_few_days)
self.print_log(log_list)
if recent_few_days is None:
result = tt.get_all_daybar(self.security_code, 'qfq')
elif recent_few_days > 0:
result = tt.get_last_n_daybar(self.security_code, recent_few_days, 'qfq')
else:
return
if result.empty == False:
# 索引值为日期
indexes_values = result.index.values
# 临时存储批量更新sql的列表
upsert_sql_list = []
# 需要处理的单只股票进度计数
add_up = 0
# 需要处理的单只股票进度打印字符
process_line = '='
# 循环处理security_code的股票日K数据
if indexes_values is not None:
len_indexes = len(indexes_values)
for idx in indexes_values:
add_up += 1
# 解析股票日K数据（每行）
# 解析每行的返回值格式为list [the_date, amount, vol, open, high, low, close]
list_data = self.analysis_columns_day_kline(result, idx)
if list_data is not None:
upsert_sql = self.upsert_day_kline.format(
security_code=Utils.quotes_surround(self.security_code),
the_date=Utils.quotes_surround(list_data[0]),
amount=list_data[1],
vol=list_data[2],
open=list_data[3],
high=list_data[4],
low=list_data[5],
close=list_data[6]
)
else:
continue
# 批量(100)提交数据更新
if len(upsert_sql_list) == 200:
self.dbService.insert_many(upsert_sql_list)
process_line += '='
upsert_sql_list = []
if upsert_sql is not None:
upsert_sql_list.append(upsert_sql)
progress = Utils.base_round(Utils.division_zero(add_up, len_indexes) * 100, 2)
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append('progress')
log_list.append(add_up)
log_list.append(len_indexes)
log_list.append(process_line)
log_list.append(str(progress) + '%')
self.print_log(log_list)
else:
upsert_sql_list.append(upsert_sql)
# 处理最后一批security_code的更新语句
if len(upsert_sql_list) > 0:
self.dbService.insert_many(upsert_sql_list)
process_line += '='
progress = Utils.base_round(Utils.division_zero(add_up, len(indexes_values)) * 100, 2)
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append('progress')
log_list.append(add_up)
log_list.append(len_indexes)
log_list.append(process_line)
log_list.append(progress)
self.print_log(log_list)
else:
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append('result`s indexes_values is None')
self.print_log(log_list)
else:
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append('tt.get_all_daybar or tt.get_last_n_daybar result is None')
self.print_log(log_list)
except Exception:
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append(traceback.format_exc())
self.print_log(log_list)
def processing_average_line_avg(self, ma):
"""
股票均线数据涨跌幅平均数据处理方法
:param ma: 均线类型
:return:
"""
average_line_avg_max_the_date = self.dbService.get_average_line_avg_max_the_date(ma, self.security_code)
decline_ma_the_date = self.dbService.get_average_line_avg_decline_max_the_date(ma, average_line_avg_max_the_date, self.security_code)
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append('ma')
log_list.append(ma)
log_list.append('average_line_avg_max_the_date')
log_list.append(average_line_avg_max_the_date)
log_list.append('decline_ma_the_date')
log_list.append(decline_ma_the_date)
self.print_log(log_list)
result = self.dbService.get_average_line(ma, self.security_code, decline_ma_the_date)
len_result = len(result)
if len_result < ma:
return
try:
if result is not None and len_result > 0:
# 开始解析股票日K数据, the_date, close
# 临时存储批量更新sql的列表
upsert_sql_list = []
# 需要处理的单只股票进度计数
add_up = 0
# 需要处理的单只股票进度打印字符
process_line = '='
# 循环处理security_code的股票日K数据
i = 0
while i < len_result:
add_up += 1
# 如果切片的下标是元祖的最后一个元素，则退出，因为已经处理完毕
if (i + ma) > len_result:
add_up -= 1
break
temp_line_tuple = result[i:(i + ma)]
# 返回值list_data list [the_date,
# close_avg_chg_avg, amount_avg_chg_avg, vol_avg_chg_avg,
# price_avg_chg_avg, amount_flow_chg_avg, vol_flow_chg_avg]
list_data = self.analysis_average_line_avg(ma, temp_line_tuple)
upsert_sql = self.upsert_average_line_avg.format(security_code=Utils.quotes_surround(self.security_code),
the_date=Utils.quotes_surround(list_data[0].strftime('%Y-%m-%d')),
ma=ma,
close_avg_chg_avg=list_data[1],
amount_avg_chg_avg=list_data[2],
vol_avg_chg_avg=list_data[3],
price_avg_chg_avg=list_data[4],
amount_flow_chg_avg=list_data[5],
vol_flow_chg_avg=list_data[6]
)
# 批量(100)提交数据更新
if len(upsert_sql_list) == 200:
self.dbService.insert_many(upsert_sql_list)
process_line += '='
upsert_sql_list = []
upsert_sql_list.append(upsert_sql)
if len_result == ma:
progress = Utils.base_round(1 * 100, 2)
else:
progress = Utils.base_round(Utils.division_zero(add_up, (len_result - ma + 1)) * 100, 2)
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append('ma')
log_list.append(ma)
log_list.append('progress')
log_list.append(add_up)
log_list.append((len_result - ma + 1))
log_list.append(process_line)
log_list.append(progress)
self.print_log(log_list)
else:
if upsert_sql is not None:
upsert_sql_list.append(upsert_sql)
i += 1
# 处理最后一批security_code的更新语句
if len(upsert_sql_list) > 0:
self.dbService.insert_many(upsert_sql_list)
process_line += '='
if len_result == ma:
progress = Utils.base_round(1 * 100, 2)
else:
progress = Utils.base_round(Utils.division_zero(add_up, (len_result - ma + 1)) * 100, 2)
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append('ma')
log_list.append(ma)
log_list.append('progress')
log_list.append(add_up)
log_list.append((len_result - ma + 1))
log_list.append(process_line)
log_list.append(progress)
self.print_log(log_list)
except Exception:
log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code]
log_list.append(self.get_method_name())
log_list.append('ma')
log_list.append(ma)
log_list.append(traceback.format_exc())
self.print_log(log_list)
def processing_day_kline(self):
"""
股票日K数据处理，分全量还是增量
:return:
"""
try:
if self.is_reset:
max_the_date = datetime.datetime.now().replace(year=1970,
month=1,
day=1)
else:
max_the_date = self.get_max_the_date()
diff_days = Utils.diff_days(max_the_date)
log_list = [
Utils.get_now(),
Utils.get_info(),
self.get_classs_name(), self.security_code,
self.get_method_name(), '最大交易日', max_the_date, '距今',
Utils.format_date(datetime.date.today()), '差', diff_days, '天'
]
Utils.print_log(log_list)
dict_data_pre = None
if diff_days is None:
result = tt.get_all_daybar(self.security_code, 'bfq')
else:
result = tt.get_last_n_daybar(self.security_code, diff_days,
'bfq')
if result.empty == False:
# 按照正序排序，时间小的排前面
result.sort_index(ascending=True)
# 需要处理的单只股票进度计数
add_up = 0
# 需要处理的单只股票进度打印字符
process_line = '='
len_indexes = len(result.index.values)
for i in range(len_indexes):
add_up += 1
if i < 9:
continue
end = i + 1
start = i - 9
section = result.iloc[start:end, :]
dict_data = self.processing_section(section, dict_data_pre)
dict_data_pre = dict_data
# 批量打印日志
if add_up % 200 == 0:
process_line += '='
progress = Utils.base_round(
Utils.division_zero(add_up, len_indexes) * 100, 2)
log_list = [
Utils.get_now(),
Utils.get_info(),
self.get_classs_name(), self.security_code,
self.get_method_name(), '处理进度', add_up,
len_indexes, process_line,
str(progress) + '%'
]
Utils.print_log(log_list)
process_line += '='
progress = Utils.base_round(
Utils.division_zero(add_up, len_indexes) * 100, 2)
log_list = [
Utils.get_now(),
Utils.get_info(),
self.get_classs_name(), self.security_code,
self.get_method_name(), '处理进度', add_up, len_indexes,
process_line,
str(progress) + '%'
]
Utils.print_log(log_list)
# self.dbService.upsert_many(list_db_data, 'tquant_stock_history_quotation', ['security_code', 'the_date'])
else:
log_list = [
Utils.get_now(),
Utils.get_warn(),
self.get_classs_name(), self.security_code,
self.get_method_name(), '【日K DataFrame为空】'
]
Utils.print_log(log_list)
except Exception:
traceback.print_exc()
def processing_1_day_chg(self, section_tail1, idx, dict_data,
dict_data_pre):
try:
amount = section_tail1.at[idx, 'amount']
# amount的类型为numpy.ndarray，是一个多维数组，可能包含多个值，其他的字段也是一样，测试的时候发现有异常抛出
if isinstance(amount, numpy.ndarray) and amount.size > 1:
amount = amount.tolist()[0]
amount = Utils.base_round(amount, 2)
amount_pre = dict_data_pre['amount']
amount_chg = Utils.base_round_zero(
Utils.division_zero(amount - amount_pre, amount_pre) * 100, 2)
dict_data['amount'] = amount
dict_data['amount_chg'] = amount_chg
vol = section_tail1.at[idx, 'vol']
if isinstance(vol, numpy.ndarray) and vol.size > 1:
vol = vol.tolist()[0]
vol = Utils.base_round(vol, 2)
vol_pre = dict_data_pre['vol']
vol_chg = Utils.base_round_zero(
Utils.division_zero(vol - vol_pre, vol_pre) * 100, 2)
dict_data['vol'] = vol
dict_data['vol_chg'] = vol_chg
open = section_tail1.at[idx, 'open']
if isinstance(open, numpy.ndarray) and open.size > 1:
open = open.tolist()[0]
open = Utils.base_round(open, 2)
open_pre = dict_data_pre['open']
open_chg = Utils.base_round_zero(
Utils.division_zero(open - open_pre, open_pre) * 100, 2)
dict_data['open'] = open
dict_data['open_chg'] = open_chg
high = section_tail1.at[idx, 'high']
if isinstance(high, numpy.ndarray) and high.size > 1:
high = high.tolist()[0]
high = Utils.base_round(high, 2)
high_pre = dict_data_pre['high']
high_chg = Utils.base_round_zero(
Utils.division_zero(high - high_pre, high_pre) * 100, 2)
dict_data['high'] = high
dict_data['high_chg'] = high_chg
low = section_tail1.at[idx, 'low']
if isinstance(low, numpy.ndarray) and low.size > 1:
low = low.tolist()[0]
low = Utils.base_round(low, 2)
low_pre = dict_data_pre['low']
low_chg = Utils.base_round_zero(
Utils.division_zero(low - low_pre, low_pre) * 100, 2)
dict_data['low'] = low
dict_data['low_chg'] = low_chg
close = section_tail1.at[idx, 'close']
if isinstance(close, numpy.ndarray) and close.size > 1:
close = close.tolist()[0]
close = Utils.base_round(close, 2)
close_pre = dict_data_pre['close']
close_chg = Utils.base_round_zero(
Utils.division_zero(close - close_pre, close_pre) * 100, 2)
dict_data['close'] = close
dict_data['close_chg'] = close_chg
close_open_chg = Utils.base_round_zero(
Utils.division_zero(close - open, open) * 100, 2)
dict_data['close_open_chg'] = close_open_chg
except Exception:
traceback.print_exc()
