def processing_avg(self, section_tail, days, dict_data, dict_data_pre): """ section_tail.describe()结果示例 <class 'pandas.core.frame.DataFrame'> amount close code high low open vol count 6.067000e+03 6067.000000 6067 6067.000000 6067.000000 6067.000000 6.067000e+03 max 8.596942e+09 14.280000 1 14.460000 13.870000 14.460000 5.086050e+08 :param section_tail: :param days: :param dict_data: :param dict_data_pre: :return: """ try: section_sum = section_tail.sum() amount_count = section_sum['amount'] vol_count = section_sum['vol'] price_avg_ = "price_avg_" _chg = "_chg" _chg_diff = "_chg_diff" close_price_avg_ = "close_price_avg_" days = str(days) price_avg = Utils.base_round_zero( Utils.division_zero(amount_count, vol_count), 2) dict_data[price_avg_ + days] = price_avg price_avg_pre = dict_data_pre[price_avg_ + days] price_avg_chg = Utils.base_round_zero( Utils.division_zero(price_avg - price_avg_pre, price_avg_pre) * 100, 2) dict_data[price_avg_ + days + _chg] = price_avg_chg price_avg_chg_pre = dict_data_pre[price_avg_ + days + _chg] price_avg_chg_diff = price_avg_chg - price_avg_chg_pre dict_data[price_avg_ + days + _chg_diff] = price_avg_chg_diff close = dict_data['close'] close_price_avg_chg = Utils.base_round_zero( Utils.division_zero(close - price_avg, price_avg) * 100, 2) dict_data[close_price_avg_ + days + _chg] = close_price_avg_chg except Exception: traceback.print_exc()
def analysis_day_kline_change_percent(self, temp_kline_tuple, price_avg_pre): """ 股票日K涨跌幅计算方法 :param temp_kline_tuple: 相邻两个日K数据的列表 :return: """ try: # 需要处理的涨跌幅的交易日,即第二个元组的the_date the_date = temp_kline_tuple[1][0] # 前一日收盘价 close1 = temp_kline_tuple[0][1] # 当前收盘价 close2 = temp_kline_tuple[1][1] # 前一日交易额 amount1 = temp_kline_tuple[0][2] # 当前交易额 amount2 = temp_kline_tuple[1][2] # 前一日交易量 vol1 = temp_kline_tuple[0][3] # 当前交易量 vol2 = temp_kline_tuple[1][3] if price_avg_pre is None : price_avg_pre = temp_kline_tuple[0][4] # 涨跌幅(百分比)计算:当日(收盘价-前一日收盘价)/前一日收盘价 * 100 close_chg = None if close1 is not None and close1 != 0: close_chg = Utils.base_round(Utils.division_zero((close2 - close1), close1) * 100, 2) else: close1 = 0 close_chg = 0 amount_chg = None if amount1 is not None and amount1 != 0: amount_chg = Utils.base_round(Utils.division_zero((amount2 - amount1), amount1) * 100, 2) else: amount1 = 0 amount_chg = 0 vol_chg = None if vol1 is not None and vol1 != 0: vol_chg = Utils.base_round(Utils.division_zero((vol2 - vol1), vol1) * 100, 2) else: vol1 = 0 vol_chg = 0 price_avg = Utils.base_round_zero(Utils.division_zero(amount2, vol2), 2) close_price_avg_chg = Utils.base_round_zero(Utils.division_zero(close2 - price_avg, price_avg) * 100, 2) if price_avg_pre is None : price_avg_pre = 0 price_avg_chg = Utils.base_round_zero(Utils.division_zero(price_avg - price_avg_pre, price_avg_pre) * 100, 2) return [the_date, close1, close_chg, amount1, amount_chg, vol1, vol_chg, price_avg, close_price_avg_chg, price_avg_chg] except Exception: log_list = [self.now(), self.error(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append(traceback.format_exc()) self.print_log(log_list) return None
def processing_day_kline_chg_calculate(dict_item1, dict_item2, dict_pre_data): """ 计算股票日K涨跌幅指标,即相邻2天的变动幅度 :param dict_item1: 前一天股票数据,字典类型 :param dict_item2: 后一天股票数据,字典类型 :return: 返回后一天股票涨跌幅数据,字典类型 """ vol1 = dict_item1['vol'] vol2 = dict_item2['vol'] close1 = dict_item1['close'] close2 = dict_item2['close'] amount2 = dict_item2['amount'] vol_chg = Utils.base_round( Utils.division_zero((vol2 - vol1), vol1) * 100, 2) close_chg = Utils.base_round( Utils.division_zero((close2 - close1), close1) * 100, 2) price_avg = Utils.base_round(Utils.division_zero(amount2, vol2), 2) price_avg_pre = dict_pre_data['price_avg_pre'] if price_avg_pre is not None: price_avg_chg = Utils.base_round( Utils.division_zero( (price_avg - price_avg_pre), price_avg_pre) * 100, 2) else: price_avg_chg = 0 close_price_avg_chg = Utils.base_round_zero( Utils.division_zero(close2 - price_avg, price_avg) * 100, 2) open2 = dict_item2['open'] close_open_chg = Utils.base_round_zero( Utils.division_zero(close2 - open2, open2) * 100, 2) return { 'vol_chg': vol_chg, 'close_chg': close_chg, 'price_avg': price_avg, 'price_avg_chg': price_avg_chg, 'close_price_avg_chg': close_price_avg_chg, 'close_open_chg': close_open_chg }
def processing_average_line_avg(ma, security_code, r, queue): """ 股票均线数据涨跌幅平均数据处理方法 :param ma: 均线类型 :return: """ average_line_avg_max_the_date = StockOneStopProcessor.dbService.get_average_line_avg_max_the_date( ma, security_code) decline_ma_the_date = StockOneStopProcessor.dbService.get_average_line_avg_decline_max_the_date( ma, average_line_avg_max_the_date) r.lpush(queue, [ 'StockOneStopProcessor', StockOneStopProcessor.get_method_name(), 'ma', ma, security_code, 'average_line_avg_max_the_date', average_line_avg_max_the_date, 'decline_ma_the_date', decline_ma_the_date ]) result = StockOneStopProcessor.dbService.get_average_line( ma, security_code, decline_ma_the_date) len_result = len(result) if len_result < ma: return try: if result is not None and len_result > 0: # 开始解析股票日K数据, the_date, close # 临时存储批量更新sql的列表 upsert_sql_list = [] # 需要处理的单只股票进度计数 add_up = 0 # 需要处理的单只股票进度打印字符 process_line = '=' # 循环处理security_code的股票日K数据 i = 0 while i < len_result: add_up += 1 # 如果切片的下标是元祖的最后一个元素,则退出,因为已经处理完毕 if (i + ma) > len_result: add_up -= 1 break temp_line_tuple = result[i:(i + ma)] # 返回值list_data list [the_date, # close_avg_chg_avg, amount_avg_chg_avg, vol_avg_chg_avg, # price_avg_chg_avg, amount_flow_chg_avg, vol_flow_chg_avg] list_data = StockOneStopProcessor.analysis_average_line_avg( ma, temp_line_tuple) upsert_sql = StockOneStopProcessor.upsert_average_line_avg.format( security_code=Utils.quotes_surround(security_code), the_date=Utils.quotes_surround( list_data[0].strftime('%Y-%m-%d')), ma=ma, close_avg_chg_avg=list_data[1], amount_avg_chg_avg=list_data[2], vol_avg_chg_avg=list_data[3], price_avg_chg_avg=list_data[4], amount_flow_chg_avg=list_data[5], vol_flow_chg_avg=list_data[6]) # print(upsert_sql) # 批量(100)提交数据更新 if len(upsert_sql_list) == 1000: StockOneStopProcessor.dbService.insert_many( upsert_sql_list) process_line += '=' upsert_sql_list = [] upsert_sql_list.append(upsert_sql) if len_result == ma: progress = Utils.base_round(1 * 100, 2) else: progress = Utils.base_round( Utils.division_zero(add_up, (len_result - ma + 1)) * 100, 2) progress_log_list = ['StockOneStopProcessor'] progress_log_list.append( StockOneStopProcessor.get_method_name()) progress_log_list.append('ma') progress_log_list.append(ma) progress_log_list.append('security_code') progress_log_list.append(security_code) progress_log_list.append('progress') progress_log_list.append(add_up) progress_log_list.append((len_result - ma + 1)) progress_log_list.append(process_line) progress_log_list.append(progress) r.lpush(queue, progress_log_list) else: if upsert_sql is not None: upsert_sql_list.append(upsert_sql) i += 1 # 处理最后一批security_code的更新语句 if len(upsert_sql_list) > 0: StockOneStopProcessor.dbService.insert_many( upsert_sql_list) process_line += '=' if len_result == ma: progress = Utils.base_round(1 * 100, 2) else: progress = Utils.base_round( Utils.division_zero(add_up, (len_result - ma + 1)) * 100, 2) progress_log_list = ['StockOneStopProcessor'] progress_log_list.append( StockOneStopProcessor.get_method_name()) progress_log_list.append('ma') progress_log_list.append(ma) progress_log_list.append('security_code') progress_log_list.append(security_code) progress_log_list.append('progress') progress_log_list.append(add_up) progress_log_list.append((len_result - ma + 1)) progress_log_list.append(process_line) progress_log_list.append(progress) r.lpush(queue, progress_log_list) except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() line_no = traceback.extract_stack()[-2][1] error_log_list = ['StockOneStopProcessor'] error_log_list.append(StockOneStopProcessor.get_method_name()) error_log_list.append('ma') error_log_list.append(ma) error_log_list.append('security_code') error_log_list.append(security_code) error_log_list.append('line_no') error_log_list.append(line_no) error_log_list.append(exc_type) error_log_list.append(exc_value) error_log_list.append(exc_traceback) r.lpush(queue, error_log_list, logging.ERROR)
def processing_day_kline(security_code, r, queue): """ 股票日K数据处理,分全量还是增量 :return: """ try: recent_few_days = StockOneStopProcessor.dbService.get_day_kline_recentdays( security_code) r.lpush(queue, [ 'StockOneStopProcessor', StockOneStopProcessor.get_method_name(), security_code, 'recent_few_days', recent_few_days ]) if recent_few_days is not None and recent_few_days > 0: result = tt.get_last_n_daybar(security_code, recent_few_days, 'qfq') else: result = tt.get_all_daybar(security_code, 'qfq') # r.lpush(queue, ['result', result]) if result.empty == False: # 索引值为日期 indexes_values = result.index.values # 临时存储批量更新sql的列表 upsert_sql_list = [] # 需要处理的单只股票进度计数 add_up = 0 # 需要处理的单只股票进度打印字符 process_line = '=' # 循环处理security_code的股票日K数据 if indexes_values is not None: len_indexes = len(indexes_values) for idx in indexes_values: add_up += 1 # 解析股票日K数据(每行) # 解析每行的返回值格式为list [the_date, amount, vol, open, high, low, close] list_data = StockOneStopProcessor.analysis_columns_day_kline( result, idx) if list_data is not None: upsert_sql = StockOneStopProcessor.upsert_day_kline.format( security_code=Utils.quotes_surround( security_code), the_date=Utils.quotes_surround(list_data[0]), amount=list_data[1], vol=list_data[2], open=list_data[3], high=list_data[4], low=list_data[5], close=list_data[6]) else: continue # 批量(100)提交数据更新 if len(upsert_sql_list) == 3000: StockOneStopProcessor.dbService.insert_many( upsert_sql_list) process_line += '=' upsert_sql_list = [] if upsert_sql is not None: upsert_sql_list.append(upsert_sql) progress = Utils.base_round( Utils.division_zero(add_up, len_indexes) * 100, 2) progress_log_list = ['StockOneStopProcessor'] progress_log_list.append( StockOneStopProcessor.get_method_name()) progress_log_list.append('security_code') progress_log_list.append(security_code) progress_log_list.append('progress') progress_log_list.append(add_up) progress_log_list.append(len_indexes) progress_log_list.append(process_line) progress_log_list.append(str(progress) + '%') r.lpush(queue, progress_log_list) else: upsert_sql_list.append(upsert_sql) # 处理最后一批security_code的更新语句 if len(upsert_sql_list) > 0: StockOneStopProcessor.dbService.insert_many( upsert_sql_list) process_line += '=' progress = Utils.base_round( Utils.division_zero(add_up, len(indexes_values)) * 100, 2) progress_log_list = ['StockOneStopProcessor'] progress_log_list.append( StockOneStopProcessor.get_method_name()) progress_log_list.append('security_code') progress_log_list.append(security_code) progress_log_list.append('progress') progress_log_list.append(add_up) progress_log_list.append(len_indexes) progress_log_list.append(process_line) progress_log_list.append(progress) r.lpush(queue, progress_log_list) else: warn_log_list = ['StockOneStopProcessor'] warn_log_list.append( StockOneStopProcessor.get_method_name()) warn_log_list.append('security_code') warn_log_list.append(security_code) warn_log_list.append('result`s indexes_values is None') r.lpush(queue, warn_log_list, logging.WARNING) else: warn_log_list = ['StockOneStopProcessor'] warn_log_list.append(StockOneStopProcessor.get_method_name()) warn_log_list.append('security_code') warn_log_list.append(security_code) warn_log_list.append( 'tt.get_all_daybar or tt.get_last_n_daybar result is None') r.lpush(queue, warn_log_list, logging.WARNING) except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() line_no = traceback.extract_stack()[-2][1] error_log_list = ['StockOneStopProcessor'] error_log_list.append(StockOneStopProcessor.get_method_name()) error_log_list.append('security_code') error_log_list.append(security_code) error_log_list.append('line_no') error_log_list.append(line_no) error_log_list.append(exc_type) error_log_list.append(exc_value) error_log_list.append(exc_traceback) r.lpush(queue, error_log_list, logging.ERROR)
def analysis_average_line(ma, temp_line_tuple, previous_data): """ 股票均线数据计算方法 :param ma: 均线类型 :param temp_line_tuple: 均线数据的ma切片列表 :param previous_data: 前一交易日数据 :return: """ # 前一ma日均收盘价,默认值为0,便于写入数据库 close_pre_avg = 0 # 前一ma日均成交额(元) amount_pre_avg = 0 # 前一ma日均成交量(手) vol_pre_avg = 0 # 前一ma日均成交价 price_pre_avg = 0 # close_pre_avg, amount_pre_avg, vol_pre_avg, price_pre_avg if previous_data is not None and len(previous_data) > 0: close_pre_avg = previous_data[0][0] amount_pre_avg = previous_data[0][1] vol_pre_avg = previous_data[0][2] price_pre_avg = previous_data[0][3] # temp_line_tuple中的数据为the_date, close, amount, vol # 当日the_date为正序排序最后一天的the_date,第一个元素 the_date = temp_line_tuple[ma - 1][0] # 将元组元素转换为列表元素 # temp_items = [item for item in temp_line_tuple[0:]] # 当日收盘价=正序排序最后一天的收盘价,最后一个元素的第2个元素 close = temp_line_tuple[ma - 1][1] # ma日均收盘价=sum(前ma日(含)的收盘价)/ma close_list = [close for close in [item[1] for item in temp_line_tuple]] close_avg = Utils.base_round_zero(Utils.average_zero(close_list), 2) # 如果收盘ma日均价为None,则为异常数据,价格不可能为0 # if close_avg is None: # close_avg = StockOneStopProcessor.base_round_zero(Decimal(0), 2) # ma日均收盘价涨跌幅=(ma日均收盘价 - 前一ma日均收盘价)/前一ma日均收盘价 * 100 # 默认值为0 close_avg_chg = 0 # if close_pre_avg is not None and close_pre_avg != Decimal(0): # close_avg_chg = StockOneStopProcessor.base_round_zero(StockOneStopProcessor.division_zero((close_avg - close_pre_avg), close_pre_avg) * 100, 2) close_avg_chg = Utils.base_round_zero( Utils.division_zero( (close_avg - close_pre_avg), close_pre_avg) * 100, 2) # 当日成交额 amount = temp_line_tuple[ma - 1][2] # ma日均成交额=sum(前ma日(含)的成交额)/ma amount_list = [ amount for amount in [item[2] for item in temp_line_tuple] ] amount_avg = Utils.base_round_zero(Utils.average_zero(amount_list), 2) # if amount_avg is None : # amount_avg = Decimal(0) # ma日均成交额涨跌幅=(ma日均成交额 - 前一ma日均成交额)/前一ma日均成交额 * 100 # 默认值为0 amount_avg_chg = 0 # if amount_pre_avg is not None and amount_pre_avg != Decimal(0): # amount_avg_chg = StockOneStopProcessor.base_round_zero(StockOneStopProcessor.division_zero((amount_avg - amount_pre_avg), amount_pre_avg) * 100, 2) amount_avg_chg = Utils.base_round_zero( Utils.division_zero( (amount_avg - amount_pre_avg), amount_pre_avg) * 100, 2) # 当日成交量 vol = temp_line_tuple[ma - 1][3] # ma日均成交量=sum(前ma日(含)的成交量)/ma vol_list = [vol for vol in [item[3] for item in temp_line_tuple]] vol_avg = Utils.base_round_zero(Utils.average_zero(vol_list), 2) # if vol_avg is None: # vol_avg = Decimal(0) # ma日均成交量涨跌幅=(ma日均成交量 - 前一ma日均成交量)/前一ma日均成交量 * 100 vol_avg_chg = 0 # if vol_pre_avg is not None and vol_pre_avg != Decimal(0): # vol_avg_chg = StockOneStopProcessor.base_round_zero(StockOneStopProcessor.division_zero((vol_avg - vol_pre_avg), vol_pre_avg) * 100, 2) vol_avg_chg = Utils.base_round_zero( Utils.division_zero((vol_avg - vol_pre_avg), vol_pre_avg) * 100, 2) # ma日均成交价=sum(前ma日(含)的成交额)/sum(ma日(含)的成交量) price_avg = Utils.base_round_zero( Utils.division_zero(Utils.sum_zero(amount_list), Utils.sum_zero(vol_list)), 2) # if price_avg is None: # price_avg = Decimal(0) # ma日均成交价涨跌幅=(ma日均成交价 - 前一ma日均成交价)/前一ma日均成交价 * 100 price_avg_chg = 0 # if price_pre_avg is not None and price_pre_avg != Decimal(0): # price_avg_chg = StockOneStopProcessor.base_round_zero(StockOneStopProcessor.division_zero((price_avg - price_pre_avg), price_pre_avg) * 100, 2) price_avg_chg = Utils.base_round_zero( Utils.division_zero( (price_avg - price_pre_avg), price_pre_avg) * 100, 2) # print('price_avg', price_avg, 'price_pre_avg', price_pre_avg, 'price_avg_chg', price_avg_chg) # 日金钱流向涨跌幅=日成交额/ma日(含)均成交额 * 100 amount_flow_chg = Utils.base_round_zero( Utils.division_zero(amount - amount_avg, amount_avg), 2) # if amount_flow_chg is None: # amount_flow_chg = Decimal(0) # 日成交量流向涨跌幅=日成交量/ma日(含)均成交量 * 100 vol_flow_chg = Utils.base_round_zero( Utils.division_zero(vol - vol_avg, vol_avg), 2) # if vol_flow_chg is None: # vol_flow_chg = Decimal(0) close_ma_price_avg_chg = Utils.base_round_zero( Utils.division_zero(close - price_avg, price_avg) * 100, 2) return [ the_date, close, close_avg, close_pre_avg, close_avg_chg, amount, amount_avg, amount_pre_avg, amount_avg_chg, vol, vol_avg, vol_pre_avg, vol_avg_chg, price_avg, price_pre_avg, price_avg_chg, amount_flow_chg, vol_flow_chg, close_ma_price_avg_chg ]
def processing_average_line(ma, security_code, r, queue): """ 股票均线数据处理方法 :param ma: :return: """ average_line_max_the_date = StockOneStopProcessor.dbService.get_average_line_max_the_date( ma, security_code) decline_ma_the_date = StockOneStopProcessor.dbService.get_average_line_decline_max_the_date( ma, average_line_max_the_date) r.lpush(queue, [ 'StockOneStopProcessor', StockOneStopProcessor.get_method_name(), 'ma', ma, security_code, 'average_line_max_the_date', average_line_max_the_date, 'decline_ma_the_date', decline_ma_the_date ]) # print('decline_ma_the_date', decline_ma_the_date, 'average_line_max_the_date', average_line_max_the_date) result = StockOneStopProcessor.dbService.get_stock_day_kline( security_code, decline_ma_the_date) len_result = len(result) # print('ma', ma, 'len_result', len_result) if len_result < ma: return try: if result is not None and len_result > 0: # 开始解析股票日K数据, the_date, close # 临时存储批量更新sql的列表 upsert_sql_list = [] # 需要处理的单只股票进度计数 add_up = 0 # 需要处理的单只股票进度打印字符 process_line = '=' # 循环处理security_code的股票日K数据 i = 0 # 由于是批量提交数据,所以在查询前一日均价时,有可能还未提交, # 所以只在第一次的时候查询,其他的情况用前一次计算的均价作为前一日均价 # is_first就是是否第一次需要查询的标识 # 前一日均值 previous_data = None while i < len_result: add_up += 1 # 如果切片的下标是元祖的最后一个元素,则退出,因为已经处理完毕 if (i + ma) > len_result: add_up -= 1 break temp_line_tuple = result[i:(i + ma)] # 如果前一交易日的数据为空,则去查询一次 if previous_data is None or len(previous_data) == 0: the_date = temp_line_tuple[ma - 1][0] # close_pre_avg, amount_pre_avg, vol_pre_avg, price_pre_avg previous_data = StockOneStopProcessor.dbService.get_previous_average_line( ma, security_code, the_date) # 返回值list [the_date, # close, close_avg, close_pre_avg, close_avg_chg, # amount, amount_avg, amount_pre_avg, amount_avg_chg, # vol, vol_avg, vol_pre_avg, vol_avg_chg, # price_avg, price_pre_avg, price_avg_chg, # amount_flow_chg, vol_flow_chg, close_ma_price_avg_chg] # print('the_date', the_date, 'previous_data', previous_data) list_data = StockOneStopProcessor.analysis_average_line( ma, temp_line_tuple, previous_data) """ 均线数据入库(3,5,10日等) """ upsert_sql = 'insert into tquant_stock_average_line (security_code, the_date, ' \ 'ma, ' \ 'close, close_avg, close_pre_avg, close_avg_chg, ' \ 'amount, amount_avg, amount_pre_avg, amount_avg_chg, ' \ 'vol, vol_avg, vol_pre_avg, vol_avg_chg, ' \ 'price_avg, price_pre_avg, price_avg_chg, ' \ 'amount_flow_chg, vol_flow_chg, close_ma_price_avg_chg) ' \ 'values ({security_code}, {the_date}, ' \ '{ma}, ' \ '{close}, {close_avg}, {close_pre_avg}, {close_avg_chg}, ' \ '{amount}, {amount_avg}, {amount_pre_avg}, {amount_avg_chg}, ' \ '{vol}, {vol_avg}, {vol_pre_avg}, {vol_avg_chg}, ' \ '{price_avg}, {price_pre_avg}, {price_avg_chg}, ' \ '{amount_flow_chg}, {vol_flow_chg}, {close_ma_price_avg_chg}) ' \ 'on duplicate key update ' \ 'close=values(close), close_avg=values(close_avg), close_pre_avg=values(close_pre_avg), close_avg_chg=values(close_avg_chg), ' \ 'amount=values(amount), amount_avg=values(amount_avg), amount_pre_avg=values(amount_pre_avg), amount_avg_chg=values(amount_avg_chg), ' \ 'vol=values(vol), vol_avg=values(vol_avg), vol_pre_avg=values(vol_pre_avg), vol_avg_chg=values(vol_avg_chg), ' \ 'price_avg=values(price_avg), price_pre_avg=values(price_pre_avg), price_avg_chg=values(price_avg_chg), ' \ 'amount_flow_chg=values(amount_flow_chg), vol_flow_chg=values(vol_flow_chg), close_ma_price_avg_chg=values(close_ma_price_avg_chg) ' upsert_sql = upsert_sql.format( security_code=Utils.quotes_surround(security_code), the_date=Utils.quotes_surround( list_data[0].strftime('%Y-%m-%d')), ma=ma, close=list_data[1], close_avg=list_data[2], close_pre_avg=list_data[3], close_avg_chg=list_data[4], amount=list_data[5], amount_avg=list_data[6], amount_pre_avg=list_data[7], amount_avg_chg=list_data[8], vol=list_data[9], vol_avg=list_data[10], vol_pre_avg=list_data[11], vol_avg_chg=list_data[12], price_avg=list_data[13], price_pre_avg=list_data[14], price_avg_chg=list_data[15], amount_flow_chg=list_data[16], vol_flow_chg=list_data[17], close_ma_price_avg_chg=list_data[18]) # print(upsert_sql) # 将本次的处理结果重新赋值到previous_data中 # close_pre_avg, amount_pre_avg, vol_pre_avg, price_pre_avg previous_data = [[ list_data[2], list_data[6], list_data[10], list_data[13] ]] # 批量(100)提交数据更新 if len(upsert_sql_list) == 1000: StockOneStopProcessor.dbService.insert_many( upsert_sql_list) process_line += '=' upsert_sql_list = [] upsert_sql_list.append(upsert_sql) if len_result == ma: progress = Utils.base_round_zero(1 * 100, 2) else: progress = Utils.base_round_zero( Utils.division_zero(add_up, (len_result - ma + 1)) * 100, 2) progress_log_list = ['StockOneStopProcessor'] progress_log_list.append( StockOneStopProcessor.get_method_name()) progress_log_list.append('ma') progress_log_list.append(ma) progress_log_list.append('security_code') progress_log_list.append(security_code) progress_log_list.append('progress') progress_log_list.append(add_up) progress_log_list.append((len_result - ma + 1)) progress_log_list.append(process_line) progress_log_list.append(progress) r.lpush(queue, progress_log_list) else: if upsert_sql is not None: upsert_sql_list.append(upsert_sql) i += 1 # 处理最后一批security_code的更新语句 if len(upsert_sql_list) > 0: StockOneStopProcessor.dbService.insert_many( upsert_sql_list) process_line += '=' if len_result == ma: progress = Utils.base_round_zero(1 * 100, 2) else: progress = Utils.base_round_zero( Utils.division_zero(add_up, (len_result - ma + 1)) * 100, 2) progress_log_list = ['StockOneStopProcessor'] progress_log_list.append( StockOneStopProcessor.get_method_name()) progress_log_list.append('ma') progress_log_list.append(ma) progress_log_list.append('security_code') progress_log_list.append(security_code) progress_log_list.append('progress') progress_log_list.append(add_up) progress_log_list.append((len_result - ma + 1)) progress_log_list.append(process_line) progress_log_list.append(progress) r.lpush(queue, progress_log_list) except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() line_no = traceback.extract_stack()[-2][1] error_log_list = ['StockOneStopProcessor'] error_log_list.append(StockOneStopProcessor.get_method_name()) error_log_list.append('ma') error_log_list.append(ma) error_log_list.append('security_code') error_log_list.append(security_code) error_log_list.append('line_no') error_log_list.append(line_no) error_log_list.append(exc_type) error_log_list.append(exc_value) error_log_list.append(exc_traceback) r.lpush(queue, error_log_list, logging.ERROR)
def analysis_day_kline_change_percent(temp_kline_tuple, price_avg_pre): """ 股票日K涨跌幅计算方法 :param temp_kline_tuple: 相邻两个日K数据的列表 :return: """ try: # 需要处理的涨跌幅的交易日,即第二个元组的the_date the_date = temp_kline_tuple[1][0] # 前一日收盘价 close1 = temp_kline_tuple[0][1] # 当前收盘价 close2 = temp_kline_tuple[1][1] # 前一日交易额 amount1 = temp_kline_tuple[0][2] # 当前交易额 amount2 = temp_kline_tuple[1][2] # 前一日交易量 vol1 = temp_kline_tuple[0][3] # 当前交易量 vol2 = temp_kline_tuple[1][3] if price_avg_pre is None: price_avg_pre = temp_kline_tuple[0][4] # 涨跌幅(百分比)计算:当日(收盘价-前一日收盘价)/前一日收盘价 * 100 close_chg = None if close1 is not None and close1 != 0: close_chg = Utils.base_round( Utils.division_zero((close2 - close1), close1) * 100, 2) else: close1 = 0 close_chg = 0 amount_chg = None if amount1 is not None and amount1 != 0: amount_chg = Utils.base_round( Utils.division_zero((amount2 - amount1), amount1) * 100, 2) else: amount1 = 0 amount_chg = 0 vol_chg = None if vol1 is not None and vol1 != 0: vol_chg = Utils.base_round( Utils.division_zero((vol2 - vol1), vol1) * 100, 2) else: vol1 = 0 vol_chg = 0 price_avg = Utils.base_round_zero( Utils.division_zero(amount2, vol2), 2) close_price_avg_chg = Utils.base_round_zero( Utils.division_zero(close2 - price_avg, price_avg) * 100, 2) if price_avg_pre is None: price_avg_pre = 0 price_avg_chg = Utils.base_round_zero( Utils.division_zero(price_avg - price_avg_pre, price_avg_pre) * 100, 2) return [ the_date, close1, close_chg, amount1, amount_chg, vol1, vol_chg, price_avg, close_price_avg_chg, price_avg_chg ] except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() line_no = traceback.extract_stack()[-2][1] error_log_list = ['StockOneStopProcessor'] error_log_list.append(StockOneStopProcessor.get_method_name()) error_log_list.append('line_no') error_log_list.append(line_no) error_log_list.append(exc_type) error_log_list.append(exc_value) error_log_list.append(exc_traceback) r.lpush(queue, error_log_list, logging.ERROR) return None
def processing_day_kline_change_percent(security_code, r, queue): """ 股票日K数据涨跌幅处理方法 :return: """ day_kline_max_the_date = StockOneStopProcessor.dbService.get_day_kline_max_the_date( security_code) r.lpush(queue, [ 'StockOneStopProcessor', StockOneStopProcessor.get_method_name(), security_code, 'day_kline_max_the_date', day_kline_max_the_date ]) result = StockOneStopProcessor.dbService.get_stock_day_kline( security_code, day_kline_max_the_date) len_result = len(result) # print('result', result) if len_result == 0: return # 临时存储批量更新sql的列表 upsert_sql_list = [] # 需要处理的单只股票进度计数 add_up = 0 # 需要处理的单只股票进度打印字符 process_line = '' # 循环处理security_code的股票日K数据 i = 0 price_avg_pre = None while i < len_result: # 切片元组,每相连的2个一组 section_idx = i + 2 if section_idx > len_result: i += 1 break temp_kline_tuple = result[i:section_idx] # 返回值格式list [the_date, close1, close_chg, amount1, amount_chg, vol1, vol_chg, # price_avg, close_price_avg_chg, price_avg_chg] list_data = StockOneStopProcessor.analysis_day_kline_change_percent( temp_kline_tuple, price_avg_pre) # print(temp_kline_tuple) # print(price_avg_pre) if list_data is not None: """ 日K涨跌幅数据入库 """ upsert_sql = 'insert into tquant_stock_day_kline (security_code, the_date, ' \ 'close_pre, close_chg, ' \ 'amount_pre, amount_chg, ' \ 'vol_pre, vol_chg, ' \ 'price_avg, close_price_avg_chg, price_avg_chg) ' \ 'values ({security_code}, {the_date}, ' \ '{close_pre}, {close_chg},' \ '{amount_pre}, {amount_chg}, {vol_pre}, {vol_chg}, ' \ '{price_avg}, {close_price_avg_chg}, {price_avg_chg}) ' \ 'on duplicate key update ' \ 'close_pre=values(close_pre), close_chg=values(close_chg), ' \ 'amount_pre=values(amount_pre), amount_chg=values(amount_chg), ' \ 'vol_pre=values(vol_pre), vol_chg=values(vol_chg), ' \ 'price_avg=values(price_avg), close_price_avg_chg=values(close_price_avg_chg), price_avg_chg=values(price_avg_chg)' upsert_sql = upsert_sql.format( security_code=Utils.quotes_surround(security_code), the_date=Utils.quotes_surround( list_data[0].strftime('%Y-%m-%d')), close_pre=list_data[1], close_chg=list_data[2], amount_pre=list_data[3], amount_chg=list_data[4], vol_pre=list_data[5], vol_chg=list_data[6], price_avg=list_data[7], close_price_avg_chg=list_data[8], price_avg_chg=list_data[9]) price_avg_pre = list_data[7] else: upsert_sql = None # 批量(100)提交数据更新 if len(upsert_sql_list) == 1000: StockOneStopProcessor.dbService.insert_many(upsert_sql_list) process_line += '=' upsert_sql_list = [] if upsert_sql is not None: upsert_sql_list.append(upsert_sql) # 这个地方为什么要add_up + 1?因为第一条数据不会被处理,所以总数会少一条,所以在计算进度的时候要+1 progress = Utils.base_round( Utils.division_zero((add_up + 1), len_result) * 100, 2) progress_log_list = ['StockOneStopProcessor'] progress_log_list.append( StockOneStopProcessor.get_method_name()) progress_log_list.append('security_code') progress_log_list.append(security_code) progress_log_list.append('progress') progress_log_list.append(add_up + 1) progress_log_list.append(len_result) progress_log_list.append(process_line) progress_log_list.append(progress) r.lpush(queue, progress_log_list) else: if upsert_sql is not None: upsert_sql_list.append(upsert_sql) i += 1 add_up += 1 # 处理最后一批security_code的更新语句 if len(upsert_sql_list) > 0: StockOneStopProcessor.dbService.insert_many(upsert_sql_list) process_line += '=' progress = Utils.base_round( Utils.division_zero((add_up + 1), len_result) * 100, 2) progress_log_list = ['StockOneStopProcessor'] progress_log_list.append(StockOneStopProcessor.get_method_name()) progress_log_list.append('security_code') progress_log_list.append(security_code) progress_log_list.append('progress') progress_log_list.append(add_up + 1) progress_log_list.append(len_result) progress_log_list.append(process_line) progress_log_list.append(progress) r.lpush(queue, progress_log_list)
def processing_day_kline(self): """ 股票日K数据处理,分全量还是增量 :return: """ try: recent_few_days = self.dbService.get_day_kline_recentdays(self.security_code) log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append('recent_few_days') log_list.append(recent_few_days) self.print_log(log_list) if recent_few_days is None: result = tt.get_all_daybar(self.security_code, 'qfq') elif recent_few_days > 0: result = tt.get_last_n_daybar(self.security_code, recent_few_days, 'qfq') else: return if result.empty == False: # 索引值为日期 indexes_values = result.index.values # 临时存储批量更新sql的列表 upsert_sql_list = [] # 需要处理的单只股票进度计数 add_up = 0 # 需要处理的单只股票进度打印字符 process_line = '=' # 循环处理security_code的股票日K数据 if indexes_values is not None: len_indexes = len(indexes_values) for idx in indexes_values: add_up += 1 # 解析股票日K数据(每行) # 解析每行的返回值格式为list [the_date, amount, vol, open, high, low, close] list_data = self.analysis_columns_day_kline(result, idx) if list_data is not None: upsert_sql = self.upsert_day_kline.format( security_code=Utils.quotes_surround(self.security_code), the_date=Utils.quotes_surround(list_data[0]), amount=list_data[1], vol=list_data[2], open=list_data[3], high=list_data[4], low=list_data[5], close=list_data[6] ) else: continue # 批量(100)提交数据更新 if len(upsert_sql_list) == 200: self.dbService.insert_many(upsert_sql_list) process_line += '=' upsert_sql_list = [] if upsert_sql is not None: upsert_sql_list.append(upsert_sql) progress = Utils.base_round(Utils.division_zero(add_up, len_indexes) * 100, 2) log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append('progress') log_list.append(add_up) log_list.append(len_indexes) log_list.append(process_line) log_list.append(str(progress) + '%') self.print_log(log_list) else: upsert_sql_list.append(upsert_sql) # 处理最后一批security_code的更新语句 if len(upsert_sql_list) > 0: self.dbService.insert_many(upsert_sql_list) process_line += '=' progress = Utils.base_round(Utils.division_zero(add_up, len(indexes_values)) * 100, 2) log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append('progress') log_list.append(add_up) log_list.append(len_indexes) log_list.append(process_line) log_list.append(progress) self.print_log(log_list) else: log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append('result`s indexes_values is None') self.print_log(log_list) else: log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append('tt.get_all_daybar or tt.get_last_n_daybar result is None') self.print_log(log_list) except Exception: log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append(traceback.format_exc()) self.print_log(log_list)
def processing_average_line_avg(self, ma): """ 股票均线数据涨跌幅平均数据处理方法 :param ma: 均线类型 :return: """ average_line_avg_max_the_date = self.dbService.get_average_line_avg_max_the_date(ma, self.security_code) decline_ma_the_date = self.dbService.get_average_line_avg_decline_max_the_date(ma, average_line_avg_max_the_date, self.security_code) log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append('ma') log_list.append(ma) log_list.append('average_line_avg_max_the_date') log_list.append(average_line_avg_max_the_date) log_list.append('decline_ma_the_date') log_list.append(decline_ma_the_date) self.print_log(log_list) result = self.dbService.get_average_line(ma, self.security_code, decline_ma_the_date) len_result = len(result) if len_result < ma: return try: if result is not None and len_result > 0: # 开始解析股票日K数据, the_date, close # 临时存储批量更新sql的列表 upsert_sql_list = [] # 需要处理的单只股票进度计数 add_up = 0 # 需要处理的单只股票进度打印字符 process_line = '=' # 循环处理security_code的股票日K数据 i = 0 while i < len_result: add_up += 1 # 如果切片的下标是元祖的最后一个元素,则退出,因为已经处理完毕 if (i + ma) > len_result: add_up -= 1 break temp_line_tuple = result[i:(i + ma)] # 返回值list_data list [the_date, # close_avg_chg_avg, amount_avg_chg_avg, vol_avg_chg_avg, # price_avg_chg_avg, amount_flow_chg_avg, vol_flow_chg_avg] list_data = self.analysis_average_line_avg(ma, temp_line_tuple) upsert_sql = self.upsert_average_line_avg.format(security_code=Utils.quotes_surround(self.security_code), the_date=Utils.quotes_surround(list_data[0].strftime('%Y-%m-%d')), ma=ma, close_avg_chg_avg=list_data[1], amount_avg_chg_avg=list_data[2], vol_avg_chg_avg=list_data[3], price_avg_chg_avg=list_data[4], amount_flow_chg_avg=list_data[5], vol_flow_chg_avg=list_data[6] ) # 批量(100)提交数据更新 if len(upsert_sql_list) == 200: self.dbService.insert_many(upsert_sql_list) process_line += '=' upsert_sql_list = [] upsert_sql_list.append(upsert_sql) if len_result == ma: progress = Utils.base_round(1 * 100, 2) else: progress = Utils.base_round(Utils.division_zero(add_up, (len_result - ma + 1)) * 100, 2) log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append('ma') log_list.append(ma) log_list.append('progress') log_list.append(add_up) log_list.append((len_result - ma + 1)) log_list.append(process_line) log_list.append(progress) self.print_log(log_list) else: if upsert_sql is not None: upsert_sql_list.append(upsert_sql) i += 1 # 处理最后一批security_code的更新语句 if len(upsert_sql_list) > 0: self.dbService.insert_many(upsert_sql_list) process_line += '=' if len_result == ma: progress = Utils.base_round(1 * 100, 2) else: progress = Utils.base_round(Utils.division_zero(add_up, (len_result - ma + 1)) * 100, 2) log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append('ma') log_list.append(ma) log_list.append('progress') log_list.append(add_up) log_list.append((len_result - ma + 1)) log_list.append(process_line) log_list.append(progress) self.print_log(log_list) except Exception: log_list = [self.now(), self.info(), self.get_classs_name(), self.security_code] log_list.append(self.get_method_name()) log_list.append('ma') log_list.append(ma) log_list.append(traceback.format_exc()) self.print_log(log_list)
def processing_day_kline(self): """ 股票日K数据处理,分全量还是增量 :return: """ try: if self.is_reset: max_the_date = datetime.datetime.now().replace(year=1970, month=1, day=1) else: max_the_date = self.get_max_the_date() diff_days = Utils.diff_days(max_the_date) log_list = [ Utils.get_now(), Utils.get_info(), self.get_classs_name(), self.security_code, self.get_method_name(), '最大交易日', max_the_date, '距今', Utils.format_date(datetime.date.today()), '差', diff_days, '天' ] Utils.print_log(log_list) dict_data_pre = None if diff_days is None: result = tt.get_all_daybar(self.security_code, 'bfq') else: result = tt.get_last_n_daybar(self.security_code, diff_days, 'bfq') if result.empty == False: # 按照正序排序,时间小的排前面 result.sort_index(ascending=True) # 需要处理的单只股票进度计数 add_up = 0 # 需要处理的单只股票进度打印字符 process_line = '=' len_indexes = len(result.index.values) for i in range(len_indexes): add_up += 1 if i < 9: continue end = i + 1 start = i - 9 section = result.iloc[start:end, :] dict_data = self.processing_section(section, dict_data_pre) dict_data_pre = dict_data # 批量打印日志 if add_up % 200 == 0: process_line += '=' progress = Utils.base_round( Utils.division_zero(add_up, len_indexes) * 100, 2) log_list = [ Utils.get_now(), Utils.get_info(), self.get_classs_name(), self.security_code, self.get_method_name(), '处理进度', add_up, len_indexes, process_line, str(progress) + '%' ] Utils.print_log(log_list) process_line += '=' progress = Utils.base_round( Utils.division_zero(add_up, len_indexes) * 100, 2) log_list = [ Utils.get_now(), Utils.get_info(), self.get_classs_name(), self.security_code, self.get_method_name(), '处理进度', add_up, len_indexes, process_line, str(progress) + '%' ] Utils.print_log(log_list) # self.dbService.upsert_many(list_db_data, 'tquant_stock_history_quotation', ['security_code', 'the_date']) else: log_list = [ Utils.get_now(), Utils.get_warn(), self.get_classs_name(), self.security_code, self.get_method_name(), '【日K DataFrame为空】' ] Utils.print_log(log_list) except Exception: traceback.print_exc()
def processing_1_day_chg(self, section_tail1, idx, dict_data, dict_data_pre): try: amount = section_tail1.at[idx, 'amount'] # amount的类型为numpy.ndarray,是一个多维数组,可能包含多个值,其他的字段也是一样,测试的时候发现有异常抛出 if isinstance(amount, numpy.ndarray) and amount.size > 1: amount = amount.tolist()[0] amount = Utils.base_round(amount, 2) amount_pre = dict_data_pre['amount'] amount_chg = Utils.base_round_zero( Utils.division_zero(amount - amount_pre, amount_pre) * 100, 2) dict_data['amount'] = amount dict_data['amount_chg'] = amount_chg vol = section_tail1.at[idx, 'vol'] if isinstance(vol, numpy.ndarray) and vol.size > 1: vol = vol.tolist()[0] vol = Utils.base_round(vol, 2) vol_pre = dict_data_pre['vol'] vol_chg = Utils.base_round_zero( Utils.division_zero(vol - vol_pre, vol_pre) * 100, 2) dict_data['vol'] = vol dict_data['vol_chg'] = vol_chg open = section_tail1.at[idx, 'open'] if isinstance(open, numpy.ndarray) and open.size > 1: open = open.tolist()[0] open = Utils.base_round(open, 2) open_pre = dict_data_pre['open'] open_chg = Utils.base_round_zero( Utils.division_zero(open - open_pre, open_pre) * 100, 2) dict_data['open'] = open dict_data['open_chg'] = open_chg high = section_tail1.at[idx, 'high'] if isinstance(high, numpy.ndarray) and high.size > 1: high = high.tolist()[0] high = Utils.base_round(high, 2) high_pre = dict_data_pre['high'] high_chg = Utils.base_round_zero( Utils.division_zero(high - high_pre, high_pre) * 100, 2) dict_data['high'] = high dict_data['high_chg'] = high_chg low = section_tail1.at[idx, 'low'] if isinstance(low, numpy.ndarray) and low.size > 1: low = low.tolist()[0] low = Utils.base_round(low, 2) low_pre = dict_data_pre['low'] low_chg = Utils.base_round_zero( Utils.division_zero(low - low_pre, low_pre) * 100, 2) dict_data['low'] = low dict_data['low_chg'] = low_chg close = section_tail1.at[idx, 'close'] if isinstance(close, numpy.ndarray) and close.size > 1: close = close.tolist()[0] close = Utils.base_round(close, 2) close_pre = dict_data_pre['close'] close_chg = Utils.base_round_zero( Utils.division_zero(close - close_pre, close_pre) * 100, 2) dict_data['close'] = close dict_data['close_chg'] = close_chg close_open_chg = Utils.base_round_zero( Utils.division_zero(close - open, open) * 100, 2) dict_data['close_open_chg'] = close_open_chg except Exception: traceback.print_exc()