def get_stock_ips(): from multiprocessing import cpu_count from QUANTAXIS.QAFetch.QATdx import get_ip_list_by_multi_process_ping stock_ip_list = QASetting.stock_ip_list ips = get_ip_list_by_multi_process_ping( stock_ip_list, _type='stock')[:cpu_count() * 2 + 1] return ips
def _QA_SU_save_index_or_etf_day(index__or_etf, client, ui_log, ui_progress): index_list = QA_fetch_get_stock_list(index__or_etf).code.tolist() coll = get_coll(client) ips = get_ip_list_by_multi_process_ping(stock_ip_list, _type='stock')[ :cpu_count() * 2 + 1] ps = QA_SU_save_index_day_parallelism( processes=cpu_count() if len(ips) >= cpu_count() else len(ips), client=client, ui_log=ui_log) # 单线程测试 # ps = QA_SU_save_index_day_parallelism( # processes=1 if len(ips) >= cpu_count() else len(ips), # client=client, ui_log=ui_log) ps.total_counts = len(index_list) ps.run(index_list)
def QA_SU_save_stock_xdxr(client=DATABASE, ui_log=None, ui_progress=None): """[summary] Keyword Arguments: client {[type]} -- [description] (default: {DATABASE}) """ # stock_list = QA_fetch_get_stock_list().code.unique().tolist() stock_list = QA_fetch_get_stock_list().code.unique().tolist() coll = get_coll(client, "stock_xdxr", "stock_xdxr") coll_adj = get_coll(client, cacheName="stock_adj", tableName="stock_adj") ips = get_ip_list_by_multi_process_ping( stock_ip_list, _type='stock')[:cpu_count() * 2 + 1] # 单线程测试 ps = QA_SU_save_stock_xdxr_parallelism( processes=cpu_count() if len(ips) >= cpu_count() else len(ips), client=client, ui_log=ui_log) ps.total_counts = len(stock_list) ps.run(stock_list)
def test_gen_paramz(self): codelist = QA.QA_fetch_stock_list_adv().code.tolist() days = 300 start = datetime.datetime.now().date() - datetime.timedelta(days) end = datetime.datetime.now().date() - datetime.timedelta(10) codeListCount = 200 ips = get_ip_list_by_multi_process_ping(QA.QAUtil.stock_ip_list, filename='stock_ip_list_MP') param = gen_param(codelist[:codeListCount], start, end, IPList=ips[:cpu_count()]) a = time.time() ps = Parallelism(cpu_count()) ps.add(QA.QAFetch.QATdx.QA_fetch_get_stock_day, param) ps.run() data = ps.get_results() b = time.time() t1 = b - a data = list(data) print('返回数据{}条,用时:{}秒'.format(len(data), t1)) # print(data) # print([x.code.unique() for x in data]) self.assertTrue( len(data) == codeListCount, '返回结果和输入参数个数不匹配: {} {}'.format(len(data), codeListCount)) i, j = 0, 0 for x in data: try: # print(x) i += 1 self.assertTrue((x is None) or (len(x) > 0), '返回数据太少:{}'.format(len(x))) if not ((x is None) or (len(x) > 0)): print('data is None') if i % 10 == 0: print(x) except Exception as e: j += 1 print(i, j)
def QA_SU_save_stock_day(client=DATABASE, ui_log=None, ui_progress=None): ''' save stock_day 保存日线数据 :param client: :param ui_log: 给GUI qt 界面使用 :param ui_progress: 给GUI qt 界面使用 :param ui_progress_int_value: 给GUI qt 界面使用 ''' stock_list = QA_fetch_get_stock_list().code.unique().tolist() coll_stock_day = client.stock_day coll_stock_day.create_index([("code", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)]) err = [] # saveing result def __gen_param(stock_list, coll_stock_day, ip_list=[]): results = [] count = len(ip_list) total = len(stock_list) for item in range(len(stock_list)): try: code = stock_list[item] QA_util_log_info( '##JOB01 Now Saving STOCK_DAY=== {}'.format(str(code)), ui_log) # 首选查找数据库 是否 有 这个代码的数据 search_cond = {'code': str(code)[0:6]} ref = coll_stock_day.find(search_cond) end_date = str(now_time())[0:10] ref_count = coll_stock_day.count_documents(search_cond) # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref_count > 0: # 接着上次获取的日期继续更新 start_date = ref[ref_count - 1]['date'] # print("ref[ref.count() - 1]['date'] {} {}".format(ref.count(), coll_stock_day.count_documents({'code': str(code)[0:6]}))) else: # 当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 start_date = '1990-01-01' QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'. format(code, start_date, end_date), ui_log) if start_date != end_date: # 更新过的,不更新 results.extend([(code, start_date, end_date, '00', 'day', ip_list[item % count]['ip'], ip_list[item % count]['port'], item, total, ui_log, ui_progress)]) except Exception as error0: print('Exception:{}'.format(error0)) err.append(code) return results ips = get_ip_list_by_multi_process_ping( stock_ip_list, _type='stock')[:cpu_count() * 2 + 1] param = __gen_param(stock_list, coll_stock_day, ips) ps = QA_SU_save_stock_day_parallelism( processes=cpu_count() if len(ips) >= cpu_count() else len(ips), client=client, ui_log=ui_log) ps.run(do_saving_work, param) if len(err) < 1: QA_util_log_info('SUCCESS save stock day ^_^', ui_log) else: QA_util_log_info('ERROR CODE \n ', ui_log) QA_util_log_info(err, ui_log)
def QA_SU_save_stock_day(client=DATABASE, ui_log=None, ui_progress=None): ''' save stock_day 保存日线数据 :param client: :param ui_log: 给GUI qt 界面使用 :param ui_progress: 给GUI qt 界面使用 :param ui_progress_int_value: 给GUI qt 界面使用 ''' stock_list = QA_fetch_get_stock_list().code.unique().tolist() coll_stock_day = client.stock_day coll_stock_day.create_index( [("code", pymongo.ASCENDING), ("date_stamp", pymongo.ASCENDING)] ) err = [] # saveing result def __gen_param(stock_list, coll_stock_day, ip_list=[]): results = [] count = len(ip_list) total = len(stock_list) for item in range(len(stock_list)): try: code = stock_list[item] QA_util_log_info( '##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)), ui_log ) # 首选查找数据库 是否 有 这个代码的数据 search_cond = {'code': str(code)[0:6]} ref = coll_stock_day.find(search_cond) end_date = str(now_time())[0:10] ref_count = coll_stock_day.count_documents(search_cond) # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref_count > 0: # 接着上次获取的日期继续更新 start_date = ref[ref_count - 1]['date'] # print("ref[ref.count() - 1]['date'] {} {}".format(ref.count(), coll_stock_day.count_documents({'code': str(code)[0:6]}))) else: # 当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 start_date = '1990-01-01' QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}' .format(code, start_date, end_date), ui_log ) if start_date != end_date: # 更新过的,不更新 results.extend([(code, start_date, end_date, '00', 'day', ip_list[item % count]['ip'], ip_list[item % count]['port'], item, total, ui_log, ui_progress)]) except Exception as error0: print('Exception:{}'.format(error0)) err.append(code) return results ips = get_ip_list_by_multi_process_ping(stock_ip_list, _type='stock')[:cpu_count() * 2 + 1] param = __gen_param(stock_list, coll_stock_day, ips) ps = QA_SU_save_stock_day_parallelism(processes=cpu_count() if len(ips) >= cpu_count() else len(ips), client=client, ui_log=ui_log) ps.add(do_saving_work, param) ps.run() if len(err) < 1: QA_util_log_info('SUCCESS save stock day ^_^', ui_log) else: QA_util_log_info('ERROR CODE \n ', ui_log) QA_util_log_info(err, ui_log)