def _saving_work(code, coll_stock_day, ui_log=None, err=[]): try: QA_util_log_info( '##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)), ui_log) # 首选查找数据库 是否 有 这个代码的数据 ref = coll_stock_day.find({'ts_code': str(code)}, sort=[('trade_date', 1)]) end_date = now_time() # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref.count() > 0: # 接着上次获取的日期继续更新 start_date_new_format = ref[ref.count() - 1]['trade_date'] start_date = ref[ref.count() - 1]['date'] if date_conver_to_new_format( QA_util_get_next_day(start_date, n=100)) < end_date: QA_util_log_info( 'LAST STOCK_DAY 100 days ago {}, skip. To resume please delete {}' .format(start_date_new_format, code), ui_log) else: QA_util_log_info( 'UPDATE_BOND_DAY \n Trying updating {} from {} to {}'. format(code, start_date_new_format, end_date), ui_log) if start_date_new_format != end_date: pandas_data = QA_util_to_json_from_pandas( QA_fetch_get_stock_day( str(code), date_conver_to_new_format( QA_util_get_next_day(start_date)), end_date, 'bfq')) if len(pandas_data) > 0: coll_stock_day.insert_many(pandas_data) # 当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 else: start_date = '19900101' QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'. format(code, start_date, end_date), ui_log) if start_date != end_date: pandas_data = QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, 'bfq')) if len(pandas_data) > 0: coll_stock_day.insert_many(pandas_data) except Exception as e: print(e) err.append(str(code))
def __saving_work(code, coll): try: ref_ = coll.find({'code': str(code)[0:6]}) end_time = str(now_time())[0:10] if ref_.count() > 0: start_time = ref_[ref_.count() - 1]['date'] QA_util_log_info( '##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}' .format(code, start_time, end_time)) if start_time != end_time: coll.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_index_day( str(code), QA_util_get_next_day(start_time), end_time))) else: start_time = '1990-01-01' QA_util_log_info( '##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}' .format(code, start_time, end_time)) if start_time != end_time: coll.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_index_day(str(code), start_time, end_time))) except: err.append(str(code))
def __saving_work(code, coll_stock_day): try: QA_util_log_info('##JOB01 Now Saving STOCK_DAY==== %s' % (str(code))) ref = coll_stock_day.find({'code': str(code)[0:6]}) end_date = str(now_time())[0:10] if ref.count() > 0: # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 start_date = ref[ref.count() - 1]['date'] QA_util_log_info( ' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' % (code, start_date, end_date)) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day( str(code), QA_util_get_next_day(start_date), end_date, '00'))) else: start_date = '1990-01-01' QA_util_log_info( ' UPDATE_STOCK_DAY \n Trying updating %s from %s to %s' % (code, start_date, end_date)) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, '00'))) except: err.append(str(code))
def __saving_work(code, coll): # try: ref_ = coll.find({'code': str(code)[0:6]}) end_time = str(now_time())[0:10] if ref_.count() > 0: start_time = ref_[ref_.count() - 1]['date'] QA_util_log_info( '##JOB04 Now Saving INDEX_DAY==== \n Trying updating %s from %s to %s' % (code, start_time, end_time)) if start_time != end_time: coll.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_index_day( str(code), QA_util_get_next_day(start_time), end_time))) else: start_time = '1990-01-01' QA_util_log_info( '##JOB04 Now Saving INDEX_DAY==== \n Trying updating %s from %s to %s' % (code, start_time, end_time)) coll.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_index_day(str(code), start_time, end_time)))
def saving_work(code, coll_stock_day): try: print("entering saving_work {}".format(code)) ref = coll_stock_day.find({'code': str(code)[:6]}) end_date = str(now_time())[:10] if ref.count() > 0: start_date = ref[ref.count() - 1]['date'] QA_util_log_info( 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'. format(code, start_date, end_date)) if start_date != end_date: start_date = QA_util_get_next_day(start_date) data_json = QA_fetch_get_stock_day(str(code), start=start_date, end=end_date, type_='json') coll_stock_day.insert_many(data_json) else: start_date = '1990-01-01' data_json = QA_fetch_get_stock_day(str(code), start=start_date, type_='json') coll_stock_day.insert_many(data_json) print("done saving_work {}".format(code)) except Exception as e: QA_util_log_info('error in saving ==== %s' % str(code)) print("The exception is {}".format(str(e)))
def __saving_work(code, coll_stock_day): try: QA_util_log_info( '##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)), ui_log) # 首选查找数据库 是否 有 这个代码的数据 ref = coll_stock_day.find({'code': str(code)[0:6]}) end_date = str(now_time())[0:10] # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref.count() > 0: # 接着上次获取的日期继续更新 start_date = ref[ref.count() - 1]['date'] QA_util_log_info('UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.format( code, start_date, end_date), ui_log) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), QA_util_get_next_day(start_date), end_date, '00'))) # 当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 else: start_date = '1990-01-01' QA_util_log_info('UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'.format (code, start_date, end_date), ui_log) if start_date != end_date: coll_stock_day.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, '00'))) except Exception as error0: print(error0) err.append(str(code))
def __saving_work(code, coll_stock_year): try: QA_util_log_info( '##JOB01 Now Saving STOCK_YEAR==== {}'.format(str(code)), ui_log=ui_log) ref = coll_stock_year.find({'code': str(code)[0:6]}) end_date = str(now_time())[0:10] if ref.count() > 0: # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 start_date = ref[ref.count() - 1]['date'] QA_util_log_info('UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'.format (code, start_date, end_date), ui_log=ui_log) if start_date != end_date: coll_stock_year.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), QA_util_get_next_day(start_date), end_date, '00', frequence='year'))) else: start_date = '1990-01-01' QA_util_log_info('UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'.format (code, start_date, end_date), ui_log=ui_log) if start_date != end_date: coll_stock_year.insert_many( QA_util_to_json_from_pandas( QA_fetch_get_stock_day(str(code), start_date, end_date, '00', frequence='year'))) except: err.append(str(code))
def __saving_work(code, coll): try: QA_util_log_info(logs='##{} Now Saving {}==== {}'.format( job_id, type_, str(code)), ui_log=ui_log) # 首选查找数据库 是否 有 这个代码的数据 ref = coll.find({'code': str(code)[0:6]}) end_date = str(now_time())[0:10] # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref.count() > 0: # 接着上次获取的日期继续更新 start_date = ref[ref.count() - 1]['date'] else: # 当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 start_date = beginning_date QA_util_log_info( logs='UPDATE_{} \n Trying updating {} from {} to {}'.format( type_, code, start_date, end_date), ui_log=ui_log) if start_date != end_date: start_date = QA_util_get_next_day( start_date ) if start_date != beginning_date else beginning_date coll.insert_many( QA_util_to_json_from_pandas(config['fetch']( code=str(code), start_date=start_date, end_date=end_date, frequence=frequence))) except Exception as error0: print(error0) err.append(str(code))
def __saving_work(code, coll_option_day): try: QA_util_log_info('##JOB12 Now Saving OPTION_DAY==== {}'.format( str(code)), ui_log=ui_log) # 首选查找数据库 是否 有 这个代码的数据 # 期权代码 从 10000001 开始编码 10001228 ref = coll_option_day.find({'code': str(code)[0:8]}) end_date = str(now_time())[0:10] # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref.count() > 0: # 接着上次获取的日期继续更新 start_date = ref[ref.count() - 1]['date'] QA_util_log_info(' 上次获取期权日线数据的最后日期是 {}'.format( start_date), ui_log=ui_log) QA_util_log_info('UPDATE_OPTION_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'.format( code, start_date, end_date), ui_log=ui_log) if start_date != end_date: start_date0 = QA_util_get_next_day(start_date) df0 = QA_fetch_get_option_day(code=code, start_date=start_date0, end_date=end_date, frequence='day', ip=None, port=None) retCount = df0.iloc[:, 0].size QA_util_log_info("日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库" .format(start_date0, end_date, code, retCount), ui_log=ui_log) coll_option_day.insert_many( QA_util_to_json_from_pandas(df0)) else: QA_util_log_info("^已经获取过这天的数据了^ {}".format( start_date), ui_log=ui_log) else: start_date = '1990-01-01' QA_util_log_info('UPDATE_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'.format (code, start_date, end_date), ui_log=ui_log) if start_date != end_date: df0 = QA_fetch_get_option_day(code=code, start_date=start_date, end_date=end_date, frequence='day', ip=None, port=None) retCount = df0.iloc[:, 0].size QA_util_log_info("日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ " .format(start_date, end_date, code, retCount), ui_log=ui_log) coll_option_day.insert_many( QA_util_to_json_from_pandas(df0)) else: QA_util_log_info( "*已经获取过这天的数据了* {}".format(start_date), ui_log=ui_log) except Exception as error0: print(error0) err.append(str(code))
def __init__(self, featuredata, feature_name=None, stock_data=None, returnday=5, host=clickhouse_ip, port=clickhouse_port, user=clickhouse_user, password=clickhouse_password) -> None: self.feature = featuredata self.featurename = featuredata.name if feature_name is None else feature_name self.feature.name = self.featurename self.codelist = self.feature.index.levels[1].unique().tolist() self.start = str(self.feature.index.levels[0][0])[0:10] self.end = str(self.feature.index.levels[0][-1])[0:10] self._host = host self._port = port self._user = user self._password = password self.factorclient = clickhouse_driver.Client(host=self._host, port=self._port, user=self._user, password=self._password, database='factor') self.dataclient = QACKClient(host=self._host, port=self._port, user=self._user, password=self._password) if stock_data is None: self.stock_data = self.dataclient.get_stock_day_qfq_adv( self.codelist, self.start, QA_util_get_next_day(self.end, returnday)) else: self.stock_data = stock_data self.returns = self.make_ret(self.stock_data.data).dropna()
def _saving_work(code, coll_stock_tech_indicator): try: #logging.debug("process {} now working on stock {}".format(os.getpid(), code)) QA_util_log_info('Now Processing ==== %s' % (code)) ref = coll_stock_tech_indicator.find({'code': str(code)[:6]}) end_date = str(datetime.date.today()) if ref.count() > 0: start_date = ref[ref.count() - 1]['date'] if start_date != end_date: start_date = QA_util_get_next_day(start_date) df_indicator = QA_SU_calculate_stock_tech_indicator( code, start_date, end_date) if len(df_indicator) > 0: df_indicator['date'] = df_indicator.index.strftime( '%Y-%m-%d') df_indicator['date_stamp'] = df_indicator[ 'date'].apply(lambda x: QA_util_date_stamp(x)) indicator_json = QA_util_to_json_from_pandas( df_indicator) coll_stock_tech_indicator.insert_many(indicator_json) #logging.debug("df_indicator successfully inserted into DB") else: start_date = '1990-01-01' df_indicator = QA_SU_calculate_stock_tech_indicator( code, start_date, end_date) if len(df_indicator) > 0: df_indicator['date'] = df_indicator.index.strftime( '%Y-%m-%d') df_indicator['date_stamp'] = df_indicator['date'].apply( lambda x: QA_util_date_stamp(x)) indicator_json = QA_util_to_json_from_pandas(df_indicator) coll_stock_tech_indicator.insert_many(indicator_json) #logging.debug("df_indicator successfully inserted into DB") except Exception as e: QA_util_log_info('error in processing ==== %s' % str(code)) err.append(code) print("The exception is {}".format(str(e)))
def __saving_work(self, code): def __QA_log_info(code, end_time, start_time): def loginfo(prefix='', astr='', listCounts=5): if len(self._loginfolist) < listCounts: self._loginfolist.append(astr) else: str = '' for i in range(len(self._loginfolist)): str += self._loginfolist[i] + ' ' str += astr QA_util_log_info(prefix.format(str), self.ui_log) self._loginfolist.clear() index_or_etf = self.get_index_or_etf_from_code(code) prefix = '##JOB04 Saving {}_DAY ==== Trying updating\n{}'.format( index_or_etf, '{}') loginfo(prefix, ' {} from {} to {}'.format(code, start_time, end_time)) # log_info = '##JOB04 Saving {}_DAY====\nTrying updating {} from {} to {}'.format( # index_or_etf, # code, # start_time, # end_time # ) # QA_util_log_info( # log_info, # ui_log=self.ui_log # ) try: search_cond = {'code': str(code)[0:6]} ref_ = get_coll().find(search_cond) ref_count = get_coll().count_documents(search_cond) end_time = str(now_time())[0:10] if ref_count > 0: start_time = ref_[ref_count - 1]['date'] __QA_log_info(code, end_time, start_time) if start_time != end_time: get_coll().insert_many( QA_util_to_json_from_pandas( QA_fetch_get_index_day( str(code), QA_util_get_next_day(start_time), end_time))) else: try: start_time = '1990-01-01' __QA_log_info(code, end_time, start_time) get_coll().insert_many( QA_util_to_json_from_pandas( QA_fetch_get_index_day(str(code), start_time, end_time))) except Exception as e: start_time = '2009-01-01' __QA_log_info(code, end_time, start_time) get_coll().insert_many( QA_util_to_json_from_pandas( QA_fetch_get_index_day(str(code), start_time, end_time))) except Exception as e: QA_util_log_info(e, ui_log=self.ui_log) self.err.append(str(code)) QA_util_log_info(self.err, ui_log=self.ui_log)
def __saving_work(self, code): try: QA_util_log_info( '##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)), self.ui_log) time.sleep(0.2) # 首选查找数据库 是否 有 这个代码的数据 search_cond = {'code': str(code)[0:6]} ref = get_coll(self.client).find(search_cond) ref_count = get_coll(self.client).count_documents(search_cond) end_date = now_time() # 当前数据库已经包含了这个代码的数据, 继续增量更新 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现 if ref_count > 0: # 接着上次获取的日期继续更新 start_date_new_format = ref[ref_count - 1]['trade_date'] start_date = ref[ref_count - 1]['date'] QA_util_log_info( 'UPDATE_STOCK_DAY: Trying updating {} {} from {} to {}'. format(ref_count, code, start_date_new_format, end_date), self.ui_log) if start_date_new_format != end_date: df = QA_fetch_get_stock_day( str(code), date_conver_to_new_format( QA_util_get_next_day(start_date)), end_date, 'bfq') if not (df is None) and len(df) > 0: get_coll(self.client).insert_many( QA_util_to_json_from_pandas(df)) else: QA_util_log_info( 'UPDATE_STOCK_DAY: Trying updating {} {} from {} to {} ---- No Data' .format(ref_count, code, start_date_new_format, end_date), self.ui_log) # 当前数据库中没有这个代码的股票数据, 从1990-01-01 开始下载所有的数据 # 一次只返回 5000 条,所以有些股票得获取两次 else: start_date = '19900101' QA_util_log_info( 'FETCH_ALL_STOCK_DAY: Trying fetching all data {} from {} to {}' .format(code, start_date, end_date), self.ui_log) if start_date != end_date: # 第一次获取 df = QA_fetch_get_stock_day(str(code), start_date, end_date, 'bfq') if not (df is None): if len(df) >= 5000: again_date = QA_util_get_last_day(df['date'].min()) QA_util_log_info( 'FETCH_ALL_STOCK_DAY: Trying updating again {} from {} to {}' .format(code, start_date, again_date), self.ui_log) df2 = QA_fetch_get_stock_day( str(code), start_date, again_date, 'bfq') df = df.append(df2) get_coll(self.client).insert_many( QA_util_to_json_from_pandas(df)) else: QA_util_log_info( 'FETCH_ALL_STOCK_DAY: Trying fetching all data {} from {} to {} ---- No Data' .format(code, start_date, end_date), self.ui_log) except Exception as e: print(e) self.err.append(str(code))
#%% data = QA.QAFetch.QATdx.QA_fetch_get_stock_transaction('000001','2015-06-05','2015-06-06') data.index = pd.to_datetime(data.index) type(data.index[0])('2019-01-01') #%% data = QA.QAFetch.QATdx.QA_fetch_get_future_list() data['signal'] = data['code'].apply(lambda x:x[-2:]) data[data['signal'].isin(['L8','L9'])] 1300/8200 assert False STOCK_DAY = (['code', 'open', 'high', 'low','close', 'volume', 'amount', 'date'],'date') ['code','open'] in from QUANTAXIS.QAUtil import (DATABASE, QA_Setting, QA_util_date_stamp, QA_util_date_valid, QA_util_dict_remove_key, QA_util_log_info, QA_util_code_tostr, QA_util_date_str2int, QA_util_date_int2str, QA_util_sql_mongo_sort_DESCENDING, QA_util_time_stamp, QA_util_to_json_from_pandas, trade_date_sse,QA_util_dateordatetime_valid,QA_util_to_anyformat_from_pandas, QA_util_get_last_day,QA_util_get_next_day,QA_util_get_real_date) QA_util_time_stamp('2016-01-01 09:00:00') QA_util_time_stamp('2016-01-01') QA_util_time_stamp('2016-01-01 00:00:00') QA_util_get_real_date(QA_util_get_last_day('2019-07-01'), towards = -1) QA_util_get_real_date(QA_util_get_next_day('2019-07-01'), towards = 1)