def QA_DataAggrement_Stock_info(data): data['updated_date'] = list( map(lambda x: QA_util_date_int2str(x), data['updated_date'])) data['ipo_date'] = list( map(lambda x: QA_util_date_int2str(x), data['ipo_date'])) data['source'] = DATASOURCE.TDX return data
def QA_SU_save_financial_files(): """本地存储financialdata """ download_financialzip() coll = DATABASE.financial coll.create_index([("code", ASCENDING), ("report_date", ASCENDING)], unique=True) for item in os.listdir(download_path): if item[0:4] != 'gpcw': print( "file ", item, " is not start with gpcw , seems not a financial file , ignore!" ) continue date = int(item.split('.')[0][-8:]) print('QUANTAXIS NOW SAVING {}'.format(date)) if coll.find({'report_date': date}).count() < 3600: print(coll.find({'report_date': date}).count()) data = parse_filelist([item]).reset_index().drop_duplicates( subset=['code', 'report_date']).sort_index() data["crawl_date"] = QA_util_today_str() data = data.assign(report_date=data['report_date'].apply( lambda x: QA_util_date_stamp(QA_util_date_int2str(x)))) data = data.assign(crawl_date=data['crawl_date'].apply( lambda x: QA_util_date_stamp(QA_util_date_int2str(x)))) data = QA_util_to_json_from_pandas(data) try: coll.insert_many(data, ordered=False) except Exception as e: if isinstance(e, MemoryError): coll.insert_many(data, ordered=True) elif isinstance(e, pymongo.bulk.BulkWriteError): pass else: print('ALL READY IN DATABASE') print('SUCCESSFULLY SAVE/UPDATE FINANCIAL DATA')
def QA_fetch_financial_code(ndays=10): START = str(QA_util_get_pre_trade_date(QA_util_today_str(), ndays)) code = list( QA_fetch_stock_financial_calendar( QA.QA_fetch_stock_list_adv().code.tolist(), start=START)['code']) market_day = pd.DataFrame( QA_fetch_stock_basic_info_tushare())[['code', 'timeToMarket']] market_day['TM'] = market_day['timeToMarket'].apply(lambda x: str( QA_util_add_months(QA_util_date_int2str(int(x)), 0) if x > 0 else None)[0:10]) code = list(market_day[market_day['TM'] >= START]['code'].values) + code return (code)
def QA_fetch_get_stock_time_to_market(): data = QATs.get_stock_basics() return data[data['timeToMarket'] != 0]['timeToMarket'].apply(lambda x: QA_util_date_int2str(x))
def QA_fetch_get_future_min(code, start, end, frequence=FREQUENCE.ONE_MIN): ''' TODO 当前仅支持L8和L9 期货数据 分钟线 code = 'IL8' start = '2014-12-27' end = '2014-12-31' ''' market_type = MARKET_TYPE.FUTURE_CN try: if code[-2:] in ['L8']: if frequence == FREQUENCE.ONE_MIN: '''找到总路径''' product_type = '股指期货' if code[:-2] in ['IF', 'IC', 'IH' ] else '商品期货' data_type = '主连' if code[-2:] == 'L8' else '指数' path = cofund_data_path[(market_type, frequence, product_type, data_type)] main_code = code[:-2] '''确定品种路径''' if main_code.isupper(): file_code = main_code if main_code in QA_util_listdir( path) else main_code.lower() elif main_code.islower(): file_code = main_code if main_code in QA_util_listdir( path) else main_code.upper() else: raise NotImplementedError path = os.path.abspath(path + file_code) date_calendar = QA_util_get_trade_range( max( start[:10], QA_util_date_int2str(min(QA_util_listfile(path, 'csv')))), min((end[:10]), QA_util_date_int2str(max(QA_util_listfile(path, 'csv'))))) # date_calendar = QA_util_get_trade_range(start[:10],end[:10]) data = pd.DataFrame() for date in date_calendar: try: intdate = QA_util_date_str2int(date) file_temp = pd.read_csv( os.path.join(path, str(intdate) + '.csv')) data = data.append(file_temp) except: pass # data = data[data['Time']>=85900] data['type'] = frequence data['code'] = code data = data.drop_duplicates(subset=['RealDate', 'Time']) return select_DataAggrement(DATABASE_NAME.FUTURE_MIN)( DATASOURCE.COFUND, data)[start:end] else: print('当前仅支持1分钟的数据调用') raise NotImplementedError else: print('当前仅支持L8的数据调用') raise NotImplementedError except Exception as exp: print("code is ", code) print(exp.__str__) return None