def process_dir2file(w, mydir, myfile): df = read_data_dataframe(myfile) all_set = set(df.index) for dirpath, dirnames, filenames in os.walk(mydir): for filename in filenames: # 这个日期需要记得修改 if filename < "2017-01-01.csv": continue filepath = os.path.join(dirpath, filename) df1 = read_constituent(filepath) # print(filepath) if df1 is None: continue if df1.empty: continue curr_set = set(df1['wind_code']) diff_set = curr_set - all_set if len(diff_set) == 0: continue print(filepath) df2 = download_ipo_last_trade_trading(w, list(diff_set)) df = pd.concat([df, df2]) all_set = set(df.index) # 出于安全考虑,还是每次都保存 write_data_dataframe(myfile, df) df['wind_code'] = df.index df.sort_values(by=['ipo_date', 'wind_code'], inplace=True) del df['wind_code'] write_data_dataframe(myfile, df)
def download_000300(w, trading_days): # 4月8号是指数的发布日期 trading_days = trading_days['2005-04-08':date_str] # 下载多天数据,以另一数据做为标准来下载 # 比如交易数据是10月8号,那就得取10月7号,然后再平移到8号,如果7号没有数据那就得9月30号 path = os.path.join(__CONFIG_H5_STK_FACTOR_DIR__, 'test.csv') date_index = read_data_dataframe(path) # dates = list(date_index.index) download_index_weight(w, trading_days, "000300.SH", __CONFIG_H5_STK_WEIGHT_DIR__)
from kquant_data.api import all_instruments from kquant_data.wind_resume.wsd import download_daily_at from kquant_data.config import __CONFIG_H5_STK_FACTOR_DIR__, __CONFIG_H5_STK_DIR__ if __name__ == '__main__': w.start() path = os.path.join(__CONFIG_H5_STK_DIR__, '1day', 'Symbol.csv') Symbols = all_instruments(path) wind_codes = list(Symbols['wind_code']) # 下载多天数据,以另一数据做为标准来下载 # 比如交易数据是10月8号,那就得取10月7号,然后再平移到8号,如果7号没有数据那就得9月30号 path = os.path.join(__CONFIG_H5_STK_FACTOR_DIR__, 'roe.csv') date_index = read_data_dataframe(path) field = 'pb_lf' df = None for i in range(len(date_index)): print(date_index.index[i]) date_str = date_index.index[i].strftime('%Y-%m-%d') df_new = download_daily_at(w, wind_codes, field, date_str, "Days=Alldays") if df is None: df = df_new else: df = pd.concat([df, df_new]) path = os.path.join(__CONFIG_H5_STK_FACTOR_DIR__, '%s.csv' % field)
if __name__ == '__main__': w.start() root_path = os.path.join(__CONFIG_H5_OPT_DIR__, 'optioncontractbasicinfo', '510050.SH.csv') # 这个只要执行一次就可以了,以后每次更新时都跳过即可 if False: df_new = download_optioncontractbasicinfo(w, status='delisted') df_new = df_new.set_index('wind_code') write_data_dataframe(root_path, df_new) # 下载新数据,并合并上旧数据 if True: df_old = read_data_dataframe(root_path) df_old['listed_date'] = pd.to_datetime(df_old['listed_date']) df_old['expire_date'] = pd.to_datetime(df_old['expire_date']) df_new = download_optioncontractbasicinfo(w, status='trading') df_new = df_new.set_index('wind_code') df_new['listed_date'] = pd.to_datetime(df_new['listed_date']) df_new['expire_date'] = pd.to_datetime(df_new['expire_date']) df = pd.concat([df_old, df_new]) # df = df_old # 将老数据删除,因为合约可能因为除权而变数据 df = df[~df.index.duplicated(keep='last')] write_data_dataframe(root_path, df) df = read_optioncontractbasicinfo(root_path)
""" import os import pandas as pd from kquant_data.config import __CONFIG_H5_FUT_FACTOR_DIR__ from kquant_data.xio.csv import read_data_dataframe, write_data_dataframe from kquant_data.future.symbol import wind_code_2_InstrumentID if __name__ == '__main__': input_path = os.path.join(__CONFIG_H5_FUT_FACTOR_DIR__, 'trade_hiscode') output_path = os.path.join(__CONFIG_H5_FUT_FACTOR_DIR__, 'trade_hiscode.csv') df_csv = pd.DataFrame(columns=['trade_hiscode']) for dirpath, dirnames, filenames in os.walk(input_path): for filename in filenames: shotname, extension = os.path.splitext(filename) dirpath_filename = os.path.join(dirpath, filename) print(dirpath_filename) _df = read_data_dataframe(dirpath_filename) _df.dropna(inplace=True) df_csv.loc[shotname] = None df_csv.loc[shotname]['trade_hiscode'] = _df.iat[-1, 0] df = wind_code_2_InstrumentID(df_csv, 'trade_hiscode') df.index.name = 'product' write_data_dataframe(output_path, df) print(output_path) # print(df) debug = 1