def data_step(self, index_info: IndexInfo):
"""
数据准备阶段
:param index_info:
:return:
"""
newest_bar = database_manager.get_newest_bar_data(
index_info.symbol, index_info.exchange, Interval.DAILY)
if newest_bar is None:
print('当前数据未初始化')
filling_days = (datetime.now() -
newest_bar.datetime).days + 3 # 需要补齐数据的天数
ds = DataSource()
if index_info.cal_close:
# 保存最近收盘数据
ds.save_bar_data(index_info.symbol, index_info.alias, filling_days)
if index_info.cal_finance:
# 保存最近金融相关数据
trade_dates = [
bar.datetime for bar in database_manager.load_bar_data(
index_info.symbol, index_info.exchange, Interval.DAILY,
datetime.now() -
timedelta(days=filling_days), datetime.now())
]
ds.save_index_finance(trade_dates=trade_dates,
index_code=index_info.index_code)
def get_bar_data_available(self) -> List[Dict]:
""""""
data = database_manager.get_bar_data_statistics()
for d in data:
oldest_bar = database_manager.get_oldest_bar_data(
d["symbol"], Exchange(d["exchange"]), Interval(d["interval"]))
d["start"] = oldest_bar.datetime
newest_bar = database_manager.get_newest_bar_data(
d["symbol"], Exchange(d["exchange"]), Interval(d["interval"]))
d["end"] = newest_bar.datetime
return data
def download_data_from_tdx(download_futures, from_date=None, back_days=None):
"""
:param download_futures: ["rb2009.SHFE"]
:param from_date: 2020-7-8
:param back_days:
"""
if tdxdata_client.init():
print("数据服务器登录成功")
else:
print("数据服务器登录失败")
return
for future in download_futures:
_future = future.split(".")
symbol = _future[0]
exchange = Exchange.__dict__[_future[1]]
interval = Interval.MINUTE
if from_date:
start = datetime.datetime.strptime(from_date, "%Y-%m-%d")
else:
bar = database_manager.get_newest_bar_data(symbol, exchange,
interval)
if bar:
start = bar.datetime
else:
start = datetime.datetime(2012, 1, 1)
if back_days:
start = start.replace(tzinfo=None)
_start = datetime.datetime.now() - datetime.timedelta(days=3)
start = _start if start > _start else start
# 下载数据
req = HistoryRequest(symbol,
exchange,
start,
datetime.datetime.now(),
interval=interval)
data = tdxdata_client.query_history(req)
# 写入数据库
if data:
database_manager.save_bar_data(data)
print(f"{symbol}更新完成：{data[0].datetime} -- {data[-1].datetime}")
print("数据全部更新完毕")
def __init_tradeDay(self,start:datetime,end:datetime):
code = TRAY_DAY_SIMBOL
db_bar_start = database_manager.get_oldest_bar_data(code,Exchange.SSE,Interval.DAILY)
db_bar_end = database_manager.get_newest_bar_data(code,Exchange.SSE,Interval.DAILY)
print(f"StrategyBridage: __init_tradeDay(), start={start},end={end}")
if((not db_bar_start is None ) and (not db_bar_end is None)):
if(start >= db_bar_start.datetime and end <= db_bar_end.datetime):
return
print(f"StrategyBridage: __init_tradeDay(), update tradeDay!!!!")
database_manager.delete_bar_data(code,Exchange.SSE,Interval.DAILY)
start = start - timedelta(days=15)
end = end + timedelta(days=15)
save_tradeday_from_jqdata(start,end)
def main():
db_conn1 = sqlite3.connect(
r'D:\PythonCodes\vnpy\.vntrader\symb_list.db'
) # Connect to a database that hosts all stock symbols (i.e. corporate database)
db_conn2 = sqlite3.connect(
r'D:\PythonCodes\vnpy\.vntrader\database.db'
) # Connect to the main database that host all stock data (i.e. bar database)
# dict_example = {'name':['Microsoft Corp.'],
# 'cname': ['微软公司'],
# 'symbol':['MSFT']}
# dict_example = {'name':['Microsoft Corp.','Apple Company'],
# 'cname': ['微软公司', '苹果公司'],
# 'symbol':['MSFT', 'AAPL']}
# dict_example = {'name':['Microsoft Corp.','Apple Company', 'Facebook'],
# 'cname': ['微软公司', '苹果公司', '脸书'],
# 'symbol':['MSFT', 'AAPL', 'FB']}
# dict_example = {'name':['Microsoft Corp.','Apple Company', 'Facebook', 'Amazon'],
# 'cname': ['微软公司', '苹果公司', '脸书','亚马逊'],
# 'symbol':['MSFT', 'AAPL', 'FB', 'AMZN']}
# df_example = pd.DataFrame.from_dict(dict_example)
# df_example.to_csv(Path.cwd().joinpath('temp.csv'), encoding='utf_8_sig', index=False)
# df_example = pd.read_csv(Path.cwd().joinpath('temp.csv'), encoding='utf_8_sig')
# df_allstocks = ak.get_us_stock_name() # Download all stock symbols using AkShare service. Expect to run this line and update stock symbols periodically.
# df_allstocks.to_csv(Path.cwd().joinpath('temp.csv'), encoding='utf_8_sig', index=False) # Save all stock symbols to a csv file. This is for testing purpose.
df_example = pd.read_csv(
Path.cwd().joinpath('temp.csv'), encoding='utf_8_sig'
) # Load all stock symbols from the csv file. This is for testing purpose.
# df_example = df_example.iloc[0:2, :] # Only take a few lines for testing. This is for testing purpose.
df_example.to_sql(
"dbcorpdata", db_conn1,
if_exists='replace') # Save all stock symbols to corporate database
df_corpsdata_dl = pd.read_sql_query(
"SELECT * from dbcorpdata",
db_conn1) # Load all stock symbols from corporate database
df_bardata_ex = pd.read_sql_query(
"SELECT * from dbbardata",
db_conn2) # Load all existing stock data from bardata database
totalSymbol = len(df_corpsdata_dl.symbol)
procSymbol = 0
forcedFullLoad = False
for s in df_corpsdata_dl.symbol: # For each symbol read from the corporate database
try:
procSymbol += 1
progress(procSymbol, totalSymbol)
bars = []
bar_latestBarOneStock_ex = database_manager.get_newest_bar_data(
s, Exchange.LOCAL, Interval.DAILY
) # Find the latest bar record for that symbol from the bar database
df_allBarOneStock_dl = ak.stock_us_daily(
symbol=s, adjust="qfq"
).fillna(method='ffill').fillna(
0
) # Download the history data for that symbol using AkShare service.
# Fill NaN or Null fields with previous value, and then zero.
if (
(bar_latestBarOneStock_ex is not None) and (~forcedFullLoad)
): # If the bar database contains this symbol, and not full load we will decide if incremental data will be needed or full history data will be saved to bar database
lastDayDate_ex = bar_latestBarOneStock_ex.datetime.replace(
tzinfo=None
) # VNPY datetime is aware type, but the AkShare datetime is unaware type
latestDayDate_dl = df_allBarOneStock_dl.index[
-1] # Be careful of the difference between last day and latest date.
dailyDataMatched = comp_bar(
bar_latestBarOneStock_ex, df_allBarOneStock_dl
) # This is a simplified logic check to compare the OHLC prices and see if they are all equal.
if dailyDataMatched: # If the close prices from existing and new sources match, we assume data remain correct and will only incrementally update
time_range = DateTimeRange(
lastDayDate_ex, latestDayDate_dl
) # Find the date range for incremental update
for dt in time_range.range(timedelta(days=1)):
# print(dt)
if dt == latestDayDate_dl: # When last date equals latest date, there is still a day in the date range, and we need to break the loop
# print('I am going to break...')
break
bar = BarData(
symbol=s,
exchange=Exchange.LOCAL,
datetime=dt, # Here dt is a native datetime object
interval=Interval.DAILY,
volume=df_allBarOneStock_dl.loc[dt].volume,
open_price=df_allBarOneStock_dl.loc[dt].open,
high_price=df_allBarOneStock_dl.loc[dt].high,
low_price=df_allBarOneStock_dl.loc[dt].low,
close_price=df_allBarOneStock_dl.loc[dt].close,
open_interest=0,
gateway_name='Sim')
bars.append(bar)
# print('only add incremental updates for '+s)
else: # If the close prices from existing and new sources do not match, we assume data are corrupted and will fully update
for i, dt in enumerate(df_allBarOneStock_dl.index):
bar = BarData(
symbol=s,
exchange=Exchange.LOCAL,
datetime=dt.to_pydatetime(
), # Convert to a datetime object
interval=Interval.DAILY,
volume=df_allBarOneStock_dl.loc[dt].volume,
open_price=df_allBarOneStock_dl.loc[dt].open,
high_price=df_allBarOneStock_dl.loc[dt].high,
low_price=df_allBarOneStock_dl.loc[dt].low,
close_price=df_allBarOneStock_dl.loc[dt].close,
open_interest=0,
gateway_name='Sim')
bars.append(bar)
# print('correct database data for '+s)
else: # If bar database does not have this symbol, or just want to force full load, we will fully update
for i, dt in enumerate(df_allBarOneStock_dl.index):
bar = BarData(
symbol=s,
exchange=Exchange.LOCAL,
datetime=dt.to_pydatetime(
), # Convert to a datetime object
interval=Interval.DAILY,
volume=df_allBarOneStock_dl.loc[dt].volume,
open_price=df_allBarOneStock_dl.loc[dt].open,
high_price=df_allBarOneStock_dl.loc[dt].high,
low_price=df_allBarOneStock_dl.loc[dt].low,
close_price=df_allBarOneStock_dl.loc[dt].close,
open_interest=0,
gateway_name='Sim')
bars.append(bar)
# print('reload data for '+s)
database_manager.save_bar_data(
bars) # Push the updates to the bar database
print("Saved stock data of " + s + " into database.")
except: # When exceptoin occurs, assume it is because database buffer full, reconnect databases.
time.sleep(5)
print('Exception detected. Now reconnect to the databases.')
db_conn1.close()
db_conn2.close()
db_conn1 = sqlite3.connect(
r'D:\PythonCodes\vnpy\.vntrader\symb_list.db')
db_conn2 = sqlite3.connect(
r'D:\PythonCodes\vnpy\.vntrader\database.db')
time.sleep(5)
db_conn1.close() # When done with the database, close the connection
db_conn2.close() # When done with the database, close the connection
if tdxdata_client.init():
print("数据服务器登录成功")
else:
print("数据服务器登录失败")
import sys
sys.exit(0)
for future in futures:
symbol = future.upper() + "99"
exchange = Exchange.__dict__[future_contracts[future]["exchange"]]
interval = Interval.MINUTE
# 查询数据库中的最新数据
# start = datetime(2010, 6, 1)
# 增量更新数据
bar = database_manager.get_newest_bar_data(symbol, exchange, interval)
if bar:
start = bar.datetime
else:
start = datetime(2012, 1, 1)
# 下载数据
req = HistoryRequest(symbol,
exchange,
start,
datetime.now(),
interval=interval)
data = tdxdata_client.query_history(req)
# 写入数据库
if data:
