def update_bars_monthly(self):
prev_month = self.latest_data_month
flag = 0
for s in self.symbol_list:
if self.next_month_bar[s]:
self.latest_symbol_data[s].append(self.next_month_bar[s])
self.next_month_bar[s] = []
else:
try:
bar = next(self._get_new_bar(s))
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
cur_month = datetime.datetime.strptime(
bar[0], "%Y-%m-%d").month
if cur_month != self.latest_data_month:
self.next_month_bar[s] = bar
flag = 1
if math.isnan(self.latest_symbol_data[s][0][1].loc[self.factor]) \
and self.latest_data_month == self.start_month:
self.factor_na += 1
elif cur_month == self.latest_data_month:
self.latest_symbol_data[s].append(bar)
prev_month = cur_month
self.latest_data_month = prev_month
if flag == 1:
self.events.put(MarketEvent())
def update_bars(self) -> bool:
"""
Updates a datetime, thus shift to the next latest bar.
"""
for s in self._symbol_dict.keys():
if self._latest_idx[s] == self._symbol_dict[s]['len']:
self._latest_idx[s] = None
self._system_dt = self._next_datetime()
# find symbols, which have updated time in the data
result = False
upd_symbols = []
for s in self._symbol_dict.keys():
if self._latest_idx[s] is not None:
result = True
if self._system_dt in self._symbol_data[s].index:
self._latest_idx[s] += 1
upd_symbols.append(s)
if self._events is not None:
self._events.put(MarketEvent(upd_symbols, self._system_dt))
self.continue_backtest = result
return result
def update_bars(self):
"""
Pushes the latest bar to the latest_symbol_data structure
for all symbols in the symbol list.
"""
for s in self.symbol_list:
try:
bar = next(self._get_new_bar(s))
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
self.events.put(MarketEvent())
def get_new_data(self):
"""Return a list of market events (new bars) for all symbols from
all exchanges for the just-elapsed time period. Add new bar data
to queue for storage in DB, after current minutes cycle completes."""
# record bar parse performance
self.logger.debug("Started parsing new ticks.")
start_parse = time.time()
for exchange in self.exchanges:
exchange.parse_ticks()
end_parse = time.time()
duration = round(end_parse - start_parse, 5)
self.logger.debug("Parsed " + str(self.total_instruments) +
" instruments' ticks in " + str(duration) +
" seconds.")
self.track_performance(duration)
# wrap new 1 min bars in market events
new_market_events = []
for exchange in self.exchanges:
bars = exchange.get_new_bars()
for symbol in exchange.get_symbols():
for bar in bars[symbol]:
event = MarketEvent(exchange.get_name(), bar)
new_market_events.append(event)
# add bars to save-to-db-later queue
# TODO: store new bars concurrently with a processpool
self.bars_save_to_db.put(event)
return new_market_events
def update_bars(self):
"""
Pushes the latest bar to the latest_symbol_data structure
for all symbols in the symbol list.
"""
for s in self.symbol_list:
bar = self._get_new_bar(s)
self.latest_symbol_data[s].append(bar)
self.events.put(MarketEvent())
def update_bars(self):
for s in self.symbol_list:
try:
bar = next(self._get_new_bar(s))
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
self.events.put(MarketEvent())
def updateBars(self):
for s in self.universe:
try:
bar = next(self.getNewSymbolBar(s))
except StopIteration:
self.continueBacktest = False
else:
if bar is not None:
self.latestData[s].append(bar)
self.eventQueue.append(MarketEvent())
def update_latest_data(self):
for symbol in self.symbol_list:
data = None
try:
data = next(self._get_new_data(symbol))
except StopIteration:
self.continue_backtest = False
if data is not None:
self.latest_symbol_data[symbol].append(data)
self.events.put(MarketEvent())
def generate_timeAxis(self, start=None, end=None, freq=None):
freq = self.freq if freq is None else freq
endtime = datetime.utcnow().replace(
tzinfo=timezone('utc')) if end is None else end
starttime = datetime(
2013, 3, 31, 0, 0, 0,
tzinfo=timezone('utc')) if start is None else start
timeTbl = pd.DataFrame()
timeTbl['timeAxis'] = pd.date_range(start=starttime,
end=endtime,
freq=freq)
timeTbl['trade_date'] = timeTbl['timeAxis'].dt.strftime(
"%Y%m%d%H%M%S").astype('int64')
timeTbl['exch_timeid'] = (timeTbl['timeAxis'].astype('int32') /
1000000).astype('int64')
self.df_calTbl = timeTbl
for row in timeTbl.itertuples():
self.cursorTime = row
self.cursorBar = self.get_current_bar()
self.histBars = self.get_processedBars()
self.eventQ.put(MarketEvent(row.trade_date))
yield row
else:
# newRow= pd.DataFrame.from_dict({'timeAxis':['cash'],'trade_date':[self.initial_capital],'exch_timeid':[self.initial_capital]
t = datetime.utcnow().replace(tzinfo=timezone('utc'))
if self.freq == '1d':
t = t.replace(
second=0, microsecond=0, minute=0, hour=0,
day=t.day) + timedelta(days=1)
Row = namedtuple('Pandas',
['timeAxis', 'trade_date', 'exch_timeid'])
newRow = Row(t, t.strftime("%Y%m%d%H%M%S"),
(int(t.timestamp()) / 1000000))
while True:
self.cursorTime = newRow
self.cursorBar = self.get_current_bar()
self.eventQ.put(MarketEvent(newRow.trade_date))
yield newRow
else:
yield None
def update_bars(self):
for s in self.symbol_list:
try:
bar = self.generator_dict[s].next()
print(bar)
print(getattr(bar, 'adj_close'))
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.internal_store[s].append(bar)
self.events.put(MarketEvent())
def update_bars(self):
# pushes the most recent data to the latest_symbol_data structure for each symbol
for s in self.symbol_list:
try:
bar = self._get_new_bar(s).__next__()
except StopIteration:
self.continue_backtest = False
else:
if bar is None:
self.latest_symbol_data[s].append(bar)
market = MarketEvent()
self.events.qput(market)
def update_bars(self):
for s in self.symbol_list:
try:
# bar = self._get_new_bar(s).next() # python 2
bar = next(self._get_new_bar(s))
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s] = self.latest_symbol_data[
s].append(bar, ignore_index=True)
self.events.put(MarketEvent())
def update_bars(self):
"""
Отправляет последний бар в структуру данных инструментов для всех инструментов в списке.
"""
for s in self.symbol_list:
try:
bar = next(self._get_new_bar(s))
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
self.events.put(MarketEvent(bar[1]))
def update_bars(self):
"""
模拟实盘数据接收，从存储的数据字典中获取1条新数据，并存入latest_symbol_data以备回测使用
"""
for s in self.symbol_list:
try:
bar = next(self.symbol_iters[s])
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
self.events.put(MarketEvent())
def update_bars(self):
"""
将最近的数据条目放入到latest_symbol_data结构中。
"""
try:
# 获取最新的基准数据，格式为Dictionary
#bar_benchmark = dict(next(self.get_next_bar(self.benchmark))[1])
bar_benchmark = next(self.get_next_bar(self.benchmark))
# 将最新的基准bar写入latest_symbol_data
#self.latest_symbol_data[self.benchmark] = self.latest_symbol_data[self.benchmark].append(bar_benchmark,ignore_index=True)
self.latest_symbol_data[self.benchmark].append(bar_benchmark)
datetime = bar_benchmark['date_time']
# 获取股票池数据
for sym in self.symbol_list:
# 如果当前代码为基准代码，跳过（解决基准回测重复取数问题）
if sym == self.benchmark:
continue
# 判断sym数据起始日期是否与回测当前日期一致
if self.symbol_start[sym] == bar_benchmark['date_time']:
self.symbol_update[sym] = True # 如果日期一致，则可以开始取数
# 如果可以取数，则获取新的bar；否则根据基准日期填充数据
if self.symbol_update[sym]:
#bar = dict(next(self.get_next_bar(sym))[1])
bar = next(self.get_next_bar(sym))
else:
bar = {
'date_time': datetime,
'symbol': sym,
'name': 'Unknown',
'open': 0,
'low': 0,
'high': 0,
'close': 0,
'chg': 0,
'pct_chg': 0,
'volume': 0,
'turnover': 0,
'suspend': True
}
# 将最新的股票bar写入写入latest_symbol_data
#self.latest_symbol_data[sym] = self.latest_symbol_data[sym].append(bar,ignore_index=True)
self.latest_symbol_data[sym].append(bar)
self.events.put(MarketEvent()) # 写入市场事件
except StopIteration: # 获取不到数据，循环终止，停止回测
self.continue_backtest = False
def update_bars(self):
for s in self.symbol_list:
try:
bar = next(self._get_new_bar(s))
# bar is of form
# (index, row)
# row is a dict i belive with keys of column names
except StopIteration:
print("finished backtest")
self.finished = True
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
self.events.put(MarketEvent())
def update_bars(self):
#before updating bars, we need to grab the next timestamp and delete it from our existing ones
try:
for s in self.symbols:
bar = self.get_new_bar(s)
if bar.name not in list(self.latest_stamps):
self.latest_stamps = self.latest_stamps.append(
pd.Index([bar.name]))
self.latest_symbol_data[s] = self.latest_symbol_data[s].append(
bar)
self.events.put(MarketEvent(
self.latest_stamps[-1])) #FIRST START, IMPORTANT
except StopIteration:
self.continue_backtest = False
print('BACKTEST COMPLETE.')
def update_bars(self):
"""
Pushes the latest bar to the latest_symbol_data structure
for all symbols in the symbol list.
"""
for symbol in self.symbols:
try:
bar = next(self.bars_generator[symbol])
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_bars_data[symbol] = self.latest_bars_data[symbol].append(bar)
self.latest_bars_data[symbol].index.rename('datetime', inplace=True)
self.events.put(item=MarketEvent())
def update_bars(self):
"""
Pushes the latest bars to the bars_queue for each symbol
in a tuple OHLCVI format: (datetime, open, high, low,
close, volume, open interest).
"""
for s in self.symbol_list:
try:
bar = next(self._get_new_bar(s))
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
self.events.put(MarketEvent())
def update_bars(self):
'''
Pushes the latest bar to the latest_symbol_data structure for all symbols in the symbol list
'''
for s in self.symbol_list:
try:
bar = next(self._get_new_bar(s)) # grab the new bar
except StopIteration:
self.continue_backtest = False # of there is no next bar then the backtest is over
else:
if bar is not None:
self.latest_symbol_data[s].append(
bar) # tack the next bar onto the latest_symbol_data
self.events.put(MarketEvent())
def update_bars(self):
"""
read each row of symbol_data into latest_symbol_data
then, generate MarketEvent
it's used in backtest module
"""
for s in self.symbol_list:
try:
bar = next(self._get_new_bar(s))
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
self.events.put(MarketEvent())
def update_bars(self):
"""
向latest_symbol_data中不断推入每日股价信息
"""
for s in self.symbol_list:
try:
bar = next(
self._get_new_bar(s)
) #调用next函数以迭代dataframe，返回一个tuple(datatime,[series for stock info])
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s].append(
bar) #在latest_symbol_data中不断append最新一天的bar
self.events.put(MarketEvent())
def update_bars(self):
"""
Pushes the latest bar to the latest_symbol_data structure
for all symbols in the symbol list.
"""
for s in self.symbol_list:
try:
bar = next(self._get_new_bar(s)) # 与yeild语句对应，下一行数据
except StopIteration:
self.continue_backtest = False # 到达数据的最近日期，令flag continue_backtest转变为False
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
# 这里赋予了latest_symbol_data的值
self.events.put(
MarketEvent()) # mac（策略）执行的开端。MarketEvent的属性就是type == ‘market’
def update_bars(self):
"""
Pushes the latest bar to the latest_symbol_data structure
for all symbols in the symbol list
"""
for s in self.symbol_list:
try:
bar = self._get_new_bar(s).next()
except StopIteration:
self.continue_backtest = False
else:
#THIS IS WHERE YOU SHOULD BE CHECKING IF YOUR GETTING ANY DATA FOR THAT SYMBOL
#WHAT HAPPENS IF YOU DONT APPEND A BAR FOR THE SYMBOL ???
if bar is not None:
self.latest_symbol_data[s].append(bar)
self.events.put(MarketEvent())
def update_bars(self):
"""
将最近的数据条目放入到latest_symbol_data结构中。
"""
for s in self.symbol_list:
try:
# TODO next
bar = next(self._get_new_bar(s))
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
# 每一次心跳，发送MarketEvent
self.events.put(MarketEvent())
def get_new_data(self):
"""
Return a list of market events (new bars) for all symbols from
all exchanges for the just-elapsed time period. Add new bar data
to queue for storage in DB, after current minutes cycle completes.
Logs parse time for tick processing.
Args:
None.
Returns:
new_market_events: list containing new market events.
Raises:
None.
"""
# Record tick parse performance.
self.logger.debug("Started parsing new ticks.")
start_parse = time.time()
for exchange in self.exchanges:
exchange.parse_ticks()
end_parse = time.time()
duration = round(end_parse - start_parse, 5)
self.logger.debug(
"Parsed " + str(self.total_instruments) +
" instruments' ticks in " + str(duration) + " seconds.")
self.track_tick_processing_performance(duration)
# Wrap new 1 min bars in market events.
new_market_events = []
for exchange in self.exchanges:
bars = exchange.get_new_bars()
for symbol in exchange.get_symbols():
for bar in bars[symbol]:
event = MarketEvent(exchange, bar)
new_market_events.append(event)
# Add bars to save-to-db-later queue.
# TODO: store bars concurrently in a separate process.
self.bars_save_to_db.put(event)
return new_market_events
def update_bars(self):
"""
Pushes the latest bar to the latest_symbol_data structure for all symbols in the symbol list.
It simply generates a MarketEvent that gets added to the queue as it appends the latest bars to the latest_symbol_data dictionary
"""
for symbol in self.symbol_list:
try:
bar = next(self._get_new_bar(symbol))
except StopIteration:
self.is_running = False
else:
if bar is not None:
self.latest_symbol_data[symbol].append(bar)
date_time = bar[0]
market_event = MarketEvent(date_time)
# Put in a MarketEvent into the event queue
self.event_queue.put(market_event)
def update_bars(self):
"""
Pushes the latest bars to the bars_queue for each symbol in a tuple OHLCV format:
(datetime, open, high, low, close, volume)
"""
for s in self.symbol_list:
try:
bar = next(self._get_new_bar(s))
except StopIteration: #when next() function gets to end, it emits StopIteration error
self.continue_backtest = False
else:
if bar is not None:
self.latest_symbol_data[s].append(bar)
#?
# print("length: ", len(self.latest_symbol_data[s])) #Length가 계속 늘어남, 일정이상 커지면 과거 정보는 버려줘야하는거 아닌가?
# print(self.latest_symbol_data[s])
self.events.put(MarketEvent()) # put() 함수는 Queue에 Item을 넣는 함수
def update_bars(self):
"""
更新latest_symbol_data dict类型
push 最新的bar进入latest_symbol_data中
"""
try:
bar=next(self._get_new_bar())
except StopIteration:
self.continue_backtest=False
else:
if bar is not None:
self.latestData.append(bar)
print("data.py update_bars put MARKET!")
self.events.put(MarketEvent())
def update_bars(self):
"""
Everytime update, update bar from the 0 row for every symbol
run strategy one by one for all symbol,
need a event loop
and finally fullfill the latest_bar_dict
"""
for s in self.symbol_list:
if self.latest_bar_dict[s] == []:
self.roll_data[s] = self._get_new_bar(s)
try:
bar = self.roll_data[s].next()
except StopIteration:
self.continue_backtest = False
else:
if bar is not None:
self.latest_bar_dict[s].append(bar)
events.put(MarketEvent())
