def update_bars_monthly(self): prev_month = self.latest_data_month flag = 0 for s in self.symbol_list: if self.next_month_bar[s]: self.latest_symbol_data[s].append(self.next_month_bar[s]) self.next_month_bar[s] = [] else: try: bar = next(self._get_new_bar(s)) except StopIteration: self.continue_backtest = False else: if bar is not None: cur_month = datetime.datetime.strptime( bar[0], "%Y-%m-%d").month if cur_month != self.latest_data_month: self.next_month_bar[s] = bar flag = 1 if math.isnan(self.latest_symbol_data[s][0][1].loc[self.factor]) \ and self.latest_data_month == self.start_month: self.factor_na += 1 elif cur_month == self.latest_data_month: self.latest_symbol_data[s].append(bar) prev_month = cur_month self.latest_data_month = prev_month if flag == 1: self.events.put(MarketEvent())
def update_bars(self) -> bool: """ Updates a datetime, thus shift to the next latest bar. """ for s in self._symbol_dict.keys(): if self._latest_idx[s] == self._symbol_dict[s]['len']: self._latest_idx[s] = None self._system_dt = self._next_datetime() # find symbols, which have updated time in the data result = False upd_symbols = [] for s in self._symbol_dict.keys(): if self._latest_idx[s] is not None: result = True if self._system_dt in self._symbol_data[s].index: self._latest_idx[s] += 1 upd_symbols.append(s) if self._events is not None: self._events.put(MarketEvent(upd_symbols, self._system_dt)) self.continue_backtest = result return result
def update_bars(self): """ Pushes the latest bar to the latest_symbol_data structure for all symbols in the symbol list. """ for s in self.symbol_list: try: bar = next(self._get_new_bar(s)) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s].append(bar) self.events.put(MarketEvent())
def get_new_data(self): """Return a list of market events (new bars) for all symbols from all exchanges for the just-elapsed time period. Add new bar data to queue for storage in DB, after current minutes cycle completes.""" # record bar parse performance self.logger.debug("Started parsing new ticks.") start_parse = time.time() for exchange in self.exchanges: exchange.parse_ticks() end_parse = time.time() duration = round(end_parse - start_parse, 5) self.logger.debug("Parsed " + str(self.total_instruments) + " instruments' ticks in " + str(duration) + " seconds.") self.track_performance(duration) # wrap new 1 min bars in market events new_market_events = [] for exchange in self.exchanges: bars = exchange.get_new_bars() for symbol in exchange.get_symbols(): for bar in bars[symbol]: event = MarketEvent(exchange.get_name(), bar) new_market_events.append(event) # add bars to save-to-db-later queue # TODO: store new bars concurrently with a processpool self.bars_save_to_db.put(event) return new_market_events
def update_bars(self): """ Pushes the latest bar to the latest_symbol_data structure for all symbols in the symbol list. """ for s in self.symbol_list: bar = self._get_new_bar(s) self.latest_symbol_data[s].append(bar) self.events.put(MarketEvent())
def update_bars(self): for s in self.symbol_list: try: bar = next(self._get_new_bar(s)) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s].append(bar) self.events.put(MarketEvent())
def updateBars(self): for s in self.universe: try: bar = next(self.getNewSymbolBar(s)) except StopIteration: self.continueBacktest = False else: if bar is not None: self.latestData[s].append(bar) self.eventQueue.append(MarketEvent())
def update_latest_data(self): for symbol in self.symbol_list: data = None try: data = next(self._get_new_data(symbol)) except StopIteration: self.continue_backtest = False if data is not None: self.latest_symbol_data[symbol].append(data) self.events.put(MarketEvent())
def generate_timeAxis(self, start=None, end=None, freq=None): freq = self.freq if freq is None else freq endtime = datetime.utcnow().replace( tzinfo=timezone('utc')) if end is None else end starttime = datetime( 2013, 3, 31, 0, 0, 0, tzinfo=timezone('utc')) if start is None else start timeTbl = pd.DataFrame() timeTbl['timeAxis'] = pd.date_range(start=starttime, end=endtime, freq=freq) timeTbl['trade_date'] = timeTbl['timeAxis'].dt.strftime( "%Y%m%d%H%M%S").astype('int64') timeTbl['exch_timeid'] = (timeTbl['timeAxis'].astype('int32') / 1000000).astype('int64') self.df_calTbl = timeTbl for row in timeTbl.itertuples(): self.cursorTime = row self.cursorBar = self.get_current_bar() self.histBars = self.get_processedBars() self.eventQ.put(MarketEvent(row.trade_date)) yield row else: # newRow= pd.DataFrame.from_dict({'timeAxis':['cash'],'trade_date':[self.initial_capital],'exch_timeid':[self.initial_capital] t = datetime.utcnow().replace(tzinfo=timezone('utc')) if self.freq == '1d': t = t.replace( second=0, microsecond=0, minute=0, hour=0, day=t.day) + timedelta(days=1) Row = namedtuple('Pandas', ['timeAxis', 'trade_date', 'exch_timeid']) newRow = Row(t, t.strftime("%Y%m%d%H%M%S"), (int(t.timestamp()) / 1000000)) while True: self.cursorTime = newRow self.cursorBar = self.get_current_bar() self.eventQ.put(MarketEvent(newRow.trade_date)) yield newRow else: yield None
def update_bars(self): for s in self.symbol_list: try: bar = self.generator_dict[s].next() print(bar) print(getattr(bar, 'adj_close')) except StopIteration: self.continue_backtest = False else: if bar is not None: self.internal_store[s].append(bar) self.events.put(MarketEvent())
def update_bars(self): # pushes the most recent data to the latest_symbol_data structure for each symbol for s in self.symbol_list: try: bar = self._get_new_bar(s).__next__() except StopIteration: self.continue_backtest = False else: if bar is None: self.latest_symbol_data[s].append(bar) market = MarketEvent() self.events.qput(market)
def update_bars(self): for s in self.symbol_list: try: # bar = self._get_new_bar(s).next() # python 2 bar = next(self._get_new_bar(s)) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s] = self.latest_symbol_data[ s].append(bar, ignore_index=True) self.events.put(MarketEvent())
def update_bars(self): """ Отправляет последний бар в структуру данных инструментов для всех инструментов в списке. """ for s in self.symbol_list: try: bar = next(self._get_new_bar(s)) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s].append(bar) self.events.put(MarketEvent(bar[1]))
def update_bars(self): """ 模拟实盘数据接收,从存储的数据字典中获取1条新数据,并存入latest_symbol_data以备回测使用 """ for s in self.symbol_list: try: bar = next(self.symbol_iters[s]) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s].append(bar) self.events.put(MarketEvent())
def update_bars(self): """ 将最近的数据条目放入到latest_symbol_data结构中。 """ try: # 获取最新的基准数据,格式为Dictionary #bar_benchmark = dict(next(self.get_next_bar(self.benchmark))[1]) bar_benchmark = next(self.get_next_bar(self.benchmark)) # 将最新的基准bar写入latest_symbol_data #self.latest_symbol_data[self.benchmark] = self.latest_symbol_data[self.benchmark].append(bar_benchmark,ignore_index=True) self.latest_symbol_data[self.benchmark].append(bar_benchmark) datetime = bar_benchmark['date_time'] # 获取股票池数据 for sym in self.symbol_list: # 如果当前代码为基准代码,跳过(解决基准回测重复取数问题) if sym == self.benchmark: continue # 判断sym数据起始日期是否与回测当前日期一致 if self.symbol_start[sym] == bar_benchmark['date_time']: self.symbol_update[sym] = True # 如果日期一致,则可以开始取数 # 如果可以取数,则获取新的bar;否则根据基准日期填充数据 if self.symbol_update[sym]: #bar = dict(next(self.get_next_bar(sym))[1]) bar = next(self.get_next_bar(sym)) else: bar = { 'date_time': datetime, 'symbol': sym, 'name': 'Unknown', 'open': 0, 'low': 0, 'high': 0, 'close': 0, 'chg': 0, 'pct_chg': 0, 'volume': 0, 'turnover': 0, 'suspend': True } # 将最新的股票bar写入写入latest_symbol_data #self.latest_symbol_data[sym] = self.latest_symbol_data[sym].append(bar,ignore_index=True) self.latest_symbol_data[sym].append(bar) self.events.put(MarketEvent()) # 写入市场事件 except StopIteration: # 获取不到数据,循环终止,停止回测 self.continue_backtest = False
def update_bars(self): for s in self.symbol_list: try: bar = next(self._get_new_bar(s)) # bar is of form # (index, row) # row is a dict i belive with keys of column names except StopIteration: print("finished backtest") self.finished = True else: if bar is not None: self.latest_symbol_data[s].append(bar) self.events.put(MarketEvent())
def update_bars(self): #before updating bars, we need to grab the next timestamp and delete it from our existing ones try: for s in self.symbols: bar = self.get_new_bar(s) if bar.name not in list(self.latest_stamps): self.latest_stamps = self.latest_stamps.append( pd.Index([bar.name])) self.latest_symbol_data[s] = self.latest_symbol_data[s].append( bar) self.events.put(MarketEvent( self.latest_stamps[-1])) #FIRST START, IMPORTANT except StopIteration: self.continue_backtest = False print('BACKTEST COMPLETE.')
def update_bars(self): """ Pushes the latest bar to the latest_symbol_data structure for all symbols in the symbol list. """ for symbol in self.symbols: try: bar = next(self.bars_generator[symbol]) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_bars_data[symbol] = self.latest_bars_data[symbol].append(bar) self.latest_bars_data[symbol].index.rename('datetime', inplace=True) self.events.put(item=MarketEvent())
def update_bars(self): """ Pushes the latest bars to the bars_queue for each symbol in a tuple OHLCVI format: (datetime, open, high, low, close, volume, open interest). """ for s in self.symbol_list: try: bar = next(self._get_new_bar(s)) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s].append(bar) self.events.put(MarketEvent())
def update_bars(self): ''' Pushes the latest bar to the latest_symbol_data structure for all symbols in the symbol list ''' for s in self.symbol_list: try: bar = next(self._get_new_bar(s)) # grab the new bar except StopIteration: self.continue_backtest = False # of there is no next bar then the backtest is over else: if bar is not None: self.latest_symbol_data[s].append( bar) # tack the next bar onto the latest_symbol_data self.events.put(MarketEvent())
def update_bars(self): """ read each row of symbol_data into latest_symbol_data then, generate MarketEvent it's used in backtest module """ for s in self.symbol_list: try: bar = next(self._get_new_bar(s)) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s].append(bar) self.events.put(MarketEvent())
def update_bars(self): """ 向latest_symbol_data中不断推入每日股价信息 """ for s in self.symbol_list: try: bar = next( self._get_new_bar(s) ) #调用next函数以迭代dataframe,返回一个tuple(datatime,[series for stock info]) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s].append( bar) #在latest_symbol_data中不断append最新一天的bar self.events.put(MarketEvent())
def update_bars(self): """ Pushes the latest bar to the latest_symbol_data structure for all symbols in the symbol list. """ for s in self.symbol_list: try: bar = next(self._get_new_bar(s)) # 与yeild语句对应,下一行数据 except StopIteration: self.continue_backtest = False # 到达数据的最近日期,令flag continue_backtest转变为False else: if bar is not None: self.latest_symbol_data[s].append(bar) # 这里赋予了latest_symbol_data的值 self.events.put( MarketEvent()) # mac(策略)执行的开端。MarketEvent的属性就是type == ‘market’
def update_bars(self): """ Pushes the latest bar to the latest_symbol_data structure for all symbols in the symbol list """ for s in self.symbol_list: try: bar = self._get_new_bar(s).next() except StopIteration: self.continue_backtest = False else: #THIS IS WHERE YOU SHOULD BE CHECKING IF YOUR GETTING ANY DATA FOR THAT SYMBOL #WHAT HAPPENS IF YOU DONT APPEND A BAR FOR THE SYMBOL ??? if bar is not None: self.latest_symbol_data[s].append(bar) self.events.put(MarketEvent())
def update_bars(self): """ 将最近的数据条目放入到latest_symbol_data结构中。 """ for s in self.symbol_list: try: # TODO next bar = next(self._get_new_bar(s)) except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s].append(bar) # 每一次心跳,发送MarketEvent self.events.put(MarketEvent())
def get_new_data(self): """ Return a list of market events (new bars) for all symbols from all exchanges for the just-elapsed time period. Add new bar data to queue for storage in DB, after current minutes cycle completes. Logs parse time for tick processing. Args: None. Returns: new_market_events: list containing new market events. Raises: None. """ # Record tick parse performance. self.logger.debug("Started parsing new ticks.") start_parse = time.time() for exchange in self.exchanges: exchange.parse_ticks() end_parse = time.time() duration = round(end_parse - start_parse, 5) self.logger.debug( "Parsed " + str(self.total_instruments) + " instruments' ticks in " + str(duration) + " seconds.") self.track_tick_processing_performance(duration) # Wrap new 1 min bars in market events. new_market_events = [] for exchange in self.exchanges: bars = exchange.get_new_bars() for symbol in exchange.get_symbols(): for bar in bars[symbol]: event = MarketEvent(exchange, bar) new_market_events.append(event) # Add bars to save-to-db-later queue. # TODO: store bars concurrently in a separate process. self.bars_save_to_db.put(event) return new_market_events
def update_bars(self): """ Pushes the latest bar to the latest_symbol_data structure for all symbols in the symbol list. It simply generates a MarketEvent that gets added to the queue as it appends the latest bars to the latest_symbol_data dictionary """ for symbol in self.symbol_list: try: bar = next(self._get_new_bar(symbol)) except StopIteration: self.is_running = False else: if bar is not None: self.latest_symbol_data[symbol].append(bar) date_time = bar[0] market_event = MarketEvent(date_time) # Put in a MarketEvent into the event queue self.event_queue.put(market_event)
def update_bars(self): """ Pushes the latest bars to the bars_queue for each symbol in a tuple OHLCV format: (datetime, open, high, low, close, volume) """ for s in self.symbol_list: try: bar = next(self._get_new_bar(s)) except StopIteration: #when next() function gets to end, it emits StopIteration error self.continue_backtest = False else: if bar is not None: self.latest_symbol_data[s].append(bar) #? # print("length: ", len(self.latest_symbol_data[s])) #Length가 계속 늘어남, 일정이상 커지면 과거 정보는 버려줘야하는거 아닌가? # print(self.latest_symbol_data[s]) self.events.put(MarketEvent()) # put() 함수는 Queue에 Item을 넣는 함수
def update_bars(self): """ 更新latest_symbol_data dict类型 push 最新的bar进入latest_symbol_data中 """ try: bar=next(self._get_new_bar()) except StopIteration: self.continue_backtest=False else: if bar is not None: self.latestData.append(bar) print("data.py update_bars put MARKET!") self.events.put(MarketEvent())
def update_bars(self): """ Everytime update, update bar from the 0 row for every symbol run strategy one by one for all symbol, need a event loop and finally fullfill the latest_bar_dict """ for s in self.symbol_list: if self.latest_bar_dict[s] == []: self.roll_data[s] = self._get_new_bar(s) try: bar = self.roll_data[s].next() except StopIteration: self.continue_backtest = False else: if bar is not None: self.latest_bar_dict[s].append(bar) events.put(MarketEvent())