def __init__(self, ib: IB):
ib.connectedEvent += self.onConnected
ib.disconnectedEvent += self.onDisconnected
ib.updateEvent += self.onUpdate
ib.pendingTickersEvent += self.onPendingTickers
ib.barUpdateEvent += self.onBarUpdate
ib.newOrderEvent += self.onNewOrder
ib.orderModifyEvent += self.onOrderModify
ib.openOrderEvent += self.onOpenOrder
ib.orderStatusEvent += self.onOrderStatus
ib.execDetailsEvent += self.onExecDetails
ib.commissionReportEvent += self.onCommissionReport
ib.updatePortfolioEvent += self.onUpdatePortfolio
ib.positionEvent += self.onPosition
ib.accountValueEvent += self.onAccountValue
ib.accountSummaryEvent += self.onAccountSummary
ib.pnlEvent += self.onPnl
ib.pnlSingleEvent += self.onPnlSingle
ib.tickNewsEvent += self.onTickNews
ib.newsBulletinEvent += self.onNewsBulletin
ib.scannerDataEvent += self.onScannerData
ib.errorEvent += self.onError
ib.timeoutEvent += self.onTimeout
scheduledUpdate = Event().timerange(300, None, 600)
scheduledUpdate += self.onScheduledUpdate
self.ib = ib
self.portfolio_items = {}
def __init__(
self,
store: AbstractBaseStore,
contract: Contract,
head: datetime,
barSize: str,
wts: str,
aggression: float = 2,
now: datetime = datetime.now()) -> None:
self.store = store
self.contract = contract
self.head = head
self.barSize = bar_size_validator(barSize)
self.wts = wts_validator(wts)
self.now = now
self.aggression = aggression
self.c = self.contract.localSymbol
# start, stop, step in seconds, ie. every 15min
pulse = Event().timerange(900, None, 900)
pulse += self.onPulse
self.next_date = ''
self._objects = []
self._queue = []
self._current_object = None
self.schedule_tasks()
def _createEvents(self):
self.entrySignal = Event('entrySignal')
self.closeSignal = Event('closeSignal')
def _createEvents(self) -> None:
self.newCandle = Event('newCandle')
class StreamAggregator(BarStreamer):
def __init__(self) -> None:
self._createEvents()
self.buffer = deque()
self.new_bars = []
self.backfill = True
super().__init__()
def _createEvents(self) -> None:
self.newCandle = Event('newCandle')
def __call__(self, ib: IB, contract: Contract) -> None:
date = self.all_bars[-1].date if contract == self.contract else None
super().__call__(ib, contract, date)
self.process_back_data(date)
def process_back_data(self, date: Optional[datetime] = None) -> None:
# flag needed on re-connect
self.backfill = True
for counter, bar in enumerate(self.bars[:-1]):
# date given on reconnect only
if (date and (bar.date > date)) or not date:
self.aggregate(bar)
# prevent from blocking too long
if counter % 10000 == 0:
log.debug(f'releasing control {self.contract.localSymbol}')
util.sleep(0)
log.debug(f'startup data generated for {self.contract.localSymbol}')
self.backfill = False
self.clear_buffer()
def create_candle(self) -> None:
df = util.df(self.new_bars)
df.date = df.date.astype('datetime64')
df.set_index('date', inplace=True)
df['volume_weighted'] = df.average * df.volume
weighted_price = df.volume_weighted.sum() / df.volume.sum()
self.newCandle.emit({
'backfill': self.backfill,
'date': df.index[-1],
'open': df.open[0],
'high': df.high.max(),
'low': df.low.min(),
'close': df.close[-1],
'weighted_price': weighted_price,
# 'price': weighted_price,
'price': df.close[-1],
'volume': df.volume.sum()
})
def onNewBar(self, bars: BarDataList, hasNewBar: bool) -> None:
if hasNewBar:
if self.backfill:
log.debug(f'buffering bar for {self.contract.localSymbol}')
self.buffer.append(bars[-2])
else:
self.clear_buffer()
self.aggregate(bars[-2])
def clear_buffer(self) -> None:
"""Utilize bars that have been buffered while processing back data."""
while self.buffer:
log.debug(f'clearing buffer for {self.contract.localSymbol}')
self.aggregate(self.buffer.popleft())
class Candle(ABC):
def __init__(self,
streamer,
contract_fields: Union[List[str], str] = 'contract',
**kwargs):
if isinstance(contract_fields, str):
self.contract_fields = [contract_fields]
else:
self.contract_fields = contract_fields
self.__dict__.update(kwargs)
self.streamer = streamer
self.streamer.newCandle.connect(self.append, keep_ref=True)
self.df = None
log.debug(f'candle init for contract: {kwargs}')
self.candles = []
self._createEvents()
def _createEvents(self):
self.signal = Event('signal')
self.entrySignal = Event('entrySignal')
self.closeSignal = Event('closeSignal')
def __call__(self, ib: IB):
log.debug(
f'Candle {self.contract.localSymbol} initializing data stream...')
self.details = ib.reqContractDetails(self.contract)[0]
self.streamer(ib, self.contract)
def append(self, candle: Dict[str, Any]):
self.candles.append(candle)
if not candle['backfill']:
df = pd.DataFrame(self.candles)
df.set_index('date', inplace=True)
self.df = self.get_indicators(df)
# log_assert(not self.df.iloc[-1].isna().any(), (
# f'Not enough data for indicators for instrument'
# f' {self.contract.localSymbol} '
# f' index: {df.index[-1]}'
# f' values: {self.df.iloc[-1].to_dict()}'
# f'{self.df}'), __name__)
self.process()
def save(self, saver):
if self.df is not None:
saver.save(self.df, 'candles', self.contract.localSymbol)
saver.save(self.streamer.all_bars_df, 'all_bars',
self.contract.localSymbol)
def set_now(self, now):
self.streamer.now = now
@abstractmethod
def get_indicators(self, df):
return df
@abstractmethod
def process(self):
self.signal.emit(self)
self.entrySignal.emit(self)
self.closeSignal.emit(self)
def __repr__(self):
return f'Candle: {self.contract}'
def _createEvents(self) -> None:
self.barUpdateEvent = Event('barUpdateEvent')
self.newOrderEvent = Event('newOrderEvent')
self.orderModifyEvent = Event('orderModifyEvent')
self.orderStatusEvent = Event('orderStatusEvent')
self.cancelOrderEvent = Event('cancelOrderEvent')
class IB:
path = '/home/tomek/ib_data/b_temp'
events = ('barUpdateEvent', 'newOrderEvent', 'orderModifyEvent',
'cancelOrderEvent', 'orderStatusEvent')
ib = master_IB()
def __init__(self, datasource_manager: DataSourceManager,
mode: str = 'use_ib', index: int = -1,
field: str = 'symbol') -> None:
self.datasource = datasource_manager.get_history
self.store = datasource_manager.store
self.mode = mode
self.index = index
self.field = field
self.market = Market()
self._contracts = []
self._createEvents()
self.id = count(1, 1)
def _createEvents(self) -> None:
self.barUpdateEvent = Event('barUpdateEvent')
self.newOrderEvent = Event('newOrderEvent')
self.orderModifyEvent = Event('orderModifyEvent')
self.orderStatusEvent = Event('orderStatusEvent')
self.cancelOrderEvent = Event('cancelOrderEvent')
def read_from_file_or_ib(self, filename: str, method: str, obj: Any,
*args):
try:
with open(f'{self.path}/{filename}.pickle', 'rb') as f:
c = pickle.load(f)
except (FileNotFoundError, EOFError):
c = dict()
try:
details = c[repr(obj)]
log.debug(f'{filename} for {repr(obj)} read from file')
return details
except KeyError:
with self.ib.connect(port=4002, clientId=2) as conn:
f = getattr(conn, method)
if args:
details = f(obj, *args)
else:
details = f(obj)
log.debug(
f'{filename} for {repr(obj)} read from ib')
c[repr(obj)] = details
with open(f'{self.path}/{filename}.pickle', 'wb') as f:
log.debug(f'{filename} saved to file')
pickle.dump(c, f)
return details
def reqContractDetails(self, contract: Contract):
if self.mode == 'use_ib':
return self.read_from_file_or_ib('details', 'reqContractDetails',
contract)
elif self.mode == 'db_only':
contfuture_object = self.store.contfuture_contract_object(
contract.symbol, self.index, self.field)
meta = self.store.read_metadata(contfuture_object)
details = ContractDetails(**{'contract': contfuture_object,
'minTick': meta['min_tick'],
'longName': meta['name']})
log.debug(f'details for {contfuture_object}: {details}')
return [details]
def qualifyContracts(self, *contracts):
"""
Modified copy of:
https://ib-insync.readthedocs.io/_modules/ib_insync/ib.html#IB.qualifyContractsAsync
"""
log.debug(f'qualifying contracts: {contracts}')
detailsLists = (self.reqContractDetails(c) for c in contracts)
result = []
for contract, detailsList in zip(contracts, detailsLists):
if not detailsList:
log.error(f'unknown contract {contract}')
else:
c = detailsList[0].contract
expiry = c.lastTradeDateOrContractMonth
if expiry:
# remove time and timezone part as it will cuse problems
expiry = expiry.split()[0]
c.lastTradeDateOrContractMonth = expiry
if contract.exchange == 'SMART':
# overwriting 'SMART' exchange can create invalid contract
c.exchange = contract.exchange
contract.update(**c.dict())
result.append(contract)
# keep track of all contracts for which details must be obtained
self._contracts.extend(result)
return result
def reqCommissionsFromIB(self, contracts: List) -> Dict:
order = MarketOrder('BUY', 1)
commissions = {contract.symbol: self.read_from_file_or_ib(
'commission', 'whatIfOrder', contract, order)
for contract in contracts}
missing_commissions = []
for contract, commission in commissions.copy().items():
if not commission:
missing_commissions.append(contract)
del commissions[contract]
commissions.update(self.getCommissionBySymbol(missing_commissions))
return commissions
def getCommissionBySymbol(self, commissions: List[Contract]
) -> Dict[str, Any]:
with open(f'{self.path}/commissions_by_symbol.pickle', 'rb') as f:
c = pickle.load(f)
return {comm: c.get(comm) for comm in commissions}
def reqCommissionsFromDB(self, contracts: List[Contract]
) -> Dict[str, Any]:
log.debug(f'requesting metadata for contracts: {contracts}')
return {contract.symbol: self.store.read_metadata(contract
)['commission']
for contract in contracts}
def reqHistoricalData(self,
contract: Contract,
durationStr: str,
barSizeSetting: str,
whatToShow: str,
useRTH: str,
endDateTime: str = '',
formatDate: int = 1,
keepUpToDate: bool = True,
**kwargs
):
return self.datasource(contract, durationStr, barSizeSetting)
reqHistoricalDataAsync = reqHistoricalData
def positions(self):
try:
return self.market.account.positions.values()
except AttributeError:
# this is before Account has been instantiated
return {}
def accountValues(self):
log.info(f'cash: {self.market.account.cash}')
return [
AccountValue(tag='TotalCashBalance',
value=self.market.account.cash),
AccountValue(tag='UnrealizedPnL',
value=self.market.account.unrealizedPnL)
]
def openTrades(self):
return [v for v in self.market.trades
if v.orderStatus.status not in OrderStatus.DoneStates]
def placeOrder(self, contract: Contract, order: Order) -> Trade:
orderId = order.orderId or next(self.id)
now = self.market.date
trade = self.market.get_trade(order)
if trade:
# this is a modification of an existing order
assert trade.orderStatus.status not in OrderStatus.DoneStates
logEntry = TradeLogEntry(now, trade.orderStatus.status, 'Modify')
trade.log.append(logEntry)
trade.modifyEvent.emit(trade)
self.orderModifyEvent.emit(trade)
else:
# this is a new order
assert order.totalQuantity != 0, 'Order quantity cannot be zero'
order.orderId = orderId
order.permId = orderId
if order.parentId:
# this is for bracket order implementation TODO
orderStatus = OrderStatus(status=OrderStatus.PreSubmitted,
remaining=order.totalQuantity)
log.error('Why the f**k are we here?')
else:
orderStatus = OrderStatus(status=OrderStatus.Submitted,
remaining=order.totalQuantity)
logEntry = TradeLogEntry(now, orderStatus, '')
trade = Trade(contract, order, orderStatus, [], [logEntry])
self.newOrderEvent.emit(trade)
self.market.append_trade(trade)
return trade
def cancelOrder(self, order: Order) -> None:
trade = self.market.get_trade(order)
if trade:
self.market.cancel_trade(trade)
self.cancelOrderEvent.emit(trade)
self.orderStatusEvent.emit(trade)
else:
log.error(f'cancelOrder: Unknown orderId {order.orderId}')
def run(self):
# TODO
# This is a f*****g monkey patch, needs to be redone
if self.mode == 'use_ib':
commissions = self.reqCommissionsFromIB(
self._contracts)
commissions = {k: v.commission for k, v in commissions.items()}
elif self.mode == 'db_only':
log.debug(f'About to request commissions for {self._contracts}')
commissions = self.reqCommissionsFromDB(
self._contracts)
else:
raise ValueError('Mode should be one of "use_ib" or "db_only"')
self.market.commissions = commissions
self.market.ticks = {
cont.symbol: self.reqContractDetails(cont)[0].minTick
for cont in self._contracts}
log.debug(f'market.object.keys: {self.market.objects.keys()}')
log.debug(f'properties set on Market: {self.market}')
log.debug(f'commissions: {self.market.commissions}')
log.debug(f'ticks: {self.market.ticks}')
self.market.run()
def sleep(self, *args):
# this makes sense only if run in asyncio
util.sleep()