def test_ibgw_port(host):
account = os.environ['IB_ACCOUNT']
password = os.environ['IB_PASSWORD']
trade_mode = os.environ['TRADE_MODE']
# build local ./Dockerfile
subprocess.check_call(['docker', 'build', '-t', IMAGE_NAME, '.'])
# run a container
docker_id = subprocess.check_output(
['docker', 'run',
'--env', 'IB_ACCOUNT={}'.format(account),
'--env', 'IB_PASSWORD={}'.format(password),
'--env', 'TRADE_MODE={}'.format(trade_mode),
'-p', '4002:4002',
'-d', IMAGE_NAME,
"tail", "-f", "/dev/null"]).decode().strip()
time.sleep(60)
ib = IB()
ib.connect('localhost', 4002, clientId=1)
contract = Forex('EURUSD')
bars = ib.reqHistoricalData(
contract, endDateTime='', durationStr='30 D',
barSizeSetting='1 hour', whatToShow='MIDPOINT', useRTH=True)
# convert to pandas dataframe:
df = util.df(bars)
print(df)
# at the end of the test suite, destroy the container
subprocess.check_call(['docker', 'rm', '-f', docker_id])
async def get_ohlc(ib, contract):
ohlc = await ib.reqHistoricalDataAsync(
contract=contract,
endDateTime="",
durationStr="365 D",
barSizeSetting="1 day",
whatToShow="Trades",
useRTH=True,
)
# reverse sort to have latest date on top
df_ohlc = util.df(ohlc).sort_index(ascending=False).reset_index(drop=True)
df_ohlc.insert(0, 'symbol', contract.symbol)
df_ohlc["rise"] = [
df_ohlc["close"].rolling(i).apply(
lambda x: x[0] - x[-1], raw=True).max()
for i in range(1, len(df_ohlc) + 1)
]
df_ohlc["rise"] = df_ohlc.rise.abs()
df_ohlc["fall"] = [
df_ohlc["close"].rolling(i).apply(
lambda x: x[0] - x[-1], raw=True).min()
for i in range(1, len(df_ohlc) + 1)
]
df_ohlc["fall"] = df_ohlc.fall.abs()
df_ohlc = df_ohlc.assign(sd=df_ohlc.close.expanding(1).std(ddof=0))
# roll the standard deviation upwards
df_ohlc.sd = df_ohlc.sd.expanding(1).max()
return df_ohlc
def ib_get_historical_data(self,
ibcontract,
durationStr="1 Y",
barSizeSetting="1 day",
whatToShow="TRADES",
log=None):
"""
Returns historical prices for a contract, up to today
ibcontract is a Contract
:returns list of prices in 4 tuples: Open high low close volume
"""
if log is None:
log = self.log
last_call = self.last_historic_price_calltime
avoid_pacing_violation(last_call, log=log)
bars = self.ib.reqHistoricalData(ibcontract,
endDateTime='',
durationStr=durationStr,
barSizeSetting=barSizeSetting,
whatToShow=whatToShow,
useRTH=True,
formatDate=1)
df = util.df(bars)
self.last_historic_price_calltime = datetime.datetime.now()
return df
def download_bars_data(
self,
contract: AContract,
start_date: date,
end_date: date,
bar_size: timedelta,
rth: bool,
**kwargs,
) -> pd.DataFrame:
ib_contract = self._to_ib_contract(contract=contract)
dates = generate_trading_days(start_date=start_date, end_date=end_date)
duration = f"{len(dates)} D"
bar_size_str = self._to_ib_bar_size(bar_size=bar_size)
bar_data = self._ib_conn.reqHistoricalData(
contract=ib_contract,
endDateTime=end_date,
durationStr=duration,
barSizeSetting=bar_size_str,
whatToShow="TRADES",
useRTH=False,
)
data = util.df(objs=bar_data)
if data is not None and len(data) != 0:
data = self._format_data(data=data)
else:
data = pd.DataFrame()
return data
def all_bars_df(self):
if self.all_bars:
df = util.df(self.all_bars)
df.date = df.date.astype('datetime64')
df.set_index('date', inplace=True)
else:
df = pd.DataFrame()
return df
def reset_volume(self, avg_periods) -> int:
# TODO: make span adjust to length of requested data
bars = self.all_bars or self.bars
if bars == self.bars:
self.span = len(self.bars)
df = util.df(bars)
# last 5 days
volume = df.iloc[-14100:].volume.rolling(avg_periods).sum() \
.mean().round()
log.debug(f'volume: {volume}')
return volume
def data(self) -> Union[pd.DataFrame, pd.datetime]:
"""Return df ready to be written to datastore or date of end point
for additional downloads"""
if self.bars:
self.df = util.df([b for bars in reversed(self.bars)
for b in bars])
self.df.set_index('date', inplace=True)
if self.ok_to_write:
df = self.df
self.df = pd.DataFrame()
return df
log.debug(f'cannot write data')
def get_futures(contract_str, remove_weekly=False):
ibw = IbWrapper()
ib = ibw.ib
vix = Future(contract_str, includeExpired=False)
cds = ib.reqContractDetails(vix)
contracts = [cd.contract for cd in cds]
if remove_weekly:
contracts = [
contract for contract in contracts
if len(contract.localSymbol) <= 4
]
bars_list = []
for contract in contracts:
bars = ib.reqHistoricalData(contract,
endDateTime='',
durationStr='5 D',
barSizeSetting='1 day',
whatToShow='TRADES',
useRTH=True,
formatDate=1)
if bars:
bars_list.append(util.df(bars))
ib.disconnect()
contract_df = util.df(contracts)
close_list = [item.loc[:, ['date', 'close']] for item in bars_list]
close_list = [item.set_index('date') for item in close_list]
close_list = [
item.rename(index=str, columns={'close': name}) for item, name in zip(
close_list,
pd.to_datetime(contract_df['lastTradeDateOrContractMonth']))
]
future_series = pd.concat(close_list, axis=1, sort=False)
future_series = future_series.transpose().sort_index()
future_series.columns = pd.to_datetime(future_series.columns)
return future_series, contract_df
def get_strikes_and_expiration(self):
expiration = self.next_weekday(datetime.date.today(), self.next_exp_weekday())
chains = self.ib.reqSecDefOptParams(underlyingSymbol='ES', futFopExchange='GLOBEX', underlyingSecType='FUT',
underlyingConId=self.ES.conId)
chain = util.df(chains)
strikes = chain[chain['expirations'].astype(str).str.contains(expiration)].loc[:, 'strikes'].values[0]
[ESValue] = self.ib.reqTickers(self.ES)
ES_price = ESValue.marketPrice()
strikes = [strike for strike in strikes
if strike % 5 == 0
and ES_price - 10 < strike < ES_price + 10]
return strikes, expiration
def option_history(self, contract):
ib.qualifyContracts(contract)
df = pd.DataFrame(
util.df(
ib.reqHistoricalData(contract=contract,
endDateTime=endDateTime,
durationStr=No_days,
barSizeSetting=interval,
whatToShow='MIDPOINT',
useRTH=False,
keepUpToDate=False))[['date', 'close']])
df.columns = ['date', f"{contract.symbol}_{contract.right}_close"]
df.set_index('date', inplace=True)
return df
def ES(self):
ES = Future(symbol='ES',
lastTradeDateOrContractMonth='20201218',
exchange='GLOBEX',
currency='USD')
ib.qualifyContracts(ES)
ES_df = ib.reqHistoricalData(contract=ES,
endDateTime=endDateTime,
durationStr=No_days,
barSizeSetting=interval,
whatToShow='TRADES',
useRTH=False)
ES_df = util.df(ES_df)
ES_df.set_index('date', inplace=True)
ES_df.index = pd.to_datetime(ES_df.index)
# ES_df['hours'] = ES_df.index.strftime('%H').astype(int)
# ES_df['minutes'] = ES_df.index.strftime('%M').astype(int)
# ES_df['hours + minutes'] = ES_df['hours'] * 100 + ES_df['minutes']
# ES_df['Day_of_week'] = ES_df.index.dayofweek
ES_df['RSI'] = ta.RSI(ES_df['close'], timeperiod=5)
# ES_df['macd'], ES_df['macdsignal'], ES_df['macdhist'] = ta.MACD(ES_df['close'], fastperiod=12, slowperiod=26,
# signalperiod=9)
# ES_df['macd - macdsignal'] = ES_df['macd'] - ES_df['macdsignal']
ES_df['MA_9'] = ta.MA(ES_df['close'], timeperiod=9)
ES_df['MA_21'] = ta.MA(ES_df['close'], timeperiod=21)
# ES_df['MA_200'] = ta.MA(ES_df['close'], timeperiod=200)
# ES_df['EMA_9'] = ta.EMA(ES_df['close'], timeperiod=9)
# ES_df['EMA_26'] = ta.EMA(ES_df['close'], timeperiod=26)
# ES_df['derv_1'] = np.gradient(ES_df['EMA_9'])
# ES_df['EMA_9_26'] = ES_df['EMA_9'] / ES_df['EMA_26']
# ES_df['EMA_50'] = ta.EMA(ES_df['close'], timeperiod=50)
# ES_df['EMA_200'] = ta.EMA(ES_df['close'], timeperiod=200)
ES_df['ATR'] = ta.ATR(ES_df['high'],
ES_df['low'],
ES_df['close'],
timeperiod=20)
# ES_df['roll_max_cp'] = ES_df['high'].rolling(20).max()
# ES_df['roll_min_cp'] = ES_df['low'].rolling(20).min()
# ES_df['Mean_ATR'] = (ta.ATR(ES_df['high'], ES_df['low'], ES_df['close'], 21)).mean()
ES_df['roll_max_vol'] = ES_df['volume'].rolling(20).max()
# ES_df['vol/max_vol'] = ES_df['volume'] / ES_df['roll_max_vol']
ES_df['EMA_9-EMA_26'] = ES_df['EMA_9'] - ES_df['EMA_26']
# ES_df['EMA_200-EMA_50'] = ES_df['EMA_200'] - ES_df['EMA_50']
# ES_df['B_upper'], ES_df['B_middle'], ES_df['B_lower'] = ta.BBANDS(ES_df['close'], timeperiod=6, nbdevup=1,
# nbdevdn=1, matype=MA_Type.T3)
ES_df.dropna(inplace=True)
ES_df = renko_df(ES_df, 1.5)
return ES_df
def _fetch_to_df(ib, symbol, duration, bar_size):
logger.info(f'Downloading historical data of {symbol} for {duration} '
f'with {bar_size} resolution')
contract = Stock(symbol, exchange='SMART', currency='USD')
bars = ib.reqHistoricalData(
contract, endDateTime='', durationStr=duration,
barSizeSetting=bar_size, whatToShow='TRADES', useRTH=True)
df = util.df(bars)
df.set_index('date', inplace=True)
df.index = df.index.tz_localize(LOCAL_TZ).tz_convert('UTC')
df['volume'] *= 100
return df
def add(self, text=''):
text = text #or self.edit.text()
if text:
contract = eval(text)
if (contract and self.ib.qualifyContracts(contract)):
data = self.ib.reqHistoricalData(contract,
endDateTime='',
durationStr='30 D',
barSizeSetting='4 hours',
whatToShow='MIDPOINT',
useRTH=True)
df = util.df(data)
print(df["close"])
self.graph.plot(df["close"])
self.edit.setText(text)
def into_dataframe(data: list,
symbol,
tf,
qtrade=False,
index=True) -> pd.DataFrame:
"""Converts Binance response list into dataframe"""
crypto = 'USDT' in symbol or 'BUSD' in symbol
if crypto:
df = pd.DataFrame(data,
columns=[
"timestamp", "open", "high", "low", "close",
"volume", "", "", "", "", "", ""
])
df[["open", "high", "low", "close",
"volume"]] = df[["open", "high", "low", "close",
"volume"]].astype(float)
df['date'] = Helper.millisecond_timestamp_to_datetime(
df['timestamp'])
else:
if qtrade:
# Questrade Data Handling
df = pd.DataFrame(data).drop(
'start', axis=1).rename(columns={'end': 'date'})
df['date'] = [
pd.Timestamp(date).tz_localize(None) for date in df['date']
]
df['timestamp'] = Helper.datetime_to_millisecond_timestamp(
df['date'])
else:
# IBKR Data Handling
df = util.df(data).drop(['average', 'barCount'], axis=1)
df['timestamp'] = Helper.datetime_to_millisecond_timestamp(
df['date'])
df['symbol'] = [symbol] * len(data)
df['tf'] = [tf] * len(data)
if index:
return df[[
'symbol', 'tf', 'timestamp', 'date', 'open', 'high', 'low',
'close', 'volume'
]].set_index("timestamp")
return df[[
'symbol', 'tf', 'timestamp', 'date', 'open', 'high', 'low',
'close', 'volume'
]]
def on_bar_update(bars: BarDataList, has_new_bar: bool):
try:
logger.debug("on_bar_update: {} {} {}".format(bars.contract, bars[-1],
has_new_bar))
df = util.df([bars[-1]])
contract = bars.contract
symbol = None
if type(contract) == Forex:
symbol = contract.symbol + contract.currency
elif type(contract) == Stock:
symbol = contract.symbol + contract.currency
if has_new_bar and symbol:
df['symbol'] = symbol
update_k_bars(df=df, _type=bars.barSizeSetting)
except Exception as e:
logger.exception(e)
def nse_und(ib, contract):
'''returns the underlying details
Args:
(ib) as the active ib object
(contract) as the contract
Returns: None. The underlying is pickled to _und.pkl'''
ticker = get_div_tick(ib, contract)
df_und = util.df([ticker])
cols = [
'contract', 'time', 'bid', 'bidSize', 'ask', 'askSize', 'last',
'lastSize', 'volume', 'open', 'high', 'low', 'close', 'dividends'
]
df_und = df_und[cols]
df_und = df_und.assign(undPrice=np.where(df_und['last'].isnull(),
df_und.close, df_und['last']))
try:
divrate = df_und.dividends[0][0] / df_und.dividends[0][
0] / df_und.undPrice
except (TypeError, AttributeError) as e:
divrate = 0.0
df_und = df_und.assign(divrate=divrate)
df_und = df_und.assign(
symbol=[c[1].symbol for c in df_und.contract.items()])
#... get the lot, margin, undPrice and dividend rate
undlot = df_slm.loc[df_slm.ibSymbol == contract.symbol, 'lot'].item()
df_und['lot'] = undlot
# margin of underlying
order = Order(action='SELL', totalQuantity=undlot, orderType='MKT')
# margin = float(ib.whatIfOrder(contract, order).initMarginChange) # doesn't work because permission is not there for NRIs!!!
margin = df_slm.loc[df_slm.ibSymbol == contract.symbol, 'margin'].item()
df_und['margin'] = margin
df_und.to_pickle(fspath + contract.symbol + '_und.pkl')
return df_und
def make_clean_df(self, ES_df, hashbar=None):
ES_df = util.df(ES_df)
ES_df['RSI'] = ta.RSI(ES_df['close'])
ES_df['macd'], ES_df['macdsignal'], ES_df['macdhist'] = ta.MACD(
ES_df['close'], fastperiod=12, slowperiod=26, signalperiod=9)
ES_df['MA_9'] = ta.MA(ES_df['close'], timeperiod=9)
ES_df['MA_21'] = ta.MA(ES_df['close'], timeperiod=21)
ES_df['MA_200'] = ta.MA(ES_df['close'], timeperiod=200)
ES_df['EMA_9'] = ta.EMA(ES_df['close'], timeperiod=9)
ES_df['EMA_21'] = ta.EMA(ES_df['close'], timeperiod=21)
ES_df['EMA_200'] = ta.EMA(ES_df['close'], timeperiod=200)
ES_df['ATR'] = ta.ATR(ES_df['high'], ES_df['low'], ES_df['close'])
ES_df['roll_max_cp'] = ES_df['high'].rolling(20).max()
ES_df['roll_min_cp'] = ES_df['low'].rolling(20).min()
ES_df['roll_max_vol'] = ES_df['volume'].rolling(20).max()
ES_df.dropna(inplace=True)
self.loop_function(ES_df)
def create_candle(self) -> None:
df = util.df(self.new_bars)
df.date = df.date.astype('datetime64')
df.set_index('date', inplace=True)
df['volume_weighted'] = df.average * df.volume
weighted_price = df.volume_weighted.sum() / df.volume.sum()
self.newCandle.emit({
'backfill': self.backfill,
'date': df.index[-1],
'open': df.open[0],
'high': df.high.max(),
'low': df.low.min(),
'close': df.close[-1],
'weighted_price': weighted_price,
# 'price': weighted_price,
'price': df.close[-1],
'volume': df.volume.sum()
})
async def ohlcAsync(ib, c):
# handle errors
try:
ohlc = await ib.reqHistoricalDataAsync(contract=c,
endDateTime="",
durationStr="365 D",
barSizeSetting="1 day",
whatToShow="Trades",
useRTH=True)
except AttributeError:
return None
# reverse sort to have latest date on top
if ohlc: # ohlc has captured something!
df_ohlc = util.df(ohlc).sort_index(ascending=False).reset_index(
drop=True)
df_ohlc.insert(0, 'symbol', c.symbol)
return df_ohlc
async def basedata(ib, symbols):
contracts = [
await ib.qualifyContractsAsync(Stock(s, 'SMART', 'USD'))
for s in symbols
]
# Dictionary to eliminate duplicate symbols
contracts = {s.symbol: s for c in contracts for s in c}
d = {}
# get prices, chains and ohlcs with standard deviaPtion
for contract in contracts.values():
tick = await ib.reqTickersAsync(contract)
chain = await ib.reqSecDefOptParamsAsync(
underlyingSymbol=contract.symbol,
futFopExchange='',
underlyingSecType=contract.secType,
underlyingConId=contract.conId)
ohlc = await ib.reqHistoricalDataAsync(contract=contract,
endDateTime='',
durationStr='365 D',
barSizeSetting='1 day',
whatToShow='Trades',
useRTH=True)
ohlc = util.df(ohlc)
ohlc = ohlc.assign(dte=ohlc.index.astype(int))
ohlc = ohlc.assign(sd=ohlc.close.expanding(1).std(ddof=0))
ohlc.insert(1, 'symbol', contract.symbol)
# ohlc = ohlc.groupby('symbol').apply(lambda df: df.sort_values(
# 'date', ascending=False)).reset_index(drop=True)
# ohlcsd = ohlc.groupby('symbol').apply(
# lambda df: df.assign(sd=df.close.expanding(1).std(ddof=0)))
d[contract.symbol] = (tick[0].marketPrice(), chain[0].expirations,
chain[0].strikes, ohlc)
return d
def bars_update(
bars_: BarDataList,
has_new_bar: bool,
contract_: AContract,
):
# TODO: make sure that correct bar is received in test cases
assert contract_ in self._bars_subscriptions
if not has_new_bar:
return
bars_df = df(bars_[-2:-1])
bars_df = bars_df.set_index("date")
bar_s = bars_df.iloc[0]
funcs_dict_ = self._bars_subscriptions[contract_]["funcs"]
for func_, kwargs_ in funcs_dict_.items():
func_(bar_s, **kwargs_)
while len(bars_) != 1:
bars_.pop(0)
def test_ibgw_interactive(ib_docker):
ib = IB()
wait = 120
while not ib.isConnected():
try:
IB.sleep(1)
ib.connect('localhost', 4002, clientId=999)
except:
pass
wait -= 1
if wait <= 0:
break
contract = Forex('EURUSD')
bars = ib.reqHistoricalData(
contract, endDateTime='', durationStr='30 D',
barSizeSetting='1 hour', whatToShow='MIDPOINT', useRTH=True)
# convert to pandas dataframe:
df = util.df(bars)
print(df)
def get_strikes_and_expiration(self):
ES = Future(symbol='ES',
lastTradeDateOrContractMonth='20200918',
exchange='GLOBEX',
currency='USD')
ib.qualifyContracts(ES)
expiration = self.next_weekday(datetime.today(),
self.next_exp_weekday())
chains = ib.reqSecDefOptParams(underlyingSymbol='ES',
futFopExchange='GLOBEX',
underlyingSecType='FUT',
underlyingConId=ES.conId)
chain = util.df(chains)
strikes = chain[chain['expirations'].astype(str).str.contains(
expiration)].loc[:, 'strikes'].values[0]
[ESValue] = ib.reqTickers(ES)
ES_price = ESValue.marketPrice()
strikes = [
strike for strike in strikes
if strike % 5 == 0 and ES_price - 10 < strike < ES_price + 10
]
return strikes, expiration
def get_hist(ib, contract, duration):
'''Gets 1-day bars of contracts for the duration specified
Args:
(ib) as the active ib object
(contract) as obj
(duration) as int
Returns: dataframe of symbol, date, ohlc, avg and volume
'''
# Prepare the duration
strduration = str(duration) + ' D'
# Extract the history
hist = ib.reqHistoricalData(contract=contract, endDateTime='',
durationStr=strduration, barSizeSetting='1 day',
whatToShow='Trades', useRTH=True)
df = util.df(hist)
df.insert(0, column='symbol', value=contract.symbol)
return df
def get_ibkr_stock_candles(self, symbol, tf, start_time, end_time):
tf_map = {
1: "1 min",
5: "5 mins",
15: "15 mins",
30: "30 mins",
60: "1 hour",
240: "4 hours",
1440: "1 day"
}
start_datetime, end_datetime = dateparser.parse(
start_time), dateparser.parse(end_time)
duration = (end_datetime - start_datetime).days + 1
data = self.ib.reqHistoricalData(Stock(str(symbol), 'SMART', 'USD'),
endDateTime=end_datetime,
durationStr=f'{duration} D',
barSizeSetting=tf_map[tf],
whatToShow='TRADES',
useRTH=False,
formatDate=1)
return util.df(data)
def ES(self):
ES = Future(symbol='ES',
lastTradeDateOrContractMonth='20200918',
exchange='GLOBEX',
currency='USD')
ib.qualifyContracts(ES)
ES_df = ib.reqHistoricalData(contract=ES,
endDateTime=endDateTime,
durationStr=No_days,
barSizeSetting=interval,
whatToShow='TRADES',
useRTH=False)
ES_df = util.df(ES_df)
ES_df.set_index('date', inplace=True)
ES_df['Resistance'], ES_df['Support'] = self.res_sup(ES_df)
ES_df['RSI'] = ta.RSI(ES_df['close'])
ES_df['macd'], ES_df['macdsignal'], ES_df['macdhist'] = ta.MACD(
ES_df['close'], fastperiod=12, slowperiod=26, signalperiod=9)
ES_df['MA_9'] = ta.MA(ES_df['close'], timeperiod=9)
ES_df['MA_21'] = ta.MA(ES_df['close'], timeperiod=21)
ES_df['MA_200'] = ta.MA(ES_df['close'], timeperiod=200)
ES_df['EMA_9'] = ta.EMA(ES_df['close'], timeperiod=9)
ES_df['EMA_21'] = ta.EMA(ES_df['close'], timeperiod=21)
ES_df['EMA_50'] = ta.EMA(ES_df['close'], timeperiod=50)
ES_df['EMA_200'] = ta.EMA(ES_df['close'], timeperiod=200)
ES_df['ATR'] = ta.ATR(ES_df['high'], ES_df['low'], ES_df['close'])
ES_df['roll_max_cp'] = ES_df['high'].rolling(20).max()
ES_df['roll_min_cp'] = ES_df['low'].rolling(20).min()
ES_df['roll_max_vol'] = ES_df['volume'].rolling(20).max()
ES_df['EMA_21-EMA_9'] = ES_df['EMA_21'] - ES_df['EMA_9']
ES_df['EMA_200-EMA_50'] = ES_df['EMA_200'] - ES_df['EMA_50']
ES_df['B_upper'], ES_df['B_middle'], ES_df['B_lower'] = ta.BBANDS(
ES_df['close'], matype=MA_Type.T3)
ES_df.dropna(inplace=True)
return ES_df
def pull_from_IB(self, start, end, market='EURUSD', freq='1 hour'):
'''
Note, all times should be UTC (aka Zulu, Z, GMT)
:param start: string in datetime format, eg. '2019-07-18 17:15:00'
:param end: same format as start
:param freq: Must be one of: ‘1 secs’, ‘5 secs’, ‘10 secs’ 15 secs’,
‘30 secs’, ‘1 min’, ‘2 mins’, ‘3 mins’, ‘5 mins’, ‘10 mins’,
‘15 mins’, ‘20 mins’, ‘30 mins’, ‘1 hour’, ‘2 hours’,
‘3 hours’, ‘4 hours’, ‘8 hours’, ‘1 day’, ‘1 week’,
‘1 month’ (see IB Docs)
'''
#Parse the requested window so API calls are not as lengthy
time_list = self.parse_datetime(start, end, freq)
df_list = []
contract = Forex(market)
for time_pair in time_list:
'''
useRTH (bool) – If True then only show data from within Regular Trading Hours,
if False then show all data.
formatDate (int) – For an intraday request setting to 2 will cause the returned
date fields to be timezone-aware datetime.datetime with UTC timezone,
instead of local timezone as used by TWS.
'''
bars = self.ib.reqHistoricalData(contract,
endDateTime=time_pair[0],
durationStr=time_pair[1],
barSizeSetting=freq,
whatToShow='MIDPOINT',
useRTH=True)
# convert to pandas dataframe:
df = util.df(bars)
#drop columns ['barCount', 'average']
df_list.append(df[['date', 'open', 'high', 'low', 'close']].copy())
return df_list
async def gethists(ib, contracts, duration, timeout):
dur_str = str(duration) + ' D'
block = 20 # split into blocks to avoid max message per second error
c_blks = [contracts[i:i + block] for i in range(0, len(contracts), block)]
# split contracts to {symbol: future_pending} dictionary
aiolists = [{
c.symbol: ib.reqHistoricalDataAsync(contract=c,
endDateTime="",
durationStr=dur_str,
barSizeSetting="1 day",
whatToShow="Trades",
useRTH=True)
for c in cs
} for cs in c_blks]
histlist = []
for aios in aiolists:
for k, v in aios.items():
try:
hist = await asyncio.wait_for(v, timeout)
df = util.df(hist)
df.insert(0, 'symbol', k)
except asyncio.TimeoutError:
print(f"Timeout exception for {k}")
df = pd.DataFrame([{'symbol': k}])
except Exception as e:
print(f"Some other error {e}")
df = pd.DataFrame([{'symbol': k}])
histlist.append(df)
return histlist
from ib_insync import IB, util, Forex
if __name__ == "__main__":
ib = IB()
ib.connect('localhost', 4001, clientId=1)
contract = Forex('EURUSD')
bars = ib.reqHistoricalData(contract,
endDateTime='',
durationStr='30 D',
barSizeSetting='1 hour',
whatToShow='MIDPOINT',
useRTH=True)
# convert to pandas dataframe:
df = util.df(bars)
print(df)
def persistMarketDataBarsOnDisk(bars,
mydb: database.tradingDB,
tableSchema,
doCorrection=True):
"""write the bars to disk.
update the data.
also calculate time differences between successive rows"""
if bars is not None and len(bars) > 0:
tStart = pd.datetime.now()
# convert bars to dataframe
df = util.df(bars)
# rename columns (the 'date' column defined in the bars is the 'datetime' column in the tables)
df.rename(columns={'date': 'datetime'}, inplace=True)
# calculate the time difference between rows
# calculate the difference in time between a row and the next row
df.loc[:, 'diffToNextRowInMinutes'] = df.datetime.diff().shift(
-1) / pd.Timedelta(1, 'm')
# the last difference is zero
df.iloc[-1, df.columns.get_loc('diffToNextRowInMinutes')] = 0
# upsert the dataframe
if tableSchema is not None:
mydb.upsertDataFrame(df, tableSchema)
# repair datetime on disk.
# it can happen that the data fetched contains less or more information
# than the data on disk (That has been fetched before).
# therefore, the diffDateTImes need to be corrected for all entries between
# the first and the last entries
firstDateTime = df.iloc[
0, df.columns.get_loc('datetime')].tz_localize(None)
lastDateTime = df.iloc[
-1, df.columns.get_loc('datetime')].tz_localize(None)
# find previous DateTime
ssn = mydb.Session()
previousDateTime = ssn.query(func.max(
tableSchema.c.datetime)).filter(
tableSchema.c.datetime < firstDateTime).scalar()
ssn.close()
# this is None if no such value exists
if previousDateTime is None:
previousDateTime = firstDateTime
if doCorrection:
df = mydb.correctDiffDateTimesForMarketDataTable(
tableName=tableSchema.name,
startDateTime=previousDateTime,
endDateTime=lastDateTime,
doCorrection=doCorrection)
pass
pass
tEnd = pd.datetime.now()
a = (
f'finished treating data. Table: {tableSchema.name}; nRows: {len(bars)}; elapsed time: {tEnd-tStart}'
)
_logger.info(a)
# print(a)
pass
pass
