def test_ibgw_port(host): account = os.environ['IB_ACCOUNT'] password = os.environ['IB_PASSWORD'] trade_mode = os.environ['TRADE_MODE'] # build local ./Dockerfile subprocess.check_call(['docker', 'build', '-t', IMAGE_NAME, '.']) # run a container docker_id = subprocess.check_output( ['docker', 'run', '--env', 'IB_ACCOUNT={}'.format(account), '--env', 'IB_PASSWORD={}'.format(password), '--env', 'TRADE_MODE={}'.format(trade_mode), '-p', '4002:4002', '-d', IMAGE_NAME, "tail", "-f", "/dev/null"]).decode().strip() time.sleep(60) ib = IB() ib.connect('localhost', 4002, clientId=1) contract = Forex('EURUSD') bars = ib.reqHistoricalData( contract, endDateTime='', durationStr='30 D', barSizeSetting='1 hour', whatToShow='MIDPOINT', useRTH=True) # convert to pandas dataframe: df = util.df(bars) print(df) # at the end of the test suite, destroy the container subprocess.check_call(['docker', 'rm', '-f', docker_id])
async def get_ohlc(ib, contract): ohlc = await ib.reqHistoricalDataAsync( contract=contract, endDateTime="", durationStr="365 D", barSizeSetting="1 day", whatToShow="Trades", useRTH=True, ) # reverse sort to have latest date on top df_ohlc = util.df(ohlc).sort_index(ascending=False).reset_index(drop=True) df_ohlc.insert(0, 'symbol', contract.symbol) df_ohlc["rise"] = [ df_ohlc["close"].rolling(i).apply( lambda x: x[0] - x[-1], raw=True).max() for i in range(1, len(df_ohlc) + 1) ] df_ohlc["rise"] = df_ohlc.rise.abs() df_ohlc["fall"] = [ df_ohlc["close"].rolling(i).apply( lambda x: x[0] - x[-1], raw=True).min() for i in range(1, len(df_ohlc) + 1) ] df_ohlc["fall"] = df_ohlc.fall.abs() df_ohlc = df_ohlc.assign(sd=df_ohlc.close.expanding(1).std(ddof=0)) # roll the standard deviation upwards df_ohlc.sd = df_ohlc.sd.expanding(1).max() return df_ohlc
def ib_get_historical_data(self, ibcontract, durationStr="1 Y", barSizeSetting="1 day", whatToShow="TRADES", log=None): """ Returns historical prices for a contract, up to today ibcontract is a Contract :returns list of prices in 4 tuples: Open high low close volume """ if log is None: log = self.log last_call = self.last_historic_price_calltime avoid_pacing_violation(last_call, log=log) bars = self.ib.reqHistoricalData(ibcontract, endDateTime='', durationStr=durationStr, barSizeSetting=barSizeSetting, whatToShow=whatToShow, useRTH=True, formatDate=1) df = util.df(bars) self.last_historic_price_calltime = datetime.datetime.now() return df
def download_bars_data( self, contract: AContract, start_date: date, end_date: date, bar_size: timedelta, rth: bool, **kwargs, ) -> pd.DataFrame: ib_contract = self._to_ib_contract(contract=contract) dates = generate_trading_days(start_date=start_date, end_date=end_date) duration = f"{len(dates)} D" bar_size_str = self._to_ib_bar_size(bar_size=bar_size) bar_data = self._ib_conn.reqHistoricalData( contract=ib_contract, endDateTime=end_date, durationStr=duration, barSizeSetting=bar_size_str, whatToShow="TRADES", useRTH=False, ) data = util.df(objs=bar_data) if data is not None and len(data) != 0: data = self._format_data(data=data) else: data = pd.DataFrame() return data
def all_bars_df(self): if self.all_bars: df = util.df(self.all_bars) df.date = df.date.astype('datetime64') df.set_index('date', inplace=True) else: df = pd.DataFrame() return df
def reset_volume(self, avg_periods) -> int: # TODO: make span adjust to length of requested data bars = self.all_bars or self.bars if bars == self.bars: self.span = len(self.bars) df = util.df(bars) # last 5 days volume = df.iloc[-14100:].volume.rolling(avg_periods).sum() \ .mean().round() log.debug(f'volume: {volume}') return volume
def data(self) -> Union[pd.DataFrame, pd.datetime]: """Return df ready to be written to datastore or date of end point for additional downloads""" if self.bars: self.df = util.df([b for bars in reversed(self.bars) for b in bars]) self.df.set_index('date', inplace=True) if self.ok_to_write: df = self.df self.df = pd.DataFrame() return df log.debug(f'cannot write data')
def get_futures(contract_str, remove_weekly=False): ibw = IbWrapper() ib = ibw.ib vix = Future(contract_str, includeExpired=False) cds = ib.reqContractDetails(vix) contracts = [cd.contract for cd in cds] if remove_weekly: contracts = [ contract for contract in contracts if len(contract.localSymbol) <= 4 ] bars_list = [] for contract in contracts: bars = ib.reqHistoricalData(contract, endDateTime='', durationStr='5 D', barSizeSetting='1 day', whatToShow='TRADES', useRTH=True, formatDate=1) if bars: bars_list.append(util.df(bars)) ib.disconnect() contract_df = util.df(contracts) close_list = [item.loc[:, ['date', 'close']] for item in bars_list] close_list = [item.set_index('date') for item in close_list] close_list = [ item.rename(index=str, columns={'close': name}) for item, name in zip( close_list, pd.to_datetime(contract_df['lastTradeDateOrContractMonth'])) ] future_series = pd.concat(close_list, axis=1, sort=False) future_series = future_series.transpose().sort_index() future_series.columns = pd.to_datetime(future_series.columns) return future_series, contract_df
def get_strikes_and_expiration(self): expiration = self.next_weekday(datetime.date.today(), self.next_exp_weekday()) chains = self.ib.reqSecDefOptParams(underlyingSymbol='ES', futFopExchange='GLOBEX', underlyingSecType='FUT', underlyingConId=self.ES.conId) chain = util.df(chains) strikes = chain[chain['expirations'].astype(str).str.contains(expiration)].loc[:, 'strikes'].values[0] [ESValue] = self.ib.reqTickers(self.ES) ES_price = ESValue.marketPrice() strikes = [strike for strike in strikes if strike % 5 == 0 and ES_price - 10 < strike < ES_price + 10] return strikes, expiration
def option_history(self, contract): ib.qualifyContracts(contract) df = pd.DataFrame( util.df( ib.reqHistoricalData(contract=contract, endDateTime=endDateTime, durationStr=No_days, barSizeSetting=interval, whatToShow='MIDPOINT', useRTH=False, keepUpToDate=False))[['date', 'close']]) df.columns = ['date', f"{contract.symbol}_{contract.right}_close"] df.set_index('date', inplace=True) return df
def ES(self): ES = Future(symbol='ES', lastTradeDateOrContractMonth='20201218', exchange='GLOBEX', currency='USD') ib.qualifyContracts(ES) ES_df = ib.reqHistoricalData(contract=ES, endDateTime=endDateTime, durationStr=No_days, barSizeSetting=interval, whatToShow='TRADES', useRTH=False) ES_df = util.df(ES_df) ES_df.set_index('date', inplace=True) ES_df.index = pd.to_datetime(ES_df.index) # ES_df['hours'] = ES_df.index.strftime('%H').astype(int) # ES_df['minutes'] = ES_df.index.strftime('%M').astype(int) # ES_df['hours + minutes'] = ES_df['hours'] * 100 + ES_df['minutes'] # ES_df['Day_of_week'] = ES_df.index.dayofweek ES_df['RSI'] = ta.RSI(ES_df['close'], timeperiod=5) # ES_df['macd'], ES_df['macdsignal'], ES_df['macdhist'] = ta.MACD(ES_df['close'], fastperiod=12, slowperiod=26, # signalperiod=9) # ES_df['macd - macdsignal'] = ES_df['macd'] - ES_df['macdsignal'] ES_df['MA_9'] = ta.MA(ES_df['close'], timeperiod=9) ES_df['MA_21'] = ta.MA(ES_df['close'], timeperiod=21) # ES_df['MA_200'] = ta.MA(ES_df['close'], timeperiod=200) # ES_df['EMA_9'] = ta.EMA(ES_df['close'], timeperiod=9) # ES_df['EMA_26'] = ta.EMA(ES_df['close'], timeperiod=26) # ES_df['derv_1'] = np.gradient(ES_df['EMA_9']) # ES_df['EMA_9_26'] = ES_df['EMA_9'] / ES_df['EMA_26'] # ES_df['EMA_50'] = ta.EMA(ES_df['close'], timeperiod=50) # ES_df['EMA_200'] = ta.EMA(ES_df['close'], timeperiod=200) ES_df['ATR'] = ta.ATR(ES_df['high'], ES_df['low'], ES_df['close'], timeperiod=20) # ES_df['roll_max_cp'] = ES_df['high'].rolling(20).max() # ES_df['roll_min_cp'] = ES_df['low'].rolling(20).min() # ES_df['Mean_ATR'] = (ta.ATR(ES_df['high'], ES_df['low'], ES_df['close'], 21)).mean() ES_df['roll_max_vol'] = ES_df['volume'].rolling(20).max() # ES_df['vol/max_vol'] = ES_df['volume'] / ES_df['roll_max_vol'] ES_df['EMA_9-EMA_26'] = ES_df['EMA_9'] - ES_df['EMA_26'] # ES_df['EMA_200-EMA_50'] = ES_df['EMA_200'] - ES_df['EMA_50'] # ES_df['B_upper'], ES_df['B_middle'], ES_df['B_lower'] = ta.BBANDS(ES_df['close'], timeperiod=6, nbdevup=1, # nbdevdn=1, matype=MA_Type.T3) ES_df.dropna(inplace=True) ES_df = renko_df(ES_df, 1.5) return ES_df
def _fetch_to_df(ib, symbol, duration, bar_size): logger.info(f'Downloading historical data of {symbol} for {duration} ' f'with {bar_size} resolution') contract = Stock(symbol, exchange='SMART', currency='USD') bars = ib.reqHistoricalData( contract, endDateTime='', durationStr=duration, barSizeSetting=bar_size, whatToShow='TRADES', useRTH=True) df = util.df(bars) df.set_index('date', inplace=True) df.index = df.index.tz_localize(LOCAL_TZ).tz_convert('UTC') df['volume'] *= 100 return df
def add(self, text=''): text = text #or self.edit.text() if text: contract = eval(text) if (contract and self.ib.qualifyContracts(contract)): data = self.ib.reqHistoricalData(contract, endDateTime='', durationStr='30 D', barSizeSetting='4 hours', whatToShow='MIDPOINT', useRTH=True) df = util.df(data) print(df["close"]) self.graph.plot(df["close"]) self.edit.setText(text)
def into_dataframe(data: list, symbol, tf, qtrade=False, index=True) -> pd.DataFrame: """Converts Binance response list into dataframe""" crypto = 'USDT' in symbol or 'BUSD' in symbol if crypto: df = pd.DataFrame(data, columns=[ "timestamp", "open", "high", "low", "close", "volume", "", "", "", "", "", "" ]) df[["open", "high", "low", "close", "volume"]] = df[["open", "high", "low", "close", "volume"]].astype(float) df['date'] = Helper.millisecond_timestamp_to_datetime( df['timestamp']) else: if qtrade: # Questrade Data Handling df = pd.DataFrame(data).drop( 'start', axis=1).rename(columns={'end': 'date'}) df['date'] = [ pd.Timestamp(date).tz_localize(None) for date in df['date'] ] df['timestamp'] = Helper.datetime_to_millisecond_timestamp( df['date']) else: # IBKR Data Handling df = util.df(data).drop(['average', 'barCount'], axis=1) df['timestamp'] = Helper.datetime_to_millisecond_timestamp( df['date']) df['symbol'] = [symbol] * len(data) df['tf'] = [tf] * len(data) if index: return df[[ 'symbol', 'tf', 'timestamp', 'date', 'open', 'high', 'low', 'close', 'volume' ]].set_index("timestamp") return df[[ 'symbol', 'tf', 'timestamp', 'date', 'open', 'high', 'low', 'close', 'volume' ]]
def on_bar_update(bars: BarDataList, has_new_bar: bool): try: logger.debug("on_bar_update: {} {} {}".format(bars.contract, bars[-1], has_new_bar)) df = util.df([bars[-1]]) contract = bars.contract symbol = None if type(contract) == Forex: symbol = contract.symbol + contract.currency elif type(contract) == Stock: symbol = contract.symbol + contract.currency if has_new_bar and symbol: df['symbol'] = symbol update_k_bars(df=df, _type=bars.barSizeSetting) except Exception as e: logger.exception(e)
def nse_und(ib, contract): '''returns the underlying details Args: (ib) as the active ib object (contract) as the contract Returns: None. The underlying is pickled to _und.pkl''' ticker = get_div_tick(ib, contract) df_und = util.df([ticker]) cols = [ 'contract', 'time', 'bid', 'bidSize', 'ask', 'askSize', 'last', 'lastSize', 'volume', 'open', 'high', 'low', 'close', 'dividends' ] df_und = df_und[cols] df_und = df_und.assign(undPrice=np.where(df_und['last'].isnull(), df_und.close, df_und['last'])) try: divrate = df_und.dividends[0][0] / df_und.dividends[0][ 0] / df_und.undPrice except (TypeError, AttributeError) as e: divrate = 0.0 df_und = df_und.assign(divrate=divrate) df_und = df_und.assign( symbol=[c[1].symbol for c in df_und.contract.items()]) #... get the lot, margin, undPrice and dividend rate undlot = df_slm.loc[df_slm.ibSymbol == contract.symbol, 'lot'].item() df_und['lot'] = undlot # margin of underlying order = Order(action='SELL', totalQuantity=undlot, orderType='MKT') # margin = float(ib.whatIfOrder(contract, order).initMarginChange) # doesn't work because permission is not there for NRIs!!! margin = df_slm.loc[df_slm.ibSymbol == contract.symbol, 'margin'].item() df_und['margin'] = margin df_und.to_pickle(fspath + contract.symbol + '_und.pkl') return df_und
def make_clean_df(self, ES_df, hashbar=None): ES_df = util.df(ES_df) ES_df['RSI'] = ta.RSI(ES_df['close']) ES_df['macd'], ES_df['macdsignal'], ES_df['macdhist'] = ta.MACD( ES_df['close'], fastperiod=12, slowperiod=26, signalperiod=9) ES_df['MA_9'] = ta.MA(ES_df['close'], timeperiod=9) ES_df['MA_21'] = ta.MA(ES_df['close'], timeperiod=21) ES_df['MA_200'] = ta.MA(ES_df['close'], timeperiod=200) ES_df['EMA_9'] = ta.EMA(ES_df['close'], timeperiod=9) ES_df['EMA_21'] = ta.EMA(ES_df['close'], timeperiod=21) ES_df['EMA_200'] = ta.EMA(ES_df['close'], timeperiod=200) ES_df['ATR'] = ta.ATR(ES_df['high'], ES_df['low'], ES_df['close']) ES_df['roll_max_cp'] = ES_df['high'].rolling(20).max() ES_df['roll_min_cp'] = ES_df['low'].rolling(20).min() ES_df['roll_max_vol'] = ES_df['volume'].rolling(20).max() ES_df.dropna(inplace=True) self.loop_function(ES_df)
def create_candle(self) -> None: df = util.df(self.new_bars) df.date = df.date.astype('datetime64') df.set_index('date', inplace=True) df['volume_weighted'] = df.average * df.volume weighted_price = df.volume_weighted.sum() / df.volume.sum() self.newCandle.emit({ 'backfill': self.backfill, 'date': df.index[-1], 'open': df.open[0], 'high': df.high.max(), 'low': df.low.min(), 'close': df.close[-1], 'weighted_price': weighted_price, # 'price': weighted_price, 'price': df.close[-1], 'volume': df.volume.sum() })
async def ohlcAsync(ib, c): # handle errors try: ohlc = await ib.reqHistoricalDataAsync(contract=c, endDateTime="", durationStr="365 D", barSizeSetting="1 day", whatToShow="Trades", useRTH=True) except AttributeError: return None # reverse sort to have latest date on top if ohlc: # ohlc has captured something! df_ohlc = util.df(ohlc).sort_index(ascending=False).reset_index( drop=True) df_ohlc.insert(0, 'symbol', c.symbol) return df_ohlc
async def basedata(ib, symbols): contracts = [ await ib.qualifyContractsAsync(Stock(s, 'SMART', 'USD')) for s in symbols ] # Dictionary to eliminate duplicate symbols contracts = {s.symbol: s for c in contracts for s in c} d = {} # get prices, chains and ohlcs with standard deviaPtion for contract in contracts.values(): tick = await ib.reqTickersAsync(contract) chain = await ib.reqSecDefOptParamsAsync( underlyingSymbol=contract.symbol, futFopExchange='', underlyingSecType=contract.secType, underlyingConId=contract.conId) ohlc = await ib.reqHistoricalDataAsync(contract=contract, endDateTime='', durationStr='365 D', barSizeSetting='1 day', whatToShow='Trades', useRTH=True) ohlc = util.df(ohlc) ohlc = ohlc.assign(dte=ohlc.index.astype(int)) ohlc = ohlc.assign(sd=ohlc.close.expanding(1).std(ddof=0)) ohlc.insert(1, 'symbol', contract.symbol) # ohlc = ohlc.groupby('symbol').apply(lambda df: df.sort_values( # 'date', ascending=False)).reset_index(drop=True) # ohlcsd = ohlc.groupby('symbol').apply( # lambda df: df.assign(sd=df.close.expanding(1).std(ddof=0))) d[contract.symbol] = (tick[0].marketPrice(), chain[0].expirations, chain[0].strikes, ohlc) return d
def bars_update( bars_: BarDataList, has_new_bar: bool, contract_: AContract, ): # TODO: make sure that correct bar is received in test cases assert contract_ in self._bars_subscriptions if not has_new_bar: return bars_df = df(bars_[-2:-1]) bars_df = bars_df.set_index("date") bar_s = bars_df.iloc[0] funcs_dict_ = self._bars_subscriptions[contract_]["funcs"] for func_, kwargs_ in funcs_dict_.items(): func_(bar_s, **kwargs_) while len(bars_) != 1: bars_.pop(0)
def test_ibgw_interactive(ib_docker): ib = IB() wait = 120 while not ib.isConnected(): try: IB.sleep(1) ib.connect('localhost', 4002, clientId=999) except: pass wait -= 1 if wait <= 0: break contract = Forex('EURUSD') bars = ib.reqHistoricalData( contract, endDateTime='', durationStr='30 D', barSizeSetting='1 hour', whatToShow='MIDPOINT', useRTH=True) # convert to pandas dataframe: df = util.df(bars) print(df)
def get_strikes_and_expiration(self): ES = Future(symbol='ES', lastTradeDateOrContractMonth='20200918', exchange='GLOBEX', currency='USD') ib.qualifyContracts(ES) expiration = self.next_weekday(datetime.today(), self.next_exp_weekday()) chains = ib.reqSecDefOptParams(underlyingSymbol='ES', futFopExchange='GLOBEX', underlyingSecType='FUT', underlyingConId=ES.conId) chain = util.df(chains) strikes = chain[chain['expirations'].astype(str).str.contains( expiration)].loc[:, 'strikes'].values[0] [ESValue] = ib.reqTickers(ES) ES_price = ESValue.marketPrice() strikes = [ strike for strike in strikes if strike % 5 == 0 and ES_price - 10 < strike < ES_price + 10 ] return strikes, expiration
def get_hist(ib, contract, duration): '''Gets 1-day bars of contracts for the duration specified Args: (ib) as the active ib object (contract) as obj (duration) as int Returns: dataframe of symbol, date, ohlc, avg and volume ''' # Prepare the duration strduration = str(duration) + ' D' # Extract the history hist = ib.reqHistoricalData(contract=contract, endDateTime='', durationStr=strduration, barSizeSetting='1 day', whatToShow='Trades', useRTH=True) df = util.df(hist) df.insert(0, column='symbol', value=contract.symbol) return df
def get_ibkr_stock_candles(self, symbol, tf, start_time, end_time): tf_map = { 1: "1 min", 5: "5 mins", 15: "15 mins", 30: "30 mins", 60: "1 hour", 240: "4 hours", 1440: "1 day" } start_datetime, end_datetime = dateparser.parse( start_time), dateparser.parse(end_time) duration = (end_datetime - start_datetime).days + 1 data = self.ib.reqHistoricalData(Stock(str(symbol), 'SMART', 'USD'), endDateTime=end_datetime, durationStr=f'{duration} D', barSizeSetting=tf_map[tf], whatToShow='TRADES', useRTH=False, formatDate=1) return util.df(data)
def ES(self): ES = Future(symbol='ES', lastTradeDateOrContractMonth='20200918', exchange='GLOBEX', currency='USD') ib.qualifyContracts(ES) ES_df = ib.reqHistoricalData(contract=ES, endDateTime=endDateTime, durationStr=No_days, barSizeSetting=interval, whatToShow='TRADES', useRTH=False) ES_df = util.df(ES_df) ES_df.set_index('date', inplace=True) ES_df['Resistance'], ES_df['Support'] = self.res_sup(ES_df) ES_df['RSI'] = ta.RSI(ES_df['close']) ES_df['macd'], ES_df['macdsignal'], ES_df['macdhist'] = ta.MACD( ES_df['close'], fastperiod=12, slowperiod=26, signalperiod=9) ES_df['MA_9'] = ta.MA(ES_df['close'], timeperiod=9) ES_df['MA_21'] = ta.MA(ES_df['close'], timeperiod=21) ES_df['MA_200'] = ta.MA(ES_df['close'], timeperiod=200) ES_df['EMA_9'] = ta.EMA(ES_df['close'], timeperiod=9) ES_df['EMA_21'] = ta.EMA(ES_df['close'], timeperiod=21) ES_df['EMA_50'] = ta.EMA(ES_df['close'], timeperiod=50) ES_df['EMA_200'] = ta.EMA(ES_df['close'], timeperiod=200) ES_df['ATR'] = ta.ATR(ES_df['high'], ES_df['low'], ES_df['close']) ES_df['roll_max_cp'] = ES_df['high'].rolling(20).max() ES_df['roll_min_cp'] = ES_df['low'].rolling(20).min() ES_df['roll_max_vol'] = ES_df['volume'].rolling(20).max() ES_df['EMA_21-EMA_9'] = ES_df['EMA_21'] - ES_df['EMA_9'] ES_df['EMA_200-EMA_50'] = ES_df['EMA_200'] - ES_df['EMA_50'] ES_df['B_upper'], ES_df['B_middle'], ES_df['B_lower'] = ta.BBANDS( ES_df['close'], matype=MA_Type.T3) ES_df.dropna(inplace=True) return ES_df
def pull_from_IB(self, start, end, market='EURUSD', freq='1 hour'): ''' Note, all times should be UTC (aka Zulu, Z, GMT) :param start: string in datetime format, eg. '2019-07-18 17:15:00' :param end: same format as start :param freq: Must be one of: ‘1 secs’, ‘5 secs’, ‘10 secs’ 15 secs’, ‘30 secs’, ‘1 min’, ‘2 mins’, ‘3 mins’, ‘5 mins’, ‘10 mins’, ‘15 mins’, ‘20 mins’, ‘30 mins’, ‘1 hour’, ‘2 hours’, ‘3 hours’, ‘4 hours’, ‘8 hours’, ‘1 day’, ‘1 week’, ‘1 month’ (see IB Docs) ''' #Parse the requested window so API calls are not as lengthy time_list = self.parse_datetime(start, end, freq) df_list = [] contract = Forex(market) for time_pair in time_list: ''' useRTH (bool) – If True then only show data from within Regular Trading Hours, if False then show all data. formatDate (int) – For an intraday request setting to 2 will cause the returned date fields to be timezone-aware datetime.datetime with UTC timezone, instead of local timezone as used by TWS. ''' bars = self.ib.reqHistoricalData(contract, endDateTime=time_pair[0], durationStr=time_pair[1], barSizeSetting=freq, whatToShow='MIDPOINT', useRTH=True) # convert to pandas dataframe: df = util.df(bars) #drop columns ['barCount', 'average'] df_list.append(df[['date', 'open', 'high', 'low', 'close']].copy()) return df_list
async def gethists(ib, contracts, duration, timeout): dur_str = str(duration) + ' D' block = 20 # split into blocks to avoid max message per second error c_blks = [contracts[i:i + block] for i in range(0, len(contracts), block)] # split contracts to {symbol: future_pending} dictionary aiolists = [{ c.symbol: ib.reqHistoricalDataAsync(contract=c, endDateTime="", durationStr=dur_str, barSizeSetting="1 day", whatToShow="Trades", useRTH=True) for c in cs } for cs in c_blks] histlist = [] for aios in aiolists: for k, v in aios.items(): try: hist = await asyncio.wait_for(v, timeout) df = util.df(hist) df.insert(0, 'symbol', k) except asyncio.TimeoutError: print(f"Timeout exception for {k}") df = pd.DataFrame([{'symbol': k}]) except Exception as e: print(f"Some other error {e}") df = pd.DataFrame([{'symbol': k}]) histlist.append(df) return histlist
from ib_insync import IB, util, Forex if __name__ == "__main__": ib = IB() ib.connect('localhost', 4001, clientId=1) contract = Forex('EURUSD') bars = ib.reqHistoricalData(contract, endDateTime='', durationStr='30 D', barSizeSetting='1 hour', whatToShow='MIDPOINT', useRTH=True) # convert to pandas dataframe: df = util.df(bars) print(df)
def persistMarketDataBarsOnDisk(bars, mydb: database.tradingDB, tableSchema, doCorrection=True): """write the bars to disk. update the data. also calculate time differences between successive rows""" if bars is not None and len(bars) > 0: tStart = pd.datetime.now() # convert bars to dataframe df = util.df(bars) # rename columns (the 'date' column defined in the bars is the 'datetime' column in the tables) df.rename(columns={'date': 'datetime'}, inplace=True) # calculate the time difference between rows # calculate the difference in time between a row and the next row df.loc[:, 'diffToNextRowInMinutes'] = df.datetime.diff().shift( -1) / pd.Timedelta(1, 'm') # the last difference is zero df.iloc[-1, df.columns.get_loc('diffToNextRowInMinutes')] = 0 # upsert the dataframe if tableSchema is not None: mydb.upsertDataFrame(df, tableSchema) # repair datetime on disk. # it can happen that the data fetched contains less or more information # than the data on disk (That has been fetched before). # therefore, the diffDateTImes need to be corrected for all entries between # the first and the last entries firstDateTime = df.iloc[ 0, df.columns.get_loc('datetime')].tz_localize(None) lastDateTime = df.iloc[ -1, df.columns.get_loc('datetime')].tz_localize(None) # find previous DateTime ssn = mydb.Session() previousDateTime = ssn.query(func.max( tableSchema.c.datetime)).filter( tableSchema.c.datetime < firstDateTime).scalar() ssn.close() # this is None if no such value exists if previousDateTime is None: previousDateTime = firstDateTime if doCorrection: df = mydb.correctDiffDateTimesForMarketDataTable( tableName=tableSchema.name, startDateTime=previousDateTime, endDateTime=lastDateTime, doCorrection=doCorrection) pass pass tEnd = pd.datetime.now() a = ( f'finished treating data. Table: {tableSchema.name}; nRows: {len(bars)}; elapsed time: {tEnd-tStart}' ) _logger.info(a) # print(a) pass pass