def get_vix(): """Fetch vix from Interactive Brokers and append to history''' :return: Dataframe """ ibw = IbWrapper() ib = ibw.ib vix = Index('VIX') cds = ib.reqContractDetails(vix) # contracts = [cd.contract for cd in cds] bars = ib.reqHistoricalData(cds[0].contract, endDateTime='', durationStr='1 Y', barSizeSetting='1 day', whatToShow='TRADES', useRTH=True, formatDate=1) ib.disconnect() vix = util.df(bars) vix = vix.set_index('date') vix = vix[['open', 'high', 'low', 'close']] vix_history = read_feather(str(UpdateSP500Data.TOP_LEVEL_PATH / 'vix_history')) full_hist = vix.combine_first(vix_history) write_feather(full_hist, str(UpdateSP500Data.TOP_LEVEL_PATH / 'vix_history')) return full_hist['close']
def spx_bar_history(update_bars=True): file_name = 'sp500_5min_bars' df_hist = read_feather(UpdateSP500Data.DATA_BASE_PATH / file_name) # Download latest if update_bars: ib = IB() ib.connect('127.0.0.1', port=4001, clientId=40) contract = Index('SPX', 'CBOE', 'USD') bars = ib.reqHistoricalData( contract, endDateTime='', durationStr='1 M', barSizeSetting='5 mins', whatToShow='TRADES', useRTH=True, formatDate=1) ib.disconnect() df = util.df(bars) df = df.set_index('date') full_hist = df.combine_first(df_hist) write_feather(full_hist, UpdateSP500Data.DATA_BASE_PATH / file_name) else: full_hist = df_hist.copy() return full_hist
def get_hsi_data(self): ''' 获取 HSI 一个月5分钟的信息 ''' symbol = 'HSI' exchange = 'HKFE' currency = 'HKD' index = Index(symbol, exchange, currency) data = self.ib.reqHistoricalData(index, endDateTime='20180119 15:00:00', durationStr='900 S', barSizeSetting='5 mins', whatToShow='TRADES', useRTH=True) if not data: raise RuntimeError('HSI 数据接口返回空.') sql = 'insert into `hsi_5m` (`date`, `open`, `high`, `low`, `close`) values ' for bar_data in data: date = bar_data.date open_price = bar_data.open high = bar_data.high low = bar_data.low close = bar_data.close sql += "('{date}', {open:.4f}, {high:.4f}, {low:.4f}, {close:.4f}),".format(date=date, open=open_price, high=high, low=low, close=close) res = self.db.query(sql.rstrip(',')) if res.rowcount == 0: raise RuntimeError('SQL 语句执行异常, 插入数据库失败:%s' % sql) else: print('HSI Index 1个月5分钟数据导入完成.', flush=True)
def main(ib): from ib_insync import Stock, Forex, Index win_historical_ticks = sg.Window('IBKR Historical Ticks', layout) #win_historical_ticks.Element('_TickCount_').Update('0') # Event Loop to process "events" and get the "values" of the inputs while True: # Event Loop event, values = win_historical_ticks.Read() if event == '_CloseWindow_': #? commenting out "disconnect()" because any calling procedure would not want a child to disconnect it #ib.disconnect() logging.info("Close hit") win_historical_ticks.close() break if event == 'Qualify': # Update the "output" element to be the value of "input" element win_historical_ticks.Element('_OUTPUT_').Update(values['_Symbol_']) if values['_SecType_'] == 'STK': contract = Stock(values['_Symbol_'], values['_Exchange_'],values['_Currency_']) if values['_SecType_'] == 'CASH': contract = Forex(values['_Symbol_']) if values['_SecType_'] == 'IND': contract = Index(values['_Symbol_'], values['_Exchange_'],values['_Currency_']) qual = ib.qualifyContracts(contract) logging.info(str(qual)) if event == 'Fetch': #NOTE add validations here if values['_targetFolder_'] != '': dtmStartTime = datetime.strptime(str(values['_StartDate_']), '%Y-%m-%d %H:%M:%S') dtmStartTime = dtmStartTime.replace(hour=0, minute=1) dtmEndTime = datetime.strptime(str(values['_EndDate_']), '%Y-%m-%d %H:%M:%S') dtmEndTime = dtmEndTime.replace(hour=23, minute=59) intTicks = int(values['_tickCount_']) # whatToShow (str) – One of ‘Bid_Ask’, ‘Midpoint’ or ‘Trades’. strWhatToShow = str(values['_whatToShow_']) # useRTH – If True then only show data from within Regular Trading Hours, if False then show all data. boolUseRTH = bool(values['_useRTH_']) # ignoreSize (bool) – Ignore bid/ask ticks that only update the size. boolIgnoreSize = bool(values['_ignoreSize_']) #sg.popup(str([dtmStartTime, " - ", dtmEndTime, " - ", intTicks, " - ", strWhatToShow, " - ", boolUseRTH, " - ", boolIgnoreSize])) if dtmEndTime > dtmStartTime: listTicks = loop_reqHistoricalTicks(ib, contract, dtmStartTime, dtmEndTime, intTicks, strWhatToShow, boolUseRTH, boolIgnoreSize) #convert listTicks to a CSV and save dfTicks = pd.DataFrame(listTicks) dfTicks = dfTicks.rename(columns={'tickAttribLast':'symbol', 'specialConditions':'currency'}) dfTicks['symbol'] = contract.symbol dfTicks['exchange'] = contract.exchange dfTicks['currency'] = contract.currency dfTicks['time'] = dfTicks['time'].apply(utc2local) dfTicks['dataType'] = 'Ticks_' + str(values['_whatToShow_']) +('_RTH' if boolUseRTH else '_RHT+AMO') #For Linux uncomment below #dfTicks.to_csv(values['_targetFolder_'] +'/'+ contract.symbol +'_'+ str(dtmStartTime) +'_'+ str(dtmEndTime)+'.csv', index=False) #For Windows uncomment below dfTicks.to_csv(str_default_path +'//'+ contract.symbol +'_'+ dtmStartTime.strftime("%Y%M%d_%H%M%S") +'_'+ dtmEndTime.strftime("%Y%M%d_%H%M%S") +'.csv', index=False)
def __init__(self, mkt_symbol, vol_symbol, exchange_dict): """Abstract class for option asset container""" exchange_mkt = exchange_dict['exchange_mkt'] exchange_vol = exchange_dict['exchange_vol'] exchange_opt = exchange_dict['exchange_opt'] self.trading_class = exchange_dict['trading_class'] underlying_index = Index(mkt_symbol, exchange_mkt) ibw = IbWrapper() ib = ibw.ib self.underlying_qc = self.__get_underlying_qc(underlying_index, ib) self.sigma_qc = self.get_sigma_qc(vol_symbol, ib, exchange_vol) self.chain = self.get_option_chain(underlying_index, ib, exchange_opt) ib.disconnect()
def crawl_index_data(self): ''' 爬取 HSI Index 数据 ''' symbol = 'HSI' exchange = 'HKFE' currency = 'HKD' records = self.db.get_hsi_record() if not records.rowcount: raise Exception('获取上次爬取数据日期失败,数据库 records 表返回空.') last_time = list(records)[0].date now_time = datetime.datetime.now() diff = now_time - last_time dur = diff.days print('正在获取 HSI 数据.') index = Index(symbol, exchange, currency) data = self.ib.reqHistoricalData(index, endDateTime='', durationStr='%d D' % dur, barSizeSetting='1 day', whatToShow='TRADES', useRTH=True) if not data: raise RuntimeError('HSI 数据接口返回空.') sql = 'insert into `hsi` (`date`, `open`, `high`, `low`, `close`) values ' for bar_data in data: date_time = datetime.datetime(*bar_data.date.timetuple()[:6]) if date_time > last_time: date = bar_data.date open_price = bar_data.open high = bar_data.high low = bar_data.low close = bar_data.close sql += "('{date}', {open:.4f}, {high:.4f}, {low:.4f}, {close:.4f}),".format(date=date, open=open_price, high=high, low=low, close=close) rows = self.db.query(sql.rstrip(',')) if rows == 0: raise Exception('SQL 语句执行异常, 插入数据库失败:%s' % sql) else: now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) row = self.db.set_hsi_record(now) if not row: raise Exception('更新 HSI 日期记录失败.')
# indexes = sorted('OEX,XEO,XSP'.split(',')) indexes = [] #...Build a list of contracts stocks = [ Stock(symbol=s, currency=currency, exchange=exchange) for s in set(stocks) ] # sort the stocks s_dict = {s.symbol: s for s in stocks} od = OrderedDict(sorted(s_dict.items())) ss = [v for k, v in od.items()] # ixs = [Index(symbol=s,currency=currency, exchange='CBOE') for s in set(indexes)] ixs = [ Index(symbol=s, currency=currency, exchange='CBOE') for s in set(indexes) ] cs = ss + ixs # cs = cs[15:18] # DATA LIMITER!!! # sort in alphabetical order cs.sort(key=lambda x: x.symbol, reverse=False) def snp_list(ib): '''returns a list of qualified snp underlyings Args: ib as the active object Returns: qualified list with conID '''
# nseSymbol to ibSymbol dictionary for conversion ntoi = {'M&M': 'MM', 'M&MFIN': 'MMFIN', 'L&TFH': 'LTFH', 'NIFTY': 'NIFTY50'} # remap ibSymbol, based on the dictionary df_slm.ibSymbol = df_slm.ibSymbol.replace(ntoi) # separate indexes and equities, eliminate discards from df_slm indexes = ['NIFTY50', 'BANKNIFTY'] discards = ['NIFTYMID5', 'NIFTYIT', 'LUPIN'] equities = sorted([s for s in df_slm.ibSymbol if s not in indexes + discards]) symbols = equities + indexes cs = [ Stock(s, exchange) if s in equities else Index(s, exchange) for s in symbols ] # cs = cs[:5] # DATA LIMITER!!! def nse_list(ib): '''returns a list of qualified nse underlyings, with symbols and option chains for them Args: ib as the active ib object Returns: qualified list with conID ''' qcs = ib.qualifyContracts(*cs) # qualified underlyings qcs_chains = [] for i in range(0, len(qcs), 50):
# note that it does not work for Index symbol (ref: https://groups.io/g/insync/message/4718) import asyncio import pandas as pd from ib_insync import IB, Index, Stock, util # build up the contracts HOST = '127.0.0.1' PORT = '1300' CID = 0 with IB().connect(HOST, PORT, CID) as ib: timeout = 5 # contract = ib.qualifyContracts(Stock(symbol='SBUX', exchange='SMART', currency='USD'))[0] contract = ib.qualifyContracts( Index(symbol='VIX', exchange='CBOE', currency='USD'))[0] req = ib.reqHistoricalDataAsync(contract=contract, endDateTime="", durationStr="2 D", barSizeSetting="1 day", whatToShow="Trades", useRTH=True) try: hist = ib.run(asyncio.wait_for(req, timeout)) except asyncio.exceptions.TimeoutError: hist = [] print(contract) print(hist)
def parse_contract(ticker): """ Backtrader contract specification (https://www.backtrader.com/docu/live/ib/ib.html): TICKER # Stock type and SMART exchange TICKER-STK # Stock and SMART exchange TICKER-STK-EXCHANGE # Stock TICKER-STK-EXCHANGE-CURRENCY # Stock TICKER-CFD # CFD and SMART exchange TICKER-CFD-EXCHANGE # CFD TICKER-CDF-EXCHANGE-CURRENCY # Stock TICKER-IND-EXCHANGE # Index TICKER-IND-EXCHANGE-CURRENCY # Index TICKER-YYYYMM-EXCHANGE # Future TICKER-YYYYMM-EXCHANGE-CURRENCY # Future TICKER-YYYYMM-EXCHANGE-CURRENCY-MULT # Future TICKER-FUT-EXCHANGE-CURRENCY-YYYYMM-MULT # Future TICKER-YYYYMM-EXCHANGE-CURRENCY-STRIKE-RIGHT # FOP TICKER-YYYYMM-EXCHANGE-CURRENCY-STRIKE-RIGHT-MULT # FOP TICKER-FOP-EXCHANGE-CURRENCY-YYYYMM-STRIKE-RIGHT # FOP TICKER-FOP-EXCHANGE-CURRENCY-YYYYMM-STRIKE-RIGHT-MULT # FOP CUR1.CUR2-CASH-IDEALPRO # Forex TICKER-YYYYMMDD-EXCHANGE-CURRENCY-STRIKE-RIGHT # OPT TICKER-YYYYMMDD-EXCHANGE-CURRENCY-STRIKE-RIGHT-MULT # OPT TICKER-OPT-EXCHANGE-CURRENCY-YYYYMMDD-STRIKE-RIGHT # OPT TICKER-OPT-EXCHANGE-CURRENCY-YYYYMMDD-STRIKE-RIGHT-MULT # OPT :return: """ contract_type, symbol, exchange, currency, expire, multiplier = \ AsyncIBDataProvider.exctract_symbol(ticker) if contract_type == 'FX': return Forex(pair=symbol) if contract_type == 'IND': return Index(symbol, exchange, currency) if contract_type == 'FUT': return Future(symbol, expire, exchange, currency=currency, multiplier=multiplier) else: return Stock(symbol, exchange, currency)
LONG_BAR_FORMAT = '{desc:<25}{percentage:3.0f}%|{bar:30}{r_bar}' QBLK = 50 # block chunks for qualification start = datetime.now() # Suppress errors in console logfile = './data/test.log' with open(logfile, 'w'): pass util.logToFile(logfile, level=30) # Get the symbols df_syms = get_syms(MARKET) # Build the contracts raw_cts = [i for j in [[Stock(symbol, exchange, currency), Index(symbol, exchange, currency)] for symbol, exchange, currency in zip(df_syms.symbol, df_syms.exchange, df_syms.currency)] for i in j] c_blks = [raw_cts[i: i + QBLK] for i in range(0, len(raw_cts), QBLK)] tq_blks = tqdm(c_blks, position=0, leave=True, desc=f'Qualifying {len([i for c in c_blks for i in c])} ', bar_format=LONG_BAR_FORMAT) qunds = dict() # to store qualified underlying contracts # Qualify the underlyings with IB().connect(HOST, PORT, CID) as ib: for b in tq_blks: cts = ib.qualifyContracts(*b)
def main(ib): from ib_insync import Stock, Forex, Index win_historical = sg.Window('Historical OHLC', layout) while True: # Event Loop event, values = win_historical.Read() if event == 'Exit': #? commenting out "disconnect()" because any calling procedure would not want a child to disconnect it #ib.disconnect() logging.info("exit hit") win_historical.Close() break if event == 'Qualify': # Update the "output" element to be the value of "input" element win_historical.Element('_OUTPUT_').Update(values['_Symbol_']) if values['_SecType_'] == 'STK': contract = Stock(values['_Symbol_'], values['_Exchange_'], values['_Currency_']) if values['_SecType_'] == 'CASH': contract = Forex(values['_Symbol_']) if values['_SecType_'] == 'IND': contract = Index(values['_Symbol_'], values['_Exchange_'], values['_Currency_']) qual = ib.qualifyContracts(contract) logging.info(str(qual)) if event == 'Fetch': strDuration = str(values['_DurStr1_']) + ' ' + str( values['_DurStr2_']) strBarSize = str(values['_BarSize_']) strWhatToShow = str(values['_WhatToShow_']) logging.info('strDuration:%s , strBarSize: %s , strWhatToShow: %s', strDuration, strBarSize, strWhatToShow) barData = ib.reqHistoricalData(contract, '', strDuration, strBarSize, strWhatToShow, 1, 1, False) listData = [[ item.date.strftime("%Y%m%d, %H:%M:%S"), item.open, item.high, item.low, item.close, item.volume, item.barCount, item.average ] for item in barData] dfStockData = pd.DataFrame(listData) if dfStockData.shape[1] == 8 and dfStockData.shape[0] > 0: logging.info('dfStockData sample: %s', dfStockData.head(3)) dfStockData.columns = [ 'date', 'open', 'high', 'low', 'close', 'volume', 'barCount', 'average' ] relist = dfStockData.values.tolist() win_historical.Element('sgtable').Update(values=relist, num_rows=25, visible=True) if event == 'Download': #check if the DatFrame is minimally populated or not. #if populated: if dfStockData.shape[1] == 8 and dfStockData.shape[0] > 0: file_name = sg.PopupGetFile( 'Please enter filename to save (if file exists, it will be overwritten): ', save_as=True) fileTarget = open(file_name, 'w+') dfStockData.to_csv(fileTarget) fileTarget.close() sg.Popup("File saved.") #if NOT populated else: sg.Popup("Please fetch some data before trying to save it.")
def get_sigma_qc(self, vol_symbol, ib, exchange): """Returns implied Volatility for market""" sigma_index = Index(vol_symbol, exchange) sigma_qc = ib.qualifyContracts(sigma_index) assert (len(sigma_qc) == 1) return sigma_qc[0]
CID = 0 # Suppress errors in console logfile = './data/test.log' with open(logfile, 'w'): pass util.logToFile(logfile, level=30) # Get the symbols df_syms = get_syms(MARKET) # Build the contracts raw_cts = [ i for j in [[Stock(symbol, exchange, currency), Index(symbol, exchange, currency)] for symbol, exchange, currency in zip( df_syms.symbol, df_syms.exchange, df_syms.currency)] for i in j ] def qual(): '''Simple qualify, without async''' start = time.time() result = ib.qualifyContracts(*cts) qcon = {r.symbol: r for r in result if r} print( f'\ntook qual {time.time()-start} seconds to qualify {len(qcon)} contracts\n' ) print(qcon)