def get_vix():
"""Fetch vix from Interactive Brokers and append to history'''
:return: Dataframe
"""
ibw = IbWrapper()
ib = ibw.ib
vix = Index('VIX')
cds = ib.reqContractDetails(vix)
# contracts = [cd.contract for cd in cds]
bars = ib.reqHistoricalData(cds[0].contract,
endDateTime='',
durationStr='1 Y',
barSizeSetting='1 day',
whatToShow='TRADES',
useRTH=True,
formatDate=1)
ib.disconnect()
vix = util.df(bars)
vix = vix.set_index('date')
vix = vix[['open', 'high', 'low', 'close']]
vix_history = read_feather(str(UpdateSP500Data.TOP_LEVEL_PATH / 'vix_history'))
full_hist = vix.combine_first(vix_history)
write_feather(full_hist, str(UpdateSP500Data.TOP_LEVEL_PATH / 'vix_history'))
return full_hist['close']
def spx_bar_history(update_bars=True):
file_name = 'sp500_5min_bars'
df_hist = read_feather(UpdateSP500Data.DATA_BASE_PATH / file_name)
# Download latest
if update_bars:
ib = IB()
ib.connect('127.0.0.1', port=4001, clientId=40)
contract = Index('SPX', 'CBOE', 'USD')
bars = ib.reqHistoricalData(
contract,
endDateTime='',
durationStr='1 M',
barSizeSetting='5 mins',
whatToShow='TRADES',
useRTH=True,
formatDate=1)
ib.disconnect()
df = util.df(bars)
df = df.set_index('date')
full_hist = df.combine_first(df_hist)
write_feather(full_hist, UpdateSP500Data.DATA_BASE_PATH / file_name)
else:
full_hist = df_hist.copy()
return full_hist
def get_hsi_data(self):
''' 获取 HSI 一个月5分钟的信息
'''
symbol = 'HSI'
exchange = 'HKFE'
currency = 'HKD'
index = Index(symbol, exchange, currency)
data = self.ib.reqHistoricalData(index, endDateTime='20180119 15:00:00', durationStr='900 S',
barSizeSetting='5 mins', whatToShow='TRADES', useRTH=True)
if not data:
raise RuntimeError('HSI 数据接口返回空.')
sql = 'insert into `hsi_5m` (`date`, `open`, `high`, `low`, `close`) values '
for bar_data in data:
date = bar_data.date
open_price = bar_data.open
high = bar_data.high
low = bar_data.low
close = bar_data.close
sql += "('{date}', {open:.4f}, {high:.4f}, {low:.4f}, {close:.4f}),".format(date=date, open=open_price, high=high, low=low, close=close)
res = self.db.query(sql.rstrip(','))
if res.rowcount == 0:
raise RuntimeError('SQL 语句执行异常， 插入数据库失败：%s' % sql)
else:
print('HSI Index 1个月5分钟数据导入完成.', flush=True)
def main(ib):
from ib_insync import Stock, Forex, Index
win_historical_ticks = sg.Window('IBKR Historical Ticks', layout)
#win_historical_ticks.Element('_TickCount_').Update('0')
# Event Loop to process "events" and get the "values" of the inputs
while True: # Event Loop
event, values = win_historical_ticks.Read()
if event == '_CloseWindow_':
#? commenting out "disconnect()" because any calling procedure would not want a child to disconnect it
#ib.disconnect()
logging.info("Close hit")
win_historical_ticks.close()
break
if event == 'Qualify':
# Update the "output" element to be the value of "input" element
win_historical_ticks.Element('_OUTPUT_').Update(values['_Symbol_'])
if values['_SecType_'] == 'STK':
contract = Stock(values['_Symbol_'], values['_Exchange_'],values['_Currency_'])
if values['_SecType_'] == 'CASH':
contract = Forex(values['_Symbol_'])
if values['_SecType_'] == 'IND':
contract = Index(values['_Symbol_'], values['_Exchange_'],values['_Currency_'])
qual = ib.qualifyContracts(contract)
logging.info(str(qual))
if event == 'Fetch':
#NOTE add validations here
if values['_targetFolder_'] != '':
dtmStartTime = datetime.strptime(str(values['_StartDate_']), '%Y-%m-%d %H:%M:%S')
dtmStartTime = dtmStartTime.replace(hour=0, minute=1)
dtmEndTime = datetime.strptime(str(values['_EndDate_']), '%Y-%m-%d %H:%M:%S')
dtmEndTime = dtmEndTime.replace(hour=23, minute=59)
intTicks = int(values['_tickCount_'])
# whatToShow (str) – One of ‘Bid_Ask’, ‘Midpoint’ or ‘Trades’.
strWhatToShow = str(values['_whatToShow_'])
# useRTH – If True then only show data from within Regular Trading Hours, if False then show all data.
boolUseRTH = bool(values['_useRTH_'])
# ignoreSize (bool) – Ignore bid/ask ticks that only update the size.
boolIgnoreSize = bool(values['_ignoreSize_'])
#sg.popup(str([dtmStartTime, " - ", dtmEndTime, " - ", intTicks, " - ", strWhatToShow, " - ", boolUseRTH, " - ", boolIgnoreSize]))
if dtmEndTime > dtmStartTime:
listTicks = loop_reqHistoricalTicks(ib, contract, dtmStartTime, dtmEndTime, intTicks, strWhatToShow, boolUseRTH, boolIgnoreSize)
#convert listTicks to a CSV and save
dfTicks = pd.DataFrame(listTicks)
dfTicks = dfTicks.rename(columns={'tickAttribLast':'symbol', 'specialConditions':'currency'})
dfTicks['symbol'] = contract.symbol
dfTicks['exchange'] = contract.exchange
dfTicks['currency'] = contract.currency
dfTicks['time'] = dfTicks['time'].apply(utc2local)
dfTicks['dataType'] = 'Ticks_' + str(values['_whatToShow_']) +('_RTH' if boolUseRTH else '_RHT+AMO')
#For Linux uncomment below
#dfTicks.to_csv(values['_targetFolder_'] +'/'+ contract.symbol +'_'+ str(dtmStartTime) +'_'+ str(dtmEndTime)+'.csv', index=False)
#For Windows uncomment below
dfTicks.to_csv(str_default_path +'//'+ contract.symbol +'_'+ dtmStartTime.strftime("%Y%M%d_%H%M%S") +'_'+ dtmEndTime.strftime("%Y%M%d_%H%M%S") +'.csv', index=False)
def __init__(self, mkt_symbol, vol_symbol, exchange_dict):
"""Abstract class for option asset container"""
exchange_mkt = exchange_dict['exchange_mkt']
exchange_vol = exchange_dict['exchange_vol']
exchange_opt = exchange_dict['exchange_opt']
self.trading_class = exchange_dict['trading_class']
underlying_index = Index(mkt_symbol, exchange_mkt)
ibw = IbWrapper()
ib = ibw.ib
self.underlying_qc = self.__get_underlying_qc(underlying_index, ib)
self.sigma_qc = self.get_sigma_qc(vol_symbol, ib, exchange_vol)
self.chain = self.get_option_chain(underlying_index, ib, exchange_opt)
ib.disconnect()
def crawl_index_data(self):
''' 爬取 HSI Index 数据
'''
symbol = 'HSI'
exchange = 'HKFE'
currency = 'HKD'
records = self.db.get_hsi_record()
if not records.rowcount:
raise Exception('获取上次爬取数据日期失败，数据库 records 表返回空.')
last_time = list(records)[0].date
now_time = datetime.datetime.now()
diff = now_time - last_time
dur = diff.days
print('正在获取 HSI 数据.')
index = Index(symbol, exchange, currency)
data = self.ib.reqHistoricalData(index, endDateTime='', durationStr='%d D' % dur,
barSizeSetting='1 day', whatToShow='TRADES', useRTH=True)
if not data:
raise RuntimeError('HSI 数据接口返回空.')
sql = 'insert into `hsi` (`date`, `open`, `high`, `low`, `close`) values '
for bar_data in data:
date_time = datetime.datetime(*bar_data.date.timetuple()[:6])
if date_time > last_time:
date = bar_data.date
open_price = bar_data.open
high = bar_data.high
low = bar_data.low
close = bar_data.close
sql += "('{date}', {open:.4f}, {high:.4f}, {low:.4f}, {close:.4f}),".format(date=date, open=open_price, high=high, low=low, close=close)
rows = self.db.query(sql.rstrip(','))
if rows == 0:
raise Exception('SQL 语句执行异常， 插入数据库失败：%s' % sql)
else:
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
row = self.db.set_hsi_record(now)
if not row:
raise Exception('更新 HSI 日期记录失败.')
# indexes = sorted('OEX,XEO,XSP'.split(','))
indexes = []
#...Build a list of contracts
stocks = [
Stock(symbol=s, currency=currency, exchange=exchange) for s in set(stocks)
]
# sort the stocks
s_dict = {s.symbol: s for s in stocks}
od = OrderedDict(sorted(s_dict.items()))
ss = [v for k, v in od.items()]
# ixs = [Index(symbol=s,currency=currency, exchange='CBOE') for s in set(indexes)]
ixs = [
Index(symbol=s, currency=currency, exchange='CBOE') for s in set(indexes)
]
cs = ss + ixs
# cs = cs[15:18] # DATA LIMITER!!!
# sort in alphabetical order
cs.sort(key=lambda x: x.symbol, reverse=False)
def snp_list(ib):
'''returns a list of qualified snp underlyings
Args: ib as the active object
Returns: qualified list with conID
'''
# nseSymbol to ibSymbol dictionary for conversion
ntoi = {'M&M': 'MM', 'M&MFIN': 'MMFIN', 'L&TFH': 'LTFH', 'NIFTY': 'NIFTY50'}
# remap ibSymbol, based on the dictionary
df_slm.ibSymbol = df_slm.ibSymbol.replace(ntoi)
# separate indexes and equities, eliminate discards from df_slm
indexes = ['NIFTY50', 'BANKNIFTY']
discards = ['NIFTYMID5', 'NIFTYIT', 'LUPIN']
equities = sorted([s for s in df_slm.ibSymbol if s not in indexes + discards])
symbols = equities + indexes
cs = [
Stock(s, exchange) if s in equities else Index(s, exchange)
for s in symbols
]
# cs = cs[:5] # DATA LIMITER!!!
def nse_list(ib):
'''returns a list of qualified nse underlyings, with symbols and option chains for them
Args: ib as the active ib object
Returns: qualified list with conID
'''
qcs = ib.qualifyContracts(*cs) # qualified underlyings
qcs_chains = []
for i in range(0, len(qcs), 50):
# note that it does not work for Index symbol (ref: https://groups.io/g/insync/message/4718)
import asyncio
import pandas as pd
from ib_insync import IB, Index, Stock, util
# build up the contracts
HOST = '127.0.0.1'
PORT = '1300'
CID = 0
with IB().connect(HOST, PORT, CID) as ib:
timeout = 5
# contract = ib.qualifyContracts(Stock(symbol='SBUX', exchange='SMART', currency='USD'))[0]
contract = ib.qualifyContracts(
Index(symbol='VIX', exchange='CBOE', currency='USD'))[0]
req = ib.reqHistoricalDataAsync(contract=contract,
endDateTime="",
durationStr="2 D",
barSizeSetting="1 day",
whatToShow="Trades",
useRTH=True)
try:
hist = ib.run(asyncio.wait_for(req, timeout))
except asyncio.exceptions.TimeoutError:
hist = []
print(contract)
print(hist)
def parse_contract(ticker):
"""
Backtrader contract specification (https://www.backtrader.com/docu/live/ib/ib.html):
TICKER # Stock type and SMART exchange
TICKER-STK # Stock and SMART exchange
TICKER-STK-EXCHANGE # Stock
TICKER-STK-EXCHANGE-CURRENCY # Stock
TICKER-CFD # CFD and SMART exchange
TICKER-CFD-EXCHANGE # CFD
TICKER-CDF-EXCHANGE-CURRENCY # Stock
TICKER-IND-EXCHANGE # Index
TICKER-IND-EXCHANGE-CURRENCY # Index
TICKER-YYYYMM-EXCHANGE # Future
TICKER-YYYYMM-EXCHANGE-CURRENCY # Future
TICKER-YYYYMM-EXCHANGE-CURRENCY-MULT # Future
TICKER-FUT-EXCHANGE-CURRENCY-YYYYMM-MULT # Future
TICKER-YYYYMM-EXCHANGE-CURRENCY-STRIKE-RIGHT # FOP
TICKER-YYYYMM-EXCHANGE-CURRENCY-STRIKE-RIGHT-MULT # FOP
TICKER-FOP-EXCHANGE-CURRENCY-YYYYMM-STRIKE-RIGHT # FOP
TICKER-FOP-EXCHANGE-CURRENCY-YYYYMM-STRIKE-RIGHT-MULT # FOP
CUR1.CUR2-CASH-IDEALPRO # Forex
TICKER-YYYYMMDD-EXCHANGE-CURRENCY-STRIKE-RIGHT # OPT
TICKER-YYYYMMDD-EXCHANGE-CURRENCY-STRIKE-RIGHT-MULT # OPT
TICKER-OPT-EXCHANGE-CURRENCY-YYYYMMDD-STRIKE-RIGHT # OPT
TICKER-OPT-EXCHANGE-CURRENCY-YYYYMMDD-STRIKE-RIGHT-MULT # OPT
:return:
"""
contract_type, symbol, exchange, currency, expire, multiplier = \
AsyncIBDataProvider.exctract_symbol(ticker)
if contract_type == 'FX':
return Forex(pair=symbol)
if contract_type == 'IND':
return Index(symbol, exchange, currency)
if contract_type == 'FUT':
return Future(symbol,
expire,
exchange,
currency=currency,
multiplier=multiplier)
else:
return Stock(symbol, exchange, currency)
LONG_BAR_FORMAT = '{desc:<25}{percentage:3.0f}%|{bar:30}{r_bar}'
QBLK = 50 # block chunks for qualification
start = datetime.now()
# Suppress errors in console
logfile = './data/test.log'
with open(logfile, 'w'):
pass
util.logToFile(logfile, level=30)
# Get the symbols
df_syms = get_syms(MARKET)
# Build the contracts
raw_cts = [i for j in [[Stock(symbol, exchange, currency), Index(symbol, exchange, currency)]
for symbol, exchange, currency
in zip(df_syms.symbol, df_syms.exchange, df_syms.currency)] for i in j]
c_blks = [raw_cts[i: i + QBLK] for i in range(0, len(raw_cts), QBLK)]
tq_blks = tqdm(c_blks, position=0, leave=True,
desc=f'Qualifying {len([i for c in c_blks for i in c])} ',
bar_format=LONG_BAR_FORMAT)
qunds = dict() # to store qualified underlying contracts
# Qualify the underlyings
with IB().connect(HOST, PORT, CID) as ib:
for b in tq_blks:
cts = ib.qualifyContracts(*b)
def main(ib):
from ib_insync import Stock, Forex, Index
win_historical = sg.Window('Historical OHLC', layout)
while True: # Event Loop
event, values = win_historical.Read()
if event == 'Exit':
#? commenting out "disconnect()" because any calling procedure would not want a child to disconnect it
#ib.disconnect()
logging.info("exit hit")
win_historical.Close()
break
if event == 'Qualify':
# Update the "output" element to be the value of "input" element
win_historical.Element('_OUTPUT_').Update(values['_Symbol_'])
if values['_SecType_'] == 'STK':
contract = Stock(values['_Symbol_'], values['_Exchange_'],
values['_Currency_'])
if values['_SecType_'] == 'CASH':
contract = Forex(values['_Symbol_'])
if values['_SecType_'] == 'IND':
contract = Index(values['_Symbol_'], values['_Exchange_'],
values['_Currency_'])
qual = ib.qualifyContracts(contract)
logging.info(str(qual))
if event == 'Fetch':
strDuration = str(values['_DurStr1_']) + ' ' + str(
values['_DurStr2_'])
strBarSize = str(values['_BarSize_'])
strWhatToShow = str(values['_WhatToShow_'])
logging.info('strDuration:%s , strBarSize: %s , strWhatToShow: %s',
strDuration, strBarSize, strWhatToShow)
barData = ib.reqHistoricalData(contract, '', strDuration,
strBarSize, strWhatToShow, 1, 1,
False)
listData = [[
item.date.strftime("%Y%m%d, %H:%M:%S"), item.open, item.high,
item.low, item.close, item.volume, item.barCount, item.average
] for item in barData]
dfStockData = pd.DataFrame(listData)
if dfStockData.shape[1] == 8 and dfStockData.shape[0] > 0:
logging.info('dfStockData sample: %s', dfStockData.head(3))
dfStockData.columns = [
'date', 'open', 'high', 'low', 'close', 'volume',
'barCount', 'average'
]
relist = dfStockData.values.tolist()
win_historical.Element('sgtable').Update(values=relist,
num_rows=25,
visible=True)
if event == 'Download':
#check if the DatFrame is minimally populated or not.
#if populated:
if dfStockData.shape[1] == 8 and dfStockData.shape[0] > 0:
file_name = sg.PopupGetFile(
'Please enter filename to save (if file exists, it will be overwritten): ',
save_as=True)
fileTarget = open(file_name, 'w+')
dfStockData.to_csv(fileTarget)
fileTarget.close()
sg.Popup("File saved.")
#if NOT populated
else:
sg.Popup("Please fetch some data before trying to save it.")
def get_sigma_qc(self, vol_symbol, ib, exchange):
"""Returns implied Volatility for market"""
sigma_index = Index(vol_symbol, exchange)
sigma_qc = ib.qualifyContracts(sigma_index)
assert (len(sigma_qc) == 1)
return sigma_qc[0]
CID = 0
# Suppress errors in console
logfile = './data/test.log'
with open(logfile, 'w'):
pass
util.logToFile(logfile, level=30)
# Get the symbols
df_syms = get_syms(MARKET)
# Build the contracts
raw_cts = [
i for j in
[[Stock(symbol, exchange, currency),
Index(symbol, exchange, currency)] for symbol, exchange, currency in zip(
df_syms.symbol, df_syms.exchange, df_syms.currency)] for i in j
]
def qual():
'''Simple qualify, without async'''
start = time.time()
result = ib.qualifyContracts(*cts)
qcon = {r.symbol: r for r in result if r}
print(
f'\ntook qual {time.time()-start} seconds to qualify {len(qcon)} contracts\n'
)
print(qcon)