def runProg():
"""run program"""
util.patchAsyncio()
# log to a file
util.logToFile(f'getRecentHistoricalData2.log')
# util.logToConsole()
# set pandas option
pd.set_option('display.width', 200)
# specify connection details
host = '127.0.0.1'
port = 4002
ibcIni = '/home/bn/IBController/configPaper.ini'
tradingMode = 'paper'
clientId = 12
# start watchdog
ibc = IBC(970, gateway=True, tradingMode=tradingMode, ibcIni=ibcIni)
ib = IB()
watchdogApp = ibcontroller.Watchdog(ibc,
ib=ib,
appStartupTime=15,
host=host,
port=port,
clientId=clientId)
watchdogApp.start()
pass
# create some contracts
qcs = []
c = Contract(symbol='EUR',
currency='USD',
exchange='IDEALPRO',
secType='CASH')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
# request market data
for qc in qcs:
ib.reqMktData(contract=qc,
genericTickList='',
snapshot=False,
regulatorySnapshot=False,
mktDataOptions=None)
pass
# define some callback
def onPendingTickers(tickers):
for t in tickers:
localSymbol = t.contract.localSymbol
if localSymbol == "EUR.USD":
nowUTC = pd.to_datetime(pd.datetime.utcnow()).tz_localize(None)
nowUTCRounded = nowUTC.floor('1 min')
dateTime = pd.to_datetime(t.time).tz_localize(None)
print(localSymbol, nowUTCRounded,
((dateTime - nowUTCRounded) / pd.Timedelta('1 sec')),
t.close)
pass
pass
pass
def myErrorCallback(reqId, errorCode, errorString, contract):
# print("myErrorCallback", reqId,errorCode,errorString,contract)
if errorCode == 322:
print("myErrorCallback", reqId, errorCode, errorString, contract)
# more than 50 simultaneous historical data requests
app.ib.client.cancelHistoricalData(reqId)
# register the callbacks with ib
ib.setCallback('error', myErrorCallback)
ib.setCallback('pendingTickers', onPendingTickers)
# run and never stop
ib.run()
class trade_ES():
def __init__(self):
self.ib = IB()
self.ib.connect('127.0.0.1', 7497, clientId=np.random.randint(10, 1000))
self.tickers_ret = {}
self.endDateTime = ''
self.No_days = '43200 S'
self.interval = '30 secs'
self.tickers_signal = "Hold"
self.ES = Future(symbol='ES', lastTradeDateOrContractMonth='20200619', exchange='GLOBEX',
currency='USD')
self.ib.qualifyContracts(self.ES)
self.ES_df = self.ib.reqHistoricalData(contract=self.ES, endDateTime=self.endDateTime, durationStr=self.No_days,
barSizeSetting=self.interval, whatToShow='TRADES', useRTH=False,
keepUpToDate=True)
self.tickers_ret = []
self.options_ret = []
self.option = {'call': FuturesOption, 'put': FuturesOption}
self.options_history = {}
self.trade_options = {'call': [], 'put': []}
self.price = 0
self.i = -1
self.ES_df.updateEvent += self.make_clean_df
self.Buy = True
self.Sell = False
self.ib.positionEvent += self.order_verify
self.waitTimeInSeconds = 120
self.tradeTime = 0
def run(self):
while self.ib.waitOnUpdate():
util.allowCtrlC()
self.ib.setCallback('error', x.checkError)
self.make_clean_df(self.ES_df)
def next_exp_weekday(self):
weekdays = {2: [6, 0], 4: [0, 1, 2], 0: [3, 4]}
today = datetime.date.today().weekday()
for exp, day in weekdays.items():
if today in day:
return exp
def next_weekday(self, d, weekday):
days_ahead = weekday - d.weekday()
if days_ahead <= 0: # Target day already happened this week
days_ahead += 7
date_to_return = d + datetime.timedelta(days_ahead) # 0 = Monday, 1=Tuself.ESday, 2=Wednself.ESday...
return date_to_return.strftime('%Y%m%d')
def get_strikes_and_expiration(self):
expiration = self.next_weekday(datetime.date.today(), self.next_exp_weekday())
chains = self.ib.reqSecDefOptParams(underlyingSymbol='ES', futFopExchange='GLOBEX', underlyingSecType='FUT',
underlyingConId=self.ES.conId)
chain = util.df(chains)
strikes = chain[chain['expirations'].astype(str).str.contains(expiration)].loc[:, 'strikes'].values[0]
[ESValue] = self.ib.reqTickers(self.ES)
ES_price = ESValue.marketPrice()
strikes = [strike for strike in strikes
if strike % 5 == 0
and ES_price - 10 < strike < ES_price + 10]
return strikes, expiration
def get_contract(self, right, net_liquidation):
strikes, expiration = self.get_strikes_and_expiration()
for strike in strikes:
contract = FuturesOption(symbol='ES', lastTradeDateOrContractMonth=expiration,
strike=strike, right=right, exchange='GLOBEX')
self.ib.qualifyContracts(contract)
self.price = self.ib.reqMktData(contract, "", False, False)
if float(self.price.last) * 50 >= net_liquidation:
continue
else:
return contract
def make_clean_df(self, ES_df, hashbar=None):
ES_df = util.df(ES_df)
ES_df['RSI'] = ta.RSI(ES_df['close'])
ES_df['macd'], ES_df['macdsignal'], ES_df['macdhist'] = ta.MACD(ES_df['close'], fastperiod=12, slowperiod=26,
signalperiod=9)
ES_df['MA_9'] = ta.MA(ES_df['close'], timeperiod=9)
ES_df['MA_21'] = ta.MA(ES_df['close'], timeperiod=21)
ES_df['MA_200'] = ta.MA(ES_df['close'], timeperiod=200)
ES_df['EMA_9'] = ta.EMA(ES_df['close'], timeperiod=9)
ES_df['EMA_21'] = ta.EMA(ES_df['close'], timeperiod=21)
ES_df['EMA_200'] = ta.EMA(ES_df['close'], timeperiod=200)
ES_df['ATR'] = ta.ATR(ES_df['high'], ES_df['low'], ES_df['close'])
ES_df['roll_max_cp'] = ES_df['high'].rolling(20).max()
ES_df['roll_min_cp'] = ES_df['low'].rolling(20).min()
ES_df['roll_max_vol'] = ES_df['volume'].rolling(20).max()
ES_df.dropna(inplace=True)
self.loop_function(ES_df)
def placeOrder(self, contract, order):
trade = self.ib.placeOrder(contract, order)
tradeTime = datetime.datetime.now()
return([trade, contract, tradeTime])
def sell(self, contract, position):
self.ib.qualifyContracts(contract)
if position.position>0:
order = 'Sell'
else:
order = 'Buy'
marketorder = MarketOrder(order, abs(position.position))
if self.tradeTime!=0:
timeDelta = datetime.datetime.now() - self.tradeTime
if timeDelta.seconds > self.waitTimeInSeconds:
marketTrade, contract, self.tradeTime = self.placeOrder(contract, marketorder)
else:
marketTrade, contract, tradeTime = self.placeOrder(contract, marketorder)
condition = marketTrade.isDone
timeout = 20
for c in self.ib.loopUntil(condition=condition, timeout=timeout):
marketorder = MarketOrder('Sell', position.position)
marketTrade = self.ib.placeOrder(contract, marketorder)
if not condition == 'Filled':
self.ib.cancelOrder(marketorder)
marketorder = MarketOrder('Sell', position.position)
marketTrade = self.ib.placeOrder(contract, marketorder)
def buy(self, contract):
self.ib.qualifyContracts(contract)
marketorder = MarketOrder('Buy', 1)
if self.tradeTime!=0:
timeDelta = datetime.datetime.now() - self.tradeTime
if timeDelta.seconds > self.waitTimeInSeconds:
marketTrade, contract, self.tradeTime = self.placeOrder(contract, marketorder)
else:
marketTrade, contract, tradeTime = self.placeOrder(contract, marketorder)
condition = marketTrade.isDone
timeout = 10
for c in self.ib.loopUntil(condition=condition, timeout=timeout):
marketorder = MarketOrder('Buy', 1)
marketTrade = self.ib.placeOrder(contract, marketorder)
if not condition == 'Filled':
self.ib.cancelOrder(marketorder)
marketorder = MarketOrder('Buy', 1)
marketTrade = self.ib.placeOrder(contract, marketorder)
def order_verify(self, order):
if order.position == 0.0 or order.position < 0:
self.Buy= True
self.Sell= False
elif order.position > 0:
self.Buy = False
self.Sell = True
else:
self.Buy = False
self.Sell = False
print(f'Buy= {self.Buy}, sell = {self.Sell}')
def loop_function(self, ES_df):
df = ES_df[
['high', 'low', 'volume', 'close', 'RSI', 'ATR', 'roll_max_cp', 'roll_min_cp', 'roll_max_vol', 'EMA_9',
'EMA_21', 'macd', 'macdsignal']]
if self.tickers_signal == "Hold":
print('Hold')
if df["high"].iloc[self.i] >= df["roll_max_cp"].iloc[self.i] and \
df["volume"].iloc[self.i] > df["roll_max_vol"].iloc[self.i - 1] and df['RSI'].iloc[self.i] > 30 \
and df['macd'].iloc[self.i] > df['macdsignal'].iloc[self.i] :
self.tickers_signal = "Buy"
return
elif df["low"].iloc[self.i] <= df["roll_min_cp"].iloc[self.i] and \
df["volume"].iloc[self.i] > df["roll_max_vol"].iloc[self.i - 1] and df['RSI'].iloc[self.i] < 70 \
and df['macd'].iloc[self.i] < df['macdsignal'].iloc[self.i]:
self.tickers_signal = "Sell"
return
elif self.tickers_signal == "Buy":
print('BUY SIGNAL')
if df["close"].iloc[self.i] > df["close"].iloc[self.i - 1] - (0.75 * df["ATR"].iloc[self.i - 1]) and self.Sell and not self.Buy:
self.tickers_signal = "Hold"
positions = self.ib.positions()
for position in positions:
if position.contract.right == 'C':
self.sell(position.contract, position)
return
elif df["low"].iloc[self.i] <= df["roll_min_cp"].iloc[self.i] and \
df["volume"].iloc[self.i] > df["roll_max_vol"].iloc[self.i - 1] and df['RSI'].iloc[self.i] < 70 \
and df['macd'].iloc[self.i] < df['macdsignal'].iloc[self.i] and self.Sell and not self.Buy:
self.tickers_signal = "Sell"
positions = self.ib.positions()
for position in positions:
if position.contract.right == 'C':
self.sell(position.contract, position)
return
self.option['put'] = self.get_contract(right="P", net_liquidation=2000)
self.buy(self.option['put'])
elif not self.Sell and self.Buy:
self.option['call'] = self.get_contract(right="C", net_liquidation=2000)
self.buy(self.option['call'])
elif self.tickers_signal == "Sell":
print('SELL SIGNAL')
if df["close"].iloc[self.i] < df["close"].iloc[self.i - 1] + (0.75 * df["ATR"].iloc[self.i - 1]) and self.Sell and not self.Buy:
self.tickers_signal = "Hold"
positions = self.ib.positions()
for position in positions:
if position.contract.right == 'P':
self.sell(position.contract, position)
return
elif df["high"].iloc[self.i] >= df["roll_max_cp"].iloc[self.i] and \
df["volume"].iloc[self.i] > df["roll_max_vol"].iloc[self.i - 1] and df['RSI'].iloc[self.i] > 30 \
and df['macd'].iloc[self.i] > df['macdsignal'].iloc[self.i] and self.Sell and not self.Buy:
self.tickers_signal = "Buy"
positions = self.ib.positions()
for position in positions:
if position.contract.right == 'P':
self.sell(position.contract, position)
return
self.option['call'] = self.get_contract(right="C", net_liquidation=2000)
self.buy(self.option['call'])
elif not self.Sell and self.Buy:
self.option['put'] = self.get_contract(right="P", net_liquidation=2000)
self.buy(self.option['put'])
def checkError(self, errCode, errString):
print('Error Callback', errCode, errString)
if errCode == 2104:
print('re-connect after 5 secs')
self.ib.sleep(5)
self.ib.disconnect()
self.ib.connect('127.0.0.1', 7497, clientId=np.random.randint(10, 1000))
self.make_clean_df(self.ES)
def runProg(args):
"""run program"""
util.patchAsyncio()
# log to a file
utils.logToFile(f'getRecentHistoricalData.log')
# utils.logToConsole()
apschedulerLogger = logging.getLogger('apscheduler')
apschedulerLogger.setLevel(logging.ERROR)
tradingLogger = logging.getLogger('trading')
tradingLogger.setLevel(logging.WARNING)
pd.set_option('display.width', 200)
# load the config file
configFile = args.configFile
config = ConfigParser(interpolation=ExtendedInterpolation(),
defaults=os.environ)
config.read(configFile)
# load data from configFile
host = config.get('InteractiveBrokers', 'host')
port = config.getint('InteractiveBrokers', 'port')
DBType = config.get('DataBase', 'DBType')
DBFileName = config.get('DataBase', 'DBFileName')
clientId = config.get('InteractiveBrokers', 'clientId')
# for production mode: watchdog
if 1:
# start watchdog
# ibc = IBC(963, gateway=True, tradingMode='paper',ibcIni='/home/bn/IBController/configPaper.ini')
ibcIni = config.get('InteractiveBrokers', 'ibcIni')
tradingMode = config.get('InteractiveBrokers', 'tradingMode')
ibc = IBC(970, gateway=True, tradingMode=tradingMode, ibcIni=ibcIni)
myWatchdogapp = myWatchdog.myWatchdog(ibc,
appStartupTime=15,
port=4002)
myWatchdogapp.start()
ib = myWatchdogapp.ib
pass
if 0:
# faster way for now
ib = IB()
ib.connect(host=host, port=port, clientId=clientId)
pass
pass
# create database class
mydb = database.tradingDB(DBType=DBType, DBFileName=DBFileName)
# load existing database
mydb.instantiateExistingTablesAndClasses(ib=ib)
# set log level of sqlalchemy
mydb._loggerSQLAlchemy.setLevel(logging.WARNING)
qcs = mydb.MarketDataInfoTableDataFrame.qualifiedContract
for qc in qcs:
print(qc, type(qc))
ib.reqMktData(contract=qc,
genericTickList='',
snapshot=False,
regulatorySnapshot=False,
mktDataOptions=None)
pass
df = pd.DataFrame(
columns='symbol bidSize bid ask askSize high low close'.split())
df['symbol'] = [qc.localSymbol for qc in qcs]
contract2Row = {qc: i for (i, qc) in enumerate(qcs)}
pprint.pprint(contract2Row)
def onPendingTickers(tickers):
for t in tickers:
iRow = contract2Row[t.contract]
localSymbol = t.contract.localSymbol
if localSymbol == "EUR.USD":
nowUTC = pd.to_datetime(pd.datetime.utcnow()).tz_localize(None)
nowUTCRounded = nowUTC.floor('1 min')
dateTime = pd.to_datetime(t.time).tz_localize(None)
print(localSymbol, nowUTCRounded,
((dateTime - nowUTCRounded) / pd.Timedelta('1 sec')),
t.close)
# df.iloc[iRow, 1:] = (t.bidSize, t.bid, t.ask, t.askSize, t.high, t.low, t.close)
# print(df)
ib.setCallback('pendingTickers', onPendingTickers)
# ib.sleep(300)
if 1:
util.allowCtrlC()
# Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
try:
asyncio.get_event_loop().run_forever()
except (KeyboardInterrupt, SystemExit):
pass
def instantiateMyDB(args):
"""instantiate all SQ ORM classes using a config file passed in the arguments"""
# load the config file
configFile = args.configFile
config = ConfigParser(interpolation=ExtendedInterpolation(),
defaults=os.environ)
config.read(configFile)
# create connection to IB
ib = IB()
ib.setCallback('error', myErrorCallback)
# load data from configFile
a = config.get('MarketData', 'ConIdList')
conIdList = eval(a)
host = config.get('InteractiveBrokers', 'host')
port = config.getint('InteractiveBrokers', 'port')
clientId = config.getint('InteractiveBrokers', 'clientId')
DBType = config.get('DataBase', 'DBType')
DBFileName = config.get('DataBase', 'DBFileName')
timeOutTime = config.getint('InteractiveBrokers', 'timeOutTime')
# override configFile if clientId is given on the command line
if args.clientId is not None:
clientId = args.clientId
# override configFile if timeOutTime is given on the command line
if args.timeOutTime is not None:
timeOutTime = args.TimeOutTime
# connect to interactive brokers
ib.connect(host=host, port=port, clientId=clientId)
# create database class
mydb = tradingDB(DBType=DBType, DBFileName=DBFileName)
# loop over all conIds defined in the config File and create the sqlalchemy ORM classes
# these tables will appear in the metadata of the DBDeclarativeBase attribute of mydb
# prepare a dataframe that holds all infos that should be put into the MarketDataInfoTable on disk
# and the MarketDataInfoTableDataFrame in memory
nTables = len(conIdList)
featureList = [
'conId',
'qualifiedContract',
'earliestDateTime',
'category',
'kwargs',
'tableName',
'tableORM',
]
dfWithInfoAboutTables = pd.DataFrame(None,
index=range(nTables),
columns=featureList)
dfWithInfoAboutTables.loc[:, 'conId'] = conIdList
df = dfWithInfoAboutTables
for indx in df.index:
conId = df.at[indx, 'conId']
qc = utils.getQualifiedContractFromConId(ib=ib,
conId=conId,
timeOutTime=timeOutTime)
df.at[indx, 'qualifiedContract'] = qc
# calculate the earliest Date for this contract
earliestDateTime = utils.getEarliestDateTimeFromIB(
ib=ib, qualifiedContract=qc, timeOutTime=timeOutTime)
df.at[indx, 'earliestDateTime'] = earliestDateTime
# set the category that should be MarketData for the tables to be generated in this loop
category = mydb.tableCategories.MarketData.value
df.at[indx, 'category'] = category
# set the keyword arguments for the call to calculateTableName
kwargs = {}
kwargs['category'] = category
kwargs['earliestDateTime'] = earliestDateTime
kwargs.update(qc.dict())
df.at[indx, 'kwargs'] = kwargs
# calculate the tableName
tableName = mydb.calculateTableName(**kwargs)
df.at[indx, 'tableName'] = tableName
# create the sqlalchemy ORM class; this will write the class to the mydb.DBDeclarativeBase.metadata object
print(
f'creating MarketData Table: conId: {conId}; tableName: {tableName}'
)
tableORM = mydb.getTableORMByTablename(tableName=tableName)
df.at[indx, 'tableORM'] = tableORM
pass
# now all the ORM tables should be defined.
# they are not yet created on disk.
# also, the MarketDataInfoTable is not populated and the MarketDataInfoTableDataFrame is not populated
# loop over all conIds defined in the config File and create a row in the Market Data Info Table
# also, populate the corresponding dataframe
# create all tables on disk if they do not yet exist
mydb.createAllTables()
ssn = mydb.Session()
for indx, row in dfWithInfoAboutTables.iterrows():
tableName = row.tableName
print(f'upserting a row for {tableName} to the MarketDataInfoTable')
# create a row for each conId in the MDIT table
# first, instantiate a row in the MarketDataInfoTable
MDIT = mydb.MarketDataInfoTable(tableName=tableName)
# set all available column values
kwargs = row.kwargs
for k, v in kwargs.items():
if k in MDIT.__table__.columns:
setattr(MDIT, k, v)
pass
pass
# upsert this table Row to the table
d = utils.convertTableRowToDict(MDIT)
# only update values that are not none
rowOfTableOnDisk = ssn.query(mydb.MarketDataInfoTable).filter(
mydb.MarketDataInfoTable.tableName == tableName).first()
for k, v in d.items():
if v is None:
a = None
try:
a = getattr(rowOfTableOnDisk, 'earliestDateTime', None)
except:
pass
d[k] = a
pass
pass
ssn.execute(mydb.upsert(mydb.MarketDataInfoTable, [d]))
ssn.commit()
ssn.close()
mydb.MarketDataInfoTableDataFrame = mydb.createMarketDataInfoTableDataFrameFromMarketDataInfoTable(
ib=ib, timeOutTime=timeOutTime)
# disconnect from interactive brokers
ib.disconnect()
return (mydb)
