def doitNew():
mydb = tradingDB()
utils.logToFile('ttest.log')
# utils.logToConsole()
# if we want to reduce salAlchemy Logging
# mydb._loggerSQLAlchemy.setLevel(logging.ERROR)
mydb.dropAllTables()
# create all tables that are defined now. This is just the table that contains a single market data table in each row
mydb.createAllTables()
pass
def runProg(args):
"""run program"""
pd.set_option('display.width', 300)
# log to a file
utils.logToFile(f'createTraingData.log')
# load the config file
configFile = args.configFile
config = ConfigParser(interpolation=ExtendedInterpolation(),
defaults=os.environ)
config.read(configFile)
# load data from configFile
DBType = config.get('DataBase', 'DBType')
DBFileName = config.get('DataBase', 'DBFileName')
# create database class
mydb = database.tradingDB(DBType=DBType, DBFileName=DBFileName)
# mydb = database.tradingDB(DBType='mysql', DBFileName=DBFileName)
# load existing database
mydb.instantiateExistingTablesAndClasses()
# set log level
mydb._loggerSQLAlchemy.setLevel(logging.ERROR)
ssn = mydb.Session()
if 1:
if (mydb.DBEngine.name == 'sqlite'):
print('starting to perform vacuum ...')
stmt = sqlalchemy.text('vacuum')
mydb.DBEngine.execute(stmt)
print('... done')
ssn.commit()
ssn.close()
def runProg(args):
"""run program"""
util.patchAsyncio()
# log to a file
utils.logToFile(f'ttestTradingHours.log', level=logging.INFO)
# utils.logToConsole()
apschedulerLogger = logging.getLogger('apscheduler')
apschedulerLogger.setLevel(logging.INFO)
tradingLogger = logging.getLogger('trading')
tradingLogger.setLevel(logging.INFO)
pd.set_option('display.width', 200)
# flags
useWatchdog = False
useScheduler = True
# local timezone
tzlocal = dateutil.tz.tzlocal()
# load the config file
configFile = args.configFile
config = ConfigParser(interpolation=ExtendedInterpolation(),
defaults=os.environ)
config.read(configFile)
# load data from configFile
host = config.get('InteractiveBrokers', 'host')
port = config.getint('InteractiveBrokers', 'port')
clientId = config.getint('InteractiveBrokers', 'clientId')
DBType = config.get('DataBase', 'DBType')
DBFileName = config.get('DataBase', 'DBFileName')
# for production mode: watchdog
if useWatchdog:
# start watchdog
# ibc = IBC(963, gateway=True, tradingMode='paper',ibcIni='/home/bn/IBController/configPaper.ini')
ibcIni = config.get('InteractiveBrokers', 'ibcIni')
tradingMode = config.get('InteractiveBrokers', 'tradingMode')
ibc = IBC(970, gateway=True, tradingMode=tradingMode, ibcIni=ibcIni)
ib = IB()
watchdogApp = Watchdog(ibc,
ib=ib,
appStartupTime=15,
host=host,
port=port,
clientId=clientId)
watchdogApp.start()
else:
# faster way for now
ib = IB()
try:
ib.connect(host=host, port=port, clientId=clientId)
except:
import random
clientId = clientId + random.randint(1, 100000)
ib.connect(host=host, port=port, clientId=clientId)
pass
class myWatchdog(object):
def __init__(self):
self.ib = ib
pass
pass
watchdogApp = myWatchdog()
pass
pass
# create database class
mydb = database.tradingDB(DBType=DBType, DBFileName=DBFileName)
# load existing database
mydb.instantiateExistingTablesAndClasses(ib=ib)
# set log level of sqlalchemy
mydb._loggerSQLAlchemy.setLevel(logging.WARNING)
# set the list of qualified contracts
# get a list of qualified contracts that correspond to each row in mydb.MarketDataInfoTableDataFrame
__qcs__ = list(mydb.MarketDataInfoTableDataFrame.qualifiedContract.values)
# qcs = __qcs__[0:2]
# qcs = operator.itemgetter(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12)(__qcs__)
# qcs = operator.itemgetter(0, 13, 1, 11, 7)(__qcs__)
# qcs = operator.itemgetter(0, 12, 13, 2, 10, 3)(__qcs__)
# qcs = operator.itemgetter(0, 1, 10)(__qcs__)
qcs = __qcs__
if isinstance(qcs, Contract):
qcs = [qcs]
pass
if isinstance(qcs, tuple):
qcs = list(qcs)
pass
if None in qcs:
print('problem with connecting to IB. Now exiting')
sys.exit()
# define the container class
cc = containerClass.ContainerClass()
# add config
cc.config = config
# set watchdogapp
cc.watchdogApp = watchdogApp
# set database
cc.mydb = mydb
cc.qcs = qcs
# register callbacks with ib
cc.registerCallbacks(useWatchdog=useWatchdog)
# define a scheduler
useScheduler = True
if useScheduler:
scheduler = AsyncIOScheduler()
cc.scheduler = scheduler
cc.scheduler.start()
pass
for qc in qcs:
print(qc)
cds = watchdogApp.ib.reqContractDetails(qc)
cd = cds[0]
tHP = marketDataIB.TradingHourParser(cd)
print(tHP.timeZoneId)
for line in tHP.tradingHours.split(';'):
print(line)
break
pass
hoursParsed = tHP.parseToDF()
print(hoursParsed.head(6))
# for index, row in hoursParsed.iterrows():
# print(row)
# break
# pass
# # general setting for the density of data for historical requests
# configIB = config['InteractiveBrokers']
# barSizePandasTimeDelta = pd.Timedelta(**eval(configIB.get('densityTimeDelta', '{"minutes":1}')))
#
#
# ##############################################################
# # request recent historical bars
# ##############################################################
# # settings to request the bars (for the qcs that are a member of cc)
# # the 'short' settings are the default ones to be applied during trading hours
#
# durationPandasTimeDelta = pd.Timedelta(**eval(configIB.get('durationTimeDeltaRecentHistoricalDataShort', '{"hours":1}')))
# timeOutTime = configIB.getint('timeOutTimeShortRequests', 10)
#
# recentHistoricalDataSettingsShort = {
# 'durationPandasTimeDelta': durationPandasTimeDelta,
# 'barSizePandasTimeDelta': barSizePandasTimeDelta,
# 'timeOutTime': timeOutTime,
# 'maximumBarsLengthFactor': 2,
# }
#
# # settings for recent historical bars to be requested during off-trading hours.
# # due to performance reasons, during the trading hours we want to request
# # very short bars; during off-trading hours, we can request longer bars
# # which fill up possible gaps left by the shorter setting.
#
# durationPandasTimeDelta = pd.Timedelta(**eval(configIB.get('durationTimeDeltaRecentHistoricalDataLong', '{"days":1}')))
# timeOutTime = configIB.getint('timeOutTimeMediumRequests', 60)
#
# recentHistoricalDataSettingsLong = {
# 'durationPandasTimeDelta': durationPandasTimeDelta,
# 'barSizePandasTimeDelta': barSizePandasTimeDelta,
# 'timeOutTime': timeOutTime,
# 'maximumBarsLengthFactor': 2,
# }
#
#
# # set the current settings in the containerClass
# a = (f'Now updating the settings for the request of recent historical bars')
# logging.info(a)
# print(a)
# # set the settings
# cc.recentHistoricalDataSettings = recentHistoricalDataSettingsShort
#
# # request the bars
# a = (f'Now requesting initial recent historical bars')
# logging.info(a)
# print(a)
# orderedDictOfBars = cc.requestRecentHistoricalOrderedDictOfBars()
# cc.orderedDictOfBars = orderedDictOfBars
#
#
# for (tableName, bars) in cc.orderedDictOfBars.items():
# nBars = None
# if isinstance(bars,objects.BarDataList):
# nBars = len(bars)
# print(tableName,type(bars),nBars)
# ##############################################################
#
#
# ##############################################################
# # request historical bars
# ##############################################################
#
# # add the job requesting historical data to the scheduler
# # this setting starts at the earliestDateTime given by IB
#
# earliestPandasTimeDelta = pd.Timedelta(**eval(configIB.get('earliestTimeDeltaHistoricalData', '{"weeks":4}')))
# durationPandasTimeDelta = pd.Timedelta(**eval(configIB.get('durationTimeDeltaHistoricalData', '{"days":1}')))
# timeOutTime = configIB.getint('timeOutTimeLongRequests', 1800)
# # timeOutTime = configIB.getint('timeOutTimeMediumRequests', 60)
#
# if earliestPandasTimeDelta.total_seconds() < 0:
# earliestDateTimeUTCNaive = None
# else:
# earliestDateTimeUTCNaive = pd.to_datetime(pd.datetime.utcnow()).floor('1 min') - earliestPandasTimeDelta
# pass
#
# historicalDataGetterSettings={
# 'ib': cc.watchdogApp.ib,
# 'mydb': cc.mydb,
# 'qcs': cc.qcs,
# 'durationPandasTimeDelta': durationPandasTimeDelta,
# 'barSizePandasTimeDelta': barSizePandasTimeDelta,
# 'earliestDateTime': earliestDateTimeUTCNaive,
# 'timeOutTime': timeOutTime,
# 'jitterSpanFraction': 0.02,
# }
#
# jobSettings = {
# 'job': marketDataIB.asyncioJobGetHistoricalData,
# 'args': [],
# 'kwargs': historicalDataGetterSettings,
# 'jobRootName': None,
# 'minute': '*',
# 'second': '0',
# 'coalesce': True,
# 'misfire_grace_time': 30,
# 'trigger': 'cron',
# 'max_instances': 1,
# }
#
# if useScheduler:
# cc.addJobToScheduler(jobSettings=jobSettings)
# pass
# ##############################################################
#
#
# ##############################################################
# # change the request of recent historical bars to a longer setting during off-trading hours
# ##############################################################
# # add a scheduled job that switches from the short to the long settings
# jobSettings = {
# 'job': cc.schedulerJobSwitchRequestForRecentHistoricalDataFromOneSettingToOther,
# 'args': [],
# 'kwargs': recentHistoricalDataSettingsLong,
# 'jobRootName': 'schedulerJobSwitchRequestForRecentHistoricalDataFromShortToLong',
# 'hour': '22',
# # 'hour': '*',
# 'minute': '07',
# # 'minute': '*/2',
# 'second': '00',
# # 'second': '5-59/10',
# 'coalesce': True,
# 'misfire_grace_time': 30,
# 'trigger': 'cron',
# 'max_instances': 1,
# }
#
# if useScheduler:
# cc.addJobToScheduler(jobSettings=jobSettings)
#
# # add a scheduled job that switches from the long to the short settings
# jobSettings = {
# 'job': cc.schedulerJobSwitchRequestForRecentHistoricalDataFromOneSettingToOther,
# 'args': [],
# 'kwargs': recentHistoricalDataSettingsShort,
# 'jobRootName': 'schedulerJobSwitchRequestForRecentHistoricalDataFromLongToShort',
# 'hour': '04',
# # 'hour': '*',
# 'minute': '13',
# # 'minute': '1-59/2',
# 'second': '00',
# # 'second': '*/10',
# 'coalesce': True,
# 'misfire_grace_time': 30,
# 'trigger': 'cron',
# 'max_instances': 1,
# }
#
# if useScheduler:
# cc.addJobToScheduler(jobSettings=jobSettings)
#
# ##############################################################
if 1:
if useScheduler:
print('Press Ctrl+{0} to exit'.format('Break' if os.name ==
'nt' else 'C'))
# Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
try:
asyncio.get_event_loop().run_forever()
except (KeyboardInterrupt, SystemExit):
pass
pass
else:
util.allowCtrlC()
ib.run()
pass
pass
ib.disconnect()
def runProg():
"""run program"""
util.patchAsyncio()
# log to a file
utils.logToFile(f'getRecentHistoricalData4.log', level=logging.INFO)
# utils.logToConsole()
# set pandas option
pd.set_option('display.width', 200)
# specify connection details
host = '127.0.0.1'
port = 4002
ibcIni = '/home/bn/IBController/configPaper.ini'
tradingMode = 'paper'
clientId = 12
# start watchdog
ibc = IBC(970, gateway=True, tradingMode=tradingMode, ibcIni=ibcIni)
ib = IB()
watchdogApp = ibcontroller.Watchdog(ibc, ib=ib,appStartupTime=15, host=host, port=port, clientId=clientId)
watchdogApp.start()
# create some contracts
qcs = []
c = Contract(symbol='EUR',currency='CHF',exchange='IDEALPRO',secType='CASH')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='EUR',currency='CNH',exchange='IDEALPRO',secType='CASH')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='EUR',currency='GBP',exchange='IDEALPRO',secType='CASH')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='EUR', currency='JPY', exchange='IDEALPRO', secType='CASH')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='EUR', currency='RUB', exchange='IDEALPRO', secType='CASH')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='EUR', currency='USD', exchange='IDEALPRO', secType='CASH')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='IBDE30',currency='EUR',exchange='SMART',secType='CFD')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='DAX',currency='EUR',exchange='DTB',secType='IND')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='HSC50',currency='HKD',exchange='HKFE',secType='IND')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='INDU',currency='USD',exchange='CME',secType='IND')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='N225',currency='JPY',exchange='OSE.JPN',secType='IND')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
c = Contract(symbol='SPX',currency='USD',exchange='CBOE',secType='IND')
qc = ib.qualifyContracts(c)[0]
qcs.append(qc)
# qcs = [qc for qc in qcs if qc.localSymbol in ['N225','EUR.USD','IBDE30','DJI']]
# qcs = [qc for qc in qcs if qc.localSymbol in ['N225','DJI']]
qcs = [qc for qc in qcs if qc.localSymbol in ['N225','DAX','DJI']]
# function to request historical bars
def requestHistoricalBars(qcs):
# request historical bars
barss = []
for qc in qcs:
whatToShow = 'TRADES' if qc.secType == 'IND' else 'MIDPOINT'
bars = ib.reqHistoricalData(
qc,
endDateTime='',
durationStr='1 W',
barSizeSetting='1 min',
whatToShow=whatToShow,
useRTH=False,
formatDate=2,
keepUpToDate=True)
barss.append(bars)
pass
return barss
barss = requestHistoricalBars(qcs)
print('er', barss[0][-1])
print('er2', barss[1][-1])
print('er2', barss[2][-1])
def requestMarketData(qcs):
for qc in qcs:
ib.reqMktData(contract=qc,
genericTickList='',
snapshot=False,
regulatorySnapshot=False,
mktDataOptions=None)
pass
pass
def requestRealTimeBars(qcs):
barss = []
for qc in qcs:
bars = ib.reqRealTimeBars(contract=qc,
barSize='',
whatToShow='MIDPOINT',
useRTH=False,
realTimeBarsOptions=None)
barss.append(bars)
pass
return (barss)
# define some callback
def onBarUpdate(bars, hasNewBar):
localSymbol = bars.contract.localSymbol
secType = bars.contract.secType
b0 = bars[0]
if isinstance(b0, objects.RealTimeBar):
dateTimeAttributeName = 'time'
barType = 'RealTimeBar'
else:
dateTimeAttributeName = 'date'
barType = 'BarData'
pass
dt0 = pd.to_datetime(getattr(b0,dateTimeAttributeName)).tz_localize(None)
bm1 = bars[-1]
dtm1 = pd.to_datetime(getattr(bm1,dateTimeAttributeName)).tz_localize(None)
nowUTC = pd.to_datetime(pd.datetime.utcnow()).tz_localize(None)
diffDateTIme = (nowUTC - dtm1) / pd.Timedelta('1 sec')
if (hasNewBar or localSymbol in ['N225','DAX']):
print(f'local Symbol: {localSymbol}, hasNewBar: {hasNewBar}; barType: {barType}, nBars: {len(bars)}, diffDateTime: {diffDateTIme}, close: {bm1.close}')
def onPendingTickers(tickers):
for t in tickers:
localSymbol = t.contract.localSymbol
if localSymbol == "EUR.USD":
nowUTC = pd.to_datetime(pd.datetime.utcnow()).tz_localize(None)
nowUTCRounded = nowUTC.floor('1 min')
dateTime = pd.to_datetime(t.time).tz_localize(None)
print(localSymbol, nowUTCRounded, ((dateTime - nowUTCRounded)/pd.Timedelta('1 sec')),t.close)
pass
pass
pass
def myErrorCallback(reqId, errorCode, errorString, contract):
print("myErrorCallback", reqId,errorCode,errorString,contract)
if False:
# if errorCode == 322:
print("myErrorCallback", reqId, errorCode, errorString, contract)
# more than 50 simultaneous historical data requests
app.ib.client.cancelHistoricalData(reqId)
def onConnectedCallback():
print('connected')
barss = requestHistoricalBars(qcs)
# barss = requestRealTimeBars(qcs)
ib.barUpdateEvent.clear()
ib.barUpdateEvent += onBarUpdate
print('connected 2')
# requestMarketData(qcs)
# ib.pendingTickersEvent.clear()
# ib.pendingTickersEvent += onPendingTickers
print('connected 3')
pass
def onDisconnectedCallback():
print ('disconnected')
def myTimeoutCallback(timeout):
print (f'timeout {timeout}')
# request the bars
barss = requestHistoricalBars(qcs)
# request market data
# requestMarketData(qcs)
# request real time bars
# requestRealTimeBars(qcs)
# register the callbacks with ib
ib.connectedEvent.clear()
ib.connectedEvent += onConnectedCallback
ib.disconnectedEvent.clear
ib.disconnectedEvent = onDisconnectedCallback
ib.barUpdateEvent.clear()
ib.barUpdateEvent += onBarUpdate
ib.pendingTickersEvent.clear()
# ib.pendingTickersEvent += onPendingTickers
ib.errorEvent.clear()
ib.errorEvent += myErrorCallback
ib.timeoutEvent.clear()
ib.timeoutEvent += myTimeoutCallback
def mySoftTimeoutCallback(watchdogApp):
print (f'soft time out {watchdogApp}')
def myHardTimeoutCallback(watchdogApp):
print (f'hard time out {watchdogApp}')
watchdogApp.flush()
# def myStoppingCallback(sthg):
# print (f'Stopping Event {sthg}')
# def myStoppedCallback(sthg):
# print (f'Stopped Event {sthg}')
# watchdogApp.softTimeoutEvent.clear()
watchdogApp.softTimeoutEvent += mySoftTimeoutCallback
# watchdogApp.hardTimeoutEvent.clear()
watchdogApp.hardTimeoutEvent += myHardTimeoutCallback
# watchdogApp.stoppingEvent.clear()
# watchdogApp.stoppingEvent += myStoppingCallback
#
# watchdogApp.stoppedEvent.clear()
# watchdogApp.stoppedEvent += myStoppedCallback
# run and never stop
ib.run()
def runProg(args):
"""run program"""
util.patchAsyncio()
# log to a file
utils.logToFile(f'getRecentHistoricalData.log')
# utils.logToConsole()
apschedulerLogger = logging.getLogger('apscheduler')
apschedulerLogger.setLevel(logging.ERROR)
tradingLogger = logging.getLogger('trading')
tradingLogger.setLevel(logging.WARNING)
pd.set_option('display.width', 200)
# load the config file
configFile = args.configFile
config = ConfigParser(interpolation=ExtendedInterpolation(),
defaults=os.environ)
config.read(configFile)
# load data from configFile
host = config.get('InteractiveBrokers', 'host')
port = config.getint('InteractiveBrokers', 'port')
DBType = config.get('DataBase', 'DBType')
DBFileName = config.get('DataBase', 'DBFileName')
clientId = config.get('InteractiveBrokers', 'clientId')
# for production mode: watchdog
if 1:
# start watchdog
# ibc = IBC(963, gateway=True, tradingMode='paper',ibcIni='/home/bn/IBController/configPaper.ini')
ibcIni = config.get('InteractiveBrokers', 'ibcIni')
tradingMode = config.get('InteractiveBrokers', 'tradingMode')
ibc = IBC(970, gateway=True, tradingMode=tradingMode, ibcIni=ibcIni)
myWatchdogapp = myWatchdog.myWatchdog(ibc,
appStartupTime=15,
port=4002)
myWatchdogapp.start()
ib = myWatchdogapp.ib
pass
if 0:
# faster way for now
ib = IB()
ib.connect(host=host, port=port, clientId=clientId)
pass
pass
# create database class
mydb = database.tradingDB(DBType=DBType, DBFileName=DBFileName)
# load existing database
mydb.instantiateExistingTablesAndClasses(ib=ib)
# set log level of sqlalchemy
mydb._loggerSQLAlchemy.setLevel(logging.WARNING)
qcs = mydb.MarketDataInfoTableDataFrame.qualifiedContract
for qc in qcs:
print(qc, type(qc))
ib.reqMktData(contract=qc,
genericTickList='',
snapshot=False,
regulatorySnapshot=False,
mktDataOptions=None)
pass
df = pd.DataFrame(
columns='symbol bidSize bid ask askSize high low close'.split())
df['symbol'] = [qc.localSymbol for qc in qcs]
contract2Row = {qc: i for (i, qc) in enumerate(qcs)}
pprint.pprint(contract2Row)
def onPendingTickers(tickers):
for t in tickers:
iRow = contract2Row[t.contract]
localSymbol = t.contract.localSymbol
if localSymbol == "EUR.USD":
nowUTC = pd.to_datetime(pd.datetime.utcnow()).tz_localize(None)
nowUTCRounded = nowUTC.floor('1 min')
dateTime = pd.to_datetime(t.time).tz_localize(None)
print(localSymbol, nowUTCRounded,
((dateTime - nowUTCRounded) / pd.Timedelta('1 sec')),
t.close)
# df.iloc[iRow, 1:] = (t.bidSize, t.bid, t.ask, t.askSize, t.high, t.low, t.close)
# print(df)
ib.setCallback('pendingTickers', onPendingTickers)
# ib.sleep(300)
if 1:
util.allowCtrlC()
# Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
try:
asyncio.get_event_loop().run_forever()
except (KeyboardInterrupt, SystemExit):
pass