def get_data_ib(instrument,
start,
resolution="1 min",
blotter=None,
output_path=None):
"""
Downloads historical data from Interactive Brokers
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
start : str
Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS])
:Optional:
resolution : str
1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day
blotter : str
Store MySQL server used by this Blotter (default is "auto detect")
output_path : str
Path to the location where the resulting CSV should be saved (default: ``None``)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _ib_history_downloaded
# load blotter settings
blotter_args = load_blotter_args(blotter,
logger=logging.getLogger(__name__))
# create contract string (no need for connection)
ibConn = ezIBpy()
ibConn.ibCallback = ibCallback
if not ibConn.connected:
ibConn.connect(clientId=0,
port=int(blotter_args['ibport']),
host=str(blotter_args['ibserver']))
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
contract_string = ibConn.contractString(instrument)
contract = ibConn.createContract(instrument)
ibConn.requestHistoricalData(contracts=[contract],
data="TRADES",
resolution=resolution,
lookback=tools.ib_duration_str(start),
rth=False)
while not _ib_history_downloaded:
time.sleep(.1)
ibConn.disconnect()
data = ibConn.historicalData[contract_string]
data['datetime'] = data.index
return prepare_data(instrument, data, output_path=output_path)
def get_data_ib(instrument, start, resolution="1 min", blotter=None, output_path=None):
"""
Downloads historical data from Interactive Brokers
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
start : str
Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS])
:Optional:
resolution : str
1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day
blotter : str
Store MySQL server used by this Blotter (default is "auto detect")
output_path : str
Path to the location where the resulting CSV should be saved (default: ``None``)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _ib_history_downloaded
# load blotter settings
blotter_args = load_blotter_args(
blotter, logger=logging.getLogger(__name__))
# create contract string (no need for connection)
ibConn = ezIBpy()
ibConn.ibCallback = ibCallback
if not ibConn.connected:
ibConn.connect(clientId=0,
port=int(blotter_args['ibport']), host=str(blotter_args['ibserver']))
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
contract_string = ibConn.contractString(instrument)
contract = ibConn.createContract(instrument)
ibConn.requestHistoricalData(contracts=[contract],
data="TRADES", resolution=resolution, lookback=tools.ib_duration_str(start), rth=False)
while not _ib_history_downloaded:
time.sleep(.1)
ibConn.disconnect()
data = ibConn.historicalData[contract_string]
data['datetime'] = data.index
return prepare_data(instrument, data, output_path=output_path)
def ibCallback(caller, msg, **kwargs):
print(caller)
print(msg)
#if caller == "handleTickGeneric":
# print("====Tick generic===")
# print(msg)
# print("====Tick generic===")
#if caller == "handleTickPrice":
# print("====Tick price===")
# print(msg)
# print("====Tick price===")
# initialize ezIBpy
ibConn = ezibpy.ezIBpy()
# connect to IB (7496/7497 = TWS, 4001 = IBGateway)
ibConn.connect(clientId=111, host="localhost", port=4001)
ibConn.ibCallback = ibCallback
# create some contracts using dedicated methods
stk_contract0 = ibConn.createStockContract("AAPL")
stk_contract1 = ibConn.createStockContract("MSFT")
stk_contract2 = ibConn.createStockContract("TSLA")
stk_contract3 = ibConn.createStockContract("F")
stk_contract4 = ibConn.createStockContract("GE")
stk_contract5 = ibConn.createStockContract("F")
stk_contract6 = ibConn.createStockContract("QCOM")
stk_contract7 = ibConn.createStockContract("NFLX")
stk_contract8 = ibConn.createStockContract("GOOG")
def __init__(self, instruments, ibclient=998, ibport=4001, ibserver="localhost"):
# detect running strategy
self.strategy = str(self.__class__).split('.')[-1].split("'")[0]
# initilize class logger
self.log_broker = logging.getLogger(__name__)
# default params (overrided in algo)
self.timezone = "UTC"
self.last_price = {}
self.tick_window = 1000
self.bar_window = 100
# -----------------------------------
# connect to IB
self.ibclient = int(ibclient)
self.ibport = int(ibport)
self.ibserver = str(ibserver)
self.ibConn = ezibpy.ezIBpy()
self.ibConn.ibCallback = self.ibCallback
self.ibConnect()
# -----------------------------------
# create contracts
instrument_tuples_dict = {}
for instrument in instruments:
try:
if isinstance(instrument, ezibpy.utils.Contract):
instrument = self.ibConn.contract_to_tuple(instrument)
else:
instrument = tools.create_ib_tuple(instrument)
contractString = self.ibConn.contractString(instrument)
instrument_tuples_dict[contractString] = instrument
self.ibConn.createContract(instrument)
except Exception as e:
pass
self.instruments = instrument_tuples_dict
self.symbols = list(self.instruments.keys())
self.instrument_combos = {}
# -----------------------------------
# track orders & trades
self.active_trades = {}
self.trades = []
# shortcut
self.account = self.ibConn.account
# use: self.orders.pending...
self.orders = tools.make_object(
by_tickerid=self.ibConn.orders,
by_symbol=self.ibConn.symbol_orders,
pending_ttls={},
pending={},
filled={},
active={},
history={},
nextId=1,
recent={}
)
# -----------------------------------
self.dbcurr = None
self.dbconn = None
# -----------------------------------
# assign default vals if not propogated from algo
if not hasattr(self, 'backtest'):
self.backtest = False
if not hasattr(self, 'sms_numbers'):
self.sms_numbers = []
if not hasattr(self, 'trade_log_dir'):
self.trade_log_dir = None
if not hasattr(self, 'blotter_name'):
self.blotter_name = None
# -----------------------------------
# load blotter settings
self.blotter_args = load_blotter_args(
self.blotter_name, logger=self.log_broker)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
if not self.blotter_args['dbskip']:
self.dbconn = pymysql.connect(
host=str(self.blotter_args['dbhost']),
port=int(self.blotter_args['dbport']),
user=str(self.blotter_args['dbuser']),
passwd=str(self.blotter_args['dbpass']),
db=str(self.blotter_args['dbname']),
autocommit=True
)
self.dbcurr = self.dbconn.cursor()
# -----------------------------------
# do stuff on exit
atexit.register(self._on_exit)
def run(self):
"""Starts the blotter
Connects to the TWS/GW, processes and logs market data,
and broadcast it over TCP via ZeroMQ (which algo subscribe to)
"""
self._check_unique_blotter()
# connect to mysql
self.mysql_connect()
self.context = zmq.Context(zmq.REP)
self.socket = self.context.socket(zmq.PUB)
self.socket.bind("tcp://*:" + str(self.args['zmqport']))
db_modified = 0
contracts = []
prev_contracts = []
first_run = True
logging.info("Connecting to Interactive Brokers...")
self.ibConn = ezIBpy()
self.ibConn.ibCallback = self.ibCallback
while not self.ibConn.connected:
self.ibConn.connect(clientId=int(self.args['ibclient']),
port=int(self.args['ibport']),
host=str(self.args['ibserver']))
time.sleep(1)
if not self.ibConn.connected:
print('*', end="", flush=True)
logging.info("Connection established...")
try:
while True:
if not os.path.exists(self.args['symbols']):
pd.DataFrame(columns=[
'symbol', 'sec_type', 'exchange', 'currency', 'expiry',
'strike', 'opt_type'
]).to_csv(self.args['symbols'], header=True, index=False)
tools.chmod(self.args['symbols'])
else:
time.sleep(0.1)
# read db properties
db_data = os.stat(self.args['symbols'])
db_size = db_data.st_size
db_last_modified = db_data.st_mtime
# empty file
if db_size == 0:
if len(prev_contracts) > 0:
logging.info('Cancel market data...')
self.ibConn.cancelMarketData()
time.sleep(0.1)
prev_contracts = []
continue
# modified?
if (first_run == False) & (db_last_modified
== db_modified):
continue
# continue...
db_modified = db_last_modified
# read contructs db
df = pd.read_csv(self.args['symbols'], header=0)
if len(df.index) == 0:
continue
# removed expired
df = df[(
(df['expiry'] < 1000000) &
(df['expiry'] >= int(datetime.now().strftime('%Y%m'))))
| ((df['expiry'] >= 1000000) &
(df['expiry'] >= int(datetime.now(
).strftime('%Y%m%d')))) | isnan(df['expiry'])]
df.fillna("", inplace=True)
df.to_csv(self.args['symbols'], header=True, index=False)
tools.chmod(self.args['symbols'])
df = df[df['symbol'].str.contains("#") ==
False] # ignore commentee
contracts = [tuple(x) for x in df.values]
if first_run:
first_run = False
else:
if contracts != prev_contracts:
# cancel market data for removed contracts
for contract in prev_contracts:
if contract not in contracts:
self.ibConn.cancelMarketData(
self.ibConn.createContract(contract))
time.sleep(0.1)
contract_string = self.ibConn.contractString(
contract).split('_')[0]
logging.info('Contract Removed [' +
contract_string + ']')
# request market data
for contract in contracts:
if contract not in prev_contracts:
self.ibConn.requestMarketData(
self.ibConn.createContract(contract))
time.sleep(0.1)
contract_string = self.ibConn.contractString(
contract).split('_')[0]
logging.info('Contract Added [' + contract_string +
']')
# update latest contracts
prev_contracts = contracts
time.sleep(2)
except (KeyboardInterrupt, SystemExit):
print("\n\n>>> Interrupted with Ctrl-c...")
sys.exit(1)
import sys
import shlex
import ezibpy
import time
import prettytable
SHELL_STATUS_RUN = 1
SHELL_STATUS_STOP = 0
tickers_ = []
# initialize ezIBpy
ibConn_ = ezibpy.ezIBpy()
def ibCallback(caller, msg, table=None, **kwargs):
if table is None:
return
if caller in ["handleAccount", "handlePortfolio"]:
print("\n", caller)
prettytable.pprint_table(sys.stdout, table)
def initConnection():
global ibConn_
# connect to IB (7496/7497 = TWS, 4001 = IBGateway)
ibConn_.accountCode = 'xxxxxxx'
ibConn_.connect(clientId=100, host="localhost", port=7496)
def initCallback():
global ibConn_
def prepare_data(instrument, data, output_path=None,
index=None, colsmap=None, kind="BAR", resample="1T"):
"""
Converts given DataFrame to a QTPyLib-compatible format and timezone
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
data : pd.DataFrame
Pandas DataDrame with that instrument's market data
output_path : str
Path to where the resulting CSV should be saved (optional)
index : pd.Series
Pandas Series that will be used for df's index (optioanl)
colsmap : dict
Dict for mapping df's columns to those used by QTPyLib
(default assumes same naming convention as QTPyLib's)
kind : str
Is this ``BAR`` or ``TICK`` data
resample : str
Pandas resolution (defaults to 1min/1T)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _TICKS_COLSMAP, _BARS_COLSMAP
# work on copy
df = data.copy()
# ezibpy's csv?
if set(df.columns) == set([
'datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']):
df.rename(columns={
'datetime': 'datetime',
'O': 'open',
'H': 'high',
'L': 'low',
'C': 'close',
'OI': 'volume',
}, inplace=True)
df.index = pd.to_datetime(df['datetime'])
df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC")
index = None
# lower case columns
df.columns = map(str.lower, df.columns)
# set index
if index is None:
index = df.index
# set defaults columns
if not isinstance(colsmap, dict):
colsmap = {}
_colsmap = _TICKS_COLSMAP if kind == "TICK" else _BARS_COLSMAP
for el in _colsmap:
if el not in colsmap:
colsmap[el] = _colsmap[el]
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
# create contract string (no need for connection)
ibConn = ezIBpy()
contract_string = ibConn.contractString(instrument)
asset_class = tools.gen_asset_class(contract_string)
symbol_group = tools.gen_symbol_group(contract_string)
# add symbol data
df.loc[:, 'symbol'] = contract_string
df.loc[:, 'symbol_group'] = symbol_group
df.loc[:, 'asset_class'] = asset_class
# validate columns
valid_cols = validate_columns(df, kind)
if not valid_cols:
raise ValueError('Invalid Column list')
# rename columns to map
df.rename(columns=colsmap, inplace=True)
# force option columns on options
if asset_class == "OPT":
df = tools.force_options_columns(df)
# remove all other columns
known_cols = list(colsmap.values()) + \
['symbol', 'symbol_group', 'asset_class', 'expiry']
for col in df.columns:
if col not in known_cols:
df.drop(col, axis=1, inplace=True)
# set UTC index
df.index = pd.to_datetime(index)
df = tools.set_timezone(df, "UTC")
df.index.rename("datetime", inplace=True)
# resample
if resample and kind == "BAR":
df = tools.resample(df, resolution=resample, tz="UTC")
# add expiry
df.loc[:, 'expiry'] = np.nan
if asset_class in ("FUT", "OPT", "FOP"):
df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string)
# save csv
if output_path is not None:
output_path = output_path[
:-1] if output_path.endswith('/') else output_path
df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind))
# return df
return df
def __init__(self,
instruments,
ibclient=999,
ibport=4001,
ibserver="localhost",
**kwargs):
# -----------------------------------
# detect running strategy
self.strategy = str(self.__class__).split('.')[-1].split("'")[0]
# -----------------------------------
# connect to IB
self.ibclient = int(ibclient)
self.ibport = int(ibport)
self.ibserver = str(ibserver)
self.ibConn = ezibpy.ezIBpy()
self.ibConn.ibCallback = self.ibCallback
self.ibConnect()
# -----------------------------------
# create contracts
instrument_tuples_dict = {}
for instrument in instruments:
try:
# signgle string
if isinstance(instrument, str):
instrument = instrument.upper()
if "FUT." not in instrument:
# symbol stock
instrument = (instrument, "STK", "SMART", "USD", "",
0.0, "")
else:
# future contract
try:
symdata = instrument.split(".")
# is this a CME future?
if symdata[
1] not in futures.futures_contracts.keys():
raise ValueError(
"Un-supported symbol. Please use full contract tuple."
)
# auto get contract details
spec = futures.get_ib_futures(symdata[1])
if not isinstance(spec, dict):
raise ValueError("Un-parsable contract tuple")
# expiry specified?
if len(symdata) == 3 and symdata[2] != '':
expiry = symdata[2]
else:
# default to most active
expiry = futures.get_active_contract(
symdata[1])
instrument = (spec['symbol'].upper(), "FUT",
spec['exchange'].upper(),
spec['currency'].upper(),
int(expiry), 0.0, "")
except:
raise ValueError("Un-parsable contract tuple")
# tuples without strike/right
elif len(instrument) <= 7:
instrument_list = list(instrument)
if len(instrument_list) < 3:
instrument_list.append("SMART")
if len(instrument_list) < 4:
instrument_list.append("USD")
if len(instrument_list) < 5:
instrument_list.append("")
if len(instrument_list) < 6:
instrument_list.append(0.0)
if len(instrument_list) < 7:
instrument_list.append("")
try:
instrument_list[4] = int(instrument_list[4])
except:
pass
instrument_list[5] = 0. if isinstance(instrument_list[5], str) \
else float(instrument_list[5])
instrument = tuple(instrument_list)
contractString = self.ibConn.contractString(instrument)
instrument_tuples_dict[contractString] = instrument
self.ibConn.createContract(instrument)
except:
pass
self.instruments = instrument_tuples_dict
self.symbols = list(self.instruments.keys())
# -----------------------------------
# track orders & trades
self.active_trades = {}
self.trades = []
# shortcut
self.account = self.ibConn.account
# use: self.orders.pending...
self.orders = tools.make_object(by_tickerid=self.ibConn.orders,
by_symbol=self.ibConn.symbol_orders,
pending_ttls={},
pending={},
filled={},
active={},
history={},
nextId=1,
recent={})
# -----------------------------------
self.dbcurr = None
self.dbconn = None
# -----------------------------------
# assign default vals if not propogated from algo
if not hasattr(self, 'backtest'):
self.backtest = False
if not hasattr(self, 'sms_numbers'):
self.sms_numbers = []
if not hasattr(self, 'trade_log_dir'):
self.trade_log_dir = None
if not hasattr(self, 'blotter_name'):
self.blotter_name = None
# -----------------------------------
# load blotter settings
self.blotter_args = {}
self.load_blotter_args(self.blotter_name)
# -----------------------------------
# do stuff on exit
atexit.register(self._on_exit)
def connect_to_tws(self):
self.ibConn = ezibpy.ezIBpy()
self.ibConn.connect(clientId=self.client,
host=self.host,
port=self.port)
def prepare_data(instrument, data, output_path=None, index=None, colsmap=None, kind="BAR"):
"""
Converts given DataFrame to a QTPyLib-compatible format and timezone
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
data : pd.DataFrame
Pandas DataDrame with that instrument's market data
output_path : str
Path to the location where the resulting CSV should be saved (default: ``None``)
index : pd.Series
Pandas Series that will be used for df's index (default is to use df.index)
colsmap : dict
Dict for mapping df's columns to those used by QTPyLib (default assumes same naming convention as QTPyLib's)
kind : str
Is this ``BAR`` or ``TICK`` data
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _bars_colsmap, _ticks_colsmap
# work on copy
df = data.copy()
# ezibpy's csv?
if set(df.columns) == set(['datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']):
df.rename(columns={
'datetime': 'datetime',
'O': 'open',
'H': 'high',
'L': 'low',
'C': 'close',
'OI': 'volume',
}, inplace=True)
df.index = pd.to_datetime(df['datetime'])
df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC")
index = None
# set index
if index is None:
index = df.index
# set defaults columns
if not isinstance(colsmap, dict):
colsmap = {}
_colsmap = _ticks_colsmap if kind == "TICK" else _bars_colsmap
for el in _colsmap:
if el not in colsmap:
colsmap[el] = _colsmap[el]
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
# create contract string (no need for connection)
ibConn = ezIBpy()
contract_string = ibConn.contractString(instrument)
asset_class = tools.gen_asset_class(contract_string)
symbol_group = tools.gen_symbol_group(contract_string)
# add symbol data
df.loc[:, 'symbol'] = contract_string
df.loc[:, 'symbol_group'] = symbol_group
df.loc[:, 'asset_class'] = asset_class
# validate columns
valid_cols = validate_columns(df, kind)
if not valid_cols:
raise ValueError('Invalid Column list')
# rename columns to map
df.rename(columns=colsmap, inplace=True)
# force option columns on options
if asset_class == "OPT":
df = tools.force_options_columns(df)
# remove all other columns
known_cols = list(colsmap.values()) + ['symbol','symbol_group','asset_class','expiry']
for col in df.columns:
if col not in known_cols:
df.drop(col, axis=1, inplace=True)
# set UTC index
df.index = pd.to_datetime(index)
df = tools.set_timezone(df, "UTC")
df.index.rename("datetime", inplace=True)
# add expiry
df.loc[:, 'expiry'] = np.nan
if asset_class in ("FUT", "OPT", "FOP"):
df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string)
# save csv
if output_path is not None:
output_path = output_path[
:-1] if output_path.endswith('/') else output_path
df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind))
# return df
return df
class IBMarket(object):
conn = ezibpy.ezIBpy()
conn.connect(clientId=101, host="localhost", port=4001)
market_data_available = {}
# define custom callback
def ibCallback(self, caller, msg, **kwargs):
#print(caller)
#print(msg)
try:
if caller == "handleHistoricalData":
completed = kwargs.get('completed')
if completed == True:
symbol = IBMarket.conn.contracts[msg.reqId].m_symbol
print("Historical data downloaded for %s" % symbol)
IBMarket.market_data_available[symbol] = True
except:
print("ERROR HANDLE CALLBACK")
def __init__(self, rth=False):
self.period = None
self.data = None
self.rth = rth
IBMarket.conn.ibCallback = self.ibCallback
def isPostClose(self, timenow):
weekday = datetime.today().weekday()
postclosetime = datetime.now().replace(hour=20, minute=00)
closetime = datetime.now().replace(hour=16, minute=00)
if weekday < 5 and timenow > closetime and timenow < postclosetime:
ispostclose = True
else:
ispostclose = False
return ispostclose
def isPreOpen(self, timenow):
weekday = datetime.today().weekday()
preopentime = datetime.now().replace(hour=4, minute=00)
opentime = datetime.now().replace(hour=9, minute=30)
if weekday < 5 and timenow < opentime and timenow > preopentime:
ispreopen = True
else:
ispreopen = False
return ispreopen
def isOpen(self, timenow):
weekday = datetime.today().weekday()
opentime = datetime.now().replace(hour=9, minute=30)
closetime = datetime.now().replace(hour=16, minute=00)
if weekday < 5 and timenow > opentime and timenow < closetime:
isopen = True
else:
isopen = False
return isopen
def getDailyData(self, symbol, since, debug=False):
#try:
print("GET IBKR INTRADAY DATA")
now = datetime.now()
print("CREATE CONTRACT FOR %s" % symbol)
contract = IBMarket.conn.createStockContract(symbol)
dirname = os.path.dirname(__file__)
filename = "data/daily/" + symbol + "_" + now.strftime(
"%Y_%m_%d") + "_1_day.csv"
if os.path.isfile(filename) == True:
os.remove(filename)
print("REQUEST HISTORICAL DATA FOR %s SINCE %s" % (symbol, since))
IBMarket.market_data_available[symbol] = False
IBMarket.conn.requestHistoricalData(contract,
rth=self.rth,
resolution="1 day",
lookback="12 M",
csv_path='data/daily/')
print("WAIT FOR DATA...")
while IBMarket.market_data_available[symbol] == False:
time.sleep(5)
IBMarket.market_data_available[symbol] = False
print("RENAME CSV FILE TO %s" % filename)
os.rename("data/daily/" + symbol + ".csv", filename)
print("READ CSV FILE:%s" % filename)
self.data = pd.read_csv(filename,
index_col='datetime',
parse_dates=True)
self.data = self.data.sort_index()
if debug:
print("===VALUES===")
print(self.data)
print("===CALCULATE ALL TA===")
self.data = ta.utils.dropna(self.data)
self.data['sma20'] = SMAIndicator(close=self.data['C'],
n=20,
fillna=True).sma_indicator()
self.data['sma50'] = SMAIndicator(close=self.data['C'],
n=50,
fillna=True).sma_indicator()
self.data['sma100'] = SMAIndicator(close=self.data['C'],
n=100,
fillna=True).sma_indicator()
self.data['sma200'] = SMAIndicator(close=self.data['C'],
n=200,
fillna=True).sma_indicator()
self.data['ema9'] = EMAIndicator(close=self.data['C'],
n=9,
fillna=True).ema_indicator()
self.data['ema20'] = EMAIndicator(close=self.data['C'],
n=20,
fillna=True).ema_indicator()
self.data = ta.add_all_ta_features(self.data,
open="O",
high="H",
low="L",
close="C",
volume="V",
fillna=True)
self.data = self.data.loc[self.data.index >= since]
print("===CALCULATE DOJI===")
candle_names = talib.get_function_groups()['Pattern Recognition']
included_items = ('CDLDOJI', 'CDLHAMMER', 'CDLEVENINGSTAR',
'CDLHANGINGMAN', 'CDLSHOOTINGSTAR')
candle_names = [
candle for candle in candle_names if candle in included_items
]
for candle in candle_names:
self.data[candle] = getattr(talib,
candle)(self.data['O'], self.data['H'],
self.data['L'], self.data['C'])
if debug:
print("===VALUES===")
print(self.data[[
'C', 'V', 'WAP', 'sma20', 'sma50', 'sma100', 'sma200',
'momentum_rsi', 'trend_cci', 'momentum_stoch_signal',
'trend_adx'
]])
#except:
# print("ERROR GETTING MARKET DATA")
# time.sleep(60)
# self.data = None
#30 request / 10 mins -> need to wait 20 secs
time.sleep(20)
return self.data
def getIntraDayData(self, symbol, since, period, live=False, debug=False):
#try:
print("GET IBKR INTRADAY DATA")
if live == True:
lookback = int(period * 1600 / 60 / 24)
else:
dtime = datetime.now() - datetime.strptime(since, '%Y-%m-%d')
#lookback = dtime.total_seconds() / 60 / 60 / 24
lookback = 7
now = datetime.now()
print("CREATE CONTRACT FOR %s" % symbol)
contract = IBMarket.conn.createStockContract(symbol)
dirname = os.path.dirname(__file__)
filename = "data/intraday/" + symbol + "_" + now.strftime(
"%Y_%m_%d") + "_" + str(period) + "_min.csv"
if os.path.isfile(filename) == True:
os.remove(filename)
print(
"REQUEST HISTORICAL DATA FOR %s, LOOKBACK %d DAYS, SINCE %s, RTH:%d"
% (symbol, lookback, since, self.rth))
IBMarket.market_data_available[symbol] = False
IBMarket.conn.requestHistoricalData(contract,
rth=self.rth,
resolution=str(period) + " mins",
lookback=str(lookback) + " D",
csv_path='data/intraday/')
print("WAIT FOR DATA...")
while IBMarket.market_data_available[symbol] == False:
time.sleep(5)
IBMarket.market_data_available[symbol] = False
print("RENAME CSV FILE")
os.rename("data/intraday/" + symbol + ".csv", filename)
print("READ CSV DATA")
self.data = pd.read_csv(filename,
index_col='datetime',
parse_dates=True)
self.data = self.data.sort_index()
self.data = self.data[:-1]
if debug:
print("===VALUES===")
print(self.data)
print("===CALCULATE SMA===")
# add trend indicator.
#self.data['sma20'] = self.data['C'].rolling(20, min_periods=20).mean()
#self.data['sma50'] = self.data['C'].rolling(50, min_periods=50).mean()
#self.data['sma100'] = self.data['C'].rolling(100, min_periods=100).mean()
#self.data['sma200'] = self.data['C'].rolling(200, min_periods=200).mean()
print("===CALCULATE ALL TA===")
self.data = ta.utils.dropna(self.data)
self.data['sma20'] = SMAIndicator(close=self.data['C'],
n=20,
fillna=True).sma_indicator()
self.data['sma50'] = SMAIndicator(close=self.data['C'],
n=50,
fillna=True).sma_indicator()
self.data['sma100'] = SMAIndicator(close=self.data['C'],
n=100,
fillna=True).sma_indicator()
self.data['sma200'] = SMAIndicator(close=self.data['C'],
n=200,
fillna=True).sma_indicator()
self.data['ema9'] = EMAIndicator(close=self.data['C'],
n=9,
fillna=True).ema_indicator()
self.data['ema20'] = EMAIndicator(close=self.data['C'],
n=20,
fillna=True).ema_indicator()
self.data = ta.add_all_ta_features(self.data,
open="O",
high="H",
low="L",
close="C",
volume="V",
fillna=True)
self.data['mvwap'] = self.data['volume_vwap'].rolling(
14, min_periods=14).mean()
self.data = self.data.loc[self.data.index >= since]
print("===CALCULATE DOJI===")
candle_names = talib.get_function_groups()['Pattern Recognition']
if debug:
print(candle_names)
included_items = ('CDLDOJI', 'CDLHAMMER', 'CDLEVENINGSTAR',
'CDLHANGINGMAN', 'CDLSHOOTINGSTAR')
candle_names = [
candle for candle in candle_names if candle in included_items
]
for candle in candle_names:
self.data[candle] = getattr(talib,
candle)(self.data['O'], self.data['H'],
self.data['L'], self.data['C'])
if debug:
print("===VALUES===")
print(self.data)
#except:
# print("ERROR GETTING MARKET DATA")
# time.sleep(60)
# self.data = None
return self.data
class IBBroker(object):
conn = ezibpy.ezIBpy()
conn.connect(clientId=100, host="localhost", port=4001)
conn.requestPositionUpdates(subscribe=True)
conn.requestAccountUpdates(subscribe=True)
totalbudget = 0
totalinvested = 0
def __init__(self, totalbudget, quota, logger):
IBBroker.totalbudget = totalbudget
self.quota = quota
self.cash = quota
self.invested = 0
self.profit = 0
self.perf = 0
self.mylogger = logger
self.value = 0
self.buypending = False
self.sellpending = False
print("[IBK] quota %.3f, cash %.3f" % (self.quota, self.cash))
def buy_short_shares(self, stock, nb_shares, price=None):
return self.sell_shares(stock, nb_shares, price)
def sell_short_shares(self, stock, nb_shares, price=None):
return self.buy_shares(stock, nb_shares, price)
def buy_shares(self, stock, nb_shares, price=None):
nb_shares = math.floor(nb_shares)
if nb_shares == 0:
return -1
if IBBroker.totalinvested + nb_shares * stock.close > IBBroker.totalbudget:
self.mylogger.logger.debug("OVER BUDGET, CANNOT BUY %s" %
stock.symbol)
return -1
if self.cash < nb_shares * stock.close:
self.mylogger.logger.debug(
"NOT ENOUGH CASH LEFT %.3f$ TO BUY %.3f$" %
(self.cash, nb_shares * stock.close))
return -1
if stock.totalbuysize * stock.avgbuyprice > self.quota:
self.mylogger.logger.debug(
"HAVE %.3f$, ABOVE QUOTA OF %.3f$" %
(stock.totalbuysize * stock.avgbuyprice, self.quota))
return -1
if self.buypending == True and self.buyorder['status'] != "FILLED":
self.mylogger.logger.debug("BUY ORDER PENDING")
return -1
self.buypending = False
contract = IBBroker.conn.createStockContract(stock.symbol)
if price is None:
self.buyorder = IBBroker.conn.createOrder(quantity=nb_shares,
rth=True)
else:
self.buyorder = IBBroker.conn.createOrder(quantity=nb_shares,
price=price,
rth=True)
id = IBBroker.conn.placeOrder(contract, self.buyorder)
self.mylogger.logger.debug(
"%s [%s] IBK ORDER ID:%d, BUY %d SHARES @ %.3f$ OF %s" %
(datetime.now(), stock.symbol, id, nb_shares, stock.close,
stock.symbol))
self.mylogger.tradebook.debug(
"%s [%s] IBK ORDER ID:%d, BUY %d SHARES @ %.3f$ OF %s" %
(datetime.now(), stock.symbol, id, nb_shares, stock.close,
stock.symbol))
time.sleep(5)
self.buypending = True
result = stock.buy(nb_shares)
self.cash -= stock.close * nb_shares
self.invested = stock.avgbuyprice * stock.totalbuysize
self.calculate_profit(stock)
IBBroker.totalinvested += self.invested
return result
def sell_shares(self, stock, nb_shares, price=None):
nb_shares = math.floor(nb_shares)
if nb_shares == 0:
return -1
if self.invested < nb_shares * stock.avgbuyprice:
return -1
if self.sellpending == True and self.sellorder['status'] != "FILLED":
self.mylogger.logger.debug("SELL ORDER PENDING")
return -1
self.sellpending = False
contract = IBBroker.conn.createStockContract(stock.symbol)
if price is None:
self.sellorder = IBBroker.conn.createOrder(quantity=nb_shares * -1,
rth=True)
else:
self.sellorder = IBBroker.conn.createOrder(quantity=nb_shares * -1,
price=price,
rth=True)
id = IBBroker.conn.placeOrder(contract, self.sellorder)
self.mylogger.logger.debug(
"%s [%s] ORDER ID: %d, SELL %d SHARES @ %.3f$ OF %s" %
(datetime.now(), stock.symbol, id, nb_shares, stock.close,
stock.symbol))
self.mylogger.tradebook.debug(
"%s [%s] ORDER ID: %d, SELL %d SHARES @ %.3f$ OF %s" %
(datetime.now(), stock.symbol, id, nb_shares, stock.close,
stock.symbol))
time.sleep(5)
self.sellpending = True
result = stock.sell(nb_shares)
self.cash += stock.close * nb_shares
self.invested = stock.avgbuyprice * stock.totalbuysize
self.calculate_profit(stock)
IBBroker.totalinvested -= self.invested
return result
def calculate_profit(self, stock):
self.profit = (self.cash + self.invested) - self.quota
self.value = stock.totalbuysize * stock.close
self.unrzprofit = self.value - self.invested
self.perf = self.profit / self.quota * 100
def print_balance(self, stock):
self.calculate_profit(stock)
self.mylogger.logger.debug(
"%s [%6s] #TRANSAC = %d , CASH = %8.2f$, INVESTED = %8.2f$, VALUE = %8.2f RLZ PROFIT = %8.2f$, UNRLZ PROFIT = %8.2f$, PERF = %8.2f%%"
% (stock.tstamp, stock.symbol, stock.transaction, self.cash,
self.invested, self.value, self.profit, self.unrzprofit,
self.perf))