def __init__(self, host, user, password, logFile, logLevel):
self.Logger = SMGLogger(logFile, logLevel)
self.UserManager = UserManager(host, user, password, self.Logger)
self.Producer = None
self.Consumer = None
self.KafkaAdmin = KafkaAdminMgr()
def __init__(self, connectionName, tickerFileName, logFile, logLevel):
self.ConnectionName = connectionName
self.TickerFileName = tickerFileName
self.Tickers = []
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Logger = SMGLogger(logFile, logLevel)
self.KafkaAdmin = KafkaAdminMgr()
def __init__(self, host, user, password, database, logFile, logLevel,
omSuffix, systemName):
self.Logger = SMGLogger(logFile, logLevel)
self.BankManager = BankManager(host, user, password, self.Logger)
self.OrderMgr = SMGOrderManager(omSuffix, 0, 0, systemName)
self.DbOmWriter = DBOrderManagerWriter(host, user, password, database)
self.Database = database
self.PricingMgr = PricingManager()
self.ExtOrderId = 0
self.KafkaAdmin = KafkaAdminMgr()
def __init__(self, hostName, user, password, dbName, omSuffix, orderSeq,
fillSeq, systemName, logName, logLevel):
self.Orders = {}
self.Fills = {}
self.Bids = {}
self.Offers = {}
self.OM = SMGOrderManager(omSuffix, orderSeq, fillSeq, systemName)
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
self.DB = DBOrderManagerWriter(hostName, user, password, dbName)
self.Logger = SMGLogger(logName, logLevel)
self.RecOrderIds = {}
self.UserId = -1
self.KafkaAdmin = KafkaAdminMgr()
def __init__(self, hostName, user, password, dbName, omSuffix, orderSeq,
fillSeq, systemName, defaultSide, logName, logLevel):
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
self.Timer = threading.Timer(10, self.sendOrder)
self.OM = SMGOrderManager(omSuffix, orderSeq, fillSeq, systemName)
self.Side = defaultSide
self.DbOmWriter = DBOrderManagerWriter(hostName, user, password,
dbName)
self.Logger = SMGLogger(logName, logLevel)
self.SimTickers = [
'BTC-USD', 'ETH-USD', 'LTC-USD', 'BCH-USD', 'ZRX-USD'
]
self.SimTickerCount = 0
self.UserId = -1
self.KafkaAdmin = KafkaAdminMgr()
class SMGUserManager(object):
def __init__(self, host, user, password, logFile, logLevel):
self.Logger = SMGLogger(logFile, logLevel)
self.UserManager = UserManager(host, user, password, self.Logger)
self.Producer = None
self.Consumer = None
self.KafkaAdmin = KafkaAdminMgr()
def connect(self):
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
def run(self, database):
self.connect()
self.UserManager.connect(database)
self.KafkaAdmin.addTopics([
'NewUser', 'NewPortfolio', 'NewPosition', 'UserAddFailed',
'SMGNewUser'
])
self.Logger.info("Subscribe to SMGNewUser")
self.Consumer.subscribe(['SMGNewUser'])
self.UserManager.loadInitialData()
recovering = True
while 1:
for message in self.Consumer:
msg = message[6].decode("utf-8")
user = self.UserManager.createUserObjectFromMessage(msg)
if user is None:
continue
if self.UserManager.doesUserExist(user.UserName):
if recovering is True:
continue
errorMsg = "User already exist. Can't create user " + user.UserName
self.Logger.error(errorMsg)
self.Producer.send("UserAddFailed",
errorMsg.encode('utf-8'))
else:
self.UserManager.saveUser(user)
actuser = self.UserManager.getUser(user.UserName)
if actuser is None:
self.Logger.error("Error creating user")
retval = "User did not save " + user.UserName
self.Producer.send("UserAddFailed",
retval.encode('utf-8'))
else:
self.UserManager.updateUserHistory(actuser, "NEW")
self.UserManager.createPortfolio(actuser, 15000000.00)
self.UserManager.createInitialPosition(
actuser.UserId, "USD", 15000000.00)
self.Producer.send("NewUser",
str(actuser).encode('utf-8'))
portfolio = self.UserManager.getPortfolio(
actuser.UserId)
self.Producer.send("NewPortfolio",
str(portfolio).encode('utf-8'))
position = self.UserManager.getPosition(
actuser.UserId, "USD")
self.Producer.send("NewPosition",
str(position).encode('utf-8'))
recovering = False
class SMGExchange(object):
def __init__(self, hostName, user, password, dbName, omSuffix, orderSeq,
fillSeq, systemName, logName, logLevel):
self.Orders = {}
self.Fills = {}
self.Bids = {}
self.Offers = {}
self.OM = SMGOrderManager(omSuffix, orderSeq, fillSeq, systemName)
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
self.DB = DBOrderManagerWriter(hostName, user, password, dbName)
self.Logger = SMGLogger(logName, logLevel)
self.RecOrderIds = {}
self.UserId = -1
self.KafkaAdmin = KafkaAdminMgr()
def setUserId(self):
sqlText = "select userid from smguser where username ='******'"
results = self.DB.Db.select(sqlText)
for result in results:
self.UserId = result[0]
return True
return False
def setFillSeq(self):
try:
sqlString = "select max(created) from smgfill where ordersystem = 'SMGExchange'"
results = self.DB.Db.select(sqlString)
if len(results) == 0:
return
created = ""
for result in results:
created = result[0]
sqlString = "select fillId from smgfill where ordersystem = 'SMGExchange' and created ='%s'" % (
created)
results = self.DB.Db.select(sqlString)
if len(results) == 0:
self.Logger.info(
"Did not get back a fillId for SMGExchange. Strange!!!")
return
fillId = ""
for result in results:
fillId = result[0]
temp = fillId.split('-')
if len(temp) != 2:
self.Logger.info("Error trying to split fillId. FillId is " +
fillId)
return
self.OM.setFillSeq(int(temp[1]))
except Exception:
self.Logger.error("Error getting starting FillId")
def setOrderSeq(self):
try:
sqlString = "select max(lastupdate) from smgorder where ordersystem = 'SMGExchange'"
results = self.DB.Db.select(sqlString)
if len(results) == 0:
return
lastupdate = ""
for result in results:
lastupdate = result[0]
sqlString = "select orderId from smgorder where ordersystem = 'SMGExchange' and lastupdate = '%s'" % (
lastupdate)
results = self.DB.Db.select(sqlString)
if len(results) == 0:
self.Logger.info(
"Did not get back a orderId for SMGExchange. Strange!!!")
return
orderId = ""
for result in results:
orderId = result[0]
temp = orderId.split('-')
if len(temp) != 2:
self.Logger.info(
"Error trying to split OrderId. orderId is " + orderId)
return
self.OM.setOrderSeq(int(temp[1]))
except Exception:
self.Logger.error("Error getting starting OrderId")
def getProcessOrderIdsBySystem(self):
try:
sqlString = "select distinct extsystem from smgorder"
results = self.DB.Db.select(sqlString)
if len(results) == 0:
return
for result in results:
sqlString = "select max(lastupdate) from smgorder where ordersystem = 'SMGExchange' and extsystem ='%s'" % (
result[0])
lastupdateres = self.DB.Db.select(sqlString)
for litem in lastupdateres:
sqlString = "select extorderid from smgorder where ordersystem = 'SMGExchange'"\
" and extsystem = '%s' and lastupdate = '%s'" % (result[0], litem[0])
extresults = self.DB.Db.select(sqlString)
for extorderid in extresults:
temp = extorderid[0].split('-')
if len(temp) != 2:
self.Logger.info(
"Error splitting external orderId " +
extorderid)
continue
self.RecOrderIds[temp[0]] = int(temp[1])
except Exception:
self.Logger.error("Error processing OrderIds by Systems")
def processBidOffer(self, message):
try:
temp = message.split(',')
if len(temp) == 1:
return
symbol = temp[1]
bid = float(temp[2])
offer = float(temp[3])
self.Bids[symbol] = bid
self.Offers[symbol] = offer
self.Logger.info("Update Bid/Offer for " + symbol + " " +
str(bid) + " X " + str(offer))
except Exception:
self.Logger.error("Error processing Bid/Offer message " + message)
def getPrice(self, symbol, side):
if side == "Buy":
if symbol in self.Offers.keys():
return self.Offers[symbol]
else:
if symbol in self.Bids.keys():
return self.Bids[symbol]
return 1
def processOrder(self, message):
try:
temp = message.split(',')
if len(temp) != 18:
return
extOrderId = temp[0]
userId = int(temp[16])
etemp = extOrderId.split('-')
if len(etemp) != 2:
self.Logger.info(
"Error processing order parsing extOrderId - " +
extOrderId)
return
if etemp[0] in self.RecOrderIds:
if int(etemp[1]) <= self.RecOrderIds[etemp[0]]:
return
order = self.OM.createOrderFromMsg(message, self.UserId)
self.DB.saveNewOrder(order)
price = self.getPrice(order.Symbol, order.Side)
fill = self.OM.createFill("", order.OrderId, order.Qty,
price, order.ExtOrderId,
datetime.datetime.now(), userId)
self.DB.saveNewFill(fill)
self.DB.updateOrder(order)
topic = order.ExtSystem + "Fill"
self.Logger.info("Sending fill - Topic " + topic + " - " +
str(fill))
self.Producer.send(topic, str(fill).encode('utf-8'))
except Exception as e:
self.Logger.error("Error processing Order message " + message +
" Error:" + str(e))
def run(self):
if self.setUserId() is False:
self.Logger.error("Not able to get UserId for SMGExchange")
return
self.setFillSeq()
self.setOrderSeq()
self.getProcessOrderIdsBySystem()
self.KafkaAdmin.addTopics(['GDAXFeed', 'SMGExchangeOrder'])
self.Logger.info("Subscribing to GDAXFeed and SMGExchangeOrder")
self.Consumer.subscribe(['GDAXFeed', 'SMGExchangeOrder'])
while 1:
for message in self.Consumer:
msg = message[6].decode("utf-8")
if message[0] == "GDAXFeed":
self.processBidOffer(msg)
elif message[0] == "SMGExchangeOrder":
self.Logger.info("Got an order - " + msg)
self.processOrder(msg)
def __init__(self, host, user, password, logFile, logLevel):
self.Db = StockMarketDB(user, password, host)
self.StartSeq = {}
self.Logger = SMGLogger(logFile, logLevel)
self.KafkaAdmin = KafkaAdminMgr()
class DBWriter(object):
def __init__(self, host, user, password, logFile, logLevel):
self.Db = StockMarketDB(user, password, host)
self.StartSeq = {}
self.Logger = SMGLogger(logFile, logLevel)
self.KafkaAdmin = KafkaAdminMgr()
def getKafkaConsumer(self):
consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
return consumer
def publishUpdate(self, message):
try:
temp = message.split(',')
seq = int(temp[0])
symbol = temp[1]
if symbol in self.StartSeq:
if seq <= self.StartSeq[symbol]:
return
bid = float(temp[2])
offer = float(temp[3])
timestamp = temp[4]
sqlString = "update cryptotopofbook set sequenceno=%d,bestbid=%.8f,bestoffer=%.8f," \
"timestamp='%s' where symbol='%s'" % (seq, bid, offer, timestamp, symbol)
self.Db.update(sqlString)
self.Logger.info(sqlString)
except Exception:
self.Logger.error("Error publishing update - " + message)
def getStartSequences(self):
sqlString = "select symbol, sequenceno from cryptotopofbook"
results = self.Db.select(sqlString)
for result in results:
self.StartSeq[result[0]] = int(result[1])
def run(self, database):
self.Db.connect()
self.Db.changeDb(database)
consumer = self.getKafkaConsumer()
self.Logger.info("Subscribe to GDAXFeed")
self.KafkaAdmin.addTopics(['GDAXFeed'])
consumer.subscribe(['GDAXFeed'])
self.getStartSequences()
while 1:
for message in consumer:
msg = message[6].decode("utf-8")
if "," in msg:
self.publishUpdate(msg)
class SMGBankManager(object):
def __init__(self, host, user, password, database, logFile, logLevel,
omSuffix, systemName):
self.Logger = SMGLogger(logFile, logLevel)
self.BankManager = BankManager(host, user, password, self.Logger)
self.OrderMgr = SMGOrderManager(omSuffix, 0, 0, systemName)
self.DbOmWriter = DBOrderManagerWriter(host, user, password, database)
self.Database = database
self.PricingMgr = PricingManager()
self.ExtOrderId = 0
self.KafkaAdmin = KafkaAdminMgr()
def __exit__(self, exc_type, exc_val, exc_tb):
print("Going to exit")
def __enter__(self):
self.BankManager.connect(self.Database)
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
self.BankManager.UserMgr.loadInitialData()
self.KafkaAdmin.addTopics(
['GDAXFeed', 'SMGNewOrder', 'BankManagerFill', 'SMGPosition'])
self.Consumer.subscribe(['GDAXFeed', 'SMGNewOrder', 'BankManagerFill'])
return self
def processFill(self, message):
try:
if not self.OrderMgr.isValidFill(message):
return
temp = message.split(',')
userId = int(temp[8])
fill = self.OrderMgr.createFillFromMsg(message, userId)
if fill is None:
return
self.Logger.info("Got Execution -" + str(fill))
self.DbOmWriter.saveNewFill(fill)
order = self.OrderMgr.getOrder(fill.OrderId)
if order is None:
return
self.DbOmWriter.updateOrder(order)
self.Producer.send("SMGNewFill", str(fill).encode('utf-8'))
self.updatePositions(order, fill)
except Exception as e:
self.Logger.error("Error processing Fill - " + str(e))
def updatePositions(self, order, fill):
buyPosition, sellPosition = self.BankManager.updatePosition(
order.UserId, order.Symbol, fill.Qty, fill.Price, order.Side,
order.SecType)
if buyPosition is not None:
self.Producer.send("SMGPosition", str(buyPosition).encode('utf-8'))
if sellPosition is not None:
self.Producer.send("SMGPosition",
str(sellPosition).encode('utf-8'))
def processOrder(self, message):
try:
temp = message.split(',')
if len(temp) != 18:
self.Logger.error("Invalid Order Message")
return "0,ERROR,Invalid Order Message"
extOrderId = temp[0]
etemp = extOrderId.split('-')
if len(etemp) != 2:
self.Logger.info(
"Error processing order parsing extOrderId - " +
extOrderId)
return extOrderId + ",ERROR,Error processing order parsing"
if int(etemp[1]) <= self.ExtOrderId:
return "IGNORE"
userId = int(temp[16])
order = self.OrderMgr.createOrderFromMsg(message, userId)
price = self.PricingMgr.getPrice(order.Symbol, order.Side)
if price == 0:
return order.ExtOrderId + ",ERROR,Can't get valid price to evaluate"
if self.BankManager.canTradeCrypto(order.UserId, order.Symbol,
order.Side, order.Qty,
price) is False:
return order.ExtOrderId + ",ERROR,Not enough funds to trade"
self.DbOmWriter.saveNewOrder(order)
self.Logger.info("Sending Order - " + str(order))
self.Producer.send('SMGExchangeOrder', str(order).encode('utf-8'))
return order.ExtOrderId + ",SUCCESS,Sent Order"
except Exception as e:
self.Logger.error("Error processing Order message " + message +
" Error:" + str(e))
return "0,ERROR,Error processing Order message "
def run(self):
while 1:
for message in self.Consumer:
msg = message[6].decode("utf-8")
if message[0] == "GDAXFeed":
self.PricingMgr.processPriceMsg(msg)
elif message[0] == "SMGNewOrder":
self.Logger.info("Got an order - " + msg)
text = self.processOrder(msg)
if text != "IGNORE":
self.Producer.send('SMGNewOrderResponse',
text.encode('utf-8'))
elif message[0] == "BankManagerFill":
self.processFill(msg)
class GDAXFeedHandler(object):
def __init__(self, connectionName, tickerFileName, logFile, logLevel):
self.ConnectionName = connectionName
self.TickerFileName = tickerFileName
self.Tickers = []
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Logger = SMGLogger(logFile, logLevel)
self.KafkaAdmin = KafkaAdminMgr()
def getTickers(self):
try:
tickerPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'data'))
tickerFilename = tickerPath
if os.name == "nt":
tickerFilename += "\\" + self.TickerFileName
else:
tickerFilename += "/" + self.TickerFileName
fp = open(tickerFilename,"r")
for ticker in fp:
self.Tickers.append(ticker.strip('\n'))
fp.close()
except Exception:
self.Logger.error("Error processing Ticker FIle - " + tickerFilename)
def getSubscriptionString(self):
try:
connectionString = "{\"type\": \"subscribe\",\"product_ids\": "
count = 0
tickerString = "["
for ticker in self.Tickers:
if count > 0:
tickerString += ","
tickerString += "\"" + ticker + "\""
count += 1
tickerString += "]"
connectionString += tickerString
connectionString +=",\"channels\": [\"heartbeat\",{\"name\": \"ticker\",\"product_ids\": "
connectionString += tickerString
connectionString += "}]}"
return connectionString
except Exception:
self.Logger.error("Error processing subscription string")
def subscribe(self, ws):
try:
self.getTickers()
subscriptionString = self.getSubscriptionString()
self.Logger.info("Sending Subscription TO cointbase: " + subscriptionString)
ws.send(subscriptionString)
self.Logger.info("Sent subscription")
except Exception:
raise Exception("Error sending subscription")
def processEvent(self, data):
try:
f = "%Y-%m-%dT%H:%M:%S.%fZ"
out = datetime.strptime(data['time'], f)
output = str(data['sequence']) + "," + data['product_id'] + "," + data['best_bid'] + "," + data[
'best_ask'] + "," + str(out)
self.Logger.info("Data Received - %s - Publish to Kafka" % output)
self.Producer.send('GDAXFeed', output.encode('utf-8'))
except Exception:
self.Logger.error("Error processing event - " + str(data))
def isHeartbeatOk(self, heartbeattime):
try:
current = datetime.now()
if current.hour < heartbeattime.hour:
return True
curval = current.second + (current.minute * 60) + (current.hour * 60 * 60)
heartval = heartbeattime.second + (heartbeattime.minute * 60) + (heartbeattime.hour * 60 * 60) + 60
if curval > heartval:
return False
return True
except Exception:
self.Logger.error("Error checking heartbeat")
def connectAndSubscribe(self):
self.Logger.info("connecting to GDAX Exchange to get Market Data")
ws = create_connection(self.ConnectionName)
self.Logger.info("Subscribing to data")
self.subscribe(ws)
return ws
def run(self):
self.Logger.info("making sure topic is created")
topics = ['GDAXFeed']
self.KafkaAdmin.addTopics(topics)
ws = self.connectAndSubscribe()
self.Logger.info("Receiving Data...")
heartbeatTime = datetime.now()
while 1:
result = ws.recv()
value = json.loads(result)
if value['type'] == "ticker" and 'time' in value:
self.processEvent(value)
elif value['type'] == "heartbeat":
heartbeatTime = datetime.now()
if not self.isHeartbeatOk(heartbeatTime):
self.Logger.info("Stale heartbeat. Need to reconnect and subscribe")
ws.close()
ws = self.connectAndSubscribe()
ws.close()
class SMGOrderSimulator(object):
def __init__(self, hostName, user, password, dbName, omSuffix, orderSeq,
fillSeq, systemName, defaultSide, logName, logLevel):
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
self.Timer = threading.Timer(10, self.sendOrder)
self.OM = SMGOrderManager(omSuffix, orderSeq, fillSeq, systemName)
self.Side = defaultSide
self.DbOmWriter = DBOrderManagerWriter(hostName, user, password,
dbName)
self.Logger = SMGLogger(logName, logLevel)
self.SimTickers = [
'BTC-USD', 'ETH-USD', 'LTC-USD', 'BCH-USD', 'ZRX-USD'
]
self.SimTickerCount = 0
self.UserId = -1
self.KafkaAdmin = KafkaAdminMgr()
def setUserId(self):
sqlText = "select userid from smguser where username ='******'"
results = self.DbOmWriter.Db.select(sqlText)
for result in results:
self.UserId = result[0]
return True
return False
def setSide(self):
if self.Side == "Buy":
self.Side = "Sell"
else:
self.Side = "Buy"
def getSymbol(self):
symbol = self.SimTickers[self.SimTickerCount]
self.SimTickerCount += 1
if self.SimTickerCount == len(self.SimTickers):
self.SimTickerCount = 0
return symbol
def getQty(self):
qty = random.randrange(100, 1000, 10)
return qty
def sendOrder(self):
try:
self.setSide()
symbol = self.getSymbol()
qty = self.getQty()
order = self.OM.createOrder("", "", symbol, self.Side, qty,
SMOrderTypes.Market.value, 0, "Day",
"", "", self.UserId, "CRYPTO")
self.DbOmWriter.saveNewOrder(order)
self.Logger.info("Sending Order - " + str(order))
self.Producer.send('SMGExchangeOrder', str(order).encode('utf-8'))
self.Timer = threading.Timer(1, self.sendOrder)
self.Timer.start()
except Exception:
self.Logger.error("Error sending Order")
def isValidFill(self, message):
temp = message.split(',')
if len(temp) != 9:
return False
temp2 = temp[6].split('-')
if len(temp2) != 2:
return False
if int(temp2[1]) <= self.OM.FillCounter:
return False
return True
def processFill(self, message):
try:
if not self.isValidFill(message):
return
fill = self.OM.createFillFromMsg(message, self.UserId)
if fill is None:
return
self.Logger.info("Got Execution -" + str(fill))
self.DbOmWriter.saveNewFill(fill)
order = self.OM.getOrder(fill.OrderId)
if order is None:
return
self.DbOmWriter.updateOrder(order)
except Exception:
self.Logger.error("Error processing Fill")
def run(self):
if self.setUserId() is False:
self.Logger.error("Not able to find userId for SMGOrderSimulator")
return
self.KafkaAdmin.addTopics(['SimulatorFill', 'SMGExchangeOrder'])
self.Logger.info("Subscribe to SimulatorFill")
self.Consumer.subscribe(['SimulatorFill'])
self.Timer.start()
while 1:
for message in self.Consumer:
msg = message[6].decode("utf-8")
self.processFill(msg)