def __init__(self, hostName, user, password, dbName, omSuffix, orderSeq,
fillSeq, systemName, defaultSide, logName, logLevel):
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
self.Timer = threading.Timer(10, self.sendOrder)
self.OM = SMGOrderManager(omSuffix, orderSeq, fillSeq, systemName)
self.Side = defaultSide
self.DbOmWriter = DBOrderManagerWriter(hostName, user, password,
dbName)
self.Logger = SMGLogger(logName, logLevel)
self.SimTickers = [
'BTC-USD', 'ETH-USD', 'LTC-USD', 'BCH-USD', 'ZRX-USD'
]
self.SimTickerCount = 0
self.UserId = -1
self.KafkaAdmin = KafkaAdminMgr()
class GDAXFeedHandler(object):
def __init__(self, connectionName, tickerFileName, logFile, logLevel):
self.ConnectionName = connectionName
self.TickerFileName = tickerFileName
self.Tickers = []
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Logger = SMGLogger(logFile, logLevel)
self.KafkaAdmin = KafkaAdminMgr()
def getTickers(self):
try:
tickerPath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'data'))
tickerFilename = tickerPath
if os.name == "nt":
tickerFilename += "\\" + self.TickerFileName
else:
tickerFilename += "/" + self.TickerFileName
fp = open(tickerFilename,"r")
for ticker in fp:
self.Tickers.append(ticker.strip('\n'))
fp.close()
except Exception:
self.Logger.error("Error processing Ticker FIle - " + tickerFilename)
def getSubscriptionString(self):
try:
connectionString = "{\"type\": \"subscribe\",\"product_ids\": "
count = 0
tickerString = "["
for ticker in self.Tickers:
if count > 0:
tickerString += ","
tickerString += "\"" + ticker + "\""
count += 1
tickerString += "]"
connectionString += tickerString
connectionString +=",\"channels\": [\"heartbeat\",{\"name\": \"ticker\",\"product_ids\": "
connectionString += tickerString
connectionString += "}]}"
return connectionString
except Exception:
self.Logger.error("Error processing subscription string")
def subscribe(self, ws):
try:
self.getTickers()
subscriptionString = self.getSubscriptionString()
self.Logger.info("Sending Subscription TO cointbase: " + subscriptionString)
ws.send(subscriptionString)
self.Logger.info("Sent subscription")
except Exception:
raise Exception("Error sending subscription")
def processEvent(self, data):
try:
f = "%Y-%m-%dT%H:%M:%S.%fZ"
out = datetime.strptime(data['time'], f)
output = str(data['sequence']) + "," + data['product_id'] + "," + data['best_bid'] + "," + data[
'best_ask'] + "," + str(out)
self.Logger.info("Data Received - %s - Publish to Kafka" % output)
self.Producer.send('GDAXFeed', output.encode('utf-8'))
except Exception:
self.Logger.error("Error processing event - " + str(data))
def isHeartbeatOk(self, heartbeattime):
try:
current = datetime.now()
if current.hour < heartbeattime.hour:
return True
curval = current.second + (current.minute * 60) + (current.hour * 60 * 60)
heartval = heartbeattime.second + (heartbeattime.minute * 60) + (heartbeattime.hour * 60 * 60) + 60
if curval > heartval:
return False
return True
except Exception:
self.Logger.error("Error checking heartbeat")
def connectAndSubscribe(self):
self.Logger.info("connecting to GDAX Exchange to get Market Data")
ws = create_connection(self.ConnectionName)
self.Logger.info("Subscribing to data")
self.subscribe(ws)
return ws
def run(self):
self.Logger.info("making sure topic is created")
topics = ['GDAXFeed']
self.KafkaAdmin.addTopics(topics)
ws = self.connectAndSubscribe()
self.Logger.info("Receiving Data...")
heartbeatTime = datetime.now()
while 1:
result = ws.recv()
value = json.loads(result)
if value['type'] == "ticker" and 'time' in value:
self.processEvent(value)
elif value['type'] == "heartbeat":
heartbeatTime = datetime.now()
if not self.isHeartbeatOk(heartbeatTime):
self.Logger.info("Stale heartbeat. Need to reconnect and subscribe")
ws.close()
ws = self.connectAndSubscribe()
ws.close()
class SMGOrderSimulator(object):
def __init__(self, hostName, user, password, dbName, omSuffix, orderSeq,
fillSeq, systemName, defaultSide, logName, logLevel):
self.Producer = KafkaProducer(bootstrap_servers='localhost:9092')
self.Consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='earliest',
consumer_timeout_ms=1000)
self.Timer = threading.Timer(10, self.sendOrder)
self.OM = SMGOrderManager(omSuffix, orderSeq, fillSeq, systemName)
self.Side = defaultSide
self.DbOmWriter = DBOrderManagerWriter(hostName, user, password,
dbName)
self.Logger = SMGLogger(logName, logLevel)
self.SimTickers = [
'BTC-USD', 'ETH-USD', 'LTC-USD', 'BCH-USD', 'ZRX-USD'
]
self.SimTickerCount = 0
self.UserId = -1
self.KafkaAdmin = KafkaAdminMgr()
def setUserId(self):
sqlText = "select userid from smguser where username ='******'"
results = self.DbOmWriter.Db.select(sqlText)
for result in results:
self.UserId = result[0]
return True
return False
def setSide(self):
if self.Side == "Buy":
self.Side = "Sell"
else:
self.Side = "Buy"
def getSymbol(self):
symbol = self.SimTickers[self.SimTickerCount]
self.SimTickerCount += 1
if self.SimTickerCount == len(self.SimTickers):
self.SimTickerCount = 0
return symbol
def getQty(self):
qty = random.randrange(100, 1000, 10)
return qty
def sendOrder(self):
try:
self.setSide()
symbol = self.getSymbol()
qty = self.getQty()
order = self.OM.createOrder("", "", symbol, self.Side, qty,
SMOrderTypes.Market.value, 0, "Day",
"", "", self.UserId, "CRYPTO")
self.DbOmWriter.saveNewOrder(order)
self.Logger.info("Sending Order - " + str(order))
self.Producer.send('SMGExchangeOrder', str(order).encode('utf-8'))
self.Timer = threading.Timer(1, self.sendOrder)
self.Timer.start()
except Exception:
self.Logger.error("Error sending Order")
def isValidFill(self, message):
temp = message.split(',')
if len(temp) != 9:
return False
temp2 = temp[6].split('-')
if len(temp2) != 2:
return False
if int(temp2[1]) <= self.OM.FillCounter:
return False
return True
def processFill(self, message):
try:
if not self.isValidFill(message):
return
fill = self.OM.createFillFromMsg(message, self.UserId)
if fill is None:
return
self.Logger.info("Got Execution -" + str(fill))
self.DbOmWriter.saveNewFill(fill)
order = self.OM.getOrder(fill.OrderId)
if order is None:
return
self.DbOmWriter.updateOrder(order)
except Exception:
self.Logger.error("Error processing Fill")
def run(self):
if self.setUserId() is False:
self.Logger.error("Not able to find userId for SMGOrderSimulator")
return
self.KafkaAdmin.addTopics(['SimulatorFill', 'SMGExchangeOrder'])
self.Logger.info("Subscribe to SimulatorFill")
self.Consumer.subscribe(['SimulatorFill'])
self.Timer.start()
while 1:
for message in self.Consumer:
msg = message[6].decode("utf-8")
self.processFill(msg)