def on_message(self, raw_message):
msg = JsonMessage(raw_message)
if not msg.is_valid():
self.close()
return
if msg.type == '1': # TestRequest
# send the heart beat back
self.write_message('{"MsgType":"0", "TestReqID":"%s"}' %
msg.get("TestReqID"))
return
elif msg.type == 'V': # Market Data Request
req_id = msg.get('MDReqID')
if int(msg.get('SubscriptionRequestType')) == 0: # Snapshot
pass
elif int(msg.get(
'SubscriptionRequestType')) == 1: # Snapshot + Updates
if req_id not in self.md_subscriptions:
self.md_subscriptions[req_id] = []
elif int(msg.get('SubscriptionRequestType')
) == 2: # Disable previous Snapshot + Update Request
if req_id in self.md_subscriptions:
del self.md_subscriptions[req_id]
return
def on_md_publish(self, publish_msg):
"""" on_md_publish. """
topic = publish_msg[0]
raw_message = publish_msg[1]
self.application.log('IN', 'TRADE_PUB', raw_message)
msg = JsonMessage(raw_message)
if msg.type == 'W': # Full Refresh
self.on_md_full_refresh(msg)
elif msg.type == 'X': # Incremental
self.on_md_incremental(msg)
def sendString(self, string_msg):
if not self.isConnected() and self.reopen:
self.connect()
self.trade_in_socket.send_unicode("REQ," + self.connection_id + ',' +
string_msg)
response_message = self.trade_in_socket.recv()
raw_resp_message_header = response_message[:3]
raw_resp_message = response_message[4:].strip()
rep_msg = None
if raw_resp_message:
try:
rep_msg = JsonMessage(raw_resp_message)
except Exception:
pass
if raw_resp_message_header == 'CLS' and rep_msg and not rep_msg.isErrorMessage(
):
self.close()
if self.reopen:
self.connect()
return rep_msg
if raw_resp_message_header != 'REP':
self.close()
if self.reopen:
self.connect()
if rep_msg and rep_msg.isErrorMessage():
raise TradeClientException(rep_msg.get('Description'),
rep_msg.get('Detail'))
raise TradeClientException('Invalid request: ' + raw_resp_message)
if rep_msg and rep_msg.isUserResponse():
if rep_msg.get("UserStatus") == 1:
self.user_id = rep_msg.get("UserID")
self.is_logged = True
if self.trade_pub_socket:
self.trade_pub_socket.setsockopt(zmq.SUBSCRIBE,
str(self.user_id))
return rep_msg
def on_message(self, raw_message):
if not self.application.session.is_active:
# in case of an error on last commit, let's just rollback it.
self.application.session.rollback()
msg = JsonMessage(raw_message)
if not msg.is_valid():
print 'Invalid message', raw_message
self.close()
return
if msg.type == '1': # TestRequest
self.application.replay_log.info('IN,' + raw_message)
# send the heart beat back
self.write_message('{"MsgType":"0", "TestReqID":"%s"}' %
msg.get("TestReqID"))
return
elif msg.type == 'V': # Market Data Request
self.application.replay_log.info('IN,' + raw_message)
req_id = msg.get('MDReqID')
if int(msg.get('SubscriptionRequestType')) == 0: # Snapshot
# Generate a FullRefresh
market_depth = msg.get('MarketDepth')
instruments = msg.get('Instruments')
entries = msg.get('MDEntryTypes')
for instrument in instruments:
om = OrderMatcher.get(instrument)
md = generate_md_full_refresh(self.application.session,
instrument, market_depth, om,
entries)
self.write_message(str(json.dumps(md, cls=JsonEncoder)))
elif int(msg.get(
'SubscriptionRequestType')) == 1: # Snapshot + Updates
if req_id not in self.md_subscriptions:
self.md_subscriptions[req_id] = []
market_depth = msg.get('MarketDepth')
instruments = msg.get('Instruments')
entries = msg.get('MDEntryTypes')
for instrument in instruments:
om = OrderMatcher.get(instrument)
md = generate_md_full_refresh(self.application.session,
instrument, market_depth, om,
entries)
self.write_message(str(json.dumps(md, cls=JsonEncoder)))
for entry in entries:
self.md_subscriptions[req_id].append(
MdSubscriptionHelper(
req_id, market_depth, entry, instrument,
self.on_send_json_msg_to_user))
elif int(msg.get('SubscriptionRequestType')
) == 2: # Disable previous Snapshot + Update Request
if req_id in self.md_subscriptions:
del self.md_subscriptions[req_id]
return
if not self.is_logged:
if msg.type == 'U10': # Request password request
self.application.replay_log.info('IN,' + raw_message)
user = User.get_user(self.application.session,
email=msg.get('Email'))
user.request_reset_password(self.application.session)
self.application.session.commit()
return
if msg.type == 'U12': # Password request
self.application.replay_log.info('IN,' + raw_message)
if UserPasswordReset.change_user_password(
self.application.session, msg.get('Token'),
msg.get('NewPassword')):
response = {
'MsgType': 'U13',
'UserStatus': 1,
'UserStatusText': u'Senha alterada com sucesso!'
}
self.write_message(json.dumps(response))
self.application.session.commit()
else:
response = {
'MsgType': 'U13',
'UserStatus': 3,
'UserStatusText': u'Código de segurança inválido!'
}
self.write_message(json.dumps(response))
self.close()
return
elif msg.type == 'U0': # signup
raw_message = raw_message.replace(msg.get('Password'), '*')
self.application.replay_log.info('IN,' + raw_message)
# check if the user is already registered
if User.get_user(self.application.session, msg.get('Username'),
msg.get('Email')):
login_response = {
'MsgType':
'BF',
'Username':
'',
'UserStatus':
3,
'UserStatusText':
u'Nome de usuário ou Email já estão registrados!'
}
self.write_message(json.dumps(login_response))
self.close()
self.application.session.rollback()
return
# signup the user
# create the user on Database
u = User(username=msg.get('Username'),
email=msg.get('Email'),
password=msg.get('Password'),
balance_btc=0,
balance_brl=0)
self.application.session.add(u)
self.application.session.commit()
# The logon message must be the first message
if msg.type != 'BE' and msg.type != 'U0':
self.close()
self.application.session.rollback()
return
raw_message = raw_message.replace(msg.get('Password'), '*')
self.application.replay_log.info('IN,' + raw_message)
# Authenticate the user
need_second_factor = False
self.user = None
try:
self.user = User.authenticate(self.application.session,
msg.get('Username'),
msg.get('Password'),
msg.get('SecondFactor'))
except NeedSecondFactorException:
need_second_factor = True
if not self.user:
login_response = {
'MsgType':
'BF',
'Username':
'',
'UserStatus':
3,
'NeedSecondFactor':
need_second_factor,
'UserStatusText':
u'Nome de usuário ou senha inválidos'
if not need_second_factor else
u'Segundo fator de autenticação inválido'
}
self.write_message(json.dumps(login_response))
self.application.session.rollback()
# TODO: improve security.
# Block the user accounts after 3 attempts
# close the all connections from the blocked user
# Block the ip for 24hs
self.close()
return
self.is_logged = True
# Send the login response
login_response = {
'MsgType': 'BF',
'UserID': self.user.id,
'Username': self.user.username,
'TwoFactorEnabled': self.user.two_factor_enabled,
'BtcAddress': self.user.bitcoin_address,
'UserStatus': 1
}
self.write_message(json.dumps(login_response))
self.application.session.add(self.user)
self.application.session.commit()
# subscribe to all execution reports for this user account.
execution_report_signal.connect(self.on_execution_report,
self.user.id)
# subscribe to balance updates for this user account
balance_signal.connect(self.on_send_json_msg_to_user, self.user.id)
# add the user to the session/
self.application.session.add(self.user)
self.application.session.commit()
self.user.publish_balance_update()
return
# The user is logged
self.application.replay_log.info('IN,' + raw_message)
# handle system messages
if self.user.is_system:
if self._handle_system_messages(msg):
return
# handle staff messages
if self.user.is_staff:
if self._handle_staff_messages(msg):
return
# handle all other messages
if msg.type == 'D': # New Order Single
# process the new order.
order = Order(user_id=self.user.id,
account_id=self.user.account_id,
user=self.user,
username=self.user.username,
client_order_id=msg.get('ClOrdID'),
symbol=msg.get('Symbol'),
side=msg.get('Side'),
type=msg.get('OrdType'),
price=msg.get('Price'),
order_qty=msg.get('OrderQty'))
self.application.session.add(order)
self.application.session.flush(
) # just to assign an ID for the order.
OrderMatcher.get(msg.get('Symbol')).match(self.application.session,
order)
self.application.session.commit()
return
elif msg.type == 'F': # Cancel Order Request
order_list = []
if msg.has('OrigClOrdID'):
order = self.application.session.query(Order).\
filter(Order.status.in_(("0", "1"))).\
filter_by( user_id = self.user.id ).\
filter_by( client_order_id = msg.get('OrigClOrdID') ).first()
if order:
order_list.append(order)
elif msg.has('OrderID'):
order = self.application.session.query(Order).\
filter(Order.status.in_(("0", "1"))).\
filter_by( user_id = self.user.id ).\
filter_by( id = msg.get('OrderID') ).first()
if order:
order_list.append(order)
else:
orders = self.application.session.query(Order).\
filter(Order.status.in_(("0", "1"))).\
filter_by( user_id = self.user.id )
for order in orders:
order_list.append(order)
for order in order_list:
OrderMatcher.get(order.symbol).cancel(self.application.session,
order)
self.application.session.commit()
elif msg.type == 'U2': # Request for Balances
self.user.publish_balance_update(msg.get('BalanceReqID'))
return
elif msg.type == 'U4': # Request for Open Orders
page = msg.get('Page', 0)
page_size = msg.get('PageSize', 100)
status_list = msg.get('StatusList', ['0', '1'])
offset = page * page_size
orders = self.application.session.query(Order).\
filter(Order.status.in_( status_list )).\
filter_by( user_id = self.user.id ).\
order_by(Order.created.desc()).\
limit( page_size ).offset( offset )
order_list = []
columns = [
'ClOrdID', 'OrderID', 'CumQty', 'OrdStatus', 'LeavesQty',
'CxlQty', 'AvgPx', 'Symbol', 'Side', 'OrdType', 'OrderQty',
'Price', 'OrderDate', 'Volume'
]
for order in orders:
order_total_value = order.average_price * order.cum_qty
if order_total_value:
order_total_value /= 1.e8
order_list.append([
order.client_order_id, order.id, order.cum_qty,
order.status, order.leaves_qty, order.cxl_qty,
order.average_price, order.symbol, order.side, order.type,
order.order_qty, order.price, order.created,
order_total_value
])
open_orders_response_msg = {
'MsgType': 'U5',
'OrdersReqID': msg.get('OrdersReqID'),
'Page': page,
'PageSize': page_size,
'Columns': columns,
'OrdListGrp': order_list
}
self.write_message(
str(json.dumps(open_orders_response_msg, cls=JsonEncoder)))
return
elif msg.type == 'U6': # BTC Withdraw Request
self.user.withdraw_btc(session=self.application.session,
amount=msg.get('Amount'),
wallet=msg.get('Wallet'))
self.application.session.commit()
return
elif msg.type == 'U8': # BRL Withdraw Request
self.user.withdraw_brl(session=self.application.session,
amount=msg.get('Amount'),
bank_number=msg.get('BankNumber'),
bank_name=msg.get('BankName'),
account_name=msg.get('AccountName'),
account_number=msg.get('AccountNumber'),
account_branch=msg.get('AccountBranch'),
cpf_cnpj=msg.get('CPFCNPJ'))
self.application.session.commit()
return
elif msg.type == 'U9': # gets or create an bitcoin address for UserID
user = self.application.session.query(User).filter_by(
id=msg.get('UserID')).first()
if user:
if user.bitcoin_address is None:
pass
#btc_address = self.application.bitcoin.getnewaddress()
#user.new_address(btc_address)
#self.application.session.add(user)
#self.application.session.commit()
self.on_send_json_msg_to_user(sender=None,
json_msg={
'MsgType':
'U14',
'NewBTCReqID':
msg.get('NewBTCReqID'),
'Address':
user.bitcoin_address
})
return
elif msg.type == 'U16': #Enable Disable Two Factor Authentication
enable = msg.get('Enable')
secret = msg.get('Secret')
code = msg.get('Code')
two_factor_secret = self.user.enable_two_factor(
enable, secret, code)
self.on_send_json_msg_to_user(sender=None,
json_msg={
'MsgType': 'U17',
'TwoFactorEnabled':
self.user.two_factor_enabled,
'TwoFactorSecret':
two_factor_secret
})
self.application.session.add(self.user)
self.application.session.commit()
elif msg.type == 'U18': #Generate Boleto
boleto_option_id = msg.get('BoletoId')
value = msg.get('Value')
boleto_option = self.application.session.query(
BoletoOptions).filter_by(id=boleto_option_id).first()
if not boleto_option:
print 'boleto_option not found'
return
boleto = boleto_option.generate_boleto(self.application.session,
self.user, value)
self.application.session.commit()
print boleto.id
self.on_send_json_msg_to_user(sender=None,
json_msg={
'MsgType': 'U19',
'BoletoId': boleto.id
})
else:
print 'Invalid Message', msg
# invalid message - Close the connection ....
self.close()
def run(self):
from bitex.message import JsonMessage, InvalidMessageException
from market_data_publisher import MarketDataPublisher
from execution import OrderMatcher
from models import Order
orders = self.db_session.query(Order).filter(
Order.status.in_(("0", "1"))).order_by(Order.created)
for order in orders:
OrderMatcher.get(order.symbol).match(self.db_session, order)
while True:
raw_message = self.input_socket.recv()
msg_header = raw_message[:3]
session_id = raw_message[4:20]
json_raw_message = raw_message[21:].strip()
try:
msg = None
if json_raw_message:
try:
msg = JsonMessage(json_raw_message)
except InvalidMessageException, e:
self.log('IN', 'TRADE_IN_REQ_ERROR', raw_message)
raise InvalidMessageError()
# never write passwords in the log file
if msg.has('Password'):
raw_message = raw_message.replace(
msg.get('Password'), '*')
if msg.has('NewPassword'):
raw_message = raw_message.replace(
msg.get('NewPassword'), '*')
self.log('IN', 'TRADE_IN_REQ', raw_message)
if msg:
if msg.isMarketDataRequest(): # Market Data Request
req_id = msg.get('MDReqID')
market_depth = msg.get('MarketDepth')
instruments = msg.get('Instruments')
entries = msg.get('MDEntryTypes')
transact_time = msg.get('TransactTime')
timestamp = None
if transact_time:
timestamp = transact_time
else:
trade_date = msg.get('TradeDate')
if not trade_date:
trade_date = time.strftime(
"%Y%m%d", time.localtime())
self.log('OUT', 'TRADEDATE', trade_date)
timestamp = datetime.datetime.strptime(
trade_date, "%Y%m%d")
self.log('OUT', 'TIMESTAMP', timestamp)
if len(instruments) > 1:
raise InvalidMessageError()
instrument = instruments[0]
om = OrderMatcher.get(instrument)
response_message = MarketDataPublisher.generate_md_full_refresh(
application.db_session, instrument, market_depth,
om, entries, req_id, timestamp)
response_message = 'REP,' + json.dumps(
response_message, cls=JsonEncoder)
elif msg.isTradeHistoryRequest():
page = msg.get('Page', 0)
page_size = msg.get('PageSize', 100)
offset = page * page_size
columns = [
'TradeID', 'Market', 'Side', 'Price', 'Size',
'Buyer', 'Seller', 'Created'
]
trade_list = MarketDataPublisher.generate_trade_history(
application.db_session, page_size, offset)
response_message = 'REP,' + json.dumps(
{
'MsgType': 'U33', # TradeHistoryResponse
'TradeHistoryReqID': -1,
'Page': page,
'PageSize': page_size,
'Columns': columns,
'TradeHistoryGrp': trade_list
},
cls=JsonEncoder)
else:
response_message = self.session_manager.process_message(
msg_header, session_id, msg)
else:
response_message = self.session_manager.process_message(
msg_header, session_id, msg)
except TradeRuntimeError, e:
self.db_session.rollback()
self.session_manager.close_session(session_id)
response_message = 'ERR,{"MsgType":"ERROR", "Description":"' + e.error_description.replace(
"'", "") + '", "Detail": ""}'
def main():
tornado.options.parse_command_line()
if not options.trade_pub or\
not options.mailchimp_apikey or\
not options.mailchimp_newsletter_list_id or\
not options.mandrill_apikey or\
not options.mailer_log :
tornado.options.print_help()
return
mailchimp_api = mailchimp.Mailchimp(options.mailchimp_apikey)
try:
mailchimp_api.helper.ping()
except mailchimp.Error:
print "Invalid MailChimp API key"
return
mandrill_api = mandrill.Mandrill(options.mandrill_apikey)
try:
mandrill_api.users.ping()
except mandrill.Error:
print "Invalid Mandrill API key"
return
print mandrill_api.users.senders()
input_log_file_handler = logging.handlers.TimedRotatingFileHandler(
options.mailer_log, when='MIDNIGHT')
formatter = logging.Formatter(u"%(asctime)s - %(message)s")
input_log_file_handler.setFormatter(formatter)
mail_logger = logging.getLogger("REPLAY")
mail_logger.setLevel(logging.INFO)
mail_logger.addHandler(input_log_file_handler)
mail_logger.info('START')
def log(command, key, value=None):
log_msg = u'%s, %s, %s' % (command, key, value if value else None)
mail_logger.info(unicode(log_msg))
pass
log('PARAM', 'BEGIN')
log('PARAM', 'trade_pub', options.trade_pub)
log('PARAM', 'mailer_log', options.mailer_log)
log('PARAM', 'END')
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect(options.trade_pub)
socket.setsockopt(zmq.SUBSCRIBE, "EMAIL")
while True:
try:
raw_email_message = socket.recv()
log('IN', 'TRADE_IN_PUB', raw_email_message)
msg = JsonMessage(raw_email_message)
if not msg.isEmail():
log('ERROR', 'EXCEPTION',
'Received message is not an email message')
continue
try:
sender = u'BitEx Support <*****@*****.**>'
body = ""
msg_to = msg.get('To')
subject = msg.get('Subject')
language = msg.get('Language')
content_type = 'plain'
if msg.has('Template') and msg.get('Template'):
params = {}
if msg.has('Params') and msg.get('Params'):
params = json.loads(msg.get('Params'))
template_name = msg.get('Template')
if template_name == 'welcome':
# user signup .... let's register him on mailchimp newsletter
try:
mailchimp_api.lists.subscribe(
id=options.mailchimp_newsletter_list_id,
email={'email': params['email']},
merge_vars={
'EMAIL': params['email'],
'FNAME': params['username']
})
except mailchimp.ListAlreadySubscribedError:
log(
'ERROR', 'EXCEPTION', params['email'] +
' mailchimp.ListAlreadySubscribedError')
except mailchimp.Error, e:
log('ERROR', 'EXCEPTION', str(e))
template_content = []
for k, v in params.iteritems():
template_content.append({'name': k, 'content': v})
message = {
'to': [{
'email': msg_to,
'name': params['username'],
'type': 'to'
}],
'metadata': {
'website': 'www.bitex.com.br'
},
'global_merge_vars':
template_content
}
result = mandrill_api.messages.send_template(
template_name=(template_name + '-' + language).lower(),
template_content=template_content,
message=message)
log('INFO', 'SUCCESS', str(result))
continue
elif msg.has('RawData') and msg.get('RawData'):
body = msg.get('RawData')
log('IN', 'LOGINDEBUG START', "")
log(
'DEBUG', 'EMAIL',
u'{"Sender":"%s","To":"%s","Subject":"%s", "Body":"%s" }' %
(sender, msg_to, subject, body))
log('IN', 'LOGINDEBUG END', "")
send_email(sender, msg_to, subject, body, content_type)
log('IN', 'SENT', "")
log('INFO', 'SUCCESS', msg.get('EmailThreadID'))
except Exception as ex:
traceback.print_exc()
log('ERROR', 'EXCEPTION', str(ex))
time.sleep(1)
def on_message(self, raw_message):
if not self.trade_client.isConnected():
return
try:
req_msg = JsonMessage(raw_message)
except InvalidMessageException as e:
self.write_message(
'{"MsgType":"ERROR", "Description":"Invalid message", "Detail": "'
+ str(e) + '"}')
self.application.unregister_connection(self)
self.trade_client.close()
self.close()
return
if req_msg.isUserRequest():
if req_msg.has('Password'):
raw_message = raw_message.replace(req_msg.get('Password'), '*')
if req_msg.has('NewPassword'):
raw_message = raw_message.replace(req_msg.get('NewPassword'),
'*')
self.application.log('IN', self.trade_client.connection_id,
raw_message)
else:
self.application.log('IN', self.trade_client.connection_id,
raw_message)
if req_msg.isTestRequest() or req_msg.isHeartbeat():
dt = datetime.datetime.now()
response_msg = {
'MsgType':
'0',
'TestReqID':
req_msg.get('TestReqID'),
'ServerTimestamp':
int(mktime(dt.timetuple()) + dt.microsecond / 1000.0)
}
sendTime = req_msg.get('SendTime')
if sendTime:
response_msg['SendTime'] = sendTime
self.write_message(str(json.dumps(response_msg, cls=JsonEncoder)))
return
if req_msg.isTradeHistoryRequest(): # Trade History request
self.on_trade_history_request(req_msg)
return
if req_msg.isMarketDataRequest(): # Market Data Request
self.on_market_data_request(req_msg)
if not self.trade_client.isConnected():
self.application.log('DEBUG', self.trade_client.connection_id,
'not self.trade_client.isConnected()')
self.application.unregister_connection(self)
self.trade_client.close()
self.close()
return
if req_msg.isSecurityStatusRequest():
self.on_security_status_request(req_msg)
return
if req_msg.isDepositRequest():
if not req_msg.get('DepositMethodID') and not req_msg.get(
'DepositID'):
currency = req_msg.get('Currency')
secret = uuid.uuid4().hex
cold_wallet = self.get_broker_wallet('cold', currency)
callback_url = options.callback_url + secret
if not cold_wallet:
return
parameters = urllib.urlencode({
'method': 'create',
'address': cold_wallet,
'callback': callback_url,
'currency': currency
})
try:
url_payment_processor = options.url_payment_processor + '?' + parameters
print "invoking .. ", url_payment_processor
response = urllib2.urlopen(url_payment_processor)
data = json.load(response)
req_msg.set('InputAddress', data['input_address'])
req_msg.set('Destination', data['destination'])
req_msg.set('Secret', secret)
except urllib2.HTTPError as e:
self.write_message(
json.dumps({
'MsgType': 'ERROR',
'ReqID': req_msg.get('DepositReqID'),
'Description':
'Blockchain.info is not available at this moment, please try again within few minutes',
'Detail': str(e)
}))
return
except Exception as e:
self.write_message(
json.dumps({
'MsgType': 'ERROR',
'ReqID': req_msg.get('DepositReqID'),
'Description':
'Error retrieving a new deposit address from Blockchain.info. Please, try again',
'Detail': str(e)
}))
return
try:
resp_message = self.trade_client.sendMessage(req_msg)
if resp_message:
self.write_message(resp_message.raw_message)
if resp_message and resp_message.isUserResponse():
self.user_response = resp_message
if not self.trade_client.isConnected():
self.application.log('DEBUG', self.trade_client.connection_id,
'not self.trade_client.isConnected()')
self.application.unregister_connection(self)
self.trade_client.close()
self.close()
except TradeClientException as e:
exception_message = {
'MsgType': 'ERROR',
'Description': 'Invalid message',
'Detail': str(e)
}
self.write_message(json.dumps(exception_message))
self.application.unregister_connection(self)
self.trade_client.close()
self.close()
