def __init__(self, engine, metadata, secure=False, ttl=30000, table_prefix=''):
SubscriptionManager.__init__(self, secure=secure, ttl=ttl)
self.engine = engine
self.metadata = metadata
self.table_prefix = table_prefix and "%s_" % table_prefix.rstrip('_') or table_prefix
self.mapTables()
def post(self, request, format='json'):
try:
braintree_user = SubscriptionManager.fetch_braintree_user(
request.user)
except BraintreeError as e:
return Response(
{
'errors': [
"User's account is not set up correctly. Please contact support"
]
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
if 'payment_method_nonce' not in request.data:
return Response({'errors': ["No payment details provided."]},
status=status.HTTP_400_BAD_REQUEST)
payment_method_nonce = request.data['payment_method_nonce']
try:
SubscriptionManager.change_payment_method(braintree_user,
payment_method_nonce)
return Response({}, status=status.HTTP_202_ACCEPTED)
except BraintreeError as e:
LOGGER.exception("Failed to change user %s payment method",
str(self.request.user))
return Response(
{
'errors':
["Could not change payment method! Please try again."]
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def __init__(self, engine, metadata, secure=False, ttl=30000, table_prefix=''):
SubscriptionManager.__init__(self, secure=secure, ttl=ttl)
self.engine = engine
self.metadata = metadata
self.table_prefix = table_prefix and "%s_" % table_prefix.rstrip('_') or table_prefix
self.mapTables()
def post(self, request, format='json'):
try:
braintree_user = SubscriptionManager.fetch_braintree_user(
request.user)
except BraintreeError as e:
return Response(
{
'errors':
["User not initialized properly. Please contact support"]
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
if braintree_user.payment_method_token is None or braintree_user.payment_method_token == "":
return Response(
{'errors': ["User must set up a payment method first"]},
status=status.HTTP_400_BAD_REQUEST)
try:
SubscriptionManager.subscribe(braintree_user)
LOGGER.info("New subscription created for user %s",
str(self.request.user))
return Response({}, status.HTTP_201_CREATED)
except BraintreeError as e:
LOGGER.error("Failed to subscribed user %s",
str(self.request.user))
return Response(
{
'errors':
["Could not create subscription. Please try again."]
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except Exception as e:
LOGGER.exception("Unknown exception in subscription creation.")
return Response({'errors': [e.message]},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def __init__(self,
secure=False,
ttl=30000,
mc_servers=['127.0.0.1:11211'],
mc_debug=0):
SubscriptionManager.__init__(self, secure=secure, ttl=ttl)
self.mc = self.createMcClient(mc_servers, mc_debug)
self._lock = memcache_manager.MemcacheMutex(self.mc)
def get(self, request, format='json'):
# First fetch the users braintree account
try:
braintree_user = SubscriptionManager.fetch_braintree_user(
request.user)
except BraintreeError:
return Response(
{
'errors': [
"User's account is not set up correctly. Please contact support"
]
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
subscription_info = {}
# First case to handle is when user does not have a subscription
if braintree_user.subscription_id == "" or braintree_user.subscription_id is None:
return Response({'user_not_subscribed': True},
status=status.HTTP_200_OK)
# Further cases will require info from braintree
try:
subscription_obj = SubscriptionManager.fetch_subscription_from_braintree(
braintree_user)
except BraintreeError:
LOGGER.exception("Could not fetch user info from braintree!")
return Response(
{'errors': ["Something went wrong. Please try again later"]},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
sub_status = SubscriptionManager.convert_subscription_status_to_string(
subscription_obj.status)
first_billing_date = subscription_obj.billing_period_start_date
next_billing_or_cancel_date = subscription_obj.billing_period_end_date
created_at = subscription_obj.created_at
price = subscription_obj.price
if braintree_user.pending_cancel:
sub_status = "Pending Cancellation"
subscription_info["status"] = sub_status
subscription_info["first_billing_date"] = first_billing_date
subscription_info[
"renewal_or_cancel_date"] = next_billing_or_cancel_date
subscription_info["date_of_creation"] = created_at
subscription_info["price"] = price
return Response(subscription_info, status=status.HTTP_200_OK)
def get(self, request, format='json'):
plan = SubscriptionManager.get_subscription_plan()
if plan is None:
return Response({'errors': ['No subscription plan found']},
status=status.HTTP_404_NOT_FOUND)
else:
return Response(plan, status=status.HTTP_200_OK)
def unSubscribe(self, connection_id, client_id, topic, sub_topic=None):
"""Remove a subscription from a topic."""
topic = SubscriptionManager.getTopicKey(topic, sub_topic)
key_name = GaeSubscription.getKeyName(connection_id, client_id, topic)
subscription = GaeSubscription.get_by_key_name(key_name)
db.delete(subscription)
def test_get_instrument(self):
config = SubscriptionManager(file_name)
instmts = dict()
for instmt_id in config.get_instmt_ids():
instmts[instmt_id] = config.get_instrument(instmt_id)
# BTCC-BTCCNY
name = 'BTCC-BTCCNY-Restful'
self.assertEqual(instmts[name].get_exchange_name(), 'BTCC')
self.assertEqual(instmts[name].get_instmt_name(), 'BTCCNY')
self.assertEqual(instmts[name].get_instmt_code(), 'btccny')
self.assertEqual(instmts[name].get_order_book_link(),
'https://data.btcchina.com/data/orderbook?limit=5&market=btccny')
self.assertEqual(instmts[name].get_trades_link(),
'https://data.btcchina.com/data/historydata?limit=1000&since=<id>&market=btccny')
m = instmts[name].get_order_book_fields_mapping()
self.assertEqual(m['date'], 'TIMESTAMP')
self.assertEqual(m['bids'], 'BIDS')
self.assertEqual(m['asks'], 'ASKS')
m = instmts[name].get_trades_fields_mapping()
self.assertEqual(m['date'], 'TIMESTAMP')
self.assertEqual(m['type'], 'TRADE_SIDE')
self.assertEqual(m['tid'], 'TRADE_ID')
self.assertEqual(m['price'], 'TRADE_PRICE')
self.assertEqual(m['amount'], 'TRADE_VOLUME')
# BTCC-XBTCNY
name = 'BTCC-XBTCNY-Restful'
self.assertEqual(instmts[name].get_exchange_name(), 'BTCC')
self.assertEqual(instmts[name].get_instmt_name(), 'XBTCNY')
self.assertEqual(instmts[name].get_instmt_code(), 'xbtcny')
self.assertEqual(instmts[name].get_order_book_link(),
'https://pro-data.btcc.com/data/pro/orderbook?limit=5&symbol=xbtcny')
self.assertEqual(instmts[name].get_trades_link(),
'https://pro-data.btcc.com/data/pro/historydata?limit=1000<id>&symbol=xbtcny')
m = instmts[name].get_order_book_fields_mapping()
self.assertEqual(m['date'], 'TIMESTAMP')
self.assertEqual(m['bids'], 'BIDS')
self.assertEqual(m['asks'], 'ASKS')
self.assertEqual(m['TIMESTAMP_OFFSET'], 1000)
m = instmts[name].get_trades_fields_mapping()
self.assertEqual(m['Timestamp'], 'TIMESTAMP')
self.assertEqual(m['Side'], 'TRADE_SIDE')
self.assertEqual(m['Id'], 'TRADE_ID')
self.assertEqual(m['Price'], 'TRADE_PRICE')
self.assertEqual(m['Quantity'], 'TRADE_VOLUME')
def subscribe(self, connection_id, client_id, topic, sub_topic=None, selector=None):
"""Add a subscription to a topic."""
topic = SubscriptionManager.getTopicKey(topic, sub_topic)
key_name = GaeSubscription.getKeyName(connection_id, client_id, topic)
subscription = GaeSubscription(key_name=key_name,
connection_id=connection_id, client_id=client_id, topic=topic)
subscription.put()
def unSubscribe(self, connection_id, client_id, topic, sub_topic=None):
"""Remove a subscription from a topic."""
topic = SubscriptionManager.getTopicKey(topic, sub_topic)
key_name = GaeSubscription.getKeyName(connection_id, client_id, topic)
subscription = GaeSubscription.get_by_key_name(key_name)
db.delete(subscription)
def post(self, request, format='json'):
try:
braintree_user = SubscriptionManager.fetch_braintree_user(
request.user)
except BraintreeError:
return Response(
{
'errors':
["User not initialized properly. Please contact support"]
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
try:
SubscriptionManager.renew(braintree_user)
LOGGER.info("Renewed subscription for user %s",
str(self.request.user))
return Response({}, status=status.HTTP_202_ACCEPTED)
except Exception as e:
LOGGER.exception("Unknown exception in subscription renewal.")
return Response({'errors': [e.message]},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def post(self, request, format='json'):
try:
braintree_user = SubscriptionManager.fetch_braintree_user(
request.user)
except BraintreeError:
return Response(
{
'errors':
["User not initialized properly. Please contact support"]
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
try:
SubscriptionManager.request_cancel(braintree_user)
return Response({}, status=status.HTTP_202_ACCEPTED)
except BraintreeError as e:
LOGGER.exception(
"Failed to move subscription for %s to pending cancel.",
str(self.request.user))
return Response({'errors': [e.message]},
status=status.HTTP_400_BAD_REQUEST)
def iterSubscribers(self, topic, sub_topic=None):
"""Iterate through all connection ids subscribed to a topic."""
topic = SubscriptionManager.getTopicKey(topic, sub_topic)
connection_ids = {} # Keep track of unique IDs.
query = GaeSubscription.all()
query.filter('topic = ', topic)
for subscription in query:
if subscription.connection_id in connection_ids:
continue
connection_ids[subscription.connection_id] = True
yield subscription.connection_id
def iterSubscribers(self, topic, sub_topic=None):
"""Iterate through all connection ids subscribed to a topic."""
topic = SubscriptionManager.getTopicKey(topic, sub_topic)
connection_ids = {} # Keep track of unique IDs.
query = GaeSubscription.all()
query.filter('topic = ', topic)
for subscription in query:
if subscription.connection_id in connection_ids:
continue
connection_ids[subscription.connection_id] = True
yield subscription.connection_id
def subscribe(self,
connection_id,
client_id,
topic,
sub_topic=None,
selector=None):
"""Add a subscription to a topic."""
topic = SubscriptionManager.getTopicKey(topic, sub_topic)
key_name = GaeSubscription.getKeyName(connection_id, client_id, topic)
subscription = GaeSubscription(key_name=key_name,
connection_id=connection_id,
client_id=client_id,
topic=topic)
subscription.put()
from candle_creator import *
from subscription_manager import SubscriptionManager
instmts = SubscriptionManager("subscriptions.ini").get_subscriptions()
myDB.connect()
for instmt in instmts:
manager = CandleManager(instmt.get_candle(), instmt.get_frequency())
if instmt.get_base() == 'candle':
print "Based on candle"
print "----------------"
creator = CandleCreator(myDB, manager, instmt.get_base_candle())
creator.runOnCandle()
else:
print "Based on trade"
print "----------------"
creator = CandleCreator(myDB, manager, instmt.get_trade())
creator.run()
def main():
parser = argparse.ArgumentParser(description='Bitcoin exchange market data feed handler.')
parser.add_argument('-instmts', action='store', help='Instrument subscription file.', default='subscriptions.ini')
parser.add_argument('-exchtime', action='store_true', help='Use exchange timestamp.')
parser.add_argument('-kdb', action='store_true', help='Use Kdb+ as database.')
parser.add_argument('-csv', action='store_true', help='Use csv file as database.')
parser.add_argument('-sqlite', action='store_true', help='Use SQLite database.')
parser.add_argument('-mysql', action='store_true', help='Use MySQL.')
parser.add_argument('-zmq', action='store_true', help='Use zmq publisher.')
parser.add_argument('-mysqldest', action='store', dest='mysqldest',
help='MySQL destination. Formatted as <name:pwd@host:port>',
default='')
parser.add_argument('-mysqlschema', action='store', dest='mysqlschema',
help='MySQL schema.',
default='')
parser.add_argument('-kdbdest', action='store', dest='kdbdest',
help='Kdb+ destination. Formatted as <host:port>',
default='')
parser.add_argument('-zmqdest', action='store', dest='zmqdest',
help='Zmq destination. For example \"tcp://127.0.0.1:3306\"',
default='')
parser.add_argument('-sqlitepath', action='store', dest='sqlitepath',
help='SQLite database path',
default='')
parser.add_argument('-csvpath', action='store', dest='csvpath',
help='Csv file path',
default='')
parser.add_argument('-output', action='store', dest='output',
help='Verbose output file path')
args = parser.parse_args()
Logger.init_log(args.output)
db_clients = []
is_database_defined = False
if args.sqlite:
db_client = SqliteClient()
db_client.connect(path=args.sqlitepath)
db_clients.append(db_client)
is_database_defined = True
if args.mysql:
db_client = MysqlClient()
mysqldest = args.mysqldest
logon_credential = mysqldest.split('@')[0]
connection = mysqldest.split('@')[1]
db_client.connect(host=connection.split(':')[0],
port=int(connection.split(':')[1]),
user=logon_credential.split(':')[0],
pwd=logon_credential.split(':')[1],
schema=args.mysqlschema)
db_clients.append(db_client)
is_database_defined = True
if args.csv:
if args.csvpath != '':
db_client = FileClient(dir=args.csvpath)
else:
db_client = FileClient()
db_clients.append(db_client)
is_database_defined = True
if args.kdb:
db_client = KdbPlusClient()
db_client.connect(host=args.kdbdest.split(':')[0], port=int(args.kdbdest.split(':')[1]))
db_clients.append(db_client)
is_database_defined = True
if args.zmq:
db_client = ZmqClient()
db_client.connect(addr=args.zmqdest)
db_clients.append(db_client)
is_database_defined = True
if not is_database_defined:
print('Error: Please define which database is used.')
parser.print_help()
sys.exit(1)
# Subscription instruments
if args.instmts is None or len(args.instmts) == 0:
print('Error: Please define the instrument subscription list. You can refer to subscriptions.ini.')
parser.print_help()
sys.exit(1)
# Use exchange timestamp rather than local timestamp
if args.exchtime:
ExchangeGateway.is_local_timestamp = False
# Initialize subscriptions
subscription_instmts = SubscriptionManager(args.instmts).get_subscriptions()
if len(subscription_instmts) == 0:
print('Error: No instrument is found in the subscription file. ' +
'Please check the file path and the content of the subscription file.')
parser.print_help()
sys.exit(1)
# Initialize snapshot destination
ExchangeGateway.init_snapshot_table(db_clients)
ExchangeGateway.init_detail_snapshot_table(db_clients)
Logger.info('[main]', 'Subscription file = %s' % args.instmts)
log_str = 'Exchange/Instrument/InstrumentCode:\n'
for instmt in subscription_instmts:
log_str += '%s/%s/%s\n' % (instmt.exchange_name, instmt.instmt_name, instmt.instmt_code)
Logger.info('[main]', log_str)
exch_gws = []
##exch_gws.append(ExchGwBtccSpot(db_clients))
##exch_gws.append(ExchGwBtccFuture(db_clients))
##exch_gws.append(ExchGwBitmex(db_clients))
##exch_gws.append(ExchGwBitfinex(db_clients))
##exch_gws.append(ExchGwOkCoin(db_clients))
exch_gws.append(ExchGwKraken(db_clients))
exch_gws.append(ExchGwGdax(db_clients))
exch_gws.append(ExchGwBitstamp(db_clients))
exch_gws.append(ExchGwBitflyer(db_clients))
##exch_gws.append(ExchGwHuoBi(db_clients))
exch_gws.append(ExchGwCoincheck(db_clients))
exch_gws.append(ExchGwCoinOne(db_clients))
##exch_gws.append(ExchGwGatecoin(db_clients))
##exch_gws.append(ExchGwQuoine(db_clients))
##exch_gws.append(ExchGwPoloniex(db_clients))
exch_gws.append(ExchGwBittrex(db_clients))
##exch_gws.append(ExchGwYunbi(db_clients)) #
##exch_gws.append(ExchGwLiqui(db_clients)) #
##exch_gws.append(ExchGwBinance(db_clients)) #
exch_gws.append(ExchGwCryptopia(db_clients))
##exch_gws.append(ExchGwOkex(db_clients))
#exch_gws.append(ExchGwWex(db_clients)) # error
threads = []
for exch in exch_gws:
for instmt in subscription_instmts:
if instmt.get_exchange_name() == exch.get_exchange_name():
Logger.info("[main]", "Starting instrument %s-%s..." % \
(instmt.get_exchange_name(), instmt.get_instmt_name()))
threads += exch.start(instmt)
print(
'Error: Please define the instrument subscription list. You can refer to subscriptions.ini.'
)
parser.print_help()
sys.exit(1)
# Data mode
if args.mode is not None:
ExchangeGateway.data_mode = ExchangeGateway.DataMode.fromstring(
args.mode)
# Use exchange timestamp rather than local timestamp
if args.exchtime:
ExchangeGateway.is_local_timestamp = False
subscription_instmts = SubscriptionManager(
args.instmts).get_subscriptions()
ExchangeGateway.init_snapshot_table(ExchangeGateway.data_mode, db_client)
Logger.info('[main]', 'Current mode = %s' % args.mode)
Logger.info('[main]', 'Subscription file = %s' % args.instmts)
exch_gws = []
exch_gws.append(ExchGwBtccSpot(db_client))
exch_gws.append(ExchGwBtccFuture(db_client))
exch_gws.append(ExchGwBitmex(db_client))
exch_gws.append(ExchGwBitfinex(db_client))
exch_gws.append(ExchGwOkCoin(db_client))
exch_gws.append(ExchGwKraken(db_client))
exch_gws.append(ExchGwHuobi(db_client))
exch_gws.append(ExchGwGdax(db_client))
exch_gws.append(ExchGwBitstamp(db_client))
def __init__(self, secure=False, ttl=30000, mc_servers=['127.0.0.1:11211'], mc_debug=0):
SubscriptionManager.__init__(self, secure=secure, ttl=ttl)
self.mc = self.createMcClient(mc_servers, mc_debug)
self._lock = memcache_manager.MemcacheMutex(self.mc)
def main2(**kwargs):
output = kwargs['output']
mysqldest = kwargs['mysqldest']
mysql = kwargs['mysql']
mysqlschema = kwargs['mysqlschema']
instmts = kwargs['instmts']
exchtime = kwargs['exchtime']
Logger.init_log(output)
db_clients = []
is_database_defined = False
if mysql:
db_client = MysqlClient()
logon_credential = mysqldest.split('@')[0]
connection = mysqldest.split('@')[1]
db_client.connect(host=connection.split(':')[0],
port=int(connection.split(':')[1]),
user=logon_credential.split(':')[0],
pwd=logon_credential.split(':')[1],
schema=mysqlschema)
db_clients.append(db_client)
is_database_defined = True
if not is_database_defined:
print('Error: Please define which database is used.')
sys.exit(1)
# Subscription instruments
if instmts is None or len(instmts) == 0:
print(
'Error: Please define the instrument subscription list. You can refer to subscriptions.ini.'
)
sys.exit(1)
# Use exchange timestamp rather than local timestamp
if exchtime:
ExchangeGateway.is_local_timestamp = False
# Initialize subscriptions
subscription_instmts = SubscriptionManager(instmts).get_subscriptions()
if len(subscription_instmts) == 0:
print(
'Error: No instrument is found in the subscription file. ' +
'Please check the file path and the content of the subscription file.'
)
sys.exit(1)
# Initialize snapshot destination
ExchangeGateway.init_snapshot_table(db_clients)
Logger.info('[main]', 'Subscription file = %s' % instmts)
log_str = 'Exchange/Instrument/InstrumentCode:\n'
for instmt in subscription_instmts:
log_str += '%s/%s/%s\n' % (instmt.exchange_name, instmt.instmt_name,
instmt.instmt_code)
Logger.info('[main]', log_str)
exch_gws = []
exch_gws.append(ExchGwBitfinex(db_clients))
threads = []
for exch in exch_gws:
for instmt in subscription_instmts:
if instmt.get_exchange_name() == exch.get_exchange_name():
Logger.info("[main]", "Starting instrument %s-%s..." % \
(instmt.get_exchange_name(), instmt.get_instmt_name()))
threads += exch.start(instmt)
