def sign_message_with_transaction(transaction, key):
"""
Signs a transaction message or transaction
:param transaction dict:
:param key str: A signing key
returns message, txnid tuple: The first 16 characters
of a sha256 hexdigest.
"""
transaction['Nonce'] = time.time()
pub = pybitcointools.encode_pubkey(pybitcointools.privkey_to_pubkey(key),
"hex")
transaction["public_key"] = pub
sig = pybitcointools.ecdsa_sign(dict2cbor(transaction), key)
transaction['Signature'] = sig
txnid = hashlib.sha256(transaction['Signature']).hexdigest()[:16]
message = {
'Transaction': transaction,
'__TYPE__': "/mktplace.transactions.MarketPlace/Transaction",
'__NONCE__': time.time(),
}
cbor_serialized_mess = dict2cbor(message)
signature = pybitcointools.ecdsa_sign(cbor_serialized_mess, key)
message['__SIGNATURE__'] = signature
return message, txnid
def sign_message_with_transaction(transaction, key):
"""
Signs a transaction message or transaction
:param transaction dict:
:param key str: A signing key
returns message, txnid tuple: The first 16 characters
of a sha256 hexdigest.
"""
transaction['Nonce'] = time.time()
pub = pybitcointools.encode_pubkey(pybitcointools.privkey_to_pubkey(key),
"hex")
transaction["public_key"] = pub
sig = pybitcointools.ecdsa_sign(
dict2cbor(transaction),
key
)
transaction['Signature'] = sig
txnid = hashlib.sha256(transaction['Signature']).hexdigest()[:16]
message = {
'Transaction': transaction,
'__TYPE__': "/mktplace.transactions.MarketPlace/Transaction",
'__NONCE__': time.time(),
}
cbor_serialized_mess = dict2cbor(message)
signature = pybitcointools.ecdsa_sign(
cbor_serialized_mess,
key
)
message['__SIGNATURE__'] = signature
return message, txnid
def create_order(self,
action,
order_type,
firm_id,
quantity,
isin=None,
cusip=None,
limit_price=None,
limit_yield=None,
object_id=None):
update = {
'UpdateType': 'CreateOrder',
'Action': action,
'OrderType': order_type,
'FirmId': firm_id,
'Quantity': quantity,
'Nonce': create_nonce()
}
possible_updates = {
'Isin': isin,
'Cusip': cusip,
'LimitPrice': limit_price,
'LimitYield': limit_yield,
'ObjectId': object_id
}
for k, v in possible_updates.iteritems():
if v is not None:
update[k] = v
if "ObjectId" not in update:
update["ObjectId"] = hashlib.sha256(dict2cbor(update)).hexdigest()
return self.send_bond_update(update)
def do_load(args, config):
url = config.get('DEFAULT', 'url')
key_file = config.get('DEFAULT', 'key_file')
client = BondClient(base_url=url, keyfile=key_file)
for filename in args.filename:
try:
with open(filename) as fd:
data = yaml.load(fd)
except IOError, ioe:
raise ClientException("IOError: {}".format(str(ioe)))
print "Loading file: {}".format(filename)
with UpdateBatch(client):
for i in xrange(0, len(data['Transactions'])):
for j in xrange(0, len(data['Transactions'][i]["Updates"])):
update = data['Transactions'][i]["Updates"][j]
if "ObjectId" not in update:
update["ObjectId"] = \
hashlib.sha256(dict2cbor(update)).hexdigest()
print "Sending transaction {}, {}...".format(i, j)
print "Update ", update
client.send_bond_update(update)
if args.wait:
client.wait_for_commit()
def create_quote(self,
firm,
ask_qty,
ask_price,
bid_qty,
bid_price,
isin=None,
cusip=None,
object_id=None):
update = {
'UpdateType': 'CreateQuote',
'Firm': firm,
'AskQty': ask_qty,
'AskPrice': ask_price,
'BidQty': bid_qty,
'BidPrice': bid_price,
'Nonce': create_nonce()
}
if isin is not None:
update['Isin'] = isin
if cusip is not None:
update['Cusip'] = cusip
if object_id is not None:
update['ObjectId'] = object_id
if "ObjectId" not in update:
update["ObjectId"] = hashlib.sha256(dict2cbor(update)).hexdigest()
return self.send_bond_update(update)
def add_authorization_to_org(self, object_id, role, participant_id=None):
"""
Add an authorization (participant/role combination) to an
organization.
Args:
object_id: The object ID of the organization
role: (str) "marketmaker" or "trader"
participant_id: The participant ID that grants the authorization
or None to indicate the participant that created
the organization.
Returns:
Transaction ID
"""
update = {
'UpdateType': 'UpdateOrganizationAuthorization',
'ObjectId': object_id,
'Action': 'add',
'Role': role,
'Nonce': create_nonce()
}
if participant_id is not None:
update['ParticipantId'] = participant_id
if "ObjectId" not in update:
update["ObjectId"] = hashlib.sha256(dict2cbor(update)).hexdigest()
return self.send_bond_update(update)
def create_org(self,
name,
object_id=None,
industry=None,
ticker=None,
pricing_src=None,
authorization=None):
update = {
'UpdateType': 'CreateOrganization',
'Name': name,
'Nonce': create_nonce()
}
obj = {
'ObjectId': object_id,
'Industry': industry,
'Ticker': ticker,
'PricingSource': pricing_src,
'Authorization': authorization
}
for k, v in obj.iteritems():
if v is not None:
update[k] = v
if "ObjectId" not in update:
update["ObjectId"] = hashlib.sha256(dict2cbor(update)).hexdigest()
return self.send_bond_update(update)
def do_post(self, request):
"""
Handle two types of HTTP POST requests:
- gossip messages. relayed to the gossip network as is
- validator command and control (/command)
"""
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
try:
response = self.render_post(request, components, request.args)
encoding = request.getHeader('Content-Type')
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response)
else:
result = dict2cbor(response)
return result
except Exception as e:
LOGGER.warn('error processing http request %s; %s; %s',
request.path,
str(e),
traceback.format_exc(20))
return self._encode_error_response(
request,
http.INTERNAL_SERVER_ERROR,
e)
def do_post(self, request):
"""
Handle two types of HTTP POST requests:
- gossip messages. relayed to the gossip network as is
- validator command and control (/command)
"""
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
try:
response = self.render_post(request, components, request.args)
encoding = request.getHeader('Content-Type')
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response)
else:
result = dict2cbor(response)
return result
except Exception as e:
LOGGER.warn('error processing http request %s; %s; %s',
request.path, str(e), traceback.format_exc(20))
return self._encode_error_response(request,
http.INTERNAL_SERVER_ERROR, e)
def render_GET(self, request):
"""
Handle a GET request on the HTTP interface. Three paths are accepted:
/store[/<storename>[/<key>|*]]
/block[/<blockid>]
/transaction[/<txnid>]
"""
# pylint: disable=invalid-name
# split the request path removing leading duplicate slashes
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.GetPageMap:
return self.error_response(request, http.BAD_REQUEST,
'unknown request {0}', request.path)
testonly = (request.method == 'HEAD')
try:
response = self.GetPageMap[prefix](components, request.args,
testonly)
if testonly:
return ''
cbor = (request.getHeader('Accept') == 'application/cbor')
if cbor:
request.responseHeaders.addRawHeader(b"content-type",
b"application/cbor")
return dict2cbor(response)
request.responseHeaders.addRawHeader(b"content-type",
b"application/json")
pretty = 'p' in request.args
if pretty:
result = pretty_print_dict(response) + '\n'
else:
result = dict2json(response)
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing http request {0}; {1}',
request.path, str(e))
except:
logger.warn('error processing http request %s; %s', request.path,
traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def do_get(self, request):
"""
Handle a GET request on the HTTP interface. Three paths are accepted:
/store[/<storename>[/<key>|*]]
/block[/<blockid>]
/transaction[/<txnid>]
"""
# pylint: disable=invalid-name
# split the request path removing leading duplicate slashes
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.GetPageMap:
# attempt to serve static content if present.
resource = self.static_content.getChild(request.path[1:], request)
return resource.render(request)
test_only = (request.method == 'HEAD')
try:
response = self.GetPageMap[prefix](components, request.args,
test_only)
if test_only:
return ''
cbor = (request.getHeader('Accept') == 'application/cbor')
if cbor:
request.responseHeaders.addRawHeader(b"content-type",
b"application/cbor")
return dict2cbor(response)
request.responseHeaders.addRawHeader(b"content-type",
b"application/json")
pretty = 'p' in request.args
if pretty:
result = pretty_print_dict(response) + '\n'
else:
result = dict2json(response)
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing http request {0}; {1}',
request.path, str(e))
except:
logger.warn('error processing http request %s; %s', request.path,
traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def do_get(self, request):
"""
Handle a GET request on the HTTP interface. Three paths are accepted:
/store[/<storename>[/<key>|*]]
/block[/<blockid>]
/transaction[/<txnid>]
"""
# pylint: disable=invalid-name
# split the request path removing leading duplicate slashes
LOGGER.info("%s.do_get %s", self.__class__.__name__, request.path)
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
if components[0] != self.page_name:
return self.error_response(request, http.NOT_FOUND,
"Invalid page name: {}", request.path)
else:
components.pop(0)
test_only = (request.method == 'HEAD')
try:
response = self.render_get(request, components, request.args)
if test_only:
return ''
cbor = (request.getHeader('Accept') == 'application/cbor')
if cbor:
request.responseHeaders.addRawHeader(b"content-type",
b"application/cbor")
return dict2cbor(response)
request.responseHeaders.addRawHeader(b"content-type",
b"application/json")
pretty = False
if 'p' in request.args:
pretty = request.args['p'] == ['1']
if pretty:
result = pretty_print_dict(response) + '\n'
else:
result = dict2json(response)
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing http request {0}; {1}',
request.path, str(e))
except:
LOGGER.warn('error processing http request %s; %s', request.path,
traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def create_participant(self, username, firm_id=None, object_id=None):
update = {'UpdateType': 'CreateParticipant', 'Username': username}
if firm_id is not None:
update['FirmId'] = firm_id
if object_id is not None:
update['ObjectId'] = object_id
else:
update['ObjectId'] = hashlib.sha256(dict2cbor(update)).hexdigest()
return self.send_bond_update(update)
def postmsg(self, msgtype, info):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
logger.debug(dict2json(info))
data = dict2cbor(info)
datalen = len(data)
url = self.BaseURL + msgtype
logger.debug(
'post transaction to %s with DATALEN=%d, '
'base64(DATA)=<%s>', url, datalen, base64.b64encode(data))
try:
request = urllib2.Request(url, data, {
'Content-Type': 'application/cbor',
'Content-Length': datalen
})
opener = urllib2.build_opener(self.ProxyHandler)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
logger.warn('operation failed with response: %s', err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
logger.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
logger.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
logger.info('server responds with message %s of type %s', content,
encoding)
return None
logger.debug(pretty_print_dict(value))
return value
def test_message_len(self):
# Test the overriden len function
# First case is when the msg has no data, creates data
msg = Message({'__SIGNATURE__': "Test"})
length = len(dict2cbor(msg.dump()))
self.assertIsNone(msg._data)
self.assertEquals(len(msg), length)
# The second case is when the msg does have data
msg2 = Message({"__SIGNATURE__": "Test"})
msg2._data = "test data"
self.assertEquals(len(msg2), len("test data"))
def create_settlement(self, order_id, object_id=None):
update = {
'UpdateType': 'CreateSettlement',
'OrderId': order_id,
'Nonce': create_nonce()
}
if object_id is not None:
update['ObjectId'] = object_id
if "ObjectId" not in update:
update["ObjectId"] = hashlib.sha256(dict2cbor(update)).hexdigest()
return self.send_bond_update(update)
def do_get(self, request):
"""
Handle a GET request on the HTTP interface. Three paths are accepted:
/store[/<storename>[/<key>|*]]
/block[/<blockid>]
/transaction[/<txnid>]
"""
# pylint: disable=invalid-name
# split the request path removing leading duplicate slashes
LOGGER.info("%s.do_get %s", self.__class__.__name__, request.path)
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
if components[0] != self.page_name:
return self._error_response(
request,
http.NOT_FOUND,
"Invalid page name: {}", request.path)
else:
components.pop(0)
test_only = (request.method == 'HEAD')
try:
response = self.render_get(request, components, request.args)
if test_only:
return ''
cbor = (request.getHeader('Accept') == 'application/cbor')
if cbor:
request.responseHeaders.addRawHeader(b"content-type",
b"application/cbor")
result = dict2cbor(response)
else:
request.responseHeaders.addRawHeader(b"content-type",
b"application/json")
pretty = False
if 'p' in request.args:
pretty = request.args['p'] == ['1']
if pretty:
result = pretty_print_dict(response) + '\n'
else:
result = dict2json(response)
return result
except Exception as e:
LOGGER.warn('error processing http request %s; %s', request.path,
traceback.format_exc(20))
return self._encode_error_response(
request,
http.INTERNAL_SERVER_ERROR,
e)
def test_message_len(self):
# Test the overriden len function
# First case is when the msg has no data, creates data
msg = Message({'__SIGNATURE__': "Test"})
length = len(dict2cbor(msg.dump()))
self.assertIsNone(msg._data)
self.assertEqual(len(msg), length)
# The second case is when the msg does have data
msg2 = Message({"__SIGNATURE__": "Test"})
msg2._data = "test data"
self.assertEqual(len(msg2), len("test data"))
def postmsg(self, msgtype, info):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
logger.debug(dict2json(info))
data = dict2cbor(info)
datalen = len(data)
url = self.BaseURL + msgtype
logger.debug('post transaction to %s with DATALEN=%d, '
'base64(DATA)=<%s>', url, datalen, base64.b64encode(data))
try:
request = urllib2.Request(url, data,
{'Content-Type': 'application/cbor',
'Content-Length': datalen})
opener = urllib2.build_opener(self.ProxyHandler)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
logger.warn('operation failed with response: %s', err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
logger.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
logger.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
logger.info('server responds with message %s of type %s', content,
encoding)
return None
logger.debug(pretty_print_dict(value))
return value
def serialize(self, signable=False):
"""Generates a CBOR serialized dict containing the a SignatureKey
to Signature mapping.
Args:
signable (bool): if signable is True, self.SignatureKey is
removed from the dict prior to serialization to CBOR.
Returns:
bytes: a CBOR representation of a SignatureKey to Signature
mapping.
"""
dump = self.dump()
if signable and self.SignatureKey in dump:
del dump[self.SignatureKey]
return dict2cbor(dump)
def serialize(self, signable=False):
"""Generates a CBOR serialized dict containing the a SignatureKey
to Signature mapping.
Args:
signable (bool): if signable is True, self.SignatureKey is
removed from the dict prior to serialization to CBOR.
Returns:
bytes: a CBOR representation of a SignatureKey to Signature
mapping.
"""
dump = self.dump()
if signable and self.SignatureKey in dump:
del dump[self.SignatureKey]
return dict2cbor(dump)
def create_holding(self,
owner_id,
asset_type,
asset_id,
amount,
object_id=None):
update = {
'UpdateType': 'CreateHolding',
'OwnerId': owner_id,
'AssetType': asset_type,
'AssetId': asset_id,
'Amount': amount,
'Nonce': create_nonce()
}
if object_id is not None:
update['ObjectId'] = object_id
if "ObjectId" not in update:
update["ObjectId"] = hashlib.sha256(dict2cbor(update)).hexdigest()
return self.send_bond_update(update)
def commit_block_store(self, blockid, blockstore):
"""Associates the blockstore with the blockid and commits
the blockstore to disk.
Marks the blockstore read only as part of the process.
Args:
blockid (str): The identifier to associate with the block.
blockstore (BlockStore): An initialized blockstore to be
used as the root store.
"""
# if we commit a block then we know that either this is the genesis
# block or that the previous block is committed already
assert blockstore.PreviousBlockID in self._persistmap
blockstore.commit_block(blockid)
self._blockmap[blockid] = blockstore
self._persistmap[blockid] = dict2cbor(blockstore.dump_block(True))
self._persistmap.sync()
def commit_block_store(self, blockid, blockstore):
"""Associates the blockstore with the blockid and commits
the blockstore to disk.
Marks the blockstore read only as part of the process.
Args:
blockid (str): The identifier to associate with the block.
blockstore (global_store_manager.BlockStore): An initialized
blockstore to be used as the root store.
"""
# if we commit a block then we know that either this is the genesis
# block or that the previous block is committed already
assert blockstore.PreviousBlockID in self._persistmap
blockstore.commit_block(blockid)
self._blockmap[blockid] = blockstore
self._persistmap[blockid] = dict2cbor(blockstore.dump_block(True))
self._persistmap.sync()
def add_transaction_store(self, tname, tstore):
"""Registers a data store type with a particular transaction type.
Args:
tname (str): The name of the transaction type.
tstore (KeyValueStore): The data store to associate with the
transaction type.
"""
# we should really only be adding transaction stores to the
# root block, if this fails we need to think more about the
# initialization
assert len(self._blockmap) == 1
rootstore = self._blockmap[self.RootBlockID]
rootstore.add_transaction_store(tname, tstore)
rootstore.commit_block(self.RootBlockID)
self._blockmap[self.RootBlockID] = rootstore
self._persistmap[self.RootBlockID] = \
dict2cbor(rootstore.dump_block(True))
self._persistmap.sync()
def add_transaction_store(self, tname, tstore):
"""Registers a data store type with a particular transaction type.
Args:
tname (str): The name of the transaction type.
tstore (KeyValueStore): The data store to associate with the
transaction type.
"""
# we should really only be adding transaction stores to the
# root block, if this fails we need to think more about the
# initialization
assert len(self._blockmap) == 1
rootstore = self._blockmap[self.RootBlockID]
rootstore.add_transaction_store(tname, tstore)
rootstore.commit_block(self.RootBlockID)
self._blockmap[self.RootBlockID] = rootstore
self._persistmap[self.RootBlockID] = \
dict2cbor(rootstore.dump_block(True))
self._persistmap.sync()
def do_post(self, request):
"""
Handle two types of HTTP POST requests:
- gossip messages. relayed to the gossip network as is
- validator command and control (/command)
"""
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
try:
response = self.render_post(request, components, request.args)
encoding = request.getHeader('Content-Type')
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response.dump())
else:
result = dict2cbor(response.dump())
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}', request.path,
str(e))
except:
LOGGER.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def __init__(self, blockstorefile='blockstore', dbmode='c'):
"""Initialize a GlobalStoreManager, opening the database file.
Args:
blockstorefile (str): The name of the file to use for
persistent data.
dbmode (str): The mode used to open the file (see anydbm
parameters).
"""
logger.info('create blockstore from file %s with flag %s',
blockstorefile, dbmode)
self._blockmap = {}
self._persistmap = anydbm.open(blockstorefile, dbmode)
rootstore = BlockStore()
rootstore.commit_block(self.RootBlockID, )
self._blockmap[self.RootBlockID] = rootstore
self._persistmap[self.RootBlockID] = \
dict2cbor(rootstore.dump_block(True))
self._persistmap.sync()
logger.debug('the persistent block store has %s keys',
len(self._persistmap))
def do_post(self, request):
"""
Handle two types of HTTP POST requests:
- gossip messages. relayed to the gossip network as is
- validator command and control (/command)
"""
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
try:
response = self.render_post(request, components, request.args)
encoding = request.getHeader('Content-Type')
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response.dump())
else:
result = dict2cbor(response.dump())
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}',
request.path, str(e))
except:
LOGGER.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def __init__(self, blockstorefile='blockstore', dbmode='c'):
"""Initialize a GlobalStoreManager, opening the database file.
Args:
blockstorefile (str): The name of the file to use for
persistent data.
dbmode (str): The mode used to open the file (see anydbm
parameters).
"""
logger.info('create blockstore from file %s with flag %s',
blockstorefile, dbmode)
self._blockmap = {}
self._persistmap = anydbm.open(blockstorefile, dbmode)
rootstore = BlockStore()
rootstore.commit_block(self.RootBlockID, )
self._blockmap[self.RootBlockID] = rootstore
self._persistmap[self.RootBlockID] = \
dict2cbor(rootstore.dump_block(True))
self._persistmap.sync()
logger.debug('the persistent block store has %s keys',
len(self._persistmap))
def render_POST(self, request):
"""
Handle two types of HTTP POST requests:
- gossip messages. relayed to the gossip network as is
- validator command and control (/command)
"""
# pylint: disable=invalid-name
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.PostPageMap:
prefix = 'default'
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
# process non-gossip API requests
if prefix == 'command':
try:
if encoding == 'application/json':
minfo = json2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'bad message encoding, {0}',
encoding)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unable to decode incoming request {0}',
data)
# process /command
try:
response = self.PostPageMap[prefix](request, components, minfo)
request.responseHeaders.addRawHeader("content-type", encoding)
result = dict2json(response)
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}',
request.path, str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
else:
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'unknown message encoding, {0}',
encoding)
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
return self.error_response(
request, http.BAD_REQUEST,
'received request for unknown message type, {0}',
typename)
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unabled to decode incoming request {0}',
data)
# determine if the message contains a valid transaction before
# we send the message to the network
# we need to start with a copy of the message due to cases
# where side effects of the validity check may impact objects
# related to msg
mymsg = copy.deepcopy(msg)
if hasattr(mymsg, 'Transaction') and mymsg.Transaction is not None:
mytxn = mymsg.Transaction
logger.info('starting local validation '
'for txn id: %s type: %s',
mytxn.Identifier,
mytxn.TransactionTypeName)
blockid = self.Ledger.MostRecentCommittedBlockID
realstoremap = self.Ledger.GlobalStoreMap.get_block_store(
blockid)
tempstoremap = global_store_manager.BlockStore(realstoremap)
if not tempstoremap:
logger.info('no store map for block %s', blockid)
return self.error_response(
request, http.BAD_REQUEST,
'unable to validate enclosed transaction {0}',
data)
transtype = mytxn.TransactionTypeName
if transtype not in tempstoremap.TransactionStores:
logger.info('transaction type %s not in global store map',
transtype)
return self.error_response(
request, http.BAD_REQUEST,
'unable to validate enclosed transaction {0}',
data)
# clone a copy of the ledger's message queue so we can
# temporarily play forward all locally submitted yet
# uncommitted transactions
myqueue = copy.deepcopy(self.Ledger.MessageQueue)
# apply any enqueued messages
while len(myqueue) > 0:
qmsg = myqueue.pop()
if qmsg and \
qmsg.MessageType in self.Ledger.MessageHandlerMap:
if (hasattr(qmsg, 'Transaction') and
qmsg.Transaction is not None):
mystore = tempstoremap.get_transaction_store(
qmsg.Transaction.TransactionTypeName)
if qmsg.Transaction.is_valid(mystore):
myqtxn = copy.copy(qmsg.Transaction)
myqtxn.apply(mystore)
# apply any local pending transactions
for txnid in self.Ledger.PendingTransactions.iterkeys():
pendtxn = self.Ledger.TransactionStore[txnid]
mystore = tempstoremap.get_transaction_store(
pendtxn.TransactionTypeName)
if pendtxn and pendtxn.is_valid(mystore):
mypendtxn = copy.copy(pendtxn)
logger.debug('applying pending transaction '
'%s to temp store', txnid)
mypendtxn.apply(mystore)
# determine validity of the POSTed transaction against our
# new temporary state
mystore = tempstoremap.get_transaction_store(
mytxn.TransactionTypeName)
try:
mytxn.check_valid(mystore)
except InvalidTransactionError as e:
logger.info('submitted transaction fails transaction '
'family validation check: %s; %s',
request.path, mymsg.dump())
return self.error_response(
request, http.BAD_REQUEST,
"enclosed transaction failed transaction "
"family validation check: {}".format(str(e)),
data)
except:
logger.info('submitted transaction is '
'not valid %s; %s; %s',
request.path, mymsg.dump(),
traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
"enclosed transaction is not valid",
data)
logger.info('transaction %s is valid',
msg.Transaction.Identifier)
# and finally execute the associated method
# and send back the results
try:
response = self.PostPageMap[prefix](request, components, msg)
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response.dump())
else:
result = dict2cbor(response.dump())
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}',
request.path, str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def test_unpack_message(self):
# Make sure that the message can be unpacked after serliazation
msg = Message({'__SIGNATURE__': "Test"})
cbor = dict2cbor(msg.dump())
msgdict = unpack_message_data(cbor)
self.assertEquals(msg.dump(), msgdict)
def test_unpack_message(self):
# Make sure that the message can be unpacked after serliazation
msg = Message({'__SIGNATURE__': "Test"})
cbor = dict2cbor(msg.dump())
msgdict = unpack_message_data(cbor)
self.assertEqual(msg.dump(), msgdict)
def _posturl(self, url, info, encoding='application/cbor'):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
if encoding == 'application/json':
data = dict2json(info)
elif encoding == 'application/cbor':
data = dict2cbor(info)
else:
LOGGER.error('unknown request encoding %s', encoding)
return None
datalen = len(data)
LOGGER.debug('post transaction to %s with DATALEN=%d, DATA=<%s>', url,
datalen, data)
try:
request = urllib2.Request(url, data,
{'Content-Type': 'application/cbor',
'Content-Length': datalen})
opener = urllib2.build_opener(self.proxy_handler)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
LOGGER.error('peer operation on url %s failed with response: %d',
url, err.code)
raise MessageException('operation failed with resonse: {0}'.format(
err.code))
except urllib2.URLError as err:
LOGGER.error('peer operation on url %s failed: %s', url,
err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except NameError as err:
LOGGER.error('name error %s', err)
raise MessageException('operation failed: {0}'.format(url))
except:
LOGGER.error('no response from peer server for url %s; %s', url,
sys.exc_info()[0])
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.info('server responds with message %s of unknown type %s',
content, encoding)
value = OrderedDict()
return value
def postmsg(self, msgtype, info):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
data = dict2cbor(info)
datalen = len(data)
url = urlparse.urljoin(self._base_url, msgtype)
LOGGER.debug('post transaction to %s with DATALEN=%d, DATA=<%s>', url,
datalen, data)
try:
request = urllib2.Request(url, data, {
'Content-Type': 'application/cbor',
'Content-Length': datalen
})
if self._cookie:
request.add_header('cookie', self._cookie)
opener = urllib2.build_opener(self._proxy_handler)
response = opener.open(request, timeout=10)
if not self._cookie:
self._cookie = response.headers.get('Set-Cookie')
except urllib2.HTTPError as err:
content = err.read()
if content is not None:
headers = err.info()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.warn('operation failed with response: %s', err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
LOGGER.warn('operation failed with response: %s %s', err.code,
str(value))
if "errorType" in value:
if value['errorType'] == "InvalidTransactionError":
raise InvalidTransactionError(
value['error'] if 'error' in value else value)
else:
raise MessageException(str(value))
else:
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
LOGGER.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
LOGGER.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.info('server responds with message %s of type %s', content,
encoding)
return None
LOGGER.debug(pretty_print_dict(value))
return value
def create_id(self):
return hashlib.sha256(dict2cbor(_serialize(
self, self._update_type))).hexdigest()
def create_id(self):
return hashlib.sha256(
dict2cbor(_serialize(self, self._update_type))).hexdigest()
def _posturl(self, url, info, encoding='application/cbor'):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
if encoding == 'application/json':
data = dict2json(info)
elif encoding == 'application/cbor':
data = dict2cbor(info)
else:
LOGGER.error('unknown request encoding %s', encoding)
return None
datalen = len(data)
LOGGER.debug('post transaction to %s with DATALEN=%d, DATA=<%s>', url,
datalen, data)
try:
request = urllib2.Request(url, data, {
'Content-Type': 'application/cbor',
'Content-Length': datalen
})
opener = urllib2.build_opener(self.proxy_handler)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
LOGGER.error('peer operation on url %s failed with response: %d',
url, err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
LOGGER.error('peer operation on url %s failed: %s', url,
err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except NameError as err:
LOGGER.error('name error %s', err)
raise MessageException('operation failed: {0}'.format(url))
except:
LOGGER.error('no response from peer server for url %s; %s', url,
sys.exc_info()[0])
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.info('server responds with message %s of unknown type %s',
content, encoding)
value = OrderedDict()
return value
def __len__(self):
if not self._data:
self._data = dict2cbor(self.dump())
return len(self._data)
def do_post(self, request):
"""
Handle two types of HTTP POST requests:
- gossip messages. relayed to the gossip network as is
- validator command and control (/command)
"""
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.PostPageMap:
prefix = 'default'
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
# process non-gossip API requests
if prefix == 'command':
try:
if encoding == 'application/json':
minfo = json2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'bad message encoding, {0}',
encoding)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unable to decode incoming request {0}', data)
# process /command
try:
response = self.PostPageMap[prefix](request, components, minfo)
request.responseHeaders.addRawHeader("content-type", encoding)
result = dict2json(response)
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}',
request.path, str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'error processing http request {0}', request.path)
else:
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self.error_response(
request, http.BAD_REQUEST,
'unknown message encoding, {0}', encoding)
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
return self.error_response(
request, http.BAD_REQUEST,
'received request for unknown message type, {0}',
typename)
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unabled to decode incoming request {0}', data)
# determine if the message contains a valid transaction before
# we send the message to the network
# we need to start with a copy of the message due to cases
# where side effects of the validity check may impact objects
# related to msg
mymsg = copy.deepcopy(msg)
if hasattr(mymsg, 'Transaction') and mymsg.Transaction is not None:
mytxn = mymsg.Transaction
logger.info(
'starting local validation '
'for txn id: %s type: %s', mytxn.Identifier,
mytxn.TransactionTypeName)
block_id = self.Ledger.MostRecentCommittedBlockID
real_store_map = self.Ledger.GlobalStoreMap.get_block_store(
block_id)
temp_store_map = \
global_store_manager.BlockStore(real_store_map)
if not temp_store_map:
logger.info('no store map for block %s', block_id)
return self.error_response(
request, http.BAD_REQUEST,
'unable to validate enclosed transaction {0}', data)
transaction_type = mytxn.TransactionTypeName
if transaction_type not in temp_store_map.TransactionStores:
logger.info('transaction type %s not in global store map',
transaction_type)
return self.error_response(
request, http.BAD_REQUEST,
'unable to validate enclosed transaction {0}', data)
# clone a copy of the ledger's message queue so we can
# temporarily play forward all locally submitted yet
# uncommitted transactions
my_queue = copy.deepcopy(self.Ledger.MessageQueue)
# apply any enqueued messages
while len(my_queue) > 0:
qmsg = my_queue.pop()
if qmsg and \
qmsg.MessageType in self.Ledger.MessageHandlerMap:
if (hasattr(qmsg, 'Transaction')
and qmsg.Transaction is not None):
my_store = temp_store_map.get_transaction_store(
qmsg.Transaction.TransactionTypeName)
if qmsg.Transaction.is_valid(my_store):
myqtxn = copy.copy(qmsg.Transaction)
myqtxn.apply(my_store)
# apply any local pending transactions
for txn_id in self.Ledger.PendingTransactions.iterkeys():
pend_txn = self.Ledger.TransactionStore[txn_id]
my_store = temp_store_map.get_transaction_store(
pend_txn.TransactionTypeName)
if pend_txn and pend_txn.is_valid(my_store):
my_pend_txn = copy.copy(pend_txn)
logger.debug(
'applying pending transaction '
'%s to temp store', txn_id)
my_pend_txn.apply(my_store)
# determine validity of the POSTed transaction against our
# new temporary state
my_store = temp_store_map.get_transaction_store(
mytxn.TransactionTypeName)
try:
mytxn.check_valid(my_store)
except InvalidTransactionError as e:
logger.info(
'submitted transaction fails transaction '
'family validation check: %s; %s', request.path,
mymsg.dump())
return self.error_response(
request, http.BAD_REQUEST,
"enclosed transaction failed transaction "
"family validation check: {}".format(str(e)), data)
except:
logger.info(
'submitted transaction is '
'not valid %s; %s; %s', request.path, mymsg.dump(),
traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
"enclosed transaction is not valid", data)
logger.info('transaction %s is valid',
msg.Transaction.Identifier)
# and finally execute the associated method
# and send back the results
try:
response = self.PostPageMap[prefix](request, components, msg)
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response.dump())
else:
result = dict2cbor(response.dump())
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}',
request.path, str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'error processing http request {0}', request.path)
def __len__(self):
if not self._data:
self._data = dict2cbor(self.dump())
return len(self._data)
def render_POST(self, request):
"""
Handle a POST request on the HTTP interface. All message on the POST
interface are gossip messages that should be relayed into the gossip
network as is.
"""
# pylint: disable=invalid-name
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.PostPageMap:
prefix = 'default'
# process the message encoding
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'unknown message encoding, {0}',
encoding)
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
return self.error_response(
request, http.BAD_REQUEST,
'received request for unknown message type, {0}', typename)
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unabled to decode incoming request {0}',
data)
# and finally execute the associated method and send back the results
try:
response = self.PostPageMap[prefix](request, components, msg)
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response.dump())
else:
result = dict2cbor(response.dump())
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}', request.path,
str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def postmsg(self, msgtype, info):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
data = dict2cbor(info)
datalen = len(data)
url = urlparse.urljoin(self._base_url, msgtype)
LOGGER.debug('post transaction to %s with DATALEN=%d, DATA=<%s>', url,
datalen, data)
try:
request = urllib2.Request(url, data,
{'Content-Type': 'application/cbor',
'Content-Length': datalen})
if self._cookie:
request.add_header('cookie', self._cookie)
opener = urllib2.build_opener(self._proxy_handler)
response = opener.open(request, timeout=10)
if not self._cookie:
self._cookie = response.headers.get('Set-Cookie')
except urllib2.HTTPError as err:
content = err.read()
if content is not None:
headers = err.info()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.warn('operation failed with response: %s', err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
LOGGER.warn('operation failed with response: %s %s',
err.code, str(value))
if "errorType" in value:
if value['errorType'] == "InvalidTransactionError":
raise InvalidTransactionError(
value['error'] if 'error' in value else value)
else:
raise MessageException(str(value))
else:
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
LOGGER.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
LOGGER.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.info('server responds with message %s of type %s', content,
encoding)
return None
LOGGER.debug(pretty_print_dict(value))
return value
def render_POST(self, request):
"""
Handle a POST request on the HTTP interface. All message on the POST
interface are gossip messages that should be relayed into the gossip
network as is.
"""
# pylint: disable=invalid-name
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.PostPageMap:
prefix = 'default'
# process the message encoding
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'unknown message encoding, {0}',
encoding)
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
return self.error_response(
request, http.BAD_REQUEST,
'received request for unknown message type, {0}', typename)
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unabled to decode incoming request {0}', data)
# and finally execute the associated method and send back the results
try:
response = self.PostPageMap[prefix](request, components, msg)
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
return dict2json(response.dump())
else:
return dict2cbor(response.dump())
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}', request.path,
str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)