def require_store(self, blockid):
"""Ensure that the store for this block (including all dependent
blocks) is loaded into the _blockmap
:param str blockid: identifier to associate with the block
"""
# this is all about removing recursion... yes, recursion is a useful
# thing... however python is not really very friendly to deep recursion
# and since this might go through the entire chain of blocks... seems
# like avoiding recursion is a very useful thing
# pass 1... build the list of blocks that we need to load in order
# to load the current block
blocklist = []
while blockid not in self._blockmap:
logger.info('add block %s to the queue for loading', blockid)
blocklist.insert(0, blockid)
if blockid not in self._persistmap:
raise KeyError('unknown block', blockid)
blockinfo = cbor2dict(self._persistmap[blockid])
blockid = blockinfo['PreviousBlockID']
# pass 2... starting with the oldest block, begin to load
# the stores
for blockid in blocklist:
logger.info('load block %s from storage', blockid)
blockinfo = cbor.loads(self._persistmap[blockid])
prevstore = self._blockmap[blockinfo['PreviousBlockID']]
blockstore = prevstore.CloneBlock(blockinfo, True)
blockstore.CommitBlock(blockid)
self._blockmap[blockid] = blockstore
def require_store(self, blockid):
"""Ensure that the store for this block (including all dependent
blocks) is loaded into the _blockmap
:param str blockid: identifier to associate with the block
"""
# this is all about removing recursion... yes, recursion is a useful
# thing... however python is not really very friendly to deep recursion
# and since this might go through the entire chain of blocks... seems
# like avoiding recursion is a very useful thing
# pass 1... build the list of blocks that we need to load in order
# to load the current block
blocklist = []
while blockid not in self._blockmap:
logger.info('add block %s to the queue for loading', blockid)
blocklist.insert(0, blockid)
if blockid not in self._persistmap:
raise KeyError('unknown block', blockid)
blockinfo = cbor2dict(self._persistmap[blockid])
blockid = blockinfo['PreviousBlockID']
# pass 2... starting with the oldest block, begin to load
# the stores
for blockid in blocklist:
logger.info('load block %s from storage', blockid)
blockinfo = cbor.loads(self._persistmap[blockid])
prevstore = self._blockmap[blockinfo['PreviousBlockID']]
blockstore = prevstore.clone_block(blockinfo, True)
blockstore.commit_block(blockid)
self._blockmap[blockid] = blockstore
def test_serialize(self):
# Test that serilazation returns the correct dictionary and that
# it can be retrieved.
# create SignedObject
temp = SignedObject({"TestSignatureDictKey": "test"}, "TestSignatureDictKey")
# serlize SignedObject
cbor = temp.serialize()
# check that the unserilized serilized dictinary is the same
# as before serilazation
self.assertEqual(cbor2dict(cbor), temp.dump())
def postmsg(self, msgtype, info):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
logger.debug(dict2json(info))
data = dict2cbor(info)
datalen = len(data)
url = self.BaseURL + msgtype
logger.debug(
'post transaction to %s with DATALEN=%d, '
'base64(DATA)=<%s>', url, datalen, base64.b64encode(data))
try:
request = urllib2.Request(url, data, {
'Content-Type': 'application/cbor',
'Content-Length': datalen
})
opener = urllib2.build_opener(self.ProxyHandler)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
logger.warn('operation failed with response: %s', err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
logger.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
logger.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
logger.info('server responds with message %s of type %s', content,
encoding)
return None
logger.debug(pretty_print_dict(value))
return value
def unpack_message_data(data):
"""Unpacks CBOR encoded data into a dict.
Args:
data (bytes): CBOR encoded data.
Returns:
dict: A dict reflecting the contents of the CBOR encoded
representation.
"""
return cbor2dict(data)
def test_serialize(self):
# Test that serilazation returns the correct dictionary and that
# it can be retrieved.
# create SignedObject
signkey = SigObj.generate_signing_key()
temp = SignedObject({signkey: "test"}, signkey)
# serlize SignedObject
cbor = temp.serialize()
# check that the unserilized serilized dictinary is the same
# as before serilazation
self.assertEquals(cbor2dict(cbor), temp.dump())
def unpack_message_data(data):
"""Unpacks CBOR encoded data into a dict.
Args:
data (bytes): CBOR encoded data.
Returns:
dict: A dict reflecting the contents of the CBOR encoded
representation.
"""
return cbor2dict(data)
def test_serialize(self):
# Test that serilazation returns the correct dictionary and that
# it can be retrieved.
# create SignedObject
signkey = SigObj.generate_signing_key()
temp = SignedObject({signkey: "test"}, signkey)
# serlize SignedObject
cbor = temp.serialize()
# check that the unserilized serilized dictinary is the same
# as before serilazation
self.assertEquals(cbor2dict(cbor), temp.dump())
def postmsg(self, msgtype, info):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
logger.debug(dict2json(info))
data = dict2cbor(info)
datalen = len(data)
url = self.BaseURL + msgtype
logger.debug('post transaction to %s with DATALEN=%d, '
'base64(DATA)=<%s>', url, datalen, base64.b64encode(data))
try:
request = urllib2.Request(url, data,
{'Content-Type': 'application/cbor',
'Content-Length': datalen})
opener = urllib2.build_opener(self.ProxyHandler)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
logger.warn('operation failed with response: %s', err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
logger.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
logger.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
logger.info('server responds with message %s of type %s', content,
encoding)
return None
logger.debug(pretty_print_dict(value))
return value
def getmsg(self, path):
"""
Send an HTTP get request to the validator. If the resulting content
is in JSON form, parse it & return the corresponding dictionary.
"""
url = "{0}/{1}".format(self.BaseURL, path.strip('/'))
logger.debug('get content from url <%s>', url)
try:
request = urllib2.Request(url, headers=self.GET_HEADER)
opener = urllib2.build_opener(self.ProxyHandler)
if path == '/prevalidation':
if self._cookie:
request.add_header('cookie', self._cookie)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
logger.warn('operation failed with response: %s', err.code)
self._print_error_information_from_server(err)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
logger.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
logger.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
logger.debug('get content <%s> from url <%s>', content, url)
return content
logger.debug(pretty_print_dict(value))
return value
def getmsg(self, path):
"""
Send an HTTP get request to the validator. If the resulting content
is in JSON form, parse it & return the corresponding dictionary.
"""
url = "{0}/{1}".format(self.BaseURL, path.strip('/'))
logger.debug('get content from url <%s>', url)
try:
request = urllib2.Request(url, headers=self.GET_HEADER)
opener = urllib2.build_opener(self.ProxyHandler)
if path == '/prevalidation':
if self._cookie:
request.add_header('cookie', self._cookie)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
logger.warn('operation failed with response: %s', err.code)
self._print_error_information_from_server(err)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
logger.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
logger.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
logger.debug('get content <%s> from url <%s>', content, url)
return content
logger.debug(pretty_print_dict(value))
return value
def _geturl(self, url, verbose=True, timeout=30):
"""
Send an HTTP get request to the validator. If the resulting content is
in JSON or CBOR form, parse it & return the corresponding dictionary.
"""
if verbose:
LOGGER.debug('get content from url <%s>', url)
try:
request = urllib2.Request(url, headers=self.GET_HEADER)
opener = urllib2.build_opener(self.proxy_handler)
response = opener.open(request, timeout=timeout)
except urllib2.HTTPError as err:
if verbose:
LOGGER.error(
'peer operation on url %s failed '
'with response: %d', url, err.code)
raise MessageException('operation failed '
'with response: {0}'.format(err.code))
except urllib2.URLError as err:
if verbose:
LOGGER.error('peer operation on url %s failed: %s', url,
err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
if verbose:
LOGGER.error('no response from peer server for url %s; %s',
url,
sys.exc_info()[0])
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
return json2dict(content)
elif encoding == 'application/cbor':
return cbor2dict(content)
else:
return content
def _geturl(self, url, verbose=True, timeout=30):
"""
Send an HTTP get request to the validator. If the resulting content is
in JSON or CBOR form, parse it & return the corresponding dictionary.
"""
if verbose:
LOGGER.debug('get content from url <%s>', url)
try:
request = urllib2.Request(url, headers=self.GET_HEADER)
opener = urllib2.build_opener(self.proxy_handler)
response = opener.open(request, timeout=timeout)
except urllib2.HTTPError as err:
if verbose:
LOGGER.error('peer operation on url %s failed '
'with response: %d', url, err.code)
raise MessageException('operation failed '
'with response: {0}'.format(err.code))
except urllib2.URLError as err:
if verbose:
LOGGER.error('peer operation on url %s failed: %s',
url, err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
if verbose:
LOGGER.error('no response from peer server for url %s; %s',
url, sys.exc_info()[0])
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
return json2dict(content)
elif encoding == 'application/cbor':
return cbor2dict(content)
else:
return content
def _get_message(self, request):
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self._encode_error_response(
request, http.BAD_REQUEST, 'unknown message'
' encoding: {0}'.format(encoding))
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.gossip.MessageHandlerMap:
return self._encode_error_response(
request, http.NOT_FOUND, 'received request for unknown message'
' type, {0}'.format(typename))
return self.Ledger.gossip.MessageHandlerMap[typename][0](minfo)
def getmsg(self, path, timeout=10):
"""
Send an HTTP get request to the validator. If the resulting content
is in JSON form, parse it & return the corresponding dictionary.
"""
url = urlparse.urljoin(self._base_url, path)
LOGGER.debug('get content from url <%s>', url)
try:
request = urllib2.Request(url)
opener = urllib2.build_opener(self._proxy_handler)
response = opener.open(request, timeout=timeout)
except urllib2.HTTPError as err:
LOGGER.warn('operation failed with response: %s', err.code)
self._print_error_information_from_server(err)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
LOGGER.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
LOGGER.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
return json2dict(content)
elif encoding == 'application/cbor':
return cbor2dict(content)
else:
return content
def getmsg(self, path, timeout=10):
"""
Send an HTTP get request to the validator. If the resulting content
is in JSON form, parse it & return the corresponding dictionary.
"""
url = urlparse.urljoin(self._base_url, path)
LOGGER.debug('get content from url <%s>', url)
try:
request = urllib2.Request(url)
opener = urllib2.build_opener(self._proxy_handler)
response = opener.open(request, timeout=timeout)
except urllib2.HTTPError as err:
LOGGER.warn('operation failed with response: %s', err.code)
self._print_error_information_from_server(err)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
LOGGER.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
LOGGER.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
return json2dict(content)
elif encoding == 'application/cbor':
return cbor2dict(content)
else:
return content
def _get_message(self, request):
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self._encode_error_response(
request,
http.BAD_REQUEST,
'unknown message'
' encoding: {0}'.format(encoding))
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
return self._encode_error_response(
request,
http.NOT_FOUND,
'received request for unknown message'
' type, {0}'.format(typename))
return self.Ledger.MessageHandlerMap[typename][0](minfo)
def _get_message(self, request):
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self._encode_error_response(
request,
http.BAD_REQUEST,
'unknown message'
' encoding: {0}'.format(encoding))
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if not self.validator.gossip.dispatcher.has_message_handler(typename):
return self._encode_error_response(
request,
http.NOT_FOUND,
'received request for unknown message'
' type, {0}'.format(typename))
return self.validator.gossip.dispatcher.unpack_message(typename, minfo)
def _posturl(self, url, info, encoding='application/cbor'):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
if encoding == 'application/json':
data = dict2json(info)
elif encoding == 'application/cbor':
data = dict2cbor(info)
else:
LOGGER.error('unknown request encoding %s', encoding)
return None
datalen = len(data)
LOGGER.debug('post transaction to %s with DATALEN=%d, DATA=<%s>', url,
datalen, data)
try:
request = urllib2.Request(url, data, {
'Content-Type': 'application/cbor',
'Content-Length': datalen
})
opener = urllib2.build_opener(self.proxy_handler)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
LOGGER.error('peer operation on url %s failed with response: %d',
url, err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
LOGGER.error('peer operation on url %s failed: %s', url,
err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except NameError as err:
LOGGER.error('name error %s', err)
raise MessageException('operation failed: {0}'.format(url))
except:
LOGGER.error('no response from peer server for url %s; %s', url,
sys.exc_info()[0])
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.info('server responds with message %s of unknown type %s',
content, encoding)
value = OrderedDict()
return value
def render_POST(self, request):
"""
Handle a POST request on the HTTP interface. All message on the POST
interface are gossip messages that should be relayed into the gossip
network as is.
"""
# pylint: disable=invalid-name
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.PostPageMap:
prefix = 'default'
# process the message encoding
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'unknown message encoding, {0}',
encoding)
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
return self.error_response(
request, http.BAD_REQUEST,
'received request for unknown message type, {0}', typename)
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unabled to decode incoming request {0}',
data)
# and finally execute the associated method and send back the results
try:
response = self.PostPageMap[prefix](request, components, msg)
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response.dump())
else:
result = dict2cbor(response.dump())
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}', request.path,
str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def _posturl(self, url, info, encoding='application/cbor'):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
if encoding == 'application/json':
data = dict2json(info)
elif encoding == 'application/cbor':
data = dict2cbor(info)
else:
LOGGER.error('unknown request encoding %s', encoding)
return None
datalen = len(data)
LOGGER.debug('post transaction to %s with DATALEN=%d, DATA=<%s>', url,
datalen, data)
try:
request = urllib2.Request(url, data,
{'Content-Type': 'application/cbor',
'Content-Length': datalen})
opener = urllib2.build_opener(self.proxy_handler)
response = opener.open(request, timeout=10)
except urllib2.HTTPError as err:
LOGGER.error('peer operation on url %s failed with response: %d',
url, err.code)
raise MessageException('operation failed with resonse: {0}'.format(
err.code))
except urllib2.URLError as err:
LOGGER.error('peer operation on url %s failed: %s', url,
err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except NameError as err:
LOGGER.error('name error %s', err)
raise MessageException('operation failed: {0}'.format(url))
except:
LOGGER.error('no response from peer server for url %s; %s', url,
sys.exc_info()[0])
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.info('server responds with message %s of unknown type %s',
content, encoding)
value = OrderedDict()
return value
def render_post(self, request, components, msg):
"""
Forward a signed message through the gossip network.
"""
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
raise Error("", 'unknown message'
' encoding: {0}'.format(encoding))
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
raise Error("",
'received request for unknown message'
' type, {0}'.format(typename))
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except Error as e:
LOGGER.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
raise Error(http.BAD_REQUEST,
'unable to decode incoming request: {0}'.format(e))
if self.Validator.Config.get("LocalValidation", True):
# determine if the message contains a valid transaction before
# we send the message to the network
# we need to start with a copy of the message due to cases
# where side effects of the validity check may impact objects
# related to msg
mymsg = copy.deepcopy(msg)
if hasattr(mymsg, 'Transaction') and mymsg.Transaction is not None:
mytxn = mymsg.Transaction
LOGGER.info('starting local validation '
'for txn id: %s type: %s',
mytxn.Identifier,
mytxn.TransactionTypeName)
block_id = self.Ledger.MostRecentCommittedBlockID
real_store_map = self.Ledger.GlobalStoreMap.get_block_store(
block_id)
temp_store_map = \
global_store_manager.BlockStore(real_store_map)
if not temp_store_map:
LOGGER.info('no store map for block %s', block_id)
raise Error(http.BAD_REQUEST,
'unable to validate enclosed'
' transaction {0}'.format(data))
transaction_type = mytxn.TransactionTypeName
if transaction_type not in temp_store_map.TransactionStores:
LOGGER.info('transaction type %s not in global store map',
transaction_type)
raise Error(http.BAD_REQUEST,
'unable to validate enclosed'
' transaction {0}'.format(data))
# clone a copy of the ledger's message queue so we can
# temporarily play forward all locally submitted yet
# uncommitted transactions
my_queue = copy.deepcopy(self.Ledger.MessageQueue)
# apply any enqueued messages
while len(my_queue) > 0:
qmsg = my_queue.pop()
if qmsg and \
qmsg.MessageType in self.Ledger.MessageHandlerMap:
if (hasattr(qmsg, 'Transaction') and
qmsg.Transaction is not None):
my_store = temp_store_map.get_transaction_store(
qmsg.Transaction.TransactionTypeName)
if qmsg.Transaction.is_valid(my_store):
myqtxn = copy.copy(qmsg.Transaction)
myqtxn.apply(my_store)
# apply any local pending transactions
for txn_id in self.Ledger.PendingTransactions.iterkeys():
pend_txn = self.Ledger.TransactionStore[txn_id]
my_store = temp_store_map.get_transaction_store(
pend_txn.TransactionTypeName)
if pend_txn and pend_txn.is_valid(my_store):
my_pend_txn = copy.copy(pend_txn)
my_pend_txn.apply(my_store)
# determine validity of the POSTed transaction against our
# new temporary state
my_store = temp_store_map.get_transaction_store(
mytxn.TransactionTypeName)
try:
mytxn.check_valid(my_store)
except InvalidTransactionError as e:
LOGGER.info('submitted transaction fails transaction '
'family validation check: %s; %s',
request.path, mymsg.dump())
raise Error(http.BAD_REQUEST,
"enclosed transaction failed transaction "
"family validation check: {}".format(str(e)))
except:
LOGGER.info('submitted transaction is '
'not valid %s; %s; %s',
request.path, mymsg.dump(),
traceback.format_exc(20))
raise Error(http.BAD_REQUEST,
"enclosed transaction is not"
" valid {}".format(data))
LOGGER.info('transaction %s is valid',
msg.Transaction.Identifier)
# and finally execute the associated method
# and send back the results
self.Ledger.handle_message(msg)
return msg
def postmsg(self, msgtype, info):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
data = dict2cbor(info)
datalen = len(data)
url = urlparse.urljoin(self._base_url, msgtype)
LOGGER.debug('post transaction to %s with DATALEN=%d, DATA=<%s>', url,
datalen, data)
try:
request = urllib2.Request(url, data,
{'Content-Type': 'application/cbor',
'Content-Length': datalen})
if self._cookie:
request.add_header('cookie', self._cookie)
opener = urllib2.build_opener(self._proxy_handler)
response = opener.open(request, timeout=10)
if not self._cookie:
self._cookie = response.headers.get('Set-Cookie')
except urllib2.HTTPError as err:
content = err.read()
if content is not None:
headers = err.info()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.warn('operation failed with response: %s', err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
LOGGER.warn('operation failed with response: %s %s',
err.code, str(value))
if "errorType" in value:
if value['errorType'] == "InvalidTransactionError":
raise InvalidTransactionError(
value['error'] if 'error' in value else value)
else:
raise MessageException(str(value))
else:
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
LOGGER.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
LOGGER.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.info('server responds with message %s of type %s', content,
encoding)
return None
LOGGER.debug(pretty_print_dict(value))
return value
def do_post(self, request):
"""
Handle two types of HTTP POST requests:
- gossip messages. relayed to the gossip network as is
- validator command and control (/command)
"""
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.PostPageMap:
prefix = 'default'
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
# process non-gossip API requests
if prefix == 'command':
try:
if encoding == 'application/json':
minfo = json2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'bad message encoding, {0}',
encoding)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unable to decode incoming request {0}', data)
# process /command
try:
response = self.PostPageMap[prefix](request, components, minfo)
request.responseHeaders.addRawHeader("content-type", encoding)
result = dict2json(response)
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}',
request.path, str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'error processing http request {0}', request.path)
else:
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self.error_response(
request, http.BAD_REQUEST,
'unknown message encoding, {0}', encoding)
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
return self.error_response(
request, http.BAD_REQUEST,
'received request for unknown message type, {0}',
typename)
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unabled to decode incoming request {0}', data)
# determine if the message contains a valid transaction before
# we send the message to the network
# we need to start with a copy of the message due to cases
# where side effects of the validity check may impact objects
# related to msg
mymsg = copy.deepcopy(msg)
if hasattr(mymsg, 'Transaction') and mymsg.Transaction is not None:
mytxn = mymsg.Transaction
logger.info(
'starting local validation '
'for txn id: %s type: %s', mytxn.Identifier,
mytxn.TransactionTypeName)
block_id = self.Ledger.MostRecentCommittedBlockID
real_store_map = self.Ledger.GlobalStoreMap.get_block_store(
block_id)
temp_store_map = \
global_store_manager.BlockStore(real_store_map)
if not temp_store_map:
logger.info('no store map for block %s', block_id)
return self.error_response(
request, http.BAD_REQUEST,
'unable to validate enclosed transaction {0}', data)
transaction_type = mytxn.TransactionTypeName
if transaction_type not in temp_store_map.TransactionStores:
logger.info('transaction type %s not in global store map',
transaction_type)
return self.error_response(
request, http.BAD_REQUEST,
'unable to validate enclosed transaction {0}', data)
# clone a copy of the ledger's message queue so we can
# temporarily play forward all locally submitted yet
# uncommitted transactions
my_queue = copy.deepcopy(self.Ledger.MessageQueue)
# apply any enqueued messages
while len(my_queue) > 0:
qmsg = my_queue.pop()
if qmsg and \
qmsg.MessageType in self.Ledger.MessageHandlerMap:
if (hasattr(qmsg, 'Transaction')
and qmsg.Transaction is not None):
my_store = temp_store_map.get_transaction_store(
qmsg.Transaction.TransactionTypeName)
if qmsg.Transaction.is_valid(my_store):
myqtxn = copy.copy(qmsg.Transaction)
myqtxn.apply(my_store)
# apply any local pending transactions
for txn_id in self.Ledger.PendingTransactions.iterkeys():
pend_txn = self.Ledger.TransactionStore[txn_id]
my_store = temp_store_map.get_transaction_store(
pend_txn.TransactionTypeName)
if pend_txn and pend_txn.is_valid(my_store):
my_pend_txn = copy.copy(pend_txn)
logger.debug(
'applying pending transaction '
'%s to temp store', txn_id)
my_pend_txn.apply(my_store)
# determine validity of the POSTed transaction against our
# new temporary state
my_store = temp_store_map.get_transaction_store(
mytxn.TransactionTypeName)
try:
mytxn.check_valid(my_store)
except InvalidTransactionError as e:
logger.info(
'submitted transaction fails transaction '
'family validation check: %s; %s', request.path,
mymsg.dump())
return self.error_response(
request, http.BAD_REQUEST,
"enclosed transaction failed transaction "
"family validation check: {}".format(str(e)), data)
except:
logger.info(
'submitted transaction is '
'not valid %s; %s; %s', request.path, mymsg.dump(),
traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
"enclosed transaction is not valid", data)
logger.info('transaction %s is valid',
msg.Transaction.Identifier)
# and finally execute the associated method
# and send back the results
try:
response = self.PostPageMap[prefix](request, components, msg)
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response.dump())
else:
result = dict2cbor(response.dump())
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}',
request.path, str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'error processing http request {0}', request.path)
def render_post(self, request, components, msg):
"""
Forward a signed message through the gossip network.
"""
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
raise Error(
"", 'unknown message'
' encoding: {0}'.format(encoding))
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
raise Error(
"", 'received request for unknown message'
' type, {0}'.format(typename))
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except Error as e:
LOGGER.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
raise Error(http.BAD_REQUEST,
'unable to decode incoming request: {0}'.format(e))
if self.Validator.Config.get("LocalValidation", True):
# determine if the message contains a valid transaction before
# we send the message to the network
# we need to start with a copy of the message due to cases
# where side effects of the validity check may impact objects
# related to msg
mymsg = copy.deepcopy(msg)
if hasattr(mymsg, 'Transaction') and mymsg.Transaction is not None:
mytxn = mymsg.Transaction
LOGGER.info(
'starting local validation '
'for txn id: %s type: %s', mytxn.Identifier,
mytxn.TransactionTypeName)
pending_block_txns = None
if self.Ledger.PendingTransactionBlock is not None:
pending_block_txns = \
self.Ledger.PendingTransactionBlock.TransactionIDs
block_id = self.Ledger.MostRecentCommittedBlockID
real_store_map = \
self.Ledger.GlobalStoreMap.get_block_store(block_id)
temp_store_map = \
global_store_manager.BlockStore(real_store_map)
if not temp_store_map:
LOGGER.info('no store map for block %s', block_id)
raise Error(
http.BAD_REQUEST, 'unable to validate enclosed'
' transaction {0}'.format(data))
pending_txns = copy.copy(self.Ledger.PendingTransactions)
pending_txn_ids = [x for x in pending_txns.iterkeys()]
# clone a copy of the ledger's message queue so we can
# temporarily play forward all locally submitted yet
# uncommitted transactions
my_queue = copy.deepcopy(self.Ledger.MessageQueue)
transaction_type = mytxn.TransactionTypeName
if transaction_type not in temp_store_map.TransactionStores:
LOGGER.info('transaction type %s not in global store map',
transaction_type)
raise Error(
http.BAD_REQUEST, 'unable to validate enclosed'
' transaction {0}'.format(data))
if pending_block_txns is not None:
pending_txn_ids = pending_block_txns + pending_txn_ids
# apply any local pending transactions
for txn_id in pending_txn_ids:
pend_txn = self.Ledger.TransactionStore[txn_id]
my_store = temp_store_map.get_transaction_store(
pend_txn.TransactionTypeName)
if pend_txn and pend_txn.is_valid(my_store):
my_pend_txn = copy.copy(pend_txn)
my_pend_txn.apply(my_store)
# apply any enqueued messages
while len(my_queue) > 0:
qmsg = my_queue.pop()
if qmsg and \
qmsg.MessageType in self.Ledger.MessageHandlerMap:
if (hasattr(qmsg, 'Transaction')
and qmsg.Transaction is not None):
my_store = temp_store_map.get_transaction_store(
qmsg.Transaction.TransactionTypeName)
if qmsg.Transaction.is_valid(my_store):
myqtxn = copy.copy(qmsg.Transaction)
myqtxn.apply(my_store)
# determine validity of the POSTed transaction against our
# new temporary state
my_store = temp_store_map.get_transaction_store(
mytxn.TransactionTypeName)
try:
mytxn.check_valid(my_store)
except InvalidTransactionError as e:
LOGGER.info(
'submitted transaction fails transaction '
'family validation check: %s; %s', request.path,
mymsg.dump())
raise Error(
http.BAD_REQUEST,
"enclosed transaction failed transaction "
"family validation check: {}".format(str(e)))
except:
LOGGER.info(
'submitted transaction is '
'not valid %s; %s; %s', request.path, mymsg.dump(),
traceback.format_exc(20))
raise Error(
http.BAD_REQUEST, "enclosed transaction is not"
" valid {}".format(data))
LOGGER.info('transaction %s is valid',
msg.Transaction.Identifier)
# and finally execute the associated method
# and send back the results
self.Ledger.handle_message(msg)
return msg
def render_POST(self, request):
"""
Handle two types of HTTP POST requests:
- gossip messages. relayed to the gossip network as is
- validator command and control (/command)
"""
# pylint: disable=invalid-name
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.PostPageMap:
prefix = 'default'
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
# process non-gossip API requests
if prefix == 'command':
try:
if encoding == 'application/json':
minfo = json2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'bad message encoding, {0}',
encoding)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unable to decode incoming request {0}',
data)
# process /command
try:
response = self.PostPageMap[prefix](request, components, minfo)
request.responseHeaders.addRawHeader("content-type", encoding)
result = dict2json(response)
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}',
request.path, str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
else:
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'unknown message encoding, {0}',
encoding)
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
return self.error_response(
request, http.BAD_REQUEST,
'received request for unknown message type, {0}',
typename)
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unabled to decode incoming request {0}',
data)
# determine if the message contains a valid transaction before
# we send the message to the network
# we need to start with a copy of the message due to cases
# where side effects of the validity check may impact objects
# related to msg
mymsg = copy.deepcopy(msg)
if hasattr(mymsg, 'Transaction') and mymsg.Transaction is not None:
mytxn = mymsg.Transaction
logger.info('starting local validation '
'for txn id: %s type: %s',
mytxn.Identifier,
mytxn.TransactionTypeName)
blockid = self.Ledger.MostRecentCommittedBlockID
realstoremap = self.Ledger.GlobalStoreMap.get_block_store(
blockid)
tempstoremap = global_store_manager.BlockStore(realstoremap)
if not tempstoremap:
logger.info('no store map for block %s', blockid)
return self.error_response(
request, http.BAD_REQUEST,
'unable to validate enclosed transaction {0}',
data)
transtype = mytxn.TransactionTypeName
if transtype not in tempstoremap.TransactionStores:
logger.info('transaction type %s not in global store map',
transtype)
return self.error_response(
request, http.BAD_REQUEST,
'unable to validate enclosed transaction {0}',
data)
# clone a copy of the ledger's message queue so we can
# temporarily play forward all locally submitted yet
# uncommitted transactions
myqueue = copy.deepcopy(self.Ledger.MessageQueue)
# apply any enqueued messages
while len(myqueue) > 0:
qmsg = myqueue.pop()
if qmsg and \
qmsg.MessageType in self.Ledger.MessageHandlerMap:
if (hasattr(qmsg, 'Transaction') and
qmsg.Transaction is not None):
mystore = tempstoremap.get_transaction_store(
qmsg.Transaction.TransactionTypeName)
if qmsg.Transaction.is_valid(mystore):
myqtxn = copy.copy(qmsg.Transaction)
myqtxn.apply(mystore)
# apply any local pending transactions
for txnid in self.Ledger.PendingTransactions.iterkeys():
pendtxn = self.Ledger.TransactionStore[txnid]
mystore = tempstoremap.get_transaction_store(
pendtxn.TransactionTypeName)
if pendtxn and pendtxn.is_valid(mystore):
mypendtxn = copy.copy(pendtxn)
logger.debug('applying pending transaction '
'%s to temp store', txnid)
mypendtxn.apply(mystore)
# determine validity of the POSTed transaction against our
# new temporary state
mystore = tempstoremap.get_transaction_store(
mytxn.TransactionTypeName)
try:
mytxn.check_valid(mystore)
except InvalidTransactionError as e:
logger.info('submitted transaction fails transaction '
'family validation check: %s; %s',
request.path, mymsg.dump())
return self.error_response(
request, http.BAD_REQUEST,
"enclosed transaction failed transaction "
"family validation check: {}".format(str(e)),
data)
except:
logger.info('submitted transaction is '
'not valid %s; %s; %s',
request.path, mymsg.dump(),
traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
"enclosed transaction is not valid",
data)
logger.info('transaction %s is valid',
msg.Transaction.Identifier)
# and finally execute the associated method
# and send back the results
try:
response = self.PostPageMap[prefix](request, components, msg)
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
result = dict2json(response.dump())
else:
result = dict2cbor(response.dump())
return result
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}',
request.path, str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def render_POST(self, request):
"""
Handle a POST request on the HTTP interface. All message on the POST
interface are gossip messages that should be relayed into the gossip
network as is.
"""
# pylint: disable=invalid-name
# break the path into its component parts
components = request.path.split('/')
while components and components[0] == '':
components.pop(0)
prefix = components.pop(0) if components else 'error'
if prefix not in self.PostPageMap:
prefix = 'default'
# process the message encoding
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
return self.error_response(request, http.BAD_REQUEST,
'unknown message encoding, {0}',
encoding)
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
return self.error_response(
request, http.BAD_REQUEST,
'received request for unknown message type, {0}', typename)
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except:
logger.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(
request, http.BAD_REQUEST,
'unabled to decode incoming request {0}', data)
# and finally execute the associated method and send back the results
try:
response = self.PostPageMap[prefix](request, components, msg)
request.responseHeaders.addRawHeader("content-type", encoding)
if encoding == 'application/json':
return dict2json(response.dump())
else:
return dict2cbor(response.dump())
except Error as e:
return self.error_response(
request, int(e.status),
'exception while processing request {0}; {1}', request.path,
str(e))
except:
logger.info('exception while processing http request %s; %s',
request.path, traceback.format_exc(20))
return self.error_response(request, http.BAD_REQUEST,
'error processing http request {0}',
request.path)
def postmsg(self, msgtype, info):
"""
Post a transaction message to the validator, parse the returning CBOR
and return the corresponding dictionary.
"""
data = dict2cbor(info)
datalen = len(data)
url = urlparse.urljoin(self._base_url, msgtype)
LOGGER.debug('post transaction to %s with DATALEN=%d, DATA=<%s>', url,
datalen, data)
try:
request = urllib2.Request(url, data, {
'Content-Type': 'application/cbor',
'Content-Length': datalen
})
if self._cookie:
request.add_header('cookie', self._cookie)
opener = urllib2.build_opener(self._proxy_handler)
response = opener.open(request, timeout=10)
if not self._cookie:
self._cookie = response.headers.get('Set-Cookie')
except urllib2.HTTPError as err:
content = err.read()
if content is not None:
headers = err.info()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.warn('operation failed with response: %s', err.code)
raise MessageException(
'operation failed with response: {0}'.format(err.code))
LOGGER.warn('operation failed with response: %s %s', err.code,
str(value))
if "errorType" in value:
if value['errorType'] == "InvalidTransactionError":
raise InvalidTransactionError(
value['error'] if 'error' in value else value)
else:
raise MessageException(str(value))
else:
raise MessageException(
'operation failed with response: {0}'.format(err.code))
except urllib2.URLError as err:
LOGGER.warn('operation failed: %s', err.reason)
raise MessageException('operation failed: {0}'.format(err.reason))
except:
LOGGER.warn('no response from server')
raise MessageException('no response from server')
content = response.read()
headers = response.info()
response.close()
encoding = headers.get('Content-Type')
if encoding == 'application/json':
value = json2dict(content)
elif encoding == 'application/cbor':
value = cbor2dict(content)
else:
LOGGER.info('server responds with message %s of type %s', content,
encoding)
return None
LOGGER.debug(pretty_print_dict(value))
return value
