def __init__(self): self.cp = ConsolePrint() self.ss = SystemStats() self.ps = PlatformStats() self.psis = PlatformIntervalStats() self.ps.psis = self.psis self.previous_net_bytes_recv = 0 self.previous_net_bytes_sent = 0 self.clients = [] self.known_endpoint_urls = [] self.known_endpoint_names = [] self.stats_loop_count = 0 self.tm = TopologyManager(self.clients) self.spm = StatsPrintManager( self.ss, self.ps, self.tm.topology_stats, self.clients) self.sscm = SummaryStatsCsvManager(self.ss, self.ps) self.vscm = ValidatorStatsCsvManager(self.clients)
def __init__(self, endpointmanager): self.epm = endpointmanager self.cp = ConsolePrint() self.ss = SystemStats() self.ps = PlatformStats() self.psis = PlatformIntervalStats() self.ps.psis = self.psis self.previous_net_bytes_recv = 0 self.previous_net_bytes_sent = 0 self.clients = [] self.known_endpoint_names = [] self.endpoints = {} self.stats_loop_count = 0 self.tm = TopologyManager(self.clients) self.bm = BranchManager(self.epm, Agent(reactor)) stats_providers = [ self.ss, self.ps, self.tm.topology_stats, self.bm, self.clients ] self.spm = StatsPrintManager(*stats_providers) self.ssw = StatsSnapshotWriter(*stats_providers) self.sscm = SummaryStatsCsvManager(self.ss, self.ps) self.vscm = ValidatorStatsCsvManager(self.clients)
def __init__(self, validator): Resource.__init__(self) self.Ledger = validator.Ledger self.Validator = validator self.ps = PlatformStats() self.GetPageMap = { 'block': self._handle_blk_request, 'statistics': self._handle_stat_request, 'store': self._handle_store_request, 'transaction': self._handle_txn_request, 'status': self._hdl_status_request, } self.PostPageMap = { 'default': self._msg_forward, 'forward': self._msg_forward, 'initiate': self._msg_initiate, 'command': self._do_command, 'echo': self._msg_echo } static_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static_content") self.static_content = File(static_dir)
class StatisticsPage(BasePage): def __init__(self, validator): BasePage.__init__(self, validator) self.ps = PlatformStats() def render_get(self, request, components, args): if not components: raise Error(http.BAD_REQUEST, 'missing stat family') source = components.pop(0) result = {} if source == 'journal': for domain in self.validator.stat_domains.iterkeys(): result[domain] = self.validator.stat_domains[domain]\ .get_stats() return result if source == 'node': for peer in self.validator.gossip.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer return result if source == 'platform': self.ps.get_stats() result['platform'] = self.ps.get_data_as_dict() return result if source == 'all': for domain in self.validator.stat_domains.iterkeys(): result[domain] = self.validator.stat_domains[domain]\ .get_stats() for peer in self.validator.gossip.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer self.ps.get_stats() result['platform'] = self.ps.get_data_as_dict() return result if 'journal' in args: for domain in self.validator.stat_domains.iterkeys(): result[domain] = self.validator.stat_domains[domain]\ .get_stats() if 'node' in args: for peer in self.validator.gossip.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer if 'platform' in args: self.ps.get_stats() result['platform'] = self.ps.get_data_as_dict() elif ('ledger' not in args) & ('node' not in args) \ & ('platform' not in args): return self._encode_error_response( request, http.BAD_REQUEST, 'not valid source or arg') return result
def __init__(self): self.cp = ConsolePrint() self.ss = SystemStats() self.ps = PlatformStats() self.previous_net_bytes_recv = 0 self.previous_net_bytes_sent = 0 self.clients = [] self.known_endpoint_urls = [] self.stats_loop_count = 0 self.spm = StatsPrintManager(self.ss, self.ps, self.clients) self.csv_enabled = False
def __init__(self, validator): Resource.__init__(self) self.Ledger = validator.Ledger self.Validator = validator self.ps = PlatformStats() self.GetPageMap = { 'block': self._handle_blk_request, 'statistics': self._handle_stat_request, 'store': self._handle_store_request, 'transaction': self._handle_txn_request, 'status': self._hdl_status_request, } self.PostPageMap = { 'default': self._msg_forward, 'forward': self._msg_forward, 'initiate': self._msg_initiate, 'command': self._do_command, 'echo': self._msg_echo } static_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), "static_content") self.static_content = File(static_dir)
def __init__(self, endpointmanager): self.epm = endpointmanager self.cp = ConsolePrint() self.ss = SystemStats() self.ps = PlatformStats() self.psis = PlatformIntervalStats() self.ps.psis = self.psis self.previous_net_bytes_recv = 0 self.previous_net_bytes_sent = 0 self.clients = [] self.known_endpoint_names = [] self.endpoints = {} self.stats_loop_count = 0 self.tm = TopologyManager(self.clients) self.bm = BranchManager(self.epm, Agent(reactor)) stats_providers = [self.ss, self.ps, self.tm.topology_stats, self.bm, self.clients] self.spm = StatsPrintManager(*stats_providers) self.ssw = StatsSnapshotWriter(*stats_providers) self.sscm = SummaryStatsCsvManager(self.ss, self.ps) self.vscm = ValidatorStatsCsvManager(self.clients)
class StatisticsPage(BasePage): def __init__(self, validator): BasePage.__init__(self, validator) self.ps = PlatformStats() def render_get(self, request, components, args): if not components: raise Error(http.BAD_REQUEST, 'missing stat family') source = components.pop(0) result = {} if source == 'ledger': for domain in self.Ledger.gossip.StatDomains.iterkeys(): result[domain] = self.Ledger.gossip.StatDomains[domain]\ .get_stats() return result if source == 'node': for peer in self.Ledger.gossip.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer return result if source == 'platform': self.ps.get_stats() result['platform'] = self.ps.get_data_as_dict() return result if source == 'all': for domain in self.Ledger.gossip.StatDomains.iterkeys(): result[domain] = self.Ledger.gossip.StatDomains[domain]\ .get_stats() for peer in self.Ledger.gossip.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer self.ps.get_stats() result['platform'] = self.ps.get_data_as_dict() return result if 'ledger' in args: for domain in self.Ledger.gossip.StatDomains.iterkeys(): result[domain] = self.Ledger.gossip.StatDomains[domain]\ .get_stats() if 'node' in args: for peer in self.Ledger.gossip.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer if 'platform' in args: self.ps.get_stats() result['platform'] = self.ps.get_data_as_dict() elif ('ledger' not in args) & ('node' not in args) \ & ('platform' not in args): return self._encode_error_response(request, http.BAD_REQUEST, 'not valid source or arg') return result
def __init__(self): self.cp = ConsolePrint() self.ss = SystemStats() self.ps = PlatformStats() self.last_net_bytes_recv = 0 self.last_net_bytes_sent = 0 self.clients = [] self.known_endpoint_urls = [] self.stats_loop_count = 0 self.spm = StatsPrintManager(self.ss, self.ps, self.clients) self.csv_enabled = False
def __init__(self, validator): Resource.__init__(self) self.Ledger = validator.Ledger self.Validator = validator self.ps = PlatformStats() self.GetPageMap = { 'block': self._handleblkrequest, 'statistics': self._handlestatrequest, 'store': self._handlestorerequest, 'transaction': self._handletxnrequest, 'status': self._hdl_status_request, } self.PostPageMap = { 'default': self._msgforward, 'forward': self._msgforward, 'initiate': self._msginitiate, 'command': self._docommand, 'echo': self._msgecho }
class StatsManager(object): def __init__(self): self.cp = ConsolePrint() self.ss = SystemStats() self.ps = PlatformStats() self.psis = PlatformIntervalStats() self.ps.psis = self.psis self.previous_net_bytes_recv = 0 self.previous_net_bytes_sent = 0 self.clients = [] self.known_endpoint_urls = [] self.known_endpoint_names = [] self.stats_loop_count = 0 self.spm = StatsPrintManager(self.ss, self.ps, self.clients) self.sscm = SummaryStatsCsvManager(self.ss, self.ps) self.vscm = ValidatorStatsCsvManager(self.clients) def initialize_client_list(self, endpoints): # add validator stats client for each endpoint for val_num, endpoint in enumerate(endpoints.values()): url = 'http://{0}:{1}'.format( endpoint["Host"], endpoint["HttpPort"]) try: c = StatsClient(val_num, url) c.name = endpoint["Name"] self.known_endpoint_names.append(endpoint["Name"]) except: e = sys.exc_info()[0] print ("error creating stats clients: ", e) self.clients.append(c) def update_client_list(self, endpoints): # add validator stats client for each endpoint name for val_num, endpoint in enumerate(endpoints.values()): if endpoint["Name"] not in self.known_endpoint_names: val_num = len(self.known_endpoint_names) url = 'http://{0}:{1}'.format( endpoint["Host"], endpoint["HttpPort"]) c = StatsClient(val_num, url) c.name = endpoint["Name"] self.clients.append(c) self.known_endpoint_names.append(endpoint["Name"]) def stats_loop(self): self.process_stats(self.clients) self.print_stats() self.csv_write() for c in self.clients: c.stats_request() self.stats_loop_count += 1 return def stats_loop_done(self, result): reactor.stop() def stats_loop_failed(self, failure): print failure.getBriefTraceback() reactor.stop() def process_stats(self, statsclients): self.ss.known_validators = len(statsclients) self.ss.active_validators = 0 self.ss.collect_stats(statsclients) self.ss.calculate_stats() self.ps.get_stats() psr = {"platform": self.ps.get_data_as_dict()} self.psis.calculate_interval_stats(psr) def print_stats(self): self.spm.print_stats() def csv_init(self, enable_summary, enable_validator): if enable_summary is True: self.sscm.initialize() if enable_validator is True: self.vscm.initialize() def csv_write(self): self.sscm.write_stats() self.vscm.write_stats() def csv_stop(self): self.sscm.stop() self.vscm.stop() def stats_stop(self): print "StatsManager is stopping" self.cp.cpstop() self.csv_stop()
class StatsManager(object): def __init__(self): self.cp = ConsolePrint() self.ss = SystemStats() self.ps = PlatformStats() self.psis = PlatformIntervalStats() self.ps.psis = self.psis self.previous_net_bytes_recv = 0 self.previous_net_bytes_sent = 0 self.clients = [] self.known_endpoint_urls = [] self.known_endpoint_names = [] self.stats_loop_count = 0 self.tm = TopologyManager(self.clients) self.spm = StatsPrintManager( self.ss, self.ps, self.tm.topology_stats, self.clients) self.sscm = SummaryStatsCsvManager(self.ss, self.ps) self.vscm = ValidatorStatsCsvManager(self.clients) def initialize_client_list(self, endpoints): # add validator stats client for each endpoint for val_num, endpoint in enumerate(endpoints.values()): url = 'http://{0}:{1}'.format( endpoint["Host"], endpoint["HttpPort"]) try: c = StatsClient(val_num, url) c.name = endpoint["Name"] self.known_endpoint_names.append(endpoint["Name"]) except: e = sys.exc_info()[0] print ("error creating stats clients: ", e) self.clients.append(c) def update_client_list(self, endpoints): # add validator stats client for each endpoint name for val_num, endpoint in enumerate(endpoints.values()): if endpoint["Name"] not in self.known_endpoint_names: val_num = len(self.known_endpoint_names) url = 'http://{0}:{1}'.format( endpoint["Host"], endpoint["HttpPort"]) c = StatsClient(val_num, url) c.name = endpoint["Name"] self.clients.append(c) self.known_endpoint_names.append(endpoint["Name"]) def stats_loop(self): self.process_stats(self.clients) self.print_stats() self.csv_write() for c in self.clients: c.stats_request() self.stats_loop_count += 1 return def stats_loop_done(self, result): reactor.stop() def stats_loop_error(self, failure): self.cp.cpstop() print failure reactor.stop() def process_stats(self, statsclients): self.ss.known_validators = len(statsclients) self.ss.active_validators = 0 self.ss.collect_stats(statsclients) self.ss.calculate_stats() self.ps.get_stats() psr = {"platform": self.ps.get_data_as_dict()} self.psis.calculate_interval_stats(psr) self.tm.update_topology() def print_stats(self): self.spm.print_stats() def csv_init(self, enable_summary, enable_validator): if enable_summary is True: self.sscm.initialize() if enable_validator is True: self.vscm.initialize() def csv_write(self): self.sscm.write_stats() self.vscm.write_stats() def csv_stop(self): self.sscm.stop() self.vscm.stop() def stats_stop(self): print "StatsManager is stopping" self.cp.cpstop() self.csv_stop()
class RootPage(Resource): isLeaf = True def __init__(self, validator): Resource.__init__(self) self.Ledger = validator.Ledger self.Validator = validator self.ps = PlatformStats() self.GetPageMap = { 'block': self._handle_blk_request, 'statistics': self._handle_stat_request, 'store': self._handle_store_request, 'transaction': self._handle_txn_request, 'status': self._hdl_status_request, } self.PostPageMap = { 'default': self._msg_forward, 'forward': self._msg_forward, 'initiate': self._msg_initiate, 'command': self._do_command, 'echo': self._msg_echo } static_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static_content") self.static_content = File(static_dir) def error_response(self, request, response, *msgargs): """ Generate a common error response for broken requests """ request.setResponseCode(response) msg = msgargs[0].format(*msgargs[1:]) if response > 400: logger.warn(msg) elif response > 300: logger.debug(msg) return "" if request.method == 'HEAD' else (msg + '\n') def errback(self, failure, request): failure.printTraceback() request.processingFailed(failure) return None def do_get(self, request): """ Handle a GET request on the HTTP interface. Three paths are accepted: /store[/<storename>[/<key>|*]] /block[/<blockid>] /transaction[/<txnid>] """ # pylint: disable=invalid-name # split the request path removing leading duplicate slashes components = request.path.split('/') while components and components[0] == '': components.pop(0) prefix = components.pop(0) if components else 'error' if prefix not in self.GetPageMap: # attempt to serve static content if present. resource = self.static_content.getChild(request.path[1:], request) return resource.render(request) test_only = (request.method == 'HEAD') try: response = self.GetPageMap[prefix](components, request.args, test_only) if test_only: return '' cbor = (request.getHeader('Accept') == 'application/cbor') if cbor: request.responseHeaders.addRawHeader(b"content-type", b"application/cbor") return dict2cbor(response) request.responseHeaders.addRawHeader(b"content-type", b"application/json") pretty = 'p' in request.args if pretty: result = pretty_print_dict(response) + '\n' else: result = dict2json(response) return result except Error as e: return self.error_response( request, int(e.status), 'exception while processing http request {0}; {1}', request.path, str(e)) except: logger.warn('error processing http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response(request, http.BAD_REQUEST, 'error processing http request {0}', request.path) def do_post(self, request): """ Handle two types of HTTP POST requests: - gossip messages. relayed to the gossip network as is - validator command and control (/command) """ # break the path into its component parts components = request.path.split('/') while components and components[0] == '': components.pop(0) prefix = components.pop(0) if components else 'error' if prefix not in self.PostPageMap: prefix = 'default' encoding = request.getHeader('Content-Type') data = request.content.getvalue() # process non-gossip API requests if prefix == 'command': try: if encoding == 'application/json': minfo = json2dict(data) else: return self.error_response(request, http.BAD_REQUEST, 'bad message encoding, {0}', encoding) except: logger.info('exception while decoding http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response( request, http.BAD_REQUEST, 'unable to decode incoming request {0}', data) # process /command try: response = self.PostPageMap[prefix](request, components, minfo) request.responseHeaders.addRawHeader("content-type", encoding) result = dict2json(response) return result except Error as e: return self.error_response( request, int(e.status), 'exception while processing request {0}; {1}', request.path, str(e)) except: logger.info('exception while processing http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response( request, http.BAD_REQUEST, 'error processing http request {0}', request.path) else: try: if encoding == 'application/json': minfo = json2dict(data) elif encoding == 'application/cbor': minfo = cbor2dict(data) else: return self.error_response( request, http.BAD_REQUEST, 'unknown message encoding, {0}', encoding) typename = minfo.get('__TYPE__', '**UNSPECIFIED**') if typename not in self.Ledger.MessageHandlerMap: return self.error_response( request, http.BAD_REQUEST, 'received request for unknown message type, {0}', typename) msg = self.Ledger.MessageHandlerMap[typename][0](minfo) except: logger.info('exception while decoding http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response( request, http.BAD_REQUEST, 'unabled to decode incoming request {0}', data) # determine if the message contains a valid transaction before # we send the message to the network # we need to start with a copy of the message due to cases # where side effects of the validity check may impact objects # related to msg mymsg = copy.deepcopy(msg) if hasattr(mymsg, 'Transaction') and mymsg.Transaction is not None: mytxn = mymsg.Transaction logger.info( 'starting local validation ' 'for txn id: %s type: %s', mytxn.Identifier, mytxn.TransactionTypeName) block_id = self.Ledger.MostRecentCommittedBlockID real_store_map = self.Ledger.GlobalStoreMap.get_block_store( block_id) temp_store_map = \ global_store_manager.BlockStore(real_store_map) if not temp_store_map: logger.info('no store map for block %s', block_id) return self.error_response( request, http.BAD_REQUEST, 'unable to validate enclosed transaction {0}', data) transaction_type = mytxn.TransactionTypeName if transaction_type not in temp_store_map.TransactionStores: logger.info('transaction type %s not in global store map', transaction_type) return self.error_response( request, http.BAD_REQUEST, 'unable to validate enclosed transaction {0}', data) # clone a copy of the ledger's message queue so we can # temporarily play forward all locally submitted yet # uncommitted transactions my_queue = copy.deepcopy(self.Ledger.MessageQueue) # apply any enqueued messages while len(my_queue) > 0: qmsg = my_queue.pop() if qmsg and \ qmsg.MessageType in self.Ledger.MessageHandlerMap: if (hasattr(qmsg, 'Transaction') and qmsg.Transaction is not None): my_store = temp_store_map.get_transaction_store( qmsg.Transaction.TransactionTypeName) if qmsg.Transaction.is_valid(my_store): myqtxn = copy.copy(qmsg.Transaction) myqtxn.apply(my_store) # apply any local pending transactions for txn_id in self.Ledger.PendingTransactions.iterkeys(): pend_txn = self.Ledger.TransactionStore[txn_id] my_store = temp_store_map.get_transaction_store( pend_txn.TransactionTypeName) if pend_txn and pend_txn.is_valid(my_store): my_pend_txn = copy.copy(pend_txn) logger.debug( 'applying pending transaction ' '%s to temp store', txn_id) my_pend_txn.apply(my_store) # determine validity of the POSTed transaction against our # new temporary state my_store = temp_store_map.get_transaction_store( mytxn.TransactionTypeName) try: mytxn.check_valid(my_store) except InvalidTransactionError as e: logger.info( 'submitted transaction fails transaction ' 'family validation check: %s; %s', request.path, mymsg.dump()) return self.error_response( request, http.BAD_REQUEST, "enclosed transaction failed transaction " "family validation check: {}".format(str(e)), data) except: logger.info( 'submitted transaction is ' 'not valid %s; %s; %s', request.path, mymsg.dump(), traceback.format_exc(20)) return self.error_response( request, http.BAD_REQUEST, "enclosed transaction is not valid", data) logger.info('transaction %s is valid', msg.Transaction.Identifier) # and finally execute the associated method # and send back the results try: response = self.PostPageMap[prefix](request, components, msg) request.responseHeaders.addRawHeader("content-type", encoding) if encoding == 'application/json': result = dict2json(response.dump()) else: result = dict2cbor(response.dump()) return result except Error as e: return self.error_response( request, int(e.status), 'exception while processing request {0}; {1}', request.path, str(e)) except: logger.info('exception while processing http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response( request, http.BAD_REQUEST, 'error processing http request {0}', request.path) def final(self, message, request): request.write(message) try: request.finish() except RuntimeError: logger.error("No connection when request.finish called") def render_GET(self, request): # pylint: disable=invalid-name d = threads.deferToThread(self.do_get, request) d.addCallback(self.final, request) d.addErrback(self.errback, request) return server.NOT_DONE_YET def render_POST(self, request): # pylint: disable=invalid-name d = threads.deferToThread(self.do_post, request) d.addCallback(self.final, request) d.addErrback(self.errback, request) return server.NOT_DONE_YET def _msg_forward(self, request, components, msg): """ Forward a signed message through the gossip network. """ self.Ledger.handle_message(msg) return msg def _msg_initiate(self, request, components, msg): """ Sign and echo a message """ if request.getClientIP() != '127.0.0.1': raise Error( http.NOT_ALLOWED, '{0} not authorized for message initiation'.format( request.getClientIP())) if isinstance(msg, transaction_message.TransactionMessage): msg.Transaction.sign_from_node(self.Ledger.LocalNode) msg.sign_from_node(self.Ledger.LocalNode) self.Ledger.handle_message(msg) return msg def _msg_echo(self, request, components, msg): """ Sign and echo a message """ return msg def _do_command(self, request, components, cmd): """ Process validator control commands """ if cmd['action'] == 'start': if self.Validator.delaystart is True: self.Validator.delaystart = False logger.info("command received : %s", cmd['action']) cmd['action'] = 'started' else: logger.warn("validator startup not delayed") cmd['action'] = 'running' else: logger.warn("unknown command received") cmd['action'] = 'startup failed' return cmd def _handle_store_request(self, path_components, args, test_only): """ Handle a store request. There are four types of requests: empty path -- return a list of known stores store name -- return a list of the keys in the store store name, key == '*' -- return a complete dump of all keys in the store store name, key != '*' -- return the data associated with the key """ if not self.Ledger.GlobalStore: raise Error(http.BAD_REQUEST, 'no global store') block_id = self.Ledger.MostRecentCommittedBlockID if 'blockid' in args: block_id = args.get('blockid').pop(0) storemap = self.Ledger.GlobalStoreMap.get_block_store(block_id) if not storemap: raise Error(http.BAD_REQUEST, 'no store map for block <{0}>'.format(block_id)) if len(path_components) == 0: return storemap.TransactionStores.keys() store_name = '/' + path_components.pop(0) if store_name not in storemap.TransactionStores: raise Error(http.BAD_REQUEST, 'no such store <{0}>'.format(store_name)) store = storemap.get_transaction_store(store_name) if len(path_components) == 0: return store.keys() key = path_components[0] if key == '*': if 'delta' in args and args.get('delta').pop(0) == '1': return store.dump(True) return store.compose() if key not in store: raise Error(http.BAD_REQUEST, 'no such key {0}'.format(key)) return store[key] def _handle_blk_request(self, path_components, args, test_only): """ Handle a block request. There are three types of requests: empty path -- return a list of the committed block ids blockid -- return the contents of the specified block blockid and fieldname -- return the specific field within the block The request may specify additional parameters: blockcount -- the total number of blocks to return (newest to oldest) Blocks are returned newest to oldest. """ if not path_components: count = 0 if 'blockcount' in args: count = int(args.get('blockcount').pop(0)) block_ids = self.Ledger.committed_block_ids(count) return block_ids block_id = path_components.pop(0) if block_id not in self.Ledger.BlockStore: raise Error(http.BAD_REQUEST, 'unknown block {0}'.format(block_id)) binfo = self.Ledger.BlockStore[block_id].dump() binfo['Identifier'] = block_id if not path_components: return binfo field = path_components.pop(0) if field not in binfo: raise Error(http.BAD_REQUEST, 'unknown block field {0}'.format(field)) return binfo[field] def _handle_txn_request(self, path_components, args, test_only): """ Handle a transaction request. There are four types of requests: empty path -- return a list of the committed transactions ids txnid -- return the contents of the specified transaction txnid and field name -- return the contents of the specified transaction txnid and HEAD request -- return success only if the transaction has been committed 404 -- transaction does not exist 302 -- transaction exists but has not been committed 200 -- transaction has been committed The request may specify additional parameters: blockcount -- the number of blocks (newest to oldest) from which to pull txns Transactions are returned from oldest to newest. """ if len(path_components) == 0: blkcount = 0 if 'blockcount' in args: blkcount = int(args.get('blockcount').pop(0)) txnids = [] blockids = self.Ledger.committed_block_ids(blkcount) while blockids: blockid = blockids.pop() txnids.extend(self.Ledger.BlockStore[blockid].TransactionIDs) return txnids txnid = path_components.pop(0) if txnid not in self.Ledger.TransactionStore: raise Error(http.NOT_FOUND, 'no such transaction {0}'.format(txnid)) txn = self.Ledger.TransactionStore[txnid] if test_only: if txn.Status == transaction.Status.committed: return None else: raise Error(http.FOUND, 'transaction not committed {0}'.format(txnid)) tinfo = txn.dump() tinfo['Identifier'] = txnid tinfo['Status'] = txn.Status if txn.Status == transaction.Status.committed: tinfo['InBlock'] = txn.InBlock if not path_components: return tinfo field = path_components.pop(0) if field not in tinfo: raise Error(http.BAD_REQUEST, 'unknown transaction field {0}'.format(field)) return tinfo[field] def _handle_stat_request(self, path_components, args, testonly): if not path_components: raise Error(http.BAD_REQUEST, 'missing stat family') result = dict() source = path_components.pop(0) if source == 'ledger': for domain in self.Ledger.StatDomains.iterkeys(): result[domain] = self.Ledger.StatDomains[domain].get_stats() return result if source == 'node': for peer in self.Ledger.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer return result if source == 'platform': result['platform'] = self.ps.get_data_as_dict() return result if source == 'all': for domain in self.Ledger.StatDomains.iterkeys(): result[domain] = self.Ledger.StatDomains[domain].get_stats() for peer in self.Ledger.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer result['platform'] = self.ps.get_data_as_dict() return result if 'ledger' in args: for domain in self.Ledger.StatDomains.iterkeys(): result[domain] = self.Ledger.StatDomains[domain].get_stats() if 'node' in args: for peer in self.Ledger.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer if 'platform' in args: result['platform'] = self.ps.get_data_as_dict() elif ('ledger' not in args) & ('node' not in args) \ & ('platform' not in args): raise Error(http.NOT_FOUND, 'not valid source or arg') return result def _hdl_status_request(self, pathcomponents, args, testonly): result = dict() result['Status'] = self.Validator.status result['Domain'] = self.Validator.EndpointDomain result['Name'] = self.Ledger.LocalNode.Name result['HttpPort'] = self.Validator.Config.get('HttpPort', None) result['Host'] = self.Ledger.LocalNode.NetHost result['NodeIdentifier'] = self.Ledger.LocalNode.Identifier result['Port'] = self.Ledger.LocalNode.NetPort result['Peers'] = [ x.Name for x in self.Ledger.peer_list(allflag=False) ] return result
class StatsManager(object): def __init__(self): self.cp = ConsolePrint() self.ss = SystemStats() self.ps = PlatformStats() self.previous_net_bytes_recv = 0 self.previous_net_bytes_sent = 0 self.clients = [] self.known_endpoint_urls = [] self.stats_loop_count = 0 self.spm = StatsPrintManager(self.ss, self.ps, self.clients) self.csv_enabled = False def initialize_client_list(self, endpoint_urls): # add validator stats client for each url in endpoint_urls self.known_endpoint_urls = list(endpoint_urls) for val_num, url in enumerate(self.known_endpoint_urls): try: c = StatsClient(val_num, url) except: e = sys.exc_info()[0] print("error creating stats clients: ", e) self.clients.append(c) def update_client_list(self, endpoint_urls): # add validator stats client for each new url in endpoint_urls for url in endpoint_urls: if url not in self.known_endpoint_urls: val_num = len(self.known_endpoint_urls) c = StatsClient(val_num, url) self.clients.append(c) self.known_endpoint_urls.append(url) def stats_loop(self): self.process_stats(self.clients) self.print_stats() self.write_stats() for c in self.clients: c.stats_request() self.stats_loop_count += 1 return def stats_loop_done(self, result): reactor.stop() def stats_loop_failed(self, failure): print failure.getBriefTraceback() reactor.stop() def process_stats(self, statsclients): self.ss.known_validators = len(statsclients) self.ss.active_validators = 0 self.ss.collect_stats(statsclients) self.ss.calculate_stats() self.ps.get_stats() def print_stats(self): self.spm.print_stats() def csv_init(self): self.csv_enabled = True self.csvmgr = CsvManager() filename = "stats_client_" + str(int(time.time())) + ".csv" self.csvmgr.open_csv_file(filename) header = self.ss.get_names() self.csvmgr.csv_append(header) header = self.ps.get_names() self.csvmgr.csv_write_header(header) def write_stats(self): if self.csv_enabled: data = self.ss.get_data() self.csvmgr.csv_append(data) data = self.ps.get_data() self.csvmgr.csv_write_data(data) def stats_stop(self): print "StatsManager is stopping" self.cp.cpstop() if self.csv_enabled: self.csvmgr.close_csv_file()
def __init__(self, validator): BasePage.__init__(self, validator) self.ps = PlatformStats()
class StatsManager(object): def __init__(self): self.cp = ConsolePrint() self.ss = SystemStats() self.ps = PlatformStats() self.last_net_bytes_recv = 0 self.last_net_bytes_sent = 0 self.clients = [] self.known_endpoint_urls = [] self.stats_loop_count = 0 self.spm = StatsPrintManager(self.ss, self.ps, self.clients) self.csv_enabled = False def initialize_client_list(self, endpoint_urls): # add validator stats client for each url in endpoint_urls self.known_endpoint_urls = list(endpoint_urls) for val_num, url in enumerate(self.known_endpoint_urls): c = StatsClient(val_num, url) self.clients.append(c) def update_client_list(self, endpoint_urls): # add validator stats client for each new url in endpoint_urls for url in endpoint_urls: if url not in self.known_endpoint_urls: val_num = len(self.known_endpoint_urls) c = StatsClient(val_num, url) self.clients.append(c) self.known_endpoint_urls.append(url) def stats_loop(self): self.process_stats(self.clients) self.print_stats() self.write_stats() for c in self.clients: c.stats_request() self.stats_loop_count += 1 return def stats_loop_done(self, result): print "Stats loop done." reactor.stop() def stats_loop_failed(self, failure): print failure.getBriefTraceback() reactor.stop() def process_stats(self, statsclients): self.ss.known_validators = len(statsclients) self.ss.active_validators = 0 self.ss.collect_stats(statsclients) self.ss.calculate_stats() self.ps.get_stats() self.this_net_bytes_recv = self.ps.net_stats.bytes_recv - \ self.last_net_bytes_recv self.last_net_bytes_recv = self.ps.net_stats.bytes_recv self.this_net_bytes_sent = self.ps.net_stats.bytes_sent - \ self.last_net_bytes_sent self.last_net_bytes_sent = self.ps.net_stats.bytes_sent def print_stats(self): self.spm.print_stats() def csv_init(self): self.csv_enabled = True self.csvmgr = CsvManager() filename = "stats_client_" + str(int(time.time())) + ".csv" self.csvmgr.open_csv_file(filename) header = self.ss.get_names() self.csvmgr.csv_append(header) header = self.ps.get_names() self.csvmgr.csv_write_header(header) def write_stats(self): if self.csv_enabled: data = self.ss.get_data() self.csvmgr.csv_append(data) data = self.ps.get_data() self.csvmgr.csv_write_data(data) def stats_stop(self): print "StatsManager is stopping" self.cp.cpstop() if self.csv_enabled: self.csvmgr.close_csv_file()
class RootPage(Resource): isLeaf = True def __init__(self, validator): Resource.__init__(self) self.Ledger = validator.Ledger self.Validator = validator self.ps = PlatformStats() self.GetPageMap = { 'block': self._handleblkrequest, 'statistics': self._handlestatrequest, 'store': self._handlestorerequest, 'transaction': self._handletxnrequest, 'status': self._hdl_status_request, } self.PostPageMap = { 'default': self._msgforward, 'forward': self._msgforward, 'initiate': self._msginitiate, 'command': self._docommand, 'echo': self._msgecho } def error_response(self, request, response, *msgargs): """ Generate a common error response for broken requests """ request.setResponseCode(response) msg = msgargs[0].format(*msgargs[1:]) if response > 400: logger.warn(msg) elif response > 300: logger.debug(msg) return "" if request.method == 'HEAD' else (msg + '\n') def render_GET(self, request): """ Handle a GET request on the HTTP interface. Three paths are accepted: /store[/<storename>[/<key>|*]] /block[/<blockid>] /transaction[/<txnid>] """ # pylint: disable=invalid-name # split the request path removing leading duplicate slashes components = request.path.split('/') while components and components[0] == '': components.pop(0) prefix = components.pop(0) if components else 'error' if prefix not in self.GetPageMap: return self.error_response(request, http.BAD_REQUEST, 'unknown request {0}', request.path) testonly = (request.method == 'HEAD') try: response = self.GetPageMap[prefix](components, request.args, testonly) if testonly: return '' cbor = (request.getHeader('Accept') == 'application/cbor') if cbor: request.responseHeaders.addRawHeader(b"content-type", b"application/cbor") return dict2cbor(response) request.responseHeaders.addRawHeader(b"content-type", b"application/json") pretty = 'p' in request.args if pretty: result = pretty_print_dict(response) + '\n' else: result = dict2json(response) return result except Error as e: return self.error_response( request, int(e.status), 'exception while processing http request {0}; {1}', request.path, str(e)) except: logger.warn('error processing http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response(request, http.BAD_REQUEST, 'error processing http request {0}', request.path) def render_POST(self, request): """ Handle two types of HTTP POST requests: - gossip messages. relayed to the gossip network as is - validator command and control (/command) """ # pylint: disable=invalid-name # break the path into its component parts components = request.path.split('/') while components and components[0] == '': components.pop(0) prefix = components.pop(0) if components else 'error' if prefix not in self.PostPageMap: prefix = 'default' encoding = request.getHeader('Content-Type') data = request.content.getvalue() # process non-gossip API requests if prefix == 'command': try: if encoding == 'application/json': minfo = json2dict(data) else: return self.error_response(request, http.BAD_REQUEST, 'bad message encoding, {0}', encoding) except: logger.info('exception while decoding http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response( request, http.BAD_REQUEST, 'unable to decode incoming request {0}', data) # process /command try: response = self.PostPageMap[prefix](request, components, minfo) request.responseHeaders.addRawHeader("content-type", encoding) result = dict2json(response) return result except Error as e: return self.error_response( request, int(e.status), 'exception while processing request {0}; {1}', request.path, str(e)) except: logger.info('exception while processing http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response(request, http.BAD_REQUEST, 'error processing http request {0}', request.path) else: try: if encoding == 'application/json': minfo = json2dict(data) elif encoding == 'application/cbor': minfo = cbor2dict(data) else: return self.error_response(request, http.BAD_REQUEST, 'unknown message encoding, {0}', encoding) typename = minfo.get('__TYPE__', '**UNSPECIFIED**') if typename not in self.Ledger.MessageHandlerMap: return self.error_response( request, http.BAD_REQUEST, 'received request for unknown message type, {0}', typename) msg = self.Ledger.MessageHandlerMap[typename][0](minfo) except: logger.info('exception while decoding http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response( request, http.BAD_REQUEST, 'unabled to decode incoming request {0}', data) # determine if the message contains a valid transaction before # we send the message to the network # we need to start with a copy of the message due to cases # where side effects of the validity check may impact objects # related to msg mymsg = copy.deepcopy(msg) if hasattr(mymsg, 'Transaction') and mymsg.Transaction is not None: mytxn = mymsg.Transaction logger.info('starting local validation ' 'for txn id: %s type: %s', mytxn.Identifier, mytxn.TransactionTypeName) blockid = self.Ledger.MostRecentCommittedBlockID realstoremap = self.Ledger.GlobalStoreMap.get_block_store( blockid) tempstoremap = global_store_manager.BlockStore(realstoremap) if not tempstoremap: logger.info('no store map for block %s', blockid) return self.error_response( request, http.BAD_REQUEST, 'unable to validate enclosed transaction {0}', data) transtype = mytxn.TransactionTypeName if transtype not in tempstoremap.TransactionStores: logger.info('transaction type %s not in global store map', transtype) return self.error_response( request, http.BAD_REQUEST, 'unable to validate enclosed transaction {0}', data) # clone a copy of the ledger's message queue so we can # temporarily play forward all locally submitted yet # uncommitted transactions myqueue = copy.deepcopy(self.Ledger.MessageQueue) # apply any enqueued messages while len(myqueue) > 0: qmsg = myqueue.pop() if qmsg and \ qmsg.MessageType in self.Ledger.MessageHandlerMap: if (hasattr(qmsg, 'Transaction') and qmsg.Transaction is not None): mystore = tempstoremap.get_transaction_store( qmsg.Transaction.TransactionTypeName) if qmsg.Transaction.is_valid(mystore): myqtxn = copy.copy(qmsg.Transaction) myqtxn.apply(mystore) # apply any local pending transactions for txnid in self.Ledger.PendingTransactions.iterkeys(): pendtxn = self.Ledger.TransactionStore[txnid] mystore = tempstoremap.get_transaction_store( pendtxn.TransactionTypeName) if pendtxn and pendtxn.is_valid(mystore): mypendtxn = copy.copy(pendtxn) logger.debug('applying pending transaction ' '%s to temp store', txnid) mypendtxn.apply(mystore) # determine validity of the POSTed transaction against our # new temporary state mystore = tempstoremap.get_transaction_store( mytxn.TransactionTypeName) try: mytxn.check_valid(mystore) except InvalidTransactionError as e: logger.info('submitted transaction fails transaction ' 'family validation check: %s; %s', request.path, mymsg.dump()) return self.error_response( request, http.BAD_REQUEST, "enclosed transaction failed transaction " "family validation check: {}".format(str(e)), data) except: logger.info('submitted transaction is ' 'not valid %s; %s; %s', request.path, mymsg.dump(), traceback.format_exc(20)) return self.error_response( request, http.BAD_REQUEST, "enclosed transaction is not valid", data) logger.info('transaction %s is valid', msg.Transaction.Identifier) # and finally execute the associated method # and send back the results try: response = self.PostPageMap[prefix](request, components, msg) request.responseHeaders.addRawHeader("content-type", encoding) if encoding == 'application/json': result = dict2json(response.dump()) else: result = dict2cbor(response.dump()) return result except Error as e: return self.error_response( request, int(e.status), 'exception while processing request {0}; {1}', request.path, str(e)) except: logger.info('exception while processing http request %s; %s', request.path, traceback.format_exc(20)) return self.error_response(request, http.BAD_REQUEST, 'error processing http request {0}', request.path) def _msgforward(self, request, components, msg): """ Forward a signed message through the gossip network. """ self.Ledger.handle_message(msg) return msg def _msginitiate(self, request, components, msg): """ Sign and echo a message """ if request.getClientIP() != '127.0.0.1': raise Error(http.NOT_ALLOWED, '{0} not authorized for message initiation'.format( request.getClientIP())) if isinstance(msg, transaction_message.TransactionMessage): msg.Transaction.sign_from_node(self.Ledger.LocalNode) msg.sign_from_node(self.Ledger.LocalNode) self.Ledger.handle_message(msg) return msg def _msgecho(self, request, components, msg): """ Sign and echo a message """ return msg def _docommand(self, request, components, cmd): """ Process validator control commands """ if cmd['action'] == 'start': if self.Validator.delaystart is True: self.Validator.delaystart = False logger.info("command received : %s", cmd['action']) cmd['action'] = 'started' else: logger.warn("validator startup not delayed") cmd['action'] = 'running' else: logger.warn("unknown command received") cmd['action'] = 'startup failed' return cmd def _handlestorerequest(self, pathcomponents, args, testonly): """ Handle a store request. There are four types of requests: empty path -- return a list of known stores store name -- return a list of the keys in the store store name, key == '*' -- return a complete dump of all keys in the store store name, key != '*' -- return the data associated with the key """ if not self.Ledger.GlobalStore: raise Error(http.BAD_REQUEST, 'no global store') blockid = self.Ledger.MostRecentCommittedBlockID if 'blockid' in args: blockid = args.get('blockid').pop(0) storemap = self.Ledger.GlobalStoreMap.get_block_store(blockid) if not storemap: raise Error(http.BAD_REQUEST, 'no store map for block <{0}>'.format(blockid)) if len(pathcomponents) == 0: return storemap.TransactionStores.keys() storename = '/' + pathcomponents.pop(0) if storename not in storemap.TransactionStores: raise Error(http.BAD_REQUEST, 'no such store <{0}>'.format(storename)) store = storemap.get_transaction_store(storename) if len(pathcomponents) == 0: return store.keys() key = pathcomponents[0] if key == '*': if 'delta' in args and args.get('delta').pop(0) == '1': return store.dump(True) return store.compose() if key not in store: raise Error(http.BAD_REQUEST, 'no such key {0}'.format(key)) return store[key] def _handleblkrequest(self, pathcomponents, args, testonly): """ Handle a block request. There are three types of requests: empty path -- return a list of the committed block ids blockid -- return the contents of the specified block blockid and fieldname -- return the specific field within the block The request may specify additional parameters: blockcount -- the total number of blocks to return (newest to oldest) Blocks are returned newest to oldest. """ if not pathcomponents: count = 0 if 'blockcount' in args: count = int(args.get('blockcount').pop(0)) blockids = self.Ledger.committed_block_ids(count) return blockids blockid = pathcomponents.pop(0) if blockid not in self.Ledger.BlockStore: raise Error(http.BAD_REQUEST, 'unknown block {0}'.format(blockid)) binfo = self.Ledger.BlockStore[blockid].dump() binfo['Identifier'] = blockid if not pathcomponents: return binfo field = pathcomponents.pop(0) if field not in binfo: raise Error(http.BAD_REQUEST, 'unknown block field {0}'.format(field)) return binfo[field] def _handletxnrequest(self, pathcomponents, args, testonly): """ Handle a transaction request. There are four types of requests: empty path -- return a list of the committed transactions ids txnid -- return the contents of the specified transaction txnid and field name -- return the contents of the specified transaction txnid and HEAD request -- return success only if the transaction has been committed 404 -- transaction does not exist 302 -- transaction exists but has not been committed 200 -- transaction has been committed The request may specify additional parameters: blockcount -- the number of blocks (newest to oldest) from which to pull txns Transactions are returned from oldest to newest. """ if len(pathcomponents) == 0: blkcount = 0 if 'blockcount' in args: blkcount = int(args.get('blockcount').pop(0)) txnids = [] blockids = self.Ledger.committed_block_ids(blkcount) while blockids: blockid = blockids.pop() txnids.extend(self.Ledger.BlockStore[blockid].TransactionIDs) return txnids txnid = pathcomponents.pop(0) if txnid not in self.Ledger.TransactionStore: raise Error(http.NOT_FOUND, 'no such transaction {0}'.format(txnid)) txn = self.Ledger.TransactionStore[txnid] if testonly: if txn.Status == transaction.Status.committed: return None else: raise Error(http.FOUND, 'transaction not committed {0}'.format(txnid)) tinfo = txn.dump() tinfo['Identifier'] = txnid tinfo['Status'] = txn.Status if txn.Status == transaction.Status.committed: tinfo['InBlock'] = txn.InBlock if not pathcomponents: return tinfo field = pathcomponents.pop(0) if field not in tinfo: raise Error(http.BAD_REQUEST, 'unknown transaction field {0}'.format(field)) return tinfo[field] def _handlestatrequest(self, pathcomponents, args, testonly): if not pathcomponents: raise Error(http.BAD_REQUEST, 'missing stat family') result = dict() source = pathcomponents.pop(0) if source == 'ledger': for domain in self.Ledger.StatDomains.iterkeys(): result[domain] = self.Ledger.StatDomains[domain].get_stats() return result if source == 'node': for peer in self.Ledger.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer return result if source == 'platform': result['platform'] = self.ps.get_data_as_dict() return result if source == 'all': for domain in self.Ledger.StatDomains.iterkeys(): result[domain] = self.Ledger.StatDomains[domain].get_stats() for peer in self.Ledger.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer result['platform'] = self.ps.get_data_as_dict() return result if 'ledger' in args: for domain in self.Ledger.StatDomains.iterkeys(): result[domain] = self.Ledger.StatDomains[domain].get_stats() if 'node' in args: for peer in self.Ledger.NodeMap.itervalues(): result[peer.Name] = peer.Stats.get_stats() result[peer.Name]['IsPeer'] = peer.is_peer if 'platform' in args: result['platform'] = self.ps.get_data_as_dict() elif ('ledger' not in args) & ('node' not in args) \ & ('platform' not in args): raise Error(http.NOT_FOUND, 'not valid source or arg') return result def _hdl_status_request(self, pathcomponents, args, testonly): result = dict() result['Status'] = self.Validator.status result['Domain'] = self.Validator.EndpointDomain result['Name'] = self.Ledger.LocalNode.Name result['HttpPort'] = self.Validator.Config.get('HttpPort', None) result['Host'] = self.Ledger.LocalNode.NetHost result['NodeIdentifier'] = self.Ledger.LocalNode.Identifier result['Port'] = self.Ledger.LocalNode.NetPort return result