def remove_block(self, block_hash): # Retrieve block block = Block.query(self.db_session).filter_by(hash=block_hash).first() # Revert event processors for event in Event.query(self.db_session).filter_by(block_id=block.id): for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id): event_processor = processor_class(block, event, None) event_processor.accumulation_revert(self.db_session) # Revert extrinsic processors for extrinsic in Extrinsic.query(self.db_session).filter_by(block_id=block.id): for processor_class in ProcessorRegistry().get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id): extrinsic_processor = processor_class(block, extrinsic) extrinsic_processor.accumulation_revert(self.db_session) # Revert block processors for processor_class in ProcessorRegistry().get_block_processors(): block_processor = processor_class(block) block_processor.accumulation_revert(self.db_session) # Delete events for item in Event.query(self.db_session).filter_by(block_id=block.id): self.db_session.delete(item) # Delete extrinsics for item in Extrinsic.query(self.db_session).filter_by(block_id=block.id): self.db_session.delete(item) # Delete block self.db_session.delete(block)
def get_item(self, item_id): if '-' in item_id: st = item_id.split("-") return Event.query(self.session).filter_by(event_idx=int(st[2]), block_id=int(st[1]), shard_num=int( st[0])).first() else: return Event.query( self.session).filter_by(id=item_id.split('-')[0]).first()
def get_relationships(self, include_list, item): relationships = {} if 'extrinsics' in include_list: relationships['extrinsics'] = Extrinsic.query( self.session).filter_by( block_id=item.bid, shard_num=item.shard_num).order_by('extrinsic_idx') if 'transactions' in include_list: relationships['transactions'] = Extrinsic.query( self.session).filter_by( block_id=item.bid, signed=1, shard_num=item.shard_num).order_by('extrinsic_idx') if 'inherents' in include_list: relationships['inherents'] = Extrinsic.query(self.session).filter( Extrinsic.block_id == item.bid, Extrinsic.signed == 0, Extrinsic.shard_num == item.shard_num, Extrinsic.module_id != 'relay').order_by('extrinsic_idx') if 'relay' in include_list: relationships['relay'] = Extrinsic.query(self.session).filter( Extrinsic.block_id == item.bid, Extrinsic.signed == 0, Extrinsic.shard_num == item.shard_num, Extrinsic.module_id == 'relay').order_by('extrinsic_idx') if 'events' in include_list: relationships['events'] = Event.query(self.session).filter_by( block_id=item.bid, system=0, shard_num=item.shard_num).order_by('event_idx') if 'logs' in include_list: relationships['logs'] = Log.query(self.session).filter_by( block_id=item.bid, shard_num=item.shard_num).order_by('log_idx') return relationships
def apply_filters(self, query, params): if params.get('filter[address]'): if len(params.get('filter[address]')) == 64: account_id = params.get('filter[address]') else: try: account_id = ss58_decode(params.get('filter[address]'), settings.SUBSTRATE_ADDRESS_TYPE) except ValueError: return query.filter(False) search_index = SearchIndex.query(self.session).filter( SearchIndex.index_type_id.in_([ settings.SEARCH_INDEX_BALANCETRANSFER, settings.SEARCH_INDEX_CLAIMS_CLAIMED, settings.SEARCH_INDEX_BALANCES_DEPOSIT, settings.SEARCH_INDEX_STAKING_REWARD ]), SearchIndex.account_id == account_id).order_by( SearchIndex.sorting_value.desc()) query = Event.query(self.session).filter( tuple_(Event.block_id, Event.event_idx).in_( [[s.block_id, s.event_idx] for s in search_index])).order_by(Event.block_id.desc()) return query
def process_genesis(self, block): # Set block time of parent block child_block = Block.query( self.db_session).filter_by(parent_hash=block.hash).first() block.set_datetime(child_block.datetime) # Retrieve genesis accounts genesis_account_page_count = self.substrate.get_runtime_state( module="Indices", storage_function="NextEnumSet", block_hash=block.hash).get('result', 0) # Get Accounts on EnumSet block.count_accounts_new = 0 block.count_accounts = 0 for enum_set_nr in range(0, genesis_account_page_count + 1): genesis_accounts = self.substrate.get_runtime_state( module="Indices", storage_function="EnumSet", params=[enum_set_nr], block_hash=block.hash).get('result') if genesis_accounts: block.count_accounts_new += len(genesis_accounts) block.count_accounts += len(genesis_accounts) for idx, account_id in enumerate(genesis_accounts): account_audit = AccountAudit( account_id=account_id.replace('0x', ''), block_id=block.id, extrinsic_idx=None, event_idx=None, type_id=ACCOUNT_AUDIT_TYPE_NEW) account_audit.save(self.db_session) account_index_id = enum_set_nr * 64 + idx account_index_audit = AccountIndexAudit( account_index_id=account_index_id, account_id=account_id.replace('0x', ''), block_id=block.id, extrinsic_idx=None, event_idx=None, type_id=ACCOUNT_INDEX_AUDIT_TYPE_NEW) account_index_audit.save(self.db_session) block.save(self.db_session) # Create initial session initial_session_event = NewSessionEventProcessor(block, Event(), None) initial_session_event.add_session(db_session=self.db_session, session_id=0)
def get_relationships(self, include_list, item): relationships = {} if 'events' in include_list: relationships['events'] = Event.query(self.session).filter_by( block_id=item.block_id, extrinsic_idx=item.extrinsic_idx).order_by('event_idx') return relationships
def rebuild_search_index(self): self.db_session.execute('truncate table {}'.format( SearchIndex.__tablename__)) for block in Block.query( self.db_session).order_by('id').yield_per(1000): extrinsic_lookup = {} block._accounts_new = [] block._accounts_reaped = [] for extrinsic in Extrinsic.query(self.db_session).filter_by( block_id=block.id).order_by('extrinsic_idx'): extrinsic_lookup[extrinsic.extrinsic_idx] = extrinsic # Add search index for signed extrinsics if extrinsic.address: search_index = SearchIndex( index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC, block_id=block.id, extrinsic_idx=extrinsic.extrinsic_idx, account_id=extrinsic.address) search_index.save(self.db_session) # Process extrinsic processors for processor_class in ProcessorRegistry( ).get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id): extrinsic_processor = processor_class( block=block, extrinsic=extrinsic, substrate=self.substrate) extrinsic_processor.process_search_index(self.db_session) for event in Event.query(self.db_session).filter_by( block_id=block.id).order_by('event_idx'): extrinsic = None if event.extrinsic_idx is not None: try: extrinsic = extrinsic_lookup[event.extrinsic_idx] except (IndexError, KeyError): extrinsic = None for processor_class in ProcessorRegistry( ).get_event_processors(event.module_id, event.event_id): event_processor = processor_class( block, event, extrinsic, metadata=self.metadata_store.get( block.spec_version_id), substrate=self.substrate) event_processor.process_search_index(self.db_session) self.db_session.commit()
def sequence_block(self, block, parent_block_data=None, parent_sequenced_block_data=None): sequenced_block = BlockTotal(id=block.id) # Process block processors for processor_class in ProcessorRegistry().get_block_processors(): block_processor = processor_class(block, sequenced_block, substrate=self.substrate) block_processor.sequencing_hook(self.db_session, parent_block_data, parent_sequenced_block_data) extrinsics = Extrinsic.query(self.db_session).filter_by( block_id=block.id).order_by('extrinsic_idx') for extrinsic in extrinsics: # Process extrinsic processors for processor_class in ProcessorRegistry( ).get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id): extrinsic_processor = processor_class(block, extrinsic, substrate=self.substrate) extrinsic_processor.sequencing_hook( self.db_session, parent_block_data, parent_sequenced_block_data) events = Event.query( self.db_session).filter_by(block_id=block.id).order_by('event_idx') # Process event processors for event in events: extrinsic = None if event.extrinsic_idx is not None: try: extrinsic = extrinsics[event.extrinsic_idx] except IndexError: extrinsic = None for processor_class in ProcessorRegistry().get_event_processors( event.module_id, event.event_id): event_processor = processor_class(block, event, extrinsic, substrate=self.substrate) event_processor.sequencing_hook(self.db_session, parent_block_data, parent_sequenced_block_data) sequenced_block.save(self.db_session) return sequenced_block
def get_relationships(self, include_list, item): relationships = {} if 'attributes' in include_list: relationships['attributes'] = RuntimeEventAttribute.query(self.session).filter_by( runtime_event_id=item.id).order_by('id') if 'recent_events' in include_list: relationships['recent_events'] = Event.query(self.session).filter_by( event_id=item.event_id, module_id=item.module_id).order_by(Event.block_id.desc())[:10] return relationships
def rebuild_search_index(self, search_index_type_id): if search_index_type_id == SEARCH_INDEX_SLASHED_ACCOUNT: for event in Event.query(self.db_session).filter_by( module_id='staking', event_id='Slash'): processor = SlashEventProcessor(block=event.block, event=event) processor.process_search_index(self.db_session) self.db_session.commit() elif search_index_type_id == SEARCH_INDEX_BALANCETRANSFER: for event in Event.query(self.db_session).filter_by( module_id='balances', event_id='Transfer'): processor = BalancesTransferProcessor(block=event.block, event=event) processor.process_search_index(self.db_session) self.db_session.commit() else: raise NotImplementedError( 'Not supported search index {}'.format(search_index_type_id))
def serialize_item(self, item): data = item.serialize() runtime_call = RuntimeCall.query(self.session).filter_by( module_id=item.module_id, call_id=item.call_id, spec_version=item.spec_version_id).first() data['attributes']['documentation'] = runtime_call.documentation block = Block.query(self.session).get(item.block_id) if block.datetime: data['attributes']['datetime'] = block.datetime.replace( tzinfo=pytz.UTC).isoformat() else: data['attributes']['datetime'] = None if item.account: data['attributes']['account'] = item.account.serialize() if item.params: item.params = self.check_params(item.params, item.serialize_id()) if item.error: # Retrieve ExtrinsicFailed event extrinsic_failed_event = Event.query(self.session).filter_by( block_id=item.block_id, event_id='ExtrinsicFailed').first() # Retrieve runtime error if extrinsic_failed_event: if 'Module' in extrinsic_failed_event.attributes[0]['value']: error = RuntimeErrorMessage.query(self.session).filter_by( module_index=extrinsic_failed_event.attributes[0] ['value']['Module']['index'], index=extrinsic_failed_event.attributes[0]['value'] ['Module']['error'], spec_version=item.spec_version_id).first() if error: data['attributes'][ 'error_message'] = error.documentation elif 'BadOrigin' in extrinsic_failed_event.attributes[0][ 'value']: data['attributes']['error_message'] = 'Bad origin' elif 'CannotLookup' in extrinsic_failed_event.attributes[0][ 'value']: data['attributes']['error_message'] = 'Cannot lookup' return data
def process_reorg_block(self, block): model = ReorgBlock(**block.asdict()) model.save(self.db_session) for extrinsic in Extrinsic.query( self.db_session).filter_by(block_id=block.id): model = ReorgExtrinsic(block_hash=block.hash, **extrinsic.asdict()) model.save(self.db_session) for event in Event.query(self.db_session).filter_by(block_id=block.id): model = ReorgEvent(block_hash=block.hash, **event.asdict()) model.save(self.db_session) for log in Log.query(self.db_session).filter_by(block_id=block.id): model = ReorgLog(block_hash=block.hash, **log.asdict()) model.save(self.db_session)
def get_relationships(self, include_list, item): relationships = {} if 'extrinsics' in include_list: relationships['extrinsics'] = Extrinsic.query(self.session).filter_by(block_id=item.id).order_by( 'extrinsic_idx') if 'transactions' in include_list: relationships['transactions'] = Extrinsic.query(self.session).filter_by(block_id=item.id, signed=1).order_by( 'extrinsic_idx') if 'inherents' in include_list: relationships['inherents'] = Extrinsic.query(self.session).filter_by(block_id=item.id, signed=0).order_by( 'extrinsic_idx') if 'events' in include_list: relationships['events'] = Event.query(self.session).filter_by(block_id=item.id, system=0).order_by( 'event_idx') if 'logs' in include_list: relationships['logs'] = Log.query(self.session).filter_by(block_id=item.id).order_by( 'log_idx') return relationships
def serialize_item(self, item): data = item.serialize() runtime_call = RuntimeCall.query(self.session).filter_by( module_id=item.module_id, call_id=item.call_id, spec_version=item.spec_version_id).first() data['attributes']['documentation'] = runtime_call.documentation block = Block.query(self.session).get(item.block_id) data['attributes']['datetime'] = block.datetime.replace( tzinfo=pytz.UTC).isoformat() if item.account: data['attributes']['account'] = item.account.serialize() if item.error: # Retrieve ExtrinsicFailed event extrinsic_failed_event = Event.query(self.session).filter_by( block_id=item.block_id, event_id='ExtrinsicFailed').first() # Retrieve runtime error if extrinsic_failed_event and 'Module' in extrinsic_failed_event.attributes[ 0]['value']: error = RuntimeErrorMessage.query(self.session).filter_by( module_id=item.module_id, index=extrinsic_failed_event.attributes[0]['value'] ['Module']['error'], spec_version=item.spec_version_id).first() if error: data['attributes']['error_message'] = error.documentation return data
def add_block(self, block_hash): # Check if block is already process if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0: raise BlockAlreadyAdded(block_hash) if settings.SUBSTRATE_MOCK_EXTRINSICS: self.substrate.mock_extrinsics = settings.SUBSTRATE_MOCK_EXTRINSICS json_block = self.substrate.get_chain_block(block_hash) parent_hash = json_block['block']['header'].pop('parentHash') block_id = json_block['block']['header'].pop('number') extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot') state_root = json_block['block']['header'].pop('stateRoot') digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None) # Convert block number to numeric if not block_id.isnumeric(): block_id = int(block_id, 16) # ==== Get block runtime from Substrate ================== self.substrate.init_runtime(block_hash=block_hash) self.process_metadata(self.substrate.runtime_version, block_hash) # ==== Get parent block runtime =================== if block_id > 0: json_parent_runtime_version = self.substrate.get_block_runtime_version(parent_hash) parent_spec_version = json_parent_runtime_version.get('specVersion', 0) self.process_metadata(parent_spec_version, parent_hash) else: parent_spec_version = self.substrate.runtime_version # ==== Set initial block properties ===================== block = Block( id=block_id, parent_id=block_id - 1, hash=block_hash, parent_hash=parent_hash, state_root=state_root, extrinsics_root=extrinsics_root, count_extrinsics=0, count_events=0, count_accounts_new=0, count_accounts_reaped=0, count_accounts=0, count_events_extrinsic=0, count_events_finalization=0, count_events_module=0, count_events_system=0, count_extrinsics_error=0, count_extrinsics_signed=0, count_extrinsics_signedby_address=0, count_extrinsics_signedby_index=0, count_extrinsics_success=0, count_extrinsics_unsigned=0, count_sessions_new=0, count_contracts_new=0, count_log=0, range10000=math.floor(block_id / 10000), range100000=math.floor(block_id / 100000), range1000000=math.floor(block_id / 1000000), spec_version_id=self.substrate.runtime_version, logs=digest_logs ) # Set temp helper variables block._accounts_new = [] block._accounts_reaped = [] # ==== Get block events from Substrate ================== extrinsic_success_idx = {} events = [] try: events_decoder = self.substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version]) event_idx = 0 for event in events_decoder.elements: event.value['module_id'] = event.value['module_id'].lower() model = Event( block_id=block_id, event_idx=event_idx, phase=event.value['phase'], extrinsic_idx=event.value['extrinsic_idx'], type=event.value['type'], spec_version_id=parent_spec_version, module_id=event.value['module_id'], event_id=event.value['event_id'], system=int(event.value['module_id'] == 'system'), module=int(event.value['module_id'] != 'system'), attributes=event.value['params'], codec_error=False ) # Process event if event.value['phase'] == 0: block.count_events_extrinsic += 1 elif event.value['phase'] == 1: block.count_events_finalization += 1 if event.value['module_id'] == 'system': block.count_events_system += 1 # Store result of extrinsic if event.value['event_id'] == 'ExtrinsicSuccess': extrinsic_success_idx[event.value['extrinsic_idx']] = True block.count_extrinsics_success += 1 if event.value['event_id'] == 'ExtrinsicFailed': extrinsic_success_idx[event.value['extrinsic_idx']] = False block.count_extrinsics_error += 1 else: block.count_events_module += 1 model.save(self.db_session) events.append(model) event_idx += 1 block.count_events = len(events_decoder.elements) except SubstrateRequestException: block.count_events = 0 # === Extract extrinsics from block ==== extrinsics_data = json_block['block'].pop('extrinsics') block.count_extrinsics = len(extrinsics_data) extrinsic_idx = 0 extrinsics = [] for extrinsic in extrinsics_data: extrinsics_decoder = ExtrinsicsDecoder( data=ScaleBytes(extrinsic), metadata=self.metadata_store[parent_spec_version] ) extrinsic_data = extrinsics_decoder.decode() # Lookup result of extrinsic extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False) model = Extrinsic( block_id=block_id, extrinsic_idx=extrinsic_idx, extrinsic_hash=extrinsics_decoder.extrinsic_hash, extrinsic_length=extrinsic_data.get('extrinsic_length'), extrinsic_version=extrinsic_data.get('version_info'), signed=extrinsics_decoder.contains_transaction, unsigned=not extrinsics_decoder.contains_transaction, signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')), signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')), address_length=extrinsic_data.get('account_length'), address=extrinsic_data.get('account_id'), account_index=extrinsic_data.get('account_index'), account_idx=extrinsic_data.get('account_idx'), signature=extrinsic_data.get('signature'), nonce=extrinsic_data.get('nonce'), era=extrinsic_data.get('era'), call=extrinsic_data.get('call_code'), module_id=extrinsic_data.get('call_module'), call_id=extrinsic_data.get('call_function'), params=extrinsic_data.get('params'), spec_version_id=parent_spec_version, success=int(extrinsic_success), error=int(not extrinsic_success), codec_error=False ) model.save(self.db_session) extrinsics.append(model) extrinsic_idx += 1 # Process extrinsic if extrinsics_decoder.contains_transaction: block.count_extrinsics_signed += 1 if model.signedby_address: block.count_extrinsics_signedby_address += 1 if model.signedby_index: block.count_extrinsics_signedby_index += 1 # Add search index for signed extrinsics search_index = SearchIndex( index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC, block_id=block.id, extrinsic_idx=model.extrinsic_idx, account_id=model.address ) search_index.save(self.db_session) else: block.count_extrinsics_unsigned += 1 # Process extrinsic processors for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id): extrinsic_processor = processor_class(block, model, substrate=self.substrate) extrinsic_processor.accumulation_hook(self.db_session) extrinsic_processor.process_search_index(self.db_session) # Process event processors for event in events: extrinsic = None if event.extrinsic_idx is not None: try: extrinsic = extrinsics[event.extrinsic_idx] except IndexError: extrinsic = None for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id): event_processor = processor_class(block, event, extrinsic, metadata=self.metadata_store.get(block.spec_version_id), substrate=self.substrate) event_processor.accumulation_hook(self.db_session) event_processor.process_search_index(self.db_session) # Process block processors for processor_class in ProcessorRegistry().get_block_processors(): block_processor = processor_class(block, substrate=self.substrate, harvester=self) block_processor.accumulation_hook(self.db_session) # Debug info if settings.DEBUG: block.debug_info = json_block # ==== Save data block ================================== block.save(self.db_session) return block
def get_item(self, item_id): return Event.query(self.session).get(item_id.split('-'))
def on_get(self, req, resp, network_id=None): resp.status = falcon.HTTP_200 # TODO make caching more generic for custom resources cache_key = '{}-{}'.format(req.method, req.url) console_handler = logging.StreamHandler() console_handler.setLevel('INFO') logger = logging.getLogger('yee') logger.setLevel('DEBUG') logger.addHandler(console_handler) # logger.info(cache_key) response = self.cache_region.get(cache_key, self.cache_expiration_time) if response is NO_VALUE: best_block = Block.query(self.session).filter_by( id=self.session.query(func.max(Block.id)).one()[0]).first() total_signed_extrinsics = Extrinsic.query( self.session).filter_by(signed=1).count() total_accounts = Account.query(self.session).filter_by().count() # total_events = Event.query(self.session).count() event = Event.query(self.session).filter_by( id=self.session.query(func.max(Event.id)).one()[0]).first() if event is None: eventid = 0 else: eventid = event.id if best_block: substrate = SubstrateInterface( SUBSTRATE_RPC_URL, metadata_version=SUBSTRATE_METADATA_VERSION) print(substrate.get_ShardCount()) response = self.get_jsonapi_response(data={ 'type': 'networkstats', 'id': network_id, 'attributes': { 'best_block': best_block.id, 'total_signed_extrinsics': total_signed_extrinsics, 'total_events': eventid, 'total_events_module': int(best_block.id), 'total_blocks': 'N/A', 'total_accounts': total_accounts, 'total_runtimes': Runtime.query(self.session).count(), 'shard_count': int(substrate.get_ShardCount(), 16) } }, ) else: response = self.get_jsonapi_response(data={ 'type': 'networkstats', 'id': network_id, 'attributes': { 'best_block': 0, 'total_signed_extrinsics': 0, 'total_events': 0, 'total_events_module': 0, 'total_blocks': 'N/A', 'total_accounts': 0, 'total_runtimes': 0 } }, ) self.cache_region.set(cache_key, response) resp.set_header('X-Cache', 'MISS') else: resp.set_header('X-Cache', 'HIT') resp.media = response
def get_query(self): return Event.query(self.session).filter( Event.module_id == 'balances', Event.event_id == 'Transfer').order_by(Event.block_id.desc())
def get_item(self, item_id): if len(item_id.split('-')) != 2: return None return Event.query(self.session).get(item_id.split('-'))
def get_query(self): return Event.query(self.session).order_by(Event.block_id.desc())
def on_post(self, req, resp): blockHash = None if req.media.get('block_id'): substrate = SubstrateInterface(url=SUBSTRATE_RPC_URL, address_type=SUBSTRATE_ADDRESS_TYPE, type_registry_preset=TYPE_REGISTRY) blockHash = substrate.get_block_hash(req.media.get('block_id')) elif req.media.get('block_hash'): blockHash = req.media.get('block_hash') else: resp.status = falcon.HTTP_BAD_REQUEST resp.media = { 'errors': ['Either blockHash or block_id should be supplied'] } if blockHash: resp.status = falcon.HTTP_200 block = Block.query( self.session).filter(Block.hash == blockHash).first() blockTotal = BlockTotal.query( self.session).filter(BlockTotal.id == block.id).first() author = ss58_encode( blockTotal.author.replace('0x', '') ) if blockTotal is not None and blockTotal.author is not None else None if block: blockInfo = {} blockInfo["timestamp"] = block.datetime.strftime( "%Y-%m-%d %H:%M:%S") blockInfo["block_hash"] = block.hash blockInfo["block_id"] = block.id blockInfo["parent_id"] = block.id - 1 if block.id > 0 else 0 blockInfo["child_id"] = block.id + 1 blockInfo["parent_hash"] = block.parent_hash blockInfo["state_root"] = block.state_root blockInfo["extrinsic_root"] = block.extrinsics_root blockInfo["validator"] = author blockInfo["count_extrinsic"] = block.count_extrinsics blockInfo["count_event"] = block.count_events blockInfo["count_log"] = block.count_log # blockInfo["age"] = time.mktime(block.datetime.timetuple()) blockInfo["age"] = block.datetime.strftime("%Y-%m-%d %H:%M:%S") # 获取和区块相关的交易信息 extrinsics = Extrinsic.query( self.session).filter(Extrinsic.block_id == block.id).all() extrinsicsObj = [ { "extrinsic_id": '{}-{}'.format(block.id, extrinsic.extrinsic_idx), "hash": extrinsic.extrinsic_hash if extrinsic.extrinsic_hash else None, # "age": time.mktime(block.datetime.timetuple()), "age": block.datetime.strftime("%Y-%m-%d %H:%M:%S"), "result": extrinsic.success, # "address": extrinsic.address if extrinsic.address else None, # "module": extrinsic.module_id, # "fee": None, # "nonce": extrinsic.nonce if extrinsic.nonce else None, # "call": extrinsic.call_id, "operation": '{}({})'.format(extrinsic.module_id, extrinsic.call_id), "params": extrinsic.params # "signature": extrinsic.signature if extrinsic.signature else None } for extrinsic in extrinsics ] # 获取和区块相关的日志信息 logs = Log.query( self.session).filter(Log.block_id == block.id).all() logsObj = [{ "log_id": '{}-{}'.format(block.id, log.log_idx), "block_id": block.id, "type": log.type, "data": log.data['value'] } for log in logs] # 获取和区块相关的事件信息 events = Event.query( self.session).filter(Event.block_id == block.id).all() eventObj = [{ "id": '{}-{}'.format(block.id, event.event_idx), "block_id": block.id, "block_hash": block.hash, "module_id": event.module_id, "event_id": event.event_id, "attributes": event.attributes, "operation": '{}({})'.format(event.module_id, event.event_id), "desc": self.getEventDesc(event.module_id, event.event_id), "hash": self.getEventHash(block.id, event.extrinsic_idx) } for event in events] resp.media = { 'status': 'success', 'data': { "block_info": blockInfo, "extrinsics": extrinsicsObj, "logs": logsObj, "events": eventObj } } else: resp.status = falcon.HTTP_404 resp.media = {'result': 'Block not found'}
def on_get(self, req, resp): resp.status = falcon.HTTP_200 extrinsicId = req.params.get('extrinsic_id') if extrinsicId is None: resp.status = falcon.HTTP_BAD_REQUEST resp.media = {'errors': ['Invalid extrinsic id']} split = extrinsicId.split("-") if split is None or len(split) != 2: resp.status = falcon.HTTP_BAD_REQUEST resp.media = {'errors': ['Invalid extrinsic id']} blockId = split[0] extrinsicIdx = split[1] extrinsic = Extrinsic.extrinsic_by_id(self.session, blockId, extrinsicIdx).first() print(extrinsic.extrinsic_idx) extrinsicDetailInfo = {} if extrinsic is not None: extrinsicDetailInfo["extrinsic_id"] = '{}-{}'.format( extrinsic.block_id, extrinsic.extrinsic_idx) extrinsicDetailInfo["block_id"] = extrinsic.block_id extrinsicDetailInfo["extrinsic_idx"] = extrinsic.extrinsic_idx extrinsicDetailInfo["hash"] = "0x{}".format( extrinsic.extrinsic_hash) if extrinsic.extrinsic_hash else None extrinsicDetailInfo["age"] = extrinsic.datetime.strftime( "%Y-%m-%d %H:%M:%S") extrinsicDetailInfo["result"] = extrinsic.success extrinsicDetailInfo["operation"] = '{}({})'.format( extrinsic.module_id, extrinsic.call_id) extrinsicDetailInfo["module_id"] = extrinsic.module_id extrinsicDetailInfo["call_id"] = extrinsic.call_id extrinsicDetailInfo[ "module_name"] = extrinsic.module_name.capitalize() extrinsicDetailInfo["call_name"] = extrinsic.call_name.capitalize() extrinsicDetailInfo["call_desc"] = extrinsic.call_desc extrinsicDetailInfo["nonce"] = int( extrinsic.nonce) if extrinsic.nonce else None extrinsicDetailInfo[ "signature"] = extrinsic.signature if extrinsic.signature else None extrinsicDetailInfo["params"] = extrinsic.params extrinsicDetailInfo["from"] = extrinsic.address.replace( '0x', '') if extrinsic.address else None extrinsicDetailInfo["from_addr"] = ss58_encode( extrinsic.address.replace('0x', '')) if extrinsic.address else None params = json.loads(extrinsic.params) for param in params: name = param.get('name') if name == 'dest': toAddr = ss58_encode(param.get('value').replace('0x', '')) extrinsicDetailInfo['to'] = param.get('value').replace( '0x', '') extrinsicDetailInfo['to_addr'] = toAddr elif name == 'value': coin = param.get('value') / 1000000000000 # 转换为单位 Unit extrinsicDetailInfo['coin'] = coin # 获取和区块相关的事件信息 events = Event.query(self.session).filter( Event.block_id == extrinsic.block_id).all() eventObj = [{ "id": '{}-{}'.format(extrinsic.block_id, event.event_idx), "block_id": extrinsic.block_id, "module_id": event.module_id, "event_id": event.event_id, "attributes": event.attributes, "operation": '{}({})'.format(event.module_id, event.event_id), "desc": self.getEventDesc(event.module_id, event.event_id), "hash": self.getEventHash(extrinsic.block_id, event.extrinsic_idx) } for event in events] resp.media = { 'status': 'success', 'data': { "extrinsic_detail": extrinsicDetailInfo, "events": eventObj, "event_count": len(eventObj) } }
def get_query(self): return Event.query(self.session).filter( Event.system == False).order_by(Event.block_id.desc())
def process_genesis(self, block): substrate = SubstrateInterface(SUBSTRATE_RPC_URL) # Set block time of parent block child_block = Block.query( self.db_session).filter_by(parent_hash=block.hash).first() block.set_datetime(child_block.datetime) # Retrieve genesis accounts storage_call = RuntimeStorage.query(self.db_session).filter_by( module_id='indices', name='NextEnumSet', spec_version=block.spec_version_id).first() if storage_call: genesis_account_page_count = substrate.get_storage( block_hash=block.hash, module="Indices", function="NextEnumSet", return_scale_type=storage_call.get_return_type(), hasher=storage_call.type_hasher) or 0 # Get Accounts on EnumSet storage_call = RuntimeStorage.query(self.db_session).filter_by( module_id='indices', name='EnumSet', spec_version=block.spec_version_id).first() if storage_call: block.count_accounts_new = 0 block.count_accounts = 0 for enum_set_nr in range(0, genesis_account_page_count + 1): account_index_u32 = U32() account_index_u32.encode(enum_set_nr) genesis_accounts = substrate.get_storage( block_hash=block.hash, module="Indices", function="EnumSet", params=account_index_u32.data.data.hex(), return_scale_type=storage_call.get_return_type(), hasher=storage_call.type_hasher) if genesis_accounts: block.count_accounts_new += len(genesis_accounts) block.count_accounts += len(genesis_accounts) for idx, account_id in enumerate(genesis_accounts): account_audit = AccountAudit( account_id=account_id.replace('0x', ''), block_id=block.id, extrinsic_idx=None, event_idx=None, type_id=ACCOUNT_AUDIT_TYPE_NEW) account_audit.save(self.db_session) account_index_id = enum_set_nr * 64 + idx account_index_audit = AccountIndexAudit( account_index_id=account_index_id, account_id=account_id.replace('0x', ''), block_id=block.id, extrinsic_idx=None, event_idx=None, type_id=ACCOUNT_INDEX_AUDIT_TYPE_NEW) account_index_audit.save(self.db_session) block.save(self.db_session) # Create initial session initial_session_event = NewSessionEventProcessor(block, Event(), None) initial_session_event.add_session(db_session=self.db_session, session_id=0)
def process_genesis(self, block): # Set block time of parent block child_block = Block.query(self.db_session).filter_by(parent_hash=block.hash).first() block.set_datetime(child_block.datetime) # Retrieve genesis accounts if settings.get_versioned_setting('SUBSTRATE_STORAGE_INDICES', block.spec_version_id) == 'Accounts': # Get accounts from storage keys storage_key_prefix = self.substrate.generate_storage_hash( storage_module='System', storage_function='Account', metadata_version=settings.SUBSTRATE_METADATA_VERSION ) rpc_result = self.substrate.rpc_request( 'state_getKeys', [storage_key_prefix, block.hash] ).get('result') # Extract accounts from storage key genesis_accounts = [storage_key[-64:] for storage_key in rpc_result if len(storage_key) == 162] for account_id in genesis_accounts: account_audit = AccountAudit( account_id=account_id, block_id=block.id, extrinsic_idx=None, event_idx=None, type_id=settings.ACCOUNT_AUDIT_TYPE_NEW ) account_audit.save(self.db_session) elif settings.get_versioned_setting('SUBSTRATE_STORAGE_INDICES', block.spec_version_id) == 'EnumSet': genesis_account_page_count = self.substrate.get_runtime_state( module="Indices", storage_function="NextEnumSet", block_hash=block.hash ).get('result', 0) # Get Accounts on EnumSet block.count_accounts_new = 0 block.count_accounts = 0 for enum_set_nr in range(0, genesis_account_page_count + 1): genesis_accounts = self.substrate.get_runtime_state( module="Indices", storage_function="EnumSet", params=[enum_set_nr], block_hash=block.hash ).get('result') if genesis_accounts: block.count_accounts_new += len(genesis_accounts) block.count_accounts += len(genesis_accounts) for idx, account_id in enumerate(genesis_accounts): account_audit = AccountAudit( account_id=account_id.replace('0x', ''), block_id=block.id, extrinsic_idx=None, event_idx=None, type_id=settings.ACCOUNT_AUDIT_TYPE_NEW ) account_audit.save(self.db_session) account_index_id = enum_set_nr * 64 + idx account_index_audit = AccountIndexAudit( account_index_id=account_index_id, account_id=account_id.replace('0x', ''), block_id=block.id, extrinsic_idx=None, event_idx=None, type_id=settings.ACCOUNT_INDEX_AUDIT_TYPE_NEW ) account_index_audit.save(self.db_session) block.save(self.db_session) # Add hardcoded account like treasury stored in settings for account_id in settings.SUBSTRATE_TREASURY_ACCOUNTS: account_audit = AccountAudit( account_id=account_id, block_id=block.id, extrinsic_idx=None, event_idx=None, data={'is_treasury': True}, type_id=settings.ACCOUNT_AUDIT_TYPE_NEW ) account_audit.save(self.db_session) # Check for sudo accounts try: # Update sudo key sudo_key = self.substrate.get_runtime_state( module='Sudo', storage_function='Key', block_hash=block.hash ).get('result') account_audit = AccountAudit( account_id=sudo_key.replace('0x', ''), block_id=block.id, extrinsic_idx=None, event_idx=None, data={'is_sudo': True}, type_id=settings.ACCOUNT_AUDIT_TYPE_NEW ) account_audit.save(self.db_session) except ValueError: pass # Create initial session initial_session_event = NewSessionEventProcessor( block=block, event=Event(), substrate=self.substrate ) if settings.get_versioned_setting('NEW_SESSION_EVENT_HANDLER', block.spec_version_id): initial_session_event.add_session(db_session=self.db_session, session_id=0) else: initial_session_event.add_session_old(db_session=self.db_session, session_id=0)
def add_block(self, block_hash): # Check if block is already process if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0: raise BlockAlreadyAdded(block_hash) # Extract data from json_block substrate = SubstrateInterface(SUBSTRATE_RPC_URL) if SUBSTRATE_MOCK_EXTRINSICS: substrate.mock_extrinsics = SUBSTRATE_MOCK_EXTRINSICS json_block = substrate.get_chain_block(block_hash) parent_hash = json_block['block']['header'].pop('parentHash') block_id = json_block['block']['header'].pop('number') extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot') state_root = json_block['block']['header'].pop('stateRoot') digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None) # Convert block number to numeric if not block_id.isnumeric(): block_id = int(block_id, 16) # ==== Get block runtime from Substrate ================== json_runtime_version = substrate.get_block_runtime_version(block_hash) # Get spec version spec_version = json_runtime_version.get('specVersion', 0) self.process_metadata(json_runtime_version, block_hash) # ==== Get parent block runtime =================== if block_id > 0: json_parent_runtime_version = substrate.get_block_runtime_version(parent_hash) parent_spec_version = json_parent_runtime_version.get('specVersion', 0) self.process_metadata(json_parent_runtime_version, parent_hash) else: parent_spec_version = spec_version # ==== Set initial block properties ===================== block = Block( id=block_id, parent_id=block_id - 1, hash=block_hash, parent_hash=parent_hash, state_root=state_root, extrinsics_root=extrinsics_root, count_extrinsics=0, count_events=0, count_accounts_new=0, count_accounts_reaped=0, count_accounts=0, count_events_extrinsic=0, count_events_finalization=0, count_events_module=0, count_events_system=0, count_extrinsics_error=0, count_extrinsics_signed=0, count_extrinsics_signedby_address=0, count_extrinsics_signedby_index=0, count_extrinsics_success=0, count_extrinsics_unsigned=0, count_sessions_new=0, count_contracts_new=0, count_log=0, range10000=math.floor(block_id / 10000), range100000=math.floor(block_id / 100000), range1000000=math.floor(block_id / 1000000), spec_version_id=spec_version, logs=digest_logs ) # Set temp helper variables block._accounts_new = [] block._accounts_reaped = [] # ==== Get block events from Substrate ================== extrinsic_success_idx = {} events = [] try: events_decoder = substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version]) event_idx = 0 for event in events_decoder.elements: event.value['module_id'] = event.value['module_id'].lower() model = Event( block_id=block_id, event_idx=event_idx, phase=event.value['phase'], extrinsic_idx=event.value['extrinsic_idx'], type=event.value['type'], spec_version_id=parent_spec_version, module_id=event.value['module_id'], event_id=event.value['event_id'], system=int(event.value['module_id'] == 'system'), module=int(event.value['module_id'] != 'system'), attributes=event.value['params'], codec_error=False ) # Process event if event.value['phase'] == 0: block.count_events_extrinsic += 1 elif event.value['phase'] == 1: block.count_events_finalization += 1 if event.value['module_id'] == 'system': block.count_events_system += 1 # Store result of extrinsic if event.value['event_id'] == 'ExtrinsicSuccess': extrinsic_success_idx[event.value['extrinsic_idx']] = True block.count_extrinsics_success += 1 if event.value['event_id'] == 'ExtrinsicFailed': extrinsic_success_idx[event.value['extrinsic_idx']] = False block.count_extrinsics_error += 1 else: block.count_events_module += 1 model.save(self.db_session) events.append(model) event_idx += 1 block.count_events = len(events_decoder.elements) except SubstrateRequestException: block.count_events = 0 # === Extract extrinsics from block ==== extrinsics_data = json_block['block'].pop('extrinsics') block.count_extrinsics = len(extrinsics_data) extrinsic_idx = 0 extrinsics = [] for extrinsic in extrinsics_data: # Save to data table if block_hash == '0x911a0bf66d5494b6b24f612b3cc14841134c6b73ab9ce02f7e012973070e5661': # TODO TEMP fix for exception in Alexander network, remove when network is obsolete extrinsics_decoder = ExtrinsicsBlock61181Decoder( data=ScaleBytes(extrinsic), metadata=self.metadata_store[parent_spec_version] ) else: extrinsics_decoder = ExtrinsicsDecoder( data=ScaleBytes(extrinsic), metadata=self.metadata_store[parent_spec_version] ) extrinsic_data = extrinsics_decoder.decode() # Lookup result of extrinsic extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False) model = Extrinsic( block_id=block_id, extrinsic_idx=extrinsic_idx, extrinsic_hash=extrinsics_decoder.extrinsic_hash, extrinsic_length=extrinsic_data.get('extrinsic_length'), extrinsic_version=extrinsic_data.get('version_info'), signed=extrinsics_decoder.contains_transaction, unsigned=not extrinsics_decoder.contains_transaction, signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')), signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')), address_length=extrinsic_data.get('account_length'), address=extrinsic_data.get('account_id'), account_index=extrinsic_data.get('account_index'), account_idx=extrinsic_data.get('account_idx'), signature=extrinsic_data.get('signature'), nonce=extrinsic_data.get('nonce'), era=extrinsic_data.get('era'), call=extrinsic_data.get('call_code'), module_id=extrinsic_data.get('call_module'), call_id=extrinsic_data.get('call_module_function'), params=extrinsic_data.get('params'), spec_version_id=parent_spec_version, success=int(extrinsic_success), error=int(not extrinsic_success), codec_error=False ) model.save(self.db_session) if extrinsic_data.get('call_module_function') == 'transfer': if len(extrinsic_data.get('params')) > 1: _amount = extrinsic_data.get('params')[1].get('value') else: _amount = extrinsic_data.get('params')[0].get('value') # [{'type': 'AccountId', 'value': '0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d', # 'valueRaw': 'd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'}, # {'type': 'AccountId', 'value': '0x8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48', # 'valueRaw': '8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48'}] account_ids = events_decoder.value[1].get('params')[0:2] transfer = Transfer( block_id=block.id, extrinsic_idx=model.extrinsic_idx, data_extrinsic_idx=str(block_id) + '_' + str(model.extrinsic_idx), transfer_from=ss58_encode(account_ids[0]['valueRaw']), from_raw=account_ids[0]['valueRaw'], transfer_to=ss58_encode(account_ids[1]['valueRaw']), to_raw=account_ids[1]['valueRaw'], hash=model.extrinsic_hash, amount=decimal.Decimal(_amount), block_timestamp=block.datetime, module_id=extrinsic_data.get('call_module'), success=int(extrinsic_success), error=int(not extrinsic_success), created_at=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), updated_at=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")#time.asctime(time.localtime(time.time())) ) transfer.save(self.db_session) extrinsics.append(model) extrinsic_idx += 1 # Process extrinsic if extrinsics_decoder.contains_transaction: block.count_extrinsics_signed += 1 if model.signedby_address: block.count_extrinsics_signedby_address += 1 if model.signedby_index: block.count_extrinsics_signedby_index += 1 else: block.count_extrinsics_unsigned += 1 # Process extrinsic processors for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id): extrinsic_processor = processor_class(block, model) extrinsic_processor.accumulation_hook(self.db_session) # Process event processors for event in events: extrinsic = None if event.extrinsic_idx is not None: try: extrinsic = extrinsics[event.extrinsic_idx] except IndexError: extrinsic = None for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id): event_processor = processor_class(block, event, extrinsic, metadata=self.metadata_store.get(block.spec_version_id)) event_processor.accumulation_hook(self.db_session) # Process block processors for processor_class in ProcessorRegistry().get_block_processors(): block_processor = processor_class(block) block_processor.accumulation_hook(self.db_session) # Debug info if DEBUG: block.debug_info = json_block # ==== Save data block ================================== block.save(self.db_session) return block