def remove_block(self, block_hash): # Retrieve block block = Block.query(self.db_session).filter_by(hash=block_hash).first() # Revert event processors for event in Event.query(self.db_session).filter_by(block_id=block.id): for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id): event_processor = processor_class(block, event, None) event_processor.accumulation_revert(self.db_session) # Revert extrinsic processors for extrinsic in Extrinsic.query(self.db_session).filter_by(block_id=block.id): for processor_class in ProcessorRegistry().get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id): extrinsic_processor = processor_class(block, extrinsic) extrinsic_processor.accumulation_revert(self.db_session) # Revert block processors for processor_class in ProcessorRegistry().get_block_processors(): block_processor = processor_class(block) block_processor.accumulation_revert(self.db_session) # Delete events for item in Event.query(self.db_session).filter_by(block_id=block.id): self.db_session.delete(item) # Delete extrinsics for item in Extrinsic.query(self.db_session).filter_by(block_id=block.id): self.db_session.delete(item) # Delete block self.db_session.delete(block)
def get_relationships(self, include_list, item): relationships = {} if 'extrinsics' in include_list: relationships['extrinsics'] = Extrinsic.query( self.session).filter_by( block_id=item.bid, shard_num=item.shard_num).order_by('extrinsic_idx') if 'transactions' in include_list: relationships['transactions'] = Extrinsic.query( self.session).filter_by( block_id=item.bid, signed=1, shard_num=item.shard_num).order_by('extrinsic_idx') if 'inherents' in include_list: relationships['inherents'] = Extrinsic.query(self.session).filter( Extrinsic.block_id == item.bid, Extrinsic.signed == 0, Extrinsic.shard_num == item.shard_num, Extrinsic.module_id != 'relay').order_by('extrinsic_idx') if 'relay' in include_list: relationships['relay'] = Extrinsic.query(self.session).filter( Extrinsic.block_id == item.bid, Extrinsic.signed == 0, Extrinsic.shard_num == item.shard_num, Extrinsic.module_id == 'relay').order_by('extrinsic_idx') if 'events' in include_list: relationships['events'] = Event.query(self.session).filter_by( block_id=item.bid, system=0, shard_num=item.shard_num).order_by('event_idx') if 'logs' in include_list: relationships['logs'] = Log.query(self.session).filter_by( block_id=item.bid, shard_num=item.shard_num).order_by('log_idx') return relationships
def on_get(self, req, resp): page = int(req.params.get('page') if req.params.get('page') else 1) pageSize = int( req.params.get('page_size') if req.params.get('page_size') else 20) moduleId = req.params.get('module_id') if req.params.get( 'module_id') else None callId = req.params.get('call_id') if req.params.get( 'call_id') else None if moduleId == 'All': moduleId = None if callId == 'All': callId = None resp.status = falcon.HTTP_200 extrinsics = Extrinsic.all_extrinsics(self.session, page, pageSize, moduleId, callId) result = [{ "extrinsic_id": '{}-{}'.format(extrinsic.block_id, extrinsic.extrinsic_idx), "block_id": extrinsic.block_id, "hash": "0x{}".format(extrinsic.extrinsic_hash) if extrinsic.extrinsic_hash else None, "age": extrinsic.datetime.strftime("%Y-%m-%d %H:%M:%S"), "result": extrinsic.success, "operation": '{}({})'.format(extrinsic.module_id, extrinsic.call_id), "params": extrinsic.params, "address": extrinsic.address if extrinsic.address else None, "nonce": extrinsic.nonce if extrinsic.nonce else None, "signature": extrinsic.signature if extrinsic.signature else None } for extrinsic in extrinsics] if moduleId and callId: count = Extrinsic.query(self.session).filter( and_(Extrinsic.module_id == moduleId, Extrinsic.call_id == callId)).count() elif moduleId: count = Extrinsic.query( self.session).filter(Extrinsic.module_id == moduleId).count() else: count = Extrinsic.query(self.session).count() resp.media = { 'status': 'success', 'data': { 'result': result, 'count': count } }
def get_item(self, item_id): if item_id[0:2] == '0x': extrinsic = Extrinsic.query( self.session).filter_by(extrinsic_hash=item_id[2:]).first() else: extrinsic = Extrinsic.query(self.session).get(item_id.split('-')) return extrinsic
def on_get(self, req, resp): resp.status = falcon.HTTP_200 page = int(req.params.get('page') if req.params.get('page') else 1) pageSize = int( req.params.get('page_size') if req.params.get('page_size') else 10) extrinsics = Extrinsic.latest_extrinsics(self.session, page, pageSize) result = [] for extrinsic in extrinsics: extrinsicId = '{}-{}'.format(extrinsic.block_id, extrinsic.extrinsic_idx) fromAddr = ss58_encode(extrinsic.address.replace('0x', '')) hash = "0x{}".format(extrinsic.extrinsic_hash) timestamp = extrinsic.datetime.strftime( "%Y-%m-%d %H:%M:%S" ) if extrinsic.datetime is not None else None blockId = extrinsic.block_id success = extrinsic.success # print(extrinsic); # timestamp = time.mktime(extrinsic.datetime.timetuple()) params = json.loads(extrinsic.params) for param in params: name = param.get('name') if name == 'dest': toAddr = ss58_encode(param.get('value').replace('0x', '')) elif name == 'value': coin = param.get('value') / 1000000000000000 # 转换为单位 Unit result.append({ "extrinsic_id": extrinsicId, "block_id": blockId, "from": fromAddr, "to": toAddr, "hash": hash, "timestamp": timestamp, "coin": coin, "result": success }) count = Extrinsic.query(self.session).filter( and_(Extrinsic.module_id == "balances", Extrinsic.call_id == "transfer")).count() resp.media = { 'status': 'success', 'data': { "result": result, "count": count } }
def getEventHash(self, blockId, extrinsicIdx): extrinsic = Extrinsic.query(self.session).filter( and_(Extrinsic.block_id == blockId, Extrinsic.extrinsic_idx == extrinsicIdx)).first() if extrinsic: return extrinsic.extrinsic_hash return None
def get_relationships(self, include_list, item): relationships = {} if 'recent_extrinsics' in include_list: relationships['recent_extrinsics'] = Extrinsic.query(self.session).filter_by( address=item.account_id).order_by(Extrinsic.block_id.desc())[:10] return relationships
def on_get(self, req, resp): substrate = SubstrateInterface(url=SUBSTRATE_RPC_URL, address_type=SUBSTRATE_ADDRESS_TYPE, type_registry_preset=TYPE_REGISTRY) resp.status = falcon.HTTP_200 # head = Block.get_head(self.session); head_hash = substrate.get_chain_head() head_number = substrate.get_block_number(head_hash) finalised_head_hash = substrate.get_chain_finalised_head() finalised_head_number = substrate.get_block_number(finalised_head_hash) extrinsicCount = Extrinsic.query(self.session).count() try: validators = substrate.get_runtime_state( module="Session", storage_function="Validators", params=[], block_hash=head_hash).get('result', []) except StorageFunctionNotFound: validators = [] try: validator_count = substrate.get_runtime_state( module="Staking", storage_function="ValidatorCount", params=[], block_hash=head_hash).get('result', 0) except StorageFunctionNotFound: validator_count = 0 transfers_count = Extrinsic.query(self.session).filter( and_(Extrinsic.module_id == 'balances', Extrinsic.call_id == 'transfer')).count() resp.media = { 'status': 'success', 'data': { 'blockNumber': head_number, 'finalizedBlockNumber': finalised_head_number, 'extrinsics': extrinsicCount, 'currValidators': len(validators), 'validators': validator_count, 'transfersCount': transfers_count } }
def rebuild_search_index(self): self.db_session.execute('truncate table {}'.format( SearchIndex.__tablename__)) for block in Block.query( self.db_session).order_by('id').yield_per(1000): extrinsic_lookup = {} block._accounts_new = [] block._accounts_reaped = [] for extrinsic in Extrinsic.query(self.db_session).filter_by( block_id=block.id).order_by('extrinsic_idx'): extrinsic_lookup[extrinsic.extrinsic_idx] = extrinsic # Add search index for signed extrinsics if extrinsic.address: search_index = SearchIndex( index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC, block_id=block.id, extrinsic_idx=extrinsic.extrinsic_idx, account_id=extrinsic.address) search_index.save(self.db_session) # Process extrinsic processors for processor_class in ProcessorRegistry( ).get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id): extrinsic_processor = processor_class( block=block, extrinsic=extrinsic, substrate=self.substrate) extrinsic_processor.process_search_index(self.db_session) for event in Event.query(self.db_session).filter_by( block_id=block.id).order_by('event_idx'): extrinsic = None if event.extrinsic_idx is not None: try: extrinsic = extrinsic_lookup[event.extrinsic_idx] except (IndexError, KeyError): extrinsic = None for processor_class in ProcessorRegistry( ).get_event_processors(event.module_id, event.event_id): event_processor = processor_class( block, event, extrinsic, metadata=self.metadata_store.get( block.spec_version_id), substrate=self.substrate) event_processor.process_search_index(self.db_session) self.db_session.commit()
def get_relationships(self, include_list, item): relationships = {} if 'extrinsics' in include_list: relationships['extrinsics'] = Extrinsic.query(self.session).filter_by(block_id=item.id).order_by( 'extrinsic_idx') if 'transactions' in include_list: relationships['transactions'] = Extrinsic.query(self.session).filter_by(block_id=item.id, signed=1).order_by( 'extrinsic_idx') if 'inherents' in include_list: relationships['inherents'] = Extrinsic.query(self.session).filter_by(block_id=item.id, signed=0).order_by( 'extrinsic_idx') if 'events' in include_list: relationships['events'] = Event.query(self.session).filter_by(block_id=item.id, system=0).order_by( 'event_idx') if 'logs' in include_list: relationships['logs'] = Log.query(self.session).filter_by(block_id=item.id).order_by( 'log_idx') return relationships
def sequence_block(self, block, parent_block_data=None, parent_sequenced_block_data=None): sequenced_block = BlockTotal(id=block.id) # Process block processors for processor_class in ProcessorRegistry().get_block_processors(): block_processor = processor_class(block, sequenced_block, substrate=self.substrate) block_processor.sequencing_hook(self.db_session, parent_block_data, parent_sequenced_block_data) extrinsics = Extrinsic.query(self.db_session).filter_by( block_id=block.id).order_by('extrinsic_idx') for extrinsic in extrinsics: # Process extrinsic processors for processor_class in ProcessorRegistry( ).get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id): extrinsic_processor = processor_class(block, extrinsic, substrate=self.substrate) extrinsic_processor.sequencing_hook( self.db_session, parent_block_data, parent_sequenced_block_data) events = Event.query( self.db_session).filter_by(block_id=block.id).order_by('event_idx') # Process event processors for event in events: extrinsic = None if event.extrinsic_idx is not None: try: extrinsic = extrinsics[event.extrinsic_idx] except IndexError: extrinsic = None for processor_class in ProcessorRegistry().get_event_processors( event.module_id, event.event_id): event_processor = processor_class(block, event, extrinsic, substrate=self.substrate) event_processor.sequencing_hook(self.db_session, parent_block_data, parent_sequenced_block_data) sequenced_block.save(self.db_session) return sequenced_block
def get_relationships(self, include_list, item): relationships = {} if 'params' in include_list: relationships['params'] = RuntimeCallParam.query(self.session).filter_by( runtime_call_id=item.id).order_by('id') if 'recent_extrinsics' in include_list: relationships['recent_extrinsics'] = Extrinsic.query(self.session).filter_by( call_id=item.call_id, module_id=item.module_id).order_by(Extrinsic.block_id.desc())[:10] return relationships
def get_relationships(self, include_list, item): relationships = {} if 'recent_extrinsics' in include_list: relationships['recent_extrinsics'] = Extrinsic.query(self.session).filter_by( address=item.id).order_by(Extrinsic.block_id.desc())[:10] if 'indices' in include_list: relationships['indices'] = AccountIndex.query(self.session).filter_by( account_id=item.id).order_by(AccountIndex.updated_at_block.desc()) return relationships
def get_item(self, item_id): if 'origin-' not in item_id and '-' in item_id: st = item_id.split("-") return Extrinsic.query(self.session).filter_by( extrinsic_idx=int(st[2]), block_id=int(st[1]), shard_num=int(st[0])).first() if len(item_id) < 10: return Extrinsic.query(self.session).filter_by(id=item_id).first() if 'origin-' in item_id: if '0x' in item_id: return Extrinsic.query( self.session).filter_by(origin_hash=item_id[9:]).first() else: return Extrinsic.query( self.session).filter_by(origin_hash=item_id[7:]).first() if item_id[0:2] == '0x': return Extrinsic.query( self.session).filter_by(extrinsic_hash=item_id[2:]).first() else: return Extrinsic.query( self.session).filter_by(extrinsic_hash=item_id).first()
def accumulation_hook(self, db_session): extrinsic = Extrinsic.query(db_session).filter_by(extrinsic_idx=self.event.extrinsic_idx, block_id=self.event.block_id).first() if extrinsic.module_id == 'Utility' and extrinsic.call_id == 'batch': #XYK pool creation (aka LP token) #don't process it for now pass else: data_asset = DataAsset( asset_id=self.event.attributes[0]['value'], symbol=extrinsic.params[0]['value'], precision=18, #TODO may be changed name=extrinsic.params[1]['value'], is_mintable=True if extrinsic.params[3]['value'] == 'true' else False ) data_asset.save(db_session)
def check(self): # Extrinsic.query(self.session).filter(Extrinsic.module_id == 'balances', Extrinsic.call_id == 'transfer', # Block.shard_num == Extrinsic.shard_num, # Extrinsic.block_id > 37926).order_by( # Extrinsic.block_id.desc()) items = Extrinsic.query(self.session).join( Block, Extrinsic.block_id == Block.bid).filter( Extrinsic.module_id == 'balances', Extrinsic.call_id == 'transfer', Block.shard_num == Extrinsic.shard_num, Extrinsic.block_id > 38375).order_by(Extrinsic.block_id.desc()) # self.session.query(Extrinsic).join(Block, Extrinsic.block_id == Block.bid) print(items) data = [self.serialize_item(item) for item in items],
def process_reorg_block(self, block): model = ReorgBlock(**block.asdict()) model.save(self.db_session) for extrinsic in Extrinsic.query( self.db_session).filter_by(block_id=block.id): model = ReorgExtrinsic(block_hash=block.hash, **extrinsic.asdict()) model.save(self.db_session) for event in Event.query(self.db_session).filter_by(block_id=block.id): model = ReorgEvent(block_hash=block.hash, **event.asdict()) model.save(self.db_session) for log in Log.query(self.db_session).filter_by(block_id=block.id): model = ReorgLog(block_hash=block.hash, **log.asdict()) model.save(self.db_session)
def accumulation_hook(self, db_session): extrinsic = Extrinsic.query(db_session).filter_by(extrinsic_idx=self.event.extrinsic_idx, block_id=self.event.block_id).first() if extrinsic.module_id == 'Assets' and extrinsic.call_id == 'register': data_asset = Asset( asset_id=self.event.attributes[0]['value'], symbol=extrinsic.params[0]['value'], precision=0 if extrinsic.params[4]['value'] else 18, #TODO may be changed name=extrinsic.params[1]['value'], is_mintable=True if extrinsic.params[3]['value'] else False ) existing_assset = Asset.query(db_session).filter(Asset.asset_id == data_asset.asset_id).first() if not existing_assset: data_asset.save(db_session) else: #XYK pool creation (aka LP token) or Ethereum bridge #don't process it for now pass
def get_relationships(self, include_list, item): relationships = {} if 'recent_extrinsics' in include_list: relationships['recent_extrinsics'] = Extrinsic.query( self.session).filter_by(address=item.id).order_by( Extrinsic.block_id.desc())[:10] if 'indices' in include_list: relationships['indices'] = AccountIndex.query( self.session).filter_by(account_id=item.id).order_by( AccountIndex.updated_at_block.desc()) if 'rewards' in include_list: relationships['rewards'] = Block.query(self.session).filter_by( coinbase=bech32.encode(HRP, bytes().fromhex(item.id))).order_by( Block.id.desc()) # Block.query(self.session).order_by(Block.id.desc() # count = self.session.query(func.count(Block.id)).filter(Block.coinbase == item_id).scalar() return relationships
def add_block(self, block_hash): # Check if block is already process if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0: raise BlockAlreadyAdded(block_hash) # Extract data from json_block substrate = SubstrateInterface(SUBSTRATE_RPC_URL) if SUBSTRATE_MOCK_EXTRINSICS: substrate.mock_extrinsics = SUBSTRATE_MOCK_EXTRINSICS json_block = substrate.get_chain_block(block_hash) parent_hash = json_block['block']['header'].pop('parentHash') block_id = json_block['block']['header'].pop('number') extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot') state_root = json_block['block']['header'].pop('stateRoot') digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None) # Convert block number to numeric if not block_id.isnumeric(): block_id = int(block_id, 16) # ==== Get block runtime from Substrate ================== json_runtime_version = substrate.get_block_runtime_version(block_hash) # Get spec version spec_version = json_runtime_version.get('specVersion', 0) self.process_metadata(json_runtime_version, block_hash) # ==== Get parent block runtime =================== if block_id > 0: json_parent_runtime_version = substrate.get_block_runtime_version(parent_hash) parent_spec_version = json_parent_runtime_version.get('specVersion', 0) self.process_metadata(json_parent_runtime_version, parent_hash) else: parent_spec_version = spec_version # ==== Set initial block properties ===================== block = Block( id=block_id, parent_id=block_id - 1, hash=block_hash, parent_hash=parent_hash, state_root=state_root, extrinsics_root=extrinsics_root, count_extrinsics=0, count_events=0, count_accounts_new=0, count_accounts_reaped=0, count_accounts=0, count_events_extrinsic=0, count_events_finalization=0, count_events_module=0, count_events_system=0, count_extrinsics_error=0, count_extrinsics_signed=0, count_extrinsics_signedby_address=0, count_extrinsics_signedby_index=0, count_extrinsics_success=0, count_extrinsics_unsigned=0, count_sessions_new=0, count_contracts_new=0, count_log=0, range10000=math.floor(block_id / 10000), range100000=math.floor(block_id / 100000), range1000000=math.floor(block_id / 1000000), spec_version_id=spec_version, logs=digest_logs ) # Set temp helper variables block._accounts_new = [] block._accounts_reaped = [] # ==== Get block events from Substrate ================== extrinsic_success_idx = {} events = [] try: events_decoder = substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version]) event_idx = 0 for event in events_decoder.elements: event.value['module_id'] = event.value['module_id'].lower() model = Event( block_id=block_id, event_idx=event_idx, phase=event.value['phase'], extrinsic_idx=event.value['extrinsic_idx'], type=event.value['type'], spec_version_id=parent_spec_version, module_id=event.value['module_id'], event_id=event.value['event_id'], system=int(event.value['module_id'] == 'system'), module=int(event.value['module_id'] != 'system'), attributes=event.value['params'], codec_error=False ) # Process event if event.value['phase'] == 0: block.count_events_extrinsic += 1 elif event.value['phase'] == 1: block.count_events_finalization += 1 if event.value['module_id'] == 'system': block.count_events_system += 1 # Store result of extrinsic if event.value['event_id'] == 'ExtrinsicSuccess': extrinsic_success_idx[event.value['extrinsic_idx']] = True block.count_extrinsics_success += 1 if event.value['event_id'] == 'ExtrinsicFailed': extrinsic_success_idx[event.value['extrinsic_idx']] = False block.count_extrinsics_error += 1 else: block.count_events_module += 1 model.save(self.db_session) events.append(model) event_idx += 1 block.count_events = len(events_decoder.elements) except SubstrateRequestException: block.count_events = 0 # === Extract extrinsics from block ==== extrinsics_data = json_block['block'].pop('extrinsics') block.count_extrinsics = len(extrinsics_data) extrinsic_idx = 0 extrinsics = [] for extrinsic in extrinsics_data: # Save to data table if block_hash == '0x911a0bf66d5494b6b24f612b3cc14841134c6b73ab9ce02f7e012973070e5661': # TODO TEMP fix for exception in Alexander network, remove when network is obsolete extrinsics_decoder = ExtrinsicsBlock61181Decoder( data=ScaleBytes(extrinsic), metadata=self.metadata_store[parent_spec_version] ) else: extrinsics_decoder = ExtrinsicsDecoder( data=ScaleBytes(extrinsic), metadata=self.metadata_store[parent_spec_version] ) extrinsic_data = extrinsics_decoder.decode() # Lookup result of extrinsic extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False) model = Extrinsic( block_id=block_id, extrinsic_idx=extrinsic_idx, extrinsic_hash=extrinsics_decoder.extrinsic_hash, extrinsic_length=extrinsic_data.get('extrinsic_length'), extrinsic_version=extrinsic_data.get('version_info'), signed=extrinsics_decoder.contains_transaction, unsigned=not extrinsics_decoder.contains_transaction, signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')), signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')), address_length=extrinsic_data.get('account_length'), address=extrinsic_data.get('account_id'), account_index=extrinsic_data.get('account_index'), account_idx=extrinsic_data.get('account_idx'), signature=extrinsic_data.get('signature'), nonce=extrinsic_data.get('nonce'), era=extrinsic_data.get('era'), call=extrinsic_data.get('call_code'), module_id=extrinsic_data.get('call_module'), call_id=extrinsic_data.get('call_module_function'), params=extrinsic_data.get('params'), spec_version_id=parent_spec_version, success=int(extrinsic_success), error=int(not extrinsic_success), codec_error=False ) model.save(self.db_session) if extrinsic_data.get('call_module_function') == 'transfer': if len(extrinsic_data.get('params')) > 1: _amount = extrinsic_data.get('params')[1].get('value') else: _amount = extrinsic_data.get('params')[0].get('value') # [{'type': 'AccountId', 'value': '0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d', # 'valueRaw': 'd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'}, # {'type': 'AccountId', 'value': '0x8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48', # 'valueRaw': '8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48'}] account_ids = events_decoder.value[1].get('params')[0:2] transfer = Transfer( block_id=block.id, extrinsic_idx=model.extrinsic_idx, data_extrinsic_idx=str(block_id) + '_' + str(model.extrinsic_idx), transfer_from=ss58_encode(account_ids[0]['valueRaw']), from_raw=account_ids[0]['valueRaw'], transfer_to=ss58_encode(account_ids[1]['valueRaw']), to_raw=account_ids[1]['valueRaw'], hash=model.extrinsic_hash, amount=decimal.Decimal(_amount), block_timestamp=block.datetime, module_id=extrinsic_data.get('call_module'), success=int(extrinsic_success), error=int(not extrinsic_success), created_at=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), updated_at=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")#time.asctime(time.localtime(time.time())) ) transfer.save(self.db_session) extrinsics.append(model) extrinsic_idx += 1 # Process extrinsic if extrinsics_decoder.contains_transaction: block.count_extrinsics_signed += 1 if model.signedby_address: block.count_extrinsics_signedby_address += 1 if model.signedby_index: block.count_extrinsics_signedby_index += 1 else: block.count_extrinsics_unsigned += 1 # Process extrinsic processors for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id): extrinsic_processor = processor_class(block, model) extrinsic_processor.accumulation_hook(self.db_session) # Process event processors for event in events: extrinsic = None if event.extrinsic_idx is not None: try: extrinsic = extrinsics[event.extrinsic_idx] except IndexError: extrinsic = None for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id): event_processor = processor_class(block, event, extrinsic, metadata=self.metadata_store.get(block.spec_version_id)) event_processor.accumulation_hook(self.db_session) # Process block processors for processor_class in ProcessorRegistry().get_block_processors(): block_processor = processor_class(block) block_processor.accumulation_hook(self.db_session) # Debug info if DEBUG: block.debug_info = json_block # ==== Save data block ================================== block.save(self.db_session) return block
def get_query(self): return Extrinsic.query(self.session).order_by( Extrinsic.datetime.desc())
def add_block(self, block_hash): # Check if block is already process if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0: raise BlockAlreadyAdded(block_hash) if settings.SUBSTRATE_MOCK_EXTRINSICS: self.substrate.mock_extrinsics = settings.SUBSTRATE_MOCK_EXTRINSICS json_block = self.substrate.get_chain_block(block_hash) parent_hash = json_block['block']['header'].pop('parentHash') block_id = json_block['block']['header'].pop('number') extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot') state_root = json_block['block']['header'].pop('stateRoot') digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None) # Convert block number to numeric if not block_id.isnumeric(): block_id = int(block_id, 16) # ==== Get block runtime from Substrate ================== self.substrate.init_runtime(block_hash=block_hash) self.process_metadata(self.substrate.runtime_version, block_hash) # ==== Get parent block runtime =================== if block_id > 0: json_parent_runtime_version = self.substrate.get_block_runtime_version(parent_hash) parent_spec_version = json_parent_runtime_version.get('specVersion', 0) self.process_metadata(parent_spec_version, parent_hash) else: parent_spec_version = self.substrate.runtime_version # ==== Set initial block properties ===================== block = Block( id=block_id, parent_id=block_id - 1, hash=block_hash, parent_hash=parent_hash, state_root=state_root, extrinsics_root=extrinsics_root, count_extrinsics=0, count_events=0, count_accounts_new=0, count_accounts_reaped=0, count_accounts=0, count_events_extrinsic=0, count_events_finalization=0, count_events_module=0, count_events_system=0, count_extrinsics_error=0, count_extrinsics_signed=0, count_extrinsics_signedby_address=0, count_extrinsics_signedby_index=0, count_extrinsics_success=0, count_extrinsics_unsigned=0, count_sessions_new=0, count_contracts_new=0, count_log=0, range10000=math.floor(block_id / 10000), range100000=math.floor(block_id / 100000), range1000000=math.floor(block_id / 1000000), spec_version_id=self.substrate.runtime_version, logs=digest_logs ) # Set temp helper variables block._accounts_new = [] block._accounts_reaped = [] # ==== Get block events from Substrate ================== extrinsic_success_idx = {} events = [] try: events_decoder = self.substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version]) event_idx = 0 for event in events_decoder.elements: event.value['module_id'] = event.value['module_id'].lower() model = Event( block_id=block_id, event_idx=event_idx, phase=event.value['phase'], extrinsic_idx=event.value['extrinsic_idx'], type=event.value['type'], spec_version_id=parent_spec_version, module_id=event.value['module_id'], event_id=event.value['event_id'], system=int(event.value['module_id'] == 'system'), module=int(event.value['module_id'] != 'system'), attributes=event.value['params'], codec_error=False ) # Process event if event.value['phase'] == 0: block.count_events_extrinsic += 1 elif event.value['phase'] == 1: block.count_events_finalization += 1 if event.value['module_id'] == 'system': block.count_events_system += 1 # Store result of extrinsic if event.value['event_id'] == 'ExtrinsicSuccess': extrinsic_success_idx[event.value['extrinsic_idx']] = True block.count_extrinsics_success += 1 if event.value['event_id'] == 'ExtrinsicFailed': extrinsic_success_idx[event.value['extrinsic_idx']] = False block.count_extrinsics_error += 1 else: block.count_events_module += 1 model.save(self.db_session) events.append(model) event_idx += 1 block.count_events = len(events_decoder.elements) except SubstrateRequestException: block.count_events = 0 # === Extract extrinsics from block ==== extrinsics_data = json_block['block'].pop('extrinsics') block.count_extrinsics = len(extrinsics_data) extrinsic_idx = 0 extrinsics = [] for extrinsic in extrinsics_data: extrinsics_decoder = ExtrinsicsDecoder( data=ScaleBytes(extrinsic), metadata=self.metadata_store[parent_spec_version] ) extrinsic_data = extrinsics_decoder.decode() # Lookup result of extrinsic extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False) model = Extrinsic( block_id=block_id, extrinsic_idx=extrinsic_idx, extrinsic_hash=extrinsics_decoder.extrinsic_hash, extrinsic_length=extrinsic_data.get('extrinsic_length'), extrinsic_version=extrinsic_data.get('version_info'), signed=extrinsics_decoder.contains_transaction, unsigned=not extrinsics_decoder.contains_transaction, signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')), signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')), address_length=extrinsic_data.get('account_length'), address=extrinsic_data.get('account_id'), account_index=extrinsic_data.get('account_index'), account_idx=extrinsic_data.get('account_idx'), signature=extrinsic_data.get('signature'), nonce=extrinsic_data.get('nonce'), era=extrinsic_data.get('era'), call=extrinsic_data.get('call_code'), module_id=extrinsic_data.get('call_module'), call_id=extrinsic_data.get('call_function'), params=extrinsic_data.get('params'), spec_version_id=parent_spec_version, success=int(extrinsic_success), error=int(not extrinsic_success), codec_error=False ) model.save(self.db_session) extrinsics.append(model) extrinsic_idx += 1 # Process extrinsic if extrinsics_decoder.contains_transaction: block.count_extrinsics_signed += 1 if model.signedby_address: block.count_extrinsics_signedby_address += 1 if model.signedby_index: block.count_extrinsics_signedby_index += 1 # Add search index for signed extrinsics search_index = SearchIndex( index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC, block_id=block.id, extrinsic_idx=model.extrinsic_idx, account_id=model.address ) search_index.save(self.db_session) else: block.count_extrinsics_unsigned += 1 # Process extrinsic processors for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id): extrinsic_processor = processor_class(block, model, substrate=self.substrate) extrinsic_processor.accumulation_hook(self.db_session) extrinsic_processor.process_search_index(self.db_session) # Process event processors for event in events: extrinsic = None if event.extrinsic_idx is not None: try: extrinsic = extrinsics[event.extrinsic_idx] except IndexError: extrinsic = None for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id): event_processor = processor_class(block, event, extrinsic, metadata=self.metadata_store.get(block.spec_version_id), substrate=self.substrate) event_processor.accumulation_hook(self.db_session) event_processor.process_search_index(self.db_session) # Process block processors for processor_class in ProcessorRegistry().get_block_processors(): block_processor = processor_class(block, substrate=self.substrate, harvester=self) block_processor.accumulation_hook(self.db_session) # Debug info if settings.DEBUG: block.debug_info = json_block # ==== Save data block ================================== block.save(self.db_session) return block
def get_query(self): return Extrinsic.query(self.session).options(defer('params')).order_by( Extrinsic.block_id.desc())
def on_get(self, req, resp, network_id=None): resp.status = falcon.HTTP_200 # TODO make caching more generic for custom resources cache_key = '{}-{}'.format(req.method, req.url) console_handler = logging.StreamHandler() console_handler.setLevel('INFO') logger = logging.getLogger('yee') logger.setLevel('DEBUG') logger.addHandler(console_handler) # logger.info(cache_key) response = self.cache_region.get(cache_key, self.cache_expiration_time) if response is NO_VALUE: best_block = Block.query(self.session).filter_by( id=self.session.query(func.max(Block.id)).one()[0]).first() total_signed_extrinsics = Extrinsic.query( self.session).filter_by(signed=1).count() total_accounts = Account.query(self.session).filter_by().count() # total_events = Event.query(self.session).count() event = Event.query(self.session).filter_by( id=self.session.query(func.max(Event.id)).one()[0]).first() if event is None: eventid = 0 else: eventid = event.id if best_block: substrate = SubstrateInterface( SUBSTRATE_RPC_URL, metadata_version=SUBSTRATE_METADATA_VERSION) print(substrate.get_ShardCount()) response = self.get_jsonapi_response(data={ 'type': 'networkstats', 'id': network_id, 'attributes': { 'best_block': best_block.id, 'total_signed_extrinsics': total_signed_extrinsics, 'total_events': eventid, 'total_events_module': int(best_block.id), 'total_blocks': 'N/A', 'total_accounts': total_accounts, 'total_runtimes': Runtime.query(self.session).count(), 'shard_count': int(substrate.get_ShardCount(), 16) } }, ) else: response = self.get_jsonapi_response(data={ 'type': 'networkstats', 'id': network_id, 'attributes': { 'best_block': 0, 'total_signed_extrinsics': 0, 'total_events': 0, 'total_events_module': 0, 'total_blocks': 'N/A', 'total_accounts': 0, 'total_runtimes': 0 } }, ) self.cache_region.set(cache_key, response) resp.set_header('X-Cache', 'MISS') else: resp.set_header('X-Cache', 'HIT') resp.media = response
def get_query(self): return Extrinsic.query(self.session).filter_by( module_id='balances', call_id='transfer').order_by(Extrinsic.datetime.desc())
def get_query(self): return Extrinsic.query(self.session).order_by( Extrinsic.block_id.desc())
def get_item(self, item_id): if item_id[0:2] == '0x': return Extrinsic.query( self.session).filter_by(extrinsic_hash=item_id[2:]).first()
def get_query(self): return Extrinsic.query( self.session).filter(Extrinsic.module_id == 'balances' and Extrinsic.call_id == 'transfer').order_by( Extrinsic.block_id.desc())
def on_post(self, req, resp): blockHash = None if req.media.get('block_id'): substrate = SubstrateInterface(url=SUBSTRATE_RPC_URL, address_type=SUBSTRATE_ADDRESS_TYPE, type_registry_preset=TYPE_REGISTRY) blockHash = substrate.get_block_hash(req.media.get('block_id')) elif req.media.get('block_hash'): blockHash = req.media.get('block_hash') else: resp.status = falcon.HTTP_BAD_REQUEST resp.media = { 'errors': ['Either blockHash or block_id should be supplied'] } if blockHash: resp.status = falcon.HTTP_200 block = Block.query( self.session).filter(Block.hash == blockHash).first() blockTotal = BlockTotal.query( self.session).filter(BlockTotal.id == block.id).first() author = ss58_encode( blockTotal.author.replace('0x', '') ) if blockTotal is not None and blockTotal.author is not None else None if block: blockInfo = {} blockInfo["timestamp"] = block.datetime.strftime( "%Y-%m-%d %H:%M:%S") blockInfo["block_hash"] = block.hash blockInfo["block_id"] = block.id blockInfo["parent_id"] = block.id - 1 if block.id > 0 else 0 blockInfo["child_id"] = block.id + 1 blockInfo["parent_hash"] = block.parent_hash blockInfo["state_root"] = block.state_root blockInfo["extrinsic_root"] = block.extrinsics_root blockInfo["validator"] = author blockInfo["count_extrinsic"] = block.count_extrinsics blockInfo["count_event"] = block.count_events blockInfo["count_log"] = block.count_log # blockInfo["age"] = time.mktime(block.datetime.timetuple()) blockInfo["age"] = block.datetime.strftime("%Y-%m-%d %H:%M:%S") # 获取和区块相关的交易信息 extrinsics = Extrinsic.query( self.session).filter(Extrinsic.block_id == block.id).all() extrinsicsObj = [ { "extrinsic_id": '{}-{}'.format(block.id, extrinsic.extrinsic_idx), "hash": extrinsic.extrinsic_hash if extrinsic.extrinsic_hash else None, # "age": time.mktime(block.datetime.timetuple()), "age": block.datetime.strftime("%Y-%m-%d %H:%M:%S"), "result": extrinsic.success, # "address": extrinsic.address if extrinsic.address else None, # "module": extrinsic.module_id, # "fee": None, # "nonce": extrinsic.nonce if extrinsic.nonce else None, # "call": extrinsic.call_id, "operation": '{}({})'.format(extrinsic.module_id, extrinsic.call_id), "params": extrinsic.params # "signature": extrinsic.signature if extrinsic.signature else None } for extrinsic in extrinsics ] # 获取和区块相关的日志信息 logs = Log.query( self.session).filter(Log.block_id == block.id).all() logsObj = [{ "log_id": '{}-{}'.format(block.id, log.log_idx), "block_id": block.id, "type": log.type, "data": log.data['value'] } for log in logs] # 获取和区块相关的事件信息 events = Event.query( self.session).filter(Event.block_id == block.id).all() eventObj = [{ "id": '{}-{}'.format(block.id, event.event_idx), "block_id": block.id, "block_hash": block.hash, "module_id": event.module_id, "event_id": event.event_id, "attributes": event.attributes, "operation": '{}({})'.format(event.module_id, event.event_id), "desc": self.getEventDesc(event.module_id, event.event_id), "hash": self.getEventHash(block.id, event.extrinsic_idx) } for event in events] resp.media = { 'status': 'success', 'data': { "block_info": blockInfo, "extrinsics": extrinsicsObj, "logs": logsObj, "events": eventObj } } else: resp.status = falcon.HTTP_404 resp.media = {'result': 'Block not found'}
def on_get(self, req, resp): resp.status = falcon.HTTP_200 extrinsicId = req.params.get('extrinsic_id') if extrinsicId is None: resp.status = falcon.HTTP_BAD_REQUEST resp.media = {'errors': ['Invalid extrinsic id']} split = extrinsicId.split("-") if split is None or len(split) != 2: resp.status = falcon.HTTP_BAD_REQUEST resp.media = {'errors': ['Invalid extrinsic id']} blockId = split[0] extrinsicIdx = split[1] extrinsic = Extrinsic.extrinsic_by_id(self.session, blockId, extrinsicIdx).first() print(extrinsic.extrinsic_idx) extrinsicDetailInfo = {} if extrinsic is not None: extrinsicDetailInfo["extrinsic_id"] = '{}-{}'.format( extrinsic.block_id, extrinsic.extrinsic_idx) extrinsicDetailInfo["block_id"] = extrinsic.block_id extrinsicDetailInfo["extrinsic_idx"] = extrinsic.extrinsic_idx extrinsicDetailInfo["hash"] = "0x{}".format( extrinsic.extrinsic_hash) if extrinsic.extrinsic_hash else None extrinsicDetailInfo["age"] = extrinsic.datetime.strftime( "%Y-%m-%d %H:%M:%S") extrinsicDetailInfo["result"] = extrinsic.success extrinsicDetailInfo["operation"] = '{}({})'.format( extrinsic.module_id, extrinsic.call_id) extrinsicDetailInfo["module_id"] = extrinsic.module_id extrinsicDetailInfo["call_id"] = extrinsic.call_id extrinsicDetailInfo[ "module_name"] = extrinsic.module_name.capitalize() extrinsicDetailInfo["call_name"] = extrinsic.call_name.capitalize() extrinsicDetailInfo["call_desc"] = extrinsic.call_desc extrinsicDetailInfo["nonce"] = int( extrinsic.nonce) if extrinsic.nonce else None extrinsicDetailInfo[ "signature"] = extrinsic.signature if extrinsic.signature else None extrinsicDetailInfo["params"] = extrinsic.params extrinsicDetailInfo["from"] = extrinsic.address.replace( '0x', '') if extrinsic.address else None extrinsicDetailInfo["from_addr"] = ss58_encode( extrinsic.address.replace('0x', '')) if extrinsic.address else None params = json.loads(extrinsic.params) for param in params: name = param.get('name') if name == 'dest': toAddr = ss58_encode(param.get('value').replace('0x', '')) extrinsicDetailInfo['to'] = param.get('value').replace( '0x', '') extrinsicDetailInfo['to_addr'] = toAddr elif name == 'value': coin = param.get('value') / 1000000000000 # 转换为单位 Unit extrinsicDetailInfo['coin'] = coin # 获取和区块相关的事件信息 events = Event.query(self.session).filter( Event.block_id == extrinsic.block_id).all() eventObj = [{ "id": '{}-{}'.format(extrinsic.block_id, event.event_idx), "block_id": extrinsic.block_id, "module_id": event.module_id, "event_id": event.event_id, "attributes": event.attributes, "operation": '{}({})'.format(event.module_id, event.event_id), "desc": self.getEventDesc(event.module_id, event.event_id), "hash": self.getEventHash(extrinsic.block_id, event.extrinsic_idx) } for event in events] resp.media = { 'status': 'success', 'data': { "extrinsic_detail": extrinsicDetailInfo, "events": eventObj, "event_count": len(eventObj) } }