def add_block(self, block_hash):
# Check if block is already process
if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0:
raise BlockAlreadyAdded(block_hash)
if settings.SUBSTRATE_MOCK_EXTRINSICS:
self.substrate.mock_extrinsics = settings.SUBSTRATE_MOCK_EXTRINSICS
json_block = self.substrate.get_chain_block(block_hash)
parent_hash = json_block['block']['header'].pop('parentHash')
block_id = json_block['block']['header'].pop('number')
extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot')
state_root = json_block['block']['header'].pop('stateRoot')
digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None)
# Convert block number to numeric
if not block_id.isnumeric():
block_id = int(block_id, 16)
# ==== Get block runtime from Substrate ==================
self.substrate.init_runtime(block_hash=block_hash)
self.process_metadata(self.substrate.runtime_version, block_hash)
# ==== Get parent block runtime ===================
if block_id > 0:
json_parent_runtime_version = self.substrate.get_block_runtime_version(parent_hash)
parent_spec_version = json_parent_runtime_version.get('specVersion', 0)
self.process_metadata(parent_spec_version, parent_hash)
else:
parent_spec_version = self.substrate.runtime_version
# ==== Set initial block properties =====================
block = Block(
id=block_id,
parent_id=block_id - 1,
hash=block_hash,
parent_hash=parent_hash,
state_root=state_root,
extrinsics_root=extrinsics_root,
count_extrinsics=0,
count_events=0,
count_accounts_new=0,
count_accounts_reaped=0,
count_accounts=0,
count_events_extrinsic=0,
count_events_finalization=0,
count_events_module=0,
count_events_system=0,
count_extrinsics_error=0,
count_extrinsics_signed=0,
count_extrinsics_signedby_address=0,
count_extrinsics_signedby_index=0,
count_extrinsics_success=0,
count_extrinsics_unsigned=0,
count_sessions_new=0,
count_contracts_new=0,
count_log=0,
range10000=math.floor(block_id / 10000),
range100000=math.floor(block_id / 100000),
range1000000=math.floor(block_id / 1000000),
spec_version_id=self.substrate.runtime_version,
logs=digest_logs
)
# Set temp helper variables
block._accounts_new = []
block._accounts_reaped = []
# ==== Get block events from Substrate ==================
extrinsic_success_idx = {}
events = []
try:
events_decoder = self.substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version])
event_idx = 0
for event in events_decoder.elements:
event.value['module_id'] = event.value['module_id'].lower()
model = Event(
block_id=block_id,
event_idx=event_idx,
phase=event.value['phase'],
extrinsic_idx=event.value['extrinsic_idx'],
type=event.value['type'],
spec_version_id=parent_spec_version,
module_id=event.value['module_id'],
event_id=event.value['event_id'],
system=int(event.value['module_id'] == 'system'),
module=int(event.value['module_id'] != 'system'),
attributes=event.value['params'],
codec_error=False
)
# Process event
if event.value['phase'] == 0:
block.count_events_extrinsic += 1
elif event.value['phase'] == 1:
block.count_events_finalization += 1
if event.value['module_id'] == 'system':
block.count_events_system += 1
# Store result of extrinsic
if event.value['event_id'] == 'ExtrinsicSuccess':
extrinsic_success_idx[event.value['extrinsic_idx']] = True
block.count_extrinsics_success += 1
if event.value['event_id'] == 'ExtrinsicFailed':
extrinsic_success_idx[event.value['extrinsic_idx']] = False
block.count_extrinsics_error += 1
else:
block.count_events_module += 1
model.save(self.db_session)
events.append(model)
event_idx += 1
block.count_events = len(events_decoder.elements)
except SubstrateRequestException:
block.count_events = 0
# === Extract extrinsics from block ====
extrinsics_data = json_block['block'].pop('extrinsics')
block.count_extrinsics = len(extrinsics_data)
extrinsic_idx = 0
extrinsics = []
for extrinsic in extrinsics_data:
extrinsics_decoder = ExtrinsicsDecoder(
data=ScaleBytes(extrinsic),
metadata=self.metadata_store[parent_spec_version]
)
extrinsic_data = extrinsics_decoder.decode()
# Lookup result of extrinsic
extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False)
model = Extrinsic(
block_id=block_id,
extrinsic_idx=extrinsic_idx,
extrinsic_hash=extrinsics_decoder.extrinsic_hash,
extrinsic_length=extrinsic_data.get('extrinsic_length'),
extrinsic_version=extrinsic_data.get('version_info'),
signed=extrinsics_decoder.contains_transaction,
unsigned=not extrinsics_decoder.contains_transaction,
signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')),
signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')),
address_length=extrinsic_data.get('account_length'),
address=extrinsic_data.get('account_id'),
account_index=extrinsic_data.get('account_index'),
account_idx=extrinsic_data.get('account_idx'),
signature=extrinsic_data.get('signature'),
nonce=extrinsic_data.get('nonce'),
era=extrinsic_data.get('era'),
call=extrinsic_data.get('call_code'),
module_id=extrinsic_data.get('call_module'),
call_id=extrinsic_data.get('call_function'),
params=extrinsic_data.get('params'),
spec_version_id=parent_spec_version,
success=int(extrinsic_success),
error=int(not extrinsic_success),
codec_error=False
)
model.save(self.db_session)
extrinsics.append(model)
extrinsic_idx += 1
# Process extrinsic
if extrinsics_decoder.contains_transaction:
block.count_extrinsics_signed += 1
if model.signedby_address:
block.count_extrinsics_signedby_address += 1
if model.signedby_index:
block.count_extrinsics_signedby_index += 1
# Add search index for signed extrinsics
search_index = SearchIndex(
index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC,
block_id=block.id,
extrinsic_idx=model.extrinsic_idx,
account_id=model.address
)
search_index.save(self.db_session)
else:
block.count_extrinsics_unsigned += 1
# Process extrinsic processors
for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id):
extrinsic_processor = processor_class(block, model, substrate=self.substrate)
extrinsic_processor.accumulation_hook(self.db_session)
extrinsic_processor.process_search_index(self.db_session)
# Process event processors
for event in events:
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsics[event.extrinsic_idx]
except IndexError:
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, extrinsic,
metadata=self.metadata_store.get(block.spec_version_id),
substrate=self.substrate)
event_processor.accumulation_hook(self.db_session)
event_processor.process_search_index(self.db_session)
# Process block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block, substrate=self.substrate, harvester=self)
block_processor.accumulation_hook(self.db_session)
# Debug info
if settings.DEBUG:
block.debug_info = json_block
# ==== Save data block ==================================
block.save(self.db_session)
return block
def add_block(self, block_hash):
# Check if block is already process
if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0:
raise BlockAlreadyAdded(block_hash)
# Extract data from json_block
substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
if SUBSTRATE_MOCK_EXTRINSICS:
substrate.mock_extrinsics = SUBSTRATE_MOCK_EXTRINSICS
json_block = substrate.get_chain_block(block_hash)
parent_hash = json_block['block']['header'].pop('parentHash')
block_id = json_block['block']['header'].pop('number')
extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot')
state_root = json_block['block']['header'].pop('stateRoot')
digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None)
# Convert block number to numeric
if not block_id.isnumeric():
block_id = int(block_id, 16)
# ==== Get block runtime from Substrate ==================
json_runtime_version = substrate.get_block_runtime_version(block_hash)
# Get spec version
spec_version = json_runtime_version.get('specVersion', 0)
self.process_metadata(json_runtime_version, block_hash)
# ==== Get parent block runtime ===================
if block_id > 0:
json_parent_runtime_version = substrate.get_block_runtime_version(parent_hash)
parent_spec_version = json_parent_runtime_version.get('specVersion', 0)
self.process_metadata(json_parent_runtime_version, parent_hash)
else:
parent_spec_version = spec_version
# ==== Set initial block properties =====================
block = Block(
id=block_id,
parent_id=block_id - 1,
hash=block_hash,
parent_hash=parent_hash,
state_root=state_root,
extrinsics_root=extrinsics_root,
count_extrinsics=0,
count_events=0,
count_accounts_new=0,
count_accounts_reaped=0,
count_accounts=0,
count_events_extrinsic=0,
count_events_finalization=0,
count_events_module=0,
count_events_system=0,
count_extrinsics_error=0,
count_extrinsics_signed=0,
count_extrinsics_signedby_address=0,
count_extrinsics_signedby_index=0,
count_extrinsics_success=0,
count_extrinsics_unsigned=0,
count_sessions_new=0,
count_contracts_new=0,
count_log=0,
range10000=math.floor(block_id / 10000),
range100000=math.floor(block_id / 100000),
range1000000=math.floor(block_id / 1000000),
spec_version_id=spec_version,
logs=digest_logs
)
# Set temp helper variables
block._accounts_new = []
block._accounts_reaped = []
# ==== Get block events from Substrate ==================
extrinsic_success_idx = {}
events = []
try:
events_decoder = substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version])
event_idx = 0
for event in events_decoder.elements:
event.value['module_id'] = event.value['module_id'].lower()
model = Event(
block_id=block_id,
event_idx=event_idx,
phase=event.value['phase'],
extrinsic_idx=event.value['extrinsic_idx'],
type=event.value['type'],
spec_version_id=parent_spec_version,
module_id=event.value['module_id'],
event_id=event.value['event_id'],
system=int(event.value['module_id'] == 'system'),
module=int(event.value['module_id'] != 'system'),
attributes=event.value['params'],
codec_error=False
)
# Process event
if event.value['phase'] == 0:
block.count_events_extrinsic += 1
elif event.value['phase'] == 1:
block.count_events_finalization += 1
if event.value['module_id'] == 'system':
block.count_events_system += 1
# Store result of extrinsic
if event.value['event_id'] == 'ExtrinsicSuccess':
extrinsic_success_idx[event.value['extrinsic_idx']] = True
block.count_extrinsics_success += 1
if event.value['event_id'] == 'ExtrinsicFailed':
extrinsic_success_idx[event.value['extrinsic_idx']] = False
block.count_extrinsics_error += 1
else:
block.count_events_module += 1
model.save(self.db_session)
events.append(model)
event_idx += 1
block.count_events = len(events_decoder.elements)
except SubstrateRequestException:
block.count_events = 0
# === Extract extrinsics from block ====
extrinsics_data = json_block['block'].pop('extrinsics')
block.count_extrinsics = len(extrinsics_data)
extrinsic_idx = 0
extrinsics = []
for extrinsic in extrinsics_data:
# Save to data table
if block_hash == '0x911a0bf66d5494b6b24f612b3cc14841134c6b73ab9ce02f7e012973070e5661':
# TODO TEMP fix for exception in Alexander network, remove when network is obsolete
extrinsics_decoder = ExtrinsicsBlock61181Decoder(
data=ScaleBytes(extrinsic),
metadata=self.metadata_store[parent_spec_version]
)
else:
extrinsics_decoder = ExtrinsicsDecoder(
data=ScaleBytes(extrinsic),
metadata=self.metadata_store[parent_spec_version]
)
extrinsic_data = extrinsics_decoder.decode()
# Lookup result of extrinsic
extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False)
model = Extrinsic(
block_id=block_id,
extrinsic_idx=extrinsic_idx,
extrinsic_hash=extrinsics_decoder.extrinsic_hash,
extrinsic_length=extrinsic_data.get('extrinsic_length'),
extrinsic_version=extrinsic_data.get('version_info'),
signed=extrinsics_decoder.contains_transaction,
unsigned=not extrinsics_decoder.contains_transaction,
signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')),
signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')),
address_length=extrinsic_data.get('account_length'),
address=extrinsic_data.get('account_id'),
account_index=extrinsic_data.get('account_index'),
account_idx=extrinsic_data.get('account_idx'),
signature=extrinsic_data.get('signature'),
nonce=extrinsic_data.get('nonce'),
era=extrinsic_data.get('era'),
call=extrinsic_data.get('call_code'),
module_id=extrinsic_data.get('call_module'),
call_id=extrinsic_data.get('call_module_function'),
params=extrinsic_data.get('params'),
spec_version_id=parent_spec_version,
success=int(extrinsic_success),
error=int(not extrinsic_success),
codec_error=False
)
model.save(self.db_session)
if extrinsic_data.get('call_module_function') == 'transfer':
if len(extrinsic_data.get('params')) > 1:
_amount = extrinsic_data.get('params')[1].get('value')
else:
_amount = extrinsic_data.get('params')[0].get('value')
# [{'type': 'AccountId', 'value': '0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d',
# 'valueRaw': 'd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'},
# {'type': 'AccountId', 'value': '0x8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48',
# 'valueRaw': '8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48'}]
account_ids = events_decoder.value[1].get('params')[0:2]
transfer = Transfer(
block_id=block.id,
extrinsic_idx=model.extrinsic_idx,
data_extrinsic_idx=str(block_id) + '_' + str(model.extrinsic_idx),
transfer_from=ss58_encode(account_ids[0]['valueRaw']),
from_raw=account_ids[0]['valueRaw'],
transfer_to=ss58_encode(account_ids[1]['valueRaw']),
to_raw=account_ids[1]['valueRaw'],
hash=model.extrinsic_hash,
amount=decimal.Decimal(_amount),
block_timestamp=block.datetime,
module_id=extrinsic_data.get('call_module'),
success=int(extrinsic_success),
error=int(not extrinsic_success),
created_at=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
updated_at=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")#time.asctime(time.localtime(time.time()))
)
transfer.save(self.db_session)
extrinsics.append(model)
extrinsic_idx += 1
# Process extrinsic
if extrinsics_decoder.contains_transaction:
block.count_extrinsics_signed += 1
if model.signedby_address:
block.count_extrinsics_signedby_address += 1
if model.signedby_index:
block.count_extrinsics_signedby_index += 1
else:
block.count_extrinsics_unsigned += 1
# Process extrinsic processors
for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id):
extrinsic_processor = processor_class(block, model)
extrinsic_processor.accumulation_hook(self.db_session)
# Process event processors
for event in events:
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsics[event.extrinsic_idx]
except IndexError:
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, extrinsic,
metadata=self.metadata_store.get(block.spec_version_id))
event_processor.accumulation_hook(self.db_session)
# Process block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block)
block_processor.accumulation_hook(self.db_session)
# Debug info
if DEBUG:
block.debug_info = json_block
# ==== Save data block ==================================
block.save(self.db_session)
return block
