def rebuild_search_index(self):
self.db_session.execute('truncate table {}'.format(
SearchIndex.__tablename__))
for block in Block.query(
self.db_session).order_by('id').yield_per(1000):
extrinsic_lookup = {}
block._accounts_new = []
block._accounts_reaped = []
for extrinsic in Extrinsic.query(self.db_session).filter_by(
block_id=block.id).order_by('extrinsic_idx'):
extrinsic_lookup[extrinsic.extrinsic_idx] = extrinsic
# Add search index for signed extrinsics
if extrinsic.address:
search_index = SearchIndex(
index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC,
block_id=block.id,
extrinsic_idx=extrinsic.extrinsic_idx,
account_id=extrinsic.address)
search_index.save(self.db_session)
# Process extrinsic processors
for processor_class in ProcessorRegistry(
).get_extrinsic_processors(extrinsic.module_id,
extrinsic.call_id):
extrinsic_processor = processor_class(
block=block,
extrinsic=extrinsic,
substrate=self.substrate)
extrinsic_processor.process_search_index(self.db_session)
for event in Event.query(self.db_session).filter_by(
block_id=block.id).order_by('event_idx'):
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsic_lookup[event.extrinsic_idx]
except (IndexError, KeyError):
extrinsic = None
for processor_class in ProcessorRegistry(
).get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(
block,
event,
extrinsic,
metadata=self.metadata_store.get(
block.spec_version_id),
substrate=self.substrate)
event_processor.process_search_index(self.db_session)
self.db_session.commit()
def apply_filters(self, query, params):
if params.get('filter[address]'):
if len(params.get('filter[address]')) == 64:
account_id = params.get('filter[address]')
else:
try:
account_id = ss58_decode(params.get('filter[address]'),
settings.SUBSTRATE_ADDRESS_TYPE)
except ValueError:
return query.filter(False)
search_index = SearchIndex.query(self.session).filter(
SearchIndex.index_type_id.in_([
settings.SEARCH_INDEX_BALANCETRANSFER,
settings.SEARCH_INDEX_CLAIMS_CLAIMED,
settings.SEARCH_INDEX_BALANCES_DEPOSIT,
settings.SEARCH_INDEX_STAKING_REWARD
]), SearchIndex.account_id == account_id).order_by(
SearchIndex.sorting_value.desc())
query = Event.query(self.session).filter(
tuple_(Event.block_id, Event.event_idx).in_(
[[s.block_id, s.event_idx]
for s in search_index])).order_by(Event.block_id.desc())
return query
def add_search_index(self, index_type_id, account_id=None, sorting_value=None):
return SearchIndex(
index_type_id=index_type_id,
block_id=self.block.id,
event_idx=None,
extrinsic_idx=self.extrinsic.extrinsic_idx,
account_id=account_id,
sorting_value=sorting_value
)
def apply_filters(self, query, params):
if params.get('filter[address]'):
if len(params.get('filter[address]')) == 64:
account_id = params.get('filter[address]')
else:
try:
account_id = ss58_decode(params.get('filter[address]'),
settings.SUBSTRATE_ADDRESS_TYPE)
except ValueError:
return query.filter(False)
else:
account_id = None
if params.get('filter[search_index]'):
self.exclude_params = False
if type(params.get('filter[search_index]')) != list:
params['filter[search_index]'] = [
params.get('filter[search_index]')
]
search_index = SearchIndex.query(self.session).filter(
SearchIndex.index_type_id.in_(
params.get('filter[search_index]')),
SearchIndex.account_id == account_id).order_by(
SearchIndex.sorting_value.desc())
query = query.filter(
tuple_(Extrinsic.block_id, Extrinsic.extrinsic_idx).in_(
[[s.block_id, s.extrinsic_idx] for s in search_index]))
else:
self.exclude_params = True
if params.get('filter[signed]'):
query = query.filter_by(signed=params.get('filter[signed]'))
if params.get('filter[module_id]'):
query = query.filter_by(
module_id=params.get('filter[module_id]'))
if params.get('filter[call_id]'):
query = query.filter_by(call_id=params.get('filter[call_id]'))
if params.get('filter[address]'):
query = query.filter_by(address=account_id)
return query
def apply_filters(self, query, params):
if params.get('filter[address]'):
if len(params.get('filter[address]')) == 64:
account_id = params.get('filter[address]')
else:
try:
account_id = ss58_decode(params.get('filter[address]'),
settings.SUBSTRATE_ADDRESS_TYPE)
except ValueError:
return query.filter(False)
else:
account_id = None
if params.get('filter[search_index]'):
if type(params.get('filter[search_index]')) != list:
params['filter[search_index]'] = [
params.get('filter[search_index]')
]
search_index = SearchIndex.query(self.session).filter(
SearchIndex.index_type_id.in_(
params.get('filter[search_index]')),
SearchIndex.account_id == account_id).order_by(
SearchIndex.sorting_value.desc())
query = query.filter(
tuple_(Event.block_id,
Event.event_idx).in_([[s.block_id, s.event_idx]
for s in search_index]))
else:
if params.get('filter[module_id]'):
query = query.filter_by(
module_id=params.get('filter[module_id]'))
if params.get('filter[event_id]'):
query = query.filter_by(
event_id=params.get('filter[event_id]'))
else:
query = query.filter(
Event.event_id.notin_(
['ExtrinsicSuccess', 'ExtrinsicFailed']))
return query
def apply_filters(self, query, params):
if params.get('filter[address]'):
if len(params.get('filter[address]')) == 64:
account_id = params.get('filter[address]')
else:
try:
account_id = ss58_decode(params.get('filter[address]'),
SUBSTRATE_ADDRESS_TYPE)
except ValueError:
return query.filter(False)
else:
account_id = None
if params.get('filter[search_index]'):
search_index = SearchIndex.query(self.session).filter_by(
index_type_id=params.get('filter[search_index]'),
account_id=account_id).order_by(
SearchIndex.sorting_value.desc())
query = query.filter(
tuple_(Event.block_id,
Event.event_idx).in_([[s.block_id, s.event_idx]
for s in search_index]))
else:
if params.get('filter[module_id]'):
query = query.filter_by(
module_id=params.get('filter[module_id]'))
if params.get('filter[event_id]'):
query = query.filter_by(
event_id=params.get('filter[event_id]'))
return query
def add_block(self, block_hash):
# Check if block is already process
if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0:
raise BlockAlreadyAdded(block_hash)
if settings.SUBSTRATE_MOCK_EXTRINSICS:
self.substrate.mock_extrinsics = settings.SUBSTRATE_MOCK_EXTRINSICS
json_block = self.substrate.get_chain_block(block_hash)
parent_hash = json_block['block']['header'].pop('parentHash')
block_id = json_block['block']['header'].pop('number')
extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot')
state_root = json_block['block']['header'].pop('stateRoot')
digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None)
# Convert block number to numeric
if not block_id.isnumeric():
block_id = int(block_id, 16)
# ==== Get block runtime from Substrate ==================
self.substrate.init_runtime(block_hash=block_hash)
self.process_metadata(self.substrate.runtime_version, block_hash)
# ==== Get parent block runtime ===================
if block_id > 0:
json_parent_runtime_version = self.substrate.get_block_runtime_version(parent_hash)
parent_spec_version = json_parent_runtime_version.get('specVersion', 0)
self.process_metadata(parent_spec_version, parent_hash)
else:
parent_spec_version = self.substrate.runtime_version
# ==== Set initial block properties =====================
block = Block(
id=block_id,
parent_id=block_id - 1,
hash=block_hash,
parent_hash=parent_hash,
state_root=state_root,
extrinsics_root=extrinsics_root,
count_extrinsics=0,
count_events=0,
count_accounts_new=0,
count_accounts_reaped=0,
count_accounts=0,
count_events_extrinsic=0,
count_events_finalization=0,
count_events_module=0,
count_events_system=0,
count_extrinsics_error=0,
count_extrinsics_signed=0,
count_extrinsics_signedby_address=0,
count_extrinsics_signedby_index=0,
count_extrinsics_success=0,
count_extrinsics_unsigned=0,
count_sessions_new=0,
count_contracts_new=0,
count_log=0,
range10000=math.floor(block_id / 10000),
range100000=math.floor(block_id / 100000),
range1000000=math.floor(block_id / 1000000),
spec_version_id=self.substrate.runtime_version,
logs=digest_logs
)
# Set temp helper variables
block._accounts_new = []
block._accounts_reaped = []
# ==== Get block events from Substrate ==================
extrinsic_success_idx = {}
events = []
try:
events_decoder = self.substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version])
event_idx = 0
for event in events_decoder.elements:
event.value['module_id'] = event.value['module_id'].lower()
model = Event(
block_id=block_id,
event_idx=event_idx,
phase=event.value['phase'],
extrinsic_idx=event.value['extrinsic_idx'],
type=event.value['type'],
spec_version_id=parent_spec_version,
module_id=event.value['module_id'],
event_id=event.value['event_id'],
system=int(event.value['module_id'] == 'system'),
module=int(event.value['module_id'] != 'system'),
attributes=event.value['params'],
codec_error=False
)
# Process event
if event.value['phase'] == 0:
block.count_events_extrinsic += 1
elif event.value['phase'] == 1:
block.count_events_finalization += 1
if event.value['module_id'] == 'system':
block.count_events_system += 1
# Store result of extrinsic
if event.value['event_id'] == 'ExtrinsicSuccess':
extrinsic_success_idx[event.value['extrinsic_idx']] = True
block.count_extrinsics_success += 1
if event.value['event_id'] == 'ExtrinsicFailed':
extrinsic_success_idx[event.value['extrinsic_idx']] = False
block.count_extrinsics_error += 1
else:
block.count_events_module += 1
model.save(self.db_session)
events.append(model)
event_idx += 1
block.count_events = len(events_decoder.elements)
except SubstrateRequestException:
block.count_events = 0
# === Extract extrinsics from block ====
extrinsics_data = json_block['block'].pop('extrinsics')
block.count_extrinsics = len(extrinsics_data)
extrinsic_idx = 0
extrinsics = []
for extrinsic in extrinsics_data:
extrinsics_decoder = ExtrinsicsDecoder(
data=ScaleBytes(extrinsic),
metadata=self.metadata_store[parent_spec_version]
)
extrinsic_data = extrinsics_decoder.decode()
# Lookup result of extrinsic
extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False)
model = Extrinsic(
block_id=block_id,
extrinsic_idx=extrinsic_idx,
extrinsic_hash=extrinsics_decoder.extrinsic_hash,
extrinsic_length=extrinsic_data.get('extrinsic_length'),
extrinsic_version=extrinsic_data.get('version_info'),
signed=extrinsics_decoder.contains_transaction,
unsigned=not extrinsics_decoder.contains_transaction,
signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')),
signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')),
address_length=extrinsic_data.get('account_length'),
address=extrinsic_data.get('account_id'),
account_index=extrinsic_data.get('account_index'),
account_idx=extrinsic_data.get('account_idx'),
signature=extrinsic_data.get('signature'),
nonce=extrinsic_data.get('nonce'),
era=extrinsic_data.get('era'),
call=extrinsic_data.get('call_code'),
module_id=extrinsic_data.get('call_module'),
call_id=extrinsic_data.get('call_function'),
params=extrinsic_data.get('params'),
spec_version_id=parent_spec_version,
success=int(extrinsic_success),
error=int(not extrinsic_success),
codec_error=False
)
model.save(self.db_session)
extrinsics.append(model)
extrinsic_idx += 1
# Process extrinsic
if extrinsics_decoder.contains_transaction:
block.count_extrinsics_signed += 1
if model.signedby_address:
block.count_extrinsics_signedby_address += 1
if model.signedby_index:
block.count_extrinsics_signedby_index += 1
# Add search index for signed extrinsics
search_index = SearchIndex(
index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC,
block_id=block.id,
extrinsic_idx=model.extrinsic_idx,
account_id=model.address
)
search_index.save(self.db_session)
else:
block.count_extrinsics_unsigned += 1
# Process extrinsic processors
for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id):
extrinsic_processor = processor_class(block, model, substrate=self.substrate)
extrinsic_processor.accumulation_hook(self.db_session)
extrinsic_processor.process_search_index(self.db_session)
# Process event processors
for event in events:
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsics[event.extrinsic_idx]
except IndexError:
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, extrinsic,
metadata=self.metadata_store.get(block.spec_version_id),
substrate=self.substrate)
event_processor.accumulation_hook(self.db_session)
event_processor.process_search_index(self.db_session)
# Process block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block, substrate=self.substrate, harvester=self)
block_processor.accumulation_hook(self.db_session)
# Debug info
if settings.DEBUG:
block.debug_info = json_block
# ==== Save data block ==================================
block.save(self.db_session)
return block