def on_post(self, req, resp):
block_hash = None
if 'block_id' in req.media:
block = Block.query(self.session).filter(
Block.id == req.media.get('block_id')).first()
elif req.media.get('block_hash'):
block_hash = req.media.get('block_hash')
block = Block.query(
self.session).filter(Block.hash == block_hash).first()
else:
block = None
resp.status = falcon.HTTP_BAD_REQUEST
resp.media = {
'errors': ['Either block_hash or block_id should be supplied']
}
if block:
print('Sequencing #{} ...'.format(block.id))
harvester = PolkascanHarvesterService(self.session,
type_registry=TYPE_REGISTRY)
if block.id == 1:
# Add genesis block
parent_block = harvester.add_block(block.parent_hash)
block_total = BlockTotal.query(
self.session).filter_by(id=block.id).first()
parent_block = Block.query(
self.session).filter(Block.id == block.id - 1).first()
parent_block_total = BlockTotal.query(
self.session).filter_by(id=block.id - 1).first()
if block_total:
resp.status = falcon.HTTP_200
resp.media = {'result': 'already exists', 'blockId': block.id}
else:
if parent_block_total:
parent_block_total = parent_block_total.asdict()
if parent_block:
parent_block = parent_block.asdict()
harvester.sequence_block(block, parent_block,
parent_block_total)
self.session.commit()
resp.status = falcon.HTTP_201
resp.media = {
'result': 'added',
'parentHash': block.parent_hash
}
else:
resp.status = falcon.HTTP_404
resp.media = {'result': 'Block not found'}
def on_post(self, req, resp):
msg = "TODO"
if req.media.get('start_hash'):
block = Block.query(self.session).filter(Block.hash == req.media.get('start_hash')).first()
else:
block = Block.query(self.session).order_by(Block.id.asc()).first()
if block and block.id != 1:
harvester = PolkascanHarvesterService(self.session, type_registry=TYPE_REGISTRY)
block_hash = block.parent_hash
for nr in range(0, block.id - 1):
try:
block = harvester.add_block(block_hash)
except BlockAlreadyAdded as e:
print('Skipping {}'.format(block_hash))
block_hash = block.parent_hash
if block.id == 0:
break
self.session.commit()
resp.media = {
'status': 'success',
'data': {
'message': msg
}
}
else:
resp.status = falcon.HTTP_404
resp.media = {'result': 'Block not found'}
def get_item(self, item_id):
if item_id.isnumeric():
return Block.query(self.session).filter_by(id=item_id).first()
elif '-' in item_id:
st = item_id.split("-")
return Block.query(self.session).filter_by(bid=int(st[1]),
shard_num=int(
st[0])).first()
else:
return Block.query(self.session).filter_by(hash=item_id).first()
def sequence_block_recursive(self,
parent_block_data,
parent_sequenced_block_data=None):
harvester = PolkascanHarvesterService(self.session,
type_registry=TYPE_REGISTRY)
harvester.metadata_store = self.metadata_store
for nr in range(0, 10):
if not parent_sequenced_block_data:
# No block ever sequenced, check if chain is at genesis state
block = Block.query(self.session).order_by('id').first()
if block.id == 1:
# Add genesis block
block = harvester.add_block(block.parent_hash)
if block.id != 0:
return {'error': 'Chain not at genesis'}
harvester.process_genesis(block)
block_id = 0
else:
block_id = parent_sequenced_block_data['id'] + 1
block = Block.query(self.session).get(block_id)
if block:
try:
sequenced_block = harvester.sequence_block(
block, parent_block_data, parent_sequenced_block_data)
self.session.commit()
parent_block_data = block.asdict()
parent_sequenced_block_data = sequenced_block.asdict()
if nr == 9 or not sequenced_block:
if sequenced_block:
if nr == 9:
sequence_block_recursive.delay(
parent_block_data, parent_sequenced_block_data)
return {'processedBlockId': block.id, 'amount': nr + 1}
except IntegrityError as e:
return {
'error': 'Sequencer already started',
'exception': str(e)
}
else:
return {'error': 'Block {} not found'.format(block_id)}
def on_get(self, req, resp):
last_known_block = Block.query(self.session).order_by(Block.id.desc()).first()
if not last_known_block:
resp.media = {
'status': 'success',
'data': {
'message': 'Harvester waiting for first run'
}
}
else:
remaining_sets_result = Block.get_missing_block_ids(self.session)
resp.status = falcon.HTTP_200
resp.media = {
'status': 'success',
'data': {
'harvester_head': last_known_block.id,
'block_process_queue': [
{'from': block_set['block_from'], 'to': block_set['block_to']}
for block_set in remaining_sets_result
]
}
}
def on_get(self, req, resp):
sequencer_task = Status.get_status(self.session, 'SEQUENCER_TASK_ID')
integrity_head = Status.get_status(self.session, 'INTEGRITY_HEAD')
sequencer_head = self.session.query(func.max(BlockTotal.id)).one()[0]
best_block = Block.query(self.session).filter_by(
id=self.session.query(func.max(Block.id)).one()[0]).first()
if best_block:
best_block_datetime = best_block.datetime.replace(tzinfo=pytz.UTC).timestamp() * 1000
best_block_nr = best_block.id
else:
best_block_datetime = None
best_block_nr = None
substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
chain_head_block_id = substrate.get_block_number(substrate.get_chain_head())
chain_finalized_block_id = substrate.get_block_number(substrate.get_chain_finalised_head())
resp.media = {
'best_block_datetime': best_block_datetime,
'best_block_nr': best_block_nr,
'sequencer_task': sequencer_task.value,
'sequencer_head': sequencer_head,
'integrity_head': int(integrity_head.value),
'chain_head_block_id': chain_head_block_id,
'chain_finalized_block_id': chain_finalized_block_id
}
def get_item(self, item_id):
if item_id.isnumeric():
return BlockTotal.query(self.session).get(item_id)
else:
block = Block.query(self.session).filter_by(hash=item_id).first()
if block:
return BlockTotal.query(self.session).get(block.id)
def remove_block(self, block_hash):
# Retrieve block
block = Block.query(self.db_session).filter_by(hash=block_hash).first()
# Revert event processors
for event in Event.query(self.db_session).filter_by(block_id=block.id):
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, None)
event_processor.accumulation_revert(self.db_session)
# Revert extrinsic processors
for extrinsic in Extrinsic.query(self.db_session).filter_by(block_id=block.id):
for processor_class in ProcessorRegistry().get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id):
extrinsic_processor = processor_class(block, extrinsic)
extrinsic_processor.accumulation_revert(self.db_session)
# Revert block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block)
block_processor.accumulation_revert(self.db_session)
# Delete events
for item in Event.query(self.db_session).filter_by(block_id=block.id):
self.db_session.delete(item)
# Delete extrinsics
for item in Extrinsic.query(self.db_session).filter_by(block_id=block.id):
self.db_session.delete(item)
# Delete block
self.db_session.delete(block)
def start_init(self):
if self.init:
print('start_init task is running : '.format(self.init))
return {'result': 'waiting init task completed! '}
n = Block.query(self.session).filter_by(bid=1).count()
if n >= 4:
print(' init task is completed! count().n: {} '.format(n))
return {'result': ' init task is completed! '}
self.init = True
print("start_init")
for shard in SHARDS_TABLE:
substrate_url = SHARDS_TABLE[shard]
substrate = SubstrateInterface(substrate_url)
start_block_hash = substrate.get_block_hash(3)
print('== start_init substrate_url {} ==start_block_hash-{}'.format(
substrate_url, start_block_hash))
end_block_hash = None
accumulate_block_recursive.delay(start_block_hash, end_block_hash,
substrate_url)
return {
'result': 'start_init job started',
'SHARDS_TABLE': SHARDS_TABLE,
'init_task_id': self.request.id
}
def sync_block_account_id(self):
db_session = self.session
blocks = Block.query(db_session).filter(
Block.account_index.is_(None)).all()
for block in blocks:
log = Log.query(db_session).filter(Log.block_id == block.id).filter(
Log.type == 'PreRuntime').first()
if log:
data = log.data.get("value").get("data")
if data:
if data[0:2] != "01" and data[0:2] != "00":
continue
res = RawBabePreDigest(ScaleBytes("0x{}".format(data)))
if data[0:2] == "01" and len(data) == 34:
res.decode()
block.account_index = res.value.get("Secondary").get(
"authorityIndex")
elif data[0:2] == "00":
res.decode(check_remaining=False)
block.account_index = res.value.get("Primary").get(
"authorityIndex")
else:
raise "error log data ".format(data)
block.save(db_session)
print("...................", block.id, block.account_index)
else:
print("...................", "Blocks not found")
db_session.commit()
def on_post(self, req, resp):
block_hash = None
if req.media.get('block_id'):
substrate = SubstrateInterface(
url=SUBSTRATE_RPC_URL,
runtime_config=RuntimeConfiguration(),
type_registry_preset=settings.TYPE_REGISTRY)
block_hash = substrate.get_block_hash(req.media.get('block_id'))
elif req.media.get('block_hash'):
block_hash = req.media.get('block_hash')
else:
resp.status = falcon.HTTP_BAD_REQUEST
resp.media = {
'errors': ['Either block_hash or block_id should be supplied']
}
if block_hash:
print('Processing {} ...'.format(block_hash))
harvester = PolkascanHarvesterService(
db_session=self.session,
type_registry=TYPE_REGISTRY,
type_registry_file=TYPE_REGISTRY_FILE + '-mst')
block = Block.query(
self.session).filter(Block.hash == block_hash).first()
if block:
resp.status = falcon.HTTP_200
resp.media = {
'result': 'already exists',
'parentHash': block.parent_hash
}
else:
amount = req.media.get('amount', 1)
for nr in range(0, amount):
try:
block = harvester.add_block(block_hash)
except BlockAlreadyAdded as e:
print('Skipping {}'.format(block_hash))
block_hash = block.parent_hash
if block.id == 0:
break
self.session.commit()
resp.status = falcon.HTTP_201
resp.media = {
'result': 'added',
'parentHash': block.parent_hash
}
else:
resp.status = falcon.HTTP_404
resp.media = {'result': 'Block not found'}
def process_genesis(self, block):
# Set block time of parent block
child_block = Block.query(
self.db_session).filter_by(parent_hash=block.hash).first()
block.set_datetime(child_block.datetime)
# Retrieve genesis accounts
genesis_account_page_count = self.substrate.get_runtime_state(
module="Indices",
storage_function="NextEnumSet",
block_hash=block.hash).get('result', 0)
# Get Accounts on EnumSet
block.count_accounts_new = 0
block.count_accounts = 0
for enum_set_nr in range(0, genesis_account_page_count + 1):
genesis_accounts = self.substrate.get_runtime_state(
module="Indices",
storage_function="EnumSet",
params=[enum_set_nr],
block_hash=block.hash).get('result')
if genesis_accounts:
block.count_accounts_new += len(genesis_accounts)
block.count_accounts += len(genesis_accounts)
for idx, account_id in enumerate(genesis_accounts):
account_audit = AccountAudit(
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=ACCOUNT_AUDIT_TYPE_NEW)
account_audit.save(self.db_session)
account_index_id = enum_set_nr * 64 + idx
account_index_audit = AccountIndexAudit(
account_index_id=account_index_id,
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=ACCOUNT_INDEX_AUDIT_TYPE_NEW)
account_index_audit.save(self.db_session)
block.save(self.db_session)
# Create initial session
initial_session_event = NewSessionEventProcessor(block, Event(), None)
initial_session_event.add_session(db_session=self.db_session,
session_id=0)
def rebuild_search_index(self):
self.db_session.execute('truncate table {}'.format(
SearchIndex.__tablename__))
for block in Block.query(
self.db_session).order_by('id').yield_per(1000):
extrinsic_lookup = {}
block._accounts_new = []
block._accounts_reaped = []
for extrinsic in Extrinsic.query(self.db_session).filter_by(
block_id=block.id).order_by('extrinsic_idx'):
extrinsic_lookup[extrinsic.extrinsic_idx] = extrinsic
# Add search index for signed extrinsics
if extrinsic.address:
search_index = SearchIndex(
index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC,
block_id=block.id,
extrinsic_idx=extrinsic.extrinsic_idx,
account_id=extrinsic.address)
search_index.save(self.db_session)
# Process extrinsic processors
for processor_class in ProcessorRegistry(
).get_extrinsic_processors(extrinsic.module_id,
extrinsic.call_id):
extrinsic_processor = processor_class(
block=block,
extrinsic=extrinsic,
substrate=self.substrate)
extrinsic_processor.process_search_index(self.db_session)
for event in Event.query(self.db_session).filter_by(
block_id=block.id).order_by('event_idx'):
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsic_lookup[event.extrinsic_idx]
except (IndexError, KeyError):
extrinsic = None
for processor_class in ProcessorRegistry(
).get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(
block,
event,
extrinsic,
metadata=self.metadata_store.get(
block.spec_version_id),
substrate=self.substrate)
event_processor.process_search_index(self.db_session)
self.db_session.commit()
def find(self, bid, shard_num, substrate):
print('== find params *shardnum=*{} *==bid=*{}*'.format(shard_num, bid))
block_before = Block.query(self.session).filter_by(shard_num=shard_num,
bid=(bid - 1)).first()
print('== find sql query *==by bid-1 hash=*{}*'.format(block_before.hash))
hash_on = substrate.get_block_hash(bid - 1)
if hash_on == block_before.hash:
return bid - 1
else:
return find(bid - 1, shard_num, substrate)
def count(self):
message_count = self.session.query(func.count(Block.id)).filter(
Block.coinbase == 'yee1gjjlh3ll709jvdvwvpc0helpw8uh3fdldh3ae3a6xkdm0qu3d4zqg2d5d3').scalar()
print('message_count---', message_count)
listbl = Block.query(self.session).filter(
Block.coinbase == 'yee1gjjlh3ll709jvdvwvpc0helpw8uh3fdldh3ae3a6xkdm0qu3d4zqg2d5d3',
Block.fee_reward > 0).all()
if len(listbl) > 0:
for b in listbl:
print(b.fee_reward)
def get_relationships(self, include_list, item):
relationships = {}
if 'blocks' in include_list:
relationships['blocks'] = Block.query(self.session).filter_by(
session_id=item.id).order_by(Block.id.desc())
if 'validators' in include_list:
relationships['validators'] = SessionValidator.query(
self.session).filter_by(session_id=item.id).order_by(
SessionValidator.rank_validator)
return relationships
def on_post(self, req, resp):
harvester = PolkascanHarvesterService(self.session,
type_registry=TYPE_REGISTRY)
block = Block.query(self.session).get(1)
if block:
result = harvester.process_genesis(block=block)
else:
result = 'Block #1 required to process genesis'
self.session.commit()
resp.media = {'result': result}
def serialize_item(self, item):
data = item.serialize()
runtime_call = RuntimeCall.query(self.session).filter_by(
module_id=item.module_id,
call_id=item.call_id,
spec_version=item.spec_version_id).first()
data['attributes']['documentation'] = runtime_call.documentation
block = Block.query(self.session).get(item.block_id)
if block.datetime:
data['attributes']['datetime'] = block.datetime.replace(
tzinfo=pytz.UTC).isoformat()
else:
data['attributes']['datetime'] = None
if item.account:
data['attributes']['account'] = item.account.serialize()
if item.params:
item.params = self.check_params(item.params, item.serialize_id())
if item.error:
# Retrieve ExtrinsicFailed event
extrinsic_failed_event = Event.query(self.session).filter_by(
block_id=item.block_id, event_id='ExtrinsicFailed').first()
# Retrieve runtime error
if extrinsic_failed_event:
if 'Module' in extrinsic_failed_event.attributes[0]['value']:
error = RuntimeErrorMessage.query(self.session).filter_by(
module_index=extrinsic_failed_event.attributes[0]
['value']['Module']['index'],
index=extrinsic_failed_event.attributes[0]['value']
['Module']['error'],
spec_version=item.spec_version_id).first()
if error:
data['attributes'][
'error_message'] = error.documentation
elif 'BadOrigin' in extrinsic_failed_event.attributes[0][
'value']:
data['attributes']['error_message'] = 'Bad origin'
elif 'CannotLookup' in extrinsic_failed_event.attributes[0][
'value']:
data['attributes']['error_message'] = 'Cannot lookup'
return data
def on_get(self, req, resp, item_id=None):
resp.status = falcon.HTTP_200
# TODO make caching more generic for custom resources
cache_key = '{}-{}'.format(req.method, req.url)
console_handler = logging.StreamHandler()
console_handler.setLevel('INFO')
logger = logging.getLogger('yee')
logger.setLevel('INFO')
logger.addHandler(console_handler)
# logger.info(cache_key)
response = self.cache_region.get(cache_key, self.cache_expiration_time)
if response is NO_VALUE:
logger.info(time.strftime("%a %b %d %H:%M:%S %Y",
time.localtime()))
count = self.session.query(func.count(
Block.id)).filter(Block.coinbase == item_id).scalar()
listbl = Block.query(self.session).filter(
Block.coinbase == item_id, Block.fee_reward > 0).all()
#select sum(fee_reward) from data_block where coinbase='yee1w3hn8vhurrjf900zkzl674alsfgxf3vnj8h03run4f3nx5durqrsdsu9r8' and fee_reward>0 ;
logger.info(time.strftime("%a %b %d %H:%M:%S %Y",
time.localtime()))
sum = 0
if len(listbl) > 0:
for b in listbl:
sum = sum + b.fee_reward
sum = sum / 100000000
logger.info(count)
logger.info(sum)
logger.info(time.strftime("%a %b %d %H:%M:%S %Y",
time.localtime()))
response = self.get_jsonapi_response(data={
'type': 'AddressFeeSum',
'attributes': {
'block_reward_sum': str(count * 64),
'fee_reward_sum': str(sum)
}
}, )
self.cache_region.set(cache_key, response)
resp.set_header('X-Cache', 'MISS')
else:
resp.set_header('X-Cache', 'HIT')
resp.media = response
def start_sequencer(self):
# Start sequencer
max_sequenced_block_id = self.session.query(func.max(
BlockTotal.id)).one()[0]
if max_sequenced_block_id is not None:
sequencer_parent_block = BlockTotal.query(
self.session).filter_by(id=max_sequenced_block_id).first()
parent_block = Block.query(
self.session).filter_by(id=max_sequenced_block_id).first()
sequence_block_recursive.delay(
parent_block_data=parent_block.asdict(),
parent_sequenced_block_data=sequencer_parent_block.asdict())
else:
sequence_block_recursive.delay(parent_block_data=None)
def get_relationships(self, include_list, item):
relationships = {}
if 'recent_extrinsics' in include_list:
relationships['recent_extrinsics'] = Extrinsic.query(
self.session).filter_by(address=item.id).order_by(
Extrinsic.block_id.desc())[:10]
if 'indices' in include_list:
relationships['indices'] = AccountIndex.query(
self.session).filter_by(account_id=item.id).order_by(
AccountIndex.updated_at_block.desc())
if 'rewards' in include_list:
relationships['rewards'] = Block.query(self.session).filter_by(
coinbase=bech32.encode(HRP,
bytes().fromhex(item.id))).order_by(
Block.id.desc())
# Block.query(self.session).order_by(Block.id.desc()
# count = self.session.query(func.count(Block.id)).filter(Block.coinbase == item_id).scalar()
return relationships
def serialize_item(self, item):
data = item.serialize()
runtime_call = RuntimeCall.query(self.session).filter_by(
module_id=item.module_id,
call_id=item.call_id,
spec_version=item.spec_version_id).first()
data['attributes']['documentation'] = runtime_call.documentation
block = Block.query(self.session).get(item.block_id)
data['attributes']['datetime'] = block.datetime.replace(
tzinfo=pytz.UTC).isoformat()
if item.account:
data['attributes']['account'] = item.account.serialize()
if item.error:
# Retrieve ExtrinsicFailed event
extrinsic_failed_event = Event.query(self.session).filter_by(
block_id=item.block_id, event_id='ExtrinsicFailed').first()
# Retrieve runtime error
if extrinsic_failed_event and 'Module' in extrinsic_failed_event.attributes[
0]['value']:
error = RuntimeErrorMessage.query(self.session).filter_by(
module_id=item.module_id,
index=extrinsic_failed_event.attributes[0]['value']
['Module']['error'],
spec_version=item.spec_version_id).first()
if error:
data['attributes']['error_message'] = error.documentation
return data
def on_get(self, req, resp):
page = int(req.params.get('page') if req.params.get('page') else 1)
pageSize = int(
req.params.get('page_size') if req.params.get('page_size') else 20)
# blocks = Block.query(self.session).order_by(Block.id.desc()).limit(pageSize).offset((page - 1) * pageSize).all()
blocks = Block.latest_blocks(self.session, page, pageSize)
resp.status = falcon.HTTP_200
result = [
{
"block_num":
blockData.id,
"event_count":
blockData.count_events,
"extrinsics_count":
blockData.count_extrinsics,
"block_timestamp":
blockData.datetime.strftime("%Y-%m-%d %H:%M:%S"),
"block_hash":
blockData.hash,
"author":
ss58_encode(blockData.author.replace('0x', ''))
if blockData.author is not None else None,
# "block_timestamp": time.mktime(blockData.datetime.timetuple()),
"finalized":
"1" if blockData.author is not None else None,
} for blockData in blocks
]
count = Block.query(self.session).count()
resp.media = {
'status': 'success',
'data': {
'result': result,
'count': count
}
}
def start_harvester(self, check_gaps=False, shard=None):
shard = self.request.args[0]
if shard is None:
raise HarvesterNotshardParamsError(
'params shard is missing.. stopping harvester ')
print("start_harvester")
substrate_url = SHARDS_TABLE[shard]
print('== start_harvester substrate_url {} =='.format(substrate_url))
substrate = SubstrateInterface(substrate_url)
n = Block.query(self.session).filter_by(bid=1).count()
if n < 4:
print('waiting init task completed! count().n: {} '.format(n))
return {'result': 'waiting init task completed! '}
block_sets = []
harvester = PolkascanHarvesterService(self.session,
type_registry=TYPE_REGISTRY)
harvester.metadata_store = self.metadata_store
start_block_hash = substrate.get_chain_head()
end_block_hash = None
r = 10
block_nr = substrate.get_block_number(start_block_hash)
max_block = Block.query(self.session).filter_by(
shard_num=shard.split(".")[1]).order_by(Block.bid.desc()).first()
print('start block_nr {} =='.format(block_nr))
print('start max_block {} =='.format(max_block.bid))
if block_nr - max_block.bid < 10:
r = block_nr - max_block.bid
print('current range r: {} =='.format(max_block.bid))
try:
for nr in range(1, r + 1):
block_hash = substrate.get_block_hash(max_block.bid + nr)
if harvester.add_block(block_hash, substrate_url):
print('start_harvester+ Added {} '.format(block_hash))
self.session.commit()
# Update persistent metadata store in Celery task
self.metadata_store = harvester.metadata_store
except BlockAlreadyAdded as e:
print('. Skipped {} '.format(block_hash))
except IntegrityError as e:
print('. Skipped duplicate {}=={} '.format(block_hash, e))
except Exception as exc:
print('! ERROR adding {}'.format(block_hash))
raise HarvesterCouldNotAddBlock(block_hash) from exc
block_sets.append({
'start_block_hash': start_block_hash,
'end_block_hash': end_block_hash
})
return {
'result':
'Yee data Synchronization job SUCCESS',
'block_sets':
block_sets,
'result':
'Synch data from {} to {} blocks check by shardnum of {}'.format(
max_block.bid + 1, r + max_block.bid + 1, shard)
}
def start_sequencer(self):
integrity_status = self.integrity_checks()
self.db_session.commit()
block_nr = None
integrity_head = Status.get_status(self.db_session, 'INTEGRITY_HEAD')
if not integrity_head.value:
integrity_head.value = 0
# 3. Check sequence head
sequencer_head = self.db_session.query(func.max(BlockTotal.id)).one()[0]
if sequencer_head is None:
sequencer_head = -1
# Start sequencing process
sequencer_parent_block = BlockTotal.query(self.db_session).filter_by(id=sequencer_head).first()
parent_block = Block.query(self.db_session).filter_by(id=sequencer_head).first()
for block_nr in range(sequencer_head + 1, int(integrity_head.value) + 1):
if block_nr == 0:
# No block ever sequenced, check if chain is at genesis state
assert (not sequencer_parent_block)
block = Block.query(self.db_session).order_by('id').first()
if not block:
self.db_session.commit()
return {'error': 'Chain not at genesis'}
if block.id == 1:
# Add genesis block
block = self.add_block(block.parent_hash)
if block.id != 0:
self.db_session.commit()
return {'error': 'Chain not at genesis'}
self.process_genesis(block)
sequencer_parent_block_data = None
parent_block_data = None
else:
block_id = sequencer_parent_block.id + 1
assert (block_id == block_nr)
block = Block.query(self.db_session).get(block_nr)
if not block:
self.db_session.commit()
return {'result': 'Finished at #{}'.format(sequencer_parent_block.id)}
sequencer_parent_block_data = sequencer_parent_block.asdict()
parent_block_data = parent_block.asdict()
sequenced_block = self.sequence_block(block, parent_block_data, sequencer_parent_block_data)
self.db_session.commit()
parent_block = block
sequencer_parent_block = sequenced_block
if block_nr:
return {'result': 'Finished at #{}'.format(block_nr)}
else:
return {'result': 'Nothing to sequence'}
def integrity_checks(self):
# 1. Check finalized head
substrate = SubstrateInterface(settings.SUBSTRATE_RPC_URL)
if settings.FINALIZATION_BY_BLOCK_CONFIRMATIONS > 0:
finalized_block_hash = substrate.get_chain_head()
finalized_block_number = max(
substrate.get_block_number(finalized_block_hash) - settings.FINALIZATION_BY_BLOCK_CONFIRMATIONS, 0
)
else:
finalized_block_hash = substrate.get_chain_finalised_head()
finalized_block_number = substrate.get_block_number(finalized_block_hash)
# 2. Check integrity head
integrity_head = Status.get_status(self.db_session, 'INTEGRITY_HEAD')
if not integrity_head.value:
# Only continue if block #1 exists
if Block.query(self.db_session).filter_by(id=1).count() == 0:
raise BlockIntegrityError('Chain not at genesis')
integrity_head.value = 0
else:
integrity_head.value = int(integrity_head.value)
start_block_id = max(integrity_head.value - 1, 0)
end_block_id = finalized_block_number
chunk_size = 1000
parent_block = None
if start_block_id < end_block_id:
# Continue integrity check
# print('== Start integrity checks from {} to {} =='.format(start_block_id, end_block_id))
for block_nr in range(start_block_id, end_block_id, chunk_size):
# TODO replace limit with filter_by block range
block_range = Block.query(self.db_session).order_by('id')[block_nr:block_nr + chunk_size]
for block in block_range:
if parent_block:
if block.id != parent_block.id + 1:
# Save integrity head if block hash of parent matches with hash in node
if parent_block.hash == substrate.get_block_hash(integrity_head.value):
integrity_head.save(self.db_session)
self.db_session.commit()
raise BlockIntegrityError('Block #{} is missing.. stopping check '.format(parent_block.id + 1))
elif block.parent_hash != parent_block.hash:
self.process_reorg_block(parent_block)
self.process_reorg_block(block)
self.remove_block(block.hash)
self.remove_block(parent_block.hash)
self.db_session.commit()
self.add_block(substrate.get_block_hash(block.id))
self.add_block(substrate.get_block_hash(parent_block.id))
self.db_session.commit()
integrity_head.value = parent_block.id - 1
# Save integrity head if block hash of parent matches with hash in node
#if parent_block.parent_hash == substrate.get_block_hash(integrity_head.value):
integrity_head.save(self.db_session)
self.db_session.commit()
raise BlockIntegrityError('Block #{} failed integrity checks, Re-adding #{}.. '.format(parent_block.id, block.id))
else:
integrity_head.value = block.id
parent_block = block
if block.id == end_block_id:
break
if parent_block:
if parent_block.hash == substrate.get_block_hash(int(integrity_head.value)):
integrity_head.save(self.db_session)
self.db_session.commit()
return {'integrity_head': integrity_head.value}
def process_genesis(self, block):
# Set block time of parent block
child_block = Block.query(self.db_session).filter_by(parent_hash=block.hash).first()
block.set_datetime(child_block.datetime)
# Retrieve genesis accounts
if settings.get_versioned_setting('SUBSTRATE_STORAGE_INDICES', block.spec_version_id) == 'Accounts':
# Get accounts from storage keys
storage_key_prefix = self.substrate.generate_storage_hash(
storage_module='System',
storage_function='Account',
metadata_version=settings.SUBSTRATE_METADATA_VERSION
)
rpc_result = self.substrate.rpc_request(
'state_getKeys',
[storage_key_prefix, block.hash]
).get('result')
# Extract accounts from storage key
genesis_accounts = [storage_key[-64:] for storage_key in rpc_result if len(storage_key) == 162]
for account_id in genesis_accounts:
account_audit = AccountAudit(
account_id=account_id,
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
elif settings.get_versioned_setting('SUBSTRATE_STORAGE_INDICES', block.spec_version_id) == 'EnumSet':
genesis_account_page_count = self.substrate.get_runtime_state(
module="Indices",
storage_function="NextEnumSet",
block_hash=block.hash
).get('result', 0)
# Get Accounts on EnumSet
block.count_accounts_new = 0
block.count_accounts = 0
for enum_set_nr in range(0, genesis_account_page_count + 1):
genesis_accounts = self.substrate.get_runtime_state(
module="Indices",
storage_function="EnumSet",
params=[enum_set_nr],
block_hash=block.hash
).get('result')
if genesis_accounts:
block.count_accounts_new += len(genesis_accounts)
block.count_accounts += len(genesis_accounts)
for idx, account_id in enumerate(genesis_accounts):
account_audit = AccountAudit(
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
account_index_id = enum_set_nr * 64 + idx
account_index_audit = AccountIndexAudit(
account_index_id=account_index_id,
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=settings.ACCOUNT_INDEX_AUDIT_TYPE_NEW
)
account_index_audit.save(self.db_session)
block.save(self.db_session)
# Add hardcoded account like treasury stored in settings
for account_id in settings.SUBSTRATE_TREASURY_ACCOUNTS:
account_audit = AccountAudit(
account_id=account_id,
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
data={'is_treasury': True},
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
# Check for sudo accounts
try:
# Update sudo key
sudo_key = self.substrate.get_runtime_state(
module='Sudo',
storage_function='Key',
block_hash=block.hash
).get('result')
account_audit = AccountAudit(
account_id=sudo_key.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
data={'is_sudo': True},
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
except ValueError:
pass
# Create initial session
initial_session_event = NewSessionEventProcessor(
block=block, event=Event(), substrate=self.substrate
)
if settings.get_versioned_setting('NEW_SESSION_EVENT_HANDLER', block.spec_version_id):
initial_session_event.add_session(db_session=self.db_session, session_id=0)
else:
initial_session_event.add_session_old(db_session=self.db_session, session_id=0)
def add_block(self, block_hash):
# Check if block is already process
if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0:
raise BlockAlreadyAdded(block_hash)
if settings.SUBSTRATE_MOCK_EXTRINSICS:
self.substrate.mock_extrinsics = settings.SUBSTRATE_MOCK_EXTRINSICS
json_block = self.substrate.get_chain_block(block_hash)
parent_hash = json_block['block']['header'].pop('parentHash')
block_id = json_block['block']['header'].pop('number')
extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot')
state_root = json_block['block']['header'].pop('stateRoot')
digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None)
# Convert block number to numeric
if not block_id.isnumeric():
block_id = int(block_id, 16)
# ==== Get block runtime from Substrate ==================
self.substrate.init_runtime(block_hash=block_hash)
self.process_metadata(self.substrate.runtime_version, block_hash)
# ==== Get parent block runtime ===================
if block_id > 0:
json_parent_runtime_version = self.substrate.get_block_runtime_version(parent_hash)
parent_spec_version = json_parent_runtime_version.get('specVersion', 0)
self.process_metadata(parent_spec_version, parent_hash)
else:
parent_spec_version = self.substrate.runtime_version
# ==== Set initial block properties =====================
block = Block(
id=block_id,
parent_id=block_id - 1,
hash=block_hash,
parent_hash=parent_hash,
state_root=state_root,
extrinsics_root=extrinsics_root,
count_extrinsics=0,
count_events=0,
count_accounts_new=0,
count_accounts_reaped=0,
count_accounts=0,
count_events_extrinsic=0,
count_events_finalization=0,
count_events_module=0,
count_events_system=0,
count_extrinsics_error=0,
count_extrinsics_signed=0,
count_extrinsics_signedby_address=0,
count_extrinsics_signedby_index=0,
count_extrinsics_success=0,
count_extrinsics_unsigned=0,
count_sessions_new=0,
count_contracts_new=0,
count_log=0,
range10000=math.floor(block_id / 10000),
range100000=math.floor(block_id / 100000),
range1000000=math.floor(block_id / 1000000),
spec_version_id=self.substrate.runtime_version,
logs=digest_logs
)
# Set temp helper variables
block._accounts_new = []
block._accounts_reaped = []
# ==== Get block events from Substrate ==================
extrinsic_success_idx = {}
events = []
try:
events_decoder = self.substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version])
event_idx = 0
for event in events_decoder.elements:
event.value['module_id'] = event.value['module_id'].lower()
model = Event(
block_id=block_id,
event_idx=event_idx,
phase=event.value['phase'],
extrinsic_idx=event.value['extrinsic_idx'],
type=event.value['type'],
spec_version_id=parent_spec_version,
module_id=event.value['module_id'],
event_id=event.value['event_id'],
system=int(event.value['module_id'] == 'system'),
module=int(event.value['module_id'] != 'system'),
attributes=event.value['params'],
codec_error=False
)
# Process event
if event.value['phase'] == 0:
block.count_events_extrinsic += 1
elif event.value['phase'] == 1:
block.count_events_finalization += 1
if event.value['module_id'] == 'system':
block.count_events_system += 1
# Store result of extrinsic
if event.value['event_id'] == 'ExtrinsicSuccess':
extrinsic_success_idx[event.value['extrinsic_idx']] = True
block.count_extrinsics_success += 1
if event.value['event_id'] == 'ExtrinsicFailed':
extrinsic_success_idx[event.value['extrinsic_idx']] = False
block.count_extrinsics_error += 1
else:
block.count_events_module += 1
model.save(self.db_session)
events.append(model)
event_idx += 1
block.count_events = len(events_decoder.elements)
except SubstrateRequestException:
block.count_events = 0
# === Extract extrinsics from block ====
extrinsics_data = json_block['block'].pop('extrinsics')
block.count_extrinsics = len(extrinsics_data)
extrinsic_idx = 0
extrinsics = []
for extrinsic in extrinsics_data:
extrinsics_decoder = ExtrinsicsDecoder(
data=ScaleBytes(extrinsic),
metadata=self.metadata_store[parent_spec_version]
)
extrinsic_data = extrinsics_decoder.decode()
# Lookup result of extrinsic
extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False)
model = Extrinsic(
block_id=block_id,
extrinsic_idx=extrinsic_idx,
extrinsic_hash=extrinsics_decoder.extrinsic_hash,
extrinsic_length=extrinsic_data.get('extrinsic_length'),
extrinsic_version=extrinsic_data.get('version_info'),
signed=extrinsics_decoder.contains_transaction,
unsigned=not extrinsics_decoder.contains_transaction,
signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')),
signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')),
address_length=extrinsic_data.get('account_length'),
address=extrinsic_data.get('account_id'),
account_index=extrinsic_data.get('account_index'),
account_idx=extrinsic_data.get('account_idx'),
signature=extrinsic_data.get('signature'),
nonce=extrinsic_data.get('nonce'),
era=extrinsic_data.get('era'),
call=extrinsic_data.get('call_code'),
module_id=extrinsic_data.get('call_module'),
call_id=extrinsic_data.get('call_function'),
params=extrinsic_data.get('params'),
spec_version_id=parent_spec_version,
success=int(extrinsic_success),
error=int(not extrinsic_success),
codec_error=False
)
model.save(self.db_session)
extrinsics.append(model)
extrinsic_idx += 1
# Process extrinsic
if extrinsics_decoder.contains_transaction:
block.count_extrinsics_signed += 1
if model.signedby_address:
block.count_extrinsics_signedby_address += 1
if model.signedby_index:
block.count_extrinsics_signedby_index += 1
# Add search index for signed extrinsics
search_index = SearchIndex(
index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC,
block_id=block.id,
extrinsic_idx=model.extrinsic_idx,
account_id=model.address
)
search_index.save(self.db_session)
else:
block.count_extrinsics_unsigned += 1
# Process extrinsic processors
for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id):
extrinsic_processor = processor_class(block, model, substrate=self.substrate)
extrinsic_processor.accumulation_hook(self.db_session)
extrinsic_processor.process_search_index(self.db_session)
# Process event processors
for event in events:
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsics[event.extrinsic_idx]
except IndexError:
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, extrinsic,
metadata=self.metadata_store.get(block.spec_version_id),
substrate=self.substrate)
event_processor.accumulation_hook(self.db_session)
event_processor.process_search_index(self.db_session)
# Process block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block, substrate=self.substrate, harvester=self)
block_processor.accumulation_hook(self.db_session)
# Debug info
if settings.DEBUG:
block.debug_info = json_block
# ==== Save data block ==================================
block.save(self.db_session)
return block
def get_query(self):
return Block.query(self.session).order_by(Block.id.desc())
def get_item(self, item_id):
if item_id.isnumeric():
return Block.query(self.session).filter_by(id=item_id).first()
else:
return Block.query(self.session).filter_by(hash=item_id).first()
