def process_metadata(self, spec_version, block_hash): # Check if metadata already in store if spec_version not in self.metadata_store: print('Metadata: CACHE MISS', spec_version) runtime_version_data = self.substrate.get_block_runtime_version(block_hash) runtime = Runtime.query(self.db_session).get(spec_version) if runtime: self.metadata_store[spec_version] = self.substrate.get_block_metadata(block_hash=block_hash) else: self.db_session.begin(subtransactions=True) try: # Store metadata in database runtime = Runtime( id=spec_version, impl_name=runtime_version_data["implName"], impl_version=runtime_version_data["implVersion"], spec_name=runtime_version_data["specName"], spec_version=spec_version, json_metadata=str(self.substrate.metadata_decoder.data), json_metadata_decoded=self.substrate.metadata_decoder.value, apis=runtime_version_data["apis"], authoring_version=runtime_version_data["authoringVersion"], count_call_functions=0, count_events=0, count_modules=len(self.substrate.metadata_decoder.metadata.modules), count_storage_functions=0, count_constants=0, count_errors=0 ) runtime.save(self.db_session) print('store version to db', self.substrate.metadata_decoder.version) for module in self.substrate.metadata_decoder.metadata.modules: # Check if module exists if RuntimeModule.query(self.db_session).filter_by( spec_version=spec_version, module_id=module.get_identifier() ).count() == 0: module_id = module.get_identifier() else: module_id = '{}_1'.format(module.get_identifier()) # Storage backwards compt check if module.storage and isinstance(module.storage, list): storage_functions = module.storage elif module.storage and isinstance(getattr(module.storage, 'value'), dict): storage_functions = module.storage.items else: storage_functions = [] runtime_module = RuntimeModule( spec_version=spec_version, module_id=module_id, prefix=module.prefix, name=module.name, count_call_functions=len(module.calls or []), count_storage_functions=len(storage_functions), count_events=len(module.events or []), count_constants=len(module.constants or []), count_errors=len(module.errors or []), ) runtime_module.save(self.db_session) # Update totals in runtime runtime.count_call_functions += runtime_module.count_call_functions runtime.count_events += runtime_module.count_events runtime.count_storage_functions += runtime_module.count_storage_functions runtime.count_constants += runtime_module.count_constants runtime.count_errors += runtime_module.count_errors if len(module.calls or []) > 0: for idx, call in enumerate(module.calls): runtime_call = RuntimeCall( spec_version=spec_version, module_id=module_id, call_id=call.get_identifier(), index=idx, name=call.name, lookup=call.lookup, documentation='\n'.join(call.docs), count_params=len(call.args) ) runtime_call.save(self.db_session) for arg in call.args: runtime_call_param = RuntimeCallParam( runtime_call_id=runtime_call.id, name=arg.name, type=arg.type ) runtime_call_param.save(self.db_session) if len(module.events or []) > 0: for event_index, event in enumerate(module.events): runtime_event = RuntimeEvent( spec_version=spec_version, module_id=module_id, event_id=event.name, index=event_index, name=event.name, lookup=event.lookup, documentation='\n'.join(event.docs), count_attributes=len(event.args) ) runtime_event.save(self.db_session) for arg_index, arg in enumerate(event.args): runtime_event_attr = RuntimeEventAttribute( runtime_event_id=runtime_event.id, index=arg_index, type=arg ) runtime_event_attr.save(self.db_session) if len(storage_functions) > 0: for idx, storage in enumerate(storage_functions): # Determine type type_hasher = None type_key1 = None type_key2 = None type_value = None type_is_linked = None type_key2hasher = None if storage.type.get('PlainType'): type_value = storage.type.get('PlainType') elif storage.type.get('MapType'): type_hasher = storage.type['MapType'].get('hasher') type_key1 = storage.type['MapType'].get('key') type_value = storage.type['MapType'].get('value') type_is_linked = storage.type['MapType'].get('isLinked', False) elif storage.type.get('DoubleMapType'): type_hasher = storage.type['DoubleMapType'].get('hasher') type_key1 = storage.type['DoubleMapType'].get('key1') type_key2 = storage.type['DoubleMapType'].get('key2') type_value = storage.type['DoubleMapType'].get('value') type_key2hasher = storage.type['DoubleMapType'].get('key2Hasher') runtime_storage = RuntimeStorage( spec_version=spec_version, module_id=module_id, index=idx, name=storage.name, lookup=None, default=storage.fallback, modifier=storage.modifier, type_hasher=type_hasher, storage_key=xxh128(module.prefix.encode()) + xxh128(storage.name.encode()), type_key1=type_key1, type_key2=type_key2, type_value=type_value, type_is_linked=type_is_linked, type_key2hasher=type_key2hasher, documentation='\n'.join(storage.docs) ) runtime_storage.save(self.db_session) if len(module.constants or []) > 0: for idx, constant in enumerate(module.constants): # Decode value try: value_obj = ScaleDecoder.get_decoder_class( constant.type, ScaleBytes(constant.constant_value) ) value_obj.decode() value = value_obj.serialize() except ValueError: value = constant.constant_value except RemainingScaleBytesNotEmptyException: value = constant.constant_value except NotImplementedError: value = constant.constant_value if type(value) is list or type(value) is dict: value = json.dumps(value) runtime_constant = RuntimeConstant( spec_version=spec_version, module_id=module_id, index=idx, name=constant.name, type=constant.type, value=value, documentation='\n'.join(constant.docs) ) runtime_constant.save(self.db_session) if len(module.errors or []) > 0: for idx, error in enumerate(module.errors): runtime_error = RuntimeErrorMessage( spec_version=spec_version, module_id=module_id, index=idx, name=error.name, documentation='\n'.join(error.docs) ) runtime_error.save(self.db_session) runtime.save(self.db_session) # Process types for runtime_type_data in list(self.substrate.get_type_registry(block_hash=block_hash).values()): runtime_type = RuntimeType( spec_version=runtime_type_data["spec_version"], type_string=runtime_type_data["type_string"], decoder_class=runtime_type_data["decoder_class"], is_primitive_core=runtime_type_data["is_primitive_core"], is_primitive_runtime=runtime_type_data["is_primitive_runtime"] ) runtime_type.save(self.db_session) self.db_session.commit() # Put in local store self.metadata_store[spec_version] = self.substrate.metadata_decoder except SQLAlchemyError as e: self.db_session.rollback()
def process_metadata(self, runtime_version_data, block_hash): spec_version = runtime_version_data.get('specVersion', 0) # Check if metadata already in store if spec_version not in self.metadata_store: print('Metadata: CACHE MISS', spec_version) runtime = Runtime.query(self.db_session).get(spec_version) if runtime: metadata_decoder = MetadataDecoder( ScaleBytes(runtime.json_metadata)) metadata_decoder.decode() self.metadata_store[spec_version] = metadata_decoder else: self.db_session.begin(subtransactions=True) try: # ==== Get block Metadata from Substrate ================== substrate = SubstrateInterface(SUBSTRATE_RPC_URL) metadata_decoder = substrate.get_block_metadata(block_hash) # Store metadata in database runtime = Runtime( id=spec_version, impl_name=runtime_version_data["implName"], impl_version=runtime_version_data["implVersion"], spec_name=runtime_version_data["specName"], spec_version=spec_version, json_metadata=str(metadata_decoder.data), json_metadata_decoded=metadata_decoder.value, apis=runtime_version_data["apis"], authoring_version=runtime_version_data[ "authoringVersion"], count_call_functions=0, count_events=0, count_modules=len(metadata_decoder.metadata.modules), count_storage_functions=0) runtime.save(self.db_session) print('store version to db', metadata_decoder.version) if not metadata_decoder.version: # Legacy V0 fallback for module in metadata_decoder.metadata.modules: runtime_module = RuntimeModule( spec_version=spec_version, module_id=module.get_identifier(), prefix=module.prefix, name=module.get_identifier(), count_call_functions=len(module.functions or []), count_storage_functions=len(module.storage or []), count_events=0) runtime_module.save(self.db_session) if len(module.functions or []) > 0: for idx, call in enumerate(module.functions): runtime_call = RuntimeCall( spec_version=spec_version, module_id=module.get_identifier(), call_id=call.get_identifier(), index=idx, name=call.name, lookup=call.lookup, documentation='\n'.join(call.docs), count_params=len(call.args)) runtime_call.save(self.db_session) for arg in call.args: runtime_call_param = RuntimeCallParam( runtime_call_id=runtime_call.id, name=arg.name, type=arg.type) runtime_call_param.save( self.db_session) # Check if type already registered in database self.process_metadata_type( arg.type, spec_version) for event_module in metadata_decoder.metadata.events_modules: for event_index, event in enumerate( event_module.events): runtime_event = RuntimeEvent( spec_version=spec_version, module_id=event_module.name, event_id=event.name, index=event_index, name=event.name, lookup=event.lookup, documentation='\n'.join(event.docs), count_attributes=len(event.args)) runtime_event.save(self.db_session) runtime_module.count_events += 1 for arg_index, arg in enumerate(event.args): runtime_event_attr = RuntimeEventAttribute( runtime_event_id=runtime_event.id, index=arg_index, type=arg) runtime_event_attr.save(self.db_session) runtime_module.save(self.db_session) else: for module in metadata_decoder.metadata.modules: # Check if module exists if RuntimeModule.query(self.db_session).filter_by( spec_version=spec_version, module_id=module.get_identifier()).count( ) == 0: module_id = module.get_identifier() else: module_id = '{}_1'.format( module.get_identifier()) # Storage backwards compt check if module.storage and isinstance( module.storage, list): storage_functions = module.storage elif module.storage and isinstance( getattr(module.storage, 'value'), dict): storage_functions = module.storage.items else: storage_functions = [] runtime_module = RuntimeModule( spec_version=spec_version, module_id=module_id, prefix=module.prefix, name=module.name, count_call_functions=len(module.calls or []), count_storage_functions=len(storage_functions), count_events=len(module.events or [])) runtime_module.save(self.db_session) # Update totals in runtime runtime.count_call_functions += runtime_module.count_call_functions runtime.count_events += runtime_module.count_events runtime.count_storage_functions += runtime_module.count_storage_functions if len(module.calls or []) > 0: for idx, call in enumerate(module.calls): runtime_call = RuntimeCall( spec_version=spec_version, module_id=module_id, call_id=call.get_identifier(), index=idx, name=call.name, lookup=call.lookup, documentation='\n'.join(call.docs), count_params=len(call.args)) runtime_call.save(self.db_session) for arg in call.args: runtime_call_param = RuntimeCallParam( runtime_call_id=runtime_call.id, name=arg.name, type=arg.type) runtime_call_param.save( self.db_session) # Check if type already registered in database self.process_metadata_type( arg.type, spec_version) if len(module.events or []) > 0: for event_index, event in enumerate( module.events): runtime_event = RuntimeEvent( spec_version=spec_version, module_id=module_id, event_id=event.name, index=event_index, name=event.name, lookup=event.lookup, documentation='\n'.join(event.docs), count_attributes=len(event.args)) runtime_event.save(self.db_session) for arg_index, arg in enumerate( event.args): runtime_event_attr = RuntimeEventAttribute( runtime_event_id=runtime_event.id, index=arg_index, type=arg) runtime_event_attr.save( self.db_session) if len(storage_functions) > 0: for idx, storage in enumerate( storage_functions): # Determine type type_hasher = None type_key1 = None type_key2 = None type_value = None type_is_linked = None type_key2hasher = None if storage.type.get('PlainType'): type_value = storage.type.get( 'PlainType') elif storage.type.get('MapType'): type_hasher = storage.type[ 'MapType'].get('hasher') type_key1 = storage.type[ 'MapType'].get('key') type_value = storage.type[ 'MapType'].get('value') type_is_linked = storage.type[ 'MapType'].get('isLinked', False) elif storage.type.get('DoubleMapType'): type_hasher = storage.type[ 'DoubleMapType'].get('hasher') type_key1 = storage.type[ 'DoubleMapType'].get('key1') type_key2 = storage.type[ 'DoubleMapType'].get('key2') type_value = storage.type[ 'DoubleMapType'].get('value') type_key2hasher = storage.type[ 'DoubleMapType'].get('key2Hasher') runtime_storage = RuntimeStorage( spec_version=spec_version, module_id=module_id, index=idx, name=storage.name, lookup=None, default=storage.fallback, modifier=storage.modifier, type_hasher=type_hasher, type_key1=type_key1, type_key2=type_key2, type_value=type_value, type_is_linked=type_is_linked, type_key2hasher=type_key2hasher, documentation='\n'.join(storage.docs)) runtime_storage.save(self.db_session) # Check if types already registered in database self.process_metadata_type( type_value, spec_version) if type_key1: self.process_metadata_type( type_key1, spec_version) if type_key2: self.process_metadata_type( type_key2, spec_version) if len(module.constants or []) > 0: for idx, constant in enumerate( module.constants): # Decode value try: value_obj = ScaleDecoder.get_decoder_class( constant.type, ScaleBytes( constant.constant_value)) value_obj.decode() value = value_obj.serialize() except ValueError: value = constant.constant_value except RemainingScaleBytesNotEmptyException: value = constant.constant_value except NotImplementedError: value = constant.constant_value runtime_constant = RuntimeConstant( spec_version=spec_version, module_id=module_id, index=idx, name=constant.name, type=constant.type, value=value, documentation='\n'.join(constant.docs)) runtime_constant.save(self.db_session) # Check if types already registered in database self.process_metadata_type( constant.type, spec_version) runtime.save(self.db_session) self.db_session.commit() # Put in local store self.metadata_store[spec_version] = metadata_decoder except SQLAlchemyError as e: self.db_session.rollback()