Exemplo n.º 1
0
    def process_metadata(self, spec_version, block_hash):

        # Check if metadata already in store
        if spec_version not in self.metadata_store:
            print('Metadata: CACHE MISS', spec_version)

            runtime_version_data = self.substrate.get_block_runtime_version(block_hash)

            runtime = Runtime.query(self.db_session).get(spec_version)

            if runtime:
                self.metadata_store[spec_version] = self.substrate.get_block_metadata(block_hash=block_hash)

            else:
                self.db_session.begin(subtransactions=True)
                try:

                    # Store metadata in database
                    runtime = Runtime(
                        id=spec_version,
                        impl_name=runtime_version_data["implName"],
                        impl_version=runtime_version_data["implVersion"],
                        spec_name=runtime_version_data["specName"],
                        spec_version=spec_version,
                        json_metadata=str(self.substrate.metadata_decoder.data),
                        json_metadata_decoded=self.substrate.metadata_decoder.value,
                        apis=runtime_version_data["apis"],
                        authoring_version=runtime_version_data["authoringVersion"],
                        count_call_functions=0,
                        count_events=0,
                        count_modules=len(self.substrate.metadata_decoder.metadata.modules),
                        count_storage_functions=0,
                        count_constants=0,
                        count_errors=0
                    )

                    runtime.save(self.db_session)

                    print('store version to db', self.substrate.metadata_decoder.version)

                    for module in self.substrate.metadata_decoder.metadata.modules:

                        # Check if module exists
                        if RuntimeModule.query(self.db_session).filter_by(
                            spec_version=spec_version,
                            module_id=module.get_identifier()
                        ).count() == 0:
                            module_id = module.get_identifier()
                        else:
                            module_id = '{}_1'.format(module.get_identifier())

                        # Storage backwards compt check
                        if module.storage and isinstance(module.storage, list):
                            storage_functions = module.storage
                        elif module.storage and isinstance(getattr(module.storage, 'value'), dict):
                            storage_functions = module.storage.items
                        else:
                            storage_functions = []

                        runtime_module = RuntimeModule(
                            spec_version=spec_version,
                            module_id=module_id,
                            prefix=module.prefix,
                            name=module.name,
                            count_call_functions=len(module.calls or []),
                            count_storage_functions=len(storage_functions),
                            count_events=len(module.events or []),
                            count_constants=len(module.constants or []),
                            count_errors=len(module.errors or []),
                        )
                        runtime_module.save(self.db_session)

                        # Update totals in runtime
                        runtime.count_call_functions += runtime_module.count_call_functions
                        runtime.count_events += runtime_module.count_events
                        runtime.count_storage_functions += runtime_module.count_storage_functions
                        runtime.count_constants += runtime_module.count_constants
                        runtime.count_errors += runtime_module.count_errors

                        if len(module.calls or []) > 0:
                            for idx, call in enumerate(module.calls):
                                runtime_call = RuntimeCall(
                                    spec_version=spec_version,
                                    module_id=module_id,
                                    call_id=call.get_identifier(),
                                    index=idx,
                                    name=call.name,
                                    lookup=call.lookup,
                                    documentation='\n'.join(call.docs),
                                    count_params=len(call.args)
                                )
                                runtime_call.save(self.db_session)

                                for arg in call.args:
                                    runtime_call_param = RuntimeCallParam(
                                        runtime_call_id=runtime_call.id,
                                        name=arg.name,
                                        type=arg.type
                                    )
                                    runtime_call_param.save(self.db_session)

                        if len(module.events or []) > 0:
                            for event_index, event in enumerate(module.events):
                                runtime_event = RuntimeEvent(
                                    spec_version=spec_version,
                                    module_id=module_id,
                                    event_id=event.name,
                                    index=event_index,
                                    name=event.name,
                                    lookup=event.lookup,
                                    documentation='\n'.join(event.docs),
                                    count_attributes=len(event.args)
                                )
                                runtime_event.save(self.db_session)

                                for arg_index, arg in enumerate(event.args):
                                    runtime_event_attr = RuntimeEventAttribute(
                                        runtime_event_id=runtime_event.id,
                                        index=arg_index,
                                        type=arg
                                    )
                                    runtime_event_attr.save(self.db_session)

                        if len(storage_functions) > 0:
                            for idx, storage in enumerate(storage_functions):

                                # Determine type
                                type_hasher = None
                                type_key1 = None
                                type_key2 = None
                                type_value = None
                                type_is_linked = None
                                type_key2hasher = None

                                if storage.type.get('PlainType'):
                                    type_value = storage.type.get('PlainType')

                                elif storage.type.get('MapType'):
                                    type_hasher = storage.type['MapType'].get('hasher')
                                    type_key1 = storage.type['MapType'].get('key')
                                    type_value = storage.type['MapType'].get('value')
                                    type_is_linked = storage.type['MapType'].get('isLinked', False)

                                elif storage.type.get('DoubleMapType'):
                                    type_hasher = storage.type['DoubleMapType'].get('hasher')
                                    type_key1 = storage.type['DoubleMapType'].get('key1')
                                    type_key2 = storage.type['DoubleMapType'].get('key2')
                                    type_value = storage.type['DoubleMapType'].get('value')
                                    type_key2hasher = storage.type['DoubleMapType'].get('key2Hasher')

                                runtime_storage = RuntimeStorage(
                                    spec_version=spec_version,
                                    module_id=module_id,
                                    index=idx,
                                    name=storage.name,
                                    lookup=None,
                                    default=storage.fallback,
                                    modifier=storage.modifier,
                                    type_hasher=type_hasher,
                                    storage_key=xxh128(module.prefix.encode()) + xxh128(storage.name.encode()),
                                    type_key1=type_key1,
                                    type_key2=type_key2,
                                    type_value=type_value,
                                    type_is_linked=type_is_linked,
                                    type_key2hasher=type_key2hasher,
                                    documentation='\n'.join(storage.docs)
                                )
                                runtime_storage.save(self.db_session)

                        if len(module.constants or []) > 0:
                            for idx, constant in enumerate(module.constants):

                                # Decode value
                                try:
                                    value_obj = ScaleDecoder.get_decoder_class(
                                        constant.type,
                                        ScaleBytes(constant.constant_value)
                                    )
                                    value_obj.decode()
                                    value = value_obj.serialize()
                                except ValueError:
                                    value = constant.constant_value
                                except RemainingScaleBytesNotEmptyException:
                                    value = constant.constant_value
                                except NotImplementedError:
                                    value = constant.constant_value

                                if type(value) is list or type(value) is dict:
                                    value = json.dumps(value)

                                runtime_constant = RuntimeConstant(
                                    spec_version=spec_version,
                                    module_id=module_id,
                                    index=idx,
                                    name=constant.name,
                                    type=constant.type,
                                    value=value,
                                    documentation='\n'.join(constant.docs)
                                )
                                runtime_constant.save(self.db_session)

                        if len(module.errors or []) > 0:
                            for idx, error in enumerate(module.errors):
                                runtime_error = RuntimeErrorMessage(
                                    spec_version=spec_version,
                                    module_id=module_id,
                                    index=idx,
                                    name=error.name,
                                    documentation='\n'.join(error.docs)
                                )
                                runtime_error.save(self.db_session)

                        runtime.save(self.db_session)

                    # Process types
                    for runtime_type_data in list(self.substrate.get_type_registry(block_hash=block_hash).values()):

                        runtime_type = RuntimeType(
                            spec_version=runtime_type_data["spec_version"],
                            type_string=runtime_type_data["type_string"],
                            decoder_class=runtime_type_data["decoder_class"],
                            is_primitive_core=runtime_type_data["is_primitive_core"],
                            is_primitive_runtime=runtime_type_data["is_primitive_runtime"]
                        )
                        runtime_type.save(self.db_session)

                    self.db_session.commit()

                    # Put in local store
                    self.metadata_store[spec_version] = self.substrate.metadata_decoder
                except SQLAlchemyError as e:
                    self.db_session.rollback()
Exemplo n.º 2
0
    def process_metadata(self, runtime_version_data, block_hash):

        spec_version = runtime_version_data.get('specVersion', 0)

        # Check if metadata already in store
        if spec_version not in self.metadata_store:
            print('Metadata: CACHE MISS', spec_version)

            runtime = Runtime.query(self.db_session).get(spec_version)

            if runtime:

                metadata_decoder = MetadataDecoder(
                    ScaleBytes(runtime.json_metadata))
                metadata_decoder.decode()

                self.metadata_store[spec_version] = metadata_decoder

            else:
                self.db_session.begin(subtransactions=True)
                try:

                    # ==== Get block Metadata from Substrate ==================
                    substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
                    metadata_decoder = substrate.get_block_metadata(block_hash)

                    # Store metadata in database
                    runtime = Runtime(
                        id=spec_version,
                        impl_name=runtime_version_data["implName"],
                        impl_version=runtime_version_data["implVersion"],
                        spec_name=runtime_version_data["specName"],
                        spec_version=spec_version,
                        json_metadata=str(metadata_decoder.data),
                        json_metadata_decoded=metadata_decoder.value,
                        apis=runtime_version_data["apis"],
                        authoring_version=runtime_version_data[
                            "authoringVersion"],
                        count_call_functions=0,
                        count_events=0,
                        count_modules=len(metadata_decoder.metadata.modules),
                        count_storage_functions=0)

                    runtime.save(self.db_session)

                    print('store version to db', metadata_decoder.version)

                    if not metadata_decoder.version:
                        # Legacy V0 fallback
                        for module in metadata_decoder.metadata.modules:
                            runtime_module = RuntimeModule(
                                spec_version=spec_version,
                                module_id=module.get_identifier(),
                                prefix=module.prefix,
                                name=module.get_identifier(),
                                count_call_functions=len(module.functions
                                                         or []),
                                count_storage_functions=len(module.storage
                                                            or []),
                                count_events=0)
                            runtime_module.save(self.db_session)

                            if len(module.functions or []) > 0:
                                for idx, call in enumerate(module.functions):
                                    runtime_call = RuntimeCall(
                                        spec_version=spec_version,
                                        module_id=module.get_identifier(),
                                        call_id=call.get_identifier(),
                                        index=idx,
                                        name=call.name,
                                        lookup=call.lookup,
                                        documentation='\n'.join(call.docs),
                                        count_params=len(call.args))
                                    runtime_call.save(self.db_session)

                                    for arg in call.args:
                                        runtime_call_param = RuntimeCallParam(
                                            runtime_call_id=runtime_call.id,
                                            name=arg.name,
                                            type=arg.type)
                                        runtime_call_param.save(
                                            self.db_session)

                                        # Check if type already registered in database
                                        self.process_metadata_type(
                                            arg.type, spec_version)

                        for event_module in metadata_decoder.metadata.events_modules:
                            for event_index, event in enumerate(
                                    event_module.events):
                                runtime_event = RuntimeEvent(
                                    spec_version=spec_version,
                                    module_id=event_module.name,
                                    event_id=event.name,
                                    index=event_index,
                                    name=event.name,
                                    lookup=event.lookup,
                                    documentation='\n'.join(event.docs),
                                    count_attributes=len(event.args))
                                runtime_event.save(self.db_session)

                                runtime_module.count_events += 1

                                for arg_index, arg in enumerate(event.args):
                                    runtime_event_attr = RuntimeEventAttribute(
                                        runtime_event_id=runtime_event.id,
                                        index=arg_index,
                                        type=arg)
                                    runtime_event_attr.save(self.db_session)

                        runtime_module.save(self.db_session)

                    else:
                        for module in metadata_decoder.metadata.modules:

                            # Check if module exists
                            if RuntimeModule.query(self.db_session).filter_by(
                                    spec_version=spec_version,
                                    module_id=module.get_identifier()).count(
                                    ) == 0:
                                module_id = module.get_identifier()
                            else:
                                module_id = '{}_1'.format(
                                    module.get_identifier())

                            # Storage backwards compt check
                            if module.storage and isinstance(
                                    module.storage, list):
                                storage_functions = module.storage
                            elif module.storage and isinstance(
                                    getattr(module.storage, 'value'), dict):
                                storage_functions = module.storage.items
                            else:
                                storage_functions = []

                            runtime_module = RuntimeModule(
                                spec_version=spec_version,
                                module_id=module_id,
                                prefix=module.prefix,
                                name=module.name,
                                count_call_functions=len(module.calls or []),
                                count_storage_functions=len(storage_functions),
                                count_events=len(module.events or []))
                            runtime_module.save(self.db_session)

                            # Update totals in runtime
                            runtime.count_call_functions += runtime_module.count_call_functions
                            runtime.count_events += runtime_module.count_events
                            runtime.count_storage_functions += runtime_module.count_storage_functions

                            if len(module.calls or []) > 0:
                                for idx, call in enumerate(module.calls):
                                    runtime_call = RuntimeCall(
                                        spec_version=spec_version,
                                        module_id=module_id,
                                        call_id=call.get_identifier(),
                                        index=idx,
                                        name=call.name,
                                        lookup=call.lookup,
                                        documentation='\n'.join(call.docs),
                                        count_params=len(call.args))
                                    runtime_call.save(self.db_session)

                                    for arg in call.args:
                                        runtime_call_param = RuntimeCallParam(
                                            runtime_call_id=runtime_call.id,
                                            name=arg.name,
                                            type=arg.type)
                                        runtime_call_param.save(
                                            self.db_session)

                                        # Check if type already registered in database
                                        self.process_metadata_type(
                                            arg.type, spec_version)

                            if len(module.events or []) > 0:
                                for event_index, event in enumerate(
                                        module.events):
                                    runtime_event = RuntimeEvent(
                                        spec_version=spec_version,
                                        module_id=module_id,
                                        event_id=event.name,
                                        index=event_index,
                                        name=event.name,
                                        lookup=event.lookup,
                                        documentation='\n'.join(event.docs),
                                        count_attributes=len(event.args))
                                    runtime_event.save(self.db_session)

                                    for arg_index, arg in enumerate(
                                            event.args):
                                        runtime_event_attr = RuntimeEventAttribute(
                                            runtime_event_id=runtime_event.id,
                                            index=arg_index,
                                            type=arg)
                                        runtime_event_attr.save(
                                            self.db_session)

                            if len(storage_functions) > 0:
                                for idx, storage in enumerate(
                                        storage_functions):

                                    # Determine type
                                    type_hasher = None
                                    type_key1 = None
                                    type_key2 = None
                                    type_value = None
                                    type_is_linked = None
                                    type_key2hasher = None

                                    if storage.type.get('PlainType'):
                                        type_value = storage.type.get(
                                            'PlainType')

                                    elif storage.type.get('MapType'):
                                        type_hasher = storage.type[
                                            'MapType'].get('hasher')
                                        type_key1 = storage.type[
                                            'MapType'].get('key')
                                        type_value = storage.type[
                                            'MapType'].get('value')
                                        type_is_linked = storage.type[
                                            'MapType'].get('isLinked', False)

                                    elif storage.type.get('DoubleMapType'):
                                        type_hasher = storage.type[
                                            'DoubleMapType'].get('hasher')
                                        type_key1 = storage.type[
                                            'DoubleMapType'].get('key1')
                                        type_key2 = storage.type[
                                            'DoubleMapType'].get('key2')
                                        type_value = storage.type[
                                            'DoubleMapType'].get('value')
                                        type_key2hasher = storage.type[
                                            'DoubleMapType'].get('key2Hasher')

                                    runtime_storage = RuntimeStorage(
                                        spec_version=spec_version,
                                        module_id=module_id,
                                        index=idx,
                                        name=storage.name,
                                        lookup=None,
                                        default=storage.fallback,
                                        modifier=storage.modifier,
                                        type_hasher=type_hasher,
                                        type_key1=type_key1,
                                        type_key2=type_key2,
                                        type_value=type_value,
                                        type_is_linked=type_is_linked,
                                        type_key2hasher=type_key2hasher,
                                        documentation='\n'.join(storage.docs))
                                    runtime_storage.save(self.db_session)

                                    # Check if types already registered in database

                                    self.process_metadata_type(
                                        type_value, spec_version)

                                    if type_key1:
                                        self.process_metadata_type(
                                            type_key1, spec_version)

                                    if type_key2:
                                        self.process_metadata_type(
                                            type_key2, spec_version)

                            if len(module.constants or []) > 0:
                                for idx, constant in enumerate(
                                        module.constants):

                                    # Decode value
                                    try:
                                        value_obj = ScaleDecoder.get_decoder_class(
                                            constant.type,
                                            ScaleBytes(
                                                constant.constant_value))
                                        value_obj.decode()
                                        value = value_obj.serialize()
                                    except ValueError:
                                        value = constant.constant_value
                                    except RemainingScaleBytesNotEmptyException:
                                        value = constant.constant_value
                                    except NotImplementedError:
                                        value = constant.constant_value

                                    runtime_constant = RuntimeConstant(
                                        spec_version=spec_version,
                                        module_id=module_id,
                                        index=idx,
                                        name=constant.name,
                                        type=constant.type,
                                        value=value,
                                        documentation='\n'.join(constant.docs))
                                    runtime_constant.save(self.db_session)

                                    # Check if types already registered in database
                                    self.process_metadata_type(
                                        constant.type, spec_version)

                        runtime.save(self.db_session)

                    self.db_session.commit()

                    # Put in local store
                    self.metadata_store[spec_version] = metadata_decoder
                except SQLAlchemyError as e:
                    self.db_session.rollback()