def test_create_asset_with_different_secret_store(publisher_instance, ddo_sample): acct = publisher_instance.main_account asset = ddo_sample my_secret_store = 'http://myownsecretstore.com' auth_service = ServiceDescriptor.authorization_service_descriptor( {'main': {'service': 'SecretStore', 'publicKey': '0casd', 'threshold': '1'}}, my_secret_store) new_asset = publisher_instance.assets.create(asset.metadata, acct, [auth_service]) assert new_asset.get_service(ServiceTypes.AUTHORIZATION).service_endpoint == my_secret_store assert new_asset.get_service(ServiceTypes.ASSET_ACCESS) assert new_asset.get_service(ServiceTypes.METADATA) publisher_instance.assets.retire(new_asset.did) access_service = ServiceDescriptor.access_service_descriptor( {"main": { "name": "dataAssetAccessServiceAgreement", "creator": '0x1234', "price": '1', "timeout": 3600, "datePublished": '2019-08-30T12:19:54Z' }}, '' ) new_asset = publisher_instance.assets.create(asset.metadata, acct, [access_service]) assert new_asset.get_service(ServiceTypes.AUTHORIZATION) assert new_asset.get_service(ServiceTypes.ASSET_ACCESS) assert new_asset.get_service(ServiceTypes.METADATA) publisher_instance.assets.retire(new_asset.did) new_asset = publisher_instance.assets.create(asset.metadata, acct) assert new_asset.get_service(ServiceTypes.AUTHORIZATION) assert new_asset.get_service(ServiceTypes.ASSET_ACCESS) assert new_asset.get_service(ServiceTypes.METADATA) publisher_instance.assets.retire(new_asset.did)
def get_nft_ddo(account, providers=None, auth_service='PSK-RSA'): ddo = get_sample_nft_ddo() metadata = ddo['service'][0]['attributes'] metadata['main']['files'][0][ 'url'] = "https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos" \ "/CoverSongs/shs_dataset_test.txt?" + str(uuid.uuid4()) metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) escrow_payment_condition = ddo['service'][1]['attributes'][ 'serviceAgreementTemplate']['conditions'][2] _number_nfts = 1 # _amounts = get_param_value_by_name(escrow_payment_condition['parameters'], '_amounts') # _receivers = to_checksum_addresses( # get_param_value_by_name(escrow_payment_condition['parameters'], '_receivers')) _amounts = ['9'] _receivers = to_checksum_addresses([account.address]) transfer_nft_condition = ddo['service'][1]['attributes'][ 'serviceAgreementTemplate']['conditions'][1] # _nftHolder = get_param_value_by_name(transfer_nft_condition['parameters'], '_nftHolder') _nftHolder = to_checksum_address(account.address) _total_price = sum(int(x) for x in _amounts) asset_rewards = {"_amounts": _amounts, "_receivers": _receivers} metadata['main']['price'] = _total_price access_service_attributes = { "main": { "name": "nftAccessAgreement", "creator": account.address, "timeout": 3600, "price": _total_price, "_amounts": _amounts, "_receivers": _receivers, "_numberNfts": str(_number_nfts), "_tokenAddress": "", "datePublished": metadata['main']['dateCreated'] } } sales_service_attributes = copy.deepcopy(access_service_attributes) sales_service_attributes['main']['name'] = 'nftSalesAgreement' sales_service_attributes['main']['_nftHolder'] = _nftHolder nft_sales_service_descriptor = ServiceDescriptor.nft_sales_service_descriptor( sales_service_attributes, 'http://localhost:8030') nft_access_service_descriptor = ServiceDescriptor.nft_access_service_descriptor( access_service_attributes, 'http://localhost:8030') return register_ddo( metadata, account, providers, auth_service, [nft_access_service_descriptor, nft_sales_service_descriptor], royalties=0, cap=10, mint=10)
def get_registered_ddo_nft(nevermined_instance, account): ddo = get_sample_nft_ddo() metadata = ddo['service'][0]['attributes'] metadata['main']['files'][0][ 'url'] = "https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos" \ "/CoverSongs/shs_dataset_test.txt?" + str(uuid.uuid4()) metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) escrow_payment_condition = ddo['service'][1]['attributes'][ 'serviceAgreementTemplate']['conditions'][2] _number_nfts = 1 _amounts = get_param_value_by_name(escrow_payment_condition['parameters'], '_amounts') _receivers = to_checksum_addresses( get_param_value_by_name(escrow_payment_condition['parameters'], '_receivers')) _total_price = 0 for i in _amounts: _total_price += int(i) asset_rewards = {"_amounts": _amounts, "_receivers": _receivers} metadata['main']['price'] = _total_price access_service_attributes = { "main": { "name": "nftAccessAgreement", "creator": account.address, "timeout": 3600, "price": _total_price, "_amounts": _amounts, "_receivers": _receivers, "_numberNfts": str(_number_nfts), "datePublished": metadata['main']['dateCreated'] } } sales_service_attributes = access_service_attributes sales_service_attributes['main']['name'] = 'nftSalesAgreement' nft_sales_service_descriptor = ServiceDescriptor.nft_sales_service_descriptor( sales_service_attributes, 'http://localhost:8030') nft_access_service_descriptor = ServiceDescriptor.nft_access_service_descriptor( access_service_attributes, 'http://localhost:8030') ddo = nevermined_instance.assets.create( metadata, account, [nft_sales_service_descriptor, nft_access_service_descriptor], asset_rewards=asset_rewards, royalties=0, cap=10, mint=10) return ddo
def create_compute(self, metadata, publisher_account, asset_rewards={"_amounts": [], "_receivers": []}, service_descriptors=None, providers=None, authorization_type=ServiceAuthorizationTypes.PSK_RSA, use_secret_store=False): """ Register a compute to the data asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store. :param metadata: dict conforming to the Metadata accepted by Nevermined Protocol. :param publisher_account: Account of the publisher registering this asset :param asset_rewards: Asset rewards distribution including amounts and receivers :param service_descriptors: list of ServiceDescriptor tuples of length 2. The first item must be one of ServiceTypes and the second item is a dict of parameters and values required by the service :param providers: list of addresses of providers of this asset (a provider is an ethereum account that is authorized to provide asset services) :param authorization_type: str indicate the authorization type that is going to be used to encrypt the urls. SecretStore, PSK-RSA and PSK-ECDSA are supported. :param use_secret_store: bool indicate whether to use the secret store directly for encrypting urls (Uses Gateway provider service if set to False) :return: DDO instance """ metadata_copy = copy.deepcopy(metadata) gateway = GatewayProvider.get_gateway() compute_service_attributes = self._build_compute(metadata_copy, publisher_account, asset_rewards) service_descriptor = ServiceDescriptor.compute_service_descriptor( compute_service_attributes, gateway.get_execute_endpoint(self._config)) return self.create(metadata, publisher_account, service_descriptors=[service_descriptor], providers=providers, authorization_type=authorization_type, use_secret_store=use_secret_store)
def get_registered_ddo(account, providers=None, auth_service='PSK-RSA', url=get_file_url()): ddo = get_sample_ddo() metadata = ddo['service'][0]['attributes'] metadata['main']['files'][0]['url'] = url metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) escrow_payment_condition = ddo['service'][1]['attributes'][ 'serviceAgreementTemplate']['conditions'][2] _amounts = get_param_value_by_name(escrow_payment_condition['parameters'], '_amounts') _receivers = to_checksum_addresses( get_param_value_by_name(escrow_payment_condition['parameters'], '_receivers')) access_service_attributes = { "main": { "name": "dataAssetAccessServiceAgreement", "creator": account.address, "price": metadata[MetadataMain.KEY]['price'], "timeout": 3600, "datePublished": metadata[MetadataMain.KEY]['dateCreated'], "_amounts": _amounts, "_receivers": _receivers } } access_service_descriptor = ServiceDescriptor.access_service_descriptor( access_service_attributes, 'http://localhost:8030') return register_ddo(metadata, account, providers, auth_service, [access_service_descriptor])
def get_registered_workflow_ddo(account, compute_did, algorithm_did, providers=None, auth_service='PSK-RSA'): metadata = get_sample_workflow_ddo()['service'][0]['attributes'] metadata['main']['workflow']['stages'][0]['input'][0]['id'] = compute_did metadata['main']['workflow']['stages'][0]['transformation'][ 'id'] = algorithm_did metadata['main']['workflow']['checksum'] = str(uuid.uuid4()) access_service_attributes = { "main": { "name": "dataAssetAccessServiceAgreement", "creator": account.address, "price": metadata[MetadataMain.KEY]['price'], "timeout": 3600, "datePublished": metadata[MetadataMain.KEY]['dateCreated'] } } access_service_descriptor = ServiceDescriptor.access_service_descriptor( access_service_attributes, 'http://localhost:8030') return register_ddo(metadata, account, providers, auth_service, [access_service_descriptor])
def create_asset(publisher_instance, ddo_sample): nevermined = publisher_instance acct = nevermined.main_account asset = ddo_sample my_secret_store = 'http://myownsecretstore.com' auth_service = ServiceDescriptor.authorization_service_descriptor( {'main': {'service': 'SecretStore', 'publicKey': '0casd', 'threshold': '1'}}, my_secret_store) return nevermined.assets.create(asset.metadata, acct, [auth_service])
def create_access_service(attributes, service_endpoint): """ Publish an asset with an `Access` service according to the supplied attributes. :param attributes: attributes of the access service, dict :param service_endpoint: str URL for initiating service access request :return: Service instance or None """ service = ServiceDescriptor.access_service_descriptor(attributes, service_endpoint) return service
def test_asset_owner(publisher_instance, ddo_sample): acct = publisher_instance.main_account asset = ddo_sample my_secret_store = 'http://myownsecretstore.com' auth_service = ServiceDescriptor.authorization_service_descriptor( {'main': {'service': 'SecretStore', 'publicKey': '0casd', 'threshold': '1'}}, my_secret_store) new_asset = publisher_instance.assets.create(asset.metadata, acct, [auth_service]) assert publisher_instance.assets.owner(new_asset.did) == acct.address publisher_instance.assets.retire(new_asset.did)
def get_proof_ddo(account, providers=None, auth_service='PSK-RSA', key=get_key()): ddo = get_sample_ddo() metadata = ddo['service'][0]['attributes'] metadata['main']['files'][0]['url'] = key metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) hash = poseidon_hash(key) providerKey = get_provider_public_key() metadata['additionalInformation'] = { "providerKey": { "x": providerKey[0], "y": providerKey[1] }, "poseidonHash": hash } escrow_payment_condition = ddo['service'][1]['attributes'][ 'serviceAgreementTemplate']['conditions'][2] _amounts = get_param_value_by_name(escrow_payment_condition['parameters'], '_amounts') _receivers = to_checksum_addresses( get_param_value_by_name(escrow_payment_condition['parameters'], '_receivers')) access_service_attributes = { "main": { "name": "dataAssetAccessProofServiceAgreement", "creator": account.address, "price": metadata[MetadataMain.KEY]['price'], "timeout": 3600, "datePublished": metadata[MetadataMain.KEY]['dateCreated'], "_amounts": _amounts, "_tokenAddress": "", "_hash": hash, "_providerPub": providerKey, "_receivers": _receivers } } access_service_descriptor = ServiceDescriptor.access_proof_service_descriptor( access_service_attributes, 'http://localhost:8030') return register_ddo(metadata, account, providers, auth_service, [access_service_descriptor])
def get_registered_compute_ddo(account, providers=None, auth_service='PSK-RSA'): ddo = get_sample_ddo() metadata = ddo['service'][0]['attributes'] metadata['main']['files'][0][ 'url'] = "https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos" \ "/CoverSongs/shs_dataset_test.txt" metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) escrow_payment_condition = ddo['service'][1]['attributes'][ 'serviceAgreementTemplate']['conditions'][2] _amounts = get_param_value_by_name(escrow_payment_condition['parameters'], '_amounts') _receivers = to_checksum_addresses( get_param_value_by_name(escrow_payment_condition['parameters'], '_receivers')) compute_service_attributes = { "main": { "name": "dataAssetComputeServiceAgreement", "creator": account.address, "dataPublished": metadata[MetadataMain.KEY]["dateCreated"], "price": metadata[MetadataMain.KEY]["price"], "timeout": 86400, "_amounts": _amounts, "_receivers": _receivers, "provider": {} } } compute_service_descriptor = ServiceDescriptor.compute_service_descriptor( compute_service_attributes, "http://localhost:8050") return register_ddo(metadata, account, providers, auth_service, [compute_service_descriptor])
def create(self, metadata, publisher_account, service_descriptors=None, providers=None, authorization_type=ServiceAuthorizationTypes.PSK_RSA, use_secret_store=False, activity_id=None, attributes=None, asset_rewards={ "_amounts": [], "_receivers": [] }, cap=None, royalties=None): """ Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store. :param metadata: dict conforming to the Metadata accepted by Nevermined Protocol. :param publisher_account: Account of the publisher registering this asset :param service_descriptors: list of ServiceDescriptor tuples of length 2. The first item must be one of ServiceTypes and the second item is a dict of parameters and values required by the service :param providers: list of addresses of providers of this asset (a provider is an ethereum account that is authorized to provide asset services) :param authorization_type: str indicate the authorization type that is going to be used to encrypt the urls. SecretStore, PSK-RSA and PSK-ECDSA are supported. :param use_secret_store: bool indicate whether to use the secret store directly for encrypting urls (Uses Gateway provider service if set to False) :param activity_id: identifier of the activity creating the new entity :param attributes: attributes associated with the action :param asset_rewards: rewards distribution including the amounts and the receivers :param cap: max cap of nfts that can be minted for the asset :param royalties: royalties in the secondary market going to the original creator :return: DDO instance """ assert isinstance( metadata, dict), f'Expected metadata of type dict, got {type(metadata)}' # copy metadata so we don't change the original metadata_copy = copy.deepcopy(metadata) # Create a DDO object ddo = DDO() gateway = GatewayProvider.get_gateway() ddo_service_endpoint = self._get_metadata_provider( ).get_service_endpoint() metadata_service_desc = ServiceDescriptor.metadata_service_descriptor( metadata_copy, ddo_service_endpoint) if metadata_copy['main']['type'] == 'dataset' or metadata_copy['main'][ 'type'] == 'algorithm': access_service_attributes = self._build_access( metadata_copy, publisher_account, asset_rewards) if not service_descriptors: if authorization_type == ServiceAuthorizationTypes.PSK_RSA: service_descriptors = [ ServiceDescriptor.authorization_service_descriptor( self._build_authorization( authorization_type, public_key=gateway.get_rsa_public_key( self._config)), gateway.get_access_endpoint(self._config)) ] elif authorization_type == ServiceAuthorizationTypes.PSK_ECDSA: service_descriptors = [ ServiceDescriptor.authorization_service_descriptor( self._build_authorization( authorization_type, public_key=gateway.get_ecdsa_public_key( self._config)), gateway.get_access_endpoint(self._config)) ] else: service_descriptors = [ ServiceDescriptor.authorization_service_descriptor( self._build_authorization(authorization_type, threshold=0), self._config.secret_store_url) ] service_descriptors += [ ServiceDescriptor.access_service_descriptor( access_service_attributes, gateway.get_access_endpoint(self._config)) ] else: service_types = set(map(lambda x: x[0], service_descriptors)) if ServiceTypes.AUTHORIZATION not in service_types: if authorization_type == ServiceAuthorizationTypes.PSK_RSA: service_descriptors += [ ServiceDescriptor.authorization_service_descriptor( self._build_authorization( authorization_type, public_key=gateway.get_rsa_public_key( self._config)), gateway.get_access_endpoint(self._config)) ] elif authorization_type == ServiceAuthorizationTypes.PSK_ECDSA: service_descriptors += [ ServiceDescriptor.authorization_service_descriptor( self._build_authorization( authorization_type, public_key=gateway.get_ecdsa_public_key( self._config)), gateway.get_access_endpoint(self._config)) ] else: service_descriptors += [ ServiceDescriptor.authorization_service_descriptor( self._build_authorization(authorization_type, threshold=0), self._config.secret_store_url) ] else: service_descriptors += [ ServiceDescriptor.access_service_descriptor( access_service_attributes, gateway.get_access_endpoint(self._config)) ] else: if not service_descriptors: service_descriptors = [] else: service_descriptors += [] logger.info('registering a workflow.') # Add all services to ddo service_descriptors = [metadata_service_desc] + service_descriptors services = ServiceFactory.build_services(service_descriptors) checksums = dict() for service in services: checksums[str(service.index)] = checksum(service.main) # Adding proof to the ddo. ddo.add_proof(checksums, publisher_account) # Generating the did and adding to the ddo. did = ddo.assign_did(DID.did(ddo.proof['checksum'])) logger.debug(f'Generating new did: {did}') # Check if it's already registered first! if did in self._get_metadata_provider().list_assets(): raise DIDAlreadyExist( f'Asset id {did} is already registered to another asset.') for service in services: if service.type == ServiceTypes.ASSET_ACCESS: access_service = ServiceFactory.complete_access_service( did, gateway.get_access_endpoint(self._config), access_service_attributes, self._keeper.access_template.address, self._keeper.escrow_payment_condition.address) ddo.add_service(access_service) elif service.type == ServiceTypes.METADATA: ddo_service_endpoint = service.service_endpoint.replace( '{did}', did) service.set_service_endpoint(ddo_service_endpoint) ddo.add_service(service) elif service.type == ServiceTypes.CLOUD_COMPUTE: compute_service = ServiceFactory.complete_compute_service( did, service.service_endpoint, service.attributes, self._keeper.compute_execution_condition.address, self._keeper.escrow_payment_condition.address) ddo.add_service(compute_service) else: ddo.add_service(service) ddo.proof['signatureValue'] = self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)), publisher_account) # Add public key and authentication ddo.add_public_key(did, publisher_account.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) # Setup metadata service # First compute files_encrypted if metadata_copy['main']['type'] in ['dataset', 'algorithm']: assert metadata_copy['main'][ 'files'], 'files is required in the metadata main attributes.' logger.debug('Encrypting content urls in the metadata.') if not use_secret_store: encrypt_endpoint = gateway.get_encrypt_endpoint(self._config) files_encrypted = gateway.encrypt_files_dict( metadata_copy['main']['files'], encrypt_endpoint, ddo.asset_id, authorization_type) else: files_encrypted = self._get_secret_store(publisher_account) \ .encrypt_document( did_to_id(did), json.dumps(metadata_copy['main']['files']), ) # only assign if the encryption worked if files_encrypted: logger.debug( f'Content urls encrypted successfully {files_encrypted}') index = 0 for file in metadata_copy['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata_copy['encryptedFiles'] = files_encrypted else: raise AssertionError('Encrypting the files failed.') # DDO url and `Metadata` service logger.debug(f'Generated ddo and services, DID is {ddo.did},' f' metadata service @{ddo_service_endpoint}.') response = None # register on-chain registered_on_chain = self._keeper.did_registry.register( ddo.asset_id, checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), url=ddo_service_endpoint, account=publisher_account, cap=cap, royalties=royalties, providers=providers, activity_id=activity_id, attributes=attributes) if registered_on_chain is False: logger.warning(f'Registering {did} on-chain failed.') return None logger.info(f'Successfully registered DDO (DID={did}) on chain.') try: # publish the new ddo response = self._get_metadata_provider().publish_asset_ddo(ddo) logger.info('Asset/ddo published successfully in Metadata.') except ValueError as ve: raise ValueError( f'Invalid value to publish in the metadata: {str(ve)}') except Exception as e: logger.error(f'Publish asset in Metadata failed: {str(e)}') if not response: return None return ddo
def register_ddo(metadata, account, providers, auth_service, additional_service_descriptors, royalties=None, cap=None, mint=0): keeper = keeper_instance() metadata_api = Metadata('http://172.17.0.1:5000') ddo = DDO() ddo_service_endpoint = metadata_api.get_service_endpoint() metadata_service_desc = ServiceDescriptor.metadata_service_descriptor( metadata, ddo_service_endpoint) authorization_service_attributes = { "main": { "service": auth_service, "publicKey": "0xd7" } } service_descriptors = [ ServiceDescriptor.authorization_service_descriptor( authorization_service_attributes, 'http://localhost:12001') ] service_descriptors += [metadata_service_desc] service_descriptors += additional_service_descriptors services = ServiceFactory.build_services(service_descriptors) checksums = dict() for service in services: try: checksums[str(service.index)] = checksum(service.main) except Exception as e: pass # Adding proof to the ddo. ddo.add_proof(checksums, account) did_seed = checksum(ddo.proof['checksum']) asset_id = keeper.did_registry.hash_did(did_seed, account.address) ddo._did = DID.did(asset_id) did = ddo._did for service in services: if service.type == ServiceTypes.ASSET_ACCESS or service.type == ServiceTypes.NFT_ACCESS or service.type == ServiceTypes.ASSET_ACCESS_PROOF: access_service = ServiceFactory.complete_access_service( did, service.service_endpoint, service.attributes, keeper.access_template.address, keeper.escrow_payment_condition.address, service.type) ddo.add_service(access_service) elif service.type == ServiceTypes.CLOUD_COMPUTE: compute_service = ServiceFactory.complete_compute_service( did, service.service_endpoint, service.attributes, keeper.compute_execution_condition.address, keeper.escrow_payment_condition.address) ddo.add_service(compute_service) elif service.type == ServiceTypes.NFT_SALES: nft_sales_service = ServiceFactory.complete_nft_sales_service( did, service.service_endpoint, service.attributes, keeper.nft_sales_template.address, keeper.escrow_payment_condition.address, service.type) ddo.add_service(nft_sales_service) else: ddo.add_service(service) ddo.proof['signatureValue'] = keeper.sign_hash(did_to_id_bytes(did), account) ddo.add_public_key(did, account.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) try: _oldddo = metadata_api.get_asset_ddo(ddo.did) if _oldddo: metadata_api.retire_asset_ddo(ddo.did) except ValueError: pass if 'files' in metadata['main']: if auth_service == ServiceAuthorizationTypes.SECRET_STORE: encrypted_files = do_secret_store_encrypt( remove_0x_prefix(ddo.asset_id), json.dumps(metadata['main']['files']), account, get_config()) elif auth_service == ServiceAuthorizationTypes.PSK_RSA: encrypted_files, public_key = rsa_encryption_from_file( json.dumps(metadata['main']['files']), get_rsa_public_key_file()) else: encrypted_files, public_key = ecdsa_encryption_from_file( json.dumps(metadata['main']['files']), get_provider_key_file(), get_provider_password()) _files = metadata['main']['files'] # only assign if the encryption worked if encrypted_files: index = 0 for file in metadata['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata['encryptedFiles'] = encrypted_files ddo_with_did = DDO(did, json_text=ddo.as_text().replace('/{did}', '/' + did)) ddo_service_endpoint = ddo_service_endpoint.replace('/{did}', '/' + did) if mint > 0 or royalties is not None or cap is not None: keeper.did_registry.register_mintable_did( did_seed, checksum=web3().toBytes(hexstr=ddo.asset_id), url=ddo_service_endpoint, cap=cap, royalties=royalties, account=account, providers=providers) if mint > 0: keeper.did_registry.mint(ddo.asset_id, mint, account=account) else: keeper_instance().did_registry.register( did_seed, checksum=web3().toBytes(hexstr=ddo.asset_id), url=ddo_service_endpoint, account=account, providers=providers) metadata_api.publish_asset_ddo(ddo_with_did) return ddo_with_did