def did(seed): """ Create a did. Format of the did: did:op:cb36cf78d87f4ce4a784f17c2a4a694f19f3fbf05b814ac6b0b7197163888865 :param seed: The list of checksums that is allocated in the proof, dict :return: Asset did, str. """ return OCEAN_PREFIX + remove_0x_prefix(checksum(seed))
def test_create_ddo(metadata): pub_acc = get_publisher_account() ddo = DDO() ddo.add_service(ServiceTypes.METADATA, 'http://myaquarius.com', values=metadata, index='0') checksums = dict() for service in ddo.services: checksums[str(service.index)] = checksum(service.main) ddo.add_proof(checksums, pub_acc) did = ddo.assign_did(DID.did(ddo.proof['checksum'])) ddo.proof['signatureValue'] = Keeper.sign_hash(did_to_id_bytes(did), pub_acc) ddo.add_public_key(did, pub_acc.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
def get_ddo_sample(datatoken_address): did = f'did:op:{remove_0x_prefix(datatoken_address)}' sample_ddo_path = get_resource_path('ddo', 'ddo_sa_sample.json') assert sample_ddo_path.exists(), "{} does not exist!".format(sample_ddo_path) asset = DDO(json_filename=sample_ddo_path) asset.metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) checksum_dict = dict() for service in asset.services: checksum_dict[str(service.index)] = checksum(service.main) asset.add_proof(checksum_dict, get_publisher_wallet()) asset._did = did return asset
def get_ddo_sample(datatoken_address): """Helper function to get a sample ddo for testing.""" did = f"did:op:{remove_0x_prefix(datatoken_address)}" sample_ddo_path = get_resource_path("ddo", "ddo_sa_sample.json") assert sample_ddo_path.exists(), "{} does not exist!".format(sample_ddo_path) asset = DDO(json_filename=sample_ddo_path) asset.metadata["main"]["files"][0]["checksum"] = str(uuid.uuid4()) checksum_dict = dict() for service in asset.services: checksum_dict[str(service.index)] = checksum(service.main) asset.add_proof(checksum_dict, get_publisher_wallet()) asset._did = did return asset
def create( self, metadata: dict, publisher_wallet: Wallet, service_descriptors: list = None, owner_address: str = None, data_token_address: str = None, provider_uri=None, dt_name: str = None, dt_symbol: str = None, dt_blob: str = None, dt_cap: float = None, ) -> (Asset, None): """ Register an asset on-chain by creating/deploying a DataToken contract and in the Metadata store (Aquarius). :param metadata: dict conforming to the Metadata accepted by Ocean Protocol. :param publisher_wallet: Wallet of the publisher registering this asset :param service_descriptors: list of ServiceDescriptor tuples of length 2. The first item must be one of ServiceTypes and the second item is a dict of parameters and values required by the service :param owner_address: hex str the ethereum address to assign asset ownership to. After registering the asset on-chain, the ownership is transferred to this address :param data_token_address: hex str the address of the data token smart contract. The new asset will be associated with this data token address. :param provider_uri: str URL of service provider. This will be used as base to construct the serviceEndpoint for the `access` (download) service :param dt_name: str name of DataToken if creating a new one :param dt_symbol: str symbol of DataToken if creating a new one :param dt_blob: str blob of DataToken if creating a new one. A `blob` is any text to be stored with the ERC20 DataToken contract for any purpose. :param dt_cap: float :return: DDO instance """ assert isinstance( metadata, dict), f"Expected metadata of type dict, got {type(metadata)}" assert service_descriptors is None or isinstance( service_descriptors, list ), f"bad type of `service_descriptors` {type(service_descriptors)}" # copy metadata so we don't change the original metadata_copy = copy.deepcopy(metadata) asset_type = metadata_copy["main"]["type"] assert asset_type in ( "dataset", "algorithm", ), f"Invalid/unsupported asset type {asset_type}" if not plecos.is_valid_dict_local(metadata_copy): errors = plecos.list_errors_dict_local(metadata_copy) msg = f"Metadata has validation errors: {errors}" logger.error(msg) raise ValueError(msg) service_descriptors = service_descriptors or [] services = self._process_service_descriptors(service_descriptors, metadata_copy, provider_uri, publisher_wallet) stype_to_service = {s.type: s for s in services} checksum_dict = dict() for service in services: checksum_dict[str(service.index)] = checksum(service.main) # Create a DDO object asset = Asset() # Adding proof to the ddo. asset.add_proof(checksum_dict, publisher_wallet) ################# # DataToken address = DTFactory.configured_address(Web3Helper.get_network_name(), self._config.address_file) dtfactory = DTFactory(address) if not data_token_address: blob = dt_blob or "" name = dt_name or metadata["main"]["name"] symbol = dt_symbol or name # register on-chain _cap = dt_cap if dt_cap else DataToken.DEFAULT_CAP tx_id = dtfactory.createToken(blob, name, symbol, to_base_18(_cap), from_wallet=publisher_wallet) data_token = DataToken(dtfactory.get_token_address(tx_id)) if not data_token: logger.warning("Creating new data token failed.") return None data_token_address = data_token.address logger.info(f"Successfully created data token with address " f"{data_token.address} for new dataset asset.") # owner_address is set as minter only if creating new data token. So if # `data_token_address` is set `owner_address` has no effect. if owner_address: data_token.proposeMinter(owner_address, from_wallet=publisher_wallet) else: # verify data_token_address dt = DataToken(data_token_address) minter = dt.contract_concise.minter() if not minter: raise AssertionError( f"datatoken address {data_token_address} does not seem to be a valid DataToken contract." ) elif minter.lower() != publisher_wallet.address.lower(): raise AssertionError( f"Minter of datatoken {data_token_address} is not the same as the publisher." ) elif not dtfactory.verify_data_token(data_token_address): raise AssertionError( f"datatoken address {data_token_address} is not found in the DTFactory events." ) assert ( data_token_address ), "data_token_address is required for publishing a dataset asset." # Generating the did and adding to the ddo. did = asset.assign_did( f"did:op:{remove_0x_prefix(data_token_address)}") logger.debug(f"Using datatoken address as did: {did}") # Check if it's already registered first! if did in self._get_aquarius().list_assets(): raise OceanDIDAlreadyExist( f"Asset id {did} is already registered to another asset.") md_service = stype_to_service[ServiceTypes.METADATA] ddo_service_endpoint = md_service.service_endpoint if "{did}" in ddo_service_endpoint: ddo_service_endpoint = ddo_service_endpoint.replace("{did}", did) md_service.set_service_endpoint(ddo_service_endpoint) # Populate the ddo services asset.add_service(md_service) access_service = stype_to_service.get(ServiceTypes.ASSET_ACCESS, None) compute_service = stype_to_service.get(ServiceTypes.CLOUD_COMPUTE, None) if access_service: asset.add_service(access_service) if compute_service: asset.add_service(compute_service) asset.proof["signatureValue"] = Web3Helper.sign_hash( add_ethereum_prefix_and_hash_msg(asset.asset_id), publisher_wallet) # Add public key and authentication asset.add_public_key(did, publisher_wallet.address) asset.add_authentication(did, PUBLIC_KEY_TYPE_RSA) # Setup metadata service # First compute files_encrypted assert metadata_copy["main"][ "files"], "files is required in the metadata main attributes." logger.debug("Encrypting content urls in the metadata.") publisher_signature = self._data_provider.sign_message( publisher_wallet, asset.asset_id, self._config) _, encrypt_endpoint = self._data_provider.build_encrypt_endpoint( provider_uri) files_encrypted = self._data_provider.encrypt_files_dict( metadata_copy["main"]["files"], encrypt_endpoint, asset.asset_id, publisher_wallet.address, publisher_signature, ) # only assign if the encryption worked if files_encrypted: logger.debug( f"Content urls encrypted successfully {files_encrypted}") index = 0 for file in metadata_copy["main"]["files"]: file["index"] = index index = index + 1 del file["url"] metadata_copy["encryptedFiles"] = files_encrypted else: raise AssertionError("Encrypting the files failed.") logger.debug(f"Generated asset and services, DID is {asset.did}," f" metadata service @{ddo_service_endpoint}.") # Set datatoken address in the asset asset.data_token_address = data_token_address try: # publish the new ddo in ocean-db/Aquarius ddo_registry = self.ddo_registry() web3 = Web3Provider.get_web3() tx_id = ddo_registry.create( asset.asset_id, bytes([1]), lzma.compress(web3.toBytes(text=asset.as_text())), publisher_wallet, ) if not ddo_registry.verify_tx(tx_id): raise AssertionError( f"create DDO on-chain failed, transaction status is 0. Transaction hash is {tx_id}" ) logger.info("Asset/ddo published on-chain successfully.") except ValueError as ve: raise ValueError( f"Invalid value to publish in the metadata: {str(ve)}") except Exception as e: logger.error(f"Publish asset on-chain failed: {str(e)}") raise return asset
def create(self, metadata, publisher_account, service_descriptors=None, providers=None, use_secret_store=True): """ Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store ( Aquarius). :param metadata: dict conforming to the Metadata accepted by Ocean Protocol. :param publisher_account: Account of the publisher registering this asset :param service_descriptors: list of ServiceDescriptor tuples of length 2. The first item must be one of ServiceTypes and the second item is a dict of parameters and values required by the service :param providers: list of addresses of providers of this asset (a provider is an ethereum account that is authorized to provide asset services) :param use_secret_store: bool indicate whether to use the secret store directly for encrypting urls (Uses Brizo provider service if set to False) :return: DDO instance """ assert isinstance( metadata, dict), f'Expected metadata of type dict, got {type(metadata)}' assert service_descriptors is None or isinstance(service_descriptors, list), \ f'bad type of `service_descriptors` {type(service_descriptors)}' # if not metadata or not Metadata.validate(metadata): # raise OceanInvalidMetadata('Metadata seems invalid. Please make sure' # ' the required metadata values are filled in.') # copy metadata so we don't change the original metadata_copy = copy.deepcopy(metadata) asset_type = metadata_copy['main']['type'] assert asset_type in ( 'dataset', 'algorithm'), f'Invalid/unsupported asset type {asset_type}' service_descriptors = service_descriptors or [] brizo = BrizoProvider.get_brizo() services = self._process_service_descriptors(service_descriptors, metadata_copy, publisher_account) stype_to_service = {s.type: s for s in services} checksum_dict = dict() for service in services: checksum_dict[str(service.index)] = checksum(service.main) # Create a DDO object ddo = DDO() # Adding proof to the ddo. ddo.add_proof(checksum_dict, publisher_account) # Generating the did and adding to the ddo. did = ddo.assign_did(DID.did(ddo.proof['checksum'])) logger.debug(f'Generating new did: {did}') # Check if it's already registered first! if did in self._get_aquarius().list_assets(): raise OceanDIDAlreadyExist( f'Asset id {did} is already registered to another asset.') md_service = stype_to_service[ServiceTypes.METADATA] ddo_service_endpoint = md_service.service_endpoint if '{did}' in ddo_service_endpoint: ddo_service_endpoint = ddo_service_endpoint.replace('{did}', did) md_service.set_service_endpoint(ddo_service_endpoint) # Populate the ddo services ddo.add_service(md_service) ddo.add_service(stype_to_service[ServiceTypes.AUTHORIZATION]) access_service = stype_to_service.get(ServiceTypes.ASSET_ACCESS, None) compute_service = stype_to_service.get(ServiceTypes.CLOUD_COMPUTE, None) if access_service: access_service.init_conditions_values( did, { cname: c.address for cname, c in self._keeper.contract_name_to_instance.items() }) ddo.add_service(access_service) if compute_service: compute_service.init_conditions_values( did, { cname: c.address for cname, c in self._keeper.contract_name_to_instance.items() }) ddo.add_service(compute_service) ddo.proof['signatureValue'] = self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)), publisher_account) # Add public key and authentication ddo.add_public_key(did, publisher_account.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) # Setup metadata service # First compute files_encrypted if metadata_copy['main']['type'] == 'dataset': assert metadata_copy['main']['files'], \ 'files is required in the metadata main attributes.' logger.debug('Encrypting content urls in the metadata.') if not use_secret_store: encrypt_endpoint = brizo.get_encrypt_endpoint(self._config) files_encrypted = brizo.encrypt_files_dict( metadata_copy['main']['files'], encrypt_endpoint, ddo.asset_id, publisher_account.address, self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(ddo.asset_id), publisher_account)) else: files_encrypted = self._get_secret_store(publisher_account) \ .encrypt_document( did_to_id(did), json.dumps(metadata_copy['main']['files']), ) # only assign if the encryption worked if files_encrypted: logger.debug( f'Content urls encrypted successfully {files_encrypted}') index = 0 for file in metadata_copy['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata_copy['encryptedFiles'] = files_encrypted else: raise AssertionError('Encrypting the files failed.') # DDO url and `Metadata` service logger.debug(f'Generated ddo and services, DID is {ddo.did},' f' metadata service @{ddo_service_endpoint}.') response = None # register on-chain registered_on_chain = self._keeper.did_registry.register( ddo.asset_id, checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), url=ddo_service_endpoint, account=publisher_account, providers=providers) if registered_on_chain is False: logger.warning(f'Registering {did} on-chain failed.') return None logger.info(f'Successfully registered DDO (DID={did}) on chain.') try: # publish the new ddo in ocean-db/Aquarius response = self._get_aquarius().publish_asset_ddo(ddo) logger.info('Asset/ddo published successfully in aquarius.') except ValueError as ve: raise ValueError( f'Invalid value to publish in the metadata: {str(ve)}') except Exception as e: logger.error(f'Publish asset in aquarius failed: {str(e)}') if not response: return None return ddo
def get_registered_ddo(client, wallet, metadata, service_descriptor): aqua = Aquarius('http://localhost:5000') ddo_service_endpoint = aqua.get_service_endpoint() metadata_store_url = json.dumps({'t': 1, 'url': ddo_service_endpoint}) # Create new data token contract addresses = get_contracts_addresses(get_address_file()) dt_address = addresses.get(DTFactory.CONTRACT_NAME) if dt_address: factory_contract = DTFactory(dt_address) else: factory_contract = new_factory_contract() ddo_contract_address = addresses.get(MetadataContract.CONTRACT_NAME) metadata_contract = MetadataContract(ddo_contract_address) tx_id = factory_contract.createToken(metadata_store_url, 'DataToken1', 'DT1', to_base_18(1000000), wallet) dt_contract = DataToken(factory_contract.get_token_address(tx_id)) if not dt_contract: raise AssertionError('Creation of data token contract failed.') ddo = Asset() ddo.data_token_address = dt_contract.address metadata_service_desc = ServiceDescriptor.metadata_service_descriptor( metadata, ddo_service_endpoint) service_descriptors = list([ ServiceDescriptor.authorization_service_descriptor( 'http://localhost:12001') ]) service_descriptors.append(service_descriptor) service_type = service_descriptor[0] service_descriptors = [metadata_service_desc] + service_descriptors services = ServiceFactory.build_services(service_descriptors) checksums = dict() for service in services: checksums[str(service.index)] = checksum(service.main) # Adding proof to the ddo. ddo.add_proof(checksums, wallet) did = ddo.assign_did(f'did:op:{remove_0x_prefix(ddo.data_token_address)}') ddo_service_endpoint.replace('{did}', did) services[0].set_service_endpoint(ddo_service_endpoint) stype_to_service = {s.type: s for s in services} _service = stype_to_service[service_type] for service in services: ddo.add_service(service) # ddo.proof['signatureValue'] = ocean_lib.sign_hash( # did_to_id_bytes(did), account) ddo.add_public_key(did, wallet.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) # if not plecos.is_valid_dict_local(ddo.metadata): # print(f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}') # assert False, f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}' files_list_str = json.dumps(metadata['main']['files']) encrypted_files = encrypt_document(client, did, files_list_str, wallet) # encrypted_files = do_encrypt(files_list_str, provider_wallet) # only assign if the encryption worked if encrypted_files: index = 0 for file in metadata['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata['encryptedFiles'] = encrypted_files web3 = Web3Provider.get_web3() block = web3.eth.blockNumber try: data = lzma.compress(web3.toBytes(text=ddo.as_text())) tx_id = metadata_contract.create(ddo.asset_id, bytes([1]), data, wallet) if not metadata_contract.verify_tx(tx_id): raise AssertionError( f'create DDO on-chain failed, transaction status is 0. Transaction hash is {tx_id}' ) except Exception as e: print(f'error publishing ddo {ddo.did} in Aquarius: {e}') raise log = metadata_contract.get_event_log( metadata_contract.EVENT_METADATA_CREATED, block, ddo.asset_id, 30) assert log, f'no ddo created event.' ddo = wait_for_ddo(aqua, ddo.did) assert ddo, f'resolve did {ddo.did} failed.' return ddo
def get_registered_ddo(account, providers=None): keeper = keeper_instance() aqua = Aquarius('http://localhost:5000') for did in aqua.list_assets(): aqua.retire_asset_ddo(did) metadata = get_sample_ddo()['service'][0]['attributes'] metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) ddo = DDO() ddo_service_endpoint = aqua.get_service_endpoint() metadata_service_desc = ServiceDescriptor.metadata_service_descriptor( metadata, ddo_service_endpoint) access_service_attributes = { "main": { "name": "dataAssetAccessServiceAgreement", "creator": account.address, "price": metadata[MetadataMain.KEY]['price'], "timeout": 3600, "datePublished": metadata[MetadataMain.KEY]['dateCreated'] } } service_descriptors = [ ServiceDescriptor.authorization_service_descriptor( 'http://localhost:12001') ] service_descriptors += [ ServiceDescriptor.access_service_descriptor( access_service_attributes, 'http://localhost:8030', keeper.escrow_access_secretstore_template.address) ] service_descriptors = [metadata_service_desc] + service_descriptors services = ServiceFactory.build_services(service_descriptors) checksums = dict() for service in services: checksums[str(service.index)] = checksum(service.main) # Adding proof to the ddo. ddo.add_proof(checksums, account) did = ddo.assign_did(DID.did(ddo.proof['checksum'])) stype_to_service = {s.type: s for s in services} access_service = stype_to_service[ServiceTypes.ASSET_ACCESS] name_to_address = { cname: cinst.address for cname, cinst in keeper.contract_name_to_instance.items() } access_service.init_conditions_values( did, contract_name_to_address=name_to_address) ddo.add_service(access_service) for service in services: ddo.add_service(service) ddo.proof['signatureValue'] = keeper.sign_hash(did_to_id_bytes(did), account) ddo.add_public_key(did, account.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) encrypted_files = do_secret_store_encrypt( remove_0x_prefix(ddo.asset_id), json.dumps(metadata['main']['files']), account, get_config()) _files = metadata['main']['files'] # only assign if the encryption worked if encrypted_files: index = 0 for file in metadata['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata['encryptedFiles'] = encrypted_files keeper_instance().did_registry.register( ddo.asset_id, checksum=web3().toBytes(hexstr=ddo.asset_id), url=ddo_service_endpoint, account=account, providers=providers) aqua.publish_asset_ddo(ddo) return ddo
def get_registered_ddo(account, metadata, service_descriptor, providers=None): keeper = keeper_instance() aqua = Aquarius('http://localhost:5000') ddo = DDO() ddo_service_endpoint = aqua.get_service_endpoint() metadata_service_desc = ServiceDescriptor.metadata_service_descriptor( metadata, ddo_service_endpoint) service_descriptors = list([ ServiceDescriptor.authorization_service_descriptor( 'http://localhost:12001') ]) service_descriptors.append(service_descriptor) service_type = service_descriptor[0] service_descriptors = [metadata_service_desc] + service_descriptors services = ServiceFactory.build_services(service_descriptors) checksums = dict() for service in services: checksums[str(service.index)] = checksum(service.main) # Adding proof to the ddo. ddo.add_proof(checksums, account) did = ddo.assign_did(DID.did(ddo.proof['checksum'])) ddo_service_endpoint.replace('{did}', did) services[0].set_service_endpoint(ddo_service_endpoint) stype_to_service = {s.type: s for s in services} _service = stype_to_service[service_type] name_to_address = { cname: cinst.address for cname, cinst in keeper.contract_name_to_instance.items() } _service.init_conditions_values(did, contract_name_to_address=name_to_address) for service in services: ddo.add_service(service) ddo.proof['signatureValue'] = keeper.sign_hash(did_to_id_bytes(did), account) ddo.add_public_key(did, account.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) try: _oldddo = aqua.get_asset_ddo(ddo.did) if _oldddo: aqua.retire_asset_ddo(ddo.did) except ValueError: pass if not plecos.is_valid_dict_local(ddo.metadata): print(f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}') assert False, f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}' encrypted_files = do_secret_store_encrypt( remove_0x_prefix(ddo.asset_id), json.dumps(metadata['main']['files']), account, get_config()) # only assign if the encryption worked if encrypted_files: index = 0 for file in metadata['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata['encryptedFiles'] = encrypted_files keeper_instance().did_registry.register( ddo.asset_id, checksum=web3().toBytes(hexstr=ddo.asset_id), url=ddo_service_endpoint, account=account, providers=providers) try: aqua.publish_asset_ddo(ddo) except Exception as e: print(f'error publishing ddo {ddo.did} in Aquarius: {e}') raise return ddo
def get_registered_ddo(account, metadata, service_descriptor): aqua = Aquarius('http://localhost:5000') ddo = DDO() ddo_service_endpoint = aqua.get_service_endpoint() # Create new data token contract dt_contract = FactoryContract(get_config().factory_address)\ .create_data_token(account, metadata_url=ddo_service_endpoint) if not dt_contract: raise AssertionError('Creation of data token contract failed.') ddo._other_values = {'dataTokenAddress': dt_contract.address} metadata_service_desc = ServiceDescriptor.metadata_service_descriptor( metadata, ddo_service_endpoint ) service_descriptors = list( [ServiceDescriptor.authorization_service_descriptor('http://localhost:12001')]) service_descriptors.append(service_descriptor) service_type = service_descriptor[0] service_descriptors = [metadata_service_desc] + service_descriptors services = ServiceFactory.build_services(service_descriptors) checksums = dict() for service in services: checksums[str(service.index)] = checksum(service.main) # Adding proof to the ddo. ddo.add_proof(checksums, account) did = ddo.assign_did(DID.did(ddo.proof['checksum'])) ddo_service_endpoint.replace('{did}', did) services[0].set_service_endpoint(ddo_service_endpoint) stype_to_service = {s.type: s for s in services} _service = stype_to_service[service_type] for service in services: ddo.add_service(service) # ddo.proof['signatureValue'] = ocean_lib.sign_hash( # did_to_id_bytes(did), account) ddo.add_public_key(did, account.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) try: _oldddo = aqua.get_asset_ddo(ddo.did) if _oldddo: aqua.retire_asset_ddo(ddo.did) except ValueError: pass if not plecos.is_valid_dict_local(ddo.metadata): print(f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}') assert False, f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}' encrypted_files = do_encrypt( json.dumps(metadata['main']['files']), account, ) # only assign if the encryption worked if encrypted_files: index = 0 for file in metadata['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata['encryptedFiles'] = encrypted_files # ddo._other_values try: aqua.publish_asset_ddo(ddo) except Exception as e: print(f'error publishing ddo {ddo.did} in Aquarius: {e}') raise return ddo