def test_creating_ddo_from_scratch():
# create an empty ddo
ddo = DDO()
assert ddo.did is None
assert ddo.asset_id is None
assert ddo.created is not None
did = DID.did({"0": "0x99999999999999999"})
ddo.assign_did(did)
assert ddo.did == did
pub_acc = get_publisher_account()
ddo.add_service(TEST_SERVICE_TYPE, TEST_SERVICE_URL)
# add a proof to the first public_key/authentication
ddo.add_proof('checksum', pub_acc)
ddo_text_proof = ddo.as_text()
assert ddo_text_proof
pub_acc = get_publisher_account()
assert not ddo.public_keys
ddo.add_public_key(did, pub_acc.address)
assert len(ddo.public_keys) == 1
assert ddo.get_public_key(0) == ddo.public_keys[0]
with pytest.raises(IndexError):
ddo.get_public_key(1)
assert ddo.get_public_key(did) == ddo.public_keys[0]
assert ddo.get_public_key('0x32233') is None
assert not ddo.authentications
ddo.add_authentication(did, '')
assert len(ddo.authentications) == 1
def test_create_ddo(metadata):
pub_acc = get_publisher_account()
ddo = DDO()
ddo.add_service(ServiceTypes.METADATA, 'http://myaquarius.com', values=metadata, index='0')
checksums = dict()
for service in ddo.services:
checksums[str(service.index)] = checksum(service.main)
ddo.add_proof(checksums, pub_acc)
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
ddo.proof['signatureValue'] = Keeper.sign_hash(did_to_id_bytes(did), pub_acc)
ddo.add_public_key(did, pub_acc.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
def create(self,
metadata,
publisher_account,
service_descriptors=None,
providers=None,
use_secret_store=True):
"""
Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store (
Aquarius).
:param metadata: dict conforming to the Metadata accepted by Ocean Protocol.
:param publisher_account: Account of the publisher registering this asset
:param service_descriptors: list of ServiceDescriptor tuples of length 2.
The first item must be one of ServiceTypes and the second
item is a dict of parameters and values required by the service
:param providers: list of addresses of providers of this asset (a provider is
an ethereum account that is authorized to provide asset services)
:param use_secret_store: bool indicate whether to use the secret store directly for
encrypting urls (Uses Brizo provider service if set to False)
:return: DDO instance
"""
assert isinstance(
metadata,
dict), f'Expected metadata of type dict, got {type(metadata)}'
assert service_descriptors is None or isinstance(service_descriptors, list), \
f'bad type of `service_descriptors` {type(service_descriptors)}'
# if not metadata or not Metadata.validate(metadata):
# raise OceanInvalidMetadata('Metadata seems invalid. Please make sure'
# ' the required metadata values are filled in.')
# copy metadata so we don't change the original
metadata_copy = copy.deepcopy(metadata)
asset_type = metadata_copy['main']['type']
assert asset_type in (
'dataset',
'algorithm'), f'Invalid/unsupported asset type {asset_type}'
service_descriptors = service_descriptors or []
brizo = BrizoProvider.get_brizo()
services = self._process_service_descriptors(service_descriptors,
metadata_copy,
publisher_account)
stype_to_service = {s.type: s for s in services}
checksum_dict = dict()
for service in services:
checksum_dict[str(service.index)] = checksum(service.main)
# Create a DDO object
ddo = DDO()
# Adding proof to the ddo.
ddo.add_proof(checksum_dict, publisher_account)
# Generating the did and adding to the ddo.
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
logger.debug(f'Generating new did: {did}')
# Check if it's already registered first!
if did in self._get_aquarius().list_assets():
raise OceanDIDAlreadyExist(
f'Asset id {did} is already registered to another asset.')
md_service = stype_to_service[ServiceTypes.METADATA]
ddo_service_endpoint = md_service.service_endpoint
if '{did}' in ddo_service_endpoint:
ddo_service_endpoint = ddo_service_endpoint.replace('{did}', did)
md_service.set_service_endpoint(ddo_service_endpoint)
# Populate the ddo services
ddo.add_service(md_service)
ddo.add_service(stype_to_service[ServiceTypes.AUTHORIZATION])
access_service = stype_to_service.get(ServiceTypes.ASSET_ACCESS, None)
compute_service = stype_to_service.get(ServiceTypes.CLOUD_COMPUTE,
None)
if access_service:
access_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(access_service)
if compute_service:
compute_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(compute_service)
ddo.proof['signatureValue'] = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)),
publisher_account)
# Add public key and authentication
ddo.add_public_key(did, publisher_account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
# Setup metadata service
# First compute files_encrypted
if metadata_copy['main']['type'] == 'dataset':
assert metadata_copy['main']['files'], \
'files is required in the metadata main attributes.'
logger.debug('Encrypting content urls in the metadata.')
if not use_secret_store:
encrypt_endpoint = brizo.get_encrypt_endpoint(self._config)
files_encrypted = brizo.encrypt_files_dict(
metadata_copy['main']['files'], encrypt_endpoint,
ddo.asset_id, publisher_account.address,
self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(ddo.asset_id),
publisher_account))
else:
files_encrypted = self._get_secret_store(publisher_account) \
.encrypt_document(
did_to_id(did),
json.dumps(metadata_copy['main']['files']),
)
# only assign if the encryption worked
if files_encrypted:
logger.debug(
f'Content urls encrypted successfully {files_encrypted}')
index = 0
for file in metadata_copy['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata_copy['encryptedFiles'] = files_encrypted
else:
raise AssertionError('Encrypting the files failed.')
# DDO url and `Metadata` service
logger.debug(f'Generated ddo and services, DID is {ddo.did},'
f' metadata service @{ddo_service_endpoint}.')
response = None
# register on-chain
registered_on_chain = self._keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=publisher_account,
providers=providers)
if registered_on_chain is False:
logger.warning(f'Registering {did} on-chain failed.')
return None
logger.info(f'Successfully registered DDO (DID={did}) on chain.')
try:
# publish the new ddo in ocean-db/Aquarius
response = self._get_aquarius().publish_asset_ddo(ddo)
logger.info('Asset/ddo published successfully in aquarius.')
except ValueError as ve:
raise ValueError(
f'Invalid value to publish in the metadata: {str(ve)}')
except Exception as e:
logger.error(f'Publish asset in aquarius failed: {str(e)}')
if not response:
return None
return ddo
def get_registered_ddo(account, providers=None):
keeper = keeper_instance()
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
metadata = get_sample_ddo()['service'][0]['attributes']
metadata['main']['files'][0]['checksum'] = str(uuid.uuid4())
ddo = DDO()
ddo_service_endpoint = aqua.get_service_endpoint()
metadata_service_desc = ServiceDescriptor.metadata_service_descriptor(
metadata, ddo_service_endpoint)
access_service_attributes = {
"main": {
"name": "dataAssetAccessServiceAgreement",
"creator": account.address,
"price": metadata[MetadataMain.KEY]['price'],
"timeout": 3600,
"datePublished": metadata[MetadataMain.KEY]['dateCreated']
}
}
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
'http://localhost:12001')
]
service_descriptors += [
ServiceDescriptor.access_service_descriptor(
access_service_attributes, 'http://localhost:8030',
keeper.escrow_access_secretstore_template.address)
]
service_descriptors = [metadata_service_desc] + service_descriptors
services = ServiceFactory.build_services(service_descriptors)
checksums = dict()
for service in services:
checksums[str(service.index)] = checksum(service.main)
# Adding proof to the ddo.
ddo.add_proof(checksums, account)
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
stype_to_service = {s.type: s for s in services}
access_service = stype_to_service[ServiceTypes.ASSET_ACCESS]
name_to_address = {
cname: cinst.address
for cname, cinst in keeper.contract_name_to_instance.items()
}
access_service.init_conditions_values(
did, contract_name_to_address=name_to_address)
ddo.add_service(access_service)
for service in services:
ddo.add_service(service)
ddo.proof['signatureValue'] = keeper.sign_hash(did_to_id_bytes(did),
account)
ddo.add_public_key(did, account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
encrypted_files = do_secret_store_encrypt(
remove_0x_prefix(ddo.asset_id), json.dumps(metadata['main']['files']),
account, get_config())
_files = metadata['main']['files']
# only assign if the encryption worked
if encrypted_files:
index = 0
for file in metadata['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata['encryptedFiles'] = encrypted_files
keeper_instance().did_registry.register(
ddo.asset_id,
checksum=web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=account,
providers=providers)
aqua.publish_asset_ddo(ddo)
return ddo
def get_registered_ddo(account, metadata, service_descriptor, providers=None):
keeper = keeper_instance()
aqua = Aquarius('http://localhost:5000')
ddo = DDO()
ddo_service_endpoint = aqua.get_service_endpoint()
metadata_service_desc = ServiceDescriptor.metadata_service_descriptor(
metadata, ddo_service_endpoint)
service_descriptors = list([
ServiceDescriptor.authorization_service_descriptor(
'http://localhost:12001')
])
service_descriptors.append(service_descriptor)
service_type = service_descriptor[0]
service_descriptors = [metadata_service_desc] + service_descriptors
services = ServiceFactory.build_services(service_descriptors)
checksums = dict()
for service in services:
checksums[str(service.index)] = checksum(service.main)
# Adding proof to the ddo.
ddo.add_proof(checksums, account)
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
ddo_service_endpoint.replace('{did}', did)
services[0].set_service_endpoint(ddo_service_endpoint)
stype_to_service = {s.type: s for s in services}
_service = stype_to_service[service_type]
name_to_address = {
cname: cinst.address
for cname, cinst in keeper.contract_name_to_instance.items()
}
_service.init_conditions_values(did,
contract_name_to_address=name_to_address)
for service in services:
ddo.add_service(service)
ddo.proof['signatureValue'] = keeper.sign_hash(did_to_id_bytes(did),
account)
ddo.add_public_key(did, account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
try:
_oldddo = aqua.get_asset_ddo(ddo.did)
if _oldddo:
aqua.retire_asset_ddo(ddo.did)
except ValueError:
pass
if not plecos.is_valid_dict_local(ddo.metadata):
print(f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}')
assert False, f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}'
encrypted_files = do_secret_store_encrypt(
remove_0x_prefix(ddo.asset_id), json.dumps(metadata['main']['files']),
account, get_config())
# only assign if the encryption worked
if encrypted_files:
index = 0
for file in metadata['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata['encryptedFiles'] = encrypted_files
keeper_instance().did_registry.register(
ddo.asset_id,
checksum=web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=account,
providers=providers)
try:
aqua.publish_asset_ddo(ddo)
except Exception as e:
print(f'error publishing ddo {ddo.did} in Aquarius: {e}')
raise
return ddo
def get_registered_ddo(account, metadata, service_descriptor):
aqua = Aquarius('http://localhost:5000')
ddo = DDO()
ddo_service_endpoint = aqua.get_service_endpoint()
# Create new data token contract
dt_contract = FactoryContract(get_config().factory_address)\
.create_data_token(account, metadata_url=ddo_service_endpoint)
if not dt_contract:
raise AssertionError('Creation of data token contract failed.')
ddo._other_values = {'dataTokenAddress': dt_contract.address}
metadata_service_desc = ServiceDescriptor.metadata_service_descriptor(
metadata, ddo_service_endpoint
)
service_descriptors = list(
[ServiceDescriptor.authorization_service_descriptor('http://localhost:12001')])
service_descriptors.append(service_descriptor)
service_type = service_descriptor[0]
service_descriptors = [metadata_service_desc] + service_descriptors
services = ServiceFactory.build_services(service_descriptors)
checksums = dict()
for service in services:
checksums[str(service.index)] = checksum(service.main)
# Adding proof to the ddo.
ddo.add_proof(checksums, account)
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
ddo_service_endpoint.replace('{did}', did)
services[0].set_service_endpoint(ddo_service_endpoint)
stype_to_service = {s.type: s for s in services}
_service = stype_to_service[service_type]
for service in services:
ddo.add_service(service)
# ddo.proof['signatureValue'] = ocean_lib.sign_hash(
# did_to_id_bytes(did), account)
ddo.add_public_key(did, account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
try:
_oldddo = aqua.get_asset_ddo(ddo.did)
if _oldddo:
aqua.retire_asset_ddo(ddo.did)
except ValueError:
pass
if not plecos.is_valid_dict_local(ddo.metadata):
print(f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}')
assert False, f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}'
encrypted_files = do_encrypt(
json.dumps(metadata['main']['files']),
account,
)
# only assign if the encryption worked
if encrypted_files:
index = 0
for file in metadata['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata['encryptedFiles'] = encrypted_files
# ddo._other_values
try:
aqua.publish_asset_ddo(ddo)
except Exception as e:
print(f'error publishing ddo {ddo.did} in Aquarius: {e}')
raise
return ddo