def test_encryption_content(client):
content = [
'https://github.com/nevermined-io/321321321321321321321321321321321',
'https://github.com/nevermined-io/dsadsadsadsadasdasdasdasdasdasdasdas',
'https://github.com/nevermined-io/321321321321321321321321321321321',
'https://github.com/nevermined-io/dsadsadsadsadasdasdasdasdasdasdasdas',
'https://github.com/nevermined-io/321321321321321321321321321321321',
'https://github.com/nevermined-io/dsadsadsadsadasdasdasdasdasdasdasdas',
'https://github.com/nevermined-io/321321321321321321321321321321321',
'https://github.com/nevermined-io/dsadsadsadsadasdasdasdasdasdasdasdas',
'https://github.com/nevermined-io/321321321321321321321321321321321',
'https://github.com/nevermined-io/dsadsadsadsadasdasdasdasdasdasdasdas',
'https://github.com/nevermined-io/h65h5h6hrh6rhrh6rhrhrhrh6rhr6hr66h'
]
message = json.dumps(content)
print(message)
did = DID.did(str(uuid.uuid4()))
for method in constants.ConfigSections.DECRYPTION_METHODS:
print('Testing encrypt: ' + method)
payload = {'message': message, 'method': method, 'did': did}
post_response = client.post(BaseURLs.ASSETS_URL + '/encrypt',
data=json.dumps(payload),
content_type='application/json')
assert post_response.status_code == 200
result = json.loads(post_response.data.decode('utf-8'))
assert len(result['hash']) > 1
assert len(result['public-key']) > 1
def asset1():
asset = _get_asset(
'https://raw.githubusercontent.com/nevermined-io/docs/master/docs/architecture/specs'
'/examples/access/v0.1/ddo1.json')
asset._did = DID.encoded_did(asset.proof['checksum'])
yield asset
metadata_provider.retire_all_assets()
def test_creating_ddo_from_scratch():
# create an empty ddo
ddo = DDO()
assert ddo.did is None
assert ddo.asset_id is None
assert ddo.created is not None
did = DID.did("0x99999999999999999")
ddo.assign_did(did)
assert ddo.did == did
pub_acc = get_publisher_account()
ddo.add_service(TEST_SERVICE_TYPE, TEST_SERVICE_URL)
# add a proof to the first public_key/authentication
ddo.add_proof('checksum', pub_acc)
ddo_text_proof = ddo.as_text()
assert ddo_text_proof
pub_acc = get_publisher_account()
assert not ddo.public_keys
ddo.add_public_key(did, pub_acc.address)
assert len(ddo.public_keys) == 1
assert ddo.get_public_key(0) == ddo.public_keys[0]
with pytest.raises(IndexError):
ddo.get_public_key(1)
assert ddo.get_public_key(did) == ddo.public_keys[0]
assert ddo.get_public_key('0x32233') is None
assert not ddo.authentications
ddo.add_authentication(did, '')
assert len(ddo.authentications) == 1
def test_resolve_did(publisher_instance, metadata):
# prep ddo
# metadata = Metadata.get_example()
publisher = publisher_instance.main_account
# happy path
original_ddo = publisher_instance.assets.create(metadata, publisher, activity_id=publisher.address, attributes='test123')
did = original_ddo.did
ddo = publisher_instance.assets.resolve(did).as_dictionary()
original = original_ddo.as_dictionary()
assert ddo['publicKey'] == original['publicKey']
assert ddo['authentication'] == original['authentication']
assert ddo['service']
assert original['service']
metadata = ddo['service'][0]['attributes']
if 'datePublished' in metadata['main']:
metadata['main'].pop('datePublished')
assert ddo['service'][0]['attributes']['main']['name'] == \
original['service'][0]['attributes']['main']['name']
assert ddo['service'][1] == original['service'][1]
# Can't resolve unregistered asset
unregistered_did = DID.did({"0": "0x00112233445566"})
with pytest.raises(DIDNotFound):
publisher_instance.assets.resolve(unregistered_did)
# Raise error on bad did
invalid_did = "did:nv:0123456789"
with pytest.raises(DIDNotFound):
publisher_instance.assets.resolve(invalid_did)
publisher_instance.assets.retire(did)
def test_did_resolver_library(publisher_account, metadata_instance,
ddo_sample_2):
did_registry = keeper().did_registry
checksum_test = Web3.keccak(text='checksum')
value_test = metadata_instance.root_url
did_resolver = DIDResolver(keeper().did_registry)
did_seed = generate_prefixed_id()
asset_id = did_registry.hash_did(did_seed, publisher_account.address)
asset1 = ddo_sample_2
asset1._did = DID.did(asset_id)
did_registry.register(did_seed,
checksum_test,
url=value_test,
account=publisher_account)
metadata_instance.publish_asset_ddo(asset1)
did_resolved = did_resolver.resolve(asset1.did)
assert did_resolved
assert did_resolved.did == asset1.did
with pytest.raises(ValueError):
did_resolver.resolve(asset1.asset_id)
metadata_instance.retire_asset_ddo(asset1.did)
def test_did_to_id():
did = DID.did("0x123")
_id = did_to_id(did)
assert _id is not None and len(_id) == 64, ''
test_id = '%s' % secrets.token_hex(32)
assert did_to_id(f'{NEVERMINED_PREFIX}{test_id}') == test_id
assert did_to_id('did:nv1:011') == '011'
assert did_to_id('did:nv:0') == '0'
with pytest.raises(ValueError):
did_to_id(NEVERMINED_PREFIX)
assert did_to_id(f'{NEVERMINED_PREFIX}AB*&$#') == 'AB', ''
def test_create_did():
proof = {
"type": "DDOIntegritySignature",
"created": "2016-02-08T16:02:20Z",
"creator": "0x00Bd138aBD70e2F00903268F3Db08f2D25677C9e",
"signatureValue": "0xc9eeb2b8106e…6abfdc5d1192641b",
"checksum": {
"0":
"0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377",
"1":
"0x999999952b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3"
}
}
did = DID.encoded_did(proof['checksum'])
assert did == 'did:nv:138fccf336883ae6312c9b8b375745a90be369454080e90985fb3e314ab0df25'
def test_create_ddo(metadata):
pub_acc = get_publisher_account()
ddo = DDO()
ddo.add_service(ServiceTypes.METADATA,
'http://mymetadata.com',
values=dict({"attributes": metadata}),
index='0')
checksums = dict()
for service in ddo.services:
checksums[str(service.index)] = checksum(service.main)
ddo.add_proof(checksums, pub_acc)
did = ddo.assign_did(DID.encoded_did(ddo.proof['checksum']))
ddo.proof['signatureValue'] = Keeper.sign_hash(did_to_id_bytes(did),
pub_acc)
ddo.add_public_key(did, pub_acc.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
def test_get_resolve_url(metadata_instance, publisher_account):
register_account = publisher_account
did_registry = keeper().did_registry
did_seed = generate_prefixed_id()
asset_id = did_registry.hash_did(did_seed, register_account.address)
did = DID.did(asset_id)
value_test = metadata_instance.root_url
did_resolver = DIDResolver(keeper().did_registry)
did_registry.register(did_seed,
b'test',
url=value_test,
account=register_account)
url = did_resolver.get_resolve_url(Web3.toBytes(hexstr=asset_id))
assert url == value_test
def setup_nft_sales_agreements_environment():
consumer_acc = get_consumer_account()
publisher_acc = get_publisher_account()
keeper = Keeper.get_instance()
ddo = get_ddo_nft_sample()
did_seed = generate_prefixed_id()
asset_id = keeper.did_registry.hash_did(did_seed, publisher_acc.address)
ddo._did = DID.did(asset_id)
keeper.did_registry.register_mintable_did(
did_seed,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url='http://172.17.0.1:5000',
cap=10,
royalties=10,
account=publisher_acc,
providers=None)
keeper.did_registry.mint(ddo.asset_id, 10, account=publisher_acc)
service_agreement = ServiceAgreement.from_ddo(ServiceTypes.NFT_SALES, ddo)
agreement_id = ServiceAgreement.create_new_agreement_id()
price = service_agreement.get_price()
(access_cond_id, lock_cond_id,
escrow_cond_id) = service_agreement.generate_agreement_condition_ids(
agreement_id, asset_id, consumer_acc.address, keeper)
nft_access_service_agreement = ServiceAgreement.from_ddo(
ServiceTypes.NFT_ACCESS, ddo)
nft_access_agreement_id = ServiceAgreement.create_new_agreement_id()
(nft_access_cond_id, nft_holder_cond_id
) = nft_access_service_agreement.generate_agreement_condition_ids(
nft_access_agreement_id, asset_id, consumer_acc.address, keeper)
return (keeper, ddo, publisher_acc, consumer_acc, agreement_id,
nft_access_agreement_id, asset_id, price, service_agreement,
nft_access_service_agreement, (lock_cond_id, access_cond_id,
escrow_cond_id),
(nft_access_cond_id, nft_holder_cond_id))
def setup_did_sales_agreements_environment():
consumer_acc = get_consumer_account()
publisher_acc = get_publisher_account()
keeper = Keeper.get_instance()
ddo = get_ddo_did_sales_sample()
did_seed = generate_prefixed_id()
asset_id = keeper.did_registry.hash_did(did_seed, publisher_acc.address)
ddo._did = DID.did(asset_id)
keeper.did_registry.register(
did_seed,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url='metadata:5000',
account=publisher_acc,
providers=None)
service_agreement = ServiceAgreement.from_ddo(ServiceTypes.DID_SALES, ddo)
agreement_id = ServiceAgreement.create_new_agreement_id()
price = service_agreement.get_price()
(access_cond_id, lock_cond_id,
escrow_cond_id) = service_agreement.generate_agreement_condition_ids(
agreement_id, asset_id, consumer_acc.address, keeper)
return (
keeper,
ddo,
publisher_acc,
consumer_acc,
agreement_id,
asset_id,
price,
service_agreement,
(lock_cond_id, access_cond_id, escrow_cond_id),
)
def setup_agreements_environment(ddo_sample):
consumer_acc = get_consumer_account()
publisher_acc = get_publisher_account()
keeper = Keeper.get_instance()
ddo = ddo_sample
ddo._did = DID.did({"0": generate_prefixed_id()})
keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url='localhost:5000',
account=publisher_acc,
providers=None
)
registered_ddo = ddo
asset_id = registered_ddo.asset_id
service_agreement = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
agreement_id = ServiceAgreement.create_new_agreement_id()
price = service_agreement.get_price()
access_cond_id, lock_cond_id, escrow_cond_id = \
service_agreement.generate_agreement_condition_ids(
agreement_id, asset_id, consumer_acc.address, publisher_acc.address, keeper
)
return (
keeper,
publisher_acc,
consumer_acc,
agreement_id,
asset_id,
price,
service_agreement,
(lock_cond_id, access_cond_id, escrow_cond_id),
)
def create(self,
metadata,
publisher_account,
service_descriptors=None,
providers=None,
authorization_type=ServiceAuthorizationTypes.PSK_RSA,
use_secret_store=False,
activity_id=None,
attributes=None,
asset_rewards={
"_amounts": [],
"_receivers": []
},
cap=None,
royalties=None):
"""
Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store.
:param metadata: dict conforming to the Metadata accepted by Nevermined Protocol.
:param publisher_account: Account of the publisher registering this asset
:param service_descriptors: list of ServiceDescriptor tuples of length 2.
The first item must be one of ServiceTypes and the second
item is a dict of parameters and values required by the service
:param providers: list of addresses of providers of this asset (a provider is
an ethereum account that is authorized to provide asset services)
:param authorization_type: str indicate the authorization type that is going to be used
to encrypt the urls.
SecretStore, PSK-RSA and PSK-ECDSA are supported.
:param use_secret_store: bool indicate whether to use the secret store directly for
encrypting urls (Uses Gateway provider service if set to False)
:param activity_id: identifier of the activity creating the new entity
:param attributes: attributes associated with the action
:param asset_rewards: rewards distribution including the amounts and the receivers
:param cap: max cap of nfts that can be minted for the asset
:param royalties: royalties in the secondary market going to the original creator
:return: DDO instance
"""
assert isinstance(
metadata,
dict), f'Expected metadata of type dict, got {type(metadata)}'
# copy metadata so we don't change the original
metadata_copy = copy.deepcopy(metadata)
# Create a DDO object
ddo = DDO()
gateway = GatewayProvider.get_gateway()
ddo_service_endpoint = self._get_metadata_provider(
).get_service_endpoint()
metadata_service_desc = ServiceDescriptor.metadata_service_descriptor(
metadata_copy, ddo_service_endpoint)
if metadata_copy['main']['type'] == 'dataset' or metadata_copy['main'][
'type'] == 'algorithm':
access_service_attributes = self._build_access(
metadata_copy, publisher_account, asset_rewards)
if not service_descriptors:
if authorization_type == ServiceAuthorizationTypes.PSK_RSA:
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(
authorization_type,
public_key=gateway.get_rsa_public_key(
self._config)),
gateway.get_access_endpoint(self._config))
]
elif authorization_type == ServiceAuthorizationTypes.PSK_ECDSA:
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(
authorization_type,
public_key=gateway.get_ecdsa_public_key(
self._config)),
gateway.get_access_endpoint(self._config))
]
else:
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(authorization_type,
threshold=0),
self._config.secret_store_url)
]
service_descriptors += [
ServiceDescriptor.access_service_descriptor(
access_service_attributes,
gateway.get_access_endpoint(self._config))
]
else:
service_types = set(map(lambda x: x[0], service_descriptors))
if ServiceTypes.AUTHORIZATION not in service_types:
if authorization_type == ServiceAuthorizationTypes.PSK_RSA:
service_descriptors += [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(
authorization_type,
public_key=gateway.get_rsa_public_key(
self._config)),
gateway.get_access_endpoint(self._config))
]
elif authorization_type == ServiceAuthorizationTypes.PSK_ECDSA:
service_descriptors += [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(
authorization_type,
public_key=gateway.get_ecdsa_public_key(
self._config)),
gateway.get_access_endpoint(self._config))
]
else:
service_descriptors += [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(authorization_type,
threshold=0),
self._config.secret_store_url)
]
else:
service_descriptors += [
ServiceDescriptor.access_service_descriptor(
access_service_attributes,
gateway.get_access_endpoint(self._config))
]
else:
if not service_descriptors:
service_descriptors = []
else:
service_descriptors += []
logger.info('registering a workflow.')
# Add all services to ddo
service_descriptors = [metadata_service_desc] + service_descriptors
services = ServiceFactory.build_services(service_descriptors)
checksums = dict()
for service in services:
checksums[str(service.index)] = checksum(service.main)
# Adding proof to the ddo.
ddo.add_proof(checksums, publisher_account)
# Generating the did and adding to the ddo.
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
logger.debug(f'Generating new did: {did}')
# Check if it's already registered first!
if did in self._get_metadata_provider().list_assets():
raise DIDAlreadyExist(
f'Asset id {did} is already registered to another asset.')
for service in services:
if service.type == ServiceTypes.ASSET_ACCESS:
access_service = ServiceFactory.complete_access_service(
did, gateway.get_access_endpoint(self._config),
access_service_attributes,
self._keeper.access_template.address,
self._keeper.escrow_payment_condition.address)
ddo.add_service(access_service)
elif service.type == ServiceTypes.METADATA:
ddo_service_endpoint = service.service_endpoint.replace(
'{did}', did)
service.set_service_endpoint(ddo_service_endpoint)
ddo.add_service(service)
elif service.type == ServiceTypes.CLOUD_COMPUTE:
compute_service = ServiceFactory.complete_compute_service(
did, service.service_endpoint, service.attributes,
self._keeper.compute_execution_condition.address,
self._keeper.escrow_payment_condition.address)
ddo.add_service(compute_service)
else:
ddo.add_service(service)
ddo.proof['signatureValue'] = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)),
publisher_account)
# Add public key and authentication
ddo.add_public_key(did, publisher_account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
# Setup metadata service
# First compute files_encrypted
if metadata_copy['main']['type'] in ['dataset', 'algorithm']:
assert metadata_copy['main'][
'files'], 'files is required in the metadata main attributes.'
logger.debug('Encrypting content urls in the metadata.')
if not use_secret_store:
encrypt_endpoint = gateway.get_encrypt_endpoint(self._config)
files_encrypted = gateway.encrypt_files_dict(
metadata_copy['main']['files'], encrypt_endpoint,
ddo.asset_id, authorization_type)
else:
files_encrypted = self._get_secret_store(publisher_account) \
.encrypt_document(
did_to_id(did),
json.dumps(metadata_copy['main']['files']),
)
# only assign if the encryption worked
if files_encrypted:
logger.debug(
f'Content urls encrypted successfully {files_encrypted}')
index = 0
for file in metadata_copy['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata_copy['encryptedFiles'] = files_encrypted
else:
raise AssertionError('Encrypting the files failed.')
# DDO url and `Metadata` service
logger.debug(f'Generated ddo and services, DID is {ddo.did},'
f' metadata service @{ddo_service_endpoint}.')
response = None
# register on-chain
registered_on_chain = self._keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=publisher_account,
cap=cap,
royalties=royalties,
providers=providers,
activity_id=activity_id,
attributes=attributes)
if registered_on_chain is False:
logger.warning(f'Registering {did} on-chain failed.')
return None
logger.info(f'Successfully registered DDO (DID={did}) on chain.')
try:
# publish the new ddo
response = self._get_metadata_provider().publish_asset_ddo(ddo)
logger.info('Asset/ddo published successfully in Metadata.')
except ValueError as ve:
raise ValueError(
f'Invalid value to publish in the metadata: {str(ve)}')
except Exception as e:
logger.error(f'Publish asset in Metadata failed: {str(e)}')
if not response:
return None
return ddo
def test_did():
assert DID.did("0x123").startswith(NEVERMINED_PREFIX)
assert len(DID.did("0x123")) - len(NEVERMINED_PREFIX) == 64
_id = did_to_id(DID.did("0x123"))
assert not _id.startswith(
'0x'), 'id portion of did should not have a 0x prefix.'
def asset2():
asset = _get_asset(
'https://raw.githubusercontent.com/nevermined-io/docs/master/docs/architecture/specs'
'/examples/access/v0.1/ddo2-update.json')
asset._did = DID.encoded_did(asset.proof['checksum'])
return asset
def register_ddo(metadata,
account,
providers,
auth_service,
additional_service_descriptors,
royalties=None,
cap=None,
mint=0):
keeper = keeper_instance()
metadata_api = Metadata('http://172.17.0.1:5000')
ddo = DDO()
ddo_service_endpoint = metadata_api.get_service_endpoint()
metadata_service_desc = ServiceDescriptor.metadata_service_descriptor(
metadata, ddo_service_endpoint)
authorization_service_attributes = {
"main": {
"service": auth_service,
"publicKey": "0xd7"
}
}
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
authorization_service_attributes, 'http://localhost:12001')
]
service_descriptors += [metadata_service_desc]
service_descriptors += additional_service_descriptors
services = ServiceFactory.build_services(service_descriptors)
checksums = dict()
for service in services:
try:
checksums[str(service.index)] = checksum(service.main)
except Exception as e:
pass
# Adding proof to the ddo.
ddo.add_proof(checksums, account)
did_seed = checksum(ddo.proof['checksum'])
asset_id = keeper.did_registry.hash_did(did_seed, account.address)
ddo._did = DID.did(asset_id)
did = ddo._did
for service in services:
if service.type == ServiceTypes.ASSET_ACCESS or service.type == ServiceTypes.NFT_ACCESS or service.type == ServiceTypes.ASSET_ACCESS_PROOF:
access_service = ServiceFactory.complete_access_service(
did, service.service_endpoint, service.attributes,
keeper.access_template.address,
keeper.escrow_payment_condition.address, service.type)
ddo.add_service(access_service)
elif service.type == ServiceTypes.CLOUD_COMPUTE:
compute_service = ServiceFactory.complete_compute_service(
did, service.service_endpoint, service.attributes,
keeper.compute_execution_condition.address,
keeper.escrow_payment_condition.address)
ddo.add_service(compute_service)
elif service.type == ServiceTypes.NFT_SALES:
nft_sales_service = ServiceFactory.complete_nft_sales_service(
did, service.service_endpoint, service.attributes,
keeper.nft_sales_template.address,
keeper.escrow_payment_condition.address, service.type)
ddo.add_service(nft_sales_service)
else:
ddo.add_service(service)
ddo.proof['signatureValue'] = keeper.sign_hash(did_to_id_bytes(did),
account)
ddo.add_public_key(did, account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
try:
_oldddo = metadata_api.get_asset_ddo(ddo.did)
if _oldddo:
metadata_api.retire_asset_ddo(ddo.did)
except ValueError:
pass
if 'files' in metadata['main']:
if auth_service == ServiceAuthorizationTypes.SECRET_STORE:
encrypted_files = do_secret_store_encrypt(
remove_0x_prefix(ddo.asset_id),
json.dumps(metadata['main']['files']), account, get_config())
elif auth_service == ServiceAuthorizationTypes.PSK_RSA:
encrypted_files, public_key = rsa_encryption_from_file(
json.dumps(metadata['main']['files']),
get_rsa_public_key_file())
else:
encrypted_files, public_key = ecdsa_encryption_from_file(
json.dumps(metadata['main']['files']), get_provider_key_file(),
get_provider_password())
_files = metadata['main']['files']
# only assign if the encryption worked
if encrypted_files:
index = 0
for file in metadata['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata['encryptedFiles'] = encrypted_files
ddo_with_did = DDO(did,
json_text=ddo.as_text().replace('/{did}', '/' + did))
ddo_service_endpoint = ddo_service_endpoint.replace('/{did}', '/' + did)
if mint > 0 or royalties is not None or cap is not None:
keeper.did_registry.register_mintable_did(
did_seed,
checksum=web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
cap=cap,
royalties=royalties,
account=account,
providers=providers)
if mint > 0:
keeper.did_registry.mint(ddo.asset_id, mint, account=account)
else:
keeper_instance().did_registry.register(
did_seed,
checksum=web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=account,
providers=providers)
metadata_api.publish_asset_ddo(ddo_with_did)
return ddo_with_did