def download(self,
did,
service_index,
owner_account,
destination,
index=None):
"""
Download the asset data.
Using the service endpoint defined in the ddo's service pointed to by service_definition_id.
Consumer's permissions is checked implicitly by the secret-store during decryption
of the contentUrls.
The service endpoint is expected to also verify that `owner_account` is the actual owner
of the asset.
This method downloads and saves the asset datafiles to disk.
:param did: DID, str
:param service_index: identifier of the service inside the asset DDO, str
:param owner_account: Account instance of the owner
:param destination: str path
:param index: Index of the document that is going to be downloaded, int
:return: str path to saved files
"""
ddo = self.resolve(did)
if index is not None:
assert isinstance(index,
int), logger.error('index has to be an integer.')
assert index >= 0, logger.error(
'index has to be 0 or a positive integer.')
return self._asset_consumer.download(
service_index, ddo, owner_account, destination,
GatewayProvider.get_gateway(),
self._get_secret_store(owner_account), self._config, index)
def execute(self, agreement_id, did, index, consumer_account, workflow_did):
"""
:param agreement_id:representation of `bytes32` id, hexstr
:param did: computing service did, str
:param index: id of the service within the asset DDO, str
:param consumer_account: Account instance of the consumer ordering the service
:param workflow_did: the asset did (of `workflow` type) which consist of `did:nv:` and
the assetId hex str (without `0x` prefix), str
:return: the id of the compute job, str
"""
workflow_ddo = self.resolve(workflow_did)
compute_ddo = self.resolve(did)
if index is not None:
assert isinstance(index, int), logger.error('index has to be an integer.')
assert index >= 0, logger.error('index has to be 0 or a positive integer.')
return self._asset_executor.execute(
agreement_id,
compute_ddo,
workflow_ddo,
consumer_account,
GatewayProvider.get_gateway(),
index,
self._config
)
def create_compute(self, metadata, publisher_account, asset_rewards={"_amounts": [], "_receivers": []},
service_descriptors=None, providers=None,
authorization_type=ServiceAuthorizationTypes.PSK_RSA, use_secret_store=False):
"""
Register a compute to the data asset in both the keeper's DIDRegistry (on-chain) and in
the Metadata store.
:param metadata: dict conforming to the Metadata accepted by Nevermined Protocol.
:param publisher_account: Account of the publisher registering this asset
:param asset_rewards: Asset rewards distribution including amounts and receivers
:param service_descriptors: list of ServiceDescriptor tuples of length 2.
The first item must be one of ServiceTypes and the second
item is a dict of parameters and values required by the service
:param providers: list of addresses of providers of this asset (a provider is
an ethereum account that is authorized to provide asset services)
:param authorization_type: str indicate the authorization type that is going to be used
to encrypt the urls.
SecretStore, PSK-RSA and PSK-ECDSA are supported.
:param use_secret_store: bool indicate whether to use the secret store directly for
encrypting urls (Uses Gateway provider service if set to False)
:return: DDO instance
"""
metadata_copy = copy.deepcopy(metadata)
gateway = GatewayProvider.get_gateway()
compute_service_attributes = self._build_compute(metadata_copy, publisher_account, asset_rewards)
service_descriptor = ServiceDescriptor.compute_service_descriptor(
compute_service_attributes, gateway.get_execute_endpoint(self._config))
return self.create(metadata, publisher_account, service_descriptors=[service_descriptor],
providers=providers, authorization_type=authorization_type,
use_secret_store=use_secret_store)
def get_consumer_instance(init_tokens=True,
use_ss_mock=True,
use_gateway_mock=True):
nevermined = Nevermined()
account = get_consumer_account()
nevermined.main_account = account
if init_tokens:
init_ocn_tokens(nevermined, nevermined.main_account)
if use_ss_mock:
SecretStoreProvider.set_secret_store_class(SecretStoreMock)
else:
SecretStoreProvider.set_secret_store_class(SecretStore)
if use_gateway_mock:
GatewayProvider.set_gateway_class(GatewayMock)
else:
Gateway.set_http_client(get_requests_session())
GatewayProvider.set_gateway_class(Gateway)
return nevermined
def upload_filecoin(self, file_path):
"""Upload a file to Filecoin
param file_path: The path to the file to upload, str
return: The url with the cid, str
"""
path = pathlib.Path(file_path)
if not path.is_file():
raise ValueError(f'File {file_path} not found or not a file')
with path.open('rb') as f:
gateway = GatewayProvider.get_gateway()
response = gateway.upload_filecoin(f, self._config)
return response['url']
def compute_status(self, service_agreement_id, execution_id, account):
"""
Get the status of a compute workflow.
:param service_agreement_id: The id of the service agreement that ordered the compute job, str
:param execution_id: The id of the compute job, str
:param account: Account instance that ordered the execution of the compute job
:return: str compute status
"""
return self._asset_consumer.compute_status(
service_agreement_id,
execution_id,
account,
GatewayProvider.get_gateway(),
self._config,
)
def create(self,
metadata,
publisher_account,
service_descriptors=None,
providers=None,
authorization_type=ServiceAuthorizationTypes.PSK_RSA,
use_secret_store=False,
activity_id=None,
attributes=None,
asset_rewards={
"_amounts": [],
"_receivers": []
},
cap=None,
royalties=None):
"""
Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store.
:param metadata: dict conforming to the Metadata accepted by Nevermined Protocol.
:param publisher_account: Account of the publisher registering this asset
:param service_descriptors: list of ServiceDescriptor tuples of length 2.
The first item must be one of ServiceTypes and the second
item is a dict of parameters and values required by the service
:param providers: list of addresses of providers of this asset (a provider is
an ethereum account that is authorized to provide asset services)
:param authorization_type: str indicate the authorization type that is going to be used
to encrypt the urls.
SecretStore, PSK-RSA and PSK-ECDSA are supported.
:param use_secret_store: bool indicate whether to use the secret store directly for
encrypting urls (Uses Gateway provider service if set to False)
:param activity_id: identifier of the activity creating the new entity
:param attributes: attributes associated with the action
:param asset_rewards: rewards distribution including the amounts and the receivers
:param cap: max cap of nfts that can be minted for the asset
:param royalties: royalties in the secondary market going to the original creator
:return: DDO instance
"""
assert isinstance(
metadata,
dict), f'Expected metadata of type dict, got {type(metadata)}'
# copy metadata so we don't change the original
metadata_copy = copy.deepcopy(metadata)
# Create a DDO object
ddo = DDO()
gateway = GatewayProvider.get_gateway()
ddo_service_endpoint = self._get_metadata_provider(
).get_service_endpoint()
metadata_service_desc = ServiceDescriptor.metadata_service_descriptor(
metadata_copy, ddo_service_endpoint)
if metadata_copy['main']['type'] == 'dataset' or metadata_copy['main'][
'type'] == 'algorithm':
access_service_attributes = self._build_access(
metadata_copy, publisher_account, asset_rewards)
if not service_descriptors:
if authorization_type == ServiceAuthorizationTypes.PSK_RSA:
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(
authorization_type,
public_key=gateway.get_rsa_public_key(
self._config)),
gateway.get_access_endpoint(self._config))
]
elif authorization_type == ServiceAuthorizationTypes.PSK_ECDSA:
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(
authorization_type,
public_key=gateway.get_ecdsa_public_key(
self._config)),
gateway.get_access_endpoint(self._config))
]
else:
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(authorization_type,
threshold=0),
self._config.secret_store_url)
]
service_descriptors += [
ServiceDescriptor.access_service_descriptor(
access_service_attributes,
gateway.get_access_endpoint(self._config))
]
else:
service_types = set(map(lambda x: x[0], service_descriptors))
if ServiceTypes.AUTHORIZATION not in service_types:
if authorization_type == ServiceAuthorizationTypes.PSK_RSA:
service_descriptors += [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(
authorization_type,
public_key=gateway.get_rsa_public_key(
self._config)),
gateway.get_access_endpoint(self._config))
]
elif authorization_type == ServiceAuthorizationTypes.PSK_ECDSA:
service_descriptors += [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(
authorization_type,
public_key=gateway.get_ecdsa_public_key(
self._config)),
gateway.get_access_endpoint(self._config))
]
else:
service_descriptors += [
ServiceDescriptor.authorization_service_descriptor(
self._build_authorization(authorization_type,
threshold=0),
self._config.secret_store_url)
]
else:
service_descriptors += [
ServiceDescriptor.access_service_descriptor(
access_service_attributes,
gateway.get_access_endpoint(self._config))
]
else:
if not service_descriptors:
service_descriptors = []
else:
service_descriptors += []
logger.info('registering a workflow.')
# Add all services to ddo
service_descriptors = [metadata_service_desc] + service_descriptors
services = ServiceFactory.build_services(service_descriptors)
checksums = dict()
for service in services:
checksums[str(service.index)] = checksum(service.main)
# Adding proof to the ddo.
ddo.add_proof(checksums, publisher_account)
# Generating the did and adding to the ddo.
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
logger.debug(f'Generating new did: {did}')
# Check if it's already registered first!
if did in self._get_metadata_provider().list_assets():
raise DIDAlreadyExist(
f'Asset id {did} is already registered to another asset.')
for service in services:
if service.type == ServiceTypes.ASSET_ACCESS:
access_service = ServiceFactory.complete_access_service(
did, gateway.get_access_endpoint(self._config),
access_service_attributes,
self._keeper.access_template.address,
self._keeper.escrow_payment_condition.address)
ddo.add_service(access_service)
elif service.type == ServiceTypes.METADATA:
ddo_service_endpoint = service.service_endpoint.replace(
'{did}', did)
service.set_service_endpoint(ddo_service_endpoint)
ddo.add_service(service)
elif service.type == ServiceTypes.CLOUD_COMPUTE:
compute_service = ServiceFactory.complete_compute_service(
did, service.service_endpoint, service.attributes,
self._keeper.compute_execution_condition.address,
self._keeper.escrow_payment_condition.address)
ddo.add_service(compute_service)
else:
ddo.add_service(service)
ddo.proof['signatureValue'] = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)),
publisher_account)
# Add public key and authentication
ddo.add_public_key(did, publisher_account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
# Setup metadata service
# First compute files_encrypted
if metadata_copy['main']['type'] in ['dataset', 'algorithm']:
assert metadata_copy['main'][
'files'], 'files is required in the metadata main attributes.'
logger.debug('Encrypting content urls in the metadata.')
if not use_secret_store:
encrypt_endpoint = gateway.get_encrypt_endpoint(self._config)
files_encrypted = gateway.encrypt_files_dict(
metadata_copy['main']['files'], encrypt_endpoint,
ddo.asset_id, authorization_type)
else:
files_encrypted = self._get_secret_store(publisher_account) \
.encrypt_document(
did_to_id(did),
json.dumps(metadata_copy['main']['files']),
)
# only assign if the encryption worked
if files_encrypted:
logger.debug(
f'Content urls encrypted successfully {files_encrypted}')
index = 0
for file in metadata_copy['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata_copy['encryptedFiles'] = files_encrypted
else:
raise AssertionError('Encrypting the files failed.')
# DDO url and `Metadata` service
logger.debug(f'Generated ddo and services, DID is {ddo.did},'
f' metadata service @{ddo_service_endpoint}.')
response = None
# register on-chain
registered_on_chain = self._keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=publisher_account,
cap=cap,
royalties=royalties,
providers=providers,
activity_id=activity_id,
attributes=attributes)
if registered_on_chain is False:
logger.warning(f'Registering {did} on-chain failed.')
return None
logger.info(f'Successfully registered DDO (DID={did}) on chain.')
try:
# publish the new ddo
response = self._get_metadata_provider().publish_asset_ddo(ddo)
logger.info('Asset/ddo published successfully in Metadata.')
except ValueError as ve:
raise ValueError(
f'Invalid value to publish in the metadata: {str(ve)}')
except Exception as e:
logger.error(f'Publish asset in Metadata failed: {str(e)}')
if not response:
return None
return ddo
