def get_consumer_ocean_instance(init_tokens=True, use_ss_mock=True, use_brizo_mock=False):
ocn = Ocean()
account = get_consumer_account()
ocn.main_account = account
if init_tokens:
init_ocn_tokens(ocn, ocn.main_account)
if use_ss_mock:
SecretStoreProvider.set_secret_store_class(SecretStoreMock)
if use_brizo_mock:
BrizoProvider.set_brizo_class(BrizoMock)
return ocn
def test_initialize_and_consume(client, publisher_ocean_instance, consumer_ocean_instance):
print(publisher_ocean_instance.accounts)
pub_ocn, cons_ocn = publisher_ocean_instance, consumer_ocean_instance
consumer_account = cons_ocn.main_account
publisher_account = pub_ocn.main_account
asset_price = 10
# Register asset
ddo = get_registered_ddo(pub_ocn, publisher_account)
print("did: %s" % ddo.did)
service_definition_id = \
ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS).get_values()[
'serviceDefinitionId']
agreement_id = ServiceAgreement.create_new_agreement_id()
service_def = ddo.find_service_by_id(service_definition_id).as_dictionary()
service_agreement = ServiceAgreement.from_ddo(service_definition_id, ddo)
service_agreement.validate_conditions()
agreement_hash = service_agreement.get_service_agreement_hash(agreement_id)
cons_ocn.main_account.unlock()
signature = consumer_account.sign_hash(agreement_hash)
sa = service_agreement
cons_ocn.main_account.unlock()
cons_ocn._approve_token_transfer(service_agreement.get_price(), consumer_account)
cons_ocn._http_client = client
# subscribe to events
register_service_agreement(cons_ocn.config.storage_path,
cons_ocn.main_account,
agreement_id,
ddo.did,
service_def,
'consumer',
service_definition_id,
service_agreement.get_price(),
get_metadata_url(ddo),
cons_ocn.consume_service,
0)
request_payload = BrizoProvider.get_brizo().prepare_purchase_payload(ddo.did, agreement_id,
service_definition_id,
signature,
consumer_account.address)
initialize = client.post(
sa.purchase_endpoint,
data=request_payload,
content_type='application/json'
)
print(initialize.status_code)
assert initialize.status_code == 201
assert pub_ocn.keeper.service_agreement.get_agreement_status(agreement_id) is False, ''
# wait a bit until all service agreement events are processed
time.sleep(15)
assert pub_ocn.keeper.service_agreement.get_agreement_status(agreement_id) is True, ''
print('Service agreement executed and fulfilled, all good.')
def _process_service_descriptors(self, service_descriptors, metadata,
account):
brizo = BrizoProvider.get_brizo()
ddo_service_endpoint = self._get_aquarius().get_service_endpoint()
service_type_to_descriptor = {sd[0]: sd for sd in service_descriptors}
_service_descriptors = []
metadata_service_desc = service_type_to_descriptor.pop(
ServiceTypes.METADATA,
ServiceDescriptor.metadata_service_descriptor(
metadata, ddo_service_endpoint))
auth_service_desc = service_type_to_descriptor.pop(
ServiceTypes.AUTHORIZATION,
ServiceDescriptor.authorization_service_descriptor(
self._config.secret_store_url))
_service_descriptors = [metadata_service_desc, auth_service_desc]
# Always dafault to creating a ServiceTypes.ASSET_ACCESS service if no services are specified
access_service_descriptor = service_type_to_descriptor.pop(
ServiceTypes.ASSET_ACCESS,
ServiceDescriptor.access_service_descriptor(
self._build_access_service(metadata, account),
brizo.get_consume_endpoint(self._config),
self._keeper.escrow_access_secretstore_template.address))
compute_service_descriptor = service_type_to_descriptor.pop(
ServiceTypes.CLOUD_COMPUTE, None)
if access_service_descriptor:
_service_descriptors.append(access_service_descriptor)
if compute_service_descriptor:
_service_descriptors.append(compute_service_descriptor)
_service_descriptors.extend(service_type_to_descriptor.values())
return ServiceFactory.build_services(_service_descriptors)
def consume(self,
service_agreement_id,
did,
service_index,
consumer_account,
destination,
index=None):
"""
Consume the asset data.
Using the service endpoint defined in the ddo's service pointed to by service_definition_id.
Consumer's permissions is checked implicitly by the secret-store during decryption
of the contentUrls.
The service endpoint is expected to also verify the consumer's permissions to consume this
asset.
This method downloads and saves the asset datafiles to disk.
:param service_agreement_id: str
:param did: DID, str
:param service_index: identifier of the service inside the asset DDO, str
:param consumer_account: Account instance of the consumer
:param destination: str path
:param index: Index of the document that is going to be downloaded, int
:return: str path to saved files
"""
ddo = self.resolve(did)
if index is not None:
assert isinstance(index,
int), logger.error('index has to be an integer.')
assert index >= 0, logger.error(
'index has to be 0 or a positive integer.')
return self._asset_consumer.download(
service_agreement_id, service_index, ddo, consumer_account,
destination, BrizoProvider.get_brizo(),
self._get_secret_store(consumer_account), index)
def get_consumer_ocean_instance(init_tokens=True,
use_ss_mock=True,
use_brizo_mock=True):
ocn = make_ocean_instance(CONSUMER_INDEX)
account = get_consumer_account(ConfigProvider.get_config())
if account.address in ocn.accounts.accounts_addresses:
ocn.main_account = account
else:
ocn.main_account = ocn.accounts.list()[1]
if init_tokens:
init_ocn_tokens(ocn, ocn.main_account)
if use_ss_mock:
SecretStoreProvider.set_secret_store_class(SecretStoreMock)
if use_brizo_mock:
BrizoProvider.set_brizo_class(BrizoMock)
return ocn
def send(self,
did,
agreement_id,
service_index,
signature,
consumer_account,
auto_consume=False):
"""
Send a signed service agreement to the publisher Brizo instance to
consume/access the service.
:param did: str representation fo the asset DID. Use this to retrieve the asset DDO.
:param agreement_id: 32 bytes identifier created by the consumer and will be used
on-chain for the executed agreement.
:param service_index: int identifies the specific service in
the ddo to use in this agreement.
:param signature: str the signed agreement message hash which includes
conditions and their parameters values and other details of the agreement.
:param consumer_account: Account instance of the consumer
:param auto_consume: boolean tells this function wether to automatically trigger
consuming the asset upon receiving access permission
:raises OceanInitializeServiceAgreementError: on failure
:return: bool
"""
asset = self._asset_resolver.resolve(did)
service_agreement = asset.get_service_by_index(service_index)
# subscribe to events related to this agreement_id before sending the request.
logger.debug(
f'Registering service agreement with id: {agreement_id}, auto-consume {auto_consume}'
)
# TODO: refactor this to use same code in `create`
publisher_address = self._keeper.did_registry.get_did_owner(
asset.asset_id)
condition_ids = service_agreement.generate_agreement_condition_ids(
agreement_id, asset.asset_id, consumer_account.address,
publisher_address, self._keeper)
from_block = Web3Provider.get_web3().eth.blockNumber
self._process_consumer_agreement_events(
agreement_id, did, service_agreement, consumer_account,
condition_ids, publisher_address, from_block, auto_consume)
return BrizoProvider.get_brizo().initialize_service_agreement(
did, agreement_id, service_index, signature,
consumer_account.address, service_agreement.endpoints.purchase)
def execute(self, agreement_id, did, index, consumer_account,
workflow_did):
"""
:param agreement_id:representation of `bytes32` id, hexstr
:param did: computing service did, str
:param index: id of the service within the asset DDO, str
:param consumer_account: Account instance of the consumer ordering the service
:param workflow_did: the asset did (of `workflow` type) which consist of `did:op:` and
the assetId hex str (without `0x` prefix), str
:return: local path on the file system where the asset data files are downloaded, str
"""
workflow_ddo = self.resolve(workflow_did)
compute_ddo = self.resolve(did)
if index is not None:
assert isinstance(index,
int), logger.error('index has to be an integer.')
assert index >= 0, logger.error(
'index has to be 0 or a positive integer.')
return self._asset_executor.execute(agreement_id, compute_ddo,
workflow_ddo, consumer_account,
BrizoProvider.get_brizo(), index)
def consume(self, service_agreement_id, did, service_definition_id,
consumer_account, destination):
"""
Consume the asset data.
Using the service endpoint defined in the ddo's service pointed to by service_definition_id.
Consumer's permissions is checked implicitly by the secret-store during decryption
of the contentUrls.
The service endpoint is expected to also verify the consumer's permissions to consume this
asset.
This method downloads and saves the asset datafiles to disk.
:param service_agreement_id: str
:param did: DID, str
:param service_definition_id: identifier of the service inside the asset DDO, str
:param consumer_account: Account instance of the consumer
:param destination: str path
:return: str path to saved files
"""
ddo = self.resolve(did)
return self._asset_consumer.download(
service_agreement_id, service_definition_id, ddo, consumer_account,
destination, BrizoProvider.get_brizo(),
self._get_secret_store(consumer_account))
def create(self,
metadata,
publisher_account,
service_descriptors=None,
providers=None,
use_secret_store=True):
"""
Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store (
Aquarius).
:param metadata: dict conforming to the Metadata accepted by Ocean Protocol.
:param publisher_account: Account of the publisher registering this asset
:param service_descriptors: list of ServiceDescriptor tuples of length 2.
The first item must be one of ServiceTypes and the second
item is a dict of parameters and values required by the service
:param providers: list of addresses of providers of this asset (a provider is
an ethereum account that is authorized to provide asset services)
:param use_secret_store: bool indicate whether to use the secret store directly for
encrypting urls (Uses Brizo provider service if set to False)
:return: DDO instance
"""
assert isinstance(
metadata,
dict), f'Expected metadata of type dict, got {type(metadata)}'
assert service_descriptors is None or isinstance(service_descriptors, list), \
f'bad type of `service_descriptors` {type(service_descriptors)}'
# if not metadata or not Metadata.validate(metadata):
# raise OceanInvalidMetadata('Metadata seems invalid. Please make sure'
# ' the required metadata values are filled in.')
# copy metadata so we don't change the original
metadata_copy = copy.deepcopy(metadata)
asset_type = metadata_copy['main']['type']
assert asset_type in (
'dataset',
'algorithm'), f'Invalid/unsupported asset type {asset_type}'
service_descriptors = service_descriptors or []
brizo = BrizoProvider.get_brizo()
services = self._process_service_descriptors(service_descriptors,
metadata_copy,
publisher_account)
stype_to_service = {s.type: s for s in services}
checksum_dict = dict()
for service in services:
checksum_dict[str(service.index)] = checksum(service.main)
# Create a DDO object
ddo = DDO()
# Adding proof to the ddo.
ddo.add_proof(checksum_dict, publisher_account)
# Generating the did and adding to the ddo.
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
logger.debug(f'Generating new did: {did}')
# Check if it's already registered first!
if did in self._get_aquarius().list_assets():
raise OceanDIDAlreadyExist(
f'Asset id {did} is already registered to another asset.')
md_service = stype_to_service[ServiceTypes.METADATA]
ddo_service_endpoint = md_service.service_endpoint
if '{did}' in ddo_service_endpoint:
ddo_service_endpoint = ddo_service_endpoint.replace('{did}', did)
md_service.set_service_endpoint(ddo_service_endpoint)
# Populate the ddo services
ddo.add_service(md_service)
ddo.add_service(stype_to_service[ServiceTypes.AUTHORIZATION])
access_service = stype_to_service.get(ServiceTypes.ASSET_ACCESS, None)
compute_service = stype_to_service.get(ServiceTypes.CLOUD_COMPUTE,
None)
if access_service:
access_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(access_service)
if compute_service:
compute_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(compute_service)
ddo.proof['signatureValue'] = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)),
publisher_account)
# Add public key and authentication
ddo.add_public_key(did, publisher_account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
# Setup metadata service
# First compute files_encrypted
if metadata_copy['main']['type'] == 'dataset':
assert metadata_copy['main']['files'], \
'files is required in the metadata main attributes.'
logger.debug('Encrypting content urls in the metadata.')
if not use_secret_store:
encrypt_endpoint = brizo.get_encrypt_endpoint(self._config)
files_encrypted = brizo.encrypt_files_dict(
metadata_copy['main']['files'], encrypt_endpoint,
ddo.asset_id, publisher_account.address,
self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(ddo.asset_id),
publisher_account))
else:
files_encrypted = self._get_secret_store(publisher_account) \
.encrypt_document(
did_to_id(did),
json.dumps(metadata_copy['main']['files']),
)
# only assign if the encryption worked
if files_encrypted:
logger.debug(
f'Content urls encrypted successfully {files_encrypted}')
index = 0
for file in metadata_copy['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata_copy['encryptedFiles'] = files_encrypted
else:
raise AssertionError('Encrypting the files failed.')
# DDO url and `Metadata` service
logger.debug(f'Generated ddo and services, DID is {ddo.did},'
f' metadata service @{ddo_service_endpoint}.')
response = None
# register on-chain
registered_on_chain = self._keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=publisher_account,
providers=providers)
if registered_on_chain is False:
logger.warning(f'Registering {did} on-chain failed.')
return None
logger.info(f'Successfully registered DDO (DID={did}) on chain.')
try:
# publish the new ddo in ocean-db/Aquarius
response = self._get_aquarius().publish_asset_ddo(ddo)
logger.info('Asset/ddo published successfully in aquarius.')
except ValueError as ve:
raise ValueError(
f'Invalid value to publish in the metadata: {str(ve)}')
except Exception as e:
logger.error(f'Publish asset in aquarius failed: {str(e)}')
if not response:
return None
return ddo
def create(self,
metadata,
publisher_account,
service_descriptors=None,
providers=None,
use_secret_store=True):
"""
Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store (
Aquarius).
:param metadata: dict conforming to the Metadata accepted by Ocean Protocol.
:param publisher_account: Account of the publisher registering this asset
:param service_descriptors: list of ServiceDescriptor tuples of length 2.
The first item must be one of ServiceTypes and the second
item is a dict of parameters and values required by the service
:param providers: list of addresses of providers of this asset (a provider is
an ethereum account that is authorized to provide asset services)
:return: DDO instance
"""
assert isinstance(
metadata,
dict), f'Expected metadata of type dict, got {type(metadata)}'
if not metadata or not Metadata.validate(metadata):
raise OceanInvalidMetadata(
'Metadata seems invalid. Please make sure'
' the required metadata values are filled in.')
# copy metadata so we don't change the original
metadata_copy = copy.deepcopy(metadata)
# Create a DDO object
did = DID.did()
logger.debug(f'Generating new did: {did}')
# Check if it's already registered first!
if did in self._get_aquarius().list_assets():
raise OceanDIDAlreadyExist(
f'Asset id {did} is already registered to another asset.')
ddo = DDO(did)
# Add public key and authentication
ddo.add_public_key(did, publisher_account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
priv_key = ddo.get_private_key()
# Setup metadata service
# First replace `files` with encrypted `files`
assert metadata_copy['base'][
'files'], 'files is required in the metadata base attributes.'
assert Metadata.validate(metadata), 'metadata seems invalid.'
logger.debug('Encrypting content urls in the metadata.')
brizo = BrizoProvider.get_brizo()
if not use_secret_store:
encrypt_endpoint = brizo.get_encrypt_endpoint(self._config)
files_encrypted = brizo.encrypt_files_dict(
metadata_copy['base']['files'], encrypt_endpoint, ddo.asset_id,
publisher_account.address,
self._keeper.sign_hash(ddo.asset_id, publisher_account))
else:
files_encrypted = self._get_secret_store(publisher_account) \
.encrypt_document(
did_to_id(did),
json.dumps(metadata_copy['base']['files']),
)
metadata_copy['base']['checksum'] = ddo.generate_checksum(
did, metadata)
ddo.add_proof(metadata_copy['base']['checksum'],
publisher_account.address,
private_key=priv_key)
# only assign if the encryption worked
if files_encrypted:
logger.info(
f'Content urls encrypted successfully {files_encrypted}')
index = 0
for file in metadata_copy['base']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata_copy['base']['encryptedFiles'] = files_encrypted
else:
raise AssertionError(
'Encrypting the files failed. Make sure the secret store is'
' setup properly in your config file.')
# DDO url and `Metadata` service
ddo_service_endpoint = self._get_aquarius().get_service_endpoint(did)
metadata_service_desc = ServiceDescriptor.metadata_service_descriptor(
metadata_copy, ddo_service_endpoint)
if not service_descriptors:
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
self._config.secret_store_url)
]
brizo = BrizoProvider.get_brizo()
service_descriptors += [
ServiceDescriptor.access_service_descriptor(
metadata[MetadataBase.KEY]['price'],
brizo.get_purchase_endpoint(self._config),
brizo.get_service_endpoint(self._config), 3600,
self._keeper.escrow_access_secretstore_template.address)
]
else:
service_types = set(map(lambda x: x[0], service_descriptors))
if ServiceTypes.AUTHORIZATION not in service_types:
service_descriptors += [
ServiceDescriptor.authorization_service_descriptor(
self._config.secret_store_url)
]
else:
brizo = BrizoProvider.get_brizo()
service_descriptors += [
ServiceDescriptor.access_service_descriptor(
metadata[MetadataBase.KEY]['price'],
brizo.get_purchase_endpoint(self._config),
brizo.get_service_endpoint(self._config), 3600, self.
_keeper.escrow_access_secretstore_template.address)
]
# Add all services to ddo
service_descriptors = service_descriptors + [metadata_service_desc]
for service in ServiceFactory.build_services(did, service_descriptors):
ddo.add_service(service)
logger.debug(
f'Generated ddo and services, DID is {ddo.did},'
f' metadata service @{ddo_service_endpoint}, '
f'`Access` service purchase @{ddo.services[0].endpoints.service}.')
response = None
try:
# publish the new ddo in ocean-db/Aquarius
response = self._get_aquarius().publish_asset_ddo(ddo)
logger.debug('Asset/ddo published successfully in aquarius.')
except ValueError as ve:
raise ValueError(
f'Invalid value to publish in the metadata: {str(ve)}')
except Exception as e:
logger.error(f'Publish asset in aquarius failed: {str(e)}')
if not response:
return None
# register on-chain
self._keeper.did_registry.register(
did,
checksum=Web3Provider.get_web3().sha3(
text=metadata_copy['base']['checksum']),
url=ddo_service_endpoint,
account=publisher_account,
providers=providers)
logger.info(f'DDO with DID {did} successfully registered on chain.')
return ddo