def is_access_granted(self, agreement_id, did, consumer_address):
"""
Check permission for the agreement.
Verify on-chain that the `consumer_address` has permission to access the given asset `did`
according to the `agreement_id`.
:param agreement_id: id of the agreement, hex str
:param did: DID, str
:param consumer_address: ethereum account address of consumer, hex str
:return: bool True if user has permission
"""
agreement_consumer = self._keeper.escrow_access_secretstore_template.get_agreement_consumer(
agreement_id)
if agreement_consumer is None:
return False
if agreement_consumer != consumer_address:
logger.warning(
f'Invalid consumer address {consumer_address} and/or '
f'service agreement id {agreement_id} (did {did})'
f', agreement consumer is {agreement_consumer}')
return False
document_id = did_to_id(did)
return self._keeper.access_secret_store_condition.check_permissions(
document_id, consumer_address)
def validate_usage(self):
"""Verify that the tokens have been transferred to the provider's wallet."""
tx_id = self.data.get("transferTxId")
token_address = self.asset._other_values["dataToken"]
try:
_tx, _order_log, _transfer_log = validate_order(
self.consumer_address,
token_address,
float(self.service.get_cost()),
tx_id,
add_0x_prefix(did_to_id(self.did))
if self.did.startswith("did:") else self.did,
self.service.index,
)
validate_transfer_not_used_for_other_service(
self.did,
self.service.index,
tx_id,
self.consumer_address,
token_address,
)
record_consume_request(
self.did,
self.service.index,
tx_id,
self.consumer_address,
token_address,
self.service.get_cost(),
)
except Exception:
self.error = f"Order for serviceId {self.service.index} is not valid."
return False
return True
def pay_for_service(
amount: float,
token_address: str,
did: str,
service_id: int,
fee_receiver: str,
from_wallet: Wallet,
consumer: str,
) -> str:
"""
Submits the payment for chosen service in DataTokens.
:param amount:
:param token_address:
:param did:
:param service_id:
:param fee_receiver:
:param from_wallet: Wallet instance
:param consumer: str the address of consumer of the service, defaults to the payer (the `from_wallet` address)
:return: hex str id of transfer transaction
"""
amount_base = to_base_18(amount)
dt = DataToken(token_address)
balance = dt.balanceOf(from_wallet.address)
if balance < amount_base:
raise AssertionError(
f"Your token balance {balance} is not sufficient "
f"to execute the requested service. This service "
f"requires {amount_base} number of tokens.")
if did.startswith("did:"):
did = add_0x_prefix(did_to_id(did))
if fee_receiver is None:
fee_receiver = ZERO_ADDRESS
if consumer is None:
consumer = from_wallet.address
tx_hash = dt.startOrder(consumer, amount_base, service_id,
fee_receiver, from_wallet)
try:
dt.verify_order_tx(
Web3Provider.get_web3(),
tx_hash,
did,
service_id,
amount_base,
from_wallet.address,
)
return tx_hash
except (AssertionError, Exception) as e:
msg = (
f"Downloading asset files failed. The problem is related to "
f"the transfer of the data tokens required for the download "
f"service: {e}")
logger.error(msg)
raise AssertionError(msg)
def owner(self, did):
"""
Return the owner of the asset.
:param did: DID, str
:return: the ethereum address of the owner/publisher of given asset did, hex-str
"""
# return self._get_aquarius(self._aquarius_url).get_asset_ddo(did).proof['creator']
return self._keeper.did_registry.get_did_owner(did_to_id(did))
def _get_num_assets(_minter):
dids = [
add_0x_prefix(did_to_id(a))
for a in ocn.assets.owner_assets(_minter)
]
dids = [a for a in dids if len(a) == 42]
return len([
a for a in dids if DataToken(a).contract_concise.isMinter(_minter)
])
def get_permissions(self, did, address):
"""
Gets access permission of a grantee
:param did: the id of an asset on-chain, hex str
:param address: ethereum account address, hex str
:return: true if the address has access permission to a DID
"""
asset_id = add_0x_prefix(did_to_id(did))
return self._keeper.did_registry.get_permission(asset_id, address)
def revoke_permissions(self, did, address_to_revoke, account):
"""
Revoke access permission to an address.
:param did: the id of an asset on-chain, hex str
:param address_to_revoke: ethereum account address, hex str
:param account: Account executing the action
:return: bool
"""
asset_id = add_0x_prefix(did_to_id(did))
return self._keeper.did_registry.revoke_permission(
asset_id, address_to_revoke, account)
def delegate_persmission(self, did, address_to_grant, account):
"""
Grant access permission to an address.
:param did: the id of an asset on-chain, hex str
:param address_to_grant: ethereum account address, hex str
:param account: Account executing the action
:return: bool
"""
asset_id = add_0x_prefix(did_to_id(did))
return self._keeper.did_registry.grant_permission(
asset_id, address_to_grant, account)
def transfer_ownership(self, did, new_owner_address, account):
"""
Transfer did ownership to an address.
:param did: the id of an asset on-chain, hex str
:param new_owner_address: ethereum account address, hex str
:param account: Account executing the action
:return: bool
"""
asset_id = add_0x_prefix(did_to_id(did))
return self._keeper.did_registry.transfer_did_ownership(
asset_id, new_owner_address, account)
def refund_reward(event, agreement_id, did, service_agreement, price, consumer_account,
publisher_address, condition_ids, escrow_condition_id):
"""
Refund the reward to the publisher address.
:param event: AttributeDict with the event data.
:param agreement_id: id of the agreement, hex str
:param did: DID, str
:param service_agreement: ServiceAgreement instance
:param price: Asset price, int
:param consumer_account: Account instance of the consumer
:param publisher_address: ethereum account address of publisher, hex str
:param condition_ids: is a list of bytes32 content-addressed Condition IDs, bytes32
:param escrow_condition_id: hex str the id of escrow reward condition at this `agreement_id`
"""
logger.debug(f"trigger refund (agreement {agreement_id}) after event {event}.")
if Keeper.get_instance().condition_manager.get_condition_state(escrow_condition_id) > 1:
logger.debug(
f'escrow reward condition already fulfilled/aborted: '
f'agreementId={agreement_id}, escrow reward conditionId={escrow_condition_id},'
f' publisher={publisher_address}'
)
return
access_id, lock_id = condition_ids[:2]
name_to_parameter = {param.name: param for param in
service_agreement.condition_by_name['escrowReward'].parameters}
document_id = add_0x_prefix(name_to_parameter['_documentId'].value)
asset_id = add_0x_prefix(did_to_id(did))
did_owner = Keeper.get_instance().agreement_manager.get_agreement_did_owner(agreement_id)
assert document_id == asset_id, f'document_id {document_id} <=> asset_id {asset_id} mismatch.'
assert price == service_agreement.get_price(), 'price mismatch.'
try:
escrow_condition = Keeper.get_instance().escrow_reward_condition
tx_hash = escrow_condition.fulfill(
agreement_id,
price,
Web3Provider.get_web3().toChecksumAddress(did_owner),
consumer_account.address,
lock_id,
access_id,
consumer_account
)
process_tx_receipt(
tx_hash,
getattr(escrow_condition.contract.events, escrow_condition.FULFILLED_EVENT)(),
'EscrowReward.Fulfilled'
)
except Exception as e:
logger.error(
f'Error when doing escrow_reward_condition.fulfills (agreementId {agreement_id}): {e}',
exc_info=1)
raise e
def test_publish(client):
endpoint = BaseURLs.ASSETS_URL + '/publish'
did = DID.did({"0": str(uuid.uuid4())})
asset_id = did_to_id(did)
account = get_provider_account()
test_urls = ['url 00', 'url 11', 'url 22']
keeper = keeper_instance()
urls_json = json.dumps(test_urls)
asset_id_hash = add_ethereum_prefix_and_hash_msg(asset_id)
signature = keeper.sign_hash(asset_id_hash, account)
address = web3().eth.account.recoverHash(asset_id_hash,
signature=signature)
assert address.lower() == account.address.lower()
address = keeper.personal_ec_recover(asset_id, signature)
assert address.lower() == account.address.lower()
payload = {
'documentId': asset_id,
'signature': signature,
'document': urls_json,
'publisherAddress': account.address
}
post_response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
encrypted_url = post_response.data.decode('utf-8')
assert encrypted_url.startswith('0x')
# publish using auth token
signature = generate_token(account)
payload['signature'] = signature
did = DID.did({"0": str(uuid.uuid4())})
asset_id = did_to_id(did)
payload['documentId'] = add_0x_prefix(asset_id)
post_response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
encrypted_url = post_response.data.decode('utf-8')
assert encrypted_url.startswith('0x')
def test_get_resolve_url(aquarius, publisher_account):
register_account = publisher_account
did_registry = keeper().did_registry
did = DID.did({"0": "0x1"})
asset_id = did_to_id(did)
value_test = aquarius.root_url
did_resolver = DIDResolver(keeper().did_registry)
did_registry.register(asset_id,
b'test',
url=value_test,
account=register_account)
url = did_resolver.get_resolve_url(Web3.toBytes(hexstr=asset_id))
assert url == value_test
def test_did_to_id():
did = DID.did({"0": "0x123"})
_id = did_to_id(did)
assert _id is not None and len(_id) == 64, ''
test_id = '%s' % secrets.token_hex(32)
assert did_to_id(f'{OCEAN_PREFIX}{test_id}') == test_id
assert did_to_id('did:op1:011') == '011'
assert did_to_id('did:op:0') == '0'
with pytest.raises(ValueError):
did_to_id(OCEAN_PREFIX)
assert did_to_id(f'{OCEAN_PREFIX}AB*&$#') == 'AB', ''
def init_conditions_values(self, did, contract_name_to_address):
param_map = {
'_documentId': did_to_id(did),
'_amount': self.attributes['main']['price'],
'_rewardAddress': contract_name_to_address['EscrowReward']
}
conditions = self.conditions[:]
for cond in conditions:
for param in cond.parameters:
param.value = param_map.get(param.name, '')
if cond.timeout > 0:
cond.timeout = self.attributes['main']['timeout']
self.service_agreement_template.set_conditions(conditions)
def grant_access(self, agreement_id, did, grantee_address, account):
"""
Grant access condition.
:param agreement_id: id of the agreement, hex str
:param did: DID, str
:param grantee_address: Address, hex str
:param account: Account
:return:
"""
tx_hash = self._keeper.access_secret_store_condition.fulfill(
agreement_id, add_0x_prefix(did_to_id(did)), grantee_address,
account)
receipt = self._keeper.access_secret_store_condition.get_tx_receipt(
tx_hash)
return bool(receipt and receipt.status == 1)
def grant_compute(self, agreement_id, did, grantee_address, account):
"""
Grant permission to run compute jobs
:param agreement_id: id of the agreement, hex str
:param did: DID, str
:param grantee_address: Address, hex str
:param account: Account
:return:
"""
tx_hash = self._keeper.compute_execution_condition.fulfill(
agreement_id, add_0x_prefix(did_to_id(did)), grantee_address,
account)
receipt = self._keeper.compute_execution_condition.get_tx_receipt(
tx_hash)
return bool(receipt and receipt.status == 1)
def build_stage_algorithm_dict(consumer_address,
algorithm_did,
algorithm_token_address,
algorithm_tx_id,
algorithm_meta,
provider_wallet,
receiver_address=None):
if algorithm_did is not None:
assert algorithm_token_address and algorithm_tx_id, \
'algorithm_did requires both algorithm_token_address and algorithm_tx_id.'
algo_asset = get_asset_from_metadatastore(get_metadata_url(),
algorithm_did)
service = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS,
algo_asset)
_tx, _order_log, _transfer_log = validate_order(
consumer_address, algorithm_token_address,
float(service.get_cost()), algorithm_tx_id,
add_0x_prefix(did_to_id(algorithm_did)) if
algorithm_did.startswith('did:') else algorithm_did, service.index)
validate_transfer_not_used_for_other_service(algorithm_did,
service.index,
algorithm_tx_id,
consumer_address,
algorithm_token_address)
record_consume_request(algorithm_did, service.index, algorithm_tx_id,
consumer_address, algorithm_token_address,
service.get_cost())
algo_id = algorithm_did
raw_code = ''
algo_url = get_asset_url_at_index(0, algo_asset, provider_wallet)
container = algo_asset.metadata['main']['algorithm']['container']
else:
algo_id = ''
algo_url = algorithm_meta.get('url')
raw_code = algorithm_meta.get('rawcode')
container = algorithm_meta.get('container')
return dict({
'id': algo_id,
'url': algo_url,
'rawcode': raw_code,
'container': container
})
def fulfill_access_secret_store_condition(event, agreement_id, did,
service_agreement, consumer_address,
publisher_account,
access_condition_id):
"""
Fulfill the access condition.
:param event: AttributeDict with the event data.
:param agreement_id: id of the agreement, hex str
:param did: DID, str
:param service_agreement: ServiceAgreement instance
:param consumer_address: ethereum account address of consumer, hex str
:param publisher_account: Account instance of the publisher
:param access_condition_id: hex str the id of the access secretstore condition for this
`agreement_id`
"""
if not event:
logger.debug(
f'`fulfill_access_secret_store_condition` got empty event: '
f'event listener timed out.')
return
keeper = Keeper.get_instance()
if keeper.condition_manager.get_condition_state(access_condition_id) > 1:
logger.debug(
f'access secretstore condition already fulfilled/aborted: '
f'agreementId={agreement_id}, access secretstore conditionId={access_condition_id}'
)
return
logger.debug(
f"grant access (agreement {agreement_id}) after event {event}.")
name_to_parameter = {
param.name: param
for param in
service_agreement.condition_by_name['accessSecretStore'].parameters
}
document_id = add_0x_prefix(name_to_parameter['_documentId'].value)
asset_id = add_0x_prefix(did_to_id(did))
assert document_id == asset_id, f'document_id {document_id} <=> asset_id {asset_id} mismatch.'
args = (agreement_id, document_id, consumer_address, publisher_account)
process_fulfill_condition(args, keeper.access_secret_store_condition,
access_condition_id, logger, keeper, 10)
def is_access_granted(agreement_id, did, consumer_address, keeper):
event_logs = _get_agreement_actor_event(keeper,
agreement_id).get_all_entries()
if not event_logs:
return False
actors = [log.args.actor for log in event_logs]
if not actors:
return False
if consumer_address not in actors:
logger.warning(f'Invalid consumer address {consumer_address} and/or '
f'service agreement id {agreement_id} (did {did})'
f', agreement actors are {actors}')
return False
document_id = did_to_id(did)
return keeper.access_secret_store_condition.check_permissions(
document_id, consumer_address)
def validate_agreement_condition(agreement_id, did, consumer_address, keeper):
event_logs = _get_agreement_actor_event(keeper,
agreement_id).get_all_entries()
if not event_logs:
return False
actors = [log.args.actor for log in event_logs]
if not actors:
return False
if consumer_address not in actors:
logger.warning(f'Invalid consumer address {consumer_address} and/or '
f'service agreement id {agreement_id} (did {did})'
f', agreement actors are {actors}')
return False
document_id = did_to_id(did)
return keeper.compute_execution_condition.was_compute_triggered(
document_id, consumer_address)
def pay_for_service(amount: float, token_address: str, did: str,
service_id: int, fee_receiver: str,
from_wallet: Wallet) -> str:
"""
Submits the payment for chosen service in DataTokens.
:param amount:
:param token_address:
:param did:
:param service_id:
:param fee_receiver:
:param from_wallet: Wallet instance
:return: hex str id of transfer transaction
"""
amount_base = to_base_18(amount)
dt = DataToken(token_address)
balance = dt.balanceOf(from_wallet.address)
if balance < amount_base:
raise AssertionError(
f'Your token balance {balance} is not sufficient '
f'to execute the requested service. This service '
f'requires {amount_base} number of tokens.')
if did.startswith('did:'):
did = add_0x_prefix(did_to_id(did))
tx_hash = dt.startOrder(from_wallet.address, amount_base, service_id,
fee_receiver, from_wallet)
try:
dt.verify_order_tx(Web3Provider.get_web3(), tx_hash, did,
service_id, amount_base, from_wallet.address)
return tx_hash
except (AssertionError, Exception) as e:
msg = (
f'Downloading asset files failed. The problem is related to '
f'the transfer of the data tokens required for the download '
f'service: {e}')
logger.error(msg)
raise AssertionError(msg)
def create(self,
metadata,
publisher_account,
service_descriptors=None,
providers=None,
use_secret_store=True):
"""
Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store (
Aquarius).
:param metadata: dict conforming to the Metadata accepted by Ocean Protocol.
:param publisher_account: Account of the publisher registering this asset
:param service_descriptors: list of ServiceDescriptor tuples of length 2.
The first item must be one of ServiceTypes and the second
item is a dict of parameters and values required by the service
:param providers: list of addresses of providers of this asset (a provider is
an ethereum account that is authorized to provide asset services)
:param use_secret_store: bool indicate whether to use the secret store directly for
encrypting urls (Uses Brizo provider service if set to False)
:return: DDO instance
"""
assert isinstance(
metadata,
dict), f'Expected metadata of type dict, got {type(metadata)}'
assert service_descriptors is None or isinstance(service_descriptors, list), \
f'bad type of `service_descriptors` {type(service_descriptors)}'
# if not metadata or not Metadata.validate(metadata):
# raise OceanInvalidMetadata('Metadata seems invalid. Please make sure'
# ' the required metadata values are filled in.')
# copy metadata so we don't change the original
metadata_copy = copy.deepcopy(metadata)
asset_type = metadata_copy['main']['type']
assert asset_type in (
'dataset',
'algorithm'), f'Invalid/unsupported asset type {asset_type}'
service_descriptors = service_descriptors or []
brizo = BrizoProvider.get_brizo()
services = self._process_service_descriptors(service_descriptors,
metadata_copy,
publisher_account)
stype_to_service = {s.type: s for s in services}
checksum_dict = dict()
for service in services:
checksum_dict[str(service.index)] = checksum(service.main)
# Create a DDO object
ddo = DDO()
# Adding proof to the ddo.
ddo.add_proof(checksum_dict, publisher_account)
# Generating the did and adding to the ddo.
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
logger.debug(f'Generating new did: {did}')
# Check if it's already registered first!
if did in self._get_aquarius().list_assets():
raise OceanDIDAlreadyExist(
f'Asset id {did} is already registered to another asset.')
md_service = stype_to_service[ServiceTypes.METADATA]
ddo_service_endpoint = md_service.service_endpoint
if '{did}' in ddo_service_endpoint:
ddo_service_endpoint = ddo_service_endpoint.replace('{did}', did)
md_service.set_service_endpoint(ddo_service_endpoint)
# Populate the ddo services
ddo.add_service(md_service)
ddo.add_service(stype_to_service[ServiceTypes.AUTHORIZATION])
access_service = stype_to_service.get(ServiceTypes.ASSET_ACCESS, None)
compute_service = stype_to_service.get(ServiceTypes.CLOUD_COMPUTE,
None)
if access_service:
access_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(access_service)
if compute_service:
compute_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(compute_service)
ddo.proof['signatureValue'] = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)),
publisher_account)
# Add public key and authentication
ddo.add_public_key(did, publisher_account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
# Setup metadata service
# First compute files_encrypted
if metadata_copy['main']['type'] == 'dataset':
assert metadata_copy['main']['files'], \
'files is required in the metadata main attributes.'
logger.debug('Encrypting content urls in the metadata.')
if not use_secret_store:
encrypt_endpoint = brizo.get_encrypt_endpoint(self._config)
files_encrypted = brizo.encrypt_files_dict(
metadata_copy['main']['files'], encrypt_endpoint,
ddo.asset_id, publisher_account.address,
self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(ddo.asset_id),
publisher_account))
else:
files_encrypted = self._get_secret_store(publisher_account) \
.encrypt_document(
did_to_id(did),
json.dumps(metadata_copy['main']['files']),
)
# only assign if the encryption worked
if files_encrypted:
logger.debug(
f'Content urls encrypted successfully {files_encrypted}')
index = 0
for file in metadata_copy['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata_copy['encryptedFiles'] = files_encrypted
else:
raise AssertionError('Encrypting the files failed.')
# DDO url and `Metadata` service
logger.debug(f'Generated ddo and services, DID is {ddo.did},'
f' metadata service @{ddo_service_endpoint}.')
response = None
# register on-chain
registered_on_chain = self._keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=publisher_account,
providers=providers)
if registered_on_chain is False:
logger.warning(f'Registering {did} on-chain failed.')
return None
logger.info(f'Successfully registered DDO (DID={did}) on chain.')
try:
# publish the new ddo in ocean-db/Aquarius
response = self._get_aquarius().publish_asset_ddo(ddo)
logger.info('Asset/ddo published successfully in aquarius.')
except ValueError as ve:
raise ValueError(
f'Invalid value to publish in the metadata: {str(ve)}')
except Exception as e:
logger.error(f'Publish asset in aquarius failed: {str(e)}')
if not response:
return None
return ddo
def computeStart():
"""Call the execution of a workflow.
---
tags:
- services
consumes:
- application/json
parameters:
- name: signature
in: query
description: Signature of (consumerAddress+jobId+documentId) to verify the consumer of
this asset/compute job. The signature uses ethereum based signing method
(see https://github.com/ethereum/EIPs/pull/683)
type: string
- name: consumerAddress
in: query
description: The consumer ethereum address.
required: true
type: string
- name: algorithmDid
in: query
description: The DID of the algorithm Asset to be executed
required: false
type: string
- name: algorithmMeta
in: query
description: json object that define the algorithm attributes and url or raw code
required: false
type: json string
- name: output
in: query
description: json object that define the output section
required: true
type: json string
responses:
200:
description: Call to the operator-service was successful.
400:
description: One of the required attributes is missing.
401:
description: Consumer signature is invalid or failed verification
500:
description: General server error
"""
data = get_request_data(request)
try:
asset, service, did, consumer_address, token_address = process_consume_request( # noqa
data,
'compute_start_job',
additional_params=["transferTxId", "output"],
require_signature=False)
service_id = data.get('serviceId')
service_type = data.get('serviceType')
signature = data.get('signature')
tx_id = data.get("transferTxId")
# Verify that the number of required tokens has been
# transferred to the provider's wallet.
_tx, _order_log, _transfer_log = validate_order(
consumer_address, token_address, float(service.get_cost()), tx_id,
add_0x_prefix(did_to_id(did)) if did.startswith('did:') else did,
service_id)
validate_transfer_not_used_for_other_service(did, service_id, tx_id,
consumer_address,
token_address)
record_consume_request(did, service_id, tx_id, consumer_address,
token_address, service.get_cost())
algorithm_did = data.get('algorithmDid')
algorithm_token_address = data.get('algorithmDataToken')
algorithm_meta = data.get('algorithmMeta')
algorithm_tx_id = data.get('algorithmTransferTxId')
output_def = data.get('output', dict())
assert service_type == ServiceTypes.CLOUD_COMPUTE
# Validate algorithm choice
if not (algorithm_meta or algorithm_did):
msg = f'Need an `algorithmMeta` or `algorithmDid` to run, otherwise don\'t bother.' # noqa
logger.error(msg, exc_info=1)
return jsonify(error=msg), 400
# algorithmDid also requires algorithmDataToken
# and algorithmTransferTxId
if algorithm_did:
if not (algorithm_token_address and algorithm_tx_id):
msg = (
f'Using `algorithmDid` requires the `algorithmDataToken` and ' # noqa
f'`algorithmTransferTxId` values in the request payload. '
f'algorithmDataToken is the DataToken address for the algorithm asset. ' # noqa
f'algorithmTransferTxId is the transaction id (hash) of transferring ' # noqa
f'data tokens from consumer wallet to this providers wallet.'
)
logger.error(msg, exc_info=1)
return jsonify(error=msg), 400
# Consumer signature
original_msg = f'{consumer_address}{did}'
verify_signature(consumer_address, signature, original_msg,
user_nonce.get_nonce(consumer_address))
########################
# Valid service?
if service is None:
return jsonify(
error=f'This DID has no compute service {did}.'), 400
#########################
# Check privacy
privacy_options = service.main.get('privacy', {})
if (algorithm_meta
and privacy_options.get('allowRawAlgorithm', True) is False):
return jsonify(
error=f'cannot run raw algorithm on this did {did}.'), 400
trusted_algorithms = privacy_options.get('trustedAlgorithms', [])
if (algorithm_did and trusted_algorithms
and algorithm_did not in trusted_algorithms):
return jsonify(
error=f'cannot run raw algorithm on this did {did}.'), 400
#########################
# Validate ALGORITHM meta
if algorithm_meta:
algorithm_meta = json.loads(algorithm_meta) if isinstance(
algorithm_meta, str) else algorithm_meta
algorithm_dict = build_stage_algorithm_dict(
consumer_address, algorithm_did, algorithm_token_address,
algorithm_tx_id, algorithm_meta, provider_wallet)
error_msg, status_code = validate_algorithm_dict(
algorithm_dict, algorithm_did)
if error_msg:
return jsonify(error=error_msg), status_code
#########################
# INPUT
asset_urls = get_asset_download_urls(
asset, provider_wallet, config_file=app.config['CONFIG_FILE'])
if not asset_urls:
return jsonify(error=f'cannot get url(s) in input did {did}.'), 400
input_dict = dict({'index': 0, 'id': did, 'url': asset_urls})
#########################
# OUTPUT
if output_def:
output_def = json.loads(output_def) if isinstance(
output_def, str) else output_def
output_dict = build_stage_output_dict(output_def, asset,
consumer_address,
provider_wallet)
#########################
# STAGE
stage = build_stage_dict(input_dict, algorithm_dict, output_dict)
#########################
# WORKFLOW
workflow = dict({'stages': list([stage])})
# workflow is ready, push it to operator
logger.info('Sending: %s', workflow)
msg_to_sign = f'{provider_wallet.address}{did}'
msg_hash = add_ethereum_prefix_and_hash_msg(msg_to_sign)
payload = {
'workflow': workflow,
'providerSignature': Web3Helper.sign_hash(msg_hash,
provider_wallet),
'documentId': did,
'agreementId': tx_id,
'owner': consumer_address,
'providerAddress': provider_wallet.address
}
response = requests_session.post(
get_compute_endpoint(),
data=json.dumps(payload),
headers={'content-type': 'application/json'})
user_nonce.increment_nonce(consumer_address)
return Response(response.content,
response.status_code,
headers={'content-type': 'application/json'})
except InvalidSignatureError as e:
msg = f'Consumer signature failed verification: {e}'
logger.error(msg, exc_info=1)
return jsonify(error=msg), 401
except (ValueError, KeyError, Exception) as e:
logger.error(f'Error- {str(e)}', exc_info=1)
return jsonify(error=f'Error : {str(e)}'), 500
def download():
"""Allows download of asset data file.
---
tags:
- services
consumes:
- application/json
parameters:
- name: consumerAddress
in: query
description: The consumer address.
required: true
type: string
- name: documentId
in: query
description: The ID of the asset/document (the DID).
required: true
type: string
- name: url
in: query
description: This URL is only valid if Provider acts as a proxy.
Consumer can't download using the URL if it's not through the Provider.
required: true
type: string
- name: signature
in: query
description: Signature of the documentId to verify that the consumer has rights to download the asset.
- name: index
in: query
description: Index of the file in the array of files.
responses:
200:
description: Redirect to valid asset url.
400:
description: One of the required attributes is missing.
401:
description: Invalid asset data.
500:
description: Error
"""
data = get_request_data(request)
try:
asset, service, did, consumer_address, token_address = process_consume_request( # noqa
data,
'download',
user_nonce=user_nonce,
additional_params=["transferTxId", "fileIndex"])
service_id = data.get('serviceId')
service_type = data.get('serviceType')
signature = data.get('signature')
tx_id = data.get("transferTxId")
if did.startswith('did:'):
did = add_0x_prefix(did_to_id(did))
_tx, _order_log, _transfer_log = validate_order(
consumer_address, token_address, float(service.get_cost()), tx_id,
did, service_id)
validate_transfer_not_used_for_other_service(did, service_id, tx_id,
consumer_address,
token_address)
record_consume_request(did, service_id, tx_id, consumer_address,
token_address, service.get_cost())
assert service_type == ServiceTypes.ASSET_ACCESS
file_index = int(data.get('fileIndex'))
file_attributes = asset.metadata['main']['files'][file_index]
content_type = file_attributes.get('contentType', None)
url = get_asset_url_at_index(file_index, asset, provider_wallet)
download_url = get_download_url(url, app.config['CONFIG_FILE'])
logger.info(f'Done processing consume request for asset {did}, '
f' url {download_url}')
user_nonce.increment_nonce(consumer_address)
return build_download_response(request, requests_session, url,
download_url, content_type)
except InvalidSignatureError as e:
msg = f'Consumer signature failed verification: {e}'
logger.error(msg, exc_info=1)
return jsonify(error=msg), 401
except Exception as e:
logger.error(
f'Error: {e}. \n'
f'Payload was: documentId={did}, '
f'consumerAddress={consumer_address},'
f'signature={signature}'
f'serviceId={service_id}'
f'serviceType={service_type}',
exc_info=1)
return jsonify(error=str(e)), 500
def _build_and_validate_algo(self, algo_data):
"""Returns False if invalid, otherwise sets the validated_algo_dict attribute."""
algorithm_did = algo_data.get("algorithmDid")
self.algo_service = None
if algorithm_did and not algo_data.get("algorithmMeta"):
algorithm_token_address = algo_data.get("algorithmDataToken")
algorithm_tx_id = algo_data.get("algorithmTransferTxId")
algo = get_asset_from_metadatastore(get_metadata_url(),
algorithm_did)
try:
asset_type = algo.metadata["main"]["type"]
except ValueError:
asset_type = None
if asset_type != "algorithm":
self.error = f"DID {algorithm_did} is not a valid algorithm"
return False
try:
dt = DataToken(self.consumer_address)
tx_receipt = dt.get_tx_receipt(algorithm_tx_id)
event_logs = dt.events.OrderStarted().processReceipt(
tx_receipt)
order_log = event_logs[0] if event_logs else None
algo_service_id = order_log.args.serviceId
self.algo_service = get_service_at_index(algo, algo_service_id)
if self.algo_service.type == ServiceTypes.CLOUD_COMPUTE:
asset_urls = get_asset_download_urls(
algo,
self.provider_wallet,
config_file=app.config["CONFIG_FILE"],
)
if not asset_urls:
self.error = "Services in algorithm with compute type must be in the same provider you are calling."
return False
if not self.algo_service:
self.error = "Failed to retrieve purchased algorithm service id."
return False
_tx, _order_log, _transfer_log = validate_order(
self.consumer_address,
algorithm_token_address,
float(self.algo_service.get_cost()),
algorithm_tx_id,
add_0x_prefix(did_to_id(algorithm_did))
if algorithm_did.startswith("did:") else algorithm_did,
self.algo_service.index,
)
validate_transfer_not_used_for_other_service(
algorithm_did,
self.algo_service.index,
algorithm_tx_id,
self.consumer_address,
algorithm_token_address,
)
record_consume_request(
algorithm_did,
self.algo_service.index,
algorithm_tx_id,
self.consumer_address,
algorithm_token_address,
self.algo_service.get_cost(),
)
except Exception:
self.error = "Algorithm is already in use or can not be found on chain."
return False
algorithm_dict = StageAlgoSerializer(self.consumer_address,
self.provider_wallet, algo_data,
self.algo_service).serialize()
valid, error_msg = validate_formatted_algorithm_dict(
algorithm_dict, algorithm_did)
if not valid:
self.error = error_msg
return False
self.validated_algo_dict = algorithm_dict
return True
def asset_id(self):
"""The asset id part of the DID"""
if not self._did:
return None
return add_0x_prefix(did_to_id(self._did))
def test_get_resolve_multiple_urls(publisher_account):
register_account = publisher_account
did_registry = keeper().did_registry
did = DID.did({"0": "0x1"})
did2 = DID.did({"0": "0x2"})
did3 = DID.did({"0": "0x3"})
did4 = DID.did({"0": "0x4"})
did5 = DID.did({"0": "0x5"})
did6 = DID.did({"0": "0x6"})
did7 = DID.did({"0": "0x7"})
did8 = DID.did({"0": "0x8"})
did9 = DID.did({"0": "0x9"})
did10 = DID.did({"0": "0x10"})
value_test = 'http://localhost:5000'
value_test2 = 'http://localhost:5001'
value_test3 = 'http://localhost:5002'
value_test4 = 'http://localhost:5003'
value_test5 = 'http://localhost:5004'
value_test6 = 'http://localhost:5005'
value_test7 = 'http://localhost:5006'
value_test8 = 'http://localhost:5007'
value_test9 = 'http://localhost:5008'
value_test10 = 'http://localhost:5009'
did_id = did_to_id(did)
did_id2 = did_to_id(did2)
did_id3 = did_to_id(did3)
did_id4 = did_to_id(did4)
did_id5 = did_to_id(did5)
did_id6 = did_to_id(did6)
did_id7 = did_to_id(did7)
did_id8 = did_to_id(did8)
did_id9 = did_to_id(did9)
did_id10 = did_to_id(did10)
did_resolver = DIDResolver(keeper().did_registry)
did_registry.register(did_id,
b'test',
url=value_test,
account=register_account)
did_registry.register(did_id2,
b'test',
url=value_test2,
account=register_account)
did_registry.register(did_id3,
b'test',
url=value_test3,
account=register_account)
did_registry.register(did_id4,
b'test',
url=value_test4,
account=register_account)
did_registry.register(did_id5,
b'test',
url=value_test5,
account=register_account)
did_registry.register(did_id6,
b'test',
url=value_test6,
account=register_account)
did_registry.register(did_id7,
b'test',
url=value_test7,
account=register_account)
did_registry.register(did_id8,
b'test',
url=value_test8,
account=register_account)
did_registry.register(did_id9,
b'test',
url=value_test9,
account=register_account)
did_registry.register(did_id10,
b'test',
url=value_test10,
account=register_account)
url = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id))
url2 = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id2))
url3 = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id3))
url4 = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id4))
url5 = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id5))
url6 = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id6))
url7 = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id7))
url8 = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id8))
url9 = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id9))
url10 = did_resolver.get_resolve_url(Web3.toBytes(hexstr=did_id10))
assert url == value_test
assert url2 == value_test2
assert url3 == value_test3
assert url4 == value_test4
assert url5 == value_test5
assert url6 == value_test6
assert url7 == value_test7
assert url8 == value_test8
assert url9 == value_test9
assert url10 == value_test10
def test_consume(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
endpoint = BaseURLs.ASSETS_URL + '/consume'
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
ddo = get_dataset_ddo_with_access_service(pub_acc,
providers=[pub_acc.address])
# initialize an agreement
agreement_id = place_order(pub_acc, ddo, cons_acc,
ServiceTypes.ASSET_ACCESS)
payload = dict({
'serviceAgreementId': agreement_id,
'consumerAddress': cons_acc.address
})
agr_id_hash = add_ethereum_prefix_and_hash_msg(agreement_id)
signature = keeper.sign_hash(agr_id_hash, cons_acc)
index = 0
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_access(agreement_id, ddo, cons_acc, pub_acc)
event = keeper.access_secret_store_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.access_secret_store_condition.check_permissions(
ddo.asset_id, cons_acc.address
), f'Failed to get access permission: agreement_id={agreement_id}, ' \
f'did={ddo.did}, consumer={cons_acc.address}'
# Consume using decrypted url
files_list = json.loads(
do_secret_store_decrypt(did_to_id(ddo.did), ddo.encrypted_files,
pub_acc, get_config()))
payload['url'] = files_list[index]['url']
request_url = endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
# Consume using url index and signature (let brizo do the decryption)
payload.pop('url')
payload['signature'] = signature
payload['index'] = index
request_url = endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
def secret_store_encrypt(did, document, account):
return get_secret_store_client(account).publish_document(
did_to_id(did), document)
def secret_store_decrypt(did, encrypted_document, account):
return get_secret_store_client(account).decrypt_document(
did_to_id(did), encrypted_document)
