def generate_token(account):
raw_msg = get_config(
).auth_token_message or "Ocean Protocol Authentication"
_time = int(datetime.now().timestamp())
_message = f'{raw_msg}\n{_time}'
prefixed_msg_hash = add_ethereum_prefix_and_hash_msg(_message)
return f'{keeper_instance().sign_hash(prefixed_msg_hash, account)}-{_time}'
def sign(self, agreement_id, did, consumer_account, service_index):
"""
Sign a service agreement.
:param agreement_id: 32 bytes identifier created by the consumer and will be used
on-chain for the executed agreement.
:param did: str representation fo the asset DID. Use this to retrieve the asset DDO.
:param consumer_account: Account instance of the consumer
:param service_index: int identifies the specific service in
the ddo to use in this agreement.
:return: signature
"""
asset = self._asset_resolver.resolve(did)
service_agreement = asset.get_service_by_index(service_index)
publisher_address = self._keeper.did_registry.get_did_owner(
asset.asset_id)
agreement_hash = service_agreement.get_service_agreement_hash(
agreement_id, asset.asset_id, consumer_account.address,
publisher_address, self._keeper)
signature = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(agreement_hash), consumer_account)
address = self._keeper.personal_ec_recover(agreement_hash, signature)
assert address == consumer_account.address
logger.debug(
f'agreement-signature={signature}, agreement-hash={agreement_hash}'
)
return signature
def get(self, account):
"""
:param account: Account instance signing the token
:return: hex str the token generated/signed by account
"""
_message, _time = self._get_message_and_time()
try:
prefixed_msg_hash = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(_message), account)
return f'{prefixed_msg_hash}-{_time}'
except Exception as e:
logging.error(f'Error signing token: {str(e)}')
def consume_service(service_agreement_id,
service_endpoint,
account,
files,
destination_folder,
index=None):
"""
Call the brizo endpoint to get access to the different files that form the asset.
:param service_agreement_id: Service Agreement Id, str
:param service_endpoint: Url to consume, str
:param account: Account instance of the consumer signing this agreement, hex-str
:param files: List containing the files to be consumed, list
:param index: Index of the document that is going to be downloaded, int
:param destination_folder: Path, str
:return: True if was downloaded, bool
"""
signature = Keeper.get_instance().sign_hash(
add_ethereum_prefix_and_hash_msg(service_agreement_id), account)
if index is not None:
assert isinstance(index,
int), logger.error('index has to be an integer.')
assert index >= 0, logger.error(
'index has to be 0 or a positive integer.')
assert index < len(files), logger.error(
'index can not be bigger than the number of files')
consume_url = Brizo._create_consume_url(service_endpoint,
service_agreement_id,
account, None, signature,
index)
logger.info(
f'invoke consume endpoint with this url: {consume_url}')
response = Brizo._http_client.get(consume_url, stream=True)
file_name = Brizo._get_file_name(response)
Brizo.write_file(response, destination_folder, file_name)
else:
for i, _file in enumerate(files):
consume_url = Brizo._create_consume_url(
service_endpoint, service_agreement_id, account, _file,
signature, i)
logger.info(
f'invoke consume endpoint with this url: {consume_url}')
response = Brizo._http_client.get(consume_url, stream=True)
file_name = Brizo._get_file_name(response)
Brizo.write_file(response, destination_folder, file_name
or f'file-{i}')
def test_publish(client):
endpoint = BaseURLs.ASSETS_URL + '/publish'
did = DID.did({"0": str(uuid.uuid4())})
asset_id = did_to_id(did)
account = get_provider_account()
test_urls = ['url 00', 'url 11', 'url 22']
keeper = keeper_instance()
urls_json = json.dumps(test_urls)
asset_id_hash = add_ethereum_prefix_and_hash_msg(asset_id)
signature = keeper.sign_hash(asset_id_hash, account)
address = web3().eth.account.recoverHash(asset_id_hash,
signature=signature)
assert address.lower() == account.address.lower()
address = keeper.personal_ec_recover(asset_id, signature)
assert address.lower() == account.address.lower()
payload = {
'documentId': asset_id,
'signature': signature,
'document': urls_json,
'publisherAddress': account.address
}
post_response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
encrypted_url = post_response.data.decode('utf-8')
assert encrypted_url.startswith('0x')
# publish using auth token
signature = generate_token(account)
payload['signature'] = signature
did = DID.did({"0": str(uuid.uuid4())})
asset_id = did_to_id(did)
payload['documentId'] = add_0x_prefix(asset_id)
post_response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
encrypted_url = post_response.data.decode('utf-8')
assert encrypted_url.startswith('0x')
def execute_service(service_agreement_id, service_endpoint, account,
workflow_ddo):
"""
:param service_agreement_id:
:param service_endpoint:
:param account:
:return:
"""
signature = Keeper.get_instance().sign_hash(
add_ethereum_prefix_and_hash_msg(service_agreement_id), account)
execute_url = Brizo._create_execute_url(service_endpoint,
service_agreement_id, account,
workflow_ddo.did, signature)
logger.info(f'invoke execute endpoint with this url: {execute_url}')
response = Brizo._http_client.post(execute_url)
print(
f'got brizo execute response: {response.content} with status-code {response.status_code} '
)
if response.status_code != 201:
raise Exception(response.content.decode('utf-8'))
return json.loads(response.content)['workflowId']
def _check_job_id(client, job_id, agreement_id, wait_time=20):
endpoint = BaseURLs.ASSETS_URL + '/compute'
cons_acc = get_consumer_account()
keeper = keeper_instance()
msg = f'{cons_acc.address}{job_id}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
payload = dict({
'signature': signature,
'serviceAgreementId': agreement_id,
'consumerAddress': cons_acc.address,
'jobId': job_id,
})
job_info = get_compute_job_info(client, endpoint, payload)
assert job_info, f'Failed to get job info for jobId {job_id}'
print(f'got info for compute job {job_id}: {job_info}')
assert job_info['statusText'] in get_possible_compute_job_status_text()
did = None
# get did of results
for _ in range(wait_time * 4):
job_info = get_compute_job_info(client, endpoint, payload)
did = job_info['did']
if did:
break
time.sleep(0.25)
assert did, f'Compute job has no results, job info {job_info}.'
# check results ddo
ddo = DIDResolver(keeper.did_registry).resolve(did)
assert ddo, f'Failed to resolve ddo for did {did}'
consumer_permission = keeper.did_registry.get_permission(
did, cons_acc.address)
assert consumer_permission is True, \
f'Consumer address {cons_acc.address} has no permissions on the results ' \
f'did {did}. This is required, the consumer must be able to access the results'
def create(self,
metadata,
publisher_account,
service_descriptors=None,
providers=None,
use_secret_store=True):
"""
Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store (
Aquarius).
:param metadata: dict conforming to the Metadata accepted by Ocean Protocol.
:param publisher_account: Account of the publisher registering this asset
:param service_descriptors: list of ServiceDescriptor tuples of length 2.
The first item must be one of ServiceTypes and the second
item is a dict of parameters and values required by the service
:param providers: list of addresses of providers of this asset (a provider is
an ethereum account that is authorized to provide asset services)
:param use_secret_store: bool indicate whether to use the secret store directly for
encrypting urls (Uses Brizo provider service if set to False)
:return: DDO instance
"""
assert isinstance(
metadata,
dict), f'Expected metadata of type dict, got {type(metadata)}'
assert service_descriptors is None or isinstance(service_descriptors, list), \
f'bad type of `service_descriptors` {type(service_descriptors)}'
# if not metadata or not Metadata.validate(metadata):
# raise OceanInvalidMetadata('Metadata seems invalid. Please make sure'
# ' the required metadata values are filled in.')
# copy metadata so we don't change the original
metadata_copy = copy.deepcopy(metadata)
asset_type = metadata_copy['main']['type']
assert asset_type in (
'dataset',
'algorithm'), f'Invalid/unsupported asset type {asset_type}'
service_descriptors = service_descriptors or []
brizo = BrizoProvider.get_brizo()
services = self._process_service_descriptors(service_descriptors,
metadata_copy,
publisher_account)
stype_to_service = {s.type: s for s in services}
checksum_dict = dict()
for service in services:
checksum_dict[str(service.index)] = checksum(service.main)
# Create a DDO object
ddo = DDO()
# Adding proof to the ddo.
ddo.add_proof(checksum_dict, publisher_account)
# Generating the did and adding to the ddo.
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
logger.debug(f'Generating new did: {did}')
# Check if it's already registered first!
if did in self._get_aquarius().list_assets():
raise OceanDIDAlreadyExist(
f'Asset id {did} is already registered to another asset.')
md_service = stype_to_service[ServiceTypes.METADATA]
ddo_service_endpoint = md_service.service_endpoint
if '{did}' in ddo_service_endpoint:
ddo_service_endpoint = ddo_service_endpoint.replace('{did}', did)
md_service.set_service_endpoint(ddo_service_endpoint)
# Populate the ddo services
ddo.add_service(md_service)
ddo.add_service(stype_to_service[ServiceTypes.AUTHORIZATION])
access_service = stype_to_service.get(ServiceTypes.ASSET_ACCESS, None)
compute_service = stype_to_service.get(ServiceTypes.CLOUD_COMPUTE,
None)
if access_service:
access_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(access_service)
if compute_service:
compute_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(compute_service)
ddo.proof['signatureValue'] = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)),
publisher_account)
# Add public key and authentication
ddo.add_public_key(did, publisher_account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
# Setup metadata service
# First compute files_encrypted
if metadata_copy['main']['type'] == 'dataset':
assert metadata_copy['main']['files'], \
'files is required in the metadata main attributes.'
logger.debug('Encrypting content urls in the metadata.')
if not use_secret_store:
encrypt_endpoint = brizo.get_encrypt_endpoint(self._config)
files_encrypted = brizo.encrypt_files_dict(
metadata_copy['main']['files'], encrypt_endpoint,
ddo.asset_id, publisher_account.address,
self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(ddo.asset_id),
publisher_account))
else:
files_encrypted = self._get_secret_store(publisher_account) \
.encrypt_document(
did_to_id(did),
json.dumps(metadata_copy['main']['files']),
)
# only assign if the encryption worked
if files_encrypted:
logger.debug(
f'Content urls encrypted successfully {files_encrypted}')
index = 0
for file in metadata_copy['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata_copy['encryptedFiles'] = files_encrypted
else:
raise AssertionError('Encrypting the files failed.')
# DDO url and `Metadata` service
logger.debug(f'Generated ddo and services, DID is {ddo.did},'
f' metadata service @{ddo_service_endpoint}.')
response = None
# register on-chain
registered_on_chain = self._keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=publisher_account,
providers=providers)
if registered_on_chain is False:
logger.warning(f'Registering {did} on-chain failed.')
return None
logger.info(f'Successfully registered DDO (DID={did}) on chain.')
try:
# publish the new ddo in ocean-db/Aquarius
response = self._get_aquarius().publish_asset_ddo(ddo)
logger.info('Asset/ddo published successfully in aquarius.')
except ValueError as ve:
raise ValueError(
f'Invalid value to publish in the metadata: {str(ve)}')
except Exception as e:
logger.error(f'Publish asset in aquarius failed: {str(e)}')
if not response:
return None
return ddo
def test_consume(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
endpoint = BaseURLs.ASSETS_URL + '/consume'
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
ddo = get_dataset_ddo_with_access_service(pub_acc,
providers=[pub_acc.address])
# initialize an agreement
agreement_id = place_order(pub_acc, ddo, cons_acc,
ServiceTypes.ASSET_ACCESS)
payload = dict({
'serviceAgreementId': agreement_id,
'consumerAddress': cons_acc.address
})
agr_id_hash = add_ethereum_prefix_and_hash_msg(agreement_id)
signature = keeper.sign_hash(agr_id_hash, cons_acc)
index = 0
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_access(agreement_id, ddo, cons_acc, pub_acc)
event = keeper.access_secret_store_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.access_secret_store_condition.check_permissions(
ddo.asset_id, cons_acc.address
), f'Failed to get access permission: agreement_id={agreement_id}, ' \
f'did={ddo.did}, consumer={cons_acc.address}'
# Consume using decrypted url
files_list = json.loads(
do_secret_store_decrypt(did_to_id(ddo.did), ddo.encrypted_files,
pub_acc, get_config()))
payload['url'] = files_list[index]['url']
request_url = endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
# Consume using url index and signature (let brizo do the decryption)
payload.pop('url')
payload['signature'] = signature
payload['index'] = index
request_url = endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
def test_compute_norawalgo_allowed(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
# publish a dataset asset
dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service_no_rawalgo(
pub_acc, providers=[pub_acc.address])
# CHECKPOINT 1
algorithmMeta = {
"rawcode": "console.log('Hello world'!)",
"format": 'docker-image',
"version": '0.1',
"container": {
"entrypoint": 'node $ALGO',
"image": 'node',
"tag": '10'
}
}
# prepare parameter values for the compute endpoint
# signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta
# initialize an agreement
agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service,
cons_acc, ServiceTypes.CLOUD_COMPUTE)
# CHECKPOINT 2
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE,
dataset_ddo_w_compute_service)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id,
cons_acc, pub_acc)
event = keeper.compute_execution_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.compute_execution_condition.was_compute_triggered(
dataset_ddo_w_compute_service.asset_id, cons_acc.address
), (f'Failed to compute: agreement_id={agreement_id}, '
f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}'
)
# prepare consumer signature on agreement_id
msg = f'{cons_acc.address}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
# Start the compute job
payload = dict({
'signature':
signature,
'serviceAgreementId':
agreement_id,
'consumerAddress':
cons_acc.address,
'algorithmDid':
None,
'algorithmMeta':
algorithmMeta,
'output':
build_stage_output_dict(dict(), dataset_ddo_w_compute_service,
cons_acc.address, pub_acc)
})
endpoint = BaseURLs.ASSETS_URL + '/compute'
response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
assert response.status == '400 BAD REQUEST', f'start compute job failed: {response.status} , { response.data}'
def test_compute(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
# publish a dataset asset
dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service(
pub_acc, providers=[pub_acc.address])
# publish an algorithm asset (asset with metadata of type `algorithm`)
alg_ddo = get_algorithm_ddo(cons_acc, providers=[pub_acc.address])
# CHECKPOINT 1
# prepare parameter values for the compute endpoint
# signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta
# initialize an agreement
agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service,
cons_acc, ServiceTypes.CLOUD_COMPUTE)
# CHECKPOINT 2
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE,
dataset_ddo_w_compute_service)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id,
cons_acc, pub_acc)
event = keeper.compute_execution_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.compute_execution_condition.was_compute_triggered(
dataset_ddo_w_compute_service.asset_id, cons_acc.address
), (f'Failed to compute: agreement_id={agreement_id}, '
f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}'
)
# prepare consumer signature on agreement_id
msg = f'{cons_acc.address}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
# Start the compute job
payload = dict({
'signature':
signature,
'serviceAgreementId':
agreement_id,
'consumerAddress':
cons_acc.address,
'algorithmDid':
alg_ddo.did,
'algorithmMeta': {},
'output':
build_stage_output_dict(dict(), dataset_ddo_w_compute_service,
cons_acc.address, pub_acc)
})
endpoint = BaseURLs.ASSETS_URL + '/compute'
response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
assert response.status == '200 OK', f'start compute job failed: {response.data}'
job_info = response.json[0]
print(f'got response from starting compute job: {job_info}')
job_id = job_info.get('jobId', '')
msg = f'{cons_acc.address}{job_id}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
payload = dict({
'signature': signature,
'serviceAgreementId': agreement_id,
'consumerAddress': cons_acc.address,
'jobId': job_id,
})
job_info = get_compute_job_info(client, endpoint, payload)
assert job_info, f'Failed to get job info for jobId {job_id}'
print(f'got info for compute job {job_id}: {job_info}')
assert job_info['statusText'] in get_possible_compute_job_status_text()