def place_order(publisher_account, ddo, consumer_account, service_type):
keeper = keeper_instance()
agreement_id = ServiceAgreement.create_new_agreement_id()
publisher_address = publisher_account.address
# balance = keeper.token.get_token_balance(consumer_account.address)/(2**18)
# if balance < 20:
# keeper.dispenser.request_tokens(100, consumer_account)
service_agreement = ServiceAgreement.from_ddo(service_type, ddo)
condition_ids = service_agreement.generate_agreement_condition_ids(
agreement_id, ddo.asset_id, consumer_account.address,
publisher_address, keeper)
time_locks = service_agreement.conditions_timelocks
time_outs = service_agreement.conditions_timeouts
template_name = keeper.template_manager.SERVICE_TO_TEMPLATE_NAME[
service_type]
template_id = keeper.template_manager.create_template_id(template_name)
actor_map = {
'consumer': consumer_account.address,
'provider': publisher_address
}
actors = [
actor_map[_type]
for _type in get_template_actor_types(keeper, template_id)
]
assert keeper.template_manager.contract_concise.isTemplateIdApproved(
template_id), f'template {template_id} is not approved.'
keeper_instance().agreement_manager.create_agreement(
agreement_id, ddo.asset_id, template_id, condition_ids, time_locks,
time_outs, actors, consumer_account)
return agreement_id
def get_dataset_ddo_with_access_service(account, providers=None):
keeper = keeper_instance()
metadata = get_sample_ddo()['service'][0]['attributes']
metadata['main']['files'][0]['checksum'] = str(uuid.uuid4())
service_descriptor = get_access_service_descriptor(keeper, account,
metadata)
return get_registered_ddo(account, metadata, service_descriptor, providers)
def get_dataset_ddo_with_compute_service_no_rawalgo(account, providers=None):
keeper = keeper_instance()
metadata = get_sample_ddo_with_compute_service(
)['service'][0]['attributes']
metadata['main']['files'][0]['checksum'] = str(uuid.uuid4())
service_descriptor = get_compute_service_descriptor_no_rawalgo(
keeper, account, metadata[MetadataMain.KEY]['price'], metadata)
return get_registered_ddo(account, metadata, service_descriptor, providers)
def lock_reward(agreement_id, service_agreement, consumer_account):
keeper = keeper_instance()
price = service_agreement.get_price()
keeper.token.token_approve(keeper.lock_reward_condition.address, price,
consumer_account)
time.sleep(3)
tx_hash = keeper.lock_reward_condition.fulfill(
agreement_id, keeper.escrow_reward_condition.address, price,
consumer_account)
keeper.lock_reward_condition.get_tx_receipt(tx_hash)
def test_agreement_expiry():
pub_acc = get_publisher_account()
keeper = keeper_instance()
metadata = get_sample_ddo()['service'][0]['attributes']
metadata['main']['files'][0]['checksum'] = str(uuid.uuid4())
service_descriptor = get_access_service_descriptor(keeper, pub_acc,
metadata)
service_descriptor[1]['attributes']['main']['timeout'] = 2
agreement = ServiceFactory.build_service(service_descriptor)
start_time = datetime.now().timestamp()
not_expired = validate_agreement_expiry(agreement, start_time)
assert not_expired, 'Agreement should not be expired at this point.'
time.sleep(3)
with pytest.raises(ServiceAgreementExpired):
validate_agreement_expiry(agreement, start_time)
def test_publish(client):
endpoint = BaseURLs.ASSETS_URL + '/publish'
did = DID.did({"0": str(uuid.uuid4())})
asset_id = did_to_id(did)
account = get_provider_account()
test_urls = ['url 00', 'url 11', 'url 22']
keeper = keeper_instance()
urls_json = json.dumps(test_urls)
asset_id_hash = add_ethereum_prefix_and_hash_msg(asset_id)
signature = keeper.sign_hash(asset_id_hash, account)
address = web3().eth.account.recoverHash(asset_id_hash,
signature=signature)
assert address.lower() == account.address.lower()
address = keeper.personal_ec_recover(asset_id, signature)
assert address.lower() == account.address.lower()
payload = {
'documentId': asset_id,
'signature': signature,
'document': urls_json,
'publisherAddress': account.address
}
post_response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
encrypted_url = post_response.data.decode('utf-8')
assert encrypted_url.startswith('0x')
# publish using auth token
signature = generate_token(account)
payload['signature'] = signature
did = DID.did({"0": str(uuid.uuid4())})
asset_id = did_to_id(did)
payload['documentId'] = add_0x_prefix(asset_id)
post_response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
encrypted_url = post_response.data.decode('utf-8')
assert encrypted_url.startswith('0x')
def version():
keeper = keeper_instance()
info = dict()
info['software'] = Metadata.TITLE
info['version'] = get_version()
info['keeper-url'] = config.keeper_url
info['network'] = keeper.network_name
info['contracts'] = dict()
info['contracts']['AccessSecretStoreCondition'] = keeper.access_secret_store_condition.address
info['contracts']['AgreementStoreManager'] = keeper.agreement_manager.address
info['contracts']['ConditionStoreManager'] = keeper.condition_manager.address
info['contracts']['DIDRegistry'] = keeper.did_registry.address
if keeper.network_name != 'pacific':
info['contracts']['Dispenser'] = keeper.dispenser.address
info['contracts']['EscrowReward'] = keeper.escrow_reward_condition.address
info['contracts']['LockRewardCondition'] = keeper.lock_reward_condition.address
info['contracts']['OceanToken'] = keeper.token.address
info['contracts']['TemplateStoreManager'] = keeper.template_manager.address
info['contracts']['ComputeExecutionCondition'] = keeper.compute_execution_condition.address
info['keeper-version'] = get_latest_keeper_version()
info['provider-address'] = get_provider_account().address
return jsonify(info)
def _check_job_id(client, job_id, agreement_id, wait_time=20):
endpoint = BaseURLs.ASSETS_URL + '/compute'
cons_acc = get_consumer_account()
keeper = keeper_instance()
msg = f'{cons_acc.address}{job_id}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
payload = dict({
'signature': signature,
'serviceAgreementId': agreement_id,
'consumerAddress': cons_acc.address,
'jobId': job_id,
})
job_info = get_compute_job_info(client, endpoint, payload)
assert job_info, f'Failed to get job info for jobId {job_id}'
print(f'got info for compute job {job_id}: {job_info}')
assert job_info['statusText'] in get_possible_compute_job_status_text()
did = None
# get did of results
for _ in range(wait_time * 4):
job_info = get_compute_job_info(client, endpoint, payload)
did = job_info['did']
if did:
break
time.sleep(0.25)
assert did, f'Compute job has no results, job info {job_info}.'
# check results ddo
ddo = DIDResolver(keeper.did_registry).resolve(did)
assert ddo, f'Failed to resolve ddo for did {did}'
consumer_permission = keeper.did_registry.get_permission(
did, cons_acc.address)
assert consumer_permission is True, \
f'Consumer address {cons_acc.address} has no permissions on the results ' \
f'did {did}. This is required, the consumer must be able to access the results'
def test_consume(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
endpoint = BaseURLs.ASSETS_URL + '/consume'
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
ddo = get_dataset_ddo_with_access_service(pub_acc,
providers=[pub_acc.address])
# initialize an agreement
agreement_id = place_order(pub_acc, ddo, cons_acc,
ServiceTypes.ASSET_ACCESS)
payload = dict({
'serviceAgreementId': agreement_id,
'consumerAddress': cons_acc.address
})
agr_id_hash = add_ethereum_prefix_and_hash_msg(agreement_id)
signature = keeper.sign_hash(agr_id_hash, cons_acc)
index = 0
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_access(agreement_id, ddo, cons_acc, pub_acc)
event = keeper.access_secret_store_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.access_secret_store_condition.check_permissions(
ddo.asset_id, cons_acc.address
), f'Failed to get access permission: agreement_id={agreement_id}, ' \
f'did={ddo.did}, consumer={cons_acc.address}'
# Consume using decrypted url
files_list = json.loads(
do_secret_store_decrypt(did_to_id(ddo.did), ddo.encrypted_files,
pub_acc, get_config()))
payload['url'] = files_list[index]['url']
request_url = endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
# Consume using url index and signature (let brizo do the decryption)
payload.pop('url')
payload['signature'] = signature
payload['index'] = index
request_url = endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
def test_compute_norawalgo_allowed(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
# publish a dataset asset
dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service_no_rawalgo(
pub_acc, providers=[pub_acc.address])
# CHECKPOINT 1
algorithmMeta = {
"rawcode": "console.log('Hello world'!)",
"format": 'docker-image',
"version": '0.1',
"container": {
"entrypoint": 'node $ALGO',
"image": 'node',
"tag": '10'
}
}
# prepare parameter values for the compute endpoint
# signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta
# initialize an agreement
agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service,
cons_acc, ServiceTypes.CLOUD_COMPUTE)
# CHECKPOINT 2
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE,
dataset_ddo_w_compute_service)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id,
cons_acc, pub_acc)
event = keeper.compute_execution_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.compute_execution_condition.was_compute_triggered(
dataset_ddo_w_compute_service.asset_id, cons_acc.address
), (f'Failed to compute: agreement_id={agreement_id}, '
f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}'
)
# prepare consumer signature on agreement_id
msg = f'{cons_acc.address}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
# Start the compute job
payload = dict({
'signature':
signature,
'serviceAgreementId':
agreement_id,
'consumerAddress':
cons_acc.address,
'algorithmDid':
None,
'algorithmMeta':
algorithmMeta,
'output':
build_stage_output_dict(dict(), dataset_ddo_w_compute_service,
cons_acc.address, pub_acc)
})
endpoint = BaseURLs.ASSETS_URL + '/compute'
response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
assert response.status == '400 BAD REQUEST', f'start compute job failed: {response.status} , { response.data}'
def test_compute(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
# publish a dataset asset
dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service(
pub_acc, providers=[pub_acc.address])
# publish an algorithm asset (asset with metadata of type `algorithm`)
alg_ddo = get_algorithm_ddo(cons_acc, providers=[pub_acc.address])
# CHECKPOINT 1
# prepare parameter values for the compute endpoint
# signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta
# initialize an agreement
agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service,
cons_acc, ServiceTypes.CLOUD_COMPUTE)
# CHECKPOINT 2
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE,
dataset_ddo_w_compute_service)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id,
cons_acc, pub_acc)
event = keeper.compute_execution_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.compute_execution_condition.was_compute_triggered(
dataset_ddo_w_compute_service.asset_id, cons_acc.address
), (f'Failed to compute: agreement_id={agreement_id}, '
f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}'
)
# prepare consumer signature on agreement_id
msg = f'{cons_acc.address}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
# Start the compute job
payload = dict({
'signature':
signature,
'serviceAgreementId':
agreement_id,
'consumerAddress':
cons_acc.address,
'algorithmDid':
alg_ddo.did,
'algorithmMeta': {},
'output':
build_stage_output_dict(dict(), dataset_ddo_w_compute_service,
cons_acc.address, pub_acc)
})
endpoint = BaseURLs.ASSETS_URL + '/compute'
response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
assert response.status == '200 OK', f'start compute job failed: {response.data}'
job_info = response.json[0]
print(f'got response from starting compute job: {job_info}')
job_id = job_info.get('jobId', '')
msg = f'{cons_acc.address}{job_id}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
payload = dict({
'signature': signature,
'serviceAgreementId': agreement_id,
'consumerAddress': cons_acc.address,
'jobId': job_id,
})
job_info = get_compute_job_info(client, endpoint, payload)
assert job_info, f'Failed to get job info for jobId {job_id}'
print(f'got info for compute job {job_id}: {job_info}')
assert job_info['statusText'] in get_possible_compute_job_status_text()
def grant_compute(agreement_id, asset_id, consumer_account, publisher_account):
keeper = keeper_instance()
tx_hash = keeper.compute_execution_condition.fulfill(
agreement_id, asset_id, consumer_account.address, publisher_account)
keeper.compute_execution_condition.get_tx_receipt(tx_hash)
def grant_access(agreement_id, ddo, consumer_account, publisher_account):
keeper = keeper_instance()
tx_hash = keeper.access_secret_store_condition.fulfill(
agreement_id, ddo.asset_id, consumer_account.address,
publisher_account)
keeper.access_secret_store_condition.get_tx_receipt(tx_hash)
def get_registered_ddo(account, metadata, service_descriptor, providers=None):
keeper = keeper_instance()
aqua = Aquarius('http://localhost:5000')
ddo = DDO()
ddo_service_endpoint = aqua.get_service_endpoint()
metadata_service_desc = ServiceDescriptor.metadata_service_descriptor(
metadata, ddo_service_endpoint)
service_descriptors = list([
ServiceDescriptor.authorization_service_descriptor(
'http://localhost:12001')
])
service_descriptors.append(service_descriptor)
service_type = service_descriptor[0]
service_descriptors = [metadata_service_desc] + service_descriptors
services = ServiceFactory.build_services(service_descriptors)
checksums = dict()
for service in services:
checksums[str(service.index)] = checksum(service.main)
# Adding proof to the ddo.
ddo.add_proof(checksums, account)
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
ddo_service_endpoint.replace('{did}', did)
services[0].set_service_endpoint(ddo_service_endpoint)
stype_to_service = {s.type: s for s in services}
_service = stype_to_service[service_type]
name_to_address = {
cname: cinst.address
for cname, cinst in keeper.contract_name_to_instance.items()
}
_service.init_conditions_values(did,
contract_name_to_address=name_to_address)
for service in services:
ddo.add_service(service)
ddo.proof['signatureValue'] = keeper.sign_hash(did_to_id_bytes(did),
account)
ddo.add_public_key(did, account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
try:
_oldddo = aqua.get_asset_ddo(ddo.did)
if _oldddo:
aqua.retire_asset_ddo(ddo.did)
except ValueError:
pass
if not plecos.is_valid_dict_local(ddo.metadata):
print(f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}')
assert False, f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}'
encrypted_files = do_secret_store_encrypt(
remove_0x_prefix(ddo.asset_id), json.dumps(metadata['main']['files']),
account, get_config())
# only assign if the encryption worked
if encrypted_files:
index = 0
for file in metadata['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata['encryptedFiles'] = encrypted_files
keeper_instance().did_registry.register(
ddo.asset_id,
checksum=web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=account,
providers=providers)
try:
aqua.publish_asset_ddo(ddo)
except Exception as e:
print(f'error publishing ddo {ddo.did} in Aquarius: {e}')
raise
return ddo
