def test_download_service(client):
aqua = Aquarius("http://localhost:5000")
try:
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
except (ValueError, Exception):
pass
pub_wallet = get_publisher_wallet()
cons_wallet = get_consumer_wallet()
ddo = get_dataset_ddo_with_access_service(client, pub_wallet)
dt_address = ddo.as_dictionary()["dataToken"]
dt_token = DataToken(dt_address)
mint_tokens_and_wait(dt_token, cons_wallet, pub_wallet)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
tx_id = send_order(client, ddo, dt_token, sa, cons_wallet)
index = 0
download_endpoint = BaseURLs.ASSETS_URL + "/download"
# Consume using url index and auth token
# (let the provider do the decryption)
payload = dict({
"documentId": ddo.did,
"serviceId": sa.index,
"serviceType": sa.type,
"dataToken": dt_address,
"consumerAddress": cons_wallet.address,
})
payload["signature"] = generate_auth_token(cons_wallet)
payload["transferTxId"] = tx_id
payload["fileIndex"] = index
request_url = (download_endpoint + "?" +
"&".join([f"{k}={v}" for k, v in payload.items()]))
response = client.get(request_url)
assert response.status_code == 200, f"{response.data}"
# Consume using url index and signature (withOUT nonce), should fail
_hash = add_ethereum_prefix_and_hash_msg(ddo.did)
payload["signature"] = Web3Helper.sign_hash(_hash, cons_wallet)
request_url = (download_endpoint + "?" +
"&".join([f"{k}={v}" for k, v in payload.items()]))
print(
">>>> Expecting InvalidSignatureError from the download endpoint <<<<"
) # noqa
response = client.get(request_url)
assert response.status_code == 400, f"{response.data}"
# Consume using url index and signature (with nonce)
nonce = get_nonce(client, cons_wallet.address)
_hash = add_ethereum_prefix_and_hash_msg(f"{ddo.did}{nonce}")
payload["signature"] = Web3Helper.sign_hash(_hash, cons_wallet)
request_url = (download_endpoint + "?" +
"&".join([f"{k}={v}" for k, v in payload.items()]))
response = client.get(request_url)
assert response.status_code == 200, f"{response.data}"
def test_download_service(client):
aqua = Aquarius('http://localhost:5000')
try:
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
except (ValueError, Exception):
pass
pub_wallet = get_publisher_wallet()
cons_wallet = get_consumer_wallet()
ddo = get_dataset_ddo_with_access_service(client, pub_wallet)
dt_address = ddo.as_dictionary()['dataToken']
dt_token = DataToken(dt_address)
mint_tokens_and_wait(dt_token, cons_wallet, pub_wallet)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
tx_id = send_order(client, ddo, dt_token, sa, cons_wallet)
index = 0
download_endpoint = BaseURLs.ASSETS_URL + '/download'
# Consume using url index and auth token (let the provider do the decryption)
payload = dict({
'documentId': ddo.did,
'serviceId': sa.index,
'serviceType': sa.type,
'dataToken': dt_address,
'consumerAddress': cons_wallet.address
})
payload['signature'] = generate_auth_token(cons_wallet)
payload['transferTxId'] = tx_id
payload['fileIndex'] = index
request_url = download_endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status_code == 200, f'{response.data}'
# Consume using url index and signature (withOUT nonce), should fail
_hash = add_ethereum_prefix_and_hash_msg(ddo.did)
payload['signature'] = Web3Helper.sign_hash(_hash, cons_wallet)
request_url = download_endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
print(
'>>>> Expecting InvalidSignatureError from the download endpoint <<<<')
response = client.get(request_url)
assert response.status_code == 401, f'{response.data}'
# Consume using url index and signature (with nonce)
nonce = get_nonce(client, cons_wallet.address)
_hash = add_ethereum_prefix_and_hash_msg(f'{ddo.did}{nonce}')
payload['signature'] = Web3Helper.sign_hash(_hash, cons_wallet)
request_url = download_endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status_code == 200, f'{response.data}'
def test_consume(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
endpoint = BaseURLs.ASSETS_URL + '/consume'
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
ddo = get_dataset_ddo_with_access_service(pub_acc,
providers=[pub_acc.address])
# initialize an agreement
agreement_id = place_order(pub_acc, ddo, cons_acc,
ServiceTypes.ASSET_ACCESS)
payload = dict({
'serviceAgreementId': agreement_id,
'consumerAddress': cons_acc.address
})
agr_id_hash = add_ethereum_prefix_and_hash_msg(agreement_id)
signature = keeper.sign_hash(agr_id_hash, cons_acc)
index = 0
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_access(agreement_id, ddo, cons_acc, pub_acc)
event = keeper.access_secret_store_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.access_secret_store_condition.check_permissions(
ddo.asset_id, cons_acc.address
), f'Failed to get access permission: agreement_id={agreement_id}, ' \
f'did={ddo.did}, consumer={cons_acc.address}'
# Consume using decrypted url
files_list = json.loads(
do_secret_store_decrypt(did_to_id(ddo.did), ddo.encrypted_files,
pub_acc, get_config()))
payload['url'] = files_list[index]['url']
request_url = endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
# Consume using url index and signature (let brizo do the decryption)
payload.pop('url')
payload['signature'] = signature
payload['index'] = index
request_url = endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status == '200 OK'
def get_registered_ddo(account, providers=None):
keeper = keeper_instance()
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
metadata = get_sample_ddo()['service'][0]['attributes']
metadata['main']['files'][0]['checksum'] = str(uuid.uuid4())
ddo = DDO()
ddo_service_endpoint = aqua.get_service_endpoint()
metadata_service_desc = ServiceDescriptor.metadata_service_descriptor(
metadata, ddo_service_endpoint)
access_service_attributes = {
"main": {
"name": "dataAssetAccessServiceAgreement",
"creator": account.address,
"price": metadata[MetadataMain.KEY]['price'],
"timeout": 3600,
"datePublished": metadata[MetadataMain.KEY]['dateCreated']
}
}
service_descriptors = [
ServiceDescriptor.authorization_service_descriptor(
'http://localhost:12001')
]
service_descriptors += [
ServiceDescriptor.access_service_descriptor(
access_service_attributes, 'http://localhost:8030',
keeper.escrow_access_secretstore_template.address)
]
service_descriptors = [metadata_service_desc] + service_descriptors
services = ServiceFactory.build_services(service_descriptors)
checksums = dict()
for service in services:
checksums[str(service.index)] = checksum(service.main)
# Adding proof to the ddo.
ddo.add_proof(checksums, account)
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
stype_to_service = {s.type: s for s in services}
access_service = stype_to_service[ServiceTypes.ASSET_ACCESS]
name_to_address = {
cname: cinst.address
for cname, cinst in keeper.contract_name_to_instance.items()
}
access_service.init_conditions_values(
did, contract_name_to_address=name_to_address)
ddo.add_service(access_service)
for service in services:
ddo.add_service(service)
ddo.proof['signatureValue'] = keeper.sign_hash(did_to_id_bytes(did),
account)
ddo.add_public_key(did, account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
encrypted_files = do_secret_store_encrypt(
remove_0x_prefix(ddo.asset_id), json.dumps(metadata['main']['files']),
account, get_config())
_files = metadata['main']['files']
# only assign if the encryption worked
if encrypted_files:
index = 0
for file in metadata['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata['encryptedFiles'] = encrypted_files
keeper_instance().did_registry.register(
ddo.asset_id,
checksum=web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=account,
providers=providers)
aqua.publish_asset_ddo(ddo)
return ddo
def test_compute_norawalgo_allowed(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
# publish a dataset asset
dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service_no_rawalgo(
pub_acc, providers=[pub_acc.address])
# CHECKPOINT 1
algorithmMeta = {
"rawcode": "console.log('Hello world'!)",
"format": 'docker-image',
"version": '0.1',
"container": {
"entrypoint": 'node $ALGO',
"image": 'node',
"tag": '10'
}
}
# prepare parameter values for the compute endpoint
# signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta
# initialize an agreement
agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service,
cons_acc, ServiceTypes.CLOUD_COMPUTE)
# CHECKPOINT 2
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE,
dataset_ddo_w_compute_service)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id,
cons_acc, pub_acc)
event = keeper.compute_execution_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.compute_execution_condition.was_compute_triggered(
dataset_ddo_w_compute_service.asset_id, cons_acc.address
), (f'Failed to compute: agreement_id={agreement_id}, '
f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}'
)
# prepare consumer signature on agreement_id
msg = f'{cons_acc.address}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
# Start the compute job
payload = dict({
'signature':
signature,
'serviceAgreementId':
agreement_id,
'consumerAddress':
cons_acc.address,
'algorithmDid':
None,
'algorithmMeta':
algorithmMeta,
'output':
build_stage_output_dict(dict(), dataset_ddo_w_compute_service,
cons_acc.address, pub_acc)
})
endpoint = BaseURLs.ASSETS_URL + '/compute'
response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
assert response.status == '400 BAD REQUEST', f'start compute job failed: {response.status} , { response.data}'
def test_compute(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
keeper = keeper_instance()
# publish a dataset asset
dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service(
pub_acc, providers=[pub_acc.address])
# publish an algorithm asset (asset with metadata of type `algorithm`)
alg_ddo = get_algorithm_ddo(cons_acc, providers=[pub_acc.address])
# CHECKPOINT 1
# prepare parameter values for the compute endpoint
# signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta
# initialize an agreement
agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service,
cons_acc, ServiceTypes.CLOUD_COMPUTE)
# CHECKPOINT 2
event = keeper.agreement_manager.subscribe_agreement_created(agreement_id,
15,
None, (),
wait=True,
from_block=0)
assert event, "Agreement event is not found, check the keeper node's logs"
consumer_balance = keeper.token.get_token_balance(cons_acc.address)
if consumer_balance < 50:
keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc)
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE,
dataset_ddo_w_compute_service)
lock_reward(agreement_id, sa, cons_acc)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id,
cons_acc, pub_acc)
event = keeper.compute_execution_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True, from_block=0)
assert event or keeper.compute_execution_condition.was_compute_triggered(
dataset_ddo_w_compute_service.asset_id, cons_acc.address
), (f'Failed to compute: agreement_id={agreement_id}, '
f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}'
)
# prepare consumer signature on agreement_id
msg = f'{cons_acc.address}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
# Start the compute job
payload = dict({
'signature':
signature,
'serviceAgreementId':
agreement_id,
'consumerAddress':
cons_acc.address,
'algorithmDid':
alg_ddo.did,
'algorithmMeta': {},
'output':
build_stage_output_dict(dict(), dataset_ddo_w_compute_service,
cons_acc.address, pub_acc)
})
endpoint = BaseURLs.ASSETS_URL + '/compute'
response = client.post(endpoint,
data=json.dumps(payload),
content_type='application/json')
assert response.status == '200 OK', f'start compute job failed: {response.data}'
job_info = response.json[0]
print(f'got response from starting compute job: {job_info}')
job_id = job_info.get('jobId', '')
msg = f'{cons_acc.address}{job_id}{agreement_id}'
agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = keeper.sign_hash(agreement_id_hash, cons_acc)
payload = dict({
'signature': signature,
'serviceAgreementId': agreement_id,
'consumerAddress': cons_acc.address,
'jobId': job_id,
})
job_info = get_compute_job_info(client, endpoint, payload)
assert job_info, f'Failed to get job info for jobId {job_id}'
print(f'got info for compute job {job_id}: {job_info}')
assert job_info['statusText'] in get_possible_compute_job_status_text()
def test_download_service(client):
aqua = Aquarius('http://localhost:5000')
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
init_endpoint = BaseURLs.ASSETS_URL + '/initialize'
download_endpoint = BaseURLs.ASSETS_URL + '/download'
pub_acc = get_publisher_account()
cons_acc = get_consumer_account()
request_ether('https://faucet.nile.dev-ocean.com', cons_acc)
ddo = get_dataset_ddo_with_access_service(pub_acc)
dt_address = ddo.as_dictionary()['dataTokenAddress']
dt_token = DataTokenContract(dt_address)
tx_id = dt_token.mint(cons_acc.address, 50, pub_acc)
dt_token.get_tx_receipt(tx_id)
time.sleep(2)
def verify_supply(mint_amount=50):
supply = dt_token.contract_concise.totalSupply()
if supply <= 0:
_tx_id = dt_token.mint(cons_acc.address, mint_amount, pub_acc)
dt_token.get_tx_receipt(_tx_id)
supply = dt_token.contract_concise.totalSupply()
return supply
while True:
try:
s = verify_supply()
if s > 0:
break
except (ValueError, Exception):
pass
auth_token = generate_auth_token(cons_acc)
index = 0
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
# Consume using decrypted url
files_list = json.loads(
do_decrypt(ddo.encrypted_files, pub_acc))
# initialize an agreement
payload = dict({
'documentId': ddo.did,
'serviceId': sa.index,
'serviceType': sa.type,
'tokenAddress': dt_address,
'consumerAddress': cons_acc.address
})
payload['url'] = files_list[index]['url']
request_url = init_endpoint + '?' + '&'.join([f'{k}={v}' for k, v in payload.items()])
response = client.get(
request_url
)
assert response.status == '200 OK'
tx_params = response.json
num_tokens = tx_params['numTokens']
assert tx_params['from'] == cons_acc.address
assert tx_params['to'] == pub_acc.address
assert tx_params['dataTokenAddress'] == ddo.as_dictionary()['dataTokenAddress']
# Transfer tokens to provider account
tx_id = dt_token.transfer(tx_params['to'], num_tokens, cons_acc)
dt_token.get_tx_receipt(tx_id)
# Consume using url index and signature (let the provider do the decryption)
payload.pop('url')
payload['signature'] = auth_token
payload['transferTxId'] = Web3.toHex(tx_id)
payload['fileIndex'] = index
request_url = download_endpoint + '?' + '&'.join([f'{k}={v}' for k, v in payload.items()])
response = client.get(
request_url
)
assert response.status == '200 OK'