def setup_agreements_environment():
consumer_acc = get_consumer_account()
publisher_acc = get_publisher_account()
keeper = Keeper.get_instance()
ddo = get_ddo_sample()
ddo._did = DID.did({"0": "0x12341234"})
keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url='aquarius:5000',
account=publisher_acc,
providers=None)
registered_ddo = ddo
asset_id = registered_ddo.asset_id
service_agreement = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS,
ddo)
agreement_id = ServiceAgreement.create_new_agreement_id()
price = service_agreement.get_price()
(lock_cond_id, access_cond_id,
escrow_cond_id) = service_agreement.generate_agreement_condition_ids(
agreement_id, asset_id, consumer_acc.address, publisher_acc.address,
keeper)
return (
keeper,
publisher_acc,
consumer_acc,
agreement_id,
asset_id,
price,
service_agreement,
(lock_cond_id, access_cond_id, escrow_cond_id),
)
def order(self, did: str, consumer_address: str,
service_index: [int, None]=None, service_type: str=None) -> OrderRequirements:
"""
Request a specific service from an asset, returns the service requirements that
must be met prior to consuming the service.
:param did:
:param consumer_address:
:param service_index:
:param service_type:
:return: OrderRequirements instance -- named tuple (amount, data_token_address, receiver_address, nonce),
"""
assert service_type or service_index, f'One of service_index or service_type is required.'
asset = self.resolve(did)
if service_type:
sa = ServiceAgreement.from_ddo(service_type, asset)
else:
service = asset.get_service_by_index(service_index)
sa = ServiceAgreement.from_ddo(service.type, asset)
dt_address = asset.data_token_address
_, initialize_url = self._data_provider.get_initialize_endpoint(sa.service_endpoint)
order_requirements = self._data_provider.get_order_requirements(
asset.did, initialize_url, consumer_address, sa.index, sa.type, dt_address
)
if not order_requirements:
raise AssertionError('Data service provider or service is not available.')
assert dt_address == order_requirements.data_token_address
return order_requirements
def test_agreement_hash():
"""
This test verifies generating agreement hash using fixed inputs and ddo example.
This will make it easier to compare the hash generated from different languages.
"""
did = "did:op:0bc278fee025464f8012b811d1bce8e22094d0984e4e49139df5d5ff7a028bdf"
template_id = '0x34219511357c43f4af90a4896b972a32809379bfbe8148eab3878e8406e75836'
agreement_id = '0xf136d6fadecb48fdb2fc1fb420f5a5d1c32d22d9424e47ab9461556e058fefaa'
ddo = get_ddo_sample()
sa = ServiceAgreement.from_json(
ddo.get_service(ServiceTypes.ASSET_ACCESS).as_dictionary())
assert did == ddo.did
# Don't generate condition ids, use fixed ids so we get consistent hash
# (access_id, lock_id, escrow_id) = sa.generate_agreement_condition_ids(
# agreement_id, ddo.asset_id, consumer, publisher, keeper)
access_id = '0x2d7c1d60dc0c3f52aa9bd71ffdbe434a0e58435571e64c893bc9646fea7f6ec1'
lock_id = '0x1e265c434c14e668695dda1555088f0ea4356f596bdecb8058812e7dcba9ee73'
escrow_id = '0xe487fa6d435c2f09ef14b65b34521302f1532ac82ba8f6c86116acd8566e2da3'
print(f'condition ids: \n'
f'{access_id} \n'
f'{lock_id} \n'
f'{escrow_id}')
agreement_hash = ServiceAgreement.generate_service_agreement_hash(
template_id,
(access_id, lock_id, escrow_id),
sa.conditions_timelocks,
sa.conditions_timeouts,
agreement_id,
generate_multi_value_hash
)
print('agreement hash: ', agreement_hash.hex())
expected = '0xc1bc6cdb161b9d04d5b48bceba51b9092c4da8db08003b96b93647fbe86e330c'
print('expected hash: ', expected)
assert agreement_hash.hex() == expected, 'hash does not match.'
def place_order(publisher_account, ddo, consumer_account, service_type):
keeper = keeper_instance()
agreement_id = ServiceAgreement.create_new_agreement_id()
publisher_address = publisher_account.address
# balance = keeper.token.get_token_balance(consumer_account.address)/(2**18)
# if balance < 20:
# keeper.dispenser.request_tokens(100, consumer_account)
service_agreement = ServiceAgreement.from_ddo(service_type, ddo)
condition_ids = service_agreement.generate_agreement_condition_ids(
agreement_id, ddo.asset_id, consumer_account.address,
publisher_address, keeper)
time_locks = service_agreement.conditions_timelocks
time_outs = service_agreement.conditions_timeouts
template_name = keeper.template_manager.SERVICE_TO_TEMPLATE_NAME[
service_type]
template_id = keeper.template_manager.create_template_id(template_name)
actor_map = {
'consumer': consumer_account.address,
'provider': publisher_address
}
actors = [
actor_map[_type]
for _type in get_template_actor_types(keeper, template_id)
]
assert keeper.template_manager.contract_concise.isTemplateIdApproved(
template_id), f'template {template_id} is not approved.'
keeper_instance().agreement_manager.create_agreement(
agreement_id, ddo.asset_id, template_id, condition_ids, time_locks,
time_outs, actors, consumer_account)
return agreement_id
def build_and_send_ddo_with_compute_service(client,
alg_diff=False,
asset_type=None):
pub_wallet = get_publisher_wallet()
cons_wallet = get_consumer_wallet()
# publish an algorithm asset (asset with metadata of type `algorithm`)
alg_ddo = (get_algorithm_ddo_different_provider(client, cons_wallet)
if alg_diff else get_algorithm_ddo(client, cons_wallet))
alg_data_token = alg_ddo.as_dictionary()["dataToken"]
alg_dt_contract = DataToken(alg_data_token)
mint_tokens_and_wait(alg_dt_contract, cons_wallet, cons_wallet)
# publish a dataset asset
if asset_type == "allow_all_published":
dataset_ddo_w_compute_service = comp_ds_allow_all_published(
client, pub_wallet)
elif asset_type == "specific_algo_dids":
algos = []
for _ in itertools.repeat(None, 2):
alg_ddo = get_algorithm_ddo(client, cons_wallet)
alg_data_token = alg_ddo.as_dictionary()["dataToken"]
alg_dt_contract = DataToken(alg_data_token)
mint_tokens_and_wait(alg_dt_contract, cons_wallet, cons_wallet)
algos.append(alg_ddo)
dataset_ddo_w_compute_service = comp_ds_specific_algo_dids(
client, pub_wallet, algos)
else:
dataset_ddo_w_compute_service = comp_ds(client, pub_wallet)
did = dataset_ddo_w_compute_service.did
ddo = dataset_ddo_w_compute_service
data_token = dataset_ddo_w_compute_service.data_token_address
dt_contract = DataToken(data_token)
mint_tokens_and_wait(dt_contract, cons_wallet, pub_wallet)
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE,
dataset_ddo_w_compute_service)
tx_id = send_order(client, ddo, dt_contract, sa, cons_wallet)
alg_service = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, alg_ddo)
alg_tx_id = send_order(client, alg_ddo, alg_dt_contract, alg_service,
cons_wallet)
return (
dataset_ddo_w_compute_service,
did,
tx_id,
sa,
data_token,
alg_ddo,
alg_data_token,
alg_dt_contract,
alg_tx_id,
)
def _get_service_endpoint(self, did, asset=None):
if not asset:
asset = resolve_asset(did, self._config.aquarius_url)
return self._data_provider.build_compute_endpoint(
ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE,
asset).service_endpoint)
def test_assets_consumed(publisher_ocean_instance, consumer_ocean_instance):
ocn = publisher_ocean_instance
acct = consumer_ocean_instance.main_account
consumed_assets = len(ocn.assets.consumer_assets(acct.address))
asset = create_asset(publisher_ocean_instance)
service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS)
service_dict = service.as_dictionary()
sa = ServiceAgreement.from_json(service_dict)
keeper = ocn.keeper
def grant_access(_, ocn_instance, agr_id, did, cons_address, account):
ocn_instance.agreements.conditions.grant_access(
agr_id, did, cons_address, account)
agreement_id = consumer_ocean_instance.assets.order(
asset.did, sa.index, acct)
keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id,
15,
grant_access, (publisher_ocean_instance, agreement_id, asset.did,
acct.address, publisher_ocean_instance.main_account),
wait=True)
keeper.access_secret_store_condition.subscribe_condition_fulfilled(
agreement_id,
15,
log_event(keeper.access_secret_store_condition.FULFILLED_EVENT), (),
wait=True)
assert ocn.agreements.is_access_granted(agreement_id, asset.did,
acct.address)
assert len(ocn.assets.consumer_assets(acct.address)) == consumed_assets + 1
publisher_ocean_instance.assets.retire(asset.did)
def download_asset_files(service_index: int,
asset: Asset,
consumer_wallet: Wallet,
destination: str,
token_address: str,
order_tx_id: str,
data_provider: DataServiceProvider,
index: [int, None] = None):
"""
Download asset data files or result files from a compute job.
:param service_index: identifier of the service inside the asset DDO, str
:param asset: Asset instance
:param consumer_wallet: Wallet instance of the consumer
:param destination: Path, str
:param token_address: hex str the address of the DataToken smart contract
:param order_tx_id: hex str the transaction hash of the startOrder tx
:param data_provider: DataServiceProvider instance
:param index: Index of the document that is going to be downloaded, int
:return: Asset folder path, str
"""
_files = asset.metadata['main']['files']
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, asset)
service_endpoint = sa.service_endpoint
if not service_endpoint:
logger.error(
'Consume asset failed, service definition is missing the "serviceEndpoint".'
)
raise AssertionError(
'Consume asset failed, service definition is missing the "serviceEndpoint".'
)
service_endpoint = data_provider.build_download_endpoint(service_endpoint)
if not os.path.isabs(destination):
destination = os.path.abspath(destination)
if not os.path.exists(destination):
os.mkdir(destination)
asset_folder = os.path.join(destination,
f'datafile.{asset.asset_id}.{service_index}')
if not os.path.exists(asset_folder):
os.mkdir(asset_folder)
if index is not None:
assert isinstance(index,
int), logger.error('index has to be an integer.')
assert index >= 0, logger.error(
'index has to be 0 or a positive integer.')
assert index < len(_files), logger.error(
'index can not be bigger than the number of files')
indexes = [index]
else:
indexes = range(len(_files))
for i in indexes:
data_provider.download_service(asset.did, service_endpoint,
consumer_wallet, _files, asset_folder,
service_index, token_address,
order_tx_id, i)
return asset_folder
def _get_service_endpoint_from_agreement(self, agreement_id, ddo=None):
agreement = self._keeper.agreement_manager.get_agreement(agreement_id)
_type = self._keeper.get_agreement_type(agreement.template_id)
if not ddo:
ddo = self._did_resolver.resolve(id_to_did(agreement.did))
service = ServiceAgreement.from_ddo(_type, ddo)
assert service, f'Using agreement_id {agreement_id}, the service type {_type} does not ' \
f'have a matching service in the DDO with DID {agreement.did}.'
compute_service = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE,
ddo)
assert service.service_endpoint == compute_service.service_endpoint, \
f'`Expecting agreement of type `{ServiceTypes.CLOUD_COMPUTE}`, but `agreement_id` {agreement_id} ' \
f'seems to have type {_type}.'
return service.service_endpoint
def place_order(publisher_account, ddo, consumer_account):
keeper = keeper_instance()
agreement_id = ServiceAgreement.create_new_agreement_id()
agreement_template = keeper.escrow_access_secretstore_template
publisher_address = publisher_account.address
service_agreement = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS,
ddo)
condition_ids = service_agreement.generate_agreement_condition_ids(
agreement_id, ddo.asset_id, consumer_account.address,
publisher_address, keeper)
time_locks = service_agreement.conditions_timelocks
time_outs = service_agreement.conditions_timeouts
agreement_template.create_agreement(agreement_id, ddo.asset_id,
condition_ids, time_locks, time_outs,
consumer_account.address,
consumer_account)
return agreement_id
def test_compute_norawalgo_allowed(client):
pub_wallet = get_publisher_wallet()
cons_wallet = get_consumer_wallet()
# publish a dataset asset
dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service_no_rawalgo(client, pub_wallet)
did = dataset_ddo_w_compute_service.did
ddo = dataset_ddo_w_compute_service
data_token = dataset_ddo_w_compute_service.data_token_address
dt_contract = DataToken(data_token)
mint_tokens_and_wait(dt_contract, cons_wallet, pub_wallet)
# CHECKPOINT 1
algorithm_meta = {
"rawcode": "console.log('Hello world'!)",
"format": 'docker-image',
"version": '0.1',
"container": {
"entrypoint": 'node $ALGO',
"image": 'node',
"tag": '10'
}
}
# prepare parameter values for the compute endpoint
# signature, documentId, consumerAddress, and algorithmDid or algorithmMeta
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE, dataset_ddo_w_compute_service)
tx_id = send_order(client, ddo, dt_contract, sa, cons_wallet)
nonce = get_nonce(client, cons_wallet.address)
# prepare consumer signature on did
msg = f'{cons_wallet.address}{did}{nonce}'
_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = Web3Helper.sign_hash(_hash, cons_wallet)
# Start the compute job
payload = dict({
'signature': signature,
'documentId': did,
'serviceId': sa.index,
'serviceType': sa.type,
'consumerAddress': cons_wallet.address,
'transferTxId': tx_id,
'dataToken': data_token,
'output': build_stage_output_dict(dict(), dataset_ddo_w_compute_service, cons_wallet.address, pub_wallet),
'algorithmDid': '',
'algorithmMeta': algorithm_meta,
'algorithmDataToken': ''
})
compute_endpoint = BaseURLs.ASSETS_URL + '/compute'
response = client.post(
compute_endpoint,
data=json.dumps(payload),
content_type='application/json'
)
assert response.status == '400 BAD REQUEST', f'start compute job failed: {response.status} , { response.data}'
def test_download_service(client):
aqua = Aquarius("http://localhost:5000")
try:
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
except (ValueError, Exception):
pass
pub_wallet = get_publisher_wallet()
cons_wallet = get_consumer_wallet()
ddo = get_dataset_ddo_with_access_service(client, pub_wallet)
dt_address = ddo.as_dictionary()["dataToken"]
dt_token = DataToken(dt_address)
mint_tokens_and_wait(dt_token, cons_wallet, pub_wallet)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
tx_id = send_order(client, ddo, dt_token, sa, cons_wallet)
index = 0
download_endpoint = BaseURLs.ASSETS_URL + "/download"
# Consume using url index and auth token
# (let the provider do the decryption)
payload = dict({
"documentId": ddo.did,
"serviceId": sa.index,
"serviceType": sa.type,
"dataToken": dt_address,
"consumerAddress": cons_wallet.address,
})
payload["signature"] = generate_auth_token(cons_wallet)
payload["transferTxId"] = tx_id
payload["fileIndex"] = index
request_url = (download_endpoint + "?" +
"&".join([f"{k}={v}" for k, v in payload.items()]))
response = client.get(request_url)
assert response.status_code == 200, f"{response.data}"
# Consume using url index and signature (withOUT nonce), should fail
_hash = add_ethereum_prefix_and_hash_msg(ddo.did)
payload["signature"] = Web3Helper.sign_hash(_hash, cons_wallet)
request_url = (download_endpoint + "?" +
"&".join([f"{k}={v}" for k, v in payload.items()]))
print(
">>>> Expecting InvalidSignatureError from the download endpoint <<<<"
) # noqa
response = client.get(request_url)
assert response.status_code == 400, f"{response.data}"
# Consume using url index and signature (with nonce)
nonce = get_nonce(client, cons_wallet.address)
_hash = add_ethereum_prefix_and_hash_msg(f"{ddo.did}{nonce}")
payload["signature"] = Web3Helper.sign_hash(_hash, cons_wallet)
request_url = (download_endpoint + "?" +
"&".join([f"{k}={v}" for k, v in payload.items()]))
response = client.get(request_url)
assert response.status_code == 200, f"{response.data}"
def test_initialize_on_ipfs_url(client):
pub_wallet = get_publisher_wallet()
cons_wallet = get_consumer_wallet()
ddo = get_dataset_with_ipfs_url_ddo(client, pub_wallet)
data_token = ddo.data_token_address
dt_contract = DataToken(data_token)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
send_order(client, ddo, dt_contract, sa, cons_wallet)
def test_download_service(client):
aqua = Aquarius('http://localhost:5000')
try:
for did in aqua.list_assets():
aqua.retire_asset_ddo(did)
except (ValueError, Exception):
pass
pub_wallet = get_publisher_wallet()
cons_wallet = get_consumer_wallet()
ddo = get_dataset_ddo_with_access_service(client, pub_wallet)
dt_address = ddo.as_dictionary()['dataToken']
dt_token = DataToken(dt_address)
mint_tokens_and_wait(dt_token, cons_wallet, pub_wallet)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
tx_id = send_order(client, ddo, dt_token, sa, cons_wallet)
index = 0
download_endpoint = BaseURLs.ASSETS_URL + '/download'
# Consume using url index and auth token (let the provider do the decryption)
payload = dict({
'documentId': ddo.did,
'serviceId': sa.index,
'serviceType': sa.type,
'dataToken': dt_address,
'consumerAddress': cons_wallet.address
})
payload['signature'] = generate_auth_token(cons_wallet)
payload['transferTxId'] = tx_id
payload['fileIndex'] = index
request_url = download_endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status_code == 200, f'{response.data}'
# Consume using url index and signature (withOUT nonce), should fail
_hash = add_ethereum_prefix_and_hash_msg(ddo.did)
payload['signature'] = Web3Helper.sign_hash(_hash, cons_wallet)
request_url = download_endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
print(
'>>>> Expecting InvalidSignatureError from the download endpoint <<<<')
response = client.get(request_url)
assert response.status_code == 401, f'{response.data}'
# Consume using url index and signature (with nonce)
nonce = get_nonce(client, cons_wallet.address)
_hash = add_ethereum_prefix_and_hash_msg(f'{ddo.did}{nonce}')
payload['signature'] = Web3Helper.sign_hash(_hash, cons_wallet)
request_url = download_endpoint + '?' + '&'.join(
[f'{k}={v}' for k, v in payload.items()])
response = client.get(request_url)
assert response.status_code == 200, f'{response.data}'
def start(self,
did: str,
consumer_wallet: Wallet,
order_tx_id: str,
nonce: [int, None] = None,
algorithm_did: [str, None] = None,
algorithm_meta: [AlgorithmMetadata, None] = None,
algorithm_tx_id: str = '',
algorithm_data_token: str = '',
output: dict = None,
job_id: str = None):
"""Start a remote compute job on the asset files identified by `did` after
verifying that the provider service is active and transferring the
number of data-tokens required for using this compute service.
:param did: str -- id of asset that has the compute service
:param consumer_wallet: Wallet instance of the consumer ordering the service
:param order_tx_id: hex str -- id of the startOrder transaction (tx hash)
:param nonce: int value to use in the signature
:param algorithm_did: str -- the asset did (of `algorithm` type) which consist of `did:op:` and
the assetId hex str (without `0x` prefix)
:param algorithm_meta: `AlgorithmMetadata` instance -- metadata about the algorithm being run if
`algorithm` is being used. This is ignored when `algorithm_did` is specified.
:param algorithm_tx_id: transaction hash of algorithm StartOrder tx (Required when using `algorithm_did`)
:param algorithm_data_token: datatoken address of this algorithm (Required when using `algorithm_did`)
:param output: dict object to be used in publishing mechanism, must define
:param job_id: str identifier of a compute job that was previously started and
stopped (if supported by the provider's backend)
:return: str -- id of compute job being executed
"""
assert algorithm_did or algorithm_meta, 'either an algorithm did or an algorithm meta must be provided.'
output = OceanCompute.check_output_dict(
output, consumer_wallet.address, data_provider=self._data_provider)
asset = resolve_asset(did,
metadata_store_url=self._config.aquarius_url)
service_endpoint = self._get_service_endpoint(did, asset)
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE, asset)
signature = self._sign_message(consumer_wallet,
f'{consumer_wallet.address}{did}',
nonce=nonce)
job_info = self._data_provider.start_compute_job(
did, service_endpoint, consumer_wallet.address, signature,
sa.index, asset.data_token_address, order_tx_id, algorithm_did,
algorithm_meta, algorithm_tx_id, algorithm_data_token, output,
job_id)
return job_info['jobId']
def test_compute_specific_algo_dids(client):
pub_wallet = get_publisher_wallet()
cons_wallet = get_consumer_wallet()
# publish a dataset asset
dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service_specific_algo_dids(client, pub_wallet)
did = dataset_ddo_w_compute_service.did
ddo = dataset_ddo_w_compute_service
data_token = dataset_ddo_w_compute_service.as_dictionary()['dataToken']
dt_contract = DataToken(data_token)
mint_tokens_and_wait(dt_contract, cons_wallet, pub_wallet)
# publish an algorithm asset (asset with metadata of type `algorithm`)
alg_ddo = get_algorithm_ddo(client, cons_wallet)
alg_data_token = alg_ddo.as_dictionary()['dataToken']
alg_dt_contract = DataToken(alg_data_token)
mint_tokens_and_wait(alg_dt_contract, pub_wallet, cons_wallet)
# CHECKPOINT 1
sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE, dataset_ddo_w_compute_service)
tx_id = send_order(client, ddo, dt_contract, sa, cons_wallet)
nonce = get_nonce(client, cons_wallet.address)
# prepare consumer signature on did
msg = f'{cons_wallet.address}{did}{nonce}'
_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = Web3Helper.sign_hash(_hash, cons_wallet)
# Start the compute job
payload = dict({
'signature': signature,
'documentId': did,
'serviceId': sa.index,
'serviceType': sa.type,
'consumerAddress': cons_wallet.address,
'transferTxId': tx_id,
'dataToken': data_token,
'output': build_stage_output_dict(dict(), dataset_ddo_w_compute_service, cons_wallet.address, pub_wallet),
'algorithmDid': alg_ddo.did,
'algorithmMeta': {},
'algorithmDataToken': alg_data_token
})
compute_endpoint = BaseURLs.ASSETS_URL + '/compute'
response = client.post(
compute_endpoint,
data=json.dumps(payload),
content_type='application/json'
)
assert response.status == '400 BAD REQUEST', f'start compute job failed: {response.status} , { response.data}'
def execute(agreement_id, compute_ddo, workflow_ddo, consumer_account,
brizo, index):
"""
:param agreement_id:
:param workflow_ddo:
:param consumer_account:
:param index:
:return:
"""
service_endpoint = ServiceAgreement.from_ddo(
ServiceTypes.CLOUD_COMPUTE, compute_ddo).service_endpoint
return brizo.execute_service(agreement_id, service_endpoint,
consumer_account, workflow_ddo)
def _read_dict(self, dictionary):
"""Import a JSON dict into this DDO."""
values = copy.deepcopy(dictionary)
self._did = values.pop('id')
self._created = values.pop('created', None)
if 'publicKey' in values:
self._public_keys = []
for value in values.pop('publicKey'):
if isinstance(value, str):
value = json.loads(value)
self._public_keys.append(DDO.create_public_key_from_json(value))
if 'authentication' in values:
self._authentications = []
for value in values.pop('authentication'):
if isinstance(value, str):
value = json.loads(value)
self._authentications.append(DDO.create_authentication_from_json(value))
if 'service' in values:
self._services = []
for value in values.pop('service'):
if isinstance(value, str):
value = json.loads(value)
if value['type'] == ServiceTypes.ASSET_ACCESS:
service = ServiceAgreement.from_json(value)
elif value['type'] == ServiceTypes.CLOUD_COMPUTE:
service = ServiceAgreement.from_json(value)
else:
service = Service.from_json(value)
self._services.append(service)
if 'proof' in values:
self._proof = values.pop('proof')
self._other_values = values
def build_access_service(attributes, service_endpoint, template_id):
"""
Build an authorization service.
:param attributes: attributes of access service, dict
:param service_endpoint: identifier of the service inside the asset DDO, str
:param template_id: hex str the ethereum smart contract address of the
service agreement template contract.
:return: ServiceAgreement instance
"""
sla_template_dict = get_sla_template()
sla_template = ServiceAgreementTemplate(
template_id, 'dataAssetAccessServiceAgreement',
attributes['main']['creator'], sla_template_dict)
return ServiceAgreement(attributes, sla_template, service_endpoint,
ServiceTypes.ASSET_ACCESS,
ServiceTypesIndices.DEFAULT_ACCESS_INDEX)
def build_stage_algorithm_dict(consumer_address,
algorithm_did,
algorithm_token_address,
algorithm_tx_id,
algorithm_meta,
provider_wallet,
receiver_address=None):
if algorithm_did is not None:
assert algorithm_token_address and algorithm_tx_id, \
'algorithm_did requires both algorithm_token_address and algorithm_tx_id.'
algo_asset = get_asset_from_metadatastore(get_metadata_url(),
algorithm_did)
service = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS,
algo_asset)
_tx, _order_log, _transfer_log = validate_order(
consumer_address, algorithm_token_address,
float(service.get_cost()), algorithm_tx_id,
add_0x_prefix(did_to_id(algorithm_did)) if
algorithm_did.startswith('did:') else algorithm_did, service.index)
validate_transfer_not_used_for_other_service(algorithm_did,
service.index,
algorithm_tx_id,
consumer_address,
algorithm_token_address)
record_consume_request(algorithm_did, service.index, algorithm_tx_id,
consumer_address, algorithm_token_address,
service.get_cost())
algo_id = algorithm_did
raw_code = ''
algo_url = get_asset_url_at_index(0, algo_asset, provider_wallet)
container = algo_asset.metadata['main']['algorithm']['container']
else:
algo_id = ''
algo_url = algorithm_meta.get('url')
raw_code = algorithm_meta.get('rawcode')
container = algorithm_meta.get('container')
return dict({
'id': algo_id,
'url': algo_url,
'rawcode': raw_code,
'container': container
})
def process_consume_request(data: dict):
did = data.get("documentId")
token_address = data.get("dataToken")
consumer_address = data.get("consumerAddress")
service_id = data.get("serviceId")
service_type = data.get("serviceType")
# grab asset for did from the metadatastore associated with
# the Data Token address
asset = get_asset_from_metadatastore(get_metadata_url(), did)
service = ServiceAgreement.from_ddo(service_type, asset)
if service.type != service_type:
raise AssertionError(
f"Requested service with id {service_id} has type {service.type} "
f"which does not match the requested service type {service_type}."
)
return asset, service, did, consumer_address, token_address
def process_consume_request(data: dict,
method: str,
user_nonce: UserNonce = None,
additional_params: list = None,
require_signature: bool = True):
required_attributes = [
'documentId', 'serviceId', 'serviceType', 'dataToken',
'consumerAddress'
]
if additional_params:
required_attributes += additional_params
if require_signature:
required_attributes.append('signature')
msg, status = check_required_attributes(required_attributes, data, method)
if msg:
raise AssertionError(msg)
did = data.get('documentId')
token_address = data.get('dataToken')
consumer_address = data.get('consumerAddress')
service_id = data.get('serviceId')
service_type = data.get('serviceType')
# grab asset for did from the metadatastore associated with the Data Token address
asset = get_asset_from_metadatastore(get_metadata_url(), did)
service = ServiceAgreement.from_ddo(service_type, asset)
if service.type != service_type:
raise AssertionError(
f'Requested service with id {service_id} has type {service.type} which '
f'does not match the requested service type {service_type}.')
if require_signature:
assert user_nonce, '`user_nonce` is required when signature is required.'
# Raises ValueError when signature is invalid
signature = data.get('signature')
verify_signature(consumer_address, signature, did,
user_nonce.get_nonce(consumer_address))
return asset, service, did, consumer_address, token_address
def test_agreement():
template_id = '0xda558ceb3abd482db04ea813810ae9c6cea5bf13ef9943908b1a8b51c4e56c7f'
agreement_id = '0x' + ('e' * 64)
access_id = '0x' + ('a' * 64)
lock_id = '0x' + ('b' * 64)
escrow_id = '0x' + ('c' * 64)
signature = ServiceAgreement.generate_service_agreement_hash(
template_id,
[access_id, lock_id, escrow_id],
[0, 0, 0],
[0, 0, 0],
agreement_id,
generate_multi_value_hash
)
print({signature})
assert (
signature == Web3.toBytes(
hexstr="0x3450be8127dbe156fad90956fd351df42ffcfabba89d230406a301ceb5b40f92"
)), "The signature is not correct."
def ocean_assets_download_destination_file_helper(publisher_ocean_instance,
metadata, tmpdir):
"""Tests downloading to an existing directory."""
publisher = get_publisher_wallet()
metadata_copy = metadata.copy()
data_provider = DataServiceProvider
ddo = publisher_ocean_instance.assets.create(metadata_copy, publisher)
wait_for_ddo(publisher_ocean_instance, ddo.did)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
written_path = download_asset_files(
sa.index,
ddo,
publisher,
tmpdir,
ddo.data_token_address,
"test_order_tx_id",
data_provider,
)
assert os.path.exists(written_path)
def test_ocean_assets_download_failure(publisher_ocean_instance, metadata):
"""Tests that downloading from an empty service raises an AssertionError."""
publisher = get_publisher_wallet()
metadata_copy = metadata.copy()
data_provider = DataServiceProvider
ddo = publisher_ocean_instance.assets.create(metadata_copy, publisher)
wait_for_ddo(publisher_ocean_instance, ddo.did)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
sa.__dict__["_service_endpoint"] = None
ddo.__dict__["_services"][1] = sa
with pytest.raises(AssertionError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
)
def test_market_flow():
pub_wallet = get_publisher_wallet()
publisher_ocean = get_publisher_ocean_instance()
consumer_ocean = get_consumer_ocean_instance()
# Register Asset
asset = get_registered_ddo(publisher_ocean, pub_wallet)
assert isinstance(asset, Asset)
assert asset.data_token_address
consumer_wallet = get_consumer_wallet()
service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS)
sa = ServiceAgreement.from_json(service.as_dictionary())
# Mint data tokens and assign to publisher
dt = publisher_ocean.get_data_token(asset.data_token_address)
mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet)
######
# Give the consumer some datatokens so they can order the service
try:
tx_id = dt.transfer_tokens(consumer_wallet.address, 10, pub_wallet)
dt.verify_transfer_tx(tx_id, pub_wallet.address,
consumer_wallet.address)
except (AssertionError, Exception) as e:
print(e)
raise
######
# Place order for the download service
order_requirements = consumer_ocean.assets.order(asset.did,
consumer_wallet.address,
sa.index)
######
# Pay for the service
_order_tx_id = consumer_ocean.assets.pay_for_service(
order_requirements.amount, order_requirements.data_token_address,
asset.did, service.index, '0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9',
consumer_wallet)
######
# Download the asset files
asset_folder = consumer_ocean.assets.download(
asset.did, sa.index, consumer_wallet, _order_tx_id,
consumer_ocean.config.downloads_path)
assert len(os.listdir(asset_folder)) > 1
orders = consumer_ocean.get_user_orders(consumer_wallet.address,
asset.asset_id)
assert orders, f'no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}'
orders = consumer_ocean.get_user_orders(
consumer_wallet.address,
consumer_ocean.web3.toChecksumAddress(asset.asset_id))
assert orders, f'no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}'
orders = consumer_ocean.get_user_orders(consumer_wallet.address)
assert orders, f'no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}'
def test_market_flow(order_type):
"""Tests that an order is correctly placed on the market.
The parameter implicit_none sends the payload with an empty key as the delegated consumer.
The parameter explicit_none sends None as the delegated consumer, explicitly."""
pub_wallet = get_publisher_wallet()
publisher_ocean = get_publisher_ocean_instance()
consumer_ocean = get_consumer_ocean_instance()
# Register Asset
asset = get_registered_ddo(publisher_ocean, get_metadata(), pub_wallet)
assert isinstance(asset, Asset)
assert asset.data_token_address
consumer_wallet = get_consumer_wallet()
service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS)
sa = ServiceAgreement.from_json(service.as_dictionary())
# Mint data tokens and assign to publisher
dt = publisher_ocean.get_data_token(asset.data_token_address)
mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet)
######
# Give the consumer some datatokens so they can order the service
try:
tx_id = dt.transfer_tokens(consumer_wallet.address, 10.0, pub_wallet)
dt.verify_transfer_tx(tx_id, pub_wallet.address,
consumer_wallet.address)
except (AssertionError, Exception) as e:
print(e)
raise
######
# Place order for the download service
order_requirements = consumer_ocean.assets.order(asset.did,
consumer_wallet.address,
sa.index)
######
# Pay for the service
args = [
order_requirements.amount,
order_requirements.data_token_address,
asset.did,
service.index,
"0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9",
consumer_wallet,
]
if order_type == "explicit_none":
args.append(None)
_order_tx_id = consumer_ocean.assets.pay_for_service(*args)
######
# Download the asset files
asset_folder = consumer_ocean.assets.download(
asset.did,
sa.index,
consumer_wallet,
_order_tx_id,
consumer_ocean.config.downloads_path,
)
assert len(os.listdir(asset_folder)) >= 1
if order_type == "explicit_none":
# no need to continue, order worked
return
orders = consumer_ocean.get_user_orders(consumer_wallet.address,
asset.asset_id)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(
consumer_wallet.address,
consumer_ocean.web3.toChecksumAddress(asset.asset_id))
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(consumer_wallet.address)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
def test_payer_market_flow():
"""Tests that an order can be placed for a delegated consumer, other than the payer."""
pub_wallet = get_publisher_wallet()
publisher_ocean = get_publisher_ocean_instance()
consumer_ocean = get_consumer_ocean_instance()
another_consumer_ocean = get_another_consumer_ocean_instance(
use_provider_mock=True)
# Register Asset
asset = get_registered_ddo(publisher_ocean, get_metadata(), pub_wallet)
assert isinstance(asset, Asset)
assert asset.data_token_address
another_consumer_wallet = get_another_consumer_wallet()
consumer_wallet = get_consumer_wallet()
service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS)
sa = ServiceAgreement.from_json(service.as_dictionary())
# Mint data tokens and assign to publisher
dt = publisher_ocean.get_data_token(asset.data_token_address)
mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet)
######
# Give the consumer some datatokens so they can order the service
try:
tx_id = dt.transfer_tokens(consumer_wallet.address, 10.0, pub_wallet)
dt.verify_transfer_tx(tx_id, pub_wallet.address,
consumer_wallet.address)
except (AssertionError, Exception) as e:
print(e)
raise
######
# Place order for the download service
order_requirements = consumer_ocean.assets.order(
asset.did, another_consumer_wallet.address, sa.index)
######
# Pay for the service and have another_consumer_wallet as consumer
_order_tx_id = consumer_ocean.assets.pay_for_service(
order_requirements.amount,
order_requirements.data_token_address,
asset.did,
service.index,
"0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9",
consumer_wallet,
another_consumer_wallet.address,
)
asset_folder = None
assert asset_folder is None
if asset_folder is None:
# Download the asset files
asset_folder = another_consumer_ocean.assets.download(
asset.did,
sa.index,
another_consumer_wallet,
_order_tx_id,
another_consumer_ocean.config.downloads_path,
)
assert len(os.listdir(asset_folder)) >= 1
orders = consumer_ocean.get_user_orders(consumer_wallet.address,
asset.asset_id)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(
consumer_wallet.address,
consumer_ocean.web3.toChecksumAddress(asset.asset_id))
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(consumer_wallet.address)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
def buy_asset():
"""
Requires all ocean services running.
"""
setup_logging(default_level=logging.INFO)
ConfigProvider.set_config(ExampleConfig.get_config())
config = ConfigProvider.get_config()
providers = {
'duero': '0xfEF2d5e1670342b9EF22eeeDcb287EC526B48095',
'nile': '0x4aaab179035dc57b35e2ce066919048686f82972'
}
# make ocean instance
ocn = Ocean()
acc = get_account(1)
if not acc:
acc = ([acc for acc in ocn.accounts.list() if acc.password] or ocn.accounts.list())[0]
Diagnostics.verify_contracts()
metadata = example_metadata.metadata.copy()
metadata['main']['dateCreated'] = get_timestamp()
keeper = Keeper.get_instance()
# Register ddo
did = ''
if did:
ddo = ocn.assets.resolve(did)
logging.info(f'using ddo: {did}')
else:
ddo = ocn.assets.create(metadata, acc, providers=[], use_secret_store=True)
assert ddo is not None, f'Registering asset on-chain failed.'
did = ddo.did
logging.info(f'registered ddo: {did}')
# ocn here will be used only to publish the asset. Handling the asset by the publisher
# will be performed by the Brizo server running locally
test_net = os.environ.get('TEST_NET', '')
if test_net.startswith('nile'):
provider = keeper.did_registry.to_checksum_address(providers['nile'])
elif test_net.startswith('duero'):
provider = keeper.did_registry.to_checksum_address(providers['duero'])
else:
provider = '0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0'
# Wait for did registry event
event = keeper.did_registry.subscribe_to_event(
keeper.did_registry.DID_REGISTRY_EVENT_NAME,
30,
event_filter={
'_did': Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
'_owner': acc.address},
wait=True
)
if not event:
logging.warning(f'Failed to get the did registry event for asset with did {did}.')
assert keeper.did_registry.get_block_number_updated(ddo.asset_id) > 0, \
f'There is an issue in registering asset {did} on-chain.'
keeper.did_registry.add_provider(ddo.asset_id, provider, acc)
logging.info(f'is {provider} set as did provider: '
f'{keeper.did_registry.is_did_provider(ddo.asset_id, provider)}')
_providers = keeper.did_registry.get_did_providers(ddo.asset_id)
access_template_name = keeper.template_manager.SERVICE_TO_TEMPLATE_NAME['access']
template_id = keeper.template_manager.create_template_id(access_template_name)
approved = keeper.template_manager.is_template_approved(template_id)
print(f'agreement template approved: {approved}')
template_data = keeper.template_manager.get_template(template_id)
print(f'access agreement template: {template_data}')
cons_ocn = Ocean()
consumer_account = get_account(0)
# sign agreement using the registered asset did above
service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS)
# This will send the order request to Brizo which in turn will execute the agreement on-chain
cons_ocn.accounts.request_tokens(consumer_account, 10)
sa = ServiceAgreement.from_json(service.as_dictionary())
agreement_id = ''
if not agreement_id:
agreement_id = cons_ocn.assets.order(
did, sa.index, consumer_account)
logging.info('placed order: %s, %s', did, agreement_id)
event = keeper.agreement_manager.subscribe_agreement_created(
agreement_id, 60, None, (), wait=True
)
assert event, "Agreement event is not found, check the keeper node's logs"
logging.info(f'Got agreement event, next: lock reward condition')
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 60, None, (), wait=True
)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
logging.info('Got lock reward event, next: wait for the access condition..')
event = keeper.access_secret_store_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True
)
logging.info(f'Got access event {event}')
i = 0
while ocn.agreements.is_access_granted(
agreement_id, did, consumer_account.address) is not True and i < 30:
time.sleep(1)
i += 1
assert ocn.agreements.is_access_granted(agreement_id, did, consumer_account.address)
ocn.assets.consume(
agreement_id,
did,
sa.index,
consumer_account,
config.downloads_path)
logging.info('Success buying asset.')
event = keeper.escrow_reward_condition.subscribe_condition_fulfilled(
agreement_id,
30,
None,
(),
wait=True
)
assert event, 'no event for EscrowReward.Fulfilled'
logging.info(f'got EscrowReward.FULFILLED event: {event}')
logging.info('Done buy asset.')
def test_ocean_assets_download_indexes(publisher_ocean_instance, metadata):
"""Tests different values of indexes that raise AssertionError."""
publisher = get_publisher_wallet()
metadata_copy = metadata.copy()
data_provider = DataServiceProvider
ddo = publisher_ocean_instance.assets.create(metadata_copy, publisher)
wait_for_ddo(publisher_ocean_instance, ddo.did)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
config = Config(os.getenv(ENV_CONFIG_FILE))
index = range(3)
if config["util"].getboolean("typecheck"):
with pytest.raises(TypeError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
index,
)
else:
with pytest.raises(AssertionError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
index,
)
index = -1
with pytest.raises(AssertionError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
index,
)
index = 4
with pytest.raises(AssertionError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
index,
)
