def test_did_resolver_library(publisher_account, aquarius):
did_registry = keeper().did_registry
checksum_test = Web3.sha3(text='checksum')
value_test = aquarius.root_url
did_resolver = DIDResolver(keeper().did_registry)
sample_ddo_path = get_resource_path('ddo', 'ddo_sample1.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
asset1 = DDO(json_filename=sample_ddo_path)
asset1._did = DID.did({"0": "0x1098098"})
did_registry.register(asset1.asset_id,
checksum_test,
url=value_test,
account=publisher_account)
aquarius.publish_asset_ddo(asset1)
did_resolved = did_resolver.resolve(asset1.did)
assert did_resolved
assert did_resolved.did == asset1.did
with pytest.raises(ValueError):
did_resolver.resolve(asset1.asset_id)
aquarius.retire_asset_ddo(asset1.did)
def test_register_asset(publisher_ocean_instance):
logging.debug("".format())
sample_ddo_path = get_resource_path('ddo', 'ddo_sa_sample.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
##########################################################
# Setup account
##########################################################
publisher = publisher_ocean_instance.main_account
# ensure Ocean token balance
if publisher_ocean_instance.accounts.balance(publisher).ocn == 0:
publisher_ocean_instance.accounts.request_tokens(publisher, 200)
# You will need some token to make this transfer!
assert publisher_ocean_instance.accounts.balance(publisher).ocn > 0
##########################################################
# Create an asset DDO with valid metadata
##########################################################
asset = DDO(json_filename=sample_ddo_path)
##########################################################
# Register using high-level interface
##########################################################
publisher_ocean_instance.assets.create(asset.metadata, publisher)
def _get_sample_ddo(name):
sample_ddo_path = get_resource_path('ddo', name)
assert sample_ddo_path.exists(), f'{sample_ddo_path} does not exist!'
with open(sample_ddo_path) as f:
sample_ddo_json_dict = json.load(f)
return sample_ddo_json_dict
def test_did_resolver_library(publisher_ocean_instance):
ocean = publisher_ocean_instance
register_account = ocean.main_account
did_registry = keeper().did_registry
checksum_test = Web3.sha3(text='checksum')
value_test = 'http://localhost:5000'
did_resolver = DIDResolver(keeper().did_registry)
sample_ddo_path = get_resource_path('ddo', 'ddo_sample1.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
asset1 = DDO(json_filename=sample_ddo_path)
did_registry.register(asset1.did,
checksum_test,
url=value_test,
account=register_account)
ocean.assets._get_aquarius().publish_asset_ddo(asset1)
did_resolved = did_resolver.resolve(asset1.did)
assert did_resolved
assert did_resolved.did == asset1.did
with pytest.raises(ValueError):
did_resolver.resolve(asset1.asset_id)
ocean.assets._get_aquarius().retire_asset_ddo(asset1.did)
def test_create_asset_with_different_secret_store(publisher_ocean_instance):
ocn = publisher_ocean_instance
sample_ddo_path = get_resource_path('ddo', 'ddo_sa_sample.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
acct = ocn.main_account
asset = DDO(json_filename=sample_ddo_path)
my_secret_store = 'http://myownsecretstore.com'
auth_service = ServiceDescriptor.authorization_service_descriptor(
my_secret_store)
new_asset = ocn.assets.create(asset.metadata, acct, [auth_service])
assert new_asset.get_service(
ServiceTypes.AUTHORIZATION).endpoints.consume == my_secret_store
assert new_asset.get_service(ServiceTypes.ASSET_ACCESS)
assert new_asset.get_service(ServiceTypes.METADATA)
new_asset = ocn.assets.create(asset.metadata, acct)
assert new_asset.get_service(ServiceTypes.AUTHORIZATION)
assert new_asset.get_service(ServiceTypes.ASSET_ACCESS)
assert new_asset.get_service(ServiceTypes.METADATA)
access_service = ServiceDescriptor.access_service_descriptor(
2, 'purchase', 'consume', 35, '')
new_asset = ocn.assets.create(asset.metadata, acct, [access_service])
assert new_asset.get_service(ServiceTypes.AUTHORIZATION)
assert new_asset.get_service(ServiceTypes.ASSET_ACCESS)
assert new_asset.get_service(ServiceTypes.METADATA)
def test_ddo_dict():
sample_ddo_path = get_resource_path('ddo', 'ddo_sample1.json')
assert sample_ddo_path.exists(), f'{sample_ddo_path} does not exist!'
ddo1 = DDO(json_filename=sample_ddo_path)
assert len(ddo1.public_keys) == 3
assert ddo1.did == 'did:op:0c184915b07b44c888d468be85a9b28253e80070e5294b1aaed81c2f0264e429'
def test_create_data_asset(publisher_ocean_instance, consumer_ocean_instance):
"""
Setup accounts and asset, register this asset on Aquarius (MetaData store)
"""
pub_ocn = publisher_ocean_instance
cons_ocn = consumer_ocean_instance
logging.debug("".format())
sample_ddo_path = get_resource_path('ddo', 'ddo_sa_sample.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
##########################################################
# Setup 2 accounts
##########################################################
aquarius_acct = pub_ocn.main_account
consumer_acct = cons_ocn.main_account
# ensure Ocean token balance
if pub_ocn.accounts.balance(aquarius_acct).ocn == 0:
rcpt = pub_ocn.accounts.request_tokens(aquarius_acct, 200)
Web3Provider.get_web3().eth.waitForTransactionReceipt(rcpt)
if cons_ocn.accounts.balance(consumer_acct).ocn == 0:
rcpt = cons_ocn.accounts.request_tokens(consumer_acct, 200)
Web3Provider.get_web3().eth.waitForTransactionReceipt(rcpt)
# You will need some token to make this transfer!
assert pub_ocn.accounts.balance(aquarius_acct).ocn > 0
assert cons_ocn.accounts.balance(consumer_acct).ocn > 0
##########################################################
# Create an Asset with valid metadata
##########################################################
asset = DDO(json_filename=sample_ddo_path)
##########################################################
# List currently published assets
##########################################################
meta_data_assets = pub_ocn.assets.search('')
if meta_data_assets:
print("Currently registered assets:")
print(meta_data_assets)
if asset.did in meta_data_assets:
pub_ocn.assets.resolve(asset.did)
pub_ocn.assets.retire(asset.did)
# Publish the metadata
new_asset = pub_ocn.assets.create(asset.metadata, aquarius_acct)
# TODO: Ensure returned metadata equals sent!
# get_asset_metadata only returns 'base' key, is this correct?
published_metadata = cons_ocn.assets.resolve(new_asset.did)
assert published_metadata
# only compare top level keys
assert sorted(list(
asset.metadata['base'].keys())).remove('files') == sorted(
list(published_metadata.metadata['base'].keys())).remove(
'encryptedFiles')
def create_asset(ocean, publisher):
sample_ddo_path = get_resource_path("ddo", "ddo_sa_sample.json")
assert sample_ddo_path.exists(), "{} does not exist!".format(sample_ddo_path)
asset = DDO(json_filename=sample_ddo_path)
asset.metadata["main"]["files"][0]["checksum"] = str(uuid.uuid4())
my_secret_store = "http://myownsecretstore.com"
auth_service = ServiceDescriptor.authorization_service_descriptor(my_secret_store)
return ocean.assets.create(asset.metadata, publisher, [auth_service])
def test_create_asset_ddo_file():
# An asset can be created directly from a DDO .json file
sample_ddo_path = get_resource_path('ddo', 'ddo_sample1.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
asset1 = DDO(json_filename=sample_ddo_path)
assert isinstance(asset1, DDO)
assert asset1.metadata
def create_asset(publisher_ocean_instance):
ocn = publisher_ocean_instance
sample_ddo_path = get_resource_path('ddo', 'ddo_sa_sample.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
acct = ocn.main_account
asset = DDO(json_filename=sample_ddo_path)
my_secret_store = 'http://myownsecretstore.com'
auth_service = ServiceDescriptor.authorization_service_descriptor(
my_secret_store)
return ocn.assets.create(asset.metadata, acct, [auth_service])
def test_ocean_secret_store(publisher_ocean_instance):
test_document = get_resource_path('ddo', 'ddo_sample1.json')
with open(test_document, 'r') as file_handle:
metadata = json.load(file_handle)
metadata_json = json.dumps(metadata)
document_id = hashlib.sha256(
(metadata_json + secrets.token_hex(32)).encode()).hexdigest()
publisher_account = publisher_ocean_instance.main_account
encrypt_content = publisher_ocean_instance.secret_store.encrypt(
document_id, metadata_json, publisher_account)
assert metadata_json == publisher_ocean_instance.secret_store.decrypt(
document_id, encrypt_content, publisher_account)
def get_ddo_sample(datatoken_address):
did = f'did:op:{remove_0x_prefix(datatoken_address)}'
sample_ddo_path = get_resource_path('ddo', 'ddo_sa_sample.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(sample_ddo_path)
asset = DDO(json_filename=sample_ddo_path)
asset.metadata['main']['files'][0]['checksum'] = str(uuid.uuid4())
checksum_dict = dict()
for service in asset.services:
checksum_dict[str(service.index)] = checksum(service.main)
asset.add_proof(checksum_dict, get_publisher_wallet())
asset._did = did
return asset
def test_create_asset_with_different_secret_store(publisher_ocean_instance):
ocn = publisher_ocean_instance
sample_ddo_path = get_resource_path('ddo', 'ddo_sa_sample.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
acct = ocn.main_account
aqua = AquariusProvider.get_aquarius(ocn.config.aquarius_url)
aqua.retire_all_assets()
asset = DDO(json_filename=sample_ddo_path)
my_secret_store = 'http://myownsecretstore.com'
auth_service = ServiceDescriptor.authorization_service_descriptor(
my_secret_store)
asset.metadata['main']['name'] += str(datetime.now().timestamp())
new_asset = ocn.assets.create(asset.metadata, acct, [auth_service])
assert new_asset.get_service(
ServiceTypes.AUTHORIZATION).service_endpoint == my_secret_store
assert new_asset.get_service(ServiceTypes.ASSET_ACCESS)
assert new_asset.get_service(ServiceTypes.METADATA)
publisher_ocean_instance.assets.retire(new_asset.did)
access_service = ServiceDescriptor.access_service_descriptor(
{
"main": {
"name": "dataAssetAccessServiceAgreement",
"creator": '0x1234',
"price": '1',
"timeout": 3600,
"datePublished": '2019-08-30T12:19:54Z'
}
}, 'service/endpoint', '0x0011001100110011')
asset.metadata['main']['name'] += str(datetime.now().timestamp())
new_asset = ocn.assets.create(asset.metadata, acct, [access_service])
assert new_asset.get_service(ServiceTypes.AUTHORIZATION)
assert new_asset.get_service(ServiceTypes.ASSET_ACCESS)
assert new_asset.get_service(ServiceTypes.METADATA)
publisher_ocean_instance.assets.retire(new_asset.did)
asset.metadata['main']['name'] += str(datetime.now().timestamp())
new_asset = ocn.assets.create(asset.metadata, acct)
assert new_asset.get_service(ServiceTypes.AUTHORIZATION)
assert new_asset.get_service(ServiceTypes.ASSET_ACCESS)
assert new_asset.get_service(ServiceTypes.METADATA)
publisher_ocean_instance.assets.retire(new_asset.did)
def test_secret_store_encrypt_decrypt():
test_document = get_resource_path('ddo', 'ddo_sample1.json')
with open(test_document, 'r') as file_handle:
metadata = json.load(file_handle)
metadata_json = json.dumps(metadata)
document_id = hashlib.sha256(
(metadata_json + secrets.token_hex(32)).encode()).hexdigest()
print(document_id)
config = ConfigProvider.get_config()
ss_client = Mock
ss_client.publish_document = MagicMock(return_value='!!document!!')
ss_client.decrypt_document = MagicMock(return_value=metadata_json)
SecretStore.set_client(ss_client)
ss_args = (config.secret_store_url, config.parity_url,
Account('0x0000', 'aaa'))
result = SecretStore(*ss_args).encrypt_document(document_id, metadata_json)
print(result)
assert SecretStore(*ss_args).decrypt_document(document_id,
result) == metadata_json
def test_sign_and_recover(web3_instance):
w3 = web3_instance
account = get_publisher_account()
msg = 'testing-signature-and-recovery-of-signer-address'
msg_hash = w3.keccak(text=msg)
signature = Keeper.sign_hash(msg_hash, account)
address = w3.toChecksumAddress(Keeper.ec_recover(msg_hash, signature))
assert address == account.address
# Signature created on msg with the ethereum prefix. `web3.eth.account.recoverHash` does NOT
# add any prefixes to the message, so we have to add the prefix before the call.
address = w3.eth.account.recoverHash(msg_hash, signature=signature)
assert address == account.address
# Now do the opposite, sign with eth.account.sign_hash() (using prefixed msg hash),
# then recover address with Keeper.ec_recover() on the msg hash with no prefix.
with open(get_resource_path('data', 'publisher_key_file.json')) as kf:
key = kf.read()
prvkey = w3.eth.account.decrypt(key, account.password)
account_sig_prefixed = add_0x_prefix(
w3.eth.account.signHash(msg_hash, prvkey)['signature'].hex())
assert Keeper.ec_recover(
msg_hash, account_sig_prefixed).lower() == account.address.lower()
# Test specific case where message is signed by some Wallet web3 such as Metamask or
# burner wallet. Such signature uses the `web3.personal` `sign` method which adds
# `Ethereum Signed Message` prefix in a generic way, see `add_ethereum_prefix_and_hash_msg` for details.
sig = '0xa9e78d2c088c0b17a8c35b69e0dfa774692ccabed570e40502795bd41f561cf7677ed02bf4ee7967a55979d585bbf203b4a490e1d747e5a4d60a50859d816ac51b'
publisher_address = '0x903322C7E45A60d7c8C3EA236c5beA9Af86310c7'
doc_id = '028faa498d154388a89dc0dea908a4e27700920217a44abe8f1cdd64953125b8'
prefixed_hash = add_ethereum_prefix_and_hash_msg(doc_id)
recovered_address = w3.eth.account.recoverHash(prefixed_hash,
signature=sig)
assert recovered_address == publisher_address
recovered_address = Keeper.ec_recover(prefixed_hash, sig)
assert recovered_address == publisher_address
recovered_address = Keeper.personal_ec_recover(doc_id, sig)
assert recovered_address == publisher_address
def test_order(publisher_ocean_instance, consumer_ocean_instance):
consumer_account = consumer_ocean_instance.main_account
publisher_account = publisher_ocean_instance.main_account
keeper = publisher_ocean_instance.keeper
ocean = consumer_ocean_instance
BrizoProvider.set_brizo_class(Brizo)
SecretStoreProvider.set_secret_store_class(SecretStore)
sample_ddo_path = get_resource_path('ddo', 'ddo_with_compute_service.json')
old_ddo = DDO(json_filename=sample_ddo_path)
metadata = old_ddo.metadata
metadata['main']['files'][0]['checksum'] = str(uuid.uuid4())
template_name = keeper.template_manager.SERVICE_TO_TEMPLATE_NAME[ServiceTypes.CLOUD_COMPUTE]
service = old_ddo.get_service(ServiceTypes.CLOUD_COMPUTE)
brizo = BrizoProvider.get_brizo()
compute_service = ServiceDescriptor.compute_service_descriptor(
service.attributes,
brizo.get_compute_endpoint(ocean.config),
keeper.template_manager.create_template_id(template_name)
)
compute_ddo = publisher_ocean_instance.assets.create(
metadata,
publisher_account,
providers=[],
service_descriptors=[compute_service],
)
did = compute_ddo.did
_compute_ddo = publisher_ocean_instance.assets.resolve(compute_ddo.did)
algorithm_ddo_path = get_resource_path('ddo', 'ddo_sample_algorithm.json')
algo_main = DDO(json_filename=algorithm_ddo_path).metadata['main']
algo_meta_dict = algo_main['algorithm'].copy()
algo_meta_dict['url'] = algo_main['files'][0]['url']
algorithm_meta = AlgorithmMetadata(algo_meta_dict)
try:
ocean.accounts.request_tokens(publisher_account, 50*keeper.dispenser.get_scale())
except Exception as err:
print(f'Requesting tokens failed: {err}')
agreement_id = ocean.compute.order(
compute_ddo.did,
consumer_account,
None,
None
)
assert agreement_id, f'got agreementId {agreement_id}'
def fulfill_compute_execution(_, ocn_instance, agr_id, _did, cons_address, account):
ocn_instance.agreements.conditions.grant_compute(
agr_id, _did, cons_address, account)
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id,
15,
fulfill_compute_execution,
(publisher_ocean_instance, agreement_id, compute_ddo.did,
consumer_account.address, publisher_account),
from_block=0,
wait=True
)
assert event, f'lock condition event is not found.'
print(f'processed agreement and got lockReward fulfilled: '
f'{agreement_id}, {compute_ddo.did}, ')
event = keeper.compute_execution_condition.subscribe_condition_fulfilled(
agreement_id,
20,
None,
(agreement_id, compute_ddo.did, consumer_account, None, algorithm_meta),
from_block=0,
wait=True
)
assert event, f'compute execution condition event is not found.'
job_id = consumer_ocean_instance.compute.start(agreement_id, consumer_account, algorithm_meta=algorithm_meta)
assert job_id, f'expected a job id, got {job_id}'
status = consumer_ocean_instance.compute.status(agreement_id, job_id, consumer_account)
print(f'got job status: {status}')
assert status and status['ok'], f'something not right about the compute job, got status: {status}'
status = consumer_ocean_instance.compute.stop(agreement_id, job_id, consumer_account)
print(f'got job status after requesting stop: {status}')
assert status, f'something not right about the compute job, got status: {status}'
def test_compute_flow():
######
# setup
pub_wallet = get_publisher_wallet()
p_ocean_instance = get_publisher_ocean_instance()
c_ocean_instance = get_consumer_ocean_instance()
cons_ocn = c_ocean_instance
consumer_wallet = get_consumer_wallet()
######
# Publish Assets
# Dataset with compute service
sample_ddo_path = get_resource_path('ddo', 'ddo_with_compute_service.json')
old_ddo = Asset(json_filename=sample_ddo_path)
metadata = old_ddo.metadata
metadata['main']['files'][0]['checksum'] = str(uuid.uuid4())
service = old_ddo.get_service(ServiceTypes.CLOUD_COMPUTE)
compute_service = ServiceDescriptor.compute_service_descriptor(
service.attributes,
DataServiceProvider.get_url(p_ocean_instance.config))
block = p_ocean_instance.web3.eth.blockNumber
compute_ddo = p_ocean_instance.assets.create(
metadata,
pub_wallet,
service_descriptors=[compute_service],
)
did = compute_ddo.did
ddo_reg = p_ocean_instance.assets.ddo_registry()
log = ddo_reg.get_event_log(ddo_reg.EVENT_METADATA_CREATED, block,
compute_ddo.asset_id, 30)
assert log, f'no ddo created event.'
ddo = wait_for_ddo(p_ocean_instance, compute_ddo.did)
assert ddo, f'resolve did {compute_ddo.did} failed.'
_compute_ddo = p_ocean_instance.assets.resolve(compute_ddo.did)
# algorithm with download service
algorithm_ddo_path = get_resource_path('ddo', 'ddo_sample_algorithm.json')
algo_main = Asset(json_filename=algorithm_ddo_path).metadata['main']
algo_meta_dict = algo_main['algorithm'].copy()
algo_meta_dict['url'] = algo_main['files'][0]['url']
algorithm_meta = AlgorithmMetadata(algo_meta_dict)
######
# Mint tokens for dataset and assign to publisher
dt = p_ocean_instance.get_data_token(compute_ddo.data_token_address)
mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet)
######
# Give the consumer some datatokens so they can order the service
try:
tx_id = dt.transfer_tokens(consumer_wallet.address, 10, pub_wallet)
dt.verify_transfer_tx(tx_id, pub_wallet.address,
consumer_wallet.address)
except (AssertionError, Exception) as e:
print(e)
raise
######
# Order compute service from the dataset asset
order_requirements = cons_ocn.assets.order(
compute_ddo.did,
consumer_wallet.address,
service_type=ServiceTypes.CLOUD_COMPUTE)
######
# Start the order on-chain using the `order` requirements from previous step
service = compute_ddo.get_service(ServiceTypes.CLOUD_COMPUTE)
_order_tx_id = cons_ocn.assets.pay_for_service(
order_requirements.amount, order_requirements.data_token_address,
compute_ddo.did, service.index,
'0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9', consumer_wallet)
######
job_id = cons_ocn.compute.start(did,
consumer_wallet,
_order_tx_id,
nonce=order_requirements.nonce,
algorithm_meta=algorithm_meta)
assert job_id, f'expected a job id, got {job_id}'
status = cons_ocn.compute.status(did, job_id, consumer_wallet)
print(f'got job status: {status}')
assert status and status[
'ok'], f'something not right about the compute job, got status: {status}'
status = cons_ocn.compute.stop(did, job_id, consumer_wallet)
print(f'got job status after requesting stop: {status}')
assert status, f'something not right about the compute job, got status: {status}'
def _get_asset(file_name):
sample_ddo_path = get_resource_path('ddo', file_name)
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
return DDO(json_filename=sample_ddo_path)
