def setup_network(config_file=None):
config = Config(filename=config_file) if config_file else get_config()
keeper_url = config.keeper_url
artifacts_path = get_keeper_path(config)
ContractHandler.set_artifacts_path(artifacts_path)
if keeper_url.startswith('http'):
provider = CustomHTTPProvider
elif keeper_url.startswith('wss'):
provider = WebsocketProvider
else:
raise AssertionError(f'Unsupported network url {keeper_url}. Must start with http or wss.')
Web3Provider.init_web3(provider=provider(keeper_url))
from web3.middleware import geth_poa_middleware
Web3Provider.get_web3().middleware_stack.inject(geth_poa_middleware, layer=0)
init_account_envvars()
account = get_account(0)
if account is None:
raise AssertionError(f'Ocean Provider cannot run without a valid '
f'ethereum account. Account address was not found in the environment'
f'variable `PROVIDER_ADDRESS`. Please set the following environment '
f'variables and try again: `PROVIDER_ADDRESS`, [`PROVIDER_PASSWORD`, '
f'and `PROVIDER_KEYFILE` or `PROVIDER_ENCRYPTED_KEY`] or `PROVIDER_KEY`.')
if not account._private_key and not (account.password and account._encrypted_key):
raise AssertionError(f'Ocean Provider cannot run without a valid '
f'ethereum account with either a `PROVIDER_PASSWORD` '
f'and `PROVIDER_KEYFILE`/`PROVIDER_ENCRYPTED_KEY` '
f'or private key `PROVIDER_KEY`. Current account has password {account.password}, '
f'keyfile {account.key_file}, encrypted-key {account._encrypted_key} '
f'and private-key {account._private_key}.')
def test_create_data_asset(publisher_ocean_instance, consumer_ocean_instance):
"""
Setup accounts and asset, register this asset on Aquarius (MetaData store)
"""
pub_ocn = publisher_ocean_instance
cons_ocn = consumer_ocean_instance
logging.debug("".format())
sample_ddo_path = get_resource_path('ddo', 'ddo_sa_sample.json')
assert sample_ddo_path.exists(), "{} does not exist!".format(
sample_ddo_path)
##########################################################
# Setup 2 accounts
##########################################################
aquarius_acct = pub_ocn.main_account
consumer_acct = cons_ocn.main_account
# ensure Ocean token balance
if pub_ocn.accounts.balance(aquarius_acct).ocn == 0:
rcpt = pub_ocn.accounts.request_tokens(aquarius_acct, 200)
Web3Provider.get_web3().eth.waitForTransactionReceipt(rcpt)
if cons_ocn.accounts.balance(consumer_acct).ocn == 0:
rcpt = cons_ocn.accounts.request_tokens(consumer_acct, 200)
Web3Provider.get_web3().eth.waitForTransactionReceipt(rcpt)
# You will need some token to make this transfer!
assert pub_ocn.accounts.balance(aquarius_acct).ocn > 0
assert cons_ocn.accounts.balance(consumer_acct).ocn > 0
##########################################################
# Create an Asset with valid metadata
##########################################################
asset = DDO(json_filename=sample_ddo_path)
##########################################################
# List currently published assets
##########################################################
meta_data_assets = pub_ocn.assets.search('')
if meta_data_assets:
print("Currently registered assets:")
print(meta_data_assets)
if asset.did in meta_data_assets:
pub_ocn.assets.resolve(asset.did)
pub_ocn.assets.retire(asset.did)
# Publish the metadata
new_asset = pub_ocn.assets.create(asset.metadata, aquarius_acct)
# get_asset_metadata only returns 'main' key, is this correct?
published_metadata = cons_ocn.assets.resolve(new_asset.did)
assert published_metadata
# only compare top level keys
assert sorted(list(
asset.metadata['main'].keys())).remove('files') == sorted(
list(published_metadata.metadata.keys())).remove('encryptedFiles')
publisher_ocean_instance.assets.retire(new_asset.did)
def setup_agreements_environment():
consumer_acc = get_consumer_account()
publisher_acc = get_publisher_account()
keeper = Keeper.get_instance()
ddo = get_ddo_sample()
ddo._did = DID.did({"0": "0x12341234"})
keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url='aquarius:5000',
account=publisher_acc,
providers=None)
registered_ddo = ddo
asset_id = registered_ddo.asset_id
service_agreement = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS,
ddo)
agreement_id = ServiceAgreement.create_new_agreement_id()
price = service_agreement.get_price()
(lock_cond_id, access_cond_id,
escrow_cond_id) = service_agreement.generate_agreement_condition_ids(
agreement_id, asset_id, consumer_acc.address, publisher_acc.address,
keeper)
return (
keeper,
publisher_acc,
consumer_acc,
agreement_id,
asset_id,
price,
service_agreement,
(lock_cond_id, access_cond_id, escrow_cond_id),
)
def get_agreement_block_time(agreement_id):
# get starting time from the on-chain agreement's blocknumber
keeper = keeper_instance()
w3 = Web3Provider.get_web3()
agreement = keeper.agreement_manager.get_agreement(agreement_id)
block_time = w3.eth.getBlock(agreement.block_number_updated).timestamp
return int(block_time)
def test_init_events_monitor(keeper, web3, storage_path, provider_account):
events_monitor = ProviderEventsMonitor(keeper, web3, storage_path,
provider_account)
assert events_monitor.last_n_blocks == events_monitor.LAST_N_BLOCKS
assert (Web3Provider.get_web3().eth.blockNumber -
events_monitor.latest_block) < 5
assert events_monitor.last_processed_block == 0
def get_instance(keeper, storage_path, account):
if not ProviderEventsMonitor._instance or \
ProviderEventsMonitor._instance.provider_account != account:
ProviderEventsMonitor._instance = ProviderEventsMonitor(
keeper, Web3Provider.get_web3(), storage_path, account)
return ProviderEventsMonitor._instance
def check(self, token):
"""
:param token: hex str consist of signature and timestamp
:return: hex str ethereum address
"""
parts = token.split('-')
if len(parts) < 2:
return '0x0'
sig, timestamp = parts
if self._get_timestamp() > (int(timestamp) + self._get_expiration()):
return '0x0'
message = self._get_message(timestamp)
address = self._keeper.personal_ec_recover(
Web3Provider.get_web3().sha3(text=message), sig)
return Web3Provider.get_web3().toChecksumAddress(address)
def register_compute_asset():
# get ocean instance
config = ExampleConfig.get_config()
ConfigProvider.set_config(config)
ocean = Ocean()
keeper = Keeper.get_instance()
consumer_account = get_account(0)
publisher_account = get_account(1)
# get descriptor for compute service
compute_descriptor = build_compute_descriptor(ocean)
# create an asset with compute service
ddo = ocean.assets.create(example_metadata.metadata,
publisher_account,
providers=[config.provider_address],
use_secret_store=False,
service_descriptors=[compute_descriptor])
event = keeper.did_registry.subscribe_to_event(
keeper.did_registry.DID_REGISTRY_EVENT_NAME,
15,
event_filter={
'_did': Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
'_owner': publisher_account.address
},
wait=True)
assert event, 'There was a problem registering an asset'
logging.info(f'Registered asset: did={ddo.did}')
# buy an asset access
agreement_id = ocean.compute.order(ddo.did, consumer_account)
event = keeper.compute_execution_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True)
assert event, 'There was a problem creating an agreement'
logging.info(f'Created asset agreement: agreement_id={agreement_id}')
# supply metadata describing the algorithm to run against the dataset
# you can also use an algorithm published as an Ocean asset instead
algo_meta = AlgorithmMetadata(example_metadata.algo_metadata)
# whether to publish the algorithm results as an Ocean assets
output_dict = {
'publishOutput': False,
'publishAlgorithmLog': False,
}
# start the compute job
job_id = ocean.compute.start(agreement_id,
consumer_account,
algorithm_meta=algo_meta,
output=output_dict)
logging.info(f'Started compute job: job_id={job_id}')
# query operator service for job status
status = ocean.compute.status(agreement_id, job_id, consumer_account)
logging.info(f'Job status: {status}')
def refund_reward(event, agreement_id, did, service_agreement, price, consumer_account,
publisher_address, condition_ids, escrow_condition_id):
"""
Refund the reward to the publisher address.
:param event: AttributeDict with the event data.
:param agreement_id: id of the agreement, hex str
:param did: DID, str
:param service_agreement: ServiceAgreement instance
:param price: Asset price, int
:param consumer_account: Account instance of the consumer
:param publisher_address: ethereum account address of publisher, hex str
:param condition_ids: is a list of bytes32 content-addressed Condition IDs, bytes32
:param escrow_condition_id: hex str the id of escrow reward condition at this `agreement_id`
"""
logger.debug(f"trigger refund (agreement {agreement_id}) after event {event}.")
if Keeper.get_instance().condition_manager.get_condition_state(escrow_condition_id) > 1:
logger.debug(
f'escrow reward condition already fulfilled/aborted: '
f'agreementId={agreement_id}, escrow reward conditionId={escrow_condition_id},'
f' publisher={publisher_address}'
)
return
access_id, lock_id = condition_ids[:2]
name_to_parameter = {param.name: param for param in
service_agreement.condition_by_name['escrowReward'].parameters}
document_id = add_0x_prefix(name_to_parameter['_documentId'].value)
asset_id = add_0x_prefix(did_to_id(did))
did_owner = Keeper.get_instance().agreement_manager.get_agreement_did_owner(agreement_id)
assert document_id == asset_id, f'document_id {document_id} <=> asset_id {asset_id} mismatch.'
assert price == service_agreement.get_price(), 'price mismatch.'
try:
escrow_condition = Keeper.get_instance().escrow_reward_condition
tx_hash = escrow_condition.fulfill(
agreement_id,
price,
Web3Provider.get_web3().toChecksumAddress(did_owner),
consumer_account.address,
lock_id,
access_id,
consumer_account
)
process_tx_receipt(
tx_hash,
getattr(escrow_condition.contract.events, escrow_condition.FULFILLED_EVENT)(),
'EscrowReward.Fulfilled'
)
except Exception as e:
logger.error(
f'Error when doing escrow_reward_condition.fulfills (agreementId {agreement_id}): {e}',
exc_info=1)
raise e
def get(self, account):
"""
:param account: Account instance signing the token
:return: hex str the token generated/signed by account
"""
_message, _time = self._get_message_and_time()
msg_hash = Web3Provider.get_web3().sha3(text=_message)
try:
prefixed_msg_hash = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(msg_hash), account)
return f'{prefixed_msg_hash}-{_time}'
except Exception as e:
logging.error(f'Error signing token: {str(e)}')
def ocean_agreements():
keeper = Keeper.get_instance()
w3 = Web3Provider.get_web3()
did_resolver = Mock()
ddo = get_ddo_sample()
service = ddo.get_service(ServiceTypes.ASSET_ACCESS)
service.update_value(
ServiceAgreementTemplate.TEMPLATE_ID_KEY,
w3.toChecksumAddress("0x00bd138abd70e2f00903268f3db08f2d25677c9e"))
did_resolver.resolve = MagicMock(return_value=ddo)
# consumer_class = Mock
# consumer_class.download = MagicMock(return_value='')
return OceanAgreements(keeper, did_resolver, AssetConsumer, AssetExecutor,
ConfigProvider.get_config())
def fulfill_escrow_reward_condition(event, agreement_id, service_agreement,
price, consumer_address, publisher_account,
condition_ids, escrow_condition_id):
"""
:param event: AttributeDict with the event data.
:param agreement_id: id of the agreement, hex str
:param service_agreement: ServiceAgreement instance
:param price: Asset price, int
:param consumer_address: ethereum account address of consumer, hex str
:param publisher_account: Account instance of the publisher
:param condition_ids: is a list of bytes32 content-addressed Condition IDs, bytes32
:param escrow_condition_id: hex str the id of escrow reward condition at this `agreement_id`
:return:
"""
if not event:
logger.warning(f'`fulfill_escrow_reward_condition` got empty event: '
f'event listener timed out.')
return
keeper = Keeper.get_instance()
if keeper.condition_manager.get_condition_state(escrow_condition_id) > 1:
logger.debug(
f'escrow reward condition already fulfilled/aborted: '
f'agreementId={agreement_id}, escrow reward conditionId={escrow_condition_id}'
)
return
logger.debug(
f"release reward (agreement {agreement_id}) after event {event}.")
access_id, lock_id = condition_ids[:2]
logger.debug(f'fulfill_escrow_reward_condition: '
f'agreementId={agreement_id}'
f'price={price}, {type(price)}'
f'consumer={consumer_address},'
f'publisher={publisher_account.address},'
f'conditionIds={condition_ids}')
assert price == service_agreement.get_price(), 'price mismatch.'
assert isinstance(
price, int), f'price expected to be int type, got type "{type(price)}"'
time.sleep(5)
keeper = Keeper.get_instance()
did_owner = keeper.agreement_manager.get_agreement_did_owner(agreement_id)
args = (agreement_id, price,
Web3Provider.get_web3().toChecksumAddress(did_owner),
consumer_address, lock_id, access_id, publisher_account)
process_fulfill_condition(args, keeper.escrow_reward_condition,
escrow_condition_id, logger, keeper, 10)
def __init__(self, address, abi_path=None, abi=None):
name = self.contract_name
assert name, 'contract_name property needs to be implemented in subclasses.'
if not abi_path and not abi:
abi_path = ContractHandler.artifacts_path
if abi_path and not abi:
abi = CustomContractBase.read_abi_from_file(name, abi_path)['abi']
contract = Web3Provider.get_web3().eth.contract(address=address,
abi=abi)
ContractHandler.set(name, contract)
ContractBase.__init__(self, name)
assert self.contract == contract
assert self.contract_concise is not None
assert self.address == address
def _get_agreement_actor_event(self,
agreement_id,
from_block=0,
to_block='latest'):
_filter = {
'agreementId': Web3Provider.get_web3().toBytes(hexstr=agreement_id)
}
event_filter = EventFilter(
self.agreement_manager.AGREEMENT_ACTOR_ADDED_EVENT,
self.agreement_manager._get_contract_agreement_actor_added_event(),
_filter,
from_block=from_block,
to_block=to_block)
event_filter.set_poll_interval(0.5)
return event_filter
def _get_agreement_actor_event(keeper,
agreement_id,
from_block=0,
to_block='latest'):
_filter = {
'agreementId': Web3Provider.get_web3().toBytes(hexstr=agreement_id)
}
event_filter = EventFilter(
keeper.agreement_manager.AGREEMENT_ACTOR_ADDED_EVENT,
keeper.agreement_manager.get_event_filter_for_agreement_actor(
None).event,
_filter,
from_block=from_block,
to_block=to_block)
event_filter.set_poll_interval(0.5)
return event_filter
def send(self,
did,
agreement_id,
service_index,
signature,
consumer_account,
auto_consume=False):
"""
Send a signed service agreement to the publisher Brizo instance to
consume/access the service.
:param did: str representation fo the asset DID. Use this to retrieve the asset DDO.
:param agreement_id: 32 bytes identifier created by the consumer and will be used
on-chain for the executed agreement.
:param service_index: int identifies the specific service in
the ddo to use in this agreement.
:param signature: str the signed agreement message hash which includes
conditions and their parameters values and other details of the agreement.
:param consumer_account: Account instance of the consumer
:param auto_consume: boolean tells this function wether to automatically trigger
consuming the asset upon receiving access permission
:raises OceanInitializeServiceAgreementError: on failure
:return: bool
"""
asset = self._asset_resolver.resolve(did)
service_agreement = asset.get_service_by_index(service_index)
# subscribe to events related to this agreement_id before sending the request.
logger.debug(
f'Registering service agreement with id: {agreement_id}, auto-consume {auto_consume}'
)
# TODO: refactor this to use same code in `create`
publisher_address = self._keeper.did_registry.get_did_owner(
asset.asset_id)
condition_ids = service_agreement.generate_agreement_condition_ids(
agreement_id, asset.asset_id, consumer_account.address,
publisher_address, self._keeper)
from_block = Web3Provider.get_web3().eth.blockNumber
self._process_consumer_agreement_events(
agreement_id, did, service_agreement, consumer_account,
condition_ids, publisher_address, from_block, auto_consume)
return BrizoProvider.get_brizo().initialize_service_agreement(
did, agreement_id, service_index, signature,
consumer_account.address, service_agreement.endpoints.purchase)
def release_reward(self, agreement_id, amount, account):
"""
Release reward condition.
:param agreement_id: id of the agreement, hex str
:param amount: Amount of tokens, int
:param account: Account
:return:
"""
agreement_values = self._keeper.agreement_manager.get_agreement(
agreement_id)
consumer = self._keeper.get_agreement_consumer(agreement_id)
owner = agreement_values.owner
lock_id, access_id = agreement_values.condition_ids[:2]
tx_hash = self._keeper.escrow_reward_condition.fulfill(
agreement_id, amount,
Web3Provider.get_web3().toChecksumAddress(owner), consumer,
lock_id, access_id, account)
receipt = self._keeper.escrow_reward_condition.get_tx_receipt(tx_hash)
return bool(receipt and receipt.status == 1)
def _verify_service_agreement_signature(self,
did,
agreement_id,
service_index,
consumer_address,
signature,
ddo=None):
"""
Verify service agreement signature.
Verify that the given signature is truly signed by the `consumer_address`
and represents this did's service agreement..
:param did: DID, str
:param agreement_id: id of the agreement, hex str
:param service_index: identifier of the service inside the asset DDO, str
:param consumer_address: ethereum account address of consumer, hex str
:param signature: Signature, str
:param ddo: DDO instance
:return: True if signature is legitimate, False otherwise
:raises: ValueError if service is not found in the ddo
:raises: AssertionError if conditions keys do not match the on-chain conditions keys
"""
if not ddo:
ddo = self._asset_resolver.resolve(did)
service_agreement = ddo.get_service_by_index(service_index)
agreement_hash = service_agreement.get_service_agreement_hash(
agreement_id, ddo.asset_id, consumer_address,
Web3Provider.get_web3().toChecksumAddress(ddo.proof['creator']),
self._keeper)
recovered_address = self._keeper.personal_ec_recover(
agreement_hash, signature)
is_valid = (recovered_address == consumer_address)
if not is_valid:
logger.warning(
f'Agreement signature failed: agreement hash is {agreement_hash.hex()}'
)
return is_valid
def run_events_monitor():
setup_logging()
config = get_config()
keeper_url = config.keeper_url
artifacts_path = get_keeper_path(config)
storage_path = config.get('resources',
'storage.path',
fallback='./provider-events-monitor.db')
ContractHandler.set_artifacts_path(artifacts_path)
web3 = Web3Provider.get_web3(keeper_url)
keeper = Keeper.get_instance()
init_account_envvars()
account = get_account(0)
if account is None:
raise AssertionError(
f'Provider events monitor cannot run without a valid '
f'ethereum account. Account address was not found in the environment'
f'variable `PROVIDER_ADDRESS`. Please set the following environment '
f'variables and try again: `PROVIDER_ADDRESS`, [`PROVIDER_PASSWORD`, '
f'and `PROVIDER_KEYFILE` or `PROVIDER_ENCRYPTED_KEY`] or `PROVIDER_KEY`.'
)
if not account._private_key and not (account.password
and account._encrypted_key):
raise AssertionError(
f'Provider events monitor cannot run without a valid '
f'ethereum account with either a `PROVIDER_PASSWORD` '
f'and `PROVIDER_KEYFILE`/`PROVIDER_ENCRYPTED_KEY` '
f'or private key `PROVIDER_KEY`. Current account has password {account.password}, '
f'keyfile {account.key_file}, encrypted-key {account._encrypted_key} '
f'and private-key {account._private_key}.')
monitor = ProviderEventsMonitor(keeper, web3, storage_path, account)
monitor.start_agreement_events_monitor()
while True:
time.sleep(5)
def buy_asset():
"""
Requires all ocean services running.
"""
setup_logging(default_level=logging.INFO)
ConfigProvider.set_config(ExampleConfig.get_config())
config = ConfigProvider.get_config()
providers = {
'duero': '0xfEF2d5e1670342b9EF22eeeDcb287EC526B48095',
'nile': '0x4aaab179035dc57b35e2ce066919048686f82972'
}
# make ocean instance
ocn = Ocean()
acc = get_account(1)
if not acc:
acc = ([acc for acc in ocn.accounts.list() if acc.password] or ocn.accounts.list())[0]
Diagnostics.verify_contracts()
metadata = example_metadata.metadata.copy()
metadata['main']['dateCreated'] = get_timestamp()
keeper = Keeper.get_instance()
# Register ddo
did = ''
if did:
ddo = ocn.assets.resolve(did)
logging.info(f'using ddo: {did}')
else:
ddo = ocn.assets.create(metadata, acc, providers=[], use_secret_store=True)
assert ddo is not None, f'Registering asset on-chain failed.'
did = ddo.did
logging.info(f'registered ddo: {did}')
# ocn here will be used only to publish the asset. Handling the asset by the publisher
# will be performed by the Brizo server running locally
test_net = os.environ.get('TEST_NET', '')
if test_net.startswith('nile'):
provider = keeper.did_registry.to_checksum_address(providers['nile'])
elif test_net.startswith('duero'):
provider = keeper.did_registry.to_checksum_address(providers['duero'])
else:
provider = '0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0'
# Wait for did registry event
event = keeper.did_registry.subscribe_to_event(
keeper.did_registry.DID_REGISTRY_EVENT_NAME,
30,
event_filter={
'_did': Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
'_owner': acc.address},
wait=True
)
if not event:
logging.warning(f'Failed to get the did registry event for asset with did {did}.')
assert keeper.did_registry.get_block_number_updated(ddo.asset_id) > 0, \
f'There is an issue in registering asset {did} on-chain.'
keeper.did_registry.add_provider(ddo.asset_id, provider, acc)
logging.info(f'is {provider} set as did provider: '
f'{keeper.did_registry.is_did_provider(ddo.asset_id, provider)}')
_providers = keeper.did_registry.get_did_providers(ddo.asset_id)
access_template_name = keeper.template_manager.SERVICE_TO_TEMPLATE_NAME['access']
template_id = keeper.template_manager.create_template_id(access_template_name)
approved = keeper.template_manager.is_template_approved(template_id)
print(f'agreement template approved: {approved}')
template_data = keeper.template_manager.get_template(template_id)
print(f'access agreement template: {template_data}')
cons_ocn = Ocean()
consumer_account = get_account(0)
# sign agreement using the registered asset did above
service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS)
# This will send the order request to Brizo which in turn will execute the agreement on-chain
cons_ocn.accounts.request_tokens(consumer_account, 10)
sa = ServiceAgreement.from_json(service.as_dictionary())
agreement_id = ''
if not agreement_id:
agreement_id = cons_ocn.assets.order(
did, sa.index, consumer_account)
logging.info('placed order: %s, %s', did, agreement_id)
event = keeper.agreement_manager.subscribe_agreement_created(
agreement_id, 60, None, (), wait=True
)
assert event, "Agreement event is not found, check the keeper node's logs"
logging.info(f'Got agreement event, next: lock reward condition')
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id, 60, None, (), wait=True
)
assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs"
logging.info('Got lock reward event, next: wait for the access condition..')
event = keeper.access_secret_store_condition.subscribe_condition_fulfilled(
agreement_id, 15, None, (), wait=True
)
logging.info(f'Got access event {event}')
i = 0
while ocn.agreements.is_access_granted(
agreement_id, did, consumer_account.address) is not True and i < 30:
time.sleep(1)
i += 1
assert ocn.agreements.is_access_granted(agreement_id, did, consumer_account.address)
ocn.assets.consume(
agreement_id,
did,
sa.index,
consumer_account,
config.downloads_path)
logging.info('Success buying asset.')
event = keeper.escrow_reward_condition.subscribe_condition_fulfilled(
agreement_id,
30,
None,
(),
wait=True
)
assert event, 'no event for EscrowReward.Fulfilled'
logging.info(f'got EscrowReward.FULFILLED event: {event}')
logging.info('Done buy asset.')
def web3():
return Web3Provider.get_web3(get_config().keeper_url)
def test_buy_asset(consumer_ocean_instance, publisher_ocean_instance):
config = ExampleConfig.get_config()
ConfigProvider.set_config(config)
keeper = Keeper.get_instance()
# :TODO: enable the actual SecretStore
# SecretStoreProvider.set_secret_store_class(SecretStore)
w3 = Web3Provider.get_web3()
pub_acc = get_publisher_account()
# Register ddo
ddo = get_registered_ddo(publisher_ocean_instance, pub_acc)
assert isinstance(ddo, DDO)
# ocn here will be used only to publish the asset. Handling the asset by the publisher
# will be performed by the Brizo server running locally
cons_ocn = consumer_ocean_instance
# restore the http client because we want the actual Brizo server to do the work
# not the BrizoMock.
# Brizo.set_http_client(requests)
consumer_account = get_consumer_account()
downloads_path_elements = len(
os.listdir(
consumer_ocean_instance._config.downloads_path)) if os.path.exists(
consumer_ocean_instance._config.downloads_path) else 0
# sign agreement using the registered asset did above
service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS)
sa = ServiceAgreement.from_json(service.as_dictionary())
# This will send the consume request to Brizo which in turn will execute the agreement on-chain
cons_ocn.accounts.request_tokens(consumer_account, 100)
agreement_id = cons_ocn.assets.order(ddo.did,
sa.index,
consumer_account,
auto_consume=False)
event_wait_time = 10
event = keeper.escrow_access_secretstore_template.subscribe_agreement_created(
agreement_id,
event_wait_time,
log_event(
keeper.escrow_access_secretstore_template.AGREEMENT_CREATED_EVENT),
(),
wait=True)
assert event, 'no event for EscrowAccessSecretStoreTemplate.AgreementCreated'
event = keeper.lock_reward_condition.subscribe_condition_fulfilled(
agreement_id,
event_wait_time,
log_event(keeper.lock_reward_condition.FULFILLED_EVENT), (),
wait=True)
assert event, 'no event for LockRewardCondition.Fulfilled'
# give access
publisher_ocean_instance.agreements.conditions.grant_access(
agreement_id, ddo.did, consumer_account.address, pub_acc)
event = keeper.access_secret_store_condition.subscribe_condition_fulfilled(
agreement_id,
event_wait_time,
log_event(keeper.access_secret_store_condition.FULFILLED_EVENT), (),
wait=True)
assert event, 'no event for AccessSecretStoreCondition.Fulfilled'
assert cons_ocn.agreements.is_access_granted(agreement_id, ddo.did,
consumer_account.address)
assert cons_ocn.assets.consume(agreement_id, ddo.did, sa.index,
consumer_account, config.downloads_path)
assert len(os.listdir(
config.downloads_path)) == downloads_path_elements + 1
# Check that we can consume only an specific file in passing the index.
assert cons_ocn.assets.consume(agreement_id, ddo.did, sa.index,
consumer_account, config.downloads_path, 2)
assert len(os.listdir(
config.downloads_path)) == downloads_path_elements + 1
with pytest.raises(AssertionError):
cons_ocn.assets.consume(agreement_id, ddo.did, sa.index,
consumer_account, config.downloads_path, -2)
with pytest.raises(AssertionError):
cons_ocn.assets.consume(agreement_id, ddo.did, sa.index,
consumer_account, config.downloads_path, 3)
# decrypt the contentUrls using the publisher account instead of consumer account.
# if the secret store is working and ACL check is enabled, this should fail
# since SecretStore decrypt will fail the checkPermissions check
try:
cons_ocn.assets.consume(agreement_id, ddo.did, sa.index, pub_acc,
config.downloads_path)
except RPCError:
print('hooray, secret store is working as expected.')
publisher_ocean_instance.agreements.conditions.release_reward(
agreement_id, sa.get_price(), pub_acc)
event = keeper.escrow_reward_condition.subscribe_condition_fulfilled(
agreement_id,
event_wait_time + 20,
log_event(keeper.escrow_reward_condition.FULFILLED_EVENT), (),
wait=True)
assert event, 'no event for EscrowReward.Fulfilled'
assert w3.toHex(event.args['_agreementId']) == agreement_id
def order(self,
did,
consumer_account,
algorithm_did=None,
algorithm_meta=None,
output=None,
provider_address=None):
"""
:param did:
:param consumer_account:
:param algorithm_did: str -- the asset did (of `algorithm` type) which consist of `did:op:` and
the assetId hex str (without `0x` prefix)
:param algorithm_meta: `AlgorithmMetadata` instance -- metadata about the algorithm being run if
`algorithm` is being used. This is ignored when `algorithm_did` is specified.
:param output: Output object to be used in publishing mechanism
:param provider_address: ethereum account address of provider (optional)
:return:
"""
assert isinstance(consumer_account, Account), \
f'Expected `consumer_account` of type `Account`, ' \
f'got type `{type(consumer_account)}` and value `{consumer_account}'
if algorithm_meta:
assert isinstance(algorithm_meta, AlgorithmMetadata), \
f'Expected `algorithm_meta` of type `AlgorithmMetadata`, ' \
f'got type `{type(algorithm_meta)}` and value `{algorithm_meta}`.'
assert algorithm_meta.is_valid(), f'`algorithm_meta` seems invalid'
ddo = self._did_resolver.resolve(did)
service = ddo.get_service(ServiceTypes.CLOUD_COMPUTE)
if not service:
raise ValueError(
f'order of compute service on asset {did} failed: '
f'this asset does not have a service of type {ServiceTypes.CLOUD_COMPUTE}.'
)
agreement_id = self._agreements.new()
logger.debug(
f'about to request create `{service.type}` service agreement: {agreement_id}'
)
# creating agreement will automatically fulfill the LockRewardCondition if the agreement is successful
self._agreements.create(did,
service.index,
agreement_id,
None,
consumer_account.address,
consumer_account,
provider_address,
auto_consume=False)
if agreement_id and (algorithm_meta or algorithm_did):
def _refund_callback(_price, _publisher_address, _condition_ids,
_service_agreement):
def do_refund(_agreement_id, _did, _consumer_account, *_):
refund_reward(None, _agreement_id, _did,
_service_agreement, _price,
_consumer_account, _publisher_address,
_condition_ids, _condition_ids[2])
return do_refund
def run_compute_job(_, _agreement_id, _did, _account, _algo_did,
_algo_meta, _output):
if _algo_did or _algo_meta:
return self.start(_agreement_id, _account, _algo_did,
_algo_meta, _output)
publisher_address = Web3Provider.get_web3().toChecksumAddress(
ddo.publisher)
condition_ids = service.generate_agreement_condition_ids(
agreement_id, ddo.asset_id, consumer_account.address,
publisher_address, self._keeper)
self._keeper.compute_execution_condition.subscribe_condition_fulfilled(
agreement_id,
max(service.condition_by_name['computeExecution'].timeout,
300),
run_compute_job, (agreement_id, did, consumer_account,
algorithm_did, algorithm_meta, output),
timeout_callback=_refund_callback(service.get_price(),
publisher_address,
condition_ids, service))
return agreement_id
def create(self,
metadata,
publisher_account,
service_descriptors=None,
providers=None,
use_secret_store=True):
"""
Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store (
Aquarius).
:param metadata: dict conforming to the Metadata accepted by Ocean Protocol.
:param publisher_account: Account of the publisher registering this asset
:param service_descriptors: list of ServiceDescriptor tuples of length 2.
The first item must be one of ServiceTypes and the second
item is a dict of parameters and values required by the service
:param providers: list of addresses of providers of this asset (a provider is
an ethereum account that is authorized to provide asset services)
:param use_secret_store: bool indicate whether to use the secret store directly for
encrypting urls (Uses Brizo provider service if set to False)
:return: DDO instance
"""
assert isinstance(
metadata,
dict), f'Expected metadata of type dict, got {type(metadata)}'
assert service_descriptors is None or isinstance(service_descriptors, list), \
f'bad type of `service_descriptors` {type(service_descriptors)}'
# if not metadata or not Metadata.validate(metadata):
# raise OceanInvalidMetadata('Metadata seems invalid. Please make sure'
# ' the required metadata values are filled in.')
# copy metadata so we don't change the original
metadata_copy = copy.deepcopy(metadata)
asset_type = metadata_copy['main']['type']
assert asset_type in (
'dataset',
'algorithm'), f'Invalid/unsupported asset type {asset_type}'
service_descriptors = service_descriptors or []
brizo = BrizoProvider.get_brizo()
services = self._process_service_descriptors(service_descriptors,
metadata_copy,
publisher_account)
stype_to_service = {s.type: s for s in services}
checksum_dict = dict()
for service in services:
checksum_dict[str(service.index)] = checksum(service.main)
# Create a DDO object
ddo = DDO()
# Adding proof to the ddo.
ddo.add_proof(checksum_dict, publisher_account)
# Generating the did and adding to the ddo.
did = ddo.assign_did(DID.did(ddo.proof['checksum']))
logger.debug(f'Generating new did: {did}')
# Check if it's already registered first!
if did in self._get_aquarius().list_assets():
raise OceanDIDAlreadyExist(
f'Asset id {did} is already registered to another asset.')
md_service = stype_to_service[ServiceTypes.METADATA]
ddo_service_endpoint = md_service.service_endpoint
if '{did}' in ddo_service_endpoint:
ddo_service_endpoint = ddo_service_endpoint.replace('{did}', did)
md_service.set_service_endpoint(ddo_service_endpoint)
# Populate the ddo services
ddo.add_service(md_service)
ddo.add_service(stype_to_service[ServiceTypes.AUTHORIZATION])
access_service = stype_to_service.get(ServiceTypes.ASSET_ACCESS, None)
compute_service = stype_to_service.get(ServiceTypes.CLOUD_COMPUTE,
None)
if access_service:
access_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(access_service)
if compute_service:
compute_service.init_conditions_values(
did, {
cname: c.address
for cname, c in
self._keeper.contract_name_to_instance.items()
})
ddo.add_service(compute_service)
ddo.proof['signatureValue'] = self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)),
publisher_account)
# Add public key and authentication
ddo.add_public_key(did, publisher_account.address)
ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA)
# Setup metadata service
# First compute files_encrypted
if metadata_copy['main']['type'] == 'dataset':
assert metadata_copy['main']['files'], \
'files is required in the metadata main attributes.'
logger.debug('Encrypting content urls in the metadata.')
if not use_secret_store:
encrypt_endpoint = brizo.get_encrypt_endpoint(self._config)
files_encrypted = brizo.encrypt_files_dict(
metadata_copy['main']['files'], encrypt_endpoint,
ddo.asset_id, publisher_account.address,
self._keeper.sign_hash(
add_ethereum_prefix_and_hash_msg(ddo.asset_id),
publisher_account))
else:
files_encrypted = self._get_secret_store(publisher_account) \
.encrypt_document(
did_to_id(did),
json.dumps(metadata_copy['main']['files']),
)
# only assign if the encryption worked
if files_encrypted:
logger.debug(
f'Content urls encrypted successfully {files_encrypted}')
index = 0
for file in metadata_copy['main']['files']:
file['index'] = index
index = index + 1
del file['url']
metadata_copy['encryptedFiles'] = files_encrypted
else:
raise AssertionError('Encrypting the files failed.')
# DDO url and `Metadata` service
logger.debug(f'Generated ddo and services, DID is {ddo.did},'
f' metadata service @{ddo_service_endpoint}.')
response = None
# register on-chain
registered_on_chain = self._keeper.did_registry.register(
ddo.asset_id,
checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id),
url=ddo_service_endpoint,
account=publisher_account,
providers=providers)
if registered_on_chain is False:
logger.warning(f'Registering {did} on-chain failed.')
return None
logger.info(f'Successfully registered DDO (DID={did}) on chain.')
try:
# publish the new ddo in ocean-db/Aquarius
response = self._get_aquarius().publish_asset_ddo(ddo)
logger.info('Asset/ddo published successfully in aquarius.')
except ValueError as ve:
raise ValueError(
f'Invalid value to publish in the metadata: {str(ve)}')
except Exception as e:
logger.error(f'Publish asset in aquarius failed: {str(e)}')
if not response:
return None
return ddo
def web3_instance():
config = ExampleConfig.get_config()
return Web3Provider.get_web3(config.keeper_url)
def __init__(self, config=None):
"""
Initialize Ocean class.
>> # Make a new Ocean instance
>> ocean = Ocean({...})
This class provides the main top-level functions in ocean protocol:
* Publish assets metadata and associated services
* Each asset is assigned a unique DID and a DID Document (DDO)
* The DDO contains the asset's services including the metadata
* The DID is registered on-chain with a URL of the metadata store
to retrieve the DDO from
>> ddo = ocean.assets.create(metadata, publisher_account)
* Discover/Search assets via the current configured metadata store (Aquarius)
>> assets_list = ocean.assets.search('search text')
* Purchase asset services by choosing a service agreement from the
asset's DDO. Purchase goes through the service agreements interface
and starts by signing a service agreement then sending the signature
to the publisher's Brizo server via the `purchaseEndpoint` in the service
definition:
>> service_def_id = ddo.get_service(ServiceTypes.ASSET_ACCESS).service_definition_id
>> service_agreement_id = ocean.assets.order(did, service_def_id, consumer_account)
An instance of Ocean is parameterized by a `Config` instance.
:param config: Config instance
"""
# Configuration information for the market is stored in the Config class
# config = Config(filename=config_file, options_dict=config_dict)
if not config:
config = ConfigProvider.get_config()
self._config = config
self._web3 = Web3Provider.get_web3(self._config.keeper_url)
ContractHandler.set_artifacts_path(self._config.keeper_path)
contracts = [
'DIDRegistry', 'Dispenser', 'TemplateStoreManager', 'OceanToken',
'ConditionStoreManager', 'EscrowAccessSecretStoreTemplate',
'AgreementStoreManager', 'AgreementStoreManager',
'AccessSecretStoreCondition', 'LockRewardCondition',
'HashLockCondition', 'SignCondition', 'EscrowReward'
]
self._keeper = Keeper.get_instance(contracts)
self._did_resolver = DIDResolver(self._keeper.did_registry)
# Initialize the public sub-modules
self.tokens = OceanTokens(self._keeper)
self.accounts = OceanAccounts(self._keeper, self._config, self.tokens)
self.secret_store = OceanSecretStore(self._config)
self.templates = OceanTemplates(self._keeper, config)
self.agreements = self._make_ocean_agreements()
self.assets = OceanAssets(self._keeper, self._did_resolver,
self.agreements, AssetConsumer,
AssetExecutor, self._config)
self.services = OceanServices()
self.ocean_providers = OceanProviders(self._keeper, self._did_resolver,
self._config)
self.auth = OceanAuth(self._keeper, self._config.storage_path)
logger.debug('Squid Ocean instance initialized: ')
logger.debug(
f'\tOther accounts: {sorted([a.address for a in self.accounts.list()])}'
)
logger.debug(f'\tDIDRegistry @ {self._keeper.did_registry.address}')
def create(self,
did,
index,
agreement_id,
service_agreement_signature,
consumer_address,
account,
auto_consume=False):
"""
Execute the service agreement on-chain using keeper's ServiceAgreement contract.
The on-chain executeAgreement method requires the following arguments:
templateId, signature, consumer, hashes, timeouts, serviceAgreementId, did.
`agreement_message_hash` is necessary to verify the signature.
The consumer `signature` includes the conditions timeouts and parameters values which
is usedon-chain to verify that the values actually match the signed hashes.
:param did: str representation fo the asset DID. Use this to retrieve the asset DDO.
:param index: str identifies the specific service in
the ddo to use in this agreement.
:param agreement_id: 32 bytes identifier created by the consumer and will be used
on-chain for the executed agreement.
:param service_agreement_signature: str the signed agreement message hash which includes
conditions and their parameters values and other details of the agreement.
:param consumer_address: ethereum account address of consumer, hex str
:param account: Account instance creating the agreement. Can be either the
consumer, publisher or provider
:param auto_consume: bool
:return: dict the `executeAgreement` transaction receipt
"""
assert consumer_address and Web3Provider.get_web3().isChecksumAddress(
consumer_address), f'Invalid consumer address {consumer_address}'
assert account.address in self._keeper.accounts, \
f'Unrecognized account address {account.address}'
agreement_template_approved = self._keeper.template_manager.is_template_approved(
self._keeper.escrow_access_secretstore_template.address)
agreement_exec_template_approved = self._keeper.template_manager.is_template_approved(
self._keeper.escrow_compute_execution_template.address)
if not agreement_template_approved:
msg = (
f'The EscrowAccessSecretStoreTemplate contract at address '
f'{self._keeper.escrow_access_secretstore_template.address} is not '
f'approved and cannot be used for creating service agreements.'
)
logger.warning(msg)
raise OceanInvalidAgreementTemplate(msg)
if not agreement_exec_template_approved:
msg = (
f'The EscroComputeExecutionTemplate contract at address '
f'{self._keeper.agreement_exec_template_approved.address} is not '
f'approved and cannot be used for creating service agreements.'
)
logger.warning(msg)
raise OceanInvalidAgreementTemplate(msg)
asset = self._asset_resolver.resolve(did)
asset_id = asset.asset_id
service_agreement = asset.get_service_by_index(index)
if service_agreement.type == ServiceTypes.ASSET_ACCESS:
agreement_template = self._keeper.escrow_access_secretstore_template
elif service_agreement.type == ServiceTypes.CLOUD_COMPUTE:
agreement_template = self._keeper.escrow_compute_execution_template
else:
raise Exception(
'The agreement could not be created. Review the index of your service.'
)
if agreement_template.get_agreement_consumer(agreement_id) is not None:
raise OceanServiceAgreementExists(
f'Service agreement {agreement_id} already exists, cannot reuse '
f'the same agreement id.')
if consumer_address != account.address:
if not self._verify_service_agreement_signature(
did,
agreement_id,
index,
consumer_address,
service_agreement_signature,
ddo=asset):
raise OceanInvalidServiceAgreementSignature(
f'Verifying consumer signature failed: '
f'signature {service_agreement_signature}, '
f'consumerAddress {consumer_address}')
publisher_address = Web3Provider.get_web3().toChecksumAddress(
asset.publisher)
condition_ids = service_agreement.generate_agreement_condition_ids(
agreement_id, asset_id, consumer_address, publisher_address,
self._keeper)
time_locks = service_agreement.conditions_timelocks
time_outs = service_agreement.conditions_timeouts
success = agreement_template.create_agreement(agreement_id, asset_id,
condition_ids,
time_locks, time_outs,
consumer_address,
account)
## TODO CHECK THIS FOR THE OTHER TEMPLATE
if not success:
# success is based on tx receipt which is not reliable.
# So we check on-chain directly to see if agreement_id is there
consumer = self._keeper.escrow_access_secretstore_template.get_agreement_consumer(
agreement_id)
if consumer:
success = True
else:
event_log = self._keeper.escrow_access_secretstore_template \
.subscribe_agreement_created(
agreement_id, 15, None, (), wait=True
)
success = event_log is not None
if success:
logger.info(
f'Service agreement {agreement_id} created successfully.')
else:
logger.info(f'Create agreement "{agreement_id}" failed.')
self._log_agreement_info(asset, service_agreement, agreement_id,
service_agreement_signature,
consumer_address, account, condition_ids)
if success:
# subscribe to events related to this agreement_id
if consumer_address == account.address:
from_block = Web3Provider.get_web3().eth.blockNumber - 10
self._process_consumer_agreement_events(
agreement_id, did, service_agreement, account,
condition_ids, publisher_address, from_block, auto_consume,
service_agreement.type)
return success