def setup_network(config_file=None): config = Config(filename=config_file) if config_file else get_config() keeper_url = config.keeper_url artifacts_path = get_keeper_path(config) ContractHandler.set_artifacts_path(artifacts_path) if keeper_url.startswith('http'): provider = CustomHTTPProvider elif keeper_url.startswith('wss'): provider = WebsocketProvider else: raise AssertionError(f'Unsupported network url {keeper_url}. Must start with http or wss.') Web3Provider.init_web3(provider=provider(keeper_url)) from web3.middleware import geth_poa_middleware Web3Provider.get_web3().middleware_stack.inject(geth_poa_middleware, layer=0) init_account_envvars() account = get_account(0) if account is None: raise AssertionError(f'Ocean Provider cannot run without a valid ' f'ethereum account. Account address was not found in the environment' f'variable `PROVIDER_ADDRESS`. Please set the following environment ' f'variables and try again: `PROVIDER_ADDRESS`, [`PROVIDER_PASSWORD`, ' f'and `PROVIDER_KEYFILE` or `PROVIDER_ENCRYPTED_KEY`] or `PROVIDER_KEY`.') if not account._private_key and not (account.password and account._encrypted_key): raise AssertionError(f'Ocean Provider cannot run without a valid ' f'ethereum account with either a `PROVIDER_PASSWORD` ' f'and `PROVIDER_KEYFILE`/`PROVIDER_ENCRYPTED_KEY` ' f'or private key `PROVIDER_KEY`. Current account has password {account.password}, ' f'keyfile {account.key_file}, encrypted-key {account._encrypted_key} ' f'and private-key {account._private_key}.')
def test_create_data_asset(publisher_ocean_instance, consumer_ocean_instance): """ Setup accounts and asset, register this asset on Aquarius (MetaData store) """ pub_ocn = publisher_ocean_instance cons_ocn = consumer_ocean_instance logging.debug("".format()) sample_ddo_path = get_resource_path('ddo', 'ddo_sa_sample.json') assert sample_ddo_path.exists(), "{} does not exist!".format( sample_ddo_path) ########################################################## # Setup 2 accounts ########################################################## aquarius_acct = pub_ocn.main_account consumer_acct = cons_ocn.main_account # ensure Ocean token balance if pub_ocn.accounts.balance(aquarius_acct).ocn == 0: rcpt = pub_ocn.accounts.request_tokens(aquarius_acct, 200) Web3Provider.get_web3().eth.waitForTransactionReceipt(rcpt) if cons_ocn.accounts.balance(consumer_acct).ocn == 0: rcpt = cons_ocn.accounts.request_tokens(consumer_acct, 200) Web3Provider.get_web3().eth.waitForTransactionReceipt(rcpt) # You will need some token to make this transfer! assert pub_ocn.accounts.balance(aquarius_acct).ocn > 0 assert cons_ocn.accounts.balance(consumer_acct).ocn > 0 ########################################################## # Create an Asset with valid metadata ########################################################## asset = DDO(json_filename=sample_ddo_path) ########################################################## # List currently published assets ########################################################## meta_data_assets = pub_ocn.assets.search('') if meta_data_assets: print("Currently registered assets:") print(meta_data_assets) if asset.did in meta_data_assets: pub_ocn.assets.resolve(asset.did) pub_ocn.assets.retire(asset.did) # Publish the metadata new_asset = pub_ocn.assets.create(asset.metadata, aquarius_acct) # get_asset_metadata only returns 'main' key, is this correct? published_metadata = cons_ocn.assets.resolve(new_asset.did) assert published_metadata # only compare top level keys assert sorted(list( asset.metadata['main'].keys())).remove('files') == sorted( list(published_metadata.metadata.keys())).remove('encryptedFiles') publisher_ocean_instance.assets.retire(new_asset.did)
def setup_agreements_environment(): consumer_acc = get_consumer_account() publisher_acc = get_publisher_account() keeper = Keeper.get_instance() ddo = get_ddo_sample() ddo._did = DID.did({"0": "0x12341234"}) keeper.did_registry.register( ddo.asset_id, checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), url='aquarius:5000', account=publisher_acc, providers=None) registered_ddo = ddo asset_id = registered_ddo.asset_id service_agreement = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo) agreement_id = ServiceAgreement.create_new_agreement_id() price = service_agreement.get_price() (lock_cond_id, access_cond_id, escrow_cond_id) = service_agreement.generate_agreement_condition_ids( agreement_id, asset_id, consumer_acc.address, publisher_acc.address, keeper) return ( keeper, publisher_acc, consumer_acc, agreement_id, asset_id, price, service_agreement, (lock_cond_id, access_cond_id, escrow_cond_id), )
def get_agreement_block_time(agreement_id): # get starting time from the on-chain agreement's blocknumber keeper = keeper_instance() w3 = Web3Provider.get_web3() agreement = keeper.agreement_manager.get_agreement(agreement_id) block_time = w3.eth.getBlock(agreement.block_number_updated).timestamp return int(block_time)
def test_init_events_monitor(keeper, web3, storage_path, provider_account): events_monitor = ProviderEventsMonitor(keeper, web3, storage_path, provider_account) assert events_monitor.last_n_blocks == events_monitor.LAST_N_BLOCKS assert (Web3Provider.get_web3().eth.blockNumber - events_monitor.latest_block) < 5 assert events_monitor.last_processed_block == 0
def get_instance(keeper, storage_path, account): if not ProviderEventsMonitor._instance or \ ProviderEventsMonitor._instance.provider_account != account: ProviderEventsMonitor._instance = ProviderEventsMonitor( keeper, Web3Provider.get_web3(), storage_path, account) return ProviderEventsMonitor._instance
def check(self, token): """ :param token: hex str consist of signature and timestamp :return: hex str ethereum address """ parts = token.split('-') if len(parts) < 2: return '0x0' sig, timestamp = parts if self._get_timestamp() > (int(timestamp) + self._get_expiration()): return '0x0' message = self._get_message(timestamp) address = self._keeper.personal_ec_recover( Web3Provider.get_web3().sha3(text=message), sig) return Web3Provider.get_web3().toChecksumAddress(address)
def register_compute_asset(): # get ocean instance config = ExampleConfig.get_config() ConfigProvider.set_config(config) ocean = Ocean() keeper = Keeper.get_instance() consumer_account = get_account(0) publisher_account = get_account(1) # get descriptor for compute service compute_descriptor = build_compute_descriptor(ocean) # create an asset with compute service ddo = ocean.assets.create(example_metadata.metadata, publisher_account, providers=[config.provider_address], use_secret_store=False, service_descriptors=[compute_descriptor]) event = keeper.did_registry.subscribe_to_event( keeper.did_registry.DID_REGISTRY_EVENT_NAME, 15, event_filter={ '_did': Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), '_owner': publisher_account.address }, wait=True) assert event, 'There was a problem registering an asset' logging.info(f'Registered asset: did={ddo.did}') # buy an asset access agreement_id = ocean.compute.order(ddo.did, consumer_account) event = keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True) assert event, 'There was a problem creating an agreement' logging.info(f'Created asset agreement: agreement_id={agreement_id}') # supply metadata describing the algorithm to run against the dataset # you can also use an algorithm published as an Ocean asset instead algo_meta = AlgorithmMetadata(example_metadata.algo_metadata) # whether to publish the algorithm results as an Ocean assets output_dict = { 'publishOutput': False, 'publishAlgorithmLog': False, } # start the compute job job_id = ocean.compute.start(agreement_id, consumer_account, algorithm_meta=algo_meta, output=output_dict) logging.info(f'Started compute job: job_id={job_id}') # query operator service for job status status = ocean.compute.status(agreement_id, job_id, consumer_account) logging.info(f'Job status: {status}')
def refund_reward(event, agreement_id, did, service_agreement, price, consumer_account, publisher_address, condition_ids, escrow_condition_id): """ Refund the reward to the publisher address. :param event: AttributeDict with the event data. :param agreement_id: id of the agreement, hex str :param did: DID, str :param service_agreement: ServiceAgreement instance :param price: Asset price, int :param consumer_account: Account instance of the consumer :param publisher_address: ethereum account address of publisher, hex str :param condition_ids: is a list of bytes32 content-addressed Condition IDs, bytes32 :param escrow_condition_id: hex str the id of escrow reward condition at this `agreement_id` """ logger.debug(f"trigger refund (agreement {agreement_id}) after event {event}.") if Keeper.get_instance().condition_manager.get_condition_state(escrow_condition_id) > 1: logger.debug( f'escrow reward condition already fulfilled/aborted: ' f'agreementId={agreement_id}, escrow reward conditionId={escrow_condition_id},' f' publisher={publisher_address}' ) return access_id, lock_id = condition_ids[:2] name_to_parameter = {param.name: param for param in service_agreement.condition_by_name['escrowReward'].parameters} document_id = add_0x_prefix(name_to_parameter['_documentId'].value) asset_id = add_0x_prefix(did_to_id(did)) did_owner = Keeper.get_instance().agreement_manager.get_agreement_did_owner(agreement_id) assert document_id == asset_id, f'document_id {document_id} <=> asset_id {asset_id} mismatch.' assert price == service_agreement.get_price(), 'price mismatch.' try: escrow_condition = Keeper.get_instance().escrow_reward_condition tx_hash = escrow_condition.fulfill( agreement_id, price, Web3Provider.get_web3().toChecksumAddress(did_owner), consumer_account.address, lock_id, access_id, consumer_account ) process_tx_receipt( tx_hash, getattr(escrow_condition.contract.events, escrow_condition.FULFILLED_EVENT)(), 'EscrowReward.Fulfilled' ) except Exception as e: logger.error( f'Error when doing escrow_reward_condition.fulfills (agreementId {agreement_id}): {e}', exc_info=1) raise e
def get(self, account): """ :param account: Account instance signing the token :return: hex str the token generated/signed by account """ _message, _time = self._get_message_and_time() msg_hash = Web3Provider.get_web3().sha3(text=_message) try: prefixed_msg_hash = self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(msg_hash), account) return f'{prefixed_msg_hash}-{_time}' except Exception as e: logging.error(f'Error signing token: {str(e)}')
def ocean_agreements(): keeper = Keeper.get_instance() w3 = Web3Provider.get_web3() did_resolver = Mock() ddo = get_ddo_sample() service = ddo.get_service(ServiceTypes.ASSET_ACCESS) service.update_value( ServiceAgreementTemplate.TEMPLATE_ID_KEY, w3.toChecksumAddress("0x00bd138abd70e2f00903268f3db08f2d25677c9e")) did_resolver.resolve = MagicMock(return_value=ddo) # consumer_class = Mock # consumer_class.download = MagicMock(return_value='') return OceanAgreements(keeper, did_resolver, AssetConsumer, AssetExecutor, ConfigProvider.get_config())
def fulfill_escrow_reward_condition(event, agreement_id, service_agreement, price, consumer_address, publisher_account, condition_ids, escrow_condition_id): """ :param event: AttributeDict with the event data. :param agreement_id: id of the agreement, hex str :param service_agreement: ServiceAgreement instance :param price: Asset price, int :param consumer_address: ethereum account address of consumer, hex str :param publisher_account: Account instance of the publisher :param condition_ids: is a list of bytes32 content-addressed Condition IDs, bytes32 :param escrow_condition_id: hex str the id of escrow reward condition at this `agreement_id` :return: """ if not event: logger.warning(f'`fulfill_escrow_reward_condition` got empty event: ' f'event listener timed out.') return keeper = Keeper.get_instance() if keeper.condition_manager.get_condition_state(escrow_condition_id) > 1: logger.debug( f'escrow reward condition already fulfilled/aborted: ' f'agreementId={agreement_id}, escrow reward conditionId={escrow_condition_id}' ) return logger.debug( f"release reward (agreement {agreement_id}) after event {event}.") access_id, lock_id = condition_ids[:2] logger.debug(f'fulfill_escrow_reward_condition: ' f'agreementId={agreement_id}' f'price={price}, {type(price)}' f'consumer={consumer_address},' f'publisher={publisher_account.address},' f'conditionIds={condition_ids}') assert price == service_agreement.get_price(), 'price mismatch.' assert isinstance( price, int), f'price expected to be int type, got type "{type(price)}"' time.sleep(5) keeper = Keeper.get_instance() did_owner = keeper.agreement_manager.get_agreement_did_owner(agreement_id) args = (agreement_id, price, Web3Provider.get_web3().toChecksumAddress(did_owner), consumer_address, lock_id, access_id, publisher_account) process_fulfill_condition(args, keeper.escrow_reward_condition, escrow_condition_id, logger, keeper, 10)
def __init__(self, address, abi_path=None, abi=None): name = self.contract_name assert name, 'contract_name property needs to be implemented in subclasses.' if not abi_path and not abi: abi_path = ContractHandler.artifacts_path if abi_path and not abi: abi = CustomContractBase.read_abi_from_file(name, abi_path)['abi'] contract = Web3Provider.get_web3().eth.contract(address=address, abi=abi) ContractHandler.set(name, contract) ContractBase.__init__(self, name) assert self.contract == contract assert self.contract_concise is not None assert self.address == address
def _get_agreement_actor_event(self, agreement_id, from_block=0, to_block='latest'): _filter = { 'agreementId': Web3Provider.get_web3().toBytes(hexstr=agreement_id) } event_filter = EventFilter( self.agreement_manager.AGREEMENT_ACTOR_ADDED_EVENT, self.agreement_manager._get_contract_agreement_actor_added_event(), _filter, from_block=from_block, to_block=to_block) event_filter.set_poll_interval(0.5) return event_filter
def _get_agreement_actor_event(keeper, agreement_id, from_block=0, to_block='latest'): _filter = { 'agreementId': Web3Provider.get_web3().toBytes(hexstr=agreement_id) } event_filter = EventFilter( keeper.agreement_manager.AGREEMENT_ACTOR_ADDED_EVENT, keeper.agreement_manager.get_event_filter_for_agreement_actor( None).event, _filter, from_block=from_block, to_block=to_block) event_filter.set_poll_interval(0.5) return event_filter
def send(self, did, agreement_id, service_index, signature, consumer_account, auto_consume=False): """ Send a signed service agreement to the publisher Brizo instance to consume/access the service. :param did: str representation fo the asset DID. Use this to retrieve the asset DDO. :param agreement_id: 32 bytes identifier created by the consumer and will be used on-chain for the executed agreement. :param service_index: int identifies the specific service in the ddo to use in this agreement. :param signature: str the signed agreement message hash which includes conditions and their parameters values and other details of the agreement. :param consumer_account: Account instance of the consumer :param auto_consume: boolean tells this function wether to automatically trigger consuming the asset upon receiving access permission :raises OceanInitializeServiceAgreementError: on failure :return: bool """ asset = self._asset_resolver.resolve(did) service_agreement = asset.get_service_by_index(service_index) # subscribe to events related to this agreement_id before sending the request. logger.debug( f'Registering service agreement with id: {agreement_id}, auto-consume {auto_consume}' ) # TODO: refactor this to use same code in `create` publisher_address = self._keeper.did_registry.get_did_owner( asset.asset_id) condition_ids = service_agreement.generate_agreement_condition_ids( agreement_id, asset.asset_id, consumer_account.address, publisher_address, self._keeper) from_block = Web3Provider.get_web3().eth.blockNumber self._process_consumer_agreement_events( agreement_id, did, service_agreement, consumer_account, condition_ids, publisher_address, from_block, auto_consume) return BrizoProvider.get_brizo().initialize_service_agreement( did, agreement_id, service_index, signature, consumer_account.address, service_agreement.endpoints.purchase)
def release_reward(self, agreement_id, amount, account): """ Release reward condition. :param agreement_id: id of the agreement, hex str :param amount: Amount of tokens, int :param account: Account :return: """ agreement_values = self._keeper.agreement_manager.get_agreement( agreement_id) consumer = self._keeper.get_agreement_consumer(agreement_id) owner = agreement_values.owner lock_id, access_id = agreement_values.condition_ids[:2] tx_hash = self._keeper.escrow_reward_condition.fulfill( agreement_id, amount, Web3Provider.get_web3().toChecksumAddress(owner), consumer, lock_id, access_id, account) receipt = self._keeper.escrow_reward_condition.get_tx_receipt(tx_hash) return bool(receipt and receipt.status == 1)
def _verify_service_agreement_signature(self, did, agreement_id, service_index, consumer_address, signature, ddo=None): """ Verify service agreement signature. Verify that the given signature is truly signed by the `consumer_address` and represents this did's service agreement.. :param did: DID, str :param agreement_id: id of the agreement, hex str :param service_index: identifier of the service inside the asset DDO, str :param consumer_address: ethereum account address of consumer, hex str :param signature: Signature, str :param ddo: DDO instance :return: True if signature is legitimate, False otherwise :raises: ValueError if service is not found in the ddo :raises: AssertionError if conditions keys do not match the on-chain conditions keys """ if not ddo: ddo = self._asset_resolver.resolve(did) service_agreement = ddo.get_service_by_index(service_index) agreement_hash = service_agreement.get_service_agreement_hash( agreement_id, ddo.asset_id, consumer_address, Web3Provider.get_web3().toChecksumAddress(ddo.proof['creator']), self._keeper) recovered_address = self._keeper.personal_ec_recover( agreement_hash, signature) is_valid = (recovered_address == consumer_address) if not is_valid: logger.warning( f'Agreement signature failed: agreement hash is {agreement_hash.hex()}' ) return is_valid
def run_events_monitor(): setup_logging() config = get_config() keeper_url = config.keeper_url artifacts_path = get_keeper_path(config) storage_path = config.get('resources', 'storage.path', fallback='./provider-events-monitor.db') ContractHandler.set_artifacts_path(artifacts_path) web3 = Web3Provider.get_web3(keeper_url) keeper = Keeper.get_instance() init_account_envvars() account = get_account(0) if account is None: raise AssertionError( f'Provider events monitor cannot run without a valid ' f'ethereum account. Account address was not found in the environment' f'variable `PROVIDER_ADDRESS`. Please set the following environment ' f'variables and try again: `PROVIDER_ADDRESS`, [`PROVIDER_PASSWORD`, ' f'and `PROVIDER_KEYFILE` or `PROVIDER_ENCRYPTED_KEY`] or `PROVIDER_KEY`.' ) if not account._private_key and not (account.password and account._encrypted_key): raise AssertionError( f'Provider events monitor cannot run without a valid ' f'ethereum account with either a `PROVIDER_PASSWORD` ' f'and `PROVIDER_KEYFILE`/`PROVIDER_ENCRYPTED_KEY` ' f'or private key `PROVIDER_KEY`. Current account has password {account.password}, ' f'keyfile {account.key_file}, encrypted-key {account._encrypted_key} ' f'and private-key {account._private_key}.') monitor = ProviderEventsMonitor(keeper, web3, storage_path, account) monitor.start_agreement_events_monitor() while True: time.sleep(5)
def buy_asset(): """ Requires all ocean services running. """ setup_logging(default_level=logging.INFO) ConfigProvider.set_config(ExampleConfig.get_config()) config = ConfigProvider.get_config() providers = { 'duero': '0xfEF2d5e1670342b9EF22eeeDcb287EC526B48095', 'nile': '0x4aaab179035dc57b35e2ce066919048686f82972' } # make ocean instance ocn = Ocean() acc = get_account(1) if not acc: acc = ([acc for acc in ocn.accounts.list() if acc.password] or ocn.accounts.list())[0] Diagnostics.verify_contracts() metadata = example_metadata.metadata.copy() metadata['main']['dateCreated'] = get_timestamp() keeper = Keeper.get_instance() # Register ddo did = '' if did: ddo = ocn.assets.resolve(did) logging.info(f'using ddo: {did}') else: ddo = ocn.assets.create(metadata, acc, providers=[], use_secret_store=True) assert ddo is not None, f'Registering asset on-chain failed.' did = ddo.did logging.info(f'registered ddo: {did}') # ocn here will be used only to publish the asset. Handling the asset by the publisher # will be performed by the Brizo server running locally test_net = os.environ.get('TEST_NET', '') if test_net.startswith('nile'): provider = keeper.did_registry.to_checksum_address(providers['nile']) elif test_net.startswith('duero'): provider = keeper.did_registry.to_checksum_address(providers['duero']) else: provider = '0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0' # Wait for did registry event event = keeper.did_registry.subscribe_to_event( keeper.did_registry.DID_REGISTRY_EVENT_NAME, 30, event_filter={ '_did': Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), '_owner': acc.address}, wait=True ) if not event: logging.warning(f'Failed to get the did registry event for asset with did {did}.') assert keeper.did_registry.get_block_number_updated(ddo.asset_id) > 0, \ f'There is an issue in registering asset {did} on-chain.' keeper.did_registry.add_provider(ddo.asset_id, provider, acc) logging.info(f'is {provider} set as did provider: ' f'{keeper.did_registry.is_did_provider(ddo.asset_id, provider)}') _providers = keeper.did_registry.get_did_providers(ddo.asset_id) access_template_name = keeper.template_manager.SERVICE_TO_TEMPLATE_NAME['access'] template_id = keeper.template_manager.create_template_id(access_template_name) approved = keeper.template_manager.is_template_approved(template_id) print(f'agreement template approved: {approved}') template_data = keeper.template_manager.get_template(template_id) print(f'access agreement template: {template_data}') cons_ocn = Ocean() consumer_account = get_account(0) # sign agreement using the registered asset did above service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS) # This will send the order request to Brizo which in turn will execute the agreement on-chain cons_ocn.accounts.request_tokens(consumer_account, 10) sa = ServiceAgreement.from_json(service.as_dictionary()) agreement_id = '' if not agreement_id: agreement_id = cons_ocn.assets.order( did, sa.index, consumer_account) logging.info('placed order: %s, %s', did, agreement_id) event = keeper.agreement_manager.subscribe_agreement_created( agreement_id, 60, None, (), wait=True ) assert event, "Agreement event is not found, check the keeper node's logs" logging.info(f'Got agreement event, next: lock reward condition') event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id, 60, None, (), wait=True ) assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs" logging.info('Got lock reward event, next: wait for the access condition..') event = keeper.access_secret_store_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True ) logging.info(f'Got access event {event}') i = 0 while ocn.agreements.is_access_granted( agreement_id, did, consumer_account.address) is not True and i < 30: time.sleep(1) i += 1 assert ocn.agreements.is_access_granted(agreement_id, did, consumer_account.address) ocn.assets.consume( agreement_id, did, sa.index, consumer_account, config.downloads_path) logging.info('Success buying asset.') event = keeper.escrow_reward_condition.subscribe_condition_fulfilled( agreement_id, 30, None, (), wait=True ) assert event, 'no event for EscrowReward.Fulfilled' logging.info(f'got EscrowReward.FULFILLED event: {event}') logging.info('Done buy asset.')
def web3(): return Web3Provider.get_web3(get_config().keeper_url)
def test_buy_asset(consumer_ocean_instance, publisher_ocean_instance): config = ExampleConfig.get_config() ConfigProvider.set_config(config) keeper = Keeper.get_instance() # :TODO: enable the actual SecretStore # SecretStoreProvider.set_secret_store_class(SecretStore) w3 = Web3Provider.get_web3() pub_acc = get_publisher_account() # Register ddo ddo = get_registered_ddo(publisher_ocean_instance, pub_acc) assert isinstance(ddo, DDO) # ocn here will be used only to publish the asset. Handling the asset by the publisher # will be performed by the Brizo server running locally cons_ocn = consumer_ocean_instance # restore the http client because we want the actual Brizo server to do the work # not the BrizoMock. # Brizo.set_http_client(requests) consumer_account = get_consumer_account() downloads_path_elements = len( os.listdir( consumer_ocean_instance._config.downloads_path)) if os.path.exists( consumer_ocean_instance._config.downloads_path) else 0 # sign agreement using the registered asset did above service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS) sa = ServiceAgreement.from_json(service.as_dictionary()) # This will send the consume request to Brizo which in turn will execute the agreement on-chain cons_ocn.accounts.request_tokens(consumer_account, 100) agreement_id = cons_ocn.assets.order(ddo.did, sa.index, consumer_account, auto_consume=False) event_wait_time = 10 event = keeper.escrow_access_secretstore_template.subscribe_agreement_created( agreement_id, event_wait_time, log_event( keeper.escrow_access_secretstore_template.AGREEMENT_CREATED_EVENT), (), wait=True) assert event, 'no event for EscrowAccessSecretStoreTemplate.AgreementCreated' event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id, event_wait_time, log_event(keeper.lock_reward_condition.FULFILLED_EVENT), (), wait=True) assert event, 'no event for LockRewardCondition.Fulfilled' # give access publisher_ocean_instance.agreements.conditions.grant_access( agreement_id, ddo.did, consumer_account.address, pub_acc) event = keeper.access_secret_store_condition.subscribe_condition_fulfilled( agreement_id, event_wait_time, log_event(keeper.access_secret_store_condition.FULFILLED_EVENT), (), wait=True) assert event, 'no event for AccessSecretStoreCondition.Fulfilled' assert cons_ocn.agreements.is_access_granted(agreement_id, ddo.did, consumer_account.address) assert cons_ocn.assets.consume(agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path) assert len(os.listdir( config.downloads_path)) == downloads_path_elements + 1 # Check that we can consume only an specific file in passing the index. assert cons_ocn.assets.consume(agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path, 2) assert len(os.listdir( config.downloads_path)) == downloads_path_elements + 1 with pytest.raises(AssertionError): cons_ocn.assets.consume(agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path, -2) with pytest.raises(AssertionError): cons_ocn.assets.consume(agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path, 3) # decrypt the contentUrls using the publisher account instead of consumer account. # if the secret store is working and ACL check is enabled, this should fail # since SecretStore decrypt will fail the checkPermissions check try: cons_ocn.assets.consume(agreement_id, ddo.did, sa.index, pub_acc, config.downloads_path) except RPCError: print('hooray, secret store is working as expected.') publisher_ocean_instance.agreements.conditions.release_reward( agreement_id, sa.get_price(), pub_acc) event = keeper.escrow_reward_condition.subscribe_condition_fulfilled( agreement_id, event_wait_time + 20, log_event(keeper.escrow_reward_condition.FULFILLED_EVENT), (), wait=True) assert event, 'no event for EscrowReward.Fulfilled' assert w3.toHex(event.args['_agreementId']) == agreement_id
def order(self, did, consumer_account, algorithm_did=None, algorithm_meta=None, output=None, provider_address=None): """ :param did: :param consumer_account: :param algorithm_did: str -- the asset did (of `algorithm` type) which consist of `did:op:` and the assetId hex str (without `0x` prefix) :param algorithm_meta: `AlgorithmMetadata` instance -- metadata about the algorithm being run if `algorithm` is being used. This is ignored when `algorithm_did` is specified. :param output: Output object to be used in publishing mechanism :param provider_address: ethereum account address of provider (optional) :return: """ assert isinstance(consumer_account, Account), \ f'Expected `consumer_account` of type `Account`, ' \ f'got type `{type(consumer_account)}` and value `{consumer_account}' if algorithm_meta: assert isinstance(algorithm_meta, AlgorithmMetadata), \ f'Expected `algorithm_meta` of type `AlgorithmMetadata`, ' \ f'got type `{type(algorithm_meta)}` and value `{algorithm_meta}`.' assert algorithm_meta.is_valid(), f'`algorithm_meta` seems invalid' ddo = self._did_resolver.resolve(did) service = ddo.get_service(ServiceTypes.CLOUD_COMPUTE) if not service: raise ValueError( f'order of compute service on asset {did} failed: ' f'this asset does not have a service of type {ServiceTypes.CLOUD_COMPUTE}.' ) agreement_id = self._agreements.new() logger.debug( f'about to request create `{service.type}` service agreement: {agreement_id}' ) # creating agreement will automatically fulfill the LockRewardCondition if the agreement is successful self._agreements.create(did, service.index, agreement_id, None, consumer_account.address, consumer_account, provider_address, auto_consume=False) if agreement_id and (algorithm_meta or algorithm_did): def _refund_callback(_price, _publisher_address, _condition_ids, _service_agreement): def do_refund(_agreement_id, _did, _consumer_account, *_): refund_reward(None, _agreement_id, _did, _service_agreement, _price, _consumer_account, _publisher_address, _condition_ids, _condition_ids[2]) return do_refund def run_compute_job(_, _agreement_id, _did, _account, _algo_did, _algo_meta, _output): if _algo_did or _algo_meta: return self.start(_agreement_id, _account, _algo_did, _algo_meta, _output) publisher_address = Web3Provider.get_web3().toChecksumAddress( ddo.publisher) condition_ids = service.generate_agreement_condition_ids( agreement_id, ddo.asset_id, consumer_account.address, publisher_address, self._keeper) self._keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id, max(service.condition_by_name['computeExecution'].timeout, 300), run_compute_job, (agreement_id, did, consumer_account, algorithm_did, algorithm_meta, output), timeout_callback=_refund_callback(service.get_price(), publisher_address, condition_ids, service)) return agreement_id
def create(self, metadata, publisher_account, service_descriptors=None, providers=None, use_secret_store=True): """ Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store ( Aquarius). :param metadata: dict conforming to the Metadata accepted by Ocean Protocol. :param publisher_account: Account of the publisher registering this asset :param service_descriptors: list of ServiceDescriptor tuples of length 2. The first item must be one of ServiceTypes and the second item is a dict of parameters and values required by the service :param providers: list of addresses of providers of this asset (a provider is an ethereum account that is authorized to provide asset services) :param use_secret_store: bool indicate whether to use the secret store directly for encrypting urls (Uses Brizo provider service if set to False) :return: DDO instance """ assert isinstance( metadata, dict), f'Expected metadata of type dict, got {type(metadata)}' assert service_descriptors is None or isinstance(service_descriptors, list), \ f'bad type of `service_descriptors` {type(service_descriptors)}' # if not metadata or not Metadata.validate(metadata): # raise OceanInvalidMetadata('Metadata seems invalid. Please make sure' # ' the required metadata values are filled in.') # copy metadata so we don't change the original metadata_copy = copy.deepcopy(metadata) asset_type = metadata_copy['main']['type'] assert asset_type in ( 'dataset', 'algorithm'), f'Invalid/unsupported asset type {asset_type}' service_descriptors = service_descriptors or [] brizo = BrizoProvider.get_brizo() services = self._process_service_descriptors(service_descriptors, metadata_copy, publisher_account) stype_to_service = {s.type: s for s in services} checksum_dict = dict() for service in services: checksum_dict[str(service.index)] = checksum(service.main) # Create a DDO object ddo = DDO() # Adding proof to the ddo. ddo.add_proof(checksum_dict, publisher_account) # Generating the did and adding to the ddo. did = ddo.assign_did(DID.did(ddo.proof['checksum'])) logger.debug(f'Generating new did: {did}') # Check if it's already registered first! if did in self._get_aquarius().list_assets(): raise OceanDIDAlreadyExist( f'Asset id {did} is already registered to another asset.') md_service = stype_to_service[ServiceTypes.METADATA] ddo_service_endpoint = md_service.service_endpoint if '{did}' in ddo_service_endpoint: ddo_service_endpoint = ddo_service_endpoint.replace('{did}', did) md_service.set_service_endpoint(ddo_service_endpoint) # Populate the ddo services ddo.add_service(md_service) ddo.add_service(stype_to_service[ServiceTypes.AUTHORIZATION]) access_service = stype_to_service.get(ServiceTypes.ASSET_ACCESS, None) compute_service = stype_to_service.get(ServiceTypes.CLOUD_COMPUTE, None) if access_service: access_service.init_conditions_values( did, { cname: c.address for cname, c in self._keeper.contract_name_to_instance.items() }) ddo.add_service(access_service) if compute_service: compute_service.init_conditions_values( did, { cname: c.address for cname, c in self._keeper.contract_name_to_instance.items() }) ddo.add_service(compute_service) ddo.proof['signatureValue'] = self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)), publisher_account) # Add public key and authentication ddo.add_public_key(did, publisher_account.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) # Setup metadata service # First compute files_encrypted if metadata_copy['main']['type'] == 'dataset': assert metadata_copy['main']['files'], \ 'files is required in the metadata main attributes.' logger.debug('Encrypting content urls in the metadata.') if not use_secret_store: encrypt_endpoint = brizo.get_encrypt_endpoint(self._config) files_encrypted = brizo.encrypt_files_dict( metadata_copy['main']['files'], encrypt_endpoint, ddo.asset_id, publisher_account.address, self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(ddo.asset_id), publisher_account)) else: files_encrypted = self._get_secret_store(publisher_account) \ .encrypt_document( did_to_id(did), json.dumps(metadata_copy['main']['files']), ) # only assign if the encryption worked if files_encrypted: logger.debug( f'Content urls encrypted successfully {files_encrypted}') index = 0 for file in metadata_copy['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata_copy['encryptedFiles'] = files_encrypted else: raise AssertionError('Encrypting the files failed.') # DDO url and `Metadata` service logger.debug(f'Generated ddo and services, DID is {ddo.did},' f' metadata service @{ddo_service_endpoint}.') response = None # register on-chain registered_on_chain = self._keeper.did_registry.register( ddo.asset_id, checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), url=ddo_service_endpoint, account=publisher_account, providers=providers) if registered_on_chain is False: logger.warning(f'Registering {did} on-chain failed.') return None logger.info(f'Successfully registered DDO (DID={did}) on chain.') try: # publish the new ddo in ocean-db/Aquarius response = self._get_aquarius().publish_asset_ddo(ddo) logger.info('Asset/ddo published successfully in aquarius.') except ValueError as ve: raise ValueError( f'Invalid value to publish in the metadata: {str(ve)}') except Exception as e: logger.error(f'Publish asset in aquarius failed: {str(e)}') if not response: return None return ddo
def web3_instance(): config = ExampleConfig.get_config() return Web3Provider.get_web3(config.keeper_url)
def __init__(self, config=None): """ Initialize Ocean class. >> # Make a new Ocean instance >> ocean = Ocean({...}) This class provides the main top-level functions in ocean protocol: * Publish assets metadata and associated services * Each asset is assigned a unique DID and a DID Document (DDO) * The DDO contains the asset's services including the metadata * The DID is registered on-chain with a URL of the metadata store to retrieve the DDO from >> ddo = ocean.assets.create(metadata, publisher_account) * Discover/Search assets via the current configured metadata store (Aquarius) >> assets_list = ocean.assets.search('search text') * Purchase asset services by choosing a service agreement from the asset's DDO. Purchase goes through the service agreements interface and starts by signing a service agreement then sending the signature to the publisher's Brizo server via the `purchaseEndpoint` in the service definition: >> service_def_id = ddo.get_service(ServiceTypes.ASSET_ACCESS).service_definition_id >> service_agreement_id = ocean.assets.order(did, service_def_id, consumer_account) An instance of Ocean is parameterized by a `Config` instance. :param config: Config instance """ # Configuration information for the market is stored in the Config class # config = Config(filename=config_file, options_dict=config_dict) if not config: config = ConfigProvider.get_config() self._config = config self._web3 = Web3Provider.get_web3(self._config.keeper_url) ContractHandler.set_artifacts_path(self._config.keeper_path) contracts = [ 'DIDRegistry', 'Dispenser', 'TemplateStoreManager', 'OceanToken', 'ConditionStoreManager', 'EscrowAccessSecretStoreTemplate', 'AgreementStoreManager', 'AgreementStoreManager', 'AccessSecretStoreCondition', 'LockRewardCondition', 'HashLockCondition', 'SignCondition', 'EscrowReward' ] self._keeper = Keeper.get_instance(contracts) self._did_resolver = DIDResolver(self._keeper.did_registry) # Initialize the public sub-modules self.tokens = OceanTokens(self._keeper) self.accounts = OceanAccounts(self._keeper, self._config, self.tokens) self.secret_store = OceanSecretStore(self._config) self.templates = OceanTemplates(self._keeper, config) self.agreements = self._make_ocean_agreements() self.assets = OceanAssets(self._keeper, self._did_resolver, self.agreements, AssetConsumer, AssetExecutor, self._config) self.services = OceanServices() self.ocean_providers = OceanProviders(self._keeper, self._did_resolver, self._config) self.auth = OceanAuth(self._keeper, self._config.storage_path) logger.debug('Squid Ocean instance initialized: ') logger.debug( f'\tOther accounts: {sorted([a.address for a in self.accounts.list()])}' ) logger.debug(f'\tDIDRegistry @ {self._keeper.did_registry.address}')
def create(self, did, index, agreement_id, service_agreement_signature, consumer_address, account, auto_consume=False): """ Execute the service agreement on-chain using keeper's ServiceAgreement contract. The on-chain executeAgreement method requires the following arguments: templateId, signature, consumer, hashes, timeouts, serviceAgreementId, did. `agreement_message_hash` is necessary to verify the signature. The consumer `signature` includes the conditions timeouts and parameters values which is usedon-chain to verify that the values actually match the signed hashes. :param did: str representation fo the asset DID. Use this to retrieve the asset DDO. :param index: str identifies the specific service in the ddo to use in this agreement. :param agreement_id: 32 bytes identifier created by the consumer and will be used on-chain for the executed agreement. :param service_agreement_signature: str the signed agreement message hash which includes conditions and their parameters values and other details of the agreement. :param consumer_address: ethereum account address of consumer, hex str :param account: Account instance creating the agreement. Can be either the consumer, publisher or provider :param auto_consume: bool :return: dict the `executeAgreement` transaction receipt """ assert consumer_address and Web3Provider.get_web3().isChecksumAddress( consumer_address), f'Invalid consumer address {consumer_address}' assert account.address in self._keeper.accounts, \ f'Unrecognized account address {account.address}' agreement_template_approved = self._keeper.template_manager.is_template_approved( self._keeper.escrow_access_secretstore_template.address) agreement_exec_template_approved = self._keeper.template_manager.is_template_approved( self._keeper.escrow_compute_execution_template.address) if not agreement_template_approved: msg = ( f'The EscrowAccessSecretStoreTemplate contract at address ' f'{self._keeper.escrow_access_secretstore_template.address} is not ' f'approved and cannot be used for creating service agreements.' ) logger.warning(msg) raise OceanInvalidAgreementTemplate(msg) if not agreement_exec_template_approved: msg = ( f'The EscroComputeExecutionTemplate contract at address ' f'{self._keeper.agreement_exec_template_approved.address} is not ' f'approved and cannot be used for creating service agreements.' ) logger.warning(msg) raise OceanInvalidAgreementTemplate(msg) asset = self._asset_resolver.resolve(did) asset_id = asset.asset_id service_agreement = asset.get_service_by_index(index) if service_agreement.type == ServiceTypes.ASSET_ACCESS: agreement_template = self._keeper.escrow_access_secretstore_template elif service_agreement.type == ServiceTypes.CLOUD_COMPUTE: agreement_template = self._keeper.escrow_compute_execution_template else: raise Exception( 'The agreement could not be created. Review the index of your service.' ) if agreement_template.get_agreement_consumer(agreement_id) is not None: raise OceanServiceAgreementExists( f'Service agreement {agreement_id} already exists, cannot reuse ' f'the same agreement id.') if consumer_address != account.address: if not self._verify_service_agreement_signature( did, agreement_id, index, consumer_address, service_agreement_signature, ddo=asset): raise OceanInvalidServiceAgreementSignature( f'Verifying consumer signature failed: ' f'signature {service_agreement_signature}, ' f'consumerAddress {consumer_address}') publisher_address = Web3Provider.get_web3().toChecksumAddress( asset.publisher) condition_ids = service_agreement.generate_agreement_condition_ids( agreement_id, asset_id, consumer_address, publisher_address, self._keeper) time_locks = service_agreement.conditions_timelocks time_outs = service_agreement.conditions_timeouts success = agreement_template.create_agreement(agreement_id, asset_id, condition_ids, time_locks, time_outs, consumer_address, account) ## TODO CHECK THIS FOR THE OTHER TEMPLATE if not success: # success is based on tx receipt which is not reliable. # So we check on-chain directly to see if agreement_id is there consumer = self._keeper.escrow_access_secretstore_template.get_agreement_consumer( agreement_id) if consumer: success = True else: event_log = self._keeper.escrow_access_secretstore_template \ .subscribe_agreement_created( agreement_id, 15, None, (), wait=True ) success = event_log is not None if success: logger.info( f'Service agreement {agreement_id} created successfully.') else: logger.info(f'Create agreement "{agreement_id}" failed.') self._log_agreement_info(asset, service_agreement, agreement_id, service_agreement_signature, consumer_address, account, condition_ids) if success: # subscribe to events related to this agreement_id if consumer_address == account.address: from_block = Web3Provider.get_web3().eth.blockNumber - 10 self._process_consumer_agreement_events( agreement_id, did, service_agreement, account, condition_ids, publisher_address, from_block, auto_consume, service_agreement.type) return success