def execute_computation(event, agreement_id, did, service_agreement, consumer_account, consume_callback, workflow_did): """ Consumption of an asset after get the event call. :param event: AttributeDict with the event data. :param agreement_id: id of the agreement, hex str :param did: DID, str :param service_agreement: ServiceAgreement instance :param consumer_account: Account instance of the consumer :param consume_callback: :param secret_store_url: str URL of secret store node for retrieving decryption keys :param parity_url: str URL of parity client to use for secret store encrypt/decrypt :param downloads_path: str path to save downloaded files """ logger.debug( f"consuming asset (agreementId {agreement_id}) after event {event}.") if consume_callback: gateway = GatewayProvider.get_gateway() consume_callback( agreement_id, DIDResolver(Keeper.get_instance().did_registry).resolve(did), DIDResolver( Keeper.get_instance().did_registry).resolve(workflow_did), consumer_account, gateway)
def refund_reward(event, agreement_id, did, service_agreement, price, consumer_account, publisher_address, condition_ids, escrow_condition_id): """ Refund the reward to the publisher address. :param event: AttributeDict with the event data. :param agreement_id: id of the agreement, hex str :param did: DID, str :param service_agreement: ServiceAgreement instance :param price: Asset price, int :param consumer_account: Account instance of the consumer :param publisher_address: ethereum account address of publisher, hex str :param condition_ids: is a list of bytes32 content-addressed Condition IDs, bytes32 :param escrow_condition_id: hex str the id of escrow reward condition at this `agreement_id` """ logger.debug( f"trigger refund (agreement {agreement_id}) after event {event}.") if Keeper.get_instance().condition_manager.get_condition_state( escrow_condition_id) > 1: logger.debug( f'escrow payment condition already fulfilled/aborted: ' f'agreementId={agreement_id}, escrow reward conditionId={escrow_condition_id},' f' publisher={publisher_address}') return access_id, lock_id = condition_ids[:2] name_to_parameter = { param.name: param for param in service_agreement.condition_by_name['escrowPayment'].parameters } document_id = add_0x_prefix(name_to_parameter['_documentId'].value) asset_id = add_0x_prefix(did_to_id(did)) did_owner = Keeper.get_instance( ).agreement_manager.get_agreement_did_owner(agreement_id) assert document_id == asset_id, f'document_id {document_id} <=> asset_id {asset_id} mismatch.' assert price == service_agreement.get_price(), 'price mismatch.' try: escrow_condition = Keeper.get_instance().escrow_payment_condition tx_hash = escrow_condition.fulfill( agreement_id, price, Web3Provider.get_web3().toChecksumAddress(did_owner), consumer_account.address, lock_id, access_id, consumer_account) process_tx_receipt( tx_hash, getattr(escrow_condition.contract.events, escrow_condition.FULFILLED_EVENT)(), 'EscrowReward.Fulfilled') except Exception as e: logger.error( f'Error when doing escrow_payment_condition.fulfills (agreementId {agreement_id}): {e}', exc_info=1) raise e
def test_store_token(): auth = Auth(Keeper.get_instance(), ':memory:') acc = get_publisher_account() token = auth.store(acc) assert auth.check(token) == acc.address, 'invalid token, check failed.' # verify it is saved assert auth.restore(acc) == token, 'Restoring token failed.'
def test_restore_token(publisher_instance): auth = Auth(Keeper.get_instance(), ':memory:') acc = get_publisher_account() assert auth.restore( acc) is None, 'Expecting None when restoring non-existing token.' token = auth.store(acc) assert auth.check(token) == acc.address, 'invalid token, check failed.' # verify it is saved assert auth.restore(acc) == token, 'Restoring token failed.'
def test_check_token(web3_instance): auth = Auth(Keeper.get_instance(), ':memory:') acc = get_publisher_account() token = auth.get(acc) address = auth.check(token) assert address != '0x0', 'Verifying token failed.' sig = token.split('-')[0] assert auth.check(sig) == '0x0'
def test_get_token(): auth = Auth(Keeper.get_instance(), ':memory:') acc = get_publisher_account() token = auth.get(acc) assert isinstance(token, str), 'Invalid auth token type.' assert token.startswith('0x'), 'Invalid auth token.' parts = token.split('-') assert len(parts) == 2, 'Invalid token, timestamp separator is not found.' address = auth.check(token) assert address != '0x0', 'Verifying token failed.'
def test_get_condition_name_by_address(): keeper = Keeper.get_instance() name = keeper.get_condition_name_by_address( keeper.lock_payment_condition.address) assert name == 'lockReward' name = keeper.get_condition_name_by_address( keeper.access_condition.address) assert name == 'accessSecretStore' name = keeper.get_condition_name_by_address( keeper.escrow_payment_condition.address) assert name == 'escrowReward'
def agreements(): publisher_acc = get_publisher_account() keeper = Keeper.get_instance() w3 = Web3Provider.get_web3() did_resolver = Mock() ddo = get_ddo_sample() service = ddo.get_service(ServiceTypes.ASSET_ACCESS) service.update_value(ServiceAgreementTemplate.TEMPLATE_ID_KEY, w3.toChecksumAddress(publisher_acc.address)) did_resolver.resolve = MagicMock(return_value=ddo) return Agreements(keeper, did_resolver, AssetConsumer, AssetExecutor, ConfigProvider.get_config())
def execute_service(service_agreement_id, service_endpoint, account, workflow_ddo): """ :param service_agreement_id: :param service_endpoint: :param account: :return: """ signature = Keeper.get_instance().sign_hash( add_ethereum_prefix_and_hash_msg(service_agreement_id), account) execute_url = Gateway._create_execute_url(service_endpoint, service_agreement_id, account, workflow_ddo.did, signature) logger.info(f'invoke execute endpoint with this url: {execute_url}') response = Gateway._http_client.post(execute_url) return response
def fulfill_lock_payment_condition(event, agreement_id, price, consumer_account, lock_condition_id): """ Fulfill the lock reward condition. :param event: AttributeDict with the event data. :param agreement_id: id of the agreement, hex str :param price: Asset price, int :param consumer_account: Account instance of the consumer :param lock_condition_id: hex str the id of the lock reward condition for this `agreement_id` """ if not event: logger.warning( f'`fulfill_lock_payment_condition` got empty event: event listener timed out.' ) return keeper = Keeper.get_instance() if keeper.condition_manager.get_condition_state(lock_condition_id) > 1: logger.debug( f'lock payment condition already fulfilled/aborted: ' f'agreementId={agreement_id}, lockPayment conditionId={lock_condition_id}' ) return logger.debug( f"about to lock payment (agreement {agreement_id}) after event {event}." ) approved = keeper.token.token_approve( keeper.lock_payment_condition.address, price, consumer_account) logger.info( f'approval of token transfer was {"" if approved else "NOT"} successful' ) args = (agreement_id, keeper.escrow_payment_condition.address, price, consumer_account) process_fulfill_condition(args, keeper.lock_payment_condition, lock_condition_id, logger, keeper, 10)
def setup_agreements_environment(ddo_sample): consumer_acc = get_consumer_account() publisher_acc = get_publisher_account() keeper = Keeper.get_instance() ddo = ddo_sample did_seed = generate_prefixed_id() asset_id = keeper.did_registry.hash_did(did_seed, publisher_acc.address) ddo._did = DID.did(asset_id) keeper.did_registry.register( did_seed, checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), url='localhost:5000', account=publisher_acc, providers=None) service_agreement = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo) agreement_id = ServiceAgreement.create_new_agreement_id() price = service_agreement.get_price() access_cond_id, lock_cond_id, escrow_cond_id = \ service_agreement.generate_agreement_condition_ids( agreement_id, asset_id, consumer_acc.address, keeper ) return ( keeper, publisher_acc, consumer_acc, agreement_id, asset_id, price, service_agreement, (lock_cond_id, access_cond_id, escrow_cond_id), )
def test_sign_agreement(publisher_instance, consumer_instance, registered_ddo): # point consumer_instance's Gateway mock to the publisher's nevermined instance Gateway.set_http_client( GatewayMock(publisher_instance, publisher_instance.main_account)) consumer_acc = consumer_instance.main_account keeper = Keeper.get_instance() publisher_acc = publisher_instance.main_account did = registered_ddo.did asset_id = registered_ddo.asset_id ddo = consumer_instance.assets.resolve(did) service_agreement = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo) price = service_agreement.get_price() # Give consumer some tokens keeper.dispenser.request_vodkas(price * 2, consumer_acc) agreement_id, signature = consumer_instance.agreements.prepare( did, consumer_acc, ServiceTypesIndices.DEFAULT_ACCESS_INDEX) success = publisher_instance.agreements.create( did, ServiceTypesIndices.DEFAULT_ACCESS_INDEX, agreement_id, consumer_acc.address, publisher_acc) assert success, 'createAgreement failed.' # Verify condition types (condition contracts) agreement_values = keeper.agreement_manager.get_agreement(agreement_id) assert agreement_values.did == asset_id, '' cond_types = keeper.access_template.get_condition_types() for i, cond_id in enumerate(agreement_values.condition_ids): cond = keeper.condition_manager.get_condition(cond_id) assert cond.type_ref == cond_types[i] access_cond_id, lock_cond_id, escrow_cond_id = agreement_values.condition_ids # Fulfill lock_payment_condition is done automatically when create agreement is done correctly assert 2 == keeper.condition_manager.get_condition_state(lock_cond_id), '' assert 1 == keeper.condition_manager.get_condition_state( access_cond_id), '' assert 1 == keeper.condition_manager.get_condition_state( escrow_cond_id), '' # Fulfill access_condition tx_hash = keeper.access_condition.fulfill(agreement_id, asset_id, consumer_acc.address, publisher_acc) keeper.access_condition.get_tx_receipt(tx_hash) assert 2 == keeper.condition_manager.get_condition_state( access_cond_id), '' event = keeper.access_condition.subscribe_condition_fulfilled( agreement_id, 10, log_event(keeper.access_condition.FULFILLED_EVENT), (), wait=True) assert event, 'no event for AccessSecretStoreCondition.Fulfilled' # Fulfill escrow_payment_condition amounts = service_agreement.get_amounts_int() receivers = service_agreement.get_receivers() tx_hash = keeper.escrow_payment_condition.fulfill( agreement_id, asset_id, amounts, receivers, keeper.escrow_payment_condition.address, lock_cond_id, access_cond_id, publisher_acc) keeper.escrow_payment_condition.get_tx_receipt(tx_hash) assert 2 == keeper.condition_manager.get_condition_state( escrow_cond_id), '' event = keeper.escrow_payment_condition.subscribe_condition_fulfilled( agreement_id, 10, log_event(keeper.escrow_payment_condition.FULFILLED_EVENT), (), wait=True) assert event, 'no event for EscrowReward.Fulfilled' publisher_instance.assets.retire(did)
def run(args): logging.debug(f"script callef with args: {args}") # setup config options = { "keeper-contracts": { "keeper.url": args.node, "secret_store.url": args.secretstore_url, }, "resources": { "downloads.path": args.path.as_posix(), "metadata.url": args.metadata_url, "gateway.url": args.gateway_url, }, } config = Config(options_dict=options) logging.debug(f"nevermined config: {config}") # setup paths inputs_path = args.path / "inputs" inputs_path.mkdir() outputs_path = args.path / "outputs" outputs_path.mkdir() transformations_path = args.path / "transformations" transformations_path.mkdir() # setup nevermined nevermined = Nevermined(config) keeper = Keeper.get_instance() # setup consumer # here we need to create a temporary key file from the credentials key_file = NamedTemporaryFile("w", delete=False) json.dump(args.credentials, key_file) key_file.flush() key_file.close() consumer = Account( Web3.toChecksumAddress(args.credentials["address"]), password=args.password, key_file=key_file.name, ) # resolve workflow workflow = nevermined.assets.resolve(args.workflow) logging.info(f"resolved workflow {args.workflow}") logging.debug(f"workflow ddo {workflow.as_dictionary()}") # get stages stages = workflow.get_service("metadata").main["workflow"]["stages"] logging.debug(f"stages {stages}") # get inputs and transformations inputs = [] transformations = [] for stage in stages: inputs += [input_["id"] for input_ in stage["input"]] if "transformation" in stage: transformations.append(stage["transformation"]["id"]) logging.debug(f"inputs: {inputs}") logging.debug(f"transformations: {transformations}") # download assets for did in inputs: ddo = nevermined.assets.resolve(did) service_agreement = ddo.get_service("compute") logging.info(f"downloading asset {ddo.did}") nevermined.assets.download(ddo.did, service_agreement.index, consumer, inputs_path.as_posix()) for did in transformations: ddo = nevermined.assets.resolve(did) service_agreement = ddo.get_service("access") logging.info(f"downloading asset {ddo.did}") nevermined.assets.download(ddo.did, service_agreement.index, consumer, transformations_path.as_posix())
def demo(): """The Nevermined Federated Learning demo. This demo showcases the nevermined Federated Learning capabilities. FLow: 1. Setup nevermined 2. Setup accounts 3. Publish compute to the data assets 4. Publish algorithm 5. Publish workflows 6. Order computations 7. Execute workflows """ print("Setting up...\n") date_created = dates_generator() # 1. Setup nevermined nevermined = Nevermined(Config("config.ini")) keeper = Keeper.get_instance() provider = "0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0" # 2. Setup accounts acc = Account(Web3.toChecksumAddress(PARITY_ADDRESS), PARITY_PASSWORD, PARITY_KEYFILE) nevermined.accounts.request_tokens(acc, 100) provider_data0 = acc provider_data1 = acc provider_coordinator = acc consumer = acc # 3. Publish compute to the data with open("resources/metadata/metadata0.json") as f: metadata_data0 = json.load(f) metadata_data0["main"]["dateCreated"] = next(date_created) with open("resources/metadata/metadata1.json") as f: metadata_data1 = json.load(f) metadata_data1["main"]["dateCreated"] = next(date_created) ddo_compute0 = nevermined.assets.create_compute( metadata_data0, provider_data0, providers=[provider], ) assert ddo_compute0 is not None, "Creating asset compute0 on-chain failed" print( f"[DATA_PROVIDER0 --> NEVERMINED] Publishing compute to the data asset for asset0: {ddo_compute0.did}" ) ddo_compute1 = nevermined.assets.create_compute( metadata_data1, provider_data1, providers=[provider], ) assert ddo_compute1 is not None, "Creating asset compute1 on-chain failed" print( f"[DATA_PROVIDER1 --> NEVERMINED] Publishing compute to the data asset for asset1: {ddo_compute1.did}" ) with open("resources/metadata/metadata_compute_coordinator.json") as f: metadata_compute_coordinator = json.load(f) metadata_compute_coordinator["main"]["dateCreated"] = next( date_created) ddo_compute_coordinator = nevermined.assets.create_compute( metadata_compute_coordinator, provider_coordinator, providers=[provider], ) assert (ddo_compute_coordinator is not None), "Creating asset compute_coordinator on-chain failed" print( f"[COORDINATOR_PROVIDER --> NEVERMINED] Publishing coordinator compute asset: {ddo_compute_coordinator.did}" ) # 4. Publish algorithm with open("resources/metadata/metadata_transformation.json") as f: metadata_transformation = json.load(f) metadata_transformation["main"]["dateCreated"] = next(date_created) ddo_transformation = nevermined.assets.create( metadata_transformation, consumer, providers=[provider], ) assert (ddo_transformation is not None), "Creating asset transformation on-chain failed" print( f"[DATA_SCIENTIST --> NEVERMINED] Publishing algorithm asset: {ddo_transformation.did}" ) # 5. Publish the workflows with open("resources/metadata/metadata_workflow.json") as f: metadata_workflow = json.load(f) with open("resources/metadata/metadata_workflow_coordinator.json") as f: metadata_workflow_coordinator = json.load(f) metadata_workflow0 = copy.deepcopy(metadata_workflow) metadata_workflow0["main"]["workflow"]["stages"][0]["input"][0][ "id"] = ddo_compute0.did metadata_workflow0["main"]["workflow"]["stages"][0]["transformation"][ "id"] = ddo_transformation.did metadata_workflow1 = copy.deepcopy(metadata_workflow) metadata_workflow1["main"]["workflow"]["stages"][0]["input"][0][ "id"] = ddo_compute1.did metadata_workflow1["main"]["workflow"]["stages"][0]["transformation"][ "id"] = ddo_transformation.did metadata_workflow_coordinator["main"]["dateCreated"] = next(date_created) ddo_workflow0 = nevermined.assets.create( metadata_workflow0, consumer, providers=[provider], ) assert ddo_workflow0 is not None, "Creating asset workflow0 on-chain failed" print( f"[DATA_SCIENTIST --> NEVERMINED] Publishing compute workflow for asset0: {ddo_workflow0.did}" ) ddo_workflow1 = nevermined.assets.create( metadata_workflow1, consumer, providers=[provider], ) assert ddo_workflow1 is not None, "Creating asset workflow1 on-chain failed" print( f"[DATA_SCIENTIST --> NEVERMINED] Publishing compute workflow for asset1: {ddo_workflow1.did}" ) ddo_workflow_coordinator = nevermined.assets.create( metadata_workflow_coordinator, consumer, providers=[provider], ) assert (ddo_workflow_coordinator is not None), "Creating asset workflow_coordinator on-chain failed" print( f"[DATA_SCIENTIST --> NEVERMINED] Publishing compute workflow for coordinator: {ddo_workflow_coordinator.did}" ) # 6. Order computations service0 = ddo_compute0.get_service( service_type=ServiceTypes.CLOUD_COMPUTE) service_agreement0 = ServiceAgreement.from_service_dict( service0.as_dictionary()) agreement_id0 = nevermined.assets.order(ddo_compute0.did, service_agreement0.index, consumer, consumer) print( f"[DATA_SCIENTIST --> DATA_PROVIDER0] Requesting an agreement for compute to the data for asset0: {agreement_id0}" ) event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id0, 60, None, (), wait=True) assert event is not None, "Reward condition is not found" event = keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id0, 60, None, (), wait=True) assert event is not None, "Execution condition not found" service1 = ddo_compute1.get_service( service_type=ServiceTypes.CLOUD_COMPUTE) service_agreement1 = ServiceAgreement.from_service_dict( service1.as_dictionary()) agreement_id1 = nevermined.assets.order(ddo_compute1.did, service_agreement1.index, consumer, consumer) print( f"[DATA_SCIENTIST --> DATA_PROVIDER1] Requesting an agreement for compute to the data for asset1: {agreement_id1}" ) event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id1, 60, None, (), wait=True) assert event is not None, "Reward condition is not found" event = keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id1, 60, None, (), wait=True) assert event is not None, "Execution condition not found" service_coordinator = ddo_compute_coordinator.get_service( service_type=ServiceTypes.CLOUD_COMPUTE) service_agreement_coordinator = ServiceAgreement.from_service_dict( service_coordinator.as_dictionary()) agreement_id_coordinator = nevermined.assets.order( ddo_compute_coordinator.did, service_agreement_coordinator.index, consumer, consumer) print( f"[DATA_SCIENTIST --> COORDINATOR_PROVIDER] Requesting an agreement for coordinator compute: {agreement_id_coordinator}" ) event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id_coordinator, 60, None, (), wait=True) assert event is not None, "Reward condition is not found" event = keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id_coordinator, 60, None, (), wait=True) assert event is not None, "Execution condition not found" # 7. Execute workflows compute_coordinator_id = nevermined.assets.execute( agreement_id_coordinator, ddo_compute_coordinator.did, service_agreement_coordinator.index, consumer, ddo_workflow_coordinator.did, ) print( f"[DATA_SCIENTIST --> COORDINATOR_PROVIDER] Requesting execution for coordinator compute: {compute_coordinator_id}" ) compute_asset0_id = nevermined.assets.execute( agreement_id0, ddo_compute0.did, service_agreement0.index, consumer, ddo_workflow0.did, ) print( f"[DATA_SCIENTIST --> DATA_PROVIDER0] Requesting execution for compute to data for asset0: {compute_asset0_id}" ) compute_asset1_id = nevermined.assets.execute( agreement_id1, ddo_compute1.did, service_agreement1.index, consumer, ddo_workflow1.did, ) print( f"[DATA_SCIENTIST --> DATA_PROVIDER1] Requesting execution for compute to data for asset1: {compute_asset1_id}" ) jobs = [ (agreement_id_coordinator, compute_coordinator_id), (agreement_id0, compute_asset0_id), (agreement_id1, compute_asset1_id), ] return jobs
def demo(): nevermined = Nevermined(Config(CONFIG_FILE)) provider_account = Account(PROVIDER_ADDRESS, PROVIDER_PASSWORD, PROVIDER_KEYFILE) # publish asset metadata_compute = { "main": { "name": "CIFAR-10 Part 1", "dateCreated": date_now(), "author": "Nevermined Provider", "license": "", "price": "1", "files": [{ "index": 0, "contentType": "image/png", "checksum": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", "checksumType": "MD5", "contentLength": "12057507", "url": "https://ck2a37sxobgcdarvr7jewxvrlvde6kehhoy6lmfuks4uabuavtiq.arweave.net/ErQN_ldwTCGCNY_SS16xXUZPKIc7seWwtFS5QAaArNE", }, { "index": 1, "contentType": "application/json", "checksum": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", "checksumType": "MD5", "contentLength": "12057507", "url": "https://raw.githubusercontent.com/keyko-io/eth-nft-hack/rod/artsgenerator-demo/config.json" }], "type": "dataset", } } ddo_compute = nevermined.assets.create_compute(metadata_compute, provider_account) print(f"Published asset with DID: {ddo_compute.did}") # publish algorithm metadata_algorithm = { "main": { "name": "Generative artist", "dateCreated": date_now(), "author": "Gene Kogan", "license": "", "price": "0", # This file will not be used but there is a bug on the sdk that # expects a least one file to exist in an algorithm "files": [ { "index": 0, "contentType": "text/text", "checksum": "0x52b5c93b82dd9e7ecc3d9fdf4755f7f69a54484941897dc517b4adfe3bbc3377", "checksumType": "MD5", "contentLength": "12057507", "url": "https://github.com/nevermined-io/tools/raw/master/README.md", }, ], "type": "algorithm", "algorithm": { "language": "python", "format": "py", "version": "0.1.0", "entrypoint": "pwd && ls -lR && cat /data/inputs/**/config.json && python /nevermined-demo/run.py", "requirements": { "container": { "image": "neverminedio/artgenerator", "tag": "latest", "checksum": "sha256:53ad3a03b2fb240b6c494339821e6638cd44c989bcf26ec4d51a6a52f7518c1d", } }, }, } } ddo_algorithm = nevermined.assets.create(metadata_algorithm, provider_account) print(f"Published algorithm with DID: {ddo_algorithm.did}") metadata_workflow = { "main": { "name": "Mint my NFT", "dateCreated": date_now(), "author": "Nevermined Consumer", "license": "", "price": "0", "type": "workflow", "workflow": { "stages": [{ "index": 0, "input": [{ "index": 0, "id": ddo_compute.did }], "transformation": { "id": ddo_algorithm.did }, }] }, } } ddo_workflow = nevermined.assets.create(metadata_workflow, provider_account) print(f"Published workflow with DID: {ddo_workflow.did}") # order the asset keeper = Keeper.get_instance() service_agreement_id = nevermined.assets.order( ddo_compute.did, ServiceTypesIndices.DEFAULT_COMPUTING_INDEX, provider_account, provider_account, ) print() print("Ordering Data In-Situ Compute") print(f"Service Agreement ID: {service_agreement_id}") wait_for_event(keeper, service_agreement_id) # execute workflow execution_id = nevermined.assets.execute( service_agreement_id, ddo_compute.did, ServiceTypesIndices.DEFAULT_COMPUTING_INDEX, provider_account, ddo_workflow.did, ) print("Firing up the GPUs to mine some Art...") print(f"Execution ID: {execution_id}") print("This will take a about 1h...") print() print('Monitoring compute status:') # wait for compute job outputs_did = None while True: status = nevermined.assets.compute_status(service_agreement_id, execution_id, provider_account) if status["status"] == "Failed": raise ValueError("The job failed") elif status["status"] == "Succeeded": outputs_did = status["did"] break print(f"{execution_id}: {status['status']}") time.sleep(60) print(f"Outputs DID: {outputs_did}") # download the output assets print() print('Downloading outputs...') nevermined.assets.download(outputs_did, ServiceTypesIndices.DEFAULT_ACCESS_INDEX, provider_account, "./") print("Finished!")
def test_buy_asset(publisher_instance_no_init, consumer_instance_no_init): config = ExampleConfig.get_config() ConfigProvider.set_config(config) keeper = Keeper.get_instance() # :TODO: enable the actual SecretStore # SecretStoreProvider.set_secret_store_class(SecretStore) w3 = Web3Provider.get_web3() pub_acc = get_publisher_account() # Register ddo ddo = get_registered_ddo(publisher_instance_no_init, pub_acc) assert isinstance(ddo, DDO) # nevermined here will be used only to publish the asset. Handling the asset by the publisher # will be performed by the Gateway server running locally # restore the http client because we want the actual Gateway server to do the work # not the GatewayMock. # Gateway.set_http_client(requests) consumer_account = get_consumer_account() downloads_path_elements = len( os.listdir(consumer_instance_no_init._config.downloads_path)) if os.path.exists( consumer_instance_no_init._config.downloads_path) else 0 # sign agreement using the registered asset did above service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS) sa = ServiceAgreement.from_service_dict(service.as_dictionary()) # This will send the access request to Gateway which in turn will execute the agreement on-chain consumer_instance_no_init.accounts.request_tokens(consumer_account, 100) agreement_id = consumer_instance_no_init.assets.order( ddo.did, sa.index, consumer_account, consumer_account) event_wait_time = 10 event = keeper.lock_payment_condition.subscribe_condition_fulfilled( agreement_id, event_wait_time, log_event(keeper.lock_payment_condition.FULFILLED_EVENT), (), wait=True ) assert event, 'no event for LockRewardCondition.Fulfilled' # give access publisher_instance_no_init.agreements.conditions.grant_access( agreement_id, ddo.asset_id, consumer_account.address, pub_acc) event = keeper.access_condition.subscribe_condition_fulfilled( agreement_id, event_wait_time, log_event(keeper.access_condition.FULFILLED_EVENT), (), wait=True ) assert event, 'no event for AccessSecretStoreCondition.Fulfilled' assert consumer_instance_no_init.agreements.is_access_granted(agreement_id, ddo.did, consumer_account.address) amounts = list(map(int, service.get_param_value_by_name('_amounts'))) receivers = service.get_param_value_by_name('_receivers') publisher_instance_no_init.agreements.conditions.release_reward( agreement_id, ddo.asset_id, amounts, receivers, pub_acc) assert consumer_instance_no_init.assets.access( agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path) assert len(os.listdir(config.downloads_path)) == downloads_path_elements + 1 # Check that we can access only an specific file in passing the index. assert consumer_instance_no_init.assets.access( agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path, 1 ) assert len(os.listdir(config.downloads_path)) == downloads_path_elements + 1 with pytest.raises(AssertionError): consumer_instance_no_init.assets.access( agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path, -2 ) with pytest.raises(AssertionError): consumer_instance_no_init.assets.access( agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path, 3 ) # decrypt the contentUrls using the publisher account instead of consumer account. # if the secret store is working and ACL check is enabled, this should fail # since SecretStore decrypt will fail the checkPermissions check and the gateway will return # an http error code back with pytest.raises(ValueError) as e: consumer_instance_no_init.assets.access( agreement_id, ddo.did, sa.index, pub_acc, config.downloads_path ) print(e)
def test_buy_asset_no_secret_store(publisher_instance_gateway, consumer_instance_gateway): config = ExampleConfig.get_config() ConfigProvider.set_config(config) keeper = Keeper.get_instance() w3 = Web3Provider.get_web3() pub_acc = get_publisher_account() for method in [ServiceAuthorizationTypes.SECRET_STORE, ServiceAuthorizationTypes.PSK_ECDSA, ServiceAuthorizationTypes.PSK_RSA]: # Register ddo ddo = get_registered_with_psk(publisher_instance_gateway, pub_acc, auth_method=method) assert isinstance(ddo, DDO) # nevermined here will be used only to publish the asset. Handling the asset by the publisher # will be performed by the Gateway server running locally # restore the http client because we want the actual Gateway server to do the work # not the GatewayMock. # Gateway.set_http_client(requests) consumer_account = get_consumer_account() downloads_path_elements = len( os.listdir(consumer_instance_gateway._config.downloads_path)) if os.path.exists( consumer_instance_gateway._config.downloads_path) else 0 # sign agreement using the registered asset did above service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS) sa = ServiceAgreement.from_service_dict(service.as_dictionary()) # This will send the access request to Gateway which in turn will execute the agreement on-chain consumer_instance_gateway.accounts.request_tokens(consumer_account, 100) agreement_id = consumer_instance_gateway.assets.order( ddo.did, sa.index, consumer_account, consumer_account) event_wait_time = 10 event = keeper.access_template.subscribe_agreement_created( agreement_id, event_wait_time, log_event(keeper.access_template.AGREEMENT_CREATED_EVENT), (), wait=True ) assert event, 'no event for EscrowAccessSecretStoreTemplate.AgreementCreated' event = keeper.lock_payment_condition.subscribe_condition_fulfilled( agreement_id, event_wait_time, log_event(keeper.lock_payment_condition.FULFILLED_EVENT), (), wait=True ) assert event, 'no event for LockRewardCondition.Fulfilled' # give access publisher_instance_gateway.agreements.conditions.grant_access( agreement_id, ddo.asset_id, consumer_account.address, pub_acc) event = keeper.access_condition.subscribe_condition_fulfilled( agreement_id, event_wait_time, log_event(keeper.access_condition.FULFILLED_EVENT), (), wait=True ) assert event, 'no event for AccessSecretStoreCondition.Fulfilled' assert consumer_instance_gateway.agreements.is_access_granted(agreement_id, ddo.did, consumer_account.address) publisher_instance_gateway.agreements.conditions.release_reward( agreement_id, ddo.asset_id, sa.get_amounts_int(), sa.get_receivers(), pub_acc) assert consumer_instance_gateway.assets.access( agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path) assert len(os.listdir(config.downloads_path)) == downloads_path_elements + 1 with pytest.raises(AssertionError): consumer_instance_gateway.assets.access( agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path, -2 ) with pytest.raises(AssertionError): consumer_instance_gateway.assets.access( agreement_id, ddo.did, sa.index, consumer_account, config.downloads_path, 3 )
def __init__(self, config=None): """ Initialize Nevermined class. >> # Make a new Nevermined instance >> nevermined = Nevermined({...}) This class provides the main top-level functions in nevermined protocol: * Publish assets metadata and associated services * Each asset is assigned a unique DID and a DID Document (DDO) * The DDO contains the asset's services including the metadata * The DID is registered on-chain with a URL of the metadata store to retrieve the DDO from >> ddo = nevermined.assets.create(metadata, publisher_account) * Discover/Search assets via the current configured metadata store. >> assets_list = nevermined.assets.search('search text') * Purchase asset services by choosing a service agreement from the asset's DDO. Purchase goes through the service agreements interface and starts by signing a service agreement then sending the signature to the publisher's Gateway server via the `purchaseEndpoint` in the service definition: >> service_def_id = ddo.get_service(ServiceTypes.ASSET_ACCESS).service_definition_id >> service_agreement_id = nevermined.assets.order(did, service_def_id, consumer_account) An instance of Nevermined is parameterized by a `Config` instance. :param config: Config instance """ # Configuration information for the market is stored in the Config class # config = Config(filename=config_file, options_dict=config_dict) if not config: config = ConfigProvider.get_config() self._config = config self._web3 = Web3Provider.get_web3(self._config.keeper_url) ContractHandler.artifacts_path = self._config.keeper_path contracts = [ 'DIDRegistry', 'Dispenser', 'TemplateStoreManager', 'NeverminedToken', 'ConditionStoreManager', 'EscrowAccessSecretStoreTemplate', 'AgreementStoreManager', 'AgreementStoreManager', 'AccessSecretStoreCondition', 'LockRewardCondition', 'HashLockCondition', 'SignCondition', 'EscrowReward' ] self._keeper = Keeper.get_instance(self._config.keeper_path, contracts) self._did_resolver = DIDResolver(self._keeper.did_registry) # Initialize the public sub-modules self.tokens = Tokens(self._keeper) self.accounts = Accounts(self._keeper, self._config, self.tokens, Faucet) self.templates = Templates(self._keeper, config) self.agreements = self._make_agreements() self.assets = Assets(self._keeper, self._did_resolver, self.agreements, AssetConsumer, AssetExecutor, self._config) self.nfts = NFTs(self._keeper) self.services = Services() self.secret_store = SecretStore(self._config) self.providers = Providers(self._keeper, self._did_resolver, self._config) self.auth = Auth(self._keeper, self._config.storage_path) self.provenance = Provenance(self._keeper, self._config) self.files = Files(self._config) logger.debug('Nevermined instance initialized: ') logger.debug( f'\tOther accounts: {sorted([a.address for a in self.accounts.list()])}' ) logger.debug(f'\tDIDRegistry @ {self._keeper.did_registry.address}')
def buy_asset(): """ Requires all Nevermined services running. """ ConfigProvider.set_config(ExampleConfig.get_config()) config = ConfigProvider.get_config() providers = { 'duero': '0xfEF2d5e1670342b9EF22eeeDcb287EC526B48095', 'nile': '0x4aaab179035dc57b35e2ce066919048686f82972' } # make nevermined instance nevermined = Nevermined() Diagnostics.verify_contracts() acc = get_account(1) if not acc: acc = ([acc for acc in nevermined.accounts.list() if acc.password] or nevermined.accounts.list())[0] keeper = Keeper.get_instance() # Register ddo did = '' # 'did:nv:7648596b60f74301ae1ef9baa5d637255d517ff362434754a3779e1de4c8219b' if did: ddo = nevermined.assets.resolve(did) logging.info(f'using ddo: {did}') else: ddo = nevermined.assets.create(example_metadata.metadata, acc, providers=[], authorization_type='SecretStore', use_secret_store=True) assert ddo is not None, f'Registering asset on-chain failed.' did = ddo.did logging.info(f'registered ddo: {did}') # nevermined here will be used only to publish the asset. Handling the asset by the publisher # will be performed by the Gateway server running locally test_net = os.environ.get('TEST_NET', '') if test_net.startswith('nile'): provider = keeper.did_registry.to_checksum_address(providers['nile']) elif test_net.startswith('duero'): provider = keeper.did_registry.to_checksum_address(providers['duero']) else: provider = '0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0' # Wait for did registry event event = keeper.did_registry.subscribe_to_event( keeper.did_registry.DID_REGISTRY_EVENT_NAME, 30, event_filter={ '_did': Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), '_owner': acc.address}, wait=True ) if not event: logging.warning(f'Failed to get the did registry event for asset with did {did}.') assert keeper.did_registry.get_block_number_updated(ddo.asset_id) > 0, \ f'There is an issue in registering asset {did} on-chain.' keeper.did_registry.add_provider(ddo.asset_id, provider, acc) logging.info(f'is {provider} set as did provider: ' f'{keeper.did_registry.is_did_provider(ddo.asset_id, provider)}') nevermined_cons = Nevermined() consumer_account = get_account(0) # sign agreement using the registered asset did above service = ddo.get_service(service_type=ServiceTypes.ASSET_ACCESS) # This will send the order request to Gateway which in turn will execute the agreement on-chain nevermined_cons.accounts.request_tokens(consumer_account, 10) sa = ServiceAgreement.from_service_dict(service.as_dictionary()) agreement_id = '' if not agreement_id: # Use these 2 lines to request new agreement from Gateway # agreement_id, signature = nevermined_cons.agreements.prepare(did, sa.service_definition_id, # consumer_account) # nevermined_cons.agreements.send(did, agreement_id, sa.service_definition_id, signature, # consumer_account) # assets.order now creates agreement directly using consumer account. agreement_id = nevermined_cons.assets.order( did, sa.index, consumer_account) logging.info('placed order: %s, %s', did, agreement_id) event = keeper.access_template.subscribe_agreement_created( agreement_id, 60, None, (), wait=True ) assert event, "Agreement event is not found, check the keeper node's logs" logging.info(f'Got agreement event, next: lock reward condition') event = keeper.lock_payment_condition.subscribe_condition_fulfilled( agreement_id, 60, None, (), wait=True ) assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs" logging.info('Got lock reward event, next: wait for the access condition..') event = keeper.access_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True ) logging.info(f'Got access event {event}') i = 0 while nevermined.agreements.is_access_granted( agreement_id, did, consumer_account.address) is not True and i < 15: time.sleep(1) i += 1 assert nevermined.agreements.is_access_granted(agreement_id, did, consumer_account.address) nevermined.assets.access( agreement_id, did, sa.index, consumer_account, config.downloads_path, index=0) logging.info('Success buying asset.') event = keeper.escrow_payment_condition.subscribe_condition_fulfilled( agreement_id, 30, None, (), wait=True ) assert event, 'no event for EscrowReward.Fulfilled' logging.info(f'got EscrowReward.FULFILLED event: {event}') logging.info('Done buy asset.')
def compute_example(verbose=False): print("Setting up...") if verbose: configure_logging() date_created = dates_generator() # Setup nevermined ConfigProvider.set_config(ExampleConfig.get_config()) config = ConfigProvider.get_config() nevermined = Nevermined() keeper = Keeper.get_instance() provider = "0x068Ed00cF0441e4829D9784fCBe7b9e26D4BD8d0" asset_rewards = { "_amounts": ["10", "2"], "_receivers": ["0x00Bd138aBD70e2F00903268F3Db08f2D25677C9e", "0x068ed00cf0441e4829d9784fcbe7b9e26d4bd8d0"] } # Setup accounts acc = Account( Web3.toChecksumAddress(PROVIDER_ADDRESS), PROVIDER_PASSWORD, PROVIDER_KEYFILE ) nevermined.accounts.request_tokens(acc, 10) provider_acc = acc consumer_acc = acc # Publish compute example_metadata.compute_ddo["main"]["dateCreated"] = next(date_created) ddo_compute = nevermined.assets.create_compute( example_metadata.metadata, provider_acc, asset_rewards, providers=[provider] ) assert ddo_compute is not None, "Creating compute asset on-chain failed." print( f"[PROVIDER --> NEVERMINED] Publishing compute to the data asset: {ddo_compute.did}" ) # Publish algorithm example_metadata.algo_metadata["main"]["dateCreated"] = next(date_created) ddo_transformation = nevermined.assets.create( example_metadata.algo_metadata, consumer_acc, providers=[provider] ) assert ( ddo_transformation is not None ), "Creating asset transformation on-chain failed." print( f"[CONSUMER --> NEVERMINED] Publishing algorithm asset: {ddo_transformation.did}" ) # Publish workflows workflow_metadata = example_metadata.workflow_ddo workflow_metadata["main"]["workflow"]["stages"][0]["input"][0]["id"] = ddo_compute.did workflow_metadata["main"]["workflow"]["stages"][0]["transformation"][ "id" ] = ddo_transformation.did ddo_workflow = nevermined.assets.create( workflow_metadata, consumer_acc, providers=[provider] ) assert ddo_workflow is not None, "Creating asset workflow on-chain failed." print(f"[CONSUMER --> NEVERMINED] Publishing compute workflow: {ddo_workflow.did}") # Order computation service = ddo_compute.get_service(service_type=ServiceTypes.CLOUD_COMPUTE) service_agreement = ServiceAgreement.from_service_dict(service.as_dictionary()) agreement_id = nevermined.assets.order( ddo_compute.did, service_agreement.index, consumer_acc ) print( f"[CONSUMER --> PROVIDER] Requesting an agreement for compute to the data: {agreement_id}" ) event = keeper.lock_payment_condition.subscribe_condition_fulfilled( agreement_id, 60, None, (), wait=True ) assert event is not None, "Reward condition is not found" event = keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id, 60, None, (), wait=True ) assert event is not None, "Execution condition not found" # Execute workflow nevermined.assets.execute( agreement_id, ddo_compute.did, service_agreement.index, consumer_acc, ddo_workflow.did, ) print("[CONSUMER --> PROVIDER] Requesting execution of the compute workflow") event = keeper.escrow_payment_condition.subscribe_condition_fulfilled( agreement_id, 60, None, (), wait=True ) assert event is not None, "Escrow Reward condition not found" print("Workflow successfully executed")
def setup_all(): config = ExampleConfig.get_config() Web3Provider.get_web3(config.keeper_url) ContractHandler.artifacts_path = config.keeper_path Keeper.get_instance(artifacts_path=config.keeper_path)
def init_ocn_tokens(nevermined, account, amount=100): nevermined.accounts.request_tokens(account, amount) Keeper.get_instance().token.token_approve( Keeper.get_instance().dispenser.address, amount, account)
def test_nfts_flow(publisher_instance_no_init, consumer_instance_no_init): config = ExampleConfig.get_config() ConfigProvider.set_config(config) keeper = Keeper.get_instance() pub_acc = get_publisher_account() # Register ddo ddo = get_registered_ddo_nft(publisher_instance_no_init, pub_acc) asset_id = ddo.asset_id nft_amounts = 1 assert isinstance(ddo, DDO) consumer_account = get_consumer_account() consumer_instance_no_init.accounts.request_tokens(consumer_account, 100) service_sales = ddo.get_service(service_type=ServiceTypes.NFT_SALES) sa_sales = ServiceAgreement.from_service_dict( service_sales.as_dictionary()) amounts = sa_sales.get_amounts_int() receivers = sa_sales.get_receivers() number_nfts = sa_sales.get_number_nfts() token_address = keeper.token.address sales_agreement_id = consumer_instance_no_init.assets.order( ddo.did, sa_sales.index, consumer_account, consumer_account) sales_agreement = keeper.agreement_manager.get_agreement( sales_agreement_id) assert sales_agreement.did == asset_id, '' lock_cond_id = sales_agreement.condition_ids[0] access_cond_id = sales_agreement.condition_ids[1] escrow_cond_id = sales_agreement.condition_ids[2] # transfer the nft keeper.transfer_nft_condition.fulfill(sales_agreement_id, asset_id, consumer_account.address, nft_amounts, lock_cond_id, pub_acc) # escrow payment keeper.escrow_payment_condition.fulfill( sales_agreement_id, asset_id, amounts, receivers, keeper.escrow_payment_condition.address, token_address, lock_cond_id, access_cond_id, pub_acc) assert keeper.condition_manager.get_condition_state(lock_cond_id) == 2, '' assert keeper.condition_manager.get_condition_state( access_cond_id) == 2, '' assert keeper.condition_manager.get_condition_state( escrow_cond_id) == 2, '' assert keeper.did_registry.balance(consumer_account.address, asset_id) >= number_nfts # CHECK ACCESS CREATING THE AGREEMENT nft_access_service_agreement = ServiceAgreement.from_ddo( ServiceTypes.NFT_ACCESS, ddo) sa_access = ddo.get_service(service_type=ServiceTypes.NFT_ACCESS) nft_access_agreement_id = consumer_instance_no_init.assets.order( ddo.did, sa_access.index, consumer_account, consumer_account) event = keeper.nft_access_template.subscribe_agreement_created( nft_access_agreement_id, 10, log_event(keeper.nft_access_template.AGREEMENT_CREATED_EVENT), (), wait=True) assert event, 'no event for AgreementCreated ' time.sleep(3) keeper.nft_holder_condition.fulfill(nft_access_agreement_id, asset_id, consumer_account.address, number_nfts, consumer_account) time.sleep(3) keeper.nft_access_condition.fulfill(nft_access_agreement_id, asset_id, consumer_account.address, pub_acc) time.sleep(3) access_agreement = keeper.agreement_manager.get_agreement( nft_access_agreement_id) assert access_agreement.did == asset_id, '' nft_holder_cond_id = access_agreement.condition_ids[0] nft_access_cond_id = access_agreement.condition_ids[1] assert keeper.condition_manager.get_condition_state( nft_access_cond_id) == 2, '' assert keeper.condition_manager.get_condition_state( nft_holder_cond_id) == 2, '' # AND HERE TESTING WITHOUT CREATING THE AGREEMENT service_access = ddo.get_service(service_type=ServiceTypes.NFT_ACCESS) no_agreement_id = '0x' assert consumer_instance_no_init.assets.access( no_agreement_id, ddo.did, service_access.index, consumer_account, config.downloads_path, service_type=ServiceTypes.NFT_ACCESS) # AND HERE CHECKING CREATING AN AGREEMENT FIRST nft_access_service_agreement = ServiceAgreement.from_ddo( ServiceTypes.NFT_ACCESS, ddo) nft_access_agreement_id = ServiceAgreement.create_new_agreement_id() (nft_access_cond_id, nft_holder_cond_id ) = nft_access_service_agreement.generate_agreement_condition_ids( nft_access_agreement_id, asset_id, consumer_account.address, keeper) keeper.nft_access_template.create_agreement( nft_access_agreement_id, asset_id, [nft_holder_cond_id, nft_access_cond_id], nft_access_service_agreement.conditions_timelocks, nft_access_service_agreement.conditions_timeouts, consumer_account.address, pub_acc) event = keeper.nft_access_template.subscribe_agreement_created( nft_access_agreement_id, 10, log_event(keeper.nft_access_template.AGREEMENT_CREATED_EVENT), (), wait=True) assert event, 'no event for AgreementCreated ' # This is because in this test the gateway is executed using the same account than the consumer keeper.did_registry.add_provider(asset_id, consumer_account.address, pub_acc) time.sleep(3) assert consumer_instance_no_init.assets.access( nft_access_agreement_id, ddo.did, service_access.index, consumer_account, config.downloads_path, index=0, service_type=ServiceTypes.NFT_ACCESS)