def place_order(publisher_account, ddo, consumer_account, service_type): keeper = keeper_instance() agreement_id = ServiceAgreement.create_new_agreement_id() publisher_address = publisher_account.address # balance = keeper.token.get_token_balance(consumer_account.address)/(2**18) # if balance < 20: # keeper.dispenser.request_tokens(100, consumer_account) service_agreement = ServiceAgreement.from_ddo(service_type, ddo) condition_ids = service_agreement.generate_agreement_condition_ids( agreement_id, ddo.asset_id, consumer_account.address, publisher_address, keeper) time_locks = service_agreement.conditions_timelocks time_outs = service_agreement.conditions_timeouts template_name = keeper.template_manager.SERVICE_TO_TEMPLATE_NAME[ service_type] template_id = keeper.template_manager.create_template_id(template_name) actor_map = { 'consumer': consumer_account.address, 'provider': publisher_address } actors = [ actor_map[_type] for _type in get_template_actor_types(keeper, template_id) ] assert keeper.template_manager.contract_concise.isTemplateIdApproved( template_id), f'template {template_id} is not approved.' keeper_instance().agreement_manager.create_agreement( agreement_id, ddo.asset_id, template_id, condition_ids, time_locks, time_outs, actors, consumer_account) return agreement_id
def get_dataset_ddo_with_access_service(account, providers=None): keeper = keeper_instance() metadata = get_sample_ddo()['service'][0]['attributes'] metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) service_descriptor = get_access_service_descriptor(keeper, account, metadata) return get_registered_ddo(account, metadata, service_descriptor, providers)
def get_dataset_ddo_with_compute_service_no_rawalgo(account, providers=None): keeper = keeper_instance() metadata = get_sample_ddo_with_compute_service( )['service'][0]['attributes'] metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) service_descriptor = get_compute_service_descriptor_no_rawalgo( keeper, account, metadata[MetadataMain.KEY]['price'], metadata) return get_registered_ddo(account, metadata, service_descriptor, providers)
def lock_reward(agreement_id, service_agreement, consumer_account): keeper = keeper_instance() price = service_agreement.get_price() keeper.token.token_approve(keeper.lock_reward_condition.address, price, consumer_account) time.sleep(3) tx_hash = keeper.lock_reward_condition.fulfill( agreement_id, keeper.escrow_reward_condition.address, price, consumer_account) keeper.lock_reward_condition.get_tx_receipt(tx_hash)
def test_agreement_expiry(): pub_acc = get_publisher_account() keeper = keeper_instance() metadata = get_sample_ddo()['service'][0]['attributes'] metadata['main']['files'][0]['checksum'] = str(uuid.uuid4()) service_descriptor = get_access_service_descriptor(keeper, pub_acc, metadata) service_descriptor[1]['attributes']['main']['timeout'] = 2 agreement = ServiceFactory.build_service(service_descriptor) start_time = datetime.now().timestamp() not_expired = validate_agreement_expiry(agreement, start_time) assert not_expired, 'Agreement should not be expired at this point.' time.sleep(3) with pytest.raises(ServiceAgreementExpired): validate_agreement_expiry(agreement, start_time)
def test_publish(client): endpoint = BaseURLs.ASSETS_URL + '/publish' did = DID.did({"0": str(uuid.uuid4())}) asset_id = did_to_id(did) account = get_provider_account() test_urls = ['url 00', 'url 11', 'url 22'] keeper = keeper_instance() urls_json = json.dumps(test_urls) asset_id_hash = add_ethereum_prefix_and_hash_msg(asset_id) signature = keeper.sign_hash(asset_id_hash, account) address = web3().eth.account.recoverHash(asset_id_hash, signature=signature) assert address.lower() == account.address.lower() address = keeper.personal_ec_recover(asset_id, signature) assert address.lower() == account.address.lower() payload = { 'documentId': asset_id, 'signature': signature, 'document': urls_json, 'publisherAddress': account.address } post_response = client.post(endpoint, data=json.dumps(payload), content_type='application/json') encrypted_url = post_response.data.decode('utf-8') assert encrypted_url.startswith('0x') # publish using auth token signature = generate_token(account) payload['signature'] = signature did = DID.did({"0": str(uuid.uuid4())}) asset_id = did_to_id(did) payload['documentId'] = add_0x_prefix(asset_id) post_response = client.post(endpoint, data=json.dumps(payload), content_type='application/json') encrypted_url = post_response.data.decode('utf-8') assert encrypted_url.startswith('0x')
def version(): keeper = keeper_instance() info = dict() info['software'] = Metadata.TITLE info['version'] = get_version() info['keeper-url'] = config.keeper_url info['network'] = keeper.network_name info['contracts'] = dict() info['contracts']['AccessSecretStoreCondition'] = keeper.access_secret_store_condition.address info['contracts']['AgreementStoreManager'] = keeper.agreement_manager.address info['contracts']['ConditionStoreManager'] = keeper.condition_manager.address info['contracts']['DIDRegistry'] = keeper.did_registry.address if keeper.network_name != 'pacific': info['contracts']['Dispenser'] = keeper.dispenser.address info['contracts']['EscrowReward'] = keeper.escrow_reward_condition.address info['contracts']['LockRewardCondition'] = keeper.lock_reward_condition.address info['contracts']['OceanToken'] = keeper.token.address info['contracts']['TemplateStoreManager'] = keeper.template_manager.address info['contracts']['ComputeExecutionCondition'] = keeper.compute_execution_condition.address info['keeper-version'] = get_latest_keeper_version() info['provider-address'] = get_provider_account().address return jsonify(info)
def _check_job_id(client, job_id, agreement_id, wait_time=20): endpoint = BaseURLs.ASSETS_URL + '/compute' cons_acc = get_consumer_account() keeper = keeper_instance() msg = f'{cons_acc.address}{job_id}{agreement_id}' agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg) signature = keeper.sign_hash(agreement_id_hash, cons_acc) payload = dict({ 'signature': signature, 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address, 'jobId': job_id, }) job_info = get_compute_job_info(client, endpoint, payload) assert job_info, f'Failed to get job info for jobId {job_id}' print(f'got info for compute job {job_id}: {job_info}') assert job_info['statusText'] in get_possible_compute_job_status_text() did = None # get did of results for _ in range(wait_time * 4): job_info = get_compute_job_info(client, endpoint, payload) did = job_info['did'] if did: break time.sleep(0.25) assert did, f'Compute job has no results, job info {job_info}.' # check results ddo ddo = DIDResolver(keeper.did_registry).resolve(did) assert ddo, f'Failed to resolve ddo for did {did}' consumer_permission = keeper.did_registry.get_permission( did, cons_acc.address) assert consumer_permission is True, \ f'Consumer address {cons_acc.address} has no permissions on the results ' \ f'did {did}. This is required, the consumer must be able to access the results'
def test_consume(client): aqua = Aquarius('http://localhost:5000') for did in aqua.list_assets(): aqua.retire_asset_ddo(did) endpoint = BaseURLs.ASSETS_URL + '/consume' pub_acc = get_publisher_account() cons_acc = get_consumer_account() keeper = keeper_instance() ddo = get_dataset_ddo_with_access_service(pub_acc, providers=[pub_acc.address]) # initialize an agreement agreement_id = place_order(pub_acc, ddo, cons_acc, ServiceTypes.ASSET_ACCESS) payload = dict({ 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address }) agr_id_hash = add_ethereum_prefix_and_hash_msg(agreement_id) signature = keeper.sign_hash(agr_id_hash, cons_acc) index = 0 event = keeper.agreement_manager.subscribe_agreement_created(agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Agreement event is not found, check the keeper node's logs" consumer_balance = keeper.token.get_token_balance(cons_acc.address) if consumer_balance < 50: keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc) sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo) lock_reward(agreement_id, sa, cons_acc) event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs" grant_access(agreement_id, ddo, cons_acc, pub_acc) event = keeper.access_secret_store_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event or keeper.access_secret_store_condition.check_permissions( ddo.asset_id, cons_acc.address ), f'Failed to get access permission: agreement_id={agreement_id}, ' \ f'did={ddo.did}, consumer={cons_acc.address}' # Consume using decrypted url files_list = json.loads( do_secret_store_decrypt(did_to_id(ddo.did), ddo.encrypted_files, pub_acc, get_config())) payload['url'] = files_list[index]['url'] request_url = endpoint + '?' + '&'.join( [f'{k}={v}' for k, v in payload.items()]) response = client.get(request_url) assert response.status == '200 OK' # Consume using url index and signature (let brizo do the decryption) payload.pop('url') payload['signature'] = signature payload['index'] = index request_url = endpoint + '?' + '&'.join( [f'{k}={v}' for k, v in payload.items()]) response = client.get(request_url) assert response.status == '200 OK'
def test_compute_norawalgo_allowed(client): aqua = Aquarius('http://localhost:5000') for did in aqua.list_assets(): aqua.retire_asset_ddo(did) pub_acc = get_publisher_account() cons_acc = get_consumer_account() keeper = keeper_instance() # publish a dataset asset dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service_no_rawalgo( pub_acc, providers=[pub_acc.address]) # CHECKPOINT 1 algorithmMeta = { "rawcode": "console.log('Hello world'!)", "format": 'docker-image', "version": '0.1', "container": { "entrypoint": 'node $ALGO', "image": 'node', "tag": '10' } } # prepare parameter values for the compute endpoint # signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta # initialize an agreement agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service, cons_acc, ServiceTypes.CLOUD_COMPUTE) # CHECKPOINT 2 event = keeper.agreement_manager.subscribe_agreement_created(agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Agreement event is not found, check the keeper node's logs" consumer_balance = keeper.token.get_token_balance(cons_acc.address) if consumer_balance < 50: keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc) sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE, dataset_ddo_w_compute_service) lock_reward(agreement_id, sa, cons_acc) event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs" grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id, cons_acc, pub_acc) event = keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event or keeper.compute_execution_condition.was_compute_triggered( dataset_ddo_w_compute_service.asset_id, cons_acc.address ), (f'Failed to compute: agreement_id={agreement_id}, ' f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}' ) # prepare consumer signature on agreement_id msg = f'{cons_acc.address}{agreement_id}' agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg) signature = keeper.sign_hash(agreement_id_hash, cons_acc) # Start the compute job payload = dict({ 'signature': signature, 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address, 'algorithmDid': None, 'algorithmMeta': algorithmMeta, 'output': build_stage_output_dict(dict(), dataset_ddo_w_compute_service, cons_acc.address, pub_acc) }) endpoint = BaseURLs.ASSETS_URL + '/compute' response = client.post(endpoint, data=json.dumps(payload), content_type='application/json') assert response.status == '400 BAD REQUEST', f'start compute job failed: {response.status} , { response.data}'
def test_compute(client): aqua = Aquarius('http://localhost:5000') for did in aqua.list_assets(): aqua.retire_asset_ddo(did) pub_acc = get_publisher_account() cons_acc = get_consumer_account() keeper = keeper_instance() # publish a dataset asset dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service( pub_acc, providers=[pub_acc.address]) # publish an algorithm asset (asset with metadata of type `algorithm`) alg_ddo = get_algorithm_ddo(cons_acc, providers=[pub_acc.address]) # CHECKPOINT 1 # prepare parameter values for the compute endpoint # signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta # initialize an agreement agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service, cons_acc, ServiceTypes.CLOUD_COMPUTE) # CHECKPOINT 2 event = keeper.agreement_manager.subscribe_agreement_created(agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Agreement event is not found, check the keeper node's logs" consumer_balance = keeper.token.get_token_balance(cons_acc.address) if consumer_balance < 50: keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc) sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE, dataset_ddo_w_compute_service) lock_reward(agreement_id, sa, cons_acc) event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs" grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id, cons_acc, pub_acc) event = keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event or keeper.compute_execution_condition.was_compute_triggered( dataset_ddo_w_compute_service.asset_id, cons_acc.address ), (f'Failed to compute: agreement_id={agreement_id}, ' f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}' ) # prepare consumer signature on agreement_id msg = f'{cons_acc.address}{agreement_id}' agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg) signature = keeper.sign_hash(agreement_id_hash, cons_acc) # Start the compute job payload = dict({ 'signature': signature, 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address, 'algorithmDid': alg_ddo.did, 'algorithmMeta': {}, 'output': build_stage_output_dict(dict(), dataset_ddo_w_compute_service, cons_acc.address, pub_acc) }) endpoint = BaseURLs.ASSETS_URL + '/compute' response = client.post(endpoint, data=json.dumps(payload), content_type='application/json') assert response.status == '200 OK', f'start compute job failed: {response.data}' job_info = response.json[0] print(f'got response from starting compute job: {job_info}') job_id = job_info.get('jobId', '') msg = f'{cons_acc.address}{job_id}{agreement_id}' agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg) signature = keeper.sign_hash(agreement_id_hash, cons_acc) payload = dict({ 'signature': signature, 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address, 'jobId': job_id, }) job_info = get_compute_job_info(client, endpoint, payload) assert job_info, f'Failed to get job info for jobId {job_id}' print(f'got info for compute job {job_id}: {job_info}') assert job_info['statusText'] in get_possible_compute_job_status_text()
def grant_compute(agreement_id, asset_id, consumer_account, publisher_account): keeper = keeper_instance() tx_hash = keeper.compute_execution_condition.fulfill( agreement_id, asset_id, consumer_account.address, publisher_account) keeper.compute_execution_condition.get_tx_receipt(tx_hash)
def grant_access(agreement_id, ddo, consumer_account, publisher_account): keeper = keeper_instance() tx_hash = keeper.access_secret_store_condition.fulfill( agreement_id, ddo.asset_id, consumer_account.address, publisher_account) keeper.access_secret_store_condition.get_tx_receipt(tx_hash)
def get_registered_ddo(account, metadata, service_descriptor, providers=None): keeper = keeper_instance() aqua = Aquarius('http://localhost:5000') ddo = DDO() ddo_service_endpoint = aqua.get_service_endpoint() metadata_service_desc = ServiceDescriptor.metadata_service_descriptor( metadata, ddo_service_endpoint) service_descriptors = list([ ServiceDescriptor.authorization_service_descriptor( 'http://localhost:12001') ]) service_descriptors.append(service_descriptor) service_type = service_descriptor[0] service_descriptors = [metadata_service_desc] + service_descriptors services = ServiceFactory.build_services(service_descriptors) checksums = dict() for service in services: checksums[str(service.index)] = checksum(service.main) # Adding proof to the ddo. ddo.add_proof(checksums, account) did = ddo.assign_did(DID.did(ddo.proof['checksum'])) ddo_service_endpoint.replace('{did}', did) services[0].set_service_endpoint(ddo_service_endpoint) stype_to_service = {s.type: s for s in services} _service = stype_to_service[service_type] name_to_address = { cname: cinst.address for cname, cinst in keeper.contract_name_to_instance.items() } _service.init_conditions_values(did, contract_name_to_address=name_to_address) for service in services: ddo.add_service(service) ddo.proof['signatureValue'] = keeper.sign_hash(did_to_id_bytes(did), account) ddo.add_public_key(did, account.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) try: _oldddo = aqua.get_asset_ddo(ddo.did) if _oldddo: aqua.retire_asset_ddo(ddo.did) except ValueError: pass if not plecos.is_valid_dict_local(ddo.metadata): print(f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}') assert False, f'invalid metadata: {plecos.validate_dict_local(ddo.metadata)}' encrypted_files = do_secret_store_encrypt( remove_0x_prefix(ddo.asset_id), json.dumps(metadata['main']['files']), account, get_config()) # only assign if the encryption worked if encrypted_files: index = 0 for file in metadata['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata['encryptedFiles'] = encrypted_files keeper_instance().did_registry.register( ddo.asset_id, checksum=web3().toBytes(hexstr=ddo.asset_id), url=ddo_service_endpoint, account=account, providers=providers) try: aqua.publish_asset_ddo(ddo) except Exception as e: print(f'error publishing ddo {ddo.did} in Aquarius: {e}') raise return ddo