def generate_token(account): raw_msg = get_config( ).auth_token_message or "Ocean Protocol Authentication" _time = int(datetime.now().timestamp()) _message = f'{raw_msg}\n{_time}' prefixed_msg_hash = add_ethereum_prefix_and_hash_msg(_message) return f'{keeper_instance().sign_hash(prefixed_msg_hash, account)}-{_time}'
def sign(self, agreement_id, did, consumer_account, service_index): """ Sign a service agreement. :param agreement_id: 32 bytes identifier created by the consumer and will be used on-chain for the executed agreement. :param did: str representation fo the asset DID. Use this to retrieve the asset DDO. :param consumer_account: Account instance of the consumer :param service_index: int identifies the specific service in the ddo to use in this agreement. :return: signature """ asset = self._asset_resolver.resolve(did) service_agreement = asset.get_service_by_index(service_index) publisher_address = self._keeper.did_registry.get_did_owner( asset.asset_id) agreement_hash = service_agreement.get_service_agreement_hash( agreement_id, asset.asset_id, consumer_account.address, publisher_address, self._keeper) signature = self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(agreement_hash), consumer_account) address = self._keeper.personal_ec_recover(agreement_hash, signature) assert address == consumer_account.address logger.debug( f'agreement-signature={signature}, agreement-hash={agreement_hash}' ) return signature
def get(self, account): """ :param account: Account instance signing the token :return: hex str the token generated/signed by account """ _message, _time = self._get_message_and_time() try: prefixed_msg_hash = self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(_message), account) return f'{prefixed_msg_hash}-{_time}' except Exception as e: logging.error(f'Error signing token: {str(e)}')
def consume_service(service_agreement_id, service_endpoint, account, files, destination_folder, index=None): """ Call the brizo endpoint to get access to the different files that form the asset. :param service_agreement_id: Service Agreement Id, str :param service_endpoint: Url to consume, str :param account: Account instance of the consumer signing this agreement, hex-str :param files: List containing the files to be consumed, list :param index: Index of the document that is going to be downloaded, int :param destination_folder: Path, str :return: True if was downloaded, bool """ signature = Keeper.get_instance().sign_hash( add_ethereum_prefix_and_hash_msg(service_agreement_id), account) if index is not None: assert isinstance(index, int), logger.error('index has to be an integer.') assert index >= 0, logger.error( 'index has to be 0 or a positive integer.') assert index < len(files), logger.error( 'index can not be bigger than the number of files') consume_url = Brizo._create_consume_url(service_endpoint, service_agreement_id, account, None, signature, index) logger.info( f'invoke consume endpoint with this url: {consume_url}') response = Brizo._http_client.get(consume_url, stream=True) file_name = Brizo._get_file_name(response) Brizo.write_file(response, destination_folder, file_name) else: for i, _file in enumerate(files): consume_url = Brizo._create_consume_url( service_endpoint, service_agreement_id, account, _file, signature, i) logger.info( f'invoke consume endpoint with this url: {consume_url}') response = Brizo._http_client.get(consume_url, stream=True) file_name = Brizo._get_file_name(response) Brizo.write_file(response, destination_folder, file_name or f'file-{i}')
def test_publish(client): endpoint = BaseURLs.ASSETS_URL + '/publish' did = DID.did({"0": str(uuid.uuid4())}) asset_id = did_to_id(did) account = get_provider_account() test_urls = ['url 00', 'url 11', 'url 22'] keeper = keeper_instance() urls_json = json.dumps(test_urls) asset_id_hash = add_ethereum_prefix_and_hash_msg(asset_id) signature = keeper.sign_hash(asset_id_hash, account) address = web3().eth.account.recoverHash(asset_id_hash, signature=signature) assert address.lower() == account.address.lower() address = keeper.personal_ec_recover(asset_id, signature) assert address.lower() == account.address.lower() payload = { 'documentId': asset_id, 'signature': signature, 'document': urls_json, 'publisherAddress': account.address } post_response = client.post(endpoint, data=json.dumps(payload), content_type='application/json') encrypted_url = post_response.data.decode('utf-8') assert encrypted_url.startswith('0x') # publish using auth token signature = generate_token(account) payload['signature'] = signature did = DID.did({"0": str(uuid.uuid4())}) asset_id = did_to_id(did) payload['documentId'] = add_0x_prefix(asset_id) post_response = client.post(endpoint, data=json.dumps(payload), content_type='application/json') encrypted_url = post_response.data.decode('utf-8') assert encrypted_url.startswith('0x')
def execute_service(service_agreement_id, service_endpoint, account, workflow_ddo): """ :param service_agreement_id: :param service_endpoint: :param account: :return: """ signature = Keeper.get_instance().sign_hash( add_ethereum_prefix_and_hash_msg(service_agreement_id), account) execute_url = Brizo._create_execute_url(service_endpoint, service_agreement_id, account, workflow_ddo.did, signature) logger.info(f'invoke execute endpoint with this url: {execute_url}') response = Brizo._http_client.post(execute_url) print( f'got brizo execute response: {response.content} with status-code {response.status_code} ' ) if response.status_code != 201: raise Exception(response.content.decode('utf-8')) return json.loads(response.content)['workflowId']
def _check_job_id(client, job_id, agreement_id, wait_time=20): endpoint = BaseURLs.ASSETS_URL + '/compute' cons_acc = get_consumer_account() keeper = keeper_instance() msg = f'{cons_acc.address}{job_id}{agreement_id}' agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg) signature = keeper.sign_hash(agreement_id_hash, cons_acc) payload = dict({ 'signature': signature, 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address, 'jobId': job_id, }) job_info = get_compute_job_info(client, endpoint, payload) assert job_info, f'Failed to get job info for jobId {job_id}' print(f'got info for compute job {job_id}: {job_info}') assert job_info['statusText'] in get_possible_compute_job_status_text() did = None # get did of results for _ in range(wait_time * 4): job_info = get_compute_job_info(client, endpoint, payload) did = job_info['did'] if did: break time.sleep(0.25) assert did, f'Compute job has no results, job info {job_info}.' # check results ddo ddo = DIDResolver(keeper.did_registry).resolve(did) assert ddo, f'Failed to resolve ddo for did {did}' consumer_permission = keeper.did_registry.get_permission( did, cons_acc.address) assert consumer_permission is True, \ f'Consumer address {cons_acc.address} has no permissions on the results ' \ f'did {did}. This is required, the consumer must be able to access the results'
def create(self, metadata, publisher_account, service_descriptors=None, providers=None, use_secret_store=True): """ Register an asset in both the keeper's DIDRegistry (on-chain) and in the Metadata store ( Aquarius). :param metadata: dict conforming to the Metadata accepted by Ocean Protocol. :param publisher_account: Account of the publisher registering this asset :param service_descriptors: list of ServiceDescriptor tuples of length 2. The first item must be one of ServiceTypes and the second item is a dict of parameters and values required by the service :param providers: list of addresses of providers of this asset (a provider is an ethereum account that is authorized to provide asset services) :param use_secret_store: bool indicate whether to use the secret store directly for encrypting urls (Uses Brizo provider service if set to False) :return: DDO instance """ assert isinstance( metadata, dict), f'Expected metadata of type dict, got {type(metadata)}' assert service_descriptors is None or isinstance(service_descriptors, list), \ f'bad type of `service_descriptors` {type(service_descriptors)}' # if not metadata or not Metadata.validate(metadata): # raise OceanInvalidMetadata('Metadata seems invalid. Please make sure' # ' the required metadata values are filled in.') # copy metadata so we don't change the original metadata_copy = copy.deepcopy(metadata) asset_type = metadata_copy['main']['type'] assert asset_type in ( 'dataset', 'algorithm'), f'Invalid/unsupported asset type {asset_type}' service_descriptors = service_descriptors or [] brizo = BrizoProvider.get_brizo() services = self._process_service_descriptors(service_descriptors, metadata_copy, publisher_account) stype_to_service = {s.type: s for s in services} checksum_dict = dict() for service in services: checksum_dict[str(service.index)] = checksum(service.main) # Create a DDO object ddo = DDO() # Adding proof to the ddo. ddo.add_proof(checksum_dict, publisher_account) # Generating the did and adding to the ddo. did = ddo.assign_did(DID.did(ddo.proof['checksum'])) logger.debug(f'Generating new did: {did}') # Check if it's already registered first! if did in self._get_aquarius().list_assets(): raise OceanDIDAlreadyExist( f'Asset id {did} is already registered to another asset.') md_service = stype_to_service[ServiceTypes.METADATA] ddo_service_endpoint = md_service.service_endpoint if '{did}' in ddo_service_endpoint: ddo_service_endpoint = ddo_service_endpoint.replace('{did}', did) md_service.set_service_endpoint(ddo_service_endpoint) # Populate the ddo services ddo.add_service(md_service) ddo.add_service(stype_to_service[ServiceTypes.AUTHORIZATION]) access_service = stype_to_service.get(ServiceTypes.ASSET_ACCESS, None) compute_service = stype_to_service.get(ServiceTypes.CLOUD_COMPUTE, None) if access_service: access_service.init_conditions_values( did, { cname: c.address for cname, c in self._keeper.contract_name_to_instance.items() }) ddo.add_service(access_service) if compute_service: compute_service.init_conditions_values( did, { cname: c.address for cname, c in self._keeper.contract_name_to_instance.items() }) ddo.add_service(compute_service) ddo.proof['signatureValue'] = self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(did_to_id_bytes(did)), publisher_account) # Add public key and authentication ddo.add_public_key(did, publisher_account.address) ddo.add_authentication(did, PUBLIC_KEY_TYPE_RSA) # Setup metadata service # First compute files_encrypted if metadata_copy['main']['type'] == 'dataset': assert metadata_copy['main']['files'], \ 'files is required in the metadata main attributes.' logger.debug('Encrypting content urls in the metadata.') if not use_secret_store: encrypt_endpoint = brizo.get_encrypt_endpoint(self._config) files_encrypted = brizo.encrypt_files_dict( metadata_copy['main']['files'], encrypt_endpoint, ddo.asset_id, publisher_account.address, self._keeper.sign_hash( add_ethereum_prefix_and_hash_msg(ddo.asset_id), publisher_account)) else: files_encrypted = self._get_secret_store(publisher_account) \ .encrypt_document( did_to_id(did), json.dumps(metadata_copy['main']['files']), ) # only assign if the encryption worked if files_encrypted: logger.debug( f'Content urls encrypted successfully {files_encrypted}') index = 0 for file in metadata_copy['main']['files']: file['index'] = index index = index + 1 del file['url'] metadata_copy['encryptedFiles'] = files_encrypted else: raise AssertionError('Encrypting the files failed.') # DDO url and `Metadata` service logger.debug(f'Generated ddo and services, DID is {ddo.did},' f' metadata service @{ddo_service_endpoint}.') response = None # register on-chain registered_on_chain = self._keeper.did_registry.register( ddo.asset_id, checksum=Web3Provider.get_web3().toBytes(hexstr=ddo.asset_id), url=ddo_service_endpoint, account=publisher_account, providers=providers) if registered_on_chain is False: logger.warning(f'Registering {did} on-chain failed.') return None logger.info(f'Successfully registered DDO (DID={did}) on chain.') try: # publish the new ddo in ocean-db/Aquarius response = self._get_aquarius().publish_asset_ddo(ddo) logger.info('Asset/ddo published successfully in aquarius.') except ValueError as ve: raise ValueError( f'Invalid value to publish in the metadata: {str(ve)}') except Exception as e: logger.error(f'Publish asset in aquarius failed: {str(e)}') if not response: return None return ddo
def test_consume(client): aqua = Aquarius('http://localhost:5000') for did in aqua.list_assets(): aqua.retire_asset_ddo(did) endpoint = BaseURLs.ASSETS_URL + '/consume' pub_acc = get_publisher_account() cons_acc = get_consumer_account() keeper = keeper_instance() ddo = get_dataset_ddo_with_access_service(pub_acc, providers=[pub_acc.address]) # initialize an agreement agreement_id = place_order(pub_acc, ddo, cons_acc, ServiceTypes.ASSET_ACCESS) payload = dict({ 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address }) agr_id_hash = add_ethereum_prefix_and_hash_msg(agreement_id) signature = keeper.sign_hash(agr_id_hash, cons_acc) index = 0 event = keeper.agreement_manager.subscribe_agreement_created(agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Agreement event is not found, check the keeper node's logs" consumer_balance = keeper.token.get_token_balance(cons_acc.address) if consumer_balance < 50: keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc) sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo) lock_reward(agreement_id, sa, cons_acc) event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs" grant_access(agreement_id, ddo, cons_acc, pub_acc) event = keeper.access_secret_store_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event or keeper.access_secret_store_condition.check_permissions( ddo.asset_id, cons_acc.address ), f'Failed to get access permission: agreement_id={agreement_id}, ' \ f'did={ddo.did}, consumer={cons_acc.address}' # Consume using decrypted url files_list = json.loads( do_secret_store_decrypt(did_to_id(ddo.did), ddo.encrypted_files, pub_acc, get_config())) payload['url'] = files_list[index]['url'] request_url = endpoint + '?' + '&'.join( [f'{k}={v}' for k, v in payload.items()]) response = client.get(request_url) assert response.status == '200 OK' # Consume using url index and signature (let brizo do the decryption) payload.pop('url') payload['signature'] = signature payload['index'] = index request_url = endpoint + '?' + '&'.join( [f'{k}={v}' for k, v in payload.items()]) response = client.get(request_url) assert response.status == '200 OK'
def test_compute_norawalgo_allowed(client): aqua = Aquarius('http://localhost:5000') for did in aqua.list_assets(): aqua.retire_asset_ddo(did) pub_acc = get_publisher_account() cons_acc = get_consumer_account() keeper = keeper_instance() # publish a dataset asset dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service_no_rawalgo( pub_acc, providers=[pub_acc.address]) # CHECKPOINT 1 algorithmMeta = { "rawcode": "console.log('Hello world'!)", "format": 'docker-image', "version": '0.1', "container": { "entrypoint": 'node $ALGO', "image": 'node', "tag": '10' } } # prepare parameter values for the compute endpoint # signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta # initialize an agreement agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service, cons_acc, ServiceTypes.CLOUD_COMPUTE) # CHECKPOINT 2 event = keeper.agreement_manager.subscribe_agreement_created(agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Agreement event is not found, check the keeper node's logs" consumer_balance = keeper.token.get_token_balance(cons_acc.address) if consumer_balance < 50: keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc) sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE, dataset_ddo_w_compute_service) lock_reward(agreement_id, sa, cons_acc) event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs" grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id, cons_acc, pub_acc) event = keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event or keeper.compute_execution_condition.was_compute_triggered( dataset_ddo_w_compute_service.asset_id, cons_acc.address ), (f'Failed to compute: agreement_id={agreement_id}, ' f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}' ) # prepare consumer signature on agreement_id msg = f'{cons_acc.address}{agreement_id}' agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg) signature = keeper.sign_hash(agreement_id_hash, cons_acc) # Start the compute job payload = dict({ 'signature': signature, 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address, 'algorithmDid': None, 'algorithmMeta': algorithmMeta, 'output': build_stage_output_dict(dict(), dataset_ddo_w_compute_service, cons_acc.address, pub_acc) }) endpoint = BaseURLs.ASSETS_URL + '/compute' response = client.post(endpoint, data=json.dumps(payload), content_type='application/json') assert response.status == '400 BAD REQUEST', f'start compute job failed: {response.status} , { response.data}'
def test_compute(client): aqua = Aquarius('http://localhost:5000') for did in aqua.list_assets(): aqua.retire_asset_ddo(did) pub_acc = get_publisher_account() cons_acc = get_consumer_account() keeper = keeper_instance() # publish a dataset asset dataset_ddo_w_compute_service = get_dataset_ddo_with_compute_service( pub_acc, providers=[pub_acc.address]) # publish an algorithm asset (asset with metadata of type `algorithm`) alg_ddo = get_algorithm_ddo(cons_acc, providers=[pub_acc.address]) # CHECKPOINT 1 # prepare parameter values for the compute endpoint # signature, serviceAgreementId, consumerAddress, and algorithmDid or algorithmMeta # initialize an agreement agreement_id = place_order(pub_acc, dataset_ddo_w_compute_service, cons_acc, ServiceTypes.CLOUD_COMPUTE) # CHECKPOINT 2 event = keeper.agreement_manager.subscribe_agreement_created(agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Agreement event is not found, check the keeper node's logs" consumer_balance = keeper.token.get_token_balance(cons_acc.address) if consumer_balance < 50: keeper.dispenser.request_tokens(50 - consumer_balance, cons_acc) sa = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE, dataset_ddo_w_compute_service) lock_reward(agreement_id, sa, cons_acc) event = keeper.lock_reward_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event, "Lock reward condition fulfilled event is not found, check the keeper node's logs" grant_compute(agreement_id, dataset_ddo_w_compute_service.asset_id, cons_acc, pub_acc) event = keeper.compute_execution_condition.subscribe_condition_fulfilled( agreement_id, 15, None, (), wait=True, from_block=0) assert event or keeper.compute_execution_condition.was_compute_triggered( dataset_ddo_w_compute_service.asset_id, cons_acc.address ), (f'Failed to compute: agreement_id={agreement_id}, ' f'did={dataset_ddo_w_compute_service.did}, consumer={cons_acc.address}' ) # prepare consumer signature on agreement_id msg = f'{cons_acc.address}{agreement_id}' agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg) signature = keeper.sign_hash(agreement_id_hash, cons_acc) # Start the compute job payload = dict({ 'signature': signature, 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address, 'algorithmDid': alg_ddo.did, 'algorithmMeta': {}, 'output': build_stage_output_dict(dict(), dataset_ddo_w_compute_service, cons_acc.address, pub_acc) }) endpoint = BaseURLs.ASSETS_URL + '/compute' response = client.post(endpoint, data=json.dumps(payload), content_type='application/json') assert response.status == '200 OK', f'start compute job failed: {response.data}' job_info = response.json[0] print(f'got response from starting compute job: {job_info}') job_id = job_info.get('jobId', '') msg = f'{cons_acc.address}{job_id}{agreement_id}' agreement_id_hash = add_ethereum_prefix_and_hash_msg(msg) signature = keeper.sign_hash(agreement_id_hash, cons_acc) payload = dict({ 'signature': signature, 'serviceAgreementId': agreement_id, 'consumerAddress': cons_acc.address, 'jobId': job_id, }) job_info = get_compute_job_info(client, endpoint, payload) assert job_info, f'Failed to get job info for jobId {job_id}' print(f'got info for compute job {job_id}: {job_info}') assert job_info['statusText'] in get_possible_compute_job_status_text()