def _add_defaults( self, services: list, metadata: dict, provider_uri: str, wallet: Wallet ) -> list: ddo_service_endpoint = self._get_aquarius().get_service_endpoint() metadata_service = Service( service_endpoint=ddo_service_endpoint, service_type=ServiceTypes.METADATA, attributes=metadata, ) services.append(metadata_service) has_access_service = False for service in services: if service.type == ServiceTypes.ASSET_ACCESS: has_access_service = True if not has_access_service: access_service = self.build_access_service( self._data_provider.build_download_endpoint(provider_uri)[1], metadata["main"]["dateCreated"], 1.0, wallet.address, ) services.append(access_service) return services
def get_registered_ddo_with_compute_service( ocean_instance, wallet, provider_uri=None, trusted_algorithms=None, trusted_algorithm_publishers=None, ): old_ddo = get_sample_ddo_with_compute_service() metadata = old_ddo.metadata metadata["main"]["files"][0]["checksum"] = str(uuid.uuid4()) service = old_ddo.get_service(ServiceTypes.CLOUD_COMPUTE) compute_attributes = ocean_instance.compute.create_compute_service_attributes( service.attributes["main"]["timeout"], service.attributes["main"]["creator"], service.attributes["main"]["datePublished"], service.attributes["main"]["provider"], privacy_attributes=ocean_instance.compute.build_service_privacy_attributes( trusted_algorithms=trusted_algorithms, trusted_algorithm_publishers=trusted_algorithm_publishers, metadata_cache_uri=ocean_instance.config.metadata_cache_uri, allow_raw_algorithm=True, allow_all_published_algorithms=not bool(trusted_algorithms), ), ) compute_service = Service( service_endpoint=DataServiceProvider.get_url(ocean_instance.config), service_type=ServiceTypes.CLOUD_COMPUTE, attributes=compute_attributes, ) return get_registered_ddo( ocean_instance, metadata, wallet, compute_service, provider_uri=provider_uri )
def add_service( self, service_type: Union[str, Service], service_endpoint: Optional[str] = None, values: Optional[dict] = None, index: Optional[int] = None, ) -> None: """ Add a service to the list of services on the DDO. :param service_type: Service :param service_endpoint: Service endpoint, str :param values: Python dict with index, templateId, serviceAgreementContract, list of conditions and purchase endpoint. """ if isinstance(service_type, Service): service = service_type else: values = copy.deepcopy(values) if values else {} service = Service( service_endpoint, service_type, values.pop("attributes", None), values, index, ) logger.debug( f"Adding service with service type {service_type} with did {self.did}" ) self.services.append(service)
def test_order(web3, alice_ocean, alice_wallet): asset = get_registered_ddo(alice_ocean, get_metadata(), alice_wallet) dt = DataToken(web3, asset.data_token_address) service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS) sa = Service.from_json(service.as_dictionary()) order_requirements = alice_ocean.assets.order( asset.did, alice_wallet.address, sa.index ) assert order_requirements, "Order was unsuccessful." _order_tx_id = alice_ocean.assets.pay_for_service( web3, order_requirements.amount, order_requirements.data_token_address, asset.did, service.index, alice_wallet.address, alice_wallet, sa.get_c2d_address(), ) asset_folder = alice_ocean.assets.download( asset.did, sa.index, alice_wallet, _order_tx_id, alice_ocean.config.downloads_path, ) assert len(os.listdir(asset_folder)) >= 1, "The asset folder is empty." for order_log in dt.get_start_order_logs(): order_log_dict = dict(order_log.args.items()) order_log_dict["amount"] = int(order_log.args.amount) order_log_dict["marketFee"] = int(order_log.args.marketFee) order_args = [ order_log.address, order_log_dict["amount"], order_log_dict["timestamp"], order_log.transactionHash, f"did:op:{remove_0x_prefix(order_log.address)}", order_log_dict["payer"], order_log_dict["consumer"], order_log_dict["serviceId"], None, ] order = Order(*order_args) assert order, "The order does not exist." assert isinstance(order, tuple), "Order is not a tuple." assert ( order[0] == asset.data_token_address ), "The order data token address is different." assert order[5] == alice_wallet.address, "The payer is not the supposed one." assert order[6] == sa.get_c2d_address(), "The consumer is not the supposed one." assert len(order) == 9, "Different number of args."
def test_service(): """Tests that the get_cost function for ServiceAgreement returns the correct value.""" ddo = get_sample_ddo() sa = ddo.get_service(ServiceTypes.ASSET_ACCESS) assert sa.get_cost() == 1.0 assert sa.get_c2d_address() assert sa.main["name"] == "dataAssetAccessServiceAgreement" assert "attributes" in sa.as_dictionary() converted = Service.from_json(sa.as_dictionary()) assert converted.attributes == sa.attributes assert converted.service_endpoint == sa.service_endpoint assert converted.type == sa.type assert converted.index == sa.index
def _read_dict(self, dictionary: dict) -> None: """Import a JSON dict into this Asset.""" values = copy.deepcopy(dictionary) id_key = "id" if "id" in values else "_id" self.did = values.pop(id_key) self.created = values.pop("created", None) if "service" in values: self.services = [] for value in values.pop("service"): if isinstance(value, str): value = json.loads(value) service = Service.from_json(value) self.services.append(service) if "proof" in values: self.proof = values.pop("proof") if "credentials" in values: self.credentials = values.pop("credentials") self.other_values = values
def build_access_service( endpoint: str, date_created: str, cost: float, address: str, timeout: Optional[int] = 3600, ) -> dict: attributes = { "main": { "name": "dataAssetAccessServiceAgreement", "creator": address, "cost": cost, "timeout": timeout, "datePublished": date_created, } } return Service( service_endpoint=endpoint, service_type=ServiceTypes.ASSET_ACCESS, attributes=attributes, )
def publish_data( ocean, private_key, files, name, symbol, author, data_license='CC0: Public Domain', ): """Publish a dataset on the Ocean marketplace. Publish metadata and service attributes on-chain. The service urls will be encrypted before going on-chain. They're only decrypted for datatoken owners upon consume. Args: ocean (): private_key (str): files (list): name (str): symbol (str): author (str): data_license (str): The license for the data, `CC0: Public Domain` by default. Returns: () """ wallet = Wallet(ocean.web3, private_key, ocean.config.block_confirmations) assert wallet.web3.eth.get_balance(wallet.address) > 0, 'need ETH' print('Proceeding with wallet:', wallet.address) data_token = ocean.create_data_token(name, symbol, wallet, blob=ocean.config.metadata_cache_uri) # return data_token token_address = data_token.address print('Created token:', token_address) date_created = datetime.now().isoformat() metadata = { 'main': { 'type': 'dataset', 'name': name, 'author': author, 'license': data_license, 'dateCreated': date_created, 'files': files, } } service_attributes = { 'main': { 'name': 'dataAssetAccessServiceAgreement', 'creator': wallet.address, 'timeout': 3600 * 24, 'datePublished': date_created, 'cost': 1.0, # <don't change, this is obsolete> } } service_endpoint = DataServiceProvider.get_url(ocean.config) # FIXME: download_service = Service( service_endpoint=service_endpoint, service_type=ServiceTypes.ASSET_ACCESS, attributes=service_attributes, ) assert wallet.web3.eth.get_balance(wallet.address) > 0, 'need ETH' asset = ocean.assets.create( metadata, wallet, # services=[download_service], # service_descriptors=[], data_token_address=token_address) print('Created asset:', asset.data_token_address) assert token_address == asset.data_token_address return data_token, asset
def start( self, input_datasets: list, consumer_wallet: Wallet, nonce: Optional[int] = None, algorithm_did: Optional[str] = None, algorithm_meta: Optional[AlgorithmMetadata] = None, algorithm_tx_id: Optional[str] = None, algorithm_data_token: Optional[str] = None, output: Optional[dict] = None, job_id: Optional[str] = None, algouserdata: Optional[dict] = None, ) -> str: """ Start a remote compute job on the asset files. Files are identified by `did` after verifying that the provider service is active and transferring the number of data-tokens required for using this compute service. :param input_datasets: list of ComputeInput -- list of input datasets to the compute job. A dataset is represented with ComputeInput struct :param consumer_wallet: Wallet instance of the consumer ordering the service :param nonce: int value to use in the signature :param algorithm_did: str -- the asset did (of `algorithm` type) which consist of `did:op:` and the assetId hex str (without `0x` prefix) :param algorithm_meta: `AlgorithmMetadata` instance -- metadata about the algorithm being run if `algorithm` is being used. This is ignored when `algorithm_did` is specified. :param algorithm_tx_id: transaction hash of algorithm StartOrder tx (Required when using `algorithm_did`) :param algorithm_data_token: datatoken address of this algorithm (Required when using `algorithm_did`) :param output: dict object to be used in publishing mechanism, must define :param job_id: str identifier of a compute job that was previously started and stopped (if supported by the provider's backend) :return: str -- id of compute job being executed """ assert ( algorithm_did or algorithm_meta ), "either an algorithm did or an algorithm meta must be provided." for i in input_datasets: assert isinstance(i, ComputeInput) first_input = input_datasets[0] did = first_input.did order_tx_id = first_input.transfer_tx_id service_id = first_input.service_id output = OceanCompute.check_output_dict(output, consumer_wallet.address, self._data_provider, self._config) asset = resolve_asset( did, metadata_cache_uri=self._config.metadata_cache_uri) _, service_endpoint = self._get_service_endpoint(did, asset) service = asset.get_service_by_index(service_id) sa = Service.from_json(service.as_dictionary()) assert (ServiceTypes.CLOUD_COMPUTE == sa.type ), "service at serviceId is not of type compute service." consumable_result = asset.is_consumable( { "type": "address", "value": consumer_wallet.address }, provider_uri=sa.service_endpoint, ) if consumable_result != ConsumableCodes.OK: raise AssetNotConsumable(consumable_result) signature = self._sign_message( consumer_wallet, f"{consumer_wallet.address}{did}", nonce=nonce, service_endpoint=sa.service_endpoint, ) try: job_info = self._data_provider.start_compute_job( did, service_endpoint, consumer_wallet.address, signature, sa.index, order_tx_id, algorithm_did, algorithm_meta, algorithm_tx_id, algorithm_data_token, output, input_datasets, job_id, userdata=first_input.userdata, algouserdata=algouserdata, ) return job_info["jobId"] except ValueError: raise
def test_market_flow(): """Tests that an order is correctly placed on the market. The parameter implicit_none sends the payload with an empty key as the delegated consumer. The parameter explicit_none sends None as the delegated consumer, explicitly.""" pub_wallet = get_publisher_wallet() publisher_ocean = get_publisher_ocean_instance() consumer_ocean = get_consumer_ocean_instance() # Register asset asset = get_registered_ddo(publisher_ocean, get_metadata(), pub_wallet) assert isinstance(asset, V3Asset) assert asset.data_token_address, "The asset does not have a token address." consumer_wallet = get_consumer_wallet() service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS) sa = Service.from_json(service.as_dictionary()) # Mint data tokens and assign to publisher dt = publisher_ocean.get_data_token(asset.data_token_address) mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet) ###### # Give the consumer some datatokens so they can order the service try: tx_id = dt.transfer(consumer_wallet.address, to_wei(10), pub_wallet) dt.verify_transfer_tx(tx_id, pub_wallet.address, consumer_wallet.address) except (AssertionError, Exception) as e: print(e) raise ###### # Place order for the download service order_requirements = consumer_ocean.assets.order(asset.did, consumer_wallet.address, sa.index) ###### # Pay for the service args = [ consumer_ocean.web3, order_requirements.amount, order_requirements.data_token_address, asset.did, service.index, "0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9", consumer_wallet, sa.get_c2d_address(), ] _order_tx_id = consumer_ocean.assets.pay_for_service(*args) ###### # Download the asset files asset_folder = consumer_ocean.assets.download( asset.did, sa.index, consumer_wallet, _order_tx_id, consumer_ocean.config.downloads_path, ) assert len(os.listdir(asset_folder)) >= 1, "The asset folder is empty." orders = consumer_ocean.get_user_orders(consumer_wallet.address, asset.asset_id) assert ( orders ), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}" orders = consumer_ocean.get_user_orders( consumer_wallet.address, consumer_ocean.web3.toChecksumAddress(asset.asset_id)) assert ( orders ), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}" orders = consumer_ocean.get_user_orders(consumer_wallet.address) assert ( orders ), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}" ###### # Publisher can get the urls of the asset asset_urls = DataServiceProvider.get_asset_urls(asset.did, str(service.index), "http://172.15.0.4:8030", pub_wallet) assert len(asset_urls) == 3 for url in asset_urls: assert "10_Monkey_Species_Small" in url
def test_payer_market_flow(): """Tests that an order can be placed for a delegated consumer, other than the payer.""" pub_wallet = get_publisher_wallet() publisher_ocean = get_publisher_ocean_instance() consumer_ocean = get_consumer_ocean_instance() another_consumer_ocean = get_another_consumer_ocean_instance( use_provider_mock=True) # Register Asset asset = get_registered_ddo(publisher_ocean, get_metadata(), pub_wallet) assert isinstance(asset, V3Asset) assert asset.data_token_address, "The asset does not have a token address." another_consumer_wallet = get_another_consumer_wallet() consumer_wallet = get_consumer_wallet() service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS) sa = Service.from_json(service.as_dictionary()) # Mint data tokens and assign to publisher dt = publisher_ocean.get_data_token(asset.data_token_address) mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet) ###### # Give the consumer some datatokens so they can order the service try: tx_id = dt.transfer(consumer_wallet.address, to_wei(10), pub_wallet) dt.verify_transfer_tx(tx_id, pub_wallet.address, consumer_wallet.address) except (AssertionError, Exception) as e: print(e) raise ###### # Place order for the download service order_requirements = consumer_ocean.assets.order( asset.did, another_consumer_wallet.address, sa.index) ###### # Pay for the service and have another_consumer_wallet as consumer _order_tx_id = consumer_ocean.assets.pay_for_service( consumer_ocean.web3, order_requirements.amount, order_requirements.data_token_address, asset.did, service.index, "0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9", consumer_wallet, another_consumer_wallet.address, ) asset_folder = None assert asset_folder is None if asset_folder is None: # Download the asset files asset_folder = another_consumer_ocean.assets.download( asset.did, sa.index, another_consumer_wallet, _order_tx_id, another_consumer_ocean.config.downloads_path, ) assert len(os.listdir(asset_folder)) >= 1 orders = consumer_ocean.get_user_orders(consumer_wallet.address, asset.asset_id) assert ( orders ), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}" orders = consumer_ocean.get_user_orders( consumer_wallet.address, consumer_ocean.web3.toChecksumAddress(asset.asset_id)) assert ( orders ), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}" orders = consumer_ocean.get_user_orders(consumer_wallet.address) assert ( orders ), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"