def _add_defaults(
self, services: list, metadata: dict, provider_uri: str, wallet: Wallet
) -> list:
ddo_service_endpoint = self._get_aquarius().get_service_endpoint()
metadata_service = Service(
service_endpoint=ddo_service_endpoint,
service_type=ServiceTypes.METADATA,
attributes=metadata,
)
services.append(metadata_service)
has_access_service = False
for service in services:
if service.type == ServiceTypes.ASSET_ACCESS:
has_access_service = True
if not has_access_service:
access_service = self.build_access_service(
self._data_provider.build_download_endpoint(provider_uri)[1],
metadata["main"]["dateCreated"],
1.0,
wallet.address,
)
services.append(access_service)
return services
def get_registered_ddo_with_compute_service(
ocean_instance,
wallet,
provider_uri=None,
trusted_algorithms=None,
trusted_algorithm_publishers=None,
):
old_ddo = get_sample_ddo_with_compute_service()
metadata = old_ddo.metadata
metadata["main"]["files"][0]["checksum"] = str(uuid.uuid4())
service = old_ddo.get_service(ServiceTypes.CLOUD_COMPUTE)
compute_attributes = ocean_instance.compute.create_compute_service_attributes(
service.attributes["main"]["timeout"],
service.attributes["main"]["creator"],
service.attributes["main"]["datePublished"],
service.attributes["main"]["provider"],
privacy_attributes=ocean_instance.compute.build_service_privacy_attributes(
trusted_algorithms=trusted_algorithms,
trusted_algorithm_publishers=trusted_algorithm_publishers,
metadata_cache_uri=ocean_instance.config.metadata_cache_uri,
allow_raw_algorithm=True,
allow_all_published_algorithms=not bool(trusted_algorithms),
),
)
compute_service = Service(
service_endpoint=DataServiceProvider.get_url(ocean_instance.config),
service_type=ServiceTypes.CLOUD_COMPUTE,
attributes=compute_attributes,
)
return get_registered_ddo(
ocean_instance, metadata, wallet, compute_service, provider_uri=provider_uri
)
def add_service(
self,
service_type: Union[str, Service],
service_endpoint: Optional[str] = None,
values: Optional[dict] = None,
index: Optional[int] = None,
) -> None:
"""
Add a service to the list of services on the DDO.
:param service_type: Service
:param service_endpoint: Service endpoint, str
:param values: Python dict with index, templateId, serviceAgreementContract,
list of conditions and purchase endpoint.
"""
if isinstance(service_type, Service):
service = service_type
else:
values = copy.deepcopy(values) if values else {}
service = Service(
service_endpoint,
service_type,
values.pop("attributes", None),
values,
index,
)
logger.debug(
f"Adding service with service type {service_type} with did {self.did}"
)
self.services.append(service)
def test_order(web3, alice_ocean, alice_wallet):
asset = get_registered_ddo(alice_ocean, get_metadata(), alice_wallet)
dt = DataToken(web3, asset.data_token_address)
service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS)
sa = Service.from_json(service.as_dictionary())
order_requirements = alice_ocean.assets.order(
asset.did, alice_wallet.address, sa.index
)
assert order_requirements, "Order was unsuccessful."
_order_tx_id = alice_ocean.assets.pay_for_service(
web3,
order_requirements.amount,
order_requirements.data_token_address,
asset.did,
service.index,
alice_wallet.address,
alice_wallet,
sa.get_c2d_address(),
)
asset_folder = alice_ocean.assets.download(
asset.did,
sa.index,
alice_wallet,
_order_tx_id,
alice_ocean.config.downloads_path,
)
assert len(os.listdir(asset_folder)) >= 1, "The asset folder is empty."
for order_log in dt.get_start_order_logs():
order_log_dict = dict(order_log.args.items())
order_log_dict["amount"] = int(order_log.args.amount)
order_log_dict["marketFee"] = int(order_log.args.marketFee)
order_args = [
order_log.address,
order_log_dict["amount"],
order_log_dict["timestamp"],
order_log.transactionHash,
f"did:op:{remove_0x_prefix(order_log.address)}",
order_log_dict["payer"],
order_log_dict["consumer"],
order_log_dict["serviceId"],
None,
]
order = Order(*order_args)
assert order, "The order does not exist."
assert isinstance(order, tuple), "Order is not a tuple."
assert (
order[0] == asset.data_token_address
), "The order data token address is different."
assert order[5] == alice_wallet.address, "The payer is not the supposed one."
assert order[6] == sa.get_c2d_address(), "The consumer is not the supposed one."
assert len(order) == 9, "Different number of args."
def test_service():
"""Tests that the get_cost function for ServiceAgreement returns the correct value."""
ddo = get_sample_ddo()
sa = ddo.get_service(ServiceTypes.ASSET_ACCESS)
assert sa.get_cost() == 1.0
assert sa.get_c2d_address()
assert sa.main["name"] == "dataAssetAccessServiceAgreement"
assert "attributes" in sa.as_dictionary()
converted = Service.from_json(sa.as_dictionary())
assert converted.attributes == sa.attributes
assert converted.service_endpoint == sa.service_endpoint
assert converted.type == sa.type
assert converted.index == sa.index
def _read_dict(self, dictionary: dict) -> None:
"""Import a JSON dict into this Asset."""
values = copy.deepcopy(dictionary)
id_key = "id" if "id" in values else "_id"
self.did = values.pop(id_key)
self.created = values.pop("created", None)
if "service" in values:
self.services = []
for value in values.pop("service"):
if isinstance(value, str):
value = json.loads(value)
service = Service.from_json(value)
self.services.append(service)
if "proof" in values:
self.proof = values.pop("proof")
if "credentials" in values:
self.credentials = values.pop("credentials")
self.other_values = values
def build_access_service(
endpoint: str,
date_created: str,
cost: float,
address: str,
timeout: Optional[int] = 3600,
) -> dict:
attributes = {
"main": {
"name": "dataAssetAccessServiceAgreement",
"creator": address,
"cost": cost,
"timeout": timeout,
"datePublished": date_created,
}
}
return Service(
service_endpoint=endpoint,
service_type=ServiceTypes.ASSET_ACCESS,
attributes=attributes,
)
def publish_data(
ocean,
private_key,
files,
name,
symbol,
author,
data_license='CC0: Public Domain',
):
"""Publish a dataset on the Ocean marketplace.
Publish metadata and service attributes on-chain.
The service urls will be encrypted before going on-chain.
They're only decrypted for datatoken owners upon consume.
Args:
ocean ():
private_key (str):
files (list):
name (str):
symbol (str):
author (str):
data_license (str): The license for the data,
`CC0: Public Domain` by default.
Returns:
()
"""
wallet = Wallet(ocean.web3, private_key, ocean.config.block_confirmations)
assert wallet.web3.eth.get_balance(wallet.address) > 0, 'need ETH'
print('Proceeding with wallet:', wallet.address)
data_token = ocean.create_data_token(name,
symbol,
wallet,
blob=ocean.config.metadata_cache_uri)
# return data_token
token_address = data_token.address
print('Created token:', token_address)
date_created = datetime.now().isoformat()
metadata = {
'main': {
'type': 'dataset',
'name': name,
'author': author,
'license': data_license,
'dateCreated': date_created,
'files': files,
}
}
service_attributes = {
'main': {
'name': 'dataAssetAccessServiceAgreement',
'creator': wallet.address,
'timeout': 3600 * 24,
'datePublished': date_created,
'cost': 1.0, # <don't change, this is obsolete>
}
}
service_endpoint = DataServiceProvider.get_url(ocean.config)
# FIXME:
download_service = Service(
service_endpoint=service_endpoint,
service_type=ServiceTypes.ASSET_ACCESS,
attributes=service_attributes,
)
assert wallet.web3.eth.get_balance(wallet.address) > 0, 'need ETH'
asset = ocean.assets.create(
metadata,
wallet,
# services=[download_service],
# service_descriptors=[],
data_token_address=token_address)
print('Created asset:', asset.data_token_address)
assert token_address == asset.data_token_address
return data_token, asset
def start(
self,
input_datasets: list,
consumer_wallet: Wallet,
nonce: Optional[int] = None,
algorithm_did: Optional[str] = None,
algorithm_meta: Optional[AlgorithmMetadata] = None,
algorithm_tx_id: Optional[str] = None,
algorithm_data_token: Optional[str] = None,
output: Optional[dict] = None,
job_id: Optional[str] = None,
algouserdata: Optional[dict] = None,
) -> str:
"""
Start a remote compute job on the asset files.
Files are identified by `did` after verifying that the provider service is active and transferring the
number of data-tokens required for using this compute service.
:param input_datasets: list of ComputeInput -- list of input datasets to the compute job. A dataset is
represented with ComputeInput struct
:param consumer_wallet: Wallet instance of the consumer ordering the service
:param nonce: int value to use in the signature
:param algorithm_did: str -- the asset did (of `algorithm` type) which consist of `did:op:` and
the assetId hex str (without `0x` prefix)
:param algorithm_meta: `AlgorithmMetadata` instance -- metadata about the algorithm being run if
`algorithm` is being used. This is ignored when `algorithm_did` is specified.
:param algorithm_tx_id: transaction hash of algorithm StartOrder tx (Required when using `algorithm_did`)
:param algorithm_data_token: datatoken address of this algorithm (Required when using `algorithm_did`)
:param output: dict object to be used in publishing mechanism, must define
:param job_id: str identifier of a compute job that was previously started and
stopped (if supported by the provider's backend)
:return: str -- id of compute job being executed
"""
assert (
algorithm_did or algorithm_meta
), "either an algorithm did or an algorithm meta must be provided."
for i in input_datasets:
assert isinstance(i, ComputeInput)
first_input = input_datasets[0]
did = first_input.did
order_tx_id = first_input.transfer_tx_id
service_id = first_input.service_id
output = OceanCompute.check_output_dict(output,
consumer_wallet.address,
self._data_provider,
self._config)
asset = resolve_asset(
did, metadata_cache_uri=self._config.metadata_cache_uri)
_, service_endpoint = self._get_service_endpoint(did, asset)
service = asset.get_service_by_index(service_id)
sa = Service.from_json(service.as_dictionary())
assert (ServiceTypes.CLOUD_COMPUTE == sa.type
), "service at serviceId is not of type compute service."
consumable_result = asset.is_consumable(
{
"type": "address",
"value": consumer_wallet.address
},
provider_uri=sa.service_endpoint,
)
if consumable_result != ConsumableCodes.OK:
raise AssetNotConsumable(consumable_result)
signature = self._sign_message(
consumer_wallet,
f"{consumer_wallet.address}{did}",
nonce=nonce,
service_endpoint=sa.service_endpoint,
)
try:
job_info = self._data_provider.start_compute_job(
did,
service_endpoint,
consumer_wallet.address,
signature,
sa.index,
order_tx_id,
algorithm_did,
algorithm_meta,
algorithm_tx_id,
algorithm_data_token,
output,
input_datasets,
job_id,
userdata=first_input.userdata,
algouserdata=algouserdata,
)
return job_info["jobId"]
except ValueError:
raise
def test_market_flow():
"""Tests that an order is correctly placed on the market.
The parameter implicit_none sends the payload with an empty key as the delegated consumer.
The parameter explicit_none sends None as the delegated consumer, explicitly."""
pub_wallet = get_publisher_wallet()
publisher_ocean = get_publisher_ocean_instance()
consumer_ocean = get_consumer_ocean_instance()
# Register asset
asset = get_registered_ddo(publisher_ocean, get_metadata(), pub_wallet)
assert isinstance(asset, V3Asset)
assert asset.data_token_address, "The asset does not have a token address."
consumer_wallet = get_consumer_wallet()
service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS)
sa = Service.from_json(service.as_dictionary())
# Mint data tokens and assign to publisher
dt = publisher_ocean.get_data_token(asset.data_token_address)
mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet)
######
# Give the consumer some datatokens so they can order the service
try:
tx_id = dt.transfer(consumer_wallet.address, to_wei(10), pub_wallet)
dt.verify_transfer_tx(tx_id, pub_wallet.address,
consumer_wallet.address)
except (AssertionError, Exception) as e:
print(e)
raise
######
# Place order for the download service
order_requirements = consumer_ocean.assets.order(asset.did,
consumer_wallet.address,
sa.index)
######
# Pay for the service
args = [
consumer_ocean.web3,
order_requirements.amount,
order_requirements.data_token_address,
asset.did,
service.index,
"0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9",
consumer_wallet,
sa.get_c2d_address(),
]
_order_tx_id = consumer_ocean.assets.pay_for_service(*args)
######
# Download the asset files
asset_folder = consumer_ocean.assets.download(
asset.did,
sa.index,
consumer_wallet,
_order_tx_id,
consumer_ocean.config.downloads_path,
)
assert len(os.listdir(asset_folder)) >= 1, "The asset folder is empty."
orders = consumer_ocean.get_user_orders(consumer_wallet.address,
asset.asset_id)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(
consumer_wallet.address,
consumer_ocean.web3.toChecksumAddress(asset.asset_id))
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(consumer_wallet.address)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
######
# Publisher can get the urls of the asset
asset_urls = DataServiceProvider.get_asset_urls(asset.did,
str(service.index),
"http://172.15.0.4:8030",
pub_wallet)
assert len(asset_urls) == 3
for url in asset_urls:
assert "10_Monkey_Species_Small" in url
def test_payer_market_flow():
"""Tests that an order can be placed for a delegated consumer, other than the payer."""
pub_wallet = get_publisher_wallet()
publisher_ocean = get_publisher_ocean_instance()
consumer_ocean = get_consumer_ocean_instance()
another_consumer_ocean = get_another_consumer_ocean_instance(
use_provider_mock=True)
# Register Asset
asset = get_registered_ddo(publisher_ocean, get_metadata(), pub_wallet)
assert isinstance(asset, V3Asset)
assert asset.data_token_address, "The asset does not have a token address."
another_consumer_wallet = get_another_consumer_wallet()
consumer_wallet = get_consumer_wallet()
service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS)
sa = Service.from_json(service.as_dictionary())
# Mint data tokens and assign to publisher
dt = publisher_ocean.get_data_token(asset.data_token_address)
mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet)
######
# Give the consumer some datatokens so they can order the service
try:
tx_id = dt.transfer(consumer_wallet.address, to_wei(10), pub_wallet)
dt.verify_transfer_tx(tx_id, pub_wallet.address,
consumer_wallet.address)
except (AssertionError, Exception) as e:
print(e)
raise
######
# Place order for the download service
order_requirements = consumer_ocean.assets.order(
asset.did, another_consumer_wallet.address, sa.index)
######
# Pay for the service and have another_consumer_wallet as consumer
_order_tx_id = consumer_ocean.assets.pay_for_service(
consumer_ocean.web3,
order_requirements.amount,
order_requirements.data_token_address,
asset.did,
service.index,
"0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9",
consumer_wallet,
another_consumer_wallet.address,
)
asset_folder = None
assert asset_folder is None
if asset_folder is None:
# Download the asset files
asset_folder = another_consumer_ocean.assets.download(
asset.did,
sa.index,
another_consumer_wallet,
_order_tx_id,
another_consumer_ocean.config.downloads_path,
)
assert len(os.listdir(asset_folder)) >= 1
orders = consumer_ocean.get_user_orders(consumer_wallet.address,
asset.asset_id)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(
consumer_wallet.address,
consumer_ocean.web3.toChecksumAddress(asset.asset_id))
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(consumer_wallet.address)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"