def test_no_service_key_ddo(publisher_ocean_instance, metadata):
"""Tests that ServiceAgreements are not created with bad indices."""
# from init
with pytest.raises(ValueError):
ServiceAgreement(attributes=None, service_index="not_a_real_index")
# from ddo
ddo = get_sample_ddo()
with pytest.raises(ValueError):
ServiceAgreement.from_ddo(service_type="not_a_real_index", ddo=ddo)
def _get_service_endpoint(self, did, asset=None):
if not asset:
asset = resolve_asset(did, self._config.metadata_cache_uri)
return self._data_provider.build_compute_endpoint(
ServiceAgreement.from_ddo(
ServiceTypes.CLOUD_COMPUTE, asset
).service_endpoint
)
def order(
self,
did: str,
consumer_address: str,
service_index: Optional[int] = None,
service_type: str = None,
) -> OrderRequirements:
"""
Request a specific service from an asset, returns the service requirements that
must be met prior to consuming the service.
:param did:
:param consumer_address:
:param service_index:
:param service_type:
:return: OrderRequirements instance -- named tuple (amount, data_token_address, receiver_address, nonce),
"""
assert (service_type or service_index
), "One of service_index or service_type is required."
asset = self.resolve(did)
if service_type:
sa = ServiceAgreement.from_ddo(service_type, asset)
else:
service = asset.get_service_by_index(service_index)
sa = ServiceAgreement.from_ddo(service.type, asset)
dt_address = asset.data_token_address
_, initialize_url = self._data_provider.build_initialize_endpoint(
sa.service_endpoint)
order_requirements = self._data_provider.get_order_requirements(
asset.did, initialize_url, consumer_address, sa.index, sa.type,
dt_address)
if not order_requirements:
raise AssertionError(
"Data service provider or service is not available.")
assert dt_address == order_requirements.data_token_address
return order_requirements
def _read_dict(self, dictionary):
"""Import a JSON dict into this DDO."""
values = copy.deepcopy(dictionary)
self._did = values.pop("id")
self._created = values.pop("created", None)
if "publicKey" in values:
self._public_keys = []
for value in values.pop("publicKey"):
if isinstance(value, str):
value = json.loads(value)
self._public_keys.append(
DDO.create_public_key_from_json(value))
if "authentication" in values:
self._authentications = []
for value in values.pop("authentication"):
if isinstance(value, str):
value = json.loads(value)
self._authentications.append(
DDO.create_authentication_from_json(value))
if "service" in values:
self._services = []
for value in values.pop("service"):
if isinstance(value, str):
value = json.loads(value)
if value["type"] == ServiceTypes.ASSET_ACCESS:
service = ServiceAgreement.from_json(value)
elif value["type"] == ServiceTypes.CLOUD_COMPUTE:
service = ServiceAgreement.from_json(value)
else:
service = Service.from_json(value)
self._services.append(service)
if "proof" in values:
self._proof = values.pop("proof")
self._other_values = values
def build_compute_service(attributes, service_endpoint):
"""
Build an authorization service.
:param attributes: attributes of compute service, dict
:param service_endpoint: identifier of the service inside the asset DDO, str
:return: ServiceAgreement instance
"""
return ServiceAgreement(
attributes,
service_endpoint,
ServiceTypes.CLOUD_COMPUTE,
ServiceTypesIndices.DEFAULT_COMPUTING_INDEX,
)
def build_access_service(attributes, service_endpoint):
"""
Build an authorization service.
:param attributes: attributes of access service, dict
:param service_endpoint: identifier of the service inside the asset DDO, str
:return: ServiceAgreement instance
"""
return ServiceAgreement(
attributes,
service_endpoint,
ServiceTypes.ASSET_ACCESS,
ServiceTypesIndices.DEFAULT_ACCESS_INDEX,
)
def ocean_assets_download_destination_file_helper(publisher_ocean_instance,
metadata, tmpdir):
"""Tests downloading to an existing directory."""
publisher = get_publisher_wallet()
metadata_copy = metadata.copy()
data_provider = DataServiceProvider
ddo = publisher_ocean_instance.assets.create(metadata_copy, publisher)
wait_for_ddo(publisher_ocean_instance, ddo.did)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
written_path = download_asset_files(
sa.index,
ddo,
publisher,
tmpdir,
ddo.data_token_address,
"test_order_tx_id",
data_provider,
)
assert os.path.exists(written_path)
def test_ocean_assets_download_failure(publisher_ocean_instance, metadata):
"""Tests that downloading from an empty service raises an AssertionError."""
publisher = get_publisher_wallet()
metadata_copy = metadata.copy()
data_provider = DataServiceProvider
ddo = publisher_ocean_instance.assets.create(metadata_copy, publisher)
wait_for_ddo(publisher_ocean_instance, ddo.did)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
sa.__dict__["_service_endpoint"] = None
ddo.__dict__["_services"][1] = sa
with pytest.raises(AssertionError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
)
def start(
self,
input_datasets: list,
consumer_wallet: Wallet,
nonce: Optional[int] = None,
algorithm_did: Optional[str] = None,
algorithm_meta: Optional[AlgorithmMetadata] = None,
algorithm_tx_id: str = None,
algorithm_data_token: str = None,
output: dict = None,
job_id: str = None,
):
"""
Start a remote compute job on the asset files.
Files are identified by `did` after verifying that the provider service is active and transferring the
number of data-tokens required for using this compute service.
:param input_datasets: list of ComputeInput -- list of input datasets to the compute job. A dataset is
represented with ComputeInput struct
:param consumer_wallet: Wallet instance of the consumer ordering the service
:param nonce: int value to use in the signature
:param algorithm_did: str -- the asset did (of `algorithm` type) which consist of `did:op:` and
the assetId hex str (without `0x` prefix)
:param algorithm_meta: `AlgorithmMetadata` instance -- metadata about the algorithm being run if
`algorithm` is being used. This is ignored when `algorithm_did` is specified.
:param algorithm_tx_id: transaction hash of algorithm StartOrder tx (Required when using `algorithm_did`)
:param algorithm_data_token: datatoken address of this algorithm (Required when using `algorithm_did`)
:param output: dict object to be used in publishing mechanism, must define
:param job_id: str identifier of a compute job that was previously started and
stopped (if supported by the provider's backend)
:return: str -- id of compute job being executed
"""
assert (
algorithm_did or algorithm_meta
), "either an algorithm did or an algorithm meta must be provided."
for i in input_datasets:
assert isinstance(i, ComputeInput)
first_input = input_datasets[0]
did = first_input.did
order_tx_id = first_input.transfer_tx_id
service_id = first_input.service_id
output = OceanCompute.check_output_dict(
output, consumer_wallet.address, data_provider=self._data_provider
)
asset = resolve_asset(did, metadata_cache_uri=self._config.metadata_cache_uri)
_, service_endpoint = self._get_service_endpoint(did, asset)
service = asset.get_service_by_index(service_id)
sa = ServiceAgreement.from_json(service.as_dictionary())
assert (
ServiceTypes.CLOUD_COMPUTE == sa.type
), "service at serviceId is not of type compute service."
signature = self._sign_message(
consumer_wallet,
f"{consumer_wallet.address}{did}",
nonce=nonce,
service_endpoint=sa.service_endpoint,
)
try:
job_info = self._data_provider.start_compute_job(
did,
service_endpoint,
consumer_wallet.address,
signature,
sa.index,
order_tx_id,
algorithm_did,
algorithm_meta,
algorithm_tx_id,
algorithm_data_token,
output,
input_datasets,
job_id,
)
return job_info["jobId"]
except ValueError:
raise
def test_ocean_assets_download_indexes(publisher_ocean_instance, metadata):
"""Tests different values of indexes that raise AssertionError."""
publisher = get_publisher_wallet()
metadata_copy = metadata.copy()
data_provider = DataServiceProvider
ddo = publisher_ocean_instance.assets.create(metadata_copy, publisher)
wait_for_ddo(publisher_ocean_instance, ddo.did)
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, ddo)
config = Config(os.getenv(ENV_CONFIG_FILE))
index = range(3)
if config["util"].getboolean("typecheck"):
with pytest.raises(TypeError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
index,
)
else:
with pytest.raises(AssertionError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
index,
)
index = -1
with pytest.raises(AssertionError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
index,
)
index = 4
with pytest.raises(AssertionError):
download_asset_files(
sa.index,
ddo,
publisher,
"test_destination",
ddo.data_token_address,
"test_order_tx_id",
data_provider,
index,
)
def test_cost():
"""Tests that the get_cost function for ServiceAgreement returns the correct value."""
ddo = get_sample_ddo()
sa = ServiceAgreement.from_ddo(service_type=ServiceTypes.ASSET_ACCESS,
ddo=ddo)
assert sa.get_cost() == 1.0
def test_market_flow(order_type):
"""Tests that an order is correctly placed on the market.
The parameter implicit_none sends the payload with an empty key as the delegated consumer.
The parameter explicit_none sends None as the delegated consumer, explicitly."""
pub_wallet = get_publisher_wallet()
publisher_ocean = get_publisher_ocean_instance()
consumer_ocean = get_consumer_ocean_instance()
# Register Asset
asset = get_registered_ddo(publisher_ocean, get_metadata(), pub_wallet)
assert isinstance(asset, Asset)
assert asset.data_token_address
consumer_wallet = get_consumer_wallet()
service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS)
sa = ServiceAgreement.from_json(service.as_dictionary())
# Mint data tokens and assign to publisher
dt = publisher_ocean.get_data_token(asset.data_token_address)
mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet)
######
# Give the consumer some datatokens so they can order the service
try:
tx_id = dt.transfer_tokens(consumer_wallet.address, 10.0, pub_wallet)
dt.verify_transfer_tx(tx_id, pub_wallet.address,
consumer_wallet.address)
except (AssertionError, Exception) as e:
print(e)
raise
######
# Place order for the download service
order_requirements = consumer_ocean.assets.order(asset.did,
consumer_wallet.address,
sa.index)
######
# Pay for the service
args = [
order_requirements.amount,
order_requirements.data_token_address,
asset.did,
service.index,
"0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9",
consumer_wallet,
]
if order_type == "explicit_none":
args.append(None)
_order_tx_id = consumer_ocean.assets.pay_for_service(*args)
######
# Download the asset files
asset_folder = consumer_ocean.assets.download(
asset.did,
sa.index,
consumer_wallet,
_order_tx_id,
consumer_ocean.config.downloads_path,
)
assert len(os.listdir(asset_folder)) >= 1
if order_type == "explicit_none":
# no need to continue, order worked
return
orders = consumer_ocean.get_user_orders(consumer_wallet.address,
asset.asset_id)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(
consumer_wallet.address,
consumer_ocean.web3.toChecksumAddress(asset.asset_id))
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(consumer_wallet.address)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
def test_payer_market_flow():
"""Tests that an order can be placed for a delegated consumer, other than the payer."""
pub_wallet = get_publisher_wallet()
publisher_ocean = get_publisher_ocean_instance()
consumer_ocean = get_consumer_ocean_instance()
another_consumer_ocean = get_another_consumer_ocean_instance(
use_provider_mock=True)
# Register Asset
asset = get_registered_ddo(publisher_ocean, get_metadata(), pub_wallet)
assert isinstance(asset, Asset)
assert asset.data_token_address
another_consumer_wallet = get_another_consumer_wallet()
consumer_wallet = get_consumer_wallet()
service = asset.get_service(service_type=ServiceTypes.ASSET_ACCESS)
sa = ServiceAgreement.from_json(service.as_dictionary())
# Mint data tokens and assign to publisher
dt = publisher_ocean.get_data_token(asset.data_token_address)
mint_tokens_and_wait(dt, pub_wallet.address, pub_wallet)
######
# Give the consumer some datatokens so they can order the service
try:
tx_id = dt.transfer_tokens(consumer_wallet.address, 10.0, pub_wallet)
dt.verify_transfer_tx(tx_id, pub_wallet.address,
consumer_wallet.address)
except (AssertionError, Exception) as e:
print(e)
raise
######
# Place order for the download service
order_requirements = consumer_ocean.assets.order(
asset.did, another_consumer_wallet.address, sa.index)
######
# Pay for the service and have another_consumer_wallet as consumer
_order_tx_id = consumer_ocean.assets.pay_for_service(
order_requirements.amount,
order_requirements.data_token_address,
asset.did,
service.index,
"0xF9f2DB837b3db03Be72252fAeD2f6E0b73E428b9",
consumer_wallet,
another_consumer_wallet.address,
)
asset_folder = None
assert asset_folder is None
if asset_folder is None:
# Download the asset files
asset_folder = another_consumer_ocean.assets.download(
asset.did,
sa.index,
another_consumer_wallet,
_order_tx_id,
another_consumer_ocean.config.downloads_path,
)
assert len(os.listdir(asset_folder)) >= 1
orders = consumer_ocean.get_user_orders(consumer_wallet.address,
asset.asset_id)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(
consumer_wallet.address,
consumer_ocean.web3.toChecksumAddress(asset.asset_id))
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
orders = consumer_ocean.get_user_orders(consumer_wallet.address)
assert (
orders
), f"no orders found using the order history: datatoken {asset.asset_id}, consumer {consumer_wallet.address}"
def download_asset_files(
service_index: int,
asset: Asset,
consumer_wallet: Wallet,
destination: str,
token_address: str,
order_tx_id: str,
data_provider: Type[DataServiceProvider],
index: Optional[int] = None,
):
"""Download asset data files or result files from a compute job.
:param service_index: identifier of the service inside the asset DDO, str
:param asset: Asset instance
:param consumer_wallet: Wallet instance of the consumer
:param destination: Path, str
:param token_address: hex str the address of the DataToken smart contract
:param order_tx_id: hex str the transaction hash of the startOrder tx
:param data_provider: DataServiceProvider class object
:param index: Index of the document that is going to be downloaded, int
:return: Asset folder path, str
"""
_files = asset.metadata["main"]["files"]
sa = ServiceAgreement.from_ddo(ServiceTypes.ASSET_ACCESS, asset)
service_endpoint = sa.service_endpoint
if not service_endpoint:
logger.error(
'Consume asset failed, service definition is missing the "serviceEndpoint".'
)
raise AssertionError(
'Consume asset failed, service definition is missing the "serviceEndpoint".'
)
_, service_endpoint = data_provider.build_download_endpoint(
service_endpoint)
if not os.path.isabs(destination):
destination = os.path.abspath(destination)
if not os.path.exists(destination):
os.mkdir(destination)
asset_folder = os.path.join(destination,
f"datafile.{asset.asset_id}.{service_index}")
if not os.path.exists(asset_folder):
os.mkdir(asset_folder)
if index is not None:
assert isinstance(index,
int), logger.error("index has to be an integer.")
assert index >= 0, logger.error(
"index has to be 0 or a positive integer.")
assert index < len(_files), logger.error(
"index can not be bigger than the number of files")
indexes = [index]
else:
indexes = range(len(_files))
for i in indexes:
data_provider.download_service(
asset.did,
service_endpoint,
consumer_wallet,
_files,
asset_folder,
service_index,
token_address,
order_tx_id,
i,
)
return asset_folder