def __init__(self, config):
keeper = Keeper(config.keeper_options)
self.asset_provider = keeper.asset_provider
self.op_template = keeper.op_template
self.verifier = VerifierService(config)
self.config = config
def __init__(self, config):
keeper = Keeper(config.keeper_options)
self.dt_factory = keeper.dt_factory
self.asset_provider = keeper.asset_provider
self.verifier = VerifierService(config)
self.tracer = TracerService(config)
self.config = config
def __init__(self, config):
keeper = Keeper(config.keeper_options)
self.dt_factory = keeper.dt_factory
self.op_template = keeper.op_template
self.task_market = keeper.task_market
self.verifier = VerifierService(config)
self.config = config
class TracerService(object):
"""The entry point for accessing the tracer service."""
TERMINAL = 'Algorithm'
def __init__(self, config):
keeper = Keeper(config.keeper_options)
self.asset_provider = keeper.asset_provider
self.op_template = keeper.op_template
self.dt_factory = keeper.dt_factory
self.task_market = keeper.task_market
self.verifier = VerifierService(config)
self.config = config
def get_enterprise(self, id):
"""Get the enterprise info."""
return self.asset_provider.get_enterprise(id)
def get_task(self, task_id):
"""Get task info."""
return self.task_market.get_task(task_id)
def get_job(self, job_id):
"""Get job info."""
return self.task_market.get_job(job_id)
def get_dt_owner(self, dt):
"""Get the owner for a data token."""
_dt = DTHelper.dt_to_id_bytes(dt)
return self.dt_factory.get_dt_owner(_dt)
def get_marketplace_stat(self):
"""Get the statistics information."""
dt_nums = self.dt_factory.get_dt_num()
template_nums = self.op_template.get_template_num()
task_nums = self.task_market.get_task_num()
job_nums = self.task_market.get_job_num()
stats = (dt_nums, template_nums, task_nums, job_nums)
return stats
def trace_owner_assets(self, address):
"""Get all assets for a given owner."""
return self.dt_factory.get_owner_assets(address)
def trace_dt_grantees(self, dt):
"""Get the list of granteed father for a dt."""
_dt = DTHelper.dt_to_id_bytes(dt)
return self.dt_factory.get_dt_grantees(_dt)
def trace_cdt_jobs(self, cdt):
"""Get the list of previous jobs for a given cdt."""
return self.task_market.get_cdt_jobs(cdt)
def trace_data_union(self, ddo, prefix):
"""
Trace the data union structure.
:param ddo: metadata object.
:param prefix: fixed prefix path, then find its subsequent paths.
:return all_paths: a list of found prefix + subsequent paths
"""
all_paths = []
if ddo.is_cdt:
for child_dt in ddo.child_dts:
new_path = prefix.copy()
_, child_ddo = resolve_asset(child_dt, self.dt_factory)
asset_name = child_ddo.metadata["main"].get("name")
if child_ddo.is_cdt:
owner = self.get_dt_owner(child_ddo.dt)
owner_info = self.get_enterprise(owner)[0]
new_path.append({
"dt": child_dt,
"name": asset_name,
"aggregator": owner_info
})
path_lists = self.trace_data_union(child_ddo, new_path)
all_paths.extend(path_lists)
else:
asset_type = child_ddo.metadata['main'].get('type')
new_path.append({
"dt": child_dt,
"name": asset_name,
"type": asset_type
})
all_paths.append(new_path)
return all_paths
def trace_dt_lifecycle(self, dt, prefix: list):
"""
Trace the whole lifecycle for a dt using dfs recursive search. Only when an
algorithm cdt is submitted for solving tasks, the terminal state is reached.
:param dt: data token identifier.
:param prefix: fixed prefix path, then find its subsequent paths.
:return all_paths: a list of found prefix + subsequent paths
"""
prefix = prefix.copy()
if len(prefix):
owner = self.get_dt_owner(dt)
owner_info = self.get_enterprise(owner)[0]
prefix.append({
"dt": DTHelper.id_bytes_to_dt(dt),
"aggregator": owner_info,
"aggrement": 0
})
else:
prefix.append({"dt": dt})
dt = DTHelper.dt_to_id_bytes(dt)
_, ddo = resolve_asset(dt, self.dt_factory)
all_paths = []
if self.verifier.check_asset_type(ddo, self.TERMINAL):
jobs = self.trace_cdt_jobs(dt)
if len(jobs):
for job in jobs:
job_id, solver, task_id, demander, task_name, task_desc = job
demander_info = self.get_enterprise(demander)[0]
solver_info = self.get_enterprise(solver)[0]
text = {
"task_name": task_name,
"task_desc": task_desc,
"solver": solver_info,
"demander": demander_info,
"task_id": task_id,
"job_id": job_id
}
new_path = prefix.copy()
new_path.append(text)
all_paths.append(new_path)
return all_paths
grantees = self.trace_dt_grantees(dt)
for cdt in grantees:
path_lists = self.trace_dt_lifecycle(cdt, prefix)
all_paths.extend(path_lists)
return all_paths
def job_list_format(self, paths):
"""
Convert paths to a formated job list table.
:param paths: a list of dt->...->dt-> [job, ..., job] authorization chains, with the same root dt
:return: list
"""
if len(paths) == 0:
print('Do not find any data linking path')
return None
job_list = []
root = paths[0][0]
for path in paths:
if path[0] != root:
raise AssertionError(f'A tree can only contain one root')
job_list.append(path[-1])
return job_list
def tree_format(self, paths):
"""
Convert paths to a formated hierarchical tree using Node class.
:param paths: a list of dt->...->dt->... authorization chains, with the same root dt
:return: root Node instance
"""
if len(paths) == 0:
print('Do not find any data linking path')
return None
root = paths[0][0]
for path in paths:
if path[0] != root:
raise AssertionError(f'A tree can only contain one root')
root_node = Node(text=root, level=0)
for path in paths:
tmp_node = root_node
level = 1
index = 0
for path_value in path[1:]:
child_node = tmp_node.get_child(text=path_value)
if not child_node:
child_node = Node(text=path_value, level=level)
tmp_node.add_child(child_node)
tmp_node = child_node
level += 1
index += 1
return root_node
def tree_to_json(self, node):
data = {"values": node.text}
if len(node.child_nodes):
data["children"] = []
for n in node.child_nodes:
data["children"].append(self.tree_to_json(n))
return data
def print_tree(self, node, indent: list, final_node=True):
"""Recursively output the node text and its child node."""
for i in range(node.level):
print(indent[i], end='')
if final_node:
print('└──', end='')
else:
print('├──', end='')
print(node.text)
if node.empty():
return
else:
cnt = len(node.child_nodes)
for i, n in enumerate(node.child_nodes):
c = ' ' if final_node else '│ '
indent.append(c)
last_node = i == cnt - 1
self.print_tree(n, indent, last_node)
del indent[-1]
class SystemService:
"""The entry point for accessing the system service."""
def __init__(self, config):
keeper = Keeper(config.keeper_options)
self.asset_provider = keeper.asset_provider
self.op_template = keeper.op_template
self.verifier = VerifierService(config)
self.config = config
def register_enterprise(self, address, name, desc, from_wallet):
"""
Register a new enterprise on-chain.
:param address: refers to the enterprise address
:param name: refers to the enterprise name
:param desc: refers to the enterprise description
:param from_wallet: the system account
:return
"""
if not self.verifier.check_enterprise(address):
self.asset_provider.register_enterprise(address, name, desc,
from_wallet)
else:
self.asset_provider.update_enterprise(address, name, desc,
from_wallet)
return
def add_provider(self, address, from_wallet):
"""
Add a new provider on-chain.
:param address: refers to the provider address
:param from_wallet: the system account
:return
"""
if not self.verifier.check_provider(address):
self.asset_provider.add_provider(address, from_wallet)
else:
self.asset_provider.update_provider(address, from_wallet)
return
def publish_template(self, metadata, operation, params, from_wallet):
"""
Publish the op template on chain.
:param metadata: refers to the template metadata
:param operation: refers to the code template
:param params: refers to the code parameters
:param from_wallet: the system account
:return
"""
op = OpTemplate()
op.add_metadata(metadata)
op.add_template(operation, params)
op.add_creator(from_wallet.address)
op.assign_tid(DTHelper.generate_new_dt())
op.create_proof()
ipfs_client = IPFSProvider(self.config)
ipfs_path = ipfs_client.add(op.to_dict())
tid = DTHelper.dt_to_id(op.tid)
name = metadata['main']['name']
checksum = op.proof['checksum']
if not self.verifier.check_op_exist(op.tid):
self.op_template.publish_template(tid, name, checksum, ipfs_path,
from_wallet)
else:
self.op_template.update_template(tid, name, checksum, ipfs_path,
from_wallet)
return op
class AssetService(object):
"""The entry point for accessing the asset service."""
def __init__(self, config):
keeper = Keeper(config.keeper_options)
self.dt_factory = keeper.dt_factory
self.asset_provider = keeper.asset_provider
self.verifier = VerifierService(config)
self.tracer = TracerService(config)
self.config = config
def generate_ddo(self,
metadata,
services,
owner_address,
child_dts=None,
verify=True):
"""
Create an asset document and declare its services.
:param metadata: refers to the asset metadata
:param services: list of asset services
:param owner_address: refers to the asset owner
:param child_dts: list of child asset identifiers
:param verify: check the correctness of asset services
:return ddo: DDO instance
"""
ddo = DDO()
ddo.add_metadata(metadata, child_dts)
ddo.add_creator(owner_address)
for service in services:
ddo.add_service(service)
ddo.assign_dt(DTHelper.generate_new_dt())
ddo.create_proof()
# make sure the generated ddo is under system constraits
if verify and not self.verifier.verify_services(ddo):
raise AssertionError(f'Service agreements are not satisfied')
return ddo
def publish_dt(self, ddo, issuer_wallet):
"""
Publish a ddo to the decentralized storage network and register its
data token on the smart-contract chain.
:param ddo: refers to the asset DDO document
:param issuer_wallet: issuer account, enterprize now
:return
"""
ipfs_client = IPFSProvider(self.config)
ipfs_path = ipfs_client.add(ddo.to_dict())
dt = DTHelper.dt_to_id(ddo.dt)
owner = ddo.creator
isLeaf = not bool(ddo.child_dts)
checksum = ddo.proof['checksum']
self.dt_factory.mint_dt(dt, owner, isLeaf, checksum, ipfs_path,
issuer_wallet)
return
def grant_dt_perm(self, dt, grantee, owner_wallet):
"""
Grant one dt to other dt.
:param dt: refers to data token identifier
:param grantee: refers to granted dt identifier
:param owner_wallet: owner account
:return
"""
_dt = DTHelper.dt_to_id(dt)
_grantee = DTHelper.dt_to_id(grantee)
self.dt_factory.grant_dt(_dt, _grantee, owner_wallet)
return
def activate_cdt(self, cdt, child_dts, aggregator_wallet):
"""
Activate cdt when all perms are ready.
:param cdt: refers to cdt identifier
:param child_dts: associated with child_dts identifier
:param aggregator_wallet: aggregator account
:return
"""
_cdt = DTHelper.dt_to_id(cdt)
_child_dts = [DTHelper.dt_to_id(dt) for dt in child_dts]
self.dt_factory.start_compose_dt(_cdt, _child_dts, aggregator_wallet)
return
def check_service_terms(self, cdt, dt, owner_address, signature):
"""
Check service agreements automatically when receiving a remote permission
authorization request, used by Compute-to-Data.
:param cdt: refers to cdt identifier provided by aggregator
:param dt: refers to dt identifier owned by the provider grid
:param owner_address: asset owner address
:param signature: signed by aggregator, [consume_address, cdt]
:return: bool
"""
if self.verifier.check_dt_perm(dt, cdt):
return True
if not self.verifier.check_dt_owner(dt, owner_address):
return False
data, cdt_ddo = resolve_asset(cdt, self.dt_factory)
if not data or not cdt_ddo:
return False
consume_address = data[1]
original_msg = f'{consume_address}{cdt}'
if not self.verifier.verify_signature(consume_address, signature,
original_msg):
return False
checksum = data[2]
if not self.verifier.verify_ddo_integrity(cdt_ddo, checksum):
return False
if not self.verifier.verify_services(cdt_ddo, [dt], False):
return False
return True
def get_dt_marketplace(self):
"""
Get all available dts in the marketplace.
:return: list
"""
dt_idx, _, issuers, checksums, _, ipfs_paths, _ = self.dt_factory.get_available_dts(
)
issuer_names = self.asset_provider.get_issuer_names(issuers)
marketplace_list = []
for dt, issuer_name, ipfs_path, checksum in zip(
dt_idx, issuer_names, ipfs_paths, checksums):
ddo = resolve_asset_by_url(ipfs_path)
if ddo and ddo.metadata['main'].get('type') != "Algorithm":
if self.verifier.verify_ddo_integrity(ddo, checksum):
dt = DTHelper.id_bytes_to_dt(dt)
asset_name = ddo.metadata["main"].get("name")
asset_fig = ddo.metadata['main'].get('fig')
union_or_not = ddo.is_cdt
marketplace_list.append({
"dt": dt,
"issuer": issuer_name,
"name": asset_name,
"fig": asset_fig,
"union_or_not": union_or_not
})
return marketplace_list
def get_dt_details(self, dt):
"""
Get the detailed information given a datatoken.
:param dt: refers to dt identifier
:return: tuple
"""
data, ddo = resolve_asset(dt, self.dt_factory)
if not data or not ddo:
return None
checksum = data[2]
if not self.verifier.verify_ddo_integrity(ddo, checksum):
return None
owner = data[0]
issuer = data[1]
issuer_name = self.asset_provider.get_enterprise(issuer)[0]
asset_name = ddo.metadata['main'].get('name')
asset_desc = ddo.metadata['main'].get('desc')
asset_type = ddo.metadata['main'].get('type')
asset_fig = ddo.metadata['main'].get('fig')
dt_info = {
"name": asset_name,
"owner": owner,
"issuer": issuer_name,
"desc": asset_desc,
"type": asset_type,
"fig": asset_fig
}
union_data = None
if ddo.is_cdt:
union_paths = self.tracer.trace_data_union(ddo, [ddo.dt])
tree = self.tracer.tree_format(union_paths)
union_data = self.tracer.tree_to_json(tree)
# self.tracer.print_tree(tree, indent=[], final_node=True)
service_lists = []
for service in ddo.services:
sid = service.index
op_name = service.attributes.get('op_name')
price = service.attributes['price']
constrains = service.descriptor
service_lists.append({
"sid": sid,
"op": op_name,
"price": price,
"constrains": constrains
})
return (dt_info, service_lists, union_data)
class JobService():
"""The entry point for accessing the job service."""
def __init__(self, config):
keeper = Keeper(config.keeper_options)
self.dt_factory = keeper.dt_factory
self.op_template = keeper.op_template
self.task_market = keeper.task_market
self.verifier = VerifierService(config)
self.config = config
def create_task(self, name, desc, demander_wallet):
"""
Add a new task on chain.
:param name: refers to the task name
:param desc: refers to the task description
:param demander_wallet: demander account
:return: int task_id
"""
task_id = self.task_market.create_task(name, desc, demander_wallet)
return task_id
def add_job(self, task_id, cdt, solver_wallet):
"""
Create a new job on chain with the algorithm cdt.
:param cdt: refers to the algorithm composable data token
:param task_id: refers to the task id that be solved
:param from_wallet: solver account
:return: int job_id
"""
_id = DTHelper.dt_to_id(cdt)
job_id = self.task_market.add_job(_id, task_id, solver_wallet)
return job_id
def check_remote_compute(self, cdt, dt, job_id, owner_address, signature):
"""
Check job status and resource permissions automatically when receiving an
on-premise computation request, used by Compute-to-Data.
:param cdt: refers to cdt identifier provided by solver
:param dt: refers to dt identifier owned by the provider grid
:param job_id: refers to job identifier in the task market
:param owner_address: asset owner address
:param signature: signed by solver, [solver_address, job_id]
:return: bool
"""
if not self.verifier.verify_job_registered(job_id, cdt):
return False
if not self.verifier.check_dt_owner(dt, owner_address):
return False
data, cdt_ddo = resolve_asset(cdt, self.dt_factory)
if not data or not cdt_ddo:
return False
if not self.verifier.check_asset_type(cdt_ddo, 'Algorithm'):
return False
solver_address = data[1]
checksum = data[2]
original_msg = f'{solver_address}{job_id}'
if not self.verifier.verify_signature(solver_address, signature,
original_msg):
return False
if not self.verifier.verify_ddo_integrity(cdt_ddo, checksum):
return False
if not self.verifier.verify_perms_ready(cdt_ddo, required_dt=dt):
return False
return True
def fetch_exec_code(self, cdt, leaf_dt):
"""
Get the code template and its fulfiled arguments, given a father cdt and a
leaf dt. The father ddo specifies which child service/template to use.
:param cdt: father composable data token
:param leaf_dt: child data token, must be leaf
:return: bool
"""
_, cdt_ddo = resolve_asset(cdt, self.dt_factory)
_, dt_ddo = resolve_asset(leaf_dt, self.dt_factory)
fulfilled = cdt_ddo.services[0].descriptor['workflow'].get(leaf_dt)
if not fulfilled:
return None, None
sid = fulfilled.get('service')
args = fulfilled.get('constraint')
tid = dt_ddo.get_service_by_index(sid).descriptor['template']
_, op = resolve_op(tid, self.op_template)
return op.operation, args