def start_exec(args):
account = Wallet(config.web3, private_key=args.private_key)
job_id = job_service.add_job(args.task_id, args.cdt, account)
logger.info(f'submit job with job_id: {job_id}')
_, cdt_ddo = resolve_asset(args.cdt, asset_service.dt_factory)
for child_dt in cdt_ddo.child_dts:
_, ddo = resolve_asset(child_dt, asset_service.dt_factory)
endpoint = ddo.services[0].endpoint
msg = f'{account.address}{job_id}'
msg_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = account.sign(msg_hash).signature.hex()
data = {
'job_id': job_id,
'algo_dt': cdt_ddo.dt,
'dt': child_dt,
'signature': signature
}
response = requests.post(f"http://{endpoint}/onPremiseCompute",
data=json.dumps(data),
headers={'content-type': 'application/json'},
timeout=5)
if response.status_code != 200:
raise Exception(response.content.decode('utf-8'))
logger.info(f'start job execution for cdt: {args.cdt}')
return
def fetch_exec_code(self, cdt, leaf_dt):
"""
Get the code template and its fulfiled arguments, given a father cdt and a
leaf dt. The father ddo specifies which child service/template to use.
:param cdt: father composable data token
:param leaf_dt: child data token, must be leaf
:return: bool
"""
_, cdt_ddo = resolve_asset(cdt, self.dt_factory)
_, dt_ddo = resolve_asset(leaf_dt, self.dt_factory)
fulfilled = cdt_ddo.services[0].descriptor['workflow'].get(leaf_dt)
if not fulfilled:
return None, None
sid = fulfilled.get('service')
args = fulfilled.get('constraint')
tid = dt_ddo.get_service_by_index(sid).descriptor['template']
_, op = resolve_op(tid, self.op_template)
return op.operation, args
def check_service_terms(self, cdt, dt, owner_address, signature):
"""
Check service agreements automatically when receiving a remote permission
authorization request, used by Compute-to-Data.
:param cdt: refers to cdt identifier provided by aggregator
:param dt: refers to dt identifier owned by the provider grid
:param owner_address: asset owner address
:param signature: signed by aggregator, [consume_address, cdt]
:return: bool
"""
if self.verifier.check_dt_perm(dt, cdt):
return True
if not self.verifier.check_dt_owner(dt, owner_address):
return False
data, cdt_ddo = resolve_asset(cdt, self.dt_factory)
if not data or not cdt_ddo:
return False
consume_address = data[1]
original_msg = f'{consume_address}{cdt}'
if not self.verifier.verify_signature(consume_address, signature,
original_msg):
return False
checksum = data[2]
if not self.verifier.verify_ddo_integrity(cdt_ddo, checksum):
return False
if not self.verifier.verify_services(cdt_ddo, [dt], False):
return False
return True
def compose_cdt(args):
cdt_ddo, account = mint_dt(args)
for child_dt in cdt_ddo.child_dts:
data, ddo = resolve_asset(child_dt, asset_service.dt_factory)
endpoint = ddo.services[0].endpoint
msg = f'{account.address}{cdt_ddo.dt}'
msg_hash = add_ethereum_prefix_and_hash_msg(msg)
signature = account.sign(msg_hash).signature.hex()
data = {'algo_dt': cdt_ddo.dt, 'dt': child_dt, 'signature': signature}
response = requests.post(f"http://{endpoint}/grantPermission",
data=json.dumps(data),
headers={'content-type': 'application/json'},
timeout=5)
if response.status_code != 200:
raise Exception(response.content.decode('utf-8'))
asset_service.activate_cdt(cdt_ddo.dt, cdt_ddo.child_dts, account)
logger.info(f'compose data token with cdt: {cdt_ddo.dt}')
return
def trace_dt_lifecycle(self, dt, prefix: list):
"""
Trace the whole lifecycle for a dt using dfs recursive search. Only when an
algorithm cdt is submitted for solving tasks, the terminal state is reached.
:param dt: data token identifier.
:param prefix: fixed prefix path, then find its subsequent paths.
:return all_paths: a list of found prefix + subsequent paths
"""
prefix = prefix.copy()
if len(prefix):
owner = self.get_dt_owner(dt)
owner_info = self.get_enterprise(owner)[0]
prefix.append({
"dt": DTHelper.id_bytes_to_dt(dt),
"aggregator": owner_info,
"aggrement": 0
})
else:
prefix.append({"dt": dt})
dt = DTHelper.dt_to_id_bytes(dt)
_, ddo = resolve_asset(dt, self.dt_factory)
all_paths = []
if self.verifier.check_asset_type(ddo, self.TERMINAL):
jobs = self.trace_cdt_jobs(dt)
if len(jobs):
for job in jobs:
job_id, solver, task_id, demander, task_name, task_desc = job
demander_info = self.get_enterprise(demander)[0]
solver_info = self.get_enterprise(solver)[0]
text = {
"task_name": task_name,
"task_desc": task_desc,
"solver": solver_info,
"demander": demander_info,
"task_id": task_id,
"job_id": job_id
}
new_path = prefix.copy()
new_path.append(text)
all_paths.append(new_path)
return all_paths
grantees = self.trace_dt_grantees(dt)
for cdt in grantees:
path_lists = self.trace_dt_lifecycle(cdt, prefix)
all_paths.extend(path_lists)
return all_paths
def get_dt_details(self, dt):
"""
Get the detailed information given a datatoken.
:param dt: refers to dt identifier
:return: tuple
"""
data, ddo = resolve_asset(dt, self.dt_factory)
if not data or not ddo:
return None
checksum = data[2]
if not self.verifier.verify_ddo_integrity(ddo, checksum):
return None
owner = data[0]
issuer = data[1]
issuer_name = self.asset_provider.get_enterprise(issuer)[0]
asset_name = ddo.metadata['main'].get('name')
asset_desc = ddo.metadata['main'].get('desc')
asset_type = ddo.metadata['main'].get('type')
asset_fig = ddo.metadata['main'].get('fig')
dt_info = {
"name": asset_name,
"owner": owner,
"issuer": issuer_name,
"desc": asset_desc,
"type": asset_type,
"fig": asset_fig
}
union_data = None
if ddo.is_cdt:
union_paths = self.tracer.trace_data_union(ddo, [ddo.dt])
tree = self.tracer.tree_format(union_paths)
union_data = self.tracer.tree_to_json(tree)
# self.tracer.print_tree(tree, indent=[], final_node=True)
service_lists = []
for service in ddo.services:
sid = service.index
op_name = service.attributes.get('op_name')
price = service.attributes['price']
constrains = service.descriptor
service_lists.append({
"sid": sid,
"op": op_name,
"price": price,
"constrains": constrains
})
return (dt_info, service_lists, union_data)
def check_remote_compute(self, cdt, dt, job_id, owner_address, signature):
"""
Check job status and resource permissions automatically when receiving an
on-premise computation request, used by Compute-to-Data.
:param cdt: refers to cdt identifier provided by solver
:param dt: refers to dt identifier owned by the provider grid
:param job_id: refers to job identifier in the task market
:param owner_address: asset owner address
:param signature: signed by solver, [solver_address, job_id]
:return: bool
"""
if not self.verifier.verify_job_registered(job_id, cdt):
return False
if not self.verifier.check_dt_owner(dt, owner_address):
return False
data, cdt_ddo = resolve_asset(cdt, self.dt_factory)
if not data or not cdt_ddo:
return False
if not self.verifier.check_asset_type(cdt_ddo, 'Algorithm'):
return False
solver_address = data[1]
checksum = data[2]
original_msg = f'{solver_address}{job_id}'
if not self.verifier.verify_signature(solver_address, signature,
original_msg):
return False
if not self.verifier.verify_ddo_integrity(cdt_ddo, checksum):
return False
if not self.verifier.verify_perms_ready(cdt_ddo, required_dt=dt):
return False
return True
def trace_data_union(self, ddo, prefix):
"""
Trace the data union structure.
:param ddo: metadata object.
:param prefix: fixed prefix path, then find its subsequent paths.
:return all_paths: a list of found prefix + subsequent paths
"""
all_paths = []
if ddo.is_cdt:
for child_dt in ddo.child_dts:
new_path = prefix.copy()
_, child_ddo = resolve_asset(child_dt, self.dt_factory)
asset_name = child_ddo.metadata["main"].get("name")
if child_ddo.is_cdt:
owner = self.get_dt_owner(child_ddo.dt)
owner_info = self.get_enterprise(owner)[0]
new_path.append({
"dt": child_dt,
"name": asset_name,
"aggregator": owner_info
})
path_lists = self.trace_data_union(child_ddo, new_path)
all_paths.extend(path_lists)
else:
asset_type = child_ddo.metadata['main'].get('type')
new_path.append({
"dt": child_dt,
"name": asset_name,
"type": asset_type
})
all_paths.append(new_path)
return all_paths
def verify_services(self, ddo, wrt_dts=None, integrity_check=True):
"""
Ensure the service constraints are fulfilled. For a given leaf ddo, we check
the parameter consistency of its constraints and used templates. For a given
composable ddo, we first check the availability of its childs, and then check
the fulfilled constraints for each workflow service.
:param ddo: a candidate DDO object, composable or leaf
:param wrt_dts: a list of dts to be fulfilled, all child dts if None
:param integrity_check: verify child ddo integrity if True
:return: bool
"""
if not ddo.is_cdt:
return validate_leaf_template(ddo, self.op_template)
if not wrt_dts:
wrt_dts = ddo.child_dts
for dt in wrt_dts:
data, child_ddo = resolve_asset(dt, self.dt_factory)
if not data or not child_ddo:
return False
if integrity_check and not self.verify_ddo_integrity(child_ddo, data[2]):
return False
if not child_ddo.is_cdt:
if not validate_leaf_template(child_ddo, self.op_template):
return False
else:
if not self.check_cdt_composed(child_ddo.dt) or (
not self.verify_perms_ready(child_ddo)):
return False
if not validate_service_agreement(ddo, child_ddo):
return False
return True