def add_in_data(self,
data,
data_hash=None,
encrypted_data_encryption_key=None,
data_iv=None):
"""Add inData work order parameter."""
if data is None:
return util.create_error_response(
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE, 0,
"Invalid data format for in data")
in_data_copy = self.params_obj["inData"]
new_data_list = self.__add_data_params(in_data_copy, data, data_hash,
encrypted_data_encryption_key,
data_iv)
self.params_obj["inData"] = new_data_list
code, err_msg = WOcheck.schema_validation("sdk_inData",
self.params_obj["inData"])
if not code:
return util.create_error_response(
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE, 0, err_msg)
return None
def _process_request(self, input_json_str):
"""
Handles incoming requests or rather dispatches to appropriate
rpc method
Parameters :
input_json_str - JSON formatted str of the request
Returns :
response - data field from the response received which is a dict
"""
response = {}
response['error'] = {}
response['error']['code'] = \
JRPCErrorCodes.INVALID_PARAMETER_FORMAT_OR_VALUE
try:
input_json = json.loads(input_json_str)
valid, err_msg = \
Validator.schema_validation("tc_methods", input_json)
if not valid:
raise ValueError(err_msg)
valid, err_msg = \
Validator.schema_validation(
input_json["method"],
input_json["params"])
if not valid:
raise ValueError(err_msg)
except Exception as err:
logger.error("Exception while processing Json: %s", str(err))
response["error"]["message"] = \
"{}".format(str(err))
return response
# save the full json for WorkOrderSubmit
input_json["params"]["raw"] = input_json_str
data = json.dumps(input_json).encode('utf-8')
response = JSONRPCResponseManager.handle(data, self.dispatcher)
return response.data
def WorkerLookUpNext(self, **params):
"""
Function to look the set of worker newly added
Parameters:
- param is the 'param' object in the a worker request as per TCF
API 5.3.5 Worker Lookup Next JSON Payload
"""
input_json_str = params["raw"]
input_value_json = json.loads(input_json_str)
valid, err_msg = \
Validator.schema_validation(
"WorkerLookUpNext",
input_value_json["params"])
if not valid:
raise JSONRPCDispatchException(
WorkerError.INVALID_PARAMETER_FORMAT_OR_VALUE, err_msg)
return self.__lookup_basic(True, params)
def WorkerRetrieve(self, **params):
"""
Function to retrieve the details of worker
Parameters:
- param is the 'param' object in the a worker request as per
Trusted Compute EEA API 5.3.7 Worker Retrieve JSON Payload
"""
input_json_str = params["raw"]
input_value_json = json.loads(input_json_str)
valid, err_msg = \
Validator.schema_validation(
"WorkerRetrieve",
input_value_json["params"])
if not valid:
raise JSONRPCDispatchException(
WorkerError.INVALID_PARAMETER_FORMAT_OR_VALUE, err_msg)
# value retrieved is 'result' field as per Spec 5.3.8 Worker Retrieve
# Response Payload
worker_id = str(params['workerId'])
value = self.kv_helper.get("workers", worker_id)
if value is None:
raise JSONRPCDispatchException(
WorkerError.INVALID_PARAMETER_FORMAT_OR_VALUE,
"Worker Id not found in the database. Hence invalid parameter")
json_dict = json.loads(value)
result = {
"workerType": json_dict["workerType"],
"organizationId": json_dict["organizationId"],
"applicationTypeId": json_dict["applicationTypeId"],
"details": json_dict["details"],
"status": json_dict["status"],
}
return result
def WorkOrderSubmit(self, **params):
"""
Function to process work order request
Parameters:
- params is variable-length argument list containing work request
as defined in EEA spec 6.1.1
Returns jrpc response as defined in EEA spec 6.1.3
"""
wo_id = params["workOrderId"]
input_json_str = params["raw"]
input_value_json = json.loads(input_json_str)
# Work order status payload should have
# data filed with work order id as defined in EEA spec section 6.
data = {"workOrderId": wo_id}
valid, err_msg = \
Validator.schema_validation(
"WorkOrderSubmit",
input_value_json["params"])
if not valid:
raise JSONRPCDispatchException(JsonRpcErrorCode.INVALID_PARAMETER,
err_msg, data)
worker_id = input_value_json["params"]["workerId"]
# Check if workerId is exists in avalon
if not self._is_worker_exists(worker_id):
raise JSONRPCDispatchException(
JsonRpcErrorCode.INVALID_PARAMETER,
"worker {} doesn't exists".format(worker_id), data)
if ((self.workorder_count + 1) > self.max_workorder_count):
# wo_ids is a csv of work order ids retrieved from database
wo_ids = self.kv_helper.get("wo-worker-processed", worker_id)
processed_wo_ids = [] if wo_ids is None else wo_ids.split(",")
# if max count reached clear a processed entry
work_orders = self.kv_helper.lookup("wo-timestamps")
for id in work_orders:
# If work order is processed then remove from table
if id in processed_wo_ids:
self.kv_helper.csv_search_delete("wo-worker-processed",
worker_id, id)
self.kv_helper.remove("wo-requests", id)
self.kv_helper.remove("wo-responses", id)
self.kv_helper.remove("wo-receipts", id)
self.kv_helper.remove("wo-timestamps", id)
self.workorder_list.remove(id)
self.workorder_count -= 1
break
# If no work order is processed then return busy
if ((self.workorder_count + 1) > self.max_workorder_count):
raise JSONRPCDispatchException(
WorkOrderStatus.BUSY,
"Work order handler is busy updating the result", data)
if (self.kv_helper.get("wo-timestamps", wo_id) is None):
# Create a new work order entry.
# Don't change the order of table updates.
# The order is important for clean up if the TCS is restarted in
# the middle.
# Add entry to wo-worker-scheduled which holds all the work order
# id separated by comma(csv) to be processed by corresponding
# worker. i.e. - <worker_id> -> <wo_id>,<wo_id>,<wo_id>...
epoch_time = str(time.time())
# Update the tables
self.kv_helper.set("wo-timestamps", wo_id, epoch_time)
self.kv_helper.set("wo-requests", wo_id, input_json_str)
self.kv_helper.csv_append("wo-worker-scheduled", worker_id, wo_id)
# Add to the internal FIFO
self.workorder_list.append(wo_id)
self.workorder_count += 1
# ZeroMQ for sync workorder processing
try:
socket = context.socket(zmq.REQ)
socket.connect(self.zmq_url)
socket.send_string(wo_id, flags=0, encoding='utf-8')
replymessage = socket.recv()
logger.info(replymessage)
socket.disconnect(self.zmq_url)
except Exception as er:
raise JSONRPCDispatchException(
WorkOrderStatus.UNKNOWN_ERROR,
"Failed to connect with enclave-manager socket: " + er,
data)
# Work order is processed. Fetch result from wo-response table
value = self.kv_helper.get("wo-responses", wo_id)
if value:
response = json.loads(value)
if 'result' in response:
return response['result']
# response without result should have an error
# return error
err_code = response["error"]["code"]
err_msg = response["error"]["message"]
if err_code == EnclaveError.ENCLAVE_ERR_VALUE:
err_code = \
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE
elif err_code == EnclaveError.ENCLAVE_ERR_UNKNOWN:
err_code = WorkOrderStatus.UNKNOWN_ERROR
else:
err_code = WorkOrderStatus.FAILED
raise JSONRPCDispatchException(err_code, err_msg, data)
else:
# Workorder id already exists
raise JSONRPCDispatchException(
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE,
"Work order id already exists in the database. \
Hence invalid parameter", data)
def create_request(self,
work_order_id,
worker_id,
workload_id,
requester_id,
session_key,
session_iv,
requester_nonce,
verifying_key=None,
payload_format="JSON-RPC",
response_timeout_msecs=6000,
result_uri=None,
notify_uri=None,
worker_encryption_key=None,
data_encryption_algorithm=None,
encrypted_session_key=None):
"""validate and creates workorder request with received values"""
if work_order_id:
self.set_work_order_id(work_order_id)
self.set_response_timeout_msecs(response_timeout_msecs)
self.set_payload_format(payload_format)
self.set_requester_nonce(requester_nonce)
self.session_key = session_key
self.set_workload_id(workload_id)
self.set_worker_id(worker_id)
if requester_id is not None:
self.set_requester_id(requester_id)
if session_iv:
self.set_session_key_iv(
crypto_utility.byte_array_to_hex(session_iv))
if result_uri:
self.set_result_uri(result_uri)
if notify_uri:
self.set_notify_uri(notify_uri)
if worker_encryption_key:
self.set_worker_encryption_key(worker_encryption_key)
if data_encryption_algorithm:
self.set_data_encryption_algorithm(data_encryption_algorithm)
self.set_encrypted_session_key(encrypted_session_key)
code, err_msg = WOcheck.schema_validation("sdk_WorkOrderSubmit",
self.params_obj)
# When the WorkorderSubmit request fails basic Json Validation
# the init object created is deleted to avoid futhur processing
# on that object by the user.
if not code:
return util.create_error_response(
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE, 0, err_msg)
self.set_worker_encryption_key(
worker_encryption_key.encode("UTF-8").hex())
self.session_iv = session_iv
self.params_obj["encryptedRequestHash"] = ""
self.params_obj["requesterSignature"] = ""
self.params_obj["inData"] = list()
if encrypted_session_key is None:
try:
encrypted_session_key = crypto_utility.generate_encrypted_key(
session_key, worker_encryption_key)
self.set_encrypted_session_key(
crypto_utility.byte_array_to_hex(encrypted_session_key))
except Exception as err:
return util.create_error_response(
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE, 0, err)
return None
def WorkOrderSubmit(self, **params):
"""
Function to process work order request
Parameters:
- params is variable-length argument list containing work request
as defined in EEA spec 6.1.1
Returns jrpc response as defined in EEA spec 6.1.3
"""
wo_id = params["workOrderId"]
input_json_str = params["raw"]
input_value_json = json.loads(input_json_str)
# Work order status payload should have
# data filed with work order id as defined in EEA spec section 6.
data = {"workOrderId": wo_id}
valid, err_msg = \
Validator.schema_validation(
"WorkOrderSubmit",
input_value_json["params"])
if not valid:
raise JSONRPCDispatchException(JsonRpcErrorCode.INVALID_PARAMETER,
err_msg, data)
worker_id = input_value_json["params"]["workerId"]
# Check if workerId is exists in avalon
if not self._is_worker_exists(worker_id):
raise JSONRPCDispatchException(
JsonRpcErrorCode.INVALID_PARAMETER,
"worker {} doesn't exists".format(worker_id), data)
if "requesterSignature" in params:
try:
decoded_str = base64.b64decode(params["requesterSignature"],
validate=True)
except Exception:
raise JSONRPCDispatchException(
JsonRpcErrorCode.INVALID_PARAMETER,
"Invalid data format for requesterSignature", data)
if ((self.workorder_count + 1) > self.max_workorder_count):
# Lookup all workers.
workers = self.kv_helper.lookup("worker-pool")
wo_worker_map = dict()
processed_wo_ids = []
for worker in workers:
# wo_ids_csv is a csv of work order ids retrieved from database
wo_ids_csv = self.kv_helper.get("wo-worker-processed", worker)
wo_ids = [] if wo_ids_csv is None else wo_ids_csv.split(",")
# Create a reverse map for search delete in wo-worker-processed
for w in wo_ids:
wo_worker_map[w] = worker
processed_wo_ids.extend(wo_ids)
# if max count reached clear a processed entry
work_orders = self.kv_helper.lookup("wo-timestamps")
for id in work_orders:
# If work order is processed then remove from table
if id in processed_wo_ids:
self.kv_helper.csv_search_delete("wo-worker-processed",
wo_worker_map[id], id)
self.kv_helper.remove("wo-requests", id)
self.kv_helper.remove("wo-responses", id)
self.kv_helper.remove("wo-receipts", id)
self.kv_helper.remove("wo-timestamps", id)
self.workorder_list.remove(id)
logger.info(
"Purged work order {} from database".format(id))
self.workorder_count -= 1
# @TODO : Need to rethink if deleting just one entry
# suffices as the operations performed are costly
break
# If no work order is processed then return busy
if ((self.workorder_count + 1) > self.max_workorder_count):
raise JSONRPCDispatchException(
WorkOrderStatus.BUSY,
"Work order handler is busy updating the result", data)
if (self.kv_helper.get("wo-timestamps", wo_id) is None):
# Create a new work order entry.
# Don't change the order of table updation.
# The order is important for clean up if the TCS is restarted in
# the middle.
# Add entry to wo-worker-scheduled which holds all the work order
# id separated by comma(csv) to be processed by corresponding
# worker. i.e. - <worker_id> -> <wo_id>,<wo_id>,<wo_id>...
epoch_time = str(time.time())
# Update the tables
self.kv_helper.set("wo-timestamps", wo_id, epoch_time)
self.kv_helper.set("wo-requests", wo_id, input_json_str)
self.kv_helper.csv_append("wo-worker-scheduled", worker_id, wo_id)
# Add to the internal FIFO
self.workorder_list.append(wo_id)
self.workorder_count += 1
raise JSONRPCDispatchException(
WorkOrderStatus.PENDING,
"Work order is computing. Please query for WorkOrderGetResult"
+ " to view the result", data)
# Workorder id already exists
raise JSONRPCDispatchException(
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE,
"Work order id already exists in the database. " +
"Hence invalid parameter", data)
def WorkOrderGetResult(self, **params):
"""
Function to process work order get result.
This API corresponds to Trusted Compute EEA API 6.1.4
Work Order Pull Request Payload
Parameters:
- params is variable-length argument list containing work request
as defined in EEA spec 6.1.4
Returns jrpc response as defined in EEA spec 6.1.2
"""
input_json_str = params["raw"]
input_value_json = json.loads(input_json_str)
valid, err_msg = \
Validator.schema_validation(
"WorkOrderGetResult",
input_value_json["params"])
if not valid:
raise JSONRPCDispatchException(
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE, err_msg)
wo_id = params["workOrderId"]
# Work order status payload should have
# data field with work order id as defined in EEA spec section 6.
data = {"workOrderId": wo_id}
# Work order is processed if it is in wo-response table
value = self.kv_helper.get("wo-responses", wo_id)
# Work order not in 'wo-timestamps' table
if not value:
if (self.kv_helper.get("wo-timestamps", wo_id) is not None):
# work order is yet to be processed
raise JSONRPCDispatchException(
WorkOrderStatus.PENDING,
"Work order result is yet to be updated", data)
raise JSONRPCDispatchException(
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE,
"Work order Id not found in the database. " +
"Hence invalid parameter", data)
# Worker order is processed and result is avalibale
response = json.loads(value)
if 'result' in response:
return response['result']
# response without a result should have an error
err_code = response["error"]["code"]
err_msg = response["error"]["message"]
if err_code == EnclaveError.ENCLAVE_ERR_VALUE:
err_code = \
WorkOrderStatus.INVALID_PARAMETER_FORMAT_OR_VALUE
elif err_code == EnclaveError.ENCLAVE_ERR_UNKNOWN:
err_code = WorkOrderStatus.UNKNOWN_ERROR
elif err_code == EnclaveError.ENCLAVE_ERR_INVALID_WORKLOAD:
err_code = WorkOrderStatus.INVALID_WORKLOAD
else:
err_code = WorkOrderStatus.FAILED
raise JSONRPCDispatchException(err_code, err_msg, data)