def get_worker_details(self, blockchain_type,
worker_registry, worker_id):
# get the worker details for direct/proxy model.
# Retrieve worker details
jrpc_req_id = 342
worker_retrieve_result = worker_registry.worker_retrieve(
worker_id, jrpc_req_id
)
logger.info("\n Worker retrieve response: {}\n".format(
json.dumps(worker_retrieve_result, indent=4)
))
if worker_retrieve_result is None or "error" in worker_retrieve_result:
logger.error("Unable to retrieve worker details\n")
sys.exit(1)
# Initializing Worker Object
worker_obj1 = worker_details.SGXWorkerDetails()
if blockchain_type:
worker_obj1.load_worker(
json.loads(worker_retrieve_result[4]))
else:
worker_obj1.load_worker(
worker_retrieve_result["result"]["details"])
return worker_obj1
def test_generate_client_signature():
logging.info("Testing generate_client_signature...")
worker_obj = worker.SGXWorkerDetails()
worker_obj.encryption_key = worker_enc_key
encrypted_session_key = crypto_utility.generate_encrypted_key(
session_key, worker_obj.encryption_key)
encrypted_session_key_hex = \
crypto_utility.byte_array_to_hex(encrypted_session_key)
read_json = read_json_file("wo_request.json", ["./"])
wo_submit_request = json.loads(read_json)
wo_request = wo_submit_request["params"]
wo_request["workOrderId"] = work_order_id
wo_request["workerId"] = worker_id
wo_request["requesterId"] = requester_id
wo_request["sessionKeyIv"] = session_key_iv
wo_request["encryptedSessionKey"] = encrypted_session_key_hex
wo_request["requesterNonce"] = requester_nonce
try:
input_json_str = json.dumps(wo_submit_request)
input_json_str, status = sig_obj.generate_client_signature(
input_json_str, worker_obj, client_private_key, session_key,
session_iv, encrypted_session_key)
if status == SignatureStatus.PASSED:
logging.info("PASSED: generate_client_signature")
return 0
else:
logging.info("FAILED: generate_client_signature")
return 1
except Exception as err:
return 1
def get_worker_by_id(self, worker_id):
"""
Get worker instance from database
Parameters :
@param worker_id - Id of worker to be retrieved
Returns :
@returns worker_obj - A worker retrieved from kv storage
"""
json_dict = json.loads(self._kv_helper.get("workers", worker_id))
worker_obj = worker_details.SGXWorkerDetails()
worker_obj.load_worker(json_dict['details'])
return worker_obj
def _get_first_active_worker(worker_registry, worker_id, config):
"""
This function looks up all the workers registered. It then filters
this data to get the first worker that has Active status.
"""
jrpc_req_id = random.randint(0, 100000)
worker_retrieve_result = None
if not worker_id:
# Lookup all worker id present on blockchain
worker_lookup_result = worker_registry.worker_lookup(
WorkerType.TEE_SGX,
config["WorkerConfig"]["OrganizationId"],
config["WorkerConfig"]["ApplicationTypeId"],
jrpc_req_id
)
logger.info("\n Worker lookup response: {}\n".format(
json.dumps(worker_lookup_result, indent=4)
))
if "result" in worker_lookup_result and \
"ids" in worker_lookup_result["result"].keys():
if worker_lookup_result["result"]["totalCount"] != 0:
worker_ids = worker_lookup_result["result"]["ids"]
# Filter workers by status(active) field
# Return first worker whose status is active
for w_id in worker_ids:
jrpc_req_id += 1
worker = worker_registry\
.worker_retrieve(w_id, jrpc_req_id)
if worker["result"]["status"] == WorkerStatus.ACTIVE.value:
worker_retrieve_result = worker
worker_id = w_id
break
else:
logger.error("No workers found")
else:
logger.error("Failed to lookup worker")
else:
worker_retrieve_result = worker_registry\
.worker_retrieve(worker_id, jrpc_req_id)
if worker_retrieve_result is None:
logger.error("Failed to lookup worker")
return None, None
# Initializing Worker Object
logger.info("\n Worker retrieve response: {}\n".format(
json.dumps(worker_retrieve_result, indent=4)
))
worker_obj = worker_details.SGXWorkerDetails()
worker_obj.load_worker(worker_retrieve_result["result"]["details"])
return worker_obj, worker_id
def configure_data(
self, input_json, worker_obj, pre_test_response):
worker_obj = worker.SGXWorkerDetails()
if input_json is not None:
self.add_json_values(
input_json, worker_obj, self.tamper)
self.set_worker_id(
crypto_utils.strip_begin_end_public_key
(pre_test_response["result"]["ids"][0]))
input_worker_retrieve = json.loads(self.to_string())
logger.info('*****Worker details Updated with Worker ID***** \
\n%s\n', input_worker_retrieve)
return input_worker_retrieve
def worker_retrieve(self, lookup_response):
worker_obj = worker.SGXWorkerDetails()
retrieve_obj = WorkerRetrieve()
worker_id = retrieve_obj.configure_data_sdk(
input_json=None,
worker_obj=None,
pre_test_response=lookup_response)
logger.info(
'*****Worker details Updated with Worker ID***** \
\n%s\n', worker_id)
worker_obj = submit_retrieve_sdk(worker_id)
# worker_obj.load_worker(retrieve_response['result']['details'])
# if constants.proxy_mode and \
# globals.blockchain_type == "ethereum":
return worker_obj
def worker_retrieve(self, lookup_response):
worker_obj = worker.SGXWorkerDetails()
retrieve_obj = WorkerRetrieve()
input_worker_retrieve = retrieve_obj.configure_data(
input_json=None, worker_obj=None,
pre_test_response=lookup_response)
logger.info('*****Worker details Updated with Worker ID***** \
\n%s\n', input_worker_retrieve)
retrieve_response = submit_request_listener(
globals.uri_client, input_worker_retrieve,
constants.worker_retrieve_output_json_file_name)
logger.info('*****Worker retrieve response***** \
\n%s\n', retrieve_response)
worker_obj.load_worker(retrieve_response['result']['details'])
return worker_obj
def _get_first_active_worker(worker_registry, worker_id):
"""
This function looks up all the workers registered. It then filters
this data to get the first worker that has Active status.
"""
jrpc_req_id = random.randint(0, 100000)
worker_retrieve_result = None
if not worker_id:
# Lookup all worker id present on blockchain
worker_lookup_result = worker_registry.worker_lookup(
WorkerType.TEE_SGX,
'aabbcc1234ddeeff',
'11aa22bb33cc44dd',
jrpc_req_id
)
logger.info(worker_lookup_result)
if "result" in worker_lookup_result and \
"ids" in worker_lookup_result["result"].keys():
if worker_lookup_result["result"]["totalCount"] != 0:
worker_ids = worker_lookup_result["result"]["ids"]
# Filter workers by status(active) field
# Return first worker whose status is active
for w_id in worker_ids:
jrpc_req_id += 1
worker = worker_registry\
.worker_retrieve(w_id, jrpc_req_id)
if worker["result"]["status"] == WorkerStatus.ACTIVE.value:
worker_retrieve_result = worker
worker_id = w_id
break
else:
logger.error("No workers found")
else:
logger.error("Failed to lookup worker")
else:
worker_retrieve_result = worker_registry\
.worker_retrieve(worker_id, jrpc_req_id)
if worker_retrieve_result is None:
logger.error("Failed to lookup worker")
return None, None
# Initializing Worker Object
worker_obj = worker_details.SGXWorkerDetails()
worker_obj.load_worker(worker_retrieve_result["result"]["details"])
return worker_obj, worker_id
def _get_first_active_worker(worker_registry, worker_id, config):
"""
This function looks up all the workers registered. It then filters
this data to get the first worker that has Active status.
"""
jrpc_req_id = random.randint(0, 100000)
worker_retrieve_result = None
if not worker_id:
# Lookup all worker id present on blockchain
# return tuple (workers count, lookup tag and list of
# workers)
count, _, worker_ids = worker_registry.worker_lookup(
WorkerType.TEE_SGX, config["WorkerConfig"]["OrganizationId"],
config["WorkerConfig"]["ApplicationTypeId"], jrpc_req_id)
logger.info("\n Worker lookup response: count {}"
"\n Workers {}\n".format(count, worker_ids))
if count > 0:
# Filter workers by status(active) field
# Return first worker whose status is active
for w_id in worker_ids:
jrpc_req_id += 1
# worker retrieve return tuple of worker
# status, worker type, Org id, app type ids
# details of worker.
w_status, _, _, _, w_details = worker_registry.worker_retrieve(
w_id, jrpc_req_id)
if w_status == WorkerStatus.ACTIVE.value:
worker_id = w_id
break
else:
logger.error("No workers found")
else:
w_status, _, _, _, w_details = worker_registry.worker_retrieve(
worker_id, jrpc_req_id)
if w_status != WorkerStatus.ACTIVE.value:
logger.info("Worker {} is not active".format(worker_id))
sys.exit(-1)
# Initializing Worker Object
logger.info("\n Worker retrieve response: {} {}\n".format(
worker_id, w_details))
worker_obj = worker_details.SGXWorkerDetails()
worker_obj.load_worker(json.loads(w_details))
return worker_obj, worker_id
def submit_retrieve_sdk(worker_id, input_json=None):
logger.info("SDK code path\n")
worker_obj = worker_details.SGXWorkerDetails()
if input_json is None:
jrpc_req_id = 11
else:
jrpc_req_id = input_json["id"]
config = pconfig.parse_configuration_files(constants.conffiles,
constants.confpaths)
logger.info(" URI client %s \n", config["tcf"]["json_rpc_uri"])
worker_registry = _create_worker_registry_instance(globals.blockchain_type,
config)
if constants.proxy_mode and globals.blockchain_type == 'ethereum':
for w_id in worker_id:
worker = worker_registry\
.worker_retrieve(w_id, jrpc_req_id)
if worker["result"]["status"] == \
WorkerStatus.ACTIVE.value:
worker_retrieve_result = worker
worker_id = w_id
break
logger.info("\n Worker ID\n%s\n", worker_id)
logger.info("\n Worker retrieve response: {}\n".format(
json.dumps(worker_retrieve_result, indent=4)))
else:
worker_retrieve_result = worker_registry.worker_retrieve(
worker_id, jrpc_req_id)
logger.info("\n Worker retrieve response: {}\n".format(
json.dumps(worker_retrieve_result, indent=4)))
if "error" in worker_retrieve_result:
logger.error("Unable to retrieve worker details\n")
return worker_retrieve_result
if constants.proxy_mode and globals.blockchain_type == 'fabric':
worker_obj.load_worker(json.loads(worker_retrieve_result[4]))
else:
worker_obj.load_worker(worker_retrieve_result['result']['details'])
worker_obj.worker_id = worker_id
logger.info("\n Worker ID\n%s\n", worker_obj.worker_id)
if input_json is None:
return worker_obj
else:
return worker_retrieve_result
def submit_update_sdk(update_params, input_json=None):
logger.info("SDK code path\n")
logger.info("Worker update params %s \n", update_params)
worker_obj = worker_details.SGXWorkerDetails()
# update_params = json.loads(update_params)
if input_json is None:
jrpc_req_id = 11
else:
jrpc_req_id = input_json["id"]
config = pconfig.parse_configuration_files(constants.conffiles,
constants.confpaths)
logger.info(" URI client %s \n", config["tcf"]["json_rpc_uri"])
worker_registry = _create_worker_registry_instance(globals.blockchain_type,
config)
if not constants.proxy_mode:
worker_update_result = worker_registry.worker_update(
eval(str(update_params))["worker_id"],
eval(str(update_params))["details"], jrpc_req_id)
logger.info("\n Worker update response: {}\n".format(
json.dumps(worker_update_result, indent=4)))
return worker_update_result
def get_worker_by_id(self, worker_id):
"""
Get worker instance from database
Parameters :
@param worker_id - Id of worker to be retrieved
Returns :
@returns worker_obj - A worker retrieved from kv storage
"""
worker_from_kv = None
# Retry infinitely if worker is not found in KV storage
while True:
worker_from_kv = self._kv_helper.get("workers", worker_id)
if worker_from_kv is None:
logger.warn(
"Could not find worker in database. Will retry in 10s")
time.sleep(10)
else:
break
json_dict = json.loads(worker_from_kv)
worker_obj = worker_details.SGXWorkerDetails()
worker_obj.load_worker(json_dict['details'])
return worker_obj
def Main(args=None):
options = _parse_command_line(args)
config = _parse_config_file(options.config)
if config is None:
logger.error("\n Error in parsing config file: {}\n".format(
options.config))
sys.exit(-1)
# mode should be one of listing or registry (default)
mode = options.mode
# Http JSON RPC listener uri
uri = options.uri
if uri:
config["tcf"]["json_rpc_uri"] = uri
# Address of smart contract
address = options.address
if address:
if mode == "listing":
config["ethereum"]["direct_registry_contract_address"] = \
address
elif mode == "registry":
logger.error(
"\n Only Worker registry listing address is supported." +
"Worker registry address is unsupported \n")
sys.exit(-1)
# worker id
worker_id = options.worker_id
worker_id_hex = options.worker_id_hex
worker_id = worker_id_hex if not worker_id \
else hex_utils.get_worker_id_from_name(worker_id)
# work load id of worker
workload_id = options.workload_id
if not workload_id:
logger.error("\nWorkload id is mandatory\n")
sys.exit(-1)
# work order input data
in_data = options.in_data
# Option to send input data in plain text
in_data_plain_text = options.in_data_plain
# show receipt in output
show_receipt = options.receipt
# show decrypted result in output
show_decrypted_output = options.decrypted_output
# requester signature for work order requests
requester_signature = options.requester_signature
# setup logging
config["Logging"] = {"LogFile": "__screen__", "LogLevel": "INFO"}
plogger.setup_loggers(config.get("Logging", {}))
sys.stdout = plogger.stream_to_logger(logging.getLogger("STDOUT"),
logging.DEBUG)
sys.stderr = plogger.stream_to_logger(logging.getLogger("STDERR"),
logging.WARN)
logger.info("******* Hyperledger Avalon Generic client *******")
if mode == "registry" and address:
logger.error("\n Worker registry contract address is unsupported \n")
sys.exit(-1)
# Retrieve JSON RPC uri from registry list
if not uri and mode == "listing":
uri = _retrieve_uri_from_registry_list(config)
if uri is None:
logger.error("\n Unable to get http JSON RPC uri \n")
sys.exit(-1)
# Prepare worker
# JRPC request id. Choose any integer value
jrpc_req_id = 31
worker_registry = JRPCWorkerRegistryImpl(config)
if not worker_id:
# Get first worker from worker registry
worker_id = _lookup_first_worker(worker_registry, jrpc_req_id)
if worker_id is None:
logger.error("\n Unable to get worker \n")
sys.exit(-1)
# Retrieve worker details
jrpc_req_id += 1
worker_retrieve_result = worker_registry.worker_retrieve(
worker_id, jrpc_req_id)
logger.info("\n Worker retrieve response: {}\n".format(
json.dumps(worker_retrieve_result, indent=4)))
if "error" in worker_retrieve_result:
logger.error("Unable to retrieve worker details\n")
sys.exit(1)
# Create session key and iv to sign work order request
session_key = crypto_utility.generate_key()
session_iv = crypto_utility.generate_iv()
# Initializing Worker Object
worker_obj = worker_details.SGXWorkerDetails()
worker_obj.load_worker(worker_retrieve_result['result']['details'])
# Do worker verification
_do_worker_verification(worker_obj)
logger.info(
"**********Worker details Updated with Worker ID" + "*********\n%s\n",
worker_id)
# Create work order
if in_data_plain_text:
# As per TC spec, if encryptedDataEncryptionKey is "-" then
# input data is not encrypted
encrypted_data_encryption_key = "-"
else:
# As per TC spec, if encryptedDataEncryptionKey is not
# provided then set it to None which means
# use default session key to encrypt input data
encrypted_data_encryption_key = None
wo_params = _create_work_order_params(worker_id, workload_id, in_data,
worker_obj.encryption_key,
session_key, session_iv,
encrypted_data_encryption_key)
client_private_key = crypto_utility.generate_signing_keys()
if requester_signature:
# Add requester signature and requester verifying_key
if wo_params.add_requester_signature(client_private_key) is False:
logger.info("Work order request signing failed")
exit(1)
# Submit work order
logger.info("Work order submit request : %s, \n \n ",
wo_params.to_jrpc_string(jrpc_req_id))
work_order = JRPCWorkOrderImpl(config)
jrpc_req_id += 1
response = work_order.work_order_submit(wo_params.get_work_order_id(),
wo_params.get_worker_id(),
wo_params.get_requester_id(),
wo_params.to_string(),
id=jrpc_req_id)
logger.info("Work order submit response : {}\n ".format(
json.dumps(response, indent=4)))
if "error" in response and response["error"]["code"] != \
WorkOrderStatus.PENDING:
sys.exit(1)
# Create receipt
wo_receipt = JRPCWorkOrderReceiptImpl(config)
if show_receipt:
jrpc_req_id += 1
_create_work_order_receipt(wo_receipt, wo_params, client_private_key,
jrpc_req_id)
# Retrieve work order result
jrpc_req_id += 1
res = work_order.work_order_get_result(wo_params.get_work_order_id(),
jrpc_req_id)
logger.info("Work order get result : {}\n ".format(
json.dumps(res, indent=4)))
# Check if result field is present in work order response
if "result" in res:
# Verify work order response signature
if _verify_wo_res_signature(res['result'], worker_obj.verification_key,
wo_params.get_requester_nonce()) is False:
logger.error("Work order response signature verification Failed")
sys.exit(1)
# Decrypt work order response
if show_decrypted_output:
decrypted_res = crypto_utility.decrypted_response(
res['result'], session_key, session_iv)
logger.info("\nDecrypted response:\n {}".format(decrypted_res))
else:
logger.error("\n Work order get result failed {}\n".format(res))
sys.exit(1)
if show_receipt:
# Retrieve receipt
jrpc_req_id += 1
retrieve_wo_receipt \
= _retrieve_work_order_receipt(wo_receipt,
wo_params, jrpc_req_id)
# Verify receipt signature
if "result" in retrieve_wo_receipt:
if _verify_receipt_signature(retrieve_wo_receipt) is False:
logger.error("Receipt signature verification Failed")
sys.exit(1)
else:
logger.info("Work Order receipt retrieve failed")
sys.exit(1)
def ParseCommandLine(args):
global worker_obj
global worker_id
global message
global config
global off_chain
global requester_signature
global input_data_hash
parser = argparse.ArgumentParser()
use_service = parser.add_mutually_exclusive_group()
parser.add_argument(
"-c",
"--config",
help="the config file containing the Ethereum contract information",
type=str)
use_service.add_argument("-r",
"--registry-list",
help="the Ethereum address of the registry list",
type=str)
use_service.add_argument("-s",
"--service-uri",
help="skip URI lookup and send to specified URI",
type=str)
use_service.add_argument(
"-o",
"--off-chain",
help="skip URI lookup and use the registry in the config file",
action="store_true")
parser.add_argument(
"-w",
"--worker-id",
help="skip worker lookup and retrieve specified worker",
type=str)
parser.add_argument(
"-m",
"--message",
help='text message to be included in the JSON request payload',
type=str)
parser.add_argument(
"-rs",
"--requester-signature",
help="Enable requester signature for work order requests",
action="store_true")
parser.add_argument("-dh",
"--data-hash",
help="Enable input data hash for work order requests",
action="store_true")
options = parser.parse_args(args)
if options.config:
conf_files = [options.config]
else:
conf_files = [TCFHOME + "/sdk/avalon_sdk/tcf_connector.toml"]
confpaths = ["."]
try:
config = pconfig.parse_configuration_files(conf_files, confpaths)
config_json_str = json.dumps(config)
except pconfig.ConfigurationException as e:
logger.error(str(e))
sys.exit(-1)
global direct_jrpc
direct_jrpc = AvalonDirectClient(conf_files[0])
# Whether or not to connect to the registry list on the blockchain
off_chain = False
if options.registry_list:
config["ethereum"]["direct_registry_contract_address"] = \
options.registry_list
if options.service_uri:
config["tcf"]["json_rpc_uri"] = options.service_uri
off_chain = True
if options.off_chain:
off_chain = True
requester_signature = options.requester_signature
input_data_hash = options.data_hash
worker_id = options.worker_id
message = options.message
if options.message is None or options.message == "":
message = "Test Message"
# Initializing Worker Object
worker_obj = worker.SGXWorkerDetails()
def parse_command_line(args):
"""Setup and parse command line arguments and help information."""
global worker_obj
global worker_id
global verbose
global config
global off_chain
global requester_signature
parser = argparse.ArgumentParser()
use_service = parser.add_mutually_exclusive_group()
parser.add_argument("-c",
"--config",
help="the config file containing the" +
" Ethereum contract information",
type=str)
use_service.add_argument("-r",
"--registry-list",
help="the Ethereum address of the registry list",
type=str)
use_service.add_argument("-s",
"--service-uri",
help="skip URI lookup and send to specified URI",
type=str)
use_service.add_argument(
"-o",
"--off-chain",
help="skip URI lookup and use the registry in the config file",
action="store_true")
parser.add_argument(
"-w",
"--worker-id",
help="skip worker lookup and retrieve specified worker",
type=str)
parser.add_argument("-v",
"--verbose",
help="increase output verbosity",
action="store_true")
parser.add_argument(
"-rs",
"--requester_signature",
help="Enable requester signature for work order requests",
action="store_true")
options = parser.parse_args(args)
if options.config:
conf_files = [options.config]
else:
conf_files = [TCFHOME + "/sdk/avalon_sdk/tcf_connector.toml"]
conf_paths = ["."]
try:
config = pconfig.parse_configuration_files(conf_files, conf_paths)
json.dumps(config, indent=4)
except pconfig.ConfigurationException as e:
logger.error(str(e))
sys.exit(-1)
global direct_jrpc
direct_jrpc = AvalonDirectClient(conf_files[0])
# Whether or not to connect to the registry list on the blockchain
off_chain = False
if options.registry_list:
config["ethereum"]["direct_registry_contract_address"] = \
options.registry_list
if options.service_uri:
service_uri = options.service_uri
off_chain = True
if options.off_chain:
service_uri = config["tcf"].get("json_rpc_uri")
off_chain = True
requester_signature = options.requester_signature
verbose = options.verbose
worker_id = options.worker_id
# Initializing Worker Object
worker_obj = worker.SGXWorkerDetails()
def parse_command_line(config, args):
LOGGER.info("***************** AVALON *****************")
global input_json_str
global input_json_dir
global server_uri
global output_json_file_name
global consensus_file_name
global sig_obj
global worker_obj
global private_key
global encrypted_session_key
global session_iv
parser = argparse.ArgumentParser()
parser.add_argument("--logfile", help="Name of the log file. " +
"Use __screen__ for standard output", type=str)
parser.add_argument("-p", "--private_key",
help="Private Key of the Client",
type=str, default=None)
parser.add_argument("--loglevel", help="Logging level", type=str)
parser.add_argument("-i", "--input_file", help="JSON input file name",
type=str, default="input.json")
parser.add_argument("--input_dir", help="Logging level", type=str,
default=[])
parser.add_argument("-c", "--connect_uri",
help="URI to send requests to", type=str, default=[])
parser.add_argument(
"output_file",
help="JSON output file name",
type=str,
default="output.json",
nargs="?")
options = parser.parse_args(args)
if config.get("Logging") is None:
config["Logging"] = {
"LogFile": "__screen__",
"LogLevel": "INFO"
}
if options.logfile:
config["Logging"]["LogFile"] = options.logfile
if options.loglevel:
config["Logging"]["LogLevel"] = options.loglevel.upper()
input_json_str = None
input_json_dir = None
if options.connect_uri:
server_uri = options.connect_uri
else:
LOGGER.error("ERROR: Please enter the server URI")
if options.input_dir:
LOGGER.info("Load Json Directory from %s", options.input_dir)
input_json_dir = options.input_dir
elif options.input_file:
try:
LOGGER.info("load JSON input from %s", options.input_file)
with open(options.input_file, "r") as file:
input_json_str = file.read()
except Exception as err:
LOGGER.error("ERROR: Failed to read from file %s: %s",
options.input_file, str(err))
else:
LOGGER.info("No input found")
if options.output_file:
output_json_file_name = options.output_file
else:
output_json_file_name = None
if options.private_key:
private_key = options.private_key
else:
# Generating the private Key for the client
private_key = enclave_helper.generate_signing_keys()
# Initializing Signature object, Worker Object
sig_obj = signature.ClientSignature()
worker_obj = worker.SGXWorkerDetails()
