def __init__(self, config_file): super(TestFabricWorkerRegistryImpl, self).__init__() if not path.isfile(config_file): raise FileNotFoundError("File not found at path: {0}".format( path.realpath(config_file))) try: with open(config_file) as fd: self.__config = toml.load(fd) except IOError as e: if e.errno != errno.ENOENT: raise Exception("Could not open config file: %s", e) self.__eth_conn = FabricWorkerRegistryImpl(self.__config)
def main(args=None): parser = argparse.ArgumentParser() parser.add_argument( "-c", "--config", help="The config file containing the Fabric network information", type=str) parser.add_argument( "-u", "--uri", help="Direct API listener endpoint", default="http://avalon-listener:1947", type=str) options = parser.parse_args(args) config = _parse_config_file(options.config) if config is None: logging.error("\n Error in parsing config file: {}\n".format( options.config )) sys.exit(-1) # Http JSON RPC listener uri uri = options.uri if uri: config["tcf"]["json_rpc_uri"] = uri fabric_worker = FabricWorkerRegistryImpl(config) fabric_work_order = FabricWorkOrderImpl(config) nest_asyncio.apply() fabric_connector_svc = FabricConnector( config, None, fabric_worker, fabric_work_order, None) fabric_connector_svc.start()
def create_worker_registry_instance(self, blockchain_type, config): # create worker registry instance for direct/proxy model if blockchain_type == 'fabric': return FabricWorkerRegistryImpl(config) elif blockchain_type == 'ethereum': return EthereumWorkerRegistryImpl(config) else: return JRPCWorkerRegistryImpl(config)
def _create_worker_registry_instance(blockchain_type, config): # create worker registry instance for direct/proxy model if constants.proxy_mode and blockchain_type == 'fabric': return FabricWorkerRegistryImpl(config) elif constants.proxy_mode and blockchain_type == 'ethereum': return EthereumWorkerRegistryImpl(config) else: logger.info("Direct SDK code path\n") return JRPCWorkerRegistryImpl(config)
def __init__(self, config, blockchain_type): super().__init__() self._config = config if blockchain_type.lower() == 'fabric': self._worker_instance = FabricWorkerRegistryImpl(self._config) self._work_order_instance = FabricWorkOrderImpl(self._config) elif blockchain_type.lower() == 'ethereum': self._worker_instance = EthereumWorkerRegistryImpl(self._config) self._work_order_instance = EthereumWorkOrderProxyImpl( self._config) else: logging.error("Invalid blockchain type")
def __init__(self, listener_url): tcf_home = environ.get("TCF_HOME", "../../../") config_file = tcf_home + "/sdk/avalon_sdk/tcf_connector.toml" if not path.isfile(config_file): raise FileNotFoundError("File not found at path: {0}".format( path.realpath(config_file))) try: with open(config_file) as fd: self.__config = toml.load(fd) except IOError as e: if e.errno != errno.ENOENT: raise Exception("Could not open config file: %s", e) self.__config['tcf']['json_rpc_uri'] = listener_url self.__fabric_worker = FabricWorkerRegistryImpl(self.__config) self.__fabric_work_order = FabricWorkOrderImpl(self.__config) self.__jrpc_worker = JRPCWorkerRegistryImpl(self.__config) self.__jrpc_work_order = JRPCWorkOrderImpl(self.__config) # List of active available worker ids in Avalon self.__worker_ids = [] # Wait time in sec self.WAIT_TIME = 31536000 nest_asyncio.apply()
class TestFabricWorkerRegistryImpl(unittest.TestCase): def __init__(self, config_file): super(TestFabricWorkerRegistryImpl, self).__init__() if not path.isfile(config_file): raise FileNotFoundError("File not found at path: {0}".format( path.realpath(config_file))) try: with open(config_file) as fd: self.__config = toml.load(fd) except IOError as e: if e.errno != errno.ENOENT: raise Exception("Could not open config file: %s", e) self.__eth_conn = FabricWorkerRegistryImpl(self.__config) def test_worker_register(self): self.__worker_id = urandom(32) self.__worker_type = WorkerType.TEE_SGX self.__details = json.dumps({ "workOrderSyncUri": "http://worker-order:8008".encode("utf-8").hex() }) self.__org_id = urandom(32) self.__application_ids = [urandom(32), urandom(32)] logging.info( "Calling worker_register contract..\n worker_id: %s\n " + "worker_type: %d\n " + "orgId: %s\n applicationIds %s\n details %s", hex_to_utf8(self.__worker_id), self.__worker_type.value, hex_to_utf8(self.__org_id), pretty_ids(self.__application_ids), self.__details) result = self.__eth_conn.worker_register(self.__worker_id, self.__worker_type, self.__org_id, self.__application_ids, self.__details) logging.info("worker_register status \n %s", result) self.assertIsNotNone(result, "transaction execution failed") def test_worker_set_status(self): self.__status = WorkerStatus.DECOMMISSIONED logging.info( "Calling worker_set_status..\n worker_id: %s\n status: %d", hex_to_utf8(self.__worker_id), self.__status.value) result = self.__eth_conn.worker_set_status(self.__worker_id, self.__status) logging.info("worker_set_status status \n%s", result) self.assertIsNotNone(result, "worker set status response not matched") def test_worker_update(self): self.__new_details = json.dumps({ "workOrderSyncUri": "http://worker-order:8008".encode("utf-8").hex(), "workOrderNotifyUri": "http://worker-order-notify:9909".encode("utf-8").hex() }) logging.info("Calling worker_update..\n worker_id: %s\n details: %s", hex_to_utf8(self.__worker_id), self.__new_details) result = self.__eth_conn.worker_update(self.__worker_id, self.__new_details) logging.info("worker_update status \n %s", result) self.assertIsNotNone(result, "worker update response not matched") def test_worker_lookup(self): logging.info( "Calling worker_lookup..\n worker_type: %d\n orgId: %s\n " + "applicationId: %s", self.__worker_type.value, hex_to_utf8(self.__org_id), hex_to_utf8(self.__application_ids[0])) result = self.__eth_conn.worker_lookup( self.__worker_type, '419c007ce1f6ecb2e52a830b03a0c8be36438b94c950d9cf2aeb48b0f99a8276', '7b1d714fc499ddc59dde26683a0bf928848801087dc5f0372c340c120848ed7b') logging.info("worker_lookup result {} {}".format(result, type(result))) logging.info("worker_lookup status {} {} {}".format( result[0], result[1], result[2])) match = byte_array_to_hex_str(self.__worker_id) in result[2] self.assertEqual(result[0], 1, "Worker lookup response count doesn't match") self.assertTrue(match, "Worker lookup response worker id doesn't match") def test_worker_retrieve(self): logging.info("Calling worker_retrieve..\n worker_id: %s", hex_to_utf8(self.__worker_id)) result = self.__eth_conn.worker_retrieve(self.__worker_id) logging.info("worker_retrieve status [%d, %d, %s, %s, %s]", result[0], result[1], result[2], result[3], result[4]) self.assertEqual( result[0], self.__status.value, "Worker retrieve response worker status doesn't match") self.assertEqual(result[1], self.__worker_type.value, "Worker retrieve response worker type doesn't match") self.assertEqual(result[2], self.__org_id, "Worker retrieve response org id doesn't match") self.assertEqual( result[3][0], self.__application_ids[0], "Worker retrieve response application id[0] doesn't match") self.assertEqual( result[3][0], self.__application_ids[0], "Worker retrieve response application id[1] doesn't match") self.assertEqual( result[4], self.__new_details, "Worker retrieve response worker details doesn't match") def test_worker_lookup_next(self): lookUpTag = "" logging.info( "Calling worker_lookup_next..\n worker_type: %d\n" + "orgId: %s\n applicationId:%s\n lookUpTag: %s", self.__worker_type.value, hex_to_utf8(self.__org_id), hex_to_utf8(self.__application_ids[0]), lookUpTag) result = self.__eth_conn.worker_lookup_next(self.__worker_type, self.__org_id, self.__application_ids[0], lookUpTag) logging.info("worker_lookup_next status [%d, %s, %s]", result[0], result[1], pretty_ids(result[2])) self.assertEqual(result[0], 0, "worker_lookup_next response count doesn't match")
class FabricConnector(): """ Fabric blockchain connector """ def __init__(self, listener_url): tcf_home = environ.get("TCF_HOME", "../../../") config_file = tcf_home + "/sdk/avalon_sdk/tcf_connector.toml" if not path.isfile(config_file): raise FileNotFoundError("File not found at path: {0}".format( path.realpath(config_file))) try: with open(config_file) as fd: self.__config = toml.load(fd) except IOError as e: if e.errno != errno.ENOENT: raise Exception("Could not open config file: %s", e) self.__config['tcf']['json_rpc_uri'] = listener_url self.__fabric_worker = FabricWorkerRegistryImpl(self.__config) self.__fabric_work_order = FabricWorkOrderImpl(self.__config) self.__jrpc_worker = JRPCWorkerRegistryImpl(self.__config) self.__jrpc_work_order = JRPCWorkOrderImpl(self.__config) # List of active available worker ids in Avalon self.__worker_ids = [] # Wait time in sec self.WAIT_TIME = 31536000 nest_asyncio.apply() def start(self): self.sync_worker() loop = asyncio.get_event_loop() tasks = self.get_work_order_event_handler_tasks() loop.run_until_complete( asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)) loop.close() def sync_worker(self): """ Check for existing worker and update worker to fabric blockchain """ # Get all TEE Intel SGX based workers ids from the Fabric blockchain worker_ids_onchain = self._lookup_workers_onchain() # Get all Intel SGX TEE based worker ids from shared kv self.__worker_ids = self._lookup_workers_in_kv_storage() # If worker id exists in shared kv then update details of # worker to with details field. # otherwise add worker to blockchain # Update all worker which are not in shared kv and # present in blockchain to Decommissioned status self._add_update_worker_to_chain(worker_ids_onchain, self.__worker_ids) def get_work_order_event_handler_tasks(self): """ Sync work order with blockchain 1. listen to work order submit event 2. Submit work order request to listener 3. Wait for a work order result 4. Update work order result to fabric """ event_handler = self.__fabric_work_order.\ get_work_order_submitted_event_handler( self.workorder_event_handler_func ) if event_handler: tasks = [ event_handler.start_event_handling(), event_handler.stop_event_handling(int(self.WAIT_TIME)) ] return tasks else: logging.info("get work order submitted event handler failed") return None def workorder_event_handler_func(self, event, block_num, txn_id, status): logging.info("Event payload: {}\n Block number: {}\n" "Transaction id: {}\n Status {}".format( event, block_num, txn_id, status)) jrpc_req_id = 301 # Add workorder id to work order list payload_string = event['payload'].decode("utf-8") work_order_req = json.loads(payload_string) work_order_id = work_order_req['workOrderId'] # Submit the work order to listener if worker id from the event # matches with available worker ids if work_order_req['workerId'] in self.__worker_ids: logging.info("Submitting to work order to listener") response = self.__jrpc_work_order.work_order_submit( work_order_req['workOrderId'], work_order_req['workerId'], work_order_req['requesterId'], work_order_req["workOrderRequest"], id=jrpc_req_id) logging.info("Work order submit response {}".format(response)) if response and 'error' in response and \ response['error']['code'] == \ WorkOrderStatus.PENDING.value: # get the work order result jrpc_req_id += 1 work_order_result = \ self.__jrpc_work_order.work_order_get_result( work_order_req['workOrderId'], jrpc_req_id ) logging.info( "Work order get result {}".format(work_order_result)) # With Synchronous work order processing work order submit # return result elif response and 'result' in response: work_order_result = response else: logging.info("work_order_submit is failed") work_order_result = None if work_order_result: logging.info("Commit work order result to blockchain") # call to chain code to store result to blockchain status = self.__fabric_work_order.work_order_complete( work_order_id, json.dumps(work_order_result)) if status == ContractResponse.SUCCESS: # remove the entry from work order list logging.info( "Chaincode invoke call work_order_complete success") else: logging.info( "Chaincode invoke call work_order_complete failed") def _lookup_workers_in_kv_storage(self): """ Retrieves the worker ids from shared kv using worker_lookup direct API. Returns list of worker ids """ jrpc_req_id = random.randint(0, 100000) worker_lookup_result = self.__jrpc_worker.worker_lookup( worker_type=WorkerType.TEE_SGX, id=jrpc_req_id) logging.info("\nWorker lookup response from kv storage : {}\n".format( json.dumps(worker_lookup_result, indent=4))) if "result" in worker_lookup_result and \ "ids" in worker_lookup_result["result"].keys(): if worker_lookup_result["result"]["totalCount"] != 0: return worker_lookup_result["result"]["ids"] else: logging.error("No workers found in kv storage") else: logging.error("Failed to lookup worker in kv storage") return [] def _retrieve_worker_details_from_kv_storage(self, worker_id): """ Retrieve worker details from shared kv using direct json rpc API Returns the worker details in json string format """ jrpc_req_id = random.randint(0, 100000) worker_info = self.__jrpc_worker.worker_retrieve( worker_id, jrpc_req_id) logging.info("Worker retrieve response from kv storage: {}".format( json.dumps(worker_info, indent=4))) if "error" in worker_info: logging.error("Unable to retrieve worker details from kv storage") return "" else: return worker_info["result"] def _lookup_workers_onchain(self): """ Lookup all workers on chain to sync up with kv storage Return list of worker ids """ worker_lookup_result = self.__fabric_worker.worker_lookup( worker_type=WorkerType.TEE_SGX) logging.info("Worker lookup response from blockchain: {}\n".format( json.dumps(worker_lookup_result, indent=4))) if worker_lookup_result and worker_lookup_result[0] > 0: return worker_lookup_result[2] else: logging.info("No workers found in fabric blockchain") return [] def _add_update_worker_to_chain(self, wids_onchain, wids_kv): """ This function adds/updates a worker in the fabric blockchain """ for wid in wids_kv: worker_info = self._retrieve_worker_details_from_kv_storage(wid) worker_id = wid worker_type = WorkerType(worker_info["workerType"]) org_id = worker_info["organizationId"] app_type_id = worker_info["applicationTypeId"] details = json.dumps(worker_info["details"]) result = None if wid in wids_onchain: logging.info( "Updating worker {} on fabric blockchain".format(wid)) result = self.__fabric_worker.worker_update(worker_id, details) else: logging.info( "Adding new worker {} to fabric blockchain".format(wid)) result = self.__fabric_worker.worker_register( worker_id, worker_type, org_id, [app_type_id], details) if result != ContractResponse.SUCCESS: logging.error("Error while adding/updating worker to fabric" + " blockchain") for wid in wids_onchain: # Mark all stale workers on blockchain as decommissioned if wid not in wids_kv: worker = self.__fabric_worker.worker_retrieve(wid) # worker_retrieve returns tuple and first element # denotes status of worker. worker_status_onchain = worker[0] # If worker is not already decommissioned, # mark it decommissioned # as it is no longer available in the kv storage if worker_status_onchain != WorkerStatus.DECOMMISSIONED.value: update_status = self.__fabric_worker.worker_set_status( wid, WorkerStatus.DECOMMISSIONED) if update_status == ContractResponse.SUCCESS: logging.info("Marked worker " + wid + " as decommissioned on" + " fabric blockchain") else: logging.info("Update worker " + wid + " is failed")