def handle(self): # Receive and interpret the request data data = bytearray() """ Discard heading for now data += self.request.recv(16) # Interpret the header for file size file_size = struct.unpack(">Q", data[8:16])[0] """ try: c = True while c != b"": c = self.request.recv(2048) data += c # Extract byte header 3 bytes image_name_length = int(data[0:3].decode('UTF-8')) tcr = image_name_length + 3 image_name_string = data[3:tcr].decode('UTF-8') # Then, push data messaging system. MessagesQueue.push_to_queue(image_name_string, data[tcr:]) except: from harmonicIO.general.services import Services Services.e_print("Insufficient memory for storing g object.")
def __init__(self, server_addr, server_port, token="None", std_idle_time=0, max_try=9, source_name=None): # Check instance type if not isinstance(server_port, int): LocalError.err_invalid_port() # Check for server address if not (Services.is_valid_ipv4(server_addr) or Services.is_valid_ipv6(server_addr)): LocalError.err_invalid_ip() # Check token type if not isinstance(token, str): LocalError.err_invalid_token_type() # Check max try type if not isinstance(max_try, int): LocalError.err_invalid_max_try_type() """ System definitions. """ self.__master_addr = server_addr self.__master_port = server_port self.__master_token = token self.__std_idle_time = std_idle_time self.__max_try = max_try if source_name: self.__source_name = source_name else: import socket self.__source_name = socket.gethostname() # Connection string from harmonicIO.general.definition import Definition self.__str_master_status = Definition.Master.get_str_check_master( server_addr, server_port, "None") self.__str_push_request = Definition.Master.get_str_push_req( server_addr, server_port, "None") # URL Request self.__connector = urllib3.PoolManager()
def update_worker_status(): """ Update the worker status to the master as well as container info. """ threading.Timer(5, update_worker_status).start() """ Get machine status by calling a unix command and fetch for load average """ content = Services.get_machine_status(Setting, CRole.WORKER) content[Definition.REST.get_str_docker( )] = DockerService.get_containers_status() content[Definition.REST.get_str_local_imgs( )] = DockerService.get_local_images() s_content = bytes(json.dumps(content), 'utf-8') html = urllib3.PoolManager() try: r = html.request('PUT', Definition.Master.get_str_check_master( Setting.get_master_addr(), Setting.get_master_port(), Setting.get_token()), body=s_content) if r.status != 200: SysOut.err_string("Cannot update worker status to the master!") else: SysOut.debug_string("Reports status to master node complete.") except Exception as e: SysOut.err_string("Master is not available!") print(e)
def set_node_addr(addr=None): if addr: Setting.__node_addr = addr else: import socket from harmonicIO.general.services import Services Setting.__node_addr = socket.gethostname() # if addr is valid if Services.is_valid_ipv4( Setting.__node_addr) or Services.is_valid_ipv6( Setting.__node_addr): return None # if addr is not valid Setting.__node_addr = Services.get_host_name_i() if Services.is_valid_ipv4( Setting.__node_addr) or Services.is_valid_ipv6( Setting.__node_addr): return None SysOut.terminate_string("Cannot get node ip address!")
def on_get(self, req, res): """ GET: /status?token={None} """ if not Definition.get_str_token() in req.params: format_response_string(res, falcon.HTTP_401, "Token is required") return if req.params[Definition.get_str_token()] == Setting.get_token(): result = LService.get_machine_status(Setting, CRole.MASTER) format_response_string(res, falcon.HTTP_200, str(result)) else: format_response_string(res, falcon.HTTP_401, "Invalid token ID")
def get_container_object(req): ret = dict() ret[Definition.REST.Batch.get_str_batch_addr()] = req.params[ Definition.REST.Batch.get_str_batch_addr()].strip() ret[Definition.REST.Batch.get_str_batch_port()] = int( req.params[Definition.REST.Batch.get_str_batch_port()]) ret[Definition.REST.Batch.get_str_batch_status()] = int( req.params[Definition.REST.Batch.get_str_batch_status()]) ret[Definition.Container.get_str_con_image_name()] = req.params[ Definition.Container.get_str_con_image_name()].strip() ret[Definition.get_str_last_update( )] = Services.get_current_timestamp() return ret
def get_tuple_object(req): # parameters ret = dict() ret[Definition.Container.get_str_data_digest()] = req.params[ Definition.Container.get_str_data_digest()].strip() ret[Definition.Container.get_str_con_image_name()] = req.params[ Definition.Container.get_str_con_image_name()].strip() ret[Definition.Container.get_str_container_os()] = req.params[ Definition.Container.get_str_container_os()].strip() ret[Definition.Container.get_str_data_source()] = req.params[ Definition.Container.get_str_data_source()].strip() ret[Definition.Container.get_str_container_priority()] = 0 ret[Definition.REST.get_str_status()] = CTuple.SC ret[Definition.get_str_last_update( )] = Services.get_current_timestamp() return ret
def on_get(self, req, res): """ GET: /status?token={None} """ if not Definition.get_str_token() in req.params: res.body = "Token is required." res.content_type = "String" res.status = falcon.HTTP_401 return if req.params[Definition.get_str_token()] == Setting.get_token(): result = LService.get_machine_status(Setting, CRole.MASTER) res.body = str(result) res.content_type = "String" res.status = falcon.HTTP_200 else: res.body = "Invalid token ID." res.content_type = "String" res.status = falcon.HTTP_401
def on_get(self, req, res): """ GET: /status?token={None} """ if not Definition.get_str_token() in req.params: res.body = "Token is required." res.content_type = "String" res.status = falcon.HTTP_401 return if req.params[Definition.get_str_token()] == Setting.get_token(): s_content = Services.get_machine_status(Setting, CRole.WORKER) s_content[Definition.REST.get_str_docker( )] = DockerService.get_containers_status() res.body = str(s_content) res.content_type = "String" res.status = falcon.HTTP_200 else: res.body = "Invalid token ID." res.content_type = "String" res.status = falcon.HTTP_401
def update_container(dict_input): def cont_in_table(dict_input): conts = LookUpTable.Containers.__containers[dict_input[ Definition.Container.get_str_con_image_name()]] for cont in conts: if dict_input.get(Definition.Container.Status.get_str_sid( )) == cont.get(Definition.Container.Status.get_str_sid()): return cont return None if dict_input[Definition.Container.get_str_con_image_name( )] not in LookUpTable.Containers.__containers: # no containers for this image exist new_cont = [dict_input] LookUpTable.Containers.__containers[dict_input[ Definition.Container.get_str_con_image_name()]] = new_cont else: cont = cont_in_table(dict_input) if not cont: # this specific container is not already in table LookUpTable.Containers.__containers[dict_input[ Definition.Container.get_str_con_image_name()]].append( dict_input) else: # container was already in table, update timestamp cont[Definition.get_str_last_update( )] = Services.get_current_timestamp()
def read_cfg_from_file(): from harmonicIO.general.services import Services if not Services.is_file_exist('harmonicIO/worker/configuration.json'): SysOut.terminate_string( 'harmonicIO/worker/configuration.json does not exist') else: with open('harmonicIO/worker/configuration.json', 'rt') as t: import json cfg = json.loads(t.read()) try: from harmonicIO.general.definition import Definition # Check for the json structure if Definition.get_str_node_name() in cfg and \ Definition.get_str_node_port() in cfg and \ Definition.get_str_data_port_range() in cfg and \ Definition.get_str_idle_time() in cfg and \ Definition.get_str_master_addr() in cfg and \ Definition.get_str_master_port() in cfg and \ Definition.get_str_node_external_addr() in cfg and \ Definition.get_str_node_internal_addr(): # Check port number is int or not if not isinstance(cfg[Definition.get_str_node_port()], int): SysOut.terminate_string( "Node port must be integer.") elif not isinstance( cfg[Definition.get_str_data_port_range()], list): SysOut.terminate_string("Port range must be list.") elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \ isinstance(cfg[Definition.get_str_data_port_range()][1], int)): SysOut.terminate_string( "Port range must be integer.") elif not isinstance( cfg[Definition.get_str_master_port()], int): SysOut.terminate_string( "Master port must be integer.") elif len(cfg[ Definition.get_str_data_port_range()]) != 2: SysOut.terminate_string( "Port range must compost of two elements: start, stop." ) elif not isinstance( cfg[Definition.get_str_idle_time()], int): SysOut.terminate_string( "Idle time must be integer.") elif cfg[Definition.get_str_data_port_range()][0] > \ cfg[Definition.get_str_data_port_range()][1]: SysOut.terminate_string( "Start port range must greater than stop port range." ) else: Setting.set_node_addr() import multiprocessing Setting.__node_name = cfg[ Definition.get_str_node_name()].strip() Setting.__node_port = cfg[ Definition.get_str_node_port()] Setting.__node_data_port_start = cfg[ Definition.get_str_data_port_range()][0] Setting.__node_data_port_stop = cfg[ Definition.get_str_data_port_range()][1] Setting.__std_idle_time = cfg[ Definition.get_str_idle_time()] Setting.__master_addr = cfg[ Definition.get_str_master_addr()].strip() Setting.__master_port = cfg[ Definition.get_str_master_port()] Setting.__node_external_addr = cfg[ Definition.get_str_node_external_addr()].strip( ).lower() # Check for auto node name if Setting.__node_name.lower() == "auto": # Get node name from host name import socket Setting.__node_name = socket.gethostname() # Check for overriding node address if cfg[Definition.get_str_node_internal_addr()] and \ cfg[Definition.get_str_node_internal_addr()] != "auto": # Set node name automatically from hostname from harmonicIO.general.services import Services if Services.is_valid_ipv4(cfg[Definition.get_str_node_internal_addr()]) or \ Services.is_valid_ipv6(cfg[Definition.get_str_node_internal_addr()]): Setting.__node_internal_addr = cfg[ Definition.get_str_node_internal_addr( )] # Check for node address validity if Setting.get_node_external_addr() != "none": from harmonicIO.general.services import Services if Services.is_valid_ipv4(Setting.get_node_external_addr()) or \ Services.is_valid_ipv6(Setting.get_node_external_addr()): SysOut.out_string( "By pass request with external address." ) else: SysOut.terminate_string( "Invaliid external ip address!") else: Setting.__node_external_addr = None SysOut.out_string("Load setting successful.") else: SysOut.terminate_string( "Required parameters are not present.") except Exception as e: print(e) SysOut.terminate_string( "Invalid data in configuration file.")
def on_get(self, req, res): """ return "&c_name=" + container_name + "&c_os=" + container_os + "&priority=" + str(priority) GET: /streamRequest?token=None This function is mainly respond with the available channel for streaming from data source. """ if not Definition.get_str_token() in req.params: res.body = "Token is required." res.content_type = "String" res.status = falcon.HTTP_401 return # Check for required parameter. if not Definition.Container.get_str_con_image_name() in req.params: res.body = "Container name is required." res.content_type = "String" res.status = falcon.HTTP_401 return if not Definition.Container.get_str_container_os() in req.params: res.body = "Container os is required." res.content_type = "String" res.status = falcon.HTTP_401 return if not Definition.Container.get_str_data_source() in req.params: res.body = "Data digest is required." res.content_type = "String" res.status = falcon.HTTP_401 return # Parse to dict object ret = LookUpTable.Tuples.get_tuple_object(req) if Definition.Container.get_str_container_priority() in req.params: if LService.is_str_is_digit(req.params[ Definition.Container.get_str_container_priority()]): ret[Definition.Container.get_str_container_priority()] = int( req.params[ Definition.Container.get_str_container_priority()]) else: res.body = "Container priority is not digit." res.content_type = "String" res.status = falcon.HTTP_401 return # Register item into tuples LookUpTable.Tuples.add_tuple_info(ret) # Check for the availability of the container ret = LookUpTable.get_candidate_container( ret[Definition.Container.get_str_con_image_name()]) if ret: res.body = Definition.Master.get_str_end_point(ret) res.content_type = "String" res.status = falcon.HTTP_200 return else: # No streaming end-point available res.body = Definition.Master.get_str_end_point_MS(Setting) res.content_type = "String" res.status = falcon.HTTP_200 return
def add_worker(dict_input): dict_input[Definition.get_str_last_update( )] = Services.get_current_timestamp() LookUpTable.Workers.__workers[dict_input[ Definition.get_str_node_addr()]] = dict_input
def read_cfg_from_file(): from harmonicIO.general.services import Services, SysOut if not Services.is_file_exist('harmonicIO/master/configuration.json'): SysOut.terminate_string( 'harmonicIO/master/configuration.json does not exist!') else: with open('harmonicIO/master/configuration.json', 'rt') as t: import json cfg = json.loads(t.read()) try: from harmonicIO.general.definition import Definition # Check for the json structure if Definition.get_str_node_name() in cfg and \ Definition.get_str_node_port() in cfg and \ Definition.get_str_master_addr() in cfg and \ Definition.get_str_data_port_range() in cfg and \ Definition.get_str_idle_time() in cfg: # Check port number is int or not if not isinstance(cfg[Definition.get_str_node_port()], int): SysOut.terminate_string( "Node port must be integer!") if not isinstance( cfg[Definition.get_str_master_addr()], str): SysOut.terminate_string( "Master address must be string!") elif not isinstance( cfg[Definition.get_str_data_port_range()], list): SysOut.terminate_string("Port range must be list!") elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \ isinstance(cfg[Definition.get_str_data_port_range()][1], int)): SysOut.terminate_string( "Port range must be integer!") elif len(cfg[ Definition.get_str_data_port_range()]) != 2: SysOut.terminate_string( "Port range must compost of two elements: start, stop!" ) elif not isinstance( cfg[Definition.get_str_idle_time()], int): SysOut.terminate_string( "Idle time must be integer!") elif cfg[Definition.get_str_data_port_range()][0] > \ cfg[Definition.get_str_data_port_range()][1]: SysOut.terminate_string( "Start port range must greater than stop port range!" ) else: Setting.__node_name = cfg[ Definition.get_str_node_name()].strip() Setting.__node_port = cfg[ Definition.get_str_node_port()] Setting.__node_data_port_start = cfg[ Definition.get_str_data_port_range()][0] Setting.__node_data_port_stop = cfg[ Definition.get_str_data_port_range()][1] Setting.__std_idle_time = cfg[ Definition.get_str_idle_time()] SysOut.out_string("Load setting successful.") try: if cfg[Definition.get_str_master_addr()].lower( ) == "auto": Setting.__node_addr = Services.get_host_name_i( ) SysOut.out_string( "Assigning master ip address automatically." ) elif Services.is_valid_ipv4(cfg[Definition.get_str_master_addr()]) or \ Services.is_valid_ipv6(cfg[Definition.get_str_master_addr()]): Setting.set_node_addr( cfg[Definition.get_str_master_addr()]) else: SysOut.terminate_string( "Invalid master IP address format!") except: SysOut.terminate_string( "Cannot assign IP address to the master!") else: SysOut.terminate_string( "Invalid data in configuration file.") except: SysOut.terminate_string( "Invalid data in configuration file.")