def handle(self):
# Receive and interpret the request data
data = bytearray()
"""
Discard heading for now
data += self.request.recv(16)
# Interpret the header for file size
file_size = struct.unpack(">Q", data[8:16])[0]
"""
try:
c = True
while c != b"":
c = self.request.recv(2048)
data += c
# Extract byte header 3 bytes
image_name_length = int(data[0:3].decode('UTF-8'))
tcr = image_name_length + 3
image_name_string = data[3:tcr].decode('UTF-8')
# Then, push data messaging system.
MessagesQueue.push_to_queue(image_name_string, data[tcr:])
except:
from harmonicIO.general.services import Services
Services.e_print("Insufficient memory for storing g object.")
def __init__(self,
server_addr,
server_port,
token="None",
std_idle_time=0,
max_try=9,
source_name=None):
# Check instance type
if not isinstance(server_port, int):
LocalError.err_invalid_port()
# Check for server address
if not (Services.is_valid_ipv4(server_addr)
or Services.is_valid_ipv6(server_addr)):
LocalError.err_invalid_ip()
# Check token type
if not isinstance(token, str):
LocalError.err_invalid_token_type()
# Check max try type
if not isinstance(max_try, int):
LocalError.err_invalid_max_try_type()
"""
System definitions.
"""
self.__master_addr = server_addr
self.__master_port = server_port
self.__master_token = token
self.__std_idle_time = std_idle_time
self.__max_try = max_try
if source_name:
self.__source_name = source_name
else:
import socket
self.__source_name = socket.gethostname()
# Connection string
from harmonicIO.general.definition import Definition
self.__str_master_status = Definition.Master.get_str_check_master(
server_addr, server_port, "None")
self.__str_push_request = Definition.Master.get_str_push_req(
server_addr, server_port, "None")
# URL Request
self.__connector = urllib3.PoolManager()
def update_worker_status():
"""
Update the worker status to the master as well as container info.
"""
threading.Timer(5, update_worker_status).start()
"""
Get machine status by calling a unix command and fetch for load average
"""
content = Services.get_machine_status(Setting, CRole.WORKER)
content[Definition.REST.get_str_docker(
)] = DockerService.get_containers_status()
content[Definition.REST.get_str_local_imgs(
)] = DockerService.get_local_images()
s_content = bytes(json.dumps(content), 'utf-8')
html = urllib3.PoolManager()
try:
r = html.request('PUT',
Definition.Master.get_str_check_master(
Setting.get_master_addr(),
Setting.get_master_port(), Setting.get_token()),
body=s_content)
if r.status != 200:
SysOut.err_string("Cannot update worker status to the master!")
else:
SysOut.debug_string("Reports status to master node complete.")
except Exception as e:
SysOut.err_string("Master is not available!")
print(e)
def set_node_addr(addr=None):
if addr:
Setting.__node_addr = addr
else:
import socket
from harmonicIO.general.services import Services
Setting.__node_addr = socket.gethostname()
# if addr is valid
if Services.is_valid_ipv4(
Setting.__node_addr) or Services.is_valid_ipv6(
Setting.__node_addr):
return None
# if addr is not valid
Setting.__node_addr = Services.get_host_name_i()
if Services.is_valid_ipv4(
Setting.__node_addr) or Services.is_valid_ipv6(
Setting.__node_addr):
return None
SysOut.terminate_string("Cannot get node ip address!")
def on_get(self, req, res):
"""
GET: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
format_response_string(res, falcon.HTTP_401, "Token is required")
return
if req.params[Definition.get_str_token()] == Setting.get_token():
result = LService.get_machine_status(Setting, CRole.MASTER)
format_response_string(res, falcon.HTTP_200, str(result))
else:
format_response_string(res, falcon.HTTP_401, "Invalid token ID")
def get_container_object(req):
ret = dict()
ret[Definition.REST.Batch.get_str_batch_addr()] = req.params[
Definition.REST.Batch.get_str_batch_addr()].strip()
ret[Definition.REST.Batch.get_str_batch_port()] = int(
req.params[Definition.REST.Batch.get_str_batch_port()])
ret[Definition.REST.Batch.get_str_batch_status()] = int(
req.params[Definition.REST.Batch.get_str_batch_status()])
ret[Definition.Container.get_str_con_image_name()] = req.params[
Definition.Container.get_str_con_image_name()].strip()
ret[Definition.get_str_last_update(
)] = Services.get_current_timestamp()
return ret
def get_tuple_object(req):
# parameters
ret = dict()
ret[Definition.Container.get_str_data_digest()] = req.params[
Definition.Container.get_str_data_digest()].strip()
ret[Definition.Container.get_str_con_image_name()] = req.params[
Definition.Container.get_str_con_image_name()].strip()
ret[Definition.Container.get_str_container_os()] = req.params[
Definition.Container.get_str_container_os()].strip()
ret[Definition.Container.get_str_data_source()] = req.params[
Definition.Container.get_str_data_source()].strip()
ret[Definition.Container.get_str_container_priority()] = 0
ret[Definition.REST.get_str_status()] = CTuple.SC
ret[Definition.get_str_last_update(
)] = Services.get_current_timestamp()
return ret
def on_get(self, req, res):
"""
GET: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if req.params[Definition.get_str_token()] == Setting.get_token():
result = LService.get_machine_status(Setting, CRole.MASTER)
res.body = str(result)
res.content_type = "String"
res.status = falcon.HTTP_200
else:
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
def on_get(self, req, res):
"""
GET: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if req.params[Definition.get_str_token()] == Setting.get_token():
s_content = Services.get_machine_status(Setting, CRole.WORKER)
s_content[Definition.REST.get_str_docker(
)] = DockerService.get_containers_status()
res.body = str(s_content)
res.content_type = "String"
res.status = falcon.HTTP_200
else:
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
def update_container(dict_input):
def cont_in_table(dict_input):
conts = LookUpTable.Containers.__containers[dict_input[
Definition.Container.get_str_con_image_name()]]
for cont in conts:
if dict_input.get(Definition.Container.Status.get_str_sid(
)) == cont.get(Definition.Container.Status.get_str_sid()):
return cont
return None
if dict_input[Definition.Container.get_str_con_image_name(
)] not in LookUpTable.Containers.__containers: # no containers for this image exist
new_cont = [dict_input]
LookUpTable.Containers.__containers[dict_input[
Definition.Container.get_str_con_image_name()]] = new_cont
else:
cont = cont_in_table(dict_input)
if not cont: # this specific container is not already in table
LookUpTable.Containers.__containers[dict_input[
Definition.Container.get_str_con_image_name()]].append(
dict_input)
else: # container was already in table, update timestamp
cont[Definition.get_str_last_update(
)] = Services.get_current_timestamp()
def read_cfg_from_file():
from harmonicIO.general.services import Services
if not Services.is_file_exist('harmonicIO/worker/configuration.json'):
SysOut.terminate_string(
'harmonicIO/worker/configuration.json does not exist')
else:
with open('harmonicIO/worker/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from harmonicIO.general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_master_port() in cfg and \
Definition.get_str_node_external_addr() in cfg and \
Definition.get_str_node_internal_addr():
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()],
int):
SysOut.terminate_string(
"Node port must be integer.")
elif not isinstance(
cfg[Definition.get_str_data_port_range()],
list):
SysOut.terminate_string("Port range must be list.")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
SysOut.terminate_string(
"Port range must be integer.")
elif not isinstance(
cfg[Definition.get_str_master_port()], int):
SysOut.terminate_string(
"Master port must be integer.")
elif len(cfg[
Definition.get_str_data_port_range()]) != 2:
SysOut.terminate_string(
"Port range must compost of two elements: start, stop."
)
elif not isinstance(
cfg[Definition.get_str_idle_time()], int):
SysOut.terminate_string(
"Idle time must be integer.")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
SysOut.terminate_string(
"Start port range must greater than stop port range."
)
else:
Setting.set_node_addr()
import multiprocessing
Setting.__node_name = cfg[
Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[
Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[
Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[
Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[
Definition.get_str_idle_time()]
Setting.__master_addr = cfg[
Definition.get_str_master_addr()].strip()
Setting.__master_port = cfg[
Definition.get_str_master_port()]
Setting.__node_external_addr = cfg[
Definition.get_str_node_external_addr()].strip(
).lower()
# Check for auto node name
if Setting.__node_name.lower() == "auto":
# Get node name from host name
import socket
Setting.__node_name = socket.gethostname()
# Check for overriding node address
if cfg[Definition.get_str_node_internal_addr()] and \
cfg[Definition.get_str_node_internal_addr()] != "auto":
# Set node name automatically from hostname
from harmonicIO.general.services import Services
if Services.is_valid_ipv4(cfg[Definition.get_str_node_internal_addr()]) or \
Services.is_valid_ipv6(cfg[Definition.get_str_node_internal_addr()]):
Setting.__node_internal_addr = cfg[
Definition.get_str_node_internal_addr(
)]
# Check for node address validity
if Setting.get_node_external_addr() != "none":
from harmonicIO.general.services import Services
if Services.is_valid_ipv4(Setting.get_node_external_addr()) or \
Services.is_valid_ipv6(Setting.get_node_external_addr()):
SysOut.out_string(
"By pass request with external address."
)
else:
SysOut.terminate_string(
"Invaliid external ip address!")
else:
Setting.__node_external_addr = None
SysOut.out_string("Load setting successful.")
else:
SysOut.terminate_string(
"Required parameters are not present.")
except Exception as e:
print(e)
SysOut.terminate_string(
"Invalid data in configuration file.")
def on_get(self, req, res):
"""
return "&c_name=" + container_name + "&c_os=" + container_os + "&priority=" + str(priority)
GET: /streamRequest?token=None
This function is mainly respond with the available channel for streaming from data source.
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Check for required parameter.
if not Definition.Container.get_str_con_image_name() in req.params:
res.body = "Container name is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not Definition.Container.get_str_container_os() in req.params:
res.body = "Container os is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not Definition.Container.get_str_data_source() in req.params:
res.body = "Data digest is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Parse to dict object
ret = LookUpTable.Tuples.get_tuple_object(req)
if Definition.Container.get_str_container_priority() in req.params:
if LService.is_str_is_digit(req.params[
Definition.Container.get_str_container_priority()]):
ret[Definition.Container.get_str_container_priority()] = int(
req.params[
Definition.Container.get_str_container_priority()])
else:
res.body = "Container priority is not digit."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Register item into tuples
LookUpTable.Tuples.add_tuple_info(ret)
# Check for the availability of the container
ret = LookUpTable.get_candidate_container(
ret[Definition.Container.get_str_con_image_name()])
if ret:
res.body = Definition.Master.get_str_end_point(ret)
res.content_type = "String"
res.status = falcon.HTTP_200
return
else:
# No streaming end-point available
res.body = Definition.Master.get_str_end_point_MS(Setting)
res.content_type = "String"
res.status = falcon.HTTP_200
return
def add_worker(dict_input):
dict_input[Definition.get_str_last_update(
)] = Services.get_current_timestamp()
LookUpTable.Workers.__workers[dict_input[
Definition.get_str_node_addr()]] = dict_input
def read_cfg_from_file():
from harmonicIO.general.services import Services, SysOut
if not Services.is_file_exist('harmonicIO/master/configuration.json'):
SysOut.terminate_string(
'harmonicIO/master/configuration.json does not exist!')
else:
with open('harmonicIO/master/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from harmonicIO.general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()],
int):
SysOut.terminate_string(
"Node port must be integer!")
if not isinstance(
cfg[Definition.get_str_master_addr()], str):
SysOut.terminate_string(
"Master address must be string!")
elif not isinstance(
cfg[Definition.get_str_data_port_range()],
list):
SysOut.terminate_string("Port range must be list!")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
SysOut.terminate_string(
"Port range must be integer!")
elif len(cfg[
Definition.get_str_data_port_range()]) != 2:
SysOut.terminate_string(
"Port range must compost of two elements: start, stop!"
)
elif not isinstance(
cfg[Definition.get_str_idle_time()], int):
SysOut.terminate_string(
"Idle time must be integer!")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
SysOut.terminate_string(
"Start port range must greater than stop port range!"
)
else:
Setting.__node_name = cfg[
Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[
Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[
Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[
Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[
Definition.get_str_idle_time()]
SysOut.out_string("Load setting successful.")
try:
if cfg[Definition.get_str_master_addr()].lower(
) == "auto":
Setting.__node_addr = Services.get_host_name_i(
)
SysOut.out_string(
"Assigning master ip address automatically."
)
elif Services.is_valid_ipv4(cfg[Definition.get_str_master_addr()]) or \
Services.is_valid_ipv6(cfg[Definition.get_str_master_addr()]):
Setting.set_node_addr(
cfg[Definition.get_str_master_addr()])
else:
SysOut.terminate_string(
"Invalid master IP address format!")
except:
SysOut.terminate_string(
"Cannot assign IP address to the master!")
else:
SysOut.terminate_string(
"Invalid data in configuration file.")
except:
SysOut.terminate_string(
"Invalid data in configuration file.")