def update_worker_status():
"""
Update the worker status to the master as well as container info.
"""
threading.Timer(5, update_worker_status).start()
"""
Get machine status by calling a unix command and fetch for load average
"""
content = Services.get_machine_status(Setting, CRole.WORKER)
content[Definition.REST.get_str_docker(
)] = DockerService.get_containers_status()
content[Definition.REST.get_str_local_imgs(
)] = DockerService.get_local_images()
s_content = bytes(json.dumps(content), 'utf-8')
html = urllib3.PoolManager()
try:
r = html.request('PUT',
Definition.Master.get_str_check_master(
Setting.get_master_addr(),
Setting.get_master_port(), Setting.get_token()),
body=s_content)
if r.status != 200:
SysOut.err_string("Cannot update worker status to the master!")
else:
SysOut.debug_string("Reports status to master node complete.")
except Exception as e:
SysOut.err_string("Master is not available!")
print(e)
def handle(self):
# Receive and interpret the request data
data = bytearray()
"""
Discard heading for now
data += self.request.recv(16)
# Interpret the header for file size
file_size = struct.unpack(">Q", data[8:16])[0]
"""
try:
c = True
while c != b"":
c = self.request.recv(2048)
data += c
# Extract byte header 3 bytes
image_name_length = int(data[0:3].decode('UTF-8'))
tcr = image_name_length + 3
image_name_string = data[3:tcr].decode('UTF-8')
# Then, push data messaging system.
MessagesQueue.push_to_queue(image_name_string, data[tcr:])
except:
from harmonicIO.general.services import Services
SysOut.err_string("Insufficient memory for storing g object.")
def on_post(self, req, res):
# check token and request type is provided
req_raw = (str(req.stream.read(req.content_length or 0),
'utf-8')) # create dict of body data if they exist
req_data = json.loads(req_raw)
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not "type" in req.params:
res.body = "No command specified."
res.content_type = "String"
res.status = falcon.HTTP_406
return
# request to create new job - create ID for job, add to lookup table, queue creation of the job
if req.params['type'] == 'new_job':
job = new_job(
req_data) # attempt to create new job from provided parameters
if not job:
SysOut.err_string("New job could not be added!")
format_response_string(res, falcon.HTTP_500,
"Could not create job.")
return
job_status = job.get('job_status')
format_response_string(
res, falcon.HTTP_200,
"Job request received, container status: {}\nJob ID: {}".
format(job_status, job.get('job_id')))
return
return
def __get_stream_end_point(self, container_name, container_os, priority,
digest):
"""
Request for the stream end point from the master.
:return: Boolean(False) when the system is busy.
Tuple(batch_addr, batch_port, tuple_id) if the batch or messaging system is available.
"""
if not priority:
priority = 0
else:
if not isinstance(priority, int):
LocalError.err_invalid_priority_type()
try:
url = self.__str_push_request + Definition.Master.get_str_push_req_container_ext(
container_name, container_os, priority, self.__source_name,
digest)
print('Sending request..')
print(url)
response = self.__connector.request('GET', url)
# print(response.status)
# print(response.text)
if response.status == 406:
# Messages in queue is full. Result in queue lock.
SysOut.warn_string("Queue in master is full.")
return False
if response.status == 500:
SysOut.warn_string(
"System internal error! Please consult documentation.")
return False
elif response.status != 200:
SysOut.warn_string("something else went wrong")
return False
except Exception as ex:
print(ex)
SysOut.err_string(
"Couldn't connect to the master at {0}:{1}.".format(
self.__master_addr, self.__master_port))
return False
try:
content = eval(response.data.decode('utf-8'))
return content
except:
SysOut.warn_string("JSON content error from the master!")
return False
def run_container(self, container_name):
def get_ports_setting(expose, ports):
return {str(expose) + '/tcp': ports}
def get_env_setting(expose, a_port):
ret = dict()
ret[Definition.Docker.HDE.get_str_node_name()] = container_name
ret[Definition.Docker.HDE.get_str_node_addr(
)] = Setting.get_node_addr()
ret[Definition.Docker.HDE.get_str_node_data_port()] = expose
ret[Definition.Docker.HDE.get_str_node_forward_port()] = a_port
ret[Definition.Docker.HDE.get_str_master_addr(
)] = Setting.get_master_addr()
ret[Definition.Docker.HDE.get_str_master_port(
)] = Setting.get_master_port()
ret[Definition.Docker.HDE.get_str_std_idle_time(
)] = Setting.get_std_idle_time()
ret[Definition.Docker.HDE.get_str_token()] = Setting.get_token()
return ret
port = self.__get_available_port()
expose_port = 80
if not port:
SysOut.err_string("No more port available!")
return False
else:
print('starting container ' + container_name)
res = self.__client.containers.run(
container_name,
detach=True,
stderr=True,
stdout=True,
ports=get_ports_setting(expose_port, port),
environment=get_env_setting(expose_port, port))
import time
time.sleep(1)
print('..created container, logs:')
print(res.logs(stdout=True, stderr=True))
if res:
SysOut.out_string("Container " + container_name +
" is created!")
SysOut.out_string("Container " + container_name + " is " +
res.status + " ")
return True
else:
SysOut.out_string("Container " + container_name +
" cannot be created!")
return False
def send_data(self, container_name, container_os, data, priority=None):
# The data must be byte array
if not isinstance(data, bytearray):
LocalError.err_invalid_data_container_type()
if len(data) == 0:
SysOut.err_string("No content in byte array.")
return None
digest = hashlib.md5(data).hexdigest()
end_point = None
counter = self.__max_try
while not end_point:
end_point = self.__get_stream_end_point(container_name,
container_os, priority,
digest)
counter -= 1
if counter == 0:
SysOut.err_string(
"Cannot contact server. Exceed maximum retry {0}!".format(
self.__max_try))
return False
# Send data to worker for processing directly
counter = self.__max_try
if end_point[Definition.get_str_node_role()] == CRole.WORKER:
while not self.__push_stream_end_point(
end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()], data):
time.sleep(self.__std_idle_time)
counter -= 1
if counter == 0:
SysOut.err_string(
"Cannot contact server. Exceed maximum retry {0}!".
format(self.__max_try))
return False
# Send data to master for queuing (?)
elif end_point[
Definition.get_str_node_role()] == CRole.MESSAGING_SYSTEM:
while not self.__push_stream_end_point_MS(
end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()], data,
container_name):
time.sleep(self.__std_idle_time)
counter -= 1
if counter == 0:
SysOut.err_string(
"Cannot contact server. Exceed maximum retry {0}!".
format(self.__max_try))
return False
else:
return False
if end_point[Definition.get_str_node_role()] == CRole.WORKER:
SysOut.out_string(
"Push data to worker ({0}:{1}>{2}) successful.".format(
end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()], container_name))
elif end_point[
Definition.get_str_node_role()] == CRole.MESSAGING_SYSTEM:
SysOut.out_string(
"Push data to messaging system ({0}:{1}>{2}) successful.".
format(end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()],
container_name))
else:
SysOut.out_string(
"Push data to unknown ({0}:{1}>{2}) successful.".format(
end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()], container_name))