def start_gc_thread():
garbage_collector = GarbageCollector(10)
gc_thread = threading.Thread(garbage_collector.collect_exited_containers())
gc_thread.daemon = True
gc_thread.start()
SysOut.out_string("Garbage collector started")
def run_queue_manager(manager):
"""
Run job queue manager thread
can be several managers to manage large amount of queued jobs
"""
import threading
for i in range(manager.queuer_threads):
manager_thread = threading.Thread(target=manager.job_queuer)
manager_thread.daemon = True
manager_thread.start()
SysOut.out_string("Job queue started")
if Setting.get_autoscaling():
supervisor_thread = threading.Thread(target=manager.queue_supervisor)
supervisor_thread.daemon = True
supervisor_thread.start()
SysOut.out_string("Autoscaling supervisor started")
def __init__(self):
self.__ports = []
self.__client = docker.from_env()
SysOut.out_string("Docker master initialization complete.")
# Define port status
for port_num in range(Setting.get_data_port_start(),
Setting.get_data_port_stop()):
self.__ports += [ChannelStatus(port_num)]
# Check number of available port
available_port = 0
for item in self.__ports:
if item.status == CStatus.AVAILABLE:
available_port += 1
self.__available_port = available_port
def run_msg_service():
"""
Run msg service to eliminate back pressure
"""
from .configuration import Setting
from .server_socket import ThreadedTCPServer, ThreadedTCPRequestHandler
import threading
server = ThreadedTCPServer((Setting.get_node_addr(), Setting.get_data_port_start()),
ThreadedTCPRequestHandler, bind_and_activate=True)
# Start a thread with the server -- that thread will then start one
server_thread = threading.Thread(target=server.serve_forever)
# Exit the server thread when the main thread terminates
server_thread.daemon = True
SysOut.out_string("Enable Messaging System on port: " + str(Setting.get_data_port_start()))
server_thread.start()
""" Have to test for graceful termination. """
def run_container(self, container_name):
def get_ports_setting(expose, ports):
return {str(expose) + '/tcp': ports}
def get_env_setting(expose, a_port):
ret = dict()
ret[Definition.Docker.HDE.get_str_node_name()] = container_name
ret[Definition.Docker.HDE.get_str_node_addr(
)] = Setting.get_node_addr()
ret[Definition.Docker.HDE.get_str_node_data_port()] = expose
ret[Definition.Docker.HDE.get_str_node_forward_port()] = a_port
ret[Definition.Docker.HDE.get_str_master_addr(
)] = Setting.get_master_addr()
ret[Definition.Docker.HDE.get_str_master_port(
)] = Setting.get_master_port()
ret[Definition.Docker.HDE.get_str_std_idle_time(
)] = Setting.get_std_idle_time()
ret[Definition.Docker.HDE.get_str_token()] = Setting.get_token()
return ret
port = self.__get_available_port()
expose_port = 80
if not port:
SysOut.err_string("No more port available!")
return False
else:
print('starting container ' + container_name)
res = self.__client.containers.run(
container_name,
detach=True,
stderr=True,
stdout=True,
ports=get_ports_setting(expose_port, port),
environment=get_env_setting(expose_port, port))
import time
time.sleep(1)
print('..created container, logs:')
print(res.logs(stdout=True, stderr=True))
if res:
SysOut.out_string("Container " + container_name +
" is created!")
SysOut.out_string("Container " + container_name + " is " +
res.status + " ")
return True
else:
SysOut.out_string("Container " + container_name +
" cannot be created!")
return False
def send_data(self, container_name, container_os, data, priority=None):
# The data must be byte array
if not isinstance(data, bytearray):
LocalError.err_invalid_data_container_type()
if len(data) == 0:
SysOut.err_string("No content in byte array.")
return None
digest = hashlib.md5(data).hexdigest()
end_point = None
counter = self.__max_try
while not end_point:
end_point = self.__get_stream_end_point(container_name,
container_os, priority,
digest)
counter -= 1
if counter == 0:
SysOut.err_string(
"Cannot contact server. Exceed maximum retry {0}!".format(
self.__max_try))
return False
# Send data to worker for processing directly
counter = self.__max_try
if end_point[Definition.get_str_node_role()] == CRole.WORKER:
while not self.__push_stream_end_point(
end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()], data):
time.sleep(self.__std_idle_time)
counter -= 1
if counter == 0:
SysOut.err_string(
"Cannot contact server. Exceed maximum retry {0}!".
format(self.__max_try))
return False
# Send data to master for queuing (?)
elif end_point[
Definition.get_str_node_role()] == CRole.MESSAGING_SYSTEM:
while not self.__push_stream_end_point_MS(
end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()], data,
container_name):
time.sleep(self.__std_idle_time)
counter -= 1
if counter == 0:
SysOut.err_string(
"Cannot contact server. Exceed maximum retry {0}!".
format(self.__max_try))
return False
else:
return False
if end_point[Definition.get_str_node_role()] == CRole.WORKER:
SysOut.out_string(
"Push data to worker ({0}:{1}>{2}) successful.".format(
end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()], container_name))
elif end_point[
Definition.get_str_node_role()] == CRole.MESSAGING_SYSTEM:
SysOut.out_string(
"Push data to messaging system ({0}:{1}>{2}) successful.".
format(end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()],
container_name))
else:
SysOut.out_string(
"Push data to unknown ({0}:{1}>{2}) successful.".format(
end_point[Definition.get_str_node_addr()],
end_point[Definition.get_str_node_port()], container_name))
def read_cfg_from_file():
from harmonicIO.general.services import Services
if not Services.is_file_exist('harmonicIO/worker/configuration.json'):
SysOut.terminate_string(
'harmonicIO/worker/configuration.json does not exist')
else:
with open('harmonicIO/worker/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from harmonicIO.general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_master_port() in cfg and \
Definition.get_str_node_external_addr() in cfg and \
Definition.get_str_node_internal_addr():
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()],
int):
SysOut.terminate_string(
"Node port must be integer.")
elif not isinstance(
cfg[Definition.get_str_data_port_range()],
list):
SysOut.terminate_string("Port range must be list.")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
SysOut.terminate_string(
"Port range must be integer.")
elif not isinstance(
cfg[Definition.get_str_master_port()], int):
SysOut.terminate_string(
"Master port must be integer.")
elif len(cfg[
Definition.get_str_data_port_range()]) != 2:
SysOut.terminate_string(
"Port range must compost of two elements: start, stop."
)
elif not isinstance(
cfg[Definition.get_str_idle_time()], int):
SysOut.terminate_string(
"Idle time must be integer.")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
SysOut.terminate_string(
"Start port range must greater than stop port range."
)
else:
Setting.set_node_addr()
import multiprocessing
Setting.__node_name = cfg[
Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[
Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[
Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[
Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[
Definition.get_str_idle_time()]
Setting.__master_addr = cfg[
Definition.get_str_master_addr()].strip()
Setting.__master_port = cfg[
Definition.get_str_master_port()]
Setting.__node_external_addr = cfg[
Definition.get_str_node_external_addr()].strip(
).lower()
# Check for auto node name
if Setting.__node_name.lower() == "auto":
# Get node name from host name
import socket
Setting.__node_name = socket.gethostname()
# Check for overriding node address
if cfg[Definition.get_str_node_internal_addr()] and \
cfg[Definition.get_str_node_internal_addr()] != "auto":
# Set node name automatically from hostname
from harmonicIO.general.services import Services
if Services.is_valid_ipv4(cfg[Definition.get_str_node_internal_addr()]) or \
Services.is_valid_ipv6(cfg[Definition.get_str_node_internal_addr()]):
Setting.__node_internal_addr = cfg[
Definition.get_str_node_internal_addr(
)]
# Check for node address validity
if Setting.get_node_external_addr() != "none":
from harmonicIO.general.services import Services
if Services.is_valid_ipv4(Setting.get_node_external_addr()) or \
Services.is_valid_ipv6(Setting.get_node_external_addr()):
SysOut.out_string(
"By pass request with external address."
)
else:
SysOut.terminate_string(
"Invaliid external ip address!")
else:
Setting.__node_external_addr = None
SysOut.out_string("Load setting successful.")
else:
SysOut.terminate_string(
"Required parameters are not present.")
except Exception as e:
print(e)
SysOut.terminate_string(
"Invalid data in configuration file.")
# Generate a sample stream order
stream_order = [0] * ITEM_NUMBER
import random
for i in range(ITEM_NUMBER):
stream_order[i] = (i, 'daemon_test' if
(random.randrange(1, 100) %
len(d_list)) == 0 else 'daemon_test')
return stream_order, d_list
if __name__ == '__main__':
# Initialize connector driver
SysOut.out_string("Running Harmonic Stream Connector")
sc = StreamConnector(MASTER_DATA["MASTER_ADDR"],
MASTER_DATA["MASTER_PORT"],
token=SETTING["TOKEN"],
std_idle_time=SETTING["IDLE_TIME"],
max_try=SETTING["MAX_TRY"],
source_name=SETTING["SOURCE_NAME"])
if sc.is_master_alive():
SysOut.out_string(
"Connection to the master ({0}:{1}) is successful.".format(
MASTER_DATA["MASTER_ADDR"], MASTER_DATA["MASTER_PORT"]))
else:
SysOut.terminate_string("Master at ({0}:{1}) is not alive!".format(
MASTER_DATA["MASTER_ADDR"], MASTER_DATA["MASTER_PORT"]))
server_thread.daemon = True
SysOut.out_string("Enable Messaging System on port: " + str(Setting.get_data_port_start()))
server_thread.start()
""" Have to test for graceful termination. """
# server.shutdown()
# server.server_close()
if __name__ == '__main__':
"""
Entry point
"""
SysOut.out_string("Running Harmonic Master")
# Load configuration from file
from .configuration import Setting
Setting.read_cfg_from_file()
# Print instance information
SysOut.out_string("Node name: " + Setting.get_node_name())
SysOut.out_string("Node address: " + Setting.get_node_addr())
SysOut.out_string("Node port: " + str(Setting.get_node_port()))
# Create thread for handling REST Service
from concurrent.futures import ThreadPoolExecutor
pool = ThreadPoolExecutor()
# Run messaging system service
if r.status != 200:
SysOut.err_string("Cannot update worker status to the master!")
else:
SysOut.debug_string("Reports status to master node complete.")
except Exception as e:
SysOut.err_string("Master is not available!")
print(e)
if __name__ == "__main__":
"""
Entry point
"""
SysOut.out_string("Running Harmonic Worker")
# Load configuration from file
Setting.read_cfg_from_file()
# Override master and repo address
# Setting.set_variables_from_ev()
# Print instance information
SysOut.out_string("Node name: " + Setting.get_node_name())
SysOut.out_string("Node internal address: " +
Setting.get_node_internal_addr())
if Setting.get_node_external_addr():
SysOut.out_string("Node external address: " +
Setting.get_node_external_addr())
SysOut.out_string("Node port: " + str(Setting.get_node_port()))
def run(self):
SysOut.out_string("REST Ready.....")
self.__server.serve_forever()
def read_cfg_from_file():
from harmonicIO.general.services import Services, SysOut
if not Services.is_file_exist('harmonicIO/master/configuration.json'):
SysOut.terminate_string(
'harmonicIO/master/configuration.json does not exist!')
else:
with open('harmonicIO/master/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from harmonicIO.general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()],
int):
SysOut.terminate_string(
"Node port must be integer!")
if not isinstance(
cfg[Definition.get_str_master_addr()], str):
SysOut.terminate_string(
"Master address must be string!")
elif not isinstance(
cfg[Definition.get_str_data_port_range()],
list):
SysOut.terminate_string("Port range must be list!")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
SysOut.terminate_string(
"Port range must be integer!")
elif len(cfg[
Definition.get_str_data_port_range()]) != 2:
SysOut.terminate_string(
"Port range must compost of two elements: start, stop!"
)
elif not isinstance(
cfg[Definition.get_str_idle_time()], int):
SysOut.terminate_string(
"Idle time must be integer!")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
SysOut.terminate_string(
"Start port range must greater than stop port range!"
)
else:
Setting.__node_name = cfg[
Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[
Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[
Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[
Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[
Definition.get_str_idle_time()]
SysOut.out_string("Load setting successful.")
try:
if cfg[Definition.get_str_master_addr()].lower(
) == "auto":
Setting.__node_addr = Services.get_host_name_i(
)
SysOut.out_string(
"Assigning master ip address automatically."
)
elif Services.is_valid_ipv4(cfg[Definition.get_str_master_addr()]) or \
Services.is_valid_ipv6(cfg[Definition.get_str_master_addr()]):
Setting.set_node_addr(
cfg[Definition.get_str_master_addr()])
else:
SysOut.terminate_string(
"Invalid master IP address format!")
except:
SysOut.terminate_string(
"Cannot assign IP address to the master!")
else:
SysOut.terminate_string(
"Invalid data in configuration file.")
except:
SysOut.terminate_string(
"Invalid data in configuration file.")