def main():
# Update configuration from the local file
from .configuration import Setting
Setting.read_cfg_from_file()
# Print instance information
print("Node name: {0}\nNode address: {1}".format(Setting.get_node_name(), Setting.get_node_addr()))
# Reset data in the database
from .meta_storage import MetaStorage
meta_storage = MetaStorage()
meta_storage.drop_database()
meta_storage.close_connection()
print("Clear data in the database complete.")
# Reset data in the local storage
from general.services import Services
if not Services.is_folder_exist(Setting.get_local_storage()):
Services.t_print(Setting.get_local_storage() + " does not exist! (Local Storage).")
# Get file from the folder
import glob
import os
files = glob.glob(Setting.get_local_storage() + "*")
for file in files:
os.remove(file)
print("Clear {0} files in the local storage complete.".format(len(files)))
# Create a thread for running REST service
from concurrent.futures import ThreadPoolExecutor
pool = ThreadPoolExecutor()
pool.submit(run_rest_service)
def __push_stream_end_point(self, target, data):
# Create a client socket to connect to server
s = None
for res in socket.getaddrinfo(target[0], target[1], socket.AF_UNSPEC, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
try:
s = socket.socket(af, socktype, proto)
except OSError as msg:
s = None
continue
try:
s.connect(sa)
except OSError as msg:
s.close()
s = None
continue
break
if s is None:
print('could not open socket')
Services.e_print("Cannot connect to " + target[0] + ":" + str(target[1]))
return False
with s:
# Identifying object id
data[0:8] = struct.pack(">Q", target[2])
s.sendall(data)
s.sendall(b'')
s.close()
return True
def __init__(self, source_folder='/', file_extension='*'):
# Check that folder is exist or not, if not throw exception
if not Services.is_folder_exist(source_folder):
Services.t_print("{0} does not exist!".format(source_folder))
if source_folder[-1] != '/':
source_folder += '/'
import time
import glob
self.__source_files = glob.glob(source_folder + '*.' + file_extension)
self.__source_folder = source_folder
self.__index = 0
print("{0} local files register.".format(len(self.__source_files)))
if len(self.__source_files) == 0:
Services.t_print("No files in {0}".format(source_folder))
# Start reading data into memory
time1 = time.time()
self.__data = list()
for i, file in enumerate(self.__source_files):
b = bytearray()
self.__get_data(b, i)
self.__data.append(b)
time2 = time.time()
print("Load all {0} files complete in {1} seconds.".format(len(self.__source_files), time2 - time1))
def __init__(self, tuple_rate_file):
if Services.is_folder_exist(tuple_rate_file):
Services.t_print("{0} does not exist!".format(tuple_rate_file))
with open(tuple_rate_file) as t:
self.__tuple_rate = eval(t.read())
self.__cur_rate = 0
self.__next_frame()
def run_pe(self):
# For production setting
for i in range(Setting.get_max_worker()):
# For testing setting
port = self.__get_available_port()
if not port:
Services.e_print("Important: no more port available.")
break
self.__pe_pool.map(run_microbatch, (str(port), ))
def __init__(self, source_folder='/', file_extension='*'):
# Check that folder is exist or not, if not throw exception
if not Services.is_folder_exist(source_folder):
Services.t_print("{0} does not exist!".format(source_folder))
if source_folder[-1] != '/':
source_folder += '/'
import glob
self.__source_files = glob.glob(source_folder + '*.' + file_extension)
self.__source_folder = source_folder
self.__index = 0
print("{0} local files register.".format(len(self.__source_files)))
if len(self.__source_files) == 0:
Services.t_print("No files in {0}".format(source_folder))
def send_data(self, data):
# The data must be byte array
if not isinstance(data, bytearray):
Services.t_print("Data type must by byte array in send_data method in StreamConnector")
if len(data) == 0:
print("No content in byte array.")
return None
c_target = self.__get_stream_end_point()
while not c_target:
time.sleep(Setting.get_std_idle_time())
c_target = self.__get_stream_end_point()
while not self.__push_stream_end_point(c_target, data):
time.sleep(Setting.get_std_idle_time())
print("Send data to " + c_target[0] + ":" + str(c_target[1]) + " successful.")
def __get_stream_end_point(self):
response = self.__connector.request('GET', Definition.Server.get_str_push_req(Setting.get_server_addr(),
Setting.get_server_port(),
Setting.get_token()))
if response.status == 406:
# Messages in queue is full. Result in queue lock.
Services.e_print("Queue in master is full.")
return False
elif response.status != 200:
return False
try:
content = json.loads(response.data.decode('utf-8'))
return (content['c_addr'], int(content['c_port']), int(content['t_id']), )
except:
Services.t_print("JSON content error from the master!\n" + response.data.decode('utf-8'))
def handle(self):
# Receive and interpret the request data
data = bytearray()
"""
Discard heading for now
data += self.request.recv(16)
# Interpret the header for file size
file_size = struct.unpack(">Q", data[8:16])[0]
"""
try:
c = True
while c != b"":
c = self.request.recv(2048)
data += c
# Then, push data messaging system.
MessagesQueue.push_to_queue(data)
except:
from general.services import Services
Services.e_print("Insufficient memory for storing g object.")
def handle(self):
# Receive and interpret the request data
data = bytearray()
"""
Discard heading for now
data += self.request.recv(16)
# Interpret the header for file size
file_size = struct.unpack(">Q", data[8:16])[0]
"""
try:
c = True
while c != b"":
c = self.request.recv(2048)
data += c
# Then, push data messaging system.
MessagesQueue.push_to_queue(data)
except:
from general.services import Services
Services.e_print("Insufficient memory for storing g object.")
def __init__(self):
self.__pe_pool = concurrent.futures.ProcessPoolExecutor(
max_workers=Setting.get_max_worker())
self.__ports = []
# Define port status
for port_num in range(Setting.get_data_port_start(),
Setting.get_data_port_stop()):
self.__ports += [ChannelStatus(port_num)]
# Check number of available port
available_port = 0
for item in self.__ports:
if item.status == CStatus.AVAILABLE:
available_port += 1
if available_port < Setting.get_max_worker():
Services.e_print(
"Important: Port number that can be used is less than the number of workers!"
)
self.__available_port = available_port
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('data_repository/configuration.json'):
Services.t_print(
'data_repository/configuration.json does not exist')
else:
with open('data_repository/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
from general.definition import Definition
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_mongodb_setting() in cfg:
if not isinstance(cfg[Definition.get_str_node_port()],
int):
Services.t_print("Node port must be number!")
Setting.__node_name = cfg[
Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.set_node_addr()
# Check for the internal structure of database setting
db_setting = cfg[Definition.get_str_mongodb_setting()]
if Definition.MongoDB.get_str_connection_string() in db_setting and \
Definition.MongoDB.get_str_db_name() in db_setting and \
Definition.MongoDB.get_str_db_feature() in db_setting and \
Definition.MongoDB.get_str_db_tree() in db_setting and \
Definition.MongoDB.get_str_db_meta() in db_setting and \
Definition.MongoDB.get_str_lc_storage() in db_setting:
# Setting mongodb table
Setting.__mg_connection_string = db_setting[
Definition.MongoDB.get_str_connection_string(
)].strip()
Setting.__db_name = db_setting[
Definition.MongoDB.get_str_db_name()].strip()
Setting.__table_feature_name = db_setting[
Definition.MongoDB.get_str_db_feature()].strip()
Setting.__table_tree_name = db_setting[
Definition.MongoDB.get_str_db_tree()].strip()
Setting.__table_meta_name = db_setting[
Definition.MongoDB.get_str_db_meta()].strip()
Setting.__lc_storage = db_setting[
Definition.MongoDB.get_str_lc_storage()].strip()
try:
pass
except:
Services.t_print("Invalid data in configuration file.")
def set_node_addr(addr=None):
if addr:
Setting.__node_addr = addr
else:
import socket
from general.services import Services
Setting.__node_addr = socket.gethostname()
# if addr is valid
if Services.is_valid_ipv4(Setting.__node_addr) or Services.is_valid_ipv6(Setting.__node_addr):
return None
# if addr is not valid
Setting.__node_addr = Services.get_host_name_i()
if Services.is_valid_ipv4(Setting.__node_addr) or Services.is_valid_ipv6(Setting.__node_addr):
return None
Services.t_print("Cannot get node ip address!")
def set_node_addr(addr=None):
if addr:
Setting.__node_addr = addr
else:
import socket
from general.services import Services
Setting.__node_addr = socket.gethostname()
# if addr is valid
if Services.is_valid_ipv4(Setting.__node_addr) or Services.is_valid_ipv6(Setting.__node_addr):
return None
# if addr is not valid
Setting.__node_addr = Services.get_host_name_i()
if Services.is_valid_ipv4(Setting.__node_addr) or Services.is_valid_ipv6(Setting.__node_addr):
return None
Services.t_print("Cannot get node ip address!")
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('data_repository/configuration.json'):
Services.t_print('data_repository/configuration.json does not exist')
else:
with open('data_repository/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
from general.definition import Definition
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_mongodb_setting() in cfg:
if not isinstance(cfg[Definition.get_str_node_port()], int):
Services.t_print("Node port must be number!")
Setting.__node_name = cfg[Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.set_node_addr()
# Check for the internal structure of database setting
db_setting = cfg[Definition.get_str_mongodb_setting()]
if Definition.MongoDB.get_str_connection_string() in db_setting and \
Definition.MongoDB.get_str_db_name() in db_setting and \
Definition.MongoDB.get_str_db_feature() in db_setting and \
Definition.MongoDB.get_str_db_tree() in db_setting and \
Definition.MongoDB.get_str_db_meta() in db_setting and \
Definition.MongoDB.get_str_lc_storage() in db_setting:
# Setting mongodb table
Setting.__mg_connection_string = db_setting[Definition.MongoDB.get_str_connection_string()].strip()
Setting.__db_name = db_setting[Definition.MongoDB.get_str_db_name()].strip()
Setting.__table_feature_name = db_setting[Definition.MongoDB.get_str_db_feature()].strip()
Setting.__table_tree_name = db_setting[Definition.MongoDB.get_str_db_tree()].strip()
Setting.__table_meta_name = db_setting[Definition.MongoDB.get_str_db_meta()].strip()
Setting.__lc_storage = db_setting[Definition.MongoDB.get_str_lc_storage()].strip()
try:
pass
except:
Services.t_print("Invalid data in configuration file.")
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('master/configuration.json'):
Services.t_print('master/configuration.json does not exist')
else:
with open('master/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()], int):
Services.t_print("Node port must be integer")
elif not isinstance(cfg[Definition.get_str_data_port_range()], list):
Services.t_print("Port range must be list")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
Services.t_print("Port range must be integer")
elif len(cfg[Definition.get_str_data_port_range()]) != 2:
Services.t_print("Port range must compost of two elements: start, stop")
elif not isinstance(cfg[Definition.get_str_idle_time()], int):
Services.t_print("Idle time must be integer")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
Services.t_print("Start port range must greater than stop port range")
else:
Setting.__node_name = cfg[Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[Definition.get_str_idle_time()]
print("Load setting successful")
if Services.is_valid_ipv4(cfg[Definition.get_str_master_addr()]) or \
Services.is_valid_ipv6(cfg[Definition.get_str_master_addr()]):
Setting.set_node_addr(cfg[Definition.get_str_master_addr()])
else:
print("Assigning master ip address automatically.")
Setting.set_node_addr()
else:
Services.t_print("Invalid data in configuration file.")
except:
Services.t_print("Invalid data in configuration file.")
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('master/configuration.json'):
Services.t_print('master/configuration.json does not exist')
else:
with open('master/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()],
int):
Services.t_print("Node port must be integer")
elif not isinstance(
cfg[Definition.get_str_data_port_range()],
list):
Services.t_print("Port range must be list")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
Services.t_print("Port range must be integer")
elif len(cfg[
Definition.get_str_data_port_range()]) != 2:
Services.t_print(
"Port range must compost of two elements: start, stop"
)
elif not isinstance(
cfg[Definition.get_str_idle_time()], int):
Services.t_print("Idle time must be integer")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
Services.t_print(
"Start port range must greater than stop port range"
)
else:
Setting.__node_name = cfg[
Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[
Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[
Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[
Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[
Definition.get_str_idle_time()]
print("Load setting successful")
if Services.is_valid_ipv4(cfg[Definition.get_str_master_addr()]) or \
Services.is_valid_ipv6(cfg[Definition.get_str_master_addr()]):
Setting.set_node_addr(
cfg[Definition.get_str_master_addr()])
else:
print("Assigning master ip address automatically.")
Setting.set_node_addr()
else:
Services.t_print("Invalid data in configuration file.")
except:
Services.t_print("Invalid data in configuration file.")
def __next_frame(self):
if len(self.__tuple_rate) > 0:
self.__cur_rate = self.__tuple_rate.pop(0)
else:
Services.t_print("No tuple rate in the next frame")
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('worker/configuration.json'):
Services.t_print('worker/configuration.json does not exist')
else:
with open('worker/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg and \
Definition.get_str_ext_process() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_master_port() in cfg and \
Definition.get_str_repo_addr() in cfg and \
Definition.get_str_repo_port() in cfg and \
Definition.get_str_workers_num() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()], int):
Services.t_print("Node port must be integer")
elif not isinstance(cfg[Definition.get_str_data_port_range()], list):
Services.t_print("Port range must be list")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
Services.t_print("Port range must be integer")
elif not isinstance(cfg[Definition.get_str_master_port()], int):
Services.t_print("Master port must be integer")
elif len(cfg[Definition.get_str_data_port_range()]) != 2:
Services.t_print("Port range must compost of two elements: start, stop")
elif not isinstance(cfg[Definition.get_str_idle_time()], int):
Services.t_print("Idle time must be integer")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
Services.t_print("Start port range must greater than stop port range")
elif not isinstance(cfg[Definition.get_str_repo_port()], int):
Services.t_print("Repository port must be integer")
else:
Setting.set_node_addr()
import multiprocessing
Setting.__node_name = cfg[Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[Definition.get_str_idle_time()]
Setting.__ext_process = cfg[Definition.get_str_ext_process()]
Setting.__master_addr = cfg[Definition.get_str_master_addr()].strip()
Setting.__master_port = cfg[Definition.get_str_master_port()]
Setting.__repo_addr = cfg[Definition.get_str_repo_addr()].strip()
Setting.__repo_port = cfg[Definition.get_str_repo_port()]
# Check for the number of worker
if isinstance(cfg[Definition.get_str_workers_num()], str):
if cfg[Definition.get_str_workers_num()].lower() == "auto" or \
cfg[Definition.get_str_workers_num()].lower() == "max":
Setting.__max_workers = multiprocessing.cpu_count()
elif cfg[Definition.get_str_workers_num()].lower() == "min":
Setting.__max_workers = Setting.get_min_worker()
else:
Services.t_print("Invalid number of PCs.")
elif isinstance(cfg[Definition.get_str_workers_num()], int):
Setting.__max_workers = cfg[Definition.get_str_workers_num()]
if Setting.__max_workers < 1:
Services.t_print("Invalid number of PCs.")
else:
print("Number of PCs is overridden to " + str(cfg[Definition.get_str_workers_num()]) + ".")
else:
Services.t_print("Invalid number of PCs.")
# Check for auto node name
if Setting.__node_name.lower() == "auto":
# Get node name from host name
import socket
Setting.__node_name = socket.gethostname()
# Check for overriding node address
if cfg["node_addr"] and cfg["node_addr"] != "auto":
# Set node name automatically from hostname
from general.services import Services
if Services.is_valid_ipv4(cfg["node_addr"]) or Services.is_valid_ipv6(cfg["node_addr"]):
Setting.__node_addr = cfg["node_addr"]
print("Load setting successful")
else:
Services.t_print("Required parameters are not present.")
except:
Services.t_print("Invalid data in configuration file.")
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('data_source/configuration.json'):
Services.t_print('data_source/configuration.json does not exist')
else:
with open('data_source/configuration.json', 'rt') as t:
try:
import json
cfg = json.loads(t.read())
from general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_server_addr() in cfg and \
Definition.get_str_server_port() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_idle_time() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_server_port()], int):
Services.t_print("Server port must be integer")
elif not isinstance(cfg[Definition.get_str_node_port()], int):
Services.t_print("Node port must be integer")
elif not isinstance(cfg[Definition.get_str_idle_time()], int):
Services.t_print("Node port must be integer")
else:
Setting.set_node_addr()
Setting.__node_name = cfg[Definition.get_str_node_name()].strip()
Setting.__server_addr = cfg[Definition.get_str_server_addr()].strip()
Setting.__server_port = cfg[Definition.get_str_server_port()]
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.__std_idle_time = cfg[Definition.get_str_idle_time()]
print("Load setting successful")
except:
Services.t_print("Invalid setting in configuration file.")