def on_get(self, req, res):
"""
GET: /streamRequest?token=None
This function is mainly respond with the available channel for streaming from data source.
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Check for the available channel
channel = PEChannels.get_available_channel(group="optimize")
if channel:
# If channel is available
res.body = Definition.get_channel_response(channel[0], channel[1], MessagingServices.get_new_msg_id())
res.content_type = "String"
res.status = falcon.HTTP_200
else:
if MessagesQueue.is_queue_available():
# Channel is not available, respond with messaging system channel
res.body = Definition.get_channel_response(Setting.get_node_addr(), Setting.get_data_port_start(),
MessagingServices.get_new_msg_id())
res.content_type = "String"
res.status = falcon.HTTP_200
else:
# Message in queue is full
res.body = Definition.get_channel_response("0.0.0.0", 0, 0)
res.content_type = "String"
res.status = falcon.HTTP_406
def on_get(self, req, res):
"""
Purpose: Get data from the database
GET: /dataRepository?token={None}
GET: /dataRepository?token={None}&command={count}
GET: /dataRepository?token={None}&command={get_features}
"""
# Check for the presence of token attribute
if Definition.get_str_token() not in req.params:
"""Token is required"""
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return None
# Check for the value of token
token_value = req.params[Definition.get_str_token()].strip()
if token_value == Setting.get_token():
"""If the token is valid"""
if df.Rest.get_string_req_command() in req.params:
"""If the request contain command option """
if req.params[df.Rest.get_string_req_command()] == df.Rest.get_string_count_features():
"""
GET: /dataRepository?token={None}&command={count}
Meaning: Count total record in the table
"""
res.body = str(self.__meta_storage.count_total_features())
res.content_type = "String"
res.status = falcon.HTTP_200
elif req.params[df.Rest.get_string_req_command()] == df.Rest.get_string_dump_features():
"""
GET: /dataRepository?token={None}&command={get_features}
Meaning: Get all features from the database. The content will be form in an tar format.
"""
res.data = self.__meta_storage.get_all_features().getvalue()
res.content_type = "Byte"
res.status = falcon.HTTP_200
else:
res.body = "Unknown command"
res.content_type = "String"
res.status = falcon.HTTP_401
else:
"""
GET: /dataRepository?token={None}
Meaning: Select all from the table, because no parameter is specified
"""
res.body = str(self.__meta_storage.dump_feature_table())
res.content_type = "String"
res.status = falcon.HTTP_200
else:
"""
Invalid Token
"""
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
def on_get(self, req, res):
"""
GET: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if req.params[Definition.get_str_token()] == Setting.get_token():
result = self.get_machine_status()
res.body = '{ "' + Definition.get_str_node_name(
) + '": "' + Setting.get_node_name() + '", \
"' + Definition.get_str_node_role(
) + '": "data_source", \
"' + Definition.get_str_node_addr(
) + '": "' + Setting.get_node_addr() + '", \
"' + Definition.get_str_load1(
) + '": ' + result[0] + ', \
"' + Definition.get_str_load5(
) + '": ' + result[1] + ', \
"' + Definition.get_str_load15(
) + '": ' + result[2] + ' }'
res.content_type = "String"
res.status = falcon.HTTP_200
else:
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
def on_get(self, req, res):
"""
GET: /messagesQuery?token=None&command=queueLength
This function inquiry about the number of messages in queue. For dealing with create a new instance.
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not Definition.MessagesQueue.get_str_command() in req.params:
res.body = "No command specified."
res.content_type = "String"
res.status = falcon.HTTP_406
return
if req.params[Definition.MessagesQueue.get_str_command()] == Definition.MessagesQueue.get_str_queue_length():
res.body = str(MessagesQueue.get_queue_length())
res.content_type = "String"
res.status = falcon.HTTP_200
return
if req.params[Definition.MessagesQueue.get_str_command()] == Definition.MessagesQueue.get_str_current_id():
res.body = str(MessagingServices.get_current_id())
res.content_type = "String"
res.status = falcon.HTTP_200
return
if req.params[Definition.MessagesQueue.get_str_command()] == Definition.ChannelStatus.get_str_pe_status():
res.body = str(PEChannels.view_available_channel())
res.content_type = "String"
res.status = falcon.HTTP_200
return
def get_machine_status(self):
"""
Get machine status by calling a unix command and fetch for load average
"""
res = str(subprocess.check_output(Definition.get_cpu_load_command())).strip()
res = res.replace(",", "").replace("\\n", "").replace("'", "")
*_, load1, load5, load15 = res.split(" ")
return load1, load5, load15
def get_machine_status(self):
"""
Get machine status by calling a unix command and fetch for load average
"""
res = str(subprocess.check_output(Definition.get_cpu_load_command())).strip()
res = res.replace(",", "").replace("\\n", "").replace("'", "")
*_, load1, load5, load15 = res.split(" ")
return load1, load5, load15
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('data_repository/configuration.json'):
Services.t_print(
'data_repository/configuration.json does not exist')
else:
with open('data_repository/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
from general.definition import Definition
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_mongodb_setting() in cfg:
if not isinstance(cfg[Definition.get_str_node_port()],
int):
Services.t_print("Node port must be number!")
Setting.__node_name = cfg[
Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.set_node_addr()
# Check for the internal structure of database setting
db_setting = cfg[Definition.get_str_mongodb_setting()]
if Definition.MongoDB.get_str_connection_string() in db_setting and \
Definition.MongoDB.get_str_db_name() in db_setting and \
Definition.MongoDB.get_str_db_feature() in db_setting and \
Definition.MongoDB.get_str_db_tree() in db_setting and \
Definition.MongoDB.get_str_db_meta() in db_setting and \
Definition.MongoDB.get_str_lc_storage() in db_setting:
# Setting mongodb table
Setting.__mg_connection_string = db_setting[
Definition.MongoDB.get_str_connection_string(
)].strip()
Setting.__db_name = db_setting[
Definition.MongoDB.get_str_db_name()].strip()
Setting.__table_feature_name = db_setting[
Definition.MongoDB.get_str_db_feature()].strip()
Setting.__table_tree_name = db_setting[
Definition.MongoDB.get_str_db_tree()].strip()
Setting.__table_meta_name = db_setting[
Definition.MongoDB.get_str_db_meta()].strip()
Setting.__lc_storage = db_setting[
Definition.MongoDB.get_str_lc_storage()].strip()
try:
pass
except:
Services.t_print("Invalid data in configuration file.")
def on_post(self, req, res):
"""
POST: /streamRequest?token=None
This function respond with getting a stream from data source or from messaging system.
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Check that the PE is existing or not, if not insert and respond
if Definition.REST.Batch.get_str_batch_addr() in req.params and \
Definition.REST.Batch.get_str_batch_port() in req.params and \
Definition.REST.Batch.get_str_batch_status() in req.params:
# Check for data type
if req.params[Definition.REST.Batch.get_str_batch_port()].isdigit() and \
req.params[Definition.REST.Batch.get_str_batch_status()].isdigit():
batch_port = int(req.params[Definition.REST.Batch.get_str_batch_port()])
batch_status = int(req.params[Definition.REST.Batch.get_str_batch_status()])
print("There are {0} messages in queue.".format(MessagesQueue.get_queue_length()))
# If queue contain data, ignore update and stream from queue
if MessagesQueue.get_queue_length() > 0 and batch_status == CStatus.AVAILABLE:
res.data = bytes(MessagesQueue.pop_queue(0)[0])
res.content_type = "Bytes"
res.status = falcon.HTTP_203
else:
# Register channel
PEChannels.register_channel(req.params[Definition.REST.Batch.get_str_batch_addr()],
batch_port, batch_status)
res.body = "OK"
res.content_type = "String"
res.status = falcon.HTTP_200
else:
res.body = "Invalid data type!"
res.content_type = "String"
res.status = falcon.HTTP_406
else:
res.body = "Invalid parameters!"
res.content_type = "String"
res.status = falcon.HTTP_406
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('data_repository/configuration.json'):
Services.t_print('data_repository/configuration.json does not exist')
else:
with open('data_repository/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
from general.definition import Definition
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_mongodb_setting() in cfg:
if not isinstance(cfg[Definition.get_str_node_port()], int):
Services.t_print("Node port must be number!")
Setting.__node_name = cfg[Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.set_node_addr()
# Check for the internal structure of database setting
db_setting = cfg[Definition.get_str_mongodb_setting()]
if Definition.MongoDB.get_str_connection_string() in db_setting and \
Definition.MongoDB.get_str_db_name() in db_setting and \
Definition.MongoDB.get_str_db_feature() in db_setting and \
Definition.MongoDB.get_str_db_tree() in db_setting and \
Definition.MongoDB.get_str_db_meta() in db_setting and \
Definition.MongoDB.get_str_lc_storage() in db_setting:
# Setting mongodb table
Setting.__mg_connection_string = db_setting[Definition.MongoDB.get_str_connection_string()].strip()
Setting.__db_name = db_setting[Definition.MongoDB.get_str_db_name()].strip()
Setting.__table_feature_name = db_setting[Definition.MongoDB.get_str_db_feature()].strip()
Setting.__table_tree_name = db_setting[Definition.MongoDB.get_str_db_tree()].strip()
Setting.__table_meta_name = db_setting[Definition.MongoDB.get_str_db_meta()].strip()
Setting.__lc_storage = db_setting[Definition.MongoDB.get_str_lc_storage()].strip()
try:
pass
except:
Services.t_print("Invalid data in configuration file.")
def on_get(self, req, res):
"""
GET: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if req.params[Definition.get_str_token()] == Setting.get_token():
result = self.get_machine_status()
res.body = '{ "' + Definition.get_str_node_name() + '": "' + Setting.get_node_name() + '", \
"' + Definition.get_str_node_role() + '": "data_repository", \
"' + Definition.get_str_node_addr() + '": "' + Setting.get_node_addr() + '", \
"' + Definition.get_str_load1() + '": ' + result[0] + ', \
"' + Definition.get_str_load5() + '": ' + result[1] + ', \
"' + Definition.get_str_load15() + '": ' + result[2] + ' }'
res.content_type = "String"
res.status = falcon.HTTP_200
else:
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('master/configuration.json'):
Services.t_print('master/configuration.json does not exist')
else:
with open('master/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()], int):
Services.t_print("Node port must be integer")
elif not isinstance(cfg[Definition.get_str_data_port_range()], list):
Services.t_print("Port range must be list")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
Services.t_print("Port range must be integer")
elif len(cfg[Definition.get_str_data_port_range()]) != 2:
Services.t_print("Port range must compost of two elements: start, stop")
elif not isinstance(cfg[Definition.get_str_idle_time()], int):
Services.t_print("Idle time must be integer")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
Services.t_print("Start port range must greater than stop port range")
else:
Setting.__node_name = cfg[Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[Definition.get_str_idle_time()]
print("Load setting successful")
if Services.is_valid_ipv4(cfg[Definition.get_str_master_addr()]) or \
Services.is_valid_ipv6(cfg[Definition.get_str_master_addr()]):
Setting.set_node_addr(cfg[Definition.get_str_master_addr()])
else:
print("Assigning master ip address automatically.")
Setting.set_node_addr()
else:
Services.t_print("Invalid data in configuration file.")
except:
Services.t_print("Invalid data in configuration file.")
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('data_source/configuration.json'):
Services.t_print('data_source/configuration.json does not exist')
else:
with open('data_source/configuration.json', 'rt') as t:
try:
import json
cfg = json.loads(t.read())
from general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_server_addr() in cfg and \
Definition.get_str_server_port() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_idle_time() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_server_port()], int):
Services.t_print("Server port must be integer")
elif not isinstance(cfg[Definition.get_str_node_port()], int):
Services.t_print("Node port must be integer")
elif not isinstance(cfg[Definition.get_str_idle_time()], int):
Services.t_print("Node port must be integer")
else:
Setting.set_node_addr()
Setting.__node_name = cfg[Definition.get_str_node_name()].strip()
Setting.__server_addr = cfg[Definition.get_str_server_addr()].strip()
Setting.__server_port = cfg[Definition.get_str_server_port()]
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.__std_idle_time = cfg[Definition.get_str_idle_time()]
print("Load setting successful")
except:
Services.t_print("Invalid setting in configuration file.")
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('master/configuration.json'):
Services.t_print('master/configuration.json does not exist')
else:
with open('master/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()],
int):
Services.t_print("Node port must be integer")
elif not isinstance(
cfg[Definition.get_str_data_port_range()],
list):
Services.t_print("Port range must be list")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
Services.t_print("Port range must be integer")
elif len(cfg[
Definition.get_str_data_port_range()]) != 2:
Services.t_print(
"Port range must compost of two elements: start, stop"
)
elif not isinstance(
cfg[Definition.get_str_idle_time()], int):
Services.t_print("Idle time must be integer")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
Services.t_print(
"Start port range must greater than stop port range"
)
else:
Setting.__node_name = cfg[
Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[
Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[
Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[
Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[
Definition.get_str_idle_time()]
print("Load setting successful")
if Services.is_valid_ipv4(cfg[Definition.get_str_master_addr()]) or \
Services.is_valid_ipv6(cfg[Definition.get_str_master_addr()]):
Setting.set_node_addr(
cfg[Definition.get_str_master_addr()])
else:
print("Assigning master ip address automatically.")
Setting.set_node_addr()
else:
Services.t_print("Invalid data in configuration file.")
except:
Services.t_print("Invalid data in configuration file.")
def on_get(self, req, res):
"""
Purpose: Get data from the database
GET: /dataRepository?token={None}
GET: /dataRepository?token={None}&command={count}
GET: /dataRepository?token={None}&command={get_features}
"""
# Check for the presence of token attribute
if Definition.get_str_token() not in req.params:
"""Token is required"""
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return None
# Check for the value of token
token_value = req.params[Definition.get_str_token()].strip()
if token_value == Setting.get_token():
"""If the token is valid"""
if df.Rest.get_string_req_command() in req.params:
"""If the request contain command option """
if req.params[df.Rest.get_string_req_command(
)] == df.Rest.get_string_count_features():
"""
GET: /dataRepository?token={None}&command={count}
Meaning: Count total record in the table
"""
res.body = str(self.__meta_storage.count_total_features())
res.content_type = "String"
res.status = falcon.HTTP_200
elif req.params[df.Rest.get_string_req_command(
)] == df.Rest.get_string_dump_features():
"""
GET: /dataRepository?token={None}&command={get_features}
Meaning: Get all features from the database. The content will be form in an tar format.
"""
res.data = self.__meta_storage.get_all_features().getvalue(
)
res.content_type = "Byte"
res.status = falcon.HTTP_200
else:
res.body = "Unknown command"
res.content_type = "String"
res.status = falcon.HTTP_401
else:
"""
GET: /dataRepository?token={None}
Meaning: Select all from the table, because no parameter is specified
"""
res.body = str(self.__meta_storage.dump_feature_table())
res.content_type = "String"
res.status = falcon.HTTP_200
else:
"""
Invalid Token
"""
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
def read_cfg_from_file():
from general.services import Services
if not Services.is_file_exist('worker/configuration.json'):
Services.t_print('worker/configuration.json does not exist')
else:
with open('worker/configuration.json', 'rt') as t:
import json
cfg = json.loads(t.read())
try:
from general.definition import Definition
# Check for the json structure
if Definition.get_str_node_name() in cfg and \
Definition.get_str_node_port() in cfg and \
Definition.get_str_data_port_range() in cfg and \
Definition.get_str_idle_time() in cfg and \
Definition.get_str_ext_process() in cfg and \
Definition.get_str_master_addr() in cfg and \
Definition.get_str_master_port() in cfg and \
Definition.get_str_repo_addr() in cfg and \
Definition.get_str_repo_port() in cfg and \
Definition.get_str_workers_num() in cfg:
# Check port number is int or not
if not isinstance(cfg[Definition.get_str_node_port()], int):
Services.t_print("Node port must be integer")
elif not isinstance(cfg[Definition.get_str_data_port_range()], list):
Services.t_print("Port range must be list")
elif not (isinstance(cfg[Definition.get_str_data_port_range()][0], int) and \
isinstance(cfg[Definition.get_str_data_port_range()][1], int)):
Services.t_print("Port range must be integer")
elif not isinstance(cfg[Definition.get_str_master_port()], int):
Services.t_print("Master port must be integer")
elif len(cfg[Definition.get_str_data_port_range()]) != 2:
Services.t_print("Port range must compost of two elements: start, stop")
elif not isinstance(cfg[Definition.get_str_idle_time()], int):
Services.t_print("Idle time must be integer")
elif cfg[Definition.get_str_data_port_range()][0] > \
cfg[Definition.get_str_data_port_range()][1]:
Services.t_print("Start port range must greater than stop port range")
elif not isinstance(cfg[Definition.get_str_repo_port()], int):
Services.t_print("Repository port must be integer")
else:
Setting.set_node_addr()
import multiprocessing
Setting.__node_name = cfg[Definition.get_str_node_name()].strip()
Setting.__node_port = cfg[Definition.get_str_node_port()]
Setting.__node_data_port_start = cfg[Definition.get_str_data_port_range()][0]
Setting.__node_data_port_stop = cfg[Definition.get_str_data_port_range()][1]
Setting.__std_idle_time = cfg[Definition.get_str_idle_time()]
Setting.__ext_process = cfg[Definition.get_str_ext_process()]
Setting.__master_addr = cfg[Definition.get_str_master_addr()].strip()
Setting.__master_port = cfg[Definition.get_str_master_port()]
Setting.__repo_addr = cfg[Definition.get_str_repo_addr()].strip()
Setting.__repo_port = cfg[Definition.get_str_repo_port()]
# Check for the number of worker
if isinstance(cfg[Definition.get_str_workers_num()], str):
if cfg[Definition.get_str_workers_num()].lower() == "auto" or \
cfg[Definition.get_str_workers_num()].lower() == "max":
Setting.__max_workers = multiprocessing.cpu_count()
elif cfg[Definition.get_str_workers_num()].lower() == "min":
Setting.__max_workers = Setting.get_min_worker()
else:
Services.t_print("Invalid number of PCs.")
elif isinstance(cfg[Definition.get_str_workers_num()], int):
Setting.__max_workers = cfg[Definition.get_str_workers_num()]
if Setting.__max_workers < 1:
Services.t_print("Invalid number of PCs.")
else:
print("Number of PCs is overridden to " + str(cfg[Definition.get_str_workers_num()]) + ".")
else:
Services.t_print("Invalid number of PCs.")
# Check for auto node name
if Setting.__node_name.lower() == "auto":
# Get node name from host name
import socket
Setting.__node_name = socket.gethostname()
# Check for overriding node address
if cfg["node_addr"] and cfg["node_addr"] != "auto":
# Set node name automatically from hostname
from general.services import Services
if Services.is_valid_ipv4(cfg["node_addr"]) or Services.is_valid_ipv6(cfg["node_addr"]):
Setting.__node_addr = cfg["node_addr"]
print("Load setting successful")
else:
Services.t_print("Required parameters are not present.")
except:
Services.t_print("Invalid data in configuration file.")