def __init__(self):
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# do not change those information
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
self.PORT = 9001
self._client_socket = None
# this DB is using the TalkTalkProtocol
# so each net log message should contain this tag
self.protocol_name = "TalkTalkProtocol - Client: "
# if the target cannot be connected in 10 sec
# then there is a problem
self.socket_timeout = 10
file_helper = DirFileHelper()
self.yaml_file_path = file_helper.get_home_path(
) + "/var/iDDB/iddb.yaml"
# maximum buffer that server can send to this client
self.MAX_RECV_BUFFER = 8192
# reference to the C debug library
self.so_debug_file = '../../out/so_files/log_reader.so'
self.c_db_debug = CDLL(self.so_debug_file)
def get_all_tables(self, database_name, table_to_compare_to_insert):
"""Displays all existing tables from an existing database.
Returns 1 if an error occured, 2 otherwise and we don't do an insert
operation - just listing tables, 3 if the table for the insert operation exists,
4 if the table for the insert operation does not exist"""
error = False
try:
if len(database_name) == 0 or database_name is None:
error = True
except:
error = True
if error:
print("You must specify a database first. Status (-1).\n")
logger = PythonLogger("ERROR")
logger.write_log("TableUtility class - an user's trying to \
list all tables but without specifying an existing database")
return 1
helper_obj = DirFileHelper()
root_db_path = helper_obj.get_home_path() + "var/iDDB/database/"
db_name = root_db_path + database_name
all_table = []
for root, dirs, files in os.walk(db_name):
for file in files:
if file.endswith('.iddb'):
all_table.append(file)
if len(all_table) == 0:
print("There are no tables yet")
else:
# we don't want to display also the file extension
no_extension = 5
if table_to_compare_to_insert is None:
print("All existing tables from database '" + database_name +
"'...")
for i in range(0, len(all_table)):
print all_table[i][:-no_extension]
else:
for i in range(0, len(all_table)):
if all_table[
i][:-no_extension] == table_to_compare_to_insert:
return 3
return 4
return 2
def get_columns_from_specified_table():
global DATABASE_NAME
global TABLE_NAME
global ERROR_CODE
global TABLE_FULL_PATH
global column_names
global column_types
helper_obj = DirFileHelper()
root_db_path = helper_obj.get_home_path() + "var/iDDB/database/"
db_name = root_db_path + DATABASE_NAME + "/"
table_name = db_name + TABLE_NAME + ".iddb"
if path.exists(table_name) is False:
print("Table provided does not exist!")
sys.exit(ERROR_CODE)
columns = []
stop_reading = "###########################################"
stop_reading1 = "table_name:"
counter = 0
f_table = open(table_name, "r")
l_table = f_table.readlines()
for line in l_table:
temp_line = line.strip()
if stop_reading1 in temp_line:
break
if counter == 2:
break
if temp_line == stop_reading:
counter += 1
continue
columns.append(temp_line)
for i in range(0, len(columns)):
column_names.append(find_between(columns[i], ":", "-"))
column_types.append(find_between(columns[i], "-", " "))
TABLE_FULL_PATH = table_name
f_table.close()
def get_server_ip_address():
all_ip = []
# get yaml file path, if doesn't exist, exit script...
file_helper = DirFileHelper()
yaml_file_path = file_helper.get_home_path() + "var/iDDB/iddb.yaml"
try:
with open(yaml_file_path) as f:
for line in f:
curr_line = line.strip()
if "#" in curr_line or len(curr_line) <= 0:
continue
# this is what we need
if "ip_node=" in curr_line:
temp_result = find_between(curr_line, "=", " ")
all_ip = temp_result.split(";")
break
except IOError:
print(
"It looks like iddb is not installed in this system or the yaml config file is missing. Please resolve them before executing this script"
)
print("Nothing inserted...")
sys.exit(ERROR_CODE)
return all_ip
class PythonLogger:
def __init__(self, log_level, started_db=False):
self.log_level = log_level
if started_db == False:
self.system_file_log = "system.log"
else:
self.system_file_log = "iDDB_init.log"
# TODO - modify this path when available
self.helper_obj = DirFileHelper()
self.system_log_path = self.helper_obj.get_home_path(
) + "var/log/iDDB/"
self.log_file = self.system_log_path + self.system_file_log
"""
Log levels accepted:
1. INFO
2. DEBUG
3. WARN (WARNING)
4. ERROR
They must be specified for each info that will be added/appended
into the log file (system.log)
If an invalid log level will be specified the the INFO one will
be considered being by default
"""
if self.log_level != "INFO" and self.log_level != "WARN" and \
self.log_level != "DEBUG" and self.log_level != "ERROR":
self.log_level = "INFO"
def write_log(self, info, mode="a"):
"""
This is the main method that will be used in this class - its aim is to
write the actual info into the system.log file (the default one) or into
the iDDB_init.log file
:info - actual information to be added into the specific file
:mode - this can have the following values:
1) a - append info
2) w - truncate the file before adding new info in it (usually
this will be used when thr database process starts)
Returns:
"""
if mode != "a" and mode != "w":
mode = "a"
if info is None or len(info) == 0:
info = "Internal error occurred..."
initial_info = self.get_current_timestamp(
) + "[" + self.log_level + "]" + ": "
try:
log_file = open(self.log_file, mode)
except IOError:
open(self.log_file, 'a').close()
log_file = open(self.log_file, mode)
usefull_info = initial_info + info + "\n"
log_file.write(usefull_info)
log_file.close()
def get_current_timestamp(self):
ts = time.time()
timestamp = datetime.datetime.fromtimestamp(ts).strftime(
'%Y-%m-%d %H:%M:%S')
return timestamp
def test_home_path(self):
# Tests if the Python API used for getting user's HOME
# variable value (home path) works (e.g. it shouldn't return
# an empty value)
dir_helper = DirFileHelper()
self.assertIsNotNone(dir_helper.get_home_path())
import sys
sys.path.insert(0, "../db_helper/python_helper/file_helper/")
from dir_file_helper import DirFileHelper
PYTHON_LOGGER_INIT = "Python: "
# get $HOME path
print (PYTHON_LOGGER_INIT + "Getting the $HOME path...")
dir_file_helper_object = DirFileHelper()
home_path = dir_file_helper_object.get_home_path()
# let's do our job from now on...
def find_between(s, start, end):
"""Utility function used to extract the desired substring
from a given string. Needed, for example, to extract the table name"""
try:
return (s.split(start))[1].split(end)[0]
except IndexError:
return None
if len(sys.argv) != 4:
print("Unexpected error occurred. Exiting...")
sys.exit(-1)
helper_obj = DirFileHelper()
CACHE_CSV_PATH = helper_obj.get_home_path() + "var/iDDB/" + sys.argv[3]
DATABASE_NAME = sys.argv[1]
TABLE_NAME = sys.argv[2]
# we'll wait between iterations 1 sec, in this way we make
# sure that the remotes can handle this request
WAIT_BEFORE_CACHE_ITERATION = 1
def remove_cached_csv(csv_file):
command = os.popen("rm -rf " + csv_file)
def handle_csv_file():
bulk_string_protocol = ""
cache_content = []
def start_real_server(self):
if self.start_server_preconditions():
logger = PythonLogger("INFO")
logger.write_log(self.protocol_name + "Starting the server...")
while True:
c, addr = self._server_socket.accept()
if self.get_server_status() is False:
self.stop_real_server()
break
# we know that the client sent an action
# so we should handle it here, depending on the
# header message
data = c.recv(self.MAX_RECV_BUFFER).decode()
if len(data) > 0:
identifier = data.split("#$")[0]
body = data.split("#$")[1]
isTalkTalkProtocolMessageOK = False
try:
unknown_part = data.split("#$")[2]
except IndexError:
isTalkTalkProtocolMessageOK = True
if isTalkTalkProtocolMessageOK is False:
c.send(self.NOK_MSG)
else:
# get a reference to the C driver
so_file = '../out/so_files/database_manipulation.so'
c_db = CDLL(so_file)
if "create_db" in identifier:
c_return = c_db.create_database(str(body))
if c_return != 1:
c.send(self.NOK_MSG)
else:
c.send(self.OK_MSG)
elif "remove_db" in identifier:
c_return = c_db.delete_empty_database(str(body))
if c_return != 1:
c.send(self.NOK_MSG)
else:
c.send(self.OK_MSG)
elif "create_tb" in identifier:
body_parts = body.split("!")
isBodyPartOk = False
try:
aux_part = body_parts[3]
except IndexError:
isBodyPartOk = True
if isBodyPartOk:
# we must override this because
# there's another section from the C driver
# which handles table operations
so_file = '../out/so_files/table_manipulation.so'
c_db = CDLL(so_file)
c_return = c_db.create_empty_table(
str(body_parts[0]), str(body_parts[1]),
str(body_parts[2]))
if c_return != 1:
c.send(self.NOK_MSG)
else:
c.send(self.OK_MSG)
else:
c.send(self.NOK_MSG)
elif "remove_tb" in identifier:
body_parts = body.split("!")
isBodyPartOk = False
try:
aux_part = body_parts[2]
except IndexError:
isBodyPartOk = True
if isBodyPartOk:
# we must override this because
# there's another section from the C driver
# which handles table operations
so_file = '../out/so_files/table_manipulation.so'
c_db = CDLL(so_file)
c_return = c_db.remove_table(
str(body_parts[0]), str(body_parts[1]))
if c_return != 1:
c.send(self.NOK_MSG)
else:
c.send(self.OK_MSG)
else:
c.send(self.NOK_MSG)
elif "insert_tb" in identifier:
# we don't need to validate input here because
# we did this in the user_cli.py phase (if we insert
# data via CLI)
message_to_be_inserted = body.split("&*()")
# when doing a bulk insert we don't want to send a lot of
# message through Internet, in this way we're trying to avoid
# congestion of the network
should_message_back = True
final_content_on_bulk_insert = ""
content = None
db_name = None
table_name = None
if "insert_tb_bulk" in identifier:
should_message_back = False
for i in range(0, len(message_to_be_inserted)):
# this case if possible when doing a bulk insert
if len(message_to_be_inserted[i]) == 0:
continue
body_parts = message_to_be_inserted[i].split(
"!")
db_name = body_parts[0]
table_name = body_parts[1]
if db_name is None or table_name is None:
c.send(self.NOK_MSG)
else:
content = body_parts[2]
try:
aux_content = body_parts[3]
c.send(self.NOK_MSG)
except IndexError:
final_content_on_bulk_insert += content + "\n"
# we should go here, otherwise something is really wrong
# and send NOK to the client
if should_message_back:
so_file = '../out/so_files/table_manipulation.so'
c_db = CDLL(so_file)
c_return = c_db.do_insert_db(
str(db_name), str(table_name),
str(content))
if c_return != 1:
c.send(self.NOK_MSG)
else:
c.send(self.OK_MSG)
else:
tb_utility = HelpingServer()
helper_obj = DirFileHelper()
table_path = helper_obj.get_home_path(
) + "var/iDDB/database/" + db_name + "/" + table_name + ".iddb"
tb_utility.do_insert(
table_path,
final_content_on_bulk_insert[:-1])
elif "truncate_tb" in identifier:
# FIXME - when enabling the failing import
# enable also the following line (this is the correct way
# to truncate a table, now we only use a hacky way)
#tb_utility = TableUtility(None)
tb_utility = HelpingServer()
helper_obj = DirFileHelper()
table_path = helper_obj.get_home_path() + body
if tb_utility.delete_from_table(table_path) == 0:
c.send(self.NOK_MSG)
else:
c.send(self.OK_MSG)
elif "select_tb" in identifier:
body_parts = body.split("!")
so_file = '../out/so_files/table_manipulation.so'
c_db = CDLL(so_file)
c_return = c_db.select_all_from_table(
str(body_parts[0]), str(body_parts[1]), 3, 1)
# this is a special case, we must take into consideration all risks when doing this
c.send(str(c_return))
else:
c.send(self.NOK_MSG)
if self.c_db_debug.is_debug() == 1:
logger = PythonLogger("DEBUG")
logger.write_log(self.protocol_name +
"Got connection from " + str(addr))
c.close()
else:
logger = PythonLogger("ERROR")
logger.write_log(self.protocol_name + "Communication error...")
class DatabaseUtility:
def __init__(self, database=None):
self.database = database
# FIXME - very important
# in the C driver, there's a function which returns
# the current user path: char * home_path()
# please use it instead of hardcoding this
self.helper_obj = DirFileHelper()
self.database_path = self.helper_obj.get_home_path(
) + "var/iDDB/database"
self.current_database_file = self.helper_obj.get_home_path(
) + "var/iDDB/current_database.idbb"
self.so_file = '../../out/so_files/log_reader.so'
self.c_db = CDLL(self.so_file)
def get_all_databases(self):
"""Returns all existing databases"""
folders = list(filter(lambda x: os.path.isdir \
(os.path.join(self.database_path, x)), \
os.listdir(self.database_path)))
if len(folders) == 0:
return "There are no databases yet"
else:
return folders
def convert_to_binary(self, db_name):
message_bytes = str(db_name).encode('ascii')
base64_bytes = base64.b64encode(message_bytes)
base64_db_name = base64_bytes.decode('ascii')
return base64_db_name
def save_current_database(self, db_name):
# we want to overwrite the file content
# so every time the database will change
f = open(self.current_database_file, "w")
f.write(self.convert_to_binary(db_name))
f.close()
def get_current_database(self):
try:
f = open(self.current_database_file, "r")
file_content = f.read()
f.close()
base64_bytes = str(file_content).encode('ascii')
message_bytes = base64.b64decode(base64_bytes)
message = message_bytes.decode('ascii')
return message
except IOError:
return None
def remove_database_file_content(self):
if os.path.exists(self.current_database_file):
os.remove(self.current_database_file)
c_return = self.c_db.is_debug()
if c_return == 1:
logger = PythonLogger("DEBUG")
logger.write_log(
"Current/existing database reference was removed...")