def safe_delete(conn, db, table, item_id, debug=False):
"""
safely deleting row in table
Args:
* [conn] connection of sql
* [db] database name
* [table] table name
* [item_id] the item id want to delete
Returns:
* [sql] str, the command of sql
"""
res = _sql_args_check(table, item_id)
if res is False:
raise Exceptions.SQLInjectionException(
f"SQL injection detected, params: table[{table}] item_id[{item_id}]"
)
if conn is None:
raise Exceptions.InvalidParamException("Conn cannot be null")
# generate sql
str_sql = f"DELETE FROM `{db}`.`{table}` WHERE `id` = '{item_id}'"
if debug: # for deubg
print(str_sql)
# executing sql
return PySQLConnection.execute(conn, str_sql)
def safe_query_tables(conn, db):
"""
safely show tables in schema
Args:
* [conn] connection of sql
* [db] database name
Returns:
* [rows] dict, the execution results
* [sql] str, the command of sql
"""
res = _sql_args_check(db)
if res is False:
raise Exceptions.SQLInjectionException(
f"SQL injection detected, params: db[{db}]")
if conn is None:
raise Exceptions.InvalidParamException("Conn cannot be null")
# generating sql
str_sql = f"SHOW TABLES IN `{db}`"
# executing sql
final_results = []
for i in PySQLConnection.query(conn, str_sql): # obtaining a list
for k, v in i.items():
final_results.append(v)
return final_results
def safe_query_columns(conn, db, table):
"""
safely show columns in table
Args:
* [conn] connection of sql
* [db] database name
* [table] table name
Returns:
* [rows] dict, the execution results
* [sql] str, the command of sql
"""
res = _sql_args_check(db, table)
if res is False:
raise Exceptions.SQLInjectionException(
f"SQL injection detected, params: db[{db}] table[{table}]")
if conn is None:
raise Exceptions.InvalidParamException("Conn cannot be null")
# generating sql
str_sql = f'SHOW COLUMNS IN `{db}`.`{table}`'
# executing sql
return PySQLConnection.query(conn, str_sql) # obtaining a list
def safe_query_id(conn, db, table, item_id, debug=False):
"""
safely querying database, and avoid sql injection attack
Args:
* [conn] connection of sql
* [db] database name
* [table] table name
* [item_id] the item id want to search
* [debug] default to False, if you wannar to see the output sql statement, make it to True
Returns:
* [rows(dict)] the execution results
"""
res = _sql_args_check(db, table, item_id)
if res is False:
raise Exceptions.SQLInjectionException(
f"SQL injection detected, params: table[{table}], id[{item_id}]")
if conn is None:
raise Exceptions.InvalidParamException("Conn cannot be null")
# generate query sentence
str_sql = f"SELECT * FROM `{db}`.`{table}` WHERE `id`='{item_id}'"
if debug: # for debug only
print(str_sql)
# executing sql
return PySQLConnection.query(conn, str_sql)
def create_dict_by_list(key_list, default=None):
"""
given a list, and extend the list to a dictionary type, with all values are set to None (default)
"""
output = {}
if type(default) is dict:
raise Exceptions.InvalidParamException("dict type cannot be set to a new dict value")
if type(default) is list:
if len(default) != len(key_list):
raise Exceptions.InvalidParamException("default as a list have different length to key list")
else:
for i in range(0, len(key_list)):
output[key_list[i]] = default[i]
# return from two list
return output
# else, normal cases
for key in key_list:
output[key] = default
# success
return output
def _append_node(self, leaf: object):
if not isinstance(leaf, CommonTreeNode):
raise Exceptions.InvalidParamException(
"node must be an instance of TreeNode")
if self._check_id_valid(leaf._item_id):
leaf._item_root = self
self._item_leaves.append(leaf)
else:
raise Exceptions.InvalidParamException(
"node id can not be the same in tree")
def encode(self, data: object):
"""
convert given data into redis token
Args:
* [data(any)] data can be anything, bytes, int, float or whatever
Returns:
* [token(bytes)] composed token consisted of data, type, timestamp
"""
# reset
self.reset()
if data is None:
raise excepts.NullPointerException("data cannot be a null type")
# 1. create timestamp
self.timestamp = ticker.time_since1970()
# 2. assign bytes to self.data
if type(data) is bytes:
self.dtype = "bytes"
self.data = convert.binary_to_string(base64.b64encode(data))
else: # other types
self.data = data
if type(data) is int:
self.dtype = "int"
elif type(data) is float:
self.dtype = "float"
elif type(data) is str:
self.dtype = "str"
elif type(data) is list:
self.dtype = "list"
elif type(data) is dict:
self.dtype = "dict"
else:
raise excepts.InvalidParamException(
"{} cannot be processed into string nor bytes".format(
type(data)))
# 3. composing a token dictionary type
self.token = {
"data": self.data,
"type": self.dtype,
"timestamp": self.timestamp
}
# return to caller
return convert.dict_to_binary(self.token)
def safe_multiple_tables_query(conn,
db: str,
tables: list,
select_con,
where_con=None,
order_by=None,
debug=False):
"""
safely multiple table cross querying, not allow nested querying to avoid sql injection.
Args:
* [conn] connection of sql
* [db] database name
* [tables] list of tables to query
* [select_con] selection condition
* [where_con] where condition
* [order_by] order by command, default sequency is asc, if you want a desc results, append "DESC" to your command
* [debug] default to False, if you wannar to see the output sql statement, make it to True
Returns:
* [rows(dict/list)] dict, the execution results
"""
if not isinstance(tables, list):
raise Exceptions.InvalidParamException("tables must be a list")
res = _sql_args_check(db, *tables, select_con, where_con, order_by)
if res is False:
raise Exceptions.SQLInjectionException(
f"SQL injection detected, params: table[{tables}] select[{select_con}] where[{where_con}] order[{order_by}]"
)
if conn is None:
raise Exceptions.InvalidParamException("Conn cannot be null")
# generate simple querying
str_tables = ''
for t in tables:
str_tables += f"`{db}`.`{t}`,"
str_sql = f"SELECT {select_con} FROM {str_tables[:-1]}"
if where_con is not None:
str_sql += f' WHERE {where_con}'
if order_by is not None:
str_sql += f' ORDER BY {order_by}'
if debug: # for debug only
print(str_sql)
# executing sql
return PySQLConnection.query(conn, str_sql)
def parsing_reg_rules(self, rule_list: list, args: dict):
"""
Parse the configuration list, and then verify the validity of the input parameters
the format of rule list:
[argument key] [mapping key] [regular expression]
@Args:
* [ruleList] list
* [args] dict, data contained with key and value
@Returns:
* [dict(str:obj)] dict type
"""
# parse the file
for line in rule_list:
# split token from spaces
seps = line.split(' ')
# check size
if len(seps) != 3:
raise Exceptions.InvalidArithException(
"regular line: {} is broken, [argument key] [mapping key] [regular expression]"
.format(line))
# append regular token to list
self.tokens.append(RegularToken(seps[0], seps[1], seps[2]))
# After parsing the file, first extract the dictionary consisting of {old key-regular match} from the file,
# and then perform regular filtering on the parameters
filtered_args = self.use_args_regular_check(args)
return filtered_args
def reflect_request_with_database_callback(database: PySQLPool,
do_action: object, *args):
"""
用于数据库连接池自释放，提供一个可供用户使用的数据库连接和回调函数
@Args:
* [database] 用于访问的数据库
* [do_action] 回调函数, 函数结构为callback(conn, [args]), 如果不需要附加参数，那么回调函数只为callback(conn)
* [args] 用户需要添加的参数
"""
if not isinstance(database, PySQLPool):
raise Exceptions.NoAvailableResourcesFoundException(
"database is not available")
conn = None
try:
# get connection from pool
conn = database.get_connection()
# check the connection status
if not PySQLConnection.check_connection(conn):
PySQLConnection.reconnect(conn)
# send database connection to callback function
if args is None:
return do_action(conn)
else:
return do_action(conn, args)
finally:
if conn:
database.put_connection(conn)
def update_id(self, new_id):
if type(new_id) is int:
self.lastid = new_id
elif type(new_id) is str:
self.lastid = int(new_id)
else:
raise excepts.InvalidParamException(
"given id must be string or integer")
def cherry_pick(sets: list, item: object):
"""
cherry pick item from sets. For example [1, 2, 3, 4, 5].
After calling cherry_pick(sample, 3).
The rest should be: sample = [1, 2, 4, 5]
"""
from siki.basics import Exceptions
if not sets or len(sets) <= 0:
raise Exceptions.InvalidParamException("the sets is empty")
if not isinstance(item, type(sets[0])):
raise Exceptions.InvalidParamException("the type of element to pick from sets is invalid")
for index in range(0, len(sets)):
if sets[index] == item: # element found
return sets.pop(index)
return None
def safe_update(conn, db, table, item_id, args, debug=False):
"""
safely updating database, and avoid sql injection attack
Args:
* [conn] connection of sql
* [db] database name
* [table] table name
* [item_id] the item id want to update
* [args(dict)] the data to update, something like: {id:1, key1:val1, key2:val2, ...}
* [debug] default to False, if you wannar to see the output sql statement, make it to True
Returns:
* [sql] str, the command of sql
"""
res = _sql_args_check(db, table, item_id, args.keys(), args.values())
if res is False:
raise Exceptions.SQLInjectionException(
f"SQL injection detected, params: table[{table}] item_id[{item_id}] vals[{args}]"
)
if conn is None:
raise Exceptions.InvalidParamException("Conn cannot be null")
# generate sql querying
setval = []
for key, val in args.items():
if val:
setval.append("`" + str(key) + "`='" + str(val) + "'")
else:
setval.append("`" + str(key) + "`=NULL")
p_vals = ", ".join(setval)
# generate sql
str_sql = f"UPDATE `{db}`.`{table}` SET {p_vals} WHERE `id`='{item_id}'"
if debug: # for debug only
print(str_sql)
# executing sql
return PySQLConnection.execute(conn, str_sql)
def _check_id_valid(self, leaf_id):
if not isinstance(leaf_id, type(self._item_id)):
raise Exceptions.InvalidParamException(
"leaf id must be the same type of TreeNode id")
if leaf_id == self._item_id:
return False
for leaf in self._item_leaves:
if leaf.self_id() == leaf_id:
return False
return True
def dequeue(self):
"""
Deleting element from queue
"""
try:
self.lock.acquire()
if len(self.queue) > 0:
return self.queue.pop()
else:
raise Exceptions.EmptyCollectionElementException("Cannot dequeue an empty queue")
finally:
self.lock.release()
def safe_insert(conn, db, table, args, debug=False):
"""
safely inserting database, and avoid sql injection attack
Args:
* [conn] connection of sql
* [db] database name
* [table] table name
* [args(dict)] the data to insert, something like: {id:1, key1:val1, key2:val2, ...}
* [debug] default to False, if you wannar to see the output sql statement, make it to True
Returns:
* [sql] str, command of sql
"""
res = _sql_args_check(db, table, args.keys(), args.values())
if res is False:
raise Exceptions.SQLInjectionException(
f"SQL injection detected, params: table[{table}], args[{args}]")
if conn is None:
raise Exceptions.InvalidParamException("conn cannot be null")
if type(args) is not dict:
raise Exceptions.InvalidParamException("args must be dict type")
# generate a insert sql command
keys = "`" + "`, `".join(args.keys()) + "`"
values = "'" + "', '".join(args.values()) + "'"
# generate insert sentence
str_sql = f"INSERT INTO `{db}`.`{table}` ({keys}) VALUES ({values})"
if debug: # for debug only
print(str_sql)
# executing sql command
return PySQLConnection.execute(conn, str_sql)
def parsing_reg_file(self, rule_file: str, args: dict):
"""
Parse the configuration file, and then verify the validity of the input parameters
@Args:
* [ruleFile] str, path of file
* [args] dict, data contained with key and value
@Returns:
* [dict(str:obj)] dict type
"""
if not FileUtils.exists(rule_file):
raise Exceptions.NoAvailableResourcesFoundException(
"file: {} not found".format(rule_file))
# parse the file
for line in open(rule_file, "rt"):
# trim tailor
line = self.trim_tailer_from_text(line)
# split token from spaces
seps = line.split(' ')
# check size
if len(seps) != 3:
raise Exceptions.InvalidArithException(
"regular line: {} is broken, [argument key] [mapping key] ["
"regular expression]".format(line))
# append regular token to list
self.tokens.append(
RegularToken(seps[0], seps[1], R"{}".format(seps[2])))
# After parsing the file, first extract the dictionary consisting of {old key-regular match} from the file,
# and then perform regular filtering on the parameters
filtered_args = self.use_args_regular_check(args)
return filtered_args
def _id_generate(self):
"""
generate the id in format of YYYYmmDDHHMMSS00000000
"""
id_strings = ""
if self.bits <= 12 and self.bits > 0:
id_strings = self._datetime() + "000000000000000000"[0:self.bits]
else:
raise excepts.ArrayIndexOutOfBoundsException(
"bits of seed to generate id is out of range")
return int(id_strings)
def merge(self, queue):
"""
merge two queues into one
"""
try:
self.lock.acquire()
if type(queue) is list or type(queue) is Queue:
for item in queue:
self.enqueue(item)
else:
raise Exceptions.InvalidParamException("cannot merge a non-list or non-queue type")
finally:
self.lock.release()
def search_delete(self, leaf_id):
if not isinstance(leaf_id, type(self._item_id)):
raise Exceptions.InvalidParamException(
"leaf id must be the same type of TreeNode id")
leaf = self.search_tree(leaf_id)
if leaf is None:
return None # nothing to delete
root = leaf._item_root
if isinstance(root, CommonTreeNode):
return ListExtern.cherry_pick(root._item_leaves, leaf)
else: # the leaf is not real leaf, it is the root of tree
return self
def reflect_request_with_cache_callback(cache: object, do_action: object,
*args):
"""
用于缓存池的回调函数
@Args:
* [cache] 用于访问的缓存
* [do_action] 回调函数, 函数结构为callback(conn, [args]), 如果不需要附加参数，那么回调函数只为callback(conn)
* [args] 用户需要添加的参数
"""
if cache is None:
raise Exceptions.NullPointerException("cache is not available")
if args is None:
return do_action(cache)
else:
return do_action(cache, args)
def search_tree(self, leaf_id):
if not isinstance(leaf_id, type(self._item_id)):
raise Exceptions.InvalidParamException(
"leaf id must be the same type of TreeNode id")
if self._item_id == leaf_id: # found
return self
for leaf in self._item_leaves: # this node is not we want
if leaf.self_id() == leaf_id: # found
return leaf
else:
if len(leaf) > 0:
sub_leaf = leaf.search_tree(leaf_id)
if isinstance(sub_leaf, CommonTreeNode):
return sub_leaf
return None # nothing found
def decode(self, token: bytes):
"""
since obtained token from redis, use this method to
convert the bytes into correct data type
Args:
* [token(bytes)] obtained bytes from redis
Returns:
* [token(dict)] contains data, timestamp, type infomation
"""
# reset all
self.reset()
if token is not None:
# 1, convert the bytes into dictionary
self.token = convert.binary_to_dict(token)
# 2. assign to attributes
self.dtype = self.token["type"]
self.timestamp = self.token["timestamp"]
# 3. if bytes, process them carefully
if self.dtype == "bytes":
self.data = base64.b64decode(
convert.string_to_binary(self.token["data"]))
else: # normal data
self.data = self.token["data"]
# 4. return to caller
self.token = {
"data": self.data,
"type": self.dtype,
"timestamp": self.timestamp
}
return self.token
else:
raise excepts.NullPointerException("token cannot be a null type")