def test_normal(self, tmpdir, value, expected):
p_db = tmpdir.join("tmp.db")
con = SimpleSQLite(str(p_db), "w")
con.create_table_from_tabledata(value)
assert con.select_as_dict(table_name=value.table_name) == expected
def test_normal(self, tmpdir, value, expected):
p_db = tmpdir.join("tmp.db")
con = SimpleSQLite(str(p_db), "w")
con.create_table_from_tabledata(value)
assert con.select_as_dict(table_name=value.table_name) == expected
def main():
con = SimpleSQLite("sample.sqlite", "w", profile=True)
con.create_table_from_data_matrix(
"sample_table",
["a", "b", "c", "d", "e"],
[[1, 1.1, "aaa", 1, 1], [2, 2.2, "bbb", 2.2, 2.2],
[3, 3.3, "ccc", 3, "ccc"]],
)
for record in con.select_as_dict(table_name="sample_table"):
print(record)
def test_normal_type_hints(self, tmpdir, table_name, attr_names,
data_matrix, type_hints, expected):
p = tmpdir.join("tmp_type_hints.db")
con = SimpleSQLite(str(p), "w")
con.create_table_from_data_matrix(table_name,
attr_names,
data_matrix,
type_hints=type_hints)
assert con.select_as_dict(table_name) == [
OrderedDict([("int", 1), ("text", "001")]),
OrderedDict([("int", 2), ("text", "010")]),
]
class TcShapingRuleParser:
@property
def con(self):
return self.__con
@property
def device(self):
return self.__device
@property
def ifb_device(self):
return self.__ifb_device
def __init__(
self,
device,
ip_version,
logger,
tc_command_output,
export_path=None,
is_parse_filter_id=True,
dump_db_path=None,
):
if dump_db_path is None:
self.__con = connect_memdb()
else:
self.__con = SimpleSQLite(dump_db_path, "w")
Filter.attach(self.__con)
Filter.create()
Qdisc.attach(self.__con)
Qdisc.create()
self.__device = device
self.__ip_version = ip_version
self.__tc_command_output = tc_command_output
self.__logger = logger
self.__export_path = export_path
self.clear()
self.__ifb_device = self.__get_ifb_from_device()
self.__iptables_ctrl = IptablesMangleController(True, ip_version)
self.is_parse_filter_id = is_parse_filter_id
def clear(self):
self.__filter_parser = TcFilterParser(self.__con, self.__ip_version)
self.__parsed_mappings = {}
def extract_export_parameters(self):
_, out_rules = self.__get_shaping_rule(self.device)
_, in_rules = self.__get_shaping_rule(self.ifb_device)
for out_rule in out_rules:
out_rule.update({
Tc.Param.DEVICE: self.device,
Tc.Param.DIRECTION: TrafficDirection.OUTGOING
})
for in_rule in in_rules:
in_rule.update({
Tc.Param.DEVICE: self.ifb_device,
Tc.Param.DIRECTION: TrafficDirection.INCOMING
})
return (out_rules, in_rules)
def get_tc_parameter(self):
out_rule_maps, _ = self.__get_shaping_rule(self.device)
in_rule_maps, _ = self.__get_shaping_rule(self.ifb_device)
return {
self.device: {
TrafficDirection.OUTGOING: out_rule_maps,
TrafficDirection.INCOMING: in_rule_maps,
}
}
def parse(self):
self.__parse_device(self.device)
self.__parse_device(self.ifb_device)
def __parse_device(self, device):
if not device:
return
if self.__parsed_mappings.get(device):
return
self.__parse_tc_class(device)
self.__parse_tc_filter(device)
self.__parse_tc_qdisc(device)
self.__parsed_mappings[device] = True
def __get_ifb_from_device(self):
if not is_execute_tc_command(self.__tc_command_output):
return None
filter_runner = subprocrunner.SubprocessRunner(
"{:s} show dev {:s} root".format(
get_tc_base_command(TcSubCommand.FILTER), self.device),
error_log_level="QUIET",
dry_run=False,
)
if filter_runner.run() != 0 and filter_runner.stderr.find(
"Cannot find device") != -1:
raise NetworkInterfaceNotFoundError(target=self.device)
return self.__filter_parser.parse_incoming_device(filter_runner.stdout)
def __get_filter_key(self, filter_param):
key_items = OrderedDict()
if Tc.Param.HANDLE in filter_param:
handle = filter_param.get(Tc.Param.HANDLE)
typepy.Integer(handle).validate()
handle = int(handle)
for mangle in self.__iptables_ctrl.parse():
if mangle.mark_id != handle:
continue
key_items[Tc.Param.DST_NETWORK] = mangle.destination
if typepy.is_not_null_string(mangle.source):
key_items[Tc.Param.SRC_NETWORK] = mangle.source
key_items[Tc.Param.PROTOCOL] = mangle.protocol
break
else:
raise ValueError("mangle mark not found: {}".format(mangle))
else:
src_network = filter_param.get(Tc.Param.SRC_NETWORK)
if typepy.is_not_null_string(
src_network) and not is_anywhere_network(
src_network, self.__ip_version):
key_items[Tc.Param.SRC_NETWORK] = src_network
dst_network = filter_param.get(Tc.Param.DST_NETWORK)
if typepy.is_not_null_string(
dst_network) and not is_anywhere_network(
dst_network, self.__ip_version):
key_items[Tc.Param.DST_NETWORK] = dst_network
src_port = filter_param.get(Tc.Param.SRC_PORT)
if typepy.Integer(src_port).is_type():
key_items[Tc.Param.SRC_PORT] = "{}".format(src_port)
elif src_port is not None:
self.__logger.warning(
"expected a integer value for {}, actual {}: {}".format(
Tc.Param.SRC_PORT,
type(src_port),
src_port,
))
dst_port = filter_param.get(Tc.Param.DST_PORT)
if typepy.Integer(dst_port).is_type():
key_items[Tc.Param.DST_PORT] = "{}".format(dst_port)
elif src_port is not None:
self.__logger.warning(
"expected a integer value for {}, actual {}".format(
Tc.Param.DST_PORT, type(dst_port)))
protocol = filter_param.get(Tc.Param.PROTOCOL)
if typepy.is_not_null_string(protocol):
key_items[Tc.Param.PROTOCOL] = protocol
key = ", ".join(
["{}={}".format(key, value) for key, value in key_items.items()])
return key, key_items
def __get_shaping_rule(self, device):
if typepy.is_null_string(device):
return ({}, [])
self.__parse_device(device)
where_dev_query = Where(Tc.Param.DEVICE, device)
try:
class_params = self.__con.select_as_dict(
table_name=TcSubCommand.CLASS.value, where=where_dev_query)
except TableNotFoundError:
class_params = []
try:
filter_params = Filter.select(where=where_dev_query)
except TableNotFoundError:
filter_params = []
shaping_rule_mapping = {}
shaping_rules = []
for filter_param in filter_params:
filter_param = filter_param.as_dict()
self.__logger.debug("{:s} param: {}".format(
TcSubCommand.FILTER, filter_param))
shaping_rule = {}
filter_key, rule_with_keys = self.__get_filter_key(filter_param)
if typepy.is_null_string(filter_key):
self.__logger.debug(
"empty filter key: {}".format(filter_param))
continue
qdisc_id = filter_param.get(Tc.Param.FLOW_ID)
if qdisc_id is None:
qdisc_id = filter_param.get(Tc.Param.CLASS_ID)
try:
qdisc_params = Qdisc.select(where=And(
[where_dev_query,
Where(Tc.Param.PARENT, qdisc_id)]))
except TableNotFoundError:
qdisc_params = []
for qdisc_param in qdisc_params:
qdisc_param = qdisc_param.as_dict()
self.__logger.debug("{:s} param: {}".format(
TcSubCommand.QDISC, qdisc_param))
if self.is_parse_filter_id:
shaping_rule[Tc.Param.FILTER_ID] = filter_param.get(
Tc.Param.FILTER_ID)
# shaping_rule[Tc.Param.PRIORITY] = filter_param.get(
# Tc.Param.PRIORITY)
shaping_rule.update(
self.__strip_param(
qdisc_param,
[
Tc.Param.DEVICE, Tc.Param.PARENT, Tc.Param.HANDLE,
"direct_qlen"
],
))
for class_param in class_params:
self.__logger.debug("{:s} param: {}".format(
TcSubCommand.CLASS, class_param))
if class_param.get(Tc.Param.CLASS_ID) not in (
filter_param.get(Tc.Param.FLOW_ID),
filter_param.get(Tc.Param.CLASS_ID),
):
continue
if self.is_parse_filter_id:
shaping_rule[Tc.Param.FILTER_ID] = filter_param.get(
Tc.Param.FILTER_ID)
# shaping_rule[Tc.Param.PRIORITY] = filter_param.get(
# Tc.Param.PRIORITY)
shaping_rule.update(
self.__strip_param(class_param,
[Tc.Param.DEVICE, Tc.Param.CLASS_ID]))
if not shaping_rule:
self.__logger.debug(
"shaping rule not found for '{}'".format(filter_param))
continue
self.__logger.debug("shaping rule found: {} {}".format(
filter_key, shaping_rule))
rule_with_keys.update(shaping_rule)
shaping_rules.append(rule_with_keys)
shaping_rule_mapping[filter_key] = shaping_rule
return (shaping_rule_mapping, shaping_rules)
def __parse_tc_qdisc(self, device):
TcQdiscParser(self.__con).parse(
device,
run_tc_show(TcSubCommand.QDISC, device, self.__tc_command_output))
def __parse_tc_filter(self, device):
self.__filter_parser.parse(
device,
run_tc_show(TcSubCommand.FILTER, device, self.__tc_command_output))
def __parse_tc_class(self, device):
TcClassParser(self.__con).parse(
device,
run_tc_show(TcSubCommand.CLASS, device, self.__tc_command_output))
@staticmethod
def __strip_param(params, strip_params):
work_params = copy.deepcopy(params)
for strip_param in strip_params:
try:
del work_params[strip_param]
except KeyError:
pass
return {
key: value
for key, value in work_params.items() if value is not None
}
#!/usr/bin/env python
# encoding: utf-8
from __future__ import print_function
from simplesqlite import SimpleSQLite
con = SimpleSQLite("sample.sqlite", "w", profile=True)
con.create_table_from_data_matrix(
"sample_table",
["a", "b", "c", "d", "e"],
[[1, 1.1, "aaa", 1, 1], [2, 2.2, "bbb", 2.2, 2.2], [3, 3.3, "ccc", 3, "ccc"]],
)
for record in con.select_as_dict(table_name="sample_table"):
print(record)
#!/usr/bin/env python
# encoding: utf-8
from __future__ import print_function
from simplesqlite import SimpleSQLite
con = SimpleSQLite("sample.sqlite", "w", profile=True)
con.create_table_from_data_matrix(
"sample_table",
["a", "b", "c", "d", "e"],
[[1, 1.1, "aaa", 1, 1], [2, 2.2, "bbb", 2.2, 2.2],
[3, 3.3, "ccc", 3, "ccc"]],
)
for record in con.select_as_dict(table_name="sample_table"):
print(record)
