def parse_filter(self, text):
self.__clear()
if dataproperty.is_empty_string(text):
return []
filter_data_matrix = []
for line in text.splitlines():
if dataproperty.is_empty_string(line):
continue
try:
self.__parse_flow_id(line)
continue
except pp.ParseException:
pass
try:
self.__parse_filter(line)
continue
except pp.ParseException:
pass
if self.flow_id is not None:
filter_data_matrix.append(self.__get_filter())
self.__clear()
if self.flow_id is not None:
filter_data_matrix.append(self.__get_filter())
return filter_data_matrix
def parse_filter(self, text):
self.__clear()
if dataproperty.is_empty_string(text):
return []
filter_data_matrix = []
for line in text.splitlines():
if dataproperty.is_empty_string(line):
continue
try:
self.__parse_flow_id(line)
continue
except pp.ParseException:
pass
try:
self.__parse_filter(line)
continue
except pp.ParseException:
pass
if self.flow_id is not None:
filter_data_matrix.append(self.__get_filter())
self.__clear()
if self.flow_id is not None:
filter_data_matrix.append(self.__get_filter())
return filter_data_matrix
def parse(self, text):
if dataproperty.is_empty_string(text):
raise ValueError("empty text")
text = text.strip()
for line in text.splitlines():
if dataproperty.is_empty_string(line):
continue
line = _to_unicode(line.lstrip())
if re.search("qdisc netem|qdisc tbf", line) is None:
continue
self.__clear()
if re.search("qdisc netem", line) is not None:
self.__parse_netem_param(line, "parent", pp.hexnums + ":")
self.__parse_netem_param(line, "delay", pp.nums + ".")
self.__parse_netem_delay_distro(line)
self.__parse_netem_param(line, "loss", pp.nums + ".")
self.__parse_netem_param(line, "corrupt", pp.nums + ".")
self.__parse_tbf_rate(line)
yield self.__parsed_param
def __get_shaping_rule(self, device):
if dataproperty.is_empty_string(device):
return {}
class_param_list = self.__parse_tc_class(device)
filter_param_list = self.__parse_tc_filter(device)
qdisc_param_list = self.__parse_tc_qdisc(device)
shaping_rule_mapping = {}
for filter_param in filter_param_list:
logger.debug("{:s} param: {}".format(Tc.Subcommand.FILTER,
filter_param))
shaping_rule = {}
filter_key = self.__get_filter_key(filter_param)
if dataproperty.is_empty_string(filter_key):
logger.debug("empty filter key: {}".format(filter_param))
continue
for qdisc_param in qdisc_param_list:
logger.debug("{:s} param: {}".format(Tc.Subcommand.QDISC,
qdisc_param))
if qdisc_param.get(Tc.Param.PARENT) not in (filter_param.get(
Tc.Param.FLOW_ID), filter_param.get(
Tc.Param.CLASS_ID)):
continue
work_qdisc_param = copy.deepcopy(qdisc_param)
del work_qdisc_param[Tc.Param.PARENT]
shaping_rule.update(work_qdisc_param)
for class_param in class_param_list:
logger.debug("{:s} param: {}".format(Tc.Subcommand.CLASS,
class_param))
if class_param.get(Tc.Param.CLASS_ID) not in (filter_param.get(
Tc.Param.FLOW_ID), filter_param.get(
Tc.Param.CLASS_ID)):
continue
work_class_param = copy.deepcopy(class_param)
del work_class_param[Tc.Param.CLASS_ID]
shaping_rule.update(work_class_param)
if not shaping_rule:
continue
logger.debug("rule found: {} {}".format(filter_key, shaping_rule))
shaping_rule_mapping[filter_key] = shaping_rule
return shaping_rule_mapping
def __write_separator_row(self, value_list):
if dataproperty.is_empty_list_or_tuple(value_list):
return
left_cross_point = self.char_cross_point
right_cross_point = self.char_cross_point
if dataproperty.is_empty_string(self.char_left_side_row):
left_cross_point = u""
if dataproperty.is_empty_string(self.char_right_side_row):
right_cross_point = u""
self._write_line(left_cross_point +
self.char_cross_point.join(value_list) +
right_cross_point)
def __write_separator_row(self, value_list):
if dataproperty.is_empty_list_or_tuple(value_list):
return
left_cross_point = self.char_cross_point
right_cross_point = self.char_cross_point
if dataproperty.is_empty_string(self.char_left_side_row):
left_cross_point = u""
if dataproperty.is_empty_string(self.char_right_side_row):
right_cross_point = u""
self._write_line(
left_cross_point +
self.char_cross_point.join(value_list) +
right_cross_point)
def parse_filter(self, text):
self.__clear()
if dataproperty.is_empty_string(text):
return []
filter_data_matrix = []
for line in text.splitlines():
if dataproperty.is_empty_string(line):
continue
try:
self.__parse_mangle_mark(line)
except pp.ParseException:
logger.debug("failed to parse mangle: {}".format(line))
else:
filter_data_matrix.append({
"classid": self.classid,
"handle": self.handle,
})
self.__clear()
continue
tc_filter = self.__get_filter()
try:
self.__parse_flow_id(line)
if tc_filter.get(Tc.Param.FLOW_ID):
logger.debug("store filter: {}".format(tc_filter))
filter_data_matrix.append(tc_filter)
self.__clear()
self.__parse_flow_id(line)
continue
except pp.ParseException:
logger.debug("failed to parse flow id: {}".format(line))
try:
self.__parse_filter(line)
except pp.ParseException:
logger.debug("failed to parse filter: {}".format(line))
if self.flow_id:
filter_data_matrix.append(self.__get_filter())
return filter_data_matrix
def test_const_packet_loss(self, device_option, dst_host_option,
transmitter, pingparser, option, value):
if dataproperty.is_empty_string(dst_host_option):
# alternative to pytest.mark.skipif
return
SubprocessRunner("tcdel --device " + device_option).run()
transmitter.destination_host = dst_host_option
# w/o packet loss tc ---
result = transmitter.ping()
pingparser.parse(result.stdout)
without_tc_loss = (pingparser.packet_receive /
pingparser.packet_transmit) * 100
# w/ packet loss tc ---
command_list = [
"tcset",
"--device " + device_option,
"{:s} {:f}".format(option, value),
]
assert SubprocessRunner(" ".join(command_list)).run() == 0
result = transmitter.ping()
pingparser.parse(result.stdout)
with_tc_loss = (pingparser.packet_receive /
pingparser.packet_transmit) * 100
# assertion ---
loss_diff = without_tc_loss - with_tc_loss
assert loss_diff > (value / 2.0)
# finalize ---
SubprocessRunner("tcdel --device " + device_option).run()
def __setup_ifb(self):
if self.direction != TrafficDirection.INCOMING:
return
if dataproperty.is_empty_string(self.ifb_device):
return
return_code = 0
command = "modprobe ifb"
return_code |= SubprocessRunner(command).run()
command = "ip link add {:s} type ifb".format(self.ifb_device)
return_code |= SubprocessRunner(command).run()
command = "ip link set dev {:s} up".format(self.ifb_device)
return_code |= SubprocessRunner(command).run()
command = "tc qdisc add dev {:s} ingress".format(self.__device)
return_code |= SubprocessRunner(command).run()
command_list = [
"tc filter add",
"dev " + self.__device,
"parent ffff: protocol ip u32 match u32 0 0",
"flowid {:x}:".format(self.__get_device_qdisc_major_id()),
"action mirred egress redirect",
"dev " + self.ifb_device,
]
return_code |= SubprocessRunner(" ".join(command_list)).run()
return return_code
def __validate_rate(self):
if dataproperty.is_empty_string(self.rate):
return
rate = thutils.common.humanreadable_to_byte(self.rate)
if rate <= 0:
raise ValueError("rate must be greater than zero")
def test_const_packet_loss(self, dst_host_option, subproc_wrapper,
transmitter, pingparser, option, value):
if dataproperty.is_empty_string(dst_host_option):
# alternative to pytest.mark.skipif
return
subproc_wrapper.run("tcdel --device " + DEVICE)
transmitter.destination_host = dst_host_option
# w/o packet loss tc ---
result = transmitter.ping()
pingparser.parse(result)
without_tc_loss = (pingparser.packet_receive /
float(pingparser.packet_transmit)) * 100.0
# w/ packet loss tc ---
command_list = [
"tcset",
"--device " + DEVICE,
"%s %f" % (option, value),
]
assert subproc_wrapper.run(" ".join(command_list)) == 0
result = transmitter.ping()
pingparser.parse(result)
with_tc_loss = (pingparser.packet_receive /
float(pingparser.packet_transmit)) * 100.0
# assertion ---
loss_diff = without_tc_loss - with_tc_loss
assert loss_diff > (value / 2.0)
# finalize ---
subproc_wrapper.run("tcdel --device " + DEVICE)
def test_const_latency(self, device_option, dst_host_option, transmitter,
pingparser, delay):
if device_option is None:
pytest.skip("device option is null")
if dataproperty.is_empty_string(dst_host_option):
pytest.skip("destination host is null")
SubprocessRunner("tcdel --device " + device_option).run()
transmitter.destination_host = dst_host_option
# w/o latency tc ---
result = transmitter.ping()
pingparser.parse(result.stdout)
without_tc_rtt_avg = pingparser.rtt_avg
# w/ latency tc ---
command_list = [
"tcset",
"--device " + device_option,
"--delay {:d}".format(delay),
]
assert SubprocessRunner(" ".join(command_list)).run() == 0
result = transmitter.ping()
pingparser.parse(result.stdout)
with_tc_rtt_avg = pingparser.rtt_avg
# assertion ---
rtt_diff = with_tc_rtt_avg - without_tc_rtt_avg
assert rtt_diff > (delay / 2.0)
# finalize ---
SubprocessRunner("tcdel --device " + device_option).run()
def make_update(cls, table, set_query, where=None):
"""
Make UPDATE query.
:param str table: Table name of execute query.
:param str set_query: SET part of UPDATE query.
:return: Query of SQLite.
:rtype: str
:raises ValueError: If ``set_query`` is empty string.
.. seealso::
:py:func:`validate_table_name() <simplesqlite.validate_table_name>`
"""
validate_table_name(table)
if dataproperty.is_empty_string(set_query):
raise ValueError("SET query is null")
query_list = [
"UPDATE " + cls.to_table_str(table),
"SET " + set_query,
]
if dataproperty.is_not_empty_string(where):
query_list.append("WHERE " + where)
return " ".join(query_list)
def __set_pre_network_filter(self, qdisc_major_id):
if self.__is_use_iptables():
return 0
if all([
dataproperty.is_empty_string(self.network),
not IntegerType(self.port).is_type(),
]):
flowid = "{:x}:{:d}".format(
qdisc_major_id, self.__get_qdisc_minor_id())
else:
flowid = "{:x}:2".format(qdisc_major_id)
command_list = [
"tc filter add",
"dev " + self.__get_tc_device(),
"protocol ip",
"parent {:x}:".format(qdisc_major_id),
"prio 2 u32 match ip {:s} {:s}".format(
self.__get_network_direction_str(),
ANYWHERE_NETWORK),
"flowid " + flowid,
]
return SubprocessRunner(" ".join(command_list)).run()
def test_const_latency(
self, dst_host_option, subproc_wrapper,
transmitter, pingparser, delay):
if dataproperty.is_empty_string(dst_host_option):
# alternative to pytest.mark.skipif
return
subproc_wrapper.run("tcdel --device " + DEVICE)
transmitter.destination_host = dst_host_option
# w/o latency tc ---
result = transmitter.ping()
pingparser.parse(result)
without_tc_rtt_avg = pingparser.rtt_avg
# w/ latency tc ---
command_list = [
"tcset",
"--device " + DEVICE,
"--delay %d" % (delay),
]
assert subproc_wrapper.run(" ".join(command_list)) == 0
result = transmitter.ping()
pingparser.parse(result)
with_tc_rtt_avg = pingparser.rtt_avg
# assertion ---
rtt_diff = with_tc_rtt_avg - without_tc_rtt_avg
assert rtt_diff > (delay / 2.0)
# finalize ---
assert subproc_wrapper.run("tcdel --device " + DEVICE) == 0
def test_const_packet_loss(
self, dst_host_option, subproc_wrapper,
transmitter, pingparser, option, value):
if dataproperty.is_empty_string(dst_host_option):
# alternative to pytest.mark.skipif
return
subproc_wrapper.run("tcdel --device " + DEVICE)
transmitter.destination_host = dst_host_option
# w/o packet loss tc ---
result = transmitter.ping()
pingparser.parse(result)
without_tc_loss = (
pingparser.packet_receive / float(pingparser.packet_transmit)) * 100.0
# w/ packet loss tc ---
command_list = [
"tcset",
"--device " + DEVICE,
"%s %f" % (option, value),
]
assert subproc_wrapper.run(" ".join(command_list)) == 0
result = transmitter.ping()
pingparser.parse(result)
with_tc_loss = (
pingparser.packet_receive / float(pingparser.packet_transmit)) * 100.0
# assertion ---
loss_diff = without_tc_loss - with_tc_loss
assert loss_diff > (value / 2.0)
# finalize ---
subproc_wrapper.run("tcdel --device " + DEVICE)
def __validate_bandwidth_rate(self):
if dataproperty.is_empty_string(self.bandwidth_rate):
return
rate = thutils.common.humanreadable_to_byte(self.bandwidth_rate)
if rate <= 0:
raise ValueError("rate must be greater than zero")
def __setup_ifb(self):
if self.direction != TrafficDirection.INCOMING:
return
if dataproperty.is_empty_string(self.ifb_device):
return
return_code = 0
command = "modprobe ifb"
return_code |= self.__subproc_wrapper.run(command)
command = "ip link set dev %s up" % (self.ifb_device)
return_code |= self.__subproc_wrapper.run(command)
command = "tc qdisc add dev %s ingress" % (self.__device)
return_code |= self.__subproc_wrapper.run(command)
command_list = [
"tc filter add",
"dev " + self.__device,
"parent ffff: protocol ip u32 match u32 0 0",
"flowid %d:" % (self.__IN_DEVICE_QDISC_MAJOR_ID),
"action mirred egress redirect",
"dev " + self.ifb_device,
]
return_code |= self.__subproc_wrapper.run(" ".join(command_list))
return return_code
def __setup_ifb(self):
if self.direction != TrafficDirection.INCOMING:
return
if dataproperty.is_empty_string(self.ifb_device):
return
return_code = 0
command = "modprobe ifb"
return_code |= self.__subproc_wrapper.run(command)
command = "ip link set dev %s up" % (self.ifb_device)
return_code |= self.__subproc_wrapper.run(command)
command = "tc qdisc add dev %s ingress" % (self.__device)
return_code |= self.__subproc_wrapper.run(command)
command_list = [
"tc filter add",
"dev " + self.__device,
"parent ffff: protocol ip u32 match u32 0 0",
"flowid %d:" % (self.__IN_DEVICE_QDISC_MAJOR_ID),
"action mirred egress redirect",
"dev " + self.ifb_device,
]
return_code |= self.__subproc_wrapper.run(" ".join(command_list))
return return_code
def make_select(cls, select, table, where=None, extra=None):
"""
Make SELECT query.
:param str select: Attribute for SELECT query
:param str table: Table name of execute query.
:param str where: Add WHERE clause to execute query if not ``None``
:param extra extra: Add additional clause to execute query if not ``None``
:return: Query of SQLite.
:rtype: str
:raises ValueError: ``select`` is empty string.
.. seealso::
:py:func:`validate_table_name() <simplesqlite.validate_table_name>`
"""
validate_table_name(table)
if dataproperty.is_empty_string(select):
raise ValueError("SELECT query is null")
query_list = [
"SELECT " + select,
"FROM " + cls.to_table_str(table),
]
if dataproperty.is_not_empty_string(where):
query_list.append("WHERE " + where)
if dataproperty.is_not_empty_string(extra):
query_list.append(extra)
return " ".join(query_list)
def to_table_data(self):
self._validate_source_data()
if dataproperty.is_empty_sequence(self._loader.header_list):
header_list = self._source_data[0]
if any([
dataproperty.is_empty_string(header) for header in header_list
]):
raise InvalidDataError(
"the first line includes empty string item: "
"the first line expected to contain header data.")
data_matrix = self._source_data[1:]
else:
header_list = self._loader.header_list
data_matrix = self._source_data
if len(data_matrix) == 0:
raise InvalidDataError(
"data row must be greater or equal than one")
yield TableData(
self._loader.make_table_name(),
header_list, data_matrix)
def test_const_latency(self, dst_host_option, subproc_wrapper, transmitter,
pingparser, delay):
if dataproperty.is_empty_string(dst_host_option):
# alternative to pytest.mark.skipif
return
subproc_wrapper.run("tcdel --device " + DEVICE)
transmitter.destination_host = dst_host_option
# w/o latency tc ---
result = transmitter.ping()
pingparser.parse(result)
without_tc_rtt_avg = pingparser.rtt_avg
# w/ latency tc ---
command_list = [
"tcset",
"--device " + DEVICE,
"--delay %d" % (delay),
]
assert subproc_wrapper.run(" ".join(command_list)) == 0
result = transmitter.ping()
pingparser.parse(result)
with_tc_rtt_avg = pingparser.rtt_avg
# assertion ---
rtt_diff = with_tc_rtt_avg - without_tc_rtt_avg
assert rtt_diff > (delay / 2.0)
# finalize ---
subproc_wrapper.run("tcdel --device " + DEVICE)
def __set_network_filter(self, qdisc_major_id):
command_list = [
"tc filter add",
"dev " + self.__get_tc_device(),
"protocol ip",
"parent {:x}:".format(qdisc_major_id),
"prio 1",
]
if self.__is_use_iptables():
mark_id = (IptablesMangleController.get_unique_mark_id() +
self.__FILTER_IPTABLES_MARK_ID_OFFSET)
command_list.append("handle {:d} fw".format(mark_id))
self.__add_mangle_mark(mark_id)
else:
if all([
dataproperty.is_empty_string(self.network),
self.port is None,
]):
return 0
command_list.append("u32")
if dataproperty.is_not_empty_string(self.network):
command_list.append("match ip {:s} {:s}".format(
self.__get_network_direction_str(), self.network))
if self.port is not None:
command_list.append("match ip dport {:d} 0xffff".format(
self.port))
command_list.append("flowid {:x}:{:d}".format(
qdisc_major_id, self.__get_qdisc_minor_id()))
return SubprocessRunner(" ".join(command_list)).run()
def make_select(cls, select, table, where=None, extra=None):
"""
Make SELECT query.
:param str select: Attribute for SELECT query
:param str table: Table name of execute query.
:param str where: Add WHERE clause to execute query if not ``None``
:param extra extra: Add additional clause to execute query if not ``None``
:return: Query of SQLite.
:rtype: str
:raises ValueError: ``select`` is empty string.
.. seealso::
:py:func:`validate_table_name() <simplesqlite.validate_table_name>`
"""
validate_table_name(table)
if dataproperty.is_empty_string(select):
raise ValueError("SELECT query is null")
query_list = [
"SELECT " + select,
"FROM " + cls.to_table_str(table),
]
if dataproperty.is_not_empty_string(where):
query_list.append("WHERE " + where)
if dataproperty.is_not_empty_string(extra):
query_list.append(extra)
return " ".join(query_list)
def make_update(cls, table, set_query, where=None):
"""
Make UPDATE query.
:param str table: Table name of execute query.
:param str set_query: SET part of UPDATE query.
:return: Query of SQLite.
:rtype: str
:raises ValueError: If ``set_query`` is empty string.
.. seealso::
:py:func:`validate_table_name() <simplesqlite.validate_table_name>`
"""
validate_table_name(table)
if dataproperty.is_empty_string(set_query):
raise ValueError("SET query is null")
query_list = [
"UPDATE " + cls.to_table_str(table),
"SET " + set_query,
]
if dataproperty.is_not_empty_string(where):
query_list.append("WHERE " + where)
return " ".join(query_list)
def test_const_latency(
self, device_option, dst_host_option, transmitter, pingparser,
delay):
if device_option is None:
pytest.skip("device option is null")
if dataproperty.is_empty_string(dst_host_option):
pytest.skip("destination host is null")
SubprocessRunner("tcdel --device " + device_option).run()
transmitter.destination_host = dst_host_option
# w/o latency tc ---
result = transmitter.ping()
pingparser.parse(result.stdout)
without_tc_rtt_avg = pingparser.rtt_avg
# w/ latency tc ---
command_list = [
"tcset",
"--device " + device_option,
"--delay {:d}".format(delay),
]
assert SubprocessRunner(" ".join(command_list)).run() == 0
result = transmitter.ping()
pingparser.parse(result.stdout)
with_tc_rtt_avg = pingparser.rtt_avg
# assertion ---
rtt_diff = with_tc_rtt_avg - without_tc_rtt_avg
assert rtt_diff > (delay / 2.0)
# finalize ---
SubprocessRunner("tcdel --device " + device_option).run()
def add_filter(self):
command_list = [
"tc filter add",
self.dev,
"protocol ip",
"parent {:s}:".format(self._tc_obj.qdisc_major_id_str),
"prio 1",
]
if self._is_use_iptables():
command_list.append("handle {:d} fw".format(
self._get_unique_mangle_mark_id()))
else:
if dataproperty.is_empty_string(self._tc_obj.network):
network = ANYWHERE_NETWORK
else:
network = self._tc_obj.network
command_list.extend([
"u32",
"match ip {:s} {:s}".format(self._get_network_direction_str(),
network),
])
if self._tc_obj.port is not None:
command_list.append("match ip dport {:d} 0xffff".format(
self._tc_obj.port))
command_list.append("flowid {:s}:{:d}".format(
self._tc_obj.qdisc_major_id_str, self.get_qdisc_minor_id()))
return SubprocessRunner(" ".join(command_list)).run()
def __set_network_filter(self, qdisc_major_id):
command_list = [
"tc filter add",
"dev " + self.__get_tc_device(),
"protocol ip",
"parent {:x}:".format(qdisc_major_id),
"prio 1",
]
if self.__is_use_iptables():
mark_id = (
IptablesMangleController.get_unique_mark_id() +
self.__FILTER_IPTABLES_MARK_ID_OFFSET)
command_list.append("handle {:d} fw".format(mark_id))
self.__add_mangle_mark(mark_id)
else:
if all([
dataproperty.is_empty_string(self.network),
self.port is None,
]):
return 0
command_list.append("u32")
if dataproperty.is_not_empty_string(self.network):
command_list.append("match ip {:s} {:s}".format(
self.__get_network_direction_str(), self.network))
if self.port is not None:
command_list.append(
"match ip dport {:d} 0xffff".format(self.port))
command_list.append("flowid {:x}:{:d}".format(
qdisc_major_id, self.__get_qdisc_minor_id()))
return SubprocessRunner(" ".join(command_list)).run()
def test_const_packet_loss(
self, device_option, dst_host_option, transmitter, pingparser,
option, value):
if dataproperty.is_empty_string(dst_host_option):
# alternative to pytest.mark.skipif
return
SubprocessRunner("tcdel --device " + device_option).run()
transmitter.destination_host = dst_host_option
# w/o packet loss tc ---
result = transmitter.ping()
pingparser.parse(result.stdout)
without_tc_loss = (
pingparser.packet_receive / pingparser.packet_transmit) * 100
# w/ packet loss tc ---
command_list = [
"tcset",
"--device " + device_option,
"{:s} {:f}".format(option, value),
]
assert SubprocessRunner(" ".join(command_list)).run() == 0
result = transmitter.ping()
pingparser.parse(result.stdout)
with_tc_loss = (
pingparser.packet_receive / pingparser.packet_transmit) * 100
# assertion ---
loss_diff = without_tc_loss - with_tc_loss
assert loss_diff > (value / 2.0)
# finalize ---
SubprocessRunner("tcdel --device " + device_option).run()
def __validate_bandwidth_rate(self):
if dataproperty.is_empty_string(self.bandwidth_rate):
return
rate = Humanreadable().humanreadable_to_byte(self.bandwidth_rate)
if rate <= 0:
raise ValueError("rate must be greater than zero")
def make_update(cls, table, set_query, where=None):
"""
Make UPDATE query.
:param str table: Table name of executing the query.
:param str set_query: SET part of the UPDATE query.
:param str where:
Add a WHERE clause to execute query,
if the value is not |None|.
:return: Query of SQLite.
:rtype: str
:raises ValueError: If ``set_query`` is empty string.
:raises ValueError: |raises_validate_table_name|
"""
validate_table_name(table)
if dataproperty.is_empty_string(set_query):
raise ValueError("SET query is null")
query_list = [
"UPDATE " + cls.to_table_str(table),
"SET " + set_query,
]
if dataproperty.is_not_empty_string(where):
query_list.append("WHERE " + where)
return " ".join(query_list)
def validate_python_var_name(var_name):
"""
:param str var_name: Name to validate.
:raises ValueError: If the ``var_name`` is
**a)** empty.
**b)** invalid as
`Python identifier <https://docs.python.org/3/reference/lexical_analysis.html#identifiers>`__.
**c)** equals to
`reserved keywords <https://docs.python.org/3/reference/lexical_analysis.html#keywords>`__ or
`built-in constants <https://docs.python.org/3/library/constants.html>`__.
"""
if dataproperty.is_empty_string(var_name):
raise ValueError("null name")
if var_name in __RESERVED_KEYWORDS + __BUILT_CONSTANTS:
raise ValueError("%s is a reserved keyword by pyhon" % (var_name))
match = __RE_INVALID_VAR_NAME.search(var_name)
if match is not None:
raise ValueError("invalid char found in the variable name: '%s'" %
(re.escape(match.group())))
match = __RE_INVALID_VAR_NAME_HEAD.search(var_name)
if match is not None:
raise ValueError(
"the first char of the variable name is invalid: '%s'" %
(re.escape(match.group())))
def validate_python_var_name(var_name):
"""
:param str var_name: Name to validate.
:raises ValueError: If the ``var_name`` is
**a)** empty.
**b)** invalid as
`Python identifier <https://docs.python.org/3/reference/lexical_analysis.html#identifiers>`__.
**c)** equals to
`reserved keywords <https://docs.python.org/3/reference/lexical_analysis.html#keywords>`__ or
`built-in constants <https://docs.python.org/3/library/constants.html>`__.
"""
if dataproperty.is_empty_string(var_name):
raise ValueError("null name")
if var_name in __RESERVED_KEYWORDS + __BUILT_CONSTANTS:
raise ValueError(
"%s is a reserved keyword by pyhon" % (var_name))
match = __RE_INVALID_VAR_NAME.search(var_name)
if match is not None:
raise ValueError(
"invalid char found in the variable name: '%s'" % (
re.escape(match.group())))
match = __RE_INVALID_VAR_NAME_HEAD.search(var_name)
if match is not None:
raise ValueError(
"the first char of the variable name is invalid: '%s'" % (
re.escape(match.group())))
def to_attr_str_list(cls, name_list, operation_query=""):
"""
:param list/tuple name_list: List of attribute names.
:param str operation_query:
Used as a SQLite function if the value is not empty.
:return:
List of strings that suitable for
attribute names of a SQLite query.
:rtype: list/itertools.imap
:Examples:
>>> from simplesqlite.sqlquery import SqlQuery
>>> list(SqlQuery.to_attr_str_list(["key", "a+b"]))
['key', '[a+b]']
>>> SqlQuery.to_attr_str_list(["key", "a+b"], operation_query="AVG")
['AVG(key)', 'AVG([a+b])']
.. seealso::
:py:meth:`.to_attr_str`
"""
if dataproperty.is_empty_string(operation_query):
return list(map(cls.to_attr_str, name_list))
return [
"%s(%s)" % (operation_query, cls.to_attr_str(name))
for name in name_list
]
def validate_table_name(name):
"""
:param str name: Table name to validate.
:raises ValueError: If ``name`` is empty.
"""
if dataproperty.is_empty_string(name):
raise ValueError("table name is empty")
def __validate_db_path(database_path):
if dataproperty.is_empty_string(database_path):
raise ValueError("null path")
if database_path == MEMORY_DB_NAME:
return
pathvalidate.validate_filename(os.path.basename(database_path))
def __validate_src_network(self):
if dataproperty.is_empty_string(self.src_network):
return
if not self.is_enable_iptables:
raise InvalidParameterError(
"--iptables option will be required to use --src-network option"
)
def validate(self):
if dataproperty.is_empty_string(self.__device):
raise ValueError("device name is empty")
self.__validate_network_delay()
self.__validate_packet_loss_rate()
self.network = self.__validate_network()
self.__validate_port()
def validate_table_name(name):
"""
:param str name: Table name to validate.
:raises ValueError: If ``name`` is empty.
"""
if dataproperty.is_empty_string(name):
raise ValueError("table name is empty")
def is_invalid_param(rate, delay, loss, corrupt):
params = [
rate,
delay,
loss,
corrupt,
]
return all([dataproperty.is_empty_string(param) for param in params])
def parse_incoming_device(self, text):
if dataproperty.is_empty_string(text):
return None
match = re.search(
"Egress Redirect to device ifb[\d]+", _to_unicode(text), re.MULTILINE)
if match is None:
return None
return re.search("ifb[\d]+", match.group()).group()
def parse_incoming_device(self, text):
if dataproperty.is_empty_string(text):
return None
match = re.search("Egress Redirect to device ifb[\d]+",
_to_unicode(text), re.MULTILINE)
if match is None:
return None
return re.search("ifb[\d]+", match.group()).group()
def validate(self):
if dataproperty.is_empty_string(self.__device):
raise ValueError("device name is empty")
self.__validate_bandwidth_rate()
self.__validate_network_delay()
self.__validate_packet_loss_rate()
self.__validate_curruption_rate()
self.network = self.__validate_network()
self.__validate_port()
def test_network(
self, device_option, dst_host_option, dst_host_ex_option,
transmitter, pingparser):
if device_option is None:
pytest.skip("device option is null")
if any([
dataproperty.is_empty_string(dst_host_option),
dataproperty.is_empty_string(dst_host_ex_option),
]):
pytest.skip("destination host is null")
SubprocessRunner("tcdel --device " + device_option).run()
delay = 100
# tc to specific network ---
command_list = [
"tcset",
"--device " + device_option,
"--delay {:d}".format(delay),
"--network " + dst_host_ex_option,
]
assert SubprocessRunner(" ".join(command_list)).run() == 0
# w/o tc network ---
transmitter.destination_host = dst_host_option
result = transmitter.ping()
pingparser.parse(result.stdout)
without_tc_rtt_avg = pingparser.rtt_avg
# w/ tc network ---
transmitter.destination_host = dst_host_ex_option
result = transmitter.ping()
pingparser.parse(result.stdout)
with_tc_rtt_avg = pingparser.rtt_avg
# assertion ---
rtt_diff = with_tc_rtt_avg - without_tc_rtt_avg
assert rtt_diff > (delay / 2.0)
# finalize ---
SubprocessRunner("tcdel --device " + device_option).run()
def add_argument_group(self, group_name):
if dataproperty.is_empty_string(group_name):
raise ValueError("null argument group name")
if group_name not in self.dict_group:
group = self.parser.add_argument_group(group_name)
self.dict_group[group_name] = group
else:
return self.dict_group.get(group_name)
return group
def is_invalid_param(rate, delay, loss, corrupt):
params = [
rate,
delay,
loss,
corrupt,
]
return all([
dataproperty.is_empty_string(param) for param in params
])
def execute_query(self, query, caller=None):
"""
Execute arbitrary SQLite query.
:param str query: Query to be executed.
:param tuple caller:
Caller information.
Expects the return value of :py:meth:`logging.Logger.findCaller`.
:return: Result of the query execution.
:rtype: sqlite3.Cursor
:raises simplesqlite.NullDatabaseConnectionError:
|raises_check_connection|
:raises sqlite3.OperationalError: |raises_operational_error|
.. warning::
This method can execute an arbitrary query.
i.e. No access permissions check by |attr_mode|.
"""
import time
self.check_connection()
if dataproperty.is_empty_string(query):
return None
if self.__is_profile:
exec_start_time = time.time()
try:
result = self.connection.execute(query)
except sqlite3.OperationalError:
_, e, _ = sys.exc_info() # for python 2.5 compatibility
if caller is None:
caller = logging.getLogger().findCaller()
file_path, line_no, func_name = caller[:3]
message_list = [
"failed to execute query at %s(%d) %s" % (
file_path, line_no, func_name),
" - query: %s" % (query),
" - msg: %s" % (e),
" - db: %s" % (self.database_path),
]
raise sqlite3.OperationalError(os.linesep.join(message_list))
if self.__is_profile:
self.__dict_query_count[query] = (
self.__dict_query_count.get(query, 0) + 1)
elapse_time = time.time() - exec_start_time
self.__dict_query_totalexectime[query] = (
self.__dict_query_totalexectime.get(query, 0) + elapse_time)
return result
def check_connection(self):
"""
:raises NullDatabaseConnectionError:
If not connected to a SQLite database file.
"""
if self.connection is None:
raise NullDatabaseConnectionError("null database connection")
if dataproperty.is_empty_string(self.database_path):
raise NullDatabaseConnectionError("null database file path")
def check_connection(self):
"""
:raises NullDatabaseConnectionError:
If not connected to a SQLite database file.
"""
if self.connection is None:
raise NullDatabaseConnectionError("null database connection")
if dataproperty.is_empty_string(self.database_path):
raise NullDatabaseConnectionError("null database file path")
def execute_query(self, query, caller=None):
"""
Execute arbitrary SQLite query.
:param str query: Query to be executed.
:param str tuple:
Caller information.
Retuen value of Logger.findCaller().
:return: Result of the query execution.
:rtype: sqlite3.Cursor
:raises sqlite3.OperationalError:
If failed to execute query.
.. seealso::
:py:meth:`check_connection`
:py:meth:`validate_file_path`
"""
import time
self.check_connection()
if dataproperty.is_empty_string(query):
return None
if self.__is_profile:
exec_start_time = time.time()
try:
result = self.connection.execute(query)
except sqlite3.OperationalError:
_, e, _ = sys.exc_info() # for python 2.5 compatibility
if caller is None:
caller = logging.getLogger().findCaller()
file_path, line_no, func_name = caller[:3]
message_list = [
"failed to execute query at %s(%d) %s" % (
file_path, line_no, func_name),
" - query: %s" % (query),
" - msg: %s" % (e),
" - db: %s" % (self.database_path),
]
raise sqlite3.OperationalError(os.linesep.join(message_list))
if self.__is_profile:
self.__dict_query_count[query] = (
self.__dict_query_count.get(query, 0) + 1)
elapse_time = time.time() - exec_start_time
self.__dict_query_totalexectime[query] = (
self.__dict_query_totalexectime.get(query, 0) + elapse_time)
return result
def execute_query(self, query, caller=None):
"""
Execute arbitrary SQLite query.
:param str query: Query to be executed.
:param str tuple:
Caller information.
Retuen value of Logger.findCaller().
:return: Result of the query execution.
:rtype: sqlite3.Cursor
:raises sqlite3.OperationalError:
If failed to execute query.
.. seealso::
:py:meth:`check_connection`
:py:meth:`validate_file_path`
"""
import time
self.check_connection()
if dataproperty.is_empty_string(query):
return None
if self.__is_profile:
exec_start_time = time.time()
try:
result = self.connection.execute(query)
except sqlite3.OperationalError:
_, e, _ = sys.exc_info() # for python 2.5 compatibility
if caller is None:
caller = logging.getLogger().findCaller()
file_path, line_no, func_name = caller[:3]
message_list = [
"failed to execute query at %s(%d) %s" %
(file_path, line_no, func_name),
" - query: %s" % (query),
" - msg: %s" % (e),
" - db: %s" % (self.database_path),
]
raise sqlite3.OperationalError(os.linesep.join(message_list))
if self.__is_profile:
self.__dict_query_count[query] = (
self.__dict_query_count.get(query, 0) + 1)
elapse_time = time.time() - exec_start_time
self.__dict_query_totalexectime[query] = (
self.__dict_query_totalexectime.get(query, 0) + elapse_time)
return result
def convertHumanReadableToSecond(readable_time):
if dataproperty.is_empty_string(readable_time):
raise ValueError("empty input")
size = float(readable_time[:-1])
unit = readable_time[-1]
if size < 0:
raise ValueError("minus size")
return size * getTimeUnitSecondsCoefficient(unit)
def test_network(self, device_option, dst_host_option, dst_host_ex_option,
transmitter, pingparser):
if device_option is None:
pytest.skip("device option is null")
if any([
dataproperty.is_empty_string(dst_host_option),
dataproperty.is_empty_string(dst_host_ex_option),
]):
pytest.skip("destination host is null")
SubprocessRunner("tcdel --device " + device_option).run()
delay = 100
# tc to specific network ---
command_list = [
"tcset",
"--device " + device_option,
"--delay {:d}".format(delay),
"--network " + dst_host_ex_option,
]
assert SubprocessRunner(" ".join(command_list)).run() == 0
# w/o tc network ---
transmitter.destination_host = dst_host_option
result = transmitter.ping()
pingparser.parse(result.stdout)
without_tc_rtt_avg = pingparser.rtt_avg
# w/ tc network ---
transmitter.destination_host = dst_host_ex_option
result = transmitter.ping()
pingparser.parse(result.stdout)
with_tc_rtt_avg = pingparser.rtt_avg
# assertion ---
rtt_diff = with_tc_rtt_avg - without_tc_rtt_avg
assert rtt_diff > (delay / 2.0)
# finalize ---
SubprocessRunner("tcdel --device " + device_option).run()
def _write_table(self):
self._verify_property()
if dataproperty.is_empty_string(self.table_name):
self._write_line(u".. table:: ")
else:
self._write_line(u".. table:: " + self.table_name)
self._write_line()
self.inc_indent_level()
super(RstTableWriter, self).write_table()
self.dec_indent_level()
def _write_table(self):
self._verify_property()
if dataproperty.is_empty_string(self.table_name):
self._write_line(u".. table:: ")
else:
self._write_line(u".. table:: " + self.table_name)
self._write_line()
self.inc_indent_level()
super(RstTableWriter, self).write_table()
self.dec_indent_level()
def test_network(
self, dst_host_option, dst_host_ex_option, subproc_wrapper,
transmitter, pingparser):
if any([
dataproperty.is_empty_string(dst_host_option),
dataproperty.is_empty_string(dst_host_ex_option),
]):
# alternative to pytest.mark.skipif
return
subproc_wrapper.run("tcdel --device " + DEVICE)
delay = 100
# tc to specific network ---
command_list = [
"tcset",
"--device " + DEVICE,
"--delay %d" % (delay),
"--network " + dst_host_ex_option,
]
assert subproc_wrapper.run(" ".join(command_list)) == 0
# w/o tc network ---
transmitter.destination_host = dst_host_option
result = transmitter.ping()
pingparser.parse(result)
without_tc_rtt_avg = pingparser.rtt_avg
# w/ tc network ---
transmitter.destination_host = dst_host_ex_option
result = transmitter.ping()
pingparser.parse(result)
with_tc_rtt_avg = pingparser.rtt_avg
# assertion ---
rtt_diff = with_tc_rtt_avg - without_tc_rtt_avg
assert rtt_diff > (delay / 2.0)
# finalize ---
subproc_wrapper.run("tcdel --device " + DEVICE)
def test_network(self, dst_host_option, dst_host_ex_option,
subproc_wrapper, transmitter, pingparser):
if any([
dataproperty.is_empty_string(dst_host_option),
dataproperty.is_empty_string(dst_host_ex_option),
]):
# alternative to pytest.mark.skipif
return
subproc_wrapper.run("tcdel --device " + DEVICE)
delay = 100
# tc to specific network ---
command_list = [
"tcset",
"--device " + DEVICE,
"--delay %d" % (delay),
"--network " + dst_host_ex_option,
]
assert subproc_wrapper.run(" ".join(command_list)) == 0
# w/o tc network ---
transmitter.destination_host = dst_host_option
result = transmitter.ping()
pingparser.parse(result)
without_tc_rtt_avg = pingparser.rtt_avg
# w/ tc network ---
transmitter.destination_host = dst_host_ex_option
result = transmitter.ping()
pingparser.parse(result)
with_tc_rtt_avg = pingparser.rtt_avg
# assertion ---
rtt_diff = with_tc_rtt_avg - without_tc_rtt_avg
assert rtt_diff > (delay / 2.0)
# finalize ---
subproc_wrapper.run("tcdel --device " + DEVICE)
def parse(self, text):
for line in text.splitlines():
self.__clear()
if dataproperty.is_empty_string(line):
continue
line = _to_unicode(line.lstrip())
self.__parse_classid(line)
self.__parse_rate(line)
yield self.__parsed_param
def __validate_network(self):
if dataproperty.is_empty_string(self.network):
return ""
try:
ipaddress.IPv4Address(six.u(self.network))
return self.network + "/32"
except ipaddress.AddressValueError:
pass
ipaddress.IPv4Network(six.u(self.network))
return self.network
raise ValueError("unrecognizable network: " + self.network)
def __set_pre_network_filter(self, qdisc_major_id):
if dataproperty.is_empty_string(self.network):
flowid = "%d:%d" % (qdisc_major_id, self.__get_qdisc_minor_id())
else:
flowid = "%d:2" % (qdisc_major_id)
command_list = [
"tc filter add", "dev " + self.__get_tc_device(), "protocol ip",
"parent %d:" % (qdisc_major_id),
"prio 2 u32 match ip %s 0.0.0.0/0" %
(self.__get_network_direction_str()), "flowid " + flowid
]
return self.__subproc_wrapper.run(" ".join(command_list))