def __get_filter_key(self, filter_param):
key_items = OrderedDict()
if Tc.Param.HANDLE in filter_param:
handle = filter_param.get(Tc.Param.HANDLE)
typepy.Integer(handle).validate()
handle = int(handle)
for mangle in self.__iptables_ctrl.parse():
if mangle.mark_id != handle:
continue
key_items[Tc.Param.DST_NETWORK] = mangle.destination
if typepy.is_not_null_string(mangle.source):
key_items[Tc.Param.SRC_NETWORK] = mangle.source
key_items[Tc.Param.PROTOCOL] = mangle.protocol
break
else:
raise ValueError("mangle mark not found: {}".format(mangle))
else:
src_network = filter_param.get(Tc.Param.SRC_NETWORK)
if typepy.is_not_null_string(
src_network) and not is_anywhere_network(
src_network, self.__ip_version):
key_items[Tc.Param.SRC_NETWORK] = src_network
dst_network = filter_param.get(Tc.Param.DST_NETWORK)
if typepy.is_not_null_string(
dst_network) and not is_anywhere_network(
dst_network, self.__ip_version):
key_items[Tc.Param.DST_NETWORK] = dst_network
src_port = filter_param.get(Tc.Param.SRC_PORT)
if typepy.Integer(src_port).is_type():
key_items[Tc.Param.SRC_PORT] = "{}".format(src_port)
elif src_port is not None:
self.__logger.warning(
"expected a integer value for {}, actual {}: {}".format(
Tc.Param.SRC_PORT,
type(src_port),
src_port,
))
dst_port = filter_param.get(Tc.Param.DST_PORT)
if typepy.Integer(dst_port).is_type():
key_items[Tc.Param.DST_PORT] = "{}".format(dst_port)
elif src_port is not None:
self.__logger.warning(
"expected a integer value for {}, actual {}".format(
Tc.Param.DST_PORT, type(dst_port)))
protocol = filter_param.get(Tc.Param.PROTOCOL)
if typepy.is_not_null_string(protocol):
key_items[Tc.Param.PROTOCOL] = protocol
key = ", ".join(
["{}={}".format(key, value) for key, value in key_items.items()])
return key, key_items
def create_table_from_json(self, json_source, table_name=""):
"""
Create a table from a JSON file/text.
:param str json_source: Path to the JSON file or JSON text.
:param str table_name: Table name to create.
:Examples:
:ref:`example-create-table-from-json`
.. seealso::
:py:meth:`.pytablereader.JsonTableFileLoader.load`
:py:meth:`.pytablereader.JsonTableTextLoader.load`
"""
loader = ptr.JsonTableFileLoader(json_source)
if typepy.is_not_null_string(table_name):
loader.table_name = table_name
try:
for tabledata in loader.load():
self.create_table_from_tabledata(tabledata)
return
except (ptr.InvalidFilePathError, IOError):
pass
loader = ptr.JsonTableTextLoader(json_source)
if typepy.is_not_null_string(table_name):
loader.table_name = table_name
for tabledata in loader.load():
self.create_table_from_tabledata(tabledata)
def create_table_from_json(self, json_source, table_name=""):
"""
Create a table from a JSON file/text.
:param str json_source: Path to the JSON file or JSON text.
:param str table_name: Table name to create.
:Examples:
:ref:`example-create-table-from-json`
.. seealso::
:py:meth:`.pytablereader.JsonTableFileLoader.load`
:py:meth:`.pytablereader.JsonTableTextLoader.load`
"""
loader = ptr.JsonTableFileLoader(json_source)
if typepy.is_not_null_string(table_name):
loader.table_name = table_name
try:
for tabledata in loader.load():
self.create_table_from_tabledata(tabledata)
return
except (ptr.InvalidFilePathError, IOError):
pass
loader = ptr.JsonTableTextLoader(json_source)
if typepy.is_not_null_string(table_name):
loader.table_name = table_name
for tabledata in loader.load():
self.create_table_from_tabledata(tabledata)
def __get_filter_key(self, filter_param):
network_format = "network={:s}"
port_format = "port={:d}"
key_item_list = []
if Tc.Param.HANDLE in filter_param:
handle = filter_param.get(Tc.Param.HANDLE)
Integer(handle).validate()
handle = int(handle)
for mangle in IptablesMangleController.parse():
if mangle.mark_id != handle:
continue
key_item_list.append(network_format.format(mangle.destination))
if typepy.is_not_null_string(mangle.source):
key_item_list.append("source={:s}".format(mangle.source))
key_item_list.append("protocol={}".format(mangle.protocol))
break
else:
raise ValueError("mangle mark not found: {}".format(mangle))
else:
network = filter_param.get(Tc.Param.NETWORK)
if typepy.is_not_null_string(network):
key_item_list.append(network_format.format(network))
port = filter_param.get(Tc.Param.PORT)
if Integer(port).is_type():
key_item_list.append(port_format.format(port))
return ", ".join(key_item_list)
def create_table_from_csv(self,
csv_source,
table_name="",
attr_name_list=(),
delimiter=",",
quotechar='"',
encoding="utf-8"):
"""
Create a table from a CSV file/text.
:param str csv_source: Path to the CSV file or CSV text.
:param str table_name:
Table name to create.
Using CSV file basename as the table name if the value is empty.
:param list attr_name_list:
Attribute names of the table.
Use the first line of the CSV file as attribute list
if attribute_name_list is empty.
:param str delimiter:
A one-character string used to separate fields.
:param str quotechar:
A one-character string used to quote fields containing special
characters, such as the ``delimiter`` or ``quotechar``,
or which contain new-line characters.
:param str encoding: CSV file encoding.
:raises ValueError: If the CSV data is invalid.
:Example:
:ref:`example-create-table-from-csv`
.. seealso::
:py:meth:`.create_table_from_data_matrix`
:py:func:`csv.reader`
:py:meth:`.pytablereader.CsvTableFileLoader.load`
:py:meth:`.pytablereader.CsvTableTextLoader.load`
"""
loader = ptr.CsvTableFileLoader(csv_source)
if typepy.is_not_null_string(table_name):
loader.table_name = table_name
loader.header_list = attr_name_list
loader.delimiter = delimiter
loader.quotechar = quotechar
loader.encoding = encoding
try:
for tabledata in loader.load():
self.create_table_from_tabledata(tabledata)
return
except (ptr.InvalidFilePathError, IOError):
pass
loader = ptr.CsvTableTextLoader(csv_source)
if typepy.is_not_null_string(table_name):
loader.table_name = table_name
loader.header_list = attr_name_list
loader.delimiter = delimiter
loader.quotechar = quotechar
loader.encoding = encoding
for tabledata in loader.load():
self.create_table_from_tabledata(tabledata)
def __get_ping_command(self):
command_list = []
if self.__is_windows() and self.auto_codepage:
command_list.append("chcp 437 &")
command_list.extend(
[
self.__get_builtin_ping_command(),
self.__get_deadline_option(),
self.__get_timeout_option(),
self.__get_count_option(),
self.__get_quiet_option(),
]
)
if self.__is_linux() and typepy.is_not_null_string(self.interface):
command_list.append("-I {}".format(self.interface))
if typepy.is_not_null_string(self.ping_option):
command_list.append(self.ping_option)
command_list.append(self.__get_destination_host())
return " ".join(command_list)
def _add_exclude_filter(self):
import subprocrunner
if all(
[
typepy.is_null_string(param)
for param in (
self._tc_obj.exclude_dst_network,
self._tc_obj.exclude_src_network,
self._tc_obj.exclude_dst_port,
self._tc_obj.exclude_src_port,
)
]
):
logger.debug("no exclude filter found")
return
command_item_list = [
self._tc_obj.get_tc_command(TcSubCommand.FILTER),
self._dev,
"protocol {:s}".format(self._tc_obj.protocol),
"parent {:s}:".format(self._tc_obj.qdisc_major_id_str),
"prio 1",
"u32",
]
if typepy.is_not_null_string(self._tc_obj.exclude_dst_network):
command_item_list.append(
"match {:s} {:s} {:s}".format(
self._tc_obj.protocol_match, "dst", self._tc_obj.exclude_dst_network
)
)
if typepy.is_not_null_string(self._tc_obj.exclude_src_network):
command_item_list.append(
"match {:s} {:s} {:s}".format(
self._tc_obj.protocol_match, "src", self._tc_obj.exclude_src_network
)
)
if typepy.is_not_null_string(self._tc_obj.exclude_dst_port):
command_item_list.append(
"match {:s} {:s} {:s} 0xffff".format(
self._tc_obj.protocol_match, "dport", self._tc_obj.exclude_dst_port
)
)
if typepy.is_not_null_string(self._tc_obj.exclude_src_port):
command_item_list.append(
"match {:s} {:s} {:s} 0xffff".format(
self._tc_obj.protocol_match, "sport", self._tc_obj.exclude_src_port
)
)
command_item_list.append("flowid {:s}".format(self.__classid_wo_shaping))
return subprocrunner.SubprocessRunner(" ".join(command_item_list)).run()
def create_table_from_csv(
self, csv_source, table_name="", attr_name_list=(),
delimiter=",", quotechar='"', encoding="utf-8"):
"""
Create a table from a CSV file/text.
:param str csv_source: Path to the CSV file or CSV text.
:param str table_name:
Table name to create.
Using CSV file basename as the table name if the value is empty.
:param list attr_name_list:
Attribute names of the table.
Use the first line of the CSV file as attribute list
if attribute_name_list is empty.
:param str delimiter:
A one-character string used to separate fields.
:param str quotechar:
A one-character string used to quote fields containing special
characters, such as the ``delimiter`` or ``quotechar``,
or which contain new-line characters.
:param str encoding: CSV file encoding.
:raises ValueError: If the CSV data is invalid.
:Example:
:ref:`example-create-table-from-csv`
.. seealso::
:py:meth:`.create_table_from_data_matrix`
:py:func:`csv.reader`
:py:meth:`.pytablereader.CsvTableFileLoader.load`
:py:meth:`.pytablereader.CsvTableTextLoader.load`
"""
loader = ptr.CsvTableFileLoader(csv_source)
if typepy.is_not_null_string(table_name):
loader.table_name = table_name
loader.header_list = attr_name_list
loader.delimiter = delimiter
loader.quotechar = quotechar
loader.encoding = encoding
try:
for tabledata in loader.load():
self.create_table_from_tabledata(tabledata)
return
except (ptr.InvalidFilePathError, IOError):
pass
loader = ptr.CsvTableTextLoader(csv_source)
if typepy.is_not_null_string(table_name):
loader.table_name = table_name
loader.header_list = attr_name_list
loader.delimiter = delimiter
loader.quotechar = quotechar
loader.encoding = encoding
for tabledata in loader.load():
self.create_table_from_tabledata(tabledata)
def __get_filter_key(self, filter_param):
src_network_format = Tc.Param.SRC_NETWORK + "={:s}"
dst_network_format = Tc.Param.DST_NETWORK + "={:s}"
protocol_format = Tc.Param.PROTOCOL + "={:s}"
key_item_list = []
if Tc.Param.HANDLE in filter_param:
handle = filter_param.get(Tc.Param.HANDLE)
Integer(handle).validate()
handle = int(handle)
for mangle in self.__iptables_ctrl.parse():
if mangle.mark_id != handle:
continue
key_item_list.append(
dst_network_format.format(mangle.destination))
if typepy.is_not_null_string(mangle.source):
key_item_list.append("{:s}={:s}".format(
Tc.Param.SRC_NETWORK, mangle.source))
key_item_list.append(protocol_format.format(mangle.protocol))
break
else:
raise ValueError("mangle mark not found: {}".format(mangle))
else:
src_network = filter_param.get(Tc.Param.SRC_NETWORK)
if typepy.is_not_null_string(
src_network) and not is_anywhere_network(
src_network, self.__ip_version):
key_item_list.append(src_network_format.format(src_network))
dst_network = filter_param.get(Tc.Param.DST_NETWORK)
if typepy.is_not_null_string(
dst_network) and not is_anywhere_network(
dst_network, self.__ip_version):
key_item_list.append(dst_network_format.format(dst_network))
src_port = filter_param.get(Tc.Param.SRC_PORT)
if Integer(src_port).is_type():
port_format = Tc.Param.SRC_PORT + "={:d}"
key_item_list.append(port_format.format(src_port))
dst_port = filter_param.get(Tc.Param.DST_PORT)
if Integer(dst_port).is_type():
port_format = Tc.Param.DST_PORT + "={:d}"
key_item_list.append(port_format.format(dst_port))
protocol = filter_param.get(Tc.Param.PROTOCOL)
if typepy.is_not_null_string(protocol):
key_item_list.append(protocol_format.format(protocol))
return ", ".join(key_item_list)
def write_table(self):
"""
|write_table| with HTML table format.
:Example:
:ref:`example-html-table-writer`
.. note::
- |None| is not written
"""
tags = _get_tags_module()
with self._logger:
self._verify_property()
self._preprocess()
if typepy.is_not_null_string(self.table_name):
self._table_tag = tags.table(id=sanitize_python_var_name(self.table_name))
self._table_tag += tags.caption(MultiByteStrDecoder(self.table_name).unicode_str)
else:
self._table_tag = tags.table()
try:
self._write_header()
except EmptyHeaderError:
pass
self._write_body()
def test_stderr(self, capsys, command, ignore_stderr_regexp, out_regexp,
expected):
from loguru import logger
import subprocrunner
logger.remove()
logger.add(sys.stderr, level="DEBUG")
logger.enable("test")
subprocrunner.set_logger(True)
runner = SubprocessRunner(command,
ignore_stderr_regexp=ignore_stderr_regexp)
runner.run()
assert is_null_string(runner.stdout.strip())
assert is_not_null_string(runner.stderr.strip())
assert runner.returncode != 0
out, err = capsys.readouterr()
print("[sys stdout]\n{}\n".format(out))
print("[sys stderr]\n{}\n".format(err))
print("[proc stdout]\n{}\n".format(runner.stdout))
print("[proc stderr]\n{}\n".format(runner.stderr))
actual = out_regexp.search(err) is not None
assert actual == expected
def test_normal_stdin(self, command, pipe_input, expected):
proc = SubprocessRunner(command).popen(PIPE)
ret_stdout, ret_stderr = proc.communicate(input=pipe_input)
assert is_not_null_string(ret_stdout)
assert is_null_string(ret_stderr)
assert proc.returncode == expected
def __strip_empty_col(self):
from simplesqlite import connect_memdb
from simplesqlite.query import Attr, AttrList
con = connect_memdb()
tmp_table_name = "tmp"
headers = ["a{:d}".format(i) for i in range(len(self.__all_values[0]))]
con.create_table_from_data_matrix(tmp_table_name, headers,
self.__all_values)
for col_idx, header in enumerate(headers):
result = con.select(select=Attr(header), table_name=tmp_table_name)
if any([
typepy.is_not_null_string(record[0])
for record in result.fetchall()
]):
break
strip_headers = headers[col_idx:]
if typepy.is_empty_sequence(strip_headers):
raise ValueError()
result = con.select(select=AttrList(strip_headers),
table_name=tmp_table_name)
self.__all_values = result.fetchall()
def to_bit(self):
"""
:raises ParameterError:
"""
logger.debug("human readable size to bit: {}".format(
self.__readable_size))
if not typepy.is_not_null_string(self.__readable_size):
raise TypeError("readable_size must be a string: actual={}".format(
self.__readable_size))
self.__readable_size = self.__readable_size.strip()
try:
size = _RE_NUMBER.search(self.__readable_size).group()
except AttributeError:
raise ParameterError("invalid value", value=self.__readable_size)
size = float(size)
if size < 0:
raise ParameterError("size must be greater or equals to zero",
value=size)
unit = _RE_NUMBER.sub("", self.__readable_size).strip().lower()
return size * self.__get_coefficient(unit)
def write_tc_script(tcconfig_command, command_history, filename_suffix=None):
filename_item_list = [tcconfig_command]
if typepy.is_not_null_string(filename_suffix):
filename_item_list.append(filename_suffix)
script_line_list = ["#!/bin/sh", ""]
org_tcconfig_cmd = _get_original_tcconfig_command(tcconfig_command)
if tcconfig_command != Tc.Command.TCSHOW:
script_line_list.extend([
"# command sequence in this script attempt to simulate the following command:",
"# '{:s}'.".format(org_tcconfig_cmd),
])
script_line_list.extend([
"#",
"# note: the command sequence execution may result different, compared to '{}.'"
.format(org_tcconfig_cmd),
"#",
"# created by {:s} on {:s}.".format(
tcconfig_command,
datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S%z")),
"",
command_history,
])
filename = "_".join(filename_item_list) + ".sh"
with io.open(filename, "w", encoding="utf8") as fp:
fp.write("\n".join(script_line_list) + "\n")
os.chmod(filename, 0o755)
logger.info("written a tc script to '{:s}'".format(filename))
def _write_table(self, **kwargs) -> None:
sort_keys = kwargs.get("sort_keys", False)
self._preprocess()
with self._logger:
self._write_opening_row()
json_text_list = []
for json_data in self._table_value_matrix:
json_text = json.dumps(json_data,
indent=self._indent_level,
ensure_ascii=False,
sort_keys=sort_keys)
json_text_list.append(json_text)
joint_text = self.char_right_side_row + "\n"
json_text = joint_text.join(json_text_list)
if all([
not self.is_write_closing_row,
typepy.is_not_null_string(json_text)
]):
json_text += joint_text
self.stream.write(indent(json_text, " " * self._indent_level))
self._write_closing_row()
def write_table(self, **kwargs) -> None:
"""
|write_table| with HTML table format.
Args:
write_css (bool):
If |True|, write CSS corresponding to the specified styles,
instead of attributes of HTML tags.
Example:
:ref:`example-html-table-writer`
.. note::
- |None| values will be replaced with an empty value
"""
tags, raw = _get_tags_module()
write_css = kwargs.get("write_css", False)
with self._logger:
try:
self._verify_property()
except EmptyTableDataError:
self._logger.logger.debug("no tabular data found")
return
self._preprocess()
css_class = None
if write_css:
css_class = kwargs.get(
"css_class",
"{}-css".format(
replace_symbol(self.table_name, replacement_text="-")),
)
css_writer = CssTableWriter(table_name=css_class)
css_writer.from_writer(self, is_overwrite_table_name=False)
css_writer.write_table(write_style_tag=True)
if typepy.is_not_null_string(self.table_name):
if css_class:
self._table_tag = tags.table(id=sanitize_python_var_name(
self.table_name),
class_name=css_class)
else:
self._table_tag = tags.table(
id=sanitize_python_var_name(self.table_name))
self._table_tag += tags.caption(
MultiByteStrDecoder(self.table_name).unicode_str)
else:
self._table_tag = tags.table()
try:
self._write_header()
except ValueError:
pass
self._write_body(not write_css)
def write_table(self):
"""
|write_table| with
`Labeled Tab-separated Values (LTSV) <http://ltsv.org/>`__ format.
Invalid characters in labels/data are removed.
:raises pytablewriter.EmptyHeaderError: If the |headers| is empty.
:Example:
:ref:`example-ltsv-table-writer`
"""
with self._logger:
self._verify_property()
self._preprocess()
for values in self._table_value_matrix:
ltsv_item_list = [
"{:s}:{}".format(pathvalidate.sanitize_ltsv_label(header_name), value)
for header_name, value in zip(self.headers, values)
if typepy.is_not_null_string(value)
]
if typepy.is_empty_sequence(ltsv_item_list):
continue
self._write_line("\t".join(ltsv_item_list))
def make_update(cls, table, set_query, where=None):
"""
Make UPDATE query.
:param str table: Table name of executing the query.
:param str set_query: SET part of the UPDATE query.
:param str where:
Add a WHERE clause to execute query,
if the value is not |None|.
:return: Query of SQLite.
:rtype: str
:raises ValueError: If ``set_query`` is empty string.
:raises simplesqlite.InvalidTableNameError:
|raises_validate_table_name|
"""
validate_table_name(table)
if typepy.is_null_string(set_query):
raise ValueError("SET query is null")
query_list = [
"UPDATE {:s}".format(cls.to_table_str(table)),
"SET {:s}".format(set_query),
]
if typepy.is_not_null_string(where):
query_list.append("WHERE {:s}".format(where))
return " ".join(query_list)
def write_table(self):
"""
|write_table| with
`Labeled Tab-separated Values (LTSV) <http://ltsv.org/>`__ format.
Invalid characters in labels/data are removed.
:raises pytablewriter.EmptyHeaderError: If the |header_list| is empty.
:Example:
:ref:`example-ltsv-table-writer`
"""
with self._logger:
self._verify_property()
self._preprocess()
for value_list in self._table_value_matrix:
ltsv_item_list = [
"{:s}:{}".format(pathvalidate.sanitize_ltsv_label(header_name), value)
for header_name, value in zip(self.header_list, value_list)
if typepy.is_not_null_string(value)
]
if typepy.is_empty_sequence(ltsv_item_list):
continue
self._write_line("\t".join(ltsv_item_list))
def _get_opening_row_item_list(self):
array_def = "{:s}.array([".format(self.import_numpy_as)
if typepy.is_not_null_string(self.table_name):
return ["{} = {}".format(self.variable_name, array_def)]
return [array_def]
def write_table(self, **kwargs) -> None:
"""
|write_table| with
`Labeled Tab-separated Values (LTSV) <http://ltsv.org/>`__ format.
Invalid characters in labels/data are removed.
:Example:
:ref:`example-ltsv-table-writer`
"""
with self._logger:
self._verify_property()
self._preprocess()
for values in self._table_value_matrix:
ltsv_item_list = [
f"{pathvalidate.sanitize_ltsv_label(header_name):s}:{value}"
for header_name, value in zip(self.headers, values)
if typepy.is_not_null_string(value)
]
if typepy.is_empty_sequence(ltsv_item_list):
continue
self._write_line("\t".join(ltsv_item_list))
def __strip_empty_col(self):
from simplesqlite import connect_sqlite_db_mem
from simplesqlite.sqlquery import SqlQuery
con = connect_sqlite_db_mem()
tmp_table_name = "tmp"
header_list = [
"a{:d}".format(i) for i in range(len(self.__all_values[0]))
]
con.create_table_from_data_matrix(table_name=tmp_table_name,
attr_name_list=header_list,
data_matrix=self.__all_values)
for col_idx, header in enumerate(header_list):
result = con.select(select=SqlQuery.to_attr_str(header),
table_name=tmp_table_name)
if any([
typepy.is_not_null_string(record[0])
for record in result.fetchall()
]):
break
strip_header_list = header_list[col_idx:]
if typepy.is_empty_sequence(strip_header_list):
raise ValueError()
result = con.select(select=",".join(
SqlQuery.to_attr_str_list(strip_header_list)),
table_name=tmp_table_name)
self.__all_values = result.fetchall()
def _get_opening_row_items(self) -> List[str]:
array_def = f"{self.import_numpy_as:s}.array(["
if typepy.is_not_null_string(self.table_name):
return [f"{self.variable_name} = {array_def}"]
return [array_def]
def _get_opening_row_items(self):
array_def = "{:s}.array([".format(self.import_numpy_as)
if typepy.is_not_null_string(self.table_name):
return ["{} = {}".format(self.variable_name, array_def)]
return [array_def]
def to_query(self) -> str:
name = self.sanitize(self._value)
need_quote = self.__RE_NEED_QUOTE.search(name) is not None
try:
validate_sqlite_attr_name(name)
except ValidationError as e:
if e.reason == ErrorReason.RESERVED_NAME and e.reusable_name is False:
need_quote = True
elif e.reason in (
ErrorReason.RESERVED_NAME,
ErrorReason.NULL_NAME,
ErrorReason.INVALID_CHARACTER,
):
pass
else:
raise
if need_quote:
sql_name = f'"{name:s}"'
elif self.__RE_NEED_BRACKET.search(name):
sql_name = f"[{name:s}]"
elif name == "join":
sql_name = f"[{name:s}]"
else:
sql_name = name
if typepy.is_not_null_string(self.__operation):
sql_name = f"{self.__operation:s}({sql_name:s})"
return sql_name
def run_command_helper(command, error_regexp, message, exception=None):
if logger.level != logbook.DEBUG:
spr.set_logger(is_enable=False)
proc = spr.SubprocessRunner(command)
proc.run()
if logger.level != logbook.DEBUG:
spr.set_logger(is_enable=True)
if proc.returncode == 0:
return 0
match = error_regexp.search(proc.stderr)
if match is None:
logger.error(proc.stderr)
return proc.returncode
if typepy.is_not_null_string(message):
logger.notice(message)
if exception is not None:
raise exception(command)
return proc.returncode
def make_update(cls, table, set_query, where=None):
"""
Make UPDATE query.
:param str table: Table name of executing the query.
:param str set_query: SET part of the UPDATE query.
:param str where:
Add a WHERE clause to execute query,
if the value is not |None|.
:return: Query of SQLite.
:rtype: str
:raises ValueError: If ``set_query`` is empty string.
:raises simplesqlite.InvalidTableNameError:
|raises_validate_table_name|
"""
validate_table_name(table)
if typepy.is_null_string(set_query):
raise ValueError("SET query is null")
query_list = [
"UPDATE {:s}".format(cls.to_table_str(table)),
"SET {:s}".format(set_query),
]
if typepy.is_not_null_string(where):
query_list.append("WHERE {:s}".format(where))
return " ".join(query_list)
def run_command_helper(command,
ignore_error_msg_regexp,
notice_msg,
exception_class=None):
proc = spr.SubprocessRunner(command, error_log_level="QUIET")
proc.run()
if proc.returncode == 0:
return 0
if ignore_error_msg_regexp:
match = ignore_error_msg_regexp.search(proc.stderr)
if match is None:
error_msg = "\n".join([
"command execution failed",
" command={}".format(command),
" stderr={}".format(proc.stderr),
])
if re.search("RTNETLINK answers: Operation not permitted",
proc.stderr):
logger.error(error_msg)
sys.exit(proc.returncode)
logger.error(error_msg)
return proc.returncode
if typepy.is_not_null_string(notice_msg):
logger.warning(notice_msg)
if exception_class is not None:
raise exception_class(command)
return proc.returncode
def ping(self):
"""
Sending ICMP packets.
:return: ``ping`` command execution result.
:rtype: :py:class:`.PingResult`
:raises ValueError: If parameters not valid.
"""
import subprocess
self.__validate_ping_param()
command_list = self.__get_base_ping_command_list()
if typepy.is_not_null_string(self.ping_option):
command_list.append(self.ping_option)
command_list.extend([
self.__get_waittime_option(),
self.__get_count_option(),
])
command = " ".join(command_list)
logger.debug(command)
ping_proc = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = ping_proc.communicate()
return PingResult(stdout, stderr, ping_proc.returncode)
def write_table(self):
"""
|write_table| with HTML table format.
:Example:
:ref:`example-html-table-writer`
.. note::
- |None| is not written
"""
self._logger.logging_start_write()
self._verify_property()
self._preprocess()
if typepy.is_not_null_string(self.table_name):
self._table_tag = tags.table(
id=pathvalidate.sanitize_python_var_name(self.table_name))
self._table_tag += tags.caption(
MultiByteStrDecoder(self.table_name).unicode_str)
else:
self._table_tag = tags.table()
try:
self._write_header()
except EmptyHeaderError:
pass
self._write_body()
self._logger.logging_complete_write()
def _get_opening_row_items(self) -> List[str]:
if typepy.is_not_null_string(self.table_name):
return [
f'{{ "{MultiByteStrDecoder(self.table_name).unicode_str:s}" : ['
]
return ["["]
def _write_table(self):
self._preprocess_value_matrix()
with self._logger:
self._write_opening_row()
self.inc_indent_level()
json_text_list = []
for json_data in self._table_value_matrix:
json_text = json.dumps(json_data,
sort_keys=True,
indent=4 * self._indent_level)
json_text = strip_quote(
json_text,
self._dp_extractor.type_value_map.get(Typecode.NONE))
json_text = strip_quote(json_text, "true")
json_text = strip_quote(json_text, "false")
json_text_list.append(json_text)
joint_text = self.char_right_side_row + "\n"
json_text = joint_text.join(json_text_list)
if all([
not self.is_write_closing_row,
typepy.is_not_null_string(json_text)
]):
json_text += joint_text
self.stream.write(json_text)
self.dec_indent_level()
self._write_closing_row()
def _write_header(self):
if not self.is_write_header:
return
if typepy.is_not_null_string(self.table_name):
self._write_line("|+" + MultiByteStrDecoder(self.table_name).unicode_str)
super(MediaWikiTableWriter, self)._write_header()
def _write_header(self):
if not self.is_write_header:
return
if typepy.is_not_null_string(self.table_name):
self._write_line("|+" + MultiByteStrDecoder(self.table_name).unicode_str)
super(MediaWikiTableWriter, self)._write_header()
def _get_opening_row_items(self):
if typepy.is_not_null_string(self.table_name):
return [
'{{ "{:s}" : ['.format(
MultiByteStrDecoder(self.table_name).unicode_str)
]
return ["["]
def _get_start_row_idx(self):
row_idx = 0
for row_values in self.__all_values:
if all([typepy.is_not_null_string(value) for value in row_values]):
break
row_idx += 1
return self.start_row + row_idx
def __init__(self, url, format_name=None, encoding=None, proxies=None):
loader_factory = TableUrlLoaderFactory(url, encoding, proxies)
if typepy.is_not_null_string(format_name):
loader = loader_factory.create_from_format_name(format_name)
else:
loader = loader_factory.create_from_path()
super(TableUrlLoader, self).__init__(loader)
def make_select(cls, select, table, where=None, extra=None):
"""
Make SELECT query.
:param str select: Attribute for SELECT query.
:param str table: Table name of executing the query.
:param str where:
Add a WHERE clause to execute query,
if the value is not |None|.
:param extra extra:
Add additional clause to execute query,
if the value is not |None|.
:return: Query of SQLite.
:rtype: str
:raises ValueError: ``select`` is empty string.
:raises simplesqlite.InvalidTableNameError:
|raises_validate_table_name|
:Examples:
>>> from simplesqlite.sqlquery import SqlQuery
>>> SqlQuery.make_select(select="value", table="example")
'SELECT value FROM example'
>>> SqlQuery.make_select(select="value", table="example", where=SqlQuery.make_where("key", 1))
'SELECT value FROM example WHERE key = 1'
>>> SqlQuery.make_select(select="value", table="example", where=SqlQuery.make_where("key", 1), extra="ORDER BY value")
'SELECT value FROM example WHERE key = 1 ORDER BY value'
"""
validate_table_name(table)
if typepy.is_null_string(select):
raise ValueError("SELECT query is null")
query_list = [
"SELECT {:s}".format(select),
"FROM {:s}".format(cls.to_table_str(table)),
]
if typepy.is_not_null_string(where):
query_list.append("WHERE {:s}".format(where))
if typepy.is_not_null_string(extra):
query_list.append(extra)
return " ".join(query_list)
def _get_opening_row_items(self):
row_item_list = []
if typepy.is_not_null_string(self.table_name):
row_item_list.append(self.table_name + r" = \left( ")
else:
row_item_list.append(r"\left( ")
row_item_list.extend(
[r"\begin{array}{", "{:s}".format("".join(self._get_col_align_char_list())), "}"]
)
return ["".join(row_item_list)]
def create_database(database_path, dup_table):
db_path = path.Path(database_path)
dir_path = db_path.dirname()
if typepy.is_not_null_string(dir_path):
dir_path.makedirs_p()
is_create_db = not db_path.isfile()
if dup_table == DupDatabase.APPEND:
return (sqlite.SimpleSQLite(db_path, "a"), is_create_db)
return (sqlite.SimpleSQLite(db_path, "w"), is_create_db)
def parse(self, ping_message):
"""
Parse ping command output.
Args:
ping_message (str or :py:class:`~pingparsing.PingResult`):
``ping`` command output.
Returns:
:py:class:`~pingparsing.PingStats`: Parsed result.
"""
try:
# accept PingResult instance as an input
if typepy.is_not_null_string(ping_message.stdout):
ping_message = ping_message.stdout
except AttributeError:
pass
logger.debug("parsing ping result: {}".format(ping_message))
self.__parser = NullPingParser()
if typepy.is_null_string(ping_message):
logger.debug("ping_message is empty")
self.__stats = PingStats()
return self.__stats
ping_lines = _to_unicode(ping_message).splitlines()
parser_class_list = (
LinuxPingParser,
WindowsPingParser,
MacOsPingParser,
AlpineLinuxPingParser,
)
for parser_class in parser_class_list:
self.__parser = parser_class()
try:
self.__stats = self.__parser.parse(ping_lines)
return self.__stats
except ParseError as e:
if e.reason != ParseErrorReason.HEADER_NOT_FOUND:
raise e
except pp.ParseException:
pass
self.__parser = NullPingParser()
return self.__stats
def add_worksheet(self, worksheet_name):
worksheet_name = sanitize_excel_sheet_name(worksheet_name)
if typepy.is_not_null_string(worksheet_name):
if worksheet_name in self._worksheet_table:
# the work sheet is already exists
return self._worksheet_table.get(worksheet_name)
else:
worksheet_name = None
worksheet = self.workbook.add_worksheet(worksheet_name)
self._worksheet_table[worksheet_name] = worksheet
return worksheet
def add_worksheet(self, worksheet_name):
worksheet_name = sanitize_excel_sheet_name(worksheet_name)
if typepy.is_not_null_string(worksheet_name):
if worksheet_name in self._worksheet_table:
# the work sheet is already exists
return self._worksheet_table.get(worksheet_name)
else:
sheet_id = 1
while True:
worksheet_name = "Sheet{:d}".format(sheet_id)
if worksheet_name not in self._worksheet_table:
break
sheet_id += 1
worksheet = self.workbook.add_sheet(worksheet_name)
self._worksheet_table[worksheet_name] = worksheet
return worksheet
def to_attr_str(cls, name, operation_query=""):
"""
:param str name: Attribute name.
:param str operation_query:
Used as a SQLite function if the value is not empty.
:return: String that suitable for attribute name of a SQLite query.
:rtype: str
:Examples:
>>> from simplesqlite.sqlquery import SqlQuery
>>> SqlQuery.to_attr_str("key")
'key'
>>> SqlQuery.to_attr_str("a+b")
'[a+b]'
>>> SqlQuery.to_attr_str("key", operation_query="SUM")
'SUM(key)'
"""
name = cls.sanitize_attr(name)
is_quote = cls.__RE_TO_ATTR_QUOTE.search(name) is not None
try:
pv.validate_sqlite_attr_name(name)
except pv.InvalidReservedNameError:
is_quote = True
except (pv.ValidReservedNameError, pv.NullNameError, pv.InvalidCharError):
pass
if is_quote:
sql_name = '"{:s}"'.format(name)
elif cls.__RE_TO_ATTR_BRACKET.search(name):
sql_name = "[{:s}]".format(name)
elif name == "join":
sql_name = "[{:s}]".format(name)
else:
sql_name = name
if typepy.is_not_null_string(operation_query):
sql_name = "{:s}({:s})".format(operation_query, sql_name)
return sql_name
def to_query(self):
name = self.sanitize(self._value)
need_quote = self.__RE_NEED_QUOTE.search(name) is not None
try:
validate_sqlite_attr_name(name)
except InvalidReservedNameError:
need_quote = True
except (ValidReservedNameError, NullNameError, InvalidCharError):
pass
if need_quote:
sql_name = '"{:s}"'.format(name)
elif self.__RE_NEED_BRACKET.search(name):
sql_name = "[{:s}]".format(name)
elif name == "join":
sql_name = "[{:s}]".format(name)
else:
sql_name = name
if typepy.is_not_null_string(self.__operation):
sql_name = "{:s}({:s})".format(self.__operation, sql_name)
return sql_name
