def __setup_ifb(self):
if self.direction != TrafficDirection.INCOMING:
return 0
if typepy.is_null_string(self.ifb_device):
return -1
return_code = 0
modprobe_proc = spr.SubprocessRunner("modprobe ifb")
try:
if modprobe_proc.run() != 0:
logger.error(modprobe_proc.stderr)
except spr.CommandError as e:
logger.debug(msgfy.to_debug_message(e))
if self.is_add_shaping_rule or self.is_change_shaping_rule:
notice_message = None
else:
notice_message = self.EXISTS_MSG_TEMPLATE.format(
"failed to add ip link: ip link already exists."
)
return_code |= run_command_helper(
"{:s} link add {:s} type ifb".format(find_bin_path("ip"), self.ifb_device),
ignore_error_msg_regexp=self.REGEXP_FILE_EXISTS,
notice_msg=notice_message,
)
return_code |= spr.SubprocessRunner(
"{:s} link set dev {:s} up".format(find_bin_path("ip"), self.ifb_device)
).run()
base_command = "{:s} add".format(get_tc_base_command(TcSubCommand.QDISC))
if self.is_add_shaping_rule or self.is_change_shaping_rule:
notice_message = None
else:
notice_message = self.EXISTS_MSG_TEMPLATE.format(
"failed to '{:s}': ingress qdisc already exists.".format(base_command)
)
return_code |= run_command_helper(
"{:s} dev {:s} ingress".format(base_command, self.device),
ignore_error_msg_regexp=self.REGEXP_FILE_EXISTS,
notice_msg=notice_message,
)
return_code |= spr.SubprocessRunner(
" ".join(
[
"{:s} add".format(get_tc_base_command(TcSubCommand.FILTER)),
"dev {:s}".format(self.device),
"parent ffff: protocol {:s} u32 match u32 0 0".format(self.protocol),
"flowid {:x}:".format(self.__qdisc_major_id),
"action mirred egress redirect",
"dev {:s}".format(self.ifb_device),
]
)
).run()
return return_code
def __is_ipv6(self):
try:
network = ipaddress.ip_address(six.text_type(self.destination_host))
except ValueError as e:
logger.debug(msgfy.to_debug_message(e))
return False
logger.debug(
"IP address: version={}, address={}".format(network.version, self.destination_host)
)
return network.version == 6
def url(ctx, url, format_name, encoding, proxy):
"""
Scrape tabular data from a URL and convert data to a SQLite database file.
"""
if typepy.is_empty_sequence(url):
sys.exit(ExitCode.NO_INPUT)
initialize_log_handler(ctx.obj[Context.LOG_LEVEL])
logger = make_logger("{:s} url".format(PROGRAM_NAME),
ctx.obj[Context.LOG_LEVEL])
try:
app_configs = app_config_mgr.load()
except ValueError as e:
logger.debug(msgfy.to_debug_message(e))
app_configs = {}
if typepy.is_empty_sequence(encoding):
encoding = app_configs.get(ConfigKey.DEFAULT_ENCODING)
logger.debug("use default encoding: {}".format(encoding))
if typepy.is_null_string(proxy):
proxy = app_configs.get(ConfigKey.PROXY_SERVER)
convert_configs = load_convert_config(logger,
ctx.obj[Context.CONVERT_CONFIG],
subcommand="url")
con, is_create_db = create_database(ctx.obj[Context.OUTPUT_PATH],
ctx.obj[Context.DUP_DATABASE])
converter = UrlConverter(
logger=logger,
con=con,
symbol_replace_value=ctx.obj[Context.SYMBOL_REPLACE_VALUE],
add_pri_key_name=ctx.obj[Context.ADD_PRIMARY_KEY_NAME],
convert_configs=convert_configs,
index_list=ctx.obj.get(Context.INDEX_LIST),
is_type_inference=ctx.obj[Context.TYPE_INFERENCE],
is_type_hint_header=ctx.obj[Context.TYPE_HINT_HEADER],
verbosity_level=ctx.obj.get(Context.VERBOSITY_LEVEL),
format_name=format_name,
encoding=encoding,
proxy=proxy,
)
converter.convert(url)
sys.exit(finalize(con, converter, is_create_db))
def verify_netem_module():
import re
runner = spr.SubprocessRunner("lsmod")
try:
if runner.run() != 0:
raise OSError(runner.returncode, "failed to execute lsmod")
except spr.CommandError as e:
# reach here when the kmod package not installed.
# this kind of environments could exist such as slim containers.
logger.debug(msgfy.to_debug_message(e))
else:
if re.search(r"\bsch_netem\b", runner.stdout) is None:
raise ModuleNotFoundError("sch_netem module not found")
def extract_starred_info(self, pypi_pkg_name):
cache_filepath = self.__pypi_cache_mgr.get_pkg_cache_filepath(
pypi_pkg_name, "starred_info")
if self.__github_cache_mgr.is_cache_available(cache_filepath):
cache_data = self.__github_cache_mgr.load_json(cache_filepath)
if cache_data:
try:
info = GitHubStarredInfo(**cache_data)
info.validate()
return info
except (TypeError, ValueError) as e:
logger.debug("failed to load cache: {}".format(
msgfy.to_debug_message(e)))
pip_show = PipShow.execute(pypi_pkg_name)
github_repo_info = self.__find_github_repo_info_from_text(
pip_show.content)
if github_repo_info:
return self.__register_starred_status(pypi_pkg_name,
github_repo_info,
depth=0)
try:
starred_info = self.__traverse_github_repo(pip_show,
pypi_pkg_name,
depth=0)
if starred_info:
return starred_info
return GitHubStarredInfo(
pypi_pkg_name=pypi_pkg_name,
github_repo_id="[Repository not found]",
star_status=StarStatus.NOT_FOUND,
is_owned=None,
url=None,
)
except RateLimitExceededException as e:
logger.error(msgfy.to_error_message(e))
return GitHubStarredInfo(
pypi_pkg_name=pypi_pkg_name,
github_repo_id="Exceed API rate limit",
star_status=StarStatus.NOT_AVAILABLE,
is_owned=None,
url=None,
)
def to_sqlite_table(self, data: OrderedDict, keys: List[str]) -> None:
if not data:
return
self.__logger.debug("to_sqlite_table: {}, keys={}".format(
type(data), keys))
if isinstance(data, (list, tuple)):
for s in data:
self.to_sqlite_table(s, keys)
return
root_maps = {}
for key, v in data.items():
if isinstance(v, (str, float) + (int, )) or v is None:
root_maps[key] = v
continue
loader = ptr.JsonTableDictLoader(v)
try:
for table_data in loader.load():
if re.search("json[0-9]+", table_data.table_name):
table_data.table_name = self.__make_table_name(keys +
[key])
else:
table_data.table_name = self.__make_table_name(
keys + [key, table_data.table_name])
self.__convert(table_data)
except ptr.DataError:
self.to_sqlite_table(v, keys + [key])
except ptr.ValidationError as e:
self.__logger.debug(msgfy.to_debug_message(e))
if not root_maps:
return
loader = ptr.JsonTableDictLoader(root_maps)
for table_data in loader.load():
if keys:
table_data.table_name = self.__make_table_name(keys)
else:
table_data.table_name = "root"
self.__convert(table_data)
def url(ctx, url, format_name, encoding, proxy):
"""
Scrape tabular data from a URL and convert data to a SQLite database file.
"""
if typepy.is_empty_sequence(url):
sys.exit(ExitCode.NO_INPUT)
initialize_log_handler(ctx.obj[Context.LOG_LEVEL])
logger = make_logger("{:s} url".format(PROGRAM_NAME), ctx.obj[Context.LOG_LEVEL])
try:
app_configs = app_config_mgr.load()
except ValueError as e:
logger.debug(msgfy.to_debug_message(e))
app_configs = {}
if typepy.is_empty_sequence(encoding):
encoding = app_configs.get(ConfigKey.DEFAULT_ENCODING)
logger.debug("use default encoding: {}".format(encoding))
if typepy.is_null_string(proxy):
proxy = app_configs.get(ConfigKey.PROXY_SERVER)
convert_configs = load_convert_config(logger, ctx.obj[Context.CONVERT_CONFIG], subcommand="url")
con, is_create_db = create_database(ctx.obj[Context.OUTPUT_PATH], ctx.obj[Context.DUP_DATABASE])
converter = UrlConverter(
logger=logger,
con=con,
symbol_replace_value=ctx.obj[Context.SYMBOL_REPLACE_VALUE],
add_pri_key_name=ctx.obj[Context.ADD_PRIMARY_KEY_NAME],
convert_configs=convert_configs,
index_list=ctx.obj.get(Context.INDEX_LIST),
is_type_inference=ctx.obj[Context.TYPE_INFERENCE],
is_type_hint_header=ctx.obj[Context.TYPE_HINT_HEADER],
verbosity_level=ctx.obj.get(Context.VERBOSITY_LEVEL),
format_name=format_name,
encoding=encoding,
proxy=proxy,
)
converter.convert(url)
sys.exit(finalize(con, converter, is_create_db))
def delete_all_tc(self):
result_list = []
result_list.append(self.__delete_qdisc())
result_list.append(self.__delete_ingress_qdisc())
try:
result_list.append(self.__delete_ifb_device() == 0)
except NetworkInterfaceNotFoundError as e:
logger.debug(msgfy.to_debug_message(e))
result_list.append(False)
with logging_context("delete iptables mangle table entries"):
try:
self.iptables_ctrl.clear()
except OSError as e:
logger.warn("{} (can not delete iptables entries)".format(e))
return any(result_list)
def delete_all_tc(self):
result_list = []
with logging_context("delete qdisc"):
proc = spr.SubprocessRunner("{:s} del dev {:s} root".format(
get_tc_base_command(TcSubCommand.QDISC), self.device))
proc.run()
if re.search("RTNETLINK answers: No such file or directory",
proc.stderr):
logger.notice("no qdisc to delete for the outgoing device.")
result_list.append(False)
elif re.search("Cannot find device", proc.stderr):
raise NetworkInterfaceNotFoundError(device=self.device)
else:
result_list.append(proc.returncode == 0)
with logging_context("delete ingress qdisc"):
returncode = run_command_helper(
"{:s} del dev {:s} ingress".format(
get_tc_base_command(TcSubCommand.QDISC), self.device),
ignore_error_msg_regexp=re.compile("|".join([
"RTNETLINK answers: Invalid argument",
"RTNETLINK answers: No such file or directory",
])),
notice_msg="no qdisc to delete for the incoming device.",
)
result_list.append(returncode == 0)
with logging_context("delete ifb device"):
try:
result_list.append(self.__delete_ifb_device() == 0)
except NetworkInterfaceNotFoundError as e:
logger.debug(msgfy.to_debug_message(e))
result_list.append(False)
with logging_context("delete iptables mangle table entries"):
try:
self.iptables_ctrl.clear()
except OSError as e:
logger.warn("{} (can not delete iptables entries)".format(e))
return any(result_list)
def to_sqlite_table(self, data, key_list):
if not data:
return
root_maps = {}
for key, v in data.items():
if isinstance(v, (six.text_type, float) + six.integer_types) or v is None:
root_maps[key] = v
continue
loader = ptr.JsonTableDictLoader(v)
try:
for table_data in loader.load():
if re.search("json[0-9]+", table_data.table_name):
table_data.table_name = self.__make_table_name(key_list + [key])
else:
table_data.table_name = self.__make_table_name(
key_list + [key, table_data.table_name]
)
self.__convert(table_data)
except ptr.DataError:
self.to_sqlite_table(v, key_list + [key])
except ptr.ValidationError as e:
self.__logger.debug(msgfy.to_debug_message(e))
if not root_maps:
return
loader = ptr.JsonTableDictLoader(root_maps)
for table_data in loader.load():
if key_list:
table_data.table_name = self.__make_table_name(key_list)
else:
table_data.table_name = "root"
self.__convert(table_data)
def main():
options = parse_option()
initialize_cli(options)
check_command_installation("tc")
if options.tc_command_output != TcCommandOutput.NOT_SET:
spr.SubprocessRunner.default_is_dry_run = True
tc_param = {}
for device in options.device:
try:
verify_network_interface(device)
tc_param.update(
TcShapingRuleParser(device, options.ip_version,
options.tc_command_output,
logger).get_tc_parameter())
except NetworkInterfaceNotFoundError as e:
logger.debug(msgfy.to_debug_message(e))
continue
command_history = "\n".join(spr.SubprocessRunner.get_history())
if options.tc_command_output == TcCommandOutput.STDOUT:
print(command_history)
return 0
if options.tc_command_output == TcCommandOutput.SCRIPT:
write_tc_script(Tc.Command.TCSHOW,
command_history,
filename_suffix="-".join(options.device))
return 0
logger.debug("command history\n{}".format(command_history))
print(json.dumps(tc_param, indent=4))
return 0
def file(ctx, files, format_name, output_path, encoding):
"""
Convert tabular data within
CSV/Excel/HTML/JSON/Jupyter Notebook/LTSV/Markdown/Mediawiki/SQLite/SSV/TSV
file(s) to a SQLite database file.
"""
from ._ipynb_converter import is_ipynb_file_path, load_ipynb_file, convert_nb
if typepy.is_empty_sequence(files):
sys.exit(ExitCode.NO_INPUT)
con = create_database(ctx, output_path)
verbosity_level = ctx.obj.get(Context.VERBOSITY_LEVEL)
schema_extractor = get_schema_extractor(con, verbosity_level)
result_counter = ResultCounter()
logger = make_logger("{:s} file".format(PROGRAM_NAME),
ctx.obj[Context.LOG_LEVEL])
table_creator = TableCreator(logger=logger, dst_con=con)
for file_path in files:
file_path = path.Path(file_path)
if not file_path.isfile():
logger.error(u"file not found: {}".format(file_path))
result_counter.inc_fail()
continue
if file_path == output_path:
logger.warn(
u"skip a file which has the same path as the output file ({})".
format(file_path))
continue
logger.debug(u"converting '{}'".format(file_path))
convert_count = result_counter.total_count
if format_name in IPYNB_FORMAT_NAME_LIST or is_ipynb_file_path(
file_path):
convert_nb(logger,
con,
result_counter,
nb=load_ipynb_file(file_path, encoding=encoding))
for table_name in con.get_table_name_list():
logger.info(
get_success_message(
verbosity_level, file_path,
schema_extractor.get_table_schema_text(table_name)))
result_counter.inc_success()
if result_counter.total_count == convert_count:
table_not_found_msg_format.format(file_path)
continue
try:
loader = ptr.TableFileLoader(file_path,
format_name=format_name,
encoding=encoding)
except ptr.InvalidFilePathError as e:
logger.debug(msgfy.to_debug_message(e))
result_counter.inc_fail()
continue
except ptr.LoaderNotFoundError:
logger.debug(
u"loader not found that coincide with '{}'".format(file_path))
result_counter.inc_fail()
continue
try:
for table_data in loader.load():
logger.debug(u"loaded tabledata: {}".format(
six.text_type(table_data)))
sqlite_tabledata = SQLiteTableDataSanitizer(
table_data).normalize()
try:
table_creator.create(sqlite_tabledata,
ctx.obj.get(Context.INDEX_LIST))
result_counter.inc_success()
except (ValueError, IOError) as e:
logger.debug(u"exception={:s}, path={}, message={}".format(
type(e).__name__, file_path, e))
result_counter.inc_fail()
continue
logger.info(
get_success_message(
verbosity_level, file_path,
schema_extractor.get_table_schema_text(
sqlite_tabledata.table_name)))
except ptr.OpenError as e:
logger.error(u"{:s}: open error: file={}, message='{}'".format(
e.__class__.__name__, file_path, str(e)))
result_counter.inc_fail()
except ptr.ValidationError as e:
if loader.format_name == "json":
dict_converter = DictConverter(logger,
table_creator,
result_counter,
schema_extractor,
verbosity_level,
source=file_path,
index_list=ctx.obj.get(
Context.INDEX_LIST))
try:
dict_converter.to_sqlite_table(loader.loader.load_dict(),
[])
except AttributeError:
pass
else:
continue
logger.error(
u"{:s}: invalid {} data format: path={}, message={}".format(
e.__class__.__name__,
_get_format_type_from_path(file_path), file_path, str(e)))
result_counter.inc_fail()
except ptr.DataError as e:
logger.error(u"{:s}: invalid {} data: path={}, message={}".format(
e.__class__.__name__, _get_format_type_from_path(file_path),
file_path, str(e)))
result_counter.inc_fail()
if result_counter.total_count == convert_count:
logger.warn(table_not_found_msg_format.format(file_path))
write_completion_message(logger, output_path, result_counter)
sys.exit(result_counter.get_return_code())
def test_exception_format_str(self, format_str, expected):
with pytest.raises(expected):
assert msgfy.to_debug_message(ValueError("test"), format_str)
def test_exception_e_object(self, exception_obj, expected):
with pytest.raises(expected):
assert msgfy.to_debug_message(exception_obj)
def test_normal_format_str(self, format_str):
assert msgfy.to_debug_message(ValueError("test"),
format_str) != format_str
def test_normal_smoke(self):
assert msgfy.to_debug_message(ValueError("test"))
def __convert(self, file_path, source_info_record_base):
logger = self._logger
result_counter = self._result_counter
try:
loader = ptr.TableFileLoader(
file_path,
format_name=self._format_name,
encoding=self._encoding,
type_hint_rules=TYPE_HINT_FROM_HEADER_RULES
if self._is_type_hint_header else None,
)
except ptr.InvalidFilePathError as e:
logger.debug(msgfy.to_debug_message(e))
result_counter.inc_fail()
return
except ptr.LoaderNotFoundError:
logger.warn("not supported file format: ext={}, path={}".format(
file_path.ext, file_path))
result_counter.inc_fail()
return
source_info_record_base.format_name = loader.format_name
try:
for table_data in loader.load():
logger.debug("loaded tabledata: {}".format(
six.text_type(table_data)))
sqlite_tabledata = self.normalize_table(table_data)
try:
self._table_creator.create(
sqlite_tabledata,
self._index_list,
source_info=source_info_record_base)
except (ValueError, IOError) as e:
logger.debug("exception={:s}, path={}, message={}".format(
type(e).__name__, file_path, e))
result_counter.inc_fail()
return
record = deepcopy(source_info_record_base)
record.dst_table = sqlite_tabledata.table_name
SourceInfo.insert(record)
except ptr.OpenError as e:
logger.error("{:s}: open error: file={}, message='{}'".format(
e.__class__.__name__, file_path, str(e)))
result_counter.inc_fail()
except ptr.ValidationError as e:
if loader.format_name == "json":
for table_name in self._convert_complex_json(
loader.loader, source_info_record_base):
record = deepcopy(source_info_record_base)
record.dst_table = table_name
SourceInfo.insert(record)
else:
logger.error(
"{:s}: invalid {} data format: path={}, message={}".format(
e.__class__.__name__,
_get_format_type_from_path(file_path),
file_path,
str(e),
))
result_counter.inc_fail()
except ptr.DataError as e:
logger.error("{:s}: invalid {} data: path={}, message={}".format(
e.__class__.__name__, _get_format_type_from_path(file_path),
file_path, str(e)))
result_counter.inc_fail()
