def __setup_ifb(self): if self.direction != TrafficDirection.INCOMING: return 0 if typepy.is_null_string(self.ifb_device): return -1 return_code = 0 modprobe_proc = spr.SubprocessRunner("modprobe ifb") try: if modprobe_proc.run() != 0: logger.error(modprobe_proc.stderr) except spr.CommandError as e: logger.debug(msgfy.to_debug_message(e)) if self.is_add_shaping_rule or self.is_change_shaping_rule: notice_message = None else: notice_message = self.EXISTS_MSG_TEMPLATE.format( "failed to add ip link: ip link already exists." ) return_code |= run_command_helper( "{:s} link add {:s} type ifb".format(find_bin_path("ip"), self.ifb_device), ignore_error_msg_regexp=self.REGEXP_FILE_EXISTS, notice_msg=notice_message, ) return_code |= spr.SubprocessRunner( "{:s} link set dev {:s} up".format(find_bin_path("ip"), self.ifb_device) ).run() base_command = "{:s} add".format(get_tc_base_command(TcSubCommand.QDISC)) if self.is_add_shaping_rule or self.is_change_shaping_rule: notice_message = None else: notice_message = self.EXISTS_MSG_TEMPLATE.format( "failed to '{:s}': ingress qdisc already exists.".format(base_command) ) return_code |= run_command_helper( "{:s} dev {:s} ingress".format(base_command, self.device), ignore_error_msg_regexp=self.REGEXP_FILE_EXISTS, notice_msg=notice_message, ) return_code |= spr.SubprocessRunner( " ".join( [ "{:s} add".format(get_tc_base_command(TcSubCommand.FILTER)), "dev {:s}".format(self.device), "parent ffff: protocol {:s} u32 match u32 0 0".format(self.protocol), "flowid {:x}:".format(self.__qdisc_major_id), "action mirred egress redirect", "dev {:s}".format(self.ifb_device), ] ) ).run() return return_code
def __is_ipv6(self): try: network = ipaddress.ip_address(six.text_type(self.destination_host)) except ValueError as e: logger.debug(msgfy.to_debug_message(e)) return False logger.debug( "IP address: version={}, address={}".format(network.version, self.destination_host) ) return network.version == 6
def url(ctx, url, format_name, encoding, proxy): """ Scrape tabular data from a URL and convert data to a SQLite database file. """ if typepy.is_empty_sequence(url): sys.exit(ExitCode.NO_INPUT) initialize_log_handler(ctx.obj[Context.LOG_LEVEL]) logger = make_logger("{:s} url".format(PROGRAM_NAME), ctx.obj[Context.LOG_LEVEL]) try: app_configs = app_config_mgr.load() except ValueError as e: logger.debug(msgfy.to_debug_message(e)) app_configs = {} if typepy.is_empty_sequence(encoding): encoding = app_configs.get(ConfigKey.DEFAULT_ENCODING) logger.debug("use default encoding: {}".format(encoding)) if typepy.is_null_string(proxy): proxy = app_configs.get(ConfigKey.PROXY_SERVER) convert_configs = load_convert_config(logger, ctx.obj[Context.CONVERT_CONFIG], subcommand="url") con, is_create_db = create_database(ctx.obj[Context.OUTPUT_PATH], ctx.obj[Context.DUP_DATABASE]) converter = UrlConverter( logger=logger, con=con, symbol_replace_value=ctx.obj[Context.SYMBOL_REPLACE_VALUE], add_pri_key_name=ctx.obj[Context.ADD_PRIMARY_KEY_NAME], convert_configs=convert_configs, index_list=ctx.obj.get(Context.INDEX_LIST), is_type_inference=ctx.obj[Context.TYPE_INFERENCE], is_type_hint_header=ctx.obj[Context.TYPE_HINT_HEADER], verbosity_level=ctx.obj.get(Context.VERBOSITY_LEVEL), format_name=format_name, encoding=encoding, proxy=proxy, ) converter.convert(url) sys.exit(finalize(con, converter, is_create_db))
def verify_netem_module(): import re runner = spr.SubprocessRunner("lsmod") try: if runner.run() != 0: raise OSError(runner.returncode, "failed to execute lsmod") except spr.CommandError as e: # reach here when the kmod package not installed. # this kind of environments could exist such as slim containers. logger.debug(msgfy.to_debug_message(e)) else: if re.search(r"\bsch_netem\b", runner.stdout) is None: raise ModuleNotFoundError("sch_netem module not found")
def extract_starred_info(self, pypi_pkg_name): cache_filepath = self.__pypi_cache_mgr.get_pkg_cache_filepath( pypi_pkg_name, "starred_info") if self.__github_cache_mgr.is_cache_available(cache_filepath): cache_data = self.__github_cache_mgr.load_json(cache_filepath) if cache_data: try: info = GitHubStarredInfo(**cache_data) info.validate() return info except (TypeError, ValueError) as e: logger.debug("failed to load cache: {}".format( msgfy.to_debug_message(e))) pip_show = PipShow.execute(pypi_pkg_name) github_repo_info = self.__find_github_repo_info_from_text( pip_show.content) if github_repo_info: return self.__register_starred_status(pypi_pkg_name, github_repo_info, depth=0) try: starred_info = self.__traverse_github_repo(pip_show, pypi_pkg_name, depth=0) if starred_info: return starred_info return GitHubStarredInfo( pypi_pkg_name=pypi_pkg_name, github_repo_id="[Repository not found]", star_status=StarStatus.NOT_FOUND, is_owned=None, url=None, ) except RateLimitExceededException as e: logger.error(msgfy.to_error_message(e)) return GitHubStarredInfo( pypi_pkg_name=pypi_pkg_name, github_repo_id="Exceed API rate limit", star_status=StarStatus.NOT_AVAILABLE, is_owned=None, url=None, )
def to_sqlite_table(self, data: OrderedDict, keys: List[str]) -> None: if not data: return self.__logger.debug("to_sqlite_table: {}, keys={}".format( type(data), keys)) if isinstance(data, (list, tuple)): for s in data: self.to_sqlite_table(s, keys) return root_maps = {} for key, v in data.items(): if isinstance(v, (str, float) + (int, )) or v is None: root_maps[key] = v continue loader = ptr.JsonTableDictLoader(v) try: for table_data in loader.load(): if re.search("json[0-9]+", table_data.table_name): table_data.table_name = self.__make_table_name(keys + [key]) else: table_data.table_name = self.__make_table_name( keys + [key, table_data.table_name]) self.__convert(table_data) except ptr.DataError: self.to_sqlite_table(v, keys + [key]) except ptr.ValidationError as e: self.__logger.debug(msgfy.to_debug_message(e)) if not root_maps: return loader = ptr.JsonTableDictLoader(root_maps) for table_data in loader.load(): if keys: table_data.table_name = self.__make_table_name(keys) else: table_data.table_name = "root" self.__convert(table_data)
def delete_all_tc(self): result_list = [] result_list.append(self.__delete_qdisc()) result_list.append(self.__delete_ingress_qdisc()) try: result_list.append(self.__delete_ifb_device() == 0) except NetworkInterfaceNotFoundError as e: logger.debug(msgfy.to_debug_message(e)) result_list.append(False) with logging_context("delete iptables mangle table entries"): try: self.iptables_ctrl.clear() except OSError as e: logger.warn("{} (can not delete iptables entries)".format(e)) return any(result_list)
def delete_all_tc(self): result_list = [] with logging_context("delete qdisc"): proc = spr.SubprocessRunner("{:s} del dev {:s} root".format( get_tc_base_command(TcSubCommand.QDISC), self.device)) proc.run() if re.search("RTNETLINK answers: No such file or directory", proc.stderr): logger.notice("no qdisc to delete for the outgoing device.") result_list.append(False) elif re.search("Cannot find device", proc.stderr): raise NetworkInterfaceNotFoundError(device=self.device) else: result_list.append(proc.returncode == 0) with logging_context("delete ingress qdisc"): returncode = run_command_helper( "{:s} del dev {:s} ingress".format( get_tc_base_command(TcSubCommand.QDISC), self.device), ignore_error_msg_regexp=re.compile("|".join([ "RTNETLINK answers: Invalid argument", "RTNETLINK answers: No such file or directory", ])), notice_msg="no qdisc to delete for the incoming device.", ) result_list.append(returncode == 0) with logging_context("delete ifb device"): try: result_list.append(self.__delete_ifb_device() == 0) except NetworkInterfaceNotFoundError as e: logger.debug(msgfy.to_debug_message(e)) result_list.append(False) with logging_context("delete iptables mangle table entries"): try: self.iptables_ctrl.clear() except OSError as e: logger.warn("{} (can not delete iptables entries)".format(e)) return any(result_list)
def to_sqlite_table(self, data, key_list): if not data: return root_maps = {} for key, v in data.items(): if isinstance(v, (six.text_type, float) + six.integer_types) or v is None: root_maps[key] = v continue loader = ptr.JsonTableDictLoader(v) try: for table_data in loader.load(): if re.search("json[0-9]+", table_data.table_name): table_data.table_name = self.__make_table_name(key_list + [key]) else: table_data.table_name = self.__make_table_name( key_list + [key, table_data.table_name] ) self.__convert(table_data) except ptr.DataError: self.to_sqlite_table(v, key_list + [key]) except ptr.ValidationError as e: self.__logger.debug(msgfy.to_debug_message(e)) if not root_maps: return loader = ptr.JsonTableDictLoader(root_maps) for table_data in loader.load(): if key_list: table_data.table_name = self.__make_table_name(key_list) else: table_data.table_name = "root" self.__convert(table_data)
def main(): options = parse_option() initialize_cli(options) check_command_installation("tc") if options.tc_command_output != TcCommandOutput.NOT_SET: spr.SubprocessRunner.default_is_dry_run = True tc_param = {} for device in options.device: try: verify_network_interface(device) tc_param.update( TcShapingRuleParser(device, options.ip_version, options.tc_command_output, logger).get_tc_parameter()) except NetworkInterfaceNotFoundError as e: logger.debug(msgfy.to_debug_message(e)) continue command_history = "\n".join(spr.SubprocessRunner.get_history()) if options.tc_command_output == TcCommandOutput.STDOUT: print(command_history) return 0 if options.tc_command_output == TcCommandOutput.SCRIPT: write_tc_script(Tc.Command.TCSHOW, command_history, filename_suffix="-".join(options.device)) return 0 logger.debug("command history\n{}".format(command_history)) print(json.dumps(tc_param, indent=4)) return 0
def file(ctx, files, format_name, output_path, encoding): """ Convert tabular data within CSV/Excel/HTML/JSON/Jupyter Notebook/LTSV/Markdown/Mediawiki/SQLite/SSV/TSV file(s) to a SQLite database file. """ from ._ipynb_converter import is_ipynb_file_path, load_ipynb_file, convert_nb if typepy.is_empty_sequence(files): sys.exit(ExitCode.NO_INPUT) con = create_database(ctx, output_path) verbosity_level = ctx.obj.get(Context.VERBOSITY_LEVEL) schema_extractor = get_schema_extractor(con, verbosity_level) result_counter = ResultCounter() logger = make_logger("{:s} file".format(PROGRAM_NAME), ctx.obj[Context.LOG_LEVEL]) table_creator = TableCreator(logger=logger, dst_con=con) for file_path in files: file_path = path.Path(file_path) if not file_path.isfile(): logger.error(u"file not found: {}".format(file_path)) result_counter.inc_fail() continue if file_path == output_path: logger.warn( u"skip a file which has the same path as the output file ({})". format(file_path)) continue logger.debug(u"converting '{}'".format(file_path)) convert_count = result_counter.total_count if format_name in IPYNB_FORMAT_NAME_LIST or is_ipynb_file_path( file_path): convert_nb(logger, con, result_counter, nb=load_ipynb_file(file_path, encoding=encoding)) for table_name in con.get_table_name_list(): logger.info( get_success_message( verbosity_level, file_path, schema_extractor.get_table_schema_text(table_name))) result_counter.inc_success() if result_counter.total_count == convert_count: table_not_found_msg_format.format(file_path) continue try: loader = ptr.TableFileLoader(file_path, format_name=format_name, encoding=encoding) except ptr.InvalidFilePathError as e: logger.debug(msgfy.to_debug_message(e)) result_counter.inc_fail() continue except ptr.LoaderNotFoundError: logger.debug( u"loader not found that coincide with '{}'".format(file_path)) result_counter.inc_fail() continue try: for table_data in loader.load(): logger.debug(u"loaded tabledata: {}".format( six.text_type(table_data))) sqlite_tabledata = SQLiteTableDataSanitizer( table_data).normalize() try: table_creator.create(sqlite_tabledata, ctx.obj.get(Context.INDEX_LIST)) result_counter.inc_success() except (ValueError, IOError) as e: logger.debug(u"exception={:s}, path={}, message={}".format( type(e).__name__, file_path, e)) result_counter.inc_fail() continue logger.info( get_success_message( verbosity_level, file_path, schema_extractor.get_table_schema_text( sqlite_tabledata.table_name))) except ptr.OpenError as e: logger.error(u"{:s}: open error: file={}, message='{}'".format( e.__class__.__name__, file_path, str(e))) result_counter.inc_fail() except ptr.ValidationError as e: if loader.format_name == "json": dict_converter = DictConverter(logger, table_creator, result_counter, schema_extractor, verbosity_level, source=file_path, index_list=ctx.obj.get( Context.INDEX_LIST)) try: dict_converter.to_sqlite_table(loader.loader.load_dict(), []) except AttributeError: pass else: continue logger.error( u"{:s}: invalid {} data format: path={}, message={}".format( e.__class__.__name__, _get_format_type_from_path(file_path), file_path, str(e))) result_counter.inc_fail() except ptr.DataError as e: logger.error(u"{:s}: invalid {} data: path={}, message={}".format( e.__class__.__name__, _get_format_type_from_path(file_path), file_path, str(e))) result_counter.inc_fail() if result_counter.total_count == convert_count: logger.warn(table_not_found_msg_format.format(file_path)) write_completion_message(logger, output_path, result_counter) sys.exit(result_counter.get_return_code())
def test_exception_format_str(self, format_str, expected): with pytest.raises(expected): assert msgfy.to_debug_message(ValueError("test"), format_str)
def test_exception_e_object(self, exception_obj, expected): with pytest.raises(expected): assert msgfy.to_debug_message(exception_obj)
def test_normal_format_str(self, format_str): assert msgfy.to_debug_message(ValueError("test"), format_str) != format_str
def test_normal_smoke(self): assert msgfy.to_debug_message(ValueError("test"))
def __convert(self, file_path, source_info_record_base): logger = self._logger result_counter = self._result_counter try: loader = ptr.TableFileLoader( file_path, format_name=self._format_name, encoding=self._encoding, type_hint_rules=TYPE_HINT_FROM_HEADER_RULES if self._is_type_hint_header else None, ) except ptr.InvalidFilePathError as e: logger.debug(msgfy.to_debug_message(e)) result_counter.inc_fail() return except ptr.LoaderNotFoundError: logger.warn("not supported file format: ext={}, path={}".format( file_path.ext, file_path)) result_counter.inc_fail() return source_info_record_base.format_name = loader.format_name try: for table_data in loader.load(): logger.debug("loaded tabledata: {}".format( six.text_type(table_data))) sqlite_tabledata = self.normalize_table(table_data) try: self._table_creator.create( sqlite_tabledata, self._index_list, source_info=source_info_record_base) except (ValueError, IOError) as e: logger.debug("exception={:s}, path={}, message={}".format( type(e).__name__, file_path, e)) result_counter.inc_fail() return record = deepcopy(source_info_record_base) record.dst_table = sqlite_tabledata.table_name SourceInfo.insert(record) except ptr.OpenError as e: logger.error("{:s}: open error: file={}, message='{}'".format( e.__class__.__name__, file_path, str(e))) result_counter.inc_fail() except ptr.ValidationError as e: if loader.format_name == "json": for table_name in self._convert_complex_json( loader.loader, source_info_record_base): record = deepcopy(source_info_record_base) record.dst_table = table_name SourceInfo.insert(record) else: logger.error( "{:s}: invalid {} data format: path={}, message={}".format( e.__class__.__name__, _get_format_type_from_path(file_path), file_path, str(e), )) result_counter.inc_fail() except ptr.DataError as e: logger.error("{:s}: invalid {} data: path={}, message={}".format( e.__class__.__name__, _get_format_type_from_path(file_path), file_path, str(e))) result_counter.inc_fail()