Ejemplo n.º 1
0
    def make_select(cls, select, table, where=None, extra=None):
        """
        Make SELECT query.

        :param str select: Attribute for SELECT query
        :param str table: Table name of execute query.
        :param str where: Add WHERE clause to execute query if not ``None``
        :param extra extra: Add additional clause to execute query if not ``None``
        :return: Query of SQLite.
        :rtype: str

        :raises ValueError: ``select`` is empty string.

        .. seealso::

            :py:func:`validate_table_name() <simplesqlite.validate_table_name>`
        """

        validate_table_name(table)
        if dataproperty.is_empty_string(select):
            raise ValueError("SELECT query is null")

        query_list = [
            "SELECT " + select,
            "FROM " + cls.to_table_str(table),
        ]
        if dataproperty.is_not_empty_string(where):
            query_list.append("WHERE " + where)
        if dataproperty.is_not_empty_string(extra):
            query_list.append(extra)

        return " ".join(query_list)
Ejemplo n.º 2
0
    def __get_filter_key(self, filter_param):
        network_format = "network={:s}"
        port_format = "port={:d}"
        key_item_list = []

        if "handle" in filter_param:
            handle = filter_param.get("handle")

            IntegerType(handle).validate()

            handle = int(handle)
            for mangle in IptablesMangleController.parse():
                if mangle.mark_id != handle:
                    continue

                key_item_list.append(network_format.format(mangle.destination))
                if dataproperty.is_not_empty_string(mangle.source):
                    key_item_list.append("source={:s}".format(mangle.source))
                key_item_list.append("protocol={}".format(mangle.protocol))

                break
            else:
                raise ValueError("mangle mark not found: {}".format(mangle))
        else:
            if dataproperty.is_not_empty_string(filter_param.get("network")):
                key_item_list.append(
                    network_format.format(filter_param.get("network")))

            if IntegerType(filter_param.get("port")).is_type():
                key_item_list.append(
                    port_format.format(filter_param.get("port")))

        return ", ".join(key_item_list)
Ejemplo n.º 3
0
    def create_table_from_json(self, json_source, table_name=""):
        """
        Create a table from a JSON file/text.

        :param str json_source: Path to the JSON file or JSON text.
        :param str table_name: Table name to create.

        .. seealso::

            :py:meth:`.loader.JsonTableFileLoader.load`
            :py:meth:`.loader.JsonTableTextLoader.load`
        """

        from .loader import JsonTableFileLoader
        from .loader import JsonTableTextLoader

        loader = JsonTableFileLoader(json_source)
        if dataproperty.is_not_empty_string(table_name):
            loader.table_name = table_name
        try:
            for tabledata in loader.load():
                self.create_table_from_tabledata(tabledata)
            return
        except IOError:
            pass

        loader = JsonTableTextLoader(json_source)
        if dataproperty.is_not_empty_string(table_name):
            loader.table_name = table_name
        for tabledata in loader.load():
            self.create_table_from_tabledata(tabledata)
Ejemplo n.º 4
0
    def __get_filter_key(self, filter_param):
        network_format = "network={:s}"
        port_format = "port={:d}"
        key_item_list = []

        if "handle" in filter_param:
            handle = filter_param.get("handle")

            IntegerType(handle).validate()

            handle = int(handle)
            for mangle in IptablesMangleController.parse():
                if mangle.mark_id != handle:
                    continue

                key_item_list.append(network_format.format(mangle.destination))
                if dataproperty.is_not_empty_string(mangle.source):
                    key_item_list.append("source={:s}".format(mangle.source))
                key_item_list.append("protocol={}".format(mangle.protocol))

                break
            else:
                raise ValueError("mangle mark not found: {}".format(mangle))
        else:
            if dataproperty.is_not_empty_string(filter_param.get("network")):
                key_item_list.append(network_format.format(filter_param.get("network")))

            if IntegerType(filter_param.get("port")).is_type():
                key_item_list.append(port_format.format(filter_param.get("port")))

        return ", ".join(key_item_list)
Ejemplo n.º 5
0
    def make_select(cls, select, table, where=None, extra=None):
        """
        Make SELECT query.

        :param str select: Attribute for SELECT query
        :param str table: Table name of execute query.
        :param str where: Add WHERE clause to execute query if not ``None``
        :param extra extra: Add additional clause to execute query if not ``None``
        :return: Query of SQLite.
        :rtype: str

        :raises ValueError: ``select`` is empty string.

        .. seealso::

            :py:func:`validate_table_name() <simplesqlite.validate_table_name>`
        """

        validate_table_name(table)
        if dataproperty.is_empty_string(select):
            raise ValueError("SELECT query is null")

        query_list = [
            "SELECT " + select,
            "FROM " + cls.to_table_str(table),
        ]
        if dataproperty.is_not_empty_string(where):
            query_list.append("WHERE " + where)
        if dataproperty.is_not_empty_string(extra):
            query_list.append(extra)

        return " ".join(query_list)
Ejemplo n.º 6
0
    def create_table_from_csv(
            self, csv_source, table_name="",
            attribute_name_list=(),
            delimiter=",", quotechar='"', encoding="utf-8"):
        """
        Create a table from a CSV file/text.

        :param str csv_source: Path to the CSV file or CSV text.
        :param str table_name:
            Table name to create.
            Use csv file basename as the table name if the value is empty.
        :param list attribute_name_list:
            Attribute names of the table.
            Use the first line of the csv file as attribute list
            if attribute_name_list is empty.
        :param str delimiter:
            A one-character string used to separate fields.
        :param str quotechar:
            A one-character string used to quote fields
            containing special characters, such as the delimiter or quotechar,
            or which contain new-line characters.
        :param str encoding: csv file encoding.
        :raises ValueError: If the csv data is invalid.

        .. seealso::

            :py:meth:`.create_table_with_data`
            :py:func:`csv.reader`
            :py:meth:`.loader.CsvTableFileLoader.load`
            :py:meth:`.loader.CsvTableTextLoader.load`
        """

        from .loader import CsvTableFileLoader
        from .loader import CsvTableTextLoader

        loader = CsvTableFileLoader(csv_source)
        if dataproperty.is_not_empty_string(table_name):
            loader.table_name = table_name
        loader.header_list = attribute_name_list
        loader.delimiter = delimiter
        loader.quotechar = quotechar
        loader.encoding = encoding
        try:
            for tabledata in loader.load():
                self.create_table_from_tabledata(tabledata)
            return
        except IOError:
            pass

        loader = CsvTableTextLoader(csv_source)
        if dataproperty.is_not_empty_string(table_name):
            loader.table_name = table_name
        loader.header_list = attribute_name_list
        loader.delimiter = delimiter
        loader.quotechar = quotechar
        loader.encoding = encoding
        for tabledata in loader.load():
            self.create_table_from_tabledata(tabledata)
Ejemplo n.º 7
0
    def write_table(self):
        """
        |write_table| with Pandas DataFrame variable definition format.
        """

        import pprint

        self._verify_property()
        self._preprocess()

        if dataproperty.is_not_empty_string(self.table_name):
            self._write_line(self.table_name + u" = pandas.DataFrame(")
        else:
            self._write_line(u"pandas.DataFrame(")

        self.inc_indent_level()
        data_frame_text = (
            u"\n".join([
                self._get_indent_string() + line
                for line in
                pprint.pformat(self._value_matrix, indent=1).splitlines()
            ]) +
            u")"
        )
        self.dec_indent_level()

        self.dec_indent_level()
        self._write_line(data_frame_text)
        self.inc_indent_level()
Ejemplo n.º 8
0
def create_database(database_path):
    db_path = path.Path(database_path)
    dir_path = db_path.dirname()
    if dataproperty.is_not_empty_string(dir_path):
        dir_path.makedirs_p()

    return simplesqlite.SimpleSQLite(db_path, "w")
Ejemplo n.º 9
0
    def write_table(self):
        """
        |write_table| with Pandas DataFrame variable definition format.
        """

        import pprint

        self._verify_property()
        self._preprocess()

        if dataproperty.is_not_empty_string(self.table_name):
            self._write_line(self.variable_name + u" = pandas.DataFrame(")
        else:
            self._write_line(u"pandas.DataFrame(")

        self.inc_indent_level()
        data_frame_text = (u"\n".join([
            self._get_indent_string() + line for line in pprint.pformat(
                self._value_matrix, indent=1).splitlines()
        ]) + u")")
        self.dec_indent_level()

        self.dec_indent_level()
        self._write_line(data_frame_text)
        self.inc_indent_level()
Ejemplo n.º 10
0
    def __get_filter(self, device):
        qdisc_parser = tcconfig.parser.TcQdiscParser()
        filter_parser = tcconfig.parser.TcFilterParser()

        # parse qdisc ---
        command = "tc qdisc show dev %s" % (device)
        proc = self.__subproc_wrapper.popen_command(command)
        qdisc_stdout, _stderr = proc.communicate()
        qdisc_param = qdisc_parser.parse(qdisc_stdout)

        # parse filter ---
        command = "tc filter show dev %s" % (device)
        proc = self.__subproc_wrapper.popen_command(command)
        filter_stdout, _stderr = proc.communicate()

        filter_table = {}

        for filter_param in filter_parser.parse_filter(filter_stdout):
            key_item_list = []

            if dataproperty.is_not_empty_string(filter_param.get("network")):
                key_item_list.append("network=" + filter_param.get("network"))

            if dataproperty.is_integer(filter_param.get("port")):
                key_item_list.append("port=%d" % (filter_param.get("port")))

            filter_key = ", ".join(key_item_list)
            filter_table[filter_key] = {}
            if filter_param.get("flowid") == qdisc_param.get("parent"):
                work_qdisc_param = dict(qdisc_param)
                del work_qdisc_param["parent"]
                filter_table[filter_key] = work_qdisc_param

        return filter_table
Ejemplo n.º 11
0
def run_command_helper(command, error_regexp, message, exception=None):
    if logger.level != logbook.DEBUG:
        spr.set_logger(is_enable=False)

    proc = spr.SubprocessRunner(command)
    proc.run()

    if logger.level != logbook.DEBUG:
        spr.set_logger(is_enable=True)

    if proc.returncode == 0:
        return 0

    match = error_regexp.search(proc.stderr)
    if match is None:
        logger.error(proc.stderr)
        return proc.returncode

    if dataproperty.is_not_empty_string(message):
        logger.notice(message)

    if exception is not None:
        raise exception(command)

    return proc.returncode
Ejemplo n.º 12
0
    def make_worksheet(self, sheet_name):
        """
        Make a worksheet to the current workbook.

        :param str sheet_name:
            Name of the worksheet to create. Name of the work sheet will 
            automatically decided (like ``"Sheet1"``)
            if the ``sheet_name`` is empty.
        """

        if dataproperty.is_not_empty_string(sheet_name):
            if sheet_name in self.__sheet_table:
                # the sheet is already exists
                self.stream = self.__sheet_table.get(sheet_name)
                return

            work_sheet_name = sheet_name
        else:
            work_sheet_name = None

        worksheet = self.workbook.workbook.add_worksheet(
            work_sheet_name)

        self.__sheet_table[worksheet.name] = worksheet
        self.stream = worksheet
Ejemplo n.º 13
0
    def to_attr_str(cls, name, operation_query=""):
        """
        :param str name: Attribute name.
        :param str operation_query:
            Used as a SQLite function if the value is not empty.
        :return: String that suitable for attribute name of a SQLite query.
        :rtype: str

        :Examples:

            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.to_attr_str("key")
            'key'
            >>> SqlQuery.to_attr_str("a+b")
            '[a+b]'
            >>> SqlQuery.to_attr_str("key", operation_query="SUM")
            'SUM(key)'
        """

        name = cls.sanitize_attr(name)

        if cls.__RE_TO_ATTR_QUOTE.search(name):
            sql_name = '"%s"' % (name)
        elif cls.__RE_TO_ATTR_BRACKET.search(name):
            sql_name = "[%s]" % (name)
        elif name == "join":
            sql_name = "[%s]" % (name)
        else:
            sql_name = name

        if dataproperty.is_not_empty_string(operation_query):
            sql_name = "%s(%s)" % (operation_query, sql_name)

        return sql_name
Ejemplo n.º 14
0
def create_database(database_path):
    db_path = path.Path(database_path)
    dir_path = db_path.dirname()
    if dataproperty.is_not_empty_string(dir_path):
        dir_path.makedirs_p()

    return simplesqlite.SimpleSQLite(db_path, "w")
Ejemplo n.º 15
0
    def make_update(cls, table, set_query, where=None):
        """
        Make UPDATE query.

        :param str table: Table name of execute query.
        :param str set_query: SET part of UPDATE query.
        :return: Query of SQLite.
        :rtype: str

        :raises ValueError: If ``set_query`` is empty string.

        .. seealso::

            :py:func:`validate_table_name() <simplesqlite.validate_table_name>`
        """

        validate_table_name(table)
        if dataproperty.is_empty_string(set_query):
            raise ValueError("SET query is null")

        query_list = [
            "UPDATE " + cls.to_table_str(table),
            "SET " + set_query,
        ]
        if dataproperty.is_not_empty_string(where):
            query_list.append("WHERE " + where)

        return " ".join(query_list)
Ejemplo n.º 16
0
    def make_update(cls, table, set_query, where=None):
        """
        Make UPDATE query.

        :param str table: Table name of execute query.
        :param str set_query: SET part of UPDATE query.
        :return: Query of SQLite.
        :rtype: str

        :raises ValueError: If ``set_query`` is empty string.

        .. seealso::

            :py:func:`validate_table_name() <simplesqlite.validate_table_name>`
        """

        validate_table_name(table)
        if dataproperty.is_empty_string(set_query):
            raise ValueError("SET query is null")

        query_list = [
            "UPDATE " + cls.to_table_str(table),
            "SET " + set_query,
        ]
        if dataproperty.is_not_empty_string(where):
            query_list.append("WHERE " + where)

        return " ".join(query_list)
Ejemplo n.º 17
0
    def make_update(cls, table, set_query, where=None):
        """
        Make UPDATE query.

        :param str table: Table name of executing the query.
        :param str set_query: SET part of the UPDATE query.
        :param str where:
            Add a WHERE clause to execute query,
            if the value is not |None|.
        :return: Query of SQLite.
        :rtype: str
        :raises ValueError: If ``set_query`` is empty string.
        :raises ValueError: |raises_validate_table_name|
        """

        validate_table_name(table)
        if dataproperty.is_empty_string(set_query):
            raise ValueError("SET query is null")

        query_list = [
            "UPDATE " + cls.to_table_str(table),
            "SET " + set_query,
        ]
        if dataproperty.is_not_empty_string(where):
            query_list.append("WHERE " + where)

        return " ".join(query_list)
Ejemplo n.º 18
0
    def __set_network_filter(self, qdisc_major_id):
        command_list = [
            "tc filter add",
            "dev " + self.__get_tc_device(),
            "protocol ip",
            "parent {:x}:".format(qdisc_major_id),
            "prio 1",
        ]

        if self.__is_use_iptables():
            mark_id = (
                IptablesMangleController.get_unique_mark_id() +
                self.__FILTER_IPTABLES_MARK_ID_OFFSET)
            command_list.append("handle {:d} fw".format(mark_id))

            self.__add_mangle_mark(mark_id)
        else:
            if all([
                dataproperty.is_empty_string(self.network),
                self.port is None,
            ]):
                return 0

            command_list.append("u32")
            if dataproperty.is_not_empty_string(self.network):
                command_list.append("match ip {:s} {:s}".format(
                    self.__get_network_direction_str(), self.network))
            if self.port is not None:
                command_list.append(
                    "match ip dport {:d} 0xffff".format(self.port))

        command_list.append("flowid {:x}:{:d}".format(
            qdisc_major_id, self.__get_qdisc_minor_id()))

        return SubprocessRunner(" ".join(command_list)).run()
Ejemplo n.º 19
0
    def removeMatchFileRecursively(cls, search_dir_path, re_remove_list):
        logger.debug("remove matched file: search-root=%s, re=%s" % (
            search_dir_path, str(re_remove_list)))

        re_compile_list = [
            re.compile(re_pattern)
            for re_pattern in re_remove_list
            if dataproperty.is_not_empty_string(re_pattern)
        ]

        dict_result_pathlist = {}

        for dir_path, _dir_name_list, filename_list in os.walk(search_dir_path):
            for filename in filename_list:
                for re_pattern in re_compile_list:
                    if re_pattern.search(filename):
                        break
                else:
                    continue

                remove_path = os.path.join(dir_path, filename)
                result = cls.remove_object(remove_path)
                dict_result_pathlist.setdefault(result, []).append(remove_path)

        return dict_result_pathlist
Ejemplo n.º 20
0
    def __set_network_filter(self, qdisc_major_id):
        command_list = [
            "tc filter add",
            "dev " + self.__get_tc_device(),
            "protocol ip",
            "parent {:x}:".format(qdisc_major_id),
            "prio 1",
        ]

        if self.__is_use_iptables():
            mark_id = (IptablesMangleController.get_unique_mark_id() +
                       self.__FILTER_IPTABLES_MARK_ID_OFFSET)
            command_list.append("handle {:d} fw".format(mark_id))

            self.__add_mangle_mark(mark_id)
        else:
            if all([
                    dataproperty.is_empty_string(self.network),
                    self.port is None,
            ]):
                return 0

            command_list.append("u32")
            if dataproperty.is_not_empty_string(self.network):
                command_list.append("match ip {:s} {:s}".format(
                    self.__get_network_direction_str(), self.network))
            if self.port is not None:
                command_list.append("match ip dport {:d} 0xffff".format(
                    self.port))

        command_list.append("flowid {:x}:{:d}".format(
            qdisc_major_id, self.__get_qdisc_minor_id()))

        return SubprocessRunner(" ".join(command_list)).run()
Ejemplo n.º 21
0
def main():
	
    options = parse_option()

    if dataproperty.is_not_empty_string(options.config_file):
        return_code = 0

        loader = TcConfigLoader()
        loader.is_overwrite = options.overwrite
        loader.load_pynetcm(options.config_file)

        for pynetcm_command in loader.get_pynetcm_command_list():
            return_code |= subprocrunner.SubprocessRunner(
                pynetcm_command).run()

        return return_code

    tc = TrafficControl(options.device)
    tc.direction = options.direction
    tc.bandwidth_rate = options.bandwidth_rate
    tc.latency_ms = options.network_latency
    tc.latency_distro_ms = options.latency_distro_ms
    tc.packet_loss_rate = options.packet_loss_rate
    tc.corruption_rate = options.corruption_rate
    tc.network = options.network
    tc.port = options.port

    tc.validate()

    if options.overwrite:
        tc.delete_tc()

    tc.set_tc()

    return 0
Ejemplo n.º 22
0
    def write_table(self):
        """
        |write_table| with Markdown table format.
        """

        if dataproperty.is_not_empty_string(self.table_name):
            self.__write_chapter(self.table_name)

        super(MarkdownTableWriter, self).write_table()
Ejemplo n.º 23
0
    def write_table(self):
        """
        |write_table| with Markdown table format.
        """

        if dataproperty.is_not_empty_string(self.table_name):
            self.__write_chapter(self.table_name)

        super(MarkdownTableWriter, self).write_table()
Ejemplo n.º 24
0
    def __parse_netem_param(self, line, parse_param_name, word_pattern):
        pattern = (pp.SkipTo(parse_param_name, include=True) +
                   pp.Word(word_pattern))

        try:
            result = pattern.parseString(_to_unicode(line))[-1]
            if dataproperty.is_not_empty_string(result):
                self.__parsed_param[parse_param_name] = result
        except pp.ParseException:
            pass
Ejemplo n.º 25
0
    def make_select(cls, select, table, where=None, extra=None):
        """
        Make SELECT query.

        :param str select: Attribute for SELECT query.
        :param str table: Table name of executing the query.
        :param str where:
            Add a WHERE clause to execute query,
            if the value is not |None|.
        :param extra extra:
            Add additional clause to execute query,
            if the value is not |None|.
        :return: Query of SQLite.
        :rtype: str
        :raises ValueError: ``select`` is empty string.
        :raises ValueError: |raises_validate_table_name|

        :Examples:

            >>> from simplesqlite.sqlquery import SqlQuery
            >>> SqlQuery.make_select(select="value", table="example")
            'SELECT value FROM example'
            >>> SqlQuery.make_select(select="value", table="example", where=SqlQuery.make_where("key", 1))
            'SELECT value FROM example WHERE key = 1'
            >>> SqlQuery.make_select(select="value", table="example", where=SqlQuery.make_where("key", 1), extra="ORDER BY value")
            'SELECT value FROM example WHERE key = 1 ORDER BY value'
        """

        validate_table_name(table)
        if dataproperty.is_empty_string(select):
            raise ValueError("SELECT query is null")

        query_list = [
            "SELECT " + select,
            "FROM " + cls.to_table_str(table),
        ]
        if dataproperty.is_not_empty_string(where):
            query_list.append("WHERE " + where)
        if dataproperty.is_not_empty_string(extra):
            query_list.append(extra)

        return " ".join(query_list)
Ejemplo n.º 26
0
    def __parse_netem_param(self, line, parse_param_name, word_pattern):
        pattern = (
            pp.SkipTo(parse_param_name, include=True) +
            pp.Word(word_pattern))

        try:
            result = pattern.parseString(_to_unicode(line))[-1]
            if dataproperty.is_not_empty_string(result):
                self.__parsed_param[parse_param_name] = result
        except pp.ParseException:
            pass
Ejemplo n.º 27
0
    def __parse_tbf_rate(self, line):
        parse_param_name = "rate"
        pattern = (pp.SkipTo(parse_param_name, include=True) +
                   pp.Word(pp.alphanums + "." + ":"))

        try:
            result = pattern.parseString(line)[-1]
            if dataproperty.is_not_empty_string(result):
                result = result.rstrip("bit")
                self.__parsed_param[parse_param_name] = result
        except pp.ParseException:
            pass
Ejemplo n.º 28
0
    def _get_start_row_idx(self):
        row_idx = 0
        for row_value_list in self.__all_values:
            if all([
                dataproperty.is_not_empty_string(value)
                for value in row_value_list
            ]):
                break

            row_idx += 1

        return self.start_row + row_idx
Ejemplo n.º 29
0
def create_database(ctx, database_path):
    is_append_table = ctx.obj.get(Context.IS_APPEND_TABLE)

    db_path = path.Path(database_path)
    dir_path = db_path.dirname()
    if dataproperty.is_not_empty_string(dir_path):
        dir_path.makedirs_p()

    if is_append_table:
        return simplesqlite.SimpleSQLite(db_path, "a")
    else:
        return simplesqlite.SimpleSQLite(db_path, "w")
Ejemplo n.º 30
0
    def __parse_tbf_rate(self, line):
        parse_param_name = "rate"
        pattern = (
            pp.SkipTo(parse_param_name, include=True) +
            pp.Word(pp.alphanums + "." + ":"))

        try:
            result = pattern.parseString(line)[-1]
            if dataproperty.is_not_empty_string(result):
                result = result.rstrip("bit")
                self.__parsed_param[parse_param_name] = result
        except pp.ParseException:
            pass
Ejemplo n.º 31
0
def command_to_filename(command, suffix=""):
    sep_char = "/\\"

    command = command.strip()
    filename = command.replace(" ", "_")
    filename = filename.replace("-", "")
    filename = filename.strip(sep_char).lstrip(sep_char)
    filename = re.sub("[%s]" % re.escape("/\\"), "-", filename)
    filename = pathvalidate.sanitize_filename(filename)
    if dataproperty.is_not_empty_string(suffix):
        filename += "_" + suffix

    return filename
Ejemplo n.º 32
0
def main():
    options = parse_option()
    logger = logbook.Logger("tcset")
    logger.level = options.log_level

    subprocrunner.logger.level = options.log_level
    if options.quiet:
        subprocrunner.logger.disable()
    else:
        subprocrunner.logger.enable()

    subprocrunner.Which("tc").verify()
    try:
        verify_netem_module()
    except ModuleNotFoundError as e:
        logger.debug(str(e))
    except subprocrunner.CommandNotFoundError as e:
        logger.error(str(e))

    if dataproperty.is_not_empty_string(options.config_file):
        return set_tc_from_file(logger, options.config_file, options.overwrite)

    tc = TrafficControl(
        options.device,
        direction=options.direction,
        bandwidth_rate=options.bandwidth_rate,
        latency_ms=options.network_latency,
        latency_distro_ms=options.latency_distro_ms,
        packet_loss_rate=options.packet_loss_rate,
        corruption_rate=options.corruption_rate,
        network=options.network,
        src_network=options.src_network,
        port=options.port,
        is_enable_iptables=options.is_enable_iptables
    )

    try:
        tc.validate()
    except (NetworkInterfaceNotFoundError, ValueError) as e:
        logger.error(str(e))
        return 1

    if options.overwrite:
        try:
            tc.delete_tc()
        except NetworkInterfaceNotFoundError:
            pass

    tc.set_tc()

    return 0
Ejemplo n.º 33
0
    def __strip_empty_col(self):
        col_idx = 0
        t_value_matrix = zip(*self.__all_values)

        for col_value_list in t_value_matrix:
            if any([
                dataproperty.is_not_empty_string(value)
                for value in col_value_list
            ]):
                break

            col_idx += 1

        self.__all_values = zip(*t_value_matrix[col_idx:])
Ejemplo n.º 34
0
def main():
    options = parse_option()

    set_log_level(options.log_level)

    subprocrunner.Which("tc").verify()
    try:
        verify_netem_module()
    except ModuleNotFoundError as e:
        logger.debug(str(e))
    except subprocrunner.CommandNotFoundError as e:
        logger.error(str(e))

    if dataproperty.is_not_empty_string(options.config_file):
        return set_tc_from_file(logger, options.config_file, options.overwrite)

    tc = TrafficControl(options.device,
                        direction=options.direction,
                        bandwidth_rate=options.bandwidth_rate,
                        latency_ms=options.network_latency,
                        latency_distro_ms=options.latency_distro_ms,
                        packet_loss_rate=options.packet_loss_rate,
                        corruption_rate=options.corruption_rate,
                        network=options.network,
                        src_network=options.src_network,
                        port=options.port,
                        is_add_shaper=options.is_add_shaper,
                        is_enable_iptables=options.is_enable_iptables,
                        shaping_algorithm=options.shaping_algorithm)

    try:
        tc.validate()
    except (NetworkInterfaceNotFoundError, ValueError) as e:
        logger.error(str(e))
        return 1

    if options.overwrite:
        if options.log_level == logbook.INFO:
            set_log_level(logbook.ERROR)

        try:
            tc.delete_tc()
        except NetworkInterfaceNotFoundError:
            pass

        set_log_level(options.log_level)

    tc.set_tc()

    return 0
Ejemplo n.º 35
0
    def write_table(self):
        """
        |write_table| with Python nested list variable definition format.
        """

        self._verify_property()

        if dataproperty.is_not_empty_string(self.table_name):
            self._write_line(self.table_name + u" = [")
        else:
            self._write_line(u"[")

        self.inc_indent_level()
        super(PythonCodeTableWriter, self).write_table()
        self.dec_indent_level()
        self._write_line(u"]")
Ejemplo n.º 36
0
    def write_table(self):
        """
        |write_table| with Python nested list variable definition format.
        """

        self._verify_property()

        if dataproperty.is_not_empty_string(self.table_name):
            self._write_line(self.table_name + u" = [")
        else:
            self._write_line(u"[")

        self.inc_indent_level()
        super(PythonCodeTableWriter, self).write_table()
        self.dec_indent_level()
        self._write_line(u"]")
Ejemplo n.º 37
0
    def write_table(self):
        """
        |write_table| with HTML table format.
        """

        self._verify_property()
        self._preprocess()

        if dataproperty.is_not_empty_string(self.table_name):
            self._table_tag = tags.table(
                id=pathvalidate.sanitize_python_var_name(self.table_name))
            self._table_tag += tags.caption(self.table_name)
        else:
            self._table_tag = tags.table()

        self._write_header()
        self._write_body()
Ejemplo n.º 38
0
    def write_table(self):
        """
        |write_table| with HTML table format.
        """

        self._verify_property()
        self._preprocess()

        if dataproperty.is_not_empty_string(self.table_name):
            self._table_tag = tags.table(
                id=pathvalidate.sanitize_python_var_name(self.table_name))
            self._table_tag += tags.caption(self.table_name)
        else:
            self._table_tag = tags.table()

        self._write_header()
        self._write_body()
Ejemplo n.º 39
0
    def to_attr_str(cls, name, operation_query=""):
        """
        :param str name: Base name of attribute.
        :param str operation_query: 
        :return: String that suitable for attribute name.
        :rtype: str
        """

        if cls.__RE_TO_ATTR_STR.search(name):
            sql_name = "[%s]" % (name)
        elif name == "join":
            sql_name = "[%s]" % (name)
        else:
            sql_name = name

        if dataproperty.is_not_empty_string(operation_query):
            sql_name = "%s(%s)" % (operation_query, sql_name)

        return sql_name
Ejemplo n.º 40
0
    def to_attr_str(cls, name, operation_query=""):
        """
        :param str name: Base name of attribute.
        :param str operation_query: 
        :return: String that suitable for attribute name.
        :rtype: str
        """

        if cls.__RE_TO_ATTR_STR.search(name):
            sql_name = "[%s]" % (name)
        elif name == "join":
            sql_name = "[%s]" % (name)
        else:
            sql_name = name

        if dataproperty.is_not_empty_string(operation_query):
            sql_name = "%s(%s)" % (operation_query, sql_name)

        return sql_name
Ejemplo n.º 41
0
    def __set_network_filter(self):
        if all([
            dataproperty.is_empty_string(self.network),
            self.port is None,
        ]):
            return 0

        command_list = [
            "tc filter add",
            "dev " + self.__device,
            "protocol ip parent 1: prio 1 u32",
            "flowid 1:1",
        ]
        if dataproperty.is_not_empty_string(self.network):
            command_list.append("match ip dst " + self.network)
        if self.port is not None:
            command_list.append("match ip dport %d 0xffff" % (self.port))

        return self.__subproc_wrapper.run(" ".join(command_list))
Ejemplo n.º 42
0
    def to_append_command(self):
        IntegerType(self.mark_id).validate()

        command_item_list = [
            "iptables -A {:s} -t mangle -j MARK".format(self.chain),
            "--set-mark {}".format(self.mark_id),
        ]

        if any([
            dataproperty.is_not_empty_string(self.protocol),
            IntegerType(self.protocol).is_type(),
        ]):
            command_item_list.append("-p {}".format(self.protocol))
        if self.__is_valid_srcdst(self.source):
            command_item_list.append(
                "-s {:s}".format(self.source))
        if self.__is_valid_srcdst(self.destination):
            command_item_list.append(
                "-d {:s}".format(self.destination))

        return " ".join(command_item_list)
Ejemplo n.º 43
0
    def to_append_command(self):
        IntegerType(self.mark_id).validate()

        command_item_list = [
            "iptables -A {:s} -t mangle -j MARK".format(self.chain),
            "--set-mark {}".format(self.mark_id),
        ]

        if any([
            dataproperty.is_not_empty_string(self.protocol),
            IntegerType(self.protocol).is_type(),
        ]):
            command_item_list.append("-p {}".format(self.protocol))
        if self.__is_valid_srcdst(self.source):
            command_item_list.append(
                "-s {:s}".format(self.source))
        if self.__is_valid_srcdst(self.destination):
            command_item_list.append(
                "-d {:s}".format(self.destination))

        return " ".join(command_item_list)
Ejemplo n.º 44
0
    def __set_network_filter(self, qdisc_major_id):
        if all([
            dataproperty.is_empty_string(self.network),
            self.port is None,
        ]):
            return 0

        command_list = [
            "tc filter add",
            "dev " + self.__get_tc_device(),
            "protocol ip",
            "parent %d:" % (qdisc_major_id),
            "prio 1 u32",
            "flowid %d:%d" % (qdisc_major_id, self.__get_qdisc_minor_id()),
        ]
        if dataproperty.is_not_empty_string(self.network):
            command_list.append("match ip %s %s" % (
                self.__get_network_direction_str(), self.network))
        if self.port is not None:
            command_list.append("match ip dport %d 0xffff" % (self.port))

        return self.__subproc_wrapper.run(" ".join(command_list))
Ejemplo n.º 45
0
    def __get_filter(self, device):
        if dataproperty.is_empty_string(device):
            return {}

        qdisc_parser = TcQdiscParser()
        filter_parser = TcFilterParser()

        # parse qdisc ---
        command = "tc qdisc show dev {:s}".format(device)
        qdisk_show_runner = SubprocessRunner(command)
        qdisk_show_runner.run()
        qdisc_param = qdisc_parser.parse(qdisk_show_runner.stdout)

        # parse filter ---
        command = "tc filter show dev {:s}".format(device)
        filter_show_runner = SubprocessRunner(command)
        filter_show_runner.run()

        filter_table = {}
        for filter_param in filter_parser.parse_filter(
                filter_show_runner.stdout):
            key_item_list = []

            if dataproperty.is_not_empty_string(filter_param.get("network")):
                key_item_list.append("network=" + filter_param.get("network"))

            if dataproperty.is_integer(filter_param.get("port")):
                key_item_list.append("port={:d}".format(
                    filter_param.get("port")))

            filter_key = ", ".join(key_item_list)
            filter_table[filter_key] = {}
            if filter_param.get("flowid") == qdisc_param.get("parent"):
                work_qdisc_param = dict(qdisc_param)
                del work_qdisc_param["parent"]
                filter_table[filter_key] = work_qdisc_param

        return filter_table
Ejemplo n.º 46
0
    def __set_network_filter(self, qdisc_major_id):
        if all([
                dataproperty.is_empty_string(self.network),
                self.port is None,
        ]):
            return 0

        command_list = [
            "tc filter add",
            "dev " + self.__get_tc_device(),
            "protocol ip",
            "parent %d:" % (qdisc_major_id),
            "prio 1 u32",
            "flowid %d:%d" % (qdisc_major_id, self.__get_qdisc_minor_id()),
        ]
        if dataproperty.is_not_empty_string(self.network):
            command_list.append(
                "match ip %s %s" %
                (self.__get_network_direction_str(), self.network))
        if self.port is not None:
            command_list.append("match ip dport %d 0xffff" % (self.port))

        return self.__subproc_wrapper.run(" ".join(command_list))
Ejemplo n.º 47
0
    def __set_network_filter(self, qdisc_major_id):
        if all([
                dataproperty.is_empty_string(self.network),
                self.port is None,
        ]):
            return 0

        command_list = [
            "tc filter add",
            "dev " + self.__get_tc_device(),
            "protocol ip",
            "parent {:x}:".format(qdisc_major_id),
            "prio 1 u32",
            "flowid {:x}:{:d}".format(qdisc_major_id,
                                      self.__get_qdisc_minor_id()),
        ]
        if dataproperty.is_not_empty_string(self.network):
            command_list.append("match ip {:s} {:s}".format(
                self.__get_network_direction_str(), self.network))
        if self.port is not None:
            command_list.append("match ip dport {:d} 0xffff".format(self.port))

        return SubprocessRunner(" ".join(command_list)).run()
Ejemplo n.º 48
0
    def make_worksheet(self, sheet_name):
        """
        Make a worksheet to the current workbook.

        :param str sheet_name:
            Name of the worksheet to create. Name of the work sheet will 
            automatically decided (like ``"Sheet1"``)
            if the ``sheet_name`` is empty.
        """

        if dataproperty.is_not_empty_string(sheet_name):
            if sheet_name in self.__sheet_table:
                # the sheet is already exists
                self.stream = self.__sheet_table.get(sheet_name)
                return

            work_sheet_name = sheet_name
        else:
            work_sheet_name = None

        worksheet = self.workbook.workbook.add_worksheet(work_sheet_name)

        self.__sheet_table[worksheet.name] = worksheet
        self.stream = worksheet
Ejemplo n.º 49
0
 def __is_valid_srcdst(srcdst):
     return (
         dataproperty.is_not_empty_string(srcdst) and
         srcdst != ANYWHERE_NETWORK
     )
Ejemplo n.º 50
0
 def test_normal(self):
     assert dataproperty.is_not_empty_string(get_execution_command())
Ejemplo n.º 51
0
 def __is_valid_srcdst(srcdst):
     return (
         dataproperty.is_not_empty_string(srcdst) and
         srcdst != ANYWHERE_NETWORK
     )
Ejemplo n.º 52
0
def url(ctx, url, format_name, output_path, encoding, proxy):
    """
    Fetch data from a URL and convert data to a SQLite database file.
    """

    if dataproperty.is_empty_sequence(url):
        sys.exit(ExitCode.NO_INPUT)

    con = create_database(ctx, output_path)
    verbosity_level = ctx.obj.get(Context.VERBOSITY_LEVEL)
    extractor = get_schema_extractor(con, verbosity_level)
    result_counter = ResultCounter()

    logger = logbook.Logger("sqlitebiter url")
    _setup_logger_from_context(logger, ctx.obj[Context.LOG_LEVEL])

    proxies = {}
    if dataproperty.is_not_empty_string(proxy):
        proxies = {
            "http": proxy,
            "https": proxy,
        }

    try:
        loader = ptr.TableUrlLoader(url,
                                    format_name,
                                    encoding=encoding,
                                    proxies=proxies)
    except ptr.LoaderNotFoundError as e:
        try:
            loader = ptr.TableUrlLoader(url,
                                        "html",
                                        encoding=encoding,
                                        proxies=proxies)
        except (ptr.LoaderNotFoundError, ptr.HTTPError):
            logger.error(e)
            sys.exit(ExitCode.FAILED_LOADER_NOT_FOUND)
    except ptr.HTTPError as e:
        logger.error(e)
        sys.exit(ExitCode.FAILED_HTTP)

    try:
        for tabledata in loader.load():
            sqlite_tabledata = ptr.SQLiteTableDataSanitizer(
                tabledata).sanitize()

            try:
                con.create_table_from_tabledata(sqlite_tabledata)
                result_counter.inc_success()
            except (ValueError) as e:
                logger.debug(u"url={}, message={}".format(url, str(e)))
                result_counter.inc_fail()
                continue

            log_message = get_success_log_format(verbosity_level).format(
                url,
                extractor.get_table_schema_text(
                    sqlite_tabledata.table_name).strip())
            logger.info(log_message)
    except ptr.InvalidDataError as e:
        logger.error(u"invalid data: url={}, message={}".format(url, str(e)))
        result_counter.inc_fail()

    write_completion_message(logger, output_path, result_counter)

    sys.exit(result_counter.get_return_code())