Beispiel #1
0
    def remote_profiles(self):
        """
        A list of remote profiles on the device.
        """
        remote_ini = self.app_ctx.remote_profiles_ini
        if not self.device.is_file(remote_ini):
            raise IOError("Remote file '%s' not found" % remote_ini)

        local_ini = tempfile.NamedTemporaryFile()
        self.device.pull(remote_ini, local_ini.name)
        cfg = ConfigParser()
        cfg.read(local_ini.name)

        profiles = []
        for section in cfg.sections():
            if cfg.has_option(section, "Path"):
                if cfg.has_option(section, "IsRelative") and cfg.getint(
                    section, "IsRelative"
                ):
                    profiles.append(
                        posixpath.join(
                            posixpath.dirname(remote_ini), cfg.get(section, "Path")
                        )
                    )
                else:
                    profiles.append(cfg.get(section, "Path"))
        return profiles
Beispiel #2
0
def load_config():
    """Load configuration.

    :returns:
        Current configuration based on configuration file and environment variables.
    :rtype: dict

    """
    config_parser = ConfigParser(
        {key: str(value) for key, value in DEFAULT_CONFIG.items()}
    )
    config_parser.add_section("greynoise")

    if os.path.isfile(CONFIG_FILE):
        LOGGER.debug("Parsing configuration file: %s...", CONFIG_FILE, path=CONFIG_FILE)
        with open(CONFIG_FILE) as config_file:
            config_parser.readfp(config_file)
    else:
        LOGGER.warning(
            "Configuration file not found: %s", CONFIG_FILE, path=CONFIG_FILE
        )

    if "GREYNOISE_API_KEY" in os.environ:
        api_key = os.environ["GREYNOISE_API_KEY"]
        LOGGER.debug(
            "API key found in environment variable: %s", api_key, api_key=api_key
        )
        # Environment variable takes precedence over configuration file content
        config_parser.set("greynoise", "api_key", api_key)

    if "GREYNOISE_TIMEOUT" in os.environ:
        timeout = os.environ["GREYNOISE_TIMEOUT"]
        try:
            int(timeout)
        except ValueError:
            LOGGER.error(
                "GREYNOISE_TIMEOUT environment variable "
                "cannot be converted to an integer: %r",
                timeout,
                timeout=timeout,
            )
        else:
            LOGGER.debug(
                "Timeout found in environment variable: %s", timeout, timeout=timeout
            )
            # Environment variable takes precedence over configuration file content
            config_parser.set("greynoise", "timeout", timeout)

    return {
        "api_key": config_parser.get("greynoise", "api_key"),
        "timeout": config_parser.getint("greynoise", "timeout"),
    }
Beispiel #3
0
    def __init__(self, config_file):
        config = ConfigParser(allow_no_value=True)
        config.read(config_file)

        self.serial = {
            "port": config.get('device', 'port'),
            "baudrate": config.getint('device', 'baudrate'),
            "parity": config.get('device', 'parity'),
            "bytesize": config.getint('device', 'bytesize'),
            "stopbits": config.getint('device', 'stopbits'),
            "timeout": config.getfloat('device', 'timeout'),
            "interCharTimeout": config.getfloat('device', 'char_delay'),
        }

        self.serial_char_delay = self.serial["interCharTimeout"]

        self.storage_dir = config.get("storage", "dir")

        self.sdb_enabled = config.has_section("simpledb")

        if self.sdb_enabled:
            self.sdb_domain = config.get("simpledb", "domain")

        if config.has_section("aws"):
            self.aws_access_key_id = config.get("aws", "access_key_id")
            self.aws_secret_access_key = config.get("aws", "secret_access_key")
            self.aws_region = config.get("aws", "region")

        self.station_id = config.get("site", "station_id")

        self.mail_enabled = config.has_section("mail")
        if self.mail_enabled:
            self.mail_host = config.get("mail", "host") or "localhost"
            self.mail_port = config.getint(
                "mail", "port") if config.has_option("mail", "port") else 587
            self.mail_username = config.get("mail", "username")
            self.mail_password = config.get("mail", "password")
            self.mail_to = config.get("mail", "to")
Beispiel #4
0
  def load(self):
    schemes = [defaultScheme]
    parser = ConfigParser()
    parser.read(settings.DASHBOARD_CONF)

    for option, default_value in defaultUIConfig.items():
      if parser.has_option('ui', option):
        try:
          self.ui_config[option] = parser.getint('ui', option)
        except ValueError:
          self.ui_config[option] = parser.get('ui', option)
      else:
        self.ui_config[option] = default_value

    if parser.has_option('ui', 'automatic_variants'):
      self.ui_config['automatic_variants']   = parser.getboolean('ui', 'automatic_variants')
    else:
      self.ui_config['automatic_variants'] = True

    self.ui_config['keyboard_shortcuts'] = defaultKeyboardShortcuts.copy()
    if parser.has_section('keyboard-shortcuts'):
      self.ui_config['keyboard_shortcuts'].update( parser.items('keyboard-shortcuts') )

    for section in parser.sections():
      if section in ('ui', 'keyboard-shortcuts'):
        continue

      scheme = parser.get(section, 'scheme')
      fields = []

      for match in fieldRegex.finditer(scheme):
        field = match.group(1)
        if parser.has_option(section, '%s.label' % field):
          label = parser.get(section, '%s.label' % field)
        else:
          label = field

        fields.append({
          'name' : field,
          'label' : label
        })

      schemes.append({
        'name' : section,
        'pattern' : scheme,
        'fields' : fields,
      })

    self.schemes = schemes
Beispiel #5
0
  def load(self):
    schemes = [defaultScheme]
    parser = ConfigParser()
    parser.read(settings.DASHBOARD_CONF)

    for option, default_value in defaultUIConfig.items():
      if parser.has_option('ui', option):
        try:
          self.ui_config[option] = parser.getint('ui', option)
        except ValueError:
          self.ui_config[option] = parser.get('ui', option)
      else:
        self.ui_config[option] = default_value

    if parser.has_option('ui', 'automatic_variants'):
      self.ui_config['automatic_variants']   = parser.getboolean('ui', 'automatic_variants')
    else:
      self.ui_config['automatic_variants'] = True

    self.ui_config['keyboard_shortcuts'] = defaultKeyboardShortcuts.copy()
    if parser.has_section('keyboard-shortcuts'):
      self.ui_config['keyboard_shortcuts'].update( parser.items('keyboard-shortcuts') )

    for section in parser.sections():
      if section in ('ui', 'keyboard-shortcuts'):
        continue

      scheme = parser.get(section, 'scheme')
      fields = []

      for match in fieldRegex.finditer(scheme):
        field = match.group(1)
        if parser.has_option(section, '%s.label' % field):
          label = parser.get(section, '%s.label' % field)
        else:
          label = field

        fields.append({
          'name' : field,
          'label' : label
        })

      schemes.append({
        'name' : section,
        'pattern' : scheme,
        'fields' : fields,
      })

    self.schemes = schemes
Beispiel #6
0
def load_config():
    """Load configuration.

    :returns:
        Current configuration based on configuration file and environment variables.
    :rtype: dict

    """
    config_parser = ConfigParser(
        {key: str(value) for key, value in DEFAULT_CONFIG.items()}
    )
    config_parser.add_section("greynoise")

    if os.path.isfile(CONFIG_FILE):
        LOGGER.debug("Parsing configuration file: %s...", CONFIG_FILE, path=CONFIG_FILE)
        with open(CONFIG_FILE) as config_file:
            # cannot update this to read_file() until py27 support is removed
            config_parser.readfp(config_file)
    else:
        LOGGER.debug("Configuration file not found: %s", CONFIG_FILE, path=CONFIG_FILE)

    if "GREYNOISE_API_KEY" in os.environ:
        api_key = os.environ["GREYNOISE_API_KEY"]
        LOGGER.debug(
            "API key found in environment variable: %s", api_key, api_key=api_key
        )
        # Environment variable takes precedence over configuration file content
        config_parser.set("greynoise", "api_key", api_key)

    if "GREYNOISE_API_SERVER" in os.environ:
        api_server = os.environ["GREYNOISE_API_SERVER"]
        LOGGER.debug(
            "API server found in environment variable: %s",
            api_server,
            api_server=api_server,
        )
        # Environment variable takes precedence over configuration file content
        config_parser.set("greynoise", "api_server", api_server)

    if "GREYNOISE_TIMEOUT" in os.environ:
        timeout = os.environ["GREYNOISE_TIMEOUT"]
        try:
            int(timeout)
        except ValueError:
            LOGGER.error(
                "GREYNOISE_TIMEOUT environment variable "
                "cannot be converted to an integer: %r",
                timeout,
                timeout=timeout,
            )
        else:
            LOGGER.debug(
                "Timeout found in environment variable: %s", timeout, timeout=timeout
            )
            # Environment variable takes precedence over configuration file content
            config_parser.set("greynoise", "timeout", timeout)

    if "GREYNOISE_PROXY" in os.environ:
        proxy = os.environ["GREYNOISE_PROXY"]
        LOGGER.debug(
            "Proxy found in environment variable: %s",
            proxy,
            proxy=proxy,
        )
        # Environment variable takes precedence over configuration file content
        config_parser.set("greynoise", "proxy", proxy)

    if "GREYNOISE_OFFERING" in os.environ:
        offering = os.environ["GREYNOISE_OFFERING"]
        LOGGER.debug(
            "Offering found in environment variable: %s",
            offering,
            offering=offering,
        )
        # Environment variable takes precedence over configuration file content
        config_parser.set("greynoise", "offering", offering)

    return {
        "api_key": config_parser.get("greynoise", "api_key"),
        "api_server": config_parser.get("greynoise", "api_server"),
        "timeout": config_parser.getint("greynoise", "timeout"),
        "proxy": config_parser.get("greynoise", "proxy"),
        "offering": config_parser.get("greynoise", "offering"),
    }
Beispiel #7
0
class PypiConfig(object):
    """Wrapper around the pypi config file"""
    def __init__(self, config_filename=DIST_CONFIG_FILE, use_setup_cfg=True):
        """Grab the PyPI configuration.

        This is .pypirc in the home directory.  It is overridable for
        test purposes.

        If there is a setup.cfg file in the current directory, we read
        it too.
        """
        self.config_filename = config_filename
        self.use_setup_cfg = use_setup_cfg
        self.reload()

    def reload(self):
        """Load the config.

        Do the initial load of the config.

        Or reload it in case of problems: this is needed when a pypi
        upload fails, you edit the .pypirc file to fix the account
        settings, and tell release to retry the command.
        """
        self._read_configfile(use_setup_cfg=self.use_setup_cfg)

    def _read_configfile(self, use_setup_cfg=True):
        """Read the PyPI config file and store it (when valid).

        Usually read the setup.cfg too.
        """
        rc = self.config_filename
        if not os.path.isabs(rc):
            rc = os.path.join(os.path.expanduser('~'), self.config_filename)
        filenames = [rc]
        if use_setup_cfg:
            # If there is a setup.cfg in the package, parse it
            filenames.append('setup.cfg')
        files = [f for f in filenames if os.path.exists(f)]
        if not files:
            self.config = None
            return
        self.config = ConfigParser()
        for filename in files:
            with codecs.open(filename, 'r', 'utf8') as fp:
                self.config.readfp(fp)

    def is_pypi_configured(self):
        # Do we have configuration for releasing to at least one
        # pypi-compatible server?
        if self.config is None:
            return False
        return len(self.distutils_servers()) > 0

    def get_server_config(self, server):
        """Get url, username, password for server.
        """
        repository_url = DEFAULT_REPOSITORY
        username = None
        password = None
        if self.config.has_section(server):
            if self.config.has_option(server, 'repository'):
                repository_url = self.config.get(server, 'repository')
            if self.config.has_option(server, 'username'):
                username = self.config.get(server, 'username')
            if self.config.has_option(server, 'password'):
                password = self.config.get(server, 'password')
        if not username and self.config.has_option('server-login', 'username'):
            username = self.config.get('server-login', 'username')
        if not password and self.config.has_option('server-login', 'password'):
            password = self.config.get('server-login', 'password')
        return {
            'repository_url': repository_url,
            'username': username,
            'password': password,
        }

    def distutils_servers(self):
        """Return a list of known distutils servers.

        If the config has an old pypi config, remove the default pypi
        server from the list.
        """
        try:
            index_servers = self.config.get('distutils',
                                            'index-servers').split()
        except (NoSectionError, NoOptionError):
            index_servers = []
        if not index_servers:
            # If no distutils index-servers have been given, 'pypi' should be
            # the default.  This is what twine does.
            if self.config.has_option('server-login', 'username'):
                # We have a username, so upload to pypi should work fine, even
                # when no explicit pypi section is in the file.
                return ['pypi']
            # https://github.com/zestsoftware/zest.releaser/issues/199
            index_servers = ['pypi']
        # The servers all need to have a section in the config file.
        return [
            server for server in index_servers
            if self.config.has_section(server)
        ]

    def want_release(self):
        """Does the user normally want to release this package.

        Some colleagues find it irritating to have to remember to
        answer the question "Check out the tag (for tweaks or
        pypi/distutils server upload)" with the non-default 'no' when
        in 99 percent of the cases they just make a release specific
        for a customer, so they always answer 'no' here.  This is
        where an extra config option comes in handy: you can influence
        the default answer so you can just keep hitting 'Enter' until
        zest.releaser is done.

        Either in your ~/.pypirc or in a setup.cfg in a specific
        package, add this when you want the default answer to this
        question to be 'no':

        [zest.releaser]
        release = no

        The default when this option has not been set is True.

        Standard config rules apply, so you can use upper or lower or
        mixed case and specify 0, false, no or off for boolean False,
        and 1, on, true or yes for boolean True.
        """
        return self._get_boolean('zest.releaser', 'release', default=True)

    def extra_message(self):
        """Return extra text to be added to commit messages.

        This can for example be used to skip CI builds.  This at least
        works for Travis.  See
        http://docs.travis-ci.com/user/how-to-skip-a-build/

        Enable this mode by adding a ``extra-message`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            extra-message = [ci skip]
        """
        default = ''
        if self.config is None:
            return default
        try:
            result = self.config.get('zest.releaser', 'extra-message')
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def create_wheel(self):
        """Should we create a Python wheel for this package?

        Either in your ~/.pypirc or in a setup.cfg in a specific
        package, add this when you want to create a Python wheel, next
        to a standard sdist:

        [zest.releaser]
        create-wheel = yes

        """
        if not USE_WHEEL:
            # If the wheel package is not available, we obviously
            # cannot create wheels.
            return False
        return self._get_boolean('zest.releaser', 'create-wheel')

    def register_package(self):
        """Should we try to register this package with a package server?

        For the standard Python Package Index (PyPI), registering a
        package is no longer needed: this is done automatically when
        uploading a distribution for a package.  In fact, trying to
        register may fail.  See
        https://github.com/zestsoftware/zest.releaser/issues/191
        So by default zest.releaser will no longer register a package.

        But you may be using your own package server, and registering
        may be wanted or even required there.  In this case
        you will need to turn on the register function.
        In your setup.cfg or ~/.pypirc, use the following to ensure that
        register is called on the package server:

        [zest.releaser]
        register = yes

        Note that if you have specified multiple package servers, this
        option is used for all of them.  There is no way to register and
        upload to server A, and only upload to server B.
        """
        return self._get_boolean('zest.releaser', 'register')

    def no_input(self):
        """Return whether the user wants to run in no-input mode.

        Enable this mode by adding a ``no-input`` option::

            [zest.releaser]
            no-input = yes

        The default when this option has not been set is False.
        """
        return self._get_boolean('zest.releaser', 'no-input')

    def development_marker(self):
        """Return development marker to be appended in postrelease.

        Override the default ``.dev0`` in setup.cfg using
        a ``development-marker`` option::

            [zest.releaser]
            development-marker = .dev1

        Returns default of ``.dev0`` when nothing has been configured.
        """
        default = '.dev0'
        if self.config is None:
            return default
        try:
            result = self.config.get('zest.releaser', 'development-marker')
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def push_changes(self):
        """Return whether the user wants to push the changes to the remote.

        Configure this mode by adding a ``push-changes`` option::

            [zest.releaser]
            push-changes = no

        The default when this option has not been set is True.
        """
        return self._get_boolean('zest.releaser', 'push-changes', default=True)

    def less_zeroes(self):
        """Return whether the user prefers less zeroes at the end of a version.

        Configure this mode by adding a ``less-zeroes`` option::

            [zest.releaser]
            less-zeroes = yes

        The default when this option has not been set is False.

        When set to true:
        - Instead of 1.3.0 we will suggest 1.3.
        - Instead of 2.0.0 we will suggest 2.0.

        This only makes sense for the bumpversion command.
        In the postrelease command we read this option too,
        but with the current logic it has no effect there.
        """
        return self._get_boolean('zest.releaser', 'less-zeroes')

    def version_levels(self):
        """How many levels does the user prefer in a version number?

        Configure this mode by adding a ``version-levels`` option::

            [zest.releaser]
            version-levels = 3

        The default when this option has not been set is 0, which means:
        no preference, so use the length of the current number.

        This means when suggesting a next version after 1.2:
        - with levels=0 we will suggest 1.3: no change
        - with levels=1 we will still suggest 1.3, as we will not
          use this to remove numbers, only to add them
        - with levels=2 we will suggest 1.3
        - with levels=3 we will suggest 1.2.1

        If the current version number has more levels, we keep them.
        So next version for 1.2.3.4 with levels=1 is 1.2.3.5.

        Tweaking version-levels and less-zeroes should give you the
        version number strategy that you prefer.
        """
        default = 0
        if self.config is None:
            return default
        try:
            result = self.config.getint('zest.releaser', 'version-levels')
        except (NoSectionError, NoOptionError, ValueError):
            return default
        if result < 0:
            return default
        return result

    def _get_boolean(self, section, key, default=False):
        """Get a boolean from the config.

        Standard config rules apply, so you can use upper or lower or
        mixed case and specify 0, false, no or off for boolean False,
        and 1, on, true or yes for boolean True.
        """
        result = default
        if self.config is not None:
            try:
                result = self.config.getboolean(section, key)
            except (NoSectionError, NoOptionError, ValueError):
                return result
        return result
Beispiel #8
0
class MultiPortConfig(object):

    HW_LB = "HW"

    @staticmethod
    def float_x_plus_one_tenth_of_y(x, y):
        return float(x) + float(y) / 10.0

    @staticmethod
    def make_str(base, iterator):
        return ' '.join((base.format(x) for x in iterator))

    @classmethod
    def make_range_str(cls, base, start, stop=0, offset=0):
        if offset and not stop:
            stop = start + offset
        return cls.make_str(base, range(start, stop))

    @staticmethod
    def parser_get(parser, section, key, default=None):
        if parser.has_option(section, key):
            return parser.get(section, key)
        return default

    @staticmethod
    def make_ip_addr(ip, mask):
        """
        :param ip: ip adddress
        :type ip: str
        :param mask: /24 prefix of 255.255.255.0 netmask
        :type mask: str
        :return: interface
        :rtype: IPv4Interface
        """

        try:
            return ipaddress.ip_interface(six.text_type('/'.join([ip, mask])))
        except (TypeError, ValueError):
            # None so we can skip later
            return None

    @classmethod
    def validate_ip_and_prefixlen(cls, ip_addr, prefixlen):
        ip_addr = cls.make_ip_addr(ip_addr, prefixlen)
        return ip_addr.ip.exploded, ip_addr.network.prefixlen

    def __init__(self,
                 topology_file,
                 config_tpl,
                 tmp_file,
                 vnfd_helper,
                 vnf_type='CGNAT',
                 lb_count=2,
                 worker_threads=3,
                 worker_config='1C/1T',
                 lb_config='SW',
                 socket=0):

        super(MultiPortConfig, self).__init__()
        self.topology_file = topology_file
        self.worker_config = worker_config.split('/')[1].lower()
        self.worker_threads = self.get_worker_threads(worker_threads)
        self.vnf_type = vnf_type
        self.pipe_line = 0
        self.vnfd_helper = vnfd_helper
        self.write_parser = ConfigParser()
        self.read_parser = ConfigParser()
        self.read_parser.read(config_tpl)
        self.master_core = self.read_parser.get("PIPELINE0", "core")
        self.master_tpl = self.get_config_tpl_data('MASTER')
        self.arpicmp_tpl = self.get_config_tpl_data('ARPICMP')
        self.txrx_tpl = self.get_config_tpl_data('TXRX')
        self.loadb_tpl = self.get_config_tpl_data('LOADB')
        self.vnf_tpl = self.get_config_tpl_data(vnf_type)
        self.swq = 0
        self.lb_count = int(lb_count)
        self.lb_config = lb_config
        self.tmp_file = os.path.join("/tmp", tmp_file)
        self.pktq_out_os = []
        self.socket = socket
        self.start_core = ""
        self.pipeline_counter = ""
        self.txrx_pipeline = ""
        self._port_pairs = None
        self.all_ports = []
        self.port_pair_list = []
        self.lb_to_port_pair_mapping = {}
        self.init_eal()

        self.lb_index = None
        self.mul = 0
        self.port_pairs = []
        self.ports_len = 0
        self.prv_que_handler = None
        self.vnfd = None
        self.rules = None
        self.pktq_out = []

    @staticmethod
    def gen_core(core):
        # return "s{}c{}".format(self.socket, core)
        # don't use sockets for VNFs, because we don't want to have to
        # adjust VM CPU topology.  It is virtual anyway
        return str(core)

    def make_port_pairs_iter(self, operand, iterable):
        return (operand(self.vnfd_helper.port_num(x), y) for y in iterable
                for x in chain.from_iterable(self.port_pairs))

    def make_range_port_pairs_iter(self, operand, start, end):
        return self.make_port_pairs_iter(operand, range(start, end))

    def init_eal(self):
        lines = ['[EAL]\n']
        vpci = (v['virtual-interface']["vpci"]
                for v in self.vnfd_helper.interfaces)
        lines.extend('w = {0}\n'.format(item) for item in vpci)
        lines.append('\n')
        with open(self.tmp_file, 'w') as fh:
            fh.writelines(lines)

    def update_timer(self):
        timer_tpl = self.get_config_tpl_data('TIMER')
        timer_tpl['core'] = self.gen_core(self.start_core)
        self.update_write_parser(timer_tpl)
        self.start_core += 1

    def get_config_tpl_data(self, type_value):
        for section in self.read_parser.sections():
            if self.read_parser.has_option(section, 'type'):
                if type_value == self.read_parser.get(section, 'type'):
                    tpl = OrderedDict(self.read_parser.items(section))
                    return tpl

    def get_txrx_tpl_data(self, value):
        for section in self.read_parser.sections():
            if self.read_parser.has_option(section, 'pipeline_txrx_type'):
                if value == self.read_parser.get(section,
                                                 'pipeline_txrx_type'):
                    tpl = OrderedDict(self.read_parser.items(section))
                    return tpl

    def init_write_parser_template(self, type_value='ARPICMP'):
        for section in self.read_parser.sections():
            if type_value == self.parser_get(self.read_parser, section, 'type',
                                             object()):
                self.start_core = self.read_parser.getint(section, 'core')
                self.pipeline_counter = self.read_parser.getint(
                    section, 'core')
                self.txrx_pipeline = self.read_parser.getint(section, 'core')
                return
            self.write_parser.add_section(section)
            for name, value in self.read_parser.items(section):
                self.write_parser.set(section, name, value)

    def update_write_parser(self, data):
        section = "PIPELINE{0}".format(self.pipeline_counter)
        self.write_parser.add_section(section)
        for name, value in data.items():
            self.write_parser.set(section, name, value)

    def get_worker_threads(self, worker_threads):
        if self.worker_config == '1t':
            return worker_threads
        else:
            return worker_threads - worker_threads % 2

    def generate_next_core_id(self):
        if self.worker_config == '1t':
            self.start_core += 1
            return

        try:
            self.start_core = '{}h'.format(int(self.start_core))
        except ValueError:
            self.start_core = int(self.start_core[:-1]) + 1

    def get_lb_count(self):
        self.lb_count = int(min(len(self.port_pair_list), self.lb_count))

    def generate_lb_to_port_pair_mapping(self):
        self.lb_to_port_pair_mapping = defaultdict(int)
        port_pair_count = len(self.port_pair_list)
        lb_pair_count = int(port_pair_count / self.lb_count)
        extra = port_pair_count % self.lb_count
        extra_iter = repeat(lb_pair_count + 1, extra)
        norm_iter = repeat(lb_pair_count, port_pair_count - extra)
        new_values = {
            i: v
            for i, v in enumerate(chain(extra_iter, norm_iter), 1)
        }
        self.lb_to_port_pair_mapping.update(new_values)

    def set_priv_to_pub_mapping(self):
        port_nums = [
            tuple(self.vnfd_helper.port_nums(x)) for x in self.port_pair_list
        ]
        return "".join(str(y).replace(" ", "") for y in port_nums)

    def set_priv_que_handler(self):
        # iterated twice, can't be generator
        priv_to_pub_map = [
            tuple(self.vnfd_helper.port_nums(x)) for x in self.port_pairs
        ]
        # must be list to use .index()
        port_list = list(chain.from_iterable(priv_to_pub_map))
        uplink_ports = (x[0] for x in priv_to_pub_map)
        self.prv_que_handler = '({})'.format("".join(
            ("{},".format(port_list.index(x)) for x in uplink_ports)))

    def generate_arp_route_tbl(self):
        arp_route_tbl_tmpl = "({port0_dst_ip_hex},{port0_netmask_hex},{port_num}," \
                             "{next_hop_ip_hex})"

        def build_arp_config(port):
            dpdk_port_num = self.vnfd_helper.port_num(port)
            interface = self.vnfd_helper.find_interface(
                name=port)["virtual-interface"]
            # We must use the dst because we are on the VNF and we need to
            # reach the TG.
            dst_port0_ip = ipaddress.ip_interface(
                six.text_type("%s/%s" %
                              (interface["dst_ip"], interface["netmask"])))

            arp_vars = {
                "port0_dst_ip_hex":
                ip_to_hex(dst_port0_ip.network.network_address.exploded),
                "port0_netmask_hex":
                ip_to_hex(dst_port0_ip.network.netmask.exploded),
                # this is the port num that contains port0 subnet and next_hop_ip_hex
                # this is LINKID which should be based on DPDK port number
                "port_num":
                dpdk_port_num,
                # next hop is dst in this case
                # must be within subnet
                "next_hop_ip_hex":
                ip_to_hex(dst_port0_ip.ip.exploded),
            }
            return arp_route_tbl_tmpl.format(**arp_vars)

        return ' '.join(build_arp_config(port) for port in self.all_ports)

    def generate_arpicmp_data(self):
        swq_in_str = self.make_range_str('SWQ{}',
                                         self.swq,
                                         offset=self.lb_count)
        self.swq += self.lb_count
        swq_out_str = self.make_range_str('SWQ{}',
                                          self.swq,
                                          offset=self.lb_count)
        self.swq += self.lb_count
        # ports_mac_list is disabled for some reason

        # mac_iter = (self.vnfd_helper.find_interface(name=port)['virtual-interface']['local_mac']
        #             for port in self.all_ports)
        pktq_in_iter = ('RXQ{}.0'.format(self.vnfd_helper.port_num(x[0]))
                        for x in self.port_pair_list)

        arpicmp_data = {
            'core': self.gen_core(self.start_core),
            'pktq_in': swq_in_str,
            'pktq_out': swq_out_str,
            # we need to disable ports_mac_list?
            # it looks like ports_mac_list is no longer required
            # 'ports_mac_list': ' '.join(mac_iter),
            'pktq_in_prv': ' '.join(pktq_in_iter),
            'prv_to_pub_map': self.set_priv_to_pub_mapping(),
            'arp_route_tbl': self.generate_arp_route_tbl(),
            # nd_route_tbl must be set or we get segault on random OpenStack IPv6 traffic
            # 'nd_route_tbl': "(0064:ff9b:0:0:0:0:9810:6414,120,0,0064:ff9b:0:0:0:0:9810:6414)"
            # safe default?  route discard prefix to localhost
            'nd_route_tbl': "(0100::,64,0,::1)"
        }
        self.pktq_out_os = swq_out_str.split(' ')
        # HWLB is a run to complition. So override the pktq_in/pktq_out
        if self.lb_config == self.HW_LB:
            self.swq = 0
            swq_in_str = \
                self.make_range_str('SWQ{}', self.swq,
                                    offset=(self.lb_count * self.worker_threads))
            arpicmp_data['pktq_in'] = swq_in_str
            # WA: Since port_pairs will not be populated during arp pipeline
            self.port_pairs = self.port_pair_list
            port_iter = \
                self.make_port_pairs_iter(self.float_x_plus_one_tenth_of_y, [self.mul])
            pktq_out = self.make_str('TXQ{}', port_iter)
            arpicmp_data['pktq_out'] = pktq_out

        return arpicmp_data

    def generate_final_txrx_data(self):
        swq_start = self.swq - self.ports_len * self.worker_threads

        txq_start = 0
        txq_end = self.worker_threads

        pktq_out_iter = self.make_range_port_pairs_iter(
            self.float_x_plus_one_tenth_of_y, txq_start, txq_end)

        swq_str = self.make_range_str('SWQ{}', swq_start, self.swq)
        txq_str = self.make_str('TXQ{}', pktq_out_iter)
        rxtx_data = {
            'pktq_in': swq_str,
            'pktq_out': txq_str,
            'pipeline_txrx_type': 'TXTX',
            'core': self.gen_core(self.start_core),
        }
        pktq_in = rxtx_data['pktq_in']
        pktq_in = '{0} {1}'.format(pktq_in,
                                   self.pktq_out_os[self.lb_index - 1])
        rxtx_data['pktq_in'] = pktq_in
        self.pipeline_counter += 1
        return rxtx_data

    def generate_initial_txrx_data(self):
        pktq_iter = self.make_range_port_pairs_iter(
            self.float_x_plus_one_tenth_of_y, 0, self.worker_threads)

        rxq_str = self.make_str('RXQ{}', pktq_iter)
        swq_str = self.make_range_str('SWQ{}', self.swq, offset=self.ports_len)
        txrx_data = {
            'pktq_in': rxq_str,
            'pktq_out': swq_str + ' SWQ{0}'.format(self.lb_index - 1),
            'pipeline_txrx_type': 'RXRX',
            'core': self.gen_core(self.start_core),
        }
        self.pipeline_counter += 1
        return txrx_data

    def generate_lb_data(self):
        pktq_in = self.make_range_str('SWQ{}', self.swq, offset=self.ports_len)
        self.swq += self.ports_len

        offset = self.ports_len * self.worker_threads
        pktq_out = self.make_range_str('SWQ{}', self.swq, offset=offset)
        self.pktq_out = pktq_out.split()

        self.swq += (self.ports_len * self.worker_threads)
        lb_data = {
            'prv_que_handler': self.prv_que_handler,
            'pktq_in': pktq_in,
            'pktq_out': pktq_out,
            'n_vnf_threads': str(self.worker_threads),
            'core': self.gen_core(self.start_core),
        }
        self.pipeline_counter += 1
        return lb_data

    def generate_vnf_data(self):
        if self.lb_config == self.HW_LB:
            port_iter = self.make_port_pairs_iter(
                self.float_x_plus_one_tenth_of_y, [self.mul])
            pktq_in = self.make_str('RXQ{}', port_iter)

            self.mul += 1
            port_iter = self.make_port_pairs_iter(
                self.float_x_plus_one_tenth_of_y, [self.mul])
            pktq_out = self.make_str('TXQ{}', port_iter)

            pipe_line_data = {
                'pktq_in': pktq_in,
                'pktq_out': pktq_out + ' SWQ{0}'.format(self.swq),
                'prv_que_handler': self.prv_que_handler,
                'core': self.gen_core(self.start_core),
            }
            self.swq += 1
        else:
            pipe_line_data = {
                'pktq_in':
                ' '.join(
                    (self.pktq_out.pop(0) for _ in range(self.ports_len))),
                'pktq_out':
                self.make_range_str('SWQ{}', self.swq, offset=self.ports_len),
                'prv_que_handler':
                self.prv_que_handler,
                'core':
                self.gen_core(self.start_core),
            }
            self.swq += self.ports_len

        if self.vnf_type in ('ACL', 'VFW'):
            pipe_line_data.pop('prv_que_handler')

        if self.vnf_tpl.get('vnf_set'):
            public_ip_port_range_list = self.vnf_tpl[
                'public_ip_port_range'].split(':')
            ip_in_hex = '{:x}'.format(
                int(public_ip_port_range_list[0], 16) + self.lb_index - 1)
            public_ip_port_range_list[0] = ip_in_hex
            self.vnf_tpl['public_ip_port_range'] = ':'.join(
                public_ip_port_range_list)

        self.pipeline_counter += 1
        return pipe_line_data

    def generate_config_data(self):
        self.init_write_parser_template()

        # use master core for master, don't use self.start_core
        self.write_parser.set('PIPELINE0', 'core',
                              self.gen_core(self.master_core))
        arpicmp_data = self.generate_arpicmp_data()
        self.arpicmp_tpl.update(arpicmp_data)
        self.update_write_parser(self.arpicmp_tpl)

        self.start_core += 1
        if self.vnf_type == 'CGNAPT':
            self.pipeline_counter += 1
            self.update_timer()

        for lb in self.lb_to_port_pair_mapping:
            self.lb_index = lb
            self.mul = 0
            port_pair_count = self.lb_to_port_pair_mapping[lb]
            if not self.port_pair_list:
                continue

            self.port_pairs = self.port_pair_list[:port_pair_count]
            self.port_pair_list = self.port_pair_list[port_pair_count:]
            self.ports_len = port_pair_count * 2
            self.set_priv_que_handler()
            if self.lb_config == 'SW':
                txrx_data = self.generate_initial_txrx_data()
                self.txrx_tpl.update(txrx_data)
                self.update_write_parser(self.txrx_tpl)
                self.start_core += 1
                lb_data = self.generate_lb_data()
                self.loadb_tpl.update(lb_data)
                self.update_write_parser(self.loadb_tpl)
                self.start_core += 1

            for i in range(self.worker_threads):
                vnf_data = self.generate_vnf_data()
                if not self.vnf_tpl:
                    self.vnf_tpl = {}
                self.vnf_tpl.update(vnf_data)
                self.update_write_parser(self.vnf_tpl)
                try:
                    self.vnf_tpl.pop('vnf_set')
                except KeyError:
                    pass
                else:
                    self.vnf_tpl.pop('public_ip_port_range')
                self.generate_next_core_id()

            if self.lb_config == 'SW':
                txrx_data = self.generate_final_txrx_data()
                self.txrx_tpl.update(txrx_data)
                self.update_write_parser(self.txrx_tpl)
                self.start_core += 1
            self.vnf_tpl = self.get_config_tpl_data(self.vnf_type)

    def generate_config(self):
        self._port_pairs = PortPairs(self.vnfd_helper.interfaces)
        self.port_pair_list = self._port_pairs.port_pair_list
        self.all_ports = self._port_pairs.all_ports

        self.get_lb_count()
        self.generate_lb_to_port_pair_mapping()
        self.generate_config_data()
        self.write_parser.write(sys.stdout)
        with open(self.tmp_file, 'a') as tfh:
            self.write_parser.write(tfh)

    def generate_link_config(self):
        def build_args(port):
            # lookup interface by name
            virtual_interface = self.vnfd_helper.find_interface(
                name=port)["virtual-interface"]
            local_ip = virtual_interface["local_ip"]
            netmask = virtual_interface["netmask"]
            port_num = self.vnfd_helper.port_num(port)
            port_ip, prefix_len = self.validate_ip_and_prefixlen(
                local_ip, netmask)
            return LINK_CONFIG_TEMPLATE.format(port_num, port_ip, prefix_len)

        return ''.join(build_args(port) for port in self.all_ports)

    def get_route_data(self, src_key, data_key, port):
        route_list = self.vnfd['vdu'][0].get(src_key, [])
        try:
            return next((route[data_key]
                         for route in route_list if route['if'] == port), None)
        except (TypeError, StopIteration, KeyError):
            return None

    def get_ports_gateway(self, port):
        return self.get_route_data('routing_table', 'gateway', port)

    def get_ports_gateway6(self, port):
        return self.get_route_data('nd_route_tbl', 'gateway', port)

    def get_netmask_gateway(self, port):
        return self.get_route_data('routing_table', 'netmask', port)

    def get_netmask_gateway6(self, port):
        return self.get_route_data('nd_route_tbl', 'netmask', port)

    def generate_arp_config(self):
        arp_config = []
        for port in self.all_ports:
            # ignore gateway, always use TG IP
            # gateway = self.get_ports_gateway(port)
            vintf = self.vnfd_helper.find_interface(
                name=port)["virtual-interface"]
            dst_mac = vintf["dst_mac"]
            dst_ip = vintf["dst_ip"]
            # arp_config.append(
            #     (self.vnfd_helper.port_num(port), gateway, dst_mac, self.txrx_pipeline))
            # so dst_mac is the TG dest mac, so we need TG dest IP.
            # should be dpdk_port_num
            arp_config.append((self.vnfd_helper.port_num(port), dst_ip,
                               dst_mac, self.txrx_pipeline))

        return '\n'.join(('p {3} arpadd {0} {1} {2}'.format(*values)
                          for values in arp_config))

    def generate_arp_config6(self):
        arp_config6 = []
        for port in self.all_ports:
            # ignore gateway, always use TG IP
            # gateway6 = self.get_ports_gateway6(port)
            vintf = self.vnfd_helper.find_interface(
                name=port)["virtual-interface"]
            dst_mac6 = vintf["dst_mac"]
            dst_ip6 = vintf["dst_ip"]
            # arp_config6.append(
            #     (self.vnfd_helper.port_num(port), gateway6, dst_mac6, self.txrx_pipeline))
            arp_config6.append((self.vnfd_helper.port_num(port), dst_ip6,
                                dst_mac6, self.txrx_pipeline))

        return '\n'.join(('p {3} arpadd {0} {1} {2}'.format(*values)
                          for values in arp_config6))

    def generate_action_config(self):
        port_list = (self.vnfd_helper.port_num(p) for p in self.all_ports)
        if self.vnf_type == "VFW":
            template = FW_ACTION_TEMPLATE
        else:
            template = ACTION_TEMPLATE

        return ''.join((template.format(port) for port in port_list))

    def get_ip_from_port(self, port):
        # we can't use gateway because in OpenStack gateways interfer with floating ip routing
        # return self.make_ip_addr(self.get_ports_gateway(port), self.get_netmask_gateway(port))
        vintf = self.vnfd_helper.find_interface(name=port)["virtual-interface"]
        ip = vintf["local_ip"]
        netmask = vintf["netmask"]
        return self.make_ip_addr(ip, netmask)

    def get_network_and_prefixlen_from_ip_of_port(self, port):
        ip_addr = self.get_ip_from_port(port)
        # handle cases with no gateway
        if ip_addr:
            return ip_addr.network.network_address.exploded, ip_addr.network.prefixlen
        else:
            return None, None

    def generate_rule_config(self):
        cmd = 'acl' if self.vnf_type == "ACL" else "vfw"
        rules_config = self.rules if self.rules else ''
        new_rules = []
        new_ipv6_rules = []
        pattern = 'p {0} add {1} {2} {3} {4} {5} 0 65535 0 65535 0 0 {6}'
        for src_intf, dst_intf in self.port_pair_list:
            src_port = self.vnfd_helper.port_num(src_intf)
            dst_port = self.vnfd_helper.port_num(dst_intf)

            src_net, src_prefix_len = self.get_network_and_prefixlen_from_ip_of_port(
                src_intf)
            dst_net, dst_prefix_len = self.get_network_and_prefixlen_from_ip_of_port(
                dst_intf)
            # ignore entires with empty values
            if all((src_net, src_prefix_len, dst_net, dst_prefix_len)):
                new_rules.append(
                    (cmd, self.txrx_pipeline, src_net, src_prefix_len, dst_net,
                     dst_prefix_len, dst_port))
                new_rules.append(
                    (cmd, self.txrx_pipeline, dst_net, dst_prefix_len, src_net,
                     src_prefix_len, src_port))

            # src_net = self.get_ports_gateway6(port_pair[0])
            # src_prefix_len = self.get_netmask_gateway6(port_pair[0])
            # dst_net = self.get_ports_gateway6(port_pair[1])
            # dst_prefix_len = self.get_netmask_gateway6(port_pair[0])
            # # ignore entires with empty values
            # if all((src_net, src_prefix_len, dst_net, dst_prefix_len)):
            #     new_ipv6_rules.append((cmd, self.txrx_pipeline, src_net, src_prefix_len,
            #                            dst_net, dst_prefix_len, dst_port))
            #     new_ipv6_rules.append((cmd, self.txrx_pipeline, dst_net, dst_prefix_len,
            #                            src_net, src_prefix_len, src_port))

        acl_apply = "\np %s applyruleset" % cmd
        new_rules_config = '\n'.join(
            pattern.format(*values)
            for values in chain(new_rules, new_ipv6_rules))
        return ''.join([rules_config, new_rules_config, acl_apply])

    def generate_script_data(self):
        self._port_pairs = PortPairs(self.vnfd_helper.interfaces)
        self.port_pair_list = self._port_pairs.port_pair_list
        self.get_lb_count()
        script_data = {
            'link_config': self.generate_link_config(),
            'arp_config': self.generate_arp_config(),
            # disable IPv6 for now
            # 'arp_config6': self.generate_arp_config6(),
            'arp_config6': "",
            'actions': '',
            'rules': '',
        }

        if self.vnf_type in ('ACL', 'VFW'):
            script_data.update({
                'actions': self.generate_action_config(),
                'rules': self.generate_rule_config(),
            })

        return script_data

    def generate_script(self, vnfd, rules=None):
        self.vnfd = vnfd
        self.rules = rules
        script_data = self.generate_script_data()
        script = SCRIPT_TPL.format(**script_data)
        if self.lb_config == self.HW_LB:
            script += 'set fwd rxonly'
            hwlb_tpl = """
set_sym_hash_ena_per_port {0} enable
set_hash_global_config {0} simple_xor ipv4-udp enable
set_sym_hash_ena_per_port {1} enable
set_hash_global_config {1} simple_xor ipv4-udp enable
set_hash_input_set {0} ipv4-udp src-ipv4 udp-src-port add
set_hash_input_set {1} ipv4-udp dst-ipv4 udp-dst-port add
set_hash_input_set {0} ipv6-udp src-ipv6 udp-src-port add
set_hash_input_set {1} ipv6-udp dst-ipv6 udp-dst-port add
"""
            for port_pair in self.port_pair_list:
                script += hwlb_tpl.format(
                    *(self.vnfd_helper.port_nums(port_pair)))
        return script
Beispiel #9
0
def loadini(struct, configfile):
    """Loads .ini configuration file and stores its values in struct"""

    config_path = os.path.expanduser(configfile)

    config = ConfigParser()
    defaults = {
        "general": {
            "arg_spec": True,
            "auto_display_list": True,
            "autocomplete_mode": default_completion,
            "color_scheme": "default",
            "complete_magic_methods": True,
            "dedent_after": 1,
            "default_autoreload": False,
            "editor": os.environ.get("VISUAL", os.environ.get("EDITOR", "vi")),
            "flush_output": True,
            "highlight_show_source": True,
            "hist_duplicates": True,
            "hist_file": "~/.pythonhist",
            "hist_length": 1000,
            "paste_time": 0.02,
            "pastebin_confirm": True,
            "pastebin_expiry": "1week",
            "pastebin_helper": "",
            "pastebin_url": "https://bpaste.net",
            "save_append_py": False,
            "single_undo_time": 1.0,
            "syntax": True,
            "tab_length": 4,
            "unicode_box": True,
        },
        "keyboard": {
            "backspace": "C-h",
            "beginning_of_line": "C-a",
            "clear_line": "C-u",
            "clear_screen": "C-l",
            "clear_word": "C-w",
            "copy_clipboard": "F10",
            "cut_to_buffer": "C-k",
            "delete": "C-d",
            "down_one_line": "C-n",
            "edit_config": "F3",
            "edit_current_block": "C-x",
            "end_of_line": "C-e",
            "exit": "",
            "external_editor": "F7",
            "help": "F1",
            "incremental_search": "M-s",
            "last_output": "F9",
            "left": "C-b",
            "pastebin": "F8",
            "reimport": "F6",
            "reverse_incremental_search": "M-r",
            "right": "C-f",
            "save": "C-s",
            "search": "C-o",
            "show_source": "F2",
            "suspend": "C-z",
            "toggle_file_watch": "F5",
            "transpose_chars": "C-t",
            "undo": "C-r",
            "up_one_line": "C-p",
            "yank_from_buffer": "C-y",
        },
        "cli": {
            "suggestion_width": 0.8,
            "trim_prompts": False,
        },
        "curtsies": {
            "list_above": False,
            "right_arrow_completion": True,
        },
    }

    default_keys_to_commands = dict(
        (value, key) for (key, value) in iteritems(defaults["keyboard"]))

    fill_config_with_default_values(config, defaults)
    try:
        if not config.read(config_path):
            # No config file. If the user has it in the old place then complain
            if os.path.isfile(os.path.expanduser("~/.bpython.ini")):
                sys.stderr.write(
                    "Error: It seems that you have a config file at "
                    "~/.bpython.ini. Please move your config file to "
                    "%s\n" % default_config_path())
                sys.exit(1)
    except UnicodeDecodeError as e:
        sys.stderr.write(
            "Error: Unable to parse config file at '{}' due to an "
            "encoding issue. Please make sure to fix the encoding "
            "of the file or remove it and then try again.\n".format(
                config_path))
        sys.exit(1)

    def get_key_no_doublebind(command):
        default_commands_to_keys = defaults["keyboard"]
        requested_key = config.get("keyboard", command)

        try:
            default_command = default_keys_to_commands[requested_key]

            if default_commands_to_keys[default_command] == config.get(
                    "keyboard", default_command):
                setattr(struct, "%s_key" % default_command, "")
        except KeyError:
            pass

        return requested_key

    struct.config_path = config_path

    struct.dedent_after = config.getint("general", "dedent_after")
    struct.tab_length = config.getint("general", "tab_length")
    struct.auto_display_list = config.getboolean("general",
                                                 "auto_display_list")
    struct.syntax = config.getboolean("general", "syntax")
    struct.arg_spec = config.getboolean("general", "arg_spec")
    struct.paste_time = config.getfloat("general", "paste_time")
    struct.single_undo_time = config.getfloat("general", "single_undo_time")
    struct.highlight_show_source = config.getboolean("general",
                                                     "highlight_show_source")
    struct.hist_file = config.get("general", "hist_file")
    struct.editor = config.get("general", "editor")
    struct.hist_length = config.getint("general", "hist_length")
    struct.hist_duplicates = config.getboolean("general", "hist_duplicates")
    struct.flush_output = config.getboolean("general", "flush_output")
    struct.default_autoreload = config.getboolean("general",
                                                  "default_autoreload")

    struct.pastebin_key = get_key_no_doublebind("pastebin")
    struct.copy_clipboard_key = get_key_no_doublebind("copy_clipboard")
    struct.save_key = get_key_no_doublebind("save")
    struct.search_key = get_key_no_doublebind("search")
    struct.show_source_key = get_key_no_doublebind("show_source")
    struct.suspend_key = get_key_no_doublebind("suspend")
    struct.toggle_file_watch_key = get_key_no_doublebind("toggle_file_watch")
    struct.undo_key = get_key_no_doublebind("undo")
    struct.reimport_key = get_key_no_doublebind("reimport")
    struct.reverse_incremental_search_key = get_key_no_doublebind(
        "reverse_incremental_search")
    struct.incremental_search_key = get_key_no_doublebind("incremental_search")
    struct.up_one_line_key = get_key_no_doublebind("up_one_line")
    struct.down_one_line_key = get_key_no_doublebind("down_one_line")
    struct.cut_to_buffer_key = get_key_no_doublebind("cut_to_buffer")
    struct.yank_from_buffer_key = get_key_no_doublebind("yank_from_buffer")
    struct.clear_word_key = get_key_no_doublebind("clear_word")
    struct.backspace_key = get_key_no_doublebind("backspace")
    struct.clear_line_key = get_key_no_doublebind("clear_line")
    struct.clear_screen_key = get_key_no_doublebind("clear_screen")
    struct.delete_key = get_key_no_doublebind("delete")

    struct.left_key = get_key_no_doublebind("left")
    struct.right_key = get_key_no_doublebind("right")
    struct.end_of_line_key = get_key_no_doublebind("end_of_line")
    struct.beginning_of_line_key = get_key_no_doublebind("beginning_of_line")
    struct.transpose_chars_key = get_key_no_doublebind("transpose_chars")
    struct.exit_key = get_key_no_doublebind("exit")
    struct.last_output_key = get_key_no_doublebind("last_output")
    struct.edit_config_key = get_key_no_doublebind("edit_config")
    struct.edit_current_block_key = get_key_no_doublebind("edit_current_block")
    struct.external_editor_key = get_key_no_doublebind("external_editor")
    struct.help_key = get_key_no_doublebind("help")

    struct.pastebin_confirm = config.getboolean("general", "pastebin_confirm")
    struct.pastebin_url = config.get("general", "pastebin_url")
    struct.pastebin_expiry = config.get("general", "pastebin_expiry")
    struct.pastebin_helper = config.get("general", "pastebin_helper")

    struct.cli_suggestion_width = config.getfloat("cli", "suggestion_width")
    struct.cli_trim_prompts = config.getboolean("cli", "trim_prompts")

    struct.complete_magic_methods = config.getboolean(
        "general", "complete_magic_methods")
    struct.autocomplete_mode = config.get("general", "autocomplete_mode")
    struct.save_append_py = config.getboolean("general", "save_append_py")

    struct.curtsies_list_above = config.getboolean("curtsies", "list_above")
    struct.curtsies_right_arrow_completion = config.getboolean(
        "curtsies", "right_arrow_completion")

    color_scheme_name = config.get("general", "color_scheme")

    default_colors = {
        "keyword": "y",
        "name": "c",
        "comment": "b",
        "string": "m",
        "error": "r",
        "number": "G",
        "operator": "Y",
        "punctuation": "y",
        "token": "C",
        "background": "d",
        "output": "w",
        "main": "c",
        "paren": "R",
        "prompt": "c",
        "prompt_more": "g",
        "right_arrow_suggestion": "K",
    }

    if color_scheme_name == "default":
        struct.color_scheme = default_colors
    else:
        struct.color_scheme = dict()

        theme_filename = color_scheme_name + ".theme"
        path = os.path.expanduser(
            os.path.join(get_config_home(), theme_filename))
        try:
            load_theme(struct, path, struct.color_scheme, default_colors)
        except EnvironmentError:
            sys.stderr.write("Could not load theme '%s'.\n" %
                             (color_scheme_name, ))
            sys.exit(1)

    # expand path of history file
    struct.hist_file = os.path.expanduser(struct.hist_file)

    # verify completion mode
    if struct.autocomplete_mode not in ALL_MODES:
        struct.autocomplete_mode = default_completion

    # set box drawing characters
    if config.getboolean("general", "unicode_box") and supports_box_chars():
        struct.left_border = "│"
        struct.right_border = "│"
        struct.top_border = "─"
        struct.bottom_border = "─"
        struct.left_bottom_corner = "└"
        struct.right_bottom_corner = "┘"
        struct.left_top_corner = "┌"
        struct.right_top_corner = "┐"
    else:
        struct.left_border = "|"
        struct.right_border = "|"
        struct.top_border = "-"
        struct.bottom_border = "-"
        struct.left_bottom_corner = "+"
        struct.right_bottom_corner = "+"
        struct.left_top_corner = "+"
        struct.right_top_corner = "+"
Beispiel #10
0
class PypiConfig(BaseConfig):
    """Wrapper around the pypi config file"""
    def __init__(self, config_filename=DIST_CONFIG_FILE, use_setup_cfg=True):
        """Grab the PyPI configuration.

        This is .pypirc in the home directory.  It is overridable for
        test purposes.

        If there is a setup.cfg file in the current directory, we read
        it too.
        """
        self.config_filename = config_filename
        self.use_setup_cfg = use_setup_cfg
        self.reload()

    def reload(self):
        """Load the config.

        Do the initial load of the config.

        Or reload it in case of problems: this is needed when a pypi
        upload fails, you edit the .pypirc file to fix the account
        settings, and tell release to retry the command.
        """
        self._read_configfile(use_setup_cfg=self.use_setup_cfg)

    def _read_configfile(self, use_setup_cfg=True):
        """Read the PyPI config file and store it (when valid).

        Usually read the setup.cfg too.
        """
        rc = self.config_filename
        if not os.path.isabs(rc):
            rc = os.path.join(os.path.expanduser('~'), self.config_filename)
        filenames = [rc]
        if use_setup_cfg:
            # If there is a setup.cfg in the package, parse it
            filenames.append('setup.cfg')
        files = [f for f in filenames if os.path.exists(f)]
        if not files:
            self.config = None
            return
        self.config = ConfigParser()
        self.config.read(files)

    def is_pypi_configured(self):
        # Do we have configuration for releasing to at least one
        # pypi-compatible server?
        if self.config is None:
            return False
        return len(self.distutils_servers()) > 0

    def get_server_config(self, server):
        """Get url, username, password for server.
        """
        repository_url = DEFAULT_REPOSITORY
        username = None
        password = None
        if self.config.has_section(server):
            if self.config.has_option(server, 'repository'):
                repository_url = self._get_text(server, 'repository')
            if self.config.has_option(server, 'username'):
                username = self._get_text(server, 'username')
            if self.config.has_option(server, 'password'):
                password = self._get_text(server, 'password', raw=True)
        if not username and self.config.has_option('server-login', 'username'):
            username = self._get_text('server-login', 'username')
        if not password and self.config.has_option('server-login', 'password'):
            password = self._get_text('server-login', 'password', raw=True)
        return {
            'repository_url': repository_url,
            'username': username,
            'password': password,
        }

    def distutils_servers(self):
        """Return a list of known distutils servers.

        If the config has an old pypi config, remove the default pypi
        server from the list.
        """
        try:
            index_servers = self._get_text('distutils',
                                           'index-servers',
                                           default='').split()
        except (NoSectionError, NoOptionError):
            index_servers = []
        if not index_servers:
            # If no distutils index-servers have been given, 'pypi' should be
            # the default.  This is what twine does.
            if self.config.has_option('server-login', 'username'):
                # We have a username, so upload to pypi should work fine, even
                # when no explicit pypi section is in the file.
                return ['pypi']
            # https://github.com/zestsoftware/zest.releaser/issues/199
            index_servers = ['pypi']
        # The servers all need to have a section in the config file.
        return [
            server for server in index_servers
            if self.config.has_section(server)
        ]

    def want_release(self):
        """Does the user normally want to release this package.

        Some colleagues find it irritating to have to remember to
        answer the question "Check out the tag (for tweaks or
        pypi/distutils server upload)" with the non-default 'no' when
        in 99 percent of the cases they just make a release specific
        for a customer, so they always answer 'no' here.  This is
        where an extra config option comes in handy: you can influence
        the default answer so you can just keep hitting 'Enter' until
        zest.releaser is done.

        Either in your ~/.pypirc or in a setup.cfg in a specific
        package, add this when you want the default answer to this
        question to be 'no':

        [zest.releaser]
        release = no

        The default when this option has not been set is True.

        Standard config rules apply, so you can use upper or lower or
        mixed case and specify 0, false, no or off for boolean False,
        and 1, on, true or yes for boolean True.
        """
        return self._get_boolean('zest.releaser', 'release', default=True)

    def extra_message(self):
        """Return extra text to be added to commit messages.

        This can for example be used to skip CI builds.  This at least
        works for Travis.  See
        http://docs.travis-ci.com/user/how-to-skip-a-build/

        Enable this mode by adding a ``extra-message`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            extra-message = [ci skip]
        """
        default = ''
        if self.config is None:
            return default
        try:
            result = self._get_text('zest.releaser',
                                    'extra-message',
                                    default=default)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def history_file(self):
        """Return path of history file.

        Usually zest.releaser can find the correct one on its own.
        But sometimes it may not find anything, or it finds multiple
        and selects the wrong one.

        Configure this by adding a ``history-file`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            history-file = deep/down/historie.doc
        """
        default = ''
        if self.config is None:
            return default
        marker = object()
        try:
            result = self._get_text('zest.releaser',
                                    'history-file',
                                    default=marker)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        if result == marker:
            # We were reading an underscore instead of a dash at first.
            try:
                result = self._get_text('zest.releaser',
                                        'history_file',
                                        default=default)
            except (NoSectionError, NoOptionError, ValueError):
                return default
        return result

    def encoding(self):
        """Return encoding to use for text files.

        Mostly the changelog and if needed `setup.py`.

        The encoding cannot always be determined correctly.
        This setting is a way to fix that.
        See https://github.com/zestsoftware/zest.releaser/issues/264

        Configure this by adding an ``encoding`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            encoding = utf-8
        """
        default = ''
        if self.config is None:
            return default
        try:
            result = self._get_text('zest.releaser',
                                    'encoding',
                                    default=default,
                                    raw=True)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def create_wheel(self):
        """Should we create a Python wheel for this package?

        Either in your ~/.pypirc or in a setup.cfg in a specific
        package, add this when you want to create a Python wheel, next
        to a standard sdist:

        [zest.releaser]
        create-wheel = yes

        If there is no setting for ``create-wheel``, then if universal
        wheels are specified as in the following example, it is treated as if
        ``create-wheel`` was true:

        [bdist_wheel]
        universal = 1

        """
        if not USE_WHEEL:
            # If the wheel package is not available, we obviously
            # cannot create wheels.
            return False
        create_setting = self._get_boolean('zest.releaser', 'create-wheel',
                                           None)
        if create_setting is not None:
            # User specified this setting, it overrides
            # inferring from bdist_wheel
            return create_setting
        # No zest.releaser setting, are they asking for a universal wheel?
        # Then they want wheels in general.
        return self._get_boolean('bdist_wheel', 'universal')

    def register_package(self):
        """Should we try to register this package with a package server?

        For the standard Python Package Index (PyPI), registering a
        package is no longer needed: this is done automatically when
        uploading a distribution for a package.  In fact, trying to
        register may fail.  See
        https://github.com/zestsoftware/zest.releaser/issues/191
        So by default zest.releaser will no longer register a package.

        But you may be using your own package server, and registering
        may be wanted or even required there.  In this case
        you will need to turn on the register function.
        In your setup.cfg or ~/.pypirc, use the following to ensure that
        register is called on the package server:

        [zest.releaser]
        register = yes

        Note that if you have specified multiple package servers, this
        option is used for all of them.  There is no way to register and
        upload to server A, and only upload to server B.
        """
        return self._get_boolean('zest.releaser', 'register')

    def no_input(self):
        """Return whether the user wants to run in no-input mode.

        Enable this mode by adding a ``no-input`` option::

            [zest.releaser]
            no-input = yes

        The default when this option has not been set is False.
        """
        return self._get_boolean('zest.releaser', 'no-input')

    def development_marker(self):
        """Return development marker to be appended in postrelease.

        Override the default ``.dev0`` in ~/.pypirc or setup.cfg using
        a ``development-marker`` option::

            [zest.releaser]
            development-marker = .dev1

        Returns default of ``.dev0`` when nothing has been configured.
        """
        default = '.dev0'
        if self.config is None:
            return default
        try:
            result = self._get_text('zest.releaser',
                                    'development-marker',
                                    default=default)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def push_changes(self):
        """Return whether the user wants to push the changes to the remote.

        Configure this mode by adding a ``push-changes`` option::

            [zest.releaser]
            push-changes = no

        The default when this option has not been set is True.
        """
        return self._get_boolean('zest.releaser', 'push-changes', default=True)

    def less_zeroes(self):
        """Return whether the user prefers less zeroes at the end of a version.

        Configure this mode by adding a ``less-zeroes`` option::

            [zest.releaser]
            less-zeroes = yes

        The default when this option has not been set is False.

        When set to true:
        - Instead of 1.3.0 we will suggest 1.3.
        - Instead of 2.0.0 we will suggest 2.0.

        This only makes sense for the bumpversion command.
        In the postrelease command we read this option too,
        but with the current logic it has no effect there.
        """
        return self._get_boolean('zest.releaser', 'less-zeroes')

    def version_levels(self):
        """How many levels does the user prefer in a version number?

        Configure this mode by adding a ``version-levels`` option::

            [zest.releaser]
            version-levels = 3

        The default when this option has not been set is 0, which means:
        no preference, so use the length of the current number.

        This means when suggesting a next version after 1.2:
        - with levels=0 we will suggest 1.3: no change
        - with levels=1 we will still suggest 1.3, as we will not
          use this to remove numbers, only to add them
        - with levels=2 we will suggest 1.3
        - with levels=3 we will suggest 1.2.1

        If the current version number has more levels, we keep them.
        So next version for 1.2.3.4 with levels=1 is 1.2.3.5.

        Tweaking version-levels and less-zeroes should give you the
        version number strategy that you prefer.
        """
        default = 0
        if self.config is None:
            return default
        try:
            result = self.config.getint('zest.releaser', 'version-levels')
        except (NoSectionError, NoOptionError, ValueError):
            return default
        if result < 0:
            return default
        return result

    _tag_format_deprecated_message = "\n".join(line.strip() for line in """
    `tag-format` contains deprecated `%%(version)s` format. Please change to:

    [zest.releaser]
    tag-format = %s
    """.strip().splitlines())

    def tag_format(self, version):
        """Return the formatted tag that should be used in the release.

        Configure it in ~/.pypirc or setup.cfg using a ``tag-format`` option::

            [zest.releaser]
            tag-format = v{version}

        ``tag-format`` must contain exaclty one formatting instruction: for the
        ``version`` key.

        Accepts also ``%(version)s`` format for backward compatibility.

        The default format, when nothing has been configured, is ``{version}``.
        """
        fmt = '{version}'
        if self.config is not None:
            try:
                fmt = self._get_text('zest.releaser',
                                     'tag-format',
                                     default=fmt,
                                     raw=True)
            except (NoSectionError, NoOptionError, ValueError):
                pass
        if '{version}' in fmt:
            return fmt.format(version=version)
        # BBB:
        if '%(version)s' in fmt:
            proposed_fmt = fmt.replace("%(version)s", "{version}")
            print(self._tag_format_deprecated_message % proposed_fmt)
            return fmt % {'version': version}
        print("{version} needs to be part of 'tag-format': %s" % fmt)
        sys.exit(1)

    def tag_message(self, version):
        """Return the commit message to be used when tagging.

        Configure it in ~/.pypirc or setup.cfg using a ``tag-message``
        option::

            [zest.releaser]
            tag-message = Creating v{version} tag.

        ``tag-message`` must contain exaclty one formatting
        instruction: for the ``version`` key.

        The default format is ``Tagging {version}``.
        """
        fmt = 'Tagging {version}'
        if self.config:
            try:
                fmt = self._get_text('zest.releaser',
                                     'tag-message',
                                     default=fmt,
                                     raw=True)
            except (NoSectionError, NoOptionError, ValueError):
                pass
        if '{version}' not in fmt:
            print("{version} needs to be part of 'tag-message': '%s'" % fmt)
            sys.exit(1)
        return fmt.format(version=version)

    def tag_signing(self):
        """Return whether the tag should be signed.

        Configure it in ~/.pypirc or setup.cfg using a ``tag-signing`` option::

            [zest.releaser]
            tag-signing = yes

        ``tag-signing`` must contain exaclty one word which will be
        converted to a boolean. Currently are accepted (case
        insensitively): 0, false, no, off for False, and 1, true, yes,
        on for True).

        The default when this option has not been set is False.

        """
        return self._get_boolean('zest.releaser', 'tag-signing', default=False)

    def date_format(self):
        """Return the string format for the date used in the changelog.

        Override the default ``%Y-%m-%d`` in ~/.pypirc or setup.cfg using
        a ``date-format`` option::

            [zest.releaser]
            date-format = %%B %%e, %%Y

        Note: the % signs should be doubled for compatibility with other tools
        (i.e. pip) that parse setup.cfg using the interpolating ConfigParser.

        Returns default of ``%Y-%m-%d`` when nothing has been configured.
        """
        default = '%Y-%m-%d'
        if self.config is None:
            return default
        try:
            result = self._get_text('zest.releaser',
                                    'date-format',
                                    default=default).replace('%%', '%')
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result
Beispiel #11
0
def read_config(filenames=SEARCH_PATH):
    """Attempt to read local configuration files to determine spalloc client
    settings.

    Parameters
    ----------
    filenames : [str, ...]
        Filenames to attempt to read. Later config file have higher priority.

    Returns
    -------
    dict
        The configuration loaded.
    """
    parser = ConfigParser()

    # Set default config values (NB: No read_dict in Python 2.7)
    parser.add_section("spalloc")
    for key, value in iteritems({"port": "22244",
                                 "keepalive": "60.0",
                                 "reconnect_delay": "5.0",
                                 "timeout": "5.0",
                                 "machine": "None",
                                 "tags": "None",
                                 "min_ratio": "0.333",
                                 "max_dead_boards": "0",
                                 "max_dead_links": "None",
                                 "require_torus": "False"}):
        parser.set("spalloc", key, value)

    # Attempt to read from each possible file location in turn
    for filename in filenames:
        try:
            with open(filename, "r") as f:
                parser.readfp(f, filename)
        except (IOError, OSError):
            # File did not exist, keep trying
            pass

    cfg = {}

    try:
        cfg["hostname"] = parser.get("spalloc", "hostname")
    except NoOptionError:
        cfg["hostname"] = None

    cfg["port"] = parser.getint("spalloc", "port")

    try:
        cfg["owner"] = parser.get("spalloc", "owner")
    except NoOptionError:
        cfg["owner"] = None

    if parser.get("spalloc", "keepalive") == "None":
        cfg["keepalive"] = None
    else:
        cfg["keepalive"] = parser.getfloat("spalloc", "keepalive")

    cfg["reconnect_delay"] = parser.getfloat("spalloc", "reconnect_delay")

    if parser.get("spalloc", "timeout") == "None":
        cfg["timeout"] = None
    else:
        cfg["timeout"] = parser.getfloat("spalloc", "timeout")

    if parser.get("spalloc", "machine") == "None":
        cfg["machine"] = None
    else:
        cfg["machine"] = parser.get("spalloc", "machine")

    if parser.get("spalloc", "tags") == "None":
        cfg["tags"] = None
    else:
        cfg["tags"] = list(map(str.strip,
                               parser.get("spalloc", "tags").split(",")))

    cfg["min_ratio"] = parser.getfloat("spalloc", "min_ratio")

    if parser.get("spalloc", "max_dead_boards") == "None":
        cfg["max_dead_boards"] = None
    else:
        cfg["max_dead_boards"] = parser.getint("spalloc", "max_dead_boards")

    if parser.get("spalloc", "max_dead_links") == "None":
        cfg["max_dead_links"] = None
    else:
        cfg["max_dead_links"] = parser.getint("spalloc", "max_dead_links")

    cfg["require_torus"] = parser.getboolean("spalloc", "require_torus")

    return cfg
def loadini(struct, configfile):
    """Loads .ini configuration file and stores its values in struct"""

    config_path = os.path.expanduser(configfile)

    config = ConfigParser()
    defaults = {
        'general': {
            'arg_spec': True,
            'auto_display_list': True,
            'autocomplete_mode': default_completion,
            'color_scheme': 'default',
            'complete_magic_methods': True,
            'dedent_after': 1,
            'editor': os.environ.get('VISUAL', os.environ.get('EDITOR', 'vi')),
            'flush_output': True,
            'highlight_show_source': True,
            'hist_duplicates': True,
            'hist_file': '~/.pythonhist',
            'hist_length': 100,
            'paste_time': 0.02,
            'pastebin_confirm': True,
            'pastebin_expiry': '1week',
            'pastebin_helper': '',
            'pastebin_removal_url': 'https://bpaste.net/remove/$removal_id',
            'pastebin_show_url': 'https://bpaste.net/show/$paste_id',
            'pastebin_url': 'https://bpaste.net/json/new',
            'save_append_py': False,
            'single_undo_time': 1.0,
            'syntax': True,
            'tab_length': 4,
            'unicode_box': True
        },
        'keyboard': {
            'backspace': 'C-h',
            'beginning_of_line': 'C-a',
            'clear_line': 'C-u',
            'clear_screen': 'C-l',
            'clear_word': 'C-w',
            'copy_clipboard': 'F10',
            'cut_to_buffer': 'C-k',
            'delete': 'C-d',
            'down_one_line': 'C-n',
            'edit_config': 'F3',
            'edit_current_block': 'C-x',
            'end_of_line': 'C-e',
            'exit': '',
            'external_editor': 'F7',
            'help': 'F1',
            'last_output': 'F9',
            'left': 'C-b',
            'pastebin': 'F8',
            'reimport': 'F6',
            'right': 'C-f',
            'save': 'C-s',
            'search': 'C-o',
            'show_source': 'F2',
            'suspend': 'C-z',
            'toggle_file_watch': 'F5',
            'transpose_chars': 'C-t',
            'undo': 'C-r',
            'up_one_line': 'C-p',
            'yank_from_buffer': 'C-y'
        },
        'cli': {
            'suggestion_width': 0.8,
            'trim_prompts': False,
        },
        'curtsies': {
            'list_above': False,
            'right_arrow_completion': True,
        }}

    default_keys_to_commands = dict((value, key) for (key, value)
                                    in iteritems(defaults['keyboard']))

    fill_config_with_default_values(config, defaults)
    if not config.read(config_path):
        # No config file. If the user has it in the old place then complain
        if os.path.isfile(os.path.expanduser('~/.bpython.ini')):
            sys.stderr.write("Error: It seems that you have a config file at "
                             "~/.bpython.ini. Please move your config file to "
                             "%s\n" % default_config_path())
            sys.exit(1)

    def get_key_no_doublebind(command):
        default_commands_to_keys = defaults['keyboard']
        requested_key = config.get('keyboard', command)

        try:
            default_command = default_keys_to_commands[requested_key]

            if (default_commands_to_keys[default_command] ==
                    config.get('keyboard', default_command)):
                setattr(struct, '%s_key' % default_command, '')
        except KeyError:
            pass

        return requested_key

    struct.config_path = config_path

    struct.dedent_after = config.getint('general', 'dedent_after')
    struct.tab_length = config.getint('general', 'tab_length')
    struct.auto_display_list = config.getboolean('general',
                                                 'auto_display_list')
    struct.syntax = config.getboolean('general', 'syntax')
    struct.arg_spec = config.getboolean('general', 'arg_spec')
    struct.paste_time = config.getfloat('general', 'paste_time')
    struct.single_undo_time = config.getfloat('general', 'single_undo_time')
    struct.highlight_show_source = config.getboolean('general',
                                                     'highlight_show_source')
    struct.hist_file = config.get('general', 'hist_file')
    struct.editor = config.get('general', 'editor')
    struct.hist_length = config.getint('general', 'hist_length')
    struct.hist_duplicates = config.getboolean('general', 'hist_duplicates')
    struct.flush_output = config.getboolean('general', 'flush_output')

    struct.pastebin_key = get_key_no_doublebind('pastebin')
    struct.copy_clipboard_key = get_key_no_doublebind('copy_clipboard')
    struct.save_key = get_key_no_doublebind('save')
    struct.search_key = get_key_no_doublebind('search')
    struct.show_source_key = get_key_no_doublebind('show_source')
    struct.suspend_key = get_key_no_doublebind('suspend')
    struct.toggle_file_watch_key = get_key_no_doublebind('toggle_file_watch')
    struct.undo_key = get_key_no_doublebind('undo')
    struct.reimport_key = get_key_no_doublebind('reimport')
    struct.up_one_line_key = get_key_no_doublebind('up_one_line')
    struct.down_one_line_key = get_key_no_doublebind('down_one_line')
    struct.cut_to_buffer_key = get_key_no_doublebind('cut_to_buffer')
    struct.yank_from_buffer_key = get_key_no_doublebind('yank_from_buffer')
    struct.clear_word_key = get_key_no_doublebind('clear_word')
    struct.backspace_key = get_key_no_doublebind('backspace')
    struct.clear_line_key = get_key_no_doublebind('clear_line')
    struct.clear_screen_key = get_key_no_doublebind('clear_screen')
    struct.delete_key = get_key_no_doublebind('delete')

    struct.left_key = get_key_no_doublebind('left')
    struct.right_key = get_key_no_doublebind('right')
    struct.end_of_line_key = get_key_no_doublebind('end_of_line')
    struct.beginning_of_line_key = get_key_no_doublebind('beginning_of_line')
    struct.transpose_chars_key = get_key_no_doublebind('transpose_chars')
    struct.clear_line_key = get_key_no_doublebind('clear_line')
    struct.clear_screen_key = get_key_no_doublebind('clear_screen')
    struct.exit_key = get_key_no_doublebind('exit')
    struct.last_output_key = get_key_no_doublebind('last_output')
    struct.edit_config_key = get_key_no_doublebind('edit_config')
    struct.edit_current_block_key = get_key_no_doublebind('edit_current_block')
    struct.external_editor_key = get_key_no_doublebind('external_editor')
    struct.help_key = get_key_no_doublebind('help')

    struct.pastebin_confirm = config.getboolean('general', 'pastebin_confirm')
    struct.pastebin_url = config.get('general', 'pastebin_url')
    struct.pastebin_show_url = config.get('general', 'pastebin_show_url')
    struct.pastebin_removal_url = config.get('general', 'pastebin_removal_url')
    struct.pastebin_expiry = config.get('general', 'pastebin_expiry')
    struct.pastebin_helper = config.get('general', 'pastebin_helper')

    struct.cli_suggestion_width = config.getfloat('cli',
                                                  'suggestion_width')
    struct.cli_trim_prompts = config.getboolean('cli',
                                                'trim_prompts')

    struct.complete_magic_methods = config.getboolean('general',
                                                      'complete_magic_methods')
    struct.autocomplete_mode = config.get('general', 'autocomplete_mode')
    struct.save_append_py = config.getboolean('general', 'save_append_py')

    struct.curtsies_list_above = config.getboolean('curtsies', 'list_above')
    struct.curtsies_right_arrow_completion = \
        config.getboolean('curtsies', 'right_arrow_completion')

    color_scheme_name = config.get('general', 'color_scheme')

    default_colors = {
        'keyword': 'y',
        'name': 'c',
        'comment': 'b',
        'string': 'm',
        'error': 'r',
        'number': 'G',
        'operator': 'Y',
        'punctuation': 'y',
        'token': 'C',
        'background': 'd',
        'output': 'w',
        'main': 'c',
        'paren': 'R',
        'prompt': 'c',
        'prompt_more': 'g',
        'right_arrow_suggestion': 'K',
    }

    if color_scheme_name == 'default':
        struct.color_scheme = default_colors
    else:
        struct.color_scheme = dict()

        theme_filename = color_scheme_name + '.theme'
        path = os.path.expanduser(os.path.join(get_config_home(),
                                               theme_filename))
        try:
            load_theme(struct, path, struct.color_scheme, default_colors)
        except EnvironmentError:
            sys.stderr.write("Could not load theme '%s'.\n" %
                             (color_scheme_name, ))
            sys.exit(1)

    # checks for valid key configuration this part still sucks
    for key in (struct.pastebin_key, struct.save_key):
        key_dispatch[key]

    # expand path of history file
    struct.hist_file = os.path.expanduser(struct.hist_file)

    # verify completion mode
    if struct.autocomplete_mode not in bpython.autocomplete.ALL_MODES:
        struct.autocomplete_mode = default_completion

    # set box drawing characters
    if config.getboolean('general', 'unicode_box') and supports_box_chars():
        struct.left_border = u'│'
        struct.right_border = u'│'
        struct.top_border = u'─'
        struct.bottom_border = u'─'
        struct.left_bottom_corner = u'└'
        struct.right_bottom_corner = u'┘'
        struct.left_top_corner = u'┌'
        struct.right_top_corner = u'┐'
    else:
        struct.left_border = u'|'
        struct.right_border = u'|'
        struct.top_border = u'-'
        struct.bottom_border = u'-'
        struct.left_bottom_corner = u'+'
        struct.right_bottom_corner = u'+'
        struct.left_top_corner = u'+'
        struct.right_top_corner = u'+'
config.read(['config.cfg'])

# Set up job queue and worker thread
job_queue = Queue()

def start_thread():
    worker = Thread(target=worker_thread, args=(job_queue, config))
    worker.setDaemon(True)
    worker.start()
    return worker

bg_thread = start_thread()

print('Connecting to DB')
db = oursql.connect(host=config.get('db', 'host'),
                    port=config.getint('db', 'port'),
                    db='wikidatawiki_p',
                    user=config.get('db', 'user'),
                    passwd=config.get('db', 'passwd'),
                    raise_on_warnings=True,
                    autoping=True,
                    autoreconnect=True)
cur = db.cursor()

print('Querying DB')
cur.execute('select de.term_entity_id from wb_terms as de join wb_terms as def on de.term_entity_id = def.term_entity_id and de.term_type = def.term_type where de.term_entity_type = "item" and def.term_entity_type = "item" and de.term_language = "nb" and def.term_language = "no" and de.term_type != "alias" and de.term_text != def.term_text')
rows = [row[0] for row in cur.fetchall()]

completed = 0
t0 = time.time()
Beispiel #14
0
class Bot(Client):
    def __init__(self, core, configfile):
        self.core = core

        self.configfile = configfile
        self.config = ConfigParser()
        self.config.read(configfile)
        
        host = self.config.get('base', 'host')
        port = self.config.getint('base', 'port')
        try:
            ssl = self.config.getboolean('base', 'ssl')
        except:
            ssl = False
        
        Client.__init__(self, (host, port), ssl)
        self.hooks = HookManager(self)
        self.plugins = PluginManager(self)

        self.hooks.install_owner(self)

        self.nick = None
        self.channels = {}
        superuser = self.config.get('base', 'superuser')
        self.allow_rules = {'*': {'ANY': 1}, superuser: {'ANY': 1000}}
        self.deny_rules = {}
        self._name = '_bot'

        autoload = self.config.get('base', 'autoload').split()
        for name in autoload:
            self.plugins.load(name)

        self.connect()

    def set_timer(self, fn, timestamp, owner=None):
        hook = TimestampHook(timestamp)
        hook.bind(fn, owner)
        self.hooks.install(hook)
        return hook

    def set_interval(self, fn, seconds, owner=None):
        hook = TimestampHook(time() + seconds, {'repeat': seconds})
        hook.bind(fn, owner)
        self.hooks.install(hook)
        return hook

    def set_timeout(self, fn, seconds, owner=None):
        hook = TimestampHook(time() + seconds)
        hook.bind(fn, owner)
        self.hooks.install(hook)
        return hook

    def do_tick(self, timestamp):
        self.hooks.call_timestamp(timestamp)

    def privmsg(self, target, text):
        wraplen = 510
        wraplen -= 1 + len(self.nick) # ":<nick>"
        wraplen -= 1 + 10 # "!<user>"
        wraplen -= 1 + 63 # "@<host>"
        wraplen -= 9 # " PRIVMSG "
        wraplen -= len(target) # "<target>"
        wraplen -= 2 # " :"
        for line in wrap(text, wraplen):
            self.send('PRIVMSG %s :%s' % (target, line))

    def notice(self, target, text):
        wraplen = 510
        wraplen -= 1 + len(self.nick) # ":<nick>"
        wraplen -= 1 + 10 # "!<user>"
        wraplen -= 1 + 63 # "@<host>"
        wraplen -= 8 # " NOTICE "
        wraplen -= len(target) # "<target>"
        wraplen -= 2 # " :"
        for line in wrap(text, wraplen):
            self.send('NOTICE %s :%s' % (target, line))

    def join(self, channels, keys=None):
        if isinstance(channels, str):
            channels = (channels,)
        if channels:
            channel_s = ','.join(channels)
            if keys:
                if isinstance(keys, str):
                    keys = (keys,)
                key_s = ','.join(keys)
                self.send('JOIN %s %s' % (channel_s, key_s))
                pairs = list(zip(channels, keys))
                for item in pairs:
                    self.channels[item[0]] = {'key': item[1], 'joined': False, 'nicks': set()}
            else:
                self.send('JOIN %s' % channel_s)
                for channel in channels:
                    self.channels[channel] = {'joined': False, 'nicks': set()}

    def part(self, channels, message=None):
        if type(channels) == str:
            channels = (channels,)
        if channels:
            channels = ','.join(channels)
            if message:
                self.send('PART %s :%s' % (channels, message))
            else:
                self.send('PART %s' % channels)

    @hook
    @priority(0)
    def disconnect_event(self):
        for _, props in list(self.channels.items()):
            props['joined'] = False
            props['nicks'].clear()

    @hook
    @priority(0)
    def shutdown_event(self, reason):
        self.send('QUIT :%s' % reason)
        for name in self.plugins.list():
            self.plugins.unload(name, True)

    @hook
    def _001_command(self, msg):
        self.server = msg.source
        self.nick = msg.param[0]

    @hook
    def _353_command(self, msg):
        channel = msg.param[2]
        if channel in self.channels and self.channels[channel]['joined']:
            nicks = self.channels[channel]['nicks']
            for nick in msg.param[-1].split():
                if nick.startswith(('~', '&', '@', '%', '+')):
                    nicks.add(nick[1:])
                else:
                    nicks.add(nick)

    @hook
    def join_command(self, msg):
        channel = msg.param[0]
        if msg.source == self.nick:
            if channel not in self.channels:
                self.channels[channel] = {}
            self.channels[channel]['joined'] = True
        elif channel in self.channels:
            self.channels[channel]['nicks'].add(msg.source)

    @hook
    def kick_command(self, msg):
        channel = msg.param[0]
        if msg.param[1] == self.nick:
            if channel in self.channels:
                self.channels[channel]['joined'] = False
                if 'nicks' in self.channels[channel]:
                    self.channels[channel]['nicks'].clear()
        elif channel in self.channels:
            self.channels[channel]['nicks'].remove(msg.source)

    @hook
    def nick_command(self, msg):
        new_nick = msg.param[0]
        if msg.source == self.nick:
            self.nick = new_nick
        for _, props in list(self.channels.items()):
            if 'nicks' in props and msg.source in props['nicks']:
                props['nicks'].remove(msg.source)
                props['nicks'].add(new_nick)

    @hook
    @priority(0)
    def part_command(self, msg):
        channel = msg.param[0]
        if msg.source == self.nick:
            if channel in self.channels:
                self.channels[channel]['joined'] = False
                if 'nicks' in self.channels[channel]:
                    self.channels[channel]['nicks'].clear()
        elif channel in self.channels:
            self.channels[channel]['nicks'].remove(msg.source)

    @hook
    def ping_command(self, msg):
        self.send('PONG :%s' % msg.param[-1])

    @hook
    @priority(0)
    def quit_command(self, msg):
        for _, props in list(self.channels.items()):
            if 'nicks' in props and msg.source in props['nicks']:
                props['nicks'].remove(msg.source)
Beispiel #15
0
def ini_read(config_file):
    def strip_item(l):
        return [x.strip() for x in l]

    cfg = ConfigParser()
    cfg.read(config_file)
    log = {}
    log['file'] = cfg.get('log', 'file')
    log['level'] = cfg.getint('log', 'level')
    server = {}
    server['protocols'] = strip_item(cfg.get('server', 'protocols').split(','))
    server['listen_ip'] = cfg.get('server', 'listen_ip')
    server['listen_port'] = cfg.getint('server', 'listen_port')
    server['search'] = strip_item(cfg.get('server', 'search').split(','))
    server['allowed_hosts'] = strip_item(cfg.get('server', 'allowed_hosts').split(','))
    smartdns = {}
    smartdns['rules'] = []
    for rule in strip_item(cfg.get('smartdns', 'rules').split(',')):
        section = 'rules_' + rule
        smartdns['rules'].append({
            'name': rule,
            'url': cfg.get(section, 'url'),
            'proxy': cfg.getboolean(section, 'proxy'),
            'refresh': cfg.getint(section, 'refresh'),
            'dns': strip_item(cfg.get(section, 'dns').split(',')),
        })
    smartdns['hack_srv'] = strip_item(cfg.get('smartdns', 'hack_srv').split(','))
    section = 'bogus_nxdomain'
    smartdns['bogus_nxdomain'] = {
        'url': cfg.get(section, 'url'),
        'proxy': cfg.getboolean(section, 'proxy'),
        'refresh': cfg.getint(section, 'refresh'),
        'hack_ip': cfg.get(section, 'hack_ip'),
    }
    section = 'proxy'
    smartdns['proxy'] = {
        'type': cfg.get(section, 'type'),
        'ip': cfg.get(section, 'ip'),
        'port': cfg.getint(section, 'port'),
    }
    smartdns['upstreams'] = {}
    names = set()
    for rule in smartdns['rules']:
        names |= set(rule['dns'])
    for name in names:
        section = 'dns_' + name.strip()
        smartdns['upstreams'][name] = {
            'ip': cfg.get(section, 'ip').split(','),
            'port': cfg.getint(section, 'port'),
            'timeout': cfg.getint(section, 'timeout'),
            'proxy': cfg.getboolean(section, 'proxy'),
            'tcp': cfg.getboolean(section, 'tcp'),
            'priority': cfg.getint(section, 'priority'),
        }
    domains = []
    for name in strip_item(cfg.get('domains', 'domain').split(',')):
        section = 'domain_' + name
        domains.append({
            'name': cfg.get(section, 'name'),
            'url': cfg.get(section, 'url'),
            'proxy': cfg.getboolean(section, 'proxy'),
            'type': cfg.get(section, 'type'),
            'refresh': cfg.getint(section, 'refresh'),
        })
    config = {
        'log': log,
        'server': server,
        'smartdns': smartdns,
        'domains': domains,
    }
    return config
Beispiel #16
0
 def getint(self, section, key, default=0):
     try:
         return ConfigParser.getint(self, section, key)
     except Exception:
         return default
Beispiel #17
0
class MultiPortConfig(object):

    HW_LB = "HW"

    @staticmethod
    def float_x_plus_one_tenth_of_y(x, y):
        return float(x) + float(y) / 10.0

    @staticmethod
    def make_str(base, iterator):
        return ' '.join((base.format(x) for x in iterator))

    @classmethod
    def make_range_str(cls, base, start, stop=0, offset=0):
        if offset and not stop:
            stop = start + offset
        return cls.make_str(base, range(start, stop))

    @staticmethod
    def parser_get(parser, section, key, default=None):
        if parser.has_option(section, key):
            return parser.get(section, key)
        return default

    @staticmethod
    def make_ip_addr(ip, mask_len):
        try:
            return ipaddress.ip_interface(
                six.text_type('/'.join([ip, mask_len])))
        except ValueError:
            # None so we can skip later
            return None

    @classmethod
    def validate_ip_and_prefixlen(cls, ip_addr, prefixlen):
        ip_addr = cls.make_ip_addr(ip_addr, prefixlen)
        return ip_addr.ip.exploded, ip_addr.network.prefixlen

    def __init__(self,
                 topology_file,
                 config_tpl,
                 tmp_file,
                 interfaces=None,
                 vnf_type='CGNAT',
                 lb_count=2,
                 worker_threads=3,
                 worker_config='1C/1T',
                 lb_config='SW',
                 socket=0):

        super(MultiPortConfig, self).__init__()
        self.topology_file = topology_file
        self.worker_config = worker_config.split('/')[1].lower()
        self.worker_threads = self.get_worker_threads(worker_threads)
        self.vnf_type = vnf_type
        self.pipe_line = 0
        self.interfaces = interfaces if interfaces else {}
        self.networks = {}
        self.write_parser = ConfigParser()
        self.read_parser = ConfigParser()
        self.read_parser.read(config_tpl)
        self.master_core = self.read_parser.get("PIPELINE0", "core")
        self.master_tpl = self.get_config_tpl_data('MASTER')
        self.arpicmp_tpl = self.get_config_tpl_data('ARPICMP')
        self.txrx_tpl = self.get_config_tpl_data('TXRX')
        self.loadb_tpl = self.get_config_tpl_data('LOADB')
        self.vnf_tpl = self.get_config_tpl_data(vnf_type)
        self.swq = 0
        self.lb_count = int(lb_count)
        self.lb_config = lb_config
        self.tmp_file = os.path.join("/tmp", tmp_file)
        self.pktq_out_os = []
        self.socket = socket
        self.start_core = ""
        self.pipeline_counter = ""
        self.txrx_pipeline = ""
        self.port_pair_list = []
        self.lb_to_port_pair_mapping = {}
        self.init_eal()

        self.lb_index = None
        self.mul = 0
        self.port_pairs = []
        self.port_pair_list = []
        self.ports_len = 0
        self.prv_que_handler = None
        self.vnfd = None
        self.rules = None
        self.pktq_out = ''

    @staticmethod
    def gen_core(core):
        # return "s{}c{}".format(self.socket, core)
        # don't use sockets for VNFs, because we don't want to have to
        # adjust VM CPU topology.  It is virtual anyway
        return str(core)

    def make_port_pairs_iter(self, operand, iterable):
        return (operand(x[-1], y) for y in iterable
                for x in chain(*self.port_pairs))

    def make_range_port_pairs_iter(self, operand, start, end):
        return self.make_port_pairs_iter(operand, range(start, end))

    def init_eal(self):
        vpci = [v['virtual-interface']["vpci"] for v in self.interfaces]
        with open(self.tmp_file, 'w') as fh:
            fh.write('[EAL]\n')
            for item in vpci:
                fh.write('w = {0}\n'.format(item))
            fh.write('\n')

    def update_timer(self):
        timer_tpl = self.get_config_tpl_data('TIMER')
        timer_tpl['core'] = self.gen_core(self.start_core)
        self.update_write_parser(timer_tpl)
        self.start_core += 1

    def get_config_tpl_data(self, type_value):
        for section in self.read_parser.sections():
            if self.read_parser.has_option(section, 'type'):
                if type_value == self.read_parser.get(section, 'type'):
                    tpl = OrderedDict(self.read_parser.items(section))
                    return tpl

    def get_txrx_tpl_data(self, value):
        for section in self.read_parser.sections():
            if self.read_parser.has_option(section, 'pipeline_txrx_type'):
                if value == self.read_parser.get(section,
                                                 'pipeline_txrx_type'):
                    tpl = OrderedDict(self.read_parser.items(section))
                    return tpl

    def init_write_parser_template(self, type_value='ARPICMP'):
        for section in self.read_parser.sections():
            if type_value == self.parser_get(self.read_parser, section, 'type',
                                             object()):
                self.start_core = self.read_parser.getint(section, 'core')
                self.pipeline_counter = self.read_parser.getint(
                    section, 'core')
                self.txrx_pipeline = self.read_parser.getint(section, 'core')
                return
            self.write_parser.add_section(section)
            for name, value in self.read_parser.items(section):
                self.write_parser.set(section, name, value)

    def update_write_parser(self, data):
        section = "PIPELINE{0}".format(self.pipeline_counter)
        self.write_parser.add_section(section)
        for name, value in data.items():
            self.write_parser.set(section, name, value)

    def get_worker_threads(self, worker_threads):
        if self.worker_config == '1t':
            return worker_threads
        else:
            return worker_threads - worker_threads % 2

    def generate_next_core_id(self):
        if self.worker_config == '1t':
            self.start_core += 1
            return

        try:
            self.start_core = 'h{}'.format(int(self.start_core))
        except ValueError:
            self.start_core = int(self.start_core[:-1]) + 1

    @staticmethod
    def get_port_pairs(interfaces):
        port_pair_list = []
        networks = defaultdict(list)
        for private_intf in interfaces:
            vintf = private_intf['virtual-interface']
            networks[vintf['vld_id']].append(vintf)

        for name, net in networks.items():
            # partition returns a tuple
            parts = list(name.partition('private'))
            if parts[0]:
                # 'private' was not in or not leftmost in the string
                continue
            parts[1] = 'public'
            public_id = ''.join(parts)
            for private_intf in net:
                try:
                    public_peer_intfs = networks[public_id]
                except KeyError:
                    LOG.warning(
                        "private network without peer %s, %s not found", name,
                        public_id)
                    continue

                for public_intf in public_peer_intfs:
                    port_pair = private_intf["ifname"], public_intf["ifname"]
                    port_pair_list.append(port_pair)

        return port_pair_list, networks

    def get_lb_count(self):
        self.lb_count = int(min(len(self.port_pair_list), self.lb_count))

    def generate_lb_to_port_pair_mapping(self):
        self.lb_to_port_pair_mapping = defaultdict(int)
        port_pair_count = len(self.port_pair_list)
        lb_pair_count = int(port_pair_count / self.lb_count)
        for i in range(self.lb_count):
            self.lb_to_port_pair_mapping[i + 1] = lb_pair_count
        for i in range(port_pair_count % self.lb_count):
            self.lb_to_port_pair_mapping[i + 1] += 1

    def set_priv_to_pub_mapping(self):
        return "".join(
            str(y) for y in [(int(x[0][-1]), int(x[1][-1]))
                             for x in self.port_pair_list])

    def set_priv_que_handler(self):
        # iterated twice, can't be generator
        priv_to_pub_map = [(int(x[0][-1]), int(x[1][-1]))
                           for x in self.port_pairs]
        # must be list to use .index()
        port_list = list(chain.from_iterable(priv_to_pub_map))
        priv_ports = (x[0] for x in priv_to_pub_map)
        self.prv_que_handler = '({})'.format(",".join(
            (str(port_list.index(x)) for x in priv_ports)))

    def generate_arp_route_tbl(self):
        arp_config = []
        arp_route_tbl_tmpl = "({port0_dst_ip_hex},{port0_netmask_hex},{port_num}," \
                             "{next_hop_ip_hex})"
        for port_pair in self.port_pair_list:
            for port in port_pair:
                port_num = int(port[-1])
                interface = self.interfaces[port_num]
                # port0_ip = ipaddress.ip_interface(six.text_type(
                #     "%s/%s" % (interface["virtual-interface"]["local_ip"],
                #                interface["virtual-interface"]["netmask"])))
                dst_port0_ip = \
                    ipaddress.ip_interface(six.text_type(
                        "%s/%s" % (interface["virtual-interface"]["dst_ip"],
                                   interface["virtual-interface"]["netmask"])))
                arp_vars = {
                    "port0_dst_ip_hex":
                    ip_to_hex(dst_port0_ip.ip.exploded),
                    "port0_netmask_hex":
                    ip_to_hex(dst_port0_ip.network.netmask.exploded),
                    "port_num":
                    port_num,
                    # next hop is dst in this case
                    "next_hop_ip_hex":
                    ip_to_hex(dst_port0_ip.ip.exploded),
                }
                arp_config.append(arp_route_tbl_tmpl.format(**arp_vars))

        return ' '.join(arp_config)

    def generate_arpicmp_data(self):
        swq_in_str = self.make_range_str('SWQ{}',
                                         self.swq,
                                         offset=self.lb_count)
        self.swq += self.lb_count
        swq_out_str = self.make_range_str('SWQ{}',
                                          self.swq,
                                          offset=self.lb_count)
        self.swq += self.lb_count
        mac_iter = (self.interfaces[int(
            x[-1])]['virtual-interface']['local_mac']
                    for port_pair in self.port_pair_list for x in port_pair)
        pktq_in_iter = ('RXQ{}'.format(float(x[0][-1]))
                        for x in self.port_pair_list)

        arpicmp_data = {
            'core': self.gen_core(self.start_core),
            'pktq_in': swq_in_str,
            'pktq_out': swq_out_str,
            'ports_mac_list': ' '.join(mac_iter),
            'pktq_in_prv': ' '.join(pktq_in_iter),
            'prv_to_pub_map': self.set_priv_to_pub_mapping(),
            'arp_route_tbl': self.generate_arp_route_tbl(),
            # can't use empty string, defaul to ()
            'nd_route_tbl': "()",
        }
        self.pktq_out_os = swq_out_str.split(' ')
        # why?
        if self.lb_config == self.HW_LB:
            arpicmp_data['pktq_in'] = swq_in_str
            self.swq = 0
        return arpicmp_data

    def generate_final_txrx_data(self):
        swq_start = self.swq - self.ports_len * self.worker_threads

        txq_start = 0
        txq_end = self.worker_threads

        pktq_out_iter = self.make_range_port_pairs_iter(
            self.float_x_plus_one_tenth_of_y, txq_start, txq_end)

        swq_str = self.make_range_str('SWQ{}', swq_start, self.swq)
        txq_str = self.make_str('TXQ{}', pktq_out_iter)
        rxtx_data = {
            'pktq_in': swq_str,
            'pktq_out': txq_str,
            'pipeline_txrx_type': 'TXTX',
            'core': self.gen_core(self.start_core),
        }
        pktq_in = rxtx_data['pktq_in']
        pktq_in = '{0} {1}'.format(pktq_in,
                                   self.pktq_out_os[self.lb_index - 1])
        rxtx_data['pktq_in'] = pktq_in
        self.pipeline_counter += 1
        return rxtx_data

    def generate_initial_txrx_data(self):
        pktq_iter = self.make_range_port_pairs_iter(
            self.float_x_plus_one_tenth_of_y, 0, self.worker_threads)

        rxq_str = self.make_str('RXQ{}', pktq_iter)
        swq_str = self.make_range_str('SWQ{}', self.swq, offset=self.ports_len)
        txrx_data = {
            'pktq_in': rxq_str,
            'pktq_out': swq_str + ' SWQ{0}'.format(self.lb_index - 1),
            'pipeline_txrx_type': 'RXRX',
            'core': self.gen_core(self.start_core),
        }
        self.pipeline_counter += 1
        return txrx_data

    def generate_lb_data(self):
        pktq_in = self.make_range_str('SWQ{}', self.swq, offset=self.ports_len)
        self.swq += self.ports_len

        offset = self.ports_len * self.worker_threads
        pktq_out = self.make_range_str('SWQ{}', self.swq, offset=offset)
        self.pktq_out = pktq_out.split()

        self.swq += (self.ports_len * self.worker_threads)
        lb_data = {
            'prv_que_handler': self.prv_que_handler,
            'pktq_in': pktq_in,
            'pktq_out': pktq_out,
            'n_vnf_threads': str(self.worker_threads),
            'core': self.gen_core(self.start_core),
        }
        self.pipeline_counter += 1
        return lb_data

    def generate_vnf_data(self):
        if self.lb_config == self.HW_LB:
            port_iter = self.make_port_pairs_iter(
                self.float_x_plus_one_tenth_of_y, [self.mul])
            pktq_in = self.make_str('RXQ{}', port_iter)

            self.mul += 1
            port_iter = self.make_port_pairs_iter(
                self.float_x_plus_one_tenth_of_y, [self.mul])
            pktq_out = self.make_str('TXQ{}', port_iter)

            pipe_line_data = {
                'pktq_in': pktq_in,
                'pktq_out': pktq_out + ' SWQ{0}'.format(self.swq),
                'prv_que_handler': self.prv_que_handler,
                'core': self.gen_core(self.start_core),
            }
            self.swq += 1
        else:
            pipe_line_data = {
                'pktq_in':
                ' '.join(
                    (self.pktq_out.pop(0) for _ in range(self.ports_len))),
                'pktq_out':
                self.make_range_str('SWQ{}', self.swq, offset=self.ports_len),
                'prv_que_handler':
                self.prv_que_handler,
                'core':
                self.gen_core(self.start_core),
            }
            self.swq += self.ports_len

        if self.vnf_type in ('ACL', 'VFW'):
            pipe_line_data.pop('prv_que_handler')

        if self.vnf_tpl.get('vnf_set'):
            public_ip_port_range_list = self.vnf_tpl[
                'public_ip_port_range'].split(':')
            ip_in_hex = '{:x}'.format(
                int(public_ip_port_range_list[0], 16) + self.lb_index - 1)
            public_ip_port_range_list[0] = ip_in_hex
            self.vnf_tpl['public_ip_port_range'] = ':'.join(
                public_ip_port_range_list)

        self.pipeline_counter += 1
        return pipe_line_data

    def generate_config_data(self):
        self.init_write_parser_template()

        # use master core for master, don't use self.start_core
        self.write_parser.set('PIPELINE0', 'core',
                              self.gen_core(self.master_core))
        arpicmp_data = self.generate_arpicmp_data()
        self.arpicmp_tpl.update(arpicmp_data)
        self.update_write_parser(self.arpicmp_tpl)

        self.start_core += 1
        if self.vnf_type == 'CGNAPT':
            self.pipeline_counter += 1
            self.update_timer()

        for lb in self.lb_to_port_pair_mapping:
            self.lb_index = lb
            self.mul = 0
            port_pair_count = self.lb_to_port_pair_mapping[lb]
            if not self.port_pair_list:
                continue

            self.port_pairs = self.port_pair_list[:port_pair_count]
            self.port_pair_list = self.port_pair_list[port_pair_count:]
            self.ports_len = port_pair_count * 2
            self.set_priv_que_handler()
            if self.lb_config == 'SW':
                txrx_data = self.generate_initial_txrx_data()
                self.txrx_tpl.update(txrx_data)
                self.update_write_parser(self.txrx_tpl)
                self.start_core += 1
                lb_data = self.generate_lb_data()
                self.loadb_tpl.update(lb_data)
                self.update_write_parser(self.loadb_tpl)
                self.start_core += 1

            for i in range(self.worker_threads):
                vnf_data = self.generate_vnf_data()
                if not self.vnf_tpl:
                    self.vnf_tpl = {}
                self.vnf_tpl.update(vnf_data)
                self.update_write_parser(self.vnf_tpl)
                try:
                    self.vnf_tpl.pop('vnf_set')
                except KeyError:
                    pass
                else:
                    self.vnf_tpl.pop('public_ip_port_range')
                self.generate_next_core_id()

            if self.lb_config == 'SW':
                txrx_data = self.generate_final_txrx_data()
                self.txrx_tpl.update(txrx_data)
                self.update_write_parser(self.txrx_tpl)
                self.start_core += 1
            self.vnf_tpl = self.get_config_tpl_data(self.vnf_type)

    def generate_config(self):
        self.port_pair_list, self.networks = self.get_port_pairs(
            self.interfaces)
        self.get_lb_count()
        self.generate_lb_to_port_pair_mapping()
        self.generate_config_data()
        self.write_parser.write(sys.stdout)
        with open(self.tmp_file, 'a') as tfh:
            self.write_parser.write(tfh)

    def generate_link_config(self):

        link_configs = []
        for port_pair in self.port_pair_list:
            for port in port_pair:
                port = port[-1]
                virtual_interface = self.interfaces[int(
                    port)]["virtual-interface"]
                local_ip = virtual_interface["local_ip"]
                netmask = virtual_interface["netmask"]
                port_ip, prefix_len = self.validate_ip_and_prefixlen(
                    local_ip, netmask)
                link_configs.append(
                    LINK_CONFIG_TEMPLATE.format(port, port_ip, prefix_len))

        return ''.join(link_configs)

    def get_route_data(self, src_key, data_key, port):
        route_list = self.vnfd['vdu'][0].get(src_key, [])
        return next((route[data_key]
                     for route in route_list if route['if'] == port), None)

    def get_ports_gateway(self, port):
        return self.get_route_data('routing_table', 'gateway', port)

    def get_ports_gateway6(self, port):
        return self.get_route_data('nd_route_tbl', 'gateway', port)

    def get_netmask_gateway(self, port):
        return self.get_route_data('routing_table', 'netmask', port)

    def get_netmask_gateway6(self, port):
        return self.get_route_data('nd_route_tbl', 'netmask', port)

    def generate_arp_config(self):
        arp_config = []
        for port_pair in self.port_pair_list:
            for port in port_pair:
                gateway = self.get_ports_gateway(port)
                # omit entries with no gateway
                if not gateway:
                    continue
                dst_mac = self.interfaces[int(
                    port[-1])]["virtual-interface"]["dst_mac"]
                arp_config.append(
                    (port[-1], gateway, dst_mac, self.txrx_pipeline))

        return '\n'.join(('p {3} arpadd {0} {1} {2}'.format(*values)
                          for values in arp_config))

    def generate_arp_config6(self):
        arp_config6 = []
        for port_pair in self.port_pair_list:
            for port in port_pair:
                gateway6 = self.get_ports_gateway6(port)
                # omit entries with no gateway
                if not gateway6:
                    continue
                dst_mac6 = self.interfaces[int(
                    port[-1])]["virtual-interface"]["dst_mac"]
                arp_config6.append(
                    (port[-1], gateway6, dst_mac6, self.txrx_pipeline))

        return '\n'.join(('p {3} arpadd {0} {1} {2}'.format(*values)
                          for values in arp_config6))

    def generate_action_config(self):
        port_list = []
        for port_pair in self.port_pair_list:
            for port in port_pair:
                port_list.append(port[-1])

        if self.vnf_type == "VFW":
            template = FW_ACTION_TEMPLATE
        else:
            template = ACTION_TEMPLATE

        return ''.join((template.format(port) for port in port_list))

    def get_ip_from_port(self, port):
        return self.make_ip_addr(self.get_ports_gateway(port),
                                 self.get_netmask_gateway(port))

    def get_ip_and_prefixlen_from_ip_of_port(self, port):
        ip_addr = self.get_ip_from_port(port)
        # handle cases with no gateway
        if ip_addr:
            return ip_addr.ip.exploded, ip_addr.network.prefixlen
        else:
            return None, None

    def generate_rule_config(self):
        cmd = 'acl' if self.vnf_type == "ACL" else "vfw"
        rules_config = self.rules if self.rules else ''
        new_rules = []
        new_ipv6_rules = []
        pattern = 'p {0} add {1} {2} {3} {4} {5} 0 65535 0 65535 0 0 {6}'
        for port_pair in self.port_pair_list:
            src_port = int(port_pair[0][-1])
            dst_port = int(port_pair[1][-1])

            src_ip, src_prefix_len = self.get_ip_and_prefixlen_from_ip_of_port(
                port_pair[0])
            dst_ip, dst_prefix_len = self.get_ip_and_prefixlen_from_ip_of_port(
                port_pair[1])
            # ignore entires with empty values
            if all((src_ip, src_prefix_len, dst_ip, dst_prefix_len)):
                new_rules.append(
                    (cmd, self.txrx_pipeline, src_ip, src_prefix_len, dst_ip,
                     dst_prefix_len, dst_port))
                new_rules.append(
                    (cmd, self.txrx_pipeline, dst_ip, dst_prefix_len, src_ip,
                     src_prefix_len, src_port))

            src_ip = self.get_ports_gateway6(port_pair[0])
            src_prefix_len = self.get_netmask_gateway6(port_pair[0])
            dst_ip = self.get_ports_gateway6(port_pair[1])
            dst_prefix_len = self.get_netmask_gateway6(port_pair[0])
            # ignore entires with empty values
            if all((src_ip, src_prefix_len, dst_ip, dst_prefix_len)):
                new_ipv6_rules.append(
                    (cmd, self.txrx_pipeline, src_ip, src_prefix_len, dst_ip,
                     dst_prefix_len, dst_port))
                new_ipv6_rules.append(
                    (cmd, self.txrx_pipeline, dst_ip, dst_prefix_len, src_ip,
                     src_prefix_len, src_port))

        acl_apply = "\np %s applyruleset" % cmd
        new_rules_config = '\n'.join(
            pattern.format(*values)
            for values in chain(new_rules, new_ipv6_rules))
        return ''.join([rules_config, new_rules_config, acl_apply])

    def generate_script_data(self):
        self.port_pair_list, self.networks = self.get_port_pairs(
            self.interfaces)
        self.get_lb_count()
        script_data = {
            'link_config': self.generate_link_config(),
            'arp_config': self.generate_arp_config(),
            'arp_config6': self.generate_arp_config6(),
            'actions': '',
            'rules': '',
        }

        if self.vnf_type in ('ACL', 'VFW'):
            script_data.update({
                'actions': self.generate_action_config(),
                'rules': self.generate_rule_config(),
            })

        return script_data

    def generate_script(self, vnfd, rules=None):
        self.vnfd = vnfd
        self.rules = rules
        script_data = self.generate_script_data()
        script = SCRIPT_TPL.format(**script_data)
        if self.lb_config == self.HW_LB:
            script += 'set fwd rxonly'
            hwlb_tpl = """
set_sym_hash_ena_per_port {0} enable
set_hash_global_config {0} simple_xor ipv4-udp enable
set_sym_hash_ena_per_port {1} enable
set_hash_global_config {1} simple_xor ipv4-udp enable
set_hash_input_set {0} ipv4-udp src-ipv4 udp-src-port add
set_hash_input_set {1} ipv4-udp dst-ipv4 udp-dst-port add
set_hash_input_set {0} ipv6-udp src-ipv6 udp-src-port add
set_hash_input_set {1} ipv6-udp dst-ipv6 udp-dst-port add
"""
            for port_pair in self.port_pair_list:
                script += hwlb_tpl.format(port_pair[0][-1], port_pair[1][-1])
        return script
Beispiel #18
0
 def getint(self, section, key, default=0):
     try:
         return ConfigParser.getint(self, section, key)
     except Exception:
         return default
Beispiel #19
0
def loadini(struct, configfile):
    """Loads .ini configuration file and stores its values in struct"""

    config_path = os.path.expanduser(configfile)

    config = ConfigParser()
    defaults = {
        'general': {
            'arg_spec': True,
            'auto_display_list': True,
            'color_scheme': 'default',
            'complete_magic_methods': True,
            'autocomplete_mode': default_completion,
            'dedent_after': 1,
            'flush_output': True,
            'highlight_show_source': True,
            'hist_file': '~/.pythonhist',
            'hist_length': 100,
            'hist_duplicates': True,
            'paste_time': 0.02,
            'single_undo_time': 1.0,
            'syntax': True,
            'tab_length': 4,
            'pastebin_confirm': True,
            'pastebin_url': 'https://bpaste.net/json/new',
            'pastebin_show_url': 'https://bpaste.net/show/$paste_id',
            'pastebin_removal_url': 'https://bpaste.net/remove/$removal_id',
            'pastebin_expiry': '1week',
            'pastebin_helper': '',
            'save_append_py': False,
            'editor': os.environ.get('VISUAL', os.environ.get('EDITOR', 'vi')),
            'unicode_box': True
        },
        'keyboard': {
            'backspace': 'C-h',
            'left': 'C-b',
            'right': 'C-f',
            'beginning_of_line': 'C-a',
            'end_of_line': 'C-e',
            'transpose_chars': 'C-t',
            'clear_line': 'C-u',
            'clear_screen': 'C-l',
            'clear_word': 'C-w',
            'cut_to_buffer': 'C-k',
            'delete': 'C-d',
            'down_one_line': 'C-n',
            'exit': '',
            'external_editor': 'F7',
            'edit_config': 'F3',
            'edit_current_block': 'C-x',
            'help': 'F1',
            'last_output': 'F9',
            'copy_clipboard': 'F10',
            'pastebin': 'F8',
            'save': 'C-s',
            'show_source': 'F2',
            'suspend': 'C-z',
            'toggle_file_watch': 'F5',
            'undo': 'C-r',
            'reimport': 'F6',
            'search': 'C-o',
            'up_one_line': 'C-p',
            'yank_from_buffer': 'C-y'},
        'cli': {
            'suggestion_width': 0.8,
            'trim_prompts': False,
        },
        'curtsies': {
            'list_above': False,
            'right_arrow_completion': True,
        }}

    default_keys_to_commands = dict((value, key) for (key, value)
                                    in iteritems(defaults['keyboard']))

    fill_config_with_default_values(config, defaults)
    if not config.read(config_path):
        # No config file. If the user has it in the old place then complain
        if os.path.isfile(os.path.expanduser('~/.bpython.ini')):
            sys.stderr.write("Error: It seems that you have a config file at "
                             "~/.bpython.ini. Please move your config file to "
                             "%s\n" % default_config_path())
            sys.exit(1)

    def get_key_no_doublebind(command):
        default_commands_to_keys = defaults['keyboard']
        requested_key = config.get('keyboard', command)

        try:
            default_command = default_keys_to_commands[requested_key]

            if (default_commands_to_keys[default_command] ==
                    config.get('keyboard', default_command)):
                setattr(struct, '%s_key' % default_command, '')
        except KeyError:
            pass

        return requested_key

    struct.config_path = config_path

    struct.dedent_after = config.getint('general', 'dedent_after')
    struct.tab_length = config.getint('general', 'tab_length')
    struct.auto_display_list = config.getboolean('general',
                                                 'auto_display_list')
    struct.syntax = config.getboolean('general', 'syntax')
    struct.arg_spec = config.getboolean('general', 'arg_spec')
    struct.paste_time = config.getfloat('general', 'paste_time')
    struct.single_undo_time = config.getfloat('general', 'single_undo_time')
    struct.highlight_show_source = config.getboolean('general',
                                                     'highlight_show_source')
    struct.hist_file = config.get('general', 'hist_file')
    struct.editor = config.get('general', 'editor')
    struct.hist_length = config.getint('general', 'hist_length')
    struct.hist_duplicates = config.getboolean('general', 'hist_duplicates')
    struct.flush_output = config.getboolean('general', 'flush_output')

    struct.pastebin_key = get_key_no_doublebind('pastebin')
    struct.copy_clipboard_key = get_key_no_doublebind('copy_clipboard')
    struct.save_key = get_key_no_doublebind('save')
    struct.search_key = get_key_no_doublebind('search')
    struct.show_source_key = get_key_no_doublebind('show_source')
    struct.suspend_key = get_key_no_doublebind('suspend')
    struct.toggle_file_watch_key = get_key_no_doublebind('toggle_file_watch')
    struct.undo_key = get_key_no_doublebind('undo')
    struct.reimport_key = get_key_no_doublebind('reimport')
    struct.up_one_line_key = get_key_no_doublebind('up_one_line')
    struct.down_one_line_key = get_key_no_doublebind('down_one_line')
    struct.cut_to_buffer_key = get_key_no_doublebind('cut_to_buffer')
    struct.yank_from_buffer_key = get_key_no_doublebind('yank_from_buffer')
    struct.clear_word_key = get_key_no_doublebind('clear_word')
    struct.backspace_key = get_key_no_doublebind('backspace')
    struct.clear_line_key = get_key_no_doublebind('clear_line')
    struct.clear_screen_key = get_key_no_doublebind('clear_screen')
    struct.delete_key = get_key_no_doublebind('delete')

    struct.left_key = get_key_no_doublebind('left')
    struct.right_key = get_key_no_doublebind('right')
    struct.end_of_line_key = get_key_no_doublebind('end_of_line')
    struct.beginning_of_line_key = get_key_no_doublebind('beginning_of_line')
    struct.transpose_chars_key = get_key_no_doublebind('transpose_chars')
    struct.clear_line_key = get_key_no_doublebind('clear_line')
    struct.clear_screen_key = get_key_no_doublebind('clear_screen')
    struct.exit_key = get_key_no_doublebind('exit')
    struct.last_output_key = get_key_no_doublebind('last_output')
    struct.edit_config_key = get_key_no_doublebind('edit_config')
    struct.edit_current_block_key = get_key_no_doublebind('edit_current_block')
    struct.external_editor_key = get_key_no_doublebind('external_editor')
    struct.help_key = get_key_no_doublebind('help')

    struct.pastebin_confirm = config.getboolean('general', 'pastebin_confirm')
    struct.pastebin_url = config.get('general', 'pastebin_url')
    struct.pastebin_show_url = config.get('general', 'pastebin_show_url')
    struct.pastebin_removal_url = config.get('general', 'pastebin_removal_url')
    struct.pastebin_expiry = config.get('general', 'pastebin_expiry')
    struct.pastebin_helper = config.get('general', 'pastebin_helper')

    struct.cli_suggestion_width = config.getfloat('cli',
                                                  'suggestion_width')
    struct.cli_trim_prompts = config.getboolean('cli',
                                                'trim_prompts')

    struct.complete_magic_methods = config.getboolean('general',
                                                      'complete_magic_methods')
    struct.autocomplete_mode = config.get('general', 'autocomplete_mode')
    struct.save_append_py = config.getboolean('general', 'save_append_py')

    struct.curtsies_list_above = config.getboolean('curtsies', 'list_above')
    struct.curtsies_right_arrow_completion = \
        config.getboolean('curtsies', 'right_arrow_completion')

    color_scheme_name = config.get('general', 'color_scheme')

    default_colors = {
        'keyword': 'y',
        'name': 'c',
        'comment': 'b',
        'string': 'm',
        'error': 'r',
        'number': 'G',
        'operator': 'Y',
        'punctuation': 'y',
        'token': 'C',
        'background': 'd',
        'output': 'w',
        'main': 'c',
        'paren': 'R',
        'prompt': 'c',
        'prompt_more': 'g',
        'right_arrow_suggestion': 'K',
    }

    if color_scheme_name == 'default':
        struct.color_scheme = default_colors
    else:
        struct.color_scheme = dict()

        theme_filename = color_scheme_name + '.theme'
        path = os.path.expanduser(os.path.join(get_config_home(),
                                               theme_filename))
        try:
            load_theme(struct, path, struct.color_scheme, default_colors)
        except EnvironmentError:
            sys.stderr.write("Could not load theme '%s'.\n" %
                             (color_scheme_name, ))
            sys.exit(1)

    # checks for valid key configuration this part still sucks
    for key in (struct.pastebin_key, struct.save_key):
        key_dispatch[key]

    # expand path of history file
    struct.hist_file = os.path.expanduser(struct.hist_file)

    # verify completion mode
    if struct.autocomplete_mode not in bpython.autocomplete.ALL_MODES:
        struct.autocomplete_mode = default_completion

    # set box drawing characters
    if config.getboolean('general', 'unicode_box') and supports_box_chars():
        struct.left_border = u'│'
        struct.right_border = u'│'
        struct.top_border = u'─'
        struct.bottom_border = u'─'
        struct.left_bottom_corner = u'└'
        struct.right_bottom_corner = u'┘'
        struct.left_top_corner = u'┌'
        struct.right_top_corner = u'┐'
    else:
        struct.left_border = u'|'
        struct.right_border = u'|'
        struct.top_border = u'-'
        struct.bottom_border = u'-'
        struct.left_bottom_corner = u'+'
        struct.right_bottom_corner = u'+'
        struct.left_top_corner = u'+'
        struct.right_top_corner = u'+'
Beispiel #20
0
class PypiConfig(BaseConfig):
    """Wrapper around the pypi config file"""

    def __init__(self, config_filename=DIST_CONFIG_FILE, use_setup_cfg=True):
        """Grab the PyPI configuration.

        This is .pypirc in the home directory.  It is overridable for
        test purposes.

        If there is a setup.cfg file in the current directory, we read
        it too.
        """
        self.config_filename = config_filename
        self.use_setup_cfg = use_setup_cfg
        self.reload()

    def reload(self):
        """Load the config.

        Do the initial load of the config.

        Or reload it in case of problems: this is needed when a pypi
        upload fails, you edit the .pypirc file to fix the account
        settings, and tell release to retry the command.
        """
        self._read_configfile(use_setup_cfg=self.use_setup_cfg)

    def _read_configfile(self, use_setup_cfg=True):
        """Read the PyPI config file and store it (when valid).

        Usually read the setup.cfg too.
        """
        rc = self.config_filename
        if not os.path.isabs(rc):
            rc = os.path.join(os.path.expanduser('~'), self.config_filename)
        filenames = [rc]
        if use_setup_cfg:
            # If there is a setup.cfg in the package, parse it
            filenames.append('setup.cfg')
        files = [f for f in filenames if os.path.exists(f)]
        if not files:
            self.config = None
            return
        self.config = ConfigParser()
        self.config.read(files)

    def is_pypi_configured(self):
        # Do we have configuration for releasing to at least one
        # pypi-compatible server?
        if self.config is None:
            return False
        return len(self.distutils_servers()) > 0

    def distutils_servers(self):
        """Return a list of known distutils servers.

        If the config has an old pypi config, remove the default pypi
        server from the list.
        """
        try:
            index_servers = self._get_text(
                'distutils', 'index-servers', default='').split()
        except (NoSectionError, NoOptionError):
            index_servers = []
        if not index_servers:
            # If no distutils index-servers have been given, 'pypi' should be
            # the default.  This is what twine does.
            if self.config.has_option('server-login', 'username'):
                # We have a username, so upload to pypi should work fine, even
                # when no explicit pypi section is in the file.
                return ['pypi']
            # https://github.com/zestsoftware/zest.releaser/issues/199
            index_servers = ['pypi']
        # The servers all need to have a section in the config file.
        return [server for server in index_servers
                if self.config.has_section(server)]

    def want_release(self):
        """Does the user normally want to release this package.

        Some colleagues find it irritating to have to remember to
        answer the question "Check out the tag (for tweaks or
        pypi/distutils server upload)" with the non-default 'no' when
        in 99 percent of the cases they just make a release specific
        for a customer, so they always answer 'no' here.  This is
        where an extra config option comes in handy: you can influence
        the default answer so you can just keep hitting 'Enter' until
        zest.releaser is done.

        Either in your ~/.pypirc or in a setup.cfg in a specific
        package, add this when you want the default answer to this
        question to be 'no':

        [zest.releaser]
        release = no

        The default when this option has not been set is True.

        Standard config rules apply, so you can use upper or lower or
        mixed case and specify 0, false, no or off for boolean False,
        and 1, on, true or yes for boolean True.
        """
        return self._get_boolean('zest.releaser', 'release', default=True)

    def extra_message(self):
        """Return extra text to be added to commit messages.

        This can for example be used to skip CI builds.  This at least
        works for Travis.  See
        http://docs.travis-ci.com/user/how-to-skip-a-build/

        Enable this mode by adding a ``extra-message`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            extra-message = [ci skip]
        """
        default = ''
        if self.config is None:
            return default
        try:
            result = self._get_text(
                'zest.releaser', 'extra-message', default=default)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def history_file(self):
        """Return path of history file.

        Usually zest.releaser can find the correct one on its own.
        But sometimes it may not find anything, or it finds multiple
        and selects the wrong one.

        Configure this by adding a ``history-file`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            history-file = deep/down/historie.doc
        """
        default = ''
        if self.config is None:
            return default
        marker = object()
        try:
            result = self._get_text(
                'zest.releaser', 'history-file', default=marker)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        if result == marker:
            # We were reading an underscore instead of a dash at first.
            try:
                result = self._get_text(
                    'zest.releaser', 'history_file', default=default)
            except (NoSectionError, NoOptionError, ValueError):
                return default
        return result

    def encoding(self):
        """Return encoding to use for text files.

        Mostly the changelog and if needed `setup.py`.

        The encoding cannot always be determined correctly.
        This setting is a way to fix that.
        See https://github.com/zestsoftware/zest.releaser/issues/264

        Configure this by adding an ``encoding`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            encoding = utf-8
        """
        default = ''
        if self.config is None:
            return default
        try:
            result = self._get_text(
                'zest.releaser', 'encoding', default=default, raw=True)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def create_wheel(self):
        """Should we create a Python wheel for this package?

        Either in your ~/.pypirc or in a setup.cfg in a specific
        package, add this when you want to create a Python wheel, next
        to a standard sdist:

        [zest.releaser]
        create-wheel = yes

        If there is no setting for ``create-wheel``, then if there is a
        ``[bdist_wheel]`` section, it is treated as if
        ``create-wheel`` was true.  We used to look at the value of
        the ``universal`` option, but that no longer matters.
        This will still create a wheel:

        [bdist_wheel]
        universal = 0

        See https://github.com/zestsoftware/zest.releaser/issues/315
        """
        if not USE_WHEEL:
            # If the wheel package is not available, we obviously
            # cannot create wheels.
            return False
        create_setting = self._get_boolean(
            'zest.releaser', 'create-wheel', None)
        if create_setting is not None:
            # User specified this setting, it overrides
            # inferring from bdist_wheel
            return create_setting
        # No zest.releaser setting, are they asking for a universal wheel?
        # Then they want wheels in general.
        return self.config.has_section('bdist_wheel')

    def register_package(self):
        """Should we try to register this package with a package server?

        For the standard Python Package Index (PyPI), registering a
        package is no longer needed: this is done automatically when
        uploading a distribution for a package.  In fact, trying to
        register may fail.  See
        https://github.com/zestsoftware/zest.releaser/issues/191
        So by default zest.releaser will no longer register a package.

        But you may be using your own package server, and registering
        may be wanted or even required there.  In this case
        you will need to turn on the register function.
        In your setup.cfg or ~/.pypirc, use the following to ensure that
        register is called on the package server:

        [zest.releaser]
        register = yes

        Note that if you have specified multiple package servers, this
        option is used for all of them.  There is no way to register and
        upload to server A, and only upload to server B.
        """
        return self._get_boolean('zest.releaser', 'register')

    def no_input(self):
        """Return whether the user wants to run in no-input mode.

        Enable this mode by adding a ``no-input`` option::

            [zest.releaser]
            no-input = yes

        The default when this option has not been set is False.
        """
        return self._get_boolean('zest.releaser', 'no-input')

    def development_marker(self):
        """Return development marker to be appended in postrelease.

        Override the default ``.dev0`` in ~/.pypirc or setup.cfg using
        a ``development-marker`` option::

            [zest.releaser]
            development-marker = .dev1

        Returns default of ``.dev0`` when nothing has been configured.
        """
        default = '.dev0'
        if self.config is None:
            return default
        try:
            result = self._get_text(
                'zest.releaser', 'development-marker', default=default)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def push_changes(self):
        """Return whether the user wants to push the changes to the remote.

        Configure this mode by adding a ``push-changes`` option::

            [zest.releaser]
            push-changes = no

        The default when this option has not been set is True.
        """
        return self._get_boolean('zest.releaser', 'push-changes', default=True)

    def less_zeroes(self):
        """Return whether the user prefers less zeroes at the end of a version.

        Configure this mode by adding a ``less-zeroes`` option::

            [zest.releaser]
            less-zeroes = yes

        The default when this option has not been set is False.

        When set to true:
        - Instead of 1.3.0 we will suggest 1.3.
        - Instead of 2.0.0 we will suggest 2.0.

        This only makes sense for the bumpversion command.
        In the postrelease command we read this option too,
        but with the current logic it has no effect there.
        """
        return self._get_boolean('zest.releaser', 'less-zeroes')

    def version_levels(self):
        """How many levels does the user prefer in a version number?

        Configure this mode by adding a ``version-levels`` option::

            [zest.releaser]
            version-levels = 3

        The default when this option has not been set is 0, which means:
        no preference, so use the length of the current number.

        This means when suggesting a next version after 1.2:
        - with levels=0 we will suggest 1.3: no change
        - with levels=1 we will still suggest 1.3, as we will not
          use this to remove numbers, only to add them
        - with levels=2 we will suggest 1.3
        - with levels=3 we will suggest 1.2.1

        If the current version number has more levels, we keep them.
        So next version for 1.2.3.4 with levels=1 is 1.2.3.5.

        Tweaking version-levels and less-zeroes should give you the
        version number strategy that you prefer.
        """
        default = 0
        if self.config is None:
            return default
        try:
            result = self.config.getint('zest.releaser', 'version-levels')
        except (NoSectionError, NoOptionError, ValueError):
            return default
        if result < 0:
            return default
        return result

    _tag_format_deprecated_message = "\n".join(line.strip() for line in """
    `tag-format` contains deprecated `%%(version)s` format. Please change to:

    [zest.releaser]
    tag-format = %s
    """.strip().splitlines())

    def tag_format(self, version):
        """Return the formatted tag that should be used in the release.

        Configure it in ~/.pypirc or setup.cfg using a ``tag-format`` option::

            [zest.releaser]
            tag-format = v{version}

        ``tag-format`` must contain exaclty one formatting instruction: for the
        ``version`` key.

        Accepts also ``%(version)s`` format for backward compatibility.

        The default format, when nothing has been configured, is ``{version}``.
        """
        fmt = '{version}'
        if self.config is not None:
            try:
                fmt = self._get_text(
                    'zest.releaser', 'tag-format', default=fmt, raw=True)
            except (NoSectionError, NoOptionError, ValueError):
                pass
        if '{version}' in fmt:
            return fmt.format(version=version)
        # BBB:
        if '%(version)s' in fmt:
            proposed_fmt = fmt.replace("%(version)s", "{version}")
            print(self._tag_format_deprecated_message % proposed_fmt)
            return fmt % {'version': version}
        print("{version} needs to be part of 'tag-format': %s" % fmt)
        sys.exit(1)

    def tag_message(self, version):
        """Return the commit message to be used when tagging.

        Configure it in ~/.pypirc or setup.cfg using a ``tag-message``
        option::

            [zest.releaser]
            tag-message = Creating v{version} tag.

        ``tag-message`` must contain exaclty one formatting
        instruction: for the ``version`` key.

        The default format is ``Tagging {version}``.
        """
        fmt = 'Tagging {version}'
        if self.config:
            try:
                fmt = self._get_text(
                    'zest.releaser', 'tag-message', default=fmt, raw=True)
            except (NoSectionError, NoOptionError, ValueError):
                pass
        if '{version}' not in fmt:
            print("{version} needs to be part of 'tag-message': '%s'" % fmt)
            sys.exit(1)
        return fmt.format(version=version)

    def tag_signing(self):
        """Return whether the tag should be signed.

        Configure it in ~/.pypirc or setup.cfg using a ``tag-signing`` option::

            [zest.releaser]
            tag-signing = yes

        ``tag-signing`` must contain exaclty one word which will be
        converted to a boolean. Currently are accepted (case
        insensitively): 0, false, no, off for False, and 1, true, yes,
        on for True).

        The default when this option has not been set is False.

        """
        return self._get_boolean('zest.releaser', 'tag-signing', default=False)

    def date_format(self):
        """Return the string format for the date used in the changelog.

        Override the default ``%Y-%m-%d`` in ~/.pypirc or setup.cfg using
        a ``date-format`` option::

            [zest.releaser]
            date-format = %%B %%e, %%Y

        Note: the % signs should be doubled for compatibility with other tools
        (i.e. pip) that parse setup.cfg using the interpolating ConfigParser.

        Returns default of ``%Y-%m-%d`` when nothing has been configured.
        """
        default = '%Y-%m-%d'
        if self.config is None:
            return default
        try:
            result = self._get_text(
                'zest.releaser', 'date-format', default=default
            ).replace('%%', '%')
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result