Example #1
0
def _load_ec_as_default_policy(proxy_conf_file, swift_conf_file, **kwargs):
    """
    Override swift.conf [storage-policy:0] section to use a 2+1 EC policy.

    :param proxy_conf_file: Source proxy conf filename
    :param swift_conf_file: Source swift conf filename
    :returns: Tuple of paths to the proxy conf file and swift conf file to use
    """
    _debug('Setting configuration for default EC policy')

    conf = ConfigParser()
    conf.read(swift_conf_file)
    # remove existing policy sections that came with swift.conf-sample
    for section in list(conf.sections()):
        if section.startswith('storage-policy'):
            conf.remove_section(section)
    # add new policy 0 section for an EC policy
    conf.add_section('storage-policy:0')
    ec_policy_spec = {
        'name': 'ec-test',
        'policy_type': 'erasure_coding',
        'ec_type': 'liberasurecode_rs_vand',
        'ec_num_data_fragments': 2,
        'ec_num_parity_fragments': 1,
        'ec_object_segment_size': 1048576,
        'default': True
    }

    for k, v in ec_policy_spec.items():
        conf.set('storage-policy:0', k, str(v))

    with open(swift_conf_file, 'w') as fp:
        conf.write(fp)
    return proxy_conf_file, swift_conf_file
Example #2
0
def load_config(config_file=_CONFIG_FILE):
    cfg_parser = ConfigParser()
    if not os.path.isfile(config_file):
        raise Exception(
            'configuration file not found: {0}'.format(config_file))
    cfg_parser.read(config_file)
    _cfg.load_config(cfg_parser)
Example #3
0
def get_config(options):
    S3CFG = os.getenv("S3CFG", None)
    if options.config:
        # Command-line overrides everything
        cfg = options.config
    elif S3CFG is not None:
        # Environment variable overrides defaults
        cfg = S3CFG
    else:
        # New default
        new_default = os.path.expanduser("~/.pegasus/s3cfg")
        if os.path.isfile(new_default):
            cfg = new_default
        else:
            # If the new default doesn't exist, try the old default
            cfg = os.path.expanduser("~/.s3cfg")

    if not os.path.isfile(cfg):
        raise Exception("Config file not found")

    debug("Found config file: %s" % cfg)

    # Make sure nobody else can read the file
    mode = os.stat(cfg).st_mode
    if mode & (stat.S_IRWXG | stat.S_IRWXO):
        raise Exception("Permissions of config file %s are too liberal" % cfg)

    config = ConfigParser(DEFAULT_CONFIG)
    config.read(cfg)

    return config
Example #4
0
File: copy.py Project: clayg/swift
    def _load_object_post_as_copy_conf(self, conf):
        if ('object_post_as_copy' in conf or '__file__' not in conf):
            # Option is explicitly set in middleware conf. In that case,
            # we assume operator knows what he's doing.
            # This takes preference over the one set in proxy app
            return

        cp = ConfigParser()
        if os.path.isdir(conf['__file__']):
            read_conf_dir(cp, conf['__file__'])
        else:
            cp.read(conf['__file__'])

        try:
            pipe = cp.get("pipeline:main", "pipeline")
        except (NoSectionError, NoOptionError):
            return

        proxy_name = pipe.rsplit(None, 1)[-1]
        proxy_section = "app:" + proxy_name

        try:
            conf['object_post_as_copy'] = cp.get(proxy_section,
                                                 'object_post_as_copy')
        except (NoSectionError, NoOptionError):
            pass
Example #5
0
File: shell.py Project: nzlosh/st2
def get_stackstorm_version():
    """
    Return StackStorm version including git commit revision if running a dev release and a file
    with package metadata which includes git revision is available.

    :rtype: ``str``
    """
    if 'dev' in __version__:
        version = __version__

        if not os.path.isfile(PACKAGE_METADATA_FILE_PATH):
            return version

        config = ConfigParser()

        try:
            config.read(PACKAGE_METADATA_FILE_PATH)
        except Exception:
            return version

        try:
            git_revision = config.get('server', 'git_sha')
        except Exception:
            return version

        version = '%s (%s)' % (version, git_revision)
    else:
        version = __version__

    return version
Example #6
0
    def loadConfigs(self):
        """Entry point to load the l10n.ini file this Parser refers to.

        This implementation uses synchronous loads, subclasses might overload
        this behaviour. If you do, make sure to pass a file-like object
        to onLoadConfig.
        """
        cp = ConfigParser(self.defaults)
        cp.read(self.inipath)
        depth = self.getDepth(cp)
        self.base = mozpath.join(mozpath.dirname(self.inipath), depth)
        # create child loaders for any other l10n.ini files to be included
        try:
            for title, path in cp.items('includes'):
                # skip default items
                if title in self.defaults:
                    continue
                # add child config parser
                self.addChild(title, path, cp)
        except NoSectionError:
            pass
        # try to load the "dirs" defined in the "compare" section
        try:
            self.dirs.extend(cp.get('compare', 'dirs').split())
        except (NoOptionError, NoSectionError):
            pass
        # try to set "all_path" and "all_url"
        try:
            self.all_path = mozpath.join(self.base, cp.get('general', 'all'))
        except (NoOptionError, NoSectionError):
            self.all_path = None
        return cp
Example #7
0
    def restore_rois(self, roifile):
        """restore ROI setting from ROI.dat file"""
        cp =  ConfigParser()
        cp.read(roifile)
        rois = []
        self.mcas[0].clear_rois()
        prefix = self.mcas[0]._prefix
        if prefix.endswith('.'):
            prefix = prefix[:-1]
        iroi = 0
        for a in cp.options('rois'):
            if a.lower().startswith('roi'):
                name, dat = cp.get('rois', a).split('|')
                lims = [int(i) for i in dat.split()]
                lo, hi = lims[0], lims[1]
                # print('ROI ', name, lo, hi)
                roi = ROI(prefix=prefix, roi=iroi)
                roi.LO = lo
                roi.HI = hi
                roi.NM = name.strip()
                rois.append(roi)
                iroi += 1

        poll(0.050, 1.0)
        self.mcas[0].set_rois(rois)
        cal0 = self.mcas[0].get_calib()
        for mca in self.mcas[1:]:
            mca.set_rois(rois, calib=cal0)
Example #8
0
    def setUp(self):
        rid = '60754-10'
        config = ConfigParser()
        p = '/Users/ross/Sandbox/pychron_validation_data.cfg'
        config.read(p)

        signals = [list(map(float, x.split(','))) for x in [config.get('Signals-{}'.format(rid), k)
                        for k in ['ar40', 'ar39', 'ar38', 'ar37', 'ar36']]]

        blanks = [list(map(float, x.split(','))) for x in [config.get('Blanks-{}'.format(rid), k)
                        for k in ['ar40', 'ar39', 'ar38', 'ar37', 'ar36']]]

        irradinfo = [list(map(float, x.split(','))) for x in [config.get('irrad-{}'.format(rid), k) for k in ['k4039', 'k3839', 'ca3937', 'ca3837', 'ca3637', 'cl3638']]]

        j = config.get('irrad-{}'.format(rid), 'j')
        j = [float(x) for x in j.split(',')]
        baselines = [(0, 0), (0, 0), (0, 0), (0, 0), (0, 0)]
        backgrounds = [(0, 0), (0, 0), (0, 0), (0, 0), (0, 0)]

        ar37df = config.getfloat('irrad-{}'.format(rid), 'ar37df')
        t = math.log(ar37df) / (constants.lambda_37.nominal_value * 365.25)
        irradinfo.append(t)

        # load results
        r = 'results-{}'.format(rid)
        self.age = config.getfloat(r, 'age')
        self.rad4039 = config.getfloat(r, 'rad4039')
        self.ca37k39 = config.getfloat(r, 'ca37k39')


        self.age_dict = calculate_arar_age(signals, baselines, blanks, backgrounds, j, irradinfo,
                               )
Example #9
0
def get_configuration(configuration=None):
    """
    Parse a configuration file

    Parameters
    ----------
    configuration : str or list, optional
        A configuration file or list of configuration files to parse,
        defaults to the deploy_default.conf file in the package and
        deploy.conf in the current working directory.

    Returns
    -------
    configparser
        The parsed configuration
    """
    if not configuration:  # pragma: no cover
        configuration = [
            # Config file that is part of the package
            # PACKAGE_DEFAULT_CONFIG,

            # Any deploy.conf files in the current directory
            'deploy.conf'
        ]
    config = ConfigParser()

    # Set the config defaults
    try:
        config.read_string(config_defaults())
    except AttributeError:
        config.readfp(io.BytesIO(config_defaults()))

    logger.debug('Working with default dict: %r', config_defaults())
    config.read(configuration)
    return config
Example #10
0
def pytest_collect_file(path, parent):
    """Handle running pylint on files discovered"""
    config = parent.config
    if not config.option.pylint:
        return
    if not path.ext == ".py":
        return
    # Find pylintrc to check ignore list
    pylintrc_file = config.option.pylint_rcfile or PYLINTRC
    # No pylintrc, therefore no ignores, so return the item.
    if not pylintrc_file or not exists(pylintrc_file):
        return PyLintItem(path, parent)

    pylintrc = ConfigParser()
    pylintrc.read(pylintrc_file)
    ignore_list = []
    try:
        ignore_string = pylintrc.get('MASTER', 'ignore')
        if len(ignore_string) > 0:
            ignore_list = ignore_string.split(',')
    except (NoSectionError, NoOptionError):
        pass
    rel_path = path.strpath.replace(parent.fspath.strpath, '', 1)[1:]
    if not any(basename in rel_path for basename in ignore_list):
        return PyLintItem(path, parent)
Example #11
0
def loadKeysConfig(path=None):
	"""Load keys config file.

	If path is ``None``, a file named :any:`DEFAULT_KEYS_FILE` will be looked for in the config
	directory.

	:param path: path of the keyboard configuration file
	"""

	if path is None:
		path = getConfigFilePath(DEFAULT_KEYS_FILE)

	cfg = ConfigParser()
	cfg.optionxform = str
	cfg.read([path])

	for category in cfg.sections():
		for actionName in cfg.options(category):
			keystr = cfg.get(category, actionName)

			context = Qt.WidgetShortcut
			if keystr.startswith('widget:'):
				keystr = keystr.split(':', 1)[1]
			elif keystr.startswith('window:'):
				keystr = keystr.split(':', 1)[1]
				context = Qt.WindowShortcut
			elif keystr.startswith('children:'):
				keystr = keystr.split(':', 1)[1]
				context = Qt.WidgetWithChildrenShortcut
			elif keystr.startswith('application:'):
				keystr = keystr.split(':', 1)[1]
				context = Qt.ApplicationShortcut
			qks = QKeySequence(keystr)

			registerActionShortcut(category, actionName, qks, context)
Example #12
0
    def get_ic_factor(self, det):
        # storing ic_factor in preferences causing issues
        # ic_factor stored in detectors.cfg

        p = os.path.join(paths.spectrometer_dir, 'detectors.cfg')
        # factors=None
        ic = 1, 0
        if os.path.isfile(p):
            c = ConfigParser()
            c.read(p)
            det = det.lower()
            for si in c.sections():
                if si.lower() == det:
                    v, e = 1, 0
                    if c.has_option(si, 'ic_factor'):
                        v = c.getfloat(si, 'ic_factor')
                    if c.has_option(si, 'ic_factor_err'):
                        e = c.getfloat(si, 'ic_factor_err')
                    ic = v, e
                    break
        else:
            self.debug('no detector file {}. cannot retrieve ic_factor'.format(p))

        r = ufloat(*ic)
        return r
Example #13
0
File: conf.py Project: gwpy/gwpy
def build_cli_examples(_):
    logger = logging.getLogger('cli-examples')

    clidir = os.path.join(SPHINX_DIR, 'cli')
    exini = os.path.join(clidir, 'examples.ini')
    exdir = os.path.join(clidir, 'examples')
    if not os.path.isdir(exdir):
        os.makedirs(exdir)

    config = ConfigParser()
    config.read(exini)

    rsts = []
    for sect in config.sections():
        rst, cmd = _build_cli_example(config, sect, exdir, logger)
        if cmd:
            logger.info('[cli] running example {0!r}'.format(sect))
            logger.debug('[cli] $ {0}'.format(cmd))
            subprocess.check_call(cmd, shell=True)
            logger.debug('[cli] wrote {0}'.format(cmd.split()[-1]))
        rsts.append(rst)

    with open(os.path.join(exdir, 'examples.rst'), 'w') as f:
        f.write('.. toctree::\n   :glob:\n\n')
        for rst in rsts:
            f.write('   {0}\n'.format(rst[len(SPHINX_DIR):]))
Example #14
0
    def update_from(self, file_path):
        """
        Reads and loads user customised settings from the given file path.

        :param file_path: Absolute path to user settings file conf.cfg.
        """
        assert isinstance(file_path, six.text_type)

        cfg_parser = ConfigParser(inline_comment_prefixes=("#",))
        cfg_parser.read(file_path)
        ptpycfg = cfg_parser["ptpython"]
        converters = [ConfigParser.getboolean, ConfigParser.getint, ConfigParser.getfloat]

        for key in ptpycfg:
            converted = False

            if key not in self.user_defined:
                # Only settings provided in initial defaults dict can get
                # customised with user defined values from conf.cfg file.
                continue

            for func in converters:
                try:
                    value = func(cfg_parser, "ptpython", key)
                except ValueError:
                    continue
                else:
                    self.user_defined[key] = value
                    converted = True
                    break

            if not converted:
                self.user_defined[key] = ptpycfg.get(key, "")
Example #15
0
    def _populate_config_from_old_location(self, conf):
        if ('rate_limit_after_segment' in conf or
                'rate_limit_segments_per_sec' in conf or
                'max_get_time' in conf or
                '__file__' not in conf):
            return

        cp = ConfigParser()
        if os.path.isdir(conf['__file__']):
            read_conf_dir(cp, conf['__file__'])
        else:
            cp.read(conf['__file__'])

        try:
            pipe = cp.get("pipeline:main", "pipeline")
        except (NoSectionError, NoOptionError):
            return

        proxy_name = pipe.rsplit(None, 1)[-1]
        proxy_section = "app:" + proxy_name
        for setting in ('rate_limit_after_segment',
                        'rate_limit_segments_per_sec',
                        'max_get_time'):
            try:
                conf[setting] = cp.get(proxy_section, setting)
            except (NoSectionError, NoOptionError):
                pass
Example #16
0
def reader(filename=None):
    if filename is None:
        filename = ini_file
    cfg = ConfigParser()
    log.debug("Reading configuration from {} ...".format(ini_file))
    cfg.read(filename)
    return cfg
def sync():
    # Add or replace the relevant properites from galaxy.ini
    # into reports.ini
    reports_config_file = "config/reports.ini"
    if len(argv) > 1:
        reports_config_file = argv[1]

    universe_config_file = "config/galaxy.ini"
    if len(argv) > 2:
        universe_config_file = argv[2]

    parser = ConfigParser()
    parser.read(universe_config_file)

    with open(reports_config_file, "r") as f:
        reports_config_lines = f.readlines()

    replaced_properties = set([])
    with open(reports_config_file, "w") as f:
        # Write all properties from reports config replacing as
        # needed.
        for reports_config_line in reports_config_lines:
            (line, replaced_property) = get_synced_line(reports_config_line, parser)
            if replaced_property:
                replaced_properties.add(replaced_property)
            f.write(line)

        # If any properties appear in universe config and not in
        # reports write these as well.
        for replacement_property in REPLACE_PROPERTIES:
            if parser.has_option(MAIN_SECTION, replacement_property) and \
                    not (replacement_property in replaced_properties):
                f.write(get_universe_line(replacement_property, parser))
Example #18
0
def log_in(client):
    """Authorizes ImgurClient to use user account"""
    config = ConfigParser()
    config.read('auth.ini')
    access_token = config.get('credentials', 'access_token')
    refresh_token = config.get('credentials', 'refresh_token')
    if len(access_token) > 0 and len(refresh_token) > 0:
        client.set_user_auth(access_token, refresh_token)
        return client

    authorization_url = client.get_auth_url('pin')
    webbrowser.open(authorization_url)
    pin = input('Please input your pin\n>\t')

    credentials = client.authorize(pin)  # grant_type default is 'pin'

    access_token = credentials['access_token']
    refresh_token = credentials['refresh_token']

    config.set('credentials', 'access_token', access_token)
    config.set('credentials', 'refresh_token', refresh_token)

    save_config(config)
    client.set_user_auth(access_token, refresh_token)
    return client
    def _generate_appdata(self, prefix, activity_path):
        info = ConfigParser()
        info.read(os.path.join(activity_path, 'activity', 'activity.info'))

        required_fields = ['metadata_license', 'license', 'name', 'icon',
                           'description']
        for name in required_fields:
            if not info.has_option('Activity', name):
                print('[WARNING] Activity needs more metadata for AppStream '
                      'file')
                print('  Without an AppStream file, the activity will NOT '
                      'show in software stores!')
                print('  Please `pydoc sugar3.activity.bundlebuilder` for'
                      'more info')
                return

        # See https://www.freedesktop.org/software/appstream/docs/
        root = ET.Element('component', type='desktop')
        ET.SubElement(root, 'project_group').text = 'Sugar'
        ET.SubElement(root, 'translation', type='gettext').text = \
            self.config.bundle_id
        ET.SubElement(root, 'id').text = \
            self.config.bundle_id + '.activity.desktop'
        desc = ET.fromstring('<description>{}</description>'.format(
            info.get('Activity', 'description')))
        root.append(desc)

        copy_pairs = [('metadata_license', 'metadata_license'),
                      ('license', 'project_license'),
                      ('summary', 'summary'),
                      ('name', 'name')]
        for key, ename in copy_pairs:
            ET.SubElement(root, ename).text = info.get('Activity', key)

        if info.has_option('Activity', 'screenshots'):
            screenshots = info.get('Activity', 'screenshots').split()
            ss_root = ET.SubElement(root, 'screenshots')
            for i, screenshot in enumerate(screenshots):
                e = ET.SubElement(ss_root, 'screenshot')
                if i == 0:
                    e.set('type', 'default')
                ET.SubElement(e, 'image').text = screenshot

        if info.has_option('Activity', 'url'):
            ET.SubElement(root, 'url', type='homepage').text = \
                info.get('Activity', 'url')
        if info.has_option('Activity', 'repository_url'):
            ET.SubElement(root, 'url', type='bugtracker').text = \
                info.get('Activity', 'repository_url')
        elif info.has_option('Activity', 'repository'):
            ET.SubElement(root, 'url', type='bugtracker').text = \
                info.get('Activity', 'repository')

        path = os.path.join(prefix, 'share', 'metainfo',
                            self.config.bundle_id + '.appdata.xml')
        if not os.path.isdir(os.path.dirname(path)):
            os.makedirs(os.path.dirname(path))
        tree = ET.ElementTree(root)
        tree.write(path, encoding='UTF-8')
    def _save(self, nv):
        p = get_spectrometer_config_path()
        config = ConfigParser()
        config.read(p)

        config.set('CDDParameters', 'OperatingVoltage', nv)
        config.write(open(p, 'w'))
        self.info('saving new operating voltage {:0.1f} to {}'.format(nv, p))
Example #21
0
 def __new__(cls):
     ctx, config = cls.ctx_and_config
     current_ctx = click.get_current_context(silent=True)
     if current_ctx != ctx:
         config = ConfigParser()
         config.read(cls.filenames)
         cls.ctx_and_config = (current_ctx, config)
     return config
Example #22
0
    def test_list_with_no_entries(self):
        name = self.make_empty_temp_file()
        to_config_file(name, "section.name", {"ports": []})

        self.assertTrue(os.path.isfile(name))

        config = ConfigParser()
        config.read(name)
        self.assertEqual("", config.get("section.name", "ports"))
Example #23
0
    def test_list_with_one_entry(self):
        name = self.make_empty_temp_file()
        to_config_file(name, "section.name", {"ports": ["port1"]})

        self.assertTrue(os.path.isfile(name))

        config = ConfigParser()
        config.read(name)
        self.assertListEqual(["port1"], config.get("section.name", "ports").split(","))
Example #24
0
def parse_config_file(file_path):
    config = ConfigParser()
    config.read(file_path)
    if 'batch_scoring' not in config.sections():
        #  We are return empty dict, because there is nothing in this file
        #  that related to arguments to batch scoring.
        return {}
    parsed_dict = dict(config.items('batch_scoring'))
    return config_validator(parsed_dict)
Example #25
0
def get_configuration_dict(configuration=None, value_types=None):
    """
    Parse the configuration files

    Parameters
    ----------
    configuration : str or list, optional
        A configuration file or list of configuration files to parse,
        defaults to the deploy_default.conf file in the package and
        deploy.conf in the current working directory.

    value_types : dict, optional
        Dictionary containing classes to apply to specific items

    Returns
    -------
    dict
        Configuration dictionary
    """
    if not value_types:  # pragma: no cover
        value_types = config_types()
    if configuration is None or configuration is '':  # pragma: no cover
        configuration = [
            # Config file that is part of the package
            # PACKAGE_DEFAULT_CONFIG,

            # Any deploy.conf files in the current directory
            'deploy.conf'
        ]
    config = ConfigParser()

    # Set the config defaults
    try:
        config.read_string(config_defaults())
    except AttributeError:
        config.readfp(io.BytesIO(config_defaults()))

    logger.debug('Working with default dict: %r', config_defaults())
    config.read(configuration)
    result_dict = {}
    for section in config.sections():
        result_dict[section] = {}
        for key, val in config.items(section):
            result_dict[section][key] = str_format_env(val)

    config_update(result_dict)

    if 'locations' not in result_dict.keys():
        result_dict['locations'] = {}
    result_dict['locations']['package_scripts'] = package_scripts_directory()
    if not result_dict['global'].get('virtualenv_dir', None):
        result_dict['global']['virtualenv_dir'] = \
            default_virtualenv_directory()

    cast_types(result_dict)

    return result_dict
Example #26
0
 def __init__(self, config=None, ipaconf=paths.IPA_DEFAULT_CONF):
     super(IPAKEMKeys, self).__init__(config)
     conf = ConfigParser()
     conf.read(ipaconf)
     self.host = conf.get('global', 'host')
     self.realm = conf.get('global', 'realm')
     self.ldap_uri = config.get('ldap_uri', None)
     if self.ldap_uri is None:
         self.ldap_uri = conf.get('global', 'ldap_uri', None)
     self._server_keys = None
def open_buildout_configfile(filepath="buildout.cfg", write_on_exit=False):
    parser = ConfigParser()
    parser.read(filepath)
    try:
        yield parser
    finally:
        if not write_on_exit:
            return
        with open(filepath, 'w') as fd:
            parser.write(fd)
Example #28
0
    def _get_config(self):
        config = ConfigParser()
        try:
            p = get_spectrometer_config_path()
        except IOError:
            p = os.path.join(paths.spectrometer_dir, 'config.cfg')

        config.read(p)

        return config
Example #29
0
    def test_empty_dict(self):
        name = self.make_empty_temp_file()
        to_config_file(name, "section.name", {})

        self.assertTrue(os.path.isfile(name))

        config = ConfigParser()
        config.read(name)
        self.assertTrue(config.has_section("section.name"))
        self.assertSetEqual(_VALID_KEYS, set(config.options("section.name")))
Example #30
0
def main():
    # Setup an argparser and parse the known commands to get the config file
    parser = argparse.ArgumentParser(add_help=False)
    parser.add_argument('-C', '--config', help='Specify a config file to use',
                        default='/etc/rpkg/rfpkg.conf')

    (args, other) = parser.parse_known_args()

    # Make sure we have a sane config file
    if not os.path.exists(args.config) and \
       not other[-1] in ['--help', '-h', 'help']:
        sys.stderr.write('Invalid config file %s\n' % args.config)
        sys.exit(1)

    # Setup a configuration object and read config file data
    config = ConfigParser()
    config.read(args.config)

    client = rfpkg.cli.rfpkgClient(config)
    client.do_imports(site='rfpkg')
    client.parse_cmdline()

    if not client.args.path:
        try:
            client.args.path = pyrpkg.utils.getcwd()
        except:
            print('Could not get current path, have you deleted it?')
            sys.exit(1)

    # setup the logger -- This logger will take things of INFO or DEBUG and
    # log it to stdout.  Anything above that (WARN, ERROR, CRITICAL) will go
    # to stderr.  Normal operation will show anything INFO and above.
    # Quiet hides INFO, while Verbose exposes DEBUG.  In all cases WARN or
    # higher are exposed (via stderr).
    log = pyrpkg.log
    client.setupLogging(log)

    if client.args.v:
        log.setLevel(logging.DEBUG)
    elif client.args.q:
        log.setLevel(logging.WARNING)
    else:
        log.setLevel(logging.INFO)

    # Run the necessary command
    try:
        sys.exit(client.args.command())
    except KeyboardInterrupt:
        pass
    except Exception as e:
        log.error('Could not execute %s: %s' %
                  (client.args.command.__name__, e))
        if client.args.v:
            raise
        sys.exit(1)
Example #31
0
if debug.lower() == 'true':
    loglevel = logging.DEBUG
else:
    loglevel = logging.INFO

logging.basicConfig(stream=sys.stdout, level=loglevel)
LOG = logging.getLogger('placement_wait_for_service')

iterations = 60
timeout = 10
placement_cfg = '/etc/placement/placement.conf'

if __name__ == '__main__':
    if os.path.isfile(placement_cfg):
        try:
            config.read(placement_cfg)
        except Exception:
            LOG.exception('Error while reading placement.conf:')
    else:
        LOG.error('Placement configuration file %s does not exist',
                  placement_cfg)
        sys.exit(1)

    # get keystone client with details from [keystone_authtoken] section
    auth = v3.Password(
        user_domain_name=config.get('keystone_authtoken', 'user_domain_name'),
        username=config.get('keystone_authtoken', 'username'),
        password=config.get('keystone_authtoken', 'password'),
        project_name=config.get('keystone_authtoken', 'project_name'),
        project_domain_name=config.get('keystone_authtoken',
                                       'project_domain_name'),
if __name__ == '__main__':
    parser = OptionParser(usage="usage: %prog [options]")
    parser.add_option('-k', '--insecure',
                      action="store_false",
                      dest='insecure',
                      default=True,
                      help='Allow insecure connection when using SSL')

    (options, args) = parser.parse_args()
    LOG.debug('Running with parameter insecure = %s',
              options.insecure)

    if os.path.isfile(nova_cfg):
        try:
            config.read(nova_cfg)
        except Exception:
            LOG.exception('Error while reading nova.conf:')
    else:
        LOG.error('Nova configuration file %s does not exist', nova_cfg)
        sys.exit(1)

    my_host = config.get('DEFAULT', 'host')
    if not my_host:
        # If host isn't set nova defaults to this
        my_host = socket.gethostname()

    loader = loading.get_plugin_loader('password')
    auth = loader.load_from_options(
        auth_url=config.get('neutron',
                            'auth_url'),
Example #33
0
def read_conf():

    parser = OptionParser()
    parser.add_option("--cfg")  # Mandatory
    (options, args) = parser.parse_args()

    cfg_file = options.cfg
    Config = ConfigParser()
    Config.read(cfg_file)

    # DATA
    if Config.has_option('data', 'out_file'):
        options.out_file = Config.get('data', 'out_file')

    if Config.has_option('data', 'fea_scp'):
        options.fea_scp = Config.get('data', 'fea_scp')

    if Config.has_option('data', 'fea_opts'):
        options.fea_opts = Config.get('data', 'fea_opts')

    if Config.has_option('data', 'lab_folder'):
        options.lab_folder = Config.get('data', 'lab_folder')

    if Config.has_option('data', 'lab_opts'):
        options.lab_opts = Config.get('data', 'lab_opts')

    if Config.has_option('data', 'pt_file'):
        options.pt_file = Config.get('data', 'pt_file')

    if Config.has_option('data', 'count_file'):
        options.count_file = Config.get('data', 'count_file')

    # TO DO

    if Config.has_option('todo', 'do_training'):
        options.do_training = Config.get('todo', 'do_training')

    if Config.has_option('todo', 'do_eval'):
        options.do_eval = Config.get('todo', 'do_eval')

    if Config.has_option('todo', 'do_forward'):
        options.do_forward = Config.get('todo', 'do_forward')

    # ARCHITECTURE

    if Config.has_option('architecture', 'hidden_dim'):
        options.hidden_dim = Config.get('architecture', 'hidden_dim')

    if Config.has_option('architecture', 'N_hid'):
        options.N_hid = Config.get('architecture', 'N_hid')

    if Config.has_option('architecture', 'drop_rate'):
        options.drop_rate = Config.get('architecture', 'drop_rate')

    if Config.has_option('architecture', 'use_batchnorm'):
        options.use_batchnorm = Config.get('architecture', 'use_batchnorm')

    if Config.has_option('architecture', 'cw_left'):
        options.cw_left = Config.get('architecture', 'cw_left')

    if Config.has_option('architecture', 'cw_right'):
        options.cw_right = Config.get('architecture', 'cw_right')

    if Config.has_option('architecture', 'use_seed'):
        options.seed = Config.get('architecture', 'seed')

    if Config.has_option('architecture', 'use_cuda'):
        options.use_cuda = Config.get('architecture', 'use_cuda')

    if Config.has_option('architecture', 'multi_gpu'):
        options.multi_gpu = Config.get('architecture', 'multi_gpu')

    if Config.has_option('architecture', 'bidir'):
        options.bidir = Config.get('architecture', 'bidir')

    if Config.has_option('architecture', 'resnet'):
        options.resnet = Config.get('architecture', 'resnet')

    if Config.has_option('architecture', 'act'):
        options.act = Config.get('architecture', 'act')

    if Config.has_option('architecture', 'resgate'):
        options.resgate = Config.get('architecture', 'resgate')

    if Config.has_option('architecture', 'minimal_gru'):
        options.minimal_gru = Config.get('architecture', 'minimal_gru')

    if Config.has_option('architecture', 'skip_conn'):
        options.skip_conn = Config.get('architecture', 'skip_conn')

    if Config.has_option('architecture', 'act_gate'):
        options.act_gate = Config.get('architecture', 'act_gate')

    if Config.has_option('architecture', 'use_laynorm'):
        options.use_laynorm = Config.get('architecture', 'use_laynorm')

    if Config.has_option('architecture', 'cost'):
        options.cost = Config.get('architecture', 'cost')

    if Config.has_option('architecture', 'NN_type'):
        options.NN_type = Config.get('architecture', 'NN_type')

    if Config.has_option('architecture', 'twin_reg'):
        options.twin_reg = Config.get('architecture', 'twin_reg')

    if Config.has_option('architecture', 'twin_w'):
        options.twin_w = Config.get('architecture', 'twin_w')

    if Config.has_option('architecture', 'cnn_pre'):
        options.cnn_pre = Config.get('architecture', 'cnn_pre')

    if Config.has_option('architecture', 'seed'):
        options.seed = Config.get('architecture', 'seed')

    # Optimization
    if Config.has_option('optimization', 'lr'):
        options.lr = Config.get('optimization', 'lr')

    if Config.has_option('optimization', 'batch_size'):
        options.batch_size = Config.get('optimization', 'batch_size')

    if Config.has_option('optimization', 'save_gpumem'):
        options.save_gpumem = Config.get('optimization', 'save_gpumem')

    if Config.has_option('optimization', 'optimizer'):
        options.optimizer = Config.get('optimization', 'optimizer')

    return options
Example #34
0
            conn.close()


if __name__ == '__main__':
    try:
        logging.basicConfig(filename="../output/loaddata.log",
                            level=logging.INFO)

        print("{} ** loaddata/main.py **".format(
            datetime.today().strftime('%Y-%m-%d-%H:%M:%S')))
        logging.info("{} ** loaddata/main.py **".format(
            datetime.today().strftime('%Y-%m-%d-%H:%M:%S')))

        #if you want some cache here, fill the file 'config.ini' with information you want
        config = ConfigParser()
        config.read('config.ini')

        if not config.get("GitHubConnection", "token"):
            print(
                "Connection Error: Token not provided at config.ini file. \nPlease get your token accessing GitHub User's Settings / Developer settings / Personal access tokens. \nFurther details at 'https://developer.github.com/v3/auth/#basic-authentication'"
            )
            logging.error(
                "Connection Error: Token not provided at config.ini file. \nPlease get your token accessing GitHub User's Settings / Developer settings / Personal access tokens. \nFurther details at 'https://developer.github.com/v3/auth/#basic-authentication'"
            )
        else:
            token = config.get("GitHubConnection", "token")
            repo_user = config.get("GitHubRepository", "user") if config.get(
                "GitHubRepository",
                "user") else raw_input("GitHub Repository user: "******"GitHubRepository", "projectname") if config.get(
Example #35
0
class StageConfig(object):
    def __init__(self, filename=None, text=None):
        self.config = {}
        self.cp = ConfigParser()
        self.nstages = 0
        fname = None
        if filename is not None and os.path.exists(filename):
            fname = filename
        if fname is None:
            for filename in conf_files:
                if os.path.exists(filename) and os.path.isfile(filename):
                    fname = filename
                    break

        if fname is not None:
            self.Read(fname=fname)
        else:
            self.cp.readfp(StringIO(DEFAULT_CONF))
            self._process_data()

    def Read(self, fname=None):
        if fname is not None:
            ret = self.cp.read(fname)
            if len(ret) == 0:
                time.sleep(0.5)
                ret = self.cp.read(fname)
            self.filename = fname
            self._process_data()

            stage_names = self.config['stages']
            image_folder = self.config['camera']['image_folder']
            pos = OrderedDict()
            if 'positions' not in self.config:
                self.config['positions'] = {}
            for key, dat in self.config['positions'].items():
                img_fname = dat['image']
                image = {
                    'type': 'filename',
                    'data': os.path.join(image_folder, img_fname)
                }

                poslist = dat['position']
                posdict = {}
                for name, val in zip(stage_names, poslist):
                    posdict[name] = val
                pos[key] = dict(image=image, position=posdict)
            self.config['positions'] = pos

    def _process_data(self):
        for sect, opts in conf_sects.items():
            if not self.cp.has_section(sect):
                # print 'skipping section ' ,sect
                continue
            bools = opts.get('bools', [])
            floats = opts.get('floats', [])
            ints = opts.get('ints', [])
            thissect = {}
            is_ordered = False
            if 'ordered' in opts:
                is_ordered = True

            for opt in self.cp.options(sect):
                get = self.cp.get
                if opt in bools:
                    get = self.cp.getboolean
                elif opt in floats:
                    get = self.cp.getfloat
                elif opt in ints:
                    get = self.cp.getint
                try:
                    val = get(sect, opt)
                except ValueError:
                    val = ''
                if is_ordered and '||' in val:
                    nam, val = val.split('||', 1)
                    opt = opt.strip()
                    val = nam, val.strip()
                thissect[opt] = val
            self.config[sect] = thissect

        if 'positions' in self.config:
            out = OrderedDict()
            poskeys = list(self.config['positions'].keys())
            poskeys.sort()
            for key in poskeys:
                name, val = self.config['positions'][key]
                name = name.strip()
                img, posval = val.strip().split('||')
                pos = [float(i) for i in posval.split(',')]
                out[name] = dict(image=img.strip(), position=pos)
            self.config['positions'] = out

        if 'stages' in self.config:
            out = OrderedDict()
            groups = []

            skeys = list(self.config['stages'].keys())
            skeys.sort()
            for key in skeys:
                name, val = self.config['stages'][key]
                name = normalize_pvname(name.strip())
                val = val.replace('||', ' | ')
                words = [w.strip() for w in val.split('|')]
                group = words[0]
                desc = words[1]
                if len(desc) == 0:
                    desc = None

                scale = 1.0
                if len(words) > 1 and len(words[2]) > 0:
                    scale = float(words[2])

                prec = None
                if len(words) > 2 and len(words[3]) > 0:
                    prec = int(words[3])

                maxstep = None
                if len(words) > 4 and len(words[4]) > 0:
                    maxstep = float(words[4])

                show = 1
                if len(words) > 5 and len(words[5]) > 0:
                    show = int(words[5])

                out[name] = dict(label=name,
                                 group=group,
                                 desc=desc,
                                 scale=scale,
                                 prec=prec,
                                 maxstep=maxstep,
                                 show=show)
                if group not in groups:
                    groups.append(group)
            self.config['stages'] = out
            self.config['stage_groups'] = groups
            self.nstages = len(out)

    def Save(self, fname=None, positions=None):
        o = []
        # print 'Save CONFIG FILE:', fname, os.getcwd()
        # print positions.keys()
        cnf = self.config

        if fname is not None:
            self.filename = fname
        o.append('## Sample Stage Configuration (saved: %s)' % (time.ctime()))

        if positions is None:
            positions = cnf['positions']
        for sect, optlist in conf_objs.items():
            o.append('#--------------------------#\n[%s]' % sect)
            if sect == 'positions' and positions is not None:
                o.append(POS_LEGEND)
                fmt = "%3.3i = %s || %s || %s "
                pfmt = ', '.join(['%f' for i in range(self.nstages)])
                idx = 1
                for name, val in positions.items():
                    pos = []
                    for pvname in cnf['stages']:
                        try:
                            pos.append(float(val['position'][pvname]))
                        except:
                            pass
                    pfmt = ', '.join(['%f' for i in range(len(pos))])
                    pos = pfmt % tuple(pos)
                    try:
                        tmpdir, imgfile = os.path.split(val['image'])
                    except:
                        tmpdir, imgfile = '', ''

                    o.append(fmt % (idx, name, imgfile, pos))
                    idx = idx + 1
            elif sect == 'stages':
                o.append(STAGE_LEGEND)
                fmt = "%i = %s || %s || %s || %s || %s || %s || %s"
                idx = 1
                for name, dat in cnf['stages'].items():
                    # print 'Save STAGE ', name, dat
                    # index =  motor || group   ||desc || scale || prec || maxstep || show
                    group = dat['group']
                    desc = dat['desc']
                    show = str(dat['show'])
                    scale = str(dat['scale'])
                    prec = str(dat['prec'])
                    maxstep = "%.3f" % (dat['maxstep'])
                    o.append(
                        fmt %
                        (idx, name, group, desc, scale, prec, maxstep, show))
                    idx = idx + 1
            if optlist is not None:
                for opt in optlist:
                    try:
                        val = cnf[sect].get(opt, ' ')
                        if not isinstance(val, (str, unicode)): val = str(val)
                        o.append("%s = %s" % (opt, val))
                    except:
                        pass
        o.append('#------------------#\n')
        # print 'Conf autosave ', fname
        # print os.path.abspath(fname)
        f = open(fname, 'w')
        f.write('\n'.join(o))
        f.close()

    def sections(self):
        return self.config.keys()

    def section(self, section):
        return self.config[section]

    def get(self, section, value=None):
        if value is None:
            return self.config[section]
        else:
            return self.config[section][value]
Example #36
0
    def add_repo(
        self, name, uri, repo_type='rpm-md',
        prio=None, dist=None, components=None,
        user=None, secret=None, credentials_file=None,
        repo_gpgcheck=None, pkg_gpgcheck=None
    ):
        """
        Add zypper repository

        :param string name: repository name
        :param string uri: repository URI
        :param repo_type: repostory type name
        :param int prio: zypper repostory priority
        :param dist: unused
        :param components: unused
        :param user: credentials username
        :param secret: credentials password
        :param credentials_file: zypper credentials file
        :param bool repo_gpgcheck: enable repository signature validation
        :param bool pkg_gpgcheck: enable package signature validation
        """
        if credentials_file:
            repo_secret = os.sep.join(
                [self.shared_zypper_dir['credentials-dir'], credentials_file]
            )
            if os.path.exists(repo_secret):
                Path.wipe(repo_secret)

            if user and secret:
                uri = ''.join([uri, '?credentials=', credentials_file])
                with open(repo_secret, 'w') as credentials:
                    credentials.write('username={0}'.format(user))
                    credentials.write('password={0}'.format(secret))

        repo_file = ''.join(
            [self.shared_zypper_dir['reposd-dir'], '/', name, '.repo']
        )
        self.repo_names.append(''.join([name, '.repo']))

        if os.path.exists(repo_file):
            Path.wipe(repo_file)

        self._backup_package_cache()
        Command.run(
            ['zypper'] + self.zypper_args + [
                '--root', self.root_dir,
                'addrepo',
                '--refresh',
                '--type', self._translate_repo_type(repo_type),
                '--keep-packages',
                '-C',
                uri,
                name
            ],
            self.command_env
        )
        if prio or repo_gpgcheck is not None or pkg_gpgcheck is not None:
            repo_config = ConfigParser()
            repo_config.read(repo_file)
            if repo_gpgcheck is not None:
                repo_config.set(
                    name, 'repo_gpgcheck', '1' if repo_gpgcheck else '0'
                )
            if pkg_gpgcheck is not None:
                repo_config.set(
                    name, 'pkg_gpgcheck', '1' if pkg_gpgcheck else '0'
                )
            if prio:
                repo_config.set(
                    name, 'priority', format(prio)
                )
            with open(repo_file, 'w') as repo:
                repo_config.write(repo)
        self._restore_package_cache()
Example #37
0
    def convert_profile(self):
        cp = ConfigParser()
        path = os.path.join(env.get_profile_path(), 'config')
        cp.read([path])

        settings = Gio.Settings('org.sugarlabs.user')
        if cp.has_option('Buddy', 'NickName'):
            name = cp.get('Buddy', 'NickName')
            # decode nickname from ascii-safe chars to unicode
            nick = name.decode('utf-8')
            settings.set_string('nick', nick)
        if cp.has_option('Buddy', 'Color'):
            color = cp.get('Buddy', 'Color')
            settings.set_string('color', color)

        if cp.has_option('Jabber', 'Server'):
            server = cp.get('Jabber', 'Server')
            settings = Gio.Settings('org.sugarlabs.collaboration')
            settings.set_string('jabber-server', server)

        if cp.has_option('Date', 'Timezone'):
            timezone = cp.get('Date', 'Timezone')
            settings = Gio.Settings('org.sugarlabs.date')
            settings.set_string('timezone', timezone)

        settings = Gio.Settings('org.sugarlabs.frame')
        if cp.has_option('Frame', 'HotCorners'):
            delay = float(cp.get('Frame', 'HotCorners'))
            settings.set_int('corner-delay', int(delay))
        if cp.has_option('Frame', 'WarmEdges'):
            delay = float(cp.get('Frame', 'WarmEdges'))
            settings.set_int('edge-delay', int(delay))

        if cp.has_option('Server', 'Backup1'):
            backup1 = cp.get('Server', 'Backup1')
            settings = Gio.Settings('org.sugarlabs')
            settings.set_string('backup-url', backup1)

        if cp.has_option('Sound', 'Volume'):
            volume = float(cp.get('Sound', 'Volume'))
            settings = Gio.Settings('org.sugarlabs.sound')
            settings.set_int('volume', int(volume))

        settings = Gio.Settings('org.sugarlabs.power')
        if cp.has_option('Power', 'AutomaticPM'):
            state = cp.get('Power', 'AutomaticPM')
            if state.lower() == 'true':
                settings.set_boolean('automatic', True)
        if cp.has_option('Power', 'ExtremePM'):
            state = cp.get('Power', 'ExtremePM')
            if state.lower() == 'true':
                settings.set_boolean('extreme', True)

        if cp.has_option('Shell', 'FavoritesLayout'):
            layout = cp.get('Shell', 'FavoritesLayout')
            settings = Gio.Settings('org.sugarlabs.desktop')
            settings.set_string('favorites-layout', layout)
        del cp
        try:
            os.unlink(path)
        except OSError:
            logging.error('Error removing old profile.')
Example #38
0
    def loadConfig(self):
        """Load build configuration from the spec file."""
        specfiles = [spec for spec in os.listdir(self.spec_dir) if spec.endswith('.ini')]
        if len(specfiles) == 0:
            raise BuildError('No .ini file found')
        elif len(specfiles) > 1:
            raise BuildError('Multiple .ini files found')

        conf = ConfigParser()
        conf.read(os.path.join(self.spec_dir, specfiles[0]))

        # [naming] section
        for entry in ('name', 'version', 'release', 'description'):
            setattr(self, entry, conf.get('naming', entry))
        if conf.has_option('naming', 'epoch'):
            self.epoch = conf.get('naming', 'epoch')

        # [building] section
        self.platform = conf.get('building', 'platform')

        # preinstalled are paths to files or directories that must exist
        # in the VM for it to execute the build.
        # If the path ends in / or \ it must be a directory, otherwise it must
        # be a file.
        # They may be specified as Cygwin (/cygdrive/c/...) or Windows (C:\...)
        # absolute paths, or without a path in which case it is searched for
        # on the PATH.
        if conf.has_option('building', 'preinstalled'):
            self.preinstalled.extend([e.strip() for e in conf.get('building', 'preinstalled').split('\n') if e])

        # buildrequires and provides are multi-valued (space-separated)
        for br in conf.get('building', 'buildrequires').split():
            # buildrequires is a space-separated list
            # each item in the list is in the format:
            # pkgname[:opt1:opt2=val2:...]
            # the options are put into a dict
            # if the option has no =val, the value in the dict will be None
            if br:
                br = br.split(':')
                bropts = {}
                for opt in br[1:]:
                    if '=' in opt:
                        key, val = opt.split('=', 1)
                    else:
                        key = opt
                        val = None
                    bropts[key] = val
                self.buildrequires.append((br[0], bropts))
        for prov in conf.get('building', 'provides').split():
            if prov:
                self.provides.append(prov)
        # optionally specify a shell to use (defaults to bash)
        # valid values are: cmd, cmd.exe (alias for cmd), and bash
        if conf.has_option('building', 'shell'):
            self.shell = conf.get('building', 'shell')
        else:
            self.shell = 'bash'
        # execute is multi-valued (newline-separated)
        self.execute.extend([e.strip() for e in conf.get('building', 'execute').split('\n') if e])

        # postbuild are files or directories that must exist after the build is
        # complete, but are not included in the build output
        # they are specified as paths relative the source directory, and may be
        # in Unix or Windows format
        # each entry may contain shell-style globs, and one or more files
        # matching the glob is considered valid
        if conf.has_option('building', 'postbuild'):
            for entry in conf.get('building', 'postbuild').split('\n'):
                entry = entry.strip()
                if not entry:
                    continue
                for var in ('name', 'version', 'release'):
                    entry = entry.replace('$' + var, getattr(self, var))
                self.postbuild.append(entry)

        # [files] section
        for entry in conf.get('files', 'output').split('\n'):
            entry = entry.strip()
            if not entry:
                continue
            tokens = entry.split(':')
            filename = tokens[0]
            for var in ('name', 'version', 'release'):
                filename = filename.replace('$' + var, getattr(self, var))
            metadata = {}
            metadata['platforms'] = tokens[1].split(',')
            if len(tokens) > 2:
                metadata['flags'] = tokens[2].split(',')
            else:
                metadata['flags'] = []
            self.output[filename] = metadata
        self.logs.extend([e.strip() for e in conf.get('files', 'logs').split('\n') if e])
Example #39
0
def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
    """
    If SWIFT_TEST_POLICY is set:
    - look in swift.conf file for specified policy
    - move this to be policy-0 but preserving its options
    - copy its ring file to test dir, changing its devices to suit
      in process testing, and renaming it to suit policy-0
    Otherwise, create a default ring file.
    """
    conf = ConfigParser()
    conf.read(swift_conf)
    sp_prefix = 'storage-policy:'

    try:
        # policy index 0 will be created if no policy exists in conf
        policies = parse_storage_policies(conf)
    except PolicyError as e:
        raise InProcessException(e)

    # clear all policies from test swift.conf before adding test policy back
    for policy in policies:
        conf.remove_section(sp_prefix + str(policy.idx))

    if policy_specified:
        policy_to_test = policies.get_by_name(policy_specified)
        if policy_to_test is None:
            raise InProcessException('Failed to find policy name "%s"'
                                     % policy_specified)
        _info('Using specified policy %s' % policy_to_test.name)
    else:
        policy_to_test = policies.default
        _info('Defaulting to policy %s' % policy_to_test.name)

    # make policy_to_test be policy index 0 and default for the test config
    sp_zero_section = sp_prefix + '0'
    conf.add_section(sp_zero_section)
    for (k, v) in policy_to_test.get_info(config=True).items():
        conf.set(sp_zero_section, k, v)
    conf.set(sp_zero_section, 'default', True)

    with open(swift_conf, 'w') as fp:
        conf.write(fp)

    # look for a source ring file
    ring_file_src = ring_file_test = 'object.ring.gz'
    if policy_to_test.idx:
        ring_file_src = 'object-%s.ring.gz' % policy_to_test.idx
    try:
        ring_file_src = _in_process_find_conf_file(conf_src_dir, ring_file_src,
                                                   use_sample=False)
    except InProcessException as e:
        if policy_specified:
            raise InProcessException('Failed to find ring file %s'
                                     % ring_file_src)
        ring_file_src = None

    ring_file_test = os.path.join(testdir, ring_file_test)
    if ring_file_src:
        # copy source ring file to a policy-0 test ring file, re-homing servers
        _info('Using source ring file %s' % ring_file_src)
        ring_data = ring.RingData.load(ring_file_src)
        obj_sockets = []
        for dev in ring_data.devs:
            device = 'sd%c1' % chr(len(obj_sockets) + ord('a'))
            utils.mkdirs(os.path.join(_testdir, 'sda1'))
            utils.mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
            obj_socket = eventlet.listen(('localhost', 0))
            obj_sockets.append(obj_socket)
            dev['port'] = obj_socket.getsockname()[1]
            dev['ip'] = '127.0.0.1'
            dev['device'] = device
            dev['replication_port'] = dev['port']
            dev['replication_ip'] = dev['ip']
        ring_data.save(ring_file_test)
    else:
        # make default test ring, 2 replicas, 4 partitions, 2 devices
        _info('No source object ring file, creating 2rep/4part/2dev ring')
        obj_sockets = [eventlet.listen(('localhost', 0)) for _ in (0, 1)]
        ring_data = ring.RingData(
            [[0, 1, 0, 1], [1, 0, 1, 0]],
            [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
              'port': obj_sockets[0].getsockname()[1]},
             {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
              'port': obj_sockets[1].getsockname()[1]}],
            30)
        with closing(GzipFile(ring_file_test, 'wb')) as f:
            pickle.dump(ring_data, f)

    for dev in ring_data.devs:
        _debug('Ring file dev: %s' % dev)

    return obj_sockets
Example #40
0
#

import numpy
from sherpa.utils import NoNewAttributesAfterInit
from sherpa.utils.err import StatErr
import sherpa.stats._statfcts

from sherpa import get_config
from six.moves.configparser import ConfigParser

__all__ = ('Stat', 'Cash', 'CStat', 'LeastSq', 'Chi2Gehrels', 'Chi2ConstVar',
           'Chi2DataVar', 'Chi2ModVar', 'Chi2XspecVar', 'Chi2', 'UserStat',
           'WStat')

config = ConfigParser()
config.read(get_config())

# truncation_flag indicates whether or not model truncation
# should be performed.  If true, use the truncation_value from
# the config file.
truncation_flag = config.get('statistics', 'truncate').upper()
truncation_value = float(config.get('statistics', 'trunc_value'))
if (bool(truncation_flag) is False or truncation_flag == "FALSE"
        or truncation_flag == "NONE" or truncation_flag == "0"):
    truncation_value = 1.0e-25


def get_syserror_weight_extra(dictionary):
    syserror = dictionary.get('syserror')
    weight = dictionary.get('weight')
    extra = dictionary.get('extra_args')
Example #41
0
def main():
    patcher.monkey_patch()
    hubs.get_hub().debug_exceptions = False

    conffile = '/etc/swift/dispersion.conf'

    parser = OptionParser(usage='''
Usage: %%prog [options] [conf_file]

[conf_file] defaults to %s'''.strip() % conffile)
    parser.add_option('-j',
                      '--dump-json',
                      action='store_true',
                      default=False,
                      help='dump dispersion report in json format')
    parser.add_option('-d',
                      '--debug',
                      action='store_true',
                      default=False,
                      help='print 404s to standard error')
    parser.add_option('-p',
                      '--partitions',
                      action='store_true',
                      default=False,
                      help='print missing partitions to standard error')
    parser.add_option('--container-only',
                      action='store_true',
                      default=False,
                      help='Only run container report')
    parser.add_option('--object-only',
                      action='store_true',
                      default=False,
                      help='Only run object report')
    parser.add_option('--insecure',
                      action='store_true',
                      default=False,
                      help='Allow accessing insecure keystone server. '
                      'The keystone\'s certificate will not be verified.')
    parser.add_option('-P',
                      '--policy-name',
                      dest='policy_name',
                      help="Specify storage policy name")

    options, args = parser.parse_args()
    if args:
        conffile = args.pop(0)

    if options.debug:
        global debug
        debug = True

    c = ConfigParser()
    if not c.read(conffile):
        exit('Unable to read config file: %s' % conffile)
    conf = dict(c.items('dispersion'))

    if options.dump_json:
        conf['dump_json'] = 'yes'
    if options.object_only:
        conf['container_report'] = 'no'
    if options.container_only:
        conf['object_report'] = 'no'
    if options.insecure:
        conf['keystone_api_insecure'] = 'yes'
    if options.partitions:
        conf['partitions'] = 'yes'

    output = generate_report(conf, options.policy_name)

    if json_output:
        print(json.dumps(output))
Example #42
0
def loadini(struct, configfile):
    """Loads .ini configuration file and stores its values in struct"""

    config_path = os.path.expanduser(configfile)

    config = ConfigParser()
    defaults = {
        'general': {
            'arg_spec': True,
            'auto_display_list': True,
            'autocomplete_mode': default_completion,
            'color_scheme': 'default',
            'complete_magic_methods': True,
            'dedent_after': 1,
            'editor': os.environ.get('VISUAL', os.environ.get('EDITOR', 'vi')),
            'flush_output': True,
            'highlight_show_source': True,
            'hist_duplicates': True,
            'hist_file': '~/.pythonhist',
            'hist_length': 100,
            'paste_time': 0.02,
            'pastebin_confirm': True,
            'pastebin_expiry': '1week',
            'pastebin_helper': '',
            'pastebin_removal_url': 'https://bpaste.net/remove/$removal_id',
            'pastebin_show_url': 'https://bpaste.net/show/$paste_id',
            'pastebin_url': 'https://bpaste.net/json/new',
            'save_append_py': False,
            'single_undo_time': 1.0,
            'syntax': True,
            'tab_length': 4,
            'unicode_box': True
        },
        'keyboard': {
            'backspace': 'C-h',
            'beginning_of_line': 'C-a',
            'clear_line': 'C-u',
            'clear_screen': 'C-l',
            'clear_word': 'C-w',
            'copy_clipboard': 'F10',
            'cut_to_buffer': 'C-k',
            'delete': 'C-d',
            'down_one_line': 'C-n',
            'edit_config': 'F3',
            'edit_current_block': 'C-x',
            'end_of_line': 'C-e',
            'exit': '',
            'external_editor': 'F7',
            'help': 'F1',
            'last_output': 'F9',
            'left': 'C-b',
            'pastebin': 'F8',
            'reimport': 'F6',
            'right': 'C-f',
            'save': 'C-s',
            'search': 'C-o',
            'show_source': 'F2',
            'suspend': 'C-z',
            'toggle_file_watch': 'F5',
            'transpose_chars': 'C-t',
            'undo': 'C-r',
            'up_one_line': 'C-p',
            'yank_from_buffer': 'C-y'
        },
        'cli': {
            'suggestion_width': 0.8,
            'trim_prompts': False,
        },
        'curtsies': {
            'list_above': False,
            'right_arrow_completion': True,
        }
    }

    default_keys_to_commands = dict(
        (value, key) for (key, value) in iteritems(defaults['keyboard']))

    fill_config_with_default_values(config, defaults)
    if not config.read(config_path):
        # No config file. If the user has it in the old place then complain
        if os.path.isfile(os.path.expanduser('~/.bpython.ini')):
            sys.stderr.write("Error: It seems that you have a config file at "
                             "~/.bpython.ini. Please move your config file to "
                             "%s\n" % default_config_path())
            sys.exit(1)

    def get_key_no_doublebind(command):
        default_commands_to_keys = defaults['keyboard']
        requested_key = config.get('keyboard', command)

        try:
            default_command = default_keys_to_commands[requested_key]

            if (default_commands_to_keys[default_command] == config.get(
                    'keyboard', default_command)):
                setattr(struct, '%s_key' % default_command, '')
        except KeyError:
            pass

        return requested_key

    struct.config_path = config_path

    struct.dedent_after = config.getint('general', 'dedent_after')
    struct.tab_length = config.getint('general', 'tab_length')
    struct.auto_display_list = config.getboolean('general',
                                                 'auto_display_list')
    struct.syntax = config.getboolean('general', 'syntax')
    struct.arg_spec = config.getboolean('general', 'arg_spec')
    struct.paste_time = config.getfloat('general', 'paste_time')
    struct.single_undo_time = config.getfloat('general', 'single_undo_time')
    struct.highlight_show_source = config.getboolean('general',
                                                     'highlight_show_source')
    struct.hist_file = config.get('general', 'hist_file')
    struct.editor = config.get('general', 'editor')
    struct.hist_length = config.getint('general', 'hist_length')
    struct.hist_duplicates = config.getboolean('general', 'hist_duplicates')
    struct.flush_output = config.getboolean('general', 'flush_output')

    struct.pastebin_key = get_key_no_doublebind('pastebin')
    struct.copy_clipboard_key = get_key_no_doublebind('copy_clipboard')
    struct.save_key = get_key_no_doublebind('save')
    struct.search_key = get_key_no_doublebind('search')
    struct.show_source_key = get_key_no_doublebind('show_source')
    struct.suspend_key = get_key_no_doublebind('suspend')
    struct.toggle_file_watch_key = get_key_no_doublebind('toggle_file_watch')
    struct.undo_key = get_key_no_doublebind('undo')
    struct.reimport_key = get_key_no_doublebind('reimport')
    struct.up_one_line_key = get_key_no_doublebind('up_one_line')
    struct.down_one_line_key = get_key_no_doublebind('down_one_line')
    struct.cut_to_buffer_key = get_key_no_doublebind('cut_to_buffer')
    struct.yank_from_buffer_key = get_key_no_doublebind('yank_from_buffer')
    struct.clear_word_key = get_key_no_doublebind('clear_word')
    struct.backspace_key = get_key_no_doublebind('backspace')
    struct.clear_line_key = get_key_no_doublebind('clear_line')
    struct.clear_screen_key = get_key_no_doublebind('clear_screen')
    struct.delete_key = get_key_no_doublebind('delete')

    struct.left_key = get_key_no_doublebind('left')
    struct.right_key = get_key_no_doublebind('right')
    struct.end_of_line_key = get_key_no_doublebind('end_of_line')
    struct.beginning_of_line_key = get_key_no_doublebind('beginning_of_line')
    struct.transpose_chars_key = get_key_no_doublebind('transpose_chars')
    struct.clear_line_key = get_key_no_doublebind('clear_line')
    struct.clear_screen_key = get_key_no_doublebind('clear_screen')
    struct.exit_key = get_key_no_doublebind('exit')
    struct.last_output_key = get_key_no_doublebind('last_output')
    struct.edit_config_key = get_key_no_doublebind('edit_config')
    struct.edit_current_block_key = get_key_no_doublebind('edit_current_block')
    struct.external_editor_key = get_key_no_doublebind('external_editor')
    struct.help_key = get_key_no_doublebind('help')

    struct.pastebin_confirm = config.getboolean('general', 'pastebin_confirm')
    struct.pastebin_url = config.get('general', 'pastebin_url')
    struct.pastebin_show_url = config.get('general', 'pastebin_show_url')
    struct.pastebin_removal_url = config.get('general', 'pastebin_removal_url')
    struct.pastebin_expiry = config.get('general', 'pastebin_expiry')
    struct.pastebin_helper = config.get('general', 'pastebin_helper')

    struct.cli_suggestion_width = config.getfloat('cli', 'suggestion_width')
    struct.cli_trim_prompts = config.getboolean('cli', 'trim_prompts')

    struct.complete_magic_methods = config.getboolean(
        'general', 'complete_magic_methods')
    struct.autocomplete_mode = config.get('general', 'autocomplete_mode')
    struct.save_append_py = config.getboolean('general', 'save_append_py')

    struct.curtsies_list_above = config.getboolean('curtsies', 'list_above')
    struct.curtsies_right_arrow_completion = \
        config.getboolean('curtsies', 'right_arrow_completion')

    color_scheme_name = config.get('general', 'color_scheme')

    default_colors = {
        'keyword': 'y',
        'name': 'c',
        'comment': 'b',
        'string': 'm',
        'error': 'r',
        'number': 'G',
        'operator': 'Y',
        'punctuation': 'y',
        'token': 'C',
        'background': 'd',
        'output': 'w',
        'main': 'c',
        'paren': 'R',
        'prompt': 'c',
        'prompt_more': 'g',
        'right_arrow_suggestion': 'K',
    }

    if color_scheme_name == 'default':
        struct.color_scheme = default_colors
    else:
        struct.color_scheme = dict()

        theme_filename = color_scheme_name + '.theme'
        path = os.path.expanduser(
            os.path.join(get_config_home(), theme_filename))
        try:
            load_theme(struct, path, struct.color_scheme, default_colors)
        except EnvironmentError:
            sys.stderr.write("Could not load theme '%s'.\n" %
                             (color_scheme_name, ))
            sys.exit(1)

    # checks for valid key configuration this part still sucks
    for key in (struct.pastebin_key, struct.save_key):
        key_dispatch[key]

    # expand path of history file
    struct.hist_file = os.path.expanduser(struct.hist_file)

    # verify completion mode
    if struct.autocomplete_mode not in bpython.autocomplete.ALL_MODES:
        struct.autocomplete_mode = default_completion

    # set box drawing characters
    if config.getboolean('general', 'unicode_box') and supports_box_chars():
        struct.left_border = u'│'
        struct.right_border = u'│'
        struct.top_border = u'─'
        struct.bottom_border = u'─'
        struct.left_bottom_corner = u'└'
        struct.right_bottom_corner = u'┘'
        struct.left_top_corner = u'┌'
        struct.right_top_corner = u'┐'
    else:
        struct.left_border = u'|'
        struct.right_border = u'|'
        struct.top_border = u'-'
        struct.bottom_border = u'-'
        struct.left_bottom_corner = u'+'
        struct.right_bottom_corner = u'+'
        struct.left_top_corner = u'+'
        struct.right_top_corner = u'+'
Example #43
0
def update(version, expected_md5):
    """
    Execute the actual update: extract the archive and execute the bash update script.

    :param version: the new version (after the update).
    :param expected_md5: the md5 sum provided by the server.
    """
    version_mapping = {}

    try:
        config = ConfigParser()
        config.read(constants.get_config_file())
        from_version = config.get('OpenMotics', 'version')
        logger.info('==================================')
        logger.info('Starting update {} -> {}'.format(from_version, version))

        update_file = constants.get_update_file()
        update_dir = os.path.dirname(update_file)
        # Change to update directory.
        os.chdir(update_dir)

        if os.path.exists(update_file):
            logger.info(' -> Extracting update.tgz')
            extract_legacy_update(update_file, expected_md5)
        else:
            logger.info(' -> Fetching metadata')
            meta = fetch_metadata(config, version, expected_md5)
            logger.info(' -> Downloading firmware for update {}'.format(meta['version']))
            for data in meta['firmwares']:
                download_firmware(data['type'], data['url'], data['sha256'])
                version_mapping[data['type']] = data['version']
    except Exception:
        logger.exception('failed to preprepare update')
        raise SystemExit(EXIT_CODES['failed_preprepare_update'])

    errors = []
    services_running = True
    try:
        date = datetime.now().strftime('%Y%m%d%H%M%S')

        # TODO: should update and re-execute itself before proceeding?

        logger.info(' -> Checking services')
        check_services()

        logger.info(' -> Stopping services')
        stop_services()
        services_running = False

        gateway_os = FIRMWARE_FILES['gateway_os']
        if os.path.exists(gateway_os):
            os_version = version_mapping.get('gateway_os')
            logger.info(' -> Updating Gateway OS to {0}'.format(os_version if os_version else 'unknown version'))
            error = update_gateway_os(gateway_os, os_version)
            if error:
                errors.append(error)

        gateway_service = FIRMWARE_FILES['gateway_service']
        if os.path.exists(gateway_service):
            service_version = version_mapping.get('gateway_service')
            logger.info(' -> Updating Gateway service to {0}'.format(service_version if service_version else 'unknown version'))
            error = update_gateway_backend(gateway_service, date, service_version)
            if error:
                errors.append(error)

        master_type = get_master_type()
        master_firmware = FIRMWARE_FILES[master_type]
        if os.path.exists(master_firmware):
            master_version = version_mapping.get(master_type)
            logger.info(' -> Updating Master firmware to {0}'.format(master_version if master_version else 'unknown version'))
            error = update_master_firmware(master_type, master_firmware, master_version)
            if error:
                errors.append(error)

        for module, filename, arguments in [('energy', FIRMWARE_FILES['energy'], []),
                                            ('power', FIRMWARE_FILES['power'], ['--8'])]:
            if os.path.exists(filename):
                energy_version = version_mapping.get(module)
                logger.info(' -> Updating {0} firmware to {1}'.format(module, energy_version if energy_version else 'unknown version'))
                error = update_energy_firmware(module, filename, energy_version, arguments)
                if error:
                    errors.append(error)

        for module in MODULE_TYPES:
            module_firmware = FIRMWARE_FILES[module]
            module_version = version_mapping.get(module)
            if os.path.exists(module_firmware):
                logger.info(' -> Updating {0} firmware to {1}'.format(module, module_version if module_version else 'unknown version'))
                error = update_module_firmware(module, module_firmware, module_version)
                if error:
                    errors.append(error)

        logger.info('Checking master communication')
        check_master_communication()

        gateway_frontend = FIRMWARE_FILES['gateway_frontend']
        if os.path.exists(gateway_frontend):
            frontend_version = version_mapping.get('gateway_frontend')
            logger.info(' -> Updating Gateway frontend to {0}'.format(frontend_version if frontend_version else 'unknown version'))
            error = update_gateway_frontend(gateway_frontend, date, frontend_version)
            if error:
                errors.append(error)

        if os.path.exists(gateway_frontend) or os.path.exists(gateway_service):
            clean_update_backups()

        logger.info(' -> Starting services')
        start_services()
        services_running = True

        logger.info(' -> Waiting for health check')
        check_gateway_health()

    except Exception as exc:
        logger.exception('Unexpected exception updating')
        errors.append(exc)
        # TODO: rollback
    finally:
        if not services_running:
            logger.info(' -> Starting services')
            start_services()

        logger.info(' -> Running cleanup')
        cmd('rm -v -rf {}/*'.format(update_dir), shell=True)

        if errors:
            logger.error('Exceptions:')
            for error in errors:
                logger.error('- {0}'.format(error))
            raise errors[0]

        config.set('OpenMotics', 'version', version)
        temp_file = constants.get_config_file() + '.update'
        with open(temp_file, 'wb') as configfile:
            config.write(configfile)
        shutil.move(temp_file, constants.get_config_file())
        cmd(['sync'])

        if os.path.exists('/tmp/post_update_reboot'):
            logger.info('Scheduling reboot in 5 minutes')
            subprocess.Popen('sleep 300 && reboot', close_fds=True, shell=True)

        logger.info('DONE')
        logger.info('exit 0')
from cbw_api_toolbox.cbw_api import CBWApi


def post_splunk(url, token, payload):
    """
        Send a post request to Splunk API
    """
    headers = {'Authorization': 'Splunk {}'.format(token)}
    res = requests.post(url=url, headers=headers, data=payload, verify=False)
    res.raise_for_status()
    return res.json()


# Load configuration from file api.conf
conf = ConfigParser()
conf.read(
    os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'api.conf'))

# Retrieve the computer from Cyberwatch API
CLIENT = CBWApi(conf.get('cyberwatch', 'url'),
                conf.get('cyberwatch', 'api_key'),
                conf.get('cyberwatch', 'secret_key'))
SERVER_ID = ''
server = CLIENT.server(SERVER_ID)

# Send the data to Splunk
post_splunk(
    conf.get('splunk', 'url'), conf.get('splunk', 'token'),
    '- Hostname: {}\n- Vulnerabilities count: {}'.format(
        server.hostname, server.cve_announcements_count))
    def __init__(self, app, conf):
        self.app = app
        self.logger = get_logger(conf, log_route='memcache')
        self.memcache_servers = conf.get('memcache_servers')
        serialization_format = conf.get('memcache_serialization_support')
        try:
            # Originally, while we documented using memcache_max_connections
            # we only accepted max_connections
            max_conns = int(
                conf.get('memcache_max_connections',
                         conf.get('max_connections', 0)))
        except ValueError:
            max_conns = 0

        memcache_options = {}
        if (not self.memcache_servers or serialization_format is None
                or max_conns <= 0):
            path = os.path.join(conf.get('swift_dir', '/etc/swift'),
                                'memcache.conf')
            memcache_conf = ConfigParser()
            if memcache_conf.read(path):
                # if memcache.conf exists we'll start with those base options
                try:
                    memcache_options = dict(memcache_conf.items('memcache'))
                except NoSectionError:
                    pass

                if not self.memcache_servers:
                    try:
                        self.memcache_servers = \
                            memcache_conf.get('memcache', 'memcache_servers')
                    except (NoSectionError, NoOptionError):
                        pass
                if serialization_format is None:
                    try:
                        serialization_format = \
                            memcache_conf.get('memcache',
                                              'memcache_serialization_support')
                    except (NoSectionError, NoOptionError):
                        pass
                if max_conns <= 0:
                    try:
                        new_max_conns = \
                            memcache_conf.get('memcache',
                                              'memcache_max_connections')
                        max_conns = int(new_max_conns)
                    except (NoSectionError, NoOptionError, ValueError):
                        pass

        # while memcache.conf options are the base for the memcache
        # middleware, if you set the same option also in the filter
        # section of the proxy config it is more specific.
        memcache_options.update(conf)
        connect_timeout = float(
            memcache_options.get('connect_timeout', CONN_TIMEOUT))
        pool_timeout = float(memcache_options.get('pool_timeout',
                                                  POOL_TIMEOUT))
        tries = int(memcache_options.get('tries', TRY_COUNT))
        io_timeout = float(memcache_options.get('io_timeout', IO_TIMEOUT))

        if not self.memcache_servers:
            self.memcache_servers = '127.0.0.1:11211'
        if max_conns <= 0:
            max_conns = 2
        if serialization_format is None:
            serialization_format = 2
        else:
            serialization_format = int(serialization_format)

        self.memcache = MemcacheRing(
            [s.strip() for s in self.memcache_servers.split(',') if s.strip()],
            connect_timeout=connect_timeout,
            pool_timeout=pool_timeout,
            tries=tries,
            io_timeout=io_timeout,
            allow_pickle=(serialization_format == 0),
            allow_unpickle=(serialization_format <= 1),
            max_conns=max_conns,
            logger=self.logger)
Example #46
0
class ToxIni(base.ConfigBase):

    _ini = None
    tox_ini_file = 'tox.ini'
    default_section = 'testenv'

    def _open_tox_ini(self):
        if self._ini is None:
            self._ini = ConfigParser()
            self._ini.read(self.tox_ini_file)
        return self._ini

    def source_name(self):
        return self.tox_ini_file

    def exists(self):
        return os.path.exists(self.tox_ini_file)

    def get_images(self):
        ini = self._open_tox_ini()
        if ini.has_option('docker', 'images'):
            return ini.get('docker', 'images').split(',')

    def get_commands(self, extra_args):
        """Get commands to run from the config file.

        If any of the commands contain the string '{posargs}', then this
        is replaced with the extra_args value.
        """
        section = self.options.get('section', self.default_section)

        if section == '_default':
            section = self.default_section

        ini = self._open_tox_ini()
        commands = ini.get(section, 'commands').split("\n")
        extra_args = " ".join(extra_args)

        scrubbed = []
        for cmd in commands:
            if '{posargs}' in cmd:
                scrubbed.append(cmd.replace('{posargs}', extra_args))
            else:
                scrubbed.append(cmd)
        return scrubbed

    def get_prep_commands(self):
        ini = self._open_tox_ini()
        deps = ""
        if ini.has_option('testenv', 'deps'):
            deps = ini.get('testenv', 'deps')
        deps = deps.replace('{toxinidir}', '/dox').replace('\n', ' ')
        if deps.strip() == '':
            return []
        install_command = "pip install -U"
        if ini.has_option('testenv', 'install_command'):
            install_command = ini.get('testenv', 'install_command')
        install_command = install_command.replace('{opts}', '')
        install_command = install_command.replace('{packages}', deps)
        return [install_command]

    def get_add_files(self):
        return [d for d in os.listdir('.') if d.endswith('requirements.txt')]
Example #47
0
class GlobusConfigParser(object):
    """
    Wraps a ConfigParser to do modified get()s and config file loading.
    """
    _GENERAL_CONF_SECTION = 'general'

    def __init__(self):
        logger.debug("Loading SDK Config parser")
        self._parser = ConfigParser()
        self._load_config()
        logger.debug("Config load succeeded")

    def _load_config(self):
        # TODO: /etc is not windows friendly, not sure about expanduser
        try:
            self._parser.read([_get_lib_config_path(), "/etc/globus.cfg",
                               os.path.expanduser("~/.globus.cfg")])
        except MissingSectionHeaderError:
            logger.error(("MissingSectionHeader means invalid config "
                          "somewhere, and is often an indicator of a stale "
                          "early form of the Globus SDK config"))
            raise GlobusError(
                "Failed to parse your ~/.globus.cfg Your config file may be "
                "in an old format. Please ensure that the file's first line "
                "is \"[general]\"")

    def get(self, option,
            section=None, environment=None,
            failover_to_general=False, check_env=False,
            type_cast=str):
        """
        Attempt to lookup an option in the config file. Optionally failover to
        the general section if the option is not found.

        Also optionally, check for a relevant environment variable, which is
        named always as GLOBUS_SDK_{option.upper()}. Note that 'section'
        doesn't slot into the naming at all. Otherwise, we'd have to contend
        with GLOBUS_SDK_GENERAL_... for almost everything, and
        GLOBUS_SDK_ENVIRONMENT\ PROD_... which is awful.

        Returns None for an unfound key, rather than raising a NoOptionError.
        """
        # envrionment is just a fancy name for sections that start with
        # 'environment '
        if environment:
            section = 'environment ' + environment
        # if you don't specify a section or an environment, assume it's the
        # general conf section
        if section is None:
            section = self._GENERAL_CONF_SECTION

        # if this is a config option which checks the shell env, look there
        # *first* for a value -- env values have higher precedence than config
        # files so that you can locally override the behavior of a command in a
        # given shell or subshell
        env_option_name = 'GLOBUS_SDK_{}'.format(option.upper())
        value = None
        if check_env and env_option_name in os.environ:
            logger.debug("Getting config value from environment: {}={}"
                         .format(env_option_name, value))
            value = os.environ[env_option_name]
        else:
            try:
                value = self._parser.get(section, option)
            except (NoOptionError, NoSectionError):
                if failover_to_general:
                    logger.debug("Config lookup of [{}]:{} failed, checking "
                                 "[general] for a value as well"
                                 .format(section, option))
                    value = self.get(option,
                                     section=self._GENERAL_CONF_SECTION)

        if value is not None:
            value = type_cast(value)
        return value
Example #48
0
        return "Public name cannot be more than 255 characters in length"
    if not (VALID_PUBLICNAME_RE.match(username)):
        return "Public name must contain only lower-case letters, numbers and '-'"
    return ''


if __name__ == "__main__":
    parser = optparse.OptionParser(description='Create a user with API key.')
    parser.add_option('-c',
                      dest='config',
                      action='store',
                      help='.ini file to retried toolshed configuration from')
    (args, options) = parser.parse_args()
    ini_file = args.config
    config_parser = ConfigParser({'here': os.getcwd()})
    print("Reading ini file: ", ini_file)
    config_parser.read(ini_file)
    config_dict = {}
    for key, value in config_parser.items("app:main"):
        config_dict[key] = value
    config = tool_shed_config.Configuration(**config_dict)
    app = BootstrapApplication(config)
    user = create_user(app)
    if user is not None:
        api_key = create_api_key(app, user)
        print("Created new user with public username '", user.username,
              ".  An API key was also created and associated with the user.")
        sys.exit(0)
    else:
        sys.exit("Problem creating a new user and an associated API key.")
Example #49
0
 def get_main_version(self):
     """ Gets reported main version """
     _ = self
     config = ConfigParser()
     config.read(constants.get_config_file())
     return str(config.get('OpenMotics', 'version'))
Example #50
0
def main():
    '''Main(). Commandline parsing and stalker startup.'''

    parser = argparse.ArgumentParser()

    parser.add_argument("-p",
                        "--posttroll_port",
                        dest="posttroll_port",
                        help="Local port where messages are published")
    parser.add_argument("-t",
                        "--topic",
                        dest="topic",
                        help="Topic of the sent messages")
    parser.add_argument("-c",
                        "--configuration_file",
                        help="Name of the config.ini configuration file")
    parser.add_argument("-C",
                        "--config_item",
                        help="Name of the configuration item to use")
    parser.add_argument("-e",
                        "--event_names",
                        help="Name of the events to monitor")
    parser.add_argument("-f",
                        "--filepattern",
                        help="Filepath pattern used to parse "
                        "satellite/orbit/date/etc information")
    parser.add_argument("-i",
                        "--instrument",
                        help="Instrument name in the satellite")

    if len(sys.argv) <= 1:
        parser.print_help()
        sys.exit()
    else:
        args = parser.parse_args()

    # Parse commandline arguments.  If args are given, they override
    # the configuration file.

    args_dict = vars(args)
    args_dict = {k: args_dict[k] for k in args_dict if args_dict[k] != None}

    config = {}

    if args.configuration_file is not None:
        config_fname = args.configuration_file

        if "template" in config_fname:
            print("Template file given as trollstalker logging config,"
                  " aborting!")
            sys.exit()

        cparser = ConfigParser()
        cparser.read(config_fname)
        config = dict(cparser.items(args.config_item, vars=args_dict))

    config.update(args_dict)

    config.update(
        {k: config[k].split(",")
         for k in config if "," in config[k]})

    config.setdefault("posttroll_port", "0")

    try:
        log_config = config["stalker_log_config"]
    except KeyError:
        try:
            loglevel = getattr(logging, config.get("loglevel", "DEBUG"))
            if loglevel == "":
                raise AttributeError
        except AttributeError:
            loglevel = logging.DEBUG

        LOGGER.setLevel(loglevel)
        rootlogger = logging.getLogger("")
        rootlogger.setLevel(loglevel)
        strhndl = logging.StreamHandler()
        strhndl.setLevel(loglevel)
        log_format = "[%(asctime)s %(levelname)-8s %(name)s] %(message)s"
        formatter = logging.Formatter(log_format)

        strhndl.setFormatter(formatter)
        rootlogger.addHandler(strhndl)
    else:
        logging.config.fileConfig(log_config)

    LOGGER.debug("Logger started")

    # Start watching for new files
    notifier = FilePublisher(config)
    notifier.start()

    try:
        while True:
            time.sleep(6000000)
    except KeyboardInterrupt:
        LOGGER.info("Interrupting TrollStalker")
    finally:
        notifier.stop()
Example #51
0
    'AAAA': ':{domain}:28:{value}:{ttl}',
    'MX': '@{domain}::{value}:{dist}:{ttl}',
    'NS': '.{domain}::{value}:{ttl}',
    'TXT': '\'{domain}:{value}:{ttl}',
    'CNAME': 'C{domain}:{value}:{ttl}',
}
MAX_RECORDS_COUNT = 3000

DNSPOD_TIMEOUT = 5
DNSPOD_RETRIES = 3

dns_monitor_cb_key_func = lambda: get_config('dns_monitor_callback_key'
                                             )  # NOQA

external_domains_config = ConfigParser(allow_no_value=True)
external_domains_config.read(EXTERNAL_DOMAINS_CONFIG_FILE)


def get_api_token_by_domain(domain):
    """
    config file example:
    token_id,token,domain
    111,yyy,*
    222,xxx,domain1.com|domain2.com|domain3.co
    """
    result = {}
    for conf in get_config('dnspod').splitlines():
        id_token, domains = conf.rsplit(',', 1)
        for scope in domains.split('|'):
            result[scope] = id_token
    logger.debug("domain config is %s", result)
Example #52
0
class PypiConfig(BaseConfig):
    """Wrapper around the pypi config file"""
    def __init__(self, config_filename=DIST_CONFIG_FILE, use_setup_cfg=True):
        """Grab the PyPI configuration.

        This is .pypirc in the home directory.  It is overridable for
        test purposes.

        If there is a setup.cfg file in the current directory, we read
        it too.
        """
        self.config_filename = config_filename
        self.use_setup_cfg = use_setup_cfg
        self.reload()

    def reload(self):
        """Load the config.

        Do the initial load of the config.

        Or reload it in case of problems: this is needed when a pypi
        upload fails, you edit the .pypirc file to fix the account
        settings, and tell release to retry the command.
        """
        self._read_configfile(use_setup_cfg=self.use_setup_cfg)

    def _read_configfile(self, use_setup_cfg=True):
        """Read the PyPI config file and store it (when valid).

        Usually read the setup.cfg too.
        """
        rc = self.config_filename
        if not os.path.isabs(rc):
            rc = os.path.join(os.path.expanduser('~'), self.config_filename)
        filenames = [rc]
        if use_setup_cfg:
            # If there is a setup.cfg in the package, parse it
            filenames.append('setup.cfg')
        files = [f for f in filenames if os.path.exists(f)]
        if not files:
            self.config = None
            return
        self.config = ConfigParser()
        self.config.read(files)

    def is_pypi_configured(self):
        # Do we have configuration for releasing to at least one
        # pypi-compatible server?
        if self.config is None:
            return False
        return len(self.distutils_servers()) > 0

    def distutils_servers(self):
        """Return a list of known distutils servers.

        If the config has an old pypi config, remove the default pypi
        server from the list.
        """
        try:
            index_servers = self._get_text('distutils',
                                           'index-servers',
                                           default='').split()
        except (NoSectionError, NoOptionError):
            index_servers = []
        if not index_servers:
            # If no distutils index-servers have been given, 'pypi' should be
            # the default.  This is what twine does.
            if self.config.has_option('server-login', 'username'):
                # We have a username, so upload to pypi should work fine, even
                # when no explicit pypi section is in the file.
                return ['pypi']
            # https://github.com/zestsoftware/zest.releaser/issues/199
            index_servers = ['pypi']
        # The servers all need to have a section in the config file.
        return [
            server for server in index_servers
            if self.config.has_section(server)
        ]

    def want_release(self):
        """Does the user normally want to release this package.

        Some colleagues find it irritating to have to remember to
        answer the question "Check out the tag (for tweaks or
        pypi/distutils server upload)" with the non-default 'no' when
        in 99 percent of the cases they just make a release specific
        for a customer, so they always answer 'no' here.  This is
        where an extra config option comes in handy: you can influence
        the default answer so you can just keep hitting 'Enter' until
        zest.releaser is done.

        Either in your ~/.pypirc or in a setup.cfg in a specific
        package, add this when you want the default answer to this
        question to be 'no':

        [zest.releaser]
        release = no

        The default when this option has not been set is True.

        Standard config rules apply, so you can use upper or lower or
        mixed case and specify 0, false, no or off for boolean False,
        and 1, on, true or yes for boolean True.
        """
        return self._get_boolean('zest.releaser', 'release', default=True)

    def extra_message(self):
        """Return extra text to be added to commit messages.

        This can for example be used to skip CI builds.  This at least
        works for Travis.  See
        http://docs.travis-ci.com/user/how-to-skip-a-build/

        Enable this mode by adding a ``extra-message`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            extra-message = [ci skip]
        """
        default = ''
        if self.config is None:
            return default
        try:
            result = self._get_text('zest.releaser',
                                    'extra-message',
                                    default=default)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def history_file(self):
        """Return path of history file.

        Usually zest.releaser can find the correct one on its own.
        But sometimes it may not find anything, or it finds multiple
        and selects the wrong one.

        Configure this by adding a ``history-file`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            history-file = deep/down/historie.doc
        """
        default = ''
        if self.config is None:
            return default
        marker = object()
        try:
            result = self._get_text('zest.releaser',
                                    'history-file',
                                    default=marker)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        if result == marker:
            # We were reading an underscore instead of a dash at first.
            try:
                result = self._get_text('zest.releaser',
                                        'history_file',
                                        default=default)
            except (NoSectionError, NoOptionError, ValueError):
                return default
        return result

    def encoding(self):
        """Return encoding to use for text files.

        Mostly the changelog and if needed `setup.py`.

        The encoding cannot always be determined correctly.
        This setting is a way to fix that.
        See https://github.com/zestsoftware/zest.releaser/issues/264

        Configure this by adding an ``encoding`` option, either in the
        package you want to release, or in your ~/.pypirc::

            [zest.releaser]
            encoding = utf-8
        """
        default = ''
        if self.config is None:
            return default
        try:
            result = self._get_text('zest.releaser',
                                    'encoding',
                                    default=default,
                                    raw=True)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def create_wheel(self):
        """Should we create a Python wheel for this package?

        Either in your ~/.pypirc or in a setup.cfg in a specific
        package, add this when you want to create a Python wheel, next
        to a standard sdist:

        [zest.releaser]
        create-wheel = yes

        If there is no setting for ``create-wheel``, then if there is a
        ``[bdist_wheel]`` section, it is treated as if
        ``create-wheel`` was true.  We used to look at the value of
        the ``universal`` option, but that no longer matters.
        This will still create a wheel:

        [bdist_wheel]
        universal = 0

        See https://github.com/zestsoftware/zest.releaser/issues/315
        """
        if not USE_WHEEL:
            # If the wheel package is not available, we obviously
            # cannot create wheels.
            return False
        create_setting = self._get_boolean('zest.releaser', 'create-wheel',
                                           None)
        if create_setting is not None:
            # User specified this setting, it overrides
            # inferring from bdist_wheel
            return create_setting
        # No zest.releaser setting, are they asking for a universal wheel?
        # Then they want wheels in general.
        return self.config.has_section('bdist_wheel')

    def register_package(self):
        """Should we try to register this package with a package server?

        For the standard Python Package Index (PyPI), registering a
        package is no longer needed: this is done automatically when
        uploading a distribution for a package.  In fact, trying to
        register may fail.  See
        https://github.com/zestsoftware/zest.releaser/issues/191
        So by default zest.releaser will no longer register a package.

        But you may be using your own package server, and registering
        may be wanted or even required there.  In this case
        you will need to turn on the register function.
        In your setup.cfg or ~/.pypirc, use the following to ensure that
        register is called on the package server:

        [zest.releaser]
        register = yes

        Note that if you have specified multiple package servers, this
        option is used for all of them.  There is no way to register and
        upload to server A, and only upload to server B.
        """
        return self._get_boolean('zest.releaser', 'register')

    def no_input(self):
        """Return whether the user wants to run in no-input mode.

        Enable this mode by adding a ``no-input`` option::

            [zest.releaser]
            no-input = yes

        The default when this option has not been set is False.
        """
        return self._get_boolean('zest.releaser', 'no-input')

    def development_marker(self):
        """Return development marker to be appended in postrelease.

        Override the default ``.dev0`` in ~/.pypirc or setup.cfg using
        a ``development-marker`` option::

            [zest.releaser]
            development-marker = .dev1

        Returns default of ``.dev0`` when nothing has been configured.
        """
        default = '.dev0'
        if self.config is None:
            return default
        try:
            result = self._get_text('zest.releaser',
                                    'development-marker',
                                    default=default)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result

    def push_changes(self):
        """Return whether the user wants to push the changes to the remote.

        Configure this mode by adding a ``push-changes`` option::

            [zest.releaser]
            push-changes = no

        The default when this option has not been set is True.
        """
        return self._get_boolean('zest.releaser', 'push-changes', default=True)

    def less_zeroes(self):
        """Return whether the user prefers less zeroes at the end of a version.

        Configure this mode by adding a ``less-zeroes`` option::

            [zest.releaser]
            less-zeroes = yes

        The default when this option has not been set is False.

        When set to true:
        - Instead of 1.3.0 we will suggest 1.3.
        - Instead of 2.0.0 we will suggest 2.0.

        This only makes sense for the bumpversion command.
        In the postrelease command we read this option too,
        but with the current logic it has no effect there.
        """
        return self._get_boolean('zest.releaser', 'less-zeroes')

    def version_levels(self):
        """How many levels does the user prefer in a version number?

        Configure this mode by adding a ``version-levels`` option::

            [zest.releaser]
            version-levels = 3

        The default when this option has not been set is 0, which means:
        no preference, so use the length of the current number.

        This means when suggesting a next version after 1.2:
        - with levels=0 we will suggest 1.3: no change
        - with levels=1 we will still suggest 1.3, as we will not
          use this to remove numbers, only to add them
        - with levels=2 we will suggest 1.3
        - with levels=3 we will suggest 1.2.1

        If the current version number has more levels, we keep them.
        So next version for 1.2.3.4 with levels=1 is 1.2.3.5.

        Tweaking version-levels and less-zeroes should give you the
        version number strategy that you prefer.
        """
        default = 0
        if self.config is None:
            return default
        try:
            result = self.config.getint('zest.releaser', 'version-levels')
        except (NoSectionError, NoOptionError, ValueError):
            return default
        if result < 0:
            return default
        return result

    _tag_format_deprecated_message = "\n".join(line.strip() for line in """
    `tag-format` contains deprecated `%%(version)s` format. Please change to:

    [zest.releaser]
    tag-format = %s
    """.strip().splitlines())

    def tag_format(self, version):
        """Return the formatted tag that should be used in the release.

        Configure it in ~/.pypirc or setup.cfg using a ``tag-format`` option::

            [zest.releaser]
            tag-format = v{version}

        ``tag-format`` must contain exaclty one formatting instruction: for the
        ``version`` key.

        Accepts also ``%(version)s`` format for backward compatibility.

        The default format, when nothing has been configured, is ``{version}``.
        """
        fmt = '{version}'
        if self.config is not None:
            try:
                fmt = self._get_text('zest.releaser',
                                     'tag-format',
                                     default=fmt,
                                     raw=True)
            except (NoSectionError, NoOptionError, ValueError):
                pass
        if '{version}' in fmt:
            return fmt.format(version=version)
        # BBB:
        if '%(version)s' in fmt:
            proposed_fmt = fmt.replace("%(version)s", "{version}")
            print(self._tag_format_deprecated_message % proposed_fmt)
            return fmt % {'version': version}
        print("{version} needs to be part of 'tag-format': %s" % fmt)
        sys.exit(1)

    def tag_message(self, version):
        """Return the commit message to be used when tagging.

        Configure it in ~/.pypirc or setup.cfg using a ``tag-message``
        option::

            [zest.releaser]
            tag-message = Creating v{version} tag.

        ``tag-message`` must contain exaclty one formatting
        instruction: for the ``version`` key.

        The default format is ``Tagging {version}``.
        """
        fmt = 'Tagging {version}'
        if self.config:
            try:
                fmt = self._get_text('zest.releaser',
                                     'tag-message',
                                     default=fmt,
                                     raw=True)
            except (NoSectionError, NoOptionError, ValueError):
                pass
        if '{version}' not in fmt:
            print("{version} needs to be part of 'tag-message': '%s'" % fmt)
            sys.exit(1)
        return fmt.format(version=version)

    def tag_signing(self):
        """Return whether the tag should be signed.

        Configure it in ~/.pypirc or setup.cfg using a ``tag-signing`` option::

            [zest.releaser]
            tag-signing = yes

        ``tag-signing`` must contain exaclty one word which will be
        converted to a boolean. Currently are accepted (case
        insensitively): 0, false, no, off for False, and 1, true, yes,
        on for True).

        The default when this option has not been set is False.

        """
        return self._get_boolean('zest.releaser', 'tag-signing', default=False)

    def date_format(self):
        """Return the string format for the date used in the changelog.

        Override the default ``%Y-%m-%d`` in ~/.pypirc or setup.cfg using
        a ``date-format`` option::

            [zest.releaser]
            date-format = %%B %%e, %%Y

        Note: the % signs should be doubled for compatibility with other tools
        (i.e. pip) that parse setup.cfg using the interpolating ConfigParser.

        Returns default of ``%Y-%m-%d`` when nothing has been configured.
        """
        default = '%Y-%m-%d'
        if self.config is None:
            return default
        try:
            result = self._get_text('zest.releaser',
                                    'date-format',
                                    default=default).replace('%%', '%')
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result
Example #53
0
def bodhi_container(
    docker_backend, docker_network, db_container, resultsdb_container,
    waiverdb_container, greenwave_container
):
    """Fixture preparing and yielding a Bodhi container to test against.

    Args:
        docker_backend (conu.DockerBackend): The Docker backend (fixture).
        docker_network (str): The Docker network ID (fixture).
        db_container (conu.DockerContainer): The PostgreSQL container (fixture).
        resultsdb_container (conu.DockerContainer): The ResultsDB container
            (fixture).
        waiverdb_container (conu.DockerContainer): The WaiverDB container
            (fixture).
        greenwave_container (conu.DockerContainer): The Greenwave container
            (fixture).

    Yields:
        conu.DockerContainer: The Bodhi container.
    """
    # Prepare the database
    make_db_and_user(db_container, "bodhi2", True)
    image = docker_backend.ImageClass(
        os.environ.get("BODHI_INTEGRATION_IMAGE", "bodhi-ci-integration-bodhi")
    )
    container = image.run_via_api()
    config_override = {
        "base_address": "https://bodhi.fedoraproject.org/",
        "sqlalchemy.url": "postgresql://bodhi2@db/bodhi2",
        "pungi.cmd": "/bin/true",
        "dogpile.cache.backend": "dogpile.cache.memory_pickle",
        "legal_link": "https://fedoraproject.org/wiki/Legal:Main",
        "privacy_link": "https://fedoraproject.org/wiki/Legal:PrivacyPolicy",
        "datagrepper_url": "https://apps.fedoraproject.org/datagrepper",
        "authtkt.secret": uuid4().hex,
        "session.secret": uuid4().hex,
        "query_wiki_test_cases": "True",
        "wiki_url": "https://fedoraproject.org/w/api.php",
        "test_case_base_url": "https://fedoraproject.org/wiki/",
        "resultsdb_url": "http://resultsdb/resultsdb/",
        "test_gating.required": "True",
        "greenwave_api_url": "http://greenwave:8080/api/v1.0",
        "waiverdb_api_url": "http://waiverdb:8080/api/v1.0",
        "default_email_domain": "fedoraproject.org",
        # Only on bodhi-backend
        # "compose_dir": "/mnt/koji/compose/updates/",
        # "compose_stage_dir": "/mnt/koji/compose/updates/",
        "max_concurrent_composes": "3",
        "clean_old_composes": "false",
        "pungi.conf.rpm": "pungi.rpm.conf.j2",
        "pungi.conf.module": "pungi.module.conf.j2",
        "pungi.extracmdline":
            "--notification-script=/usr/bin/pungi-fedmsg-notification "
            "--notification-script=pungi-wait-for-signed-ostree-handler",
        "max_update_length_for_ui": "70",
        "top_testers_timeframe": "900",
        "buildsystem": "koji",
        "fedmenu.url": "https://apps.fedoraproject.org/fedmenu",
        "fedmenu.data_url": "https://apps.fedoraproject.org/js/data.js",
        "acl_system": "pagure",
        "bugtracker": "bugzilla",
        "bz_products": "Fedora,Fedora EPEL",
        "reboot_pkgs": "kernel kernel-smp kernel-PAE glibc hal dbus",
        "critpath.type": "pdc",
        "critpath.num_admin_approvals": "0",
        "fedora_modular.mandatory_days_in_testing": "7",
        "buildroot_overrides.expire_after": "1",
        "pyramid.reload_templates": "false",
        "pyramid.debug_authorization": "false",
        "pyramid.debug_notfound": "false",
        "pyramid.debug_routematch": "false",
        "authtkt.secure": "true",
        "authtkt.timeout": "1209600",
    }
    with edit_file(container, "/etc/bodhi/production.ini") as config_path:
        config = ConfigParser()
        config.read(config_path)
        for key, value in config_override.items():
            config.set("app:main", key, value)

        with open(config_path, "w") as config_file:
            config.write(config_file)

    container.start()
    docker_backend.d.connect_container_to_network(
        container.get_id(), docker_network["Id"], aliases=["bodhi"],
    )
    # Update the database schema
    container.execute(["alembic-3", "-c", "/bodhi/alembic.ini", "upgrade", "head"])
    # we need to wait for the webserver to start serving
    container.wait_for_port(8080, timeout=30)
    yield container
    container.kill()
    container.delete()
Example #54
0
class SetupConfig(BaseConfig):
    """Wrapper around the setup.cfg file if available.

    One reason is to cleanup setup.cfg from these settings::

        [egg_info]
        tag_build = dev
        tag_svn_revision = true

    Another is for optional zest.releaser-specific settings::

        [zest.releaser]
        python-file-with-version = reinout/maurits.py


    """

    config_filename = SETUP_CONFIG_FILE

    def __init__(self):
        """Grab the configuration (overridable for test purposes)"""
        # If there is a setup.cfg in the package, parse it
        if not os.path.exists(self.config_filename):
            self.config = None
            return
        self.config = ConfigParser()
        self.config.read(self.config_filename)

    def has_bad_commands(self):
        if self.config is None:
            return False
        if not self.config.has_section('egg_info'):
            # bail out early as the main section is not there
            return False
        bad = False
        # Check 1.
        if self.config.has_option('egg_info', 'tag_build'):
            # Might still be empty.
            value = self._get_text('egg_info', 'tag_build')
            if value:
                logger.warning("%s has [egg_info] tag_build set to '%s'",
                               self.config_filename, value)
                bad = True
        # Check 2.
        if self.config.has_option('egg_info', 'tag_svn_revision'):
            if self.config.getboolean('egg_info', 'tag_svn_revision'):
                value = self._get_text('egg_info', 'tag_svn_revision')
                logger.warning(
                    "%s has [egg_info] tag_svn_revision set to '%s'",
                    self.config_filename, value)
                bad = True
        return bad

    def fix_config(self):
        if not self.has_bad_commands():
            logger.warning("Cannot fix already fine %s.", self.config_filename)
            return
        if self.config.has_option('egg_info', 'tag_build'):
            self.config.set('egg_info', 'tag_build', '')
        if self.config.has_option('egg_info', 'tag_svn_revision'):
            self.config.set('egg_info', 'tag_svn_revision', 'false')
        new_setup = open(self.config_filename, 'w')
        try:
            self.config.write(new_setup)
        finally:
            new_setup.close()
        logger.info("New setup.cfg contents:")
        with open(self.config_filename) as config_file:
            print(''.join(config_file.readlines()))

    def python_file_with_version(self):
        """Return Python filename with ``__version__`` marker, if configured.

        Enable this by adding a ``python-file-with-version`` option::

            [zest.releaser]
            python-file-with-version = reinout/maurits.py

        Return None when nothing has been configured.

        """
        default = None
        if self.config is None:
            return default
        try:
            result = self._get_text('zest.releaser',
                                    'python-file-with-version',
                                    default=default)
        except (NoSectionError, NoOptionError, ValueError):
            return default
        return result
Example #55
0
        ],
        keywords=
        "deep learning artificial intelligence machine learning neural network",
        python_requires='>=3.5',
    )

    ############################################################################
    # Parse setup.cfg
    from six.moves.configparser import ConfigParser
    path_cfg = os.path.join(os.path.dirname(__file__), "setup.cfg")
    if not os.path.isfile(path_cfg):
        raise ValueError(
            "`setup.cfg` does not exist. Read installation document and install using CMake."
        )
    cfgp = ConfigParser()
    cfgp.read(path_cfg)
    build_dir = cfgp.get("cmake", "build_dir")

    ############################################################################
    # Extension module information
    src_dir = os.path.join(root_dir, 'src')
    path_pkg = os.path.join(src_dir, 'nnabla')

    library_name = cfgp.get("cmake", "target_name")
    library_file_name = cfgp.get("cmake", "target_file_name")
    library_path = cfgp.get("cmake", "target_file")
    library_dir = os.path.dirname(library_path)

    ext_opts = extopts(library_name, library_dir)

    module_names = [
Example #56
0
def read_opts():

    parser = OptionParser()
    parser.add_option("--cfg")  # Mandatory
    (options, args) = parser.parse_args()

    cfg_file = options.cfg
    Config = ConfigParser()
    Config.read(cfg_file)

    # DATA
    options.out_folder = Config.get('data', 'out_folder')
    options.tr_fea_scp = Config.get('data', 'tr_fea_scp')
    options.tr_fea_opts = Config.get('data', 'tr_fea_opts')
    options.tr_lab_folder = Config.get('data', 'tr_lab_folder')
    options.tr_lab_opts = Config.get('data', 'tr_lab_opts')

    options.dev_fea_scp = Config.get('data', 'dev_fea_scp')
    options.dev_fea_opts = Config.get('data', 'dev_fea_opts')
    options.dev_lab_folder = Config.get('data', 'dev_lab_folder')
    options.dev_lab_opts = Config.get('data', 'dev_lab_opts')

    options.te_fea_scp = Config.get('data', 'te_fea_scp')
    options.te_fea_opts = Config.get('data', 'te_fea_opts')
    options.te_lab_folder = Config.get('data', 'te_lab_folder')
    options.te_lab_opts = Config.get('data', 'te_lab_opts')

    options.count_file = Config.get('data', 'count_file')
    options.pt_file = Config.get('data', 'pt_file')

    # ARCHITECTURE
    options.hidden_dim = Config.get('architecture', 'hidden_dim')
    options.N_hid = Config.get('architecture', 'N_hid')
    options.drop_rate = Config.get('architecture', 'drop_rate')
    options.use_batchnorm = Config.get('architecture', 'use_batchnorm')
    options.cw_left = Config.get('architecture', 'cw_left')
    options.cw_right = Config.get('architecture', 'cw_right')
    options.seed = Config.get('architecture', 'seed')
    options.use_cuda = Config.get('architecture', 'use_cuda')
    options.multi_gpu = Config.get('architecture', 'multi_gpu')

    if Config.has_option('architecture', 'bidir'):
        options.bidir = Config.get('architecture', 'bidir')

    if Config.has_option('architecture', 'resnet'):
        options.resnet = Config.get('architecture', 'resnet')

    if Config.has_option('architecture', 'act'):
        options.act = Config.get('architecture', 'act')

    if Config.has_option('architecture', 'resgate'):
        options.resgate = Config.get('architecture', 'resgate')

    if Config.has_option('architecture', 'minimal_gru'):
        options.minimal_gru = Config.get('architecture', 'minimal_gru')

    if Config.has_option('architecture', 'skip_conn'):
        options.skip_conn = Config.get('architecture', 'skip_conn')

    if Config.has_option('architecture', 'act_gate'):
        options.act_gate = Config.get('architecture', 'act_gate')

    if Config.has_option('architecture', 'use_laynorm'):
        options.use_laynorm = Config.get('architecture', 'use_laynorm')

    if Config.has_option('architecture', 'cost'):
        options.cost = Config.get('architecture', 'cost')

    if Config.has_option('architecture', 'NN_type'):
        options.NN_type = Config.get('architecture', 'NN_type')

    if Config.has_option('architecture', 'twin_reg'):
        options.twin_reg = Config.get('architecture', 'twin_reg')

    if Config.has_option('architecture', 'twin_w'):
        options.twin_w = Config.get('architecture', 'twin_w')

    if Config.has_option('architecture', 'cnn_pre'):
        options.cnn_pre = Config.get('architecture', 'cnn_pre')

    options.N_ep = Config.get('optimization', 'N_ep')
    options.lr = Config.get('optimization', 'lr')
    options.halving_factor = Config.get('optimization', 'halving_factor')
    options.improvement_threshold = Config.get('optimization',
                                               'improvement_threshold')
    options.batch_size = Config.get('optimization', 'batch_size')
    options.save_gpumem = Config.get('optimization', 'save_gpumem')
    options.optimizer = Config.get('optimization', 'optimizer')

    return options
Example #57
0
    def add_repo(self,
                 name,
                 uri,
                 repo_type='rpm-md',
                 prio=None,
                 dist=None,
                 components=None,
                 user=None,
                 secret=None,
                 credentials_file=None,
                 repo_gpgcheck=None,
                 pkg_gpgcheck=None):
        """
        Add zypper repository

        :param str name: repository name
        :param str uri: repository URI
        :param repo_type: repostory type name
        :param int prio: zypper repostory priority
        :param dist: unused
        :param components: unused
        :param user: credentials username
        :param secret: credentials password
        :param credentials_file: zypper credentials file
        :param bool repo_gpgcheck: enable repository signature validation
        :param bool pkg_gpgcheck: enable package signature validation
        """
        if credentials_file:
            repo_secret = os.sep.join(
                [self.shared_zypper_dir['credentials-dir'], credentials_file])
            if os.path.exists(repo_secret):
                Path.wipe(repo_secret)

            if user and secret:
                uri = ''.join([uri, '?credentials=', credentials_file])
                with open(repo_secret, 'w') as credentials:
                    credentials.write('username={0}{1}'.format(
                        user, os.linesep))
                    credentials.write('password={0}{1}'.format(
                        secret, os.linesep))

        repo_file = ''.join(
            [self.shared_zypper_dir['reposd-dir'], '/', name, '.repo'])
        self.repo_names.append(''.join([name, '.repo']))

        if os.path.exists(repo_file):
            Path.wipe(repo_file)

        self._backup_package_cache()
        zypper_addrepo_command = ['zypper'] + self.zypper_args + [
            '--root', self.root_dir, 'addrepo', '--refresh', '--keep-packages'
            if Uri(uri).is_remote() else '--no-keep-packages', '--no-check',
            uri, name
        ]
        try:
            Command.run(zypper_addrepo_command, self.command_env)
        except Exception:
            # for whatever reason zypper sometimes failes with
            # a 'failed to cache rpm database' error. I could not
            # find any reason why and a simple recall of the exact
            # same command in the exact same environment works.
            # Thus the stupid but simple workaround to this problem
            # is try one recall before really failing
            Command.run(zypper_addrepo_command, self.command_env)

        if prio or repo_gpgcheck is not None or pkg_gpgcheck is not None:
            repo_config = ConfigParser()
            repo_config.read(repo_file)
            if repo_gpgcheck is not None:
                repo_config.set(name, 'repo_gpgcheck',
                                '1' if repo_gpgcheck else '0')
            if pkg_gpgcheck is not None:
                repo_config.set(name, 'pkg_gpgcheck',
                                '1' if pkg_gpgcheck else '0')
            if prio:
                repo_config.set(name, 'priority', format(prio))
            with open(repo_file, 'w') as repo:
                repo_config.write(repo)
        self._restore_package_cache()
Example #58
0
def main():
    conf = ConfigParser()

    parser = argparse.ArgumentParser()
    parser.add_argument("configuration_file",
                        help="the configuration file to use")
    parser.add_argument("--dry-run",
                        help="do not actually run, just fake it",
                        action="store_true")
    parser.add_argument(
        "-c",
        "--config-item",
        help="just run this config_item, can be provided several times",
        default=[],
        action="append")
    parser.add_argument("-l",
                        "--logfile",
                        help="file to log to (stdout by default)")
    parser.add_argument("-v",
                        "--verbose",
                        help="increase the verbosity of the script",
                        action="store_true")
    parser.add_argument("-q",
                        "--quiet",
                        help="decrease the verbosity of the script",
                        action="store_true")
    parser.add_argument("-m",
                        "--mail",
                        help="send errors and warning via mail",
                        action="store_true")

    args = parser.parse_args()

    logger = logging.getLogger("remove_it")

    if args.verbose:
        logger.setLevel(logging.DEBUG)
    elif args.quiet:
        logger.setLevel(logging.ERROR)
    else:
        logger.setLevel(logging.INFO)

    if args.logfile:
        handler = logging.handlers.RotatingFileHandler(args.logfile,
                                                       maxBytes=1000000,
                                                       backupCount=10)
    else:
        handler = logging.StreamHandler()
    handler.setLevel(logging.DEBUG)
    handler.setFormatter(
        logging.Formatter('[%(asctime)-15s %(levelname)-8s] %(message)s'))

    logger.addHandler(handler)

    conf.read(args.configuration_file)

    info = {"hostname": socket.gethostname(), "user": getpass.getuser()}

    if args.mail:
        try:
            mailhandler = BufferingSMTPHandler(
                conf.get("DEFAULT", "mailhost"),
                "{user}@{hostname}".format(**info),
                conf.get("DEFAULT", "to").split(","),
                conf.get("DEFAULT", "subject").format(**info), 500)
        except NoOptionError:
            logger.info("Mail information missing, won't send emails")
        else:
            mailhandler.setLevel(logging.WARNING)
            logger.addHandler(mailhandler)

    logger.info("Starting cleanup as {user} on {hostname}".format(**info))

    config_items = []

    if args.config_item:
        for config_item in args.config_item:
            if config_item not in conf.sections():
                logger.error("No section named %s in %s", config_item,
                             args.configuration_file)
            else:
                config_items.append(config_item)
    else:
        config_items = conf.sections()

    logger.debug("Setting up posttroll connexion...")
    with Publish("remover") as pub:
        time.sleep(3)
        logger.debug("Ready")
        tot_size = 0
        tot_files = 0
        for section in config_items:
            info = dict(conf.items(section))
            base_dir = info.get("base_dir", "")
            if not os.path.exists(base_dir):
                logger.warning("Path %s missing, skipping section %s",
                               base_dir, section)
                continue
            logger.info("Cleaning in %s", base_dir)
            templates = (item.strip() for item in info["templates"].split(","))
            kws = {}
            for key in ["days", "hours", "minutes", "seconds"]:
                try:
                    kws[key] = int(info[key])
                except KeyError:
                    pass
            ref_time = datetime.utcnow() - timedelta(**kws)

            section_files = 0
            section_size = 0
            for template in templates:
                pathname = os.path.join(base_dir, template)
                logger.info("  Cleaning %s", pathname)
                flist = glob(pathname)
                for filename in flist:
                    if not os.path.exists(filename):
                        continue
                    try:
                        stat = os.lstat(filename)
                    except OSError:
                        logger.warning("Couldn't lstat path=%s", str(filename))
                        continue

                    if datetime.fromtimestamp(stat.st_ctime) < ref_time:
                        if not args.dry_run:
                            try:
                                if os.path.isdir(filename):
                                    if not os.listdir(filename):
                                        os.rmdir(filename)
                                    else:
                                        logger.info("%s not empty.", filename)
                                else:
                                    os.remove(filename)
                                    pub.send(
                                        str(
                                            Message("deletion", "del",
                                                    {"uri": filename})))
                                logger.debug("    Removed %s", filename)
                            except (IOError, OSError) as err:
                                logger.warning("Can't remove " + filename +
                                               ": " + str(err))
                                continue
                        else:
                            logger.debug("Would remove %s", filename)
                        section_files += 1
                        section_size += stat.st_size

            logger.info("# removed files: %s", section_files)
            logger.info("MB removed: %s", section_size / 1000000)
            tot_size += section_size
            tot_files += section_files
    logger.info(
        "Thanks for using pytroll/remove_it. See you soon on pytroll.org!")
class MultiPortConfig(object):

    HW_LB = "HW"

    @staticmethod
    def float_x_plus_one_tenth_of_y(x, y):
        return float(x) + float(y) / 10.0

    @staticmethod
    def make_str(base, iterator):
        return ' '.join((base.format(x) for x in iterator))

    @classmethod
    def make_range_str(cls, base, start, stop=0, offset=0):
        if offset and not stop:
            stop = start + offset
        return cls.make_str(base, range(start, stop))

    @staticmethod
    def parser_get(parser, section, key, default=None):
        if parser.has_option(section, key):
            return parser.get(section, key)
        return default

    @staticmethod
    def make_ip_addr(ip, mask):
        """
        :param ip: ip adddress
        :type ip: str
        :param mask: /24 prefix of 255.255.255.0 netmask
        :type mask: str
        :return: interface
        :rtype: IPv4Interface
        """

        try:
            return ipaddress.ip_interface(six.text_type('/'.join([ip, mask])))
        except (TypeError, ValueError):
            # None so we can skip later
            return None

    @classmethod
    def validate_ip_and_prefixlen(cls, ip_addr, prefixlen):
        ip_addr = cls.make_ip_addr(ip_addr, prefixlen)
        return ip_addr.ip.exploded, ip_addr.network.prefixlen

    def __init__(self,
                 topology_file,
                 config_tpl,
                 tmp_file,
                 vnfd_helper,
                 vnf_type='CGNAT',
                 lb_count=2,
                 worker_threads=3,
                 worker_config='1C/1T',
                 lb_config='SW',
                 socket=0):

        super(MultiPortConfig, self).__init__()
        self.topology_file = topology_file
        self.worker_config = worker_config.split('/')[1].lower()
        self.worker_threads = self.get_worker_threads(worker_threads)
        self.vnf_type = vnf_type
        self.pipe_line = 0
        self.vnfd_helper = vnfd_helper
        self.write_parser = ConfigParser()
        self.read_parser = ConfigParser()
        self.read_parser.read(config_tpl)
        self.master_core = self.read_parser.get("PIPELINE0", "core")
        self.master_tpl = self.get_config_tpl_data('MASTER')
        self.arpicmp_tpl = self.get_config_tpl_data('ARPICMP')
        self.txrx_tpl = self.get_config_tpl_data('TXRX')
        self.loadb_tpl = self.get_config_tpl_data('LOADB')
        self.vnf_tpl = self.get_config_tpl_data(vnf_type)
        self.swq = 0
        self.lb_count = int(lb_count)
        self.lb_config = lb_config
        self.tmp_file = os.path.join("/tmp", tmp_file)
        self.pktq_out_os = []
        self.socket = socket
        self.start_core = 0
        self.pipeline_counter = ""
        self.txrx_pipeline = ""
        self._port_pairs = None
        self.all_ports = []
        self.port_pair_list = []
        self.lb_to_port_pair_mapping = {}
        self.init_eal()

        self.lb_index = None
        self.mul = 0
        self.port_pairs = []
        self.ports_len = 0
        self.prv_que_handler = None
        self.vnfd = None
        self.rules = None
        self.pktq_out = []

    @staticmethod
    def gen_core(core):
        # return "s{}c{}".format(self.socket, core)
        # don't use sockets for VNFs, because we don't want to have to
        # adjust VM CPU topology.  It is virtual anyway
        return str(core)

    def make_port_pairs_iter(self, operand, iterable):
        return (operand(self.vnfd_helper.port_num(x), y) for y in iterable
                for x in chain.from_iterable(self.port_pairs))

    def make_range_port_pairs_iter(self, operand, start, end):
        return self.make_port_pairs_iter(operand, range(start, end))

    def init_eal(self):
        lines = ['[EAL]\n']
        vpci = (v['virtual-interface']["vpci"]
                for v in self.vnfd_helper.interfaces)
        lines.extend('w = {0}\n'.format(item) for item in vpci)
        lines.append('\n')
        with open(self.tmp_file, 'w') as fh:
            fh.writelines(lines)

    def update_timer(self):
        timer_tpl = self.get_config_tpl_data('TIMER')
        timer_tpl['core'] = self.gen_core(0)
        self.update_write_parser(timer_tpl)

    def get_config_tpl_data(self, type_value):
        for section in self.read_parser.sections():
            if self.read_parser.has_option(section, 'type'):
                if type_value == self.read_parser.get(section, 'type'):
                    tpl = OrderedDict(self.read_parser.items(section))
                    return tpl

    def get_txrx_tpl_data(self, value):
        for section in self.read_parser.sections():
            if self.read_parser.has_option(section, 'pipeline_txrx_type'):
                if value == self.read_parser.get(section,
                                                 'pipeline_txrx_type'):
                    tpl = OrderedDict(self.read_parser.items(section))
                    return tpl

    def init_write_parser_template(self, type_value='ARPICMP'):
        for section in self.read_parser.sections():
            if type_value == self.parser_get(self.read_parser, section, 'type',
                                             object()):
                self.pipeline_counter = self.read_parser.getint(
                    section, 'core')
                self.txrx_pipeline = self.read_parser.getint(section, 'core')
                return
            self.write_parser.add_section(section)
            for name, value in self.read_parser.items(section):
                self.write_parser.set(section, name, value)

    def update_write_parser(self, data):
        section = "PIPELINE{0}".format(self.pipeline_counter)
        self.write_parser.add_section(section)
        for name, value in data.items():
            self.write_parser.set(section, name, value)

    def get_worker_threads(self, worker_threads):
        if self.worker_config == '1t':
            return worker_threads
        else:
            return worker_threads - worker_threads % 2

    def generate_next_core_id(self):
        if self.worker_config == '1t':
            self.start_core += 1
            return

        try:
            self.start_core = '{}h'.format(int(self.start_core))
        except ValueError:
            self.start_core = int(self.start_core[:-1]) + 1

    def get_lb_count(self):
        self.lb_count = int(min(len(self.port_pair_list), self.lb_count))

    def generate_lb_to_port_pair_mapping(self):
        self.lb_to_port_pair_mapping = defaultdict(int)
        port_pair_count = len(self.port_pair_list)
        lb_pair_count = int(port_pair_count / self.lb_count)
        extra = port_pair_count % self.lb_count
        extra_iter = repeat(lb_pair_count + 1, extra)
        norm_iter = repeat(lb_pair_count, port_pair_count - extra)
        new_values = {
            i: v
            for i, v in enumerate(chain(extra_iter, norm_iter), 1)
        }
        self.lb_to_port_pair_mapping.update(new_values)

    def set_priv_to_pub_mapping(self):
        port_nums = [
            tuple(self.vnfd_helper.port_nums(x)) for x in self.port_pair_list
        ]
        return "".join(str(y).replace(" ", "") for y in port_nums)

    def set_priv_que_handler(self):
        # iterated twice, can't be generator
        priv_to_pub_map = [
            tuple(self.vnfd_helper.port_nums(x)) for x in self.port_pairs
        ]
        # must be list to use .index()
        port_list = list(chain.from_iterable(priv_to_pub_map))
        uplink_ports = (x[0] for x in priv_to_pub_map)
        self.prv_que_handler = '({})'.format("".join(
            ("{},".format(port_list.index(x)) for x in uplink_ports)))

    def generate_arp_route_tbl(self):
        arp_route_tbl_tmpl = "({port0_dst_ip_hex},{port0_netmask_hex},{port_num}," \
                             "{next_hop_ip_hex})"

        def build_arp_config(port):
            dpdk_port_num = self.vnfd_helper.port_num(port)
            interface = self.vnfd_helper.find_interface(
                name=port)["virtual-interface"]
            # We must use the dst because we are on the VNF and we need to
            # reach the TG.
            dst_port0_ip = ipaddress.ip_interface(
                six.text_type("%s/%s" %
                              (interface["dst_ip"], interface["netmask"])))

            arp_vars = {
                "port0_dst_ip_hex":
                ip_to_hex(dst_port0_ip.network.network_address.exploded),
                "port0_netmask_hex":
                ip_to_hex(dst_port0_ip.network.netmask.exploded),
                # this is the port num that contains port0 subnet and next_hop_ip_hex
                # this is LINKID which should be based on DPDK port number
                "port_num":
                dpdk_port_num,
                # next hop is dst in this case
                # must be within subnet
                "next_hop_ip_hex":
                ip_to_hex(dst_port0_ip.ip.exploded),
            }
            return arp_route_tbl_tmpl.format(**arp_vars)

        return ' '.join(build_arp_config(port) for port in self.all_ports)

    def generate_arpicmp_data(self):
        swq_in_str = self.make_range_str('SWQ{}',
                                         self.swq,
                                         offset=self.lb_count)
        self.swq += self.lb_count
        swq_out_str = self.make_range_str('SWQ{}',
                                          self.swq,
                                          offset=self.lb_count)
        self.swq += self.lb_count
        # ports_mac_list is disabled for some reason

        # mac_iter = (self.vnfd_helper.find_interface(name=port)['virtual-interface']['local_mac']
        #             for port in self.all_ports)
        pktq_in_iter = ('RXQ{}.0'.format(self.vnfd_helper.port_num(x[0]))
                        for x in self.port_pair_list)

        arpicmp_data = {
            'core': self.gen_core(0),
            'pktq_in': swq_in_str,
            'pktq_out': swq_out_str,
            # we need to disable ports_mac_list?
            # it looks like ports_mac_list is no longer required
            # 'ports_mac_list': ' '.join(mac_iter),
            'pktq_in_prv': ' '.join(pktq_in_iter),
            'prv_to_pub_map': self.set_priv_to_pub_mapping(),
            'arp_route_tbl': self.generate_arp_route_tbl(),
            # nd_route_tbl must be set or we get segault on random OpenStack IPv6 traffic
            # 'nd_route_tbl': "(0064:ff9b:0:0:0:0:9810:6414,120,0,0064:ff9b:0:0:0:0:9810:6414)"
            # safe default?  route discard prefix to localhost
            'nd_route_tbl': "(0100::,64,0,::1)"
        }
        self.pktq_out_os = swq_out_str.split(' ')
        # HWLB is a run to complition. So override the pktq_in/pktq_out
        if self.lb_config == self.HW_LB:
            self.swq = 0
            swq_in_str = \
                self.make_range_str('SWQ{}', self.swq,
                                    offset=(self.lb_count * self.worker_threads))
            arpicmp_data['pktq_in'] = swq_in_str
            # WA: Since port_pairs will not be populated during arp pipeline
            self.port_pairs = self.port_pair_list
            port_iter = \
                self.make_port_pairs_iter(self.float_x_plus_one_tenth_of_y, [self.mul])
            pktq_out = self.make_str('TXQ{}', port_iter)
            arpicmp_data['pktq_out'] = pktq_out

        return arpicmp_data

    def generate_final_txrx_data(self, core=0):
        swq_start = self.swq - self.ports_len * self.worker_threads

        txq_start = 0
        txq_end = self.worker_threads

        pktq_out_iter = self.make_range_port_pairs_iter(
            self.float_x_plus_one_tenth_of_y, txq_start, txq_end)

        swq_str = self.make_range_str('SWQ{}', swq_start, self.swq)
        txq_str = self.make_str('TXQ{}', pktq_out_iter)
        rxtx_data = {
            'pktq_in': swq_str,
            'pktq_out': txq_str,
            'pipeline_txrx_type': 'TXTX',
            'core': self.gen_core(core),
        }
        pktq_in = rxtx_data['pktq_in']
        pktq_in = '{0} {1}'.format(pktq_in,
                                   self.pktq_out_os[self.lb_index - 1])
        rxtx_data['pktq_in'] = pktq_in
        self.pipeline_counter += 1
        return rxtx_data

    def generate_initial_txrx_data(self):
        pktq_iter = self.make_range_port_pairs_iter(
            self.float_x_plus_one_tenth_of_y, 0, self.worker_threads)

        rxq_str = self.make_str('RXQ{}', pktq_iter)
        swq_str = self.make_range_str('SWQ{}', self.swq, offset=self.ports_len)
        txrx_data = {
            'pktq_in': rxq_str,
            'pktq_out': swq_str + ' SWQ{0}'.format(self.lb_index - 1),
            'pipeline_txrx_type': 'RXRX',
            'core': self.gen_core(self.start_core),
        }
        self.pipeline_counter += 1
        return self.start_core, txrx_data

    def generate_lb_data(self):
        pktq_in = self.make_range_str('SWQ{}', self.swq, offset=self.ports_len)
        self.swq += self.ports_len

        offset = self.ports_len * self.worker_threads
        pktq_out = self.make_range_str('SWQ{}', self.swq, offset=offset)
        self.pktq_out = pktq_out.split()

        self.swq += (self.ports_len * self.worker_threads)
        lb_data = {
            'prv_que_handler': self.prv_que_handler,
            'pktq_in': pktq_in,
            'pktq_out': pktq_out,
            'n_vnf_threads': str(self.worker_threads),
            'core': self.gen_core(self.start_core),
        }
        self.pipeline_counter += 1
        return lb_data

    def generate_vnf_data(self):
        if self.lb_config == self.HW_LB:
            port_iter = self.make_port_pairs_iter(
                self.float_x_plus_one_tenth_of_y, [self.mul])
            pktq_in = self.make_str('RXQ{}', port_iter)

            self.mul += 1
            port_iter = self.make_port_pairs_iter(
                self.float_x_plus_one_tenth_of_y, [self.mul])
            pktq_out = self.make_str('TXQ{}', port_iter)

            pipe_line_data = {
                'pktq_in': pktq_in,
                'pktq_out': pktq_out + ' SWQ{0}'.format(self.swq),
                'prv_que_handler': self.prv_que_handler,
                'core': self.gen_core(self.start_core),
            }
            self.swq += 1
        else:
            pipe_line_data = {
                'pktq_in':
                ' '.join(
                    (self.pktq_out.pop(0) for _ in range(self.ports_len))),
                'pktq_out':
                self.make_range_str('SWQ{}', self.swq, offset=self.ports_len),
                'prv_que_handler':
                self.prv_que_handler,
                'core':
                self.gen_core(self.start_core),
            }
            self.swq += self.ports_len

        if self.vnf_type in ('ACL', 'VFW'):
            pipe_line_data.pop('prv_que_handler')

        if self.vnf_tpl.get('vnf_set'):
            public_ip_port_range_list = self.vnf_tpl[
                'public_ip_port_range'].split(':')
            ip_in_hex = '{:x}'.format(
                int(public_ip_port_range_list[0], 16) + self.lb_index - 1)
            public_ip_port_range_list[0] = ip_in_hex
            self.vnf_tpl['public_ip_port_range'] = ':'.join(
                public_ip_port_range_list)

        self.pipeline_counter += 1
        return pipe_line_data

    def generate_config_data(self):
        self.init_write_parser_template()

        # use master core for master, don't use self.start_core
        self.write_parser.set('PIPELINE0', 'core',
                              self.gen_core(self.master_core))
        arpicmp_data = self.generate_arpicmp_data()
        self.arpicmp_tpl.update(arpicmp_data)
        self.update_write_parser(self.arpicmp_tpl)

        if self.vnf_type == 'CGNAPT':
            self.pipeline_counter += 1
            self.update_timer()

        if self.lb_config == 'HW':
            self.start_core = 1

        for lb in self.lb_to_port_pair_mapping:
            self.lb_index = lb
            self.mul = 0
            port_pair_count = self.lb_to_port_pair_mapping[lb]
            if not self.port_pair_list:
                continue

            self.port_pairs = self.port_pair_list[:port_pair_count]
            self.port_pair_list = self.port_pair_list[port_pair_count:]
            self.ports_len = port_pair_count * 2
            self.set_priv_que_handler()
            if self.lb_config == 'SW':
                core, txrx_data = self.generate_initial_txrx_data()
                self.txrx_tpl.update(txrx_data)
                self.update_write_parser(self.txrx_tpl)
                self.start_core += 1
                lb_data = self.generate_lb_data()
                self.loadb_tpl.update(lb_data)
                self.update_write_parser(self.loadb_tpl)
                self.start_core += 1

            for i in range(self.worker_threads):
                vnf_data = self.generate_vnf_data()
                if not self.vnf_tpl:
                    self.vnf_tpl = {}
                self.vnf_tpl.update(vnf_data)
                self.update_write_parser(self.vnf_tpl)
                try:
                    self.vnf_tpl.pop('vnf_set')
                except KeyError:
                    pass
                else:
                    self.vnf_tpl.pop('public_ip_port_range')
                self.generate_next_core_id()

            if self.lb_config == 'SW':
                txrx_data = self.generate_final_txrx_data(core)
                self.txrx_tpl.update(txrx_data)
                self.update_write_parser(self.txrx_tpl)
            self.vnf_tpl = self.get_config_tpl_data(self.vnf_type)

    def generate_config(self):
        self._port_pairs = PortPairs(self.vnfd_helper.interfaces)
        self.port_pair_list = self._port_pairs.port_pair_list
        self.all_ports = self._port_pairs.all_ports

        self.get_lb_count()
        self.generate_lb_to_port_pair_mapping()
        self.generate_config_data()
        self.write_parser.write(sys.stdout)
        with open(self.tmp_file, 'a') as tfh:
            self.write_parser.write(tfh)

    def generate_link_config(self):
        def build_args(port):
            # lookup interface by name
            virtual_interface = self.vnfd_helper.find_interface(
                name=port)["virtual-interface"]
            local_ip = virtual_interface["local_ip"]
            netmask = virtual_interface["netmask"]
            port_num = self.vnfd_helper.port_num(port)
            port_ip, prefix_len = self.validate_ip_and_prefixlen(
                local_ip, netmask)
            return LINK_CONFIG_TEMPLATE.format(port_num, port_ip, prefix_len)

        return ''.join(build_args(port) for port in self.all_ports)

    def get_route_data(self, src_key, data_key, port):
        route_list = self.vnfd['vdu'][0].get(src_key, [])
        try:
            return next((route[data_key]
                         for route in route_list if route['if'] == port), None)
        except (TypeError, StopIteration, KeyError):
            return None

    def get_ports_gateway(self, port):
        return self.get_route_data('routing_table', 'gateway', port)

    def get_ports_gateway6(self, port):
        return self.get_route_data('nd_route_tbl', 'gateway', port)

    def get_netmask_gateway(self, port):
        return self.get_route_data('routing_table', 'netmask', port)

    def get_netmask_gateway6(self, port):
        return self.get_route_data('nd_route_tbl', 'netmask', port)

    def generate_arp_config(self):
        arp_config = []
        for port in self.all_ports:
            # ignore gateway, always use TG IP
            # gateway = self.get_ports_gateway(port)
            vintf = self.vnfd_helper.find_interface(
                name=port)["virtual-interface"]
            dst_mac = vintf["dst_mac"]
            dst_ip = vintf["dst_ip"]
            # arp_config.append(
            #     (self.vnfd_helper.port_num(port), gateway, dst_mac, self.txrx_pipeline))
            # so dst_mac is the TG dest mac, so we need TG dest IP.
            # should be dpdk_port_num
            arp_config.append((self.vnfd_helper.port_num(port), dst_ip,
                               dst_mac, self.txrx_pipeline))

        return '\n'.join(('p {3} arpadd {0} {1} {2}'.format(*values)
                          for values in arp_config))

    def generate_arp_config6(self):
        arp_config6 = []
        for port in self.all_ports:
            # ignore gateway, always use TG IP
            # gateway6 = self.get_ports_gateway6(port)
            vintf = self.vnfd_helper.find_interface(
                name=port)["virtual-interface"]
            dst_mac6 = vintf["dst_mac"]
            dst_ip6 = vintf["dst_ip"]
            # arp_config6.append(
            #     (self.vnfd_helper.port_num(port), gateway6, dst_mac6, self.txrx_pipeline))
            arp_config6.append((self.vnfd_helper.port_num(port), dst_ip6,
                                dst_mac6, self.txrx_pipeline))

        return '\n'.join(('p {3} arpadd {0} {1} {2}'.format(*values)
                          for values in arp_config6))

    def generate_action_config(self):
        port_list = (self.vnfd_helper.port_num(p) for p in self.all_ports)
        if self.vnf_type == "VFW":
            template = FW_ACTION_TEMPLATE
        else:
            template = ACTION_TEMPLATE

        return ''.join((template.format(port) for port in port_list))

    def get_ip_from_port(self, port):
        # we can't use gateway because in OpenStack gateways interfer with floating ip routing
        # return self.make_ip_addr(self.get_ports_gateway(port), self.get_netmask_gateway(port))
        vintf = self.vnfd_helper.find_interface(name=port)["virtual-interface"]
        ip = vintf["local_ip"]
        netmask = vintf["netmask"]
        return self.make_ip_addr(ip, netmask)

    def get_network_and_prefixlen_from_ip_of_port(self, port):
        ip_addr = self.get_ip_from_port(port)
        # handle cases with no gateway
        if ip_addr:
            return ip_addr.network.network_address.exploded, ip_addr.network.prefixlen
        else:
            return None, None

    def generate_rule_config(self):
        cmd = 'acl' if self.vnf_type == "ACL" else "vfw"
        rules_config = self.rules if self.rules else ''
        new_rules = []
        new_ipv6_rules = []
        pattern = 'p {0} add {1} {2} {3} {4} {5} 0 65535 0 65535 0 0 {6}'
        for src_intf, dst_intf in self.port_pair_list:
            src_port = self.vnfd_helper.port_num(src_intf)
            dst_port = self.vnfd_helper.port_num(dst_intf)

            src_net, src_prefix_len = self.get_network_and_prefixlen_from_ip_of_port(
                src_intf)
            dst_net, dst_prefix_len = self.get_network_and_prefixlen_from_ip_of_port(
                dst_intf)
            # ignore entires with empty values
            if all((src_net, src_prefix_len, dst_net, dst_prefix_len)):
                new_rules.append(
                    (cmd, self.txrx_pipeline, src_net, src_prefix_len, dst_net,
                     dst_prefix_len, dst_port))
                new_rules.append(
                    (cmd, self.txrx_pipeline, dst_net, dst_prefix_len, src_net,
                     src_prefix_len, src_port))

            # src_net = self.get_ports_gateway6(port_pair[0])
            # src_prefix_len = self.get_netmask_gateway6(port_pair[0])
            # dst_net = self.get_ports_gateway6(port_pair[1])
            # dst_prefix_len = self.get_netmask_gateway6(port_pair[0])
            # # ignore entires with empty values
            # if all((src_net, src_prefix_len, dst_net, dst_prefix_len)):
            #     new_ipv6_rules.append((cmd, self.txrx_pipeline, src_net, src_prefix_len,
            #                            dst_net, dst_prefix_len, dst_port))
            #     new_ipv6_rules.append((cmd, self.txrx_pipeline, dst_net, dst_prefix_len,
            #                            src_net, src_prefix_len, src_port))

        acl_apply = "\np %s applyruleset" % cmd
        new_rules_config = '\n'.join(
            pattern.format(*values)
            for values in chain(new_rules, new_ipv6_rules))
        return ''.join([rules_config, new_rules_config, acl_apply])

    def generate_script_data(self):
        self._port_pairs = PortPairs(self.vnfd_helper.interfaces)
        self.port_pair_list = self._port_pairs.port_pair_list
        self.get_lb_count()
        script_data = {
            'link_config': self.generate_link_config(),
            'arp_config': self.generate_arp_config(),
            # disable IPv6 for now
            # 'arp_config6': self.generate_arp_config6(),
            'arp_config6': "",
            'actions': '',
            'rules': '',
        }

        if self.vnf_type in ('ACL', 'VFW'):
            script_data.update({
                'actions': self.generate_action_config(),
                'rules': self.generate_rule_config(),
            })

        return script_data

    def generate_script(self, vnfd, rules=None):
        self.vnfd = vnfd
        self.rules = rules
        script_data = self.generate_script_data()
        script = SCRIPT_TPL.format(**script_data)
        if self.lb_config == self.HW_LB:
            script += 'set fwd rxonly'
            hwlb_tpl = """
set_sym_hash_ena_per_port {0} enable
set_hash_global_config {0} simple_xor ipv4-udp enable
set_sym_hash_ena_per_port {1} enable
set_hash_global_config {1} simple_xor ipv4-udp enable
set_hash_input_set {0} ipv4-udp src-ipv4 udp-src-port add
set_hash_input_set {1} ipv4-udp dst-ipv4 udp-dst-port add
set_hash_input_set {0} ipv6-udp src-ipv6 udp-src-port add
set_hash_input_set {1} ipv6-udp dst-ipv6 udp-dst-port add
"""
            for port_pair in self.port_pair_list:
                script += hwlb_tpl.format(
                    *(self.vnfd_helper.port_nums(port_pair)))
        return script
Example #60
0
File: api.py Project: vdt/pyquil
import pyquil.quil as pq
from pyquil.job_results import JobResult, WavefunctionResult, recover_complexes

USER_HOMEDIR = os.path.expanduser("~")

PYQUIL_CONFIG_PATH = os.getenv('PYQUIL_CONFIG',
                               os.path.join(USER_HOMEDIR, ".pyquil_config"))
PYQUIL_CONFIG = ConfigParser()

try:
    if "~" in PYQUIL_CONFIG_PATH:
        raise RuntimeError(
            "PYQUIL_CONFIG enviroment variable contains `~`. Use $HOME instead."
        )
    if len(PYQUIL_CONFIG.read(PYQUIL_CONFIG_PATH)) == 0:
        raise RuntimeError("Error locating config file")
except:
    print(
        "! WARNING:\n"
        "!   There was an error reading your pyQuil config file.\n"
        "!   Make sure you have a .pyquil_config file either in\n"
        "!   your home directory, or you have the environment\n"
        "!   variable PYQUIL_CONFIG set to a valid path. You must\n"
        "!   have permissions to read this file.",
        file=sys.stderr)


def config_value(name, default=None):
    """
    Get a config file value for a particular name