def get_ic_factor(self, det): # storing ic_factor in preferences causing issues # ic_factor stored in detectors.cfg p = os.path.join(paths.spectrometer_dir, 'detectors.cfg') # factors=None ic = 1, 0 if os.path.isfile(p): c = ConfigParser() c.read(p) det = det.lower() for si in c.sections(): if si.lower() == det: v, e = 1, 0 if c.has_option(si, 'ic_factor'): v = c.getfloat(si, 'ic_factor') if c.has_option(si, 'ic_factor_err'): e = c.getfloat(si, 'ic_factor_err') ic = v, e break else: self.debug('no detector file {}. cannot retrieve ic_factor'.format(p)) r = ufloat(*ic) return r
def remote_profiles(self): """ A list of remote profiles on the device. """ remote_ini = self.app_ctx.remote_profiles_ini if not self.device.is_file(remote_ini): raise IOError("Remote file '%s' not found" % remote_ini) local_ini = tempfile.NamedTemporaryFile() self.device.pull(remote_ini, local_ini.name) cfg = ConfigParser() cfg.read(local_ini.name) profiles = [] for section in cfg.sections(): if cfg.has_option(section, "Path"): if cfg.has_option(section, "IsRelative") and cfg.getint( section, "IsRelative" ): profiles.append( posixpath.join( posixpath.dirname(remote_ini), cfg.get(section, "Path") ) ) else: profiles.append(cfg.get(section, "Path")) return profiles
def _generate_appdata(self, prefix, activity_path): info = ConfigParser() info.read(os.path.join(activity_path, 'activity', 'activity.info')) required_fields = ['metadata_license', 'license', 'name', 'icon', 'description'] for name in required_fields: if not info.has_option('Activity', name): print('[WARNING] Activity needs more metadata for AppStream ' 'file') print(' Without an AppStream file, the activity will NOT ' 'show in software stores!') print(' Please `pydoc sugar3.activity.bundlebuilder` for' 'more info') return # See https://www.freedesktop.org/software/appstream/docs/ root = ET.Element('component', type='desktop') ET.SubElement(root, 'project_group').text = 'Sugar' ET.SubElement(root, 'translation', type='gettext').text = \ self.config.bundle_id ET.SubElement(root, 'id').text = \ self.config.bundle_id + '.activity.desktop' desc = ET.fromstring('<description>{}</description>'.format( info.get('Activity', 'description'))) root.append(desc) copy_pairs = [('metadata_license', 'metadata_license'), ('license', 'project_license'), ('summary', 'summary'), ('name', 'name')] for key, ename in copy_pairs: ET.SubElement(root, ename).text = info.get('Activity', key) if info.has_option('Activity', 'screenshots'): screenshots = info.get('Activity', 'screenshots').split() ss_root = ET.SubElement(root, 'screenshots') for i, screenshot in enumerate(screenshots): e = ET.SubElement(ss_root, 'screenshot') if i == 0: e.set('type', 'default') ET.SubElement(e, 'image').text = screenshot if info.has_option('Activity', 'url'): ET.SubElement(root, 'url', type='homepage').text = \ info.get('Activity', 'url') if info.has_option('Activity', 'repository_url'): ET.SubElement(root, 'url', type='bugtracker').text = \ info.get('Activity', 'repository_url') elif info.has_option('Activity', 'repository'): ET.SubElement(root, 'url', type='bugtracker').text = \ info.get('Activity', 'repository') path = os.path.join(prefix, 'share', 'metainfo', self.config.bundle_id + '.appdata.xml') if not os.path.isdir(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) tree = ET.ElementTree(root) tree.write(path, encoding='UTF-8')
def _generate_appdata(self, prefix, activity_path): info = ConfigParser() info.read(os.path.join(activity_path, 'activity', 'activity.info')) required_fields = [ 'metadata_license', 'license', 'name', 'icon', 'description' ] for name in required_fields: if not info.has_option('Activity', name): print('[WARNING] Activity needs more metadata for AppStream ' 'file') print(' Without an AppStream file, the activity will NOT ' 'show in software stores!') print(' Please `pydoc sugar3.activity.bundlebuilder` for' 'more info') return # See https://www.freedesktop.org/software/appstream/docs/ root = ET.Element('component', type='desktop') ET.SubElement(root, 'project_group').text = 'Sugar' ET.SubElement(root, 'translation', type='gettext').text = \ self.config.bundle_id ET.SubElement(root, 'id').text = \ self.config.bundle_id + '.activity.desktop' desc = ET.fromstring('<description>{}</description>'.format( info.get('Activity', 'description'))) root.append(desc) copy_pairs = [('metadata_license', 'metadata_license'), ('license', 'project_license'), ('summary', 'summary'), ('name', 'name')] for key, ename in copy_pairs: ET.SubElement(root, ename).text = info.get('Activity', key) if info.has_option('Activity', 'screenshots'): screenshots = info.get('Activity', 'screenshots').split() ss_root = ET.SubElement(root, 'screenshots') for i, screenshot in enumerate(screenshots): e = ET.SubElement(ss_root, 'screenshot') if i == 0: e.set('type', 'default') ET.SubElement(e, 'image').text = screenshot if info.has_option('Activity', 'url'): ET.SubElement(root, 'url', type='homepage').text = \ info.get('Activity', 'url') if info.has_option('Activity', 'repository_url'): ET.SubElement(root, 'url', type='bugtracker').text = \ info.get('Activity', 'repository_url') elif info.has_option('Activity', 'repository'): ET.SubElement(root, 'url', type='bugtracker').text = \ info.get('Activity', 'repository') path = os.path.join(prefix, 'share', 'metainfo', self.config.bundle_id + '.appdata.xml') if not os.path.isdir(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) tree = ET.ElementTree(root) tree.write(path, encoding='UTF-8')
def _read_pypirc(self): """Reads the .pypirc file.""" rc = self._get_rc_file() if os.path.exists(rc): self.announce('Using PyPI login from %s' % rc) repository = self.repository or self.DEFAULT_REPOSITORY config = ConfigParser() config.read(rc) sections = config.sections() if 'distutils' in sections: # let's get the list of servers index_servers = config.get('distutils', 'index-servers') _servers = [ server.strip() for server in index_servers.split('\n') if server.strip() != '' ] if _servers == []: # nothing set, let's try to get the default pypi if 'pypi' in sections: _servers = ['pypi'] else: # the file is not properly defined, returning # an empty dict return {} for server in _servers: current = {'server': server} current['username'] = config.get(server, 'username') # optional params for key, default in (('repository', self.DEFAULT_REPOSITORY), ('realm', self.DEFAULT_REALM), ('password', None)): if config.has_option(server, key): current[key] = config.get(server, key) else: current[key] = default if (current['server'] == repository or current['repository'] == repository): return current elif 'server-login' in sections: # old format server = 'server-login' if config.has_option(server, 'repository'): repository = config.get(server, 'repository') else: repository = self.DEFAULT_REPOSITORY return { 'username': config.get(server, 'username'), 'password': config.get(server, 'password'), 'repository': repository, 'server': server, 'realm': self.DEFAULT_REALM } return {}
def read_config_file(cfgfile, options): config = ConfigParser() config.readfp(open(cfgfile)) if config.has_option('testflo', 'skip_dirs'): skips = config.get('testflo', 'skip_dirs') options.skip_dirs = [s.strip() for s in skips.split(',') if s.strip()] if config.has_option('testflo', 'num_procs'): options.num_procs = int(config.get('testflo', 'num_procs'))
def read_config_file(cfgfile, options): config = ConfigParser() config.readfp(open(cfgfile)) if config.has_option('testflo', 'skip_dirs'): skips = config.get('testflo', 'skip_dirs') options.skip_dirs = [s.strip() for s in skips.split(',') if s.strip()] if config.has_option('testflo', 'num_procs'): options.num_procs = int(config.get('testflo', 'num_procs')) if config.has_option('testflo', 'noreport'): options.noreport = bool(config.get('testflo', 'noreport'))
def read_param_file(self): """Read the parameter file and populate the relevant dictionaries """ self.out_values = [] # read the file params = ConfigParser(allow_no_value=True) f = params.read(self.config_file) if not f: msgr.fatal(u"File <{}> not found".format(self.config_file)) # populate dictionaries using loops instead of using update() method # in order to not add invalid key for k in self.raw_input_times: if params.has_option('time', k): self.raw_input_times[k] = params.get('time', k) for k in self.sim_param: if params.has_option('options', k): self.sim_param[k] = params.getfloat('options', k) for k in self.grass_params: if params.has_option('grass', k): self.grass_params[k] = params.get('grass', k) # check for deprecated input names if params.has_option('input', "drainage_capacity"): msgr.warning(u"'drainage_capacity' is deprecated. " u"Use 'losses' instead.") self.input_map_names['losses'] = params.get( 'input', "drainage_capacity") # search for valid inputs for k in self.input_map_names: if params.has_option('input', k): self.input_map_names[k] = params.get('input', k) # drainage parameters for k in self.drainage_params: if params.has_option('drainage', k): if k in ['swmm_inp', 'output', 'swmm_gage']: self.drainage_params[k] = params.get('drainage', k) else: self.drainage_params[k] = params.getfloat('drainage', k) # statistic file if params.has_option('statistics', 'stats_file'): self.stats_file = params.get('statistics', 'stats_file') else: self.stats_file = None # output maps if params.has_option('output', 'prefix'): self.out_prefix = params.get('output', 'prefix') if params.has_option('output', 'values'): out_values = params.get('output', 'values').split(',') self.out_values = [e.strip() for e in out_values] # check for deprecated values if 'drainage_cap' in self.out_values and 'losses' not in self.out_values: msgr.warning(u"'drainage_cap' is deprecated. " u"Use 'losses' instead.") self.out_values.append('losses') self.generate_output_name() return self
def read_printers(self): """get invalid/valid users from cups and samba config""" # read CUPS configuration if os.path.isfile(ShareConfiguration.CUPS_CONF): reg_cups = re.compile(r'\s*<Printer\s+([^>]+)>') with open("/etc/cups/printers.conf") as fd: for line in fd.readlines(): m_cups = reg_cups.match(line) if m_cups: prt = Printer(m_cups.group(1).strip()) self._printers[prt.name] = prt # samba if not os.path.exists(ShareConfiguration.PRINTERS_UDM_DIR): return for filename in os.listdir(ShareConfiguration.PRINTERS_UDM_DIR): cfg = ConfigParser() cfg.read( os.path.join(ShareConfiguration.PRINTERS_UDM_DIR, filename)) try: prt_name = cfg.sections()[0] except IndexError: continue prt = None if prt_name in self._printers: prt = self._printers[prt_name] else: if cfg.has_option(prt_name, 'printer name'): cups_name = cfg.get(prt_name, 'printer name') if cups_name in self._printers: prt = self._printers[cups_name] prt.smbname = prt_name if prt is None: continue if cfg.has_option(prt_name, Restrictions.INVALID_USERS): prt.invalid_users = shlex.split( cfg.get(prt_name, Restrictions.INVALID_USERS)) if cfg.has_option(prt_name, Restrictions.VALID_USERS): prt.valid_users = shlex.split( cfg.get(prt_name, Restrictions.VALID_USERS)) if cfg.has_option(prt_name, Restrictions.HOSTS_DENY): prt.hosts_deny = shlex.split( cfg.get(prt_name, Restrictions.HOSTS_DENY))
def test_overwrite_dict_cfg(self): c = ConfigParser(allow_no_value=True) d = { "section_a": "empty_value", "section_b": {"key_c": "val_d", "key_d": "val_d"}, "section_c": ["key_c", "key_d"], } ansible_common.overwrite_dict_to_cfg(c, d) # Python3 and Python2 convert empty values into None or '' # we don't really care but we need to compare correctly for unittest self.assertTrue(c.has_option("section_a", "empty_value")) self.assertEqual(sorted(c.items("section_b")), [('key_c', 'val_d'), ('key_d', 'val_d')]) self.assertTrue(c.has_option("section_c", "key_c")) self.assertTrue(c.has_option("section_c", "key_d"))
def load(self): schemes = [defaultScheme] parser = ConfigParser() parser.read(settings.DASHBOARD_CONF) for option, default_value in defaultUIConfig.items(): if parser.has_option('ui', option): try: self.ui_config[option] = parser.getint('ui', option) except ValueError: self.ui_config[option] = parser.get('ui', option) else: self.ui_config[option] = default_value if parser.has_option('ui', 'automatic_variants'): self.ui_config['automatic_variants'] = parser.getboolean('ui', 'automatic_variants') else: self.ui_config['automatic_variants'] = True self.ui_config['keyboard_shortcuts'] = defaultKeyboardShortcuts.copy() if parser.has_section('keyboard-shortcuts'): self.ui_config['keyboard_shortcuts'].update( parser.items('keyboard-shortcuts') ) for section in parser.sections(): if section in ('ui', 'keyboard-shortcuts'): continue scheme = parser.get(section, 'scheme') fields = [] for match in fieldRegex.finditer(scheme): field = match.group(1) if parser.has_option(section, '%s.label' % field): label = parser.get(section, '%s.label' % field) else: label = field fields.append({ 'name' : field, 'label' : label }) schemes.append({ 'name' : section, 'pattern' : scheme, 'fields' : fields, }) self.schemes = schemes
def sync(): # Add or replace the relevant properites from galaxy.ini # into reports.ini reports_config_file = "config/reports.ini" if len(argv) > 1: reports_config_file = argv[1] universe_config_file = "config/galaxy.ini" if len(argv) > 2: universe_config_file = argv[2] parser = ConfigParser() parser.read(universe_config_file) with open(reports_config_file, "r") as f: reports_config_lines = f.readlines() replaced_properties = set([]) with open(reports_config_file, "w") as f: # Write all properties from reports config replacing as # needed. for reports_config_line in reports_config_lines: (line, replaced_property) = get_synced_line(reports_config_line, parser) if replaced_property: replaced_properties.add(replaced_property) f.write(line) # If any properties appear in universe config and not in # reports write these as well. for replacement_property in REPLACE_PROPERTIES: if parser.has_option(MAIN_SECTION, replacement_property) and \ not (replacement_property in replaced_properties): f.write(get_universe_line(replacement_property, parser))
def process(self): nm_config = NetworkManagerConfig() try: # Use 'NM --print-config' to read the configurationo so # that the main configuration file and other files in # various directories get merged in the right way. r = run(['NetworkManager', '--print-config'], split=False)['stdout'] except (OSError, CalledProcessError) as e: self.log.warning( 'Error reading NetworkManager configuration: {}'.format(e)) return parser = ConfigParser() try: if hasattr(parser, 'read_string'): # Python 3 parser.read_string(r) else: # Python 2 from cStringIO import StringIO parser.readfp(StringIO(r)) except ParsingError as e: self.log.warning( 'Error parsing NetworkManager configuration: {}'.format(e)) return if parser.has_option('main', 'dhcp'): nm_config.dhcp = parser.get("main", "dhcp") self.produce(nm_config)
def _install_desktop_file(self, destdir, prefix, activity_path): cp = ConfigParser() section = 'Desktop Entry' cp.add_section(section) cp.optionxform = str # Allow CamelCase entries # Get it from the activity.info for the non-translated version info = ConfigParser() info_path = os.path.join(destdir, os.path.relpath(activity_path, '/'), 'activity', 'activity.info') info.read(info_path) cp.set(section, 'Name', info.get('Activity', 'name')) if info.has_option('Activity', 'summary'): cp.set(section, 'Comment', info.get('Activity', 'summary')) for path in sorted( glob( os.path.join(activity_path, 'locale', '*', 'activity.linfo'))): locale = path.split(os.path.sep)[-2] info = ConfigParser() info.read(path) if info.has_option('Activity', 'name'): cp.set(section, 'Name[{}]'.format(locale), info.get('Activity', 'name')) if info.has_option('Activity', 'summary'): cp.set(section, 'Comment[{}]'.format(locale), info.get('Activity', 'summary')) cp.set(section, 'Terminal', 'false') cp.set(section, 'Type', 'Application') cp.set(section, 'Categories', 'Education;') cp.set( section, 'Icon', os.path.join(activity_path, 'activity', self.config.bundle.get_icon_filename())) cp.set(section, 'Exec', self.config.bundle.get_command()) cp.set(section, 'Path', activity_path) # Path == CWD for running name = '{}.activity.desktop'.format(self.config.bundle_id) path = os.path.join(destdir, os.path.relpath(prefix, '/'), 'share', 'applications', name) if not os.path.isdir(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) with open(path, 'w') as f: cp.write(f) print('Install %s' % (path))
def __init__(self, host="127.0.0.1"): self._host = host config = ConfigParser() config.read(constants.get_config_file()) if config.has_option('OpenMotics', 'http_port'): self._port = config.get('OpenMotics', 'http_port') else: self._port = 80
def _parse_linfo(self, linfo_file): cp = ConfigParser() try: cp.readfp(linfo_file) section = 'Activity' if cp.has_option(section, 'name'): self._name = cp.get(section, 'name') if cp.has_option(section, 'summary'): self._summary = cp.get(section, 'summary') if cp.has_option(section, 'tags'): tag_list = cp.get(section, 'tags').strip(';') self._tags = [tag.strip() for tag in tag_list.split(';')] except ParsingError as e: logging.exception('Exception reading linfo file: %s', e)
def get_platform(): # type: () -> str from six.moves.configparser import ConfigParser config = ConfigParser() config.read(constants.get_config_file()) if config.has_option('OpenMotics', 'platform'): platform = config.get('OpenMotics', 'platform') if platform in Platform.Types: return platform return Platform.Type.CLASSIC
class OSCAPConfig(object): _config_file = SCAP_BASEDIR + "/config" _section = "openscap" _profile = "profile" _datastream = "datastream" _configured = "configured" def __init__(self): self._cp = ConfigParser() self._cp.read(self._config_file) if not self._cp.has_section(self._section): self._cp.add_section(self._section) def _set_value(self, key, val): self._cp.set(self._section, key, val) self._save() def _get_value(self, key, default=None): if not self._cp.has_option(self._section, key): return default return self._cp.get(self._section, key) def _save(self): with open(self._config_file, "w") as f: self._cp.write(f) @property def profile(self): return self._get_value(self._profile) @profile.setter def profile(self, value): return self._set_value(self._profile, value) @property def datastream(self): return self._get_value(self._datastream) @datastream.setter def datastream(self, value): return self._set_value(self._datastream, value) @property def registered(self): return bool(self.profile and self.datastream) @property def configured(self): return self._get_value(self._configured) @configured.setter def configured(self, value): return self._set_value(self._configured, value)
def getLayoutsFromManifest(fp, _format, directory_name): # support multiple sections with the same name in manifest.cfg if six.PY2: parser = ConfigParser(None, multidict) parser.readfp(fp) else: data = fp.read() if isinstance(data, six.binary_type): data = data.decode() parser = ConfigParser(dict_type=multidict, strict=False) parser.read_string(data) layouts = {} for section in parser.sections(): if not section.startswith(_format.resourceType) or \ ':variants' in section: continue # id is a combination of directory name + filename if parser.has_option(section, 'file'): filename = parser.get(section, 'file') else: filename = '' # this should not happen... _id = directory_name + '/' + filename if _id in layouts: # because TTW resources are created first, we consider layouts # with same id in a TTW to be taken before other resources continue data = { 'directory': directory_name } for key in _format.keys: if parser.has_option(section, key): data[key] = parser.get(section, key) else: data[key] = _format.defaults.get(key, None) layouts[_id] = data return layouts
def _install_desktop_file(self, prefix, activity_path): cp = ConfigParser() section = 'Desktop Entry' cp.add_section(section) cp.optionxform = str # Allow CamelCase entries # Get it from the activity.info for the non-translated version info = ConfigParser() info.read(os.path.join(activity_path, 'activity', 'activity.info')) cp.set(section, 'Name', info.get('Activity', 'name')) if info.has_option('Activity', 'summary'): cp.set(section, 'Comment', info.get('Activity', 'summary')) for path in sorted(glob(os.path.join(activity_path, 'locale', '*', 'activity.linfo'))): locale = path.split(os.path.sep)[-2] info = ConfigParser() info.read(path) if info.has_option('Activity', 'name'): cp.set(section, 'Name[{}]'.format(locale), info.get('Activity', 'name')) if info.has_option('Activity', 'summary'): cp.set(section, 'Comment[{}]'.format(locale), info.get('Activity', 'summary')) cp.set(section, 'Terminal', 'false') cp.set(section, 'Type', 'Application') cp.set(section, 'Categories', 'Education;') cp.set(section, 'Icon', os.path.join( activity_path, 'activity', self.config.bundle.get_icon_filename())) cp.set(section, 'Exec', self.config.bundle.get_command()) cp.set(section, 'Path', activity_path) # Path == CWD for running name = '{}.activity.desktop'.format(self.config.bundle_id) path = os.path.join(prefix, 'share', 'applications', name) if not os.path.isdir(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) with open(path, 'w') as f: cp.write(f)
def load_theme(struct, path, colors, default_colors): theme = ConfigParser() with open(path, 'r') as f: theme.readfp(f) for k, v in chain(theme.items('syntax'), theme.items('interface')): if theme.has_option('syntax', k): colors[k] = theme.get('syntax', k) else: colors[k] = theme.get('interface', k) # Check against default theme to see if all values are defined for k, v in iteritems(default_colors): if k not in colors: colors[k] = v
def test_ignore_extra_params(self): name = self.make_empty_temp_file() to_config_file( name, "section.name", {"invalid_parameter": "empty", "port_queue_dt": 1.2} ) self.assertTrue(os.path.isfile(name)) config = ConfigParser() config.read(name) self.assertTrue(config.has_section("section.name")) self.assertSetEqual(_VALID_KEYS, set(config.options("section.name"))) self.assertEqual(1.2, config.getfloat("section.name", "port_queue_dt")) self.assertFalse(config.has_option("section.name", "invalid_parameter"))
def read_shares(self): """get invalid user from samba share conf""" if not os.path.isdir(ShareConfiguration.SHARES_UDM_DIR): return for filename in os.listdir(ShareConfiguration.SHARES_UDM_DIR): filename = os.path.join(ShareConfiguration.SHARES_UDM_DIR, filename) cfg = ConfigParser() cfg.read(filename) try: share = Share(cfg.sections()[0]) except IndexError: continue if cfg.has_option(share.name, Restrictions.INVALID_USERS): share.invalid_users = shlex.split( cfg.get(share.name, Restrictions.INVALID_USERS)) if cfg.has_option(share.name, Restrictions.HOSTS_DENY): share.hosts_deny = shlex.split( cfg.get(share.name, Restrictions.HOSTS_DENY)) self._shares[share.name] = share
def test_ignore_extra_params(self): name = self.make_empty_temp_file() to_config_file(name, "section.name", { "invalid_parameter": "empty", "port_queue_dt": 1.2 }) self.assertTrue(os.path.isfile(name)) config = ConfigParser() config.read(name) self.assertTrue(config.has_section("section.name")) self.assertSetEqual(_VALID_KEYS, set(config.options("section.name"))) self.assertEqual(1.2, config.getfloat("section.name", "port_queue_dt")) self.assertFalse(config.has_option("section.name", "invalid_parameter"))
def serve(args): config = ConfigParser({'port': '8080', 'db': None}) config.read([CONFIG_FILE]) port = int(args.port or int(config.get('app', 'port'))) view_path = getattr(args, 'view_path', None) controller_path = getattr(args, 'controller_path', None) if args.cdn is None: if config.has_option('app', 'cdn'): cdn = config.getboolean('app', 'cdn') else: cdn = True else: cdn = args.cdn server.serve(db=config.get('app', 'db'), port=port, verbose=args.verbose, view_path=view_path, controller_path=controller_path, cdn=cdn)
def parse_config_file(filename): logger.debug("reading configuration file: %s", filename) cp = ConfigParser() with open(filename) as f: cp.readfp(f, filename=filename) if cp.has_option('include', 'files'): for fn in glob(cp.get('include', 'files')): cp.read(fn) cp.redis = cp.get('sup_broadcast', 'redis') cp.url = cp.get('sup_broadcast', 'supervisor_url') try: cp.ident = cp.get('supervisord', 'identifier') except Error: cp.ident = _url2name(cp.url) cp.config_file = filename return cp
def get_configured_plugins(): """Retrieves a list of all plugins that the user has configured for availability within Vigilance. This list will prefer configurations from three sources in this order: 1. The VIGILANCE_PLUGINS environment variable. 2. A .vigilance file within the current working directory. 3. A setup.cfg file within the current working directory. @returns A list of plugin specifier strings. @see load_suites """ plugins = [] parser = ConfigParser() parser.add_section('vigilance') _read_config_file('setup.cfg', parser) _read_config_file('.vigilance', parser) vEnv = os.getenv('VIGILANCE_PLUGINS', None) if vEnv: parser.set('vigilance', 'plugins', vEnv) if parser.has_option('vigilance', 'plugins'): plugins = parser.get('vigilance', 'plugins').split(',') return plugins
def parse(self): """ Parse the config and return a dict with the parsed values. :rtype: ``dict`` """ result = defaultdict(dict) if not os.path.isfile(self.config_file_path): # Config doesn't exist, return the default values return CONFIG_DEFAULT_VALUES config = ConfigParser() with open(self.config_file_path, 'r') as fp: config.readfp(fp) for section, keys in six.iteritems(CONFIG_FILE_OPTIONS): for key, options in six.iteritems(keys): key_type = options['type'] key_default_value = options['default'] if config.has_option(section, key): if key_type in ['str', 'string']: get_func = config.get elif key_type in ['int', 'integer']: get_func = config.getint elif key_type in ['float']: get_func = config.getfloat elif key_type in ['bool', 'boolean']: get_func = config.getboolean else: msg = 'Invalid type "%s" for option "%s"' % (key_type, key) raise ValueError(msg) value = get_func(section, key) result[section][key] = value else: result[section][key] = key_default_value return dict(result)
def _parse_info(self, info_file): cp = ConfigParser() cp.readfp(info_file) section = 'Library' if cp.has_option(section, 'name'): self._name = cp.get(section, 'name') else: raise MalformedBundleException( 'Content bundle %s does not specify a name' % self._path) if cp.has_option(section, 'library_version'): version = cp.get(section, 'library_version') try: NormalizedVersion(version) except InvalidVersionError: raise MalformedBundleException( 'Content bundle %s has invalid version number %s' % (self._path, version)) self._library_version = version if cp.has_option(section, 'locale'): self._locale = cp.get(section, 'locale') if cp.has_option(section, 'global_name'): self._global_name = cp.get(section, 'global_name') if cp.has_option(section, 'icon'): self._icon = cp.get(section, 'icon') # Compatibility with old content bundles if self._global_name is not None \ and cp.has_option(section, 'bundle_class'): self._global_name = cp.get(section, 'bundle_class') if cp.has_option(section, 'activity_start'): self._activity_start = cp.get(section, 'activity_start') if self._global_name is None: raise MalformedBundleException( 'Content bundle %s must specify global_name' % self._path)
def process_mistral_config(config_path): """ Remove sensitive data (credentials) from the Mistral config. :param config_path: Full absolute path to the mistral config inside /tmp. :type config_path: ``str`` """ assert config_path.startswith('/tmp') if not os.path.isfile(config_path): return config = ConfigParser() config.read(config_path) for section, options in MISTRAL_CONF_OPTIONS_TO_REMOVE.items(): for option in options: if config.has_option(section, option): config.set(section, option, REMOVED_VALUE_NAME) with open(config_path, 'w') as fp: config.write(fp)
def __init__(self, config_file): config = ConfigParser(allow_no_value=True) config.read(config_file) self.serial = { "port": config.get('device', 'port'), "baudrate": config.getint('device', 'baudrate'), "parity": config.get('device', 'parity'), "bytesize": config.getint('device', 'bytesize'), "stopbits": config.getint('device', 'stopbits'), "timeout": config.getfloat('device', 'timeout'), "interCharTimeout": config.getfloat('device', 'char_delay'), } self.serial_char_delay = self.serial["interCharTimeout"] self.storage_dir = config.get("storage", "dir") self.sdb_enabled = config.has_section("simpledb") if self.sdb_enabled: self.sdb_domain = config.get("simpledb", "domain") if config.has_section("aws"): self.aws_access_key_id = config.get("aws", "access_key_id") self.aws_secret_access_key = config.get("aws", "secret_access_key") self.aws_region = config.get("aws", "region") self.station_id = config.get("site", "station_id") self.mail_enabled = config.has_section("mail") if self.mail_enabled: self.mail_host = config.get("mail", "host") or "localhost" self.mail_port = config.getint( "mail", "port") if config.has_option("mail", "port") else 587 self.mail_username = config.get("mail", "username") self.mail_password = config.get("mail", "password") self.mail_to = config.get("mail", "to")
def convert_profile(self): cp = ConfigParser() path = os.path.join(env.get_profile_path(), 'config') cp.read([path]) settings = Gio.Settings('org.sugarlabs.user') if cp.has_option('Buddy', 'NickName'): name = cp.get('Buddy', 'NickName') # decode nickname from ascii-safe chars to unicode nick = name.decode('utf-8') settings.set_string('nick', nick) if cp.has_option('Buddy', 'Color'): color = cp.get('Buddy', 'Color') settings.set_string('color', color) if cp.has_option('Jabber', 'Server'): server = cp.get('Jabber', 'Server') settings = Gio.Settings('org.sugarlabs.collaboration') settings.set_string('jabber-server', server) if cp.has_option('Date', 'Timezone'): timezone = cp.get('Date', 'Timezone') settings = Gio.Settings('org.sugarlabs.date') settings.set_string('timezone', timezone) settings = Gio.Settings('org.sugarlabs.frame') if cp.has_option('Frame', 'HotCorners'): delay = float(cp.get('Frame', 'HotCorners')) settings.set_int('corner-delay', int(delay)) if cp.has_option('Frame', 'WarmEdges'): delay = float(cp.get('Frame', 'WarmEdges')) settings.set_int('edge-delay', int(delay)) if cp.has_option('Server', 'Backup1'): backup1 = cp.get('Server', 'Backup1') settings = Gio.Settings('org.sugarlabs') settings.set_string('backup-url', backup1) if cp.has_option('Sound', 'Volume'): volume = float(cp.get('Sound', 'Volume')) settings = Gio.Settings('org.sugarlabs.sound') settings.set_int('volume', int(volume)) settings = Gio.Settings('org.sugarlabs.power') if cp.has_option('Power', 'AutomaticPM'): state = cp.get('Power', 'AutomaticPM') if state.lower() == 'true': settings.set_boolean('automatic', True) if cp.has_option('Power', 'ExtremePM'): state = cp.get('Power', 'ExtremePM') if state.lower() == 'true': settings.set_boolean('extreme', True) if cp.has_option('Shell', 'FavoritesLayout'): layout = cp.get('Shell', 'FavoritesLayout') settings = Gio.Settings('org.sugarlabs.desktop') settings.set_string('favorites-layout', layout) del cp try: os.unlink(path) except OSError: logging.error('Error removing old profile.')
if not automated or dag.error_code != knope.KNOPE_ERROR_NO_SEGMENTS: dag.write_sub_files() dag.write_dag() print("Successfully created DAG file: '%s'" % dag.get_dag_file()) if submitdag: from subprocess import Popen x = Popen(['condor_submit_dag', dag.get_dag_file()]) x.wait() if x.returncode == 0: print("Submitted DAG file") else: print("Unable to submit DAG file") else: print("Run 'condor_submit_dag %s' to submit DAG file" % dag.get_dag_file()) # output DAG class to pickle file if given if cp.has_option('analysis', 'pickle_file'): try: pfile = cp.get('analysis', 'pickle_file') fp = open(pfile, 'wb') pickle.dump(dag, fp) fp.close() except: print("Warning... could not output analysis class to pickle file") else: print("No new science segments found in current time frame") sys.exit(0)
# Convert prior samples to injections convertsub = os.path.join(rundir, 'samples2injections.sub') converterr = os.path.join( outerlogdir, 'samples2injection-$(cluster)-$(process)-$(node).err') convertout = os.path.join( outerlogdir, 'samples2injection-$(cluster)-$(process)-$(node).out') if opts.injections: injfile = os.path.abspath(opts.injections) else: injfile = os.path.join(rundir, 'priorsamples.xml') approx = prior_cp.get('engine', 'approx') prior2injexe = prior_cp.get('condor', 'pos_to_sim_burst') prior2injjob = pipeline.CondorDAGJob('vanilla', prior2injexe) if main_cp.has_option('analysis', 'accounting_group'): prior2injjob.add_condor_cmd('accounting_group', main_cp.get('analysis', 'accounting_group')) prior2injjob.set_sub_file(convertsub) prior2injjob.set_stderr_file(converterr) prior2injjob.set_stdout_file(convertout) prior2injjob.add_condor_cmd('getenv', 'True') prior2injnode = pipeline.CondorDAGNode(prior2injjob) prior2injnode.add_var_opt('output', injfile) prior2injnode.add_var_opt('num-of-injs', str(opts.trials)) prior2injnode.add_var_opt('approx', approx) prior2injnode.add_var_arg(priorfile) prior2injnode.add_parent(priordagnode) # Create the pipeline based on the injections #main_cp.set('input','injection-file',injfile)
class PypiConfig(object): """Wrapper around the pypi config file""" def __init__(self, config_filename=DIST_CONFIG_FILE, use_setup_cfg=True): """Grab the PyPI configuration. This is .pypirc in the home directory. It is overridable for test purposes. If there is a setup.cfg file in the current directory, we read it too. """ self.config_filename = config_filename self._read_configfile(use_setup_cfg=use_setup_cfg) def _read_configfile(self, use_setup_cfg=True): """Read the PyPI config file and store it (when valid). Usually read the setup.cfg too. """ rc = self.config_filename if not os.path.isabs(rc): rc = os.path.join(os.path.expanduser('~'), self.config_filename) filenames = [rc] if use_setup_cfg: # If there is a setup.cfg in the package, parse it filenames.append('setup.cfg') files = [f for f in filenames if os.path.exists(f)] if not files: self.config = None return self.config = ConfigParser() for filename in files: with codecs.open(filename, 'r', 'utf8') as fp: self.config.readfp(fp) def is_pypi_configured(self): # Do we have configuration for releasing to at least one # pypi-compatible server? if self.config is None: return False if self.is_old_pypi_config(): return True if self.is_new_pypi_config() and len(self.distutils_servers()) > 0: return True return False def is_old_pypi_config(self): if self.config is None: return False try: self.config.get('server-login', 'username') except (NoSectionError, NoOptionError): return False return True def is_new_pypi_config(self): try: self.config.get('distutils', 'index-servers') except (NoSectionError, NoOptionError): return False return True def get_server_config(self, server): """Get url, username, password for server. """ repository_url = DEFAULT_REPOSITORY username = None password = None if self.config.has_section(server): if self.config.has_option(server, 'repository'): repository_url = self.config.get(server, 'repository') if self.config.has_option(server, 'username'): username = self.config.get(server, 'username') if self.config.has_option(server, 'password'): password = self.config.get(server, 'password') if not username and self.config.has_option('server-login', 'username'): username = self.config.get('server-login', 'username') if not password and self.config.has_option('server-login', 'password'): password = self.config.get('server-login', 'password') return { 'repository_url': repository_url, 'username': username, 'password': password, } def distutils_servers(self): """Return a list of known distutils servers. If the config has an old pypi config, remove the default pypi server from the list. """ try: raw_index_servers = self.config.get('distutils', 'index-servers') except (NoSectionError, NoOptionError): return [] ignore_servers = [''] if self.is_old_pypi_config(): # We have already asked about uploading to pypi using the normal # upload. ignore_servers.append('pypi') # Yes, you can even have an old pypi config with a # [distutils] server list. index_servers = [ server.strip() for server in raw_index_servers.split('\n') if server.strip() not in ignore_servers] return index_servers def want_release(self): """Does the user normally want to release this package. Some colleagues find it irritating to have to remember to answer the question "Check out the tag (for tweaks or pypi/distutils server upload)" with the non-default 'no' when in 99 percent of the cases they just make a release specific for a customer, so they always answer 'no' here. This is where an extra config option comes in handy: you can influence the default answer so you can just keep hitting 'Enter' until zest.releaser is done. Either in your ~/.pypirc or in a setup.cfg in a specific package, add this when you want the default answer to this question to be 'no': [zest.releaser] release = no The default when this option has not been set is True. Standard config rules apply, so you can use upper or lower or mixed case and specify 0, false, no or off for boolean False, and 1, on, true or yes for boolean True. """ default = True if self.config is None: return default try: result = self.config.getboolean('zest.releaser', 'release') except (NoSectionError, NoOptionError, ValueError): return default return result def extra_message(self): """Return extra text to be added to commit messages. This can for example be used to skip CI builds. This at least works for Travis. See http://docs.travis-ci.com/user/how-to-skip-a-build/ Enable this mode by adding a ``extra-message`` option, either in the package you want to release, or in your ~/.pypirc:: [zest.releaser] extra-message = [ci skip] """ default = '' if self.config is None: return default try: result = self.config.get('zest.releaser', 'extra-message') except (NoSectionError, NoOptionError, ValueError): return default return result def create_wheel(self): """Should we create a Python wheel for this package? Either in your ~/.pypirc or in a setup.cfg in a specific package, add this when you want to create a Python wheel, next to a standard sdist: [zest.releaser] create-wheel = yes """ if not USE_WHEEL: # If the wheel package is not available, we obviously # cannot create wheels. return False default = False if self.config is None: return default try: result = self.config.getboolean('zest.releaser', 'create-wheel') except (NoSectionError, NoOptionError, ValueError): return default return result def no_input(self): """Return whether the user wants to run in no-input mode. Enable this mode by adding a ``no-input`` option:: [zest.releaser] no-input = yes The default when this option has not been set is False. Standard config rules apply, so you can use upper or lower or mixed case and specify 0, false, no or off for boolean False, and 1, on, true or yes for boolean True. """ default = False if self.config is None: return default try: result = self.config.getboolean('zest.releaser', 'no-input') except (NoSectionError, NoOptionError, ValueError): return default return result
class PypiConfig(object): """Wrapper around the pypi config file""" def __init__(self, config_filename=DIST_CONFIG_FILE, use_setup_cfg=True): """Grab the PyPI configuration. This is .pypirc in the home directory. It is overridable for test purposes. If there is a setup.cfg file in the current directory, we read it too. """ self.config_filename = config_filename self.use_setup_cfg = use_setup_cfg self.reload() def reload(self): """Load the config. Do the initial load of the config. Or reload it in case of problems: this is needed when a pypi upload fails, you edit the .pypirc file to fix the account settings, and tell release to retry the command. """ self._read_configfile(use_setup_cfg=self.use_setup_cfg) def _read_configfile(self, use_setup_cfg=True): """Read the PyPI config file and store it (when valid). Usually read the setup.cfg too. """ rc = self.config_filename if not os.path.isabs(rc): rc = os.path.join(os.path.expanduser('~'), self.config_filename) filenames = [rc] if use_setup_cfg: # If there is a setup.cfg in the package, parse it filenames.append('setup.cfg') files = [f for f in filenames if os.path.exists(f)] if not files: self.config = None return self.config = ConfigParser() for filename in files: with codecs.open(filename, 'r', 'utf8') as fp: self.config.readfp(fp) def is_pypi_configured(self): # Do we have configuration for releasing to at least one # pypi-compatible server? if self.config is None: return False return len(self.distutils_servers()) > 0 def get_server_config(self, server): """Get url, username, password for server. """ repository_url = DEFAULT_REPOSITORY username = None password = None if self.config.has_section(server): if self.config.has_option(server, 'repository'): repository_url = self.config.get(server, 'repository') if self.config.has_option(server, 'username'): username = self.config.get(server, 'username') if self.config.has_option(server, 'password'): password = self.config.get(server, 'password') if not username and self.config.has_option('server-login', 'username'): username = self.config.get('server-login', 'username') if not password and self.config.has_option('server-login', 'password'): password = self.config.get('server-login', 'password') return { 'repository_url': repository_url, 'username': username, 'password': password, } def distutils_servers(self): """Return a list of known distutils servers. If the config has an old pypi config, remove the default pypi server from the list. """ try: index_servers = self.config.get('distutils', 'index-servers').split() except (NoSectionError, NoOptionError): index_servers = [] if not index_servers: # If no distutils index-servers have been given, 'pypi' should be # the default. This is what twine does. if self.config.has_option('server-login', 'username'): # We have a username, so upload to pypi should work fine, even # when no explicit pypi section is in the file. return ['pypi'] # https://github.com/zestsoftware/zest.releaser/issues/199 index_servers = ['pypi'] # The servers all need to have a section in the config file. return [ server for server in index_servers if self.config.has_section(server) ] def want_release(self): """Does the user normally want to release this package. Some colleagues find it irritating to have to remember to answer the question "Check out the tag (for tweaks or pypi/distutils server upload)" with the non-default 'no' when in 99 percent of the cases they just make a release specific for a customer, so they always answer 'no' here. This is where an extra config option comes in handy: you can influence the default answer so you can just keep hitting 'Enter' until zest.releaser is done. Either in your ~/.pypirc or in a setup.cfg in a specific package, add this when you want the default answer to this question to be 'no': [zest.releaser] release = no The default when this option has not been set is True. Standard config rules apply, so you can use upper or lower or mixed case and specify 0, false, no or off for boolean False, and 1, on, true or yes for boolean True. """ return self._get_boolean('zest.releaser', 'release', default=True) def extra_message(self): """Return extra text to be added to commit messages. This can for example be used to skip CI builds. This at least works for Travis. See http://docs.travis-ci.com/user/how-to-skip-a-build/ Enable this mode by adding a ``extra-message`` option, either in the package you want to release, or in your ~/.pypirc:: [zest.releaser] extra-message = [ci skip] """ default = '' if self.config is None: return default try: result = self.config.get('zest.releaser', 'extra-message') except (NoSectionError, NoOptionError, ValueError): return default return result def create_wheel(self): """Should we create a Python wheel for this package? Either in your ~/.pypirc or in a setup.cfg in a specific package, add this when you want to create a Python wheel, next to a standard sdist: [zest.releaser] create-wheel = yes """ if not USE_WHEEL: # If the wheel package is not available, we obviously # cannot create wheels. return False return self._get_boolean('zest.releaser', 'create-wheel') def register_package(self): """Should we try to register this package with a package server? For the standard Python Package Index (PyPI), registering a package is no longer needed: this is done automatically when uploading a distribution for a package. In fact, trying to register may fail. See https://github.com/zestsoftware/zest.releaser/issues/191 So by default zest.releaser will no longer register a package. But you may be using your own package server, and registering may be wanted or even required there. In this case you will need to turn on the register function. In your setup.cfg or ~/.pypirc, use the following to ensure that register is called on the package server: [zest.releaser] register = yes Note that if you have specified multiple package servers, this option is used for all of them. There is no way to register and upload to server A, and only upload to server B. """ return self._get_boolean('zest.releaser', 'register') def no_input(self): """Return whether the user wants to run in no-input mode. Enable this mode by adding a ``no-input`` option:: [zest.releaser] no-input = yes The default when this option has not been set is False. """ return self._get_boolean('zest.releaser', 'no-input') def development_marker(self): """Return development marker to be appended in postrelease. Override the default ``.dev0`` in setup.cfg using a ``development-marker`` option:: [zest.releaser] development-marker = .dev1 Returns default of ``.dev0`` when nothing has been configured. """ default = '.dev0' if self.config is None: return default try: result = self.config.get('zest.releaser', 'development-marker') except (NoSectionError, NoOptionError, ValueError): return default return result def push_changes(self): """Return whether the user wants to push the changes to the remote. Configure this mode by adding a ``push-changes`` option:: [zest.releaser] push-changes = no The default when this option has not been set is True. """ return self._get_boolean('zest.releaser', 'push-changes', default=True) def less_zeroes(self): """Return whether the user prefers less zeroes at the end of a version. Configure this mode by adding a ``less-zeroes`` option:: [zest.releaser] less-zeroes = yes The default when this option has not been set is False. When set to true: - Instead of 1.3.0 we will suggest 1.3. - Instead of 2.0.0 we will suggest 2.0. This only makes sense for the bumpversion command. In the postrelease command we read this option too, but with the current logic it has no effect there. """ return self._get_boolean('zest.releaser', 'less-zeroes') def version_levels(self): """How many levels does the user prefer in a version number? Configure this mode by adding a ``version-levels`` option:: [zest.releaser] version-levels = 3 The default when this option has not been set is 0, which means: no preference, so use the length of the current number. This means when suggesting a next version after 1.2: - with levels=0 we will suggest 1.3: no change - with levels=1 we will still suggest 1.3, as we will not use this to remove numbers, only to add them - with levels=2 we will suggest 1.3 - with levels=3 we will suggest 1.2.1 If the current version number has more levels, we keep them. So next version for 1.2.3.4 with levels=1 is 1.2.3.5. Tweaking version-levels and less-zeroes should give you the version number strategy that you prefer. """ default = 0 if self.config is None: return default try: result = self.config.getint('zest.releaser', 'version-levels') except (NoSectionError, NoOptionError, ValueError): return default if result < 0: return default return result def _get_boolean(self, section, key, default=False): """Get a boolean from the config. Standard config rules apply, so you can use upper or lower or mixed case and specify 0, false, no or off for boolean False, and 1, on, true or yes for boolean True. """ result = default if self.config is not None: try: result = self.config.getboolean(section, key) except (NoSectionError, NoOptionError, ValueError): return result return result
def _parse_info(self, info_file): cp = ConfigParser() cp.readfp(info_file) section = 'Activity' if cp.has_option(section, 'bundle_id'): self._bundle_id = cp.get(section, 'bundle_id') else: if cp.has_option(section, 'service_name'): self._bundle_id = cp.get(section, 'service_name') logging.error('ATTENTION: service_name property in the ' 'activity.info file is deprecated, should be ' 'changed to bundle_id') else: raise MalformedBundleException( 'Activity bundle %s does not specify a bundle_id' % self.get_path()) if ' ' in self._bundle_id: raise MalformedBundleException('Space in bundle_id') if cp.has_option(section, 'name'): self._name = cp.get(section, 'name') else: raise MalformedBundleException( 'Activity bundle %s does not specify a name' % self.get_path()) if cp.has_option(section, 'exec'): self.bundle_exec = cp.get(section, 'exec') else: if cp.has_option(section, 'class'): self.bundle_exec = 'sugar-activity ' + cp.get(section, 'class') logging.error('ATTENTION: class property in the ' 'activity.info file is deprecated, should be ' 'changed to exec') else: raise MalformedBundleException( 'Activity bundle %s must specify exec' % self.get_path()) if cp.has_option(section, 'mime_types'): mime_list = cp.get(section, 'mime_types').strip(';') self._mime_types = [mime.strip() for mime in mime_list.split(';')] if cp.has_option(section, 'show_launcher'): if cp.get(section, 'show_launcher') == 'no': self._show_launcher = False if cp.has_option(section, 'tags'): tag_list = cp.get(section, 'tags').strip(';') self._tags = [tag.strip() for tag in tag_list.split(';')] if cp.has_option(section, 'icon'): self._icon = cp.get(section, 'icon') else: logging.warning( 'Activity bundle %s does not specify an icon' % self.get_path()) if cp.has_option(section, 'activity_version'): version = cp.get(section, 'activity_version') try: NormalizedVersion(version) except InvalidVersionError: raise MalformedBundleException( 'Activity bundle %s has invalid version number %s' % (self.get_path(), version)) self._activity_version = version else: logging.warning( 'Activity bundle %s does not specify an activity_version, ' 'assuming %s' % (self.get_path(), self._activity_version)) if cp.has_option(section, 'summary'): self._summary = cp.get(section, 'summary') if cp.has_option(section, 'description'): self._description = cp.get(section, 'description') if cp.has_option(section, 'single_instance'): if cp.get(section, 'single_instance') == 'yes': self._single_instance = True if cp.has_option(section, 'max_participants'): max_participants = cp.get(section, 'max_participants') try: self._max_participants = int(max_participants) except ValueError: raise MalformedBundleException( 'Activity bundle %s has invalid max_participants %s' % (self.get_path(), max_participants)) if not cp.has_option(section, 'license'): logging.warning( 'Activity bundle %s does not specify a license' % self.get_path())
class SetupConfig(object): """Wrapper around the setup.cfg file if available. One reason is to cleanup setup.cfg from these settings:: [egg_info] tag_build = dev tag_svn_revision = true Another is for optional zest.releaser-specific settings:: [zest.releaser] python-file-with-version = reinout/maurits.py """ config_filename = SETUP_CONFIG_FILE def __init__(self): """Grab the configuration (overridable for test purposes)""" # If there is a setup.cfg in the package, parse it if not os.path.exists(self.config_filename): self.config = None return self.config = ConfigParser() with codecs.open(self.config_filename, 'r', 'utf8') as fp: self.config.readfp(fp) def development_marker(self): """Return development marker to be appended in postrelease Override the default ``.dev0`` in setup.cfg using a ``development-marker`` option:: [zest.releaser] development-marker = .dev1 Returns default of `.dev0` when nothing has been configured. """ try: result = self.config.get('zest.releaser', 'development-marker') except (NoSectionError, NoOptionError, ValueError): result = ".dev0" return result def has_bad_commands(self): if self.config is None: return False if not self.config.has_section('egg_info'): # bail out early as the main section is not there return False bad = False # Check 1. if self.config.has_option('egg_info', 'tag_build'): # Might still be empty. value = self.config.get('egg_info', 'tag_build') if value: logger.warn("%s has [egg_info] tag_build set to %r", self.config_filename, value) bad = True # Check 2. if self.config.has_option('egg_info', 'tag_svn_revision'): if self.config.getboolean('egg_info', 'tag_svn_revision'): value = self.config.get('egg_info', 'tag_svn_revision') logger.warn("%s has [egg_info] tag_svn_revision set to %r", self.config_filename, value) bad = True return bad def fix_config(self): if not self.has_bad_commands(): logger.warn("Cannot fix already fine %s.", self.config_filename) return if self.config.has_option('egg_info', 'tag_build'): self.config.set('egg_info', 'tag_build', '') if self.config.has_option('egg_info', 'tag_svn_revision'): self.config.set('egg_info', 'tag_svn_revision', 'false') new_setup = open(self.config_filename, 'w') try: self.config.write(new_setup) finally: new_setup.close() logger.info("New setup.cfg contents:") print(''.join(open(self.config_filename).readlines())) def python_file_with_version(self): """Return Python filename with ``__version__`` marker, if configured. Enable this by adding a ``python-file-with-version`` option:: [zest.releaser] python-file-with-version = reinout/maurits.py Return None when nothing has been configured. """ default = None if self.config is None: return default try: result = self.config.get( 'zest.releaser', 'python-file-with-version') except (NoSectionError, NoOptionError, ValueError): return default return result
class SetupConfig(object): """Wrapper around the setup.cfg file if available. One reason is to cleanup setup.cfg from these settings:: [egg_info] tag_build = dev tag_svn_revision = true Another is for optional zest.releaser-specific settings:: [zest.releaser] no-input = yes """ config_filename = SETUP_CONFIG_FILE def __init__(self): """Grab the configuration (overridable for test purposes)""" # If there is a setup.cfg in the package, parse it if not os.path.exists(os.path.join(utils.PACKAGE_ROOT, self.config_filename)): self.config = None return self.config = ConfigParser() with codecs.open(self.config_filename, 'r', 'utf8') as fp: self.config.readfp(fp) def has_bad_commands(self): if self.config is None: return False if not self.config.has_section('egg_info'): # bail out early as the main section is not there return False bad = False # Check 1. if self.config.has_option('egg_info', 'tag_build'): # Might still be empty. value = self.config.get('egg_info', 'tag_build') if value: logger.warn("%s has [egg_info] tag_build set to %r", self.config_filename, value) bad = True # Check 2. if self.config.has_option('egg_info', 'tag_svn_revision'): if self.config.getboolean('egg_info', 'tag_svn_revision'): value = self.config.get('egg_info', 'tag_svn_revision') logger.warn("%s has [egg_info] tag_svn_revision set to %r", self.config_filename, value) bad = True return bad def fix_config(self): if not self.has_bad_commands(): logger.warn("Cannot fix already fine %s.", self.config_filename) return if self.config.has_option('egg_info', 'tag_build'): self.config.set('egg_info', 'tag_build', '') if self.config.has_option('egg_info', 'tag_svn_revision'): self.config.set('egg_info', 'tag_svn_revision', 'false') new_setup = open(self.config_filename, 'w') try: self.config.write(new_setup) finally: new_setup.close() logger.info("New setup.cfg contents:") print(''.join(open(self.config_filename).readlines())) def no_input(self): """Return whether the user wants to run in no-input mode. Enable this mode by adding a ``no-input`` option:: [zest.releaser] no-input = yes The default when this option has not been set is False. Standard config rules apply, so you can use upper or lower or mixed case and specify 0, false, no or off for boolean False, and 1, on, true or yes for boolean True. """ default = False if self.config is None: return default try: result = self.config.getboolean('zest.releaser', 'no-input') except (NoSectionError, NoOptionError, ValueError): return default return result def python_file_with_version(self): """Return Python filename with ``__version__`` marker, if configured. Enable this by adding a ``python-file-with-version`` option:: [zest.releaser] python-file-with-version = reinout/maurits.py Return None when nothing has been configured. """ default = None if self.config is None: return default try: result = self.config.get( 'zest.releaser', 'python-file-with-version') except (NoSectionError, NoOptionError, ValueError): return default return result
class MultiPortConfig(object): HW_LB = "HW" @staticmethod def float_x_plus_one_tenth_of_y(x, y): return float(x) + float(y) / 10.0 @staticmethod def make_str(base, iterator): return ' '.join((base.format(x) for x in iterator)) @classmethod def make_range_str(cls, base, start, stop=0, offset=0): if offset and not stop: stop = start + offset return cls.make_str(base, range(start, stop)) @staticmethod def parser_get(parser, section, key, default=None): if parser.has_option(section, key): return parser.get(section, key) return default @staticmethod def make_ip_addr(ip, mask): """ :param ip: ip adddress :type ip: str :param mask: /24 prefix of 255.255.255.0 netmask :type mask: str :return: interface :rtype: IPv4Interface """ try: return ipaddress.ip_interface(six.text_type('/'.join([ip, mask]))) except (TypeError, ValueError): # None so we can skip later return None @classmethod def validate_ip_and_prefixlen(cls, ip_addr, prefixlen): ip_addr = cls.make_ip_addr(ip_addr, prefixlen) return ip_addr.ip.exploded, ip_addr.network.prefixlen def __init__(self, topology_file, config_tpl, tmp_file, vnfd_helper, vnf_type='CGNAT', lb_count=2, worker_threads=3, worker_config='1C/1T', lb_config='SW', socket=0): super(MultiPortConfig, self).__init__() self.topology_file = topology_file self.worker_config = worker_config.split('/')[1].lower() self.worker_threads = self.get_worker_threads(worker_threads) self.vnf_type = vnf_type self.pipe_line = 0 self.vnfd_helper = vnfd_helper self.write_parser = ConfigParser() self.read_parser = ConfigParser() self.read_parser.read(config_tpl) self.master_core = self.read_parser.get("PIPELINE0", "core") self.master_tpl = self.get_config_tpl_data('MASTER') self.arpicmp_tpl = self.get_config_tpl_data('ARPICMP') self.txrx_tpl = self.get_config_tpl_data('TXRX') self.loadb_tpl = self.get_config_tpl_data('LOADB') self.vnf_tpl = self.get_config_tpl_data(vnf_type) self.swq = 0 self.lb_count = int(lb_count) self.lb_config = lb_config self.tmp_file = os.path.join("/tmp", tmp_file) self.pktq_out_os = [] self.socket = socket self.start_core = "" self.pipeline_counter = "" self.txrx_pipeline = "" self._port_pairs = None self.all_ports = [] self.port_pair_list = [] self.lb_to_port_pair_mapping = {} self.init_eal() self.lb_index = None self.mul = 0 self.port_pairs = [] self.ports_len = 0 self.prv_que_handler = None self.vnfd = None self.rules = None self.pktq_out = [] @staticmethod def gen_core(core): # return "s{}c{}".format(self.socket, core) # don't use sockets for VNFs, because we don't want to have to # adjust VM CPU topology. It is virtual anyway return str(core) def make_port_pairs_iter(self, operand, iterable): return (operand(self.vnfd_helper.port_num(x), y) for y in iterable for x in chain.from_iterable(self.port_pairs)) def make_range_port_pairs_iter(self, operand, start, end): return self.make_port_pairs_iter(operand, range(start, end)) def init_eal(self): lines = ['[EAL]\n'] vpci = (v['virtual-interface']["vpci"] for v in self.vnfd_helper.interfaces) lines.extend('w = {0}\n'.format(item) for item in vpci) lines.append('\n') with open(self.tmp_file, 'w') as fh: fh.writelines(lines) def update_timer(self): timer_tpl = self.get_config_tpl_data('TIMER') timer_tpl['core'] = self.gen_core(self.start_core) self.update_write_parser(timer_tpl) self.start_core += 1 def get_config_tpl_data(self, type_value): for section in self.read_parser.sections(): if self.read_parser.has_option(section, 'type'): if type_value == self.read_parser.get(section, 'type'): tpl = OrderedDict(self.read_parser.items(section)) return tpl def get_txrx_tpl_data(self, value): for section in self.read_parser.sections(): if self.read_parser.has_option(section, 'pipeline_txrx_type'): if value == self.read_parser.get(section, 'pipeline_txrx_type'): tpl = OrderedDict(self.read_parser.items(section)) return tpl def init_write_parser_template(self, type_value='ARPICMP'): for section in self.read_parser.sections(): if type_value == self.parser_get(self.read_parser, section, 'type', object()): self.start_core = self.read_parser.getint(section, 'core') self.pipeline_counter = self.read_parser.getint( section, 'core') self.txrx_pipeline = self.read_parser.getint(section, 'core') return self.write_parser.add_section(section) for name, value in self.read_parser.items(section): self.write_parser.set(section, name, value) def update_write_parser(self, data): section = "PIPELINE{0}".format(self.pipeline_counter) self.write_parser.add_section(section) for name, value in data.items(): self.write_parser.set(section, name, value) def get_worker_threads(self, worker_threads): if self.worker_config == '1t': return worker_threads else: return worker_threads - worker_threads % 2 def generate_next_core_id(self): if self.worker_config == '1t': self.start_core += 1 return try: self.start_core = '{}h'.format(int(self.start_core)) except ValueError: self.start_core = int(self.start_core[:-1]) + 1 def get_lb_count(self): self.lb_count = int(min(len(self.port_pair_list), self.lb_count)) def generate_lb_to_port_pair_mapping(self): self.lb_to_port_pair_mapping = defaultdict(int) port_pair_count = len(self.port_pair_list) lb_pair_count = int(port_pair_count / self.lb_count) extra = port_pair_count % self.lb_count extra_iter = repeat(lb_pair_count + 1, extra) norm_iter = repeat(lb_pair_count, port_pair_count - extra) new_values = { i: v for i, v in enumerate(chain(extra_iter, norm_iter), 1) } self.lb_to_port_pair_mapping.update(new_values) def set_priv_to_pub_mapping(self): port_nums = [ tuple(self.vnfd_helper.port_nums(x)) for x in self.port_pair_list ] return "".join(str(y).replace(" ", "") for y in port_nums) def set_priv_que_handler(self): # iterated twice, can't be generator priv_to_pub_map = [ tuple(self.vnfd_helper.port_nums(x)) for x in self.port_pairs ] # must be list to use .index() port_list = list(chain.from_iterable(priv_to_pub_map)) uplink_ports = (x[0] for x in priv_to_pub_map) self.prv_que_handler = '({})'.format("".join( ("{},".format(port_list.index(x)) for x in uplink_ports))) def generate_arp_route_tbl(self): arp_route_tbl_tmpl = "({port0_dst_ip_hex},{port0_netmask_hex},{port_num}," \ "{next_hop_ip_hex})" def build_arp_config(port): dpdk_port_num = self.vnfd_helper.port_num(port) interface = self.vnfd_helper.find_interface( name=port)["virtual-interface"] # We must use the dst because we are on the VNF and we need to # reach the TG. dst_port0_ip = ipaddress.ip_interface( six.text_type("%s/%s" % (interface["dst_ip"], interface["netmask"]))) arp_vars = { "port0_dst_ip_hex": ip_to_hex(dst_port0_ip.network.network_address.exploded), "port0_netmask_hex": ip_to_hex(dst_port0_ip.network.netmask.exploded), # this is the port num that contains port0 subnet and next_hop_ip_hex # this is LINKID which should be based on DPDK port number "port_num": dpdk_port_num, # next hop is dst in this case # must be within subnet "next_hop_ip_hex": ip_to_hex(dst_port0_ip.ip.exploded), } return arp_route_tbl_tmpl.format(**arp_vars) return ' '.join(build_arp_config(port) for port in self.all_ports) def generate_arpicmp_data(self): swq_in_str = self.make_range_str('SWQ{}', self.swq, offset=self.lb_count) self.swq += self.lb_count swq_out_str = self.make_range_str('SWQ{}', self.swq, offset=self.lb_count) self.swq += self.lb_count # ports_mac_list is disabled for some reason # mac_iter = (self.vnfd_helper.find_interface(name=port)['virtual-interface']['local_mac'] # for port in self.all_ports) pktq_in_iter = ('RXQ{}.0'.format(self.vnfd_helper.port_num(x[0])) for x in self.port_pair_list) arpicmp_data = { 'core': self.gen_core(self.start_core), 'pktq_in': swq_in_str, 'pktq_out': swq_out_str, # we need to disable ports_mac_list? # it looks like ports_mac_list is no longer required # 'ports_mac_list': ' '.join(mac_iter), 'pktq_in_prv': ' '.join(pktq_in_iter), 'prv_to_pub_map': self.set_priv_to_pub_mapping(), 'arp_route_tbl': self.generate_arp_route_tbl(), # nd_route_tbl must be set or we get segault on random OpenStack IPv6 traffic # 'nd_route_tbl': "(0064:ff9b:0:0:0:0:9810:6414,120,0,0064:ff9b:0:0:0:0:9810:6414)" # safe default? route discard prefix to localhost 'nd_route_tbl': "(0100::,64,0,::1)" } self.pktq_out_os = swq_out_str.split(' ') # HWLB is a run to complition. So override the pktq_in/pktq_out if self.lb_config == self.HW_LB: self.swq = 0 swq_in_str = \ self.make_range_str('SWQ{}', self.swq, offset=(self.lb_count * self.worker_threads)) arpicmp_data['pktq_in'] = swq_in_str # WA: Since port_pairs will not be populated during arp pipeline self.port_pairs = self.port_pair_list port_iter = \ self.make_port_pairs_iter(self.float_x_plus_one_tenth_of_y, [self.mul]) pktq_out = self.make_str('TXQ{}', port_iter) arpicmp_data['pktq_out'] = pktq_out return arpicmp_data def generate_final_txrx_data(self): swq_start = self.swq - self.ports_len * self.worker_threads txq_start = 0 txq_end = self.worker_threads pktq_out_iter = self.make_range_port_pairs_iter( self.float_x_plus_one_tenth_of_y, txq_start, txq_end) swq_str = self.make_range_str('SWQ{}', swq_start, self.swq) txq_str = self.make_str('TXQ{}', pktq_out_iter) rxtx_data = { 'pktq_in': swq_str, 'pktq_out': txq_str, 'pipeline_txrx_type': 'TXTX', 'core': self.gen_core(self.start_core), } pktq_in = rxtx_data['pktq_in'] pktq_in = '{0} {1}'.format(pktq_in, self.pktq_out_os[self.lb_index - 1]) rxtx_data['pktq_in'] = pktq_in self.pipeline_counter += 1 return rxtx_data def generate_initial_txrx_data(self): pktq_iter = self.make_range_port_pairs_iter( self.float_x_plus_one_tenth_of_y, 0, self.worker_threads) rxq_str = self.make_str('RXQ{}', pktq_iter) swq_str = self.make_range_str('SWQ{}', self.swq, offset=self.ports_len) txrx_data = { 'pktq_in': rxq_str, 'pktq_out': swq_str + ' SWQ{0}'.format(self.lb_index - 1), 'pipeline_txrx_type': 'RXRX', 'core': self.gen_core(self.start_core), } self.pipeline_counter += 1 return txrx_data def generate_lb_data(self): pktq_in = self.make_range_str('SWQ{}', self.swq, offset=self.ports_len) self.swq += self.ports_len offset = self.ports_len * self.worker_threads pktq_out = self.make_range_str('SWQ{}', self.swq, offset=offset) self.pktq_out = pktq_out.split() self.swq += (self.ports_len * self.worker_threads) lb_data = { 'prv_que_handler': self.prv_que_handler, 'pktq_in': pktq_in, 'pktq_out': pktq_out, 'n_vnf_threads': str(self.worker_threads), 'core': self.gen_core(self.start_core), } self.pipeline_counter += 1 return lb_data def generate_vnf_data(self): if self.lb_config == self.HW_LB: port_iter = self.make_port_pairs_iter( self.float_x_plus_one_tenth_of_y, [self.mul]) pktq_in = self.make_str('RXQ{}', port_iter) self.mul += 1 port_iter = self.make_port_pairs_iter( self.float_x_plus_one_tenth_of_y, [self.mul]) pktq_out = self.make_str('TXQ{}', port_iter) pipe_line_data = { 'pktq_in': pktq_in, 'pktq_out': pktq_out + ' SWQ{0}'.format(self.swq), 'prv_que_handler': self.prv_que_handler, 'core': self.gen_core(self.start_core), } self.swq += 1 else: pipe_line_data = { 'pktq_in': ' '.join( (self.pktq_out.pop(0) for _ in range(self.ports_len))), 'pktq_out': self.make_range_str('SWQ{}', self.swq, offset=self.ports_len), 'prv_que_handler': self.prv_que_handler, 'core': self.gen_core(self.start_core), } self.swq += self.ports_len if self.vnf_type in ('ACL', 'VFW'): pipe_line_data.pop('prv_que_handler') if self.vnf_tpl.get('vnf_set'): public_ip_port_range_list = self.vnf_tpl[ 'public_ip_port_range'].split(':') ip_in_hex = '{:x}'.format( int(public_ip_port_range_list[0], 16) + self.lb_index - 1) public_ip_port_range_list[0] = ip_in_hex self.vnf_tpl['public_ip_port_range'] = ':'.join( public_ip_port_range_list) self.pipeline_counter += 1 return pipe_line_data def generate_config_data(self): self.init_write_parser_template() # use master core for master, don't use self.start_core self.write_parser.set('PIPELINE0', 'core', self.gen_core(self.master_core)) arpicmp_data = self.generate_arpicmp_data() self.arpicmp_tpl.update(arpicmp_data) self.update_write_parser(self.arpicmp_tpl) self.start_core += 1 if self.vnf_type == 'CGNAPT': self.pipeline_counter += 1 self.update_timer() for lb in self.lb_to_port_pair_mapping: self.lb_index = lb self.mul = 0 port_pair_count = self.lb_to_port_pair_mapping[lb] if not self.port_pair_list: continue self.port_pairs = self.port_pair_list[:port_pair_count] self.port_pair_list = self.port_pair_list[port_pair_count:] self.ports_len = port_pair_count * 2 self.set_priv_que_handler() if self.lb_config == 'SW': txrx_data = self.generate_initial_txrx_data() self.txrx_tpl.update(txrx_data) self.update_write_parser(self.txrx_tpl) self.start_core += 1 lb_data = self.generate_lb_data() self.loadb_tpl.update(lb_data) self.update_write_parser(self.loadb_tpl) self.start_core += 1 for i in range(self.worker_threads): vnf_data = self.generate_vnf_data() if not self.vnf_tpl: self.vnf_tpl = {} self.vnf_tpl.update(vnf_data) self.update_write_parser(self.vnf_tpl) try: self.vnf_tpl.pop('vnf_set') except KeyError: pass else: self.vnf_tpl.pop('public_ip_port_range') self.generate_next_core_id() if self.lb_config == 'SW': txrx_data = self.generate_final_txrx_data() self.txrx_tpl.update(txrx_data) self.update_write_parser(self.txrx_tpl) self.start_core += 1 self.vnf_tpl = self.get_config_tpl_data(self.vnf_type) def generate_config(self): self._port_pairs = PortPairs(self.vnfd_helper.interfaces) self.port_pair_list = self._port_pairs.port_pair_list self.all_ports = self._port_pairs.all_ports self.get_lb_count() self.generate_lb_to_port_pair_mapping() self.generate_config_data() self.write_parser.write(sys.stdout) with open(self.tmp_file, 'a') as tfh: self.write_parser.write(tfh) def generate_link_config(self): def build_args(port): # lookup interface by name virtual_interface = self.vnfd_helper.find_interface( name=port)["virtual-interface"] local_ip = virtual_interface["local_ip"] netmask = virtual_interface["netmask"] port_num = self.vnfd_helper.port_num(port) port_ip, prefix_len = self.validate_ip_and_prefixlen( local_ip, netmask) return LINK_CONFIG_TEMPLATE.format(port_num, port_ip, prefix_len) return ''.join(build_args(port) for port in self.all_ports) def get_route_data(self, src_key, data_key, port): route_list = self.vnfd['vdu'][0].get(src_key, []) try: return next((route[data_key] for route in route_list if route['if'] == port), None) except (TypeError, StopIteration, KeyError): return None def get_ports_gateway(self, port): return self.get_route_data('routing_table', 'gateway', port) def get_ports_gateway6(self, port): return self.get_route_data('nd_route_tbl', 'gateway', port) def get_netmask_gateway(self, port): return self.get_route_data('routing_table', 'netmask', port) def get_netmask_gateway6(self, port): return self.get_route_data('nd_route_tbl', 'netmask', port) def generate_arp_config(self): arp_config = [] for port in self.all_ports: # ignore gateway, always use TG IP # gateway = self.get_ports_gateway(port) vintf = self.vnfd_helper.find_interface( name=port)["virtual-interface"] dst_mac = vintf["dst_mac"] dst_ip = vintf["dst_ip"] # arp_config.append( # (self.vnfd_helper.port_num(port), gateway, dst_mac, self.txrx_pipeline)) # so dst_mac is the TG dest mac, so we need TG dest IP. # should be dpdk_port_num arp_config.append((self.vnfd_helper.port_num(port), dst_ip, dst_mac, self.txrx_pipeline)) return '\n'.join(('p {3} arpadd {0} {1} {2}'.format(*values) for values in arp_config)) def generate_arp_config6(self): arp_config6 = [] for port in self.all_ports: # ignore gateway, always use TG IP # gateway6 = self.get_ports_gateway6(port) vintf = self.vnfd_helper.find_interface( name=port)["virtual-interface"] dst_mac6 = vintf["dst_mac"] dst_ip6 = vintf["dst_ip"] # arp_config6.append( # (self.vnfd_helper.port_num(port), gateway6, dst_mac6, self.txrx_pipeline)) arp_config6.append((self.vnfd_helper.port_num(port), dst_ip6, dst_mac6, self.txrx_pipeline)) return '\n'.join(('p {3} arpadd {0} {1} {2}'.format(*values) for values in arp_config6)) def generate_action_config(self): port_list = (self.vnfd_helper.port_num(p) for p in self.all_ports) if self.vnf_type == "VFW": template = FW_ACTION_TEMPLATE else: template = ACTION_TEMPLATE return ''.join((template.format(port) for port in port_list)) def get_ip_from_port(self, port): # we can't use gateway because in OpenStack gateways interfer with floating ip routing # return self.make_ip_addr(self.get_ports_gateway(port), self.get_netmask_gateway(port)) vintf = self.vnfd_helper.find_interface(name=port)["virtual-interface"] ip = vintf["local_ip"] netmask = vintf["netmask"] return self.make_ip_addr(ip, netmask) def get_network_and_prefixlen_from_ip_of_port(self, port): ip_addr = self.get_ip_from_port(port) # handle cases with no gateway if ip_addr: return ip_addr.network.network_address.exploded, ip_addr.network.prefixlen else: return None, None def generate_rule_config(self): cmd = 'acl' if self.vnf_type == "ACL" else "vfw" rules_config = self.rules if self.rules else '' new_rules = [] new_ipv6_rules = [] pattern = 'p {0} add {1} {2} {3} {4} {5} 0 65535 0 65535 0 0 {6}' for src_intf, dst_intf in self.port_pair_list: src_port = self.vnfd_helper.port_num(src_intf) dst_port = self.vnfd_helper.port_num(dst_intf) src_net, src_prefix_len = self.get_network_and_prefixlen_from_ip_of_port( src_intf) dst_net, dst_prefix_len = self.get_network_and_prefixlen_from_ip_of_port( dst_intf) # ignore entires with empty values if all((src_net, src_prefix_len, dst_net, dst_prefix_len)): new_rules.append( (cmd, self.txrx_pipeline, src_net, src_prefix_len, dst_net, dst_prefix_len, dst_port)) new_rules.append( (cmd, self.txrx_pipeline, dst_net, dst_prefix_len, src_net, src_prefix_len, src_port)) # src_net = self.get_ports_gateway6(port_pair[0]) # src_prefix_len = self.get_netmask_gateway6(port_pair[0]) # dst_net = self.get_ports_gateway6(port_pair[1]) # dst_prefix_len = self.get_netmask_gateway6(port_pair[0]) # # ignore entires with empty values # if all((src_net, src_prefix_len, dst_net, dst_prefix_len)): # new_ipv6_rules.append((cmd, self.txrx_pipeline, src_net, src_prefix_len, # dst_net, dst_prefix_len, dst_port)) # new_ipv6_rules.append((cmd, self.txrx_pipeline, dst_net, dst_prefix_len, # src_net, src_prefix_len, src_port)) acl_apply = "\np %s applyruleset" % cmd new_rules_config = '\n'.join( pattern.format(*values) for values in chain(new_rules, new_ipv6_rules)) return ''.join([rules_config, new_rules_config, acl_apply]) def generate_script_data(self): self._port_pairs = PortPairs(self.vnfd_helper.interfaces) self.port_pair_list = self._port_pairs.port_pair_list self.get_lb_count() script_data = { 'link_config': self.generate_link_config(), 'arp_config': self.generate_arp_config(), # disable IPv6 for now # 'arp_config6': self.generate_arp_config6(), 'arp_config6': "", 'actions': '', 'rules': '', } if self.vnf_type in ('ACL', 'VFW'): script_data.update({ 'actions': self.generate_action_config(), 'rules': self.generate_rule_config(), }) return script_data def generate_script(self, vnfd, rules=None): self.vnfd = vnfd self.rules = rules script_data = self.generate_script_data() script = SCRIPT_TPL.format(**script_data) if self.lb_config == self.HW_LB: script += 'set fwd rxonly' hwlb_tpl = """ set_sym_hash_ena_per_port {0} enable set_hash_global_config {0} simple_xor ipv4-udp enable set_sym_hash_ena_per_port {1} enable set_hash_global_config {1} simple_xor ipv4-udp enable set_hash_input_set {0} ipv4-udp src-ipv4 udp-src-port add set_hash_input_set {1} ipv4-udp dst-ipv4 udp-dst-port add set_hash_input_set {0} ipv6-udp src-ipv6 udp-src-port add set_hash_input_set {1} ipv6-udp dst-ipv6 udp-dst-port add """ for port_pair in self.port_pair_list: script += hwlb_tpl.format( *(self.vnfd_helper.port_nums(port_pair))) return script
from sherpa.data import Data2D, Data1D, BaseData, Data2DInt from sherpa.astro.data import DataIMG, DataIMGInt, DataARF, DataRMF, DataPHA, DataRosatRMF from sherpa import get_config import importlib config = ConfigParser() config.read(get_config()) io_opt = config.get('options', 'io_pkg') io_opt = str(io_opt).strip().lower() # Python 3 allows the following (although should move to the dictionary # style provided by the mapping protocal access. # # ogip_emin = config.get('ogip', 'minimum_energy', fallback='1.0e-10') if config.has_option('ogip', 'minimum_energy'): ogip_emin = config.get('ogip', 'minimum_energy') else: # The original version of minimum_energy is 1e-10 keV, so use it as # the default value. ogip_emin = '1.0e-10' if ogip_emin.upper() == 'NONE': ogip_emin = None else: emsg = "Invalid value for [ogip] minimum_energy config value; " + \ "it must be None or a float > 0" try: ogip_emin = float(ogip_emin) except ValueError: raise ValueError(emsg)
class SetupConfig(object): """Wrapper around the setup.cfg file if available. One reason is to cleanup setup.cfg from these settings:: [egg_info] tag_build = dev tag_svn_revision = true Another is for optional zest.releaser-specific settings:: [zest.releaser] python-file-with-version = reinout/maurits.py """ config_filename = SETUP_CONFIG_FILE def __init__(self): """Grab the configuration (overridable for test purposes)""" # If there is a setup.cfg in the package, parse it if not os.path.exists(self.config_filename): self.config = None return self.config = ConfigParser() with codecs.open(self.config_filename, 'r', 'utf8') as fp: self.config.readfp(fp) def has_bad_commands(self): if self.config is None: return False if not self.config.has_section('egg_info'): # bail out early as the main section is not there return False bad = False # Check 1. if self.config.has_option('egg_info', 'tag_build'): # Might still be empty. value = self.config.get('egg_info', 'tag_build') if value: logger.warn("%s has [egg_info] tag_build set to %r", self.config_filename, value) bad = True # Check 2. if self.config.has_option('egg_info', 'tag_svn_revision'): if self.config.getboolean('egg_info', 'tag_svn_revision'): value = self.config.get('egg_info', 'tag_svn_revision') logger.warn("%s has [egg_info] tag_svn_revision set to %r", self.config_filename, value) bad = True return bad def fix_config(self): if not self.has_bad_commands(): logger.warn("Cannot fix already fine %s.", self.config_filename) return if self.config.has_option('egg_info', 'tag_build'): self.config.set('egg_info', 'tag_build', '') if self.config.has_option('egg_info', 'tag_svn_revision'): self.config.set('egg_info', 'tag_svn_revision', 'false') new_setup = open(self.config_filename, 'w') try: self.config.write(new_setup) finally: new_setup.close() logger.info("New setup.cfg contents:") print(''.join(open(self.config_filename).readlines())) def python_file_with_version(self): """Return Python filename with ``__version__`` marker, if configured. Enable this by adding a ``python-file-with-version`` option:: [zest.releaser] python-file-with-version = reinout/maurits.py Return None when nothing has been configured. """ default = None if self.config is None: return default try: result = self.config.get('zest.releaser', 'python-file-with-version') except (NoSectionError, NoOptionError, ValueError): return default return result
def loadConfig(self): """Load build configuration from the spec file.""" specfiles = [spec for spec in os.listdir(self.spec_dir) if spec.endswith('.ini')] if len(specfiles) == 0: raise BuildError('No .ini file found') elif len(specfiles) > 1: raise BuildError('Multiple .ini files found') conf = ConfigParser() conf.read(os.path.join(self.spec_dir, specfiles[0])) # [naming] section for entry in ('name', 'version', 'release', 'description'): setattr(self, entry, conf.get('naming', entry)) if conf.has_option('naming', 'epoch'): self.epoch = conf.get('naming', 'epoch') # [building] section self.platform = conf.get('building', 'platform') # preinstalled are paths to files or directories that must exist # in the VM for it to execute the build. # If the path ends in / or \ it must be a directory, otherwise it must # be a file. # They may be specified as Cygwin (/cygdrive/c/...) or Windows (C:\...) # absolute paths, or without a path in which case it is searched for # on the PATH. if conf.has_option('building', 'preinstalled'): self.preinstalled.extend([e.strip() for e in conf.get('building', 'preinstalled').split('\n') if e]) # buildrequires and provides are multi-valued (space-separated) for br in conf.get('building', 'buildrequires').split(): # buildrequires is a space-separated list # each item in the list is in the format: # pkgname[:opt1:opt2=val2:...] # the options are put into a dict # if the option has no =val, the value in the dict will be None if br: br = br.split(':') bropts = {} for opt in br[1:]: if '=' in opt: key, val = opt.split('=', 1) else: key = opt val = None bropts[key] = val self.buildrequires.append((br[0], bropts)) for prov in conf.get('building', 'provides').split(): if prov: self.provides.append(prov) # optionally specify a shell to use (defaults to bash) # valid values are: cmd, cmd.exe (alias for cmd), and bash if conf.has_option('building', 'shell'): self.shell = conf.get('building', 'shell') else: self.shell = 'bash' # execute is multi-valued (newline-separated) self.execute.extend([e.strip() for e in conf.get('building', 'execute').split('\n') if e]) # postbuild are files or directories that must exist after the build is # complete, but are not included in the build output # they are specified as paths relative the source directory, and may be # in Unix or Windows format # each entry may contain shell-style globs, and one or more files # matching the glob is considered valid if conf.has_option('building', 'postbuild'): for entry in conf.get('building', 'postbuild').split('\n'): entry = entry.strip() if not entry: continue for var in ('name', 'version', 'release'): entry = entry.replace('$' + var, getattr(self, var)) self.postbuild.append(entry) # [files] section for entry in conf.get('files', 'output').split('\n'): entry = entry.strip() if not entry: continue tokens = entry.split(':') filename = tokens[0] for var in ('name', 'version', 'release'): filename = filename.replace('$' + var, getattr(self, var)) metadata = {} metadata['platforms'] = tokens[1].split(',') if len(tokens) > 2: metadata['flags'] = tokens[2].split(',') else: metadata['flags'] = [] self.output[filename] = metadata self.logs.extend([e.strip() for e in conf.get('files', 'logs').split('\n') if e])
def parse(self): """ Parse the config and return a dict with the parsed values. :rtype: ``dict`` """ result = defaultdict(dict) if not os.path.isfile(self.config_file_path): # Config doesn't exist, return the default values return CONFIG_DEFAULT_VALUES config_dir_path = os.path.dirname(self.config_file_path) if self.validate_config_permissions: # Make sure the directory permissions == 0o770 if bool(os.stat(config_dir_path).st_mode & 0o7): self.LOG.warn( "The StackStorm configuration directory permissions are " "insecure (too permissive): others have access.") # Make sure the setgid bit is set on the directory if not bool(os.stat(config_dir_path).st_mode & 0o2000): self.LOG.info( "The SGID bit is not set on the StackStorm configuration " "directory.") # Make sure the file permissions == 0o660 if bool(os.stat(self.config_file_path).st_mode & 0o7): self.LOG.warn( "The StackStorm configuration file permissions are " "insecure: others have access.") config = ConfigParser() with io.open(self.config_file_path, 'r', encoding='utf8') as fp: config.readfp(fp) for section, keys in six.iteritems(CONFIG_FILE_OPTIONS): for key, options in six.iteritems(keys): key_type = options['type'] key_default_value = options['default'] if config.has_option(section, key): if key_type in ['str', 'string']: get_func = config.get elif key_type in ['int', 'integer']: get_func = config.getint elif key_type in ['float']: get_func = config.getfloat elif key_type in ['bool', 'boolean']: get_func = config.getboolean else: msg = 'Invalid type "%s" for option "%s"' % (key_type, key) raise ValueError(msg) value = get_func(section, key) result[section][key] = value else: result[section][key] = key_default_value return dict(result)
include_dirs = [] extra_compile_args = [] extra_link_args = [] extra_objects = [] libs = ['ldap', 'lber'] defines = [ ] extra_files = [] LDAP_CLASS = OpenLDAP2 #-- Read the [_ldap] section of setup.cfg cfg = ConfigParser() cfg.read('setup.cfg') if cfg.has_section('_ldap'): for name in dir(LDAP_CLASS): if cfg.has_option('_ldap', name): print(name + ': ' + cfg.get('_ldap', name)) setattr(LDAP_CLASS, name, cfg.get('_ldap', name).split()) for i in range(len(LDAP_CLASS.defines)): LDAP_CLASS.defines[i]=((LDAP_CLASS.defines[i],None)) for i in range(len(LDAP_CLASS.extra_files)): destdir, origfiles = LDAP_CLASS.extra_files[i].split(':') origfileslist = origfiles.split(',') LDAP_CLASS.extra_files[i]=(destdir, origfileslist) #-- Let distutils/setuptools do the rest name = 'python-ldap' # Python 2.3.6+ and setuptools are needed to build eggs, so
def parse(self): """ Parse the config and return a dict with the parsed values. :rtype: ``dict`` """ result = defaultdict(dict) if not os.path.isfile(self.config_file_path): # Config doesn't exist, return the default values return CONFIG_DEFAULT_VALUES config_dir_path = os.path.dirname(self.config_file_path) if self.validate_config_permissions: # Make sure the directory permissions == 0o770 if bool(os.stat(config_dir_path).st_mode & 0o7): self.LOG.warn( "The StackStorm configuration directory permissions are " "insecure (too permissive): others have access." ) # Make sure the setgid bit is set on the directory if not bool(os.stat(config_dir_path).st_mode & 0o2000): self.LOG.info( "The SGID bit is not set on the StackStorm configuration " "directory." ) # Make sure the file permissions == 0o660 if bool(os.stat(self.config_file_path).st_mode & 0o7): self.LOG.warn( "The StackStorm configuration file permissions are " "insecure: others have access." ) config = ConfigParser() with io.open(self.config_file_path, "r", encoding="utf8") as fp: config.readfp(fp) for section, keys in six.iteritems(CONFIG_FILE_OPTIONS): for key, options in six.iteritems(keys): key_type = options["type"] key_default_value = options["default"] if config.has_option(section, key): if key_type in ["str", "string"]: get_func = config.get elif key_type in ["int", "integer"]: get_func = config.getint elif key_type in ["float"]: get_func = config.getfloat elif key_type in ["bool", "boolean"]: get_func = config.getboolean else: msg = 'Invalid type "%s" for option "%s"' % (key_type, key) raise ValueError(msg) value = get_func(section, key, raw=True) result[section][key] = value else: result[section][key] = key_default_value return dict(result)
class PypiConfig(BaseConfig): """Wrapper around the pypi config file""" def __init__(self, config_filename=DIST_CONFIG_FILE, use_setup_cfg=True): """Grab the PyPI configuration. This is .pypirc in the home directory. It is overridable for test purposes. If there is a setup.cfg file in the current directory, we read it too. """ self.config_filename = config_filename self.use_setup_cfg = use_setup_cfg self.reload() def reload(self): """Load the config. Do the initial load of the config. Or reload it in case of problems: this is needed when a pypi upload fails, you edit the .pypirc file to fix the account settings, and tell release to retry the command. """ self._read_configfile(use_setup_cfg=self.use_setup_cfg) def _read_configfile(self, use_setup_cfg=True): """Read the PyPI config file and store it (when valid). Usually read the setup.cfg too. """ rc = self.config_filename if not os.path.isabs(rc): rc = os.path.join(os.path.expanduser('~'), self.config_filename) filenames = [rc] if use_setup_cfg: # If there is a setup.cfg in the package, parse it filenames.append('setup.cfg') files = [f for f in filenames if os.path.exists(f)] if not files: self.config = None return self.config = ConfigParser() self.config.read(files) def is_pypi_configured(self): # Do we have configuration for releasing to at least one # pypi-compatible server? if self.config is None: return False return len(self.distutils_servers()) > 0 def distutils_servers(self): """Return a list of known distutils servers. If the config has an old pypi config, remove the default pypi server from the list. """ try: index_servers = self._get_text( 'distutils', 'index-servers', default='').split() except (NoSectionError, NoOptionError): index_servers = [] if not index_servers: # If no distutils index-servers have been given, 'pypi' should be # the default. This is what twine does. if self.config.has_option('server-login', 'username'): # We have a username, so upload to pypi should work fine, even # when no explicit pypi section is in the file. return ['pypi'] # https://github.com/zestsoftware/zest.releaser/issues/199 index_servers = ['pypi'] # The servers all need to have a section in the config file. return [server for server in index_servers if self.config.has_section(server)] def want_release(self): """Does the user normally want to release this package. Some colleagues find it irritating to have to remember to answer the question "Check out the tag (for tweaks or pypi/distutils server upload)" with the non-default 'no' when in 99 percent of the cases they just make a release specific for a customer, so they always answer 'no' here. This is where an extra config option comes in handy: you can influence the default answer so you can just keep hitting 'Enter' until zest.releaser is done. Either in your ~/.pypirc or in a setup.cfg in a specific package, add this when you want the default answer to this question to be 'no': [zest.releaser] release = no The default when this option has not been set is True. Standard config rules apply, so you can use upper or lower or mixed case and specify 0, false, no or off for boolean False, and 1, on, true or yes for boolean True. """ return self._get_boolean('zest.releaser', 'release', default=True) def extra_message(self): """Return extra text to be added to commit messages. This can for example be used to skip CI builds. This at least works for Travis. See http://docs.travis-ci.com/user/how-to-skip-a-build/ Enable this mode by adding a ``extra-message`` option, either in the package you want to release, or in your ~/.pypirc:: [zest.releaser] extra-message = [ci skip] """ default = '' if self.config is None: return default try: result = self._get_text( 'zest.releaser', 'extra-message', default=default) except (NoSectionError, NoOptionError, ValueError): return default return result def history_file(self): """Return path of history file. Usually zest.releaser can find the correct one on its own. But sometimes it may not find anything, or it finds multiple and selects the wrong one. Configure this by adding a ``history-file`` option, either in the package you want to release, or in your ~/.pypirc:: [zest.releaser] history-file = deep/down/historie.doc """ default = '' if self.config is None: return default marker = object() try: result = self._get_text( 'zest.releaser', 'history-file', default=marker) except (NoSectionError, NoOptionError, ValueError): return default if result == marker: # We were reading an underscore instead of a dash at first. try: result = self._get_text( 'zest.releaser', 'history_file', default=default) except (NoSectionError, NoOptionError, ValueError): return default return result def encoding(self): """Return encoding to use for text files. Mostly the changelog and if needed `setup.py`. The encoding cannot always be determined correctly. This setting is a way to fix that. See https://github.com/zestsoftware/zest.releaser/issues/264 Configure this by adding an ``encoding`` option, either in the package you want to release, or in your ~/.pypirc:: [zest.releaser] encoding = utf-8 """ default = '' if self.config is None: return default try: result = self._get_text( 'zest.releaser', 'encoding', default=default, raw=True) except (NoSectionError, NoOptionError, ValueError): return default return result def create_wheel(self): """Should we create a Python wheel for this package? Either in your ~/.pypirc or in a setup.cfg in a specific package, add this when you want to create a Python wheel, next to a standard sdist: [zest.releaser] create-wheel = yes If there is no setting for ``create-wheel``, then if there is a ``[bdist_wheel]`` section, it is treated as if ``create-wheel`` was true. We used to look at the value of the ``universal`` option, but that no longer matters. This will still create a wheel: [bdist_wheel] universal = 0 See https://github.com/zestsoftware/zest.releaser/issues/315 """ if not USE_WHEEL: # If the wheel package is not available, we obviously # cannot create wheels. return False create_setting = self._get_boolean( 'zest.releaser', 'create-wheel', None) if create_setting is not None: # User specified this setting, it overrides # inferring from bdist_wheel return create_setting # No zest.releaser setting, are they asking for a universal wheel? # Then they want wheels in general. return self.config.has_section('bdist_wheel') def register_package(self): """Should we try to register this package with a package server? For the standard Python Package Index (PyPI), registering a package is no longer needed: this is done automatically when uploading a distribution for a package. In fact, trying to register may fail. See https://github.com/zestsoftware/zest.releaser/issues/191 So by default zest.releaser will no longer register a package. But you may be using your own package server, and registering may be wanted or even required there. In this case you will need to turn on the register function. In your setup.cfg or ~/.pypirc, use the following to ensure that register is called on the package server: [zest.releaser] register = yes Note that if you have specified multiple package servers, this option is used for all of them. There is no way to register and upload to server A, and only upload to server B. """ return self._get_boolean('zest.releaser', 'register') def no_input(self): """Return whether the user wants to run in no-input mode. Enable this mode by adding a ``no-input`` option:: [zest.releaser] no-input = yes The default when this option has not been set is False. """ return self._get_boolean('zest.releaser', 'no-input') def development_marker(self): """Return development marker to be appended in postrelease. Override the default ``.dev0`` in ~/.pypirc or setup.cfg using a ``development-marker`` option:: [zest.releaser] development-marker = .dev1 Returns default of ``.dev0`` when nothing has been configured. """ default = '.dev0' if self.config is None: return default try: result = self._get_text( 'zest.releaser', 'development-marker', default=default) except (NoSectionError, NoOptionError, ValueError): return default return result def push_changes(self): """Return whether the user wants to push the changes to the remote. Configure this mode by adding a ``push-changes`` option:: [zest.releaser] push-changes = no The default when this option has not been set is True. """ return self._get_boolean('zest.releaser', 'push-changes', default=True) def less_zeroes(self): """Return whether the user prefers less zeroes at the end of a version. Configure this mode by adding a ``less-zeroes`` option:: [zest.releaser] less-zeroes = yes The default when this option has not been set is False. When set to true: - Instead of 1.3.0 we will suggest 1.3. - Instead of 2.0.0 we will suggest 2.0. This only makes sense for the bumpversion command. In the postrelease command we read this option too, but with the current logic it has no effect there. """ return self._get_boolean('zest.releaser', 'less-zeroes') def version_levels(self): """How many levels does the user prefer in a version number? Configure this mode by adding a ``version-levels`` option:: [zest.releaser] version-levels = 3 The default when this option has not been set is 0, which means: no preference, so use the length of the current number. This means when suggesting a next version after 1.2: - with levels=0 we will suggest 1.3: no change - with levels=1 we will still suggest 1.3, as we will not use this to remove numbers, only to add them - with levels=2 we will suggest 1.3 - with levels=3 we will suggest 1.2.1 If the current version number has more levels, we keep them. So next version for 1.2.3.4 with levels=1 is 1.2.3.5. Tweaking version-levels and less-zeroes should give you the version number strategy that you prefer. """ default = 0 if self.config is None: return default try: result = self.config.getint('zest.releaser', 'version-levels') except (NoSectionError, NoOptionError, ValueError): return default if result < 0: return default return result _tag_format_deprecated_message = "\n".join(line.strip() for line in """ `tag-format` contains deprecated `%%(version)s` format. Please change to: [zest.releaser] tag-format = %s """.strip().splitlines()) def tag_format(self, version): """Return the formatted tag that should be used in the release. Configure it in ~/.pypirc or setup.cfg using a ``tag-format`` option:: [zest.releaser] tag-format = v{version} ``tag-format`` must contain exaclty one formatting instruction: for the ``version`` key. Accepts also ``%(version)s`` format for backward compatibility. The default format, when nothing has been configured, is ``{version}``. """ fmt = '{version}' if self.config is not None: try: fmt = self._get_text( 'zest.releaser', 'tag-format', default=fmt, raw=True) except (NoSectionError, NoOptionError, ValueError): pass if '{version}' in fmt: return fmt.format(version=version) # BBB: if '%(version)s' in fmt: proposed_fmt = fmt.replace("%(version)s", "{version}") print(self._tag_format_deprecated_message % proposed_fmt) return fmt % {'version': version} print("{version} needs to be part of 'tag-format': %s" % fmt) sys.exit(1) def tag_message(self, version): """Return the commit message to be used when tagging. Configure it in ~/.pypirc or setup.cfg using a ``tag-message`` option:: [zest.releaser] tag-message = Creating v{version} tag. ``tag-message`` must contain exaclty one formatting instruction: for the ``version`` key. The default format is ``Tagging {version}``. """ fmt = 'Tagging {version}' if self.config: try: fmt = self._get_text( 'zest.releaser', 'tag-message', default=fmt, raw=True) except (NoSectionError, NoOptionError, ValueError): pass if '{version}' not in fmt: print("{version} needs to be part of 'tag-message': '%s'" % fmt) sys.exit(1) return fmt.format(version=version) def tag_signing(self): """Return whether the tag should be signed. Configure it in ~/.pypirc or setup.cfg using a ``tag-signing`` option:: [zest.releaser] tag-signing = yes ``tag-signing`` must contain exaclty one word which will be converted to a boolean. Currently are accepted (case insensitively): 0, false, no, off for False, and 1, true, yes, on for True). The default when this option has not been set is False. """ return self._get_boolean('zest.releaser', 'tag-signing', default=False) def date_format(self): """Return the string format for the date used in the changelog. Override the default ``%Y-%m-%d`` in ~/.pypirc or setup.cfg using a ``date-format`` option:: [zest.releaser] date-format = %%B %%e, %%Y Note: the % signs should be doubled for compatibility with other tools (i.e. pip) that parse setup.cfg using the interpolating ConfigParser. Returns default of ``%Y-%m-%d`` when nothing has been configured. """ default = '%Y-%m-%d' if self.config is None: return default try: result = self._get_text( 'zest.releaser', 'date-format', default=default ).replace('%%', '%') except (NoSectionError, NoOptionError, ValueError): return default return result