def read_reg(filename, encoding='utf-16'): with codecs.open(filename, encoding=encoding) as f: data = f.read() # removes the header containing the regedit info at the top of file data = re.sub(r'^[^\[]*\n', '', data, flags=re.S) # using the configuration parser to identify the sections of the file based on [section name] = section config = RawConfigParser(strict=False, allow_no_value=True) # dirty hack for "disabling" case-insensitive keys in "configparser" config.optionxform=str config.read_string(data) data = [] # iterate over sections and keys and generate `data` for pandas.DataFrame for section in config.sections(): # sets the stage for just the section or the Path if there is no data below it in the reg file if not config[section]: data.append([section, None, None, None]) for key in config[section]: tp = val = None if config[section][key]: # This if statement identifies the value types and values # still have issues with value type if the value contains a path for windows removes the drive letter if ':' in config[section][key]: tp = config[section][key].split(':')[0] val = config[section][key].split(':')[1] else: val = config[section][key].replace('"', '') data.append([section, key.replace('"', ''), tp, val]) # This sets the dataframe with the data collected from above to the columns required. df = pd.DataFrame(data, columns=['Path', 'Key', 'Type', 'Value']) df.loc[df.Type.notnull() & df.Type.str.contains(r'^hex'), 'Value'] = \ df.loc[df.Type.notnull() & df.Type.str.contains(r'^hex'), 'Value'].str.replace(r'\\\n', '') df.to_csv(r'.\\Registry_keys\by host converted\%s.csv' % (str(filename).split('\\')[-1].strip(".reg")), encoding='UTF-8', index=False)
def load(self, csv): conf = RawConfigParser() with open(csv, 'rb') as f: input_bytes = f.read() decoded = input_bytes.decode(chardet.detect(input_bytes)['encoding']) decoded = '[__global__]\n' + decoded conf.read_string(decoded) for sec in conf.sections(): if not self.conf.has_section(sec): self.conf.add_section(sec) self.crap.add_section(sec) for k, v in conf.items(sec): is_crap = False if '__' in v: is_crap = True if not is_crap: if self.conf.has_option(sec, k): if self.conf.get(sec, k).lower() != v.lower(): print('Overwriting locale %s (%r -> %r)' % (k, self.conf.get(sec, k), v)) self.conf.set(sec, k, v) else: if self.crap.has_option(sec, k): print('Overwriting crap locale %s (%r -> %r)' % (k, self.crap.get(sec, k), v)) self.crap.set(sec, k, v)
class KeyStore(object): """Store OAuth2 client ids and client 'secrets'. Google recognise that client secrets can't be kept secret in an application that runs on a user's computer. See https://developers.google.com/identity/protocols/OAuth2InstalledApp for more background. However, they also say the secret "may not be embedded in open source projects" (see section 4.b.1 of https://developers.google.com/terms/). Photini stores the client credentials in a separate file, using mild obfuscation to hide the actual values. If this is insufficient to satisfy Google then the keys file will have to be removed from open source and distributed by other means. Or users will need to create their own by registering as a developer at Google. The position with Flickr keys is less clear, but there's no harm in obfuscating them as well. """ def __init__(self): self.config = RawConfigParser() if sys.version_info >= (3, 2): data = pkg_resources.resource_string('photini', 'data/keys.txt') data = data.decode('utf-8') self.config.read_string(data) else: data = pkg_resources.resource_stream('photini', 'data/keys.txt') self.config.readfp(data) def get(self, section, option): value = self.config.get(section, option) value = value.encode('ascii') value = codecs.decode(value, 'base64_codec') return value.decode('ascii')
def test_defaultreturn(self): """base tests for var with defaults""" class _Example(DefConfigMixin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.requiredvars = { 'var1': { 'default': 'default string', 'description': 'description for var1', } } configtext = """ [test] var1 = bla """ config = RawConfigParser() config.read_string(configtext) ex = _Example(config=config) var1 = ex.config.get("test", "var1") self.assertEqual("bla", var1) with self.assertRaises(NoOptionError): _ = ex.config.getboolean("test", "var2")
def test_plugin_defaults(self): """Test if parser is wrapped due to mixin""" class _Plugin(ScannerPlugin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.requiredvars = { 'var1': { 'default': 'default string', 'description': 'description for var1', } } configtext = """ [test] var1 = bla """ config = RawConfigParser() config.read_string(configtext) pl = _Plugin(config=config, section="test") # The RawConfigParser should have been wrapped into # a ConfigWrapper object which is returned if accessing config self.assertIsInstance(pl.config, ConfigWrapper) var1 = pl.config.get("test", "var1") print(f"Scanner plugin -> var1: {var1}") self.assertEqual("bla", var1)
def test_default_realbool(self): """ Test if default boolean variable is returned if given as bool Note: In the configuration file, a boolean has to be given as string, so the string should be converted to a boolean. For the wrapper a boolean value has to work as well. """ config = RawConfigParser() configtext = """ [test] """ config.read_string(configtext) with self.assertRaises(NoOptionError): _ = config.getboolean("test", "var1") wrap = ConfigWrapper( config, { "var1": { "default": False, "description": "this is just a test for integer var1" } }) var1 = wrap.getboolean("test", "var1") print(f"From ConfigWrapper -> var1: {var1}") self.assertIsInstance(var1, bool) self.assertEqual(var1, False)
def test_defaultdictchange(self): """Test if parser is wrapped due to mixin""" class _Example(DefConfigMixin): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.requiredvars = { 'var1': { 'default': 'default string', 'description': 'description for var1', } } configtext = """ [test] """ config = RawConfigParser() config.read_string(configtext) ex = _Example(config=config) var1 = ex.config.get("test", "var1") print(f"After initial -> var1: {var1}") self.assertEqual("default string", var1) ex.requiredvars = { 'var1': { 'default': 'new default', 'description': 'description for var1', } } var1 = ex.config.get("test", "var1") print(f"After new dict -> var1: {var1}") self.assertEqual("new default", var1)
def test_existing_bool(self): """Test if float variable given in config file is correctly returned""" config = RawConfigParser() configtext = """ [test] var1 = true """ config.read_string(configtext) var1 = config.getboolean("test", "var1") print(f"From RawConfigParser -> var1: {var1}") self.assertEqual(var1, True) wrap = ConfigWrapper( config, { "var1": { "default": "false", "description": "this is just a test for float var1" } }) var1 = wrap.getboolean("test", "var1") print(f"From ConfigWrapper -> var1: {var1}") self.assertIsInstance(var1, bool) self.assertEqual(var1, True)
def test_default_realfloat(self): """ Test if default float variable is returned if given as string Note: In the configuration file, a float can be given as string, so the string should be converted to a float. For the wrapper a string value has to work as well. """ config = RawConfigParser() configtext = """ [test] """ config.read_string(configtext) with self.assertRaises(NoOptionError): _ = config.getfloat("test", "var1") wrap = ConfigWrapper( config, { "var1": { "default": "1.1", "description": "this is just a test for float var1" } }) var1 = wrap.getfloat("test", "var1") print(f"From ConfigWrapper -> var1: {var1}") self.assertIsInstance(var1, float) self.assertEqual(var1, 1.1)
def test_existing_string(self): """Test if string variable given in config file is correctly returned""" config = RawConfigParser() configtext = """ [test] var1 = test """ config.read_string(configtext) var1 = config.get("test", "var1") print(f"From RawConfigParser -> var1: {var1}") self.assertEqual(var1, "test") wrap = ConfigWrapper( config, { "var1": { "default": "<undefined>", "description": "this is just a test for string var1" } }) var1 = wrap.get("test", "var1") print(f"From ConfigWrapper -> var1: {var1}") self.assertEqual(var1, "test") self.assertIsInstance(var1, str)
def load(self, csv): conf = RawConfigParser() with open(csv, 'rb') as f: input_bytes = f.read() decoded = input_bytes.decode( chardet.detect(input_bytes)['encoding']) decoded = '[__global__]\n' + decoded conf.read_string(decoded) for sec in conf.sections(): if not self.conf.has_section(sec): self.conf.add_section(sec) self.crap.add_section(sec) for k, v in conf.items(sec): is_crap = False if '__' in v: is_crap = True if not is_crap: if self.conf.has_option(sec, k): if self.conf.get(sec, k).lower() != v.lower(): print('Overwriting locale %s (%r -> %r)' % (k, self.conf.get(sec, k), v)) self.conf.set(sec, k, v) else: if self.crap.has_option(sec, k): print('Overwriting crap locale %s (%r -> %r)' % (k, self.crap.get(sec, k), v)) self.crap.set(sec, k, v)
def get_config(basename, path=None): if not path: path = [ p % basename for p in ['%s.ini', '~/%s.ini', '%s.yaml', '~/%s.yaml'] ] elif isinstance(path, str): path = [path] for filename in map(os.path.expanduser, path): if os.path.exists(filename): if filename.endswith('.ini'): parser = RawConfigParser() parser.read_string('[_default_section]\n' + open(filename, 'r').read()) section = basename if basename in parser else '_default_section' return filename, dict(parser[section]) elif filename.endswith('.yaml') and yaml: data = yaml.load(open(filename, 'rb').read()) if not isinstance(data, Mapping): raise ValueError('Expected mapping in %s, got %s' % (filename, type(data))) if basename in data and isinstance(data[basename], Mapping): return filename, data[basename] return filename, data else: raise ValueError('Unknown config extension: %s' % filename) raise RuntimeError('Config not found in %s (pyyaml installed %s)' % (path, bool(yaml)))
def test_str_writer(self, arg: str) -> None: c = ConfigFile() c.set_value("a", "b", StrWriter().to_ini(arg)) buf = StringIO() c.build(buf) rcp = RawConfigParser(strict=False) rcp.read_string(buf.getvalue()) self.assertEqual(arg, rcp["a"]["b"])
def test_list_semi_writer(self, arg: List[str], expected: str) -> None: c = ConfigFile() c.set_value("a", "b", ListSemiWriter().to_ini(arg)) buf = StringIO() c.build(buf) rcp = RawConfigParser(strict=False) rcp.read_string(buf.getvalue()) self.assertEqual(expected, rcp["a"]["b"])
def test_dict_writer(self, arg: Dict[str, str], expected: str) -> None: c = ConfigFile() c.set_value("a", "b", DictWriter().to_ini(arg)) buf = StringIO() c.build(buf) rcp = RawConfigParser(strict=False) rcp.read_string(buf.getvalue()) # I would prefer this be dangling lines self.assertEqual(expected, rcp["a"]["b"])
def test_list_comma_writer_compat(self, arg: Union[str, List[str]], expected: str) -> None: c = ConfigFile() c.set_value("a", "b", ListCommaWriterCompat().to_ini(arg)) buf = StringIO() c.build(buf) rcp = RawConfigParser(strict=False) rcp.read_string(buf.getvalue()) # I would prefer this be dangling lines self.assertEqual(expected, rcp["a"]["b"])
def parse(login_path: str, path=None) -> dict: if path is None: path = _get_login_path_file() parser = RawConfigParser(dict_type=dict, allow_no_value=True, default_section="~~~UNUSED~~~") parser.read_string(read(path), source=path) data = dict(parser.items(login_path)) if 'port' in data: data['port'] = int(data['port']) return data
def parse(login_path: str, path=None) -> dict: if path is None: path = _get_login_path_file() parser = RawConfigParser( dict_type=dict, allow_no_value=True, default_section="~~~UNUSED~~~" ) parser.read_string(read(path), source=path) data = dict(parser.items(login_path)) data = {key: _strip_quotes(value) for key, value in data.items()} if "port" in data: data["port"] = int(data["port"]) return data
def get_user_dir(dir_name, default=None): ''' Get path to XDG's user directory ''' config = RawConfigParser(allow_no_value=True) userdirs_path = os.path.join(xdg_config_home, 'user-dirs.dirs') try: with open(userdirs_path, 'r') as f: config.read_string('[DEFAULT]\n' + f.read()) return config.get(DEFAULTSECT, 'XDG_DESKTOP_DIR') except Exception as exc: return default
def get_config(conf): config = RawConfigParser() with open(conf, 'r') as f: config_string = '[asection]\n' + f.read() try: config.read_string(config_string) except Exception: config_fp = StringIO.StringIO(config_string) config.read_file(config_fp) return config
def atest_parse(self, text: str) -> None: print("Validating", repr(text)) rcp = RawConfigParser(strict=True) rcp.read_string(text) conf = parse_string(text) for section in rcp: if section != "DEFAULT": self.assertIn(section, conf) # type: ignore self.assertEqual( conf[section].keys(), list(rcp[section].keys()), )
def set_from_config(self, path=['amocrm.ini', '~/amocrm.ini']): if isinstance(path, str): path = [path] for config in map(os.path.expanduser, path): if os.path.exists(config): parser = RawConfigParser() parser.read_string('[_default_section]\n' + open(config).read()) section = 'amocrm' if 'amocrm' in parser else '_default_section' self.set(parser[section]['user_login'], parser[section]['user_hash'], parser[section]['domain'], parser[section].get('responsible_user'))
def load_config(config_file, config_type=None, log=logging): """Read and parse configuration data from the file. Args: config_file (str or Path): Absolute path to the configuration file. config_type (str): The type the configuration is in (config or yaml). If not set (or set to None) the file extension is used for automatic detection. log (optional): A log handler to use. Returns: Configuration dictionary. """ config_file = Path(config_file) # Auto-detection if config_type is None: file_type = _detect_config_type(config_file, log) else: file_type = config_type try: if file_type in [".conf", "conf"]: configparser = RawConfigParser() try: # pylint: disable=deprecated-method configparser.readfp(_FakeSecHead(config_file.open('r'))) # TODO why was this necessary? except Exception: with config_file.open('r') as f: config_string = '[asection]\n' + f.read() configparser.read_string(config_string) config = parse_parameters(configparser)["asection"] elif file_type in [".yaml", "yaml"]: with config_file.open('r') as f: config = yaml.safe_load(f) # check for "None" entries _fix_none_entries(dictionary=config) else: raise Exception() except Exception: log.error("Could not load config file %s", config_file) raise return config
def get_conf_directory(): default_config = '/etc/default/dmd' home_dir = os.path.expanduser("~") + '/.dmd/' project_cfg = os.getcwd() + '/src/cfg/' if os.path.isfile(default_config): with open(default_config) as f: file_content = '[dummy_section]\n' + f.read() config_parser = RawConfigParser() config_parser.read_string(file_content) return config_parser.get('dummy_section', 'conf_directory') elif os.path.isfile(home_dir + 'controller'): return home_dir else: return project_cfg
def read_info(filepath, to_dict=False): with open(filepath, "rb") as f: info_string = f.read().decode("utf-8") if not re.search(r"^\[.*?]", info_string): # print(filepath + " has no section") info_string = "[DEFAULT]\n" + info_string info = RawConfigParser() info.optionxform = str info.read_string(info_string) if to_dict: # TODO: Support multiple sections if not list(info.keys()) == ["DEFAULT"]: raise NotImplementedError("Configs with multiple sections not yet supported") return dict(info["DEFAULT"]) return info
def test_parse(self, lines: List[str]) -> None: text = "\n".join(lines) rcp = RawConfigParser(strict=True) rcp.read_string(text) print("Validating", repr(text)) print(" ", {k: dict(rcp[k]) for k in rcp if k != "DEFAULT"}) conf = parse_string(text) for section in rcp: if section != "DEFAULT": self.assertIn(section, conf) # type: ignore self.assertEqual( conf[section].keys(), list(rcp[section].keys()), )
class INITokenizer(BaseTokenizer): """ .ini file parser. Yields (section, key, value) """ name = "ini" def __init__(self, data): super().__init__(data) self.config = RawConfigParser() self.config.read_string(data) def __iter__(self): for section in sorted(self.config.sections()): for k, v in sorted(self.config.items(section)): yield section, k, v
def __init__(self, config_str, setup_function): self.__tests = [] config_parser = RawConfigParser() config_parser.read_string(config_str) app_path = config_parser.get('main', 'app_path') build_type = config_parser.get('main', 'build_type') platform = config_parser.get('main', 'platform') ''' get_changed_files возвращает список изменённых файлов и зависит от используемой CVS ''' changed_files = None if build_type != 'hook' else get_changed_files() self.__context = VerificationContext(app_path, build_type, platform, changed_files) setup_function(self)
def get_configfile_from_config(configuration): default_config = '/etc/default/jaws' project_cfg = os.getcwd() + '/src/cfg/' path = "" config_file = "" if os.path.isfile(default_config): with open(default_config) as f: file_content = '[dummy_section]\n' + f.read() config_parser = RawConfigParser() config_parser.read_string(file_content) path = config_parser.get('dummy_section', 'conf_directory') else: path = project_cfg main_parser = ConfigParser() main_config = main_parser.read(path + 'main.ini') if configuration == "storages": config_file = main_parser.get('CONFIGFILES', 'STORAGES') if configuration == "acl": config_file = main_parser.get('CONFIGFILES', 'ACL') if configuration == "application": config_file = main_parser.get('CONFIGFILES', 'APPLICATION') if configuration == "copytools": config_file = main_parser.get('CONFIGFILES', 'COPYTOOLS') if configuration == "controller": config_file = main_parser.get('CONFIGFILES', 'CONTROLLER') if configuration == "master": config_file = main_parser.get('CONFIGFILES', 'MASTER') if configuration == "view": config_file = main_parser.get('CONFIGFILES', 'MASTER') if configuration == "worker": config_file = main_parser.get('CONFIGFILES', 'WORKER') if configuration == "main": config_file = path + 'main.ini' config = ConfigParser() config.read(path + config_file) configuration_sections = config.sections() configuration = dict() for section in configuration_sections: configuration[section] = dict(config.items(section)) return configuration
def _load(source, *args, **kwargs): """Return a dict of sections and corresponding key-value pairs.""" sections = collections.defaultdict(dict) try: with source(*args, **kwargs) as file_object: content = file_object.read().decode('utf-8') parser = RawConfigParser() parser.read_string(content) for name in parser.sections(): for key, value in parser.items(name): sections[_decode(name)][_decode(key)] = _decode(value) except IOError: logger.error('Unable to load configuration from %s', *args) return sections
def read_info(filepath, to_dict=False, might_be_scheduled=True): if might_be_scheduled and not isfile(filepath): scheduled_files = glob(filepath + ".*") if not scheduled_files: raise FileNotFoundError(filepath) filepath = scheduled_files[0] with open(filepath) as f: info_string = f.read() if not re.search(r"^\[.*?\]", info_string): # print(filepath + " has no section") info_string = "[DEFAULT]\n" + info_string info = RawConfigParser() info.optionxform = str info.read_string(info_string) if to_dict: # TODO: Support multiple sections if not list(info.keys()) == ["DEFAULT"]: raise NotImplementedError( "Configs with multiple sections not yet supported") return dict(info["DEFAULT"]) return info
class Xhtml2PdfReport(Report): def __init__(self, options): self.options = RawConfigParser(allow_no_value=True) try: self.options.read_string(options) except AttributeError: self.options.readfp(BytesIO(options.encode("UTF-8"))) def get_default_options(self): return DEFAULT_OPTS def render(self, request, context): model = context["bmfmodule"]["model"]._meta template_name = "%s/%s_htmlreport.html" % (model.app_label, model.model_name) pages_file = None try: bg_pk = self.options.getint("pages", "pdf_background_pk") file = Document.objects.get(pk=bg_pk) pages_file = codecs.encode(file.file.read(), "base64").decode().replace("\n", "") except (Document.DoesNotExist, ValueError): pass letter_file = None try: bg_pk = self.options.getint("letter_page", "pdf_background_pk") file = Document.objects.get(pk=bg_pk) letter_file = codecs.encode(file.file.read(), "base64").decode().replace("\n", "") except (Document.DoesNotExist, ValueError): pass options = { "template_name": template_name, "size": self.options.get("layout", "size"), "form": self.options.get("layout", "form"), "letter": self.options.getboolean("layout", "letter"), "template_letter": letter_file, "template_pages": pages_file, "letter_margin_right": self.options.get("letter_page", "margin_right"), "letter_margin_bottom": self.options.get("letter_page", "margin_bottom"), "letter_extra": self.options.getboolean("letter_page", "extra"), "letter_extra_right": self.options.get("letter_page", "extra_right"), "letter_extra_top": self.options.get("letter_page", "extra_top"), "page_margin_top": self.options.get("pages", "margin_top"), "page_margin_right": self.options.get("pages", "margin_right"), "page_margin_bottom": self.options.get("pages", "margin_bottom"), "footer_height": self.options.get("footer", "height"), "footer_right": self.options.get("footer", "right"), } context["options"] = options template = select_template([template_name, "djangobmf/report_html_base.html"]) # pdf won't be in UTF-8 html = template.render(Context(context)).encode("ISO-8859-1") if settings.REPORTING_SERVER: response = requests.post(settings.REPORTING_SERVER, data=html, timeout=5.0) return "pdf", "application/pdf", response.content, True elif XHTML2PDF: buffer = BytesIO() pdf = pisa.pisaDocument(BytesIO(html), buffer) pdf = buffer.getvalue() buffer.close() return "pdf", "application/pdf", pdf, True else: return "html", "text/html", html, False
def __parse_configurations(self, __confpath, __isastream): '''Parse the different configurations''' try: # check if the path to the confs is a directory or a file if os.path.isdir(__confpath): __confs = [__file for __file in os.listdir(__confpath) if __file.endswith('.conf')] else: __confpath, __conft = os.path.split(__confpath) __confs = [__conft] # check if at least one configuration file is availabe if not __confs: __errmsg = 'Could not find any .conf file in {}' print(__errmsg.format(__confpath)) sys.exit(1) # parse the configuration files for __conf in __confs: __currentconf = {} __config = RawConfigParser() __fullconfpath = os.path.join('/'.join([__confpath, __conf])) try: with open(__fullconfpath, 'r') as __file: # strip GPG/PGP header and footer if it is a signed file __stripres = self.strip_gpg_header(__file, __fullconfpath) __config.read_string(__stripres) except UnicodeDecodeError as __err: __msg = 'Error while parsing the configuration file {}:'.format(__fullconfpath) print(__msg) print(__err) sys.exit(1) # Common information for the backups # The name of the backup __currentconf['name'] = __config.get('main', 'name') ### The type of the backups __currentconf['type'] = __config.get('main', 'type') # Common information for the archives ### The archive path __confsettings = [{'main': 'path'}, ### The list of the expected files in the archive {'main': 'files_list'}, ### The delimiter to use in the list of files {'main': 'delimiter'}, ### The hash sum to identify the list of files {'main': 'sha512'} ] for __element in __confsettings: __key, __value = __element.popitem() if __config.has_option(__key, __value): __currentconf[__value] = __config.get( __key, __value) else: __currentconf[__value] = __config.set( __key, __value, '') # Checking the information ### Check the paths in the configuration __confkeys= ('path', 'files_list') for __confkey in __confkeys: if __confkey == 'path' and __isastream: break else: __path = __currentconf[__confkey] if not __path: print('A path is missing in {}.'.format(__config.get('main', 'name'))) sys.exit(1) if not os.path.isabs(__path): __path = os.path.normpath(os.path.join(os.path.abspath(__confpath), __path)) __currentconf[__confkey] = __path # placeholder should be here plh = PlaceHolder(__currentconf[__confkey]) __currentconf[__confkey] = plh.realpath # test if the path exists if not os.path.exists(__currentconf[__confkey]): print('{} does not exist.'.format(__path)) sys.exit(1) # If the backup type is archive, path must not be a directory if not __isastream and __currentconf['type'] == 'archive' and os.path.isdir(__currentconf['path']): __errmsg = '{} is a directory but appears as an archive in configuration {}.' print(__errmsg.format(__currentconf['path'], __config.get('main', 'name'))) sys.exit(1) # check if the name of the conf does not exist yet if __config.get('main', 'name') in self.__configs: __errmsg = 'The configuration name in {} already exists. Please rename it.' print(__errmsg.format(__fullconfpath)) sys.exit(1) else: self.__configs[__config.get('main', 'name')] = __currentconf except (ParsingError, NoSectionError, NoOptionError, OSError, IOError) as __err: print(__err) sys.exit(1)