def _parse_config(conf_content):
conf_ast = ast.parse(conf_content)
config = {}
for statement in conf_ast.body:
if not isinstance(statement, ast.Assign):
# ignore complicated statements
continue
target = statement.targets[0]
if isinstance(target, ast.Name):
name = target.id
elif (isinstance(target, ast.Subscript) and
isinstance(target.value, ast.Name) and
isinstance(target.slice, ast.Index) and
isinstance(target.slice.value, ast.Str)):
# cheat a bit since this name is illegal for variable
name = "%s[%s]" % (target.value.id, target.slice.value.s)
else:
logger.warning('cannot parse assignment at line %i',
statement.lineno)
continue
try:
config[name] = _resolve_constant(statement.value)
except NotConstant:
logger.warning('value assigned to %s in horizon config could not '
'be parsed as a constant', name)
continue
return config
def _parse_config(conf_content):
conf_ast = ast.parse(conf_content)
config = {}
for statement in conf_ast.body:
if not isinstance(statement, ast.Assign):
# ignore complicated statements
continue
target = statement.targets[0]
if isinstance(target, ast.Name):
name = target.id
elif (isinstance(target, ast.Subscript)
and isinstance(target.value, ast.Name)
and isinstance(target.slice, ast.Index)
and isinstance(target.slice.value, ast.Str)):
# cheat a bit since this name is illegal for variable
name = "%s[%s]" % (target.value.id, target.slice.value.s)
else:
logger.warning('cannot parse assignment at line %i',
statement.lineno)
continue
try:
config[name] = _resolve_constant(statement.value)
except NotConstant:
logger.warning(
'value assigned to %s in horizon config could not '
'be parsed as a constant', name)
continue
return config
def _get_deb_repos():
repos = []
files = ['/etc/apt/sources.list']
try:
for name in os.listdir('/etc/apt/sources.list.d'):
if not name.endswith('.list'):
continue
if set(name).difference(SOURCES_LIST_CHARS):
# unexpected characters in the name, ignored by apt by default
continue
files.append('/etc/apt/sources.list.d/' + name)
except OSError:
# if the directory cannot be read, it's ok to ignore it
pass
for name in files:
try:
with open(name, 'r') as f:
for line in f:
repo = _parse_deb_repo_line(line)
if repo:
repos.append(repo)
except EnvironmentError:
logger.warning('cannot read repo list "%s"', name)
continue
return repos
def _find_all_inis(config_set):
if not config_set:
return []
found = []
# the first item should be just the main ini
if os.path.isfile(config_set['ini_file']):
found.append(config_set['ini_file'])
else:
logger.warning('expected "%s" to be a file, ignoring',
config_set['ini_file'])
scan_dirs = config_set['ini_dirs'].split(':')
for scan_dir in scan_dirs:
if not os.path.isdir(scan_dir):
continue
for entry in sorted(os.listdir(scan_dir)):
if not entry.endswith('.ini'):
continue
full_path = os.path.join(scan_dir, entry)
if not os.path.isfile(full_path):
continue
found.append(full_path)
return found
def _find_all_inis(config_set):
if not config_set:
return []
found = []
# the first item should be just the main ini
if os.path.isfile(config_set['ini_file']):
found.append(config_set['ini_file'])
else:
logger.warning('expected "%s" to be a file, ignoring',
config_set['ini_file'])
scan_dirs = config_set['ini_dirs'].split(':')
for scan_dir in scan_dirs:
if not os.path.isdir(scan_dir):
continue
for entry in sorted(os.listdir(scan_dir)):
if not entry.endswith('.ini'):
continue
full_path = os.path.join(scan_dir, entry)
if not os.path.isfile(full_path):
continue
found.append(full_path)
return found
def _parse_openstack_ini_contents(fobj):
section = 'DEFAULT'
config = collections.defaultdict(dict)
for line in fobj:
line = line.strip()
if not line:
continue
if line.startswith('#'):
continue
if line.startswith('[') and line.endswith(']'):
section = line[1:-1]
continue
parts = line.split('=', 1)
if len(parts) != 2:
logger.warning("line cannot be parsed: '%s'", line)
continue
key, value = parts
key = key.strip()
value = value.strip()
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
elif value.startswith("'") and value.endswith("'"):
value = value[1:-1]
config[section][key] = value
return config
def _get_deb_repos():
repos = []
files = ['/etc/apt/sources.list']
try:
for name in os.listdir('/etc/apt/sources.list.d'):
if not name.endswith('.list'):
continue
if set(name).difference(SOURCES_LIST_CHARS):
# unexpected characters in the name, ignored by apt by default
continue
files.append('/etc/apt/sources.list.d/' + name)
except OSError:
# if the directory cannot be read, it's ok to ignore it
pass
for name in files:
try:
with open(name, 'r') as f:
for line in f:
repo = _parse_deb_repo_line(line)
if repo:
repos.append(repo)
except EnvironmentError:
logger.warning('cannot read repo list "%s"', name)
continue
return repos
def _read_config(path):
with open(path, 'r') as f:
conf_lines = f.readlines()
config = collections.defaultdict(dict)
section = None
for lineno, line in enumerate(conf_lines):
# strip comment
try:
comment_start = line.index('#')
except ValueError:
pass # no comment found
else:
line = line[:comment_start]
line = line.strip()
if not line:
continue
parts = [p.strip() for p in line.split(' ')]
if parts[0] in ('global', 'defaults'):
section = parts[0]
continue
elif parts[0] in ('listen', 'frontend', 'backend'):
if len(line) == 1:
logger.warning("section '%s' is missing a name, ignoring line",
parts[0])
continue
else:
section = '/'.join(parts)
continue
if section is None:
logger.warning("option outside of any section, ignoring")
continue
key = parts[0]
if len(parts) == 1:
val = None
else:
val = parts[1:]
if key in REPEATING_OPTIONS:
if key not in config[section]:
config[section][key] = []
config[section][key].append(val)
else:
config[section][key] = val
return config
def _read_config(path):
with open(path, 'r') as f:
conf_lines = f.readlines()
config = collections.defaultdict(dict)
section = None
for lineno, line in enumerate(conf_lines):
# strip comment
try:
comment_start = line.index('#')
except ValueError:
pass # no comment found
else:
line = line[:comment_start]
line = line.strip()
if not line:
continue
parts = [p.strip() for p in line.split(' ')]
if parts[0] in ('global', 'defaults'):
section = parts[0]
continue
elif parts[0] in ('listen', 'frontend', 'backend'):
if len(line) == 1:
logger.warning("section '%s' is missing a name, ignoring line",
parts[0])
continue
else:
section = '/'.join(parts)
continue
if section is None:
logger.warning("option outside of any section, ignoring")
continue
key = parts[0]
if len(parts) == 1:
val = None
else:
val = parts[1:]
if key in REPEATING_OPTIONS:
if key not in config[section]:
config[section][key] = []
config[section][key].append(val)
else:
config[section][key] = val
return config
def _nginx_parse(tokens):
config = []
statement = []
while True:
try:
tok = next(tokens)
except StopIteration:
return config
if tok is TOK_BEGIN_BLOCK:
statement.append(_nginx_parse(tokens))
config.append(statement)
statement = []
elif tok is TOK_END_BLOCK:
if statement:
config.append(statement)
statement = []
return config
elif tok is TOK_END_OF_STATEMENT:
if statement:
# special-case config includes
if statement[0] == 'include':
if len(statement) != 2:
raise ParsingError("include option must be followed "
"by one path, got %s" % statement)
else:
if os.path.isabs(statement[1]):
glob_str = statement[1]
else:
glob_str = os.path.join(
os.path.dirname(NGINX_CONFIG_PATH),
statement[1])
inc_paths = glob.glob(glob_str)
if not inc_paths:
logger.warning(
"include name '%s' did not resolve "
"to any existing files", statement[1])
for inc_path in inc_paths:
for sub_statement in _read_nginx_config(inc_path):
config.append(sub_statement)
else:
config.append(statement)
statement = []
else:
statement.append(tok)
def _nginx_parse(tokens):
config = []
statement = []
while True:
try:
tok = next(tokens)
except StopIteration:
return config
if tok is TOK_BEGIN_BLOCK:
statement.append(_nginx_parse(tokens))
config.append(statement)
statement = []
elif tok is TOK_END_BLOCK:
if statement:
config.append(statement)
statement = []
return config
elif tok is TOK_END_OF_STATEMENT:
if statement:
# special-case config includes
if statement[0] == 'include':
if len(statement) != 2:
raise ParsingError("include option must be followed "
"by one path, got %s" % statement)
else:
if os.path.isabs(statement[1]):
glob_str = statement[1]
else:
glob_str = os.path.join(
os.path.dirname(NGINX_CONFIG_PATH),
statement[1])
inc_paths = glob.glob(glob_str)
if not inc_paths:
logger.warning("include name '%s' did not resolve "
"to any existing files",
statement[1])
for inc_path in inc_paths:
for sub_statement in _read_nginx_config(inc_path):
config.append(sub_statement)
else:
config.append(statement)
statement = []
else:
statement.append(tok)
def _do_read_config(path, config=collections.defaultdict(dict), section=None):
with open(path, 'r') as f:
conf_lines = f.readlines()
for lineno, line in enumerate(conf_lines):
# strip comment
try:
comment_start = line.rindex(';')
except ValueError:
pass # no comment found
else:
line = line[:comment_start]
line = line.strip()
if not line:
continue
if line.startswith('[') and line.endswith(']'):
section = line[1:-1]
continue
parts = line.split('=', 1)
if len(parts) != 2:
logger.warning("Could not parse line %i in config '%s'",
lineno + 1, path)
continue
key = parts[0].strip()
val = parts[1].strip()
if key == 'include':
for d_path in os.listdir(val):
conf_path = os.path.join(d_path, path)
_read_config(conf_path, config, section)
if key == 'options':
# special case, there can be multiple values
if key in config[section]:
config[section][key].append(val)
else:
config[section][key] = [val]
else:
config[section][key] = val
return config
def _do_read_config(path, config=collections.defaultdict(dict), section=None):
with open(path, 'r') as f:
conf_lines = f.readlines()
for lineno, line in enumerate(conf_lines):
# strip comment
try:
comment_start = line.rindex(';')
except ValueError:
pass # no comment found
else:
line = line[:comment_start]
line = line.strip()
if not line:
continue
if line.startswith('[') and line.endswith(']'):
section = line[1:-1]
continue
parts = line.split('=', 1)
if len(parts) != 2:
logger.warning("Could not parse line %i in config '%s'",
lineno + 1, path)
continue
key = parts[0].strip()
val = parts[1].strip()
if key == 'include':
for d_path in os.listdir(val):
conf_path = os.path.join(d_path, path)
_read_config(conf_path, config, section)
if key == 'options':
# special case, there can be multiple values
if key in config[section]:
config[section][key].append(val)
else:
config[section][key] = [val]
else:
config[section][key] = val
return config
def php_ini(ini_paths):
results = GroupTestResult()
for set_name, config_set in ini_paths.items():
inis = _find_all_inis(config_set)
if not inis:
logger.warning('no php config paths found for %s', set_name)
results.add_result(set_name, TestResult(Result.SKIP,
'config not found'))
continue
config = {}
for ini in inis:
config = _parse_php_config(ini, config)
for test, res in utils.verify_config(
set_name, config, config_set['options'], needs_parsing=False):
results.add_result(test, res)
return results
def _parse_php_config(path, config):
# php ini files are nothing like ini files, they need special treatment
# like skipping sections and converting values to booleans
with open(path, 'r') as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if not line:
continue
if line.startswith(';'): # skip comment
continue
if line.startswith('['): # skip sections... because php
continue
key, _, val = line.partition('=')
key = key.strip()
val = val.strip()
if not key:
logger.warning('line "%s" is invalid php config, skipped', line)
continue
if not val:
config[key] = None
elif val == 'None':
config[key] = None
elif val in ('1', 'On', 'True', 'Yes'):
config[key] = True
elif val in ('0', 'Off', 'False', 'No'):
config[key] = False
elif val[0] == '"' and val[-1] == '"':
config[key] = val[1:-1]
else:
config[key] = val
return config
def _parse_php_config(path, config):
# php ini files are nothing like ini files, they need special treatment
# like skipping sections and converting values to booleans
with open(path, 'r') as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if not line:
continue
if line.startswith(';'): # skip comment
continue
if line.startswith('['): # skip sections... because php
continue
key, _, val = line.partition('=')
key = key.strip()
val = val.strip()
if not key:
logger.warning('line "%s" is invalid php config, skipped', line)
continue
if not val:
config[key] = None
elif val == 'None':
config[key] = None
elif val in ('1', 'On', 'True', 'Yes'):
config[key] = True
elif val in ('0', 'Off', 'False', 'No'):
config[key] = False
elif val[0] == '"' and val[-1] == '"':
config[key] = val[1:-1]
else:
config[key] = val
return config
def _get_mounts():
try:
mounts = subprocess.check_output(['mount'])
except (subprocess.CalledProcessError, OSError):
return None
results = []
for mount in mounts.splitlines():
m = MOUNT_RE.match(mount.strip())
if not m:
logger.warning("could not parse mount line '%s'", mount)
continue
results.append((
m.group(1),
m.group(2),
m.group(3),
m.group(4).split(b','),
))
return results
def _get_mounts():
try:
mounts = subprocess.check_output(['mount'])
except (subprocess.CalledProcessError, OSError):
return None
results = []
for mount in mounts.splitlines():
m = MOUNT_RE.match(mount.strip())
if not m:
logger.warning("could not parse mount line '%s'", mount)
continue
results.append((
m.group(1),
m.group(2),
m.group(3),
m.group(4).split(b','),
))
return results
def _parse_deb_repo_line(line):
# this parses only the one-line format, because honestly, who uses deb822
# in their sources file...
line = line.strip()
# cut off the comments
comment_pos = line.find('#')
if comment_pos != -1:
line = line[:comment_pos]
# ignore empty lines
if not line:
return
# everything's split by whitespace
parts = line.split()
pkg_type = parts.pop(0)
while parts:
if '=' in parts[0]:
# just ignore the options...
parts.pop(0)
else:
break
if not parts:
logger.warning('deb entry "%s" missing uri, ignoring', line)
return
uri = parts.pop(0)
if not parts:
logger.warning('deb entry "%s" missing suite, ignoring', line)
return
suite = parts.pop(0)
components = parts
return {
'type': pkg_type,
'uri': uri,
'suite': suite,
'components': components,
}
def _parse_deb_repo_line(line):
# this parses only the one-line format, because honestly, who uses deb822
# in their sources file...
line = line.strip()
# cut off the comments
comment_pos = line.find('#')
if comment_pos != -1:
line = line[:comment_pos]
# ignore empty lines
if not line:
return
# everything's split by whitespace
parts = line.split()
pkg_type = parts.pop(0)
while parts:
if '=' in parts[0]:
# just ignore the options...
parts.pop(0)
else:
break
if not parts:
logger.warning('deb entry "%s" missing uri, ignoring', line)
return
uri = parts.pop(0)
if not parts:
logger.warning('deb entry "%s" missing suite, ignoring', line)
return
suite = parts.pop(0)
components = parts
return {
'type': pkg_type,
'uri': uri,
'suite': suite,
'components': components,
}
def php_ini(ini_paths):
results = GroupTestResult()
for set_name, config_set in ini_paths.items():
inis = _find_all_inis(config_set)
if not inis:
logger.warning('no php config paths found for %s', set_name)
results.add_result(set_name,
TestResult(Result.SKIP, 'config not found'))
continue
config = {}
for ini in inis:
config = _parse_php_config(ini, config)
for test, res in utils.verify_config(set_name,
config,
config_set['options'],
needs_parsing=False):
results.add_result(test, res)
return results
def _get_login_defs_config():
config = {}
try:
with open('/etc/login.defs', 'r') as f:
for line in f:
line = line.strip()
if not line:
continue
if line.startswith('#'):
continue
try:
key, val = line.split()
config[key] = val
except ValueError:
logger.debug("could not parse '%s'", line)
continue
except EnvironmentError:
logger.warning("cannot read the login.defs config")
return None
return config
def _get_login_defs_config():
config = {}
try:
with open('/etc/login.defs', 'r') as f:
for line in f:
line = line.strip()
if not line:
continue
if line.startswith('#'):
continue
try:
key, val = line.split()
config[key] = val
except ValueError:
logger.debug("could not parse '%s'", line)
continue
except EnvironmentError:
logger.warning("cannot read the login.defs config")
return None
return config